{
 "questions": 500,
 "accuracy": 0.93,
 "accuracy_ci95": [
  0.9042,
  0.9492
 ],
 "accuracy_ci_method": "wilson",
 "task_averaged_accuracy": 0.9366,
 "by_type": {
  "knowledge-update": {
   "accuracy": 0.9359,
   "n": 78
  },
  "multi-session": {
   "accuracy": 0.8647,
   "n": 133
  },
  "single-session-assistant": {
   "accuracy": 1.0,
   "n": 56
  },
  "single-session-preference": {
   "accuracy": 0.9,
   "n": 30
  },
  "single-session-user": {
   "accuracy": 0.9714,
   "n": 70
  },
  "temporal-reasoning": {
   "accuracy": 0.9474,
   "n": 133
  }
 },
 "abstention": {
  "accuracy": 0.9333,
  "n": 30
 },
 "evidence_recall_at_k": 0.9847,
 "evidence_turn_coverage": 0.9871,
 "avg_prompt_tokens_est": 33973.074,
 "cost_usd": 2.6035,
 "cost_basis": "provider-reported USD",
 "answers_cut_off": 0,
 "answers_empty": 0,
 "reader_tokens": 4000,
 "reader_tokens_enforced": true,
 "judge_tokens": 10,
 "judge_tokens_enforced": true,
 "reader": "anthropic/claude-haiku-5.5",
 "judge": "openai/gpt-4o-mini-2024-07-18",
 "inference_provider": "openrouter",
 "model_names_are_requested": false,
 "mode": "dense",
 "embedding": "cloud:qwen/qwen3-embedding-8b",
 "context": "sessions",
 "top_k": 10,
 "budget": null,
 "cot": true,
 "split": "all"
}