{
  "schema": "mnemetric.longmemeval-development/v1",
  "updated": "2026-10-30",
  "benchmark": "LongMemEval-S (cleaned), 500 questions",
  "classification": "Mnemosyne LongMemEval-S result using benchmark grading prompts",
  "comparison_eligible": false,
  "independently_reproduced": false,
  "signed_admitted_result": false,
  "protocol_review": {
    "review_date": "2026-10-30",
    "source_repository": "onfire7777/Mnemosyne",
    "source_head_at_review": "aa666bc458867c1da9ca65a14ffdbd6e9d264d05",
    "source_file": "eval/longmemeval_s/run_bench_LongMemEval.py",
    "source_sha256_at_review": "101d1a7af692bb3bcc4dba7a087efa62bf65b7d0dfe02688aae3d25b8ae5d7e1",
    "source_was_modified_at_review": true,
    "run_time_source_binding_available": false,
    "benchmark_specific_answer_examples_in_current_reader_prompt": true,
    "example_topics": [
      "shampoo brand",
      "novel page-count sum",
      "follower change",
      "gift giver"
    ],
    "diagnostic_cases": 31,
    "diagnostic_partition_counts": {
      "heldout": 24,
      "dev": 7
    },
    "limitations": [
      "The current source includes benchmark-specific answer examples; the execution-time prompt snapshot was not retained.",
      "The intermediate 31-case diagnostic includes 7 dev and 24 heldout-labelled records.",
      "The source hash identifies the review snapshot rather than proven execution-time bytes.",
      "Independent reproduction and matched competitor measurements are not included in these artifacts.",
      "The Wilson interval describes the recorded verdict proportion; it does not include tuning or judge uncertainty.",
      "Cost is the sum of retained reader and judge charges; embedding preparation and wider experimentation are excluded."
    ],
    "result_description": "Mnemosyne LongMemEval-S result: 97.20% (486/500), using LongMemEval grading prompts.",
    "clarification": "Custom system architecture and lack of independent reproduction do not automatically invalidate a LongMemEval result. The concerns here are exposure of test-labelled cases and the absence of a bound execution-time prompt snapshot."
  },
  "runs": [
    {
      "run_id": "dense-top10-claude-haiku-5.5-all",
      "label": "Dense baseline",
      "date": "2026-10-28",
      "questions": 500,
      "correct": 465,
      "incorrect": 35,
      "accuracy": 0.93,
      "accuracy_ci95": [
        0.9042,
        0.9492
      ],
      "task_averaged_accuracy": 0.9366,
      "by_type": {
        "knowledge-update": {
          "correct": 73,
          "n": 78,
          "accuracy": 0.9359
        },
        "multi-session": {
          "correct": 115,
          "n": 133,
          "accuracy": 0.8647
        },
        "single-session-assistant": {
          "correct": 56,
          "n": 56,
          "accuracy": 1.0
        },
        "single-session-preference": {
          "correct": 27,
          "n": 30,
          "accuracy": 0.9
        },
        "single-session-user": {
          "correct": 68,
          "n": 70,
          "accuracy": 0.9714
        },
        "temporal-reasoning": {
          "correct": 126,
          "n": 133,
          "accuracy": 0.9474
        }
      },
      "abstention": {
        "correct": 28,
        "n": 30,
        "accuracy": 0.9333
      },
      "evidence_recall_at_k": 0.9847,
      "evidence_turn_coverage": 0.9871,
      "retrieval_denominator": 500,
      "cost_usd": 2.6035,
      "answers_cut_off": 0,
      "answers_empty": 0,
      "configuration": {
        "reader": "anthropic/claude-haiku-5.5",
        "reasoning_effort": null,
        "judge": "openai/gpt-4o-mini-2024-07-18",
        "inference_provider": "openrouter",
        "embedding": "cloud:qwen/qwen3-embedding-8b",
        "mode": "dense",
        "context": "sessions",
        "top_k": 10,
        "reader_tokens": 4000,
        "judge_tokens": 10,
        "cot": true,
        "split": "all"
      },
      "partitions": {
        "dev": {
          "questions": 100,
          "correct": 94,
          "incorrect": 6,
          "accuracy": 0.94,
          "accuracy_ci95": [
            0.8752,
            0.9722
          ],
          "task_averaged_accuracy": 0.9373,
          "by_type": {
            "knowledge-update": {
              "correct": 9,
              "n": 11,
              "accuracy": 0.8182
            },
            "multi-session": {
              "correct": 24,
              "n": 27,
              "accuracy": 0.8889
            },
            "single-session-assistant": {
              "correct": 13,
              "n": 13,
              "accuracy": 1.0
            },
            "single-session-preference": {
              "correct": 5,
              "n": 5,
              "accuracy": 1.0
            },
            "single-session-user": {
              "correct": 11,
              "n": 12,
              "accuracy": 0.9167
            },
            "temporal-reasoning": {
              "correct": 32,
              "n": 32,
              "accuracy": 1.0
            }
          },
          "abstention": {
            "correct": 4,
            "n": 5,
            "accuracy": 0.8
          },
          "evidence_recall_at_k": 1.0,
          "evidence_turn_coverage": 1.0,
          "retrieval_denominator": 100,
          "cost_usd": 0.5212,
          "answers_cut_off": 0,
          "answers_empty": 0
        },
        "heldout": {
          "questions": 400,
          "correct": 371,
          "incorrect": 29,
          "accuracy": 0.9275,
          "accuracy_ci95": [
            0.8978,
            0.949
          ],
          "task_averaged_accuracy": 0.9345,
          "by_type": {
            "knowledge-update": {
              "correct": 64,
              "n": 67,
              "accuracy": 0.9552
            },
            "multi-session": {
              "correct": 91,
              "n": 106,
              "accuracy": 0.8585
            },
            "single-session-assistant": {
              "correct": 43,
              "n": 43,
              "accuracy": 1.0
            },
            "single-session-preference": {
              "correct": 22,
              "n": 25,
              "accuracy": 0.88
            },
            "single-session-user": {
              "correct": 57,
              "n": 58,
              "accuracy": 0.9828
            },
            "temporal-reasoning": {
              "correct": 94,
              "n": 101,
              "accuracy": 0.9307
            }
          },
          "abstention": {
            "correct": 24,
            "n": 25,
            "accuracy": 0.96
          },
          "evidence_recall_at_k": 0.9808,
          "evidence_turn_coverage": 0.9838,
          "retrieval_denominator": 400,
          "cost_usd": 2.0823,
          "answers_cut_off": 0,
          "answers_empty": 0
        }
      },
      "source_files": {
        ".jsonl": {
          "sha256": "bae23c27a393bee5cda21e378f3e47d08383a54bee86123afa06e0f25ff6af06",
          "bytes": 948516
        },
        ".summary.json": {
          "sha256": "387c9ed8312c215a116df9f5ab2a991781d166916672455c34754028918cb711",
          "bytes": 1243
        }
      }
    },
    {
      "run_id": "dense-top10-claude-haiku-5.5-xhigh-all",
      "label": "Hybrid + strict abstention",
      "date": "2026-10-29",
      "questions": 500,
      "correct": 469,
      "incorrect": 31,
      "accuracy": 0.938,
      "accuracy_ci95": [
        0.9133,
        0.956
      ],
      "task_averaged_accuracy": 0.9485,
      "by_type": {
        "knowledge-update": {
          "correct": 76,
          "n": 78,
          "accuracy": 0.9744
        },
        "multi-session": {
          "correct": 116,
          "n": 133,
          "accuracy": 0.8722
        },
        "single-session-assistant": {
          "correct": 56,
          "n": 56,
          "accuracy": 1.0
        },
        "single-session-preference": {
          "correct": 28,
          "n": 30,
          "accuracy": 0.9333
        },
        "single-session-user": {
          "correct": 68,
          "n": 70,
          "accuracy": 0.9714
        },
        "temporal-reasoning": {
          "correct": 125,
          "n": 133,
          "accuracy": 0.9398
        }
      },
      "abstention": {
        "correct": 30,
        "n": 30,
        "accuracy": 1.0
      },
      "evidence_recall_at_k": 0.9935,
      "evidence_turn_coverage": 0.9954,
      "retrieval_denominator": 500,
      "cost_usd": 3.1017,
      "answers_cut_off": 0,
      "answers_empty": 0,
      "configuration": {
        "reader": "anthropic/claude-haiku-5.5",
        "reasoning_effort": "xhigh",
        "judge": "openai/gpt-4o-mini-2024-07-18",
        "inference_provider": "openrouter",
        "embedding": "cloud:qwen/qwen3-embedding-8b",
        "mode": "dense",
        "context": "sessions",
        "top_k": 10,
        "reader_tokens": 32000,
        "judge_tokens": 10,
        "cot": true,
        "split": "all"
      },
      "partitions": {
        "dev": {
          "questions": 100,
          "correct": 93,
          "incorrect": 7,
          "accuracy": 0.93,
          "accuracy_ci95": [
            0.8625,
            0.9657
          ],
          "task_averaged_accuracy": 0.951,
          "by_type": {
            "knowledge-update": {
              "correct": 11,
              "n": 11,
              "accuracy": 1.0
            },
            "multi-session": {
              "correct": 23,
              "n": 27,
              "accuracy": 0.8519
            },
            "single-session-assistant": {
              "correct": 13,
              "n": 13,
              "accuracy": 1.0
            },
            "single-session-preference": {
              "correct": 5,
              "n": 5,
              "accuracy": 1.0
            },
            "single-session-user": {
              "correct": 11,
              "n": 12,
              "accuracy": 0.9167
            },
            "temporal-reasoning": {
              "correct": 30,
              "n": 32,
              "accuracy": 0.9375
            }
          },
          "abstention": {
            "correct": 5,
            "n": 5,
            "accuracy": 1.0
          },
          "evidence_recall_at_k": 1.0,
          "evidence_turn_coverage": 1.0,
          "retrieval_denominator": 100,
          "cost_usd": 0.6196,
          "answers_cut_off": 0,
          "answers_empty": 0
        },
        "heldout": {
          "questions": 400,
          "correct": 376,
          "incorrect": 24,
          "accuracy": 0.94,
          "accuracy_ci95": [
            0.9123,
            0.9594
          ],
          "task_averaged_accuracy": 0.9485,
          "by_type": {
            "knowledge-update": {
              "correct": 65,
              "n": 67,
              "accuracy": 0.9701
            },
            "multi-session": {
              "correct": 93,
              "n": 106,
              "accuracy": 0.8774
            },
            "single-session-assistant": {
              "correct": 43,
              "n": 43,
              "accuracy": 1.0
            },
            "single-session-preference": {
              "correct": 23,
              "n": 25,
              "accuracy": 0.92
            },
            "single-session-user": {
              "correct": 57,
              "n": 58,
              "accuracy": 0.9828
            },
            "temporal-reasoning": {
              "correct": 95,
              "n": 101,
              "accuracy": 0.9406
            }
          },
          "abstention": {
            "correct": 25,
            "n": 25,
            "accuracy": 1.0
          },
          "evidence_recall_at_k": 0.9919,
          "evidence_turn_coverage": 0.9943,
          "retrieval_denominator": 400,
          "cost_usd": 2.4821,
          "answers_cut_off": 0,
          "answers_empty": 0
        }
      },
      "source_files": {
        ".jsonl": {
          "sha256": "2e5e9497fa92356ded23fddb2378b7af229fca178f22d165d9a3c2c83d24d551",
          "bytes": 1132053
        },
        ".summary.json": {
          "sha256": "714356c3cba74e10ab41c7605a6b042b3863a45c543a09833406b36fb2a0956f",
          "bytes": 1274
        }
      }
    },
    {
      "run_id": "dense-top10-claude-haiku-5.5-xhigh-v2-all",
      "label": "Hybrid + revised synthesis",
      "date": "2026-10-29",
      "questions": 500,
      "correct": 486,
      "incorrect": 14,
      "accuracy": 0.972,
      "accuracy_ci95": [
        0.9536,
        0.9832
      ],
      "task_averaged_accuracy": 0.9793,
      "by_type": {
        "knowledge-update": {
          "correct": 77,
          "n": 78,
          "accuracy": 0.9872
        },
        "multi-session": {
          "correct": 126,
          "n": 133,
          "accuracy": 0.9474
        },
        "single-session-assistant": {
          "correct": 56,
          "n": 56,
          "accuracy": 1.0
        },
        "single-session-preference": {
          "correct": 30,
          "n": 30,
          "accuracy": 1.0
        },
        "single-session-user": {
          "correct": 68,
          "n": 70,
          "accuracy": 0.9714
        },
        "temporal-reasoning": {
          "correct": 129,
          "n": 133,
          "accuracy": 0.9699
        }
      },
      "abstention": {
        "correct": 26,
        "n": 30,
        "accuracy": 0.8667
      },
      "evidence_recall_at_k": 0.997,
      "evidence_turn_coverage": 0.999,
      "retrieval_denominator": 500,
      "cost_usd": 3.1106,
      "answers_cut_off": 0,
      "answers_empty": 0,
      "configuration": {
        "reader": "anthropic/claude-haiku-5.5",
        "reasoning_effort": "xhigh",
        "judge": "openai/gpt-4o-mini-2024-07-18",
        "inference_provider": "openrouter",
        "embedding": "cloud:qwen/qwen3-embedding-8b",
        "mode": "dense",
        "context": "sessions",
        "top_k": 10,
        "reader_tokens": 32000,
        "judge_tokens": 10,
        "cot": true,
        "split": "all"
      },
      "partitions": {
        "dev": {
          "questions": 100,
          "correct": 98,
          "incorrect": 2,
          "accuracy": 0.98,
          "accuracy_ci95": [
            0.93,
            0.9945
          ],
          "task_averaged_accuracy": 0.9787,
          "by_type": {
            "knowledge-update": {
              "correct": 10,
              "n": 11,
              "accuracy": 0.9091
            },
            "multi-session": {
              "correct": 26,
              "n": 27,
              "accuracy": 0.963
            },
            "single-session-assistant": {
              "correct": 13,
              "n": 13,
              "accuracy": 1.0
            },
            "single-session-preference": {
              "correct": 5,
              "n": 5,
              "accuracy": 1.0
            },
            "single-session-user": {
              "correct": 12,
              "n": 12,
              "accuracy": 1.0
            },
            "temporal-reasoning": {
              "correct": 32,
              "n": 32,
              "accuracy": 1.0
            }
          },
          "abstention": {
            "correct": 4,
            "n": 5,
            "accuracy": 0.8
          },
          "evidence_recall_at_k": 1.0,
          "evidence_turn_coverage": 1.0,
          "retrieval_denominator": 100,
          "cost_usd": 0.6234,
          "answers_cut_off": 0,
          "answers_empty": 0
        },
        "heldout": {
          "questions": 400,
          "correct": 388,
          "incorrect": 12,
          "accuracy": 0.97,
          "accuracy_ci95": [
            0.9483,
            0.9828
          ],
          "task_averaged_accuracy": 0.9782,
          "by_type": {
            "knowledge-update": {
              "correct": 67,
              "n": 67,
              "accuracy": 1.0
            },
            "multi-session": {
              "correct": 100,
              "n": 106,
              "accuracy": 0.9434
            },
            "single-session-assistant": {
              "correct": 43,
              "n": 43,
              "accuracy": 1.0
            },
            "single-session-preference": {
              "correct": 25,
              "n": 25,
              "accuracy": 1.0
            },
            "single-session-user": {
              "correct": 56,
              "n": 58,
              "accuracy": 0.9655
            },
            "temporal-reasoning": {
              "correct": 97,
              "n": 101,
              "accuracy": 0.9604
            }
          },
          "abstention": {
            "correct": 22,
            "n": 25,
            "accuracy": 0.88
          },
          "evidence_recall_at_k": 0.9962,
          "evidence_turn_coverage": 0.9987,
          "retrieval_denominator": 400,
          "cost_usd": 2.4872,
          "answers_cut_off": 0,
          "answers_empty": 0
        }
      },
      "source_files": {
        ".jsonl": {
          "sha256": "082b1fd23071899a68ff75d6d476a8c4f58ee64d5c7734e6e78a0386fbabb1cd",
          "bytes": 1031310
        },
        ".summary.json": {
          "sha256": "caf6a6a932d56d628fd9925136083b90e732f4fcdd37e425bd8baf57159521bc",
          "bytes": 1273
        }
      }
    }
  ]
}
