{
  "review_date": "2026-10-30",
  "source_repository": "onfire7777/Mnemosyne",
  "source_head_at_review": "aa666bc458867c1da9ca65a14ffdbd6e9d264d05",
  "source_file": "eval/longmemeval_s/run_bench_LongMemEval.py",
  "source_sha256_at_review": "101d1a7af692bb3bcc4dba7a087efa62bf65b7d0dfe02688aae3d25b8ae5d7e1",
  "source_was_modified_at_review": true,
  "run_time_source_binding_available": false,
  "benchmark_specific_answer_examples_in_current_reader_prompt": true,
  "example_topics": [
    "shampoo brand",
    "novel page-count sum",
    "follower change",
    "gift giver"
  ],
  "diagnostic_cases": 31,
  "diagnostic_partition_counts": {
    "heldout": 24,
    "dev": 7
  },
  "limitations": [
    "The current source includes benchmark-specific answer examples; the execution-time prompt snapshot was not retained.",
    "The intermediate 31-case diagnostic includes 7 dev and 24 heldout-labelled records.",
    "The source hash identifies the review snapshot rather than proven execution-time bytes.",
    "Independent reproduction and matched competitor measurements are not included in these artifacts.",
    "The Wilson interval describes the recorded verdict proportion; it does not include tuning or judge uncertainty.",
    "Cost is the sum of retained reader and judge charges; embedding preparation and wider experimentation are excluded."
  ],
  "result_description": "Mnemosyne LongMemEval-S result: 97.20% (486/500), using LongMemEval grading prompts.",
  "clarification": "Custom system architecture and lack of independent reproduction do not automatically invalidate a LongMemEval result. The concerns here are exposure of test-labelled cases and the absence of a bound execution-time prompt snapshot."
}
