Official benchmark setup required. Current development checks and adapted results are not official benchmark runs. Official comparisons must use the upstream code, datasets, scoring and prescribed setup, with versions and deviations disclosed. Protocol status →
Trace e3fc4d6e
Stored context/evidence
["ultrachat_473192","9db30f1c_2","ultrachat_253808","ultrachat_61952","ultrachat_325941","sharegpt_kkzJOUc_0","ultrachat_328443","c3437f49","sharegpt_ipe1nhQ_0","ultrachat_161238","sharegpt_nAacldL_21","05be3ff0_4","d31c3ec8_1","c6bed037_2","ultrachat_478030","4d261642","ultrachat_428408","ultrachat_443354","bec86aec","f3d43a12_2","fc6549b2","sharegpt_71QV6Zs_7","14f9ee3c","95ae7b23_1","sharegpt_h3YKuXq_0","ultrachat_299058","ultrachat_385629","9b72b151","ultrachat_498574","ultrachat_495326","1bf761b8_2","6399f4dd_2","answer_sharegpt_5m7gg5F_0","sharegpt_1IIU5Zx_0","6aaf298d","sharegpt_GZP0zo9_1","42adb80d_3","ultrachat_179461","sharegpt_ltqfPQc_0","ultrachat_47024","ultrachat_31614","3b38bf46","ultrachat_334058","ultrachat_442052","ultrachat_3370","ultrachat_38181","ultrachat_563692","sharegpt_TVK4gjw_24","sharegpt_sXKNzPE_0","sharegpt_ySGu6N8_0","ultrachat_301922","sharegpt_80RCktD_0","ad44b3e1_2"]
Retrieved context/evidence
["answer_sharegpt_5m7gg5F_0","ultrachat_563692","sharegpt_1IIU5Zx_0","9db30f1c_2"]
Retrieval output (not a generated answer)
answer_sharegpt_5m7gg5F_0
Complete stored trace (JSON)
{
"answer": "answer_sharegpt_5m7gg5F_0",
"answer_session_ids": [
"answer_sharegpt_5m7gg5F_0"
],
"gold_references": [
"answer_sharegpt_5m7gg5F_0"
],
"oracle_has_answer_turns": {
"05be3ff0_4": [],
"14f9ee3c": [],
"1bf761b8_2": [],
"3b38bf46": [],
"42adb80d_3": [],
"4d261642": [],
"6399f4dd_2": [],
"6aaf298d": [],
"95ae7b23_1": [],
"9b72b151": [],
"9db30f1c_2": [],
"ad44b3e1_2": [],
"answer_sharegpt_5m7gg5F_0": [
1
],
"bec86aec": [],
"c3437f49": [],
"c6bed037_2": [],
"d31c3ec8_1": [],
"f3d43a12_2": [],
"fc6549b2": [],
"sharegpt_1IIU5Zx_0": [],
"sharegpt_71QV6Zs_7": [],
"sharegpt_80RCktD_0": [],
"sharegpt_GZP0zo9_1": [],
"sharegpt_TVK4gjw_24": [],
"sharegpt_h3YKuXq_0": [],
"sharegpt_ipe1nhQ_0": [],
"sharegpt_kkzJOUc_0": [],
"sharegpt_ltqfPQc_0": [],
"sharegpt_nAacldL_21": [],
"sharegpt_sXKNzPE_0": [],
"sharegpt_ySGu6N8_0": [],
"ultrachat_161238": [],
"ultrachat_179461": [],
"ultrachat_253808": [],
"ultrachat_299058": [],
"ultrachat_301922": [],
"ultrachat_31614": [],
"ultrachat_325941": [],
"ultrachat_328443": [],
"ultrachat_334058": [],
"ultrachat_3370": [],
"ultrachat_38181": [],
"ultrachat_385629": [],
"ultrachat_428408": [],
"ultrachat_442052": [],
"ultrachat_443354": [],
"ultrachat_47024": [],
"ultrachat_473192": [],
"ultrachat_478030": [],
"ultrachat_495326": [],
"ultrachat_498574": [],
"ultrachat_563692": [],
"ultrachat_61952": []
},
"question_id": "e3fc4d6e",
"ranked_retrieved_hits": [
"answer_sharegpt_5m7gg5F_0",
"ultrachat_563692",
"sharegpt_1IIU5Zx_0",
"9db30f1c_2"
],
"scoring_family": "deterministic-retrieval",
"stored_records": [
"ultrachat_473192",
"9db30f1c_2",
"ultrachat_253808",
"ultrachat_61952",
"ultrachat_325941",
"sharegpt_kkzJOUc_0",
"ultrachat_328443",
"c3437f49",
"sharegpt_ipe1nhQ_0",
"ultrachat_161238",
"sharegpt_nAacldL_21",
"05be3ff0_4",
"d31c3ec8_1",
"c6bed037_2",
"ultrachat_478030",
"4d261642",
"ultrachat_428408",
"ultrachat_443354",
"bec86aec",
"f3d43a12_2",
"fc6549b2",
"sharegpt_71QV6Zs_7",
"14f9ee3c",
"95ae7b23_1",
"sharegpt_h3YKuXq_0",
"ultrachat_299058",
"ultrachat_385629",
"9b72b151",
"ultrachat_498574",
"ultrachat_495326",
"1bf761b8_2",
"6399f4dd_2",
"answer_sharegpt_5m7gg5F_0",
"sharegpt_1IIU5Zx_0",
"6aaf298d",
"sharegpt_GZP0zo9_1",
"42adb80d_3",
"ultrachat_179461",
"sharegpt_ltqfPQc_0",
"ultrachat_47024",
"ultrachat_31614",
"3b38bf46",
"ultrachat_334058",
"ultrachat_442052",
"ultrachat_3370",
"ultrachat_38181",
"ultrachat_563692",
"sharegpt_TVK4gjw_24",
"sharegpt_sXKNzPE_0",
"sharegpt_ySGu6N8_0",
"ultrachat_301922",
"sharegpt_80RCktD_0",
"ad44b3e1_2"
]
}