Official benchmark setup required. Current development checks and adapted results are not official benchmark runs. Official comparisons must use the upstream code, datasets, scoring and prescribed setup, with versions and deviations disclosed. Protocol status →
Trace 778164c6
Stored context/evidence
["ultrachat_169952","ultrachat_336538","490bb46d_1","2d74df23_3","ultrachat_578090","ultrachat_423899","0ba46bdc_2","eccc6e84_3","sharegpt_VEoWAUD_0","05b551b6","sharegpt_CMe0Odo_0","sharegpt_xasomVD_35","a2053427_1","81e406dd_2","ultrachat_419925","ultrachat_144598","c6bd5277","sharegpt_uzwG36E_0","023641cc_3","sharegpt_n42dRB1_0","sharegpt_fnuLBKl_129","ultrachat_346016","sharegpt_ki9IVDq_6","c15dadce_4","sharegpt_CyJ3dal_43","sharegpt_4eVd8BN_0","ultrachat_460345","sharegpt_QizzJ7Z_0","b605741a_1","025836be_1","ultrachat_254669","1340826c","sharegpt_dUoAhvI_23","483ddd90","ultrachat_9855","sharegpt_In7us5d_0","sharegpt_luC18OZ_0","ddaa5a15_2","ultrachat_543871","e42d0467_1","ultrachat_313916","sharegpt_ty6EeyH_0","0c28f7fb","ultrachat_274821","sharegpt_mWsFRK3_127","sharegpt_Z622Ill_0","35201d43","answer_ultrachat_399000","sharegpt_84De5zv_0","sharegpt_WXCvFir_11"]
Retrieved context/evidence
["answer_ultrachat_399000","ultrachat_144598","sharegpt_CMe0Odo_0","ultrachat_254669","sharegpt_QizzJ7Z_0"]
Retrieval output (not a generated answer)
answer_ultrachat_399000
Complete stored trace (JSON)
{
"answer": "answer_ultrachat_399000",
"answer_session_ids": [
"answer_ultrachat_399000"
],
"gold_references": [
"answer_ultrachat_399000"
],
"oracle_has_answer_turns": {
"023641cc_3": [],
"025836be_1": [],
"05b551b6": [],
"0ba46bdc_2": [],
"0c28f7fb": [],
"1340826c": [],
"2d74df23_3": [],
"35201d43": [],
"483ddd90": [],
"490bb46d_1": [],
"81e406dd_2": [],
"a2053427_1": [],
"answer_ultrachat_399000": [
3
],
"b605741a_1": [],
"c15dadce_4": [],
"c6bd5277": [],
"ddaa5a15_2": [],
"e42d0467_1": [],
"eccc6e84_3": [],
"sharegpt_4eVd8BN_0": [],
"sharegpt_84De5zv_0": [],
"sharegpt_CMe0Odo_0": [],
"sharegpt_CyJ3dal_43": [],
"sharegpt_In7us5d_0": [],
"sharegpt_QizzJ7Z_0": [],
"sharegpt_VEoWAUD_0": [],
"sharegpt_WXCvFir_11": [],
"sharegpt_Z622Ill_0": [],
"sharegpt_dUoAhvI_23": [],
"sharegpt_fnuLBKl_129": [],
"sharegpt_ki9IVDq_6": [],
"sharegpt_luC18OZ_0": [],
"sharegpt_mWsFRK3_127": [],
"sharegpt_n42dRB1_0": [],
"sharegpt_ty6EeyH_0": [],
"sharegpt_uzwG36E_0": [],
"sharegpt_xasomVD_35": [],
"ultrachat_144598": [],
"ultrachat_169952": [],
"ultrachat_254669": [],
"ultrachat_274821": [],
"ultrachat_313916": [],
"ultrachat_336538": [],
"ultrachat_346016": [],
"ultrachat_419925": [],
"ultrachat_423899": [],
"ultrachat_460345": [],
"ultrachat_543871": [],
"ultrachat_578090": [],
"ultrachat_9855": []
},
"question_id": "778164c6",
"ranked_retrieved_hits": [
"answer_ultrachat_399000",
"ultrachat_144598",
"sharegpt_CMe0Odo_0",
"ultrachat_254669",
"sharegpt_QizzJ7Z_0"
],
"scoring_family": "deterministic-retrieval",
"stored_records": [
"ultrachat_169952",
"ultrachat_336538",
"490bb46d_1",
"2d74df23_3",
"ultrachat_578090",
"ultrachat_423899",
"0ba46bdc_2",
"eccc6e84_3",
"sharegpt_VEoWAUD_0",
"05b551b6",
"sharegpt_CMe0Odo_0",
"sharegpt_xasomVD_35",
"a2053427_1",
"81e406dd_2",
"ultrachat_419925",
"ultrachat_144598",
"c6bd5277",
"sharegpt_uzwG36E_0",
"023641cc_3",
"sharegpt_n42dRB1_0",
"sharegpt_fnuLBKl_129",
"ultrachat_346016",
"sharegpt_ki9IVDq_6",
"c15dadce_4",
"sharegpt_CyJ3dal_43",
"sharegpt_4eVd8BN_0",
"ultrachat_460345",
"sharegpt_QizzJ7Z_0",
"b605741a_1",
"025836be_1",
"ultrachat_254669",
"1340826c",
"sharegpt_dUoAhvI_23",
"483ddd90",
"ultrachat_9855",
"sharegpt_In7us5d_0",
"sharegpt_luC18OZ_0",
"ddaa5a15_2",
"ultrachat_543871",
"e42d0467_1",
"ultrachat_313916",
"sharegpt_ty6EeyH_0",
"0c28f7fb",
"ultrachat_274821",
"sharegpt_mWsFRK3_127",
"sharegpt_Z622Ill_0",
"35201d43",
"answer_ultrachat_399000",
"sharegpt_84De5zv_0",
"sharegpt_WXCvFir_11"
]
}