Official benchmark setup required. Current development checks and adapted results are not official benchmark runs. Official comparisons must use the upstream code, datasets, scoring and prescribed setup, with versions and deviations disclosed. Protocol status →
Trace 36580ce8
Stored context/evidence
["sharegpt_MN6prau_0","ca0a84a3_1","sharegpt_ZbUcxUa_0","9325144f","sharegpt_6I57AvU_77","ultrachat_318891","14076ea0_1","ultrachat_138369","sharegpt_rAlDk6n_0","ultrachat_272552","60153a02_4","ultrachat_53784","e42d0467_2","sharegpt_nYWWPnO_0","375e27b9_1","ce7fedf8_2","ultrachat_106408","sharegpt_1QPXgB0_0","ultrachat_344096","fc9a271e_2","9f8fc173_3","sharegpt_s48W0eC_0","ultrachat_31614","2aee3e70","sharegpt_Q9iK2Qt_51","ultrachat_32917","9b38bda6_2","ultrachat_299779","sharegpt_ZoYI78l_1","37e50ff6_2","ultrachat_521047","ultrachat_471245","ultrachat_451944","sharegpt_FpTfRvR_0","ultrachat_149170","sharegpt_dWV6erT_32","sharegpt_HNy24nG_0","f62c04c6","sharegpt_A6FQIZB_0","526354c8_2","sharegpt_1HdhKiU_0","sharegpt_tzk1Zm8_7","sharegpt_KH9Qr5r_0","2865bf22_2","ultrachat_270984","sharegpt_yI9gjck_62","sharegpt_5L3F069_69","answer_93e1bd22","sharegpt_bMeRfJz_0","sharegpt_cUXo8S0_0","ultrachat_28133","sharegpt_WmCiERa_25","sharegpt_vKQRlls_0","7b1a2831","sharegpt_i8xL0ih_109"]
Retrieved context/evidence
["sharegpt_cUXo8S0_0","sharegpt_1QPXgB0_0","sharegpt_WmCiERa_25","sharegpt_rAlDk6n_0"]
Retrieval output (not a generated answer)
sharegpt_cUXo8S0_0
Complete stored trace (JSON)
{
"answer": "sharegpt_cUXo8S0_0",
"answer_session_ids": [
"answer_93e1bd22"
],
"gold_references": [
"answer_93e1bd22"
],
"oracle_has_answer_turns": {
"14076ea0_1": [],
"2865bf22_2": [],
"2aee3e70": [],
"375e27b9_1": [],
"37e50ff6_2": [],
"526354c8_2": [],
"60153a02_4": [],
"7b1a2831": [],
"9325144f": [],
"9b38bda6_2": [],
"9f8fc173_3": [],
"answer_93e1bd22": [
4
],
"ca0a84a3_1": [],
"ce7fedf8_2": [],
"e42d0467_2": [],
"f62c04c6": [],
"fc9a271e_2": [],
"sharegpt_1HdhKiU_0": [],
"sharegpt_1QPXgB0_0": [],
"sharegpt_5L3F069_69": [],
"sharegpt_6I57AvU_77": [],
"sharegpt_A6FQIZB_0": [],
"sharegpt_FpTfRvR_0": [],
"sharegpt_HNy24nG_0": [],
"sharegpt_KH9Qr5r_0": [],
"sharegpt_MN6prau_0": [],
"sharegpt_Q9iK2Qt_51": [],
"sharegpt_WmCiERa_25": [],
"sharegpt_ZbUcxUa_0": [],
"sharegpt_ZoYI78l_1": [],
"sharegpt_bMeRfJz_0": [],
"sharegpt_cUXo8S0_0": [],
"sharegpt_dWV6erT_32": [],
"sharegpt_i8xL0ih_109": [],
"sharegpt_nYWWPnO_0": [],
"sharegpt_rAlDk6n_0": [],
"sharegpt_s48W0eC_0": [],
"sharegpt_tzk1Zm8_7": [],
"sharegpt_vKQRlls_0": [],
"sharegpt_yI9gjck_62": [],
"ultrachat_106408": [],
"ultrachat_138369": [],
"ultrachat_149170": [],
"ultrachat_270984": [],
"ultrachat_272552": [],
"ultrachat_28133": [],
"ultrachat_299779": [],
"ultrachat_31614": [],
"ultrachat_318891": [],
"ultrachat_32917": [],
"ultrachat_344096": [],
"ultrachat_451944": [],
"ultrachat_471245": [],
"ultrachat_521047": [],
"ultrachat_53784": []
},
"question_id": "36580ce8",
"ranked_retrieved_hits": [
"sharegpt_cUXo8S0_0",
"sharegpt_1QPXgB0_0",
"sharegpt_WmCiERa_25",
"sharegpt_rAlDk6n_0"
],
"scoring_family": "deterministic-retrieval",
"stored_records": [
"sharegpt_MN6prau_0",
"ca0a84a3_1",
"sharegpt_ZbUcxUa_0",
"9325144f",
"sharegpt_6I57AvU_77",
"ultrachat_318891",
"14076ea0_1",
"ultrachat_138369",
"sharegpt_rAlDk6n_0",
"ultrachat_272552",
"60153a02_4",
"ultrachat_53784",
"e42d0467_2",
"sharegpt_nYWWPnO_0",
"375e27b9_1",
"ce7fedf8_2",
"ultrachat_106408",
"sharegpt_1QPXgB0_0",
"ultrachat_344096",
"fc9a271e_2",
"9f8fc173_3",
"sharegpt_s48W0eC_0",
"ultrachat_31614",
"2aee3e70",
"sharegpt_Q9iK2Qt_51",
"ultrachat_32917",
"9b38bda6_2",
"ultrachat_299779",
"sharegpt_ZoYI78l_1",
"37e50ff6_2",
"ultrachat_521047",
"ultrachat_471245",
"ultrachat_451944",
"sharegpt_FpTfRvR_0",
"ultrachat_149170",
"sharegpt_dWV6erT_32",
"sharegpt_HNy24nG_0",
"f62c04c6",
"sharegpt_A6FQIZB_0",
"526354c8_2",
"sharegpt_1HdhKiU_0",
"sharegpt_tzk1Zm8_7",
"sharegpt_KH9Qr5r_0",
"2865bf22_2",
"ultrachat_270984",
"sharegpt_yI9gjck_62",
"sharegpt_5L3F069_69",
"answer_93e1bd22",
"sharegpt_bMeRfJz_0",
"sharegpt_cUXo8S0_0",
"ultrachat_28133",
"sharegpt_WmCiERa_25",
"sharegpt_vKQRlls_0",
"7b1a2831",
"sharegpt_i8xL0ih_109"
]
}