{ "format": "anan-context-cache-scenario-1", "recorded_on": "2026-10-17", "source_sha256": "17d82772cfb20e42054b3357acd9e58127ae0288a0b6c2e7b9f39fb843f64893", "scenario": "cold first request; reuse only an exact whole-request message prefix with unchanged model/tools; otherwise cold; output and compaction preparation included; no partial-prefix or generated-output caching assumed", "selection": "earliest complete three-strategy cohort per task instance; latest submitted run per strategy within that cohort; one run per strategy per task; selection independent of outcome", "excluded_instances": [ "context.retain-procurement-exceptions-001" ], "pairs": [ { "instance_id": "context.keep-old-constraints-001", "case": "C01", "cohort": "535f854331e152e9", "pair_id": "context.keep-old-constraints-001:535f854331e152e9", "group": "standard", "strategies": { "original": { "sample_id": "sample-31973714ce0d9447", "status": "failed", "goal_correct": false, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 48477, "cached_input_tokens": 48256, "uncached_input_tokens": 221, "output_tokens": 380, "input_plus_output_tokens": 48857, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 48477 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 48927, "cached_input_tokens": 48768, "uncached_input_tokens": 159, "output_tokens": 74, "input_plus_output_tokens": 49001, "scenario_cached_input_tokens": 48477, "scenario_uncached_input_tokens": 450 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 49131, "cached_input_tokens": 48896, "uncached_input_tokens": 235, "output_tokens": 2489, "input_plus_output_tokens": 51620, "scenario_cached_input_tokens": 48927, "scenario_uncached_input_tokens": 204 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 51691, "cached_input_tokens": 51328, "uncached_input_tokens": 363, "output_tokens": 321, "input_plus_output_tokens": 52012, "scenario_cached_input_tokens": 49131, "scenario_uncached_input_tokens": 2560 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 52039, "cached_input_tokens": 51584, "uncached_input_tokens": 455, "output_tokens": 291, "input_plus_output_tokens": 52330, "scenario_cached_input_tokens": 51691, "scenario_uncached_input_tokens": 348 } ], "measured": { "input_tokens": 250265, "cached_input_tokens": 248832, "uncached_input_tokens": 1433, "output_tokens": 3555, "input_plus_output_tokens": 253820 }, "scenario": { "uncached_input_tokens": 52039, "cached_input_tokens": 198226, "output_tokens": 3555 } }, "compact": { "sample_id": "sample-9d05c7df4bf72215", "status": "failed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "compaction", "reason": "cold_start", "input_tokens": 25373, "cached_input_tokens": 25216, "uncached_input_tokens": 157, "output_tokens": 1171, "input_plus_output_tokens": 26544, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 25373 }, { "sequence": 2, "phase": "compaction", "reason": "context_changed", "input_tokens": 4264, "cached_input_tokens": 4096, "uncached_input_tokens": 168, "output_tokens": 595, "input_plus_output_tokens": 4859, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 4264 }, { "sequence": 3, "phase": "mainChat", "reason": "context_changed", "input_tokens": 28777, "cached_input_tokens": 4352, "uncached_input_tokens": 24425, "output_tokens": 544, "input_plus_output_tokens": 29321, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 28777 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29375, "cached_input_tokens": 29056, "uncached_input_tokens": 319, "output_tokens": 82, "input_plus_output_tokens": 29457, "scenario_cached_input_tokens": 28777, "scenario_uncached_input_tokens": 598 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29587, "cached_input_tokens": 29440, "uncached_input_tokens": 147, "output_tokens": 1657, "input_plus_output_tokens": 31244, "scenario_cached_input_tokens": 29375, "scenario_uncached_input_tokens": 212 }, { "sequence": 6, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 31289, "cached_input_tokens": 30976, "uncached_input_tokens": 313, "output_tokens": 624, "input_plus_output_tokens": 31913, "scenario_cached_input_tokens": 29587, "scenario_uncached_input_tokens": 1702 }, { "sequence": 7, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 31935, "cached_input_tokens": 31488, "uncached_input_tokens": 447, "output_tokens": 318, "input_plus_output_tokens": 32253, "scenario_cached_input_tokens": 31289, "scenario_uncached_input_tokens": 646 } ], "measured": { "input_tokens": 180600, "cached_input_tokens": 154624, "uncached_input_tokens": 25976, "output_tokens": 4991, "input_plus_output_tokens": 185591 }, "scenario": { "uncached_input_tokens": 61572, "cached_input_tokens": 119028, "output_tokens": 4991 } }, "rebuild": { "sample_id": "sample-9f05cda62dc8f58f", "status": "failed", "goal_correct": false, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 16784, "cached_input_tokens": 4352, "uncached_input_tokens": 12432, "output_tokens": 1893, "input_plus_output_tokens": 18677, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 16784 } ], "measured": { "input_tokens": 16784, "cached_input_tokens": 4352, "uncached_input_tokens": 12432, "output_tokens": 1893, "input_plus_output_tokens": 18677 }, "scenario": { "uncached_input_tokens": 16784, "cached_input_tokens": 0, "output_tokens": 1893 } } } }, { "instance_id": "context.recover-current-memory-001", "case": "C02", "cohort": "9633647ffed0634c", "pair_id": "context.recover-current-memory-001:9633647ffed0634c", "group": "standard", "strategies": { "original": { "sample_id": "sample-19eeb61194ad3883", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 48825, "cached_input_tokens": 25088, "uncached_input_tokens": 23737, "output_tokens": 104, "input_plus_output_tokens": 48929, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 48825 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 48991, "cached_input_tokens": 48768, "uncached_input_tokens": 223, "output_tokens": 213, "input_plus_output_tokens": 49204, "scenario_cached_input_tokens": 48825, "scenario_uncached_input_tokens": 166 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 49256, "cached_input_tokens": 49024, "uncached_input_tokens": 232, "output_tokens": 250, "input_plus_output_tokens": 49506, "scenario_cached_input_tokens": 48991, "scenario_uncached_input_tokens": 265 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 49535, "cached_input_tokens": 49280, "uncached_input_tokens": 255, "output_tokens": 229, "input_plus_output_tokens": 49764, "scenario_cached_input_tokens": 49256, "scenario_uncached_input_tokens": 279 } ], "measured": { "input_tokens": 196607, "cached_input_tokens": 172160, "uncached_input_tokens": 24447, "output_tokens": 796, "input_plus_output_tokens": 197403 }, "scenario": { "uncached_input_tokens": 49535, "cached_input_tokens": 147072, "output_tokens": 796 } }, "compact": { "sample_id": "sample-902ec11daaa6d60c", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "compaction", "reason": "cold_start", "input_tokens": 25552, "cached_input_tokens": 3968, "uncached_input_tokens": 21584, "output_tokens": 681, "input_plus_output_tokens": 26233, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 25552 }, { "sequence": 2, "phase": "compaction", "reason": "context_changed", "input_tokens": 4264, "cached_input_tokens": 4096, "uncached_input_tokens": 168, "output_tokens": 319, "input_plus_output_tokens": 4583, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 4264 }, { "sequence": 3, "phase": "mainChat", "reason": "context_changed", "input_tokens": 29015, "cached_input_tokens": 4352, "uncached_input_tokens": 24663, "output_tokens": 165, "input_plus_output_tokens": 29180, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 29015 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29290, "cached_input_tokens": 29056, "uncached_input_tokens": 234, "output_tokens": 251, "input_plus_output_tokens": 29541, "scenario_cached_input_tokens": 29015, "scenario_uncached_input_tokens": 275 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29570, "cached_input_tokens": 29312, "uncached_input_tokens": 258, "output_tokens": 63, "input_plus_output_tokens": 29633, "scenario_cached_input_tokens": 29290, "scenario_uncached_input_tokens": 280 }, { "sequence": 6, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29715, "cached_input_tokens": 29568, "uncached_input_tokens": 147, "output_tokens": 131, "input_plus_output_tokens": 29846, "scenario_cached_input_tokens": 29570, "scenario_uncached_input_tokens": 145 } ], "measured": { "input_tokens": 147406, "cached_input_tokens": 100352, "uncached_input_tokens": 47054, "output_tokens": 1610, "input_plus_output_tokens": 149016 }, "scenario": { "uncached_input_tokens": 59531, "cached_input_tokens": 87875, "output_tokens": 1610 } }, "rebuild": { "sample_id": "sample-f8ee1c2a0a47a13e", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 17061, "cached_input_tokens": 4352, "uncached_input_tokens": 12709, "output_tokens": 280, "input_plus_output_tokens": 17341, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 17061 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 17451, "cached_input_tokens": 17280, "uncached_input_tokens": 171, "output_tokens": 293, "input_plus_output_tokens": 17744, "scenario_cached_input_tokens": 17061, "scenario_uncached_input_tokens": 390 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 17813, "cached_input_tokens": 17664, "uncached_input_tokens": 149, "output_tokens": 162, "input_plus_output_tokens": 17975, "scenario_cached_input_tokens": 17451, "scenario_uncached_input_tokens": 362 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 18004, "cached_input_tokens": 17792, "uncached_input_tokens": 212, "output_tokens": 220, "input_plus_output_tokens": 18224, "scenario_cached_input_tokens": 17813, "scenario_uncached_input_tokens": 191 } ], "measured": { "input_tokens": 70329, "cached_input_tokens": 57088, "uncached_input_tokens": 13241, "output_tokens": 955, "input_plus_output_tokens": 71284 }, "scenario": { "uncached_input_tokens": 18004, "cached_input_tokens": 52325, "output_tokens": 955 } } } }, { "instance_id": "context.prefer-current-file-001", "case": "C03", "cohort": "1035264d2ee6c175", "pair_id": "context.prefer-current-file-001:1035264d2ee6c175", "group": "standard", "strategies": { "original": { "sample_id": "sample-b070204e1e4a917e", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 48613, "cached_input_tokens": 48384, "uncached_input_tokens": 229, "output_tokens": 343, "input_plus_output_tokens": 48956, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 48613 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 49095, "cached_input_tokens": 48896, "uncached_input_tokens": 199, "output_tokens": 172, "input_plus_output_tokens": 49267, "scenario_cached_input_tokens": 48613, "scenario_uncached_input_tokens": 482 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 49295, "cached_input_tokens": 49152, "uncached_input_tokens": 143, "output_tokens": 279, "input_plus_output_tokens": 49574, "scenario_cached_input_tokens": 49095, "scenario_uncached_input_tokens": 200 } ], "measured": { "input_tokens": 147003, "cached_input_tokens": 146432, "uncached_input_tokens": 571, "output_tokens": 794, "input_plus_output_tokens": 147797 }, "scenario": { "uncached_input_tokens": 49295, "cached_input_tokens": 97708, "output_tokens": 794 } }, "compact": { "sample_id": "sample-882b430d9ead5734", "status": "failed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "compaction", "reason": "cold_start", "input_tokens": 25506, "cached_input_tokens": 25344, "uncached_input_tokens": 162, "output_tokens": 981, "input_plus_output_tokens": 26487, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 25506 }, { "sequence": 2, "phase": "compaction", "reason": "context_changed", "input_tokens": 4264, "cached_input_tokens": 4096, "uncached_input_tokens": 168, "output_tokens": 442, "input_plus_output_tokens": 4706, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 4264 }, { "sequence": 3, "phase": "mainChat", "reason": "context_changed", "input_tokens": 28783, "cached_input_tokens": 4352, "uncached_input_tokens": 24431, "output_tokens": 243, "input_plus_output_tokens": 29026, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 28783 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29165, "cached_input_tokens": 28928, "uncached_input_tokens": 237, "output_tokens": 176, "input_plus_output_tokens": 29341, "scenario_cached_input_tokens": 28783, "scenario_uncached_input_tokens": 382 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29369, "cached_input_tokens": 29056, "uncached_input_tokens": 313, "output_tokens": 217, "input_plus_output_tokens": 29586, "scenario_cached_input_tokens": 29165, "scenario_uncached_input_tokens": 204 }, { "sequence": 6, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29698, "cached_input_tokens": 29440, "uncached_input_tokens": 258, "output_tokens": 556, "input_plus_output_tokens": 30254, "scenario_cached_input_tokens": 29369, "scenario_uncached_input_tokens": 329 }, { "sequence": 7, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 30276, "cached_input_tokens": 29824, "uncached_input_tokens": 452, "output_tokens": 155, "input_plus_output_tokens": 30431, "scenario_cached_input_tokens": 29698, "scenario_uncached_input_tokens": 578 } ], "measured": { "input_tokens": 177061, "cached_input_tokens": 151040, "uncached_input_tokens": 26021, "output_tokens": 2770, "input_plus_output_tokens": 179831 }, "scenario": { "uncached_input_tokens": 60046, "cached_input_tokens": 117015, "output_tokens": 2770 } }, "rebuild": { "sample_id": "sample-b0112a7f535c8fd8", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 17018, "cached_input_tokens": 4352, "uncached_input_tokens": 12666, "output_tokens": 428, "input_plus_output_tokens": 17446, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 17018 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 17541, "cached_input_tokens": 17280, "uncached_input_tokens": 261, "output_tokens": 227, "input_plus_output_tokens": 17768, "scenario_cached_input_tokens": 17018, "scenario_uncached_input_tokens": 523 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 17823, "cached_input_tokens": 17664, "uncached_input_tokens": 159, "output_tokens": 364, "input_plus_output_tokens": 18187, "scenario_cached_input_tokens": 17541, "scenario_uncached_input_tokens": 282 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 18235, "cached_input_tokens": 18048, "uncached_input_tokens": 187, "output_tokens": 197, "input_plus_output_tokens": 18432, "scenario_cached_input_tokens": 17823, "scenario_uncached_input_tokens": 412 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 18460, "cached_input_tokens": 18176, "uncached_input_tokens": 284, "output_tokens": 61, "input_plus_output_tokens": 18521, "scenario_cached_input_tokens": 18235, "scenario_uncached_input_tokens": 225 }, { "sequence": 6, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 18617, "cached_input_tokens": 18432, "uncached_input_tokens": 185, "output_tokens": 617, "input_plus_output_tokens": 19234, "scenario_cached_input_tokens": 18460, "scenario_uncached_input_tokens": 157 } ], "measured": { "input_tokens": 107694, "cached_input_tokens": 93952, "uncached_input_tokens": 13742, "output_tokens": 1894, "input_plus_output_tokens": 109588 }, "scenario": { "uncached_input_tokens": 18617, "cached_input_tokens": 89077, "output_tokens": 1894 } } } }, { "instance_id": "context.skip-completed-work-001", "case": "C04", "cohort": "9cc68c4da5cbbdea", "pair_id": "context.skip-completed-work-001:9cc68c4da5cbbdea", "group": "standard", "strategies": { "original": { "sample_id": "sample-26b34f533a1edcf3", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 48585, "cached_input_tokens": 24960, "uncached_input_tokens": 23625, "output_tokens": 205, "input_plus_output_tokens": 48790, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 48585 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 48864, "cached_input_tokens": 48512, "uncached_input_tokens": 352, "output_tokens": 64, "input_plus_output_tokens": 48928, "scenario_cached_input_tokens": 48585, "scenario_uncached_input_tokens": 279 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 49013, "cached_input_tokens": 48768, "uncached_input_tokens": 245, "output_tokens": 189, "input_plus_output_tokens": 49202, "scenario_cached_input_tokens": 48864, "scenario_uncached_input_tokens": 149 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 49278, "cached_input_tokens": 49024, "uncached_input_tokens": 254, "output_tokens": 177, "input_plus_output_tokens": 49455, "scenario_cached_input_tokens": 49013, "scenario_uncached_input_tokens": 265 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 49482, "cached_input_tokens": 49280, "uncached_input_tokens": 202, "output_tokens": 242, "input_plus_output_tokens": 49724, "scenario_cached_input_tokens": 49278, "scenario_uncached_input_tokens": 204 } ], "measured": { "input_tokens": 245222, "cached_input_tokens": 220544, "uncached_input_tokens": 24678, "output_tokens": 877, "input_plus_output_tokens": 246099 }, "scenario": { "uncached_input_tokens": 49482, "cached_input_tokens": 195740, "output_tokens": 877 } }, "compact": { "sample_id": "sample-95bdaf086a63cd07", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "compaction", "reason": "cold_start", "input_tokens": 25495, "cached_input_tokens": 10880, "uncached_input_tokens": 14615, "output_tokens": 1355, "input_plus_output_tokens": 26850, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 25495 }, { "sequence": 2, "phase": "compaction", "reason": "context_changed", "input_tokens": 4264, "cached_input_tokens": 4096, "uncached_input_tokens": 168, "output_tokens": 469, "input_plus_output_tokens": 4733, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 4264 }, { "sequence": 3, "phase": "mainChat", "reason": "context_changed", "input_tokens": 28967, "cached_input_tokens": 4352, "uncached_input_tokens": 24615, "output_tokens": 303, "input_plus_output_tokens": 29270, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 28967 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29423, "cached_input_tokens": 29184, "uncached_input_tokens": 239, "output_tokens": 109, "input_plus_output_tokens": 29532, "scenario_cached_input_tokens": 28967, "scenario_uncached_input_tokens": 456 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29689, "cached_input_tokens": 29440, "uncached_input_tokens": 249, "output_tokens": 279, "input_plus_output_tokens": 29968, "scenario_cached_input_tokens": 29423, "scenario_uncached_input_tokens": 266 }, { "sequence": 6, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 30006, "cached_input_tokens": 29824, "uncached_input_tokens": 182, "output_tokens": 275, "input_plus_output_tokens": 30281, "scenario_cached_input_tokens": 29689, "scenario_uncached_input_tokens": 317 }, { "sequence": 7, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 30308, "cached_input_tokens": 30080, "uncached_input_tokens": 228, "output_tokens": 82, "input_plus_output_tokens": 30390, "scenario_cached_input_tokens": 30006, "scenario_uncached_input_tokens": 302 }, { "sequence": 8, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 30465, "cached_input_tokens": 30208, "uncached_input_tokens": 257, "output_tokens": 275, "input_plus_output_tokens": 30740, "scenario_cached_input_tokens": 30308, "scenario_uncached_input_tokens": 157 } ], "measured": { "input_tokens": 208617, "cached_input_tokens": 168064, "uncached_input_tokens": 40553, "output_tokens": 3147, "input_plus_output_tokens": 211764 }, "scenario": { "uncached_input_tokens": 60224, "cached_input_tokens": 148393, "output_tokens": 3147 } }, "rebuild": { "sample_id": "sample-9b2fe9a18e9b7b81", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 16953, "cached_input_tokens": 4352, "uncached_input_tokens": 12601, "output_tokens": 346, "input_plus_output_tokens": 17299, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 16953 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 17445, "cached_input_tokens": 17280, "uncached_input_tokens": 165, "output_tokens": 118, "input_plus_output_tokens": 17563, "scenario_cached_input_tokens": 16953, "scenario_uncached_input_tokens": 492 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 17727, "cached_input_tokens": 17536, "uncached_input_tokens": 191, "output_tokens": 340, "input_plus_output_tokens": 18067, "scenario_cached_input_tokens": 17445, "scenario_uncached_input_tokens": 282 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 18094, "cached_input_tokens": 17792, "uncached_input_tokens": 302, "output_tokens": 307, "input_plus_output_tokens": 18401, "scenario_cached_input_tokens": 17727, "scenario_uncached_input_tokens": 367 } ], "measured": { "input_tokens": 70219, "cached_input_tokens": 56960, "uncached_input_tokens": 13259, "output_tokens": 1111, "input_plus_output_tokens": 71330 }, "scenario": { "uncached_input_tokens": 18094, "cached_input_tokens": 52125, "output_tokens": 1111 } } } }, { "instance_id": "context.resolve-final-delivery-001", "case": "C06", "cohort": "c8b661ad13c1ae61", "pair_id": "context.resolve-final-delivery-001:c8b661ad13c1ae61", "group": "standard", "strategies": { "original": { "sample_id": "sample-8841bdf4abf8439f", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 48632, "cached_input_tokens": 10880, "uncached_input_tokens": 37752, "output_tokens": 214, "input_plus_output_tokens": 48846, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 48632 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 49035, "cached_input_tokens": 48768, "uncached_input_tokens": 267, "output_tokens": 66, "input_plus_output_tokens": 49101, "scenario_cached_input_tokens": 48632, "scenario_uncached_input_tokens": 403 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 49184, "cached_input_tokens": 49024, "uncached_input_tokens": 160, "output_tokens": 189, "input_plus_output_tokens": 49373, "scenario_cached_input_tokens": 49035, "scenario_uncached_input_tokens": 149 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 49400, "cached_input_tokens": 49152, "uncached_input_tokens": 248, "output_tokens": 60, "input_plus_output_tokens": 49460, "scenario_cached_input_tokens": 49184, "scenario_uncached_input_tokens": 216 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 49544, "cached_input_tokens": 49408, "uncached_input_tokens": 136, "output_tokens": 175, "input_plus_output_tokens": 49719, "scenario_cached_input_tokens": 49400, "scenario_uncached_input_tokens": 144 } ], "measured": { "input_tokens": 245795, "cached_input_tokens": 207232, "uncached_input_tokens": 38563, "output_tokens": 704, "input_plus_output_tokens": 246499 }, "scenario": { "uncached_input_tokens": 49544, "cached_input_tokens": 196251, "output_tokens": 704 } }, "compact": { "sample_id": "sample-acc40786f32cc08b", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "compaction", "reason": "cold_start", "input_tokens": 25506, "cached_input_tokens": 25088, "uncached_input_tokens": 418, "output_tokens": 1085, "input_plus_output_tokens": 26591, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 25506 }, { "sequence": 2, "phase": "compaction", "reason": "context_changed", "input_tokens": 4264, "cached_input_tokens": 4096, "uncached_input_tokens": 168, "output_tokens": 355, "input_plus_output_tokens": 4619, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 4264 }, { "sequence": 3, "phase": "mainChat", "reason": "context_changed", "input_tokens": 28798, "cached_input_tokens": 4352, "uncached_input_tokens": 24446, "output_tokens": 367, "input_plus_output_tokens": 29165, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 28798 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29354, "cached_input_tokens": 29056, "uncached_input_tokens": 298, "output_tokens": 157, "input_plus_output_tokens": 29511, "scenario_cached_input_tokens": 28798, "scenario_uncached_input_tokens": 556 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29594, "cached_input_tokens": 29440, "uncached_input_tokens": 154, "output_tokens": 314, "input_plus_output_tokens": 29908, "scenario_cached_input_tokens": 29354, "scenario_uncached_input_tokens": 240 }, { "sequence": 6, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29935, "cached_input_tokens": 29696, "uncached_input_tokens": 239, "output_tokens": 59, "input_plus_output_tokens": 29994, "scenario_cached_input_tokens": 29594, "scenario_uncached_input_tokens": 341 }, { "sequence": 7, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 30078, "cached_input_tokens": 29824, "uncached_input_tokens": 254, "output_tokens": 266, "input_plus_output_tokens": 30344, "scenario_cached_input_tokens": 29935, "scenario_uncached_input_tokens": 143 } ], "measured": { "input_tokens": 177529, "cached_input_tokens": 151552, "uncached_input_tokens": 25977, "output_tokens": 2603, "input_plus_output_tokens": 180132 }, "scenario": { "uncached_input_tokens": 59848, "cached_input_tokens": 117681, "output_tokens": 2603 } }, "rebuild": { "sample_id": "sample-5b6774b6f846b3fc", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 17067, "cached_input_tokens": 4352, "uncached_input_tokens": 12715, "output_tokens": 393, "input_plus_output_tokens": 17460, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 17067 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 17649, "cached_input_tokens": 17408, "uncached_input_tokens": 241, "output_tokens": 65, "input_plus_output_tokens": 17714, "scenario_cached_input_tokens": 17067, "scenario_uncached_input_tokens": 582 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 17797, "cached_input_tokens": 17536, "uncached_input_tokens": 261, "output_tokens": 246, "input_plus_output_tokens": 18043, "scenario_cached_input_tokens": 17649, "scenario_uncached_input_tokens": 148 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 18070, "cached_input_tokens": 17920, "uncached_input_tokens": 150, "output_tokens": 60, "input_plus_output_tokens": 18130, "scenario_cached_input_tokens": 17797, "scenario_uncached_input_tokens": 273 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 18214, "cached_input_tokens": 18048, "uncached_input_tokens": 166, "output_tokens": 227, "input_plus_output_tokens": 18441, "scenario_cached_input_tokens": 18070, "scenario_uncached_input_tokens": 144 } ], "measured": { "input_tokens": 88797, "cached_input_tokens": 75264, "uncached_input_tokens": 13533, "output_tokens": 991, "input_plus_output_tokens": 89788 }, "scenario": { "uncached_input_tokens": 18214, "cached_input_tokens": 70583, "output_tokens": 991 } } } }, { "instance_id": "context.reconcile-revised-receipts-001", "case": "C07", "cohort": "c589e3574bb1445a", "pair_id": "context.reconcile-revised-receipts-001:c589e3574bb1445a", "group": "standard", "strategies": { "original": { "sample_id": "sample-b00200d15e70375f", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 49080, "cached_input_tokens": 10880, "uncached_input_tokens": 38200, "output_tokens": 649, "input_plus_output_tokens": 49729, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 49080 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 50293, "cached_input_tokens": 49664, "uncached_input_tokens": 629, "output_tokens": 150, "input_plus_output_tokens": 50443, "scenario_cached_input_tokens": 49080, "scenario_uncached_input_tokens": 1213 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 50472, "cached_input_tokens": 50176, "uncached_input_tokens": 296, "output_tokens": 256, "input_plus_output_tokens": 50728, "scenario_cached_input_tokens": 50293, "scenario_uncached_input_tokens": 179 } ], "measured": { "input_tokens": 149845, "cached_input_tokens": 110720, "uncached_input_tokens": 39125, "output_tokens": 1055, "input_plus_output_tokens": 150900 }, "scenario": { "uncached_input_tokens": 50472, "cached_input_tokens": 99373, "output_tokens": 1055 } }, "compact": { "sample_id": "sample-00865f25eab09bef", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "compaction", "reason": "cold_start", "input_tokens": 25968, "cached_input_tokens": 25472, "uncached_input_tokens": 496, "output_tokens": 863, "input_plus_output_tokens": 26831, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 25968 }, { "sequence": 2, "phase": "compaction", "reason": "context_changed", "input_tokens": 4264, "cached_input_tokens": 4096, "uncached_input_tokens": 168, "output_tokens": 299, "input_plus_output_tokens": 4563, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 4264 }, { "sequence": 3, "phase": "mainChat", "reason": "context_changed", "input_tokens": 29042, "cached_input_tokens": 4352, "uncached_input_tokens": 24690, "output_tokens": 595, "input_plus_output_tokens": 29637, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 29042 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 30125, "cached_input_tokens": 29568, "uncached_input_tokens": 557, "output_tokens": 384, "input_plus_output_tokens": 30509, "scenario_cached_input_tokens": 29042, "scenario_uncached_input_tokens": 1083 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 30589, "cached_input_tokens": 30336, "uncached_input_tokens": 253, "output_tokens": 123, "input_plus_output_tokens": 30712, "scenario_cached_input_tokens": 30125, "scenario_uncached_input_tokens": 464 }, { "sequence": 6, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 30741, "cached_input_tokens": 30592, "uncached_input_tokens": 149, "output_tokens": 62, "input_plus_output_tokens": 30803, "scenario_cached_input_tokens": 30589, "scenario_uncached_input_tokens": 152 }, { "sequence": 7, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 30881, "cached_input_tokens": 30720, "uncached_input_tokens": 161, "output_tokens": 469, "input_plus_output_tokens": 31350, "scenario_cached_input_tokens": 30741, "scenario_uncached_input_tokens": 140 } ], "measured": { "input_tokens": 181610, "cached_input_tokens": 155136, "uncached_input_tokens": 26474, "output_tokens": 2795, "input_plus_output_tokens": 184405 }, "scenario": { "uncached_input_tokens": 61113, "cached_input_tokens": 120497, "output_tokens": 2795 } }, "rebuild": { "sample_id": "sample-4d80c43c40595e8e", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 17886, "cached_input_tokens": 4352, "uncached_input_tokens": 13534, "output_tokens": 649, "input_plus_output_tokens": 18535, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 17886 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 19023, "cached_input_tokens": 18432, "uncached_input_tokens": 591, "output_tokens": 67, "input_plus_output_tokens": 19090, "scenario_cached_input_tokens": 17886, "scenario_uncached_input_tokens": 1137 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 19170, "cached_input_tokens": 18944, "uncached_input_tokens": 226, "output_tokens": 101, "input_plus_output_tokens": 19271, "scenario_cached_input_tokens": 19023, "scenario_uncached_input_tokens": 147 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 19300, "cached_input_tokens": 19072, "uncached_input_tokens": 228, "output_tokens": 57, "input_plus_output_tokens": 19357, "scenario_cached_input_tokens": 19170, "scenario_uncached_input_tokens": 130 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 19435, "cached_input_tokens": 19200, "uncached_input_tokens": 235, "output_tokens": 436, "input_plus_output_tokens": 19871, "scenario_cached_input_tokens": 19300, "scenario_uncached_input_tokens": 135 } ], "measured": { "input_tokens": 94814, "cached_input_tokens": 80000, "uncached_input_tokens": 14814, "output_tokens": 1310, "input_plus_output_tokens": 96124 }, "scenario": { "uncached_input_tokens": 19435, "cached_input_tokens": 75379, "output_tokens": 1310 } } } }, { "instance_id": "context.distinguish-work-outcomes-001", "case": "C08", "cohort": "5e92dc6c4f4ceaae", "pair_id": "context.distinguish-work-outcomes-001:5e92dc6c4f4ceaae", "group": "standard", "strategies": { "original": { "sample_id": "sample-a679b02deb930069", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 48855, "cached_input_tokens": 10880, "uncached_input_tokens": 37975, "output_tokens": 340, "input_plus_output_tokens": 49195, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 48855 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 49406, "cached_input_tokens": 49152, "uncached_input_tokens": 254, "output_tokens": 127, "input_plus_output_tokens": 49533, "scenario_cached_input_tokens": 48855, "scenario_uncached_input_tokens": 551 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 49700, "cached_input_tokens": 49408, "uncached_input_tokens": 292, "output_tokens": 191, "input_plus_output_tokens": 49891, "scenario_cached_input_tokens": 49406, "scenario_uncached_input_tokens": 294 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 49918, "cached_input_tokens": 49664, "uncached_input_tokens": 254, "output_tokens": 55, "input_plus_output_tokens": 49973, "scenario_cached_input_tokens": 49700, "scenario_uncached_input_tokens": 218 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 50056, "cached_input_tokens": 49920, "uncached_input_tokens": 136, "output_tokens": 271, "input_plus_output_tokens": 50327, "scenario_cached_input_tokens": 49918, "scenario_uncached_input_tokens": 138 } ], "measured": { "input_tokens": 247935, "cached_input_tokens": 209024, "uncached_input_tokens": 38911, "output_tokens": 984, "input_plus_output_tokens": 248919 }, "scenario": { "uncached_input_tokens": 50056, "cached_input_tokens": 197879, "output_tokens": 984 } }, "compact": { "sample_id": "sample-d38f639b894c2d38", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "compaction", "reason": "cold_start", "input_tokens": 25733, "cached_input_tokens": 25216, "uncached_input_tokens": 517, "output_tokens": 1532, "input_plus_output_tokens": 27265, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 25733 }, { "sequence": 2, "phase": "compaction", "reason": "context_changed", "input_tokens": 4264, "cached_input_tokens": 4096, "uncached_input_tokens": 168, "output_tokens": 434, "input_plus_output_tokens": 4698, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 4264 }, { "sequence": 3, "phase": "mainChat", "reason": "context_changed", "input_tokens": 28967, "cached_input_tokens": 4352, "uncached_input_tokens": 24615, "output_tokens": 165, "input_plus_output_tokens": 29132, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 28967 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29226, "cached_input_tokens": 29056, "uncached_input_tokens": 170, "output_tokens": 127, "input_plus_output_tokens": 29353, "scenario_cached_input_tokens": 28967, "scenario_uncached_input_tokens": 259 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29531, "cached_input_tokens": 29312, "uncached_input_tokens": 219, "output_tokens": 191, "input_plus_output_tokens": 29722, "scenario_cached_input_tokens": 29226, "scenario_uncached_input_tokens": 305 }, { "sequence": 6, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29749, "cached_input_tokens": 29440, "uncached_input_tokens": 309, "output_tokens": 55, "input_plus_output_tokens": 29804, "scenario_cached_input_tokens": 29531, "scenario_uncached_input_tokens": 218 }, { "sequence": 7, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 29887, "cached_input_tokens": 29696, "uncached_input_tokens": 191, "output_tokens": 249, "input_plus_output_tokens": 30136, "scenario_cached_input_tokens": 29749, "scenario_uncached_input_tokens": 138 } ], "measured": { "input_tokens": 177357, "cached_input_tokens": 151168, "uncached_input_tokens": 26189, "output_tokens": 2753, "input_plus_output_tokens": 180110 }, "scenario": { "uncached_input_tokens": 59884, "cached_input_tokens": 117473, "output_tokens": 2753 } }, "rebuild": { "sample_id": "sample-1497235c9ac910a5", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 17275, "cached_input_tokens": 4352, "uncached_input_tokens": 12923, "output_tokens": 150, "input_plus_output_tokens": 17425, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 17275 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 17512, "cached_input_tokens": 17280, "uncached_input_tokens": 232, "output_tokens": 93, "input_plus_output_tokens": 17605, "scenario_cached_input_tokens": 17275, "scenario_uncached_input_tokens": 237 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 17699, "cached_input_tokens": 17536, "uncached_input_tokens": 163, "output_tokens": 102, "input_plus_output_tokens": 17801, "scenario_cached_input_tokens": 17512, "scenario_uncached_input_tokens": 187 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 17896, "cached_input_tokens": 17664, "uncached_input_tokens": 232, "output_tokens": 178, "input_plus_output_tokens": 18074, "scenario_cached_input_tokens": 17699, "scenario_uncached_input_tokens": 197 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 18101, "cached_input_tokens": 17792, "uncached_input_tokens": 309, "output_tokens": 214, "input_plus_output_tokens": 18315, "scenario_cached_input_tokens": 17896, "scenario_uncached_input_tokens": 205 } ], "measured": { "input_tokens": 88483, "cached_input_tokens": 74624, "uncached_input_tokens": 13859, "output_tokens": 737, "input_plus_output_tokens": 89220 }, "scenario": { "uncached_input_tokens": 18101, "cached_input_tokens": 70382, "output_tokens": 737 } } } }, { "instance_id": "context.restore-booking-card-001", "case": "C09", "cohort": "27e94fcde419b229", "pair_id": "context.restore-booking-card-001:27e94fcde419b229", "group": "standard", "strategies": { "original": { "sample_id": "sample-c6627f39a445268c", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 95887, "cached_input_tokens": 4352, "uncached_input_tokens": 91535, "output_tokens": 273, "input_plus_output_tokens": 96160, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 95887 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 96189, "cached_input_tokens": 96000, "uncached_input_tokens": 189, "output_tokens": 134, "input_plus_output_tokens": 96323, "scenario_cached_input_tokens": 95887, "scenario_uncached_input_tokens": 302 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 96351, "cached_input_tokens": 96128, "uncached_input_tokens": 223, "output_tokens": 60, "input_plus_output_tokens": 96411, "scenario_cached_input_tokens": 96189, "scenario_uncached_input_tokens": 162 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 96522, "cached_input_tokens": 96384, "uncached_input_tokens": 138, "output_tokens": 140, "input_plus_output_tokens": 96662, "scenario_cached_input_tokens": 96351, "scenario_uncached_input_tokens": 171 } ], "measured": { "input_tokens": 384949, "cached_input_tokens": 292864, "uncached_input_tokens": 92085, "output_tokens": 607, "input_plus_output_tokens": 385556 }, "scenario": { "uncached_input_tokens": 96522, "cached_input_tokens": 288427, "output_tokens": 607 } }, "compact": { "sample_id": "sample-0305621721633e4e", "status": "failed", "goal_correct": false, "requests": [ { "sequence": 1, "phase": "compaction", "reason": "cold_start", "input_tokens": 57992, "cached_input_tokens": 57472, "uncached_input_tokens": 520, "output_tokens": 1477, "input_plus_output_tokens": 59469, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 57992 }, { "sequence": 2, "phase": "mainChat", "reason": "context_changed", "input_tokens": 43376, "cached_input_tokens": 4352, "uncached_input_tokens": 39024, "output_tokens": 203, "input_plus_output_tokens": 43579, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 43376 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 43608, "cached_input_tokens": 43392, "uncached_input_tokens": 216, "output_tokens": 593, "input_plus_output_tokens": 44201, "scenario_cached_input_tokens": 43376, "scenario_uncached_input_tokens": 232 } ], "measured": { "input_tokens": 144976, "cached_input_tokens": 105216, "uncached_input_tokens": 39760, "output_tokens": 2273, "input_plus_output_tokens": 147249 }, "scenario": { "uncached_input_tokens": 101600, "cached_input_tokens": 43376, "output_tokens": 2273 } }, "rebuild": { "sample_id": "sample-91fb0fe9bf08528b", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 54185, "cached_input_tokens": 4352, "uncached_input_tokens": 49833, "output_tokens": 305, "input_plus_output_tokens": 54490, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 54185 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 54519, "cached_input_tokens": 54272, "uncached_input_tokens": 247, "output_tokens": 209, "input_plus_output_tokens": 54728, "scenario_cached_input_tokens": 54185, "scenario_uncached_input_tokens": 334 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 54756, "cached_input_tokens": 54400, "uncached_input_tokens": 356, "output_tokens": 56, "input_plus_output_tokens": 54812, "scenario_cached_input_tokens": 54519, "scenario_uncached_input_tokens": 237 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 54923, "cached_input_tokens": 54784, "uncached_input_tokens": 139, "output_tokens": 177, "input_plus_output_tokens": 55100, "scenario_cached_input_tokens": 54756, "scenario_uncached_input_tokens": 167 } ], "measured": { "input_tokens": 218383, "cached_input_tokens": 167808, "uncached_input_tokens": 50575, "output_tokens": 747, "input_plus_output_tokens": 219130 }, "scenario": { "uncached_input_tokens": 54923, "cached_input_tokens": 163460, "output_tokens": 747 } } } }, { "instance_id": "context.reconcile-event-revisions-001", "case": "C10", "cohort": "e1a466bd470c6fe2", "pair_id": "context.reconcile-event-revisions-001:e1a466bd470c6fe2", "group": "standard", "strategies": { "original": { "sample_id": "sample-352884b8b49d9ea6", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 96208, "cached_input_tokens": 49920, "uncached_input_tokens": 46288, "output_tokens": 344, "input_plus_output_tokens": 96552, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 96208 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 96604, "cached_input_tokens": 96384, "uncached_input_tokens": 220, "output_tokens": 57, "input_plus_output_tokens": 96661, "scenario_cached_input_tokens": 96208, "scenario_uncached_input_tokens": 396 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 96954, "cached_input_tokens": 96640, "uncached_input_tokens": 314, "output_tokens": 595, "input_plus_output_tokens": 97549, "scenario_cached_input_tokens": 96604, "scenario_uncached_input_tokens": 350 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 97577, "cached_input_tokens": 97280, "uncached_input_tokens": 297, "output_tokens": 605, "input_plus_output_tokens": 98182, "scenario_cached_input_tokens": 96954, "scenario_uncached_input_tokens": 623 } ], "measured": { "input_tokens": 387343, "cached_input_tokens": 340224, "uncached_input_tokens": 47119, "output_tokens": 1601, "input_plus_output_tokens": 388944 }, "scenario": { "uncached_input_tokens": 97577, "cached_input_tokens": 289766, "output_tokens": 1601 } }, "compact": { "sample_id": "sample-8e623d33850a1fe0", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "compaction", "reason": "cold_start", "input_tokens": 58100, "cached_input_tokens": 57600, "uncached_input_tokens": 500, "output_tokens": 1145, "input_plus_output_tokens": 59245, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 58100 }, { "sequence": 2, "phase": "mainChat", "reason": "context_changed", "input_tokens": 43544, "cached_input_tokens": 4352, "uncached_input_tokens": 39192, "output_tokens": 416, "input_plus_output_tokens": 43960, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 43544 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 44028, "cached_input_tokens": 43776, "uncached_input_tokens": 252, "output_tokens": 66, "input_plus_output_tokens": 44094, "scenario_cached_input_tokens": 43544, "scenario_uncached_input_tokens": 484 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 44387, "cached_input_tokens": 44032, "uncached_input_tokens": 355, "output_tokens": 692, "input_plus_output_tokens": 45079, "scenario_cached_input_tokens": 44028, "scenario_uncached_input_tokens": 359 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 45107, "cached_input_tokens": 44800, "uncached_input_tokens": 307, "output_tokens": 60, "input_plus_output_tokens": 45167, "scenario_cached_input_tokens": 44387, "scenario_uncached_input_tokens": 720 }, { "sequence": 6, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 45279, "cached_input_tokens": 45056, "uncached_input_tokens": 223, "output_tokens": 636, "input_plus_output_tokens": 45915, "scenario_cached_input_tokens": 45107, "scenario_uncached_input_tokens": 172 } ], "measured": { "input_tokens": 280445, "cached_input_tokens": 239616, "uncached_input_tokens": 40829, "output_tokens": 3015, "input_plus_output_tokens": 283460 }, "scenario": { "uncached_input_tokens": 103379, "cached_input_tokens": 177066, "output_tokens": 3015 } }, "rebuild": { "sample_id": "sample-3203af6fe1614019", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 54506, "cached_input_tokens": 4352, "uncached_input_tokens": 50154, "output_tokens": 139, "input_plus_output_tokens": 54645, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 54506 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 54986, "cached_input_tokens": 54528, "uncached_input_tokens": 458, "output_tokens": 737, "input_plus_output_tokens": 55723, "scenario_cached_input_tokens": 54506, "scenario_uncached_input_tokens": 480 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 55751, "cached_input_tokens": 55424, "uncached_input_tokens": 327, "output_tokens": 121, "input_plus_output_tokens": 55872, "scenario_cached_input_tokens": 54986, "scenario_uncached_input_tokens": 765 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 55984, "cached_input_tokens": 55808, "uncached_input_tokens": 176, "output_tokens": 184, "input_plus_output_tokens": 56168, "scenario_cached_input_tokens": 55751, "scenario_uncached_input_tokens": 233 } ], "measured": { "input_tokens": 221227, "cached_input_tokens": 170112, "uncached_input_tokens": 51115, "output_tokens": 1181, "input_plus_output_tokens": 222408 }, "scenario": { "uncached_input_tokens": 55984, "cached_input_tokens": 165243, "output_tokens": 1181 } } } }, { "instance_id": "context.restore-booking-card-128k-001", "case": "C09", "cohort": "99d1be600defd350", "pair_id": "context.restore-booking-card-128k-001:99d1be600defd350", "group": "extended", "strategies": { "original": { "sample_id": "sample-d9d468229c9eeb1a", "status": "failed", "goal_correct": false, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 308038, "cached_input_tokens": 4352, "uncached_input_tokens": 303686, "output_tokens": 5285, "input_plus_output_tokens": 313323, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 308038 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 313352, "cached_input_tokens": 307968, "uncached_input_tokens": 5384, "output_tokens": 231, "input_plus_output_tokens": 313583, "scenario_cached_input_tokens": 308038, "scenario_uncached_input_tokens": 5314 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 313611, "cached_input_tokens": 313472, "uncached_input_tokens": 139, "output_tokens": 60, "input_plus_output_tokens": 313671, "scenario_cached_input_tokens": 313352, "scenario_uncached_input_tokens": 259 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 313785, "cached_input_tokens": 313600, "uncached_input_tokens": 185, "output_tokens": 212, "input_plus_output_tokens": 313997, "scenario_cached_input_tokens": 313611, "scenario_uncached_input_tokens": 174 } ], "measured": { "input_tokens": 1248786, "cached_input_tokens": 939392, "uncached_input_tokens": 309394, "output_tokens": 5788, "input_plus_output_tokens": 1254574 }, "scenario": { "uncached_input_tokens": 313785, "cached_input_tokens": 935001, "output_tokens": 5788 } }, "compact": { "sample_id": "sample-c927ef11af5aa91d", "status": "failed", "goal_correct": false, "requests": [ { "sequence": 1, "phase": "compaction", "reason": "cold_start", "input_tokens": 270143, "cached_input_tokens": 269696, "uncached_input_tokens": 447, "output_tokens": 501, "input_plus_output_tokens": 270644, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 270143 }, { "sequence": 2, "phase": "mainChat", "reason": "context_changed", "input_tokens": 43196, "cached_input_tokens": 4352, "uncached_input_tokens": 38844, "output_tokens": 257, "input_plus_output_tokens": 43453, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 43196 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 43498, "cached_input_tokens": 43264, "uncached_input_tokens": 234, "output_tokens": 668, "input_plus_output_tokens": 44166, "scenario_cached_input_tokens": 43196, "scenario_uncached_input_tokens": 302 } ], "measured": { "input_tokens": 356837, "cached_input_tokens": 317312, "uncached_input_tokens": 39525, "output_tokens": 1426, "input_plus_output_tokens": 358263 }, "scenario": { "uncached_input_tokens": 313641, "cached_input_tokens": 43196, "output_tokens": 1426 } }, "rebuild": { "sample_id": "sample-f16e8937203aa783", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 54185, "cached_input_tokens": 4352, "uncached_input_tokens": 49833, "output_tokens": 307, "input_plus_output_tokens": 54492, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 54185 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 54521, "cached_input_tokens": 54272, "uncached_input_tokens": 249, "output_tokens": 133, "input_plus_output_tokens": 54654, "scenario_cached_input_tokens": 54185, "scenario_uncached_input_tokens": 336 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 54682, "cached_input_tokens": 54528, "uncached_input_tokens": 154, "output_tokens": 60, "input_plus_output_tokens": 54742, "scenario_cached_input_tokens": 54521, "scenario_uncached_input_tokens": 161 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 54853, "cached_input_tokens": 54656, "uncached_input_tokens": 197, "output_tokens": 208, "input_plus_output_tokens": 55061, "scenario_cached_input_tokens": 54682, "scenario_uncached_input_tokens": 171 } ], "measured": { "input_tokens": 218241, "cached_input_tokens": 167808, "uncached_input_tokens": 50433, "output_tokens": 708, "input_plus_output_tokens": 218949 }, "scenario": { "uncached_input_tokens": 54853, "cached_input_tokens": 163388, "output_tokens": 708 } } } }, { "instance_id": "context.reconcile-event-revisions-128k-001", "case": "C10", "cohort": "1623e098fac59c6f", "pair_id": "context.reconcile-event-revisions-128k-001:1623e098fac59c6f", "group": "extended", "strategies": { "original": { "sample_id": "sample-eb2c06c271a8bac5", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 308359, "cached_input_tokens": 308224, "uncached_input_tokens": 135, "output_tokens": 356, "input_plus_output_tokens": 308715, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 308359 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 308809, "cached_input_tokens": 308608, "uncached_input_tokens": 201, "output_tokens": 61, "input_plus_output_tokens": 308870, "scenario_cached_input_tokens": 308359, "scenario_uncached_input_tokens": 450 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 309163, "cached_input_tokens": 308864, "uncached_input_tokens": 299, "output_tokens": 822, "input_plus_output_tokens": 309985, "scenario_cached_input_tokens": 308809, "scenario_uncached_input_tokens": 354 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 310013, "cached_input_tokens": 309632, "uncached_input_tokens": 381, "output_tokens": 56, "input_plus_output_tokens": 310069, "scenario_cached_input_tokens": 309163, "scenario_uncached_input_tokens": 850 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 310181, "cached_input_tokens": 310016, "uncached_input_tokens": 165, "output_tokens": 327, "input_plus_output_tokens": 310508, "scenario_cached_input_tokens": 310013, "scenario_uncached_input_tokens": 168 } ], "measured": { "input_tokens": 1546525, "cached_input_tokens": 1545344, "uncached_input_tokens": 1181, "output_tokens": 1622, "input_plus_output_tokens": 1548147 }, "scenario": { "uncached_input_tokens": 310181, "cached_input_tokens": 1236344, "output_tokens": 1622 } }, "compact": { "sample_id": "sample-f18f7413ce4129e9", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "compaction", "reason": "cold_start", "input_tokens": 270251, "cached_input_tokens": 269824, "uncached_input_tokens": 427, "output_tokens": 1521, "input_plus_output_tokens": 271772, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 270251 }, { "sequence": 2, "phase": "mainChat", "reason": "context_changed", "input_tokens": 43799, "cached_input_tokens": 4352, "uncached_input_tokens": 39447, "output_tokens": 369, "input_plus_output_tokens": 44168, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 43799 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 44220, "cached_input_tokens": 44032, "uncached_input_tokens": 188, "output_tokens": 57, "input_plus_output_tokens": 44277, "scenario_cached_input_tokens": 43799, "scenario_uncached_input_tokens": 421 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 44570, "cached_input_tokens": 44160, "uncached_input_tokens": 410, "output_tokens": 481, "input_plus_output_tokens": 45051, "scenario_cached_input_tokens": 44220, "scenario_uncached_input_tokens": 350 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 45079, "cached_input_tokens": 44928, "uncached_input_tokens": 151, "output_tokens": 60, "input_plus_output_tokens": 45139, "scenario_cached_input_tokens": 44570, "scenario_uncached_input_tokens": 509 }, { "sequence": 6, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 45251, "cached_input_tokens": 45056, "uncached_input_tokens": 195, "output_tokens": 585, "input_plus_output_tokens": 45836, "scenario_cached_input_tokens": 45079, "scenario_uncached_input_tokens": 172 } ], "measured": { "input_tokens": 493170, "cached_input_tokens": 452352, "uncached_input_tokens": 40818, "output_tokens": 3073, "input_plus_output_tokens": 496243 }, "scenario": { "uncached_input_tokens": 315502, "cached_input_tokens": 177668, "output_tokens": 3073 } }, "rebuild": { "sample_id": "sample-a1419f2972a55a87", "status": "passed", "goal_correct": true, "requests": [ { "sequence": 1, "phase": "mainChat", "reason": "cold_start", "input_tokens": 54506, "cached_input_tokens": 4352, "uncached_input_tokens": 50154, "output_tokens": 247, "input_plus_output_tokens": 54753, "scenario_cached_input_tokens": 0, "scenario_uncached_input_tokens": 54506 }, { "sequence": 2, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 54805, "cached_input_tokens": 54656, "uncached_input_tokens": 149, "output_tokens": 57, "input_plus_output_tokens": 54862, "scenario_cached_input_tokens": 54506, "scenario_uncached_input_tokens": 299 }, { "sequence": 3, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 55155, "cached_input_tokens": 54784, "uncached_input_tokens": 371, "output_tokens": 491, "input_plus_output_tokens": 55646, "scenario_cached_input_tokens": 54805, "scenario_uncached_input_tokens": 350 }, { "sequence": 4, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 55674, "cached_input_tokens": 55296, "uncached_input_tokens": 378, "output_tokens": 56, "input_plus_output_tokens": 55730, "scenario_cached_input_tokens": 55155, "scenario_uncached_input_tokens": 519 }, { "sequence": 5, "phase": "mainChat", "reason": "append_only_prefix", "input_tokens": 55842, "cached_input_tokens": 55680, "uncached_input_tokens": 162, "output_tokens": 486, "input_plus_output_tokens": 56328, "scenario_cached_input_tokens": 55674, "scenario_uncached_input_tokens": 168 } ], "measured": { "input_tokens": 275982, "cached_input_tokens": 224768, "uncached_input_tokens": 51214, "output_tokens": 1337, "input_plus_output_tokens": 277319 }, "scenario": { "uncached_input_tokens": 55842, "cached_input_tokens": 220140, "output_tokens": 1337 } } } } ] }