{ "schema": "mithril.loop-efficiency/v1", "generated_at_utc": "2026-10-15T12:46:43.943843+00:00", "plan": { "schema": "mithril.efficiency-plan/v1", "selection": "Prospective first four IDs per domain by SHA256(mithril-efficiency-refined-v2:domain:id), excluding all earlier used AND planned IDs; no answer/outcome selection. Public tasks new to this comparison series, not globally uncontaminated.", "bank_tasks": [ "task_034", "task_089", "task_095", "task_093" ], "it_incidents": [ "Scenario-16", "Scenario-17", "Scenario-29", "Scenario-38" ], "seeds": [ 612 ], "arms": [ "actor-only", "perplexity/pplx-decider-v1.1-27b", "cloudflare/clef-flash" ], "primary_episodes": 24, "bank_max_steps": 200, "actor_model": "google/gemini-3.8-flash", "provider": "Google AI Studio", "allow_fallbacks": false, "new_experiment_cap_usd": 60, "cache": "Separate fresh application namespace per case/seed/arm; no prior/shared response cache. Counterbalanced first pass; exact-cache replay reported separately. Provider cache cannot be reset and is measured.", "latency": "Observed episode wall, API wait, tool/local time, call counts, median and p90; grader time separate. Never substitute cache playback time for initial execution.", "scoring": "Original tau2 evaluator with each original task reward_basis (DB or ACTION); IT exact/filter/alias root-group scorer after all predictions, unsupported label schemas remain unscored; no hidden labels enter inference.", "development_only": [ "task_064", "Scenario-11" ], "dataset_sha": "76df38a82288f75ba9e41dc8c515033332497473", "development_allocation_amendment": { "development_subcap_usd": 8, "total_experiment_cap_usd": 40, "before_any_unused_task_inference": true, "reason": "Earlier development allocation reached during banking calibration; preserve all failures/costs and allocate within the unchanged overall cap for operational qualification and source-precedence fix", "at_utc": "2026-10-15T07:56:13.396278+00:00" }, "development_allocation_amendments": [ { "development_subcap_usd": 10, "total_experiment_cap_usd": 40, "before_any_unused_task_inference": true, "reason": "Reserve within the unchanged overall cap for completion of known-case operational canaries after provider-error/cache repair; no score-based retuning or ledger reset", "at_utc": "2026-10-15T08:23:16.035498+00:00" } ], "round": "refined-v2", "created_at_utc": "2026-10-15T09:11:30.618768+00:00", "global_cap_usd": 60, "round1_plan_sha256": "e2418ba7477485590f3c019d4c1b9f12691b66ac0d263ba9d1e50abacabeca21", "reason": "Observed tool-only JSON recovery crash and cosmetic answer alternatives in initial comparison; fix those generically, use critic-first revisions and verify on another unused cohort. Initial 84 outputs remain separate.", "budget_amendment": { "at_utc": "2026-10-15T10:00:57.200568+00:00", "previous_cap_usd": 40, "approved_maximum_usd": 60, "scope": "Finish the preselected initial84 and refined24 primary comparisons, development qualification and exact-cache replays; no third experimental round. Actual necessary invoices only; ledger never reset.", "status": "Approved by the user before the additional charges; the original frozen initial plan is preserved." }, "evaluator_metadata_audit": { "before_refined_inference": true, "task_034_reward_basis": [ "ACTION" ], "other_selected_bank_task_reward_basis": [ "DB" ], "llm_assertion_grading": false, "boundary": "Only evaluation-type metadata inspected; no expected actions, target values or private user goals enter inference. No selected ID changed." }, "initial_new_experiment_cap_usd": 40, "prospective_refinement_amendments": [ { "at_utc": "2026-10-15T11:42:25.586582+00:00", "before_any_refined_primary_inference": true, "observation": "All 36 initial IT trajectories admitted only one distinct causal candidate, so no native decision API was called. This is a trace-structure observation, not an expected-answer change.", "change": "The existing IT critique call also proposes an optional source-checked distinct competing diagnosis. Admit it only after the same literal-source checks and entity-set deduplication; an admitted joint competitor skips the separate revision-generation call. Shared critic max_tokens 1536 to 2048; original actor and evidence budgets remain unchanged.", "scope": "Same preselected 24 executions, same models and shared conditions, same global maximum USD60. No extra cohort; no target entities or label values enter inference." } ] }, "conditions": [ { "phase": "primary", "domain": "banking_knowledge", "arm": "actor-only", "planned": 4, "attempted": 4, "scored": 4, "successful_executions": 4, "mean_score": 0.5, "latency_median_seconds": 99.59311625000555, "latency_p90_seconds": 160.05579762489944, "cost_per_attempt_usd": 0.2811526125, "calls_per_attempt": 34.0, "metering": { "response_observations": 136, "valid_response_observations": 134, "application_cache_hits": 0, "application_cache_denominator": 134, "physical_responses": 136, "provider_error_responses": 2, "actual_cost_usd": 1.12461045, "provider_cache_metadata_responses": 136, "provider_cache_hit_responses": 11, "provider_cached_input_tokens": 161206, "provider_reported_prompt_tokens": 1593421, "physical_api_seconds": 378.06885183601116, "elapsed_response_seconds": 387.06669661701017, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 111, "valid_response_observations": 109, "application_cache_hits": 0, "application_cache_denominator": 109, "physical_responses": 111, "provider_error_responses": 2, "actual_cost_usd": 1.06333695, "provider_cache_metadata_responses": 111, "provider_cache_hit_responses": 11, "provider_cached_input_tokens": 161206, "provider_reported_prompt_tokens": 1525508, "physical_api_seconds": 308.14781787797983, "elapsed_response_seconds": 316.07525074502337, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 0, "application_cache_denominator": 2, "physical_responses": 2, "provider_error_responses": 0, "actual_cost_usd": 0.01672875, "provider_cache_metadata_responses": 2, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 21715, "physical_api_seconds": 5.150595834013075, "elapsed_response_seconds": 5.38267320698651, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 23, "valid_response_observations": 23, "application_cache_hits": 0, "application_cache_denominator": 23, "physical_responses": 23, "provider_error_responses": 0, "actual_cost_usd": 0.04454475, "provider_cache_metadata_responses": 23, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 46198, "physical_api_seconds": 64.77043812401826, "elapsed_response_seconds": 65.60877266500029, "backoff_seconds": 0 } }, "controller_steps": 108, "reviewed_steps": 0, "selector_bypassed_steps": 108, "latency_scope": "Completed execution wall (including evaluated zeros), including cache/API/tool work; original bank grader excluded. Infrastructure failures excluded from latency summaries but retained in attempts and fees.", "served_models": { "google/gemini-3.8-flash": { "response_observations": 136, "valid_response_observations": 134, "application_cache_hits": 0, "application_cache_denominator": 134, "physical_responses": 136, "provider_error_responses": 2, "actual_cost_usd": 1.12461045, "provider_cache_metadata_responses": 136, "provider_cache_hit_responses": 11, "provider_cached_input_tokens": 161206, "provider_reported_prompt_tokens": 1593421, "physical_api_seconds": 378.06885183601116, "elapsed_response_seconds": 387.06669661701017, "backoff_seconds": 0 } }, "format_failure_clarifications": 1 }, { "phase": "primary", "domain": "banking_knowledge", "arm": "perplexity/pplx-decider-v1.1-27b", "planned": 4, "attempted": 4, "scored": 4, "successful_executions": 4, "mean_score": 0.25, "latency_median_seconds": 153.53178097899945, "latency_p90_seconds": 196.78567747509516, "cost_per_attempt_usd": 0.3883613525, "calls_per_attempt": 45.5, "metering": { "response_observations": 182, "valid_response_observations": 180, "application_cache_hits": 0, "application_cache_denominator": 180, "physical_responses": 182, "provider_error_responses": 2, "actual_cost_usd": 1.55344541, "provider_cache_metadata_responses": 175, "provider_cache_hit_responses": 14, "provider_cached_input_tokens": 99592, "provider_reported_prompt_tokens": 2068718, "physical_api_seconds": 528.8849142559193, "elapsed_response_seconds": 534.6530258308921, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 103, "valid_response_observations": 101, "application_cache_hits": 0, "application_cache_denominator": 101, "physical_responses": 103, "provider_error_responses": 2, "actual_cost_usd": 0.973919175, "provider_cache_metadata_responses": 103, "provider_cache_hit_responses": 8, "provider_cached_input_tokens": 94169, "provider_reported_prompt_tokens": 1352946, "physical_api_seconds": 284.2809534979169, "elapsed_response_seconds": 287.46337933493487, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 4, "valid_response_observations": 4, "application_cache_hits": 0, "application_cache_denominator": 4, "physical_responses": 4, "provider_error_responses": 0, "actual_cost_usd": 0.03630525, "provider_cache_metadata_responses": 4, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 47022, "physical_api_seconds": 11.056060000977595, "elapsed_response_seconds": 11.150099375998252, "backoff_seconds": 0 }, "decision": { "response_observations": 7, "valid_response_observations": 7, "application_cache_hits": 0, "application_cache_denominator": 7, "physical_responses": 7, "provider_error_responses": 0, "actual_cost_usd": 0.00028525999999999996, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 5.005285584004014, "elapsed_response_seconds": 5.167018289997941, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 36, "valid_response_observations": 36, "application_cache_hits": 0, "application_cache_denominator": 36, "physical_responses": 36, "provider_error_responses": 0, "actual_cost_usd": 0.411790425, "provider_cache_metadata_responses": 36, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 1839, "provider_reported_prompt_tokens": 527694, "physical_api_seconds": 117.77956746099517, "elapsed_response_seconds": 119.12760408097529, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 22, "valid_response_observations": 22, "application_cache_hits": 0, "application_cache_denominator": 22, "physical_responses": 22, "provider_error_responses": 0, "actual_cost_usd": 0.0415185, "provider_cache_metadata_responses": 22, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 44978, "physical_api_seconds": 57.0586019600305, "elapsed_response_seconds": 57.7328922089946, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 10, "valid_response_observations": 10, "application_cache_hits": 0, "application_cache_denominator": 10, "physical_responses": 10, "provider_error_responses": 0, "actual_cost_usd": 0.0896268, "provider_cache_metadata_responses": 10, "provider_cache_hit_responses": 1, "provider_cached_input_tokens": 3584, "provider_reported_prompt_tokens": 96078, "physical_api_seconds": 53.70444575199508, "elapsed_response_seconds": 54.012032539991196, "backoff_seconds": 0 } }, "controller_steps": 97, "reviewed_steps": 36, "selector_bypassed_steps": 90, "latency_scope": "Completed execution wall (including evaluated zeros), including cache/API/tool work; original bank grader excluded. Infrastructure failures excluded from latency summaries but retained in attempts and fees.", "served_models": { "google/gemini-3.8-flash": { "response_observations": 175, "valid_response_observations": 173, "application_cache_hits": 0, "application_cache_denominator": 173, "physical_responses": 175, "provider_error_responses": 2, "actual_cost_usd": 1.55316015, "provider_cache_metadata_responses": 175, "provider_cache_hit_responses": 14, "provider_cached_input_tokens": 99592, "provider_reported_prompt_tokens": 2068718, "physical_api_seconds": 523.8796286719153, "elapsed_response_seconds": 529.4860075408942, "backoff_seconds": 0 }, "perplexity/pplx-decider-v1.1-27b-20261006": { "response_observations": 7, "valid_response_observations": 7, "application_cache_hits": 0, "application_cache_denominator": 7, "physical_responses": 7, "provider_error_responses": 0, "actual_cost_usd": 0.00028525999999999996, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 5.005285584004014, "elapsed_response_seconds": 5.167018289997941, "backoff_seconds": 0 } }, "format_failure_clarifications": 1 }, { "phase": "primary", "domain": "banking_knowledge", "arm": "cloudflare/clef-flash", "planned": 4, "attempted": 4, "scored": 4, "successful_executions": 4, "mean_score": 0.5, "latency_median_seconds": 209.02744139599963, "latency_p90_seconds": 391.78667191239947, "cost_per_attempt_usd": 0.83559792475, "calls_per_attempt": 75.5, "metering": { "response_observations": 302, "valid_response_observations": 300, "application_cache_hits": 0, "application_cache_denominator": 300, "physical_responses": 302, "provider_error_responses": 2, "actual_cost_usd": 3.342391699, "provider_cache_metadata_responses": 285, "provider_cache_hit_responses": 15, "provider_cached_input_tokens": 121437, "provider_reported_prompt_tokens": 4437121, "physical_api_seconds": 884.0785374559928, "elapsed_response_seconds": 894.7256175488437, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 186, "valid_response_observations": 184, "application_cache_hits": 0, "application_cache_denominator": 184, "physical_responses": 186, "provider_error_responses": 2, "actual_cost_usd": 2.358814425, "provider_cache_metadata_responses": 186, "provider_cache_hit_responses": 10, "provider_cached_input_tokens": 117179, "provider_reported_prompt_tokens": 3199507, "physical_api_seconds": 531.6228957910353, "elapsed_response_seconds": 537.7293654139212, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.06901425, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 89599, "physical_api_seconds": 15.25609604100464, "elapsed_response_seconds": 15.361648916979902, "backoff_seconds": 0 }, "decision": { "response_observations": 17, "valid_response_observations": 17, "application_cache_hits": 0, "application_cache_denominator": 17, "physical_responses": 17, "provider_error_responses": 0, "actual_cost_usd": 0.000764674, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 31.265439084992977, "elapsed_response_seconds": 31.70435037596326, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 51, "valid_response_observations": 51, "application_cache_hits": 0, "application_cache_denominator": 51, "physical_responses": 51, "provider_error_responses": 0, "actual_cost_usd": 0.74166765, "provider_cache_metadata_responses": 51, "provider_cache_hit_responses": 4, "provider_cached_input_tokens": 1472, "provider_reported_prompt_tokens": 954755, "physical_api_seconds": 166.58428512098908, "elapsed_response_seconds": 168.70928233899758, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 32, "valid_response_observations": 32, "application_cache_hits": 0, "application_cache_denominator": 32, "physical_responses": 32, "provider_error_responses": 0, "actual_cost_usd": 0.06123075, "provider_cache_metadata_responses": 32, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 73856, "physical_api_seconds": 78.11858241797017, "elapsed_response_seconds": 79.45271466897975, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 11, "valid_response_observations": 11, "application_cache_hits": 0, "application_cache_denominator": 11, "physical_responses": 11, "provider_error_responses": 0, "actual_cost_usd": 0.11089995, "provider_cache_metadata_responses": 11, "provider_cache_hit_responses": 1, "provider_cached_input_tokens": 2786, "provider_reported_prompt_tokens": 119404, "physical_api_seconds": 61.23123900000064, "elapsed_response_seconds": 61.76825583400205, "backoff_seconds": 0 } }, "controller_steps": 167, "reviewed_steps": 51, "selector_bypassed_steps": 150, "latency_scope": "Completed execution wall (including evaluated zeros), including cache/API/tool work; original bank grader excluded. Infrastructure failures excluded from latency summaries but retained in attempts and fees.", "served_models": { "cloudflare/clef-flash": { "response_observations": 17, "valid_response_observations": 17, "application_cache_hits": 0, "application_cache_denominator": 17, "physical_responses": 17, "provider_error_responses": 0, "actual_cost_usd": 0.000764674, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 31.265439084992977, "elapsed_response_seconds": 31.70435037596326, "backoff_seconds": 0 }, "google/gemini-3.8-flash": { "response_observations": 285, "valid_response_observations": 283, "application_cache_hits": 0, "application_cache_denominator": 283, "physical_responses": 285, "provider_error_responses": 2, "actual_cost_usd": 3.341627025, "provider_cache_metadata_responses": 285, "provider_cache_hit_responses": 15, "provider_cached_input_tokens": 121437, "provider_reported_prompt_tokens": 4437121, "physical_api_seconds": 852.8130983709998, "elapsed_response_seconds": 863.0212671728805, "backoff_seconds": 0 } }, "format_failure_clarifications": 3 }, { "phase": "primary", "domain": "ITBench", "arm": "actor-only", "planned": 4, "attempted": 4, "scored": 3, "successful_executions": 4, "mean_score": 0.3333333333333333, "latency_median_seconds": 16.08318733350461, "latency_p90_seconds": 22.33331323760067, "cost_per_attempt_usd": 0.11744355, "calls_per_attempt": 5.0, "metering": { "response_observations": 20, "valid_response_observations": 19, "application_cache_hits": 0, "application_cache_denominator": 19, "physical_responses": 20, "provider_error_responses": 1, "actual_cost_usd": 0.4697742, "provider_cache_metadata_responses": 20, "provider_cache_hit_responses": 16, "provider_cached_input_tokens": 318626, "provider_reported_prompt_tokens": 905334, "physical_api_seconds": 68.80746708299557, "elapsed_response_seconds": 69.17055350101145, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 20, "valid_response_observations": 19, "application_cache_hits": 0, "application_cache_denominator": 19, "physical_responses": 20, "provider_error_responses": 1, "actual_cost_usd": 0.4697742, "provider_cache_metadata_responses": 20, "provider_cache_hit_responses": 16, "provider_cached_input_tokens": 318626, "provider_reported_prompt_tokens": 905334, "physical_api_seconds": 68.80746708299557, "elapsed_response_seconds": 69.17055350101145, "backoff_seconds": 0 } }, "controller_steps": 0, "reviewed_steps": 0, "selector_bypassed_steps": 0, "latency_scope": "Completed execution wall (including evaluated zeros), including cache/API/tool work; original bank grader excluded. Infrastructure failures excluded from latency summaries but retained in attempts and fees.", "served_models": { "google/gemini-3.8-flash": { "response_observations": 20, "valid_response_observations": 19, "application_cache_hits": 0, "application_cache_denominator": 19, "physical_responses": 20, "provider_error_responses": 1, "actual_cost_usd": 0.4697742, "provider_cache_metadata_responses": 20, "provider_cache_hit_responses": 16, "provider_cached_input_tokens": 318626, "provider_reported_prompt_tokens": 905334, "physical_api_seconds": 68.80746708299557, "elapsed_response_seconds": 69.17055350101145, "backoff_seconds": 0 } }, "format_failure_clarifications": 0 }, { "phase": "primary", "domain": "ITBench", "arm": "perplexity/pplx-decider-v1.1-27b", "planned": 4, "attempted": 4, "scored": 2, "successful_executions": 3, "mean_score": 0.5, "latency_median_seconds": 19.753461250002147, "latency_p90_seconds": 21.67764375000552, "cost_per_attempt_usd": 0.13998590625, "calls_per_attempt": 6.0, "metering": { "response_observations": 24, "valid_response_observations": 22, "application_cache_hits": 0, "application_cache_denominator": 22, "physical_responses": 24, "provider_error_responses": 2, "actual_cost_usd": 0.559943625, "provider_cache_metadata_responses": 24, "provider_cache_hit_responses": 17, "provider_cached_input_tokens": 231225, "provider_reported_prompt_tokens": 947169, "physical_api_seconds": 75.9435682919866, "elapsed_response_seconds": 76.79897653999797, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 21, "valid_response_observations": 19, "application_cache_hits": 0, "application_cache_denominator": 19, "physical_responses": 21, "provider_error_responses": 2, "actual_cost_usd": 0.5030861249999999, "provider_cache_metadata_responses": 21, "provider_cache_hit_responses": 17, "provider_cached_input_tokens": 231225, "provider_reported_prompt_tokens": 871799, "physical_api_seconds": 67.121772124985, "elapsed_response_seconds": 67.91338370800077, "backoff_seconds": 0 }, "it-critique": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 0, "application_cache_denominator": 3, "physical_responses": 3, "provider_error_responses": 0, "actual_cost_usd": 0.056857500000000005, "provider_cache_metadata_responses": 3, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 75370, "physical_api_seconds": 8.821796167001594, "elapsed_response_seconds": 8.8855928319972, "backoff_seconds": 0 } }, "controller_steps": 0, "reviewed_steps": 0, "selector_bypassed_steps": 0, "latency_scope": "Completed execution wall (including evaluated zeros), including cache/API/tool work; original bank grader excluded. Infrastructure failures excluded from latency summaries but retained in attempts and fees.", "served_models": { "google/gemini-3.8-flash": { "response_observations": 24, "valid_response_observations": 22, "application_cache_hits": 0, "application_cache_denominator": 22, "physical_responses": 24, "provider_error_responses": 2, "actual_cost_usd": 0.559943625, "provider_cache_metadata_responses": 24, "provider_cache_hit_responses": 17, "provider_cached_input_tokens": 231225, "provider_reported_prompt_tokens": 947169, "physical_api_seconds": 75.9435682919866, "elapsed_response_seconds": 76.79897653999797, "backoff_seconds": 0 } }, "format_failure_clarifications": 0 }, { "phase": "primary", "domain": "ITBench", "arm": "cloudflare/clef-flash", "planned": 4, "attempted": 4, "scored": 3, "successful_executions": 4, "mean_score": 0.3333333333333333, "latency_median_seconds": 20.438308375007182, "latency_p90_seconds": 24.29812424989941, "cost_per_attempt_usd": 0.13131494999999999, "calls_per_attempt": 6.75, "metering": { "response_observations": 27, "valid_response_observations": 24, "application_cache_hits": 0, "application_cache_denominator": 24, "physical_responses": 27, "provider_error_responses": 3, "actual_cost_usd": 0.5252597999999999, "provider_cache_metadata_responses": 27, "provider_cache_hit_responses": 15, "provider_cached_input_tokens": 386114, "provider_reported_prompt_tokens": 1038509, "physical_api_seconds": 79.43196799806901, "elapsed_response_seconds": 80.0914529570291, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 21, "valid_response_observations": 20, "application_cache_hits": 0, "application_cache_denominator": 20, "physical_responses": 21, "provider_error_responses": 1, "actual_cost_usd": 0.45763215, "provider_cache_metadata_responses": 21, "provider_cache_hit_responses": 14, "provider_cached_input_tokens": 365512, "provider_reported_prompt_tokens": 931637, "physical_api_seconds": 60.10766716605576, "elapsed_response_seconds": 60.64603366299707, "backoff_seconds": 0 }, "it-critique": { "response_observations": 4, "valid_response_observations": 4, "application_cache_hits": 0, "application_cache_denominator": 4, "physical_responses": 4, "provider_error_responses": 0, "actual_cost_usd": 0.06762765, "provider_cache_metadata_responses": 4, "provider_cache_hit_responses": 1, "provider_cached_input_tokens": 20602, "provider_reported_prompt_tokens": 106872, "physical_api_seconds": 11.480499249009881, "elapsed_response_seconds": 11.550890960017568, "backoff_seconds": 0 }, "it-revise-alternatives": { "response_observations": 2, "valid_response_observations": 0, "application_cache_hits": 0, "application_cache_denominator": 0, "physical_responses": 2, "provider_error_responses": 2, "actual_cost_usd": 0, "provider_cache_metadata_responses": 2, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 7.843801583003369, "elapsed_response_seconds": 7.894528334014467, "backoff_seconds": 0 } }, "controller_steps": 0, "reviewed_steps": 0, "selector_bypassed_steps": 0, "latency_scope": "Completed execution wall (including evaluated zeros), including cache/API/tool work; original bank grader excluded. Infrastructure failures excluded from latency summaries but retained in attempts and fees.", "served_models": { "google/gemini-3.8-flash": { "response_observations": 27, "valid_response_observations": 24, "application_cache_hits": 0, "application_cache_denominator": 24, "physical_responses": 27, "provider_error_responses": 3, "actual_cost_usd": 0.5252597999999999, "provider_cache_metadata_responses": 27, "provider_cache_hit_responses": 15, "provider_cached_input_tokens": 386114, "provider_reported_prompt_tokens": 1038509, "physical_api_seconds": 79.43196799806901, "elapsed_response_seconds": 80.0914529570291, "backoff_seconds": 0 } }, "format_failure_clarifications": 0 }, { "phase": "warm", "domain": "banking_knowledge", "arm": "actor-only", "planned": 4, "attempted": 4, "scored": 3, "successful_executions": 3, "mean_score": 0.6666666666666666, "latency_median_seconds": 9.378686125011882, "latency_p90_seconds": 15.091517091396963, "cost_per_attempt_usd": 0.0, "calls_per_attempt": 0.0, "metering": { "response_observations": 104, "valid_response_observations": 104, "application_cache_hits": 104, "application_cache_denominator": 104, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.42828741106495727, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 85, "valid_response_observations": 85, "application_cache_hits": 85, "application_cache_denominator": 85, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.3487680750549771, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 1, "application_cache_denominator": 1, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.00219462500535883, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 18, "valid_response_observations": 18, "application_cache_hits": 18, "application_cache_denominator": 18, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.07732471100462135, "backoff_seconds": 0 } }, "controller_steps": 83, "reviewed_steps": 0, "selector_bypassed_steps": 83, "latency_scope": "Completed execution wall (including evaluated zeros), including cache/API/tool work; original bank grader excluded. Infrastructure failures excluded from latency summaries but retained in attempts and fees.", "served_models": { "google/gemini-3.8-flash": { "response_observations": 104, "valid_response_observations": 104, "application_cache_hits": 104, "application_cache_denominator": 104, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.42828741106495727, "backoff_seconds": 0 } }, "format_failure_clarifications": 0 }, { "phase": "warm", "domain": "banking_knowledge", "arm": "perplexity/pplx-decider-v1.1-27b", "planned": 4, "attempted": 4, "scored": 3, "successful_executions": 3, "mean_score": 0.3333333333333333, "latency_median_seconds": 12.43221604199789, "latency_p90_seconds": 12.485492341205827, "cost_per_attempt_usd": 0.0, "calls_per_attempt": 0.0, "metering": { "response_observations": 150, "valid_response_observations": 150, "application_cache_hits": 150, "application_cache_denominator": 150, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.5856163760181516, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 87, "valid_response_observations": 87, "application_cache_hits": 87, "application_cache_denominator": 87, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.32895679102512076, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 3, "application_cache_denominator": 3, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.009160083020105958, "backoff_seconds": 0 }, "decision": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.017824542985181324, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 28, "valid_response_observations": 28, "application_cache_hits": 28, "application_cache_denominator": 28, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.1338421269756509, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 18, "valid_response_observations": 18, "application_cache_hits": 18, "application_cache_denominator": 18, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.06328158301766962, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 8, "valid_response_observations": 8, "application_cache_hits": 8, "application_cache_denominator": 8, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.032551248994423077, "backoff_seconds": 0 } }, "controller_steps": 82, "reviewed_steps": 28, "selector_bypassed_steps": 76, "latency_scope": "Completed execution wall (including evaluated zeros), including cache/API/tool work; original bank grader excluded. Infrastructure failures excluded from latency summaries but retained in attempts and fees.", "served_models": { "google/gemini-3.8-flash": { "response_observations": 144, "valid_response_observations": 144, "application_cache_hits": 144, "application_cache_denominator": 144, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.5677918330329703, "backoff_seconds": 0 }, "perplexity/pplx-decider-v1.1-27b-20261006": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.017824542985181324, "backoff_seconds": 0 } }, "format_failure_clarifications": 0 }, { "phase": "warm", "domain": "banking_knowledge", "arm": "cloudflare/clef-flash", "planned": 4, "attempted": 4, "scored": 3, "successful_executions": 3, "mean_score": 0.6666666666666666, "latency_median_seconds": 27.533201083992026, "latency_p90_seconds": 42.64399605039216, "cost_per_attempt_usd": 0.0, "calls_per_attempt": 0.0, "metering": { "response_observations": 273, "valid_response_observations": 273, "application_cache_hits": 273, "application_cache_denominator": 273, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 1.0508962048770627, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 169, "valid_response_observations": 169, "application_cache_hits": 169, "application_cache_denominator": 169, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.6356747909012483, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 4, "valid_response_observations": 4, "application_cache_hits": 4, "application_cache_denominator": 4, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.01042433301336132, "backoff_seconds": 0 }, "decision": { "response_observations": 16, "valid_response_observations": 16, "application_cache_hits": 16, "application_cache_denominator": 16, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.07234933198196813, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 45, "valid_response_observations": 45, "application_cache_hits": 45, "application_cache_denominator": 45, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.16191970300860703, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 30, "valid_response_observations": 30, "application_cache_hits": 30, "application_cache_denominator": 30, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.12868946098024026, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 9, "valid_response_observations": 9, "application_cache_hits": 9, "application_cache_denominator": 9, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.04183858499163762, "backoff_seconds": 0 } }, "controller_steps": 151, "reviewed_steps": 45, "selector_bypassed_steps": 135, "latency_scope": "Completed execution wall (including evaluated zeros), including cache/API/tool work; original bank grader excluded. Infrastructure failures excluded from latency summaries but retained in attempts and fees.", "served_models": { "cloudflare/clef-flash": { "response_observations": 16, "valid_response_observations": 16, "application_cache_hits": 16, "application_cache_denominator": 16, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.07234933198196813, "backoff_seconds": 0 }, "google/gemini-3.8-flash": { "response_observations": 257, "valid_response_observations": 257, "application_cache_hits": 257, "application_cache_denominator": 257, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.9785468728950946, "backoff_seconds": 0 } }, "format_failure_clarifications": 2 }, { "phase": "warm", "domain": "ITBench", "arm": "actor-only", "planned": 4, "attempted": 4, "scored": 2, "successful_executions": 3, "mean_score": 0.5, "latency_median_seconds": 0.4411356250056997, "latency_p90_seconds": 0.5308959922054782, "cost_per_attempt_usd": 0.0, "calls_per_attempt": 0.0, "metering": { "response_observations": 17, "valid_response_observations": 17, "application_cache_hits": 17, "application_cache_denominator": 17, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.0693115839821985, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 17, "valid_response_observations": 17, "application_cache_hits": 17, "application_cache_denominator": 17, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.0693115839821985, "backoff_seconds": 0 } }, "controller_steps": 0, "reviewed_steps": 0, "selector_bypassed_steps": 0, "latency_scope": "Completed execution wall (including evaluated zeros), including cache/API/tool work; original bank grader excluded. Infrastructure failures excluded from latency summaries but retained in attempts and fees.", "served_models": { "google/gemini-3.8-flash": { "response_observations": 17, "valid_response_observations": 17, "application_cache_hits": 17, "application_cache_denominator": 17, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.0693115839821985, "backoff_seconds": 0 } }, "format_failure_clarifications": 0 }, { "phase": "warm", "domain": "ITBench", "arm": "perplexity/pplx-decider-v1.1-27b", "planned": 4, "attempted": 4, "scored": 1, "successful_executions": 2, "mean_score": 1.0, "latency_median_seconds": 0.46765850000520004, "latency_p90_seconds": 0.5046132664065226, "cost_per_attempt_usd": 0.0, "calls_per_attempt": 0.0, "metering": { "response_observations": 16, "valid_response_observations": 16, "application_cache_hits": 16, "application_cache_denominator": 16, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.040816957989591174, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 14, "valid_response_observations": 14, "application_cache_hits": 14, "application_cache_denominator": 14, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.03376304099219851, "backoff_seconds": 0 }, "it-critique": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.007053916997392662, "backoff_seconds": 0 } }, "controller_steps": 0, "reviewed_steps": 0, "selector_bypassed_steps": 0, "latency_scope": "Completed execution wall (including evaluated zeros), including cache/API/tool work; original bank grader excluded. Infrastructure failures excluded from latency summaries but retained in attempts and fees.", "served_models": { "google/gemini-3.8-flash": { "response_observations": 16, "valid_response_observations": 16, "application_cache_hits": 16, "application_cache_denominator": 16, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.040816957989591174, "backoff_seconds": 0 } }, "format_failure_clarifications": 0 }, { "phase": "warm", "domain": "ITBench", "arm": "cloudflare/clef-flash", "planned": 4, "attempted": 4, "scored": 1, "successful_executions": 1, "mean_score": 1.0, "latency_median_seconds": 0.48988591598754283, "latency_p90_seconds": 0.48988591598754283, "cost_per_attempt_usd": 0.0, "calls_per_attempt": 0.0, "metering": { "response_observations": 21, "valid_response_observations": 21, "application_cache_hits": 21, "application_cache_denominator": 21, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.04843137599527836, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 18, "valid_response_observations": 18, "application_cache_hits": 18, "application_cache_denominator": 18, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.04171075100020971, "backoff_seconds": 0 }, "it-critique": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 3, "application_cache_denominator": 3, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.006720624995068647, "backoff_seconds": 0 } }, "controller_steps": 0, "reviewed_steps": 0, "selector_bypassed_steps": 0, "latency_scope": "Completed execution wall (including evaluated zeros), including cache/API/tool work; original bank grader excluded. Infrastructure failures excluded from latency summaries but retained in attempts and fees.", "served_models": { "google/gemini-3.8-flash": { "response_observations": 21, "valid_response_observations": 21, "application_cache_hits": 21, "application_cache_denominator": 21, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.04843137599527836, "backoff_seconds": 0 } }, "format_failure_clarifications": 0 } ], "paired_comparisons": [ { "domain": "banking_knowledge", "arm": "perplexity/pplx-decider-v1.1-27b", "complete_pairs": 4, "unique_tasks": 4, "actor_mean_score": 0.5, "mithril_mean_score": 0.25, "mean_score_difference": -0.25, "improved": 0, "regressed": 1, "unchanged": 3, "task_cluster_bootstrap_95_percent_interval": [ -0.75, 0.0 ], "uncertainty": "Paired task-cluster percentile bootstrap, 5000 resamples, fixed seed 610. Each sampled task retains all its scored task/seed pairs, including repeated draws. Unscored pairs remain excluded. Small public slice, not a leaderboard rank.", "actor_latency_median_seconds": 99.59311625000555, "mithril_latency_median_seconds": 153.53178097899945, "actor_cost_usd": 1.12461045, "mithril_cost_usd": 1.55344541 }, { "domain": "banking_knowledge", "arm": "cloudflare/clef-flash", "complete_pairs": 4, "unique_tasks": 4, "actor_mean_score": 0.5, "mithril_mean_score": 0.5, "mean_score_difference": 0.0, "improved": 1, "regressed": 1, "unchanged": 2, "task_cluster_bootstrap_95_percent_interval": [ -0.75, 0.75 ], "uncertainty": "Paired task-cluster percentile bootstrap, 5000 resamples, fixed seed 610. Each sampled task retains all its scored task/seed pairs, including repeated draws. Unscored pairs remain excluded. Small public slice, not a leaderboard rank.", "actor_latency_median_seconds": 99.59311625000555, "mithril_latency_median_seconds": 209.02744139599963, "actor_cost_usd": 1.12461045, "mithril_cost_usd": 3.342391699 }, { "domain": "ITBench", "arm": "perplexity/pplx-decider-v1.1-27b", "complete_pairs": 2, "unique_tasks": 2, "actor_mean_score": 0.5, "mithril_mean_score": 0.5, "mean_score_difference": 0.0, "improved": 0, "regressed": 0, "unchanged": 2, "task_cluster_bootstrap_95_percent_interval": [ 0.0, 0.0 ], "uncertainty": "Paired task-cluster percentile bootstrap, 5000 resamples, fixed seed 610. Each sampled task retains all its scored task/seed pairs, including repeated draws. Unscored pairs remain excluded. Small public slice, not a leaderboard rank.", "actor_latency_median_seconds": 14.757413729508698, "mithril_latency_median_seconds": 20.956075312504254, "actor_cost_usd": 0.23332755, "mithril_cost_usd": 0.27044565 }, { "domain": "ITBench", "arm": "cloudflare/clef-flash", "complete_pairs": 3, "unique_tasks": 3, "actor_mean_score": 0.3333333333333333, "mithril_mean_score": 0.3333333333333333, "mean_score_difference": 0.0, "improved": 0, "regressed": 0, "unchanged": 3, "task_cluster_bootstrap_95_percent_interval": [ 0.0, 0.0 ], "uncertainty": "Paired task-cluster percentile bootstrap, 5000 resamples, fixed seed 610. Each sampled task retains all its scored task/seed pairs, including repeated draws. Unscored pairs remain excluded. Small public slice, not a leaderboard rank.", "actor_latency_median_seconds": 15.11749650001002, "mithril_latency_median_seconds": 18.671605667012045, "actor_cost_usd": 0.33493335, "mithril_cost_usd": 0.3946545 } ], "episodes": [ { "domain": "banking_knowledge", "task": "task_034", "seed": 612, "arm": "actor-only", "phase": "primary", "score": 1.0, "status": 0, "episode_seconds": 49.69333579200611, "controller_wall_seconds": 53.08515037500183, "termination": "user_stop", "grade_status": null, "grade_error": null, "error_type": null, "metering": { "response_observations": 12, "valid_response_observations": 12, "application_cache_hits": 0, "application_cache_denominator": 12, "physical_responses": 12, "provider_error_responses": 0, "actual_cost_usd": 0.058032, "provider_cache_metadata_responses": 12, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 63681, "physical_api_seconds": 47.203502956966986, "elapsed_response_seconds": 47.48941445698438, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 0, "application_cache_denominator": 6, "physical_responses": 6, "provider_error_responses": 0, "actual_cost_usd": 0.041937, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 52506, "physical_api_seconds": 20.35351933297352, "elapsed_response_seconds": 20.501716500002658, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 0, "application_cache_denominator": 6, "physical_responses": 6, "provider_error_responses": 0, "actual_cost_usd": 0.016094999999999998, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 11175, "physical_api_seconds": 26.849983623993467, "elapsed_response_seconds": 26.98769795698172, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 12, "valid_response_observations": 12, "application_cache_hits": 0, "application_cache_denominator": 12, "physical_responses": 12, "provider_error_responses": 0, "actual_cost_usd": 0.058032, "provider_cache_metadata_responses": 12, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 63681, "physical_api_seconds": 47.203502956966986, "elapsed_response_seconds": 47.48941445698438, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 6, "reviewed_steps": 0, "selector_bypassed_steps": 6 }, { "domain": "banking_knowledge", "task": "task_034", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "primary", "score": 1.0, "status": 0, "episode_seconds": 58.073104833005345, "controller_wall_seconds": 60.74511316700955, "termination": "user_stop", "grade_status": null, "grade_error": null, "error_type": null, "metering": { "response_observations": 17, "valid_response_observations": 17, "application_cache_hits": 0, "application_cache_denominator": 17, "physical_responses": 17, "provider_error_responses": 0, "actual_cost_usd": 0.09095625, "provider_cache_metadata_responses": 17, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 107945, "physical_api_seconds": 55.271812003993546, "elapsed_response_seconds": 55.67979858194303, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 0, "application_cache_denominator": 6, "physical_responses": 6, "provider_error_responses": 0, "actual_cost_usd": 0.04196775, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 53082, "physical_api_seconds": 17.76858091900067, "elapsed_response_seconds": 17.946767372981412, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 4, "valid_response_observations": 4, "application_cache_hits": 0, "application_cache_denominator": 4, "physical_responses": 4, "provider_error_responses": 0, "actual_cost_usd": 0.030768, "provider_cache_metadata_responses": 4, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 39404, "physical_api_seconds": 11.257218791986816, "elapsed_response_seconds": 11.325124206981855, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 0, "application_cache_denominator": 6, "physical_responses": 6, "provider_error_responses": 0, "actual_cost_usd": 0.013410749999999999, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 10666, "physical_api_seconds": 22.19353958501597, "elapsed_response_seconds": 22.318358126984094, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 0.00480975, "provider_cache_metadata_responses": 1, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 4793, "physical_api_seconds": 4.052472707990091, "elapsed_response_seconds": 4.089548874995671, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 17, "valid_response_observations": 17, "application_cache_hits": 0, "application_cache_denominator": 17, "physical_responses": 17, "provider_error_responses": 0, "actual_cost_usd": 0.09095625, "provider_cache_metadata_responses": 17, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 107945, "physical_api_seconds": 55.271812003993546, "elapsed_response_seconds": 55.67979858194303, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 6, "reviewed_steps": 4, "selector_bypassed_steps": 6 }, { "domain": "banking_knowledge", "task": "task_034", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "primary", "score": 1.0, "status": 0, "episode_seconds": 140.22660195900244, "controller_wall_seconds": 143.49530616699485, "termination": "user_stop", "grade_status": null, "grade_error": null, "error_type": null, "metering": { "response_observations": 43, "valid_response_observations": 43, "application_cache_hits": 0, "application_cache_denominator": 43, "physical_responses": 43, "provider_error_responses": 0, "actual_cost_usd": 0.26648661, "provider_cache_metadata_responses": 42, "provider_cache_hit_responses": 1, "provider_cached_input_tokens": 2786, "provider_reported_prompt_tokens": 333361, "physical_api_seconds": 130.55662308592582, "elapsed_response_seconds": 132.33147146200645, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 12, "valid_response_observations": 12, "application_cache_hits": 0, "application_cache_denominator": 12, "physical_responses": 12, "provider_error_responses": 0, "actual_cost_usd": 0.11344125, "provider_cache_metadata_responses": 12, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 143640, "physical_api_seconds": 38.83143600198673, "elapsed_response_seconds": 39.12116320802306, "backoff_seconds": 0 }, "decision": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 4.641e-05, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 7.187391125000431, "elapsed_response_seconds": 7.232056333989021, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 10, "valid_response_observations": 10, "application_cache_hits": 0, "application_cache_denominator": 10, "physical_responses": 10, "provider_error_responses": 0, "actual_cost_usd": 0.10019325, "provider_cache_metadata_responses": 10, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 127811, "physical_api_seconds": 29.05765845697897, "elapsed_response_seconds": 29.425219167984324, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 18, "valid_response_observations": 18, "application_cache_hits": 0, "application_cache_denominator": 18, "physical_responses": 18, "provider_error_responses": 0, "actual_cost_usd": 0.03766125, "provider_cache_metadata_responses": 18, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 45520, "physical_api_seconds": 43.48303679397213, "elapsed_response_seconds": 44.47945479401096, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 0, "application_cache_denominator": 2, "physical_responses": 2, "provider_error_responses": 0, "actual_cost_usd": 0.015144449999999999, "provider_cache_metadata_responses": 2, "provider_cache_hit_responses": 1, "provider_cached_input_tokens": 2786, "provider_reported_prompt_tokens": 16390, "physical_api_seconds": 11.997100707987556, "elapsed_response_seconds": 12.073577957999078, "backoff_seconds": 0 } }, "served_models": { "cloudflare/clef-flash": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 4.641e-05, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 7.187391125000431, "elapsed_response_seconds": 7.232056333989021, "backoff_seconds": 0 }, "google/gemini-3.8-flash": { "response_observations": 42, "valid_response_observations": 42, "application_cache_hits": 0, "application_cache_denominator": 42, "physical_responses": 42, "provider_error_responses": 0, "actual_cost_usd": 0.2664402, "provider_cache_metadata_responses": 42, "provider_cache_hit_responses": 1, "provider_cached_input_tokens": 2786, "provider_reported_prompt_tokens": 333361, "physical_api_seconds": 123.36923196092539, "elapsed_response_seconds": 125.09941512801743, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 12, "reviewed_steps": 10, "selector_bypassed_steps": 11 }, { "domain": "banking_knowledge", "task": "task_089", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "primary", "score": 0.0, "status": 0, "episode_seconds": 134.37759916599316, "controller_wall_seconds": 138.5116305420088, "termination": "user_stop", "grade_status": null, "grade_error": null, "error_type": null, "metering": { "response_observations": 39, "valid_response_observations": 37, "application_cache_hits": 0, "application_cache_denominator": 37, "physical_responses": 39, "provider_error_responses": 2, "actual_cost_usd": 0.30541317, "provider_cache_metadata_responses": 38, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 381624, "physical_api_seconds": 122.22944983097841, "elapsed_response_seconds": 124.10255866601074, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 19, "valid_response_observations": 17, "application_cache_hits": 0, "application_cache_denominator": 17, "physical_responses": 19, "provider_error_responses": 2, "actual_cost_usd": 0.15828825, "provider_cache_metadata_responses": 19, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 200801, "physical_api_seconds": 61.624459956001374, "elapsed_response_seconds": 62.709130748029565, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 0.00617025, "provider_cache_metadata_responses": 1, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 8142, "physical_api_seconds": 2.096523457992589, "elapsed_response_seconds": 2.1288270419900073, "backoff_seconds": 0 }, "decision": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 3.642e-05, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0.6214596670033643, "elapsed_response_seconds": 0.6457019579975167, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 9, "valid_response_observations": 9, "application_cache_hits": 0, "application_cache_denominator": 9, "physical_responses": 9, "provider_error_responses": 0, "actual_cost_usd": 0.095061, "provider_cache_metadata_responses": 9, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 122468, "physical_api_seconds": 27.215549040978658, "elapsed_response_seconds": 27.579915916969185, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 0, "application_cache_denominator": 6, "physical_responses": 6, "provider_error_responses": 0, "actual_cost_usd": 0.013459500000000001, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 16786, "physical_api_seconds": 12.386336124007357, "elapsed_response_seconds": 12.669751626017387, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 0, "application_cache_denominator": 3, "physical_responses": 3, "provider_error_responses": 0, "actual_cost_usd": 0.03239775, "provider_cache_metadata_responses": 3, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 33427, "physical_api_seconds": 18.285121584995068, "elapsed_response_seconds": 18.369231375007075, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 38, "valid_response_observations": 36, "application_cache_hits": 0, "application_cache_denominator": 36, "physical_responses": 38, "provider_error_responses": 2, "actual_cost_usd": 0.30537675, "provider_cache_metadata_responses": 38, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 381624, "physical_api_seconds": 121.60799016397505, "elapsed_response_seconds": 123.45685670801322, "backoff_seconds": 0 }, "perplexity/pplx-decider-v1.1-27b-20261006": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 3.642e-05, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0.6214596670033643, "elapsed_response_seconds": 0.6457019579975167, "backoff_seconds": 0 } }, "format_failure_clarifications": 1, "controller_steps": 18, "reviewed_steps": 9, "selector_bypassed_steps": 17 }, { "domain": "banking_knowledge", "task": "task_089", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "primary", "score": 0.0, "status": 0, "episode_seconds": 120.3860446250037, "controller_wall_seconds": 122.69416675000684, "termination": "user_stop", "grade_status": null, "grade_error": null, "error_type": null, "metering": { "response_observations": 36, "valid_response_observations": 34, "application_cache_hits": 0, "application_cache_denominator": 34, "physical_responses": 36, "provider_error_responses": 2, "actual_cost_usd": 0.271173212, "provider_cache_metadata_responses": 35, "provider_cache_hit_responses": 2, "provider_cached_input_tokens": 5292, "provider_reported_prompt_tokens": 347564, "physical_api_seconds": 104.94778020602826, "elapsed_response_seconds": 107.11395199692925, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 20, "valid_response_observations": 18, "application_cache_hits": 0, "application_cache_denominator": 18, "physical_responses": 20, "provider_error_responses": 2, "actual_cost_usd": 0.15736065, "provider_cache_metadata_responses": 20, "provider_cache_hit_responses": 2, "provider_cached_input_tokens": 5292, "provider_reported_prompt_tokens": 208307, "physical_api_seconds": 56.72931041599077, "elapsed_response_seconds": 57.92484116394189, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 0.0061695, "provider_cache_metadata_responses": 1, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 8066, "physical_api_seconds": 2.577325667007244, "elapsed_response_seconds": 2.59457062499132, "backoff_seconds": 0 }, "decision": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 3.6062e-05, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0.8605431250034599, "elapsed_response_seconds": 0.9045049170090351, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 7, "valid_response_observations": 7, "application_cache_hits": 0, "application_cache_denominator": 7, "physical_responses": 7, "provider_error_responses": 0, "actual_cost_usd": 0.068916, "provider_cache_metadata_responses": 7, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 88228, "physical_api_seconds": 20.765369041022495, "elapsed_response_seconds": 21.307747915998334, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 4, "valid_response_observations": 4, "application_cache_hits": 0, "application_cache_denominator": 4, "physical_responses": 4, "provider_error_responses": 0, "actual_cost_usd": 0.00766125, "provider_cache_metadata_responses": 4, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 9365, "physical_api_seconds": 8.524442249006825, "elapsed_response_seconds": 8.644381249992875, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 0, "application_cache_denominator": 3, "physical_responses": 3, "provider_error_responses": 0, "actual_cost_usd": 0.031029750000000002, "provider_cache_metadata_responses": 3, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 33598, "physical_api_seconds": 15.490789707997465, "elapsed_response_seconds": 15.737906124995789, "backoff_seconds": 0 } }, "served_models": { "cloudflare/clef-flash": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 3.6062e-05, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0.8605431250034599, "elapsed_response_seconds": 0.9045049170090351, "backoff_seconds": 0 }, "google/gemini-3.8-flash": { "response_observations": 35, "valid_response_observations": 33, "application_cache_hits": 0, "application_cache_denominator": 33, "physical_responses": 35, "provider_error_responses": 2, "actual_cost_usd": 0.27113715, "provider_cache_metadata_responses": 35, "provider_cache_hit_responses": 2, "provider_cached_input_tokens": 5292, "provider_reported_prompt_tokens": 347564, "physical_api_seconds": 104.0872370810248, "elapsed_response_seconds": 106.20944707992021, "backoff_seconds": 0 } }, "format_failure_clarifications": 1, "controller_steps": 19, "reviewed_steps": 7, "selector_bypassed_steps": 18 }, { "domain": "banking_knowledge", "task": "task_089", "seed": 612, "arm": "actor-only", "phase": "primary", "score": 1.0, "status": 0, "episode_seconds": 85.70217891701031, "controller_wall_seconds": 87.95040516700828, "termination": "user_stop", "grade_status": null, "grade_error": null, "error_type": null, "metering": { "response_observations": 27, "valid_response_observations": 27, "application_cache_hits": 0, "application_cache_denominator": 27, "physical_responses": 27, "provider_error_responses": 0, "actual_cost_usd": 0.21470325, "provider_cache_metadata_responses": 27, "provider_cache_hit_responses": 3, "provider_cached_input_tokens": 36830, "provider_reported_prompt_tokens": 307108, "physical_api_seconds": 76.60829862601531, "elapsed_response_seconds": 77.61920170504891, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 22, "valid_response_observations": 22, "application_cache_hits": 0, "application_cache_denominator": 22, "physical_responses": 22, "provider_error_responses": 0, "actual_cost_usd": 0.20351775, "provider_cache_metadata_responses": 22, "provider_cache_hit_responses": 3, "provider_cached_input_tokens": 36830, "provider_reported_prompt_tokens": 293374, "physical_api_seconds": 64.58491633400263, "elapsed_response_seconds": 65.43541195504076, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.011185500000000001, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 13734, "physical_api_seconds": 12.023382292012684, "elapsed_response_seconds": 12.183789750008145, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 27, "valid_response_observations": 27, "application_cache_hits": 0, "application_cache_denominator": 27, "physical_responses": 27, "provider_error_responses": 0, "actual_cost_usd": 0.21470325, "provider_cache_metadata_responses": 27, "provider_cache_hit_responses": 3, "provider_cached_input_tokens": 36830, "provider_reported_prompt_tokens": 307108, "physical_api_seconds": 76.60829862601531, "elapsed_response_seconds": 77.61920170504891, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 22, "reviewed_steps": 0, "selector_bypassed_steps": 22 }, { "domain": "banking_knowledge", "task": "task_095", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "primary", "score": 1.0, "status": 0, "episode_seconds": 440.6259823750006, "controller_wall_seconds": 444.53040720800345, "termination": "user_stop", "grade_status": null, "grade_error": null, "error_type": null, "metering": { "response_observations": 137, "valid_response_observations": 137, "application_cache_hits": 0, "application_cache_denominator": 137, "physical_responses": 137, "provider_error_responses": 0, "actual_cost_usd": 1.784775976, "provider_cache_metadata_responses": 125, "provider_cache_hit_responses": 9, "provider_cached_input_tokens": 101593, "provider_reported_prompt_tokens": 2418519, "physical_api_seconds": 399.42153050005436, "elapsed_response_seconds": 403.6261602119339, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 88, "valid_response_observations": 88, "application_cache_hits": 0, "application_cache_denominator": 88, "physical_responses": 88, "provider_error_responses": 0, "actual_cost_usd": 1.259191575, "provider_cache_metadata_responses": 88, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 100121, "provider_reported_prompt_tokens": 1747556, "physical_api_seconds": 250.59451241705392, "elapsed_response_seconds": 253.18235429194465, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 4, "valid_response_observations": 4, "application_cache_hits": 0, "application_cache_denominator": 4, "physical_responses": 4, "provider_error_responses": 0, "actual_cost_usd": 0.06284475, "provider_cache_metadata_responses": 4, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 81533, "physical_api_seconds": 12.678770373997395, "elapsed_response_seconds": 12.767078291988582, "backoff_seconds": 0 }, "decision": { "response_observations": 12, "valid_response_observations": 12, "application_cache_hits": 0, "application_cache_denominator": 12, "physical_responses": 12, "provider_error_responses": 0, "actual_cost_usd": 0.000503251, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 19.18070120998891, "elapsed_response_seconds": 19.473317998985294, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 24, "valid_response_observations": 24, "application_cache_hits": 0, "application_cache_denominator": 24, "physical_responses": 24, "provider_error_responses": 0, "actual_cost_usd": 0.41904765, "provider_cache_metadata_responses": 24, "provider_cache_hit_responses": 4, "provider_cached_input_tokens": 1472, "provider_reported_prompt_tokens": 541460, "physical_api_seconds": 84.0676970820059, "elapsed_response_seconds": 85.06690917004016, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 0, "application_cache_denominator": 6, "physical_responses": 6, "provider_error_responses": 0, "actual_cost_usd": 0.01017225, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 12358, "physical_api_seconds": 16.153215792000992, "elapsed_response_seconds": 16.280773291975493, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 0, "application_cache_denominator": 3, "physical_responses": 3, "provider_error_responses": 0, "actual_cost_usd": 0.0330165, "provider_cache_metadata_responses": 3, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 35612, "physical_api_seconds": 16.746633625007235, "elapsed_response_seconds": 16.855727166999714, "backoff_seconds": 0 } }, "served_models": { "cloudflare/clef-flash": { "response_observations": 12, "valid_response_observations": 12, "application_cache_hits": 0, "application_cache_denominator": 12, "physical_responses": 12, "provider_error_responses": 0, "actual_cost_usd": 0.000503251, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 19.18070120998891, "elapsed_response_seconds": 19.473317998985294, "backoff_seconds": 0 }, "google/gemini-3.8-flash": { "response_observations": 125, "valid_response_observations": 125, "application_cache_hits": 0, "application_cache_denominator": 125, "physical_responses": 125, "provider_error_responses": 0, "actual_cost_usd": 1.7842727250000001, "provider_cache_metadata_responses": 125, "provider_cache_hit_responses": 9, "provider_cached_input_tokens": 101593, "provider_reported_prompt_tokens": 2418519, "physical_api_seconds": 380.24082929006545, "elapsed_response_seconds": 384.1528422129486, "backoff_seconds": 0 } }, "format_failure_clarifications": 2, "controller_steps": 76, "reviewed_steps": 24, "selector_bypassed_steps": 64 }, { "domain": "banking_knowledge", "task": "task_095", "seed": 612, "arm": "actor-only", "phase": "primary", "score": 0.0, "status": 0, "episode_seconds": 113.48405358300079, "controller_wall_seconds": 117.54206058300042, "termination": "user_stop", "grade_status": null, "grade_error": null, "error_type": null, "metering": { "response_observations": 37, "valid_response_observations": 35, "application_cache_hits": 0, "application_cache_denominator": 35, "physical_responses": 37, "provider_error_responses": 2, "actual_cost_usd": 0.272343375, "provider_cache_metadata_responses": 37, "provider_cache_hit_responses": 3, "provider_cached_input_tokens": 45295, "provider_reported_prompt_tokens": 392225, "physical_api_seconds": 102.4896764580335, "elapsed_response_seconds": 103.26257141600945, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 29, "valid_response_observations": 27, "application_cache_hits": 0, "application_cache_denominator": 27, "physical_responses": 29, "provider_error_responses": 2, "actual_cost_usd": 0.25663162500000003, "provider_cache_metadata_responses": 29, "provider_cache_hit_responses": 3, "provider_cached_input_tokens": 45295, "provider_reported_prompt_tokens": 372326, "physical_api_seconds": 84.79880304200924, "elapsed_response_seconds": 85.42847537598573, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 0.00522075, "provider_cache_metadata_responses": 1, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 6866, "physical_api_seconds": 2.2634321250079665, "elapsed_response_seconds": 2.282366290994105, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 7, "valid_response_observations": 7, "application_cache_hits": 0, "application_cache_denominator": 7, "physical_responses": 7, "provider_error_responses": 0, "actual_cost_usd": 0.010491, "provider_cache_metadata_responses": 7, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 13033, "physical_api_seconds": 15.427441291016294, "elapsed_response_seconds": 15.551729749029619, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 37, "valid_response_observations": 35, "application_cache_hits": 0, "application_cache_denominator": 35, "physical_responses": 37, "provider_error_responses": 2, "actual_cost_usd": 0.272343375, "provider_cache_metadata_responses": 37, "provider_cache_hit_responses": 3, "provider_cached_input_tokens": 45295, "provider_reported_prompt_tokens": 392225, "physical_api_seconds": 102.4896764580335, "elapsed_response_seconds": 103.26257141600945, "backoff_seconds": 0 } }, "format_failure_clarifications": 1, "controller_steps": 28, "reviewed_steps": 0, "selector_bypassed_steps": 28 }, { "domain": "banking_knowledge", "task": "task_095", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "primary", "score": 0.0, "status": 0, "episode_seconds": 207.1141266249906, "controller_wall_seconds": 217.40946379199158, "termination": "user_stop", "grade_status": null, "grade_error": null, "error_type": null, "metering": { "response_observations": 67, "valid_response_observations": 67, "application_cache_hits": 0, "application_cache_denominator": 67, "physical_responses": 67, "provider_error_responses": 0, "actual_cost_usd": 0.64153321, "provider_cache_metadata_responses": 63, "provider_cache_hit_responses": 10, "provider_cached_input_tokens": 75800, "provider_reported_prompt_tokens": 894900, "physical_api_seconds": 188.77859996295592, "elapsed_response_seconds": 190.67292791702494, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 43, "valid_response_observations": 43, "application_cache_hits": 0, "application_cache_denominator": 43, "physical_responses": 43, "provider_error_responses": 0, "actual_cost_usd": 0.4275966, "provider_cache_metadata_responses": 43, "provider_cache_hit_responses": 6, "provider_cached_input_tokens": 74328, "provider_reported_prompt_tokens": 627919, "physical_api_seconds": 111.47045908294967, "elapsed_response_seconds": 112.55123512599675, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 0.01227, "provider_cache_metadata_responses": 1, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 15830, "physical_api_seconds": 3.369297208992066, "elapsed_response_seconds": 3.389260124997236, "backoff_seconds": 0 }, "decision": { "response_observations": 4, "valid_response_observations": 4, "application_cache_hits": 0, "application_cache_denominator": 4, "physical_responses": 4, "provider_error_responses": 0, "actual_cost_usd": 0.00016696, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 2.899387501005549, "elapsed_response_seconds": 2.9596156240149867, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 13, "valid_response_observations": 13, "application_cache_hits": 0, "application_cache_denominator": 13, "physical_responses": 13, "provider_error_responses": 0, "actual_cost_usd": 0.16876065, "provider_cache_metadata_responses": 13, "provider_cache_hit_responses": 4, "provider_cached_input_tokens": 1472, "provider_reported_prompt_tokens": 216144, "physical_api_seconds": 47.64848908501153, "elapsed_response_seconds": 48.1916848350229, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 0, "application_cache_denominator": 3, "physical_responses": 3, "provider_error_responses": 0, "actual_cost_usd": 0.004427250000000001, "provider_cache_metadata_responses": 3, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 5028, "physical_api_seconds": 7.22063729299407, "elapsed_response_seconds": 7.295792541000992, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 0, "application_cache_denominator": 3, "physical_responses": 3, "provider_error_responses": 0, "actual_cost_usd": 0.02831175, "provider_cache_metadata_responses": 3, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 29979, "physical_api_seconds": 16.17032979200303, "elapsed_response_seconds": 16.28533966599207, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 63, "valid_response_observations": 63, "application_cache_hits": 0, "application_cache_denominator": 63, "physical_responses": 63, "provider_error_responses": 0, "actual_cost_usd": 0.64136625, "provider_cache_metadata_responses": 63, "provider_cache_hit_responses": 10, "provider_cached_input_tokens": 75800, "provider_reported_prompt_tokens": 894900, "physical_api_seconds": 185.87921246195037, "elapsed_response_seconds": 187.71331229300995, "backoff_seconds": 0 }, "perplexity/pplx-decider-v1.1-27b-20261006": { "response_observations": 4, "valid_response_observations": 4, "application_cache_hits": 0, "application_cache_denominator": 4, "physical_responses": 4, "provider_error_responses": 0, "actual_cost_usd": 0.00016696, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 2.899387501005549, "elapsed_response_seconds": 2.9596156240149867, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 40, "reviewed_steps": 13, "selector_bypassed_steps": 36 }, { "domain": "banking_knowledge", "task": "task_093", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "primary", "score": 0.0, "status": 0, "episode_seconds": 277.8282808329968, "controller_wall_seconds": 280.5316693749919, "termination": "user_stop", "grade_status": null, "grade_error": null, "error_type": null, "metering": { "response_observations": 86, "valid_response_observations": 86, "application_cache_hits": 0, "application_cache_denominator": 86, "physical_responses": 86, "provider_error_responses": 0, "actual_cost_usd": 1.019955901, "provider_cache_metadata_responses": 83, "provider_cache_hit_responses": 3, "provider_cached_input_tokens": 11766, "provider_reported_prompt_tokens": 1337677, "physical_api_seconds": 249.15260366398434, "elapsed_response_seconds": 251.65403387797414, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 66, "valid_response_observations": 66, "application_cache_hits": 0, "application_cache_denominator": 66, "physical_responses": 66, "provider_error_responses": 0, "actual_cost_usd": 0.82882095, "provider_cache_metadata_responses": 66, "provider_cache_hit_responses": 3, "provider_cached_input_tokens": 11766, "provider_reported_prompt_tokens": 1100004, "physical_api_seconds": 185.46763695600384, "elapsed_response_seconds": 187.50100675001158, "backoff_seconds": 0 }, "decision": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 0, "application_cache_denominator": 3, "physical_responses": 3, "provider_error_responses": 0, "actual_cost_usd": 0.000178951, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 4.036803625000175, "elapsed_response_seconds": 4.094471125979908, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 10, "valid_response_observations": 10, "application_cache_hits": 0, "application_cache_denominator": 10, "physical_responses": 10, "provider_error_responses": 0, "actual_cost_usd": 0.15351075, "provider_cache_metadata_responses": 10, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 197256, "physical_api_seconds": 32.69356054098171, "elapsed_response_seconds": 32.90940608497476, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 4, "valid_response_observations": 4, "application_cache_hits": 0, "application_cache_denominator": 4, "physical_responses": 4, "provider_error_responses": 0, "actual_cost_usd": 0.005736, "provider_cache_metadata_responses": 4, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 6613, "physical_api_seconds": 9.957887582990224, "elapsed_response_seconds": 10.048105333000422, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 0, "application_cache_denominator": 3, "physical_responses": 3, "provider_error_responses": 0, "actual_cost_usd": 0.03170925, "provider_cache_metadata_responses": 3, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 33804, "physical_api_seconds": 16.996714959008386, "elapsed_response_seconds": 17.101044584007468, "backoff_seconds": 0 } }, "served_models": { "cloudflare/clef-flash": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 0, "application_cache_denominator": 3, "physical_responses": 3, "provider_error_responses": 0, "actual_cost_usd": 0.000178951, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 4.036803625000175, "elapsed_response_seconds": 4.094471125979908, "backoff_seconds": 0 }, "google/gemini-3.8-flash": { "response_observations": 83, "valid_response_observations": 83, "application_cache_hits": 0, "application_cache_denominator": 83, "physical_responses": 83, "provider_error_responses": 0, "actual_cost_usd": 1.01977695, "provider_cache_metadata_responses": 83, "provider_cache_hit_responses": 3, "provider_cached_input_tokens": 11766, "provider_reported_prompt_tokens": 1337677, "physical_api_seconds": 245.11580003898416, "elapsed_response_seconds": 247.55956275199424, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 60, "reviewed_steps": 10, "selector_bypassed_steps": 57 }, { "domain": "banking_knowledge", "task": "task_093", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "primary", "score": 0.0, "status": 0, "episode_seconds": 172.68596279200574, "controller_wall_seconds": 183.56192379099957, "termination": "user_stop", "grade_status": null, "grade_error": null, "error_type": null, "metering": { "response_observations": 59, "valid_response_observations": 59, "application_cache_hits": 0, "application_cache_denominator": 59, "physical_responses": 59, "provider_error_responses": 0, "actual_cost_usd": 0.51554278, "provider_cache_metadata_responses": 57, "provider_cache_hit_responses": 4, "provider_cached_input_tokens": 23792, "provider_reported_prompt_tokens": 684249, "physical_api_seconds": 162.6050524579914, "elapsed_response_seconds": 164.19774066591344, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 35, "valid_response_observations": 35, "application_cache_hits": 0, "application_cache_denominator": 35, "physical_responses": 35, "provider_error_responses": 0, "actual_cost_usd": 0.346066575, "provider_cache_metadata_responses": 35, "provider_cache_hit_responses": 2, "provider_cached_input_tokens": 19841, "provider_reported_prompt_tokens": 471144, "physical_api_seconds": 93.41745353996521, "elapsed_response_seconds": 94.25624608792714, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 0, "application_cache_denominator": 2, "physical_responses": 2, "provider_error_responses": 0, "actual_cost_usd": 0.017865, "provider_cache_metadata_responses": 2, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 23050, "physical_api_seconds": 5.59023933399294, "elapsed_response_seconds": 5.632012209011009, "backoff_seconds": 0 }, "decision": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 0, "application_cache_denominator": 2, "physical_responses": 2, "provider_error_responses": 0, "actual_cost_usd": 8.188e-05, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 1.4844384159951005, "elapsed_response_seconds": 1.5617007079854375, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 10, "valid_response_observations": 10, "application_cache_hits": 0, "application_cache_denominator": 10, "physical_responses": 10, "provider_error_responses": 0, "actual_cost_usd": 0.11720077500000001, "provider_cache_metadata_responses": 10, "provider_cache_hit_responses": 1, "provider_cached_input_tokens": 367, "provider_reported_prompt_tokens": 149678, "physical_api_seconds": 31.65831054301816, "elapsed_response_seconds": 32.03087912200135, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 7, "valid_response_observations": 7, "application_cache_hits": 0, "application_cache_denominator": 7, "physical_responses": 7, "provider_error_responses": 0, "actual_cost_usd": 0.010221000000000001, "provider_cache_metadata_responses": 7, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 12498, "physical_api_seconds": 15.258088958013104, "elapsed_response_seconds": 15.448989914992126, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 0, "application_cache_denominator": 3, "physical_responses": 3, "provider_error_responses": 0, "actual_cost_usd": 0.02410755, "provider_cache_metadata_responses": 3, "provider_cache_hit_responses": 1, "provider_cached_input_tokens": 3584, "provider_reported_prompt_tokens": 27879, "physical_api_seconds": 15.196521667006891, "elapsed_response_seconds": 15.26791262399638, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 57, "valid_response_observations": 57, "application_cache_hits": 0, "application_cache_denominator": 57, "physical_responses": 57, "provider_error_responses": 0, "actual_cost_usd": 0.5154609, "provider_cache_metadata_responses": 57, "provider_cache_hit_responses": 4, "provider_cached_input_tokens": 23792, "provider_reported_prompt_tokens": 684249, "physical_api_seconds": 161.1206140419963, "elapsed_response_seconds": 162.636039957928, "backoff_seconds": 0 }, "perplexity/pplx-decider-v1.1-27b-20261006": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 0, "application_cache_denominator": 2, "physical_responses": 2, "provider_error_responses": 0, "actual_cost_usd": 8.188e-05, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 1.4844384159951005, "elapsed_response_seconds": 1.5617007079854375, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 33, "reviewed_steps": 10, "selector_bypassed_steps": 31 }, { "domain": "banking_knowledge", "task": "task_093", "seed": 612, "arm": "actor-only", "phase": "primary", "score": 0.0, "status": 0, "episode_seconds": 180.01511649999884, "controller_wall_seconds": 182.661533416991, "termination": "user_stop", "grade_status": null, "grade_error": null, "error_type": null, "metering": { "response_observations": 60, "valid_response_observations": 60, "application_cache_hits": 0, "application_cache_denominator": 60, "physical_responses": 60, "provider_error_responses": 0, "actual_cost_usd": 0.579531825, "provider_cache_metadata_responses": 60, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 79081, "provider_reported_prompt_tokens": 830407, "physical_api_seconds": 151.76737379499536, "elapsed_response_seconds": 158.69550903896743, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 54, "valid_response_observations": 54, "application_cache_hits": 0, "application_cache_denominator": 54, "physical_responses": 54, "provider_error_responses": 0, "actual_cost_usd": 0.561250575, "provider_cache_metadata_responses": 54, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 79081, "provider_reported_prompt_tokens": 807302, "physical_api_seconds": 138.41057916899445, "elapsed_response_seconds": 144.70964691399422, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 0.011508, "provider_cache_metadata_responses": 1, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 14849, "physical_api_seconds": 2.887163709005108, "elapsed_response_seconds": 3.1003069159924053, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.006773250000000001, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 8256, "physical_api_seconds": 10.469630916995811, "elapsed_response_seconds": 10.885555208980804, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 60, "valid_response_observations": 60, "application_cache_hits": 0, "application_cache_denominator": 60, "physical_responses": 60, "provider_error_responses": 0, "actual_cost_usd": 0.579531825, "provider_cache_metadata_responses": 60, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 79081, "provider_reported_prompt_tokens": 830407, "physical_api_seconds": 151.76737379499536, "elapsed_response_seconds": 158.69550903896743, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 52, "reviewed_steps": 0, "selector_bypassed_steps": 52 }, { "domain": "ITBench", "task": "Scenario-16", "seed": 612, "arm": "actor-only", "phase": "primary", "score": 1.0, "status": "prediction-complete", "episode_seconds": 14.397330959007377, "controller_wall_seconds": null, "termination": null, "grade_status": "scored", "grade_error": null, "error_type": null, "metering": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.08629875, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 3, "provider_cached_input_tokens": 123670, "provider_reported_prompt_tokens": 224558, "physical_api_seconds": 13.682272125035524, "elapsed_response_seconds": 13.781953916011844, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.08629875, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 3, "provider_cached_input_tokens": 123670, "provider_reported_prompt_tokens": 224558, "physical_api_seconds": 13.682272125035524, "elapsed_response_seconds": 13.781953916011844, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.08629875, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 3, "provider_cached_input_tokens": 123670, "provider_reported_prompt_tokens": 224558, "physical_api_seconds": 13.682272125035524, "elapsed_response_seconds": 13.781953916011844, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-16", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "primary", "score": 1.0, "status": "prediction-complete", "episode_seconds": 19.753461250002147, "controller_wall_seconds": null, "termination": null, "grade_status": "scored", "grade_error": null, "error_type": null, "metering": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 0, "application_cache_denominator": 6, "physical_responses": 6, "provider_error_responses": 0, "actual_cost_usd": 0.1798959, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 2112, "provider_reported_prompt_tokens": 239677, "physical_api_seconds": 19.06062187498901, "elapsed_response_seconds": 19.15300745899731, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.1627929, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 2112, "provider_reported_prompt_tokens": 217048, "physical_api_seconds": 16.5329362499906, "elapsed_response_seconds": 16.612722500998643, "backoff_seconds": 0 }, "it-critique": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 0.017103, "provider_cache_metadata_responses": 1, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 22629, "physical_api_seconds": 2.527685624998412, "elapsed_response_seconds": 2.540284957998665, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 0, "application_cache_denominator": 6, "physical_responses": 6, "provider_error_responses": 0, "actual_cost_usd": 0.1798959, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 2112, "provider_reported_prompt_tokens": 239677, "physical_api_seconds": 19.06062187498901, "elapsed_response_seconds": 19.15300745899731, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-16", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "primary", "score": 1.0, "status": "prediction-complete", "episode_seconds": 16.630522374995053, "controller_wall_seconds": null, "termination": null, "grade_status": "scored", "grade_error": null, "error_type": null, "metering": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 0, "application_cache_denominator": 6, "physical_responses": 6, "provider_error_responses": 0, "actual_cost_usd": 0.113881575, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 6, "provider_cached_input_tokens": 109151, "provider_reported_prompt_tokens": 247983, "physical_api_seconds": 16.042943249995005, "elapsed_response_seconds": 16.139341457019327, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.107581425, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 88549, "provider_reported_prompt_tokens": 221146, "physical_api_seconds": 13.677963416994317, "elapsed_response_seconds": 13.762254790010047, "backoff_seconds": 0 }, "it-critique": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 0.00630015, "provider_cache_metadata_responses": 1, "provider_cache_hit_responses": 1, "provider_cached_input_tokens": 20602, "provider_reported_prompt_tokens": 26837, "physical_api_seconds": 2.364979833000689, "elapsed_response_seconds": 2.37708666700928, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 0, "application_cache_denominator": 6, "physical_responses": 6, "provider_error_responses": 0, "actual_cost_usd": 0.113881575, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 6, "provider_cached_input_tokens": 109151, "provider_reported_prompt_tokens": 247983, "physical_api_seconds": 16.042943249995005, "elapsed_response_seconds": 16.139341457019327, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-17", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "primary", "score": 0.0, "status": "prediction-complete", "episode_seconds": 22.15868937500636, "controller_wall_seconds": null, "termination": null, "grade_status": "scored", "grade_error": null, "error_type": null, "metering": { "response_observations": 6, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 6, "provider_error_responses": 1, "actual_cost_usd": 0.09054975, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 2, "provider_cached_input_tokens": 80360, "provider_reported_prompt_tokens": 190227, "physical_api_seconds": 21.710838500992395, "elapsed_response_seconds": 21.81077699898742, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 4, "application_cache_hits": 0, "application_cache_denominator": 4, "physical_responses": 5, "provider_error_responses": 1, "actual_cost_usd": 0.075531, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 2, "provider_cached_input_tokens": 80360, "provider_reported_prompt_tokens": 170307, "physical_api_seconds": 18.07817141698615, "elapsed_response_seconds": 18.15114054098376, "backoff_seconds": 0 }, "it-critique": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 0.01501875, "provider_cache_metadata_responses": 1, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 19920, "physical_api_seconds": 3.6326670840062434, "elapsed_response_seconds": 3.6596364580036607, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 6, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 6, "provider_error_responses": 1, "actual_cost_usd": 0.09054975, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 2, "provider_cached_input_tokens": 80360, "provider_reported_prompt_tokens": 190227, "physical_api_seconds": 21.710838500992395, "elapsed_response_seconds": 21.81077699898742, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-17", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "primary", "score": 0.0, "status": "prediction-complete", "episode_seconds": 18.671605667012045, "controller_wall_seconds": null, "termination": null, "grade_status": "scored", "grade_error": null, "error_type": null, "metering": { "response_observations": 6, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 6, "provider_error_responses": 1, "actual_cost_usd": 0.120903525, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 4, "provider_cached_input_tokens": 44687, "provider_reported_prompt_tokens": 198773, "physical_api_seconds": 18.127619248014526, "elapsed_response_seconds": 18.21905675099697, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 4, "application_cache_hits": 0, "application_cache_denominator": 4, "physical_responses": 5, "provider_error_responses": 1, "actual_cost_usd": 0.10357177499999999, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 4, "provider_cached_input_tokens": 44687, "provider_reported_prompt_tokens": 175839, "physical_api_seconds": 15.814372040011222, "elapsed_response_seconds": 15.889505916988128, "backoff_seconds": 0 }, "it-critique": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 0.01733175, "provider_cache_metadata_responses": 1, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 22934, "physical_api_seconds": 2.313247208003304, "elapsed_response_seconds": 2.3295508340088418, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 6, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 6, "provider_error_responses": 1, "actual_cost_usd": 0.120903525, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 4, "provider_cached_input_tokens": 44687, "provider_reported_prompt_tokens": 198773, "physical_api_seconds": 18.127619248014526, "elapsed_response_seconds": 18.21905675099697, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-17", "seed": 612, "arm": "actor-only", "phase": "primary", "score": 0.0, "status": "prediction-complete", "episode_seconds": 15.11749650001002, "controller_wall_seconds": null, "termination": null, "grade_status": "scored", "grade_error": null, "error_type": null, "metering": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.1470288, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 4, "provider_cached_input_tokens": 44794, "provider_reported_prompt_tokens": 234088, "physical_api_seconds": 14.571997707986156, "elapsed_response_seconds": 14.640328958019381, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.1470288, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 4, "provider_cached_input_tokens": 44794, "provider_reported_prompt_tokens": 234088, "physical_api_seconds": 14.571997707986156, "elapsed_response_seconds": 14.640328958019381, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.1470288, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 4, "provider_cached_input_tokens": 44794, "provider_reported_prompt_tokens": 234088, "physical_api_seconds": 14.571997707986156, "elapsed_response_seconds": 14.640328958019381, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-29", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "primary", "score": 0.0, "status": "prediction-complete", "episode_seconds": 25.195172749998164, "controller_wall_seconds": null, "termination": null, "grade_status": "scored", "grade_error": null, "error_type": null, "metering": { "response_observations": 8, "valid_response_observations": 7, "application_cache_hits": 0, "application_cache_denominator": 7, "physical_responses": 8, "provider_error_responses": 1, "actual_cost_usd": 0.1598694, "provider_cache_metadata_responses": 8, "provider_cache_hit_responses": 2, "provider_cached_input_tokens": 83022, "provider_reported_prompt_tokens": 285804, "physical_api_seconds": 23.77325250004651, "elapsed_response_seconds": 24.030322667007567, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 0, "application_cache_denominator": 6, "physical_responses": 6, "provider_error_responses": 0, "actual_cost_usd": 0.14194215000000002, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 2, "provider_cached_input_tokens": 83022, "provider_reported_prompt_tokens": 262756, "physical_api_seconds": 16.167413375034812, "elapsed_response_seconds": 16.37457104100031, "backoff_seconds": 0 }, "it-critique": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 0.01792725, "provider_cache_metadata_responses": 1, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 23048, "physical_api_seconds": 3.544536750006955, "elapsed_response_seconds": 3.568232083998737, "backoff_seconds": 0 }, "it-revise-alternatives": { "response_observations": 1, "valid_response_observations": 0, "application_cache_hits": 0, "application_cache_denominator": 0, "physical_responses": 1, "provider_error_responses": 1, "actual_cost_usd": 0, "provider_cache_metadata_responses": 1, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 4.061302375004743, "elapsed_response_seconds": 4.087519542008522, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 8, "valid_response_observations": 7, "application_cache_hits": 0, "application_cache_denominator": 7, "physical_responses": 8, "provider_error_responses": 1, "actual_cost_usd": 0.1598694, "provider_cache_metadata_responses": 8, "provider_cache_hit_responses": 2, "provider_cached_input_tokens": 83022, "provider_reported_prompt_tokens": 285804, "physical_api_seconds": 23.77325250004651, "elapsed_response_seconds": 24.030322667007567, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-29", "seed": 612, "arm": "actor-only", "phase": "primary", "score": 0.0, "status": "prediction-complete", "episode_seconds": 24.598071125001297, "controller_wall_seconds": null, "termination": null, "grade_status": "scored", "grade_error": null, "error_type": null, "metering": { "response_observations": 5, "valid_response_observations": 4, "application_cache_hits": 0, "application_cache_denominator": 4, "physical_responses": 5, "provider_error_responses": 1, "actual_cost_usd": 0.10160580000000001, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 4, "provider_cached_input_tokens": 43824, "provider_reported_prompt_tokens": 172451, "physical_api_seconds": 24.244075582973892, "elapsed_response_seconds": 24.318416793001234, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 4, "application_cache_hits": 0, "application_cache_denominator": 4, "physical_responses": 5, "provider_error_responses": 1, "actual_cost_usd": 0.10160580000000001, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 4, "provider_cached_input_tokens": 43824, "provider_reported_prompt_tokens": 172451, "physical_api_seconds": 24.244075582973892, "elapsed_response_seconds": 24.318416793001234, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 5, "valid_response_observations": 4, "application_cache_hits": 0, "application_cache_denominator": 4, "physical_responses": 5, "provider_error_responses": 1, "actual_cost_usd": 0.10160580000000001, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 4, "provider_cached_input_tokens": 43824, "provider_reported_prompt_tokens": 172451, "physical_api_seconds": 24.244075582973892, "elapsed_response_seconds": 24.318416793001234, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-29", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "primary", "score": null, "status": "execution-error", "episode_seconds": 20.25836716601043, "controller_wall_seconds": null, "termination": null, "grade_status": "execution-error", "grade_error": null, "error_type": "RuntimeError", "metering": { "response_observations": 6, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 6, "provider_error_responses": 1, "actual_cost_usd": 0.1334211, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 44208, "provider_reported_prompt_tokens": 216942, "physical_api_seconds": 19.047916125011398, "elapsed_response_seconds": 19.5656964580121, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 6, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 6, "provider_error_responses": 1, "actual_cost_usd": 0.1334211, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 44208, "provider_reported_prompt_tokens": 216942, "physical_api_seconds": 19.047916125011398, "elapsed_response_seconds": 19.5656964580121, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 6, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 6, "provider_error_responses": 1, "actual_cost_usd": 0.1334211, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 44208, "provider_reported_prompt_tokens": 216942, "physical_api_seconds": 19.047916125011398, "elapsed_response_seconds": 19.5656964580121, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-38", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "primary", "score": null, "status": "prediction-complete", "episode_seconds": 22.20501108300232, "controller_wall_seconds": null, "termination": null, "grade_status": "unsupported-or-ambiguous-label", "grade_error": "Unsupported original regular-expression filter syntax; no inferred wildcard correction. The entire case remains unscored for every arm", "error_type": null, "metering": { "response_observations": 7, "valid_response_observations": 6, "application_cache_hits": 0, "application_cache_denominator": 6, "physical_responses": 7, "provider_error_responses": 1, "actual_cost_usd": 0.1306053, "provider_cache_metadata_responses": 7, "provider_cache_hit_responses": 3, "provider_cached_input_tokens": 149254, "provider_reported_prompt_tokens": 305949, "physical_api_seconds": 21.48815300001297, "elapsed_response_seconds": 21.70273208200524, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.1045368, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 3, "provider_cached_input_tokens": 149254, "provider_reported_prompt_tokens": 271896, "physical_api_seconds": 14.447918334015412, "elapsed_response_seconds": 14.619701914998586, "backoff_seconds": 0 }, "it-critique": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 0.0260685, "provider_cache_metadata_responses": 1, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 34053, "physical_api_seconds": 3.2577354579989333, "elapsed_response_seconds": 3.2760213750007097, "backoff_seconds": 0 }, "it-revise-alternatives": { "response_observations": 1, "valid_response_observations": 0, "application_cache_hits": 0, "application_cache_denominator": 0, "physical_responses": 1, "provider_error_responses": 1, "actual_cost_usd": 0, "provider_cache_metadata_responses": 1, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 3.782499207998626, "elapsed_response_seconds": 3.807008792005945, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 7, "valid_response_observations": 6, "application_cache_hits": 0, "application_cache_denominator": 6, "physical_responses": 7, "provider_error_responses": 1, "actual_cost_usd": 0.1306053, "provider_cache_metadata_responses": 7, "provider_cache_hit_responses": 3, "provider_cached_input_tokens": 149254, "provider_reported_prompt_tokens": 305949, "physical_api_seconds": 21.48815300001297, "elapsed_response_seconds": 21.70273208200524, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-38", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "primary", "score": null, "status": "prediction-complete", "episode_seconds": 17.078818790992955, "controller_wall_seconds": null, "termination": null, "grade_status": "unsupported-or-ambiguous-label", "grade_error": "Unsupported original regular-expression filter syntax; no inferred wildcard correction. The entire case remains unscored for every arm", "error_type": null, "metering": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 0, "application_cache_denominator": 6, "physical_responses": 6, "provider_error_responses": 0, "actual_cost_usd": 0.156076875, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 104545, "provider_reported_prompt_tokens": 300323, "physical_api_seconds": 16.12419179099379, "elapsed_response_seconds": 16.269495624001138, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.131341125, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 104545, "provider_reported_prompt_tokens": 267502, "physical_api_seconds": 13.462748332996853, "elapsed_response_seconds": 13.583824208006263, "backoff_seconds": 0 }, "it-critique": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 0, "application_cache_denominator": 1, "physical_responses": 1, "provider_error_responses": 0, "actual_cost_usd": 0.02473575, "provider_cache_metadata_responses": 1, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 32821, "physical_api_seconds": 2.6614434579969384, "elapsed_response_seconds": 2.685671415994875, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 0, "application_cache_denominator": 6, "physical_responses": 6, "provider_error_responses": 0, "actual_cost_usd": 0.156076875, "provider_cache_metadata_responses": 6, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 104545, "provider_reported_prompt_tokens": 300323, "physical_api_seconds": 16.12419179099379, "elapsed_response_seconds": 16.269495624001138, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-38", "seed": 612, "arm": "actor-only", "phase": "primary", "score": null, "status": "prediction-complete", "episode_seconds": 17.048878166999202, "controller_wall_seconds": null, "termination": null, "grade_status": "unsupported-or-ambiguous-label", "grade_error": "Unsupported original regular-expression filter syntax; no inferred wildcard correction. The entire case remains unscored for every arm", "error_type": null, "metering": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.13484084999999998, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 106338, "provider_reported_prompt_tokens": 274237, "physical_api_seconds": 16.309121667, "elapsed_response_seconds": 16.429853833978996, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.13484084999999998, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 106338, "provider_reported_prompt_tokens": 274237, "physical_api_seconds": 16.309121667, "elapsed_response_seconds": 16.429853833978996, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 0, "application_cache_denominator": 5, "physical_responses": 5, "provider_error_responses": 0, "actual_cost_usd": 0.13484084999999998, "provider_cache_metadata_responses": 5, "provider_cache_hit_responses": 5, "provider_cached_input_tokens": 106338, "provider_reported_prompt_tokens": 274237, "physical_api_seconds": 16.309121667, "elapsed_response_seconds": 16.429853833978996, "backoff_seconds": 0 } } }, { "domain": "banking_knowledge", "task": "task_034", "seed": 612, "arm": "actor-only", "phase": "warm", "score": 1.0, "status": 0, "episode_seconds": 3.107219083001837, "controller_wall_seconds": 7.975692416992388, "termination": "user_stop", "grade_status": "same-output-cache-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 12, "valid_response_observations": 12, "application_cache_hits": 12, "application_cache_denominator": 12, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.07698300000629388, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.04277854099927936, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.03420445900701452, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 12, "valid_response_observations": 12, "application_cache_hits": 12, "application_cache_denominator": 12, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.07698300000629388, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 6, "reviewed_steps": 0, "selector_bypassed_steps": 6 }, { "domain": "banking_knowledge", "task": "task_034", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "warm", "score": 1.0, "status": 0, "episode_seconds": 3.3054286249971483, "controller_wall_seconds": 5.474269750004169, "termination": "user_stop", "grade_status": "same-output-cache-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 17, "valid_response_observations": 17, "application_cache_hits": 17, "application_cache_denominator": 17, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.10098533103882801, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.03630970801168587, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 4, "valid_response_observations": 4, "application_cache_hits": 4, "application_cache_denominator": 4, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.026367124024545774, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.033441416002460755, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 1, "application_cache_denominator": 1, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.004867083000135608, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 17, "valid_response_observations": 17, "application_cache_hits": 17, "application_cache_denominator": 17, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.10098533103882801, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 6, "reviewed_steps": 4, "selector_bypassed_steps": 6 }, { "domain": "banking_knowledge", "task": "task_034", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "warm", "score": 1.0, "status": 0, "episode_seconds": 6.391231042012805, "controller_wall_seconds": 8.28443650000554, "termination": "user_stop", "grade_status": "same-output-cache-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 43, "valid_response_observations": 43, "application_cache_hits": 43, "application_cache_denominator": 43, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.1699036650097696, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 12, "valid_response_observations": 12, "application_cache_hits": 12, "application_cache_denominator": 12, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.047726999007863924, "backoff_seconds": 0 }, "decision": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 1, "application_cache_denominator": 1, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.0032082910038297996, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 10, "valid_response_observations": 10, "application_cache_hits": 10, "application_cache_denominator": 10, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.037769208021927625, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 18, "valid_response_observations": 18, "application_cache_hits": 18, "application_cache_denominator": 18, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.07485812497907318, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.006341041997075081, "backoff_seconds": 0 } }, "served_models": { "cloudflare/clef-flash": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 1, "application_cache_denominator": 1, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.0032082910038297996, "backoff_seconds": 0 }, "google/gemini-3.8-flash": { "response_observations": 42, "valid_response_observations": 42, "application_cache_hits": 42, "application_cache_denominator": 42, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.1666953740059398, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 12, "reviewed_steps": 10, "selector_bypassed_steps": 11 }, { "domain": "banking_knowledge", "task": "task_089", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "warm", "score": null, "status": 0, "episode_seconds": 0.0, "controller_wall_seconds": 2.6037318750022678, "termination": "infrastructure_error", "grade_status": "unscored-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 7, "valid_response_observations": 7, "application_cache_hits": 7, "application_cache_denominator": 7, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.03149158399901353, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 3, "application_cache_denominator": 3, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.009590248999302275, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 1, "application_cache_denominator": 1, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.011533333992701955, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.007647959006135352, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 1, "application_cache_denominator": 1, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.0027200420008739457, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 7, "valid_response_observations": 7, "application_cache_hits": 7, "application_cache_denominator": 7, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.03149158399901353, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 3, "reviewed_steps": 1, "selector_bypassed_steps": 3 }, { "domain": "banking_knowledge", "task": "task_089", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "warm", "score": null, "status": 0, "episode_seconds": 0.0, "controller_wall_seconds": 1.7830867500015302, "termination": "infrastructure_error", "grade_status": "unscored-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 7, "valid_response_observations": 7, "application_cache_hits": 7, "application_cache_denominator": 7, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.03344712499529123, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 3, "application_cache_denominator": 3, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.011933250003494322, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 1, "application_cache_denominator": 1, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.010748417000286281, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.007983040995895863, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 1, "application_cache_denominator": 1, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.002782416995614767, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 7, "valid_response_observations": 7, "application_cache_hits": 7, "application_cache_denominator": 7, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.03344712499529123, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 3, "reviewed_steps": 1, "selector_bypassed_steps": 3 }, { "domain": "banking_knowledge", "task": "task_089", "seed": 612, "arm": "actor-only", "phase": "warm", "score": 1.0, "status": 0, "episode_seconds": 9.378686125011882, "controller_wall_seconds": 11.349210541011416, "termination": "user_stop", "grade_status": "same-output-cache-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 27, "valid_response_observations": 27, "application_cache_hits": 27, "application_cache_denominator": 27, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.1744529160350794, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 22, "valid_response_observations": 22, "application_cache_hits": 22, "application_cache_denominator": 22, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.1508217060327297, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.02363121000234969, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 27, "valid_response_observations": 27, "application_cache_hits": 27, "application_cache_denominator": 27, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.1744529160350794, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 22, "reviewed_steps": 0, "selector_bypassed_steps": 22 }, { "domain": "banking_knowledge", "task": "task_095", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "warm", "score": 1.0, "status": 0, "episode_seconds": 46.421694791992195, "controller_wall_seconds": 48.24164095798915, "termination": "user_stop", "grade_status": "same-output-cache-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 137, "valid_response_observations": 137, "application_cache_hits": 137, "application_cache_denominator": 137, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.5384121680690441, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 88, "valid_response_observations": 88, "application_cache_hits": 88, "application_cache_denominator": 88, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.3237437090429012, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 4, "valid_response_observations": 4, "application_cache_hits": 4, "application_cache_denominator": 4, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.01042433301336132, "backoff_seconds": 0 }, "decision": { "response_observations": 12, "valid_response_observations": 12, "application_cache_hits": 12, "application_cache_denominator": 12, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.059445749982842244, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 24, "valid_response_observations": 24, "application_cache_hits": 24, "application_cache_denominator": 24, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.08399895601905882, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.03660016901267227, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 3, "application_cache_denominator": 3, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.0241992509982083, "backoff_seconds": 0 } }, "served_models": { "cloudflare/clef-flash": { "response_observations": 12, "valid_response_observations": 12, "application_cache_hits": 12, "application_cache_denominator": 12, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.059445749982842244, "backoff_seconds": 0 }, "google/gemini-3.8-flash": { "response_observations": 125, "valid_response_observations": 125, "application_cache_hits": 125, "application_cache_denominator": 125, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.4789664180862019, "backoff_seconds": 0 } }, "format_failure_clarifications": 2, "controller_steps": 76, "reviewed_steps": 24, "selector_bypassed_steps": 64 }, { "domain": "banking_knowledge", "task": "task_095", "seed": 612, "arm": "actor-only", "phase": "warm", "score": null, "status": 0, "episode_seconds": 0.0, "controller_wall_seconds": 1.6305394169903593, "termination": "infrastructure_error", "grade_status": "unscored-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.014769207002245821, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 3, "application_cache_denominator": 3, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.007327916013309732, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.007441290988936089, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.014769207002245821, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 3, "reviewed_steps": 0, "selector_bypassed_steps": 3 }, { "domain": "banking_knowledge", "task": "task_095", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "warm", "score": 0.0, "status": 0, "episode_seconds": 12.498811416007811, "controller_wall_seconds": 14.052805792001891, "termination": "user_stop", "grade_status": "same-output-cache-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 67, "valid_response_observations": 67, "application_cache_hits": 67, "application_cache_denominator": 67, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.20021829401957802, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 43, "valid_response_observations": 43, "application_cache_hits": 43, "application_cache_denominator": 43, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.13395654302439652, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 1, "application_cache_denominator": 1, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.003276750008808449, "backoff_seconds": 0 }, "decision": { "response_observations": 4, "valid_response_observations": 4, "application_cache_hits": 4, "application_cache_denominator": 4, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.01402762599173002, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 13, "valid_response_observations": 13, "application_cache_hits": 13, "application_cache_denominator": 13, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.031233582994900644, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 3, "application_cache_denominator": 3, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.0084811249980703, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 3, "application_cache_denominator": 3, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.009242667001672089, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 63, "valid_response_observations": 63, "application_cache_hits": 63, "application_cache_denominator": 63, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.186190668027848, "backoff_seconds": 0 }, "perplexity/pplx-decider-v1.1-27b-20261006": { "response_observations": 4, "valid_response_observations": 4, "application_cache_hits": 4, "application_cache_denominator": 4, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.01402762599173002, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 40, "reviewed_steps": 13, "selector_bypassed_steps": 36 }, { "domain": "banking_knowledge", "task": "task_093", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "warm", "score": 0.0, "status": 0, "episode_seconds": 27.533201083992026, "controller_wall_seconds": 29.241950750001706, "termination": "user_stop", "grade_status": "same-output-cache-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 86, "valid_response_observations": 86, "application_cache_hits": 86, "application_cache_denominator": 86, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.30913324680295773, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 66, "valid_response_observations": 66, "application_cache_hits": 66, "application_cache_denominator": 66, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.2522708328469889, "backoff_seconds": 0 }, "decision": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 3, "application_cache_denominator": 3, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.00969529099529609, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 10, "valid_response_observations": 10, "application_cache_hits": 10, "application_cache_denominator": 10, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.0294031219673343, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 4, "valid_response_observations": 4, "application_cache_hits": 4, "application_cache_denominator": 4, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.009248125992598943, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 3, "application_cache_denominator": 3, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.00851587500073947, "backoff_seconds": 0 } }, "served_models": { "cloudflare/clef-flash": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 3, "application_cache_denominator": 3, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.00969529099529609, "backoff_seconds": 0 }, "google/gemini-3.8-flash": { "response_observations": 83, "valid_response_observations": 83, "application_cache_hits": 83, "application_cache_denominator": 83, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.29943795580766164, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 60, "reviewed_steps": 10, "selector_bypassed_steps": 57 }, { "domain": "banking_knowledge", "task": "task_093", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "warm", "score": 0.0, "status": 0, "episode_seconds": 12.43221604199789, "controller_wall_seconds": 14.087468625002657, "termination": "user_stop", "grade_status": "same-output-cache-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 59, "valid_response_observations": 59, "application_cache_hits": 59, "application_cache_denominator": 59, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.2529211669607321, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 35, "valid_response_observations": 35, "application_cache_hits": 35, "application_cache_denominator": 35, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.1491002909897361, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.005883333011297509, "backoff_seconds": 0 }, "decision": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.0037969169934513047, "backoff_seconds": 0 }, "review-and-revision": { "response_observations": 10, "valid_response_observations": 10, "application_cache_hits": 10, "application_cache_denominator": 10, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.06470808596350253, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 7, "valid_response_observations": 7, "application_cache_hits": 7, "application_cache_denominator": 7, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.013711083011003211, "backoff_seconds": 0 }, "working-ledger": { "response_observations": 3, "valid_response_observations": 3, "application_cache_hits": 3, "application_cache_denominator": 3, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.015721456991741434, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 57, "valid_response_observations": 57, "application_cache_hits": 57, "application_cache_denominator": 57, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.24912424996728078, "backoff_seconds": 0 }, "perplexity/pplx-decider-v1.1-27b-20261006": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.0037969169934513047, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 33, "reviewed_steps": 10, "selector_bypassed_steps": 31 }, { "domain": "banking_knowledge", "task": "task_093", "seed": 612, "arm": "actor-only", "phase": "warm", "score": 0.0, "status": 0, "episode_seconds": 16.519724832993234, "controller_wall_seconds": 18.224997249999433, "termination": "user_stop", "grade_status": "same-output-cache-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 60, "valid_response_observations": 60, "application_cache_hits": 60, "application_cache_denominator": 60, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.16208228802133817, "backoff_seconds": 0 }, "categories": { "actor": { "response_observations": 54, "valid_response_observations": 54, "application_cache_hits": 54, "application_cache_denominator": 54, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.1478399120096583, "backoff_seconds": 0 }, "actor-recovery": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 1, "application_cache_denominator": 1, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.00219462500535883, "backoff_seconds": 0 }, "user-simulator": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.01204775100632105, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 60, "valid_response_observations": 60, "application_cache_hits": 60, "application_cache_denominator": 60, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.16208228802133817, "backoff_seconds": 0 } }, "format_failure_clarifications": 0, "controller_steps": 52, "reviewed_steps": 0, "selector_bypassed_steps": 52 }, { "domain": "ITBench", "task": "Scenario-16", "seed": 612, "arm": "actor-only", "phase": "warm", "score": 1.0, "status": "prediction-complete", "episode_seconds": 0.5533360840054229, "controller_wall_seconds": null, "termination": null, "grade_status": "same-output-cache-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.029073917015921324, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.029073917015921324, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.029073917015921324, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-16", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "warm", "score": 1.0, "status": "prediction-complete", "episode_seconds": 0.5138519580068532, "controller_wall_seconds": null, "termination": null, "grade_status": "same-output-cache-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.011614167000516318, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.009707582998089492, "backoff_seconds": 0 }, "it-critique": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 1, "application_cache_denominator": 1, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.0019065840024268255, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.011614167000516318, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-16", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "warm", "score": 1.0, "status": "prediction-complete", "episode_seconds": 0.48988591598754283, "controller_wall_seconds": null, "termination": null, "grade_status": "same-output-cache-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.012479417011491023, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.011078042007284239, "backoff_seconds": 0 }, "it-critique": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 1, "application_cache_denominator": 1, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.001401375004206784, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.012479417011491023, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-17", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "warm", "score": null, "status": "execution-error", "episode_seconds": 7.791312000001199, "controller_wall_seconds": null, "termination": null, "grade_status": "unscored-replay", "grade_error": null, "error_type": "RuntimeError", "metering": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.004259415989508852, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.004259415989508852, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.004259415989508852, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-17", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "warm", "score": null, "status": "execution-error", "episode_seconds": 0.15643920899310615, "controller_wall_seconds": null, "termination": null, "grade_status": "unscored-replay", "grade_error": null, "error_type": "RuntimeError", "metering": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.004927834001136944, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.004927834001136944, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.004927834001136944, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-17", "seed": 612, "arm": "actor-only", "phase": "warm", "score": 0.0, "status": "prediction-complete", "episode_seconds": 0.4411356250056997, "controller_wall_seconds": null, "termination": null, "grade_status": "same-output-cache-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.007829998983652331, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.007829998983652331, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.007829998983652331, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-29", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "warm", "score": null, "status": "execution-error", "episode_seconds": 8.692097291001119, "controller_wall_seconds": null, "termination": null, "grade_status": "unscored-replay", "grade_error": null, "error_type": "RuntimeError", "metering": { "response_observations": 7, "valid_response_observations": 7, "application_cache_hits": 7, "application_cache_denominator": 7, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.01702558298711665, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.013174082996556535, "backoff_seconds": 0 }, "it-critique": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 1, "application_cache_denominator": 1, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.0038514999905601144, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 7, "valid_response_observations": 7, "application_cache_hits": 7, "application_cache_denominator": 7, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.01702558298711665, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-29", "seed": 612, "arm": "actor-only", "phase": "warm", "score": null, "status": "execution-error", "episode_seconds": 0.08343683400016744, "controller_wall_seconds": null, "termination": null, "grade_status": "unscored-replay", "grade_error": null, "error_type": "RuntimeError", "metering": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.004501583011006005, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.004501583011006005, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.004501583011006005, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-29", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "warm", "score": null, "status": "execution-error", "episode_seconds": 0.06077437500061933, "controller_wall_seconds": null, "termination": null, "grade_status": "unscored-replay", "grade_error": null, "error_type": "RuntimeError", "metering": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.007164834009017795, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.007164834009017795, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 2, "valid_response_observations": 2, "application_cache_hits": 2, "application_cache_denominator": 2, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.007164834009017795, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-38", "seed": 612, "arm": "cloudflare/clef-flash", "phase": "warm", "score": null, "status": "execution-error", "episode_seconds": 11.527995124997688, "controller_wall_seconds": null, "termination": null, "grade_status": "unscored-replay", "grade_error": null, "error_type": "RuntimeError", "metering": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.013998541995533742, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.012530791995231993, "backoff_seconds": 0 }, "it-critique": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 1, "application_cache_denominator": 1, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.0014677500003017485, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.013998541995533742, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-38", "seed": 612, "arm": "perplexity/pplx-decider-v1.1-27b", "phase": "warm", "score": null, "status": "prediction-complete", "episode_seconds": 0.42146504200354684, "controller_wall_seconds": null, "termination": null, "grade_status": "unscored-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.01777854099054821, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.012631207995582372, "backoff_seconds": 0 }, "it-critique": { "response_observations": 1, "valid_response_observations": 1, "application_cache_hits": 1, "application_cache_denominator": 1, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.005147332994965836, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 6, "valid_response_observations": 6, "application_cache_hits": 6, "application_cache_denominator": 6, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.01777854099054821, "backoff_seconds": 0 } } }, { "domain": "ITBench", "task": "Scenario-38", "seed": 612, "arm": "actor-only", "phase": "warm", "score": null, "status": "prediction-complete", "episode_seconds": 0.41888825000205543, "controller_wall_seconds": null, "termination": null, "grade_status": "unscored-replay", "grade_error": null, "error_type": null, "metering": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.02790608497161884, "backoff_seconds": 0 }, "categories": { "it-actor": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.02790608497161884, "backoff_seconds": 0 } }, "served_models": { "google/gemini-3.8-flash": { "response_observations": 5, "valid_response_observations": 5, "application_cache_hits": 5, "application_cache_denominator": 5, "physical_responses": 0, "provider_error_responses": 0, "actual_cost_usd": 0, "provider_cache_metadata_responses": 0, "provider_cache_hit_responses": 0, "provider_cached_input_tokens": 0, "provider_reported_prompt_tokens": 0, "physical_api_seconds": 0, "elapsed_response_seconds": 0.02790608497161884, "backoff_seconds": 0 } } } ], "accounting": { "new_experiment_actual_cost_usd": 49.8999558050001, "new_experiment_uncertain_reserve_usd": 0.10161450000000072, "primary_cost_usd": 7.575425184, "warm_cost_usd": 0, "development_and_failed_attempt_cost_usd": 42.32453062100009, "previous_experiment_actual_cost_usd": 88.7480854, "cumulative_actual_cost_usd": 138.64804120500008, "previous_unresolved_reserve_usd": 1.24745642, "all_attempt_observations": { "response_observations": 8493, "valid_response_observations": 8427, "application_cache_hits": 3619, "application_cache_denominator": 8427, "physical_responses": 4873, "provider_error_responses": 66, "actual_cost_usd": 49.899955805, "provider_cache_metadata_responses": 4540, "provider_cache_hit_responses": 844, "provider_cached_input_tokens": 12246350, "provider_reported_prompt_tokens": 75025495, "physical_api_seconds": 14293.02256992305, "elapsed_response_seconds": 14535.917017167216, "backoff_seconds": 30 }, "earlier_round_invoiced_cost_usd": 41.07253668500014, "approved_cap_usd": 60, "budget_amendment": { "at_utc": "2026-10-15T10:00:57.200568+00:00", "previous_cap_usd": 40, "approved_maximum_usd": 60, "scope": "Finish the preselected initial84 and refined24 primary comparisons, development qualification and exact-cache replays; no third experimental round. Actual necessary invoices only; ledger never reset.", "status": "Approved by the user before the additional charges; the original frozen initial plan is preserved." }, "unsettled_physical_responses": [ { "namespace": "bank-task_080-611-decider", "request_sha256": "0b7b260c3c70369ca827219ba18a32aa353e30b6f34020b43837a417add43ef2", "response_id": "gen-1792062400-9RrXsaHHakVggUJPdGWB", "model": "google/gemini-3.8-flash", "provider": "Google AI Studio", "uncertain_reserve_usd": 0.1016145, "status": "Saved physical response; invoice unavailable. Not included in settled-response counters or confirmed fees." } ], "response_count_scope": "Response counters and fee sums use settled observations. Saved responses without an invoice are listed separately and are not assumed free. Their prospective estimates remain in the unresolved-request reserve." }, "interpretation": [ "The refined banking loop critiques one proposal first; only a concrete issue can produce a revision. This avoids cosmetic alternative generation and ranking. Effective-cost checks include source-verified cash-flow terms. Initial-loop results use a different, earlier unused cohort and stay separate.", "Actor is the ordinary one-next-action text-generation/tool loop. Mithril adds conditional planning, critique, evidence lookup, revision and native selection; Decider and Clef Flash differ only by decision-model ID.", "Primary application caches started empty and were isolated by case/seed/arm. Provider caching is observed, not controlled; execution order is counterbalanced. Warm replay refuses new physical calls and is a separate execution phase, not a new independent score.", "Banking uses the original tau2 evaluator and user simulator. IT uses original objects, events, logs, traces and metrics; post-execution exact/filter/alias root-group normalization is local, not the official GPT-5.5 grader.", "Missing/ambiguous grades stay null. Complete-pair improvements exclude unscored episodes; all attempt fees and planned coverage are reported. Public cases are unused in this comparison series, not globally uncontaminated.", "The shared native decision-state brief is bounded to 1750 cl100k reference tokens, with complete choices sent separately. This bound does not establish equal provider tokenization, full effective attention or absence of provider-side truncation.", "Arms run independent full trajectories with matched tasks, seeds and shared generation settings. Separate provider generations and later actions can produce different candidate pools; this is not a fixed-candidate selector-only ablation." ], "earlier_round_receipts": "/explore/efficiency-round1.json", "provenance_seal_sha256": "08e4a93a9c134c2a15c2fb1e662a1d73459ea634a1bda48ac94640c2b279ece9", "public_redactions": [ "Operator local accounting path omitted; measurements unchanged" ], "analysis_addendum": { "file": "apps/code/bench/adaptive-loop-efficiency-20261015/paired_analysis.py", "sha256": "367c9d8d833b8d13f2a5760d28d8aeb22cdcffa716aac496b4a440fd5db0d307", "packaging_file": "apps/code/bench/adaptive-loop-efficiency-20261015/prepare_public_efficiency.py", "packaging_sha256": "11329263296cd37a32235ccf691907c08a2d2dd6151a58f0694aedb6570a9d97", "reason": "Resample whole tasks with all scored pairs rather than averaging cluster means when seed coverage is unequal. Original execution outputs, point estimates and invoices are unchanged." }, "execution_failure_diagnostics": { "banking_unscored_runs": [], "boundary": "Categories derive from original runner error logs after execution; missing scores remain null. No expected actions or private simulator goals are included." } }