{
  "schema": "mithril.comparison/v10-expanded",
  "status": "completed_with_diagnostic_failures",
  "conditions": [
    {
      "domain": "banking_knowledge",
      "controller": "v12",
      "mode": "actor-only",
      "arm": "actor-only",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 4,
      "scored": 4,
      "unscored": 0,
      "mean_score": 0.0,
      "successes": 0,
      "all_attempt_verified_success_rate": 0.0,
      "costs": {
        "logical_calls": 238,
        "application_cache_hits": 12,
        "physical_calls": 226,
        "actual_new_cost_usd": 1.5749334,
        "unique_response_value_usd": 1.639827675
      },
      "termination_counts": {
        "user_stop": 4
      },
      "final_episode_costs": {
        "logical_calls": 238,
        "application_cache_hits": 12,
        "physical_calls": 226,
        "actual_new_cost_usd": 1.5749334,
        "unique_response_value_usd": 1.639827675
      },
      "cost_scope": "All recorded same-cohort tau attempts, including interrupted originals and continuations; superseded pilots remain in the study total."
    },
    {
      "domain": "banking_knowledge",
      "controller": "v12",
      "mode": "reflection-revision",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 4,
      "scored": 4,
      "unscored": 0,
      "mean_score": 0.25,
      "successes": 1,
      "all_attempt_verified_success_rate": 0.25,
      "costs": {
        "logical_calls": 1135,
        "application_cache_hits": 278,
        "physical_calls": 857,
        "actual_new_cost_usd": 20.14037342,
        "unique_response_value_usd": 20.284987145
      },
      "termination_counts": {
        "user_stop": 4
      },
      "final_episode_costs": {
        "logical_calls": 891,
        "application_cache_hits": 248,
        "physical_calls": 643,
        "actual_new_cost_usd": 16.987809075,
        "unique_response_value_usd": 20.284987145
      },
      "cost_scope": "All recorded same-cohort tau attempts, including interrupted originals and continuations; superseded pilots remain in the study total."
    },
    {
      "domain": "banking_knowledge",
      "controller": "v12",
      "mode": "reflection-revision",
      "arm": "cloudflare/clef-flash",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 4,
      "scored": 4,
      "unscored": 0,
      "mean_score": 0.0,
      "successes": 0,
      "all_attempt_verified_success_rate": 0.0,
      "costs": {
        "logical_calls": 2010,
        "application_cache_hits": 676,
        "physical_calls": 1334,
        "actual_new_cost_usd": 39.910482867,
        "unique_response_value_usd": 40.060225842
      },
      "termination_counts": {
        "user_stop": 4
      },
      "final_episode_costs": {
        "logical_calls": 1364,
        "application_cache_hits": 663,
        "physical_calls": 701,
        "actual_new_cost_usd": 22.15426719,
        "unique_response_value_usd": 40.060225842
      },
      "cost_scope": "All recorded same-cohort tau attempts, including interrupted originals and continuations; superseded pilots remain in the study total."
    },
    {
      "domain": "retail",
      "controller": "v8",
      "mode": "actor-only",
      "arm": "actor-only",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 4,
      "scored": 4,
      "unscored": 0,
      "mean_score": 1.0,
      "successes": 4,
      "all_attempt_verified_success_rate": 1.0,
      "costs": {
        "logical_calls": 65,
        "application_cache_hits": 37,
        "physical_calls": 28,
        "actual_new_cost_usd": 0.10515785,
        "unique_response_value_usd": 0.270996125
      },
      "termination_counts": {
        "user_stop": 4
      },
      "final_episode_costs": {
        "logical_calls": 65,
        "application_cache_hits": 37,
        "physical_calls": 28,
        "actual_new_cost_usd": 0.10515785,
        "unique_response_value_usd": 0.270996125
      },
      "cost_scope": "All recorded same-cohort tau attempts, including interrupted originals and continuations; superseded pilots remain in the study total."
    },
    {
      "domain": "retail",
      "controller": "v8",
      "mode": "reflection-revision",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 4,
      "scored": 4,
      "unscored": 0,
      "mean_score": 1.0,
      "successes": 4,
      "all_attempt_verified_success_rate": 1.0,
      "costs": {
        "logical_calls": 180,
        "application_cache_hits": 28,
        "physical_calls": 152,
        "actual_new_cost_usd": 0.74895676,
        "unique_response_value_usd": 0.84893426
      },
      "termination_counts": {
        "user_stop": 4
      },
      "final_episode_costs": {
        "logical_calls": 180,
        "application_cache_hits": 28,
        "physical_calls": 152,
        "actual_new_cost_usd": 0.74895676,
        "unique_response_value_usd": 0.84893426
      },
      "cost_scope": "All recorded same-cohort tau attempts, including interrupted originals and continuations; superseded pilots remain in the study total."
    },
    {
      "domain": "retail",
      "controller": "v8",
      "mode": "reflection-revision",
      "arm": "cloudflare/clef-flash",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 4,
      "scored": 4,
      "unscored": 0,
      "mean_score": 1.0,
      "successes": 4,
      "all_attempt_verified_success_rate": 1.0,
      "costs": {
        "logical_calls": 200,
        "application_cache_hits": 28,
        "physical_calls": 172,
        "actual_new_cost_usd": 0.80459471,
        "unique_response_value_usd": 0.923161085
      },
      "termination_counts": {
        "user_stop": 4
      },
      "final_episode_costs": {
        "logical_calls": 200,
        "application_cache_hits": 28,
        "physical_calls": 172,
        "actual_new_cost_usd": 0.80459471,
        "unique_response_value_usd": 0.923161085
      },
      "cost_scope": "All recorded same-cohort tau attempts, including interrupted originals and continuations; superseded pilots remain in the study total."
    },
    {
      "domain": "airline",
      "controller": "v8",
      "mode": "actor-only",
      "arm": "actor-only",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 4,
      "scored": 4,
      "unscored": 0,
      "mean_score": 0.5,
      "successes": 2,
      "all_attempt_verified_success_rate": 0.5,
      "costs": {
        "logical_calls": 55,
        "application_cache_hits": 12,
        "physical_calls": 43,
        "actual_new_cost_usd": 0.153947775,
        "unique_response_value_usd": 0.183438525
      },
      "termination_counts": {
        "user_stop": 4
      },
      "final_episode_costs": {
        "logical_calls": 55,
        "application_cache_hits": 12,
        "physical_calls": 43,
        "actual_new_cost_usd": 0.153947775,
        "unique_response_value_usd": 0.183438525
      },
      "cost_scope": "All recorded same-cohort tau attempts, including interrupted originals and continuations; superseded pilots remain in the study total."
    },
    {
      "domain": "airline",
      "controller": "v8",
      "mode": "reflection-revision",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 4,
      "scored": 4,
      "unscored": 0,
      "mean_score": 0.5,
      "successes": 2,
      "all_attempt_verified_success_rate": 0.5,
      "costs": {
        "logical_calls": 166,
        "application_cache_hits": 23,
        "physical_calls": 143,
        "actual_new_cost_usd": 0.51087252,
        "unique_response_value_usd": 0.59142852
      },
      "termination_counts": {
        "user_stop": 4
      },
      "final_episode_costs": {
        "logical_calls": 166,
        "application_cache_hits": 23,
        "physical_calls": 143,
        "actual_new_cost_usd": 0.51087252,
        "unique_response_value_usd": 0.59142852
      },
      "cost_scope": "All recorded same-cohort tau attempts, including interrupted originals and continuations; superseded pilots remain in the study total."
    },
    {
      "domain": "airline",
      "controller": "v8",
      "mode": "reflection-revision",
      "arm": "cloudflare/clef-flash",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 4,
      "scored": 4,
      "unscored": 0,
      "mean_score": 0.5,
      "successes": 2,
      "all_attempt_verified_success_rate": 0.5,
      "costs": {
        "logical_calls": 154,
        "application_cache_hits": 57,
        "physical_calls": 97,
        "actual_new_cost_usd": 0.353123303,
        "unique_response_value_usd": 0.556636478
      },
      "termination_counts": {
        "user_stop": 4
      },
      "final_episode_costs": {
        "logical_calls": 154,
        "application_cache_hits": 57,
        "physical_calls": 97,
        "actual_new_cost_usd": 0.353123303,
        "unique_response_value_usd": 0.556636478
      },
      "cost_scope": "All recorded same-cohort tau attempts, including interrupted originals and continuations; superseded pilots remain in the study total."
    },
    {
      "domain": "ITBench",
      "controller": "utf8",
      "mode": "actor-only",
      "arm": "actor-only",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 2,
      "scored": 2,
      "unscored": 0,
      "mean_score": 0.75,
      "successes": 1,
      "all_attempt_verified_success_rate": 0.5,
      "costs": {
        "logical_calls": 0,
        "application_cache_hits": 0,
        "physical_calls": 0,
        "actual_new_cost_usd": 0,
        "unique_response_value_usd": 0
      },
      "termination_counts": {
        "prediction_completed": 2
      },
      "final_episode_costs": {
        "logical_calls": 0,
        "application_cache_hits": 0,
        "physical_calls": 0,
        "actual_new_cost_usd": 0,
        "unique_response_value_usd": 0
      },
      "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
    },
    {
      "domain": "ITBench",
      "controller": "utf8",
      "mode": "reflection-revision",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 2,
      "scored": 2,
      "unscored": 0,
      "mean_score": 0.5,
      "successes": 0,
      "all_attempt_verified_success_rate": 0.0,
      "costs": {
        "logical_calls": 1,
        "application_cache_hits": 1,
        "physical_calls": 0,
        "actual_new_cost_usd": 0,
        "unique_response_value_usd": 6.782e-05
      },
      "termination_counts": {
        "prediction_completed": 2
      },
      "final_episode_costs": {
        "logical_calls": 1,
        "application_cache_hits": 1,
        "physical_calls": 0,
        "actual_new_cost_usd": 0,
        "unique_response_value_usd": 6.782e-05
      },
      "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
    },
    {
      "domain": "ITBench",
      "controller": "utf8",
      "mode": "reflection-revision",
      "arm": "cloudflare/clef-flash",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 2,
      "scored": 2,
      "unscored": 0,
      "mean_score": 0.5,
      "successes": 0,
      "all_attempt_verified_success_rate": 0.0,
      "costs": {
        "logical_calls": 1,
        "application_cache_hits": 1,
        "physical_calls": 0,
        "actual_new_cost_usd": 0,
        "unique_response_value_usd": 7.497e-05
      },
      "termination_counts": {
        "prediction_completed": 2
      },
      "final_episode_costs": {
        "logical_calls": 1,
        "application_cache_hits": 1,
        "physical_calls": 0,
        "actual_new_cost_usd": 0,
        "unique_response_value_usd": 7.497e-05
      },
      "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
    },
    {
      "domain": "IFEval",
      "controller": "utf8",
      "mode": "actor-only",
      "arm": "actor-only",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 541,
      "scored": 541,
      "unscored": 0,
      "mean_score": 0.9426987060998152,
      "successes": 510,
      "all_attempt_verified_success_rate": 0.9426987060998152,
      "costs": {
        "logical_calls": 0,
        "application_cache_hits": 0,
        "physical_calls": 0,
        "actual_new_cost_usd": 0,
        "unique_response_value_usd": 0
      },
      "termination_counts": {
        "prediction_completed": 541
      },
      "strict_instruction_accuracy": 0.960431654676259,
      "loose_prompt_accuracy": 0.9574861367837338,
      "loose_instruction_accuracy": 0.9700239808153477,
      "repair_model_calls": 0,
      "changed_responses": 0,
      "final_episode_costs": {
        "logical_calls": 0,
        "application_cache_hits": 0,
        "physical_calls": 0,
        "actual_new_cost_usd": 0,
        "unique_response_value_usd": 0
      },
      "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
    },
    {
      "domain": "IFEval",
      "controller": "utf8",
      "mode": "reflection-revision",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 541,
      "scored": 541,
      "unscored": 0,
      "mean_score": 0.944547134935305,
      "successes": 511,
      "all_attempt_verified_success_rate": 0.944547134935305,
      "costs": {
        "logical_calls": 1215,
        "application_cache_hits": 755,
        "physical_calls": 460,
        "actual_new_cost_usd": 0.20230698,
        "unique_response_value_usd": 1.09474098
      },
      "termination_counts": {
        "prediction_completed": 541
      },
      "strict_instruction_accuracy": 0.9628297362110312,
      "loose_prompt_accuracy": 0.9537892791127541,
      "loose_instruction_accuracy": 0.9688249400479616,
      "repair_model_calls": 0,
      "changed_responses": 0,
      "final_episode_costs": {
        "logical_calls": 1215,
        "application_cache_hits": 755,
        "physical_calls": 460,
        "actual_new_cost_usd": 0.20230698,
        "unique_response_value_usd": 1.09474098
      },
      "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
    },
    {
      "domain": "IFEval",
      "controller": "utf8",
      "mode": "reflection-revision",
      "arm": "cloudflare/clef-flash",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 541,
      "scored": 541,
      "unscored": 0,
      "mean_score": 0.9390018484288355,
      "successes": 508,
      "all_attempt_verified_success_rate": 0.9390018484288355,
      "costs": {
        "logical_calls": 1215,
        "application_cache_hits": 752,
        "physical_calls": 463,
        "actual_new_cost_usd": 0.178230254,
        "unique_response_value_usd": 1.101267796
      },
      "termination_counts": {
        "prediction_completed": 541
      },
      "strict_instruction_accuracy": 0.9592326139088729,
      "loose_prompt_accuracy": 0.955637707948244,
      "loose_instruction_accuracy": 0.9700239808153477,
      "repair_model_calls": 0,
      "changed_responses": 0,
      "final_episode_costs": {
        "logical_calls": 1215,
        "application_cache_hits": 752,
        "physical_calls": 463,
        "actual_new_cost_usd": 0.178230254,
        "unique_response_value_usd": 1.101267796
      },
      "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
    },
    {
      "domain": "IFEval",
      "controller": "v10",
      "mode": "actor-with-checks",
      "arm": "actor-only",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 541,
      "scored": 541,
      "unscored": 0,
      "mean_score": 0.944547134935305,
      "successes": 511,
      "all_attempt_verified_success_rate": 0.944547134935305,
      "costs": {
        "logical_calls": 0,
        "application_cache_hits": 0,
        "physical_calls": 0,
        "actual_new_cost_usd": 0,
        "unique_response_value_usd": 0
      },
      "termination_counts": {
        "prediction_completed": 541
      },
      "strict_instruction_accuracy": 0.9616306954436451,
      "loose_prompt_accuracy": 0.9611829944547134,
      "loose_instruction_accuracy": 0.9724220623501199,
      "repair_model_calls": 0,
      "changed_responses": 7,
      "final_episode_costs": {
        "logical_calls": 0,
        "application_cache_hits": 0,
        "physical_calls": 0,
        "actual_new_cost_usd": 0,
        "unique_response_value_usd": 0
      },
      "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
    },
    {
      "domain": "IFEval",
      "controller": "v10",
      "mode": "reflection-revision-checks",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 541,
      "scored": 541,
      "unscored": 0,
      "mean_score": 0.9463955637707948,
      "successes": 512,
      "all_attempt_verified_success_rate": 0.9463955637707948,
      "costs": {
        "logical_calls": 0,
        "application_cache_hits": 0,
        "physical_calls": 0,
        "actual_new_cost_usd": 0,
        "unique_response_value_usd": 0
      },
      "termination_counts": {
        "prediction_completed": 541
      },
      "strict_instruction_accuracy": 0.9640287769784173,
      "loose_prompt_accuracy": 0.9537892791127541,
      "loose_instruction_accuracy": 0.9688249400479616,
      "repair_model_calls": 0,
      "changed_responses": 6,
      "final_episode_costs": {
        "logical_calls": 0,
        "application_cache_hits": 0,
        "physical_calls": 0,
        "actual_new_cost_usd": 0,
        "unique_response_value_usd": 0
      },
      "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
    },
    {
      "domain": "IFEval",
      "controller": "v10",
      "mode": "reflection-revision-checks",
      "arm": "cloudflare/clef-flash",
      "actor_model": "google/gemini-3.8-flash",
      "attempted": 541,
      "scored": 541,
      "unscored": 0,
      "mean_score": 0.9426987060998152,
      "successes": 510,
      "all_attempt_verified_success_rate": 0.9426987060998152,
      "costs": {
        "logical_calls": 1,
        "application_cache_hits": 1,
        "physical_calls": 0,
        "actual_new_cost_usd": 0,
        "unique_response_value_usd": 0.00270075
      },
      "termination_counts": {
        "prediction_completed": 541
      },
      "strict_instruction_accuracy": 0.9616306954436451,
      "loose_prompt_accuracy": 0.9593345656192237,
      "loose_instruction_accuracy": 0.9724220623501199,
      "repair_model_calls": 1,
      "changed_responses": 8,
      "final_episode_costs": {
        "logical_calls": 1,
        "application_cache_hits": 1,
        "physical_calls": 0,
        "actual_new_cost_usd": 0,
        "unique_response_value_usd": 0.00270075
      },
      "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
    }
  ],
  "matched_harness_comparison": [
    {
      "domain": "banking_knowledge",
      "controller": "v12",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "complete_cases": 4,
      "before": 0.0,
      "after": 0.25,
      "improved": 1,
      "regressed": 0,
      "unchanged": 3,
      "exact_paired_discordance_p": 1,
      "availability_before": 4,
      "availability_after": 4,
      "planned_cases": 4,
      "baseline": 0.0,
      "research": 0.25
    },
    {
      "domain": "banking_knowledge",
      "controller": "v12",
      "arm": "cloudflare/clef-flash",
      "complete_cases": 4,
      "before": 0.0,
      "after": 0.0,
      "improved": 0,
      "regressed": 0,
      "unchanged": 4,
      "exact_paired_discordance_p": 1,
      "availability_before": 4,
      "availability_after": 4,
      "planned_cases": 4,
      "baseline": 0.0,
      "research": 0.0
    },
    {
      "domain": "retail",
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "complete_cases": 4,
      "before": 1.0,
      "after": 1.0,
      "improved": 0,
      "regressed": 0,
      "unchanged": 4,
      "exact_paired_discordance_p": 1,
      "availability_before": 4,
      "availability_after": 4,
      "planned_cases": 4,
      "baseline": 1.0,
      "research": 1.0
    },
    {
      "domain": "retail",
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "complete_cases": 4,
      "before": 1.0,
      "after": 1.0,
      "improved": 0,
      "regressed": 0,
      "unchanged": 4,
      "exact_paired_discordance_p": 1,
      "availability_before": 4,
      "availability_after": 4,
      "planned_cases": 4,
      "baseline": 1.0,
      "research": 1.0
    },
    {
      "domain": "airline",
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "complete_cases": 4,
      "before": 0.5,
      "after": 0.5,
      "improved": 0,
      "regressed": 0,
      "unchanged": 4,
      "exact_paired_discordance_p": 1,
      "availability_before": 4,
      "availability_after": 4,
      "planned_cases": 4,
      "baseline": 0.5,
      "research": 0.5
    },
    {
      "domain": "airline",
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "complete_cases": 4,
      "before": 0.5,
      "after": 0.5,
      "improved": 0,
      "regressed": 0,
      "unchanged": 4,
      "exact_paired_discordance_p": 1,
      "availability_before": 4,
      "availability_after": 4,
      "planned_cases": 4,
      "baseline": 0.5,
      "research": 0.5
    },
    {
      "domain": "ITBench",
      "controller": "utf8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "complete_cases": 2,
      "before": 0.75,
      "after": 0.5,
      "improved": 0,
      "regressed": 1,
      "unchanged": 1,
      "exact_paired_discordance_p": 1,
      "availability_before": 2,
      "availability_after": 2,
      "planned_cases": 2,
      "baseline": 0.75,
      "research": 0.5
    },
    {
      "domain": "ITBench",
      "controller": "utf8",
      "arm": "cloudflare/clef-flash",
      "complete_cases": 2,
      "before": 0.75,
      "after": 0.5,
      "improved": 0,
      "regressed": 1,
      "unchanged": 1,
      "exact_paired_discordance_p": 1,
      "availability_before": 2,
      "availability_after": 2,
      "planned_cases": 2,
      "baseline": 0.75,
      "research": 0.5
    },
    {
      "domain": "IFEval",
      "controller": "utf8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "complete_cases": 541,
      "before": 0.9426987060998152,
      "after": 0.944547134935305,
      "improved": 9,
      "regressed": 8,
      "unchanged": 524,
      "exact_paired_discordance_p": 1,
      "availability_before": 541,
      "availability_after": 541,
      "planned_cases": 541,
      "baseline": 0.9426987060998152,
      "research": 0.944547134935305
    },
    {
      "domain": "IFEval",
      "controller": "utf8",
      "arm": "cloudflare/clef-flash",
      "complete_cases": 541,
      "before": 0.9426987060998152,
      "after": 0.9390018484288355,
      "improved": 8,
      "regressed": 10,
      "unchanged": 523,
      "exact_paired_discordance_p": 0.8145294189453125,
      "availability_before": 541,
      "availability_after": 541,
      "planned_cases": 541,
      "baseline": 0.9426987060998152,
      "research": 0.9390018484288355
    },
    {
      "domain": "IFEval \u00b7 output checks",
      "controller": "v10",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "complete_cases": 541,
      "before": 0.944547134935305,
      "after": 0.9463955637707948,
      "improved": 9,
      "regressed": 8,
      "unchanged": 524,
      "exact_paired_discordance_p": 1,
      "availability_before": 541,
      "availability_after": 541,
      "planned_cases": 541,
      "baseline": 0.944547134935305,
      "research": 0.9463955637707948
    },
    {
      "domain": "IFEval \u00b7 output checks",
      "controller": "v10",
      "arm": "cloudflare/clef-flash",
      "complete_cases": 541,
      "before": 0.944547134935305,
      "after": 0.9426987060998152,
      "improved": 7,
      "regressed": 8,
      "unchanged": 526,
      "exact_paired_discordance_p": 1,
      "availability_before": 541,
      "availability_after": 541,
      "planned_cases": 541,
      "baseline": 0.944547134935305,
      "research": 0.9426987060998152
    }
  ],
  "visible_contract_effects": [
    {
      "arm": "actor-only",
      "complete_cases": 541,
      "before": 0.9426987060998152,
      "after": 0.944547134935305,
      "improved": 1,
      "regressed": 0,
      "unchanged": 540,
      "exact_paired_discordance_p": 1,
      "availability_before": 541,
      "availability_after": 541,
      "planned_cases": 541
    },
    {
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "complete_cases": 541,
      "before": 0.944547134935305,
      "after": 0.9463955637707948,
      "improved": 1,
      "regressed": 0,
      "unchanged": 540,
      "exact_paired_discordance_p": 1,
      "availability_before": 541,
      "availability_after": 541,
      "planned_cases": 541
    },
    {
      "arm": "cloudflare/clef-flash",
      "complete_cases": 541,
      "before": 0.9390018484288355,
      "after": 0.9426987060998152,
      "improved": 2,
      "regressed": 0,
      "unchanged": 539,
      "exact_paired_discordance_p": 0.5,
      "availability_before": 541,
      "availability_after": 541,
      "planned_cases": 541
    }
  ],
  "native_serialization_effects": [
    {
      "arm": "actor-only",
      "complete_cases": 540,
      "before": 0.9425925925925925,
      "after": 0.9425925925925925,
      "improved": 1,
      "regressed": 1,
      "unchanged": 538,
      "exact_paired_discordance_p": 1,
      "availability_before": 540,
      "availability_after": 541,
      "planned_cases": 541
    },
    {
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "complete_cases": 540,
      "before": 0.9425925925925925,
      "after": 0.9444444444444444,
      "improved": 1,
      "regressed": 0,
      "unchanged": 539,
      "exact_paired_discordance_p": 1,
      "availability_before": 540,
      "availability_after": 541,
      "planned_cases": 541
    },
    {
      "arm": "cloudflare/clef-flash",
      "complete_cases": 538,
      "before": 0.9405204460966543,
      "after": 0.9386617100371747,
      "improved": 1,
      "regressed": 2,
      "unchanged": 535,
      "exact_paired_discordance_p": 1,
      "availability_before": 538,
      "availability_after": 541,
      "planned_cases": 541
    }
  ],
  "primary_actual_totals": {
    "logical_calls": 13269,
    "application_cache_hits": 5260,
    "physical_calls": 8009,
    "actual_new_cost_usd": 80.134945768,
    "unique_response_value_usd": 80.134945768
  },
  "experiment_lifetime_cost_usd": 88.7480854,
  "unknown_failed_request_reserve_usd": 1.207548719999997,
  "lifetime_unknown_failed_request_reserve_usd": 1.2474564199999971,
  "physical_provider_cache_by_served_model": {
    "cloudflare/clef-flash": {
      "physical_calls": 1130,
      "actual_new_cost_usd": 0.099383383,
      "provider_cache_metadata_reported": 0,
      "provider_cache_positive": 0,
      "cached_input_tokens": 0,
      "input_tokens": 3962472
    },
    "google/gemini-3.8-flash": {
      "physical_calls": 5299,
      "actual_new_cost_usd": 79.217065725,
      "provider_cache_metadata_reported": 5299,
      "provider_cache_positive": 1608,
      "cached_input_tokens": 39423733,
      "input_tokens": 136248859
    },
    "openai/gpt-4.1": {
      "physical_calls": 6,
      "actual_new_cost_usd": 0.032288000000000004,
      "provider_cache_metadata_reported": 6,
      "provider_cache_positive": 0,
      "cached_input_tokens": 0,
      "input_tokens": 13960
    },
    "perplexity/pplx-decider-v1.1-27b-20261006": {
      "physical_calls": 1084,
      "actual_new_cost_usd": 0.07110846,
      "provider_cache_metadata_reported": 0,
      "provider_cache_positive": 0,
      "cached_input_tokens": 0,
      "input_tokens": 3555423
    },
    "qwen/qwen3-30b-a3b-instruct-2507": {
      "physical_calls": 490,
      "actual_new_cost_usd": 0.7151002,
      "provider_cache_metadata_reported": 490,
      "provider_cache_positive": 441,
      "cached_input_tokens": 2286895,
      "input_tokens": 6933862
    }
  },
  "phase_costs": {
    "superseded_v7": {
      "logical_calls": 606,
      "application_cache_hits": 68,
      "physical_calls": 538,
      "actual_new_cost_usd": 0.719117193,
      "unique_response_value_usd": 0.719117193
    },
    "superseded_v8_short": {
      "logical_calls": 324,
      "application_cache_hits": 13,
      "physical_calls": 311,
      "actual_new_cost_usd": 2.499733636,
      "unique_response_value_usd": 2.499733636
    },
    "bank-final-recovery": {
      "logical_calls": 555,
      "application_cache_hits": 348,
      "physical_calls": 207,
      "actual_new_cost_usd": 3.789851972,
      "unique_response_value_usd": 7.671555557
    },
    "bank-original-steps": {
      "logical_calls": 1496,
      "application_cache_hits": 691,
      "physical_calls": 805,
      "actual_new_cost_usd": 16.95283895,
      "unique_response_value_usd": 34.774234352
    },
    "contract-refinement": {
      "logical_calls": 1,
      "application_cache_hits": 0,
      "physical_calls": 1,
      "actual_new_cost_usd": 0.00270075,
      "unique_response_value_usd": 0.00270075
    },
    "development": {
      "logical_calls": 33,
      "application_cache_hits": 10,
      "physical_calls": 23,
      "actual_new_cost_usd": 0.08096503000000001,
      "unique_response_value_usd": 0.10834303
    },
    "final": {
      "logical_calls": 2772,
      "application_cache_hits": 251,
      "physical_calls": 2521,
      "actual_new_cost_usd": 44.736013612,
      "unique_response_value_usd": 44.737884112
    },
    "final-output-refinement": {
      "logical_calls": 9,
      "application_cache_hits": 5,
      "physical_calls": 4,
      "actual_new_cost_usd": 0.00470925,
      "unique_response_value_usd": 0.00741
    },
    "ifeval": {
      "logical_calls": 76,
      "application_cache_hits": 14,
      "physical_calls": 62,
      "actual_new_cost_usd": 0.061933139,
      "unique_response_value_usd": 0.061933139
    },
    "ifeval-full": {
      "logical_calls": 3522,
      "application_cache_hits": 1450,
      "physical_calls": 2072,
      "actual_new_cost_usd": 2.173834074,
      "unique_response_value_usd": 3.24693009
    },
    "ifeval-utf8": {
      "logical_calls": 3512,
      "application_cache_hits": 2259,
      "physical_calls": 1253,
      "actual_new_cost_usd": 1.037323484,
      "unique_response_value_usd": 3.242368276
    },
    "itbench-invalid-tool-control-pilot": {
      "logical_calls": 12,
      "application_cache_hits": 0,
      "physical_calls": 12,
      "actual_new_cost_usd": 0.318185325,
      "unique_response_value_usd": 0.318185325
    },
    "itbench-utf8": {
      "logical_calls": 2,
      "application_cache_hits": 2,
      "physical_calls": 0,
      "actual_new_cost_usd": 0,
      "unique_response_value_usd": 0.00014279
    },
    "itbench-v8": {
      "logical_calls": 34,
      "application_cache_hits": 13,
      "physical_calls": 21,
      "actual_new_cost_usd": 0.533962115,
      "unique_response_value_usd": 0.70765919
    },
    "recovery-tau": {
      "logical_calls": 313,
      "application_cache_hits": 135,
      "physical_calls": 178,
      "actual_new_cost_usd": 7.22355832,
      "unique_response_value_usd": 9.49007464
    },
    "utf8-contract-refinement": {
      "logical_calls": 1,
      "application_cache_hits": 1,
      "physical_calls": 0,
      "actual_new_cost_usd": 0,
      "unique_response_value_usd": 0.00270075
    },
    "utf8-native-canary": {
      "logical_calls": 1,
      "application_cache_hits": 0,
      "physical_calls": 1,
      "actual_new_cost_usd": 0.000218918,
      "unique_response_value_usd": 0.000218918
    }
  },
  "final_study_costs": {
    "logical_calls": 12339,
    "application_cache_hits": 5179,
    "physical_calls": 7160,
    "actual_new_cost_usd": 76.916094939,
    "unique_response_value_usd": 76.945343439
  },
  "legacy_regrade": {
    "attempts": 96,
    "scored": 89,
    "changed_rewards": 0,
    "paid_calls": 0
  },
  "initial_tau_outcomes": [
    {
      "domain": "retail",
      "task": "74",
      "seed": 420,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 52,
      "wall_seconds": 121.3956798750005,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "74",
      "seed": 420,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 18,
      "wall_seconds": 5.334619000001112,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "74",
      "seed": 420,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 57,
      "wall_seconds": 131.1813780000084,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "74",
      "seed": 421,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 56,
      "wall_seconds": 139.6566114169982,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "74",
      "seed": 421,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 55,
      "wall_seconds": 124.79407416698814,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "74",
      "seed": 421,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 18,
      "wall_seconds": 4.197786415999872,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "60",
      "seed": 420,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 12,
      "wall_seconds": 36.126505583000835,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "60",
      "seed": 420,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 35,
      "wall_seconds": 60.53399379100301,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "60",
      "seed": 420,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 35,
      "wall_seconds": 31.597886791991186,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "60",
      "seed": 421,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 50,
      "wall_seconds": 135.23859733300924,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "60",
      "seed": 421,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 15,
      "wall_seconds": 47.001110291006626,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "60",
      "seed": 421,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 36,
      "wall_seconds": 64.91119295799581,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "32",
      "seed": 420,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 44,
      "wall_seconds": 101.00678883299406,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "32",
      "seed": 420,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 36,
      "wall_seconds": 46.78521620899846,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "32",
      "seed": 420,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 15,
      "wall_seconds": 29.88773670900264,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "32",
      "seed": 421,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 15,
      "wall_seconds": 39.19213787499757,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "32",
      "seed": 421,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 39,
      "wall_seconds": 88.041491416996,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "32",
      "seed": 421,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 48,
      "wall_seconds": 67.48581374999776,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "19",
      "seed": 420,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 39,
      "wall_seconds": 90.22761712499778,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "19",
      "seed": 420,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 12,
      "wall_seconds": 32.8267577499937,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "19",
      "seed": 420,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 39,
      "wall_seconds": 9.425805000006221,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "19",
      "seed": 421,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 40,
      "wall_seconds": 110.43312308400346,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "19",
      "seed": 421,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 35,
      "wall_seconds": 79.06977200000256,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "19",
      "seed": 421,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 13,
      "wall_seconds": 32.945430334002594,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_064",
      "seed": 420,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 64,
      "wall_seconds": 193.3003095000022,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_064",
      "seed": 420,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 136,
      "wall_seconds": 382.0738702499948,
      "failure_category": "provider_http_520",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_064",
      "seed": 420,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 227,
      "wall_seconds": 691.3173980419961,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_064",
      "seed": 421,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 299,
      "wall_seconds": 888.165334624995,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_064",
      "seed": 421,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 77,
      "wall_seconds": 209.92478679100168,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_064",
      "seed": 421,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 235,
      "wall_seconds": 665.755133041006,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_097",
      "seed": 420,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 164,
      "wall_seconds": 470.0741836249945,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_097",
      "seed": 420,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 480,
      "wall_seconds": 1547.1944672919926,
      "failure_category": "callback_guard",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_097",
      "seed": 420,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 35,
      "wall_seconds": 86.17277504199592,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_097",
      "seed": 421,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 62,
      "wall_seconds": 168.27544112500618,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_097",
      "seed": 421,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 167,
      "wall_seconds": 456.8308353750035,
      "failure_category": "missing_invoice_cost",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_097",
      "seed": 421,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 6,
      "wall_seconds": 1.0131228750105947,
      "failure_category": "cumulative_fee_guard",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_075",
      "seed": 410,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 1,
      "wall_seconds": 0.42644700000528246,
      "failure_category": "cumulative_fee_guard",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_075",
      "seed": 410,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 1,
      "wall_seconds": 0.44024266600899864,
      "failure_category": "cumulative_fee_guard",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_075",
      "seed": 410,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 1,
      "wall_seconds": 0.43751495800097473,
      "failure_category": "cumulative_fee_guard",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_039",
      "seed": 420,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 2,
      "wall_seconds": 0.43692504199862015,
      "failure_category": "cumulative_fee_guard",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_039",
      "seed": 420,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 2,
      "wall_seconds": 0.4992381670017494,
      "failure_category": "cumulative_fee_guard",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_039",
      "seed": 420,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 2,
      "wall_seconds": 0.6334966659924248,
      "failure_category": "cumulative_fee_guard",
      "attempt": "initial"
    }
  ],
  "transfer_diagnostics": [
    {
      "domain": "banking_knowledge",
      "task": "task_075",
      "seed": 410,
      "controller": "v12",
      "resource_profile": "Original200steps; nonbinding1280callbackguard",
      "retained_terminal_episode": false,
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 29,
      "wall_seconds": 84.86816491700301,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_075",
      "seed": 410,
      "controller": "v12",
      "resource_profile": "Original200steps; nonbinding1280callbackguard",
      "retained_terminal_episode": false,
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 138,
      "wall_seconds": 386.4411273750011,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_075",
      "seed": 410,
      "controller": "v12",
      "resource_profile": "Original200steps; nonbinding1280callbackguard",
      "retained_terminal_episode": false,
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 57,
      "wall_seconds": 135.32654220800032,
      "failure_category": "provider_http_422",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_039",
      "seed": 420,
      "controller": "v12",
      "resource_profile": "Original200steps; nonbinding1280callbackguard",
      "retained_terminal_episode": false,
      "transport_continuation": true,
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 189,
      "wall_seconds": 79.9741120839899,
      "failure_category": null,
      "attempt": "transport continuation"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_039",
      "seed": 420,
      "controller": "v12",
      "resource_profile": "Original200steps; nonbinding1280callbackguard",
      "retained_terminal_episode": false,
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 87,
      "wall_seconds": 272.0189974160021,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_039",
      "seed": 420,
      "controller": "v12",
      "resource_profile": "Original200steps; nonbinding1280callbackguard",
      "retained_terminal_episode": false,
      "transport_continuation": true,
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 187,
      "wall_seconds": 381.4325399160007,
      "failure_category": null,
      "attempt": "transport continuation"
    }
  ],
  "transport_continuations": [
    {
      "original": {
        "domain": "banking_knowledge",
        "task": "task_064",
        "seed": 420,
        "controller": "v8",
        "arm": "perplexity/pplx-decider-v1.1-27b",
        "mode": "reflection-revision",
        "split": "heldout",
        "actor_model": "google/gemini-3.8-flash",
        "score": null,
        "termination_reason": "infrastructure_error",
        "logical_calls": 136,
        "wall_seconds": 382.0738702499948,
        "failure_category": "provider_http_520",
        "attempt": "initial"
      },
      "continuation": {
        "domain": "banking_knowledge",
        "task": "task_064",
        "seed": 420,
        "controller": "v8",
        "arm": "perplexity/pplx-decider-v1.1-27b",
        "mode": "reflection-revision",
        "split": "heldout",
        "actor_model": "google/gemini-3.8-flash",
        "score": 0.0,
        "termination_reason": "user_stop",
        "logical_calls": 313,
        "wall_seconds": 603.2345347500086,
        "failure_category": null,
        "attempt": "transport continuation"
      },
      "reason": "Physical HTTP/spend interruption only; all such failed cases retried once, without task or outcome selection."
    }
  ],
  "historical_variants": {
    "ascii_native": [
      {
        "domain": "IFEval",
        "controller": "ascii_native",
        "mode": "actor-only",
        "arm": "actor-only",
        "actor_model": "google/gemini-3.8-flash",
        "attempted": 541,
        "scored": 540,
        "unscored": 1,
        "mean_score": 0.9425925925925925,
        "successes": 509,
        "all_attempt_verified_success_rate": 0.9408502772643254,
        "costs": {
          "logical_calls": 0,
          "application_cache_hits": 0,
          "physical_calls": 0,
          "actual_new_cost_usd": 0,
          "unique_response_value_usd": 0
        },
        "termination_counts": {
          "prediction_completed": 540,
          "unscored": 1
        },
        "strict_instruction_accuracy": 0.9602888086642599,
        "loose_prompt_accuracy": 0.9611111111111111,
        "loose_instruction_accuracy": 0.9723225030084236,
        "repair_model_calls": 0,
        "changed_responses": 0,
        "final_episode_costs": {
          "logical_calls": 0,
          "application_cache_hits": 0,
          "physical_calls": 0,
          "actual_new_cost_usd": 0,
          "unique_response_value_usd": 0
        },
        "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
      },
      {
        "domain": "IFEval",
        "controller": "ascii_native",
        "mode": "reflection-revision",
        "arm": "perplexity/pplx-decider-v1.1-27b",
        "actor_model": "google/gemini-3.8-flash",
        "attempted": 541,
        "scored": 540,
        "unscored": 1,
        "mean_score": 0.9425925925925925,
        "successes": 509,
        "all_attempt_verified_success_rate": 0.9408502772643254,
        "costs": {
          "logical_calls": 1212,
          "application_cache_hits": 555,
          "physical_calls": 657,
          "actual_new_cost_usd": 0.30301851,
          "unique_response_value_usd": 1.07987465
        },
        "termination_counts": {
          "prediction_completed": 540,
          "unscored": 1
        },
        "strict_instruction_accuracy": 0.9614921780986763,
        "loose_prompt_accuracy": 0.9555555555555556,
        "loose_instruction_accuracy": 0.9699157641395909,
        "repair_model_calls": 0,
        "changed_responses": 0,
        "final_episode_costs": {
          "logical_calls": 1212,
          "application_cache_hits": 555,
          "physical_calls": 657,
          "actual_new_cost_usd": 0.30301851,
          "unique_response_value_usd": 1.07987465
        },
        "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
      },
      {
        "domain": "IFEval",
        "controller": "ascii_native",
        "mode": "reflection-revision",
        "arm": "cloudflare/clef-flash",
        "actor_model": "google/gemini-3.8-flash",
        "attempted": 541,
        "scored": 538,
        "unscored": 3,
        "mean_score": 0.9405204460966543,
        "successes": 506,
        "all_attempt_verified_success_rate": 0.9353049907578558,
        "costs": {
          "logical_calls": 1207,
          "application_cache_hits": 527,
          "physical_calls": 680,
          "actual_new_cost_usd": 0.382977604,
          "unique_response_value_usd": 1.03941394
        },
        "termination_counts": {
          "prediction_completed": 538,
          "unscored": 3
        },
        "strict_instruction_accuracy": 0.9601930036188179,
        "loose_prompt_accuracy": 0.9553903345724907,
        "loose_instruction_accuracy": 0.9698431845597105,
        "repair_model_calls": 0,
        "changed_responses": 0,
        "final_episode_costs": {
          "logical_calls": 1207,
          "application_cache_hits": 527,
          "physical_calls": 680,
          "actual_new_cost_usd": 0.382977604,
          "unique_response_value_usd": 1.03941394
        },
        "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
      }
    ],
    "ascii_checks": [
      {
        "domain": "IFEval",
        "controller": "ascii_checks",
        "mode": "actor-only",
        "arm": "actor-only",
        "actor_model": "google/gemini-3.8-flash",
        "attempted": 541,
        "scored": 540,
        "unscored": 1,
        "mean_score": 0.9425925925925925,
        "successes": 509,
        "all_attempt_verified_success_rate": 0.9408502772643254,
        "costs": {
          "logical_calls": 0,
          "application_cache_hits": 0,
          "physical_calls": 0,
          "actual_new_cost_usd": 0,
          "unique_response_value_usd": 0
        },
        "termination_counts": {
          "prediction_completed": 540,
          "unscored": 1
        },
        "strict_instruction_accuracy": 0.9602888086642599,
        "loose_prompt_accuracy": 0.9611111111111111,
        "loose_instruction_accuracy": 0.9723225030084236,
        "repair_model_calls": 0,
        "changed_responses": 7,
        "final_episode_costs": {
          "logical_calls": 0,
          "application_cache_hits": 0,
          "physical_calls": 0,
          "actual_new_cost_usd": 0,
          "unique_response_value_usd": 0
        },
        "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
      },
      {
        "domain": "IFEval",
        "controller": "ascii_checks",
        "mode": "reflection-revision",
        "arm": "perplexity/pplx-decider-v1.1-27b",
        "actor_model": "google/gemini-3.8-flash",
        "attempted": 541,
        "scored": 540,
        "unscored": 1,
        "mean_score": 0.9444444444444444,
        "successes": 510,
        "all_attempt_verified_success_rate": 0.9426987060998152,
        "costs": {
          "logical_calls": 0,
          "application_cache_hits": 0,
          "physical_calls": 0,
          "actual_new_cost_usd": 0,
          "unique_response_value_usd": 0
        },
        "termination_counts": {
          "prediction_completed": 540,
          "unscored": 1
        },
        "strict_instruction_accuracy": 0.9626955475330926,
        "loose_prompt_accuracy": 0.9555555555555556,
        "loose_instruction_accuracy": 0.9699157641395909,
        "repair_model_calls": 0,
        "changed_responses": 6,
        "final_episode_costs": {
          "logical_calls": 0,
          "application_cache_hits": 0,
          "physical_calls": 0,
          "actual_new_cost_usd": 0,
          "unique_response_value_usd": 0
        },
        "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
      },
      {
        "domain": "IFEval",
        "controller": "ascii_checks",
        "mode": "reflection-revision",
        "arm": "cloudflare/clef-flash",
        "actor_model": "google/gemini-3.8-flash",
        "attempted": 541,
        "scored": 538,
        "unscored": 3,
        "mean_score": 0.9405204460966543,
        "successes": 506,
        "all_attempt_verified_success_rate": 0.9353049907578558,
        "costs": {
          "logical_calls": 1,
          "application_cache_hits": 0,
          "physical_calls": 1,
          "actual_new_cost_usd": 0.00270075,
          "unique_response_value_usd": 0.00270075
        },
        "termination_counts": {
          "prediction_completed": 538,
          "unscored": 3
        },
        "strict_instruction_accuracy": 0.9601930036188179,
        "loose_prompt_accuracy": 0.9591078066914498,
        "loose_instruction_accuracy": 0.9722557297949337,
        "repair_model_calls": 1,
        "changed_responses": 8,
        "final_episode_costs": {
          "logical_calls": 1,
          "application_cache_hits": 0,
          "physical_calls": 1,
          "actual_new_cost_usd": 0.00270075,
          "unique_response_value_usd": 0.00270075
        },
        "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
      }
    ],
    "rejected_extended_output_checks": [
      {
        "domain": "IFEval",
        "controller": "v11-rejected",
        "mode": "actor-with-checks",
        "arm": "actor-only",
        "actor_model": "google/gemini-3.8-flash",
        "attempted": 541,
        "scored": 541,
        "unscored": 0,
        "mean_score": 0.9426987060998152,
        "successes": 510,
        "all_attempt_verified_success_rate": 0.9426987060998152,
        "costs": {
          "logical_calls": 2,
          "application_cache_hits": 0,
          "physical_calls": 2,
          "actual_new_cost_usd": 0.00234375,
          "unique_response_value_usd": 0.00234375
        },
        "termination_counts": {
          "prediction_completed": 541
        },
        "strict_instruction_accuracy": 0.960431654676259,
        "loose_prompt_accuracy": 0.9593345656192237,
        "loose_instruction_accuracy": 0.9712230215827338,
        "repair_model_calls": 2,
        "changed_responses": 7,
        "final_episode_costs": {
          "logical_calls": 2,
          "application_cache_hits": 0,
          "physical_calls": 2,
          "actual_new_cost_usd": 0.00234375,
          "unique_response_value_usd": 0.00234375
        },
        "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
      },
      {
        "domain": "IFEval",
        "controller": "v11-rejected",
        "mode": "reflection-revision-checks",
        "arm": "perplexity/pplx-decider-v1.1-27b",
        "actor_model": "google/gemini-3.8-flash",
        "attempted": 541,
        "scored": 541,
        "unscored": 0,
        "mean_score": 0.9463955637707948,
        "successes": 512,
        "all_attempt_verified_success_rate": 0.9463955637707948,
        "costs": {
          "logical_calls": 3,
          "application_cache_hits": 3,
          "physical_calls": 0,
          "actual_new_cost_usd": 0,
          "unique_response_value_usd": 0.00398925
        },
        "termination_counts": {
          "prediction_completed": 541
        },
        "strict_instruction_accuracy": 0.9640287769784173,
        "loose_prompt_accuracy": 0.9537892791127541,
        "loose_instruction_accuracy": 0.9688249400479616,
        "repair_model_calls": 3,
        "changed_responses": 6,
        "final_episode_costs": {
          "logical_calls": 3,
          "application_cache_hits": 3,
          "physical_calls": 0,
          "actual_new_cost_usd": 0,
          "unique_response_value_usd": 0.00398925
        },
        "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
      },
      {
        "domain": "IFEval",
        "controller": "v11-rejected",
        "mode": "reflection-revision-checks",
        "arm": "cloudflare/clef-flash",
        "actor_model": "google/gemini-3.8-flash",
        "attempted": 541,
        "scored": 541,
        "unscored": 0,
        "mean_score": 0.9408502772643254,
        "successes": 509,
        "all_attempt_verified_success_rate": 0.9408502772643254,
        "costs": {
          "logical_calls": 4,
          "application_cache_hits": 2,
          "physical_calls": 2,
          "actual_new_cost_usd": 0.0023655,
          "unique_response_value_usd": 0.006234
        },
        "termination_counts": {
          "prediction_completed": 541
        },
        "strict_instruction_accuracy": 0.960431654676259,
        "loose_prompt_accuracy": 0.9574861367837338,
        "loose_instruction_accuracy": 0.9712230215827338,
        "repair_model_calls": 4,
        "changed_responses": 8,
        "final_episode_costs": {
          "logical_calls": 4,
          "application_cache_hits": 2,
          "physical_calls": 2,
          "actual_new_cost_usd": 0.0023655,
          "unique_response_value_usd": 0.006234
        },
        "cost_scope": "Incremental recorded profile charges; shared or previously metered generation remains in the study total."
      }
    ]
  },
  "scope": "Two development-disjoint public tasks per tau domain with two seeds each; one previously studied ITBench incident with two seeds and local root-group scoring; all 541 original IFEval prompts with unmodified strict/loose graders. Shared Gemini 3.8 Flash generation/simulator roles through Google AI Studio. Decider and Clef Flash differ only as the native selector within each profile. UTF-8 JSON and optional visible-output checks use the same recorded proposals and criteria. Agent-task slices and this harness do not establish an overall leaderboard rank.",
  "accounting_boundary": "Fees include all preparation, superseded pilots, failures with returned invoices, original and continuation predictions, grading and diagnostics. Tau-domain fees include all same-cohort attempts and continuations; other profiles show incremental recorded charges. Shared generation remains included in the total. Cache denominators count recorded successful responses; unresolved failed requests have a separate estimated reserve. Returned fees already include provider-cache charging. These are warm-order measurements, not cold-run prices.",
  "provider_context_caveat": "OpenRouter documents a roughly 2K-token text-state truncation caveat for the Clef route; Cloudflare describes a 24K total hosted Clef Flash window. Identical API input does not prove identical effective context. No claim that every supplied source token was read.",
  "source_freezes_verified": {
    "original": 30,
    "it_utf8": 3,
    "bank-transport-plan.json": 34,
    "bank-fullstep-plan.json": 33,
    "ifeval-full-plan.json": 32,
    "ifeval-utf8-plan.json": 33,
    "contract-refinement-plan-v1.json": 5,
    "contract-refinement-plan.json": 5,
    "utf8-contract-refinement-plan.json": 5,
    "final-output-refinement-plan.json": 6
  },
  "source_pins": {
    "tau2": "fc0055dc4e0a316c3f83133267fbd6faaa770992",
    "IFEval": "e6890f85757dd84e27ca6df2dd30651dafad28e0"
  },
  "operational_amendments": {
    "cumulative_new_spend_guard_usd": 100,
    "original_source_freeze_unchanged": true,
    "independent_ifeval_case_workers_per_variant": 4,
    "task_ids_and_seeds_preserved": true,
    "original_tau_within_case_arm_order_preserved": true,
    "native_unicode_change": "Every decoded state/candidate/review value is asserted equal; only nested Unicode representation changes.",
    "public_contract_change": "Partial checks use explicit original prompt spans. No hidden verifier IDs, kwargs or expected answers enter inference. Up to one repair, identical generator/provider, and fallback preserves the recorded response. No native action is changed."
  },
  "unassigned_shared_and_development_cost_usd": 15.451965928999996,
  "condition_cost_scope": "Tau-domain rows include every recorded same-task/seed/arm attempt, including interrupted originals and continuations. Other profile rows show incremental charges. Shared generation, diagnostics and superseded experimental fees remain in the total. Warm-order measurements are not cold-run prices.",
  "superseded_pilots": {
    "qwen_time_guard": [
      {
        "domain": "banking_knowledge",
        "task": "task_075",
        "seed": 410,
        "controller": "v6",
        "arm": "perplexity/pplx-decider-v1.1-27b",
        "score": 0.0,
        "termination_reason": "user_stop",
        "wall_seconds": 412.869428334001,
        "failure_category": null,
        "attempt": "initial"
      },
      {
        "domain": "banking_knowledge",
        "task": "task_075",
        "seed": 410,
        "controller": "v6",
        "arm": "cloudflare/clef-flash",
        "score": 0.0,
        "termination_reason": "user_stop",
        "wall_seconds": 277.6108492919957,
        "failure_category": null,
        "attempt": "initial"
      },
      {
        "domain": "banking_knowledge",
        "task": "task_075",
        "seed": 410,
        "controller": "v7",
        "arm": "perplexity/pplx-decider-v1.1-27b",
        "score": 0.0,
        "termination_reason": "timeout",
        "wall_seconds": 917.5842307499988,
        "failure_category": null,
        "attempt": "initial"
      },
      {
        "domain": "banking_knowledge",
        "task": "task_075",
        "seed": 410,
        "controller": "v7",
        "arm": "cloudflare/clef-flash",
        "score": null,
        "termination_reason": "infrastructure_error",
        "wall_seconds": 503.6952462499976,
        "failure_category": "infrastructure_error",
        "attempt": "initial"
      }
    ],
    "short_resource_guard": [
      {
        "domain": "banking_knowledge",
        "task": "task_039",
        "seed": 420,
        "controller": "v8",
        "arm": "actor-only",
        "mode": "actor-only",
        "split": "heldout",
        "actor_model": "google/gemini-3.8-flash",
        "score": 0.0,
        "termination_reason": "max_steps",
        "logical_calls": 37,
        "wall_seconds": 118.64834495799732,
        "failure_category": null,
        "attempt": "initial"
      },
      {
        "domain": "banking_knowledge",
        "task": "task_039",
        "seed": 420,
        "controller": "v8",
        "arm": "perplexity/pplx-decider-v1.1-27b",
        "mode": "reflection-revision",
        "split": "heldout",
        "actor_model": "google/gemini-3.8-flash",
        "score": null,
        "termination_reason": "infrastructure_error",
        "logical_calls": 120,
        "wall_seconds": 354.59370129200397,
        "failure_category": "callback_guard",
        "attempt": "initial"
      },
      {
        "domain": "banking_knowledge",
        "task": "task_039",
        "seed": 420,
        "controller": "v8",
        "arm": "cloudflare/clef-flash",
        "mode": "reflection-revision",
        "split": "heldout",
        "actor_model": "google/gemini-3.8-flash",
        "score": null,
        "termination_reason": "infrastructure_error",
        "logical_calls": 120,
        "wall_seconds": 269.85448233399075,
        "failure_category": "callback_guard",
        "attempt": "initial"
      }
    ]
  },
  "rejected_experiment_reason": "The broader output-contract profile scored 510/512/509, versus 511/512/510 for the minimal common profile, and used nine logical repair calls versus one. The smaller profile is retained; all rejected outcomes and fees remain included.",
  "transfer_diagnostic_coverage": {
    "attempted": 6,
    "scored": 5,
    "failure_categories": {
      "provider_http_422": 1
    }
  },
  "bank_transport_completion": {
    "status": "complete",
    "episodes": 18,
    "continuations": 3,
    "scored": 17,
    "source_freeze_verified": true
  },
  "bank_transport_recoveries": [
    {
      "original": {
        "domain": "banking_knowledge",
        "task": "task_097",
        "seed": 421,
        "controller": "v12",
        "resource_profile": "Original200steps; nonbinding1280callbackguard",
        "retained_terminal_episode": false,
        "arm": "perplexity/pplx-decider-v1.1-27b",
        "mode": "reflection-revision",
        "split": "heldout",
        "actor_model": "google/gemini-3.8-flash",
        "score": null,
        "termination_reason": "infrastructure_error",
        "logical_calls": 105,
        "wall_seconds": 209.3977751670027,
        "failure_category": "missing_invoice_cost",
        "attempt": "initial"
      },
      "continuation": {
        "domain": "banking_knowledge",
        "task": "task_097",
        "seed": 421,
        "controller": "v12",
        "resource_profile": "Original200steps; nonbinding1280callbackguard",
        "retained_terminal_episode": false,
        "transport_continuation": true,
        "arm": "perplexity/pplx-decider-v1.1-27b",
        "mode": "reflection-revision",
        "split": "heldout",
        "actor_model": "google/gemini-3.8-flash",
        "score": 1.0,
        "termination_reason": "user_stop",
        "logical_calls": 179,
        "wall_seconds": 260.8656283750024,
        "failure_category": null,
        "attempt": "transport continuation"
      },
      "reason": "Once-only physical transport or missing-invoice continuation; never retried on score."
    },
    {
      "original": {
        "domain": "banking_knowledge",
        "task": "task_039",
        "seed": 420,
        "controller": "v12",
        "resource_profile": "Original200steps; nonbinding1280callbackguard",
        "retained_terminal_episode": false,
        "arm": "cloudflare/clef-flash",
        "mode": "reflection-revision",
        "split": "transfer_diagnostic",
        "actor_model": "google/gemini-3.8-flash",
        "score": null,
        "termination_reason": "infrastructure_error",
        "logical_calls": 172,
        "wall_seconds": 507.4506176249997,
        "failure_category": "provider_incomplete_response",
        "attempt": "initial"
      },
      "continuation": {
        "domain": "banking_knowledge",
        "task": "task_039",
        "seed": 420,
        "controller": "v12",
        "resource_profile": "Original200steps; nonbinding1280callbackguard",
        "retained_terminal_episode": false,
        "transport_continuation": true,
        "arm": "cloudflare/clef-flash",
        "mode": "reflection-revision",
        "split": "transfer_diagnostic",
        "actor_model": "google/gemini-3.8-flash",
        "score": 0.0,
        "termination_reason": "user_stop",
        "logical_calls": 189,
        "wall_seconds": 79.9741120839899,
        "failure_category": null,
        "attempt": "transport continuation"
      },
      "reason": "Once-only physical transport or missing-invoice continuation; never retried on score."
    },
    {
      "original": {
        "domain": "banking_knowledge",
        "task": "task_039",
        "seed": 420,
        "controller": "v12",
        "resource_profile": "Original200steps; nonbinding1280callbackguard",
        "retained_terminal_episode": false,
        "arm": "perplexity/pplx-decider-v1.1-27b",
        "mode": "reflection-revision",
        "split": "transfer_diagnostic",
        "actor_model": "google/gemini-3.8-flash",
        "score": null,
        "termination_reason": "infrastructure_error",
        "logical_calls": 74,
        "wall_seconds": 181.31621704099234,
        "failure_category": "missing_invoice_cost",
        "attempt": "initial"
      },
      "continuation": {
        "domain": "banking_knowledge",
        "task": "task_039",
        "seed": 420,
        "controller": "v12",
        "resource_profile": "Original200steps; nonbinding1280callbackguard",
        "retained_terminal_episode": false,
        "transport_continuation": true,
        "arm": "perplexity/pplx-decider-v1.1-27b",
        "mode": "reflection-revision",
        "split": "transfer_diagnostic",
        "actor_model": "google/gemini-3.8-flash",
        "score": 0.0,
        "termination_reason": "user_stop",
        "logical_calls": 187,
        "wall_seconds": 381.4325399160007,
        "failure_category": null,
        "attempt": "transport continuation"
      },
      "reason": "Once-only physical transport or missing-invoice continuation; never retried on score."
    }
  ],
  "original_resource_tau_outcomes": [
    {
      "domain": "retail",
      "task": "74",
      "seed": 420,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 52,
      "wall_seconds": 121.3956798750005,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "74",
      "seed": 420,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 18,
      "wall_seconds": 5.334619000001112,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "74",
      "seed": 420,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 57,
      "wall_seconds": 131.1813780000084,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "74",
      "seed": 421,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 56,
      "wall_seconds": 139.6566114169982,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "74",
      "seed": 421,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 55,
      "wall_seconds": 124.79407416698814,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "74",
      "seed": 421,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 18,
      "wall_seconds": 4.197786415999872,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "60",
      "seed": 420,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 12,
      "wall_seconds": 36.126505583000835,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "60",
      "seed": 420,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 35,
      "wall_seconds": 60.53399379100301,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "60",
      "seed": 420,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 35,
      "wall_seconds": 31.597886791991186,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "60",
      "seed": 421,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 50,
      "wall_seconds": 135.23859733300924,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "60",
      "seed": 421,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 15,
      "wall_seconds": 47.001110291006626,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "retail",
      "task": "60",
      "seed": 421,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 36,
      "wall_seconds": 64.91119295799581,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "32",
      "seed": 420,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 44,
      "wall_seconds": 101.00678883299406,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "32",
      "seed": 420,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 36,
      "wall_seconds": 46.78521620899846,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "32",
      "seed": 420,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 15,
      "wall_seconds": 29.88773670900264,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "32",
      "seed": 421,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 15,
      "wall_seconds": 39.19213787499757,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "32",
      "seed": 421,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 39,
      "wall_seconds": 88.041491416996,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "32",
      "seed": 421,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 48,
      "wall_seconds": 67.48581374999776,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "19",
      "seed": 420,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 39,
      "wall_seconds": 90.22761712499778,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "19",
      "seed": 420,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 12,
      "wall_seconds": 32.8267577499937,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "19",
      "seed": 420,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 39,
      "wall_seconds": 9.425805000006221,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "19",
      "seed": 421,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 40,
      "wall_seconds": 110.43312308400346,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "19",
      "seed": 421,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 35,
      "wall_seconds": 79.06977200000256,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "airline",
      "task": "19",
      "seed": 421,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "termination_reason": "user_stop",
      "logical_calls": 13,
      "wall_seconds": 32.945430334002594,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_064",
      "seed": 420,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 64,
      "wall_seconds": 193.3003095000022,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_064",
      "seed": 420,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 313,
      "wall_seconds": 603.2345347500086,
      "failure_category": null,
      "attempt": "transport continuation"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_064",
      "seed": 420,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 227,
      "wall_seconds": 691.3173980419961,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_064",
      "seed": 421,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 299,
      "wall_seconds": 888.165334624995,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_064",
      "seed": 421,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 77,
      "wall_seconds": 209.92478679100168,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_064",
      "seed": 421,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 235,
      "wall_seconds": 665.755133041006,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_097",
      "seed": 420,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 164,
      "wall_seconds": 470.0741836249945,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_097",
      "seed": 420,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 480,
      "wall_seconds": 1547.1944672919926,
      "failure_category": "callback_guard",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_097",
      "seed": 420,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 35,
      "wall_seconds": 86.17277504199592,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_097",
      "seed": 421,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.0,
      "termination_reason": "user_stop",
      "logical_calls": 62,
      "wall_seconds": 168.27544112500618,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_097",
      "seed": 421,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 167,
      "wall_seconds": 456.8308353750035,
      "failure_category": "missing_invoice_cost",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_097",
      "seed": 421,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "heldout",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 6,
      "wall_seconds": 1.0131228750105947,
      "failure_category": "cumulative_fee_guard",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_075",
      "seed": 410,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 1,
      "wall_seconds": 0.42644700000528246,
      "failure_category": "cumulative_fee_guard",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_075",
      "seed": 410,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 1,
      "wall_seconds": 0.44024266600899864,
      "failure_category": "cumulative_fee_guard",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_075",
      "seed": 410,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 1,
      "wall_seconds": 0.43751495800097473,
      "failure_category": "cumulative_fee_guard",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_039",
      "seed": 420,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 2,
      "wall_seconds": 0.43692504199862015,
      "failure_category": "cumulative_fee_guard",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_039",
      "seed": 420,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 2,
      "wall_seconds": 0.4992381670017494,
      "failure_category": "cumulative_fee_guard",
      "attempt": "initial"
    },
    {
      "domain": "banking_knowledge",
      "task": "task_039",
      "seed": 420,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "split": "transfer_diagnostic",
      "actor_model": "google/gemini-3.8-flash",
      "score": null,
      "termination_reason": "infrastructure_error",
      "logical_calls": 2,
      "wall_seconds": 0.6334966659924248,
      "failure_category": "cumulative_fee_guard",
      "attempt": "initial"
    }
  ],
  "bank_original_step_completion": {
    "status": "complete",
    "episodes": 18,
    "source_freeze_verified": true,
    "scored": 14,
    "retained_terminal": 9,
    "max_steps": 200,
    "logical_call_guard": 1280,
    "retention": "Every naturally terminal prior episode is retained, irrespective of reward. Incomplete or operationally interrupted episodes resume identical cached requests under the common enlarged nonbinding guard."
  },
  "pool_diagnostics": {
    "boundary": "These are post-generation oracle diagnostics. The ceiling is not achieved performance and no checker output entered a model request.",
    "IFEval": [
      {
        "arm": "actor-only",
        "cases": 541,
        "achieved_mean_score": 0.9426987060998152,
        "post_execution_pool_ceiling": 0.9426987060998152,
        "missed_better_candidate_cases": 0
      },
      {
        "arm": "perplexity/pplx-decider-v1.1-27b",
        "cases": 541,
        "achieved_mean_score": 0.944547134935305,
        "post_execution_pool_ceiling": 0.9648798521256932,
        "missed_better_candidate_cases": 11
      },
      {
        "arm": "cloudflare/clef-flash",
        "cases": 541,
        "achieved_mean_score": 0.9390018484288355,
        "post_execution_pool_ceiling": 0.9648798521256932,
        "missed_better_candidate_cases": 15
      }
    ],
    "ITBench": [
      {
        "arm": "actor-only",
        "seed": 420,
        "achieved": 1.0,
        "post_execution_pool_ceiling": 1.0
      },
      {
        "arm": "perplexity/pplx-decider-v1.1-27b",
        "seed": 420,
        "achieved": 0.5,
        "post_execution_pool_ceiling": 1.0
      },
      {
        "arm": "cloudflare/clef-flash",
        "seed": 420,
        "achieved": 0.5,
        "post_execution_pool_ceiling": 1.0
      },
      {
        "arm": "actor-only",
        "seed": 421,
        "achieved": 0.5,
        "post_execution_pool_ceiling": 0.5
      },
      {
        "arm": "cloudflare/clef-flash",
        "seed": 421,
        "achieved": 0.5,
        "post_execution_pool_ceiling": 0.5
      },
      {
        "arm": "perplexity/pplx-decider-v1.1-27b",
        "seed": 421,
        "achieved": 0.5,
        "post_execution_pool_ceiling": 0.5
      }
    ]
  },
  "original_it_outcomes": [
    {
      "domain": "ITBench",
      "task": "Scenario-11",
      "seed": 420,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "actor_model": "google/gemini-3.8-flash",
      "score": 1.0,
      "logical_calls": 12,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "ITBench",
      "task": "Scenario-11",
      "seed": 420,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.5,
      "logical_calls": 15,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "ITBench",
      "task": "Scenario-11",
      "seed": 420,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.5,
      "logical_calls": 15,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "ITBench",
      "task": "Scenario-11",
      "seed": 421,
      "controller": "v8",
      "arm": "actor-only",
      "mode": "actor-only",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.5,
      "logical_calls": 4,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "ITBench",
      "task": "Scenario-11",
      "seed": 421,
      "controller": "v8",
      "arm": "cloudflare/clef-flash",
      "mode": "reflection-revision",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.5,
      "logical_calls": 14,
      "failure_category": null,
      "attempt": "initial"
    },
    {
      "domain": "ITBench",
      "task": "Scenario-11",
      "seed": 421,
      "controller": "v8",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "mode": "reflection-revision",
      "actor_model": "google/gemini-3.8-flash",
      "score": 0.5,
      "logical_calls": 14,
      "failure_category": null,
      "attempt": "initial"
    }
  ],
  "measured_public_modules": [
    {
      "path": "/explore/research-loop.mjs",
      "sha256": "39b95f47287ef880151e00b49030569c53cf671272888a34593b3b1b44e4be81"
    },
    {
      "path": "/explore/visible-response-contract.mjs",
      "sha256": "f7c523cd989da263c5d6a29e75782395edb3a348505ecff78809cc4823229b15"
    }
  ],
  "provider_context_sources": [
    "https://openrouter.ai/cloudflare/clef-flash",
    "https://blog.cloudflare.com/clef-faster-cheaper-multimodal/"
  ],
  "loop_guard_scope": "The progress guard detects exact operation repeats without changed observations. Paraphrased queries and overlapping retrieval results can still prolong an episode. Zero selected exact no-progress repeats is not proof of useful task progress.",
  "evaluation_scope": "Exploratory evaluation on reused public benchmarks. The smaller output-check profile was retained after comparing variants on the same IFEval dataset. No untouched independent holdout or generalization improvement is claimed. Original hidden checker identifiers, kwargs and target answers never enter inference.",
  "paired_test_scope": "Exploratory two-sided exact paired sign/discordance tests on complete pairs; unadjusted, without a general superiority or leaderboard claim.",
  "grader_source_verification": {
    "commit": "e6890f85757dd84e27ca6df2dd30651dafad28e0",
    "original_modules": [
      {
        "module": "evaluation_lib",
        "source_url": "https://raw.githubusercontent.com/google-research/google-research/e6890f85757dd84e27ca6df2dd30651dafad28e0/instruction_following_eval/evaluation_lib.py",
        "sha256": "35decc06000718487f44d7deafa6d3f48a8ec0886281edf40162c0265b7d248c",
        "matches_original": true
      },
      {
        "module": "instructions",
        "source_url": "https://raw.githubusercontent.com/google-research/google-research/e6890f85757dd84e27ca6df2dd30651dafad28e0/instruction_following_eval/instructions.py",
        "sha256": "60e086f5342a03ce8e18b64bbcccf86308f523c08aa826707a562150a52f3edf",
        "matches_original": true
      },
      {
        "module": "instructions_util",
        "source_url": "https://raw.githubusercontent.com/google-research/google-research/e6890f85757dd84e27ca6df2dd30651dafad28e0/instruction_following_eval/instructions_util.py",
        "sha256": "a73797261eee5bf447e279d82a2b700b1bdd3cb1193412dbab1270a85832bc6b",
        "matches_original": true
      },
      {
        "module": "instructions_registry",
        "source_url": "https://raw.githubusercontent.com/google-research/google-research/e6890f85757dd84e27ca6df2dd30651dafad28e0/instruction_following_eval/instructions_registry.py",
        "sha256": "ec92d72c264f6d906978613085db262356174300370a3fffe6fefd5969ce9cfc",
        "matches_original": true
      }
    ],
    "resource_preflight_empty_responses": 541,
    "preflight_not_model_scores": true,
    "versions": {
      "nltk": "3.10.3",
      "langdetect": "1.0.9",
      "absl-py": "2.5.1",
      "immutabledict": "4.3.1"
    }
  },
  "response_artifacts": [
    {
      "path": "/explore/responses/native-actor-only.jsonl",
      "arm": "actor-only",
      "profile": "UTF8",
      "cases": 541,
      "sha256": "d903f727751f79270b726978497138f192fe0c27a7d78ad5a63940124cd46a02",
      "bytes": 1039344
    },
    {
      "path": "/explore/responses/native-decider.jsonl",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "profile": "UTF8",
      "cases": 541,
      "sha256": "0181fd76d4378d18ea68db7ac081fe9480380944933cf5fc52af2b0149eaa591",
      "bytes": 1045272
    },
    {
      "path": "/explore/responses/native-clef-flash.jsonl",
      "arm": "cloudflare/clef-flash",
      "profile": "UTF8",
      "cases": 541,
      "sha256": "98cf90622e3f61c0b55635563a379a435d7e8d26bbe8b292e501fac355b06b98",
      "bytes": 1032858
    },
    {
      "path": "/explore/responses/output-checks-actor-only.jsonl",
      "arm": "actor-only",
      "profile": "V10 checks",
      "cases": 541,
      "sha256": "3a8a93440237df3bb7b07178bec105fa5c33979e3044b1f682e6b2e11f4f1701",
      "bytes": 1039242
    },
    {
      "path": "/explore/responses/output-checks-decider.jsonl",
      "arm": "perplexity/pplx-decider-v1.1-27b",
      "profile": "V10 checks",
      "cases": 541,
      "sha256": "2871f7c8e8d8cd79df1b012c15787a00edd141a9f86b783e75286ebe16b4b169",
      "bytes": 1045186
    },
    {
      "path": "/explore/responses/output-checks-clef-flash.jsonl",
      "arm": "cloudflare/clef-flash",
      "profile": "V10 checks",
      "cases": 541,
      "sha256": "206f2bd52d592adb992781f22a3186c6682b682fc73ba702669ec10119d0c798",
      "bytes": 1032997
    }
  ]
}