{
  "schema_version": 1,
  "report_scope": {
    "prospective_experiments": [
      1,
      2,
      3,
      4,
      5,
      6,
      9,
      11,
      12,
      13,
      14,
      15
    ],
    "retrospective_experiments": [
      7
    ],
    "post_hoc_numbered_analyses": [
      8,
      10
    ],
    "post_hoc_unnumbered_analyses": [
      "prior_art_sentinel",
      "e10b_archetype_substrate_join",
      "e9_substrate_followup"
    ],
    "candidate_dossiers": 59,
    "candidate_endpoint_counts": {
      "EMPIRICAL_PARTNER_CANDIDATE": 32,
      "POST_HOC_STRICT_INNOVATION_LIKE_SURVIVOR": 2,
      "STRICT_SUCCESS": 25
    },
    "candidate_experiment_counts": {
      "3": 2,
      "4": 6,
      "5": 4,
      "6": 47
    }
  },
  "canonical_analysis_sources": {
    "experiment_1": "experiments/inverse_innovation/experiment_01/analysis/calibration_summary.json",
    "experiment_2": "experiments/inverse_innovation/experiment_02/main_20/analysis/results.json",
    "experiment_3": "experiments/inverse_innovation/experiment_03/full_matrix_320/analysis/integrated_results.json",
    "experiment_4": "experiments/inverse_innovation/experiment_04/retrieval_first_paired_20/analysis/result.json",
    "experiment_5": "experiments/inverse_innovation/experiment_05/complete_proposal_portfolio_20/analysis/result.json",
    "experiment_6": "experiments/inverse_innovation/experiment_06/four_proposal_generalization_60/analysis/result.json",
    "experiment_7": "experiments/inverse_innovation/experiment_07/retrospective_selector_60/analysis/result.json",
    "experiment_8": "experiments/inverse_innovation/experiment_08/yield_decomposition_existing_data/results.json",
    "experiment_9": "experiments/inverse_innovation/experiment_09/archetype_breadth_probe_150/analysis/results.json",
    "experiment_10": "experiments/inverse_innovation/experiment_10/proposal_type_disposition_existing_data/results.json",
    "experiment_11": "experiments/inverse_innovation/experiment_11/mechanism_context_external_comparison_20/analysis/results.json",
    "experiment_12": "experiments/inverse_innovation/experiment_12/substrate_denial_paired_72/analysis/primary_results.json",
    "experiment_12_max_followup": "experiments/inverse_innovation/experiment_12/substrate_denial_paired_72/max_followup/analysis/results.json",
    "experiment_12_runtime": "experiments/inverse_innovation/experiment_12/substrate_denial_paired_72/analysis/runtime_summary.json",
    "experiment_13": "experiments/inverse_innovation/experiment_13/second_slot_policy_60/analysis/results.json",
    "experiment_14": "experiments/inverse_innovation/experiment_14/applicability_retrieval_40/analysis/results.json",
    "experiment_15": "experiments/inverse_innovation/experiment_15/route_aware_retrieval_40/analysis/results.json",
    "prior_art_sentinel": "experiments/inverse_innovation/report_hardening_2026-08/prior_art_sentinel/results.jsonl",
    "e10b_archetype_substrate_join": "experiments/inverse_innovation/substrate_analysis_2026-08/e10b/results.json",
    "e9_substrate_followup": "experiments/inverse_innovation/substrate_analysis_2026-08/e9_followup/results.json"
  },
  "headline_facts": {
    "experiment_1_metrics": {
      "A": {
        "n_generation": 60,
        "n_reviews": 60,
        "decisions": {
          "NO_FIT": 9,
          "INDETERMINATE": 5,
          "CANDIDATE": 46
        },
        "gate_pass_n": 34,
        "gate_pass_rate": 0.5667,
        "promoted_n": 22,
        "weighted_total": {
          "mean": 74.12,
          "median": 79.38,
          "q1": 65.0,
          "q3": 85.0
        },
        "fabrication_findings": 4
      },
      "B": {
        "n_generation": 60,
        "n_reviews": 60,
        "decisions": {
          "NO_FIT": 15,
          "INDETERMINATE": 2,
          "CANDIDATE": 43
        },
        "gate_pass_n": 38,
        "gate_pass_rate": 0.6333,
        "promoted_n": 27,
        "weighted_total": {
          "mean": 79.46,
          "median": 84.38,
          "q1": 74.69,
          "q3": 85.0
        },
        "fabrication_findings": 4
      },
      "C": {
        "n_generation": 60,
        "n_reviews": 60,
        "decisions": {
          "NO_FIT": 12,
          "CANDIDATE": 45,
          "INDETERMINATE": 3
        },
        "gate_pass_n": 41,
        "gate_pass_rate": 0.6833,
        "promoted_n": 30,
        "weighted_total": {
          "mean": 79.69,
          "median": 82.5,
          "q1": 75.94,
          "q3": 85.0
        },
        "fabrication_findings": 0
      }
    },
    "experiment_1_comparisons": {
      "C_minus_A": {
        "mean": 5.56,
        "median": 3.12,
        "q1": 0.0,
        "q3": 11.25,
        "mean_95pct_bootstrap_ci": [
          3.23,
          7.98
        ]
      },
      "C_minus_B": {
        "mean": 0.23,
        "median": 0.0,
        "q1": -2.81,
        "q3": 5.0,
        "mean_95pct_bootstrap_ci": [
          -1.67,
          2.12
        ]
      }
    },
    "experiment_1_round2_gate": "FAIL",
    "experiment_2_co_primary": {
      "R_vs_S": {
        "cells": 20,
        "wins": 15,
        "losses": 4,
        "ties": 1,
        "decisive_win_rate": 0.7895,
        "wilson_95": [
          0.5667,
          0.9149
        ],
        "one_sided_exact_p": 0.009605,
        "holm_adjusted_p": 0.01921
      },
      "R_vs_D": {
        "cells": 20,
        "wins": 15,
        "losses": 5,
        "ties": 0,
        "decisive_win_rate": 0.75,
        "wilson_95": [
          0.5313,
          0.8881
        ],
        "one_sided_exact_p": 0.020695,
        "holm_adjusted_p": 0.020695
      }
    },
    "experiment_2_evidence_band": "MECHANISM_SPECIFIC_SUPPORT",
    "experiment_2_terminal_quality_means": {
      "S": {
        "conceptual_overall": 3.8,
        "operational_overall": 3.175,
        "causal_component_testability": 3.55
      },
      "R": {
        "conceptual_overall": 3.925,
        "operational_overall": 3.65,
        "causal_component_testability": 3.7417
      },
      "D": {
        "conceptual_overall": 3.7,
        "operational_overall": 3.425,
        "causal_component_testability": 3.6
      }
    },
    "experiment_3_funnel": {
      "matrix_cells": 320,
      "stage1_success": 297,
      "stage1_research_needed": 23,
      "prospective_stage1_cells": 300,
      "prospective_stage1_success": 284,
      "closed_book_adopter_pipeline": 33,
      "external_selected": 48,
      "external_assessed": 47,
      "terminal_nonresponses": 1,
      "externally_verified_pipeline": 21,
      "adjacent_prior_art": 6,
      "no_close_match_found": 0,
      "post_hoc_strict_innovation_like_survivors": 2
    },
    "experiment_4_primary_endpoint": {
      "unit": "primary archetype-domain cell",
      "intention_to_treat_pairs": 20,
      "arms": {
        "PROPOSAL_FIRST": {
          "cells": 20,
          "strict_successes": 5,
          "strict_success_percent": 25.0,
          "strict_success_wilson_95_percent": [
            11.2,
            46.9
          ],
          "externally_verified_adopter_pipeline_regardless_of_prior_art": 11,
          "externally_verified_adopter_pipeline_percent": 55.0,
          "full_evaluations": 20,
          "no_finalist": 0,
          "first_evaluation_known_collisions": 10,
          "first_evaluation_known_collision_percent": 50.0,
          "known_by_or_before_full_evaluation": 10,
          "stop_reasons": {
            "DIFFUSION": 4,
            "EMPIRICAL_RESEARCH_NEEDED": 11,
            "STRICT_SUCCESS": 5
          },
          "final_proximities": {
            "ADJACENT_PRIOR_ART": 10,
            "ESTABLISHED_PRACTICE": 4,
            "SUBSTANTIAL_COLLISION": 6
          },
          "scientific_revisions": 0,
          "balanced_score": {
            "n": 20,
            "total": 61.221,
            "mean": 3.061,
            "median": 3.111,
            "q1": 2.889,
            "q3": 3.333,
            "min": 2.333,
            "max": 3.556
          }
        },
        "RETRIEVAL_FIRST": {
          "cells": 20,
          "strict_successes": 1,
          "strict_success_percent": 5.0,
          "strict_success_wilson_95_percent": [
            0.9,
            23.6
          ],
          "externally_verified_adopter_pipeline_regardless_of_prior_art": 6,
          "externally_verified_adopter_pipeline_percent": 30.0,
          "full_evaluations": 15,
          "no_finalist": 5,
          "first_evaluation_known_collisions": 13,
          "first_evaluation_known_collision_percent": 86.7,
          "known_by_or_before_full_evaluation": 18,
          "stop_reasons": {
            "EMPIRICAL_RESEARCH_NEEDED": 14,
            "NO_FINALIST": 5,
            "STRICT_SUCCESS": 1
          },
          "final_proximities": {
            "ADJACENT_PRIOR_ART": 2,
            "NO_FINALIST": 5,
            "SUBSTANTIAL_COLLISION": 13
          },
          "scientific_revisions": 0,
          "balanced_score": {
            "n": 15,
            "total": 45.0,
            "mean": 3.0,
            "median": 3.0,
            "q1": 2.778,
            "q3": 3.333,
            "min": 2.222,
            "max": 3.556
          }
        }
      },
      "treatment_minus_control_success_count": -4,
      "treatment_minus_control_percentage_points": -20.0,
      "paired_table": {
        "both": 0,
        "control_only": 5,
        "treatment_only": 1,
        "neither": 14,
        "matrix_control_rows_treatment_columns": [
          [
            0,
            5
          ],
          [
            1,
            14
          ]
        ]
      },
      "exact_mcnemar": {
        "discordant_pairs": 6,
        "two_sided_exact_p_value": 0.2188,
        "matched_odds_ratio_treatment_over_control": 0.2,
        "treatment_win_share_among_discordant_percent": 16.7,
        "treatment_win_share_exact_95_percent": [
          0.4,
          64.1
        ],
        "matched_odds_ratio_exact_95": [
          0.004,
          1.786
        ]
      }
    },
    "experiment_5_primary": {
      "p1_success_cells": 1,
      "p1_cell_yield": 0.05,
      "portfolio_success_cells": 4,
      "portfolio_cell_yield": 0.2,
      "absolute_cell_yield_change": 0.15,
      "rescue_count": 3,
      "rescue_denominator_p1_failures": 19,
      "rescue_rate": 0.1579,
      "rescue_rate_wilson_95": [
        0.0552,
        0.3757
      ],
      "strict_proposals_total": 4,
      "additional_strict_proposals_beyond_p1": 3,
      "strict_proposals_at_indices_3_to_5": 2,
      "cumulative_yield_curve": [
        {
          "proposal_index": 1,
          "strict_proposals_at_index": 1,
          "strict_proposal_rate_at_index": 0.05,
          "new_success_cells_at_index": 1,
          "new_success_cell_ids": [
            "layer_decay_and_expiration_management__environmental_climate"
          ],
          "cumulative_success_cells": 1,
          "cumulative_cell_yield": 0.05
        },
        {
          "proposal_index": 2,
          "strict_proposals_at_index": 1,
          "strict_proposal_rate_at_index": 0.05,
          "new_success_cells_at_index": 1,
          "new_success_cell_ids": [
            "layer_decay_and_expiration_management__cognitive_science"
          ],
          "cumulative_success_cells": 2,
          "cumulative_cell_yield": 0.1
        },
        {
          "proposal_index": 3,
          "strict_proposals_at_index": 1,
          "strict_proposal_rate_at_index": 0.05,
          "new_success_cells_at_index": 1,
          "new_success_cell_ids": [
            "layer_decay_and_expiration_management__psychology"
          ],
          "cumulative_success_cells": 3,
          "cumulative_cell_yield": 0.15
        },
        {
          "proposal_index": 4,
          "strict_proposals_at_index": 1,
          "strict_proposal_rate_at_index": 0.05,
          "new_success_cells_at_index": 1,
          "new_success_cell_ids": [
            "computability_boundary_mapping__economics_finance"
          ],
          "cumulative_success_cells": 4,
          "cumulative_cell_yield": 0.2
        },
        {
          "proposal_index": 5,
          "strict_proposals_at_index": 0,
          "strict_proposal_rate_at_index": 0.0,
          "new_success_cells_at_index": 0,
          "new_success_cell_ids": [],
          "cumulative_success_cells": 4,
          "cumulative_cell_yield": 0.2
        }
      ],
      "frozen_interpretation_rule_verdict": "NOT_SUPPORTED"
    },
    "experiment_6_primary": {
      "p1_success_cells": 6,
      "p1_cell_yield": 0.1,
      "portfolio_success_cells": 11,
      "portfolio_cell_yield": 0.1833,
      "absolute_cell_yield_change": 0.0833,
      "rescue_count": 5,
      "rescue_denominator_p1_failures": 54,
      "rescue_rate": 0.0926,
      "rescue_rate_wilson_95": [
        0.0402,
        0.1991
      ],
      "strict_proposals_total": 15,
      "additional_strict_proposals_beyond_p1": 9,
      "p4_new_strict_cells": 4,
      "cumulative_yield_curve": [
        {
          "proposal_index": 1,
          "strict_at_index": 6,
          "new_strict_cells_at_index": 6,
          "new_strict_cell_ids": [
            "bounded_rivalry_governance__mathematics",
            "bounded_rivalry_governance__nanotechnology",
            "predictive_residual_processing__criminology_forensic",
            "predictive_residual_processing__religious_studies_theology",
            "representation_independent_interface_contract__chemistry_materials",
            "representation_independent_interface_contract__criminology_forensic"
          ],
          "cumulative_strict_cells": 6,
          "cumulative_strict_cell_yield": 0.1
        },
        {
          "proposal_index": 2,
          "strict_at_index": 2,
          "new_strict_cells_at_index": 1,
          "new_strict_cell_ids": [
            "ritualized_meaning_and_commitment_enactment__aviation_aeronautics"
          ],
          "cumulative_strict_cells": 7,
          "cumulative_strict_cell_yield": 0.1167
        },
        {
          "proposal_index": 3,
          "strict_at_index": 1,
          "new_strict_cells_at_index": 0,
          "new_strict_cell_ids": [],
          "cumulative_strict_cells": 7,
          "cumulative_strict_cell_yield": 0.1167
        },
        {
          "proposal_index": 4,
          "strict_at_index": 6,
          "new_strict_cells_at_index": 4,
          "new_strict_cell_ids": [
            "bounded_rivalry_governance__film_media_production",
            "ritualized_meaning_and_commitment_enactment__accounting_auditing",
            "ritualized_meaning_and_commitment_enactment__criminology_forensic",
            "ritualized_meaning_and_commitment_enactment__environmental_climate"
          ],
          "cumulative_strict_cells": 11,
          "cumulative_strict_cell_yield": 0.1833
        }
      ],
      "frozen_interpretation_rule_verdict": "NOT_SUPPORTED"
    },
    "experiment_6_empirical_partner_lane": {
      "prefilter_eligible": 32,
      "qualified": 32,
      "qualified_archetypes": 5,
      "qualified_domains": 12,
      "audit_sample": 7,
      "percent_agreement": 1.0,
      "cohen_kappa": null,
      "disagreements": [],
      "frozen_validation_rule_verdict": "VALIDATED",
      "combined_strict_or_partner_curve": [
        {
          "proposal_index": 1,
          "combined_at_index": 12,
          "new_combined_cells_at_index": 12,
          "new_combined_cell_ids": [
            "bounded_rivalry_governance__accounting_auditing",
            "bounded_rivalry_governance__mathematics",
            "bounded_rivalry_governance__nanotechnology",
            "predictive_residual_processing__chemistry_materials",
            "predictive_residual_processing__criminology_forensic",
            "predictive_residual_processing__futurism_foresight",
            "predictive_residual_processing__religious_studies_theology",
            "representation_independent_interface_contract__chemistry_materials",
            "representation_independent_interface_contract__criminology_forensic",
            "representation_independent_interface_contract__environmental_climate",
            "ritualized_meaning_and_commitment_enactment__environmental_climate",
            "ritualized_meaning_and_commitment_enactment__nanotechnology"
          ],
          "cumulative_combined_cells": 12,
          "cumulative_combined_cell_yield": 0.2
        },
        {
          "proposal_index": 2,
          "combined_at_index": 8,
          "new_combined_cells_at_index": 6,
          "new_combined_cell_ids": [
            "bounded_rivalry_governance__aviation_aeronautics",
            "catalytic_pathway_enablement__mathematics",
            "representation_independent_interface_contract__film_media_production",
            "representation_independent_interface_contract__futurism_foresight",
            "ritualized_meaning_and_commitment_enactment__accounting_auditing",
            "ritualized_meaning_and_commitment_enactment__aviation_aeronautics"
          ],
          "cumulative_combined_cells": 18,
          "cumulative_combined_cell_yield": 0.3
        },
        {
          "proposal_index": 3,
          "combined_at_index": 10,
          "new_combined_cells_at_index": 3,
          "new_combined_cell_ids": [
            "bounded_rivalry_governance__criminology_forensic",
            "representation_independent_interface_contract__aviation_aeronautics",
            "ritualized_meaning_and_commitment_enactment__human_computer_interaction"
          ],
          "cumulative_combined_cells": 21,
          "cumulative_combined_cell_yield": 0.35
        },
        {
          "proposal_index": 4,
          "combined_at_index": 17,
          "new_combined_cells_at_index": 10,
          "new_combined_cell_ids": [
            "bounded_rivalry_governance__film_media_production",
            "bounded_rivalry_governance__human_computer_interaction",
            "bounded_rivalry_governance__religious_studies_theology",
            "bounded_rivalry_governance__sociology_anthropology",
            "predictive_residual_processing__environmental_climate",
            "predictive_residual_processing__mathematics",
            "representation_independent_interface_contract__sociology_anthropology",
            "ritualized_meaning_and_commitment_enactment__criminology_forensic",
            "ritualized_meaning_and_commitment_enactment__religious_studies_theology",
            "ritualized_meaning_and_commitment_enactment__sociology_anthropology"
          ],
          "cumulative_combined_cells": 31,
          "cumulative_combined_cell_yield": 0.5167
        }
      ]
    },
    "experiment_7_source_counts": {
      "cells": 60,
      "proposals": 240,
      "strict_proposals": 15,
      "empirical_partner_candidates": 32,
      "strict_or_partner_proposals": 47,
      "strict_cells": 11,
      "strict_or_partner_cells": 31
    },
    "experiment_7_primary_policy_result": {
      "selected_proposals": 120,
      "selected_fraction": 0.5,
      "strict_proposals_retained": 9,
      "strict_proposals_total": 15,
      "strict_proposal_recall": 0.6,
      "strict_proposal_recall_wilson95": [
        0.357468,
        0.801755
      ],
      "strict_cells_retained": 8,
      "strict_cells_total": 11,
      "strict_cell_recall": 0.727273,
      "strict_cell_recall_wilson95": [
        0.434355,
        0.902539
      ],
      "broad_proposals_retained": 25,
      "broad_proposals_total": 47,
      "broad_proposal_recall": 0.531915,
      "broad_proposal_recall_wilson95": [
        0.392324,
        0.666683
      ],
      "broad_cells_retained": 23,
      "broad_cells_total": 31,
      "broad_cell_recall": 0.741935,
      "broad_cell_recall_wilson95": [
        0.567539,
        0.862983
      ],
      "strict_precision": 0.075,
      "strict_precision_wilson95": [
        0.039957,
        0.136409
      ],
      "broad_precision": 0.208333,
      "broad_precision_wilson95": [
        0.145284,
        0.289477
      ],
      "external_evaluation_candidates_avoided": 120,
      "external_evaluation_candidate_fraction_avoided": 0.5,
      "thresholds": {
        "strict_cell_at_least_9_of_11": false,
        "broad_cell_at_least_25_of_31": false,
        "strict_proposal_at_least_11_of_15": false,
        "broad_proposal_at_least_33_of_47": false,
        "selected_no_more_than_150": true
      },
      "recall_thresholds_pass": false,
      "qualifies_for_prospective_rule": false,
      "resource_accounting": {
        "selected_external_evaluation": {
          "calls": 122,
          "summed_call_elapsed_seconds": 30749.034,
          "input_tokens": 90302953,
          "cached_input_tokens": 77485824,
          "cache_write_input_tokens": 0,
          "uncached_input_tokens": 12817129,
          "output_tokens": 1128709,
          "reasoning_output_tokens": 439106
        },
        "avoided_external_evaluation": {
          "calls": 121,
          "summed_call_elapsed_seconds": 29340.588,
          "input_tokens": 85277040,
          "cached_input_tokens": 72759040,
          "cache_write_input_tokens": 0,
          "uncached_input_tokens": 12518000,
          "output_tokens": 1090953,
          "reasoning_output_tokens": 422460
        },
        "counterfactual_selector_cost": {
          "calls": 180,
          "summed_call_elapsed_seconds": 10962.662,
          "input_tokens": 4883459,
          "cached_input_tokens": 1315328,
          "cache_write_input_tokens": 0,
          "uncached_input_tokens": 3568131,
          "output_tokens": 294142,
          "reasoning_output_tokens": 106962
        },
        "net_avoided_after_selector": {
          "calls": -59,
          "summed_call_elapsed_seconds": 18377.926,
          "input_tokens": 80393581,
          "cached_input_tokens": 71443712,
          "cache_write_input_tokens": 0,
          "uncached_input_tokens": 8949869,
          "output_tokens": 796811,
          "reasoning_output_tokens": 315498
        }
      }
    },
    "experiment_7_primary_verdict": "PRIMARY_NOT_SUPPORTED",
    "experiment_7_prospective_verdict": "PROSPECTIVE_CONFIRMATION_NOT_WARRANTED",
    "experiment_9_breadth_probe": {
      "status": "SUPPORTIVE_OF_BROAD_DISTRIBUTION",
      "frozen_gate_conditions": {
        "at_least_10_of_24_generated_archetypes_productive": true,
        "generated_fraction_within_0_20_of_reference": true,
        "at_least_two_generated_survivors_in_each_domain": true
      },
      "primary": {
        "generated_random_untested": {
          "archetypes": 24,
          "productive_archetypes": 24,
          "productive_fraction": 1.0,
          "productive_wilson_95": [
            0.8620237953250198,
            1.0
          ],
          "surviving_cells": 60,
          "cells": 72,
          "cell_survival_rate": 0.8333333333333334,
          "cell_survival_wilson_95": [
            0.7308929306946451,
            0.9020062875743832
          ],
          "prior_art_dispositions": {
            "ADJACENT_PRIOR_ART": 60,
            "SUBSTANTIAL_COLLISION": 11,
            "ESTABLISHED_PRACTICE": 1
          }
        },
        "reference_productive_fraction": 0.9,
        "generated_minus_reference": 0.09999999999999998,
        "generated_domain_survival": {
          "accounting_auditing": {
            "survivors": 23,
            "cells": 24,
            "rate": 0.9583333333333334,
            "wilson_95": [
              0.7975819352134406,
              0.9926065438344942
            ]
          },
          "chemistry_materials": {
            "survivors": 16,
            "cells": 24,
            "rate": 0.6666666666666666,
            "wilson_95": [
              0.4670631683813174,
              0.8202780967270225
            ]
          },
          "computer_science": {
            "survivors": 21,
            "cells": 24,
            "rate": 0.875,
            "wilson_95": [
              0.6899611872949993,
              0.9565566591987655
            ]
          }
        }
      },
      "all_strata": {
        "HAND_CURATED_CENSUS": {
          "archetypes": 16,
          "productive_archetypes": 16,
          "productive_fraction": 1.0,
          "productive_wilson_95": [
            0.8063923194655637,
            1.0
          ],
          "surviving_cells": 34,
          "cells": 48,
          "cell_survival_rate": 0.7083333333333334,
          "cell_survival_wilson_95": [
            0.5682074019042831,
            0.8175842107348348
          ],
          "prior_art_dispositions": {
            "ADJACENT_PRIOR_ART": 34,
            "SUBSTANTIAL_COLLISION": 10,
            "ESTABLISHED_PRACTICE": 4
          }
        },
        "GENERATED_RANDOM_UNTESTED": {
          "archetypes": 24,
          "productive_archetypes": 24,
          "productive_fraction": 1.0,
          "productive_wilson_95": [
            0.8620237953250198,
            1.0
          ],
          "surviving_cells": 60,
          "cells": 72,
          "cell_survival_rate": 0.8333333333333334,
          "cell_survival_wilson_95": [
            0.7308929306946451,
            0.9020062875743832
          ],
          "prior_art_dispositions": {
            "ADJACENT_PRIOR_ART": 60,
            "SUBSTANTIAL_COLLISION": 11,
            "ESTABLISHED_PRACTICE": 1
          }
        },
        "PREVIOUSLY_TESTED_REFERENCE": {
          "archetypes": 10,
          "productive_archetypes": 9,
          "productive_fraction": 0.9,
          "productive_wilson_95": [
            0.5958499732047615,
            0.9821237869049271
          ],
          "surviving_cells": 20,
          "cells": 30,
          "cell_survival_rate": 0.6666666666666666,
          "cell_survival_wilson_95": [
            0.4878005164454384,
            0.8076950191632386
          ],
          "prior_art_dispositions": {
            "ADJACENT_PRIOR_ART": 20,
            "SUBSTANTIAL_COLLISION": 8,
            "ESTABLISHED_PRACTICE": 2
          }
        }
      },
      "resources": {
        "transport_attempts": 300,
        "invalid_or_failed_attempts": 0,
        "summed_agent_elapsed_seconds": 40021.831,
        "token_usage": {
          "input_tokens": 52876667,
          "cached_input_tokens": 40308736,
          "cache_write_input_tokens": 0,
          "output_tokens": 1252720,
          "reasoning_output_tokens": 376597
        },
        "attempts_without_exposed_usage": 0
      }
    },
    "experiment_11_external_context_comparison": {
      "status": "NOT_SUPPORTIVE_UNDER_FROZEN_GATE",
      "frozen_gate_conditions": {
        "R_net_positive_vs_S": false,
        "R_net_positive_vs_D": true,
        "pooled_R_preference_at_least_0_65": false,
        "omnibus_exact_p_at_most_0_05": false,
        "R_safety_stop_not_higher_than_either_control": true
      },
      "primary": {
        "contrasts": {
          "R_vs_S": {
            "wins": 9,
            "losses": 11,
            "ties": 0,
            "decisive_win_rate": 0.45,
            "wilson_95": [
              0.2581978582556243,
              0.6579146575496577
            ],
            "exact_one_sided_sign_p": 0.7482776641845703,
            "holm_adjusted_p": 0.7482776641845703
          },
          "R_vs_D": {
            "wins": 12,
            "losses": 8,
            "ties": 0,
            "decisive_win_rate": 0.6,
            "wilson_95": [
              0.38658150076225317,
              0.7811934676271829
            ],
            "exact_one_sided_sign_p": 0.2517223358154297,
            "holm_adjusted_p": 0.5034446716308594
          }
        },
        "pooled_R_preference_rate": 0.525,
        "omnibus_randomization": {
          "observed_sum": 1.5,
          "observed_mean": 0.075,
          "exact_one_sided_p": 0.4458843269902537,
          "extreme_assignments": 1554702516,
          "total_assignments": 3486784401,
          "support_points": 41
        }
      },
      "by_arm": {
        "R": {
          "n": 20,
          "strict": 12,
          "screen": 13,
          "safety_stop": 0,
          "prior_art": {
            "ADJACENT_PRIOR_ART": 16,
            "ESTABLISHED_PRACTICE": 4
          },
          "research_value": {
            "MODERATE": 17,
            "HIGH": 3
          },
          "strict_rate": 0.6,
          "strict_wilson_95": [
            0.38658150076225317,
            0.7811934676271829
          ],
          "screen_rate": 0.65,
          "screen_wilson_95": [
            0.4328542766852363,
            0.8188081758989179
          ],
          "safety_stop_rate": 0.0
        },
        "S": {
          "n": 20,
          "strict": 11,
          "screen": 11,
          "safety_stop": 0,
          "prior_art": {
            "ADJACENT_PRIOR_ART": 11,
            "ESTABLISHED_PRACTICE": 9
          },
          "research_value": {
            "MODERATE": 18,
            "HIGH": 2
          },
          "strict_rate": 0.55,
          "strict_wilson_95": [
            0.3420853424503424,
            0.7418021417443758
          ],
          "screen_rate": 0.55,
          "screen_wilson_95": [
            0.3420853424503424,
            0.7418021417443758
          ],
          "safety_stop_rate": 0.0
        },
        "D": {
          "n": 20,
          "strict": 10,
          "screen": 11,
          "safety_stop": 0,
          "prior_art": {
            "ADJACENT_PRIOR_ART": 13,
            "ESTABLISHED_PRACTICE": 7
          },
          "research_value": {
            "MODERATE": 16,
            "HIGH": 4
          },
          "strict_rate": 0.5,
          "strict_wilson_95": [
            0.29929800819821234,
            0.7007019918017876
          ],
          "screen_rate": 0.55,
          "screen_wilson_95": [
            0.3420853424503424,
            0.7418021417443758
          ],
          "safety_stop_rate": 0.0
        }
      },
      "primary_judge_pair_agreement": {
        "agree": 48,
        "total": 60,
        "rate": 0.8
      },
      "resources": {
        "transport_attempts": 231,
        "invalid_or_failed_attempts": 91,
        "summed_agent_elapsed_seconds": 32682.334,
        "token_usage": {
          "input_tokens": 90356643,
          "cached_input_tokens": 75299072,
          "cache_write_input_tokens": 0,
          "output_tokens": 1169103,
          "reasoning_output_tokens": 466307
        },
        "attempts_without_exposed_usage": 3,
        "returncodes": {
          "0": 228,
          "1": 3
        }
      }
    },
    "experiment_12_substrate_denial": {
      "status": "COMPLETE",
      "primary": {
        "cells": 72,
        "valid_constrained_yield": {
          "estimate": 0.9305555555555556,
          "lower": 0.8475202051632649,
          "successes": 67,
          "total": 72,
          "upper": 0.9699746184342299
        },
        "usable_constrained_yield": {
          "estimate": 0.6805555555555556,
          "lower": 0.5660765126783435,
          "successes": 49,
          "total": 72,
          "upper": 0.7767438972173802
        },
        "ordinary_control_usable_yield": {
          "estimate": 0.8333333333333334,
          "lower": 0.7308929306946451,
          "successes": 60,
          "total": 72,
          "upper": 0.9020062875743832
        },
        "paired_usable_yield_difference_constrained_minus_ordinary": {
          "cluster_unit": "archetype",
          "estimate": -0.1527777777777778,
          "iterations": 50000,
          "lower": -0.3055555555555556,
          "seed": "eoa-exp12-archetype-bootstrap-v1-20260805",
          "upper": 0.0
        },
        "paired_usable_yield_table": {
          "both_survive": 41,
          "constrained_only": 8,
          "discordant_pairs": 27,
          "exact_mcnemar_two_sided_p": 0.052238985896110535,
          "neither_survives": 4,
          "ordinary_only": 19
        },
        "quality": {
          "constrained_wins": 4,
          "decisive_constrained_win_share": 0.05555555555555555,
          "exact_sign_test": {
            "constrained_wins": 4,
            "decisive": 72,
            "ordinary_wins": 68,
            "two_sided_p": 4.620814601991648e-16
          },
          "ordinary_wins": 68,
          "ties": 0
        },
        "failure_anatomy": {
          "ABSTENTION": 3,
          "FORBIDDEN_PRIMARY_DISGUISED": 2,
          "LIGHT_SCREEN_COLLISION": 18,
          "LIGHT_SCREEN_SURVIVOR": 49
        },
        "by_domain": {
          "accounting_auditing": {
            "cells": 24,
            "compliant": 21,
            "constrained_decisive_win_share": 0.041666666666666664,
            "constrained_quality_wins": 1,
            "constrained_usable": 19,
            "ordinary_quality_wins": 23,
            "ordinary_usable": 23,
            "quality_ties": 0
          },
          "chemistry_materials": {
            "cells": 24,
            "compliant": 23,
            "constrained_decisive_win_share": 0.125,
            "constrained_quality_wins": 3,
            "constrained_usable": 16,
            "ordinary_quality_wins": 21,
            "ordinary_usable": 16,
            "quality_ties": 0
          },
          "computer_science": {
            "cells": 24,
            "compliant": 23,
            "constrained_decisive_win_share": 0.0,
            "constrained_quality_wins": 0,
            "constrained_usable": 14,
            "ordinary_quality_wins": 24,
            "ordinary_usable": 21,
            "quality_ties": 0
          }
        },
        "by_sf_band": {
          "FRAMED": {
            "cells": 12,
            "compliant": 10,
            "constrained_decisive_win_share": 0.0,
            "constrained_quality_wins": 0,
            "constrained_usable": 6,
            "ordinary_quality_wins": 12,
            "ordinary_usable": 11,
            "quality_ties": 0
          },
          "MIDDLE": {
            "cells": 15,
            "compliant": 14,
            "constrained_decisive_win_share": 0.06666666666666667,
            "constrained_quality_wins": 1,
            "constrained_usable": 12,
            "ordinary_quality_wins": 14,
            "ordinary_usable": 11,
            "quality_ties": 0
          },
          "STRUCTURAL": {
            "cells": 45,
            "compliant": 43,
            "constrained_decisive_win_share": 0.06666666666666667,
            "constrained_quality_wins": 3,
            "constrained_usable": 31,
            "ordinary_quality_wins": 42,
            "ordinary_usable": 38,
            "quality_ties": 0
          }
        },
        "by_primary_substrate": {
          "GOVERNANCE_PROCESS": {
            "cells": 2,
            "compliant": 0,
            "constrained_decisive_win_share": 0.0,
            "constrained_quality_wins": 0,
            "constrained_usable": 0,
            "ordinary_quality_wins": 2,
            "ordinary_usable": 1,
            "quality_ties": 0
          },
          "MEASUREMENT_INSTRUMENTATION": {
            "cells": 27,
            "compliant": 27,
            "constrained_decisive_win_share": 0.037037037037037035,
            "constrained_quality_wins": 1,
            "constrained_usable": 21,
            "ordinary_quality_wins": 26,
            "ordinary_usable": 27,
            "quality_ties": 0
          },
          "None": {
            "cells": 3,
            "compliant": 0,
            "constrained_decisive_win_share": 0.0,
            "constrained_quality_wins": 0,
            "constrained_usable": 0,
            "ordinary_quality_wins": 3,
            "ordinary_usable": 3,
            "quality_ties": 0
          },
          "PHYSICAL_MATERIAL": {
            "cells": 40,
            "compliant": 40,
            "constrained_decisive_win_share": 0.075,
            "constrained_quality_wins": 3,
            "constrained_usable": 28,
            "ordinary_quality_wins": 37,
            "ordinary_usable": 29,
            "quality_ties": 0
          }
        },
        "frozen_max_trigger": {
          "condition_1_compliant_at_least_24": true,
          "condition_2_quality_or_yield_cost": true,
          "interpretation_if_not_run": null,
          "observed_compliant": 67,
          "observed_constrained_decisive_win_share": 0.05555555555555555,
          "observed_ordinary_minus_constrained_usable_yield": 0.1527777777777778,
          "quality_clause_constrained_decisive_win_share_at_most_0_40": true,
          "run_max_stage": true,
          "yield_clause_ordinary_advantage_at_least_0_15": true
        }
      },
      "max_followup": {
        "cells": 18,
        "compliance": {
          "CONSTRAINED_HIGH": {
            "abstentions": 2,
            "compliant": 16,
            "denominator": 18,
            "noncompliant_proposals": 0,
            "rate": 0.8888888888888888
          },
          "CONSTRAINED_MAX": {
            "abstentions": 1,
            "compliant": 17,
            "denominator": 18,
            "noncompliant_proposals": 0,
            "rate": 0.9444444444444444
          }
        },
        "quality_comparisons": {
          "constrained_max_vs_high": {
            "arm_a": "CONSTRAINED_MAX",
            "arm_a_decisive_win_share": 0.6666666666666666,
            "arm_a_wins": 12,
            "arm_b": "CONSTRAINED_HIGH",
            "arm_b_wins": 6,
            "comparisons": 18,
            "exact_sign_test_two_sided_p": 0.237884521484375,
            "net_advantage_per_cell_arm_a_minus_arm_b": 0.3333333333333333,
            "ties": 0
          },
          "ordinary_max_vs_high": {
            "arm_a": "ORDINARY_MAX",
            "arm_a_decisive_win_share": 0.7777777777777778,
            "arm_a_wins": 14,
            "arm_b": "ORDINARY_HIGH",
            "arm_b_wins": 4,
            "comparisons": 18,
            "exact_sign_test_two_sided_p": 0.0308837890625,
            "net_advantage_per_cell_arm_a_minus_arm_b": 0.5555555555555556,
            "ties": 0
          },
          "ordinary_vs_constrained_at_high": {
            "arm_a": "ORDINARY_HIGH",
            "arm_a_decisive_win_share": 0.8888888888888888,
            "arm_a_wins": 16,
            "arm_b": "CONSTRAINED_HIGH",
            "arm_b_wins": 2,
            "comparisons": 18,
            "exact_sign_test_two_sided_p": 0.001312255859375,
            "net_advantage_per_cell_arm_a_minus_arm_b": 0.7777777777777778,
            "ties": 0
          },
          "ordinary_vs_constrained_at_max": {
            "arm_a": "ORDINARY_MAX",
            "arm_a_decisive_win_share": 0.8888888888888888,
            "arm_a_wins": 16,
            "arm_b": "CONSTRAINED_MAX",
            "arm_b_wins": 2,
            "comparisons": 18,
            "exact_sign_test_two_sided_p": 0.001312255859375,
            "net_advantage_per_cell_arm_a_minus_arm_b": 0.7777777777777778,
            "ties": 0
          }
        },
        "principal_descriptive_interaction": {
          "constrained_max_net_advantage_over_high": 0.3333333333333333,
          "difference_constrained_minus_ordinary": -0.22222222222222227,
          "interpretation_boundary": "Descriptive only; the 18-cell subset is small and purposively balanced, and reasoning effort is not a controlled compute dose.",
          "ordinary_max_net_advantage_over_high": 0.5555555555555556
        },
        "usable_yield": {
          "by_arm": {
            "CONSTRAINED_HIGH": {
              "denominator": 18,
              "dispositions": {
                "ADJACENT_PRIOR_ART": 11,
                "AUTOMATIC_FAIL_ABSTENTION_OR_NONCOMPLIANCE": 2,
                "ESTABLISHED_PRACTICE": 1,
                "SUBSTANTIAL_COLLISION": 4
              },
              "rate": 0.6111111111111112,
              "survivors": 11
            },
            "CONSTRAINED_MAX": {
              "denominator": 18,
              "dispositions": {
                "ADJACENT_PRIOR_ART": 9,
                "AUTOMATIC_FAIL_ABSTENTION_OR_NONCOMPLIANCE": 1,
                "SUBSTANTIAL_COLLISION": 8
              },
              "rate": 0.5,
              "survivors": 9
            },
            "ORDINARY_HIGH": {
              "denominator": 18,
              "dispositions": {
                "ADJACENT_PRIOR_ART": 12,
                "ESTABLISHED_PRACTICE": 1,
                "SUBSTANTIAL_COLLISION": 5
              },
              "rate": 0.6666666666666666,
              "survivors": 12
            },
            "ORDINARY_MAX": {
              "denominator": 18,
              "dispositions": {
                "ADJACENT_PRIOR_ART": 9,
                "ESTABLISHED_PRACTICE": 2,
                "SUBSTANTIAL_COLLISION": 7
              },
              "rate": 0.5,
              "survivors": 9
            }
          },
          "by_domain": {
            "accounting_auditing": {
              "CONSTRAINED_HIGH": 5,
              "CONSTRAINED_MAX": 3,
              "ORDINARY_HIGH": 5,
              "ORDINARY_MAX": 3,
              "cells": 6
            },
            "chemistry_materials": {
              "CONSTRAINED_HIGH": 3,
              "CONSTRAINED_MAX": 3,
              "ORDINARY_HIGH": 3,
              "ORDINARY_MAX": 3,
              "cells": 6
            },
            "computer_science": {
              "CONSTRAINED_HIGH": 3,
              "CONSTRAINED_MAX": 3,
              "ORDINARY_HIGH": 4,
              "ORDINARY_MAX": 3,
              "cells": 6
            }
          },
          "high_to_max_transitions": {
            "constrained": {
              "both_survive": 7,
              "high_only": 4,
              "max_only": 2,
              "neither": 5
            },
            "ordinary": {
              "both_survive": 6,
              "high_only": 6,
              "max_only": 3,
              "neither": 3
            }
          }
        }
      },
      "resources": {
        "calls": 281,
        "failed_calls": 0,
        "first_started_at": "2026-08-06T00:09:34.549018Z",
        "last_finished_at": "2026-08-06T04:30:32.892066Z",
        "mean_elapsed_seconds": 154.88232740213522,
        "median_elapsed_seconds": 140.39,
        "phase_wall_span_seconds": 15658.343048,
        "successful_calls": 281,
        "summed_elapsed_seconds": 43521.934,
        "tokens": {
          "cache_write_input_tokens": 0,
          "cached_input_tokens": 29580032,
          "input_tokens": 44256895,
          "output_tokens": 1337531,
          "reasoning_output_tokens": 660600,
          "uncached_input_tokens": 14676863
        }
      }
    },
    "experiment_13_second_slot_policy": {
      "status": "COMPLETE",
      "cells": 60,
      "archetypes": 12,
      "domains": 5,
      "generation": {
        "common_p1_proposals": 60,
        "ordinary_p2_proposals": 60,
        "substrate_p2_abstentions": 0,
        "substrate_p2_proposals": 60
      },
      "p1_screen_survival": {
        "ci95": [
          0.5923271483560882,
          0.8149317157156729
        ],
        "rate": 0.7166666666666667,
        "successes": 43,
        "total": 60
      },
      "substrate_compliance": {
        "ci95": [
          0.93982814785791,
          0.9999999999999998
        ],
        "rate": 1.0,
        "successes": 60,
        "total": 60
      },
      "incremental_survival": {
        "archetype_cluster_bootstrap": {
          "ci95": [
            -0.31666666666666665,
            0.03333333333333333
          ],
          "cluster": "archetype",
          "estimate": -0.15,
          "iterations": 20000
        },
        "archetype_exact_sign_flip": {
          "archetype_differences_substrate_minus_ordinary": [
            -3,
            3,
            -2,
            -2,
            -1,
            -1,
            0,
            -2,
            0,
            -2,
            1,
            0
          ],
          "exact_two_sided_p": 0.1953125,
          "observed_absolute_sum": 9,
          "permutations": 4096
        },
        "cell_level_exact_mcnemar_two_sided_p_sensitivity": 0.13604594767093658,
        "difference_substrate_minus_ordinary": -0.15,
        "ordinary": {
          "ci95": [
            0.5576623297382323,
            0.7869413244763347
          ],
          "rate": 0.6833333333333333,
          "successes": 41,
          "total": 60
        },
        "paired_table": {
          "both": 22,
          "neither": 9,
          "ordinary_only": 19,
          "substrate_only": 10
        },
        "substrate": {
          "ci95": [
            0.4089342702715789,
            0.6537209395856151
          ],
          "rate": 0.5333333333333333,
          "successes": 32,
          "total": 60
        }
      },
      "p1_failure_rescue": {
        "ordinary_rescues": 5,
        "p1_failures": 17,
        "substrate_rescues": 8
      },
      "quality": {
        "cell_level_exact_sign_test_two_sided_p_sensitivity": 0.6988834276200057,
        "ordinary_wins": 32,
        "substrate_wins": 28,
        "ties": 0
      },
      "substrate_distributions": {
        "COMMON_P1": {
          "COMPUTATIONAL_INFORMATION": 36,
          "GOVERNANCE_PROCESS": 22,
          "MEASUREMENT_INSTRUMENTATION": 1,
          "PHYSICAL_MATERIAL": 1
        },
        "ORDINARY_DIVERSE_P2": {
          "COMPUTATIONAL_INFORMATION": 30,
          "GOVERNANCE_PROCESS": 24,
          "MEASUREMENT_INSTRUMENTATION": 4,
          "PHYSICAL_MATERIAL": 2
        },
        "SUBSTRATE_DIVERSE_P2": {
          "MEASUREMENT_INSTRUMENTATION": 36,
          "PHYSICAL_MATERIAL": 24
        }
      },
      "prior_art_dispositions": {
        "COMMON_P1": {
          "ADJACENT_PRIOR_ART": 43,
          "ESTABLISHED_PRACTICE": 1,
          "SUBSTANTIAL_COLLISION": 16
        },
        "ORDINARY_DIVERSE_P2": {
          "ADJACENT_PRIOR_ART": 50,
          "ESTABLISHED_PRACTICE": 3,
          "SUBSTANTIAL_COLLISION": 7
        },
        "SUBSTRATE_DIVERSE_P2": {
          "ADJACENT_PRIOR_ART": 33,
          "ESTABLISHED_PRACTICE": 11,
          "SUBSTANTIAL_COLLISION": 16
        }
      },
      "frozen_decision": {
        "complement_clause": {
          "compliant_at_least_48": true,
          "difference_at_least_minus_0_15": true,
          "substrate_only_at_least_6": true
        },
        "prefer_substrate_clause": {
          "difference_at_least_0_05": false,
          "substrate_only_exceeds_ordinary_only": false
        },
        "verdict": "USE_AS_BOUNDED_PORTFOLIO_COMPLEMENT"
      },
      "resources": {
        "calls": 458,
        "failed_calls": 0,
        "mean_call_seconds": 112.2831441048035,
        "median_call_seconds": 120.509,
        "summed_call_hours": 14.284911111111116,
        "summed_call_seconds": 51425.680000000015,
        "token_totals": {
          "cached_input_tokens": 54841600,
          "input_tokens": 80429441,
          "output_tokens": 1715925,
          "reasoning_output_tokens": 618283
        },
        "wall_span_hours": 13.262820171978738,
        "wall_span_seconds": 47746.15261912346
      }
    },
    "experiment_14_applicability_graph": {
      "status": "COMPLETE",
      "case_eligibility": {
        "audit_disagreements_any_material_field": 11,
        "audit_validity_agreement": {
          "ci95": [
            0.8951971146376326,
            0.9768856355855343
          ],
          "rate": 0.95,
          "successes": 114,
          "total": 120
        },
        "eligible": 114,
        "eligible_archetypes": 40,
        "eligible_near_miss": 37,
        "eligible_positive": 77,
        "excluded": 6,
        "information_gate_passed": true,
        "planned": 120
      },
      "retrieval": {
        "archetype_cluster_bootstrap_recall_at_5_difference": {
          "ci95": [
            -0.14102564102564102,
            -0.025974025974025976
          ],
          "cluster": "archetype",
          "estimate": -0.07792207792207792,
          "iterations": 20000
        },
        "by_positive_case_type": {
          "DIRECT_POSITIVE": {
            "diagnostic": {
              "mean_reciprocal_rank": 0.014490477319424688,
              "median_rank_when_found": 23.0,
              "n": 38,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.09180986822558396
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 38
              },
              "recall_at_10": {
                "ci95": [
                  0.027214595506987724,
                  0.20799371457771454
                ],
                "rate": 0.07894736842105263,
                "successes": 3,
                "total": 38
              },
              "recall_at_5": {
                "ci95": [
                  0.004660587685768783,
                  0.1349487611595213
                ],
                "rate": 0.02631578947368421,
                "successes": 1,
                "total": 38
              },
              "recall_at_50": {
                "ci95": [
                  0.07443863415158927,
                  0.3041681177922313
                ],
                "rate": 0.15789473684210525,
                "successes": 6,
                "total": 38
              }
            },
            "difference": -0.13157894736842105,
            "solution": {
              "mean_reciprocal_rank": 0.10219652828515911,
              "median_rank_when_found": 6.0,
              "n": 38,
              "recall_at_1": {
                "ci95": [
                  0.014554208147292413,
                  0.17285462131770377
                ],
                "rate": 0.05263157894736842,
                "successes": 2,
                "total": 38
              },
              "recall_at_10": {
                "ci95": [
                  0.11074791842995879,
                  0.363457794753274
                ],
                "rate": 0.21052631578947367,
                "successes": 8,
                "total": 38
              },
              "recall_at_5": {
                "ci95": [
                  0.07443863415158927,
                  0.3041681177922313
                ],
                "rate": 0.15789473684210525,
                "successes": 6,
                "total": 38
              },
              "recall_at_50": {
                "ci95": [
                  0.23384219247155955,
                  0.5271604044299099
                ],
                "rate": 0.3684210526315789,
                "successes": 14,
                "total": 38
              }
            }
          },
          "TRANSFER_POSITIVE": {
            "diagnostic": {
              "mean_reciprocal_rank": 0.012412267294059694,
              "median_rank_when_found": 29,
              "n": 39,
              "recall_at_1": {
                "ci95": [
                  -6.938893903907228e-18,
                  0.08966685370757144
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 39
              },
              "recall_at_10": {
                "ci95": [
                  0.004540703100990098,
                  0.13180990169850076
                ],
                "rate": 0.02564102564102564,
                "successes": 1,
                "total": 39
              },
              "recall_at_5": {
                "ci95": [
                  0.004540703100990098,
                  0.13180990169850076
                ],
                "rate": 0.02564102564102564,
                "successes": 1,
                "total": 39
              },
              "recall_at_50": {
                "ci95": [
                  0.08977265473219115,
                  0.32668045661881623
                ],
                "rate": 0.1794871794871795,
                "successes": 7,
                "total": 39
              }
            },
            "difference": -0.02564102564102564,
            "solution": {
              "mean_reciprocal_rank": 0.04788698955365622,
              "median_rank_when_found": 19.5,
              "n": 39,
              "recall_at_1": {
                "ci95": [
                  0.004540703100990098,
                  0.13180990169850076
                ],
                "rate": 0.02564102564102564,
                "successes": 1,
                "total": 39
              },
              "recall_at_10": {
                "ci95": [
                  0.02650739530377974,
                  0.20321071167954996
                ],
                "rate": 0.07692307692307693,
                "successes": 3,
                "total": 39
              },
              "recall_at_5": {
                "ci95": [
                  0.014177955415274113,
                  0.16885640047613612
                ],
                "rate": 0.05128205128205128,
                "successes": 2,
                "total": 39
              },
              "recall_at_50": {
                "ci95": [
                  0.10779718230710382,
                  0.35533968013582296
                ],
                "rate": 0.20512820512820512,
                "successes": 8,
                "total": 39
              }
            }
          }
        },
        "by_stratum": {
          "ALTERNATIVE_ROUTES": {
            "diagnostic": {
              "mean_reciprocal_rank": 0.0,
              "median_rank_when_found": null,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              }
            },
            "dnf_reranked": {
              "mean_reciprocal_rank": 0.0,
              "median_rank_when_found": null,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              }
            },
            "n": 16,
            "solution": {
              "mean_reciprocal_rank": 0.12890625,
              "median_rank_when_found": 12.5,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.10182067491213048,
                  0.49498316535508774
                ],
                "rate": 0.25,
                "successes": 4,
                "total": 16
              }
            }
          },
          "MULTI_DSA_COMPLETE": {
            "diagnostic": {
              "mean_reciprocal_rank": 0.008518518518518519,
              "median_rank_when_found": 23.0,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.03736125910306537,
                  0.37881982832322364
                ],
                "rate": 0.13333333333333333,
                "successes": 2,
                "total": 15
              }
            },
            "dnf_reranked": {
              "mean_reciprocal_rank": 0.06666666666666667,
              "median_rank_when_found": 1,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              }
            },
            "n": 15,
            "solution": {
              "mean_reciprocal_rank": 0.06487436456063907,
              "median_rank_when_found": 18.5,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.07047549346981563,
                  0.45185448715169363
                ],
                "rate": 0.2,
                "successes": 3,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.03736125910306537,
                  0.37881982832322364
                ],
                "rate": 0.13333333333333333,
                "successes": 2,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.30116980025498397,
                  0.7519046463426261
                ],
                "rate": 0.5333333333333333,
                "successes": 8,
                "total": 15
              }
            }
          },
          "MULTI_PARTIAL_OPEN": {
            "diagnostic": {
              "mean_reciprocal_rank": 0.027807328605200944,
              "median_rank_when_found": 27.5,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.03736125910306537,
                  0.37881982832322364
                ],
                "rate": 0.13333333333333333,
                "successes": 2,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.10897453325692374,
                  0.5195043405598057
                ],
                "rate": 0.26666666666666666,
                "successes": 4,
                "total": 15
              }
            },
            "dnf_reranked": {
              "mean_reciprocal_rank": 0.13333333333333333,
              "median_rank_when_found": 1.0,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.03736125910306537,
                  0.37881982832322364
                ],
                "rate": 0.13333333333333333,
                "successes": 2,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.03736125910306537,
                  0.37881982832322364
                ],
                "rate": 0.13333333333333333,
                "successes": 2,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.03736125910306537,
                  0.37881982832322364
                ],
                "rate": 0.13333333333333333,
                "successes": 2,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.03736125910306537,
                  0.37881982832322364
                ],
                "rate": 0.13333333333333333,
                "successes": 2,
                "total": 15
              }
            },
            "n": 15,
            "solution": {
              "mean_reciprocal_rank": 0.08128654970760234,
              "median_rank_when_found": 6,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.03736125910306537,
                  0.37881982832322364
                ],
                "rate": 0.13333333333333333,
                "successes": 2,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.07047549346981563,
                  0.45185448715169363
                ],
                "rate": 0.2,
                "successes": 3,
                "total": 15
              }
            }
          },
          "MULTI_PRIME_COMPLETE": {
            "diagnostic": {
              "mean_reciprocal_rank": 0.012067021755927304,
              "median_rank_when_found": 37,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.1517632443971473,
                  0.5828645226148023
                ],
                "rate": 0.3333333333333333,
                "successes": 5,
                "total": 15
              }
            },
            "dnf_reranked": {
              "mean_reciprocal_rank": 0.0,
              "median_rank_when_found": null,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              }
            },
            "n": 15,
            "solution": {
              "mean_reciprocal_rank": 0.06259259259259259,
              "median_rank_when_found": 5.0,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.07047549346981563,
                  0.45185448715169363
                ],
                "rate": 0.2,
                "successes": 3,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.03736125910306537,
                  0.37881982832322364
                ],
                "rate": 0.13333333333333333,
                "successes": 2,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.10897453325692374,
                  0.5195043405598057
                ],
                "rate": 0.26666666666666666,
                "successes": 4,
                "total": 15
              }
            }
          },
          "SINGLE_LITERAL": {
            "diagnostic": {
              "mean_reciprocal_rank": 0.019301470588235295,
              "median_rank_when_found": 10.5,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              }
            },
            "dnf_reranked": {
              "mean_reciprocal_rank": 0.0625,
              "median_rank_when_found": 1,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              }
            },
            "n": 16,
            "solution": {
              "mean_reciprocal_rank": 0.034828629032258064,
              "median_rank_when_found": 31,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.06591599071428142,
                  0.4300888096197414
                ],
                "rate": 0.1875,
                "successes": 3,
                "total": 16
              }
            }
          }
        },
        "diagnostic_max_hit": {
          "mean_reciprocal_rank": 0.013437877436447613,
          "median_rank_when_found": 29,
          "n": 77,
          "recall_at_1": {
            "ci95": [
              0.0,
              0.04751842528243408
            ],
            "rate": 0.0,
            "successes": 0,
            "total": 77
          },
          "recall_at_10": {
            "ci95": [
              0.020384718503645788,
              0.12609283142476915
            ],
            "rate": 0.05194805194805195,
            "successes": 4,
            "total": 77
          },
          "recall_at_5": {
            "ci95": [
              0.007152159807893181,
              0.08984582779753134
            ],
            "rate": 0.025974025974025976,
            "successes": 2,
            "total": 77
          },
          "recall_at_50": {
            "ci95": [
              0.10140668708276009,
              0.2677288932991118
            ],
            "rate": 0.16883116883116883,
            "successes": 13,
            "total": 77
          }
        },
        "diagnostic_minus_solution_recall_at_5": -0.07792207792207792,
        "diagnostic_top12_target_coverage": {
          "ci95": [
            0.020384718503645788,
            0.12609283142476915
          ],
          "rate": 0.05194805194805195,
          "successes": 4,
          "total": 77
        },
        "dnf_reranked_top12": {
          "mean_reciprocal_rank": 0.05194805194805195,
          "median_rank_when_found": 1.0,
          "n": 77,
          "recall_at_1": {
            "ci95": [
              0.020384718503645788,
              0.12609283142476915
            ],
            "rate": 0.05194805194805195,
            "successes": 4,
            "total": 77
          },
          "recall_at_10": {
            "ci95": [
              0.020384718503645788,
              0.12609283142476915
            ],
            "rate": 0.05194805194805195,
            "successes": 4,
            "total": 77
          },
          "recall_at_5": {
            "ci95": [
              0.020384718503645788,
              0.12609283142476915
            ],
            "rate": 0.05194805194805195,
            "successes": 4,
            "total": 77
          },
          "recall_at_50": {
            "ci95": [
              0.020384718503645788,
              0.12609283142476915
            ],
            "rate": 0.05194805194805195,
            "successes": 4,
            "total": 77
          }
        },
        "frozen_gate_passed": false,
        "paired_at_5": {
          "both": 2,
          "diagnostic_only": 0,
          "exact_mcnemar_p_two_sided": 0.03125,
          "neither": 69,
          "solution_only": 6
        },
        "solution_index": {
          "mean_reciprocal_rank": 0.07468909957699531,
          "median_rank_when_found": 9.0,
          "n": 77,
          "recall_at_1": {
            "ci95": [
              0.013337950040590962,
              0.10839981872632877
            ],
            "rate": 0.03896103896103896,
            "successes": 3,
            "total": 77
          },
          "recall_at_10": {
            "ci95": [
              0.0816833163991473,
              0.23797270165973414
            ],
            "rate": 0.14285714285714285,
            "successes": 11,
            "total": 77
          },
          "recall_at_5": {
            "ci95": [
              0.053592923647580934,
              0.19184375092681488
            ],
            "rate": 0.1038961038961039,
            "successes": 8,
            "total": 77
          },
          "recall_at_50": {
            "ci95": [
              0.1968951815913284,
              0.3948984292440005
            ],
            "rate": 0.2857142857142857,
            "successes": 22,
            "total": 77
          }
        }
      },
      "dnf_target_route_fidelity": {
        "balanced_accuracy": 0.9324324324324325,
        "frozen_gate_passed": true,
        "near_miss_false_acceptance_count": 5,
        "near_miss_specificity": {
          "ci95": [
            0.7202258821599021,
            0.9408670557426799
          ],
          "rate": 0.8648648648648649,
          "successes": 32,
          "total": 37
        },
        "positive_false_rejection_count": 0,
        "positive_sensitivity": {
          "ci95": [
            0.9524815747175658,
            0.9999999999999999
          ],
          "rate": 1.0,
          "successes": 77,
          "total": 77
        }
      },
      "frozen_verdict": "VERIFICATION_ONLY",
      "resources": {
        "calls": 215,
        "failed_calls": 0,
        "mean_call_seconds": 86.04967441860465,
        "median_call_seconds": 87.328,
        "summed_call_hours": 5.139077777777778,
        "token_totals": {
          "cached_input_tokens": 1880832,
          "input_tokens": 3584592,
          "output_tokens": 912450,
          "reasoning_output_tokens": 236247
        },
        "wall_span_hours": 1.7915853642092812
      }
    },
    "experiment_15_route_aware_retrieval": {
      "status": "COMPLETE",
      "case_eligibility": {
        "audit_disagreements_any_material_field": 12,
        "audit_validity_agreement": {
          "ci95": [
            0.8951971146376326,
            0.9768856355855343
          ],
          "rate": 0.95,
          "successes": 114,
          "total": 120
        },
        "eligible": 115,
        "eligible_archetypes": 40,
        "eligible_near_miss": 37,
        "eligible_positive": 78,
        "excluded": 5,
        "information_gate_passed": true,
        "planned": 120
      },
      "retrieval": {
        "arm_summaries": {
          "diagnostic": {
            "mean_reciprocal_rank": 0.08884773080157857,
            "median_rank": 5.0,
            "n": 78,
            "recall_at_1": {
              "ci95": [
                0.020121012473896145,
                0.1245667654729449
              ],
              "rate": 0.05128205128205128,
              "successes": 4,
              "total": 78
            },
            "recall_at_10": {
              "ci95": [
                0.12015206340148435,
                0.2933481271906353
              ],
              "rate": 0.19230769230769232,
              "successes": 15,
              "total": 78
            },
            "recall_at_5": {
              "ci95": [
                0.09025504908387116,
                0.24993266533226338
              ],
              "rate": 0.15384615384615385,
              "successes": 12,
              "total": 78
            },
            "recall_at_50": {
              "ci95": [
                0.19425334729674015,
                0.39030928770601114
              ],
              "rate": 0.28205128205128205,
              "successes": 22,
              "total": 78
            }
          },
          "full_mean": {
            "mean_reciprocal_rank": 0.025813175985763624,
            "median_rank": 148.5,
            "n": 78,
            "recall_at_1": {
              "ci95": [
                -3.469446951953614e-18,
                0.04693780971219429
              ],
              "rate": 0.0,
              "successes": 0,
              "total": 78
            },
            "recall_at_10": {
              "ci95": [
                0.052893347657463854,
                0.18954439852402394
              ],
              "rate": 0.10256410256410256,
              "successes": 8,
              "total": 78
            },
            "recall_at_5": {
              "ci95": [
                0.013165825413161053,
                0.1070844604750183
              ],
              "rate": 0.038461538461538464,
              "successes": 3,
              "total": 78
            },
            "recall_at_50": {
              "ci95": [
                0.1407681204113414,
                0.32160705429810155
              ],
              "rate": 0.21794871794871795,
              "successes": 17,
              "total": 78
            }
          },
          "full_min": {
            "mean_reciprocal_rank": 0.013001314093736057,
            "median_rank": 201.0,
            "n": 78,
            "recall_at_1": {
              "ci95": [
                -3.469446951953614e-18,
                0.04693780971219429
              ],
              "rate": 0.0,
              "successes": 0,
              "total": 78
            },
            "recall_at_10": {
              "ci95": [
                0.0022667317538337325,
                0.06910857001702223
              ],
              "rate": 0.01282051282051282,
              "successes": 1,
              "total": 78
            },
            "recall_at_5": {
              "ci95": [
                0.0022667317538337325,
                0.06910857001702223
              ],
              "rate": 0.01282051282051282,
              "successes": 1,
              "total": 78
            },
            "recall_at_50": {
              "ci95": [
                0.052893347657463854,
                0.18954439852402394
              ],
              "rate": 0.10256410256410256,
              "successes": 8,
              "total": 78
            }
          },
          "sentence_lower_half": {
            "mean_reciprocal_rank": 0.020932629034669265,
            "median_rank": 109.5,
            "n": 78,
            "recall_at_1": {
              "ci95": [
                -3.469446951953614e-18,
                0.04693780971219429
              ],
              "rate": 0.0,
              "successes": 0,
              "total": 78
            },
            "recall_at_10": {
              "ci95": [
                0.0022667317538337325,
                0.06910857001702223
              ],
              "rate": 0.01282051282051282,
              "successes": 1,
              "total": 78
            },
            "recall_at_5": {
              "ci95": [
                0.0022667317538337325,
                0.06910857001702223
              ],
              "rate": 0.01282051282051282,
              "successes": 1,
              "total": 78
            },
            "recall_at_50": {
              "ci95": [
                0.16185238513667277,
                0.3493977736900935
              ],
              "rate": 0.24358974358974358,
              "successes": 19,
              "total": 78
            }
          },
          "sentence_mean": {
            "mean_reciprocal_rank": 0.035051948513788504,
            "median_rank": 74.5,
            "n": 78,
            "recall_at_1": {
              "ci95": [
                -3.469446951953614e-18,
                0.04693780971219429
              ],
              "rate": 0.0,
              "successes": 0,
              "total": 78
            },
            "recall_at_10": {
              "ci95": [
                0.04415300851677373,
                0.17384724560605236
              ],
              "rate": 0.08974358974358974,
              "successes": 7,
              "total": 78
            },
            "recall_at_5": {
              "ci95": [
                0.007060093472759203,
                0.08875270035675845
              ],
              "rate": 0.02564102564102564,
              "successes": 2,
              "total": 78
            },
            "recall_at_50": {
              "ci95": [
                0.31959060348175483,
                0.533784444166275
              ],
              "rate": 0.4230769230769231,
              "successes": 33,
              "total": 78
            }
          },
          "sentence_min": {
            "mean_reciprocal_rank": 0.018997778367115355,
            "median_rank": 132.5,
            "n": 78,
            "recall_at_1": {
              "ci95": [
                -3.469446951953614e-18,
                0.04693780971219429
              ],
              "rate": 0.0,
              "successes": 0,
              "total": 78
            },
            "recall_at_10": {
              "ci95": [
                0.0022667317538337325,
                0.06910857001702223
              ],
              "rate": 0.01282051282051282,
              "successes": 1,
              "total": 78
            },
            "recall_at_5": {
              "ci95": [
                0.0022667317538337325,
                0.06910857001702223
              ],
              "rate": 0.01282051282051282,
              "successes": 1,
              "total": 78
            },
            "recall_at_50": {
              "ci95": [
                0.12015206340148435,
                0.2933481271906353
              ],
              "rate": 0.19230769230769232,
              "successes": 15,
              "total": 78
            }
          },
          "solution": {
            "mean_reciprocal_rank": 0.057743271718918264,
            "median_rank": 17,
            "n": 78,
            "recall_at_1": {
              "ci95": [
                0.007060093472759203,
                0.08875270035675845
              ],
              "rate": 0.02564102564102564,
              "successes": 2,
              "total": 78
            },
            "recall_at_10": {
              "ci95": [
                0.052893347657463854,
                0.18954439852402394
              ],
              "rate": 0.10256410256410256,
              "successes": 8,
              "total": 78
            },
            "recall_at_5": {
              "ci95": [
                0.035730464002355806,
                0.15783229806180862
              ],
              "rate": 0.07692307692307693,
              "successes": 6,
              "total": 78
            },
            "recall_at_50": {
              "ci95": [
                0.20524218466498595,
                0.4037579423964271
              ],
              "rate": 0.2948717948717949,
              "successes": 23,
              "total": 78
            }
          }
        },
        "by_positive_case_type": {
          "DIRECT_POSITIVE": {
            "diagnostic": {
              "mean_reciprocal_rank": 0.10387166085946574,
              "median_rank": 3.0,
              "n": 40,
              "recall_at_1": {
                "ci95": [
                  0.025836025774588198,
                  0.19864233524310543
                ],
                "rate": 0.075,
                "successes": 3,
                "total": 40
              },
              "recall_at_10": {
                "ci95": [
                  0.07061187717320358,
                  0.2907232436648971
                ],
                "rate": 0.15,
                "successes": 6,
                "total": 40
              },
              "recall_at_5": {
                "ci95": [
                  0.07061187717320358,
                  0.2907232436648971
                ],
                "rate": 0.15,
                "successes": 6,
                "total": 40
              },
              "recall_at_50": {
                "ci95": [
                  0.14187118639096302,
                  0.40193961420768026
                ],
                "rate": 0.25,
                "successes": 10,
                "total": 40
              }
            },
            "full_mean": {
              "mean_reciprocal_rank": 0.02886214490460318,
              "median_rank": 136.5,
              "n": 40,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.08762160119728664
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 40
              },
              "recall_at_10": {
                "ci95": [
                  0.03957952868260638,
                  0.23051775227522298
                ],
                "rate": 0.1,
                "successes": 4,
                "total": 40
              },
              "recall_at_5": {
                "ci95": [
                  0.013820667386148344,
                  0.16503877369140962
                ],
                "rate": 0.05,
                "successes": 2,
                "total": 40
              },
              "recall_at_50": {
                "ci95": [
                  0.10499989725437704,
                  0.347573063463995
                ],
                "rate": 0.2,
                "successes": 8,
                "total": 40
              }
            },
            "full_min": {
              "mean_reciprocal_rank": 0.015901218501791273,
              "median_rank": 203.5,
              "n": 40,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.08762160119728664
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 40
              },
              "recall_at_10": {
                "ci95": [
                  0.0044268315026814095,
                  0.1288136896347409
                ],
                "rate": 0.025,
                "successes": 1,
                "total": 40
              },
              "recall_at_5": {
                "ci95": [
                  0.0044268315026814095,
                  0.1288136896347409
                ],
                "rate": 0.025,
                "successes": 1,
                "total": 40
              },
              "recall_at_50": {
                "ci95": [
                  0.025836025774588198,
                  0.19864233524310543
                ],
                "rate": 0.075,
                "successes": 3,
                "total": 40
              }
            },
            "sentence_lower_half": {
              "mean_reciprocal_rank": 0.027116823912147807,
              "median_rank": 92.5,
              "n": 40,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.08762160119728664
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 40
              },
              "recall_at_10": {
                "ci95": [
                  0.0044268315026814095,
                  0.1288136896347409
                ],
                "rate": 0.025,
                "successes": 1,
                "total": 40
              },
              "recall_at_5": {
                "ci95": [
                  0.0044268315026814095,
                  0.1288136896347409
                ],
                "rate": 0.025,
                "successes": 1,
                "total": 40
              },
              "recall_at_50": {
                "ci95": [
                  0.12316091323540743,
                  0.37503096742310027
                ],
                "rate": 0.225,
                "successes": 9,
                "total": 40
              }
            },
            "sentence_mean": {
              "mean_reciprocal_rank": 0.039011193916401996,
              "median_rank": 73.0,
              "n": 40,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.08762160119728664
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 40
              },
              "recall_at_10": {
                "ci95": [
                  0.025836025774588198,
                  0.19864233524310543
                ],
                "rate": 0.075,
                "successes": 3,
                "total": 40
              },
              "recall_at_5": {
                "ci95": [
                  0.0044268315026814095,
                  0.1288136896347409
                ],
                "rate": 0.025,
                "successes": 1,
                "total": 40
              },
              "recall_at_50": {
                "ci95": [
                  0.3070530781090492,
                  0.6017090820106794
                ],
                "rate": 0.45,
                "successes": 18,
                "total": 40
              }
            },
            "sentence_min": {
              "mean_reciprocal_rank": 0.024847116078777935,
              "median_rank": 95.5,
              "n": 40,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.08762160119728664
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 40
              },
              "recall_at_10": {
                "ci95": [
                  0.0044268315026814095,
                  0.1288136896347409
                ],
                "rate": 0.025,
                "successes": 1,
                "total": 40
              },
              "recall_at_5": {
                "ci95": [
                  0.0044268315026814095,
                  0.1288136896347409
                ],
                "rate": 0.025,
                "successes": 1,
                "total": 40
              },
              "recall_at_50": {
                "ci95": [
                  0.05459500250945401,
                  0.261121198388511
                ],
                "rate": 0.125,
                "successes": 5,
                "total": 40
              }
            },
            "solution": {
              "mean_reciprocal_rank": 0.07801725489225489,
              "median_rank": 16.0,
              "n": 40,
              "recall_at_1": {
                "ci95": [
                  0.013820667386148344,
                  0.16503877369140962
                ],
                "rate": 0.05,
                "successes": 2,
                "total": 40
              },
              "recall_at_10": {
                "ci95": [
                  0.05459500250945401,
                  0.261121198388511
                ],
                "rate": 0.125,
                "successes": 5,
                "total": 40
              },
              "recall_at_5": {
                "ci95": [
                  0.03957952868260638,
                  0.23051775227522298
                ],
                "rate": 0.1,
                "successes": 4,
                "total": 40
              },
              "recall_at_50": {
                "ci95": [
                  0.2213452887317629,
                  0.5049411916274231
                ],
                "rate": 0.35,
                "successes": 14,
                "total": 40
              }
            }
          },
          "TRANSFER_POSITIVE": {
            "diagnostic": {
              "mean_reciprocal_rank": 0.07303306758274998,
              "median_rank": 5.5,
              "n": 38,
              "recall_at_1": {
                "ci95": [
                  0.004660587685768783,
                  0.1349487611595213
                ],
                "rate": 0.02631578947368421,
                "successes": 1,
                "total": 38
              },
              "recall_at_10": {
                "ci95": [
                  0.12993707017054082,
                  0.39206812363239807
                ],
                "rate": 0.23684210526315788,
                "successes": 9,
                "total": 38
              },
              "recall_at_5": {
                "ci95": [
                  0.07443863415158927,
                  0.3041681177922313
                ],
                "rate": 0.15789473684210525,
                "successes": 6,
                "total": 38
              },
              "recall_at_50": {
                "ci95": [
                  0.1908460367559618,
                  0.4745575989060954
                ],
                "rate": 0.3157894736842105,
                "successes": 12,
                "total": 38
              }
            },
            "full_mean": {
              "mean_reciprocal_rank": 0.022603735018564095,
              "median_rank": 188.0,
              "n": 38,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.09180986822558396
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 38
              },
              "recall_at_10": {
                "ci95": [
                  0.04170257347553526,
                  0.24130521722887316
                ],
                "rate": 0.10526315789473684,
                "successes": 4,
                "total": 38
              },
              "recall_at_5": {
                "ci95": [
                  0.004660587685768783,
                  0.1349487611595213
                ],
                "rate": 0.02631578947368421,
                "successes": 1,
                "total": 38
              },
              "recall_at_50": {
                "ci95": [
                  0.12993707017054082,
                  0.39206812363239807
                ],
                "rate": 0.23684210526315788,
                "successes": 9,
                "total": 38
              }
            },
            "full_min": {
              "mean_reciprocal_rank": 0.009948783137888464,
              "median_rank": 179.0,
              "n": 38,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.09180986822558396
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 38
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.09180986822558396
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 38
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.09180986822558396
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 38
              },
              "recall_at_50": {
                "ci95": [
                  0.05753867690120133,
                  0.2732685944229131
                ],
                "rate": 0.13157894736842105,
                "successes": 5,
                "total": 38
              }
            },
            "sentence_lower_half": {
              "mean_reciprocal_rank": 0.014422950216270802,
              "median_rank": 152.5,
              "n": 38,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.09180986822558396
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 38
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.09180986822558396
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 38
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.09180986822558396
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 38
              },
              "recall_at_50": {
                "ci95": [
                  0.14971621254798143,
                  0.42008846187466364
                ],
                "rate": 0.2631578947368421,
                "successes": 10,
                "total": 38
              }
            },
            "sentence_mean": {
              "mean_reciprocal_rank": 0.03088432177419536,
              "median_rank": 78.0,
              "n": 38,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.09180986822558396
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 38
              },
              "recall_at_10": {
                "ci95": [
                  0.04170257347553526,
                  0.24130521722887316
                ],
                "rate": 0.10526315789473684,
                "successes": 4,
                "total": 38
              },
              "recall_at_5": {
                "ci95": [
                  0.004660587685768783,
                  0.1349487611595213
                ],
                "rate": 0.02631578947368421,
                "successes": 1,
                "total": 38
              },
              "recall_at_50": {
                "ci95": [
                  0.25598075748160104,
                  0.5528213200395746
                ],
                "rate": 0.39473684210526316,
                "successes": 15,
                "total": 38
              }
            },
            "sentence_min": {
              "mean_reciprocal_rank": 0.012840580775891588,
              "median_rank": 135.0,
              "n": 38,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.09180986822558396
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 38
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.09180986822558396
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 38
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.09180986822558396
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 38
              },
              "recall_at_50": {
                "ci95": [
                  0.14971621254798143,
                  0.42008846187466364
                ],
                "rate": 0.2631578947368421,
                "successes": 10,
                "total": 38
              }
            },
            "solution": {
              "mean_reciprocal_rank": 0.036402236799616544,
              "median_rank": 17,
              "n": 38,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.09180986822558396
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 38
              },
              "recall_at_10": {
                "ci95": [
                  0.027214595506987724,
                  0.20799371457771454
                ],
                "rate": 0.07894736842105263,
                "successes": 3,
                "total": 38
              },
              "recall_at_5": {
                "ci95": [
                  0.014554208147292413,
                  0.17285462131770377
                ],
                "rate": 0.05263157894736842,
                "successes": 2,
                "total": 38
              },
              "recall_at_50": {
                "ci95": [
                  0.12993707017054082,
                  0.39206812363239807
                ],
                "rate": 0.23684210526315788,
                "successes": 9,
                "total": 38
              }
            }
          }
        },
        "by_stratum": {
          "ALTERNATIVE_ROUTES": {
            "diagnostic": {
              "mean_reciprocal_rank": 0.10714285714285714,
              "median_rank": 3.5,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.07047549346981563,
                  0.45185448715169363
                ],
                "rate": 0.2,
                "successes": 3,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.07047549346981563,
                  0.45185448715169363
                ],
                "rate": 0.2,
                "successes": 3,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.10897453325692374,
                  0.5195043405598057
                ],
                "rate": 0.26666666666666666,
                "successes": 4,
                "total": 15
              }
            },
            "full_mean": {
              "mean_reciprocal_rank": 0.015601018872004338,
              "median_rank": 135,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.10897453325692374,
                  0.5195043405598057
                ],
                "rate": 0.26666666666666666,
                "successes": 4,
                "total": 15
              }
            },
            "full_min": {
              "mean_reciprocal_rank": 0.009645098746955804,
              "median_rank": 174,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.03736125910306537,
                  0.37881982832322364
                ],
                "rate": 0.13333333333333333,
                "successes": 2,
                "total": 15
              }
            },
            "sentence_lower_half": {
              "mean_reciprocal_rank": 0.016394327176082062,
              "median_rank": 78,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.10897453325692374,
                  0.5195043405598057
                ],
                "rate": 0.26666666666666666,
                "successes": 4,
                "total": 15
              }
            },
            "sentence_mean": {
              "mean_reciprocal_rank": 0.033584832919215046,
              "median_rank": 46,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.03736125910306537,
                  0.37881982832322364
                ],
                "rate": 0.13333333333333333,
                "successes": 2,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.357468301206246,
                  0.8017550385865841
                ],
                "rate": 0.6,
                "successes": 9,
                "total": 15
              }
            },
            "sentence_min": {
              "mean_reciprocal_rank": 0.017458907797614814,
              "median_rank": 75,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.1517632443971473,
                  0.5828645226148023
                ],
                "rate": 0.3333333333333333,
                "successes": 5,
                "total": 15
              }
            },
            "solution": {
              "mean_reciprocal_rank": 0.0062055958397421816,
              "median_rank": 37,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.07047549346981563,
                  0.45185448715169363
                ],
                "rate": 0.2,
                "successes": 3,
                "total": 15
              }
            }
          },
          "MULTI_DSA_COMPLETE": {
            "diagnostic": {
              "mean_reciprocal_rank": 0.09642857142857143,
              "median_rank": 5.0,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.06591599071428142,
                  0.4300888096197414
                ],
                "rate": 0.1875,
                "successes": 3,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.10182067491213048,
                  0.49498316535508774
                ],
                "rate": 0.25,
                "successes": 4,
                "total": 16
              }
            },
            "full_mean": {
              "mean_reciprocal_rank": 0.026012675644570212,
              "median_rank": 104.5,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.10182067491213048,
                  0.49498316535508774
                ],
                "rate": 0.25,
                "successes": 4,
                "total": 16
              }
            },
            "full_min": {
              "mean_reciprocal_rank": 0.008298174090520909,
              "median_rank": 191.5,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              }
            },
            "sentence_lower_half": {
              "mean_reciprocal_rank": 0.012430085986279713,
              "median_rank": 107.0,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.10182067491213048,
                  0.49498316535508774
                ],
                "rate": 0.25,
                "successes": 4,
                "total": 16
              }
            },
            "sentence_mean": {
              "mean_reciprocal_rank": 0.035474674681730045,
              "median_rank": 80.5,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.23098652405492354,
                  0.6682144360118811
                ],
                "rate": 0.4375,
                "successes": 7,
                "total": 16
              }
            },
            "sentence_min": {
              "mean_reciprocal_rank": 0.009018770982580485,
              "median_rank": 118.0,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              }
            },
            "solution": {
              "mean_reciprocal_rank": 0.07795407014157014,
              "median_rank": 30.0,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.18481232558863633,
                  0.6135895945449727
                ],
                "rate": 0.375,
                "successes": 6,
                "total": 16
              }
            }
          },
          "MULTI_PARTIAL_OPEN": {
            "diagnostic": {
              "mean_reciprocal_rank": 0.05243584857723577,
              "median_rank": 5,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.06591599071428142,
                  0.4300888096197414
                ],
                "rate": 0.1875,
                "successes": 3,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.06591599071428142,
                  0.4300888096197414
                ],
                "rate": 0.1875,
                "successes": 3,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.14164643854782039,
                  0.5559564416525933
                ],
                "rate": 0.3125,
                "successes": 5,
                "total": 16
              }
            },
            "full_mean": {
              "mean_reciprocal_rank": 0.043959792086333355,
              "median_rank": 160.5,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.06591599071428142,
                  0.4300888096197414
                ],
                "rate": 0.1875,
                "successes": 3,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.14164643854782039,
                  0.5559564416525933
                ],
                "rate": 0.3125,
                "successes": 5,
                "total": 16
              }
            },
            "full_min": {
              "mean_reciprocal_rank": 0.006492602640270839,
              "median_rank": 309.0,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              }
            },
            "sentence_lower_half": {
              "mean_reciprocal_rank": 0.012763807648294092,
              "median_rank": 183.5,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              }
            },
            "sentence_mean": {
              "mean_reciprocal_rank": 0.029773114197963108,
              "median_rank": 117.0,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.18481232558863633,
                  0.6135895945449727
                ],
                "rate": 0.375,
                "successes": 6,
                "total": 16
              }
            },
            "sentence_min": {
              "mean_reciprocal_rank": 0.008139372018028675,
              "median_rank": 246.0,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              }
            },
            "solution": {
              "mean_reciprocal_rank": 0.011663105413105413,
              "median_rank": 26,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.06591599071428142,
                  0.4300888096197414
                ],
                "rate": 0.1875,
                "successes": 3,
                "total": 16
              }
            }
          },
          "MULTI_PRIME_COMPLETE": {
            "diagnostic": {
              "mean_reciprocal_rank": 0.0927801724137931,
              "median_rank": 6,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.06591599071428142,
                  0.4300888096197414
                ],
                "rate": 0.1875,
                "successes": 3,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.14164643854782039,
                  0.5559564416525933
                ],
                "rate": 0.3125,
                "successes": 5,
                "total": 16
              }
            },
            "full_mean": {
              "mean_reciprocal_rank": 0.014760836950584857,
              "median_rank": 173.5,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              }
            },
            "full_min": {
              "mean_reciprocal_rank": 0.008876824926960914,
              "median_rank": 385.0,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              }
            },
            "sentence_lower_half": {
              "mean_reciprocal_rank": 0.012996531379068374,
              "median_rank": 136.5,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              }
            },
            "sentence_mean": {
              "mean_reciprocal_rank": 0.029873067247413936,
              "median_rank": 87.5,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.23098652405492354,
                  0.6682144360118811
                ],
                "rate": 0.4375,
                "successes": 7,
                "total": 16
              }
            },
            "sentence_min": {
              "mean_reciprocal_rank": 0.009710559533407277,
              "median_rank": 209.5,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.01111934476464252,
                  0.28328737570298945
                ],
                "rate": 0.0625,
                "successes": 1,
                "total": 16
              }
            },
            "solution": {
              "mean_reciprocal_rank": 0.07341269841269842,
              "median_rank": 14,
              "n": 16,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.1936076805344365
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 16
              },
              "recall_at_10": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              },
              "recall_at_5": {
                "ci95": [
                  0.03497748774324047,
                  0.36022827265758695
                ],
                "rate": 0.125,
                "successes": 2,
                "total": 16
              },
              "recall_at_50": {
                "ci95": [
                  0.14164643854782039,
                  0.5559564416525933
                ],
                "rate": 0.3125,
                "successes": 5,
                "total": 16
              }
            }
          },
          "SINGLE_LITERAL": {
            "diagnostic": {
              "mean_reciprocal_rank": 0.0971111111111111,
              "median_rank": 5.0,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.07047549346981563,
                  0.45185448715169363
                ],
                "rate": 0.2,
                "successes": 3,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.03736125910306537,
                  0.37881982832322364
                ],
                "rate": 0.13333333333333333,
                "successes": 2,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.10897453325692374,
                  0.5195043405598057
                ],
                "rate": 0.26666666666666666,
                "successes": 4,
                "total": 15
              }
            },
            "full_mean": {
              "mean_reciprocal_rank": 0.028245304593712196,
              "median_rank": 357,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.03736125910306537,
                  0.37881982832322364
                ],
                "rate": 0.13333333333333333,
                "successes": 2,
                "total": 15
              }
            },
            "full_min": {
              "mean_reciprocal_rank": 0.03271629277220219,
              "median_rank": 137,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.03736125910306537,
                  0.37881982832322364
                ],
                "rate": 0.13333333333333333,
                "successes": 2,
                "total": 15
              }
            },
            "sentence_lower_half": {
              "mean_reciprocal_rank": 0.05171889045631313,
              "median_rank": 63,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.24809535365737387,
                  0.698830199745016
                ],
                "rate": 0.4666666666666667,
                "successes": 7,
                "total": 15
              }
            },
            "sentence_mean": {
              "mean_reciprocal_rank": 0.04722305281690429,
              "median_rank": 104,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.10897453325692374,
                  0.5195043405598057
                ],
                "rate": 0.26666666666666666,
                "successes": 4,
                "total": 15
              }
            },
            "sentence_min": {
              "mean_reciprocal_rank": 0.052669590341767505,
              "median_rank": 59,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.0,
                  0.20388330103584862
                ],
                "rate": 0.0,
                "successes": 0,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.24809535365737387,
                  0.698830199745016
                ],
                "rate": 0.4666666666666667,
                "successes": 7,
                "total": 15
              }
            },
            "solution": {
              "mean_reciprocal_rank": 0.12016088486676722,
              "median_rank": 8.0,
              "n": 15,
              "recall_at_1": {
                "ci95": [
                  0.011866895493268553,
                  0.2981652987378003
                ],
                "rate": 0.06666666666666667,
                "successes": 1,
                "total": 15
              },
              "recall_at_10": {
                "ci95": [
                  0.07047549346981563,
                  0.45185448715169363
                ],
                "rate": 0.2,
                "successes": 3,
                "total": 15
              },
              "recall_at_5": {
                "ci95": [
                  0.07047549346981563,
                  0.45185448715169363
                ],
                "rate": 0.2,
                "successes": 3,
                "total": 15
              },
              "recall_at_50": {
                "ci95": [
                  0.1982449614134158,
                  0.6425316987937539
                ],
                "rate": 0.4,
                "successes": 6,
                "total": 15
              }
            }
          }
        },
        "cluster_bootstrap_primary_minus_diagnostic": {
          "ci95": [
            -0.24050632911392406,
            -0.05128205128205128
          ],
          "cluster": "archetype",
          "estimate": -0.14102564102564102,
          "iterations": 20000
        },
        "cluster_bootstrap_primary_minus_solution": {
          "ci95": [
            -0.11842105263157894,
            -0.012987012987012988
          ],
          "cluster": "archetype",
          "estimate": -0.0641025641025641,
          "iterations": 20000
        },
        "primary_gate_against_diagnostic": {
          "baseline_arm": "diagnostic",
          "by_positive_case_type": {
            "DIRECT_POSITIVE": {
              "baseline_rate": 0.15,
              "difference": -0.125,
              "n": 40,
              "test_rate": 0.025
            },
            "TRANSFER_POSITIVE": {
              "baseline_rate": 0.15789473684210525,
              "difference": -0.15789473684210525,
              "n": 38,
              "test_rate": 0.0
            }
          },
          "gate_passed_before_information_gate": false,
          "paired_at_5": {
            "baseline_only": 12,
            "both": 0,
            "exact_mcnemar_p_two_sided": 0.00341796875,
            "neither": 65,
            "test_only": 1
          },
          "recall_at_5_difference": -0.14102564102564102,
          "subtype_noninferiority": false,
          "test_arm": "sentence_lower_half"
        },
        "primary_gate_against_solution": {
          "baseline_arm": "solution",
          "by_positive_case_type": {
            "DIRECT_POSITIVE": {
              "baseline_rate": 0.1,
              "difference": -0.07500000000000001,
              "n": 40,
              "test_rate": 0.025
            },
            "TRANSFER_POSITIVE": {
              "baseline_rate": 0.05263157894736842,
              "difference": -0.05263157894736842,
              "n": 38,
              "test_rate": 0.0
            }
          },
          "gate_passed_before_information_gate": false,
          "paired_at_5": {
            "baseline_only": 5,
            "both": 1,
            "exact_mcnemar_p_two_sided": 0.0625,
            "neither": 72,
            "test_only": 0
          },
          "recall_at_5_difference": -0.0641025641025641,
          "subtype_noninferiority": false,
          "test_arm": "sentence_lower_half"
        },
        "primary_route_arm": "SENTENCE_LOWER_HALF"
      },
      "near_miss_target_rank_descriptive": {
        "diagnostic": {
          "mean_reciprocal_rank": 0.05527345527345527,
          "median_rank": 4.0,
          "n": 37,
          "recall_at_1": {
            "ci95": [
              -6.938893903907228e-18,
              0.09405782583720251
            ],
            "rate": 0.0,
            "successes": 0,
            "total": 37
          },
          "recall_at_10": {
            "ci95": [
              0.07651207234011151,
              0.3113648370093497
            ],
            "rate": 0.16216216216216217,
            "successes": 6,
            "total": 37
          },
          "recall_at_5": {
            "ci95": [
              0.059132944257320186,
              0.2797741178400979
            ],
            "rate": 0.13513513513513514,
            "successes": 5,
            "total": 37
          },
          "recall_at_50": {
            "ci95": [
              0.11386511993385592,
              0.3719514839196916
            ],
            "rate": 0.21621621621621623,
            "successes": 8,
            "total": 37
          }
        },
        "full_mean": {
          "mean_reciprocal_rank": 0.03140830568825891,
          "median_rank": 175,
          "n": 37,
          "recall_at_1": {
            "ci95": [
              -6.938893903907228e-18,
              0.09405782583720251
            ],
            "rate": 0.0,
            "successes": 0,
            "total": 37
          },
          "recall_at_10": {
            "ci95": [
              0.014950976833157673,
              0.17704654350813104
            ],
            "rate": 0.05405405405405406,
            "successes": 2,
            "total": 37
          },
          "recall_at_5": {
            "ci95": [
              0.014950976833157673,
              0.17704654350813104
            ],
            "rate": 0.05405405405405406,
            "successes": 2,
            "total": 37
          },
          "recall_at_50": {
            "ci95": [
              0.07651207234011151,
              0.3113648370093497
            ],
            "rate": 0.16216216216216217,
            "successes": 6,
            "total": 37
          }
        },
        "full_min": {
          "mean_reciprocal_rank": 0.014200423038923764,
          "median_rank": 260,
          "n": 37,
          "recall_at_1": {
            "ci95": [
              -6.938893903907228e-18,
              0.09405782583720251
            ],
            "rate": 0.0,
            "successes": 0,
            "total": 37
          },
          "recall_at_10": {
            "ci95": [
              0.014950976833157673,
              0.17704654350813104
            ],
            "rate": 0.05405405405405406,
            "successes": 2,
            "total": 37
          },
          "recall_at_5": {
            "ci95": [
              -6.938893903907228e-18,
              0.09405782583720251
            ],
            "rate": 0.0,
            "successes": 0,
            "total": 37
          },
          "recall_at_50": {
            "ci95": [
              0.059132944257320186,
              0.2797741178400979
            ],
            "rate": 0.13513513513513514,
            "successes": 5,
            "total": 37
          }
        },
        "sentence_lower_half": {
          "mean_reciprocal_rank": 0.053127114507144385,
          "median_rank": 83,
          "n": 37,
          "recall_at_1": {
            "ci95": [
              -6.938893903907228e-18,
              0.09405782583720251
            ],
            "rate": 0.0,
            "successes": 0,
            "total": 37
          },
          "recall_at_10": {
            "ci95": [
              0.059132944257320186,
              0.2797741178400979
            ],
            "rate": 0.13513513513513514,
            "successes": 5,
            "total": 37
          },
          "recall_at_5": {
            "ci95": [
              0.02796057410044797,
              0.2130067934928839
            ],
            "rate": 0.08108108108108109,
            "successes": 3,
            "total": 37
          },
          "recall_at_50": {
            "ci95": [
              0.26346518192014284,
              0.5651403526977063
            ],
            "rate": 0.40540540540540543,
            "successes": 15,
            "total": 37
          }
        },
        "sentence_mean": {
          "mean_reciprocal_rank": 0.0867183914381505,
          "median_rank": 56,
          "n": 37,
          "recall_at_1": {
            "ci95": [
              -6.938893903907228e-18,
              0.09405782583720251
            ],
            "rate": 0.0,
            "successes": 0,
            "total": 37
          },
          "recall_at_10": {
            "ci95": [
              0.11386511993385592,
              0.3719514839196916
            ],
            "rate": 0.21621621621621623,
            "successes": 8,
            "total": 37
          },
          "recall_at_5": {
            "ci95": [
              0.07651207234011151,
              0.3113648370093497
            ],
            "rate": 0.16216216216216217,
            "successes": 6,
            "total": 37
          },
          "recall_at_50": {
            "ci95": [
              0.3103856980437303,
              0.6161595310782052
            ],
            "rate": 0.4594594594594595,
            "successes": 17,
            "total": 37
          }
        },
        "sentence_min": {
          "mean_reciprocal_rank": 0.04346252858103701,
          "median_rank": 82,
          "n": 37,
          "recall_at_1": {
            "ci95": [
              -6.938893903907228e-18,
              0.09405782583720251
            ],
            "rate": 0.0,
            "successes": 0,
            "total": 37
          },
          "recall_at_10": {
            "ci95": [
              0.02796057410044797,
              0.2130067934928839
            ],
            "rate": 0.08108108108108109,
            "successes": 3,
            "total": 37
          },
          "recall_at_5": {
            "ci95": [
              0.014950976833157673,
              0.17704654350813104
            ],
            "rate": 0.05405405405405406,
            "successes": 2,
            "total": 37
          },
          "recall_at_50": {
            "ci95": [
              0.24063999175798265,
              0.5389956956078235
            ],
            "rate": 0.3783783783783784,
            "successes": 14,
            "total": 37
          }
        },
        "solution": {
          "mean_reciprocal_rank": 0.07484672600469965,
          "median_rank": 16.5,
          "n": 37,
          "recall_at_1": {
            "ci95": [
              0.014950976833157673,
              0.17704654350813104
            ],
            "rate": 0.05405405405405406,
            "successes": 2,
            "total": 37
          },
          "recall_at_10": {
            "ci95": [
              0.04285189940411473,
              0.24708531544126028
            ],
            "rate": 0.10810810810810811,
            "successes": 4,
            "total": 37
          },
          "recall_at_5": {
            "ci95": [
              0.02796057410044797,
              0.2130067934928839
            ],
            "rate": 0.08108108108108109,
            "successes": 3,
            "total": 37
          },
          "recall_at_50": {
            "ci95": [
              0.15397461511049093,
              0.4297816832471427
            ],
            "rate": 0.2702702702702703,
            "successes": 10,
            "total": 37
          }
        }
      },
      "frozen_verdict": "NOT_SUPPORTIVE",
      "resources": {
        "calls": 83,
        "failed_calls": 0,
        "mean_call_seconds": 67.6876626506024,
        "median_call_seconds": 64.595,
        "summed_call_hours": 1.5605766666666667,
        "token_totals": {
          "cached_input_tokens": 457728,
          "input_tokens": 1329255,
          "output_tokens": 255170,
          "reasoning_output_tokens": 110461
        },
        "wall_span_hours": 0.5413590347766877
      }
    },
    "post_hoc_hardening": {
      "prior_art_sentinel": {
        "cases": 10,
        "queries": 40,
        "verdict_counts": {
          "CLOSER_ADJACENT_PRECEDENT_FOUND": 8,
          "NO_CLOSER_PRECEDENT_FOUND": 2
        },
        "contrast_survives": 10
      },
      "experiment_8_yield_decomposition": {
        "breadth_diagnostic_counts": {
          "BROAD_ACROSS_TESTED_ARCHETYPES": 7,
          "CONCENTRATED_IN_TESTED_ARCHETYPES": 6
        },
        "partner_e6_archetypes_with_outcome": 5,
        "strict_e5_archetypes_with_outcome": 2,
        "strict_e6_archetypes_with_outcome": 4
      },
      "experiment_10_proposal_type": {
        "universe": {
          "by_experiment": {
            "3": 47,
            "4": 35,
            "5": 100,
            "6": 240
          },
          "total": 422
        },
        "agreement": {
          "evidence_dependency": {
            "agreements": 373,
            "cohen_kappa": 0.8284,
            "confusion_pass1_rows_pass2_columns": {
              "FIELD_HUMAN_INSTITUTIONAL": {
                "FIELD_HUMAN_INSTITUTIONAL": 134,
                "LAB_TECHNICAL": 4,
                "MIXED_OR_UNCLEAR": 1,
                "PRIVATE_OPERATIONAL": 6,
                "PUBLIC_DOCUMENTARY": 0
              },
              "LAB_TECHNICAL": {
                "FIELD_HUMAN_INSTITUTIONAL": 9,
                "LAB_TECHNICAL": 144,
                "MIXED_OR_UNCLEAR": 0,
                "PRIVATE_OPERATIONAL": 11,
                "PUBLIC_DOCUMENTARY": 0
              },
              "MIXED_OR_UNCLEAR": {
                "FIELD_HUMAN_INSTITUTIONAL": 4,
                "LAB_TECHNICAL": 1,
                "MIXED_OR_UNCLEAR": 1,
                "PRIVATE_OPERATIONAL": 2,
                "PUBLIC_DOCUMENTARY": 0
              },
              "PRIVATE_OPERATIONAL": {
                "FIELD_HUMAN_INSTITUTIONAL": 3,
                "LAB_TECHNICAL": 3,
                "MIXED_OR_UNCLEAR": 0,
                "PRIVATE_OPERATIONAL": 89,
                "PUBLIC_DOCUMENTARY": 1
              },
              "PUBLIC_DOCUMENTARY": {
                "FIELD_HUMAN_INSTITUTIONAL": 0,
                "LAB_TECHNICAL": 1,
                "MIXED_OR_UNCLEAR": 0,
                "PRIVATE_OPERATIONAL": 3,
                "PUBLIC_DOCUMENTARY": 5
              }
            },
            "expected_agreement": 0.3233,
            "n": 422,
            "percent_agreement": 88.4
          },
          "rows_with_any_disagreement": 68,
          "rows_with_any_disagreement_percent": 16.1,
          "substrate": {
            "agreements": 399,
            "cohen_kappa": 0.9004,
            "confusion_pass1_rows_pass2_columns": {
              "COMPUTATIONAL_INFORMATION": {
                "COMPUTATIONAL_INFORMATION": 183,
                "GOVERNANCE_PROCESS": 7,
                "HYBRID_OTHER": 1,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 4
              },
              "GOVERNANCE_PROCESS": {
                "COMPUTATIONAL_INFORMATION": 3,
                "GOVERNANCE_PROCESS": 201,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 2
              },
              "HYBRID_OTHER": {
                "COMPUTATIONAL_INFORMATION": 1,
                "GOVERNANCE_PROCESS": 0,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 0
              },
              "MEASUREMENT_INSTRUMENTATION": {
                "COMPUTATIONAL_INFORMATION": 0,
                "GOVERNANCE_PROCESS": 0,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 4,
                "PHYSICAL_MATERIAL": 0
              },
              "PHYSICAL_MATERIAL": {
                "COMPUTATIONAL_INFORMATION": 2,
                "GOVERNANCE_PROCESS": 3,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 11
              }
            },
            "expected_agreement": 0.4527,
            "n": 422,
            "percent_agreement": 94.5
          },
          "third_adjudications": 68
        },
        "exploratory_fisher": {
          "experiment_6_governance_vs_other_by_partner_status": {
            "odds_ratio": 1.2721,
            "table": [
              [
                17,
                98
              ],
              [
                15,
                110
              ]
            ],
            "two_sided_p_value": 0.571924
          },
          "table_orientation": "rows: governance, other; columns: partner, not-partner"
        }
      },
      "e10b_archetype_substrate_join": {
        "status": "COMPLETE_EXPLICITLY_POST_HOC",
        "verdict": "DESCRIPTIVE_TWO_LEVEL_SUBSTRATE_CONSTRAINT",
        "universes": {
          "all_e10": 422,
          "archetypes": 10,
          "primary_complete_census_e5_e6": 340
        },
        "corrected_archetype_summary": {
          "archetypes": 10,
          "corpus_mean_sf_reference": 0.374,
          "deadweight_loss_reduction_included": true,
          "unweighted_mean_sf": 0.36583333333333334
        },
        "primary_archetype_level_tests": {
          "sf_vs_constraint_escape_share": {
            "extreme_assignments": 3084480,
            "leave_one_archetype_out": {
              "maximum_rho": 0.239457,
              "minimum_rho": -0.256439,
              "rows": [
                {
                  "omitted_archetype": "bounded_rivalry_governance",
                  "rho": -0.125409
                },
                {
                  "omitted_archetype": "catalytic_pathway_enablement",
                  "rho": -0.162911
                },
                {
                  "omitted_archetype": "computability_boundary_mapping",
                  "rho": 0.030523
                },
                {
                  "omitted_archetype": "deadweight_loss_reduction",
                  "rho": -0.124838
                },
                {
                  "omitted_archetype": "invariant_mode_decomposition_design",
                  "rho": -0.256439
                },
                {
                  "omitted_archetype": "layer_decay_and_expiration_management",
                  "rho": 0.030523
                },
                {
                  "omitted_archetype": "negative_space_design",
                  "rho": 0.239457
                },
                {
                  "omitted_archetype": "predictive_residual_processing",
                  "rho": 0.030523
                },
                {
                  "omitted_archetype": "representation_independent_interface_contract",
                  "rho": -0.125409
                },
                {
                  "omitted_archetype": "ritualized_meaning_and_commitment_enactment",
                  "rho": -0.124838
                }
              ]
            },
            "spearman_rho": -0.058155,
            "total_assignments": 3628800,
            "two_sided_exact_permutation_p": 0.85
          },
          "sf_vs_governance_share_within_dominant_channels": {
            "extreme_assignments": 153024,
            "leave_one_archetype_out": {
              "maximum_rho": -0.600917,
              "minimum_rho": -0.821723,
              "rows": [
                {
                  "omitted_archetype": "bounded_rivalry_governance",
                  "rho": -0.600917
                },
                {
                  "omitted_archetype": "catalytic_pathway_enablement",
                  "rho": -0.655963
                },
                {
                  "omitted_archetype": "computability_boundary_mapping",
                  "rho": -0.617391
                },
                {
                  "omitted_archetype": "deadweight_loss_reduction",
                  "rho": -0.604545
                },
                {
                  "omitted_archetype": "invariant_mode_decomposition_design",
                  "rho": -0.617391
                },
                {
                  "omitted_archetype": "layer_decay_and_expiration_management",
                  "rho": -0.821723
                },
                {
                  "omitted_archetype": "negative_space_design",
                  "rho": -0.775229
                },
                {
                  "omitted_archetype": "predictive_residual_processing",
                  "rho": -0.617391
                },
                {
                  "omitted_archetype": "representation_independent_interface_contract",
                  "rho": -0.70561
                },
                {
                  "omitted_archetype": "ritualized_meaning_and_commitment_enactment",
                  "rho": -0.604545
                }
              ]
            },
            "spearman_rho": -0.665584,
            "total_assignments": 3628800,
            "two_sided_exact_permutation_p": 0.04216931
          }
        },
        "primary_by_band": [
          {
            "constraint_escape_share": 0.045454545454545456,
            "dominant_channel_share": 0.9545454545454546,
            "governance_rate": 0.875,
            "governance_share_within_dominant_channels": 0.9166666666666666,
            "proposals": 88,
            "sf_band": "FRAMED",
            "substrate_counts": {
              "COMPUTATIONAL_INFORMATION": 7,
              "GOVERNANCE_PROCESS": 77,
              "HYBRID_OTHER": 0,
              "MEASUREMENT_INSTRUMENTATION": 1,
              "PHYSICAL_MATERIAL": 3
            }
          },
          {
            "constraint_escape_share": 0.0,
            "dominant_channel_share": 1.0,
            "governance_rate": 0.4895833333333333,
            "governance_share_within_dominant_channels": 0.4895833333333333,
            "proposals": 96,
            "sf_band": "MIDDLE",
            "substrate_counts": {
              "COMPUTATIONAL_INFORMATION": 49,
              "GOVERNANCE_PROCESS": 47,
              "HYBRID_OTHER": 0,
              "MEASUREMENT_INSTRUMENTATION": 0,
              "PHYSICAL_MATERIAL": 0
            }
          },
          {
            "constraint_escape_share": 0.10256410256410256,
            "dominant_channel_share": 0.8974358974358975,
            "governance_rate": 0.23717948717948717,
            "governance_share_within_dominant_channels": 0.2642857142857143,
            "proposals": 156,
            "sf_band": "STRUCTURAL",
            "substrate_counts": {
              "COMPUTATIONAL_INFORMATION": 103,
              "GOVERNANCE_PROCESS": 37,
              "HYBRID_OTHER": 2,
              "MEASUREMENT_INSTRUMENTATION": 3,
              "PHYSICAL_MATERIAL": 11
            }
          }
        ]
      },
      "e9_substrate_followup": {
        "status": "COMPLETE_BLINDED_POST_HOC_REANALYSIS",
        "verdict": "SUPPORTIVE_OF_TWO_LEVEL_SUBSTRATE_CONSTRAINT",
        "agreement": {
          "agreements": 140,
          "cohen_kappa": 0.8849516797054763,
          "disagreements_adjudicated": 10,
          "rate": 0.9333333333333333,
          "total": 150
        },
        "overall": {
          "archetypes": 50,
          "dominant_channel_share": 0.9266666666666666,
          "dominant_channel_wilson_95": [
            0.873464,
            0.958561
          ],
          "proposals": 150,
          "substrate_counts": {
            "COMPUTATIONAL_INFORMATION": 71,
            "GOVERNANCE_PROCESS": 68,
            "HYBRID_OTHER": 0,
            "MEASUREMENT_INSTRUMENTATION": 0,
            "PHYSICAL_MATERIAL": 11
          }
        },
        "frozen_gate_conditions": {
          "all50_source_modulation": true,
          "broad_constraint_overall_and_each_band": true,
          "negative_direction_at_least_two_domains": true
        },
        "all_50_balanced_mechanism_analysis": {
          "association": {
            "archetypes": 50,
            "sf_vs_constraint_escape_share": {
              "draws": 200000,
              "extreme_draws": 135886,
              "leave_one_archetype_out": {
                "maximum_rho": 0.125068,
                "minimum_rho": 0.014466,
                "rows": [
                  {
                    "omitted_archetype": "activation_decay_measurement",
                    "rho": 0.054271
                  },
                  {
                    "omitted_archetype": "additive_measure_space_design",
                    "rho": 0.076853
                  },
                  {
                    "omitted_archetype": "agentic_control_loop_design",
                    "rho": 0.054275
                  },
                  {
                    "omitted_archetype": "anchoring_reset",
                    "rho": 0.054271
                  },
                  {
                    "omitted_archetype": "authority_legitimacy_and_consent_foundations",
                    "rho": 0.04377
                  },
                  {
                    "omitted_archetype": "authority_mentor_relationship_anchoring",
                    "rho": 0.054271
                  },
                  {
                    "omitted_archetype": "backcasting_pathway_design",
                    "rho": 0.043769
                  },
                  {
                    "omitted_archetype": "backpressure",
                    "rho": 0.071783
                  },
                  {
                    "omitted_archetype": "bounded_rivalry_governance",
                    "rho": 0.054275
                  },
                  {
                    "omitted_archetype": "buffering",
                    "rho": 0.068876
                  },
                  {
                    "omitted_archetype": "bulkhead_isolation",
                    "rho": 0.056185
                  },
                  {
                    "omitted_archetype": "catalytic_pathway_enablement",
                    "rho": 0.041689
                  },
                  {
                    "omitted_archetype": "circuit_breaker",
                    "rho": 0.076853
                  },
                  {
                    "omitted_archetype": "computability_boundary_mapping",
                    "rho": 0.076853
                  },
                  {
                    "omitted_archetype": "constraint_envelope_adjustment",
                    "rho": 0.057775
                  },
                  {
                    "omitted_archetype": "constraint_propagation_and_decoupling",
                    "rho": 0.076853
                  },
                  {
                    "omitted_archetype": "context_keyed_representation_switching",
                    "rho": 0.063028
                  },
                  {
                    "omitted_archetype": "controlled_reentry",
                    "rho": 0.071783
                  },
                  {
                    "omitted_archetype": "deadweight_loss_reduction",
                    "rho": 0.047268
                  },
                  {
                    "omitted_archetype": "decoupling_via_interface",
                    "rho": 0.050751
                  },
                  {
                    "omitted_archetype": "disequilibrium_leverage_and_dissipation_management",
                    "rho": 0.014466
                  },
                  {
                    "omitted_archetype": "emergent_pattern_detection",
                    "rho": 0.076853
                  },
                  {
                    "omitted_archetype": "failover",
                    "rho": 0.045311
                  },
                  {
                    "omitted_archetype": "feedback_loop_redirection",
                    "rho": 0.066526
                  },
                  {
                    "omitted_archetype": "flow_diversion_or_rerouting",
                    "rho": 0.014466
                  },
                  {
                    "omitted_archetype": "formal_derivation_system_design",
                    "rho": 0.057775
                  },
                  {
                    "omitted_archetype": "graceful_degradation",
                    "rho": 0.110567
                  },
                  {
                    "omitted_archetype": "incentive_compatible_rule_design",
                    "rho": 0.043769
                  },
                  {
                    "omitted_archetype": "invariant_mode_decomposition_design",
                    "rho": 0.076853
                  },
                  {
                    "omitted_archetype": "inversion_of_control",
                    "rho": 0.076853
                  },
                  {
                    "omitted_archetype": "layer_decay_and_expiration_management",
                    "rho": 0.076853
                  },
                  {
                    "omitted_archetype": "load_balancing",
                    "rho": 0.059523
                  },
                  {
                    "omitted_archetype": "load_leveling_or_demand_smoothing",
                    "rho": 0.056025
                  },
                  {
                    "omitted_archetype": "load_shedding",
                    "rho": 0.054275
                  },
                  {
                    "omitted_archetype": "modular_decomposition",
                    "rho": 0.110567
                  },
                  {
                    "omitted_archetype": "negative_space_design",
                    "rho": 0.125068
                  },
                  {
                    "omitted_archetype": "objective_weighting_governance",
                    "rho": 0.054271
                  },
                  {
                    "omitted_archetype": "pareto_frontier_navigation",
                    "rho": 0.047272
                  },
                  {
                    "omitted_archetype": "prediction_error_learning_calibration",
                    "rho": 0.076853
                  },
                  {
                    "omitted_archetype": "predictive_residual_processing",
                    "rho": 0.076853
                  },
                  {
                    "omitted_archetype": "preimage_set_characterization",
                    "rho": 0.076853
                  },
                  {
                    "omitted_archetype": "priority_based_admission",
                    "rho": 0.054275
                  },
                  {
                    "omitted_archetype": "progressive_disclosure",
                    "rho": 0.059523
                  },
                  {
                    "omitted_archetype": "rate_limiting",
                    "rho": 0.054271
                  },
                  {
                    "omitted_archetype": "representation_independent_interface_contract",
                    "rho": 0.054271
                  },
                  {
                    "omitted_archetype": "ritualized_meaning_and_commitment_enactment",
                    "rho": 0.047268
                  },
                  {
                    "omitted_archetype": "search_space_pruning",
                    "rho": 0.066526
                  },
                  {
                    "omitted_archetype": "substrate_lineage_risk_audit",
                    "rho": 0.054275
                  },
                  {
                    "omitted_archetype": "synchronized_release_dampening",
                    "rho": 0.014466
                  },
                  {
                    "omitted_archetype": "versioning_and_quality_discrimination",
                    "rho": 0.047268
                  }
                ]
              },
              "seed": 2026080510,
              "spearman_rho": 0.060926,
              "two_sided_seeded_permutation_p": 0.6794316
            },
            "sf_vs_governance_share_within_dominant_channels": {
              "draws": 200000,
              "extreme_draws": 1,
              "leave_one_archetype_out": {
                "maximum_rho": -0.607962,
                "minimum_rho": -0.687852,
                "rows": [
                  {
                    "omitted_archetype": "activation_decay_measurement",
                    "rho": -0.622251
                  },
                  {
                    "omitted_archetype": "additive_measure_space_design",
                    "rho": -0.613376
                  },
                  {
                    "omitted_archetype": "agentic_control_loop_design",
                    "rho": -0.620505
                  },
                  {
                    "omitted_archetype": "anchoring_reset",
                    "rho": -0.622251
                  },
                  {
                    "omitted_archetype": "authority_legitimacy_and_consent_foundations",
                    "rho": -0.607978
                  },
                  {
                    "omitted_archetype": "authority_mentor_relationship_anchoring",
                    "rho": -0.617976
                  },
                  {
                    "omitted_archetype": "backcasting_pathway_design",
                    "rho": -0.607962
                  },
                  {
                    "omitted_archetype": "backpressure",
                    "rho": -0.625245
                  },
                  {
                    "omitted_archetype": "bounded_rivalry_governance",
                    "rho": -0.615673
                  },
                  {
                    "omitted_archetype": "buffering",
                    "rho": -0.619243
                  },
                  {
                    "omitted_archetype": "bulkhead_isolation",
                    "rho": -0.62146
                  },
                  {
                    "omitted_archetype": "catalytic_pathway_enablement",
                    "rho": -0.623754
                  },
                  {
                    "omitted_archetype": "circuit_breaker",
                    "rho": -0.627368
                  },
                  {
                    "omitted_archetype": "computability_boundary_mapping",
                    "rho": -0.613376
                  },
                  {
                    "omitted_archetype": "constraint_envelope_adjustment",
                    "rho": -0.628001
                  },
                  {
                    "omitted_archetype": "constraint_propagation_and_decoupling",
                    "rho": -0.613376
                  },
                  {
                    "omitted_archetype": "context_keyed_representation_switching",
                    "rho": -0.626996
                  },
                  {
                    "omitted_archetype": "controlled_reentry",
                    "rho": -0.638027
                  },
                  {
                    "omitted_archetype": "deadweight_loss_reduction",
                    "rho": -0.611564
                  },
                  {
                    "omitted_archetype": "decoupling_via_interface",
                    "rho": -0.626996
                  },
                  {
                    "omitted_archetype": "disequilibrium_leverage_and_dissipation_management",
                    "rho": -0.639618
                  },
                  {
                    "omitted_archetype": "emergent_pattern_detection",
                    "rho": -0.613376
                  },
                  {
                    "omitted_archetype": "failover",
                    "rho": -0.628315
                  },
                  {
                    "omitted_archetype": "feedback_loop_redirection",
                    "rho": -0.625062
                  },
                  {
                    "omitted_archetype": "flow_diversion_or_rerouting",
                    "rho": -0.639618
                  },
                  {
                    "omitted_archetype": "formal_derivation_system_design",
                    "rho": -0.63207
                  },
                  {
                    "omitted_archetype": "graceful_degradation",
                    "rho": -0.629328
                  },
                  {
                    "omitted_archetype": "incentive_compatible_rule_design",
                    "rho": -0.607962
                  },
                  {
                    "omitted_archetype": "invariant_mode_decomposition_design",
                    "rho": -0.613376
                  },
                  {
                    "omitted_archetype": "inversion_of_control",
                    "rho": -0.613376
                  },
                  {
                    "omitted_archetype": "layer_decay_and_expiration_management",
                    "rho": -0.677759
                  },
                  {
                    "omitted_archetype": "load_balancing",
                    "rho": -0.622117
                  },
                  {
                    "omitted_archetype": "load_leveling_or_demand_smoothing",
                    "rho": -0.623408
                  },
                  {
                    "omitted_archetype": "load_shedding",
                    "rho": -0.620505
                  },
                  {
                    "omitted_archetype": "modular_decomposition",
                    "rho": -0.629328
                  },
                  {
                    "omitted_archetype": "negative_space_design",
                    "rho": -0.687852
                  },
                  {
                    "omitted_archetype": "objective_weighting_governance",
                    "rho": -0.622251
                  },
                  {
                    "omitted_archetype": "pareto_frontier_navigation",
                    "rho": -0.620612
                  },
                  {
                    "omitted_archetype": "prediction_error_learning_calibration",
                    "rho": -0.627368
                  },
                  {
                    "omitted_archetype": "predictive_residual_processing",
                    "rho": -0.613376
                  },
                  {
                    "omitted_archetype": "preimage_set_characterization",
                    "rho": -0.613376
                  },
                  {
                    "omitted_archetype": "priority_based_admission",
                    "rho": -0.620376
                  },
                  {
                    "omitted_archetype": "progressive_disclosure",
                    "rho": -0.628592
                  },
                  {
                    "omitted_archetype": "rate_limiting",
                    "rho": -0.637029
                  },
                  {
                    "omitted_archetype": "representation_independent_interface_contract",
                    "rho": -0.637029
                  },
                  {
                    "omitted_archetype": "ritualized_meaning_and_commitment_enactment",
                    "rho": -0.611564
                  },
                  {
                    "omitted_archetype": "search_space_pruning",
                    "rho": -0.625334
                  },
                  {
                    "omitted_archetype": "substrate_lineage_risk_audit",
                    "rho": -0.615673
                  },
                  {
                    "omitted_archetype": "synchronized_release_dampening",
                    "rho": -0.613376
                  },
                  {
                    "omitted_archetype": "versioning_and_quality_discrimination",
                    "rho": -0.611564
                  }
                ]
              },
              "seed": 2026080509,
              "spearman_rho": -0.624205,
              "two_sided_seeded_permutation_p": 1e-05
            }
          },
          "by_band": [
            {
              "dominant_channel_share": 0.9523809523809523,
              "dominant_channel_wilson_95": [
                0.773306,
                0.991544
              ],
              "governance_share_within_dominant_channels": 0.9,
              "proposals": 21,
              "sf_band": "FRAMED",
              "substrate_counts": {
                "COMPUTATIONAL_INFORMATION": 2,
                "GOVERNANCE_PROCESS": 18,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 1
              }
            },
            {
              "dominant_channel_share": 0.9393939393939394,
              "dominant_channel_wilson_95": [
                0.803938,
                0.983219
              ],
              "governance_share_within_dominant_channels": 0.7096774193548387,
              "proposals": 33,
              "sf_band": "MIDDLE",
              "substrate_counts": {
                "COMPUTATIONAL_INFORMATION": 9,
                "GOVERNANCE_PROCESS": 22,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 2
              }
            },
            {
              "dominant_channel_share": 0.9166666666666666,
              "dominant_channel_wilson_95": [
                0.844101,
                0.957169
              ],
              "governance_share_within_dominant_channels": 0.3181818181818182,
              "proposals": 96,
              "sf_band": "STRUCTURAL",
              "substrate_counts": {
                "COMPUTATIONAL_INFORMATION": 60,
                "GOVERNANCE_PROCESS": 28,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 8
              }
            }
          ],
          "by_domain": [
            {
              "domain_slug": "accounting_auditing",
              "dominant_channel_share": 1.0,
              "dominant_channel_wilson_95": [
                0.928652,
                1.0
              ],
              "governance_share_within_dominant_channels": 0.66,
              "proposals": 50,
              "substrate_counts": {
                "COMPUTATIONAL_INFORMATION": 17,
                "GOVERNANCE_PROCESS": 33,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 0
              }
            },
            {
              "domain_slug": "chemistry_materials",
              "dominant_channel_share": 0.78,
              "dominant_channel_wilson_95": [
                0.647585,
                0.872461
              ],
              "governance_share_within_dominant_channels": 0.48717948717948717,
              "proposals": 50,
              "substrate_counts": {
                "COMPUTATIONAL_INFORMATION": 20,
                "GOVERNANCE_PROCESS": 19,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 11
              }
            },
            {
              "domain_slug": "computer_science",
              "dominant_channel_share": 1.0,
              "dominant_channel_wilson_95": [
                0.928652,
                1.0
              ],
              "governance_share_within_dominant_channels": 0.32,
              "proposals": 50,
              "substrate_counts": {
                "COMPUTATIONAL_INFORMATION": 34,
                "GOVERNANCE_PROCESS": 16,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 0
              }
            }
          ],
          "by_sample_stratum": [
            {
              "dominant_channel_share": 0.9583333333333334,
              "dominant_channel_wilson_95": [
                0.884507,
                0.985729
              ],
              "governance_share_within_dominant_channels": 0.4927536231884058,
              "proposals": 72,
              "sample_stratum": "GENERATED_RANDOM_UNTESTED",
              "substrate_counts": {
                "COMPUTATIONAL_INFORMATION": 35,
                "GOVERNANCE_PROCESS": 34,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 3
              }
            },
            {
              "dominant_channel_share": 0.875,
              "dominant_channel_wilson_95": [
                0.752995,
                0.941429
              ],
              "governance_share_within_dominant_channels": 0.5,
              "proposals": 48,
              "sample_stratum": "HAND_CURATED_CENSUS",
              "substrate_counts": {
                "COMPUTATIONAL_INFORMATION": 21,
                "GOVERNANCE_PROCESS": 21,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 6
              }
            },
            {
              "dominant_channel_share": 0.9333333333333333,
              "dominant_channel_wilson_95": [
                0.786765,
                0.981523
              ],
              "governance_share_within_dominant_channels": 0.4642857142857143,
              "proposals": 30,
              "sample_stratum": "PREVIOUSLY_TESTED_REFERENCE",
              "substrate_counts": {
                "COMPUTATIONAL_INFORMATION": 15,
                "GOVERNANCE_PROCESS": 13,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 2
              }
            }
          ],
          "domain_directions": [
            {
              "domain_slug": "accounting_auditing",
              "eligible_proposals": 50,
              "negative": true,
              "rho": -0.451387
            },
            {
              "domain_slug": "chemistry_materials",
              "eligible_proposals": 39,
              "negative": true,
              "rho": -0.671259
            },
            {
              "domain_slug": "computer_science",
              "eligible_proposals": 50,
              "negative": true,
              "rho": -0.586132
            }
          ]
        },
        "random_generated_primary_stratum": {
          "archetypes": 24,
          "association": {
            "archetypes": 24,
            "sf_vs_constraint_escape_share": {
              "draws": 200000,
              "extreme_draws": 86603,
              "leave_one_archetype_out": {
                "maximum_rho": 0.356866,
                "minimum_rho": 0.047242,
                "rows": [
                  {
                    "omitted_archetype": "activation_decay_measurement",
                    "rho": 0.179007
                  },
                  {
                    "omitted_archetype": "additive_measure_space_design",
                    "rho": 0.197626
                  },
                  {
                    "omitted_archetype": "agentic_control_loop_design",
                    "rho": 0.179145
                  },
                  {
                    "omitted_archetype": "anchoring_reset",
                    "rho": 0.179007
                  },
                  {
                    "omitted_archetype": "authority_legitimacy_and_consent_foundations",
                    "rho": 0.15924
                  },
                  {
                    "omitted_archetype": "authority_mentor_relationship_anchoring",
                    "rho": 0.179145
                  },
                  {
                    "omitted_archetype": "backcasting_pathway_design",
                    "rho": 0.159199
                  },
                  {
                    "omitted_archetype": "constraint_propagation_and_decoupling",
                    "rho": 0.197626
                  },
                  {
                    "omitted_archetype": "context_keyed_representation_switching",
                    "rho": 0.179145
                  },
                  {
                    "omitted_archetype": "disequilibrium_leverage_and_dissipation_management",
                    "rho": 0.047242
                  },
                  {
                    "omitted_archetype": "emergent_pattern_detection",
                    "rho": 0.197626
                  },
                  {
                    "omitted_archetype": "formal_derivation_system_design",
                    "rho": 0.179145
                  },
                  {
                    "omitted_archetype": "incentive_compatible_rule_design",
                    "rho": 0.159199
                  },
                  {
                    "omitted_archetype": "inversion_of_control",
                    "rho": 0.197626
                  },
                  {
                    "omitted_archetype": "modular_decomposition",
                    "rho": 0.356866
                  },
                  {
                    "omitted_archetype": "objective_weighting_governance",
                    "rho": 0.179007
                  },
                  {
                    "omitted_archetype": "pareto_frontier_navigation",
                    "rho": 0.15924
                  },
                  {
                    "omitted_archetype": "prediction_error_learning_calibration",
                    "rho": 0.197626
                  },
                  {
                    "omitted_archetype": "preimage_set_characterization",
                    "rho": 0.197626
                  },
                  {
                    "omitted_archetype": "progressive_disclosure",
                    "rho": 0.179145
                  },
                  {
                    "omitted_archetype": "search_space_pruning",
                    "rho": 0.179145
                  },
                  {
                    "omitted_archetype": "substrate_lineage_risk_audit",
                    "rho": 0.179145
                  },
                  {
                    "omitted_archetype": "synchronized_release_dampening",
                    "rho": 0.047242
                  },
                  {
                    "omitted_archetype": "versioning_and_quality_discrimination",
                    "rho": 0.15924
                  }
                ]
              },
              "seed": 2026080610,
              "spearman_rho": 0.176352,
              "two_sided_seeded_permutation_p": 0.43301783
            },
            "sf_vs_governance_share_within_dominant_channels": {
              "draws": 200000,
              "extreme_draws": 2,
              "leave_one_archetype_out": {
                "maximum_rho": -0.77859,
                "minimum_rho": -0.812094,
                "rows": [
                  {
                    "omitted_archetype": "activation_decay_measurement",
                    "rho": -0.79653
                  },
                  {
                    "omitted_archetype": "additive_measure_space_design",
                    "rho": -0.782757
                  },
                  {
                    "omitted_archetype": "agentic_control_loop_design",
                    "rho": -0.788775
                  },
                  {
                    "omitted_archetype": "anchoring_reset",
                    "rho": -0.79653
                  },
                  {
                    "omitted_archetype": "authority_legitimacy_and_consent_foundations",
                    "rho": -0.778791
                  },
                  {
                    "omitted_archetype": "authority_mentor_relationship_anchoring",
                    "rho": -0.791748
                  },
                  {
                    "omitted_archetype": "backcasting_pathway_design",
                    "rho": -0.77859
                  },
                  {
                    "omitted_archetype": "constraint_propagation_and_decoupling",
                    "rho": -0.782757
                  },
                  {
                    "omitted_archetype": "context_keyed_representation_switching",
                    "rho": -0.794593
                  },
                  {
                    "omitted_archetype": "disequilibrium_leverage_and_dissipation_management",
                    "rho": -0.812094
                  },
                  {
                    "omitted_archetype": "emergent_pattern_detection",
                    "rho": -0.782757
                  },
                  {
                    "omitted_archetype": "formal_derivation_system_design",
                    "rho": -0.794593
                  },
                  {
                    "omitted_archetype": "incentive_compatible_rule_design",
                    "rho": -0.77859
                  },
                  {
                    "omitted_archetype": "inversion_of_control",
                    "rho": -0.782757
                  },
                  {
                    "omitted_archetype": "modular_decomposition",
                    "rho": -0.807588
                  },
                  {
                    "omitted_archetype": "objective_weighting_governance",
                    "rho": -0.79653
                  },
                  {
                    "omitted_archetype": "pareto_frontier_navigation",
                    "rho": -0.798045
                  },
                  {
                    "omitted_archetype": "prediction_error_learning_calibration",
                    "rho": -0.804746
                  },
                  {
                    "omitted_archetype": "preimage_set_characterization",
                    "rho": -0.782757
                  },
                  {
                    "omitted_archetype": "progressive_disclosure",
                    "rho": -0.794593
                  },
                  {
                    "omitted_archetype": "search_space_pruning",
                    "rho": -0.794593
                  },
                  {
                    "omitted_archetype": "substrate_lineage_risk_audit",
                    "rho": -0.791748
                  },
                  {
                    "omitted_archetype": "synchronized_release_dampening",
                    "rho": -0.782757
                  },
                  {
                    "omitted_archetype": "versioning_and_quality_discrimination",
                    "rho": -0.778791
                  }
                ]
              },
              "seed": 2026080609,
              "spearman_rho": -0.791353,
              "two_sided_seeded_permutation_p": 1.5e-05
            }
          },
          "by_band": [
            {
              "dominant_channel_share": 1.0,
              "dominant_channel_wilson_95": [
                0.757506,
                1.0
              ],
              "governance_share_within_dominant_channels": 1.0,
              "proposals": 12,
              "sf_band": "FRAMED",
              "substrate_counts": {
                "COMPUTATIONAL_INFORMATION": 0,
                "GOVERNANCE_PROCESS": 12,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 0
              }
            },
            {
              "dominant_channel_share": 0.9333333333333333,
              "dominant_channel_wilson_95": [
                0.701835,
                0.988133
              ],
              "governance_share_within_dominant_channels": 0.7857142857142857,
              "proposals": 15,
              "sf_band": "MIDDLE",
              "substrate_counts": {
                "COMPUTATIONAL_INFORMATION": 3,
                "GOVERNANCE_PROCESS": 11,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 1
              }
            },
            {
              "dominant_channel_share": 0.9555555555555556,
              "dominant_channel_wilson_95": [
                0.851725,
                0.987726
              ],
              "governance_share_within_dominant_channels": 0.2558139534883721,
              "proposals": 45,
              "sf_band": "STRUCTURAL",
              "substrate_counts": {
                "COMPUTATIONAL_INFORMATION": 32,
                "GOVERNANCE_PROCESS": 11,
                "HYBRID_OTHER": 0,
                "MEASUREMENT_INSTRUMENTATION": 0,
                "PHYSICAL_MATERIAL": 2
              }
            }
          ],
          "proposals": 72
        }
      }
    }
  },
  "interpretation_boundary": "This file harmonizes preserved experiment artifacts for report drafting. It does not make heterogeneous experiment endpoints equivalent.",
  "dossier_ordering": {
    "status": "POST_HOC_ORDERING_AID_NOT_AN_EXPERIMENTAL_ENDPOINT",
    "profiles": {
      "balanced": {
        "meaningful_impact": 15,
        "stakeholder_pull": 10,
        "incremental_advantage": 10,
        "external_distinctiveness": 10,
        "technical_implementability": 15,
        "adoption_authority_feasibility": 10,
        "evidence_readiness": 10,
        "pilot_speed_cost": 10,
        "scalability": 5,
        "safety_net_benefit": 5
      },
      "deployment_heavy": {
        "meaningful_impact": 10,
        "stakeholder_pull": 5,
        "incremental_advantage": 5,
        "external_distinctiveness": 5,
        "technical_implementability": 20,
        "adoption_authority_feasibility": 15,
        "evidence_readiness": 15,
        "pilot_speed_cost": 15,
        "scalability": 5,
        "safety_net_benefit": 5
      },
      "impact_heavy": {
        "meaningful_impact": 25,
        "stakeholder_pull": 15,
        "incremental_advantage": 15,
        "external_distinctiveness": 5,
        "technical_implementability": 10,
        "adoption_authority_feasibility": 5,
        "evidence_readiness": 5,
        "pilot_speed_cost": 5,
        "scalability": 10,
        "safety_net_benefit": 5
      }
    },
    "first_evidence_affordability_proxy": {
      "UNDER_10K": 5,
      "10K_TO_50K": 4,
      "50K_TO_250K": 3,
      "250K_TO_1M": 2,
      "OVER_1M": 1
    },
    "balanced_rank_bands": {
      "A": "1-10",
      "B": "11-25",
      "C": "26-45",
      "D": "46-59"
    }
  }
}