diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..4795860 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +results/design-open-set-stress-n60/*.csv -text -diff diff --git a/docs/design_open_set_stress_n60_result.md b/docs/design_open_set_stress_n60_result.md new file mode 100644 index 0000000..81136cb --- /dev/null +++ b/docs/design_open_set_stress_n60_result.md @@ -0,0 +1,31 @@ +# Frozen locked-N60 open-set stress result + +The immutable package in `results/design-open-set-stress-n60/` was produced from clean +commit `c71695fda83ae93407599a909097962ee3fa9e0e`. It byte-verifies and reconstructs the +chronologically locked equal-N60 allocation (SHA-256 +`a823be49faf6c6cbebf60b11d4b5ca895cf7734d6e9c577ee98f97a5907b69b2`) from source +commit `1b2028929ac6ebc1cce0882f0c22af9918044342` and explicit allocation seed 7. + +The frozen settings use 100 calibration, 100 calibration-audit, and 200 evaluation +replicates; threshold, audit, and evaluation seeds are 104729, 130363, and 155921. The +held-out fraction is 0.35. No threshold or evaluation was changed after inspecting results. + +| Allocation | Weakest pure rate (Wilson lower; raw closed-set) | Worst 50/50 mixture false-pure rate (upper) | Null false-pure rate (upper) | Nonlinear probe false-pure rate (upper) | +|---|---:|---:|---:|---:| +| Coupled novelty | 0.030 (0.0138; 0.425) | 0.460 (0.5292) | 0.010 (0.0357) | 0.035 (0.0705) | +| Uniform factorial | 0.425 (0.3585; 0.720) | 0.545 (0.6125) | 0.015 (0.0432) | 0.005 (0.0278) | +| Locked heuristic maximin | 0.750 (0.6857; 0.820) | 0.800 (0.8495) | 0.020 (0.0503) | 0.515 (0.5833) | + +For the locked heuristic maximin allocation, the pure-over-null, winner-over-runner, and +flexible-over-pure thresholds are 1.4684378147, 0.7589681024, and 2.0743104877. Its +worst mixture is gain plus update, while its weakest pure generator is surprise. + +This is failure-boundary evidence, not open-set robustness. Null control is bounded, and +the locked maximin schedule retains reasonable matched-pure performance, but the current +adequacy rule frequently labels mixtures and the single declared nonlinear probe as pure. +The artifact does not justify claims for arbitrary mixtures, nonlinear alternatives, +serial dependence, sensor dynamics, subject hierarchy, or physical protocol feasibility. + +The package checksum-table SHA-256 is +`44a5188c43bda52e6fc9dc7007cf2de44a9671e9c5477ac88c3173c06cfdbd80`; the manifest +SHA-256 is `d840a2ec34f5a386109c7f985033b53144fdcc1b1a0e0592b3274c0e38902b64`. diff --git a/docs/design_stress.md b/docs/design_stress.md new file mode 100644 index 0000000..877c30b --- /dev/null +++ b/docs/design_stress.md @@ -0,0 +1,124 @@ +# Post-freeze design abstention stress + +This module is a **versioned sensitivity analysis**, not part of the immutable +five-seed matched-generator evidence. It asks whether a deliberately conservative +classifier can avoid confident pure-candidate calls when the generator is null or +is an unmodelled 50/50 combination of two candidates. It does not turn synthetic +recovery into biological evidence. + +## Frozen schedule + +The primary run evaluates the chronologically earlier, equal-budget 60-draw +allocation frozen for the five-seed paper benchmark. The CLI verifies the exact +allocation-file SHA-256, source-code commit, and explicit seed metadata, loads all +three designs from that file, and reconstructs all three deterministic constructors +before accepting the counts. The accepted CSV does not itself contain a seed +column, so the seed is a required CLI input recorded in provenance rather than an +invented file field. The accepted summary did not serialize the optimizer cap, so this contract +also records the source-code setting: the maximin constructor uses +`max_point_fraction=0.15` (a cap of 9 at N=60), while the cap does not apply to +the coupled-novelty or uniform-factorial comparators. In the frozen file the +observed maxima are 8, 12, and 1 respectively. Applying the optimizer cap to the +novelty comparator would therefore change the accepted benchmark rather than +validate it. Hash, seed, constructor, or unused-override mismatches are rejected. +This prevents a later certificate or stress result from silently changing the +primary schedule. + +The artifact still reports each design's population observation-equivalent index, +recomputed from its 60-draw geometry using + +```math +G(R)=\tfrac12\log\!\left(1+a^2R/\sigma^2\right). +``` + +With standardized generating signals, `a=1`, `sigma=1`, and target held-out +log-score gap 5, those indices are 45 for the heuristic maximin design, 93 for +uniform factorial, and 1,113 for coupled novelty. The common primary budget of +60 is not asserted to equal any one of those indices. Both are effectively +independent Gaussian-observation counts, not physical trials, fluorescence +samples, sessions, or animals. + +Optional `budget_factors` can generate separately labeled 0.5/1/2-type +sensitivity schedules, and a checksum-bound certified integer allocation can +replace an exactly matching optional maximin budget. Neither is part of the +primary frozen run. In particular, a certified-N45 diagnostic completed before +this primary freeze remains outside the primary artifact and was not used to +tune its thresholds. A count vector over independently instantiated grid cells +is still an allocation target, not an executable ordered history: no reset, +washout, carry-over, or history-realization protocol is provided here. + +## Train-only scoring and abstention + +Every replicate is randomly divided into training and held-out observations. +Each pure candidate is fitted with an intercept, slope, and training-residual +variance. The flexible adequacy model contains an intercept and all six +candidates. Its ridge penalty is selected only inside the training set by +three-fold cross-validation; its residual variance is also fitted on training +data. All comparisons are held-out profiled Gaussian log-score differences. + +Three inequalities are required for a pure call: + +1. the best pure candidate must beat the intercept-only model by more than the + upper null-calibration quantile. Because the statistic already maximizes over + all six pure candidates, this threshold is familywise for that candidate set; +2. the best pure candidate must beat the runner-up by more than a separately + null-calibrated ambiguity threshold; +3. the all-six flexible model may not beat the best pure model by more than the + worst-candidate pure-generator adequacy threshold. + +The third rule is one-sided. The flexible model nests the pure model, so the two +population scores tie under a correctly specified pure generator; requiring the +pure model to beat the flexible model would be invalid. The adequacy threshold is +the largest conformal upper threshold across the six matched pure generators, +which protects the worst calibrated pure candidate at the declared finite +calibration resolution. + +Threshold calibration, calibration audit, and final evaluation use three +disjoint deterministic seeds. The artifact reports the independent calibration +audit rather than treating threshold-training performance as validation. + +## Scenarios and interpretation + +Matched pure generators are evaluated with correct-call, wrong-call, abstention, +and raw closed-set winner rates. The null reports any non-null pure call as a +false call. All 15 unordered 50/50 candidate pairs are evaluated after equal +coefficients and unit-standard-deviation scaling on the full feasible grid. Any +pure call for a mixture—including a call naming one of its constituents—is +counted as false. Pointwise Wilson intervals accompany all reported rates; they +are not simultaneous confidence bounds over the many scenario cells. + +The mixture family is open-set relative to the six pure labels but remains inside +their linear span. One additional, deliberately narrow out-of-span probe takes +`tanh(standardized surprise)`, removes its full-grid OLS projection on an +intercept and all six standardized candidates, and scales the residual to unit +standard deviation. Orthogonality is checked numerically on the full grid; a pure +call is false. This is a diagnostic of one fixed saturation-shaped residual, not +coverage of a biologically defined misspecification class. + +The bounded artifact therefore does **not** certify robustness to arbitrary +out-of-span misspecification, other nonlinear/saturating combinations, serial +dependence, subject hierarchy, indicator dynamics, nuisance mismatch, or invalid +sequential histories. Matched-field simulation also cannot diagnose +misspecification of the candidate signals themselves. + +## Reproducible artifact + +From a clean checkout of the exact stress commit: + +```bash +bayesian-ach-design-stress \ + --repo-root . \ + --code-sha <40-character-stress-commit> \ + --output /absolute/path/design-stress-n60 \ + --fixed-budgets 60 \ + --locked-allocation /absolute/path/optimal_design_allocation_seed7.csv \ + --locked-allocation-sha256 \ + --locked-design-code-sha <40-character-design-commit> \ + --locked-allocation-seed 7 +``` + +The command refuses a dirty or mismatched checkout. It writes the configuration, +thresholds, independent calibration audit, pure/null/mixture and fixed nonlinear-probe evaluations, +allocations, a provenance manifest, and `SHA256SUMS.csv`. The manifest binds the +producer commit, canonical configuration digest, every payload file, and every +supplied certificate package. diff --git a/docs/design_stress_geometry_diagnostic.md b/docs/design_stress_geometry_diagnostic.md new file mode 100644 index 0000000..d84f3b9 --- /dev/null +++ b/docs/design_stress_geometry_diagnostic.md @@ -0,0 +1,21 @@ +# Post-freeze stress geometry diagnostic + +This explanatory diagnostic is downstream of the immutable locked-N60 stress package. It +does not alter its thresholds, seeds, allocations, simulations, or endpoints. + +`scripts/analyze_design_stress_geometry.py` stratifies the frozen result by allocation and +replays the 15 maximin mixture evaluations to count each decision gate independently. For +each 50/50 pair it also computes, on the locked maximin support, the weighted affine +residual against the best constituent and best pure candidate, the oracle two-component +residual, candidate correlation, covariance condition number, and the population profiled +Gaussian gap for the frozen 21-sample held-out size. + +`scripts/verify_design_stress_geometry.py` checks the immutable source-package digest, +producer/script provenance, every payload checksum, pair count, gate/false-call identities, +and the zero-residual two-component oracle. It then reruns all stratification, geometry, and +15 x 200 frozen evaluations and requires byte-identical JSON, CSV, and manifest outputs. + +The diagnostic explains whether poor mixture rejection reflects population aliasing on the +locked support or finite-sample decision behavior. It is not a new endpoint, a robustness +claim, a threshold-tuning analysis, or permission to select a replacement gate after seeing +the frozen result. diff --git a/pyproject.toml b/pyproject.toml index a0c6107..32b0827 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ plot = ["matplotlib>=3.8"] [project.scripts] bayesian-ach = "bayesian_ach.cli_ext:main" bayesian-ach-design = "bayesian_ach.design_cli:main" +bayesian-ach-design-stress = "bayesian_ach.design_stress_cli:main" bayesian-ach-replay = "bayesian_ach.replay_cli:main" [tool.setuptools.packages.find] diff --git a/results/design-open-set-stress-n60-geometry/SHA256SUMS.csv b/results/design-open-set-stress-n60-geometry/SHA256SUMS.csv new file mode 100644 index 0000000..7e7d4a1 --- /dev/null +++ b/results/design-open-set-stress-n60-geometry/SHA256SUMS.csv @@ -0,0 +1,4 @@ +file,bytes,sha256 +stress_diagnostic.json,20761,854b97eae7c873bdfc2de3b8145003028b570c65cca3774cedb32827f8fc4a2a +maximin_mixture_geometry.csv,6752,8d179c5c5ae2a1623c955e96d00bb86ea510be0a91d2326870de6ca367e144f7 +artifact_manifest.json,721,e248c92fae66931fa0fbf7a108d8bb2fe82572949c47f89e85365bed28286372 diff --git a/results/design-open-set-stress-n60-geometry/artifact_manifest.json b/results/design-open-set-stress-n60-geometry/artifact_manifest.json new file mode 100644 index 0000000..54fdd36 --- /dev/null +++ b/results/design-open-set-stress-n60-geometry/artifact_manifest.json @@ -0,0 +1,20 @@ +{ + "artifact": "post_freeze_stress_geometry_diagnostic", + "diagnostic_script_sha256": "31ff6397b5a4bd1dbdf0b83f578c0b24e8e0e36f34058758d92aa4094ec89bb7", + "files": [ + { + "bytes": 20761, + "path": "stress_diagnostic.json", + "sha256": "854b97eae7c873bdfc2de3b8145003028b570c65cca3774cedb32827f8fc4a2a" + }, + { + "bytes": 6752, + "path": "maximin_mixture_geometry.csv", + "sha256": "8d179c5c5ae2a1623c955e96d00bb86ea510be0a91d2326870de6ca367e144f7" + } + ], + "producer_commit": "88aa601dcf65a1cecf20d388ecdb2e304f6c4116", + "producer_git_dirty": false, + "schema_version": 1, + "source_artifact_sha256sums_sha256": "44a5188c43bda52e6fc9dc7007cf2de44a9671e9c5477ac88c3173c06cfdbd80" +} diff --git a/results/design-open-set-stress-n60-geometry/maximin_mixture_geometry.csv b/results/design-open-set-stress-n60-geometry/maximin_mixture_geometry.csv new file mode 100644 index 0000000..a6ab919 --- /dev/null +++ b/results/design-open-set-stress-n60-geometry/maximin_mixture_geometry.csv @@ -0,0 +1,16 @@ +first_candidate,second_candidate,false_pure_call_rate,wilson_upper,sequential_reason_counts,independent_gate_pass_counts,best_constituent,best_constituent_residual,best_any_pure,best_any_pure_residual,oracle_two_component_residual,best_pure_profiled_gap_n_test_21,oracle_two_component_profiled_gap_n_test_21,weighted_candidate_correlation,weighted_pair_covariance_condition_number +innovation_l2,surprise,0.745,0.8003950482796865,"{""flexible_model_better"":3,""null_not_rejected"":3,""pure_ambiguity"":45,""pure_call"":149}","{""all_three_pass"":149,""flexible_adequacy_pass"":197,""pure_over_null_pass"":197,""winner_over_runner_pass"":153}",surprise,0.06476225526808417,surprise,0.06476225526808417,1.7834111285025804e-31,0.6588911673833898,1.8725816849277094e-30,0.8467944254803421,12.054612191382652 +innovation_l2,gain,0.555,0.6221939899704665,"{""flexible_model_better"":31,""null_not_rejected"":6,""pure_ambiguity"":52,""pure_call"":111}","{""all_three_pass"":111,""flexible_adequacy_pass"":151,""pure_over_null_pass"":194,""winner_over_runner_pass"":143}",gain,0.44165477400362824,update_l2,0.23773950726644638,7.233792871118458e-32,2.239510748574044,7.595482514674381e-31,-0.02131984334093419,1.7613618463469893 +innovation_l2,update_l2,0.62,0.6844099415132637,"{""flexible_model_better"":25,""null_not_rejected"":1,""pure_ambiguity"":50,""pure_call"":124}","{""all_three_pass"":124,""flexible_adequacy_pass"":160,""pure_over_null_pass"":199,""winner_over_runner_pass"":149}",update_l2,0.2576905807957704,update_l2,0.2576905807957704,5.256248004221645e-31,2.4074102515385984,5.519060404432727e-30,0.4213845233433928,2.823121945397039 +innovation_l2,information_gain,0.58,0.6462640083838327,"{""flexible_model_better"":16,""null_not_rejected"":3,""pure_ambiguity"":65,""pure_call"":116}","{""all_three_pass"":116,""flexible_adequacy_pass"":170,""pure_over_null_pass"":197,""winner_over_runner_pass"":133}",information_gain,0.2286673248345743,information_gain,0.2286673248345743,2.511402930060989e-32,2.162266115687133,2.636973076564038e-31,0.4586862807960559,2.8800745735556 +innovation_l2,change_probability,0.615,0.6796669027307678,"{""flexible_model_better"":26,""null_not_rejected"":3,""pure_ambiguity"":48,""pure_call"":123}","{""all_three_pass"":123,""flexible_adequacy_pass"":157,""pure_over_null_pass"":197,""winner_over_runner_pass"":150}",change_probability,0.2715733105685233,change_probability,0.2715733105685233,3.425330603757512e-31,2.522677090253475,3.596597133945387e-30,0.3569282136837277,2.4570981315526876 +surprise,gain,0.485,0.5538915080725428,"{""flexible_model_better"":22,""null_not_rejected"":2,""pure_ambiguity"":79,""pure_call"":97}","{""all_three_pass"":97,""flexible_adequacy_pass"":151,""pure_over_null_pass"":198,""winner_over_runner_pass"":120}",gain,0.44477271898790127,update_l2,0.3334070092601476,4.191979116953257e-31,3.02124194263797,4.40157807280092e-30,-0.043754286693241884,1.755875929808455 +surprise,update_l2,0.575,0.6414638745648614,"{""flexible_model_better"":13,""null_not_rejected"":1,""pure_ambiguity"":71,""pure_call"":115}","{""all_three_pass"":115,""flexible_adequacy_pass"":174,""pure_over_null_pass"":199,""winner_over_runner_pass"":128}",update_l2,0.28827138069547065,update_l2,0.28827138069547065,3.0044507132440885e-32,2.6596637002520334,3.1546732489062928e-31,0.32590264303136707,2.3238272821220116 +surprise,information_gain,0.72,0.7776311022605097,"{""flexible_model_better"":11,""null_not_rejected"":2,""pure_ambiguity"":43,""pure_call"":144}","{""all_three_pass"":144,""flexible_adequacy_pass"":181,""pure_over_null_pass"":198,""winner_over_runner_pass"":157}",information_gain,0.1751311847924566,information_gain,0.1751311847924566,2.2232935278006252e-31,1.6944877739577782,2.3344582041906565e-30,0.6044310151366707,4.259069867164642 +surprise,change_probability,0.67,0.7314257412939605,"{""flexible_model_better"":39,""null_not_rejected"":5,""pure_ambiguity"":22,""pure_call"":134}","{""all_three_pass"":134,""flexible_adequacy_pass"":158,""pure_over_null_pass"":195,""winner_over_runner_pass"":174}",change_probability,0.24059459272699615,change_probability,0.24059459272699615,1.4359733665351229e-31,2.263703136999608,1.507772034861879e-30,0.4498740075388434,2.9744407398573025 +gain,update_l2,0.8,0.8495479907390189,"{""flexible_model_better"":5,""null_not_rejected"":1,""pure_ambiguity"":34,""pure_call"":160}","{""all_three_pass"":160,""flexible_adequacy_pass"":192,""pure_over_null_pass"":199,""winner_over_runner_pass"":165}",gain,0.1515439180125079,gain,0.1515439180125079,7.5579912038097065e-31,1.4815875834877898,7.935890764000191e-30,0.791530832607952,8.604108451760718 +gain,information_gain,0.61,0.6749165686253038,"{""flexible_model_better"":20,""pure_ambiguity"":58,""pure_call"":122}","{""all_three_pass"":122,""flexible_adequacy_pass"":166,""pure_over_null_pass"":200,""winner_over_runner_pass"":142}",gain,0.3094561867702409,update_l2,0.2454971557070175,2.429239636520434e-32,2.3051151066504176,2.5507016183464558e-31,0.4917349601612237,3.0003291825845855 +gain,change_probability,0.27,0.33543435198668425,"{""flexible_model_better"":75,""null_not_rejected"":5,""pure_ambiguity"":66,""pure_call"":54}","{""all_three_pass"":54,""flexible_adequacy_pass"":82,""pure_over_null_pass"":195,""winner_over_runner_pass"":129}",gain,0.7194446631198409,gain,0.7194446631198409,2.8000132496473827e-31,5.691014368329873,2.940013912129752e-30,-0.0444760730140199,1.1234288396957601 +update_l2,information_gain,0.745,0.8003950482796865,"{""flexible_model_better"":5,""null_not_rejected"":3,""pure_ambiguity"":43,""pure_call"":149}","{""all_three_pass"":149,""flexible_adequacy_pass"":191,""pure_over_null_pass"":197,""winner_over_runner_pass"":154}",update_l2,0.13734921104922593,update_l2,0.13734921104922593,7.415215471879735e-31,1.3513531640796759,7.785976245473722e-30,0.7713790932333591,7.802793892925121 +update_l2,change_probability,0.325,0.39268014883970037,"{""flexible_model_better"":68,""null_not_rejected"":2,""pure_ambiguity"":65,""pure_call"":65}","{""all_three_pass"":65,""flexible_adequacy_pass"":100,""pure_over_null_pass"":198,""winner_over_runner_pass"":134}",update_l2,0.6095014120398539,update_l2,0.6095014120398539,6.510927165325115e-32,4.997206715257962,6.836473523591371e-31,0.09440005532043154,1.2090358929226095 +information_gain,change_probability,0.535,0.6028143584299599,"{""flexible_model_better"":53,""null_not_rejected"":5,""pure_ambiguity"":35,""pure_call"":107}","{""all_three_pass"":107,""flexible_adequacy_pass"":131,""pure_over_null_pass"":195,""winner_over_runner_pass"":161}",change_probability,0.4585249642762335,change_probability,0.4585249642762335,8.693262582975106e-31,3.962969079130324,9.127925712123861e-30,0.22817185530184655,1.6223418188593566 diff --git a/results/design-open-set-stress-n60-geometry/stress_diagnostic.json b/results/design-open-set-stress-n60-geometry/stress_diagnostic.json new file mode 100644 index 0000000..8c936bd --- /dev/null +++ b/results/design-open-set-stress-n60-geometry/stress_diagnostic.json @@ -0,0 +1,560 @@ +{ + "calibration_audit_replicates": 100, + "calibration_audit_seed": 130363, + "calibration_replicates": 100, + "diagnostic_producer_clean_worktree": true, + "diagnostic_producer_commit": "88aa601dcf65a1cecf20d388ecdb2e304f6c4116", + "diagnostic_script_sha256": "31ff6397b5a4bd1dbdf0b83f578c0b24e8e0e36f34058758d92aa4094ec89bb7", + "evaluation_replicates": 200, + "evaluation_seed": 155921, + "kind": "read_only_post_freeze_stress_diagnostic", + "maximin_mixture_geometry": [ + { + "best_any_pure": "surprise", + "best_any_pure_residual": 0.06476225526808417, + "best_constituent": "surprise", + "best_constituent_residual": 0.06476225526808417, + "best_pure_profiled_gap_n_test_21": 0.6588911673833898, + "false_pure_call_rate": 0.745, + "first_candidate": "innovation_l2", + "independent_gate_pass_counts": { + "all_three_pass": 149, + "flexible_adequacy_pass": 197, + "pure_over_null_pass": 197, + "winner_over_runner_pass": 153 + }, + "oracle_two_component_profiled_gap_n_test_21": 1.8725816849277094e-30, + "oracle_two_component_residual": 1.7834111285025804e-31, + "second_candidate": "surprise", + "sequential_reason_counts": { + "flexible_model_better": 3, + "null_not_rejected": 3, + "pure_ambiguity": 45, + "pure_call": 149 + }, + "weighted_candidate_correlation": 0.8467944254803421, + "weighted_pair_covariance_condition_number": 12.054612191382652, + "wilson_upper": 0.8003950482796865 + }, + { + "best_any_pure": "update_l2", + "best_any_pure_residual": 0.23773950726644638, + "best_constituent": "gain", + "best_constituent_residual": 0.44165477400362824, + "best_pure_profiled_gap_n_test_21": 2.239510748574044, + "false_pure_call_rate": 0.555, + "first_candidate": "innovation_l2", + "independent_gate_pass_counts": { + "all_three_pass": 111, + "flexible_adequacy_pass": 151, + "pure_over_null_pass": 194, + "winner_over_runner_pass": 143 + }, + "oracle_two_component_profiled_gap_n_test_21": 7.595482514674381e-31, + "oracle_two_component_residual": 7.233792871118458e-32, + "second_candidate": "gain", + "sequential_reason_counts": { + "flexible_model_better": 31, + "null_not_rejected": 6, + "pure_ambiguity": 52, + "pure_call": 111 + }, + "weighted_candidate_correlation": -0.02131984334093419, + "weighted_pair_covariance_condition_number": 1.7613618463469893, + "wilson_upper": 0.6221939899704665 + }, + { + "best_any_pure": "update_l2", + "best_any_pure_residual": 0.2576905807957704, + "best_constituent": "update_l2", + "best_constituent_residual": 0.2576905807957704, + "best_pure_profiled_gap_n_test_21": 2.4074102515385984, + "false_pure_call_rate": 0.62, + "first_candidate": "innovation_l2", + "independent_gate_pass_counts": { + "all_three_pass": 124, + "flexible_adequacy_pass": 160, + "pure_over_null_pass": 199, + "winner_over_runner_pass": 149 + }, + "oracle_two_component_profiled_gap_n_test_21": 5.519060404432727e-30, + "oracle_two_component_residual": 5.256248004221645e-31, + "second_candidate": "update_l2", + "sequential_reason_counts": { + "flexible_model_better": 25, + "null_not_rejected": 1, + "pure_ambiguity": 50, + "pure_call": 124 + }, + "weighted_candidate_correlation": 0.4213845233433928, + "weighted_pair_covariance_condition_number": 2.823121945397039, + "wilson_upper": 0.6844099415132637 + }, + { + "best_any_pure": "information_gain", + "best_any_pure_residual": 0.2286673248345743, + "best_constituent": "information_gain", + "best_constituent_residual": 0.2286673248345743, + "best_pure_profiled_gap_n_test_21": 2.162266115687133, + "false_pure_call_rate": 0.58, + "first_candidate": "innovation_l2", + "independent_gate_pass_counts": { + "all_three_pass": 116, + "flexible_adequacy_pass": 170, + "pure_over_null_pass": 197, + "winner_over_runner_pass": 133 + }, + "oracle_two_component_profiled_gap_n_test_21": 2.636973076564038e-31, + "oracle_two_component_residual": 2.511402930060989e-32, + "second_candidate": "information_gain", + "sequential_reason_counts": { + "flexible_model_better": 16, + "null_not_rejected": 3, + "pure_ambiguity": 65, + "pure_call": 116 + }, + "weighted_candidate_correlation": 0.4586862807960559, + "weighted_pair_covariance_condition_number": 2.8800745735556, + "wilson_upper": 0.6462640083838327 + }, + { + "best_any_pure": "change_probability", + "best_any_pure_residual": 0.2715733105685233, + "best_constituent": "change_probability", + "best_constituent_residual": 0.2715733105685233, + "best_pure_profiled_gap_n_test_21": 2.522677090253475, + "false_pure_call_rate": 0.615, + "first_candidate": "innovation_l2", + "independent_gate_pass_counts": { + "all_three_pass": 123, + "flexible_adequacy_pass": 157, + "pure_over_null_pass": 197, + "winner_over_runner_pass": 150 + }, + "oracle_two_component_profiled_gap_n_test_21": 3.596597133945387e-30, + "oracle_two_component_residual": 3.425330603757512e-31, + "second_candidate": "change_probability", + "sequential_reason_counts": { + "flexible_model_better": 26, + "null_not_rejected": 3, + "pure_ambiguity": 48, + "pure_call": 123 + }, + "weighted_candidate_correlation": 0.3569282136837277, + "weighted_pair_covariance_condition_number": 2.4570981315526876, + "wilson_upper": 0.6796669027307678 + }, + { + "best_any_pure": "update_l2", + "best_any_pure_residual": 0.3334070092601476, + "best_constituent": "gain", + "best_constituent_residual": 0.44477271898790127, + "best_pure_profiled_gap_n_test_21": 3.02124194263797, + "false_pure_call_rate": 0.485, + "first_candidate": "surprise", + "independent_gate_pass_counts": { + "all_three_pass": 97, + "flexible_adequacy_pass": 151, + "pure_over_null_pass": 198, + "winner_over_runner_pass": 120 + }, + "oracle_two_component_profiled_gap_n_test_21": 4.40157807280092e-30, + "oracle_two_component_residual": 4.191979116953257e-31, + "second_candidate": "gain", + "sequential_reason_counts": { + "flexible_model_better": 22, + "null_not_rejected": 2, + "pure_ambiguity": 79, + "pure_call": 97 + }, + "weighted_candidate_correlation": -0.043754286693241884, + "weighted_pair_covariance_condition_number": 1.755875929808455, + "wilson_upper": 0.5538915080725428 + }, + { + "best_any_pure": "update_l2", + "best_any_pure_residual": 0.28827138069547065, + "best_constituent": "update_l2", + "best_constituent_residual": 0.28827138069547065, + "best_pure_profiled_gap_n_test_21": 2.6596637002520334, + "false_pure_call_rate": 0.575, + "first_candidate": "surprise", + "independent_gate_pass_counts": { + "all_three_pass": 115, + "flexible_adequacy_pass": 174, + "pure_over_null_pass": 199, + "winner_over_runner_pass": 128 + }, + "oracle_two_component_profiled_gap_n_test_21": 3.1546732489062928e-31, + "oracle_two_component_residual": 3.0044507132440885e-32, + "second_candidate": "update_l2", + "sequential_reason_counts": { + "flexible_model_better": 13, + "null_not_rejected": 1, + "pure_ambiguity": 71, + "pure_call": 115 + }, + "weighted_candidate_correlation": 0.32590264303136707, + "weighted_pair_covariance_condition_number": 2.3238272821220116, + "wilson_upper": 0.6414638745648614 + }, + { + "best_any_pure": "information_gain", + "best_any_pure_residual": 0.1751311847924566, + "best_constituent": "information_gain", + "best_constituent_residual": 0.1751311847924566, + "best_pure_profiled_gap_n_test_21": 1.6944877739577782, + "false_pure_call_rate": 0.72, + "first_candidate": "surprise", + "independent_gate_pass_counts": { + "all_three_pass": 144, + "flexible_adequacy_pass": 181, + "pure_over_null_pass": 198, + "winner_over_runner_pass": 157 + }, + "oracle_two_component_profiled_gap_n_test_21": 2.3344582041906565e-30, + "oracle_two_component_residual": 2.2232935278006252e-31, + "second_candidate": "information_gain", + "sequential_reason_counts": { + "flexible_model_better": 11, + "null_not_rejected": 2, + "pure_ambiguity": 43, + "pure_call": 144 + }, + "weighted_candidate_correlation": 0.6044310151366707, + "weighted_pair_covariance_condition_number": 4.259069867164642, + "wilson_upper": 0.7776311022605097 + }, + { + "best_any_pure": "change_probability", + "best_any_pure_residual": 0.24059459272699615, + "best_constituent": "change_probability", + "best_constituent_residual": 0.24059459272699615, + "best_pure_profiled_gap_n_test_21": 2.263703136999608, + "false_pure_call_rate": 0.67, + "first_candidate": "surprise", + "independent_gate_pass_counts": { + "all_three_pass": 134, + "flexible_adequacy_pass": 158, + "pure_over_null_pass": 195, + "winner_over_runner_pass": 174 + }, + "oracle_two_component_profiled_gap_n_test_21": 1.507772034861879e-30, + "oracle_two_component_residual": 1.4359733665351229e-31, + "second_candidate": "change_probability", + "sequential_reason_counts": { + "flexible_model_better": 39, + "null_not_rejected": 5, + "pure_ambiguity": 22, + "pure_call": 134 + }, + "weighted_candidate_correlation": 0.4498740075388434, + "weighted_pair_covariance_condition_number": 2.9744407398573025, + "wilson_upper": 0.7314257412939605 + }, + { + "best_any_pure": "gain", + "best_any_pure_residual": 0.1515439180125079, + "best_constituent": "gain", + "best_constituent_residual": 0.1515439180125079, + "best_pure_profiled_gap_n_test_21": 1.4815875834877898, + "false_pure_call_rate": 0.8, + "first_candidate": "gain", + "independent_gate_pass_counts": { + "all_three_pass": 160, + "flexible_adequacy_pass": 192, + "pure_over_null_pass": 199, + "winner_over_runner_pass": 165 + }, + "oracle_two_component_profiled_gap_n_test_21": 7.935890764000191e-30, + "oracle_two_component_residual": 7.5579912038097065e-31, + "second_candidate": "update_l2", + "sequential_reason_counts": { + "flexible_model_better": 5, + "null_not_rejected": 1, + "pure_ambiguity": 34, + "pure_call": 160 + }, + "weighted_candidate_correlation": 0.791530832607952, + "weighted_pair_covariance_condition_number": 8.604108451760718, + "wilson_upper": 0.8495479907390189 + }, + { + "best_any_pure": "update_l2", + "best_any_pure_residual": 0.2454971557070175, + "best_constituent": "gain", + "best_constituent_residual": 0.3094561867702409, + "best_pure_profiled_gap_n_test_21": 2.3051151066504176, + "false_pure_call_rate": 0.61, + "first_candidate": "gain", + "independent_gate_pass_counts": { + "all_three_pass": 122, + "flexible_adequacy_pass": 166, + "pure_over_null_pass": 200, + "winner_over_runner_pass": 142 + }, + "oracle_two_component_profiled_gap_n_test_21": 2.5507016183464558e-31, + "oracle_two_component_residual": 2.429239636520434e-32, + "second_candidate": "information_gain", + "sequential_reason_counts": { + "flexible_model_better": 20, + "pure_ambiguity": 58, + "pure_call": 122 + }, + "weighted_candidate_correlation": 0.4917349601612237, + "weighted_pair_covariance_condition_number": 3.0003291825845855, + "wilson_upper": 0.6749165686253038 + }, + { + "best_any_pure": "gain", + "best_any_pure_residual": 0.7194446631198409, + "best_constituent": "gain", + "best_constituent_residual": 0.7194446631198409, + "best_pure_profiled_gap_n_test_21": 5.691014368329873, + "false_pure_call_rate": 0.27, + "first_candidate": "gain", + "independent_gate_pass_counts": { + "all_three_pass": 54, + "flexible_adequacy_pass": 82, + "pure_over_null_pass": 195, + "winner_over_runner_pass": 129 + }, + "oracle_two_component_profiled_gap_n_test_21": 2.940013912129752e-30, + "oracle_two_component_residual": 2.8000132496473827e-31, + "second_candidate": "change_probability", + "sequential_reason_counts": { + "flexible_model_better": 75, + "null_not_rejected": 5, + "pure_ambiguity": 66, + "pure_call": 54 + }, + "weighted_candidate_correlation": -0.0444760730140199, + "weighted_pair_covariance_condition_number": 1.1234288396957601, + "wilson_upper": 0.33543435198668425 + }, + { + "best_any_pure": "update_l2", + "best_any_pure_residual": 0.13734921104922593, + "best_constituent": "update_l2", + "best_constituent_residual": 0.13734921104922593, + "best_pure_profiled_gap_n_test_21": 1.3513531640796759, + "false_pure_call_rate": 0.745, + "first_candidate": "update_l2", + "independent_gate_pass_counts": { + "all_three_pass": 149, + "flexible_adequacy_pass": 191, + "pure_over_null_pass": 197, + "winner_over_runner_pass": 154 + }, + "oracle_two_component_profiled_gap_n_test_21": 7.785976245473722e-30, + "oracle_two_component_residual": 7.415215471879735e-31, + "second_candidate": "information_gain", + "sequential_reason_counts": { + "flexible_model_better": 5, + "null_not_rejected": 3, + "pure_ambiguity": 43, + "pure_call": 149 + }, + "weighted_candidate_correlation": 0.7713790932333591, + "weighted_pair_covariance_condition_number": 7.802793892925121, + "wilson_upper": 0.8003950482796865 + }, + { + "best_any_pure": "update_l2", + "best_any_pure_residual": 0.6095014120398539, + "best_constituent": "update_l2", + "best_constituent_residual": 0.6095014120398539, + "best_pure_profiled_gap_n_test_21": 4.997206715257962, + "false_pure_call_rate": 0.325, + "first_candidate": "update_l2", + "independent_gate_pass_counts": { + "all_three_pass": 65, + "flexible_adequacy_pass": 100, + "pure_over_null_pass": 198, + "winner_over_runner_pass": 134 + }, + "oracle_two_component_profiled_gap_n_test_21": 6.836473523591371e-31, + "oracle_two_component_residual": 6.510927165325115e-32, + "second_candidate": "change_probability", + "sequential_reason_counts": { + "flexible_model_better": 68, + "null_not_rejected": 2, + "pure_ambiguity": 65, + "pure_call": 65 + }, + "weighted_candidate_correlation": 0.09440005532043154, + "weighted_pair_covariance_condition_number": 1.2090358929226095, + "wilson_upper": 0.39268014883970037 + }, + { + "best_any_pure": "change_probability", + "best_any_pure_residual": 0.4585249642762335, + "best_constituent": "change_probability", + "best_constituent_residual": 0.4585249642762335, + "best_pure_profiled_gap_n_test_21": 3.962969079130324, + "false_pure_call_rate": 0.535, + "first_candidate": "information_gain", + "independent_gate_pass_counts": { + "all_three_pass": 107, + "flexible_adequacy_pass": 131, + "pure_over_null_pass": 195, + "winner_over_runner_pass": 161 + }, + "oracle_two_component_profiled_gap_n_test_21": 9.127925712123861e-30, + "oracle_two_component_residual": 8.693262582975106e-31, + "second_candidate": "change_probability", + "sequential_reason_counts": { + "flexible_model_better": 53, + "null_not_rejected": 5, + "pure_ambiguity": 35, + "pure_call": 107 + }, + "weighted_candidate_correlation": 0.22817185530184655, + "weighted_pair_covariance_condition_number": 1.6223418188593566, + "wilson_upper": 0.6028143584299599 + } + ], + "n_test": 21, + "no_threshold_or_evaluation_changes": true, + "schema_version": 1, + "source_artifact_sha256sums_sha256": "44a5188c43bda52e6fc9dc7007cf2de44a9671e9c5477ac88c3173c06cfdbd80", + "source_producer_commit": "c71695fda83ae93407599a909097962ee3fa9e0e", + "stratified": [ + { + "design": "coupled_novelty", + "mixture_reason_counts": { + "flexible_model_better": 37, + "null_not_rejected": 34, + "pure_ambiguity": 1891, + "pure_call": 1038 + }, + "null_false_pure_rate": 0.01, + "null_reason_counts": { + "null_not_rejected": 184, + "pure_ambiguity": 14, + "pure_call": 2 + }, + "null_wilson_upper": 0.035721761716176796, + "out_of_span_false_pure_rate": 0.035, + "out_of_span_reason_counts": { + "flexible_model_better": 16, + "null_not_rejected": 112, + "pure_ambiguity": 65, + "pure_call": 7 + }, + "out_of_span_wilson_upper": 0.07047061152229071, + "pure_reason_counts": { + "flexible_model_better": 16, + "null_not_rejected": 45, + "pure_ambiguity": 718, + "pure_call": 421 + }, + "thresholds": { + "flexible_over_pure": 0.9358247112625193, + "pure_over_null": 0.9908053929429492, + "winner_over_runner": 0.8374072007101319 + }, + "weakest_pure_correct_rate": 0.03, + "weakest_pure_generator": "gain", + "weakest_pure_raw_closed_set_rate": 0.425, + "weakest_pure_wilson_lower": 0.013820314340111346, + "worst_mixture_false_pure_rate": 0.46, + "worst_mixture_pair": [ + "innovation_l2", + "change_probability" + ], + "worst_mixture_wilson_upper": 0.5291771081104667 + }, + { + "design": "uniform_factorial", + "mixture_reason_counts": { + "flexible_model_better": 261, + "null_not_rejected": 41, + "pure_ambiguity": 1490, + "pure_call": 1208 + }, + "null_false_pure_rate": 0.015, + "null_reason_counts": { + "null_not_rejected": 195, + "pure_ambiguity": 2, + "pure_call": 3 + }, + "null_wilson_upper": 0.043165728792690254, + "out_of_span_false_pure_rate": 0.005, + "out_of_span_reason_counts": { + "null_not_rejected": 194, + "pure_ambiguity": 5, + "pure_call": 1 + }, + "out_of_span_wilson_upper": 0.027773704397892923, + "pure_reason_counts": { + "flexible_model_better": 25, + "null_not_rejected": 47, + "pure_ambiguity": 319, + "pure_call": 809 + }, + "thresholds": { + "flexible_over_pure": 1.335738097917563, + "pure_over_null": 1.4841715232562613, + "winner_over_runner": 1.043715011057646 + }, + "weakest_pure_correct_rate": 0.425, + "weakest_pure_generator": "innovation_l2", + "weakest_pure_raw_closed_set_rate": 0.72, + "weakest_pure_wilson_lower": 0.3585361254351385, + "worst_mixture_false_pure_rate": 0.545, + "worst_mixture_pair": [ + "update_l2", + "information_gain" + ], + "worst_mixture_wilson_upper": 0.612517806901165 + }, + { + "design": "maximin_optimized", + "mixture_reason_counts": { + "flexible_model_better": 412, + "null_not_rejected": 42, + "pure_ambiguity": 776, + "pure_call": 1770 + }, + "null_false_pure_rate": 0.02, + "null_reason_counts": { + "null_not_rejected": 195, + "pure_ambiguity": 1, + "pure_call": 4 + }, + "null_wilson_upper": 0.05028708690582643, + "out_of_span_false_pure_rate": 0.515, + "out_of_span_reason_counts": { + "flexible_model_better": 30, + "null_not_rejected": 52, + "pure_ambiguity": 15, + "pure_call": 103 + }, + "out_of_span_wilson_upper": 0.5833261482812249, + "pure_reason_counts": { + "flexible_model_better": 14, + "null_not_rejected": 26, + "pure_ambiguity": 86, + "pure_call": 1074 + }, + "thresholds": { + "flexible_over_pure": 2.074310487667276, + "pure_over_null": 1.4684378147083308, + "winner_over_runner": 0.7589681023920747 + }, + "weakest_pure_correct_rate": 0.75, + "weakest_pure_generator": "surprise", + "weakest_pure_raw_closed_set_rate": 0.82, + "weakest_pure_wilson_lower": 0.6856590168795417, + "worst_mixture_false_pure_rate": 0.8, + "worst_mixture_pair": [ + "gain", + "update_l2" + ], + "worst_mixture_wilson_upper": 0.8495479907390189 + } + ], + "test_fraction": 0.35, + "threshold_seed": 104729 +} diff --git a/results/design-open-set-stress-n60/SHA256SUMS.csv b/results/design-open-set-stress-n60/SHA256SUMS.csv new file mode 100644 index 0000000..5dbe41f --- /dev/null +++ b/results/design-open-set-stress-n60/SHA256SUMS.csv @@ -0,0 +1,10 @@ +path,bytes,sha256 +summary.json,3705,c4a6dd3271506da87667ac91566e317c3b16273d6d16b5d8f24a9839288990e7 +thresholds.csv,652,e606262f2b42e1671a68b8ec4dc4b87cb38ba8f56fd0a5d9704d4652c025d513 +calibration_audit.csv,2521,148002a5d321a3a98855e690649f9ecf1384dda2d754a24021711adfc7172ff4 +pure_recovery.csv,3839,4ccbd057d609c75b39a428d2708b1aa4afba95cab9990bb9c8f772388b66c17f +null_evaluation.csv,631,25a06c37aea06319ca7bf5abfd93f5b23bfc25c82ac932e23f4d224ae2abd5a1 +mixture_evaluation.csv,12148,393b44e646417d0742e2f15a7b8aeac046cc39ff24e35d9f4b15194929e7b1db +out_of_span_evaluation.csv,1793,24054f50878222ad14478b02e8e59aec85b3d9a93d4ede447f9581531982857a +allocations.csv,5008,86e1bc633a8fe9917a8b50ee6563a38706c6fd4b6124e69a5b69257c9c2af5e4 +artifact_manifest.json,2627,d840a2ec34f5a386109c7f985033b53144fdcc1b1a0e0592b3274c0e38902b64 diff --git a/results/design-open-set-stress-n60/allocations.csv b/results/design-open-set-stress-n60/allocations.csv new file mode 100644 index 0000000..8a7333b --- /dev/null +++ b/results/design-open-set-stress-n60/allocations.csv @@ -0,0 +1,87 @@ +design,budget_schedule,budget_factor,budget,allocation_source,point_id,count +coupled_novelty,fixed_budget,,60,frozen_override,29,4 +coupled_novelty,fixed_budget,,60,frozen_override,77,5 +coupled_novelty,fixed_budget,,60,frozen_override,83,5 +coupled_novelty,fixed_budget,,60,frozen_override,129,12 +coupled_novelty,fixed_budget,,60,frozen_override,131,1 +coupled_novelty,fixed_budget,,60,frozen_override,134,3 +coupled_novelty,fixed_budget,,60,frozen_override,135,2 +coupled_novelty,fixed_budget,,60,frozen_override,158,12 +coupled_novelty,fixed_budget,,60,frozen_override,162,6 +coupled_novelty,fixed_budget,,60,frozen_override,182,1 +coupled_novelty,fixed_budget,,60,frozen_override,210,9 +uniform_factorial,fixed_budget,,60,frozen_override,0,1 +uniform_factorial,fixed_budget,,60,frozen_override,2,1 +uniform_factorial,fixed_budget,,60,frozen_override,4,1 +uniform_factorial,fixed_budget,,60,frozen_override,5,1 +uniform_factorial,fixed_budget,,60,frozen_override,7,1 +uniform_factorial,fixed_budget,,60,frozen_override,10,1 +uniform_factorial,fixed_budget,,60,frozen_override,12,1 +uniform_factorial,fixed_budget,,60,frozen_override,14,1 +uniform_factorial,fixed_budget,,60,frozen_override,21,1 +uniform_factorial,fixed_budget,,60,frozen_override,28,1 +uniform_factorial,fixed_budget,,60,frozen_override,35,1 +uniform_factorial,fixed_budget,,60,frozen_override,41,1 +uniform_factorial,fixed_budget,,60,frozen_override,46,1 +uniform_factorial,fixed_budget,,60,frozen_override,49,1 +uniform_factorial,fixed_budget,,60,frozen_override,54,1 +uniform_factorial,fixed_budget,,60,frozen_override,55,1 +uniform_factorial,fixed_budget,,60,frozen_override,57,1 +uniform_factorial,fixed_budget,,60,frozen_override,58,1 +uniform_factorial,fixed_budget,,60,frozen_override,62,1 +uniform_factorial,fixed_budget,,60,frozen_override,70,1 +uniform_factorial,fixed_budget,,60,frozen_override,71,1 +uniform_factorial,fixed_budget,,60,frozen_override,74,1 +uniform_factorial,fixed_budget,,60,frozen_override,90,1 +uniform_factorial,fixed_budget,,60,frozen_override,103,1 +uniform_factorial,fixed_budget,,60,frozen_override,108,1 +uniform_factorial,fixed_budget,,60,frozen_override,111,1 +uniform_factorial,fixed_budget,,60,frozen_override,115,1 +uniform_factorial,fixed_budget,,60,frozen_override,117,1 +uniform_factorial,fixed_budget,,60,frozen_override,132,1 +uniform_factorial,fixed_budget,,60,frozen_override,140,1 +uniform_factorial,fixed_budget,,60,frozen_override,145,1 +uniform_factorial,fixed_budget,,60,frozen_override,157,1 +uniform_factorial,fixed_budget,,60,frozen_override,160,1 +uniform_factorial,fixed_budget,,60,frozen_override,163,1 +uniform_factorial,fixed_budget,,60,frozen_override,164,1 +uniform_factorial,fixed_budget,,60,frozen_override,170,1 +uniform_factorial,fixed_budget,,60,frozen_override,175,1 +uniform_factorial,fixed_budget,,60,frozen_override,176,1 +uniform_factorial,fixed_budget,,60,frozen_override,177,1 +uniform_factorial,fixed_budget,,60,frozen_override,178,1 +uniform_factorial,fixed_budget,,60,frozen_override,179,1 +uniform_factorial,fixed_budget,,60,frozen_override,180,1 +uniform_factorial,fixed_budget,,60,frozen_override,181,1 +uniform_factorial,fixed_budget,,60,frozen_override,186,1 +uniform_factorial,fixed_budget,,60,frozen_override,195,1 +uniform_factorial,fixed_budget,,60,frozen_override,197,1 +uniform_factorial,fixed_budget,,60,frozen_override,200,1 +uniform_factorial,fixed_budget,,60,frozen_override,202,1 +uniform_factorial,fixed_budget,,60,frozen_override,204,1 +uniform_factorial,fixed_budget,,60,frozen_override,205,1 +uniform_factorial,fixed_budget,,60,frozen_override,209,1 +uniform_factorial,fixed_budget,,60,frozen_override,211,1 +uniform_factorial,fixed_budget,,60,frozen_override,217,1 +uniform_factorial,fixed_budget,,60,frozen_override,219,1 +uniform_factorial,fixed_budget,,60,frozen_override,227,1 +uniform_factorial,fixed_budget,,60,frozen_override,230,1 +uniform_factorial,fixed_budget,,60,frozen_override,231,1 +uniform_factorial,fixed_budget,,60,frozen_override,233,1 +uniform_factorial,fixed_budget,,60,frozen_override,234,1 +uniform_factorial,fixed_budget,,60,frozen_override,237,1 +maximin_optimized,fixed_budget,,60,frozen_override,0,3 +maximin_optimized,fixed_budget,,60,frozen_override,5,2 +maximin_optimized,fixed_budget,,60,frozen_override,11,1 +maximin_optimized,fixed_budget,,60,frozen_override,18,3 +maximin_optimized,fixed_budget,,60,frozen_override,23,3 +maximin_optimized,fixed_budget,,60,frozen_override,120,8 +maximin_optimized,fixed_budget,,60,frozen_override,125,8 +maximin_optimized,fixed_budget,,60,frozen_override,131,1 +maximin_optimized,fixed_budget,,60,frozen_override,138,8 +maximin_optimized,fixed_budget,,60,frozen_override,143,7 +maximin_optimized,fixed_budget,,60,frozen_override,192,3 +maximin_optimized,fixed_budget,,60,frozen_override,197,4 +maximin_optimized,fixed_budget,,60,frozen_override,210,4 +maximin_optimized,fixed_budget,,60,frozen_override,215,4 +maximin_optimized,fixed_budget,,60,frozen_override,219,1 diff --git a/results/design-open-set-stress-n60/artifact_manifest.json b/results/design-open-set-stress-n60/artifact_manifest.json new file mode 100644 index 0000000..d59560d --- /dev/null +++ b/results/design-open-set-stress-n60/artifact_manifest.json @@ -0,0 +1,77 @@ +{ + "artifact": "post_freeze_design_abstention_sensitivity", + "configuration_sha256": "a5ecb194d9ef93f336e4acd1d55fdde3b5570aa5d60d3c0dbf4eef0209590844", + "files": [ + { + "bytes": 3705, + "path": "summary.json", + "sha256": "c4a6dd3271506da87667ac91566e317c3b16273d6d16b5d8f24a9839288990e7" + }, + { + "bytes": 652, + "path": "thresholds.csv", + "sha256": "e606262f2b42e1671a68b8ec4dc4b87cb38ba8f56fd0a5d9704d4652c025d513" + }, + { + "bytes": 2521, + "path": "calibration_audit.csv", + "sha256": "148002a5d321a3a98855e690649f9ecf1384dda2d754a24021711adfc7172ff4" + }, + { + "bytes": 3839, + "path": "pure_recovery.csv", + "sha256": "4ccbd057d609c75b39a428d2708b1aa4afba95cab9990bb9c8f772388b66c17f" + }, + { + "bytes": 631, + "path": "null_evaluation.csv", + "sha256": "25a06c37aea06319ca7bf5abfd93f5b23bfc25c82ac932e23f4d224ae2abd5a1" + }, + { + "bytes": 12148, + "path": "mixture_evaluation.csv", + "sha256": "393b44e646417d0742e2f15a7b8aeac046cc39ff24e35d9f4b15194929e7b1db" + }, + { + "bytes": 1793, + "path": "out_of_span_evaluation.csv", + "sha256": "24054f50878222ad14478b02e8e59aec85b3d9a93d4ede447f9581531982857a" + }, + { + "bytes": 5008, + "path": "allocations.csv", + "sha256": "86e1bc633a8fe9917a8b50ee6563a38706c6fd4b6124e69a5b69257c9c2af5e4" + } + ], + "inputs": [ + { + "allocation_bytes": 14450, + "allocation_file": "optimal_design_allocation_seed7.csv", + "allocation_file_seed_field_present": false, + "allocation_seed": 7, + "allocation_seed_source": "explicit_cli_metadata", + "allocation_sha256": "a823be49faf6c6cbebf60b11d4b5ca895cf7734d6e9c577ee98f97a5907b69b2", + "construction_contract": { + "all_three_allocations_reconstructed": true, + "allocation_seed": 7, + "comparator_cap_semantics": "the maximin cap does not apply to the deterministic coupled-novelty or uniform-factorial constructors", + "maximin_max_point_fraction": 0.15, + "maximin_maximum_count_by_budget": { + "60": 9 + } + }, + "design_budgets": { + "coupled_novelty": 60, + "maximin_optimized": 60, + "uniform_factorial": 60 + }, + "kind": "chronologically_locked_primary_design_allocation", + "source_code_sha": "1b2028929ac6ebc1cce0882f0c22af9918044342", + "source_repository": "IPS-Stuttgart/Bayesian-ACh" + } + ], + "producer_commit": "c71695fda83ae93407599a909097962ee3fa9e0e", + "producer_git_dirty": false, + "repository": "IPS-Stuttgart/Bayesian-ACh", + "schema_version": 1 +} diff --git a/results/design-open-set-stress-n60/calibration_audit.csv b/results/design-open-set-stress-n60/calibration_audit.csv new file mode 100644 index 0000000..a9102c0 --- /dev/null +++ b/results/design-open-set-stress-n60/calibration_audit.csv @@ -0,0 +1,22 @@ +design,budget_schedule,budget_factor,budget,scenario,generator,replicates,correct_pure_calls,wrong_pure_calls,abstentions,rate,wilson_lower,wilson_upper +coupled_novelty,fixed_budget,,60,null,null,100,0,0,100,0.0,0.0,0.03699349820698566 +coupled_novelty,fixed_budget,,60,matched_pure,innovation_l2,100,47,11,42,0.47,0.3751081795934127,0.5671114302990063 +coupled_novelty,fixed_budget,,60,matched_pure,surprise,100,10,19,71,0.1,0.0552291370606751,0.17436566150491345 +coupled_novelty,fixed_budget,,60,matched_pure,gain,100,4,7,89,0.04,0.01566330399154762,0.09837071435887917 +coupled_novelty,fixed_budget,,60,matched_pure,update_l2,100,2,16,82,0.02,0.005501967550162357,0.07001179072854387 +coupled_novelty,fixed_budget,,60,matched_pure,information_gain,100,57,5,38,0.57,0.47215389549212305,0.6626670147588988 +coupled_novelty,fixed_budget,,60,matched_pure,change_probability,100,28,16,56,0.28,0.20139685211186215,0.37488028709921156 +uniform_factorial,fixed_budget,,60,null,null,100,0,0,100,0.0,0.0,0.03699349820698566 +uniform_factorial,fixed_budget,,60,matched_pure,innovation_l2,100,51,13,36,0.51,0.41348011934027173,0.6057800106955886 +uniform_factorial,fixed_budget,,60,matched_pure,surprise,100,50,12,38,0.5,0.4038315303659957,0.5961684696340044 +uniform_factorial,fixed_budget,,60,matched_pure,gain,100,77,4,19,0.77,0.6784561697712622,0.8415673411969654 +uniform_factorial,fixed_budget,,60,matched_pure,update_l2,100,43,10,47,0.43,0.33733298524110106,0.5278461045078767 +uniform_factorial,fixed_budget,,60,matched_pure,information_gain,100,60,8,32,0.6,0.5020025867910618,0.6905987135675411 +uniform_factorial,fixed_budget,,60,matched_pure,change_probability,100,87,0,13,0.87,0.7901964855992357,0.9224283257275948 +maximin_optimized,fixed_budget,,60,null,null,100,0,3,97,0.03,0.010254524024038925,0.08451936429052759 +maximin_optimized,fixed_budget,,60,matched_pure,innovation_l2,100,77,10,13,0.77,0.6784561697712622,0.8415673411969654 +maximin_optimized,fixed_budget,,60,matched_pure,surprise,100,72,9,19,0.72,0.6251197129007884,0.7986031478881379 +maximin_optimized,fixed_budget,,60,matched_pure,gain,100,89,4,7,0.89,0.8136870349691969,0.9374580364293543 +maximin_optimized,fixed_budget,,60,matched_pure,update_l2,100,85,1,14,0.85,0.7671644040916764,0.9069401471634336 +maximin_optimized,fixed_budget,,60,matched_pure,information_gain,100,81,8,11,0.81,0.7222115462093562,0.8748524849023126 +maximin_optimized,fixed_budget,,60,matched_pure,change_probability,100,96,0,4,0.96,0.9016292856411208,0.9843366960084523 diff --git a/results/design-open-set-stress-n60/mixture_evaluation.csv b/results/design-open-set-stress-n60/mixture_evaluation.csv new file mode 100644 index 0000000..dee42ad --- /dev/null +++ b/results/design-open-set-stress-n60/mixture_evaluation.csv @@ -0,0 +1,46 @@ +design,budget_schedule,budget_factor,budget,first_candidate,second_candidate,mixture_definition,replicates,false_pure_calls,constituent_pure_calls,abstentions,false_pure_call_rate,wilson_lower,wilson_upper,abstention_reasons +coupled_novelty,fixed_budget,,60,innovation_l2,surprise,"equal coefficients, then unit-SD scaling on full grid",200,70,62,130,0.35,0.28728824127358077,0.4183653566395992,"{""flexible_model_better"":2,""null_not_rejected"":4,""pure_ambiguity"":124,""pure_call"":70}" +coupled_novelty,fixed_budget,,60,innovation_l2,gain,"equal coefficients, then unit-SD scaling on full grid",200,79,57,121,0.395,0.32984101013207967,0.46411650840714636,"{""flexible_model_better"":9,""null_not_rejected"":1,""pure_ambiguity"":111,""pure_call"":79}" +coupled_novelty,fixed_budget,,60,innovation_l2,update_l2,"equal coefficients, then unit-SD scaling on full grid",200,52,20,148,0.26,0.20413830063574687,0.3249074560253411,"{""flexible_model_better"":2,""null_not_rejected"":3,""pure_ambiguity"":143,""pure_call"":52}" +coupled_novelty,fixed_budget,,60,innovation_l2,information_gain,"equal coefficients, then unit-SD scaling on full grid",200,65,25,135,0.325,0.26391571539234293,0.39268014883970037,"{""flexible_model_better"":1,""null_not_rejected"":3,""pure_ambiguity"":131,""pure_call"":65}" +coupled_novelty,fixed_budget,,60,innovation_l2,change_probability,"equal coefficients, then unit-SD scaling on full grid",200,92,63,108,0.46,0.3923305179997147,0.5291771081104667,"{""flexible_model_better"":3,""null_not_rejected"":1,""pure_ambiguity"":104,""pure_call"":92}" +coupled_novelty,fixed_budget,,60,surprise,gain,"equal coefficients, then unit-SD scaling on full grid",200,55,22,145,0.275,0.21779700329307355,0.34068339357669647,"{""null_not_rejected"":1,""pure_ambiguity"":144,""pure_call"":55}" +coupled_novelty,fixed_budget,,60,surprise,update_l2,"equal coefficients, then unit-SD scaling on full grid",200,52,10,148,0.26,0.20413830063574687,0.3249074560253411,"{""null_not_rejected"":1,""pure_ambiguity"":147,""pure_call"":52}" +coupled_novelty,fixed_budget,,60,surprise,information_gain,"equal coefficients, then unit-SD scaling on full grid",200,72,66,128,0.36,0.2966919730456654,0.42858471833996925,"{""flexible_model_better"":2,""null_not_rejected"":7,""pure_ambiguity"":119,""pure_call"":72}" +coupled_novelty,fixed_budget,,60,surprise,change_probability,"equal coefficients, then unit-SD scaling on full grid",200,79,61,121,0.395,0.32984101013207967,0.46411650840714636,"{""flexible_model_better"":3,""null_not_rejected"":1,""pure_ambiguity"":117,""pure_call"":79}" +coupled_novelty,fixed_budget,,60,gain,update_l2,"equal coefficients, then unit-SD scaling on full grid",200,38,14,162,0.19,0.14167171527790676,0.2500123870759985,"{""null_not_rejected"":4,""pure_ambiguity"":158,""pure_call"":38}" +coupled_novelty,fixed_budget,,60,gain,information_gain,"equal coefficients, then unit-SD scaling on full grid",200,76,63,124,0.38,0.31559005848673627,0.44893281984380773,"{""null_not_rejected"":3,""pure_ambiguity"":121,""pure_call"":76}" +coupled_novelty,fixed_budget,,60,gain,change_probability,"equal coefficients, then unit-SD scaling on full grid",200,79,51,121,0.395,0.32984101013207967,0.46411650840714636,"{""flexible_model_better"":6,""pure_ambiguity"":115,""pure_call"":79}" +coupled_novelty,fixed_budget,,60,update_l2,information_gain,"equal coefficients, then unit-SD scaling on full grid",200,64,51,136,0.32,0.25926533315468053,0.38751898434113546,"{""null_not_rejected"":2,""pure_ambiguity"":134,""pure_call"":64}" +coupled_novelty,fixed_budget,,60,update_l2,change_probability,"equal coefficients, then unit-SD scaling on full grid",200,75,49,125,0.375,0.31085436858409826,0.44385696301021843,"{""flexible_model_better"":3,""null_not_rejected"":2,""pure_ambiguity"":120,""pure_call"":75}" +coupled_novelty,fixed_budget,,60,information_gain,change_probability,"equal coefficients, then unit-SD scaling on full grid",200,90,72,110,0.45,0.3826406840224836,0.5192438486152431,"{""flexible_model_better"":6,""null_not_rejected"":1,""pure_ambiguity"":103,""pure_call"":90}" +uniform_factorial,fixed_budget,,60,innovation_l2,surprise,"equal coefficients, then unit-SD scaling on full grid",200,104,104,96,0.52,0.45103785072814245,0.588208336216767,"{""flexible_model_better"":1,""null_not_rejected"":6,""pure_ambiguity"":89,""pure_call"":104}" +uniform_factorial,fixed_budget,,60,innovation_l2,gain,"equal coefficients, then unit-SD scaling on full grid",200,59,16,141,0.295,0.2361394385684915,0.36158714524618774,"{""flexible_model_better"":21,""null_not_rejected"":3,""pure_ambiguity"":117,""pure_call"":59}" +uniform_factorial,fixed_budget,,60,innovation_l2,update_l2,"equal coefficients, then unit-SD scaling on full grid",200,57,36,143,0.285,0.22695003004617292,0.3511534602960517,"{""flexible_model_better"":8,""null_not_rejected"":1,""pure_ambiguity"":134,""pure_call"":57}" +uniform_factorial,fixed_budget,,60,innovation_l2,information_gain,"equal coefficients, then unit-SD scaling on full grid",200,87,46,113,0.435,0.36815725502699076,0.5042926374020539,"{""flexible_model_better"":13,""null_not_rejected"":2,""pure_ambiguity"":98,""pure_call"":87}" +uniform_factorial,fixed_budget,,60,innovation_l2,change_probability,"equal coefficients, then unit-SD scaling on full grid",200,75,54,125,0.375,0.31085436858409826,0.44385696301021843,"{""flexible_model_better"":27,""null_not_rejected"":2,""pure_ambiguity"":96,""pure_call"":75}" +uniform_factorial,fixed_budget,,60,surprise,gain,"equal coefficients, then unit-SD scaling on full grid",200,48,21,152,0.24,0.1860661599241369,0.30373340979204166,"{""flexible_model_better"":21,""null_not_rejected"":1,""pure_ambiguity"":130,""pure_call"":48}" +uniform_factorial,fixed_budget,,60,surprise,update_l2,"equal coefficients, then unit-SD scaling on full grid",200,85,44,115,0.425,0.3585361254351385,0.49429067352145145,"{""flexible_model_better"":7,""pure_ambiguity"":108,""pure_call"":85}" +uniform_factorial,fixed_budget,,60,surprise,information_gain,"equal coefficients, then unit-SD scaling on full grid",200,105,95,95,0.525,0.45597396014684755,0.5930837735342891,"{""flexible_model_better"":13,""pure_ambiguity"":82,""pure_call"":105}" +uniform_factorial,fixed_budget,,60,surprise,change_probability,"equal coefficients, then unit-SD scaling on full grid",200,92,86,108,0.46,0.3923305179997147,0.5291771081104667,"{""flexible_model_better"":25,""null_not_rejected"":6,""pure_ambiguity"":77,""pure_call"":92}" +uniform_factorial,fixed_budget,,60,gain,update_l2,"equal coefficients, then unit-SD scaling on full grid",200,99,95,101,0.495,0.4264584535990475,0.5637299996647251,"{""flexible_model_better"":8,""null_not_rejected"":3,""pure_ambiguity"":90,""pure_call"":99}" +uniform_factorial,fixed_budget,,60,gain,information_gain,"equal coefficients, then unit-SD scaling on full grid",200,103,63,97,0.515,0.4461084919274572,0.5833261482812249,"{""flexible_model_better"":9,""null_not_rejected"":2,""pure_ambiguity"":86,""pure_call"":103}" +uniform_factorial,fixed_budget,,60,gain,change_probability,"equal coefficients, then unit-SD scaling on full grid",200,63,55,137,0.315,0.2546232099022851,0.3823495608573036,"{""flexible_model_better"":42,""null_not_rejected"":6,""pure_ambiguity"":89,""pure_call"":63}" +uniform_factorial,fixed_budget,,60,update_l2,information_gain,"equal coefficients, then unit-SD scaling on full grid",200,109,105,91,0.545,0.47578611372488105,0.612517806901165,"{""null_not_rejected"":4,""pure_ambiguity"":87,""pure_call"":109}" +uniform_factorial,fixed_budget,,60,update_l2,change_probability,"equal coefficients, then unit-SD scaling on full grid",200,57,46,143,0.285,0.22695003004617292,0.3511534602960517,"{""flexible_model_better"":21,""null_not_rejected"":4,""pure_ambiguity"":118,""pure_call"":57}" +uniform_factorial,fixed_budget,,60,information_gain,change_probability,"equal coefficients, then unit-SD scaling on full grid",200,65,53,135,0.325,0.26391571539234293,0.39268014883970037,"{""flexible_model_better"":45,""null_not_rejected"":1,""pure_ambiguity"":89,""pure_call"":65}" +maximin_optimized,fixed_budget,,60,innovation_l2,surprise,"equal coefficients, then unit-SD scaling on full grid",200,149,144,51,0.745,0.6803707417954529,0.8003950482796865,"{""flexible_model_better"":3,""null_not_rejected"":3,""pure_ambiguity"":45,""pure_call"":149}" +maximin_optimized,fixed_budget,,60,innovation_l2,gain,"equal coefficients, then unit-SD scaling on full grid",200,111,24,89,0.555,0.4857330241280342,0.6221939899704665,"{""flexible_model_better"":31,""null_not_rejected"":6,""pure_ambiguity"":52,""pure_call"":111}" +maximin_optimized,fixed_budget,,60,innovation_l2,update_l2,"equal coefficients, then unit-SD scaling on full grid",200,124,118,76,0.62,0.5510671801561923,0.6844099415132637,"{""flexible_model_better"":25,""null_not_rejected"":1,""pure_ambiguity"":50,""pure_call"":124}" +maximin_optimized,fixed_budget,,60,innovation_l2,information_gain,"equal coefficients, then unit-SD scaling on full grid",200,116,90,84,0.58,0.5107207393958046,0.6462640083838327,"{""flexible_model_better"":16,""null_not_rejected"":3,""pure_ambiguity"":65,""pure_call"":116}" +maximin_optimized,fixed_budget,,60,innovation_l2,change_probability,"equal coefficients, then unit-SD scaling on full grid",200,123,112,77,0.615,0.5459986722024608,0.6796669027307678,"{""flexible_model_better"":26,""null_not_rejected"":3,""pure_ambiguity"":48,""pure_call"":123}" +maximin_optimized,fixed_budget,,60,surprise,gain,"equal coefficients, then unit-SD scaling on full grid",200,97,36,103,0.485,0.4166738517187752,0.5538915080725428,"{""flexible_model_better"":22,""null_not_rejected"":2,""pure_ambiguity"":79,""pure_call"":97}" +maximin_optimized,fixed_budget,,60,surprise,update_l2,"equal coefficients, then unit-SD scaling on full grid",200,115,55,85,0.575,0.5057093264785485,0.6414638745648614,"{""flexible_model_better"":13,""null_not_rejected"":1,""pure_ambiguity"":71,""pure_call"":115}" +maximin_optimized,fixed_budget,,60,surprise,information_gain,"equal coefficients, then unit-SD scaling on full grid",200,144,140,56,0.72,0.6540769541334929,0.7776311022605097,"{""flexible_model_better"":11,""null_not_rejected"":2,""pure_ambiguity"":43,""pure_call"":144}" +maximin_optimized,fixed_budget,,60,surprise,change_probability,"equal coefficients, then unit-SD scaling on full grid",200,134,131,66,0.67,0.602166847737769,0.7314257412939605,"{""flexible_model_better"":39,""null_not_rejected"":5,""pure_ambiguity"":22,""pure_call"":134}" +maximin_optimized,fixed_budget,,60,gain,update_l2,"equal coefficients, then unit-SD scaling on full grid",200,160,159,40,0.8,0.7391448134346212,0.8495479907390189,"{""flexible_model_better"":5,""null_not_rejected"":1,""pure_ambiguity"":34,""pure_call"":160}" +maximin_optimized,fixed_budget,,60,gain,information_gain,"equal coefficients, then unit-SD scaling on full grid",200,122,77,78,0.61,0.5409374595716976,0.6749165686253038,"{""flexible_model_better"":20,""pure_ambiguity"":58,""pure_call"":122}" +maximin_optimized,fixed_budget,,60,gain,change_probability,"equal coefficients, then unit-SD scaling on full grid",200,54,46,146,0.27,0.21323449814685846,0.33543435198668425,"{""flexible_model_better"":75,""null_not_rejected"":5,""pure_ambiguity"":66,""pure_call"":54}" +maximin_optimized,fixed_budget,,60,update_l2,information_gain,"equal coefficients, then unit-SD scaling on full grid",200,149,146,51,0.745,0.6803707417954529,0.8003950482796865,"{""flexible_model_better"":5,""null_not_rejected"":3,""pure_ambiguity"":43,""pure_call"":149}" +maximin_optimized,fixed_budget,,60,update_l2,change_probability,"equal coefficients, then unit-SD scaling on full grid",200,65,58,135,0.325,0.26391571539234293,0.39268014883970037,"{""flexible_model_better"":68,""null_not_rejected"":2,""pure_ambiguity"":65,""pure_call"":65}" +maximin_optimized,fixed_budget,,60,information_gain,change_probability,"equal coefficients, then unit-SD scaling on full grid",200,107,94,93,0.535,0.4658664687236316,0.6028143584299599,"{""flexible_model_better"":53,""null_not_rejected"":5,""pure_ambiguity"":35,""pure_call"":107}" diff --git a/results/design-open-set-stress-n60/null_evaluation.csv b/results/design-open-set-stress-n60/null_evaluation.csv new file mode 100644 index 0000000..1710f85 --- /dev/null +++ b/results/design-open-set-stress-n60/null_evaluation.csv @@ -0,0 +1,4 @@ +design,budget_schedule,budget_factor,budget,replicates,false_pure_calls,abstentions,false_pure_call_rate,wilson_lower,wilson_upper,abstention_reasons +coupled_novelty,fixed_budget,,60,200,2,198,0.01,0.0027466581335444384,0.035721761716176796,"{""null_not_rejected"":184,""pure_ambiguity"":14,""pure_call"":2}" +uniform_factorial,fixed_budget,,60,200,3,197,0.015,0.005114237793258314,0.043165728792690254,"{""null_not_rejected"":195,""pure_ambiguity"":2,""pure_call"":3}" +maximin_optimized,fixed_budget,,60,200,4,196,0.02,0.007804426416349472,0.05028708690582643,"{""null_not_rejected"":195,""pure_ambiguity"":1,""pure_call"":4}" diff --git a/results/design-open-set-stress-n60/out_of_span_evaluation.csv b/results/design-open-set-stress-n60/out_of_span_evaluation.csv new file mode 100644 index 0000000..08d0df0 --- /dev/null +++ b/results/design-open-set-stress-n60/out_of_span_evaluation.csv @@ -0,0 +1,4 @@ +design,budget_schedule,budget_factor,budget,probe,probe_definition,full_grid_prestandardization_residual_sd,full_grid_maximum_absolute_mean_inner_product,replicates,false_pure_calls,abstentions,false_pure_call_rate,wilson_lower,wilson_upper,pure_call_counts,abstention_reasons +coupled_novelty,fixed_budget,,60,full_grid_orthogonalized_tanh_surprise,unit-SD residual of tanh(standardized surprise) after full-grid OLS projection on intercept plus all six standardized candidates,0.09057218165490002,4.3261690526226935e-14,200,7,193,0.035,0.017055542008567192,0.07047061152229071,"{""change_probability"":7,""gain"":0,""information_gain"":0,""innovation_l2"":0,""surprise"":0,""update_l2"":0}","{""flexible_model_better"":16,""null_not_rejected"":112,""pure_ambiguity"":65,""pure_call"":7}" +uniform_factorial,fixed_budget,,60,full_grid_orthogonalized_tanh_surprise,unit-SD residual of tanh(standardized surprise) after full-grid OLS projection on intercept plus all six standardized candidates,0.09057218165490002,4.3261690526226935e-14,200,1,199,0.005,0.0008831687156009762,0.027773704397892923,"{""change_probability"":0,""gain"":1,""information_gain"":0,""innovation_l2"":0,""surprise"":0,""update_l2"":0}","{""null_not_rejected"":194,""pure_ambiguity"":5,""pure_call"":1}" +maximin_optimized,fixed_budget,,60,full_grid_orthogonalized_tanh_surprise,unit-SD residual of tanh(standardized surprise) after full-grid OLS projection on intercept plus all six standardized candidates,0.09057218165490002,4.3261690526226935e-14,200,103,97,0.515,0.4461084919274572,0.5833261482812249,"{""change_probability"":0,""gain"":0,""information_gain"":0,""innovation_l2"":99,""surprise"":0,""update_l2"":4}","{""flexible_model_better"":30,""null_not_rejected"":52,""pure_ambiguity"":15,""pure_call"":103}" diff --git a/results/design-open-set-stress-n60/pure_recovery.csv b/results/design-open-set-stress-n60/pure_recovery.csv new file mode 100644 index 0000000..1ce54ff --- /dev/null +++ b/results/design-open-set-stress-n60/pure_recovery.csv @@ -0,0 +1,19 @@ +design,budget_schedule,budget_factor,budget,generator,replicates,correct_pure_calls,wrong_pure_calls,abstentions,correct_call_rate,wilson_lower,wilson_upper,raw_closed_set_winner_rate,abstention_reasons +coupled_novelty,fixed_budget,,60,innovation_l2,200,96,11,93,0.48,0.41179166378323306,0.5489621492718576,0.68,"{""flexible_model_better"":6,""null_not_rejected"":11,""pure_ambiguity"":76,""pure_call"":107}" +coupled_novelty,fixed_budget,,60,surprise,200,16,29,155,0.08,0.04984058588975997,0.12598948826714396,0.28,"{""flexible_model_better"":3,""null_not_rejected"":18,""pure_ambiguity"":134,""pure_call"":45}" +coupled_novelty,fixed_budget,,60,gain,200,6,15,179,0.03,0.013820314340111346,0.06389429245451923,0.425,"{""flexible_model_better"":1,""null_not_rejected"":6,""pure_ambiguity"":172,""pure_call"":21}" +coupled_novelty,fixed_budget,,60,update_l2,200,14,29,157,0.07,0.042152171828342806,0.11405480885610644,0.41,"{""null_not_rejected"":2,""pure_ambiguity"":155,""pure_call"":43}" +coupled_novelty,fixed_budget,,60,information_gain,200,104,8,88,0.52,0.45103785072814245,0.588208336216767,0.65,"{""flexible_model_better"":3,""null_not_rejected"":7,""pure_ambiguity"":78,""pure_call"":112}" +coupled_novelty,fixed_budget,,60,change_probability,200,70,23,107,0.35,0.28728824127358077,0.4183653566395992,0.635,"{""flexible_model_better"":3,""null_not_rejected"":1,""pure_ambiguity"":103,""pure_call"":93}" +uniform_factorial,fixed_budget,,60,innovation_l2,200,85,22,93,0.425,0.3585361254351385,0.49429067352145145,0.72,"{""flexible_model_better"":4,""null_not_rejected"":7,""pure_ambiguity"":82,""pure_call"":107}" +uniform_factorial,fixed_budget,,60,surprise,200,95,24,81,0.475,0.4069162264657108,0.5440260398531525,0.73,"{""flexible_model_better"":4,""null_not_rejected"":2,""pure_ambiguity"":75,""pure_call"":119}" +uniform_factorial,fixed_budget,,60,gain,200,148,11,41,0.74,0.6750925439746589,0.7958616993642531,0.83,"{""flexible_model_better"":5,""null_not_rejected"":10,""pure_ambiguity"":26,""pure_call"":159}" +uniform_factorial,fixed_budget,,60,update_l2,200,96,22,82,0.48,0.41179166378323306,0.5489621492718576,0.675,"{""flexible_model_better"":3,""null_not_rejected"":7,""pure_ambiguity"":72,""pure_call"":118}" +uniform_factorial,fixed_budget,,60,information_gain,200,124,16,60,0.62,0.5510671801561923,0.6844099415132637,0.78,"{""flexible_model_better"":3,""null_not_rejected"":1,""pure_ambiguity"":56,""pure_call"":140}" +uniform_factorial,fixed_budget,,60,change_probability,200,166,0,34,0.83,0.7718411637168737,0.8757209208741302,0.9,"{""flexible_model_better"":6,""null_not_rejected"":20,""pure_ambiguity"":8,""pure_call"":166}" +maximin_optimized,fixed_budget,,60,innovation_l2,200,152,16,32,0.76,0.6962665902079583,0.8139338400758631,0.84,"{""flexible_model_better"":2,""null_not_rejected"":7,""pure_ambiguity"":23,""pure_call"":168}" +maximin_optimized,fixed_budget,,60,surprise,200,150,18,32,0.75,0.6856590168795417,0.8049183199318249,0.82,"{""flexible_model_better"":1,""null_not_rejected"":7,""pure_ambiguity"":24,""pure_call"":168}" +maximin_optimized,fixed_budget,,60,gain,200,175,12,13,0.875,0.8219857499741765,0.9138802552428735,0.915,"{""flexible_model_better"":4,""pure_ambiguity"":9,""pure_call"":187}" +maximin_optimized,fixed_budget,,60,update_l2,200,160,20,20,0.8,0.7391448134346212,0.8495479907390189,0.835,"{""null_not_rejected"":2,""pure_ambiguity"":18,""pure_call"":180}" +maximin_optimized,fixed_budget,,60,information_gain,200,174,9,17,0.87,0.8163364812775159,0.9097179772033067,0.935,"{""flexible_model_better"":4,""null_not_rejected"":3,""pure_ambiguity"":10,""pure_call"":183}" +maximin_optimized,fixed_budget,,60,change_probability,200,183,5,12,0.915,0.8681041292843443,0.9462542498225244,0.935,"{""flexible_model_better"":3,""null_not_rejected"":7,""pure_ambiguity"":2,""pure_call"":188}" diff --git a/results/design-open-set-stress-n60/summary.json b/results/design-open-set-stress-n60/summary.json new file mode 100644 index 0000000..01b0172 --- /dev/null +++ b/results/design-open-set-stress-n60/summary.json @@ -0,0 +1,92 @@ +{ + "base_n_eff_indices_from_n60_geometry": { + "coupled_novelty": 1113, + "maximin_optimized": 45, + "uniform_factorial": 93 + }, + "candidate_names": [ + "innovation_l2", + "surprise", + "gain", + "update_l2", + "information_gain", + "change_probability" + ], + "config": { + "allocation_seed": 7, + "alpha": 0.05, + "budget_factors": [], + "calibration_audit_replicates": 100, + "calibration_audit_seed": 130363, + "calibration_replicates": 100, + "confidence_level": 0.95, + "effect_size": 1.0, + "evaluation_replicates": 200, + "evaluation_seed": 155921, + "fixed_budgets": [ + 60 + ], + "inner_folds": 3, + "max_point_fraction": 0.15, + "noise_std": 1.0, + "ridge_lambdas": [ + 0.0, + 0.01, + 0.1, + 1.0 + ], + "target_log_score_gap": 5.0, + "test_fraction": 0.35, + "threshold_seed": 104729 + }, + "design_names": [ + "coupled_novelty", + "uniform_factorial", + "maximin_optimized" + ], + "experiment": "post_freeze_design_abstention_sensitivity", + "maximum_mixture_false_pure_wilson_upper": 0.8495479907390189, + "maximum_null_false_pure_wilson_upper": 0.05028708690582643, + "maximum_out_of_span_false_pure_wilson_upper": 0.5833261482812249, + "minimum_matched_pure_wilson_lower": 0.013820314340111346, + "primary_schedule_chronology": "The equal-N60 allocation and its source commit were frozen in the accepted five-seed benchmark before certificate and stress results; no later certified schedule replaces this primary allocation.", + "primary_schedule_provenance": { + "allocation_bytes": 14450, + "allocation_file": "optimal_design_allocation_seed7.csv", + "allocation_file_seed_field_present": false, + "allocation_seed": 7, + "allocation_seed_source": "explicit_cli_metadata", + "allocation_sha256": "a823be49faf6c6cbebf60b11d4b5ca895cf7734d6e9c577ee98f97a5907b69b2", + "construction_contract": { + "all_three_allocations_reconstructed": true, + "allocation_seed": 7, + "comparator_cap_semantics": "the maximin cap does not apply to the deterministic coupled-novelty or uniform-factorial constructors", + "maximin_max_point_fraction": 0.15, + "maximin_maximum_count_by_budget": { + "60": 9 + } + }, + "design_budgets": { + "coupled_novelty": 60, + "maximin_optimized": 60, + "uniform_factorial": 60 + }, + "kind": "chronologically_locked_primary_design_allocation", + "source_code_sha": "1b2028929ac6ebc1cce0882f0c22af9918044342", + "source_repository": "IPS-Stuttgart/Bayesian-ACh" + }, + "schema_version": 1, + "scope": "This immutable post-freeze sensitivity probes finite-training plug-in variability, a no-signal null, standardized equal 50/50 in-span mixtures, and one residualized nonlinear probe. It does not establish robustness to arbitrary out-of-span biology, other nonlinear mixtures, serial dependence, sensor dynamics, subject hierarchy, or sequential protocol feasibility. A mixture pure call is counted as false even when it names a constituent.", + "technical_gates": { + "all_fifteen_mixtures_per_design_budget": true, + "all_thresholds_finite": true, + "calibration_and_evaluation_seeds_disjoint": true, + "chronologically_locked_primary_allocation_verified": true, + "one_out_of_span_probe_per_design_budget": true + }, + "threshold_rule": { + "adequacy": "all-six train-only-selected ridge model does not beat the best pure by more than the worst-candidate pure-calibrated upper threshold", + "ambiguity": "best pure minus runner exceeds a separately null-calibrated upper threshold", + "pure_signal": "best pure minus intercept-only null exceeds a familywise null-calibrated upper threshold" + } +} diff --git a/results/design-open-set-stress-n60/thresholds.csv b/results/design-open-set-stress-n60/thresholds.csv new file mode 100644 index 0000000..656ba7d --- /dev/null +++ b/results/design-open-set-stress-n60/thresholds.csv @@ -0,0 +1,4 @@ +design,budget_schedule,budget_factor,budget,allocation_source,base_n_eff_index_from_n60_geometry,realized_minimum_pairwise_residual,realized_population_n_eff_index,pure_over_null_threshold,winner_over_runner_threshold,flexible_over_pure_threshold +coupled_novelty,fixed_budget,,60,frozen_override,1113,0.009028563962867553,1113,0.9908053929429492,0.8374072007101319,0.9358247112625193 +uniform_factorial,fixed_budget,,60,frozen_override,93,0.11410719241875189,93,1.4841715232562613,1.043715011057646,1.335738097917563 +maximin_optimized,fixed_budget,,60,frozen_override,45,0.2500365482696447,45,1.4684378147083308,0.7589681023920747,2.074310487667276 diff --git a/scripts/analyze_design_stress_geometry.py b/scripts/analyze_design_stress_geometry.py new file mode 100644 index 0000000..67135fb --- /dev/null +++ b/scripts/analyze_design_stress_geometry.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +"""Read-only stratification and mixture geometry audit for the frozen N60 stress.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import re +import subprocess +from collections import Counter +from itertools import combinations +from pathlib import Path +from typing import Any + +import numpy as np + +from bayesian_ach.design_grid import DESIGN_CANDIDATE_NAMES, generate_transition_design_grid +from bayesian_ach.design_stress import ( + DesignStressConfig, + _rng, + _Scores, + _simulate_scores, + _Thresholds, +) + +_SHA40 = re.compile(r"^[0-9a-f]{40}$") + + +def _rows(path: Path) -> list[dict[str, str]]: + with path.open(newline="", encoding="utf-8") as handle: + return list(csv.DictReader(handle)) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _git_provenance(repo_root: Path, expected_sha: str) -> None: + if _SHA40.fullmatch(expected_sha) is None: + raise ValueError("--code-sha must be a lowercase 40-character commit SHA") + head = subprocess.run( + ["git", "-C", str(repo_root), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + status = subprocess.run( + ["git", "-C", str(repo_root), "status", "--porcelain", "--untracked-files=all"], + check=True, + capture_output=True, + text=True, + ).stdout + if head != expected_sha or status: + raise RuntimeError("geometry diagnostic requires the clean exact --code-sha checkout") + + +def _reason_counts(rows: list[dict[str, str]]) -> dict[str, int]: + total: Counter[str] = Counter() + for row in rows: + total.update(json.loads(row["abstention_reasons"])) + return dict(sorted(total.items())) + + +def _weighted_residual( + response: np.ndarray, + predictors: np.ndarray, + weights: np.ndarray, +) -> float: + design = np.column_stack((np.ones(response.size), predictors)) + root = np.sqrt(weights) + coefficients, _, _, _ = np.linalg.lstsq( + design * root[:, None], + response * root, + rcond=None, + ) + residual = response - design @ coefficients + return float(np.sum(weights * residual**2)) + + +def _gate_counts( + scores: _Scores, + thresholds: _Thresholds, +) -> tuple[bool, bool, bool, bool, str]: + pure_over_null = scores.pure_over_null > thresholds.pure_over_null + winner_over_runner = scores.winner_over_runner > thresholds.winner_over_runner + flexible_adequacy = scores.flexible_over_pure <= thresholds.flexible_over_pure + all_pass = pure_over_null and winner_over_runner and flexible_adequacy + if not pure_over_null: + reason = "null_not_rejected" + elif not winner_over_runner: + reason = "pure_ambiguity" + elif not flexible_adequacy: + reason = "flexible_model_better" + else: + reason = "pure_call" + return pure_over_null, winner_over_runner, flexible_adequacy, all_pass, reason + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--artifact", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--repo-root", default=Path.cwd(), type=Path) + parser.add_argument("--code-sha", required=True) + parser.add_argument("--recompute-only", action="store_true", help=argparse.SUPPRESS) + args = parser.parse_args() + if args.recompute_only: + if _SHA40.fullmatch(args.code_sha) is None: + raise ValueError("--code-sha must be a lowercase 40-character commit SHA") + else: + _git_provenance(args.repo_root.resolve(), args.code_sha) + artifact = args.artifact.resolve() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + + pure = _rows(artifact / "pure_recovery.csv") + mixtures = _rows(artifact / "mixture_evaluation.csv") + null = _rows(artifact / "null_evaluation.csv") + out_of_span = _rows(artifact / "out_of_span_evaluation.csv") + threshold_rows = _rows(artifact / "thresholds.csv") + allocation_rows = _rows(artifact / "allocations.csv") + designs = ["coupled_novelty", "uniform_factorial", "maximin_optimized"] + stratified: list[dict[str, Any]] = [] + for design in designs: + design_pure = [row for row in pure if row["design"] == design] + design_mix = [row for row in mixtures if row["design"] == design] + design_null = next(row for row in null if row["design"] == design) + design_oos = next(row for row in out_of_span if row["design"] == design) + threshold = next(row for row in threshold_rows if row["design"] == design) + weakest = min(design_pure, key=lambda row: float(row["wilson_lower"])) + worst_mix = max(design_mix, key=lambda row: float(row["wilson_upper"])) + stratified.append( + { + "design": design, + "weakest_pure_generator": weakest["generator"], + "weakest_pure_correct_rate": float(weakest["correct_call_rate"]), + "weakest_pure_wilson_lower": float(weakest["wilson_lower"]), + "weakest_pure_raw_closed_set_rate": float( + weakest["raw_closed_set_winner_rate"] + ), + "pure_reason_counts": _reason_counts(design_pure), + "worst_mixture_pair": [ + worst_mix["first_candidate"], + worst_mix["second_candidate"], + ], + "worst_mixture_false_pure_rate": float( + worst_mix["false_pure_call_rate"] + ), + "worst_mixture_wilson_upper": float(worst_mix["wilson_upper"]), + "mixture_reason_counts": _reason_counts(design_mix), + "null_false_pure_rate": float(design_null["false_pure_call_rate"]), + "null_wilson_upper": float(design_null["wilson_upper"]), + "null_reason_counts": json.loads(design_null["abstention_reasons"]), + "out_of_span_false_pure_rate": float( + design_oos["false_pure_call_rate"] + ), + "out_of_span_wilson_upper": float(design_oos["wilson_upper"]), + "out_of_span_reason_counts": json.loads( + design_oos["abstention_reasons"] + ), + "thresholds": { + "pure_over_null": float(threshold["pure_over_null_threshold"]), + "winner_over_runner": float( + threshold["winner_over_runner_threshold"] + ), + "flexible_over_pure": float( + threshold["flexible_over_pure_threshold"] + ), + }, + } + ) + + _, _, signals = generate_transition_design_grid() + counts = np.zeros(signals.shape[0], dtype=np.int64) + for row in allocation_rows: + if row["design"] == "maximin_optimized": + counts[int(row["point_id"])] = int(row["count"]) + weights = counts.astype(float) / float(np.sum(counts)) + selected = counts > 0 + trial_indices = np.repeat(np.arange(counts.size), counts) + trial_signals = signals[trial_indices] + threshold_row = next( + row for row in threshold_rows if row["design"] == "maximin_optimized" + ) + thresholds = _Thresholds( + pure_over_null=float(threshold_row["pure_over_null_threshold"]), + winner_over_runner=float(threshold_row["winner_over_runner_threshold"]), + flexible_over_pure=float(threshold_row["flexible_over_pure_threshold"]), + ) + config = DesignStressConfig() + mixture_by_pair = { + (row["first_candidate"], row["second_candidate"]): row + for row in mixtures + if row["design"] == "maximin_optimized" + } + geometry: list[dict[str, Any]] = [] + n_test = int(round(config.test_fraction * 60)) + for first, second in combinations(range(signals.shape[1]), 2): + first_name = DESIGN_CANDIDATE_NAMES[first] + second_name = DESIGN_CANDIDATE_NAMES[second] + mixture_grid = 0.5 * (signals[:, first] + signals[:, second]) + mixture_grid /= float(np.std(mixture_grid)) + constituent_residuals = { + index: _weighted_residual( + mixture_grid, + signals[:, index : index + 1], + weights, + ) + for index in (first, second) + } + all_residuals = [ + _weighted_residual( + mixture_grid, + signals[:, index : index + 1], + weights, + ) + for index in range(signals.shape[1]) + ] + pair_residual = _weighted_residual( + mixture_grid, + signals[:, [first, second]], + weights, + ) + pair_values = signals[selected][:, [first, second]] + pair_weights = weights[selected] + means = np.sum(pair_values * pair_weights[:, None], axis=0) + centered = pair_values - means + covariance = (centered * pair_weights[:, None]).T @ centered + scales = np.sqrt(np.diag(covariance)) + correlation = float(covariance[0, 1] / (scales[0] * scales[1])) + condition = float(np.linalg.cond(covariance)) + + gate_totals: Counter[str] = Counter() + reason_totals: Counter[str] = Counter() + mixture = mixture_grid[trial_indices] + for replicate in range(config.evaluation_replicates): + scores = _simulate_scores( + trial_signals, + mixture, + config=config, + rng=_rng( + config.evaluation_seed, + 2, + 60, + 2, + first, + second, + replicate, + ), + ) + null_pass, ambiguity_pass, adequacy_pass, all_pass, reason = _gate_counts( + scores, + thresholds, + ) + gate_totals.update( + { + "pure_over_null_pass": int(null_pass), + "winner_over_runner_pass": int(ambiguity_pass), + "flexible_adequacy_pass": int(adequacy_pass), + "all_three_pass": int(all_pass), + } + ) + reason_totals[reason] += 1 + frozen = mixture_by_pair[(first_name, second_name)] + if gate_totals["all_three_pass"] != int(frozen["false_pure_calls"]): + raise RuntimeError("diagnostic replay does not reproduce frozen false-call count") + best_constituent = min(constituent_residuals, key=constituent_residuals.get) + best_any = int(np.argmin(all_residuals)) + geometry.append( + { + "first_candidate": first_name, + "second_candidate": second_name, + "false_pure_call_rate": float(frozen["false_pure_call_rate"]), + "wilson_upper": float(frozen["wilson_upper"]), + "sequential_reason_counts": dict(sorted(reason_totals.items())), + "independent_gate_pass_counts": dict(sorted(gate_totals.items())), + "best_constituent": DESIGN_CANDIDATE_NAMES[best_constituent], + "best_constituent_residual": constituent_residuals[best_constituent], + "best_any_pure": DESIGN_CANDIDATE_NAMES[best_any], + "best_any_pure_residual": all_residuals[best_any], + "oracle_two_component_residual": pair_residual, + "best_pure_profiled_gap_n_test_21": ( + 0.5 * n_test * math.log1p(all_residuals[best_any]) + ), + "oracle_two_component_profiled_gap_n_test_21": ( + 0.5 * n_test * math.log1p(max(0.0, pair_residual)) + ), + "weighted_candidate_correlation": correlation, + "weighted_pair_covariance_condition_number": condition, + } + ) + + report = { + "schema_version": 1, + "kind": "read_only_post_freeze_stress_diagnostic", + "diagnostic_producer_commit": args.code_sha, + "diagnostic_producer_clean_worktree": True, + "diagnostic_script_sha256": _sha256(Path(__file__).resolve()), + "source_artifact_sha256sums_sha256": _sha256(artifact / "SHA256SUMS.csv"), + "source_producer_commit": json.loads( + (artifact / "artifact_manifest.json").read_text(encoding="utf-8") + )["producer_commit"], + "no_threshold_or_evaluation_changes": True, + "evaluation_replicates": config.evaluation_replicates, + "calibration_replicates": config.calibration_replicates, + "calibration_audit_replicates": config.calibration_audit_replicates, + "threshold_seed": config.threshold_seed, + "calibration_audit_seed": config.calibration_audit_seed, + "evaluation_seed": config.evaluation_seed, + "test_fraction": config.test_fraction, + "n_test": n_test, + "stratified": stratified, + "maximin_mixture_geometry": geometry, + } + (output / "stress_diagnostic.json").write_text( + json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + with (output / "maximin_mixture_geometry.csv").open( + "w", newline="", encoding="utf-8" + ) as handle: + flat_rows = [] + for row in geometry: + flat = dict(row) + flat["sequential_reason_counts"] = json.dumps( + flat["sequential_reason_counts"], sort_keys=True, separators=(",", ":") + ) + flat["independent_gate_pass_counts"] = json.dumps( + flat["independent_gate_pass_counts"], sort_keys=True, separators=(",", ":") + ) + flat_rows.append(flat) + writer = csv.DictWriter(handle, fieldnames=list(flat_rows[0]), lineterminator="\n") + writer.writeheader() + writer.writerows(flat_rows) + payload_names = ("stress_diagnostic.json", "maximin_mixture_geometry.csv") + manifest = { + "schema_version": 1, + "artifact": "post_freeze_stress_geometry_diagnostic", + "producer_commit": args.code_sha, + "producer_git_dirty": False, + "diagnostic_script_sha256": report["diagnostic_script_sha256"], + "source_artifact_sha256sums_sha256": report[ + "source_artifact_sha256sums_sha256" + ], + "files": [ + { + "path": name, + "bytes": (output / name).stat().st_size, + "sha256": _sha256(output / name), + } + for name in payload_names + ], + } + (output / "artifact_manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + checksum_names = (*payload_names, "artifact_manifest.json") + with (output / "SHA256SUMS.csv").open( + "w", newline="", encoding="utf-8" + ) as handle: + writer = csv.DictWriter( + handle, + fieldnames=["file", "bytes", "sha256"], + lineterminator="\n", + ) + writer.writeheader() + writer.writerows( + { + "file": name, + "bytes": (output / name).stat().st_size, + "sha256": _sha256(output / name), + } + for name in checksum_names + ) + print(json.dumps(report, indent=2, sort_keys=True, allow_nan=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_design_stress_geometry.py b/scripts/verify_design_stress_geometry.py new file mode 100644 index 0000000..f0278a1 --- /dev/null +++ b/scripts/verify_design_stress_geometry.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Independently recompute a frozen post-stress geometry diagnostic.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +_PAYLOADS = { + "stress_diagnostic.json", + "maximin_mixture_geometry.csv", + "artifact_manifest.json", +} +_SHA40 = re.compile(r"^[0-9a-f]{40}$") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _verify_hashes(package: Path) -> dict[str, dict[str, str]]: + with (package / "SHA256SUMS.csv").open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + locked: dict[str, dict[str, str]] = {} + for row in rows: + name = str(row["file"]) + if name in locked: + raise ValueError(f"duplicate checksum row: {name}") + locked[name] = {key: str(value) for key, value in row.items()} + if set(locked) != _PAYLOADS: + raise ValueError("checksum table does not bind the exact diagnostic payload set") + for name, row in locked.items(): + path = package / name + if ( + not path.is_file() + or int(row["bytes"]) != path.stat().st_size + or row["sha256"] != _sha256(path) + ): + raise ValueError(f"locked geometry payload mismatch: {name}") + return locked + + +def verify_package( + package: Path, + *, + source_artifact: Path, + repo_root: Path, +) -> dict[str, object]: + """Verify hashes and rerun all stratification, geometry, and frozen evaluations.""" + + package = package.resolve() + source_artifact = source_artifact.resolve() + repo_root = repo_root.resolve() + locked = _verify_hashes(package) + manifest = json.loads( + (package / "artifact_manifest.json").read_text(encoding="utf-8") + ) + producer = str(manifest.get("producer_commit", "")) + analyzer = repo_root / "scripts/analyze_design_stress_geometry.py" + if ( + manifest.get("schema_version") != 1 + or manifest.get("artifact") != "post_freeze_stress_geometry_diagnostic" + or manifest.get("producer_git_dirty") is not False + or _SHA40.fullmatch(producer) is None + or manifest.get("diagnostic_script_sha256") != _sha256(analyzer) + or manifest.get("source_artifact_sha256sums_sha256") + != _sha256(source_artifact / "SHA256SUMS.csv") + ): + raise ValueError("invalid geometry diagnostic provenance contract") + if { + item["path"]: item["sha256"] for item in manifest["files"] + } != { + name: row["sha256"] + for name, row in locked.items() + if name != "artifact_manifest.json" + }: + raise ValueError("manifest and checksum table disagree") + + report = json.loads((package / "stress_diagnostic.json").read_text(encoding="utf-8")) + geometry = report["maximin_mixture_geometry"] + if ( + report.get("diagnostic_producer_commit") != producer + or report.get("diagnostic_script_sha256") != _sha256(analyzer) + or report.get("source_artifact_sha256sums_sha256") + != _sha256(source_artifact / "SHA256SUMS.csv") + or report.get("no_threshold_or_evaluation_changes") is not True + or len(geometry) != 15 + ): + raise ValueError("invalid geometry diagnostic report contract") + for row in geometry: + if ( + int(row["independent_gate_pass_counts"]["all_three_pass"]) + != int(round(float(row["false_pure_call_rate"]) * 200)) + or float(row["oracle_two_component_residual"]) > 1.0e-20 + ): + raise ValueError("geometry diagnostic gate or oracle residual is inconsistent") + + with tempfile.TemporaryDirectory(prefix="stress-geometry-verify-") as temporary: + recomputed = Path(temporary) / "recomputed" + subprocess.run( + [ + sys.executable, + str(analyzer), + "--artifact", + str(source_artifact), + "--output", + str(recomputed), + "--repo-root", + str(repo_root), + "--code-sha", + producer, + "--recompute-only", + ], + check=True, + capture_output=True, + text=True, + ) + for name in _PAYLOADS: + if (package / name).read_bytes() != (recomputed / name).read_bytes(): + raise ValueError(f"independent recomputation mismatch: {name}") + return { + "package": package.name, + "verified": True, + "producer_commit": producer, + "pair_count": len(geometry), + "source_artifact_sha256sums_sha256": manifest[ + "source_artifact_sha256sums_sha256" + ], + "sha256sums_sha256": _sha256(package / "SHA256SUMS.csv"), + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("package", type=Path) + parser.add_argument("--source-artifact", required=True, type=Path) + parser.add_argument("--repo-root", default=Path.cwd(), type=Path) + args = parser.parse_args() + report = verify_package( + args.package, + source_artifact=args.source_artifact, + repo_root=args.repo_root, + ) + print(json.dumps(report, indent=2, sort_keys=True, allow_nan=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/bayesian_ach/design_stress.py b/src/bayesian_ach/design_stress.py new file mode 100644 index 0000000..1d6181c --- /dev/null +++ b/src/bayesian_ach/design_stress.py @@ -0,0 +1,1121 @@ +"""Finite-sample abstention stress tests for maximin design evidence.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from dataclasses import asdict, dataclass +from itertools import combinations +from statistics import NormalDist +from typing import Any + +import numpy as np +from numpy.typing import NDArray + +from bayesian_ach.design_geometry import design_diagnostics +from bayesian_ach.design_grid import ( + DESIGN_CANDIDATE_NAMES, + coupled_novelty_design, + generate_transition_design_grid, + uniform_factorial_design, +) +from bayesian_ach.design_optimizer import optimize_maximin_design + +STRESS_DESIGNS = ( + "coupled_novelty", + "uniform_factorial", + "maximin_optimized", +) + + +@dataclass(frozen=True, slots=True) +class DesignStressConfig: + """Frozen simulation and calibration settings for the sensitivity artifact.""" + + fixed_budgets: tuple[int, ...] = (60,) + budget_factors: tuple[float, ...] = () + calibration_replicates: int = 100 + calibration_audit_replicates: int = 100 + evaluation_replicates: int = 200 + test_fraction: float = 0.35 + effect_size: float = 1.0 + noise_std: float = 1.0 + target_log_score_gap: float = 5.0 + alpha: float = 0.05 + confidence_level: float = 0.95 + ridge_lambdas: tuple[float, ...] = (0.0, 0.01, 0.1, 1.0) + inner_folds: int = 3 + max_point_fraction: float = 0.15 + allocation_seed: int = 7 + threshold_seed: int = 104729 + calibration_audit_seed: int = 130363 + evaluation_seed: int = 155921 + + def validate(self) -> None: + if not self.fixed_budgets and not self.budget_factors: + raise ValueError("at least one fixed budget or N_eff factor is required") + if ( + any( + value < len(DESIGN_CANDIDATE_NAMES) + 1 + for value in self.fixed_budgets + ) + or len(set(self.fixed_budgets)) != len(self.fixed_budgets) + ): + raise ValueError("fixed_budgets must be unique and exceed candidate count") + if ( + any( + not math.isfinite(value) or value <= 0.0 + for value in self.budget_factors + ) + or len(set(self.budget_factors)) != len(self.budget_factors) + ): + raise ValueError("budget_factors must be finite, positive, and unique") + for name, value in ( + ("calibration_replicates", self.calibration_replicates), + ("calibration_audit_replicates", self.calibration_audit_replicates), + ("evaluation_replicates", self.evaluation_replicates), + ): + if value < 20: + raise ValueError(f"{name} must be at least 20") + if not 0.0 < self.test_fraction < 1.0: + raise ValueError("test_fraction must lie in (0, 1)") + if ( + not math.isfinite(self.effect_size) + or not math.isfinite(self.noise_std) + or self.effect_size <= 0.0 + or self.noise_std <= 0.0 + ): + raise ValueError("effect_size and noise_std must be finite and positive") + if ( + not math.isfinite(self.target_log_score_gap) + or self.target_log_score_gap <= 0.0 + ): + raise ValueError("target_log_score_gap must be finite and positive") + if not 0.0 < self.alpha < 0.5: + raise ValueError("alpha must lie in (0, 0.5)") + if not 0.0 < self.confidence_level < 1.0: + raise ValueError("confidence_level must lie in (0, 1)") + if not self.ridge_lambdas or any( + not math.isfinite(value) or value < 0.0 + for value in self.ridge_lambdas + ): + raise ValueError("ridge_lambdas must be finite and nonnegative") + if self.inner_folds < 2: + raise ValueError("inner_folds must be at least two") + if not 0.0 < self.max_point_fraction <= 1.0: + raise ValueError("max_point_fraction must lie in (0, 1]") + seeds = { + self.threshold_seed, + self.calibration_audit_seed, + self.evaluation_seed, + } + if len(seeds) != 3: + raise ValueError("threshold, audit, and evaluation seeds must be distinct") + + +@dataclass(frozen=True, slots=True) +class DesignStressResult: + """Complete descriptive stress-test tables and summary.""" + + summary: dict[str, Any] + thresholds: tuple[dict[str, Any], ...] + calibration: tuple[dict[str, Any], ...] + pure_recovery: tuple[dict[str, Any], ...] + null_evaluation: tuple[dict[str, Any], ...] + mixture_evaluation: tuple[dict[str, Any], ...] + out_of_span_evaluation: tuple[dict[str, Any], ...] + allocations: tuple[dict[str, Any], ...] + + +@dataclass(frozen=True, slots=True) +class _Scores: + winner: int + runner: int + best_pure_score: float + runner_score: float + null_score: float + flexible_score: float + ridge_lambda: float + + @property + def pure_over_null(self) -> float: + return self.best_pure_score - self.null_score + + @property + def winner_over_runner(self) -> float: + return self.best_pure_score - self.runner_score + + @property + def flexible_over_pure(self) -> float: + return self.flexible_score - self.best_pure_score + + +@dataclass(frozen=True, slots=True) +class _Thresholds: + pure_over_null: float + winner_over_runner: float + flexible_over_pure: float + + +def _fit_variance(residual: NDArray[np.float64]) -> float: + return max(float(np.mean(residual**2)), 1.0e-8) + + +def _gaussian_score(residual: NDArray[np.float64], variance: float) -> float: + return float( + np.sum( + -0.5 + * ( + math.log(2.0 * math.pi * variance) + + residual**2 / variance + ) + ) + ) + + +def _fit_univariate( + predictor: NDArray[np.float64], + response: NDArray[np.float64], + train: NDArray[np.int64], + test: NDArray[np.int64], +) -> float: + design = np.column_stack((np.ones(train.size), predictor[train])) + coefficients, _, _, _ = np.linalg.lstsq( + design, + response[train], + rcond=None, + ) + training_residual = response[train] - design @ coefficients + variance = _fit_variance(training_residual) + prediction = coefficients[0] + coefficients[1] * predictor[test] + return _gaussian_score(response[test] - prediction, variance) + + +def _fit_null( + response: NDArray[np.float64], + train: NDArray[np.int64], + test: NDArray[np.int64], +) -> float: + mean = float(np.mean(response[train])) + variance = _fit_variance(response[train] - mean) + return _gaussian_score(response[test] - mean, variance) + + +def _ridge_coefficients( + signals: NDArray[np.float64], + response: NDArray[np.float64], + indices: NDArray[np.int64], + ridge_lambda: float, +) -> NDArray[np.float64]: + design = np.column_stack((np.ones(indices.size), signals[indices])) + penalty = np.eye(design.shape[1], dtype=float) * ridge_lambda + penalty[0, 0] = 0.0 + coefficients, _, _, _ = np.linalg.lstsq( + design.T @ design + penalty, + design.T @ response[indices], + rcond=None, + ) + return np.asarray(coefficients, dtype=float) + + +def _score_flexible_at_lambda( + signals: NDArray[np.float64], + response: NDArray[np.float64], + train: NDArray[np.int64], + test: NDArray[np.int64], + ridge_lambda: float, +) -> float: + coefficients = _ridge_coefficients( + signals, + response, + train, + ridge_lambda, + ) + train_design = np.column_stack((np.ones(train.size), signals[train])) + variance = _fit_variance(response[train] - train_design @ coefficients) + test_design = np.column_stack((np.ones(test.size), signals[test])) + return _gaussian_score(response[test] - test_design @ coefficients, variance) + + +def _select_ridge_lambda( + signals: NDArray[np.float64], + response: NDArray[np.float64], + train: NDArray[np.int64], + *, + lambdas: tuple[float, ...], + folds: int, + rng: np.random.Generator, +) -> float: + shuffled = np.asarray(rng.permutation(train), dtype=np.int64) + split = [ + np.asarray(values, dtype=np.int64) + for values in np.array_split(shuffled, min(folds, train.size)) + if values.size > 0 + ] + best_lambda = float(min(lambdas)) + best_score = -math.inf + for ridge_lambda in sorted(lambdas): + score = 0.0 + for fold_index, validation in enumerate(split): + inner_train = np.concatenate( + [values for index, values in enumerate(split) if index != fold_index] + ) + score += _score_flexible_at_lambda( + signals, + response, + inner_train, + validation, + float(ridge_lambda), + ) + if score > best_score + 1.0e-12: + best_score = score + best_lambda = float(ridge_lambda) + return best_lambda + + +def _fit_and_score( + signals: NDArray[np.float64], + response: NDArray[np.float64], + train: NDArray[np.int64], + test: NDArray[np.int64], + *, + config: DesignStressConfig, + rng: np.random.Generator, +) -> _Scores: + pure = np.asarray( + [ + _fit_univariate(signals[:, candidate], response, train, test) + for candidate in range(signals.shape[1]) + ], + dtype=float, + ) + order = np.lexsort((np.arange(pure.size), -pure)) + winner = int(order[0]) + runner = int(order[1]) + ridge_lambda = _select_ridge_lambda( + signals, + response, + train, + lambdas=config.ridge_lambdas, + folds=config.inner_folds, + rng=rng, + ) + return _Scores( + winner=winner, + runner=runner, + best_pure_score=float(pure[winner]), + runner_score=float(pure[runner]), + null_score=_fit_null(response, train, test), + flexible_score=_score_flexible_at_lambda( + signals, + response, + train, + test, + ridge_lambda, + ), + ridge_lambda=ridge_lambda, + ) + + +def _simulate_scores( + signals: NDArray[np.float64], + generator: NDArray[np.float64], + *, + config: DesignStressConfig, + rng: np.random.Generator, +) -> _Scores: + response = ( + config.effect_size * generator + + rng.normal(0.0, config.noise_std, size=generator.size) + ) + order = np.asarray(rng.permutation(generator.size), dtype=np.int64) + n_test = max( + 2, + min( + generator.size - config.inner_folds - 2, + int(round(config.test_fraction * generator.size)), + ), + ) + test = order[:n_test] + train = order[n_test:] + return _fit_and_score( + signals, + response, + train, + test, + config=config, + rng=rng, + ) + + +def _upper_conformal_quantile( + values: list[float], + *, + alpha: float, +) -> float: + ordered = np.sort(np.asarray(values, dtype=float)) + rank = int(math.ceil((ordered.size + 1) * (1.0 - alpha))) + if rank > ordered.size: + return math.inf + return float(ordered[max(0, rank - 1)]) + + +def _wilson_interval( + successes: int, + total: int, + confidence_level: float, +) -> tuple[float, float]: + if total < 1: + raise ValueError("total must be positive") + z = NormalDist().inv_cdf(0.5 + confidence_level / 2.0) + proportion = successes / total + denominator = 1.0 + z**2 / total + centre = (proportion + z**2 / (2.0 * total)) / denominator + radius = ( + z + * math.sqrt( + proportion * (1.0 - proportion) / total + + z**2 / (4.0 * total**2) + ) + / denominator + ) + return max(0.0, centre - radius), min(1.0, centre + radius) + + +def _call(scores: _Scores, thresholds: _Thresholds) -> tuple[int | None, str]: + if scores.pure_over_null <= thresholds.pure_over_null: + return None, "null_not_rejected" + if scores.winner_over_runner <= thresholds.winner_over_runner: + return None, "pure_ambiguity" + if scores.flexible_over_pure > thresholds.flexible_over_pure: + return None, "flexible_model_better" + return scores.winner, "pure_call" + + +def _rng(seed: int, *keys: int) -> np.random.Generator: + return np.random.default_rng(np.random.SeedSequence([seed, *keys])) + + +def _design_counts( + design: str, + rows: tuple[dict[str, Any], ...], + signals: NDArray[np.float64], + budget: int, + config: DesignStressConfig, +) -> NDArray[np.int64]: + if design == "coupled_novelty": + return coupled_novelty_design(rows, budget) + if design == "uniform_factorial": + return uniform_factorial_design( + len(rows), + budget, + seed=config.allocation_seed, + ) + if design == "maximin_optimized": + return optimize_maximin_design( + signals, + budget, + max_point_fraction=config.max_point_fraction, + ).counts + raise ValueError(f"unknown design: {design}") + + +def _validated_override( + value: NDArray[np.int64], + *, + design: str, + point_count: int, + budget: int, + maximum_count: int, +) -> NDArray[np.int64]: + counts = np.asarray(value, dtype=np.int64) + if counts.shape != (point_count,) or np.any(counts < 0): + raise ValueError("allocation override must be a nonnegative grid-sized vector") + if int(np.sum(counts)) != budget: + raise ValueError("allocation override must sum to its declared budget") + if design == "maximin_optimized" and np.any(counts > maximum_count): + raise ValueError("maximin allocation override exceeds the per-cell cap") + return counts + + +def _base_targets( + rows: tuple[dict[str, Any], ...], + signals: NDArray[np.float64], + config: DesignStressConfig, +) -> dict[str, int]: + targets: dict[str, int] = {} + for design in STRESS_DESIGNS: + counts = _design_counts(design, rows, signals, 60, config) + targets[design] = design_diagnostics( + signals, + counts, + effect_size=config.effect_size, + noise_std=config.noise_std, + target_log_score_gap=config.target_log_score_gap, + ).trials_for_expected_log_score_gap_target + return targets + + +def _calibrate_thresholds( + signals: NDArray[np.float64], + *, + config: DesignStressConfig, + design_index: int, + budget: int, +) -> _Thresholds: + null_over_baseline: list[float] = [] + null_ambiguity: list[float] = [] + null_generator = np.zeros(signals.shape[0], dtype=float) + for replicate in range(config.calibration_replicates): + scores = _simulate_scores( + signals, + null_generator, + config=config, + rng=_rng( + config.threshold_seed, + design_index, + budget, + 0, + replicate, + ), + ) + null_over_baseline.append(scores.pure_over_null) + null_ambiguity.append(scores.winner_over_runner) + + flexible_thresholds = [] + for generator in range(signals.shape[1]): + values = [] + for replicate in range(config.calibration_replicates): + scores = _simulate_scores( + signals, + signals[:, generator], + config=config, + rng=_rng( + config.threshold_seed, + design_index, + budget, + 1, + generator, + replicate, + ), + ) + values.append(scores.flexible_over_pure) + flexible_thresholds.append( + _upper_conformal_quantile(values, alpha=config.alpha) + ) + return _Thresholds( + pure_over_null=_upper_conformal_quantile( + null_over_baseline, + alpha=config.alpha, + ), + winner_over_runner=_upper_conformal_quantile( + null_ambiguity, + alpha=config.alpha, + ), + flexible_over_pure=max(0.0, max(flexible_thresholds)), + ) + + +def _budget_schedule(factor: float | None) -> str: + return "fixed_budget" if factor is None else "n_eff_factor" + + +def _calibration_audit_rows( + design: str, + design_index: int, + factor: float | None, + budget: int, + signals: NDArray[np.float64], + thresholds: _Thresholds, + config: DesignStressConfig, +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + null_false_calls = 0 + for replicate in range(config.calibration_audit_replicates): + scores = _simulate_scores( + signals, + np.zeros(signals.shape[0], dtype=float), + config=config, + rng=_rng( + config.calibration_audit_seed, + design_index, + budget, + 0, + replicate, + ), + ) + call, _ = _call(scores, thresholds) + null_false_calls += int(call is not None) + lower, upper = _wilson_interval( + null_false_calls, + config.calibration_audit_replicates, + config.confidence_level, + ) + rows.append( + { + "design": design, + "budget_schedule": _budget_schedule(factor), + "budget_factor": factor, + "budget": budget, + "scenario": "null", + "generator": "null", + "replicates": config.calibration_audit_replicates, + "correct_pure_calls": 0, + "wrong_pure_calls": null_false_calls, + "abstentions": config.calibration_audit_replicates - null_false_calls, + "rate": null_false_calls / config.calibration_audit_replicates, + "wilson_lower": lower, + "wilson_upper": upper, + } + ) + for generator, name in enumerate(DESIGN_CANDIDATE_NAMES): + correct = 0 + wrong = 0 + abstain = 0 + for replicate in range(config.calibration_audit_replicates): + scores = _simulate_scores( + signals, + signals[:, generator], + config=config, + rng=_rng( + config.calibration_audit_seed, + design_index, + budget, + 1, + generator, + replicate, + ), + ) + call, _ = _call(scores, thresholds) + correct += int(call == generator) + wrong += int(call is not None and call != generator) + abstain += int(call is None) + lower, upper = _wilson_interval( + correct, + config.calibration_audit_replicates, + config.confidence_level, + ) + rows.append( + { + "design": design, + "budget_schedule": _budget_schedule(factor), + "budget_factor": factor, + "budget": budget, + "scenario": "matched_pure", + "generator": name, + "replicates": config.calibration_audit_replicates, + "correct_pure_calls": correct, + "wrong_pure_calls": wrong, + "abstentions": abstain, + "rate": correct / config.calibration_audit_replicates, + "wilson_lower": lower, + "wilson_upper": upper, + } + ) + return rows + + +def _evaluation_rows( + design: str, + design_index: int, + factor: float | None, + budget: int, + trial_signals: NDArray[np.float64], + full_grid_signals: NDArray[np.float64], + trial_indices: NDArray[np.int64], + thresholds: _Thresholds, + config: DesignStressConfig, +) -> tuple[ + list[dict[str, Any]], + list[dict[str, Any]], + list[dict[str, Any]], +]: + pure_rows: list[dict[str, Any]] = [] + for generator, name in enumerate(DESIGN_CANDIDATE_NAMES): + correct = 0 + wrong = 0 + abstain = 0 + raw_correct = 0 + reasons: dict[str, int] = {} + for replicate in range(config.evaluation_replicates): + scores = _simulate_scores( + trial_signals, + trial_signals[:, generator], + config=config, + rng=_rng( + config.evaluation_seed, + design_index, + budget, + 0, + generator, + replicate, + ), + ) + call, reason = _call(scores, thresholds) + raw_correct += int(scores.winner == generator) + correct += int(call == generator) + wrong += int(call is not None and call != generator) + abstain += int(call is None) + reasons[reason] = reasons.get(reason, 0) + 1 + lower, upper = _wilson_interval( + correct, + config.evaluation_replicates, + config.confidence_level, + ) + pure_rows.append( + { + "design": design, + "budget_schedule": _budget_schedule(factor), + "budget_factor": factor, + "budget": budget, + "generator": name, + "replicates": config.evaluation_replicates, + "correct_pure_calls": correct, + "wrong_pure_calls": wrong, + "abstentions": abstain, + "correct_call_rate": correct / config.evaluation_replicates, + "wilson_lower": lower, + "wilson_upper": upper, + "raw_closed_set_winner_rate": ( + raw_correct / config.evaluation_replicates + ), + "abstention_reasons": dict(sorted(reasons.items())), + } + ) + + null_false = 0 + null_reasons: dict[str, int] = {} + for replicate in range(config.evaluation_replicates): + scores = _simulate_scores( + trial_signals, + np.zeros(trial_signals.shape[0], dtype=float), + config=config, + rng=_rng( + config.evaluation_seed, + design_index, + budget, + 1, + replicate, + ), + ) + call, reason = _call(scores, thresholds) + null_false += int(call is not None) + null_reasons[reason] = null_reasons.get(reason, 0) + 1 + lower, upper = _wilson_interval( + null_false, + config.evaluation_replicates, + config.confidence_level, + ) + null_rows = [ + { + "design": design, + "budget_schedule": _budget_schedule(factor), + "budget_factor": factor, + "budget": budget, + "replicates": config.evaluation_replicates, + "false_pure_calls": null_false, + "abstentions": config.evaluation_replicates - null_false, + "false_pure_call_rate": null_false / config.evaluation_replicates, + "wilson_lower": lower, + "wilson_upper": upper, + "abstention_reasons": dict(sorted(null_reasons.items())), + } + ] + + mixture_rows: list[dict[str, Any]] = [] + for first, second in combinations(range(full_grid_signals.shape[1]), 2): + mixture_grid = 0.5 * ( + full_grid_signals[:, first] + full_grid_signals[:, second] + ) + scale = float(np.std(mixture_grid)) + if scale <= 1.0e-12: + raise RuntimeError("a 50/50 mixture is constant on the full grid") + mixture = mixture_grid[trial_indices] / scale + false_calls = 0 + constituent_calls = 0 + mixture_reasons: dict[str, int] = {} + for replicate in range(config.evaluation_replicates): + scores = _simulate_scores( + trial_signals, + mixture, + config=config, + rng=_rng( + config.evaluation_seed, + design_index, + budget, + 2, + first, + second, + replicate, + ), + ) + call, reason = _call(scores, thresholds) + false_calls += int(call is not None) + constituent_calls += int(call in {first, second}) + mixture_reasons[reason] = mixture_reasons.get(reason, 0) + 1 + lower, upper = _wilson_interval( + false_calls, + config.evaluation_replicates, + config.confidence_level, + ) + mixture_rows.append( + { + "design": design, + "budget_schedule": _budget_schedule(factor), + "budget_factor": factor, + "budget": budget, + "first_candidate": DESIGN_CANDIDATE_NAMES[first], + "second_candidate": DESIGN_CANDIDATE_NAMES[second], + "mixture_definition": ( + "equal coefficients, then unit-SD scaling on full grid" + ), + "replicates": config.evaluation_replicates, + "false_pure_calls": false_calls, + "constituent_pure_calls": constituent_calls, + "abstentions": config.evaluation_replicates - false_calls, + "false_pure_call_rate": ( + false_calls / config.evaluation_replicates + ), + "wilson_lower": lower, + "wilson_upper": upper, + "abstention_reasons": dict(sorted(mixture_reasons.items())), + } + ) + return pure_rows, null_rows, mixture_rows + + +def _out_of_span_probe( + full_grid_signals: NDArray[np.float64], +) -> tuple[NDArray[np.float64], float, float]: + """Return a fixed nonlinear surprise probe orthogonal to the linear span.""" + + surprise_index = DESIGN_CANDIDATE_NAMES.index("surprise") + raw = np.tanh(full_grid_signals[:, surprise_index]) + design = np.column_stack( + (np.ones(full_grid_signals.shape[0]), full_grid_signals) + ) + coefficients, _, _, _ = np.linalg.lstsq(design, raw, rcond=None) + residual = raw - design @ coefficients + scale = float(np.std(residual)) + if scale <= 1.0e-12: + raise RuntimeError("the nonlinear open-set probe is numerically degenerate") + probe = residual / scale + maximum_inner_product = float( + np.max(np.abs(design.T @ probe)) / full_grid_signals.shape[0] + ) + return np.asarray(probe, dtype=float), scale, maximum_inner_product + + +def _out_of_span_rows( + design: str, + design_index: int, + factor: float | None, + budget: int, + trial_signals: NDArray[np.float64], + full_grid_signals: NDArray[np.float64], + trial_indices: NDArray[np.int64], + thresholds: _Thresholds, + config: DesignStressConfig, +) -> list[dict[str, Any]]: + probe, residual_scale, maximum_inner_product = _out_of_span_probe( + full_grid_signals + ) + generator = probe[trial_indices] + false_calls = 0 + reasons: dict[str, int] = {} + pure_call_counts = {name: 0 for name in DESIGN_CANDIDATE_NAMES} + for replicate in range(config.evaluation_replicates): + scores = _simulate_scores( + trial_signals, + generator, + config=config, + rng=_rng( + config.evaluation_seed, + design_index, + budget, + 3, + replicate, + ), + ) + call, reason = _call(scores, thresholds) + false_calls += int(call is not None) + if call is not None: + pure_call_counts[DESIGN_CANDIDATE_NAMES[call]] += 1 + reasons[reason] = reasons.get(reason, 0) + 1 + lower, upper = _wilson_interval( + false_calls, + config.evaluation_replicates, + config.confidence_level, + ) + return [ + { + "design": design, + "budget_schedule": _budget_schedule(factor), + "budget_factor": factor, + "budget": budget, + "probe": "full_grid_orthogonalized_tanh_surprise", + "probe_definition": ( + "unit-SD residual of tanh(standardized surprise) after full-grid " + "OLS projection on intercept plus all six standardized candidates" + ), + "full_grid_prestandardization_residual_sd": residual_scale, + "full_grid_maximum_absolute_mean_inner_product": ( + maximum_inner_product + ), + "replicates": config.evaluation_replicates, + "false_pure_calls": false_calls, + "abstentions": config.evaluation_replicates - false_calls, + "false_pure_call_rate": false_calls / config.evaluation_replicates, + "wilson_lower": lower, + "wilson_upper": upper, + "pure_call_counts": dict(sorted(pure_call_counts.items())), + "abstention_reasons": dict(sorted(reasons.items())), + } + ] + + +def run_design_stress( + config: DesignStressConfig | None = None, + *, + allocation_overrides: Mapping[ + tuple[str, int], + NDArray[np.int64], + ] + | None = None, +) -> DesignStressResult: + """Run a bounded post-freeze pure/null/mixture sensitivity analysis.""" + + config = DesignStressConfig() if config is None else config + config.validate() + grid_rows, _, standardized = generate_transition_design_grid() + base_targets = _base_targets(grid_rows, standardized, config) + overrides = {} if allocation_overrides is None else dict(allocation_overrides) + used_overrides: set[tuple[str, int]] = set() + + threshold_rows: list[dict[str, Any]] = [] + calibration_rows: list[dict[str, Any]] = [] + pure_rows: list[dict[str, Any]] = [] + null_rows: list[dict[str, Any]] = [] + mixture_rows: list[dict[str, Any]] = [] + out_of_span_rows: list[dict[str, Any]] = [] + allocation_rows: list[dict[str, Any]] = [] + + schedules_per_design = len(config.fixed_budgets) + len(config.budget_factors) + for design_index, design in enumerate(STRESS_DESIGNS): + schedules = [ + ("fixed_budget", None, budget) + for budget in config.fixed_budgets + ] + [ + ( + "n_eff_factor", + factor, + int(math.ceil(factor * base_targets[design])), + ) + for factor in config.budget_factors + ] + budgets = [budget for _, _, budget in schedules] + if len(set(budgets)) != len(budgets): + raise ValueError( + f"stress schedules duplicate a budget for design {design}" + ) + for budget_schedule, factor, budget in schedules: + maximum_count = max( + 1, + int(math.ceil(config.max_point_fraction * budget)), + ) + key = (design, budget) + if key in overrides: + counts = _validated_override( + overrides[key], + design=design, + point_count=len(grid_rows), + budget=budget, + maximum_count=maximum_count, + ) + allocation_source = "frozen_override" + used_overrides.add(key) + else: + counts = _design_counts( + design, + grid_rows, + standardized, + budget, + config, + ) + allocation_source = ( + "deterministic_greedy_exchange" + if design == "maximin_optimized" + else "declared_baseline_constructor" + ) + indices = np.repeat(np.arange(counts.size), counts) + trial_signals = standardized[indices] + diagnostics = design_diagnostics( + standardized, + counts, + effect_size=config.effect_size, + noise_std=config.noise_std, + target_log_score_gap=config.target_log_score_gap, + ) + for point_index in np.flatnonzero(counts): + allocation_rows.append( + { + "design": design, + "budget_schedule": budget_schedule, + "budget_factor": factor, + "budget": budget, + "allocation_source": allocation_source, + "point_id": int(point_index), + "count": int(counts[point_index]), + } + ) + + thresholds = _calibrate_thresholds( + trial_signals, + config=config, + design_index=design_index, + budget=budget, + ) + threshold_rows.append( + { + "design": design, + "budget_schedule": budget_schedule, + "budget_factor": factor, + "budget": budget, + "allocation_source": allocation_source, + "base_n_eff_index_from_n60_geometry": base_targets[design], + "realized_minimum_pairwise_residual": ( + diagnostics.minimum_pairwise_residual_variance + ), + "realized_population_n_eff_index": ( + diagnostics.trials_for_expected_log_score_gap_target + ), + "pure_over_null_threshold": thresholds.pure_over_null, + "winner_over_runner_threshold": thresholds.winner_over_runner, + "flexible_over_pure_threshold": thresholds.flexible_over_pure, + } + ) + calibration_rows.extend( + _calibration_audit_rows( + design, + design_index, + factor, + budget, + trial_signals, + thresholds, + config, + ) + ) + pure, null, mixture = _evaluation_rows( + design, + design_index, + factor, + budget, + trial_signals, + standardized, + indices, + thresholds, + config, + ) + pure_rows.extend(pure) + null_rows.extend(null) + mixture_rows.extend(mixture) + out_of_span_rows.extend( + _out_of_span_rows( + design, + design_index, + factor, + budget, + trial_signals, + standardized, + indices, + thresholds, + config, + ) + ) + + unused_overrides = set(overrides) - used_overrides + if unused_overrides: + raise ValueError( + "allocation overrides did not match a stress budget: " + f"{sorted(unused_overrides)}" + ) + + minimum_pure_lower = min(float(row["wilson_lower"]) for row in pure_rows) + maximum_null_upper = max(float(row["wilson_upper"]) for row in null_rows) + maximum_mixture_upper = max( + float(row["wilson_upper"]) for row in mixture_rows + ) + maximum_out_of_span_upper = max( + float(row["wilson_upper"]) for row in out_of_span_rows + ) + summary = { + "schema_version": 1, + "experiment": "post_freeze_design_abstention_sensitivity", + "config": asdict(config), + "candidate_names": list(DESIGN_CANDIDATE_NAMES), + "design_names": list(STRESS_DESIGNS), + "base_n_eff_indices_from_n60_geometry": base_targets, + "minimum_matched_pure_wilson_lower": minimum_pure_lower, + "maximum_null_false_pure_wilson_upper": maximum_null_upper, + "maximum_mixture_false_pure_wilson_upper": maximum_mixture_upper, + "maximum_out_of_span_false_pure_wilson_upper": maximum_out_of_span_upper, + "threshold_rule": { + "pure_signal": ( + "best pure minus intercept-only null exceeds a familywise " + "null-calibrated upper threshold" + ), + "ambiguity": ( + "best pure minus runner exceeds a separately null-calibrated " + "upper threshold" + ), + "adequacy": ( + "all-six train-only-selected ridge model does not beat the " + "best pure by more than the worst-candidate pure-calibrated " + "upper threshold" + ), + }, + "scope": ( + "This immutable post-freeze sensitivity probes finite-training " + "plug-in variability, a no-signal null, standardized equal " + "50/50 in-span mixtures, and one residualized nonlinear probe. " + "It does not establish robustness to arbitrary out-of-span " + "biology, other nonlinear mixtures, serial " + "dependence, sensor dynamics, subject hierarchy, or sequential " + "protocol feasibility. A mixture pure call is counted as false " + "even when it names a constituent." + ), + "technical_gates": { + "calibration_and_evaluation_seeds_disjoint": len( + { + config.threshold_seed, + config.calibration_audit_seed, + config.evaluation_seed, + } + ) + == 3, + "all_fifteen_mixtures_per_design_budget": len(mixture_rows) + == len(STRESS_DESIGNS) * schedules_per_design * 15, + "one_out_of_span_probe_per_design_budget": len(out_of_span_rows) + == len(STRESS_DESIGNS) * schedules_per_design, + "all_thresholds_finite": all( + np.isfinite( + [ + row["pure_over_null_threshold"], + row["winner_over_runner_threshold"], + row["flexible_over_pure_threshold"], + ] + ).all() + for row in threshold_rows + ), + }, + } + return DesignStressResult( + summary=summary, + thresholds=tuple(threshold_rows), + calibration=tuple(calibration_rows), + pure_recovery=tuple(pure_rows), + null_evaluation=tuple(null_rows), + mixture_evaluation=tuple(mixture_rows), + out_of_span_evaluation=tuple(out_of_span_rows), + allocations=tuple(allocation_rows), + ) diff --git a/src/bayesian_ach/design_stress_cli.py b/src/bayesian_ach/design_stress_cli.py new file mode 100644 index 0000000..30beac0 --- /dev/null +++ b/src/bayesian_ach/design_stress_cli.py @@ -0,0 +1,453 @@ +"""CLI for the versioned post-freeze design stress artifact.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import re +import subprocess +from collections.abc import Mapping, Sequence +from dataclasses import asdict +from pathlib import Path +from typing import Any + +import numpy as np +from numpy.typing import NDArray + +from bayesian_ach.design_grid import ( + coupled_novelty_design, + generate_transition_design_grid, + uniform_factorial_design, +) +from bayesian_ach.design_optimizer import optimize_maximin_design +from bayesian_ach.design_stress import ( + STRESS_DESIGNS, + DesignStressConfig, + run_design_stress, +) +from bayesian_ach.io import write_json, write_rows_csv + +_REPOSITORY = "IPS-Stuttgart/Bayesian-ACh" +_LOCKED_ALLOCATION_SEED = 7 +_LOCKED_MAXIMIN_MAX_POINT_FRACTION = 0.15 +_SHA_PATTERN = re.compile(r"^[0-9a-f]{40}$") +_PAYLOAD_FILENAMES = ( + "summary.json", + "thresholds.csv", + "calibration_audit.csv", + "pure_recovery.csv", + "null_evaluation.csv", + "mixture_evaluation.csv", + "out_of_span_evaluation.csv", + "allocations.csv", +) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _canonical_digest(value: Mapping[str, Any]) -> str: + payload = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _git_provenance(repo_root: Path, expected_sha: str) -> None: + if _SHA_PATTERN.fullmatch(expected_sha) is None: + raise ValueError("--code-sha must be a lowercase 40-character commit SHA") + head = subprocess.run( + ["git", "-C", str(repo_root), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if head != expected_sha: + raise RuntimeError(f"checked-out HEAD {head} does not match --code-sha {expected_sha}") + status = subprocess.run( + ["git", "-C", str(repo_root), "status", "--porcelain", "--untracked-files=all"], + check=True, + capture_output=True, + text=True, + ).stdout + if status: + raise RuntimeError("claim-bearing stress artifacts require a clean Git worktree") + + +def _csv_safe_rows( + rows: Sequence[Mapping[str, Any]], +) -> list[dict[str, Any]]: + safe: list[dict[str, Any]] = [] + for row in rows: + safe.append( + { + key: json.dumps(value, sort_keys=True, separators=(",", ":")) + if isinstance(value, (dict, list, tuple)) + else value + for key, value in row.items() + } + ) + return safe + + +def _verify_checksum_table(directory: Path) -> dict[str, str]: + checksum_path = directory / "SHA256SUMS.csv" + if not checksum_path.is_file(): + raise ValueError(f"missing certificate checksum table: {checksum_path}") + locked: dict[str, str] = {} + with checksum_path.open(newline="", encoding="utf-8") as handle: + for row in csv.DictReader(handle): + name = str(row["file"]) + if name in locked: + raise ValueError(f"duplicate checksum entry: {name}") + locked[name] = str(row["sha256"]) + for name, expected in locked.items(): + candidate = directory / name + if not candidate.is_file() or _sha256(candidate) != expected: + raise ValueError(f"certificate package checksum mismatch: {name}") + return locked + + +def _load_certified_allocation( + allocation_path: Path, +) -> tuple[tuple[str, int], NDArray[np.int64], dict[str, Any]]: + allocation_path = allocation_path.resolve() + directory = allocation_path.parent + locked = _verify_checksum_table(directory) + summary_path = directory / "certificate_summary.json" + for required in (allocation_path.name, summary_path.name): + if required not in locked: + raise ValueError(f"certificate checksum table does not bind {required}") + summary = json.loads(summary_path.read_text(encoding="utf-8")) + config = summary.get("config", {}) + if ( + summary.get("schema_version") != 1 + or summary.get("experiment") != "certified_finite_grid_maximin_allocation" + or summary.get("certified") is not True + or summary.get("direct_geometry_matches_lower_bound") is not True + or config.get("integer") is not True + ): + raise ValueError("allocation override is not a certified integer design artifact") + budget = int(config["budget"]) + point_count = int(summary["grid_point_count"]) + counts = np.zeros(point_count, dtype=np.int64) + with allocation_path.open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + if not rows: + raise ValueError("certified allocation table is empty") + for row in rows: + point = int(row["point_id"]) + value = float(row["allocation"]) + rounded = int(round(value)) + if point < 0 or point >= point_count or abs(value - rounded) > 1.0e-9: + raise ValueError("certified integer allocation contains an invalid row") + if counts[point] != 0: + raise ValueError(f"duplicate certified allocation point: {point}") + counts[point] = rounded + if int(np.sum(counts)) != budget: + raise ValueError("certified allocation does not sum to its locked budget") + provenance = { + "certificate_code_sha": summary["code_sha"], + "certificate_budget": budget, + "certificate_lower_bound": summary["lower_bound"], + "certificate_upper_bound": summary["upper_bound"], + "certificate_absolute_gap": summary["absolute_gap"], + "certificate_allocation_path": allocation_path.name, + "certificate_allocation_sha256": locked[allocation_path.name], + "certificate_summary_sha256": locked[summary_path.name], + "certificate_checksums_sha256": _sha256(directory / "SHA256SUMS.csv"), + } + return ("maximin_optimized", budget), counts, provenance + + +def _load_locked_design_allocation( + path: Path, + *, + expected_sha256: str, + source_code_sha: str, + allocation_seed: int, +) -> tuple[dict[tuple[str, int], NDArray[np.int64]], dict[str, Any]]: + path = path.resolve() + if re.fullmatch(r"[0-9a-f]{64}", expected_sha256) is None: + raise ValueError("locked allocation SHA-256 must contain 64 lowercase hex characters") + if _SHA_PATTERN.fullmatch(source_code_sha) is None: + raise ValueError( + "locked allocation source code SHA must contain 40 lowercase hex characters" + ) + if allocation_seed != _LOCKED_ALLOCATION_SEED: + raise ValueError("locked allocation seed does not match the frozen constructor") + observed_sha256 = _sha256(path) + if observed_sha256 != expected_sha256: + raise ValueError("locked design allocation SHA-256 mismatch") + grid_rows, _, standardized = generate_transition_design_grid() + point_count = len(grid_rows) + counts: dict[str, NDArray[np.int64]] = { + design: np.zeros(point_count, dtype=np.int64) + for design in STRESS_DESIGNS + } + seen: set[tuple[str, int]] = set() + seeds: set[str] = set() + with path.open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + if not rows: + raise ValueError("locked design allocation is empty") + for row in rows: + design = str(row["design"]) + if design not in counts: + raise ValueError(f"unknown locked design name: {design}") + point = int(row["point_id"]) + raw_count = row.get("count", row.get("allocation")) + if raw_count is None: + raise ValueError("locked allocation requires a count or allocation column") + value = float(raw_count) + rounded = int(round(value)) + key = (design, point) + if ( + point < 0 + or point >= point_count + or key in seen + or rounded <= 0 + or not math.isclose(value, rounded, abs_tol=1.0e-9) + ): + raise ValueError("locked design allocation has an invalid row") + counts[design][point] = rounded + seen.add(key) + if row.get("seed") not in {None, ""}: + seeds.add(str(row["seed"])) + budgets = {design: int(np.sum(value)) for design, value in counts.items()} + if any(budget <= 0 for budget in budgets.values()): + raise ValueError("locked allocation must contain every declared design") + if seeds and seeds != {str(allocation_seed)}: + raise ValueError( + "allocation-file seed does not match the explicit frozen seed metadata" + ) + expected_counts: dict[str, NDArray[np.int64]] = { + "coupled_novelty": coupled_novelty_design( + grid_rows, + budgets["coupled_novelty"], + ), + "uniform_factorial": uniform_factorial_design( + point_count, + budgets["uniform_factorial"], + seed=_LOCKED_ALLOCATION_SEED, + ), + "maximin_optimized": optimize_maximin_design( + standardized, + budgets["maximin_optimized"], + max_point_fraction=_LOCKED_MAXIMIN_MAX_POINT_FRACTION, + ).counts, + } + for design, expected in expected_counts.items(): + if not np.array_equal(counts[design], expected): + raise ValueError( + f"locked {design} allocation does not match its frozen constructor" + ) + overrides = { + (design, budgets[design]): value + for design, value in counts.items() + } + provenance = { + "kind": "chronologically_locked_primary_design_allocation", + "source_repository": _REPOSITORY, + "source_code_sha": source_code_sha, + "allocation_file": path.name, + "allocation_sha256": observed_sha256, + "allocation_bytes": path.stat().st_size, + "design_budgets": budgets, + "allocation_seed": allocation_seed, + "allocation_seed_source": "explicit_cli_metadata", + "allocation_file_seed_field_present": bool(seeds), + "construction_contract": { + "allocation_seed": allocation_seed, + "maximin_max_point_fraction": _LOCKED_MAXIMIN_MAX_POINT_FRACTION, + "maximin_maximum_count_by_budget": { + str(budgets["maximin_optimized"]): int( + math.ceil( + _LOCKED_MAXIMIN_MAX_POINT_FRACTION + * budgets["maximin_optimized"] + ) + ) + }, + "comparator_cap_semantics": ( + "the maximin cap does not apply to the deterministic " + "coupled-novelty or uniform-factorial constructors" + ), + "all_three_allocations_reconstructed": True, + }, + } + return overrides, provenance + + +def _write_artifact( + output: Path, + *, + result: Any, + config: DesignStressConfig, + code_sha: str, + input_provenance: Sequence[Mapping[str, Any]], +) -> None: + output.mkdir(parents=True, exist_ok=False) + write_json(output / "summary.json", result.summary) + write_rows_csv(output / "thresholds.csv", _csv_safe_rows(result.thresholds)) + write_rows_csv( + output / "calibration_audit.csv", + _csv_safe_rows(result.calibration), + ) + write_rows_csv( + output / "pure_recovery.csv", + _csv_safe_rows(result.pure_recovery), + ) + write_rows_csv( + output / "null_evaluation.csv", + _csv_safe_rows(result.null_evaluation), + ) + write_rows_csv( + output / "mixture_evaluation.csv", + _csv_safe_rows(result.mixture_evaluation), + ) + write_rows_csv( + output / "out_of_span_evaluation.csv", + _csv_safe_rows(result.out_of_span_evaluation), + ) + write_rows_csv(output / "allocations.csv", _csv_safe_rows(result.allocations)) + + files = [ + { + "path": name, + "bytes": (output / name).stat().st_size, + "sha256": _sha256(output / name), + } + for name in _PAYLOAD_FILENAMES + ] + manifest = { + "schema_version": 1, + "artifact": "post_freeze_design_abstention_sensitivity", + "repository": _REPOSITORY, + "producer_commit": code_sha, + "producer_git_dirty": False, + "configuration_sha256": _canonical_digest(asdict(config)), + "inputs": list(input_provenance), + "files": files, + } + write_json(output / "artifact_manifest.json", manifest) + checksum_rows = files + [ + { + "path": "artifact_manifest.json", + "bytes": (output / "artifact_manifest.json").stat().st_size, + "sha256": _sha256(output / "artifact_manifest.json"), + } + ] + write_rows_csv(output / "SHA256SUMS.csv", checksum_rows) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="bayesian-ach-design-stress", + description="Freeze post-baseline pure/null/mixture abstention sensitivity.", + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--repo-root", type=Path, default=Path.cwd()) + parser.add_argument("--code-sha", required=True) + parser.add_argument("--fixed-budgets", nargs="+", type=int, default=(60,)) + parser.add_argument("--budget-factors", nargs="*", type=float, default=()) + parser.add_argument("--calibration-replicates", type=int, default=100) + parser.add_argument("--calibration-audit-replicates", type=int, default=100) + parser.add_argument("--evaluation-replicates", type=int, default=200) + parser.add_argument("--threshold-seed", type=int, default=104729) + parser.add_argument("--calibration-audit-seed", type=int, default=130363) + parser.add_argument("--evaluation-seed", type=int, default=155921) + parser.add_argument("--locked-allocation", type=Path) + parser.add_argument("--locked-allocation-sha256") + parser.add_argument("--locked-design-code-sha") + parser.add_argument("--locked-allocation-seed", type=int) + parser.add_argument( + "--certified-allocation", + action="append", + type=Path, + default=[], + help="Checksum-bound certified integer allocation CSV; may be repeated.", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) + _git_provenance(args.repo_root.resolve(), args.code_sha) + config = DesignStressConfig( + fixed_budgets=tuple(args.fixed_budgets), + budget_factors=tuple(args.budget_factors), + calibration_replicates=args.calibration_replicates, + calibration_audit_replicates=args.calibration_audit_replicates, + evaluation_replicates=args.evaluation_replicates, + threshold_seed=args.threshold_seed, + calibration_audit_seed=args.calibration_audit_seed, + evaluation_seed=args.evaluation_seed, + ) + overrides: dict[tuple[str, int], NDArray[np.int64]] = {} + provenance: list[dict[str, Any]] = [] + locked_arguments = ( + args.locked_allocation, + args.locked_allocation_sha256, + args.locked_design_code_sha, + args.locked_allocation_seed, + ) + if any(value is not None for value in locked_arguments): + if not all(value is not None for value in locked_arguments): + raise ValueError( + "locked allocation path, SHA-256, source code SHA, and seed " + "are jointly required" + ) + locked_overrides, item = _load_locked_design_allocation( + args.locked_allocation, + expected_sha256=args.locked_allocation_sha256, + source_code_sha=args.locked_design_code_sha, + allocation_seed=args.locked_allocation_seed, + ) + overrides.update(locked_overrides) + provenance.append(item) + for path in args.certified_allocation: + key, counts, item = _load_certified_allocation(path) + if key in overrides: + raise ValueError(f"duplicate certified allocation for {key}") + overrides[key] = counts + provenance.append(item) + result = run_design_stress(config, allocation_overrides=overrides) + locked_items = [ + item + for item in provenance + if item.get("kind") == "chronologically_locked_primary_design_allocation" + ] + if locked_items: + result.summary["primary_schedule_provenance"] = locked_items[0] + result.summary["technical_gates"][ + "chronologically_locked_primary_allocation_verified" + ] = True + result.summary["primary_schedule_chronology"] = ( + "The equal-N60 allocation and its source commit were frozen in the " + "accepted five-seed benchmark before certificate and stress results; " + "no later certified schedule replaces this primary allocation." + ) + _write_artifact( + args.output.resolve(), + result=result, + config=config, + code_sha=args.code_sha, + input_provenance=provenance, + ) + print(json.dumps(result.summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/tests/test_design_stress.py b/tests/test_design_stress.py new file mode 100644 index 0000000..6fe6a02 --- /dev/null +++ b/tests/test_design_stress.py @@ -0,0 +1,492 @@ +import csv +import hashlib +import json +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest + +import bayesian_ach.design_stress as stress +import bayesian_ach.design_stress_cli as stress_cli +from bayesian_ach.design_stress import DesignStressConfig, run_design_stress +from bayesian_ach.design_stress_cli import ( + _load_certified_allocation, + _load_locked_design_allocation, + _write_artifact, +) + + +def test_gate_requires_signal_separation_and_pure_adequacy() -> None: + thresholds = stress._Thresholds( + pure_over_null=1.0, + winner_over_runner=1.0, + flexible_over_pure=1.0, + ) + decisive = stress._Scores( + winner=2, + runner=1, + best_pure_score=5.0, + runner_score=3.0, + null_score=0.0, + flexible_score=5.5, + ridge_lambda=0.1, + ) + assert stress._call(decisive, thresholds) == (2, "pure_call") + + no_signal = replace(decisive, best_pure_score=1.0) + ambiguous = replace(decisive, runner_score=4.0) + misspecified = replace(decisive, flexible_score=6.01) + assert stress._call(no_signal, thresholds) == (None, "null_not_rejected") + assert stress._call(ambiguous, thresholds) == (None, "pure_ambiguity") + assert stress._call(misspecified, thresholds) == (None, "flexible_model_better") + + +def test_conformal_threshold_and_wilson_interval_are_conservative() -> None: + assert stress._upper_conformal_quantile(list(range(20)), alpha=0.05) == 19.0 + assert stress._upper_conformal_quantile(list(range(10)), alpha=0.05) == np.inf + lower, upper = stress._wilson_interval(0, 100, 0.95) + assert lower == pytest.approx(0.0) + assert 0.03 < upper < 0.04 + + +def test_stress_config_rejects_nonfinite_and_duplicate_settings() -> None: + with pytest.raises(ValueError, match="finite, positive, and unique"): + DesignStressConfig(budget_factors=(1.0, 1.0)).validate() + with pytest.raises(ValueError, match="target_log_score_gap"): + DesignStressConfig(target_log_score_gap=np.inf).validate() + with pytest.raises(ValueError, match="ridge_lambdas"): + DesignStressConfig(ridge_lambdas=(0.0, np.nan)).validate() + + +def test_threshold_calibration_is_finite_with_minimum_replicates() -> None: + rng = np.random.default_rng(17) + signals = rng.normal(size=(36, 6)) + signals = (signals - signals.mean(axis=0)) / signals.std(axis=0) + config = DesignStressConfig( + fixed_budgets=(), + budget_factors=(1.0,), + calibration_replicates=20, + calibration_audit_replicates=20, + evaluation_replicates=20, + inner_folds=2, + ) + thresholds = stress._calibrate_thresholds( + signals, + config=config, + design_index=0, + budget=36, + ) + assert np.isfinite( + [ + thresholds.pure_over_null, + thresholds.winner_over_runner, + thresholds.flexible_over_pure, + ] + ).all() + assert thresholds.flexible_over_pure >= 0.0 + + +def test_nonlinear_probe_is_outside_full_grid_linear_span() -> None: + _, _, signals = stress.generate_transition_design_grid() + probe, scale, maximum_inner_product = stress._out_of_span_probe(signals) + assert probe.shape == (signals.shape[0],) + assert scale > 0.0 + assert np.std(probe) == pytest.approx(1.0) + assert maximum_inner_product < 1.0e-12 + + +def _mock_small_run(monkeypatch: pytest.MonkeyPatch) -> None: + rng = np.random.default_rng(23) + signals = rng.normal(size=(20, 6)) + signals = (signals - signals.mean(axis=0)) / signals.std(axis=0) + rows = tuple({"point_id": index} for index in range(20)) + monkeypatch.setattr( + stress, + "generate_transition_design_grid", + lambda: (rows, signals.copy(), signals.copy()), + ) + monkeypatch.setattr( + stress, + "_base_targets", + lambda *_: {name: 8 for name in stress.STRESS_DESIGNS}, + ) + + def counts(*args: object) -> np.ndarray: + result = np.zeros(20, dtype=np.int64) + result[: int(args[3])] = 1 + return result + + monkeypatch.setattr(stress, "_design_counts", counts) + monkeypatch.setattr( + stress, + "_calibrate_thresholds", + lambda *args, **kwargs: stress._Thresholds(1.0, 1.0, 1.0), + ) + monkeypatch.setattr( + stress, + "_calibration_audit_rows", + lambda design, _index, factor, budget, *_args: [ + { + "design": design, + "budget_factor": factor, + "budget": budget, + "scenario": "null", + "generator": "null", + "replicates": 20, + "correct_pure_calls": 0, + "wrong_pure_calls": 0, + "abstentions": 20, + "rate": 0.0, + "wilson_lower": 0.0, + "wilson_upper": 0.16, + } + ], + ) + + def evaluations( + design: str, + _index: int, + factor: float | None, + budget: int, + *_args: object, + ) -> tuple[list[dict[str, object]], list[dict[str, object]], list[dict[str, object]]]: + pure = [ + { + "design": design, + "budget_factor": factor, + "budget": budget, + "generator": name, + "replicates": 20, + "correct_pure_calls": 10, + "wrong_pure_calls": 0, + "abstentions": 10, + "correct_call_rate": 0.5, + "wilson_lower": 0.30, + "wilson_upper": 0.70, + "raw_closed_set_winner_rate": 0.7, + "abstention_reasons": {"pure_call": 10, "pure_ambiguity": 10}, + } + for name in stress.DESIGN_CANDIDATE_NAMES + ] + null = [ + { + "design": design, + "budget_factor": factor, + "budget": budget, + "replicates": 20, + "false_pure_calls": 1, + "abstentions": 19, + "false_pure_call_rate": 0.05, + "wilson_lower": 0.01, + "wilson_upper": 0.24, + "abstention_reasons": {"null_not_rejected": 19, "pure_call": 1}, + } + ] + mixture = [ + { + "design": design, + "budget_factor": factor, + "budget": budget, + "first_candidate": f"first-{index}", + "second_candidate": f"second-{index}", + "mixture_definition": "test", + "replicates": 20, + "false_pure_calls": 2, + "constituent_pure_calls": 2, + "abstentions": 18, + "false_pure_call_rate": 0.1, + "wilson_lower": 0.03, + "wilson_upper": 0.30, + "abstention_reasons": {"flexible_model_better": 18, "pure_call": 2}, + } + for index in range(15) + ] + return pure, null, mixture + + monkeypatch.setattr(stress, "_evaluation_rows", evaluations) + monkeypatch.setattr( + stress, + "_out_of_span_rows", + lambda design, _index, factor, budget, *_args: [ + { + "design": design, + "budget_factor": factor, + "budget": budget, + "probe": "test", + "probe_definition": "test", + "full_grid_prestandardization_residual_sd": 0.1, + "full_grid_maximum_absolute_mean_inner_product": 0.0, + "replicates": 20, + "false_pure_calls": 1, + "abstentions": 19, + "false_pure_call_rate": 0.05, + "wilson_lower": 0.01, + "wilson_upper": 0.24, + "pure_call_counts": {}, + "abstention_reasons": {"null_not_rejected": 19, "pure_call": 1}, + } + ], + ) + + +def test_stress_orchestration_covers_every_design_and_pair( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _mock_small_run(monkeypatch) + result = run_design_stress( + DesignStressConfig( + fixed_budgets=(), + budget_factors=(1.0,), + calibration_replicates=20, + calibration_audit_replicates=20, + evaluation_replicates=20, + ) + ) + assert len(result.thresholds) == 3 + assert len(result.pure_recovery) == 18 + assert len(result.null_evaluation) == 3 + assert len(result.mixture_evaluation) == 45 + assert result.summary["technical_gates"] == { + "calibration_and_evaluation_seeds_disjoint": True, + "all_fifteen_mixtures_per_design_budget": True, + "one_out_of_span_probe_per_design_budget": True, + "all_thresholds_finite": True, + } + + +def test_optimizer_cap_does_not_apply_to_declared_comparators() -> None: + counts = np.zeros(20, dtype=np.int64) + counts[:5] = 12 + accepted = stress._validated_override( + counts, + design="coupled_novelty", + point_count=20, + budget=60, + maximum_count=9, + ) + np.testing.assert_array_equal(accepted, counts) + with pytest.raises(ValueError, match="maximin allocation override"): + stress._validated_override( + counts, + design="maximin_optimized", + point_count=20, + budget=60, + maximum_count=9, + ) + + +def test_unused_certified_override_is_rejected( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _mock_small_run(monkeypatch) + counts = np.zeros(20, dtype=np.int64) + counts[:9] = 1 + with pytest.raises(ValueError, match="did not match"): + run_design_stress( + DesignStressConfig( + fixed_budgets=(), + budget_factors=(1.0,), + calibration_replicates=20, + calibration_audit_replicates=20, + evaluation_replicates=20, + ), + allocation_overrides={("maximin_optimized", 9): counts}, + ) + + +def test_locked_primary_allocation_is_hash_bound( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + expected_counts = np.array([1, 0, 1, 0], dtype=np.int64) + grid_rows = tuple({"point_id": index} for index in range(4)) + standardized = np.arange(24, dtype=float).reshape(4, 6) + monkeypatch.setattr( + stress_cli, + "generate_transition_design_grid", + lambda: (grid_rows, standardized.copy(), standardized.copy()), + ) + monkeypatch.setattr( + stress_cli, + "coupled_novelty_design", + lambda *_args, **_kwargs: expected_counts.copy(), + ) + monkeypatch.setattr( + stress_cli, + "uniform_factorial_design", + lambda *_args, **_kwargs: expected_counts.copy(), + ) + monkeypatch.setattr( + stress_cli, + "optimize_maximin_design", + lambda *_args, **_kwargs: SimpleNamespace(counts=expected_counts.copy()), + ) + path = tmp_path / "locked.csv" + rows = [ + {"design": design, "point_id": point, "count": 1} + for design in stress.STRESS_DESIGNS + for point in (0, 2) + ] + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter( + handle, + fieldnames=("design", "point_id", "count"), + ) + writer.writeheader() + writer.writerows(rows) + expected = hashlib.sha256(path.read_bytes()).hexdigest() + overrides, provenance = _load_locked_design_allocation( + path, + expected_sha256=expected, + source_code_sha="b" * 40, + allocation_seed=7, + ) + assert set(overrides) == {(design, 2) for design in stress.STRESS_DESIGNS} + assert provenance["allocation_sha256"] == expected + assert provenance["allocation_seed"] == 7 + assert provenance["allocation_seed_source"] == "explicit_cli_metadata" + assert provenance["allocation_file_seed_field_present"] is False + assert provenance["construction_contract"] == { + "allocation_seed": 7, + "maximin_max_point_fraction": 0.15, + "maximin_maximum_count_by_budget": {"2": 1}, + "comparator_cap_semantics": ( + "the maximin cap does not apply to the deterministic " + "coupled-novelty or uniform-factorial constructors" + ), + "all_three_allocations_reconstructed": True, + } + + with pytest.raises(ValueError, match="SHA-256 mismatch"): + _load_locked_design_allocation( + path, + expected_sha256="0" * 64, + source_code_sha="b" * 40, + allocation_seed=7, + ) + with pytest.raises(ValueError, match="seed does not match"): + _load_locked_design_allocation( + path, + expected_sha256=expected, + source_code_sha="b" * 40, + allocation_seed=8, + ) + + altered_rows = [ + {"design": "coupled_novelty", "point_id": 0, "count": 2}, + *[ + {"design": design, "point_id": point, "count": 1} + for design in ("uniform_factorial", "maximin_optimized") + for point in (0, 2) + ], + ] + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter( + handle, + fieldnames=("design", "point_id", "count"), + ) + writer.writeheader() + writer.writerows(altered_rows) + altered_sha = hashlib.sha256(path.read_bytes()).hexdigest() + with pytest.raises(ValueError, match="coupled_novelty.*frozen constructor"): + _load_locked_design_allocation( + path, + expected_sha256=altered_sha, + source_code_sha="b" * 40, + allocation_seed=7, + ) + + +def _write_certificate_package(directory: Path) -> Path: + directory.mkdir() + allocation = directory / "certified_allocation.csv" + allocation.write_text("point_id,allocation\n0,2\n2,1\n", encoding="utf-8") + summary = directory / "certificate_summary.json" + summary.write_text( + json.dumps( + { + "schema_version": 1, + "experiment": "certified_finite_grid_maximin_allocation", + "code_sha": "1" * 40, + "grid_point_count": 4, + "certified": True, + "direct_geometry_matches_lower_bound": True, + "lower_bound": 0.2, + "upper_bound": 0.2, + "absolute_gap": 0.0, + "config": {"budget": 3, "integer": True}, + }, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + with (directory / "SHA256SUMS.csv").open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=("file", "bytes", "sha256")) + writer.writeheader() + for path in (allocation, summary): + writer.writerow( + { + "file": path.name, + "bytes": path.stat().st_size, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + ) + return allocation + + +def test_certified_override_loader_checks_package_and_tampering( + tmp_path: Path, +) -> None: + allocation = _write_certificate_package(tmp_path / "certificate") + key, counts, provenance = _load_certified_allocation(allocation) + assert key == ("maximin_optimized", 3) + np.testing.assert_array_equal(counts, np.array([2, 0, 1, 0])) + assert provenance["certificate_budget"] == 3 + + allocation.write_text("point_id,allocation\n0,1\n2,2\n", encoding="utf-8") + with pytest.raises(ValueError, match="checksum mismatch"): + _load_certified_allocation(allocation) + + +def test_artifact_writer_hashes_every_payload(tmp_path: Path) -> None: + one_row = ({"name": "row", "nested": {"b": 2, "a": 1}},) + result = SimpleNamespace( + summary={"schema_version": 1, "technical_gates": {"ok": True}}, + thresholds=one_row, + calibration=one_row, + pure_recovery=one_row, + null_evaluation=one_row, + mixture_evaluation=one_row, + out_of_span_evaluation=one_row, + allocations=one_row, + ) + output = tmp_path / "artifact" + config = DesignStressConfig() + _write_artifact( + output, + result=result, + config=config, + code_sha="a" * 40, + input_provenance=(), + ) + manifest = json.loads((output / "artifact_manifest.json").read_text()) + assert manifest["producer_commit"] == "a" * 40 + assert manifest["producer_git_dirty"] is False + assert len(manifest["files"]) == 8 + checksums = list(csv.DictReader((output / "SHA256SUMS.csv").open())) + assert {row["path"] for row in checksums} == { + "summary.json", + "thresholds.csv", + "calibration_audit.csv", + "pure_recovery.csv", + "null_evaluation.csv", + "mixture_evaluation.csv", + "out_of_span_evaluation.csv", + "allocations.csv", + "artifact_manifest.json", + } + for row in checksums: + assert hashlib.sha256((output / row["path"]).read_bytes()).hexdigest() == row["sha256"] diff --git a/tests/test_design_stress_geometry.py b/tests/test_design_stress_geometry.py new file mode 100644 index 0000000..f15eb66 --- /dev/null +++ b/tests/test_design_stress_geometry.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + + +def test_geometry_diagnostic_recomputes_and_detects_tamper(tmp_path: Path) -> None: + root = Path(__file__).resolve().parents[1] + source = root / "results/design-open-set-stress-n60" + analyzer = root / "scripts/analyze_design_stress_geometry.py" + verifier = root / "scripts/verify_design_stress_geometry.py" + output = tmp_path / "geometry" + command = [ + sys.executable, + str(analyzer), + "--artifact", + str(source), + "--output", + str(output), + "--repo-root", + str(root), + "--code-sha", + "1" * 40, + "--recompute-only", + ] + subprocess.run(command, check=True, capture_output=True, text=True) + verify = [ + sys.executable, + str(verifier), + str(output), + "--source-artifact", + str(source), + "--repo-root", + str(root), + ] + subprocess.run(verify, check=True, capture_output=True, text=True) + geometry = output / "maximin_mixture_geometry.csv" + geometry.write_text( + geometry.read_text(encoding="utf-8").replace("0.8,", "0.7,", 1), + encoding="utf-8", + ) + failed = subprocess.run(verify, capture_output=True, text=True) + assert failed.returncode != 0 + assert "payload mismatch" in failed.stderr