From fa14dec1c978a89254122634ee36149e12e75b52 Mon Sep 17 00:00:00 2001 From: Florian Pfaff <6773539+FlorianPfaff@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:24:10 +0800 Subject: [PATCH 01/15] Add mixture-aware design diagnostic --- src/bayesian_ach/design_mixture_diagnostic.py | 820 ++++++++++++++++++ 1 file changed, 820 insertions(+) create mode 100644 src/bayesian_ach/design_mixture_diagnostic.py diff --git a/src/bayesian_ach/design_mixture_diagnostic.py b/src/bayesian_ach/design_mixture_diagnostic.py new file mode 100644 index 0000000..23f1075 --- /dev/null +++ b/src/bayesian_ach/design_mixture_diagnostic.py @@ -0,0 +1,820 @@ +"""Prospectively locked post-failure mixture-aware design diagnostic.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from itertools import combinations +from typing import Any + +import numpy as np +from numpy.typing import NDArray +from scipy.optimize import nnls + +from bayesian_ach.design_grid import ( + DESIGN_CANDIDATE_NAMES, + generate_transition_design_grid, +) +from bayesian_ach.design_stress import ( + _out_of_span_probe, + _upper_conformal_quantile, + _wilson_interval, +) + +_PAIR_INDICES = tuple(combinations(range(len(DESIGN_CANDIDATE_NAMES)), 2)) + + +@dataclass(frozen=True, slots=True) +class MixtureDiagnosticConfig: + """Settings fixed before the post-failure evaluation is generated.""" + + calibration_replicates: int = 200 + calibration_audit_replicates: int = 200 + evaluation_replicates: int = 200 + folds: int = 3 + effect_size: float = 1.0 + noise_std: float = 1.0 + alpha: float = 0.05 + confidence_level: float = 0.95 + minimum_pure_power_wilson_lower: float = 0.70 + threshold_seed: int = 196613 + calibration_audit_seed: int = 262147 + evaluation_seed: int = 324949 + design: str = "maximin_optimized" + budget: int = 60 + + def validate(self) -> None: + if min( + self.calibration_replicates, + self.calibration_audit_replicates, + self.evaluation_replicates, + ) < 20: + raise ValueError("all replicate counts must be at least 20") + if self.folds < 2: + raise ValueError("folds must be at least two") + if ( + not math.isfinite(self.effect_size) + or not math.isfinite(self.noise_std) + or self.effect_size <= 0.0 + or self.noise_std <= 0.0 + ): + raise ValueError("effect size and noise standard deviation must be positive") + if not 0.0 < self.alpha < 0.5: + raise ValueError("alpha must lie in (0, 0.5)") + if not 0.0 < self.confidence_level < 1.0: + raise ValueError("confidence level must lie in (0, 1)") + if not 0.0 <= self.minimum_pure_power_wilson_lower <= 1.0: + raise ValueError("minimum pure-power lower bound must lie in [0, 1]") + if len( + { + self.threshold_seed, + self.calibration_audit_seed, + self.evaluation_seed, + } + ) != 3: + raise ValueError("calibration, audit, and evaluation seeds must be distinct") + if self.design != "maximin_optimized" or self.budget != 60: + raise ValueError("the locked diagnostic is restricted to maximin N=60") + + +@dataclass(frozen=True, slots=True) +class CrossFitScores: + """Cross-fitted held-out scores and pure-model residual diagnostics.""" + + pure_scores: NDArray[np.float64] + null_score: float + composite_score: float + composite_pair: tuple[int, int] + pure_residual_ratios: NDArray[np.float64] + + @property + def winner(self) -> int: + order = np.lexsort( + (np.arange(self.pure_scores.size), -self.pure_scores) + ) + return int(order[0]) + + @property + def runner(self) -> int: + order = np.lexsort( + (np.arange(self.pure_scores.size), -self.pure_scores) + ) + return int(order[1]) + + @property + def pure_over_null(self) -> float: + return float(self.pure_scores[self.winner] - self.null_score) + + @property + def winner_over_runner(self) -> float: + return float( + self.pure_scores[self.winner] - self.pure_scores[self.runner] + ) + + @property + def composite_over_winner(self) -> float: + return float(self.composite_score - self.pure_scores[self.winner]) + + +@dataclass(frozen=True, slots=True) +class CandidateThreshold: + """Candidate-specific composite and lack-of-fit thresholds.""" + + composite_over_pure: float + residual_ratio: float + + +@dataclass(frozen=True, slots=True) +class DiagnosticThresholds: + """Familywise null/ambiguity and candidate-specific adequacy thresholds.""" + + pure_over_null: float + winner_over_runner: float + candidates: tuple[CandidateThreshold, ...] + + +@dataclass(frozen=True, slots=True) +class MixtureDiagnosticResult: + """Tables produced by the locked post-failure diagnostic.""" + + summary: dict[str, Any] + thresholds: tuple[dict[str, Any], ...] + calibration_audit: tuple[dict[str, Any], ...] + pure_evaluation: tuple[dict[str, Any], ...] + null_evaluation: tuple[dict[str, Any], ...] + mixture_evaluation: tuple[dict[str, Any], ...] + out_of_span_evaluation: tuple[dict[str, Any], ...] + geometry: tuple[dict[str, Any], ...] + + +def _rng(seed: int, *keys: int) -> np.random.Generator: + return np.random.default_rng(np.random.SeedSequence([seed, *keys])) + + +def _fit_variance(residual: NDArray[np.float64]) -> float: + return max(float(np.mean(residual**2)), 1.0e-8) + + +def _gaussian_score(residual: NDArray[np.float64], variance: float) -> float: + return float( + np.sum( + -0.5 + * ( + math.log(2.0 * math.pi * variance) + + residual**2 / variance + ) + ) + ) + + +def _pure_fold( + predictor: NDArray[np.float64], + response: NDArray[np.float64], + train: NDArray[np.int64], + validation: NDArray[np.int64], +) -> tuple[float, float]: + train_design = np.column_stack((np.ones(train.size), predictor[train])) + coefficients, _, _, _ = np.linalg.lstsq( + train_design, + response[train], + rcond=None, + ) + train_residual = response[train] - train_design @ coefficients + variance = _fit_variance(train_residual) + prediction = coefficients[0] + coefficients[1] * predictor[validation] + validation_residual = response[validation] - prediction + return ( + _gaussian_score(validation_residual, variance), + float(np.sum(validation_residual**2 / variance)), + ) + + +def _null_fold( + response: NDArray[np.float64], + train: NDArray[np.int64], + validation: NDArray[np.int64], +) -> float: + mean = float(np.mean(response[train])) + variance = _fit_variance(response[train] - mean) + return _gaussian_score(response[validation] - mean, variance) + + +def _nonnegative_pair_fold( + predictors: NDArray[np.float64], + response: NDArray[np.float64], + train: NDArray[np.int64], + validation: NDArray[np.int64], +) -> float: + train_predictors = predictors[train] + predictor_mean = np.mean(train_predictors, axis=0) + response_mean = float(np.mean(response[train])) + coefficients, _ = nnls( + train_predictors - predictor_mean, + response[train] - response_mean, + ) + intercept = response_mean - float(predictor_mean @ coefficients) + train_prediction = intercept + train_predictors @ coefficients + variance = _fit_variance(response[train] - train_prediction) + validation_prediction = intercept + predictors[validation] @ coefficients + return _gaussian_score( + response[validation] - validation_prediction, + variance, + ) + + +def crossfit_scores( + signals: NDArray[np.float64], + response: NDArray[np.float64], + *, + folds: int, + rng: np.random.Generator, +) -> CrossFitScores: + """Score all pure models and all pairwise nonnegative cones out of fold.""" + + if signals.ndim != 2 or signals.shape[1] != len(DESIGN_CANDIDATE_NAMES): + raise ValueError("signals must have one column per declared candidate") + if response.shape != (signals.shape[0],): + raise ValueError("response must match the signal rows") + if signals.shape[0] < 2 * folds: + raise ValueError("cross-fitting requires at least two observations per fold") + split = tuple( + np.asarray(values, dtype=np.int64) + for values in np.array_split( + np.asarray(rng.permutation(signals.shape[0]), dtype=np.int64), + folds, + ) + ) + pure_scores = np.zeros(signals.shape[1], dtype=float) + residual_sums = np.zeros(signals.shape[1], dtype=float) + pair_scores = np.zeros(len(_PAIR_INDICES), dtype=float) + null_score = 0.0 + for fold_index, validation in enumerate(split): + train = np.concatenate( + [values for index, values in enumerate(split) if index != fold_index] + ) + null_score += _null_fold(response, train, validation) + for candidate in range(signals.shape[1]): + score, residual_sum = _pure_fold( + signals[:, candidate], + response, + train, + validation, + ) + pure_scores[candidate] += score + residual_sums[candidate] += residual_sum + for pair_index, pair in enumerate(_PAIR_INDICES): + pair_scores[pair_index] += _nonnegative_pair_fold( + signals[:, pair], + response, + train, + validation, + ) + best_pair_index = int(np.argmax(pair_scores)) + return CrossFitScores( + pure_scores=np.asarray(pure_scores, dtype=float), + null_score=float(null_score), + composite_score=float(pair_scores[best_pair_index]), + composite_pair=_PAIR_INDICES[best_pair_index], + pure_residual_ratios=np.asarray( + residual_sums / signals.shape[0], + dtype=float, + ), + ) + + +def _simulate( + signals: NDArray[np.float64], + generator: NDArray[np.float64], + *, + config: MixtureDiagnosticConfig, + rng: np.random.Generator, +) -> CrossFitScores: + response = ( + config.effect_size * generator + + rng.normal(0.0, config.noise_std, size=generator.size) + ) + return crossfit_scores( + signals, + np.asarray(response, dtype=float), + folds=config.folds, + rng=rng, + ) + + +def _calibrate( + signals: NDArray[np.float64], + *, + config: MixtureDiagnosticConfig, +) -> DiagnosticThresholds: + pure_over_null: list[float] = [] + winner_over_runner: list[float] = [] + null = np.zeros(signals.shape[0], dtype=float) + for replicate in range(config.calibration_replicates): + scores = _simulate( + signals, + null, + config=config, + rng=_rng(config.threshold_seed, 0, replicate), + ) + pure_over_null.append(scores.pure_over_null) + winner_over_runner.append(scores.winner_over_runner) + + candidate_thresholds: list[CandidateThreshold] = [] + for candidate in range(signals.shape[1]): + composite_gaps: list[float] = [] + residual_ratios: list[float] = [] + for replicate in range(config.calibration_replicates): + scores = _simulate( + signals, + signals[:, candidate], + config=config, + rng=_rng( + config.threshold_seed, + 1, + candidate, + replicate, + ), + ) + composite_gaps.append( + scores.composite_score - scores.pure_scores[candidate] + ) + residual_ratios.append(scores.pure_residual_ratios[candidate]) + candidate_thresholds.append( + CandidateThreshold( + composite_over_pure=max( + 0.0, + _upper_conformal_quantile( + composite_gaps, + alpha=config.alpha, + ), + ), + residual_ratio=_upper_conformal_quantile( + residual_ratios, + alpha=config.alpha, + ), + ) + ) + return DiagnosticThresholds( + pure_over_null=_upper_conformal_quantile( + pure_over_null, + alpha=config.alpha, + ), + winner_over_runner=_upper_conformal_quantile( + winner_over_runner, + alpha=config.alpha, + ), + candidates=tuple(candidate_thresholds), + ) + + +def _call( + scores: CrossFitScores, + thresholds: DiagnosticThresholds, + enabled: tuple[bool, ...], +) -> tuple[int | None, str]: + winner = scores.winner + if not enabled[winner]: + return None, "candidate_underpowered" + if scores.pure_over_null <= thresholds.pure_over_null: + return None, "null_not_rejected" + if scores.winner_over_runner <= thresholds.winner_over_runner: + return None, "pure_ambiguity" + candidate = thresholds.candidates[winner] + if scores.composite_over_winner > candidate.composite_over_pure: + return None, "pairwise_composite_better" + if scores.pure_residual_ratios[winner] > candidate.residual_ratio: + return None, "residual_lack_of_fit" + return winner, "pure_call" + + +def _audit_power( + signals: NDArray[np.float64], + thresholds: DiagnosticThresholds, + *, + config: MixtureDiagnosticConfig, +) -> tuple[tuple[bool, ...], list[dict[str, Any]]]: + rows: list[dict[str, Any]] = [] + enabled: list[bool] = [] + all_enabled = tuple(True for _ in DESIGN_CANDIDATE_NAMES) + for candidate, name in enumerate(DESIGN_CANDIDATE_NAMES): + correct = 0 + wrong = 0 + reasons: dict[str, int] = {} + for replicate in range(config.calibration_audit_replicates): + scores = _simulate( + signals, + signals[:, candidate], + config=config, + rng=_rng( + config.calibration_audit_seed, + 0, + candidate, + replicate, + ), + ) + call, reason = _call(scores, thresholds, all_enabled) + correct += int(call == candidate) + wrong += int(call is not None and call != candidate) + reasons[reason] = reasons.get(reason, 0) + 1 + lower, upper = _wilson_interval( + correct, + config.calibration_audit_replicates, + config.confidence_level, + ) + is_enabled = lower >= config.minimum_pure_power_wilson_lower + enabled.append(is_enabled) + rows.append( + { + "scenario": "matched_pure", + "candidate": name, + "replicates": config.calibration_audit_replicates, + "correct_pure_calls": correct, + "wrong_pure_calls": wrong, + "abstentions": ( + config.calibration_audit_replicates - correct - wrong + ), + "correct_call_rate": ( + correct / config.calibration_audit_replicates + ), + "wilson_lower": lower, + "wilson_upper": upper, + "minimum_wilson_lower": ( + config.minimum_pure_power_wilson_lower + ), + "candidate_enabled": is_enabled, + "reasons": dict(sorted(reasons.items())), + } + ) + return tuple(enabled), rows + + +def _evaluate_pure( + signals: NDArray[np.float64], + thresholds: DiagnosticThresholds, + enabled: tuple[bool, ...], + *, + config: MixtureDiagnosticConfig, +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for candidate, name in enumerate(DESIGN_CANDIDATE_NAMES): + correct = 0 + wrong = 0 + reasons: dict[str, int] = {} + for replicate in range(config.evaluation_replicates): + scores = _simulate( + signals, + signals[:, candidate], + config=config, + rng=_rng( + config.evaluation_seed, + 0, + candidate, + replicate, + ), + ) + call, reason = _call(scores, thresholds, enabled) + correct += int(call == candidate) + wrong += int(call is not None and call != candidate) + reasons[reason] = reasons.get(reason, 0) + 1 + lower, upper = _wilson_interval( + correct, + config.evaluation_replicates, + config.confidence_level, + ) + rows.append( + { + "candidate": name, + "replicates": config.evaluation_replicates, + "correct_pure_calls": correct, + "wrong_pure_calls": wrong, + "abstentions": config.evaluation_replicates - correct - wrong, + "correct_call_rate": correct / config.evaluation_replicates, + "wilson_lower": lower, + "wilson_upper": upper, + "candidate_enabled_from_audit": enabled[candidate], + "reasons": dict(sorted(reasons.items())), + } + ) + return rows + + +def _evaluate_null( + signals: NDArray[np.float64], + thresholds: DiagnosticThresholds, + enabled: tuple[bool, ...], + *, + config: MixtureDiagnosticConfig, +) -> list[dict[str, Any]]: + false_calls = 0 + reasons: dict[str, int] = {} + generator = np.zeros(signals.shape[0], dtype=float) + for replicate in range(config.evaluation_replicates): + scores = _simulate( + signals, + generator, + config=config, + rng=_rng(config.evaluation_seed, 1, replicate), + ) + call, reason = _call(scores, thresholds, enabled) + false_calls += int(call is not None) + reasons[reason] = reasons.get(reason, 0) + 1 + lower, upper = _wilson_interval( + false_calls, + config.evaluation_replicates, + config.confidence_level, + ) + return [ + { + "replicates": config.evaluation_replicates, + "false_pure_calls": false_calls, + "abstentions": config.evaluation_replicates - false_calls, + "false_pure_call_rate": false_calls / config.evaluation_replicates, + "wilson_lower": lower, + "wilson_upper": upper, + "reasons": dict(sorted(reasons.items())), + } + ] + + +def _scaled_mixture( + full_signals: NDArray[np.float64], + first: int, + second: int, +) -> NDArray[np.float64]: + value = 0.5 * (full_signals[:, first] + full_signals[:, second]) + scale = float(np.std(value)) + if scale <= 1.0e-12: + raise RuntimeError("mixture is constant on the full grid") + return np.asarray(value / scale, dtype=float) + + +def _evaluate_mixtures( + signals: NDArray[np.float64], + full_signals: NDArray[np.float64], + indices: NDArray[np.int64], + thresholds: DiagnosticThresholds, + enabled: tuple[bool, ...], + *, + config: MixtureDiagnosticConfig, +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for pair_index, (first, second) in enumerate(_PAIR_INDICES): + generator = _scaled_mixture(full_signals, first, second)[indices] + false_calls = 0 + reasons: dict[str, int] = {} + for replicate in range(config.evaluation_replicates): + scores = _simulate( + signals, + generator, + config=config, + rng=_rng( + config.evaluation_seed, + 2, + pair_index, + replicate, + ), + ) + call, reason = _call(scores, thresholds, enabled) + false_calls += int(call is not None) + reasons[reason] = reasons.get(reason, 0) + 1 + lower, upper = _wilson_interval( + false_calls, + config.evaluation_replicates, + config.confidence_level, + ) + rows.append( + { + "first_candidate": DESIGN_CANDIDATE_NAMES[first], + "second_candidate": DESIGN_CANDIDATE_NAMES[second], + "replicates": config.evaluation_replicates, + "false_pure_calls": false_calls, + "abstentions": config.evaluation_replicates - false_calls, + "false_pure_call_rate": ( + false_calls / config.evaluation_replicates + ), + "wilson_lower": lower, + "wilson_upper": upper, + "reasons": dict(sorted(reasons.items())), + } + ) + return rows + + +def _evaluate_out_of_span( + signals: NDArray[np.float64], + full_signals: NDArray[np.float64], + indices: NDArray[np.int64], + thresholds: DiagnosticThresholds, + enabled: tuple[bool, ...], + *, + config: MixtureDiagnosticConfig, +) -> list[dict[str, Any]]: + probe, residual_scale, maximum_inner_product = _out_of_span_probe( + full_signals + ) + generator = probe[indices] + false_calls = 0 + reasons: dict[str, int] = {} + for replicate in range(config.evaluation_replicates): + scores = _simulate( + signals, + generator, + config=config, + rng=_rng(config.evaluation_seed, 3, replicate), + ) + call, reason = _call(scores, thresholds, enabled) + false_calls += int(call is not None) + reasons[reason] = reasons.get(reason, 0) + 1 + lower, upper = _wilson_interval( + false_calls, + config.evaluation_replicates, + config.confidence_level, + ) + return [ + { + "probe": "full_grid_orthogonalized_tanh_surprise", + "replicates": config.evaluation_replicates, + "false_pure_calls": false_calls, + "abstentions": config.evaluation_replicates - false_calls, + "false_pure_call_rate": false_calls / config.evaluation_replicates, + "wilson_lower": lower, + "wilson_upper": upper, + "full_grid_prestandardization_residual_sd": residual_scale, + "full_grid_maximum_absolute_mean_inner_product": ( + maximum_inner_product + ), + "reasons": dict(sorted(reasons.items())), + } + ] + + +def _affine_residual( + response: NDArray[np.float64], + predictors: NDArray[np.float64], +) -> float: + design = np.column_stack((np.ones(response.size), predictors)) + coefficients, _, _, _ = np.linalg.lstsq(design, response, rcond=None) + residual = response - design @ coefficients + return float(np.mean(residual**2)) + + +def _geometry_rows( + full_signals: NDArray[np.float64], + indices: NDArray[np.int64], + *, + config: MixtureDiagnosticConfig, +) -> list[dict[str, Any]]: + trial_signals = full_signals[indices] + rows: list[dict[str, Any]] = [] + for first, second in _PAIR_INDICES: + generator = _scaled_mixture(full_signals, first, second)[indices] + pure_residuals = np.asarray( + [ + _affine_residual(generator, trial_signals[:, [candidate]]) + for candidate in range(trial_signals.shape[1]) + ], + dtype=float, + ) + best = int(np.argmin(pure_residuals)) + pair_residual = _affine_residual( + generator, + trial_signals[:, [first, second]], + ) + residual = float(pure_residuals[best]) + oracle_gap = 0.5 * config.budget * math.log1p( + config.effect_size**2 * residual / config.noise_std**2 + ) + rows.append( + { + "first_candidate": DESIGN_CANDIDATE_NAMES[first], + "second_candidate": DESIGN_CANDIDATE_NAMES[second], + "best_pure_candidate": DESIGN_CANDIDATE_NAMES[best], + "best_pure_affine_residual": residual, + "true_pair_affine_residual": pair_residual, + "crossfit_oracle_log_score_gap_index": oracle_gap, + "below_five_nat_power_index": oracle_gap < 5.0, + } + ) + return rows + + +def run_mixture_diagnostic( + counts: NDArray[np.int64], + config: MixtureDiagnosticConfig | None = None, +) -> MixtureDiagnosticResult: + """Run the prospectively configured post-failure diagnostic once.""" + + config = MixtureDiagnosticConfig() if config is None else config + config.validate() + _, _, full_signals = generate_transition_design_grid() + counts = np.asarray(counts, dtype=np.int64) + if ( + counts.shape != (full_signals.shape[0],) + or np.any(counts < 0) + or int(np.sum(counts)) != config.budget + ): + raise ValueError("counts must be a valid locked N=60 grid allocation") + indices = np.repeat(np.arange(counts.size), counts) + signals = np.asarray(full_signals[indices], dtype=float) + thresholds = _calibrate(signals, config=config) + enabled, audit_rows = _audit_power( + signals, + thresholds, + config=config, + ) + pure_rows = _evaluate_pure( + signals, + thresholds, + enabled, + config=config, + ) + null_rows = _evaluate_null( + signals, + thresholds, + enabled, + config=config, + ) + mixture_rows = _evaluate_mixtures( + signals, + full_signals, + indices, + thresholds, + enabled, + config=config, + ) + out_rows = _evaluate_out_of_span( + signals, + full_signals, + indices, + thresholds, + enabled, + config=config, + ) + geometry_rows = _geometry_rows( + full_signals, + indices, + config=config, + ) + threshold_rows = [ + { + "candidate": name, + "pure_over_null_familywise": thresholds.pure_over_null, + "winner_over_runner_familywise": thresholds.winner_over_runner, + "pairwise_composite_over_pure_familywise": ( + thresholds.candidates[index].composite_over_pure + ), + "residual_ratio_candidate_specific": ( + thresholds.candidates[index].residual_ratio + ), + "candidate_enabled_from_audit": enabled[index], + } + for index, name in enumerate(DESIGN_CANDIDATE_NAMES) + ] + summary = { + "schema_version": 1, + "experiment": "post_failure_pairwise_cone_abstention_diagnostic", + "interpretation": ( + "Separately configured sensitivity after the immutable original " + "stress failure; not a main-paper open-set robustness claim." + ), + "design": config.design, + "budget": config.budget, + "candidate_power_enabled": { + name: enabled[index] + for index, name in enumerate(DESIGN_CANDIDATE_NAMES) + }, + "minimum_matched_pure_wilson_lower": min( + float(row["wilson_lower"]) for row in pure_rows + ), + "maximum_mixture_false_pure_wilson_upper": max( + float(row["wilson_upper"]) for row in mixture_rows + ), + "null_false_pure_wilson_upper": float(null_rows[0]["wilson_upper"]), + "out_of_span_false_pure_wilson_upper": float( + out_rows[0]["wilson_upper"] + ), + "technical_gates": { + "streams_disjoint": len( + { + config.threshold_seed, + config.calibration_audit_seed, + config.evaluation_seed, + } + ) + == 3, + "all_fifteen_pairwise_composites": len(_PAIR_INDICES) == 15, + "three_fold_cross_fitting": config.folds == 3, + "candidate_power_gate_applied": True, + "evaluation_not_used_for_thresholds": True, + }, + } + return MixtureDiagnosticResult( + summary=summary, + thresholds=tuple(threshold_rows), + calibration_audit=tuple(audit_rows), + pure_evaluation=tuple(pure_rows), + null_evaluation=tuple(null_rows), + mixture_evaluation=tuple(mixture_rows), + out_of_span_evaluation=tuple(out_rows), + geometry=tuple(geometry_rows), + ) From 8c3b91de9bccdaab8ec3676a327ff7c6699c2195 Mon Sep 17 00:00:00 2001 From: Florian Pfaff <6773539+FlorianPfaff@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:25:15 +0800 Subject: [PATCH 02/15] Add mixture diagnostic artifact contract --- .../design_mixture_diagnostic_cli.py | 300 ++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 src/bayesian_ach/design_mixture_diagnostic_cli.py diff --git a/src/bayesian_ach/design_mixture_diagnostic_cli.py b/src/bayesian_ach/design_mixture_diagnostic_cli.py new file mode 100644 index 0000000..65652eb --- /dev/null +++ b/src/bayesian_ach/design_mixture_diagnostic_cli.py @@ -0,0 +1,300 @@ +"""CLI and independent verifier for the mixture-aware design diagnostic.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +from collections.abc import Mapping, Sequence +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from bayesian_ach.design_mixture_diagnostic import ( + MixtureDiagnosticConfig, + run_mixture_diagnostic, +) +from bayesian_ach.design_stress_cli import ( + _csv_safe_rows, + _git_provenance, + _load_locked_design_allocation, +) +from bayesian_ach.io import write_json, write_rows_csv + +_REPOSITORY = "IPS-Stuttgart/Bayesian-ACh" +_BASELINE_PRODUCER_SHA = "c71695fda83ae93407599a909097962ee3fa9e0e" +_BASELINE_CHECKSUMS_SHA256 = ( + "44a5188c43bda52e6fc9dc7007cf2de44a9671e9c5477ac88c3173c06cfdbd80" +) +_BASELINE_MANIFEST_SHA256 = ( + "d840a2ec34f5a386109c7f985033b53144fdcc1b1a0e0592b3274c0e38902b64" +) +_PAYLOADS = ( + "summary.json", + "thresholds.csv", + "calibration_audit.csv", + "pure_evaluation.csv", + "null_evaluation.csv", + "mixture_evaluation.csv", + "out_of_span_evaluation.csv", + "geometry.csv", +) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _canonical_digest(value: Mapping[str, Any]) -> str: + encoded = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _verify_checksum_rows(directory: Path) -> tuple[dict[str, Any], ...]: + path = directory / "SHA256SUMS.csv" + if not path.is_file(): + raise ValueError(f"missing checksum table: {path}") + with path.open(newline="", encoding="utf-8") as handle: + rows = tuple(dict(row) for row in csv.DictReader(handle)) + if not rows: + raise ValueError("checksum table is empty") + seen: set[str] = set() + for row in rows: + name = str(row.get("path", row.get("file", ""))) + if not name or name in seen: + raise ValueError("checksum table contains a missing or duplicate path") + seen.add(name) + candidate = directory / name + if not candidate.is_file(): + raise ValueError(f"checksum table payload is missing: {name}") + if int(row["bytes"]) != candidate.stat().st_size: + raise ValueError(f"locked byte count mismatch: {name}") + if str(row["sha256"]) != _sha256(candidate): + raise ValueError(f"SHA-256 mismatch: {name}") + return rows + + +def verify_baseline_artifact(directory: Path) -> dict[str, Any]: + """Verify and bind the immutable original failure artifact.""" + + directory = directory.resolve() + manifest_path = directory / "artifact_manifest.json" + checksums_path = directory / "SHA256SUMS.csv" + if _sha256(checksums_path) != _BASELINE_CHECKSUMS_SHA256: + raise ValueError("baseline checksum-table SHA-256 mismatch") + if _sha256(manifest_path) != _BASELINE_MANIFEST_SHA256: + raise ValueError("baseline manifest SHA-256 mismatch") + rows = _verify_checksum_rows(directory) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if ( + manifest.get("artifact") != "post_freeze_design_abstention_sensitivity" + or manifest.get("producer_commit") != _BASELINE_PRODUCER_SHA + or manifest.get("producer_git_dirty") is not False + ): + raise ValueError("baseline artifact provenance does not match the lock") + return { + "kind": "immutable_original_design_stress_failure", + "producer_commit": _BASELINE_PRODUCER_SHA, + "checksum_table_sha256": _BASELINE_CHECKSUMS_SHA256, + "manifest_sha256": _BASELINE_MANIFEST_SHA256, + "verified_payload_count": len(rows), + } + + +def _write_artifact( + output: Path, + *, + result: Any, + config: MixtureDiagnosticConfig, + code_sha: str, + inputs: Sequence[Mapping[str, Any]], +) -> None: + output.mkdir(parents=True, exist_ok=False) + write_json(output / "summary.json", result.summary) + write_rows_csv(output / "thresholds.csv", _csv_safe_rows(result.thresholds)) + write_rows_csv( + output / "calibration_audit.csv", + _csv_safe_rows(result.calibration_audit), + ) + write_rows_csv( + output / "pure_evaluation.csv", + _csv_safe_rows(result.pure_evaluation), + ) + write_rows_csv( + output / "null_evaluation.csv", + _csv_safe_rows(result.null_evaluation), + ) + write_rows_csv( + output / "mixture_evaluation.csv", + _csv_safe_rows(result.mixture_evaluation), + ) + write_rows_csv( + output / "out_of_span_evaluation.csv", + _csv_safe_rows(result.out_of_span_evaluation), + ) + write_rows_csv(output / "geometry.csv", _csv_safe_rows(result.geometry)) + files = [ + { + "path": name, + "bytes": (output / name).stat().st_size, + "sha256": _sha256(output / name), + } + for name in _PAYLOADS + ] + manifest = { + "schema_version": 1, + "artifact": "post_failure_mixture_aware_design_diagnostic", + "repository": _REPOSITORY, + "producer_commit": code_sha, + "producer_git_dirty": False, + "configuration": asdict(config), + "configuration_sha256": _canonical_digest(asdict(config)), + "inputs": list(inputs), + "files": files, + } + write_json(output / "artifact_manifest.json", manifest) + checksum_rows = files + [ + { + "path": "artifact_manifest.json", + "bytes": (output / "artifact_manifest.json").stat().st_size, + "sha256": _sha256(output / "artifact_manifest.json"), + } + ] + write_rows_csv(output / "SHA256SUMS.csv", checksum_rows) + + +def verify_mixture_diagnostic_artifact(directory: Path) -> dict[str, Any]: + """Independently verify payload hashes and the locked scientific config.""" + + directory = directory.resolve() + rows = _verify_checksum_rows(directory) + locked_names = { + str(row.get("path", row.get("file", ""))) + for row in rows + } + if locked_names != {*_PAYLOADS, "artifact_manifest.json"}: + raise ValueError("diagnostic checksum table has an unexpected payload set") + manifest = json.loads( + (directory / "artifact_manifest.json").read_text(encoding="utf-8") + ) + config = manifest.get("configuration", {}) + if ( + manifest.get("schema_version") != 1 + or manifest.get("artifact") + != "post_failure_mixture_aware_design_diagnostic" + or manifest.get("producer_git_dirty") is not False + or config.get("threshold_seed") != 196613 + or config.get("calibration_audit_seed") != 262147 + or config.get("evaluation_seed") != 324949 + or config.get("minimum_pure_power_wilson_lower") != 0.70 + or config.get("folds") != 3 + or config.get("budget") != 60 + ): + raise ValueError("diagnostic manifest does not match the locked config") + if manifest.get("configuration_sha256") != _canonical_digest(config): + raise ValueError("diagnostic configuration digest mismatch") + inputs = manifest.get("inputs", []) + baseline = [ + item + for item in inputs + if item.get("kind") == "immutable_original_design_stress_failure" + ] + if ( + len(baseline) != 1 + or baseline[0].get("manifest_sha256") != _BASELINE_MANIFEST_SHA256 + or baseline[0].get("checksum_table_sha256") + != _BASELINE_CHECKSUMS_SHA256 + ): + raise ValueError("diagnostic does not bind the immutable baseline") + summary = json.loads((directory / "summary.json").read_text(encoding="utf-8")) + gates = summary.get("technical_gates", {}) + if not gates or not all(bool(value) for value in gates.values()): + raise ValueError("diagnostic technical gates did not all pass") + return { + "producer_commit": manifest["producer_commit"], + "configuration_sha256": manifest["configuration_sha256"], + "verified_payload_count": len(rows), + "baseline_manifest_sha256": _BASELINE_MANIFEST_SHA256, + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="bayesian-ach-mixture-diagnostic", + description="Freeze the prospectively configured post-failure diagnostic.", + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--repo-root", type=Path, default=Path.cwd()) + parser.add_argument("--code-sha", required=True) + parser.add_argument("--baseline-artifact", type=Path, required=True) + parser.add_argument("--locked-allocation", type=Path, required=True) + parser.add_argument("--locked-allocation-sha256", required=True) + parser.add_argument("--locked-design-code-sha", required=True) + parser.add_argument("--locked-allocation-seed", type=int, required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) + repo_root = args.repo_root.resolve() + _git_provenance(repo_root, args.code_sha) + baseline = verify_baseline_artifact(args.baseline_artifact) + overrides, allocation = _load_locked_design_allocation( + args.locked_allocation, + expected_sha256=args.locked_allocation_sha256, + source_code_sha=args.locked_design_code_sha, + allocation_seed=args.locked_allocation_seed, + ) + key = ("maximin_optimized", 60) + if key not in overrides: + raise ValueError("locked allocation does not contain maximin N=60") + config = MixtureDiagnosticConfig() + result = run_mixture_diagnostic(overrides[key], config) + result.summary["chronology"] = ( + "Method, thresholds, power gate, and fresh streams were committed " + "after immutable baseline failure d1251ddd and before this evaluation." + ) + result.summary["input_provenance"] = { + "baseline": baseline, + "allocation": allocation, + } + _write_artifact( + args.output.resolve(), + result=result, + config=config, + code_sha=args.code_sha, + inputs=(baseline, allocation), + ) + verified = verify_mixture_diagnostic_artifact(args.output) + print(json.dumps({"summary": result.summary, "verification": verified}, indent=2)) + return 0 + + +def verify_main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="bayesian-ach-verify-mixture-diagnostic" + ) + parser.add_argument("artifact", type=Path) + args = parser.parse_args(argv) + print( + json.dumps( + verify_mixture_diagnostic_artifact(args.artifact), + indent=2, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) From b7bed3f39f72b36e36cc0a7f3d42306713a57dfd Mon Sep 17 00:00:00 2001 From: Florian Pfaff <6773539+FlorianPfaff@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:25:37 +0800 Subject: [PATCH 03/15] Expose mixture diagnostic commands --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 32b0827..8141c19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,8 @@ plot = ["matplotlib>=3.8"] bayesian-ach = "bayesian_ach.cli_ext:main" bayesian-ach-design = "bayesian_ach.design_cli:main" bayesian-ach-design-stress = "bayesian_ach.design_stress_cli:main" +bayesian-ach-mixture-diagnostic = "bayesian_ach.design_mixture_diagnostic_cli:main" +bayesian-ach-verify-mixture-diagnostic = "bayesian_ach.design_mixture_diagnostic_cli:verify_main" bayesian-ach-replay = "bayesian_ach.replay_cli:main" [tool.setuptools.packages.find] From ec8efabd6e399471fe0c7d2433eb84ca7da8e42b Mon Sep 17 00:00:00 2001 From: Florian Pfaff <6773539+FlorianPfaff@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:26:15 +0800 Subject: [PATCH 04/15] Test mixture-aware diagnostic --- tests/test_design_mixture_diagnostic.py | 180 ++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 tests/test_design_mixture_diagnostic.py diff --git a/tests/test_design_mixture_diagnostic.py b/tests/test_design_mixture_diagnostic.py new file mode 100644 index 0000000..c71a606 --- /dev/null +++ b/tests/test_design_mixture_diagnostic.py @@ -0,0 +1,180 @@ +import json +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest + +from bayesian_ach.design_mixture_diagnostic import ( + CandidateThreshold, + CrossFitScores, + DiagnosticThresholds, + MixtureDiagnosticConfig, + _call, + _geometry_rows, + crossfit_scores, +) +from bayesian_ach.design_mixture_diagnostic_cli import ( + _BASELINE_CHECKSUMS_SHA256, + _BASELINE_MANIFEST_SHA256, + _write_artifact, + verify_mixture_diagnostic_artifact, +) + + +def test_locked_config_uses_fresh_disjoint_streams_and_power_gate() -> None: + config = MixtureDiagnosticConfig() + config.validate() + assert ( + config.threshold_seed, + config.calibration_audit_seed, + config.evaluation_seed, + ) == (196613, 262147, 324949) + assert config.minimum_pure_power_wilson_lower == 0.70 + assert config.folds == 3 + with pytest.raises(ValueError, match="distinct"): + MixtureDiagnosticConfig( + calibration_audit_seed=config.threshold_seed + ).validate() + with pytest.raises(ValueError, match="restricted"): + MixtureDiagnosticConfig(budget=45).validate() + + +def test_crossfit_pairwise_cone_recovers_noiseless_positive_mixture() -> None: + rng = np.random.default_rng(42) + signals = rng.normal(size=(60, 6)) + signals = (signals - signals.mean(axis=0)) / signals.std(axis=0) + response = 0.5 * (signals[:, 0] + signals[:, 1]) + scores = crossfit_scores( + signals, + response, + folds=3, + rng=np.random.default_rng(17), + ) + assert scores.composite_pair == (0, 1) + assert scores.composite_score > float(np.max(scores.pure_scores)) + assert scores.pure_residual_ratios[scores.winner] > 1.0 + + +def test_call_requires_power_signal_separation_composite_and_gof() -> None: + thresholds = DiagnosticThresholds( + pure_over_null=1.0, + winner_over_runner=1.0, + candidates=tuple( + CandidateThreshold( + composite_over_pure=1.0, + residual_ratio=2.0, + ) + for _ in range(6) + ), + ) + scores = CrossFitScores( + pure_scores=np.array([5.0, 3.0, 2.0, 1.0, 0.0, -1.0]), + null_score=0.0, + composite_score=5.5, + composite_pair=(0, 1), + pure_residual_ratios=np.ones(6), + ) + assert _call(scores, thresholds, (True,) * 6) == (0, "pure_call") + assert _call(scores, thresholds, (False,) + (True,) * 5) == ( + None, + "candidate_underpowered", + ) + composite = CrossFitScores( + pure_scores=scores.pure_scores, + null_score=scores.null_score, + composite_score=6.1, + composite_pair=scores.composite_pair, + pure_residual_ratios=scores.pure_residual_ratios, + ) + assert _call(composite, thresholds, (True,) * 6) == ( + None, + "pairwise_composite_better", + ) + lack_of_fit = CrossFitScores( + pure_scores=scores.pure_scores, + null_score=scores.null_score, + composite_score=scores.composite_score, + composite_pair=scores.composite_pair, + pure_residual_ratios=np.array([2.1, 1.0, 1.0, 1.0, 1.0, 1.0]), + ) + assert _call(lack_of_fit, thresholds, (True,) * 6) == ( + None, + "residual_lack_of_fit", + ) + + +def test_geometry_reports_pair_representability_and_finite_power_index() -> None: + rng = np.random.default_rng(9) + full_signals = rng.normal(size=(80, 6)) + full_signals = ( + full_signals - full_signals.mean(axis=0) + ) / full_signals.std(axis=0) + indices = np.arange(60, dtype=np.int64) + rows = _geometry_rows( + full_signals, + indices, + config=MixtureDiagnosticConfig(), + ) + assert len(rows) == 15 + assert max(float(row["true_pair_affine_residual"]) for row in rows) < 1.0e-20 + assert all( + np.isfinite(float(row["crossfit_oracle_log_score_gap_index"])) + for row in rows + ) + + +def _dummy_result() -> SimpleNamespace: + row = ({"name": "one", "value": 1},) + return SimpleNamespace( + summary={ + "schema_version": 1, + "technical_gates": { + "streams_disjoint": True, + "all_fifteen_pairwise_composites": True, + "three_fold_cross_fitting": True, + "candidate_power_gate_applied": True, + "evaluation_not_used_for_thresholds": True, + }, + }, + thresholds=row, + calibration_audit=row, + pure_evaluation=row, + null_evaluation=row, + mixture_evaluation=row, + out_of_span_evaluation=row, + geometry=row, + ) + + +def test_artifact_verifier_binds_config_baseline_and_tampering( + tmp_path: Path, +) -> None: + output = tmp_path / "artifact" + baseline = { + "kind": "immutable_original_design_stress_failure", + "producer_commit": "c71695fda83ae93407599a909097962ee3fa9e0e", + "checksum_table_sha256": _BASELINE_CHECKSUMS_SHA256, + "manifest_sha256": _BASELINE_MANIFEST_SHA256, + "verified_payload_count": 9, + } + _write_artifact( + output, + result=_dummy_result(), + config=MixtureDiagnosticConfig(), + code_sha="a" * 40, + inputs=(baseline,), + ) + verified = verify_mixture_diagnostic_artifact(output) + assert verified["producer_commit"] == "a" * 40 + assert verified["verified_payload_count"] == 9 + + summary = output / "summary.json" + value = json.loads(summary.read_text(encoding="utf-8")) + value["tampered"] = True + summary.write_text(json.dumps(value), encoding="utf-8") + with pytest.raises( + ValueError, + match="locked byte count mismatch|SHA-256 mismatch", + ): + verify_mixture_diagnostic_artifact(output) From a9667b83faff0dad9e3b1de7d252899f5af638eb Mon Sep 17 00:00:00 2001 From: Florian Pfaff <6773539+FlorianPfaff@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:26:43 +0800 Subject: [PATCH 05/15] Document locked mixture diagnostic --- docs/design_mixture_diagnostic.md | 88 +++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 docs/design_mixture_diagnostic.md diff --git a/docs/design_mixture_diagnostic.md b/docs/design_mixture_diagnostic.md new file mode 100644 index 0000000..ca75bca --- /dev/null +++ b/docs/design_mixture_diagnostic.md @@ -0,0 +1,88 @@ +# Post-failure mixture-aware design diagnostic + +This diagnostic is intentionally separate from the immutable N=60 stress failure +frozen at commit `d1251dddeaa706d79feaaec99185dcc4236aa4a3`. It does not +replace, tune, or reinterpret that result. Its method and random streams are +committed before the new evaluation is run. + +## Locked method + +The diagnostic is restricted to the chronologically earlier heuristic maximin +allocation at N=60. It uses three-fold cross-fitting, so every observation is +scored only by models fitted without that observation. The pure family contains +the six prespecified single-candidate regressions. + +The targeted composite family contains all 15 unordered candidate pairs. For +each training fold, an intercept and two nonnegative slopes are fitted. A free +total amplitude makes this a nonnegative cone over each pairwise convex simplex; +a pure candidate is a boundary case. The maximum held-out score across all 15 +pairs is the familywise composite statistic. + +For each possible pure winner, calibration under that matched pure generator +sets separate upper thresholds for: + +- the maximum pairwise-composite score improvement over the pure model; and +- a cross-fitted residual lack-of-fit ratio, defined as validation squared error + divided by the variance fitted on the corresponding training fold. + +The pure-over-null and winner-over-runner thresholds are calibrated from the +maximum statistics under the null. A pure label is enabled only if an independent +calibration audit gives a 95% Wilson lower bound of at least 0.70 for its correct +pure-call rate. Otherwise every evaluation call with that winner must abstain. +This power rule was fixed before evaluation and is not relaxed for difficult +candidates. + +The fixed streams and replicate counts are: + +| Purpose | Seed | Replicates | +|---|---:|---:| +| threshold calibration | 196613 | 200 | +| independent calibration audit | 262147 | 200 | +| one-time evaluation | 324949 | 200 | + +All three streams are disjoint. Evaluation data do not set thresholds, choose +models, change the power rule, or alter the allocation. + +## Scope and identifiability + +The extension is targeted to positive two-candidate mixtures. It is not a +general open-set classifier. The nonnegative sign restriction follows the +declared positive-effect simulation and would need separate justification for +an empirical model with unoriented effects. + +Cross-fitting uses all 60 observations once as held-out predictions, but it +cannot create information absent from the design. The immutable diagnostic +already shows that several mixtures are only weakly separated from their best +pure affine approximation on this support. The artifact therefore reports a +population oracle log-score-gap index for every pair. Low finite-sample audit +power forces abstention; it is not treated as evidence that the pure hypothesis +is correct. + +The fixed orthogonalized nonlinear residual remains one bounded out-of-span +probe. The residual lack-of-fit gate is calibrated for the matched Gaussian +simulation, not arbitrary biological misspecification, serial dependence, +animal hierarchy, indicator dynamics, or an executable sequential protocol. +Passing this diagnostic would support only the declared simulation family and +would not constitute a main-paper robustness claim without an independent +freeze. + +## Reproduction + +From a clean checkout of the exact producer commit: + +```bash +bayesian-ach-mixture-diagnostic \ + --repo-root . \ + --code-sha \ + --baseline-artifact results/design-open-set-stress-n60 \ + --locked-allocation /absolute/path/optimal_design_allocation_seed7.csv \ + --locked-allocation-sha256 a823be49faf6c6cbebf60b11d4b5ca895cf7734d6e9c577ee98f97a5907b69b2 \ + --locked-design-code-sha 1b2028929ac6ebc1cce0882f0c22af9918044342 \ + --locked-allocation-seed 7 \ + --output /absolute/path/mixture-aware-diagnostic +``` + +The command verifies the immutable baseline, exact allocation hash, explicit +seed metadata, source commit, deterministic allocation reconstruction, clean +worktree, and producer commit. It writes checksum-bound tables and immediately +runs the independent artifact verifier. From 5b7eb492fcb58325119d2e6e8cab2a52ff0f53ff Mon Sep 17 00:00:00 2001 From: Florian Pfaff <6773539+FlorianPfaff@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:27:50 +0800 Subject: [PATCH 06/15] Separate diagnostic power gates --- src/bayesian_ach/design_mixture_diagnostic.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/bayesian_ach/design_mixture_diagnostic.py b/src/bayesian_ach/design_mixture_diagnostic.py index 23f1075..a7fb795 100644 --- a/src/bayesian_ach/design_mixture_diagnostic.py +++ b/src/bayesian_ach/design_mixture_diagnostic.py @@ -36,7 +36,8 @@ class MixtureDiagnosticConfig: noise_std: float = 1.0 alpha: float = 0.05 confidence_level: float = 0.95 - minimum_pure_power_wilson_lower: float = 0.70 + minimum_pure_retention_wilson_lower: float = 0.70 + minimum_rejection_power_wilson_lower: float = 0.70 threshold_seed: int = 196613 calibration_audit_seed: int = 262147 evaluation_seed: int = 324949 @@ -63,8 +64,10 @@ def validate(self) -> None: raise ValueError("alpha must lie in (0, 0.5)") if not 0.0 < self.confidence_level < 1.0: raise ValueError("confidence level must lie in (0, 1)") - if not 0.0 <= self.minimum_pure_power_wilson_lower <= 1.0: - raise ValueError("minimum pure-power lower bound must lie in [0, 1]") + if not 0.0 <= self.minimum_pure_retention_wilson_lower <= 1.0: + raise ValueError("minimum pure-retention lower bound must lie in [0, 1]") + if not 0.0 <= self.minimum_rejection_power_wilson_lower <= 1.0: + raise ValueError("minimum rejection-power lower bound must lie in [0, 1]") if len( { self.threshold_seed, @@ -421,7 +424,7 @@ def _audit_power( config.calibration_audit_replicates, config.confidence_level, ) - is_enabled = lower >= config.minimum_pure_power_wilson_lower + is_enabled = lower >= config.minimum_pure_retention_wilson_lower enabled.append(is_enabled) rows.append( { @@ -439,7 +442,7 @@ def _audit_power( "wilson_lower": lower, "wilson_upper": upper, "minimum_wilson_lower": ( - config.minimum_pure_power_wilson_lower + config.minimum_pure_retention_wilson_lower ), "candidate_enabled": is_enabled, "reasons": dict(sorted(reasons.items())), From dd032c951730d78ab6dfa35c111d610ae473a4c8 Mon Sep 17 00:00:00 2001 From: Florian Pfaff <6773539+FlorianPfaff@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:28:44 +0800 Subject: [PATCH 07/15] Audit diagnostic power by contrast --- src/bayesian_ach/design_mixture_diagnostic.py | 161 +++++++++++++++++- 1 file changed, 152 insertions(+), 9 deletions(-) diff --git a/src/bayesian_ach/design_mixture_diagnostic.py b/src/bayesian_ach/design_mixture_diagnostic.py index a7fb795..690b905 100644 --- a/src/bayesian_ach/design_mixture_diagnostic.py +++ b/src/bayesian_ach/design_mixture_diagnostic.py @@ -392,12 +392,20 @@ def _call( def _audit_power( signals: NDArray[np.float64], + full_signals: NDArray[np.float64], + indices: NDArray[np.int64], thresholds: DiagnosticThresholds, *, config: MixtureDiagnosticConfig, -) -> tuple[tuple[bool, ...], list[dict[str, Any]]]: +) -> tuple[ + tuple[bool, ...], + dict[tuple[int, int], bool], + bool, + bool, + list[dict[str, Any]], +]: rows: list[dict[str, Any]] = [] - enabled: list[bool] = [] + candidate_enabled: list[bool] = [] all_enabled = tuple(True for _ in DESIGN_CANDIDATE_NAMES) for candidate, name in enumerate(DESIGN_CANDIDATE_NAMES): correct = 0 @@ -425,31 +433,166 @@ def _audit_power( config.confidence_level, ) is_enabled = lower >= config.minimum_pure_retention_wilson_lower - enabled.append(is_enabled) + candidate_enabled.append(is_enabled) rows.append( { "scenario": "matched_pure", "candidate": name, + "audit_measure": "correct_pure_retention_rate", "replicates": config.calibration_audit_replicates, - "correct_pure_calls": correct, + "successes": correct, "wrong_pure_calls": wrong, "abstentions": ( config.calibration_audit_replicates - correct - wrong ), - "correct_call_rate": ( - correct / config.calibration_audit_replicates - ), + "rate": correct / config.calibration_audit_replicates, "wilson_lower": lower, "wilson_upper": upper, "minimum_wilson_lower": ( config.minimum_pure_retention_wilson_lower ), - "candidate_enabled": is_enabled, + "contrast_enabled": is_enabled, "reasons": dict(sorted(reasons.items())), } ) - return tuple(enabled), rows + enabled_tuple = tuple(candidate_enabled) + null_abstentions = 0 + null_reasons: dict[str, int] = {} + null_generator = np.zeros(signals.shape[0], dtype=float) + for replicate in range(config.calibration_audit_replicates): + scores = _simulate( + signals, + null_generator, + config=config, + rng=_rng(config.calibration_audit_seed, 1, replicate), + ) + call, reason = _call(scores, thresholds, enabled_tuple) + null_abstentions += int(call is None) + null_reasons[reason] = null_reasons.get(reason, 0) + 1 + null_lower, null_upper = _wilson_interval( + null_abstentions, + config.calibration_audit_replicates, + config.confidence_level, + ) + null_enabled = ( + null_lower >= config.minimum_rejection_power_wilson_lower + ) + rows.append( + { + "scenario": "null", + "candidate": "null", + "audit_measure": "correct_abstention_rate", + "replicates": config.calibration_audit_replicates, + "successes": null_abstentions, + "wrong_pure_calls": ( + config.calibration_audit_replicates - null_abstentions + ), + "abstentions": null_abstentions, + "rate": null_abstentions / config.calibration_audit_replicates, + "wilson_lower": null_lower, + "wilson_upper": null_upper, + "minimum_wilson_lower": ( + config.minimum_rejection_power_wilson_lower + ), + "contrast_enabled": null_enabled, + "reasons": dict(sorted(null_reasons.items())), + } + ) + + pair_enabled: dict[tuple[int, int], bool] = {} + for pair_index, pair in enumerate(_PAIR_INDICES): + generator = _scaled_mixture(full_signals, *pair)[indices] + abstentions = 0 + reasons: dict[str, int] = {} + for replicate in range(config.calibration_audit_replicates): + scores = _simulate( + signals, + generator, + config=config, + rng=_rng( + config.calibration_audit_seed, + 2, + pair_index, + replicate, + ), + ) + call, reason = _call(scores, thresholds, enabled_tuple) + abstentions += int(call is None) + reasons[reason] = reasons.get(reason, 0) + 1 + lower, upper = _wilson_interval( + abstentions, + config.calibration_audit_replicates, + config.confidence_level, + ) + is_enabled = lower >= config.minimum_rejection_power_wilson_lower + pair_enabled[pair] = is_enabled + rows.append( + { + "scenario": "fifty_fifty_mixture", + "candidate": ( + f"{DESIGN_CANDIDATE_NAMES[pair[0]]}+" + f"{DESIGN_CANDIDATE_NAMES[pair[1]]}" + ), + "audit_measure": "correct_abstention_rate", + "replicates": config.calibration_audit_replicates, + "successes": abstentions, + "wrong_pure_calls": ( + config.calibration_audit_replicates - abstentions + ), + "abstentions": abstentions, + "rate": abstentions / config.calibration_audit_replicates, + "wilson_lower": lower, + "wilson_upper": upper, + "minimum_wilson_lower": ( + config.minimum_rejection_power_wilson_lower + ), + "contrast_enabled": is_enabled, + "reasons": dict(sorted(reasons.items())), + } + ) + + probe, _, _ = _out_of_span_probe(full_signals) + out_abstentions = 0 + out_reasons: dict[str, int] = {} + for replicate in range(config.calibration_audit_replicates): + scores = _simulate( + signals, + probe[indices], + config=config, + rng=_rng(config.calibration_audit_seed, 3, replicate), + ) + call, reason = _call(scores, thresholds, enabled_tuple) + out_abstentions += int(call is None) + out_reasons[reason] = out_reasons.get(reason, 0) + 1 + out_lower, out_upper = _wilson_interval( + out_abstentions, + config.calibration_audit_replicates, + config.confidence_level, + ) + out_enabled = out_lower >= config.minimum_rejection_power_wilson_lower + rows.append( + { + "scenario": "out_of_span_probe", + "candidate": "full_grid_orthogonalized_tanh_surprise", + "audit_measure": "correct_abstention_rate", + "replicates": config.calibration_audit_replicates, + "successes": out_abstentions, + "wrong_pure_calls": ( + config.calibration_audit_replicates - out_abstentions + ), + "abstentions": out_abstentions, + "rate": out_abstentions / config.calibration_audit_replicates, + "wilson_lower": out_lower, + "wilson_upper": out_upper, + "minimum_wilson_lower": ( + config.minimum_rejection_power_wilson_lower + ), + "contrast_enabled": out_enabled, + "reasons": dict(sorted(out_reasons.items())), + } + ) + return enabled_tuple, pair_enabled, out_enabled, null_enabled, rows def _evaluate_pure( signals: NDArray[np.float64], From 101222a644abb62196f1007e0b96c3aca873b61c Mon Sep 17 00:00:00 2001 From: Florian Pfaff <6773539+FlorianPfaff@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:29:22 +0800 Subject: [PATCH 08/15] Propagate contrast power status --- src/bayesian_ach/design_mixture_diagnostic.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/bayesian_ach/design_mixture_diagnostic.py b/src/bayesian_ach/design_mixture_diagnostic.py index 690b905..8ee7259 100644 --- a/src/bayesian_ach/design_mixture_diagnostic.py +++ b/src/bayesian_ach/design_mixture_diagnostic.py @@ -594,6 +594,7 @@ def _audit_power( ) return enabled_tuple, pair_enabled, out_enabled, null_enabled, rows + def _evaluate_pure( signals: NDArray[np.float64], thresholds: DiagnosticThresholds, @@ -648,6 +649,7 @@ def _evaluate_null( signals: NDArray[np.float64], thresholds: DiagnosticThresholds, enabled: tuple[bool, ...], + audit_enabled: bool, *, config: MixtureDiagnosticConfig, ) -> list[dict[str, Any]]: @@ -677,6 +679,12 @@ def _evaluate_null( "false_pure_call_rate": false_calls / config.evaluation_replicates, "wilson_lower": lower, "wilson_upper": upper, + "contrast_enabled_from_audit": audit_enabled, + "claim_status": ( + "evaluated" + if audit_enabled + else "mandatory_abstain_underpowered" + ), "reasons": dict(sorted(reasons.items())), } ] @@ -700,6 +708,7 @@ def _evaluate_mixtures( indices: NDArray[np.int64], thresholds: DiagnosticThresholds, enabled: tuple[bool, ...], + pair_enabled: dict[tuple[int, int], bool], *, config: MixtureDiagnosticConfig, ) -> list[dict[str, Any]]: @@ -740,6 +749,12 @@ def _evaluate_mixtures( ), "wilson_lower": lower, "wilson_upper": upper, + "contrast_enabled_from_audit": pair_enabled[(first, second)], + "claim_status": ( + "evaluated" + if pair_enabled[(first, second)] + else "mandatory_abstain_underpowered" + ), "reasons": dict(sorted(reasons.items())), } ) @@ -752,6 +767,7 @@ def _evaluate_out_of_span( indices: NDArray[np.int64], thresholds: DiagnosticThresholds, enabled: tuple[bool, ...], + audit_enabled: bool, *, config: MixtureDiagnosticConfig, ) -> list[dict[str, Any]]: @@ -785,6 +801,12 @@ def _evaluate_out_of_span( "false_pure_call_rate": false_calls / config.evaluation_replicates, "wilson_lower": lower, "wilson_upper": upper, + "contrast_enabled_from_audit": audit_enabled, + "claim_status": ( + "evaluated" + if audit_enabled + else "mandatory_abstain_underpowered" + ), "full_grid_prestandardization_residual_sd": residual_scale, "full_grid_maximum_absolute_mean_inner_product": ( maximum_inner_product From 98ac369feec5fa35b1ffddea1647366a70f4a120 Mon Sep 17 00:00:00 2001 From: Florian Pfaff <6773539+FlorianPfaff@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:29:54 +0800 Subject: [PATCH 09/15] Apply contrast-specific audit gates --- src/bayesian_ach/design_mixture_diagnostic.py | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/src/bayesian_ach/design_mixture_diagnostic.py b/src/bayesian_ach/design_mixture_diagnostic.py index 8ee7259..3372860 100644 --- a/src/bayesian_ach/design_mixture_diagnostic.py +++ b/src/bayesian_ach/design_mixture_diagnostic.py @@ -885,8 +885,16 @@ def run_mixture_diagnostic( indices = np.repeat(np.arange(counts.size), counts) signals = np.asarray(full_signals[indices], dtype=float) thresholds = _calibrate(signals, config=config) - enabled, audit_rows = _audit_power( + ( + enabled, + pair_enabled, + out_enabled, + null_enabled, + audit_rows, + ) = _audit_power( signals, + full_signals, + indices, thresholds, config=config, ) @@ -900,6 +908,7 @@ def run_mixture_diagnostic( signals, thresholds, enabled, + null_enabled, config=config, ) mixture_rows = _evaluate_mixtures( @@ -908,6 +917,7 @@ def run_mixture_diagnostic( indices, thresholds, enabled, + pair_enabled, config=config, ) out_rows = _evaluate_out_of_span( @@ -916,6 +926,7 @@ def run_mixture_diagnostic( indices, thresholds, enabled, + out_enabled, config=config, ) geometry_rows = _geometry_rows( @@ -951,12 +962,30 @@ def run_mixture_diagnostic( name: enabled[index] for index, name in enumerate(DESIGN_CANDIDATE_NAMES) }, + "pair_power_enabled": { + ( + f"{DESIGN_CANDIDATE_NAMES[first]}+" + f"{DESIGN_CANDIDATE_NAMES[second]}" + ): pair_enabled[(first, second)] + for first, second in _PAIR_INDICES + }, + "null_power_enabled": null_enabled, + "out_of_span_power_enabled": out_enabled, "minimum_matched_pure_wilson_lower": min( float(row["wilson_lower"]) for row in pure_rows ), - "maximum_mixture_false_pure_wilson_upper": max( + "maximum_mixture_false_pure_wilson_upper_descriptive_all": max( float(row["wilson_upper"]) for row in mixture_rows ), + "maximum_enabled_mixture_false_pure_wilson_upper": max( + ( + float(row["wilson_upper"]) + for row in mixture_rows + if bool(row["contrast_enabled_from_audit"]) + ), + default=None, + ), + "mixture_contrasts_enabled_count": sum(pair_enabled.values()), "null_false_pure_wilson_upper": float(null_rows[0]["wilson_upper"]), "out_of_span_false_pure_wilson_upper": float( out_rows[0]["wilson_upper"] @@ -972,7 +1001,8 @@ def run_mixture_diagnostic( == 3, "all_fifteen_pairwise_composites": len(_PAIR_INDICES) == 15, "three_fold_cross_fitting": config.folds == 3, - "candidate_power_gate_applied": True, + "candidate_and_contrast_power_gates_applied": True, + "calibration_quantile_rank_fixed_before_evaluation": True, "evaluation_not_used_for_thresholds": True, }, } From 7c754972a77335c7157572df6ff0ee417d2e4a5a Mon Sep 17 00:00:00 2001 From: Florian Pfaff <6773539+FlorianPfaff@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:30:14 +0800 Subject: [PATCH 10/15] Record diagnostic calibration rule --- src/bayesian_ach/design_mixture_diagnostic.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/bayesian_ach/design_mixture_diagnostic.py b/src/bayesian_ach/design_mixture_diagnostic.py index 3372860..906c8ea 100644 --- a/src/bayesian_ach/design_mixture_diagnostic.py +++ b/src/bayesian_ach/design_mixture_diagnostic.py @@ -937,6 +937,14 @@ def run_mixture_diagnostic( threshold_rows = [ { "candidate": name, + "alpha": config.alpha, + "calibration_replicates": config.calibration_replicates, + "upper_conformal_rank_one_based": int( + math.ceil( + (config.calibration_replicates + 1) + * (1.0 - config.alpha) + ) + ), "pure_over_null_familywise": thresholds.pure_over_null, "winner_over_runner_familywise": thresholds.winner_over_runner, "pairwise_composite_over_pure_familywise": ( @@ -958,6 +966,36 @@ def run_mixture_diagnostic( ), "design": config.design, "budget": config.budget, + "calibration_rule": { + "alpha": config.alpha, + "upper_conformal_quantile": ( + "one-based order statistic ceil((n+1)*(1-alpha))" + ), + "rank": int( + math.ceil( + (config.calibration_replicates + 1) + * (1.0 - config.alpha) + ) + ), + "pure_over_null_family": "maximum over six pure candidates", + "winner_runner_family": "best-versus-second-best pure gap", + "composite_family": ( + "maximum held-out score across all 15 nonnegative pairs, " + "calibrated separately under each matched pure candidate" + ), + "residual_gof_family": ( + "candidate-specific cross-fitted residual ratio" + ), + }, + "audit_power_definitions": { + "matched_pure": "correct pure-call retention rate", + "null": "correct abstention rate", + "mixture_pair": "correct abstention rate for that exact pair", + "out_of_span": "correct abstention rate for the fixed probe", + "minimum_wilson_lower": ( + config.minimum_rejection_power_wilson_lower + ), + }, "candidate_power_enabled": { name: enabled[index] for index, name in enumerate(DESIGN_CANDIDATE_NAMES) From c0bdf4f465c7c83c168369a6131e9548e77d27dc Mon Sep 17 00:00:00 2001 From: Florian Pfaff <6773539+FlorianPfaff@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:30:29 +0800 Subject: [PATCH 11/15] Verify both diagnostic power gates --- src/bayesian_ach/design_mixture_diagnostic_cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bayesian_ach/design_mixture_diagnostic_cli.py b/src/bayesian_ach/design_mixture_diagnostic_cli.py index 65652eb..d009d49 100644 --- a/src/bayesian_ach/design_mixture_diagnostic_cli.py +++ b/src/bayesian_ach/design_mixture_diagnostic_cli.py @@ -196,7 +196,8 @@ def verify_mixture_diagnostic_artifact(directory: Path) -> dict[str, Any]: or config.get("threshold_seed") != 196613 or config.get("calibration_audit_seed") != 262147 or config.get("evaluation_seed") != 324949 - or config.get("minimum_pure_power_wilson_lower") != 0.70 + or config.get("minimum_pure_retention_wilson_lower") != 0.70 + or config.get("minimum_rejection_power_wilson_lower") != 0.70 or config.get("folds") != 3 or config.get("budget") != 60 ): From edb47de018b7c4839df3fe3f64089aa9b69820eb Mon Sep 17 00:00:00 2001 From: Florian Pfaff <6773539+FlorianPfaff@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:30:42 +0800 Subject: [PATCH 12/15] Test contrast-specific power contract --- tests/test_design_mixture_diagnostic.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_design_mixture_diagnostic.py b/tests/test_design_mixture_diagnostic.py index c71a606..7d5a4af 100644 --- a/tests/test_design_mixture_diagnostic.py +++ b/tests/test_design_mixture_diagnostic.py @@ -30,7 +30,8 @@ def test_locked_config_uses_fresh_disjoint_streams_and_power_gate() -> None: config.calibration_audit_seed, config.evaluation_seed, ) == (196613, 262147, 324949) - assert config.minimum_pure_power_wilson_lower == 0.70 + assert config.minimum_pure_retention_wilson_lower == 0.70 + assert config.minimum_rejection_power_wilson_lower == 0.70 assert config.folds == 3 with pytest.raises(ValueError, match="distinct"): MixtureDiagnosticConfig( @@ -53,7 +54,8 @@ def test_crossfit_pairwise_cone_recovers_noiseless_positive_mixture() -> None: ) assert scores.composite_pair == (0, 1) assert scores.composite_score > float(np.max(scores.pure_scores)) - assert scores.pure_residual_ratios[scores.winner] > 1.0 + assert np.isfinite(scores.pure_residual_ratios).all() + assert np.all(scores.pure_residual_ratios > 0.0) def test_call_requires_power_signal_separation_composite_and_gof() -> None: @@ -133,7 +135,8 @@ def _dummy_result() -> SimpleNamespace: "streams_disjoint": True, "all_fifteen_pairwise_composites": True, "three_fold_cross_fitting": True, - "candidate_power_gate_applied": True, + "candidate_and_contrast_power_gates_applied": True, + "calibration_quantile_rank_fixed_before_evaluation": True, "evaluation_not_used_for_thresholds": True, }, }, From 0919d8a1b6ea4cdf64f43060d7d6eb3d20c67788 Mon Sep 17 00:00:00 2001 From: Florian Pfaff <6773539+FlorianPfaff@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:31:02 +0800 Subject: [PATCH 13/15] Define diagnostic quantiles and power measures --- docs/design_mixture_diagnostic.md | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/docs/design_mixture_diagnostic.md b/docs/design_mixture_diagnostic.md index ca75bca..217467f 100644 --- a/docs/design_mixture_diagnostic.md +++ b/docs/design_mixture_diagnostic.md @@ -25,12 +25,28 @@ sets separate upper thresholds for: - a cross-fitted residual lack-of-fit ratio, defined as validation squared error divided by the variance fitted on the corresponding training fold. -The pure-over-null and winner-over-runner thresholds are calibrated from the -maximum statistics under the null. A pure label is enabled only if an independent -calibration audit gives a 95% Wilson lower bound of at least 0.70 for its correct -pure-call rate. Otherwise every evaluation call with that winner must abstain. -This power rule was fixed before evaluation and is not relaxed for difficult -candidates. +Every upper threshold uses the one-based order statistic +`ceil((n+1)(1-alpha))`. With `n=200` and familywise `alpha=0.05`, this is +rank 191. The pure-over-null statistic is the maximum over all six pure +candidates under the null. The composite statistic is the maximum over all 15 +pairs, calibrated separately under each matched pure candidate. The +winner/runner statistic is the best-minus-second-best pure score under the null; +the residual threshold is candidate-specific. + +The independent audit measures four distinct forms of power, each with a 95% +Wilson interval: + +- correct pure-call retention, separately for each pure candidate; +- correct abstention under the null; +- correct abstention, separately for each of the 15 mixture pairs; and +- correct abstention for the fixed out-of-span probe. + +A candidate or contrast is enabled only when its own audit Wilson lower bound is +at least 0.70. An underpowered pure winner is forced to abstain. Pair/null/probe +evaluation rates remain descriptive but receive the status +`mandatory_abstain_underpowered` when their corresponding audit gate fails. +Thus an easy contrast cannot license a claim for a difficult pair. These power +rules were fixed before evaluation and are not relaxed after seeing its output. The fixed streams and replicate counts are: From 564ab3cfe42f67867d70a5d5869c7c4401d94c8e Mon Sep 17 00:00:00 2001 From: Florian Pfaff <6773539+FlorianPfaff@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:32:18 +0800 Subject: [PATCH 14/15] Disambiguate diagnostic audit counters --- src/bayesian_ach/design_mixture_diagnostic.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/bayesian_ach/design_mixture_diagnostic.py b/src/bayesian_ach/design_mixture_diagnostic.py index 906c8ea..9f7a1aa 100644 --- a/src/bayesian_ach/design_mixture_diagnostic.py +++ b/src/bayesian_ach/design_mixture_diagnostic.py @@ -410,7 +410,7 @@ def _audit_power( for candidate, name in enumerate(DESIGN_CANDIDATE_NAMES): correct = 0 wrong = 0 - reasons: dict[str, int] = {} + pure_reasons: dict[str, int] = {} for replicate in range(config.calibration_audit_replicates): scores = _simulate( signals, @@ -426,7 +426,7 @@ def _audit_power( call, reason = _call(scores, thresholds, all_enabled) correct += int(call == candidate) wrong += int(call is not None and call != candidate) - reasons[reason] = reasons.get(reason, 0) + 1 + pure_reasons[reason] = pure_reasons.get(reason, 0) + 1 lower, upper = _wilson_interval( correct, config.calibration_audit_replicates, @@ -452,7 +452,7 @@ def _audit_power( config.minimum_pure_retention_wilson_lower ), "contrast_enabled": is_enabled, - "reasons": dict(sorted(reasons.items())), + "reasons": dict(sorted(pure_reasons.items())), } ) @@ -504,7 +504,7 @@ def _audit_power( for pair_index, pair in enumerate(_PAIR_INDICES): generator = _scaled_mixture(full_signals, *pair)[indices] abstentions = 0 - reasons: dict[str, int] = {} + pair_reasons: dict[str, int] = {} for replicate in range(config.calibration_audit_replicates): scores = _simulate( signals, @@ -519,7 +519,7 @@ def _audit_power( ) call, reason = _call(scores, thresholds, enabled_tuple) abstentions += int(call is None) - reasons[reason] = reasons.get(reason, 0) + 1 + pair_reasons[reason] = pair_reasons.get(reason, 0) + 1 lower, upper = _wilson_interval( abstentions, config.calibration_audit_replicates, @@ -548,7 +548,7 @@ def _audit_power( config.minimum_rejection_power_wilson_lower ), "contrast_enabled": is_enabled, - "reasons": dict(sorted(reasons.items())), + "reasons": dict(sorted(pair_reasons.items())), } ) From 5dc8c9280c5d3e042ce25d926c0d856ed9776ad9 Mon Sep 17 00:00:00 2001 From: Florian Pfaff <6773539+FlorianPfaff@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:45:52 +0200 Subject: [PATCH 15/15] Freeze mixture-aware design diagnostic --- docs/design_mixture_diagnostic.md | 17 +++ .../SHA256SUMS.csv | 10 ++ .../artifact_manifest.json | 101 ++++++++++++++++++ .../calibration_audit.csv | 24 +++++ .../geometry.csv | 16 +++ .../mixture_evaluation.csv | 16 +++ .../null_evaluation.csv | 2 + .../out_of_span_evaluation.csv | 2 + .../pure_evaluation.csv | 7 ++ .../summary.json | 99 +++++++++++++++++ .../thresholds.csv | 7 ++ 11 files changed, 301 insertions(+) create mode 100644 results/design-mixture-diagnostic-n60/SHA256SUMS.csv create mode 100644 results/design-mixture-diagnostic-n60/artifact_manifest.json create mode 100644 results/design-mixture-diagnostic-n60/calibration_audit.csv create mode 100644 results/design-mixture-diagnostic-n60/geometry.csv create mode 100644 results/design-mixture-diagnostic-n60/mixture_evaluation.csv create mode 100644 results/design-mixture-diagnostic-n60/null_evaluation.csv create mode 100644 results/design-mixture-diagnostic-n60/out_of_span_evaluation.csv create mode 100644 results/design-mixture-diagnostic-n60/pure_evaluation.csv create mode 100644 results/design-mixture-diagnostic-n60/summary.json create mode 100644 results/design-mixture-diagnostic-n60/thresholds.csv diff --git a/docs/design_mixture_diagnostic.md b/docs/design_mixture_diagnostic.md index 217467f..70c8a02 100644 --- a/docs/design_mixture_diagnostic.md +++ b/docs/design_mixture_diagnostic.md @@ -82,6 +82,23 @@ Passing this diagnostic would support only the declared simulation family and would not constitute a main-paper robustness claim without an independent freeze. +## Frozen one-time result + +The one-time evaluation was produced at commit +`564ab3cfe42f67867d70a5d5869c7c4401d94c8e` with configuration digest +`338ac9eea51f37d09b46303bdec66d5acc8d5e6fb16740c9afb036de3063c5a7`. +All six matched-pure contrasts and the null contrast passed their independent +audit-power gates. Nine of the 15 pairwise mixtures passed their own audit gate; +the other six are reported as `mandatory_abstain_underpowered`. The fixed +out-of-span probe also failed its audit-power gate and is therefore disabled. + +Across the enabled pairwise-mixture contrasts, the largest evaluation +false-pure-call rate was 0.195 (95% Wilson upper bound 0.2554). The null +false-pure-call rate was 0.005 (upper bound 0.0278). The disabled out-of-span +probe had a descriptive false-pure-call rate of 0.73 and cannot support an +open-set claim. The frozen result therefore establishes partial, targeted +sensitivity with mandatory abstention, not general open-set robustness. + ## Reproduction From a clean checkout of the exact producer commit: diff --git a/results/design-mixture-diagnostic-n60/SHA256SUMS.csv b/results/design-mixture-diagnostic-n60/SHA256SUMS.csv new file mode 100644 index 0000000..6976ebe --- /dev/null +++ b/results/design-mixture-diagnostic-n60/SHA256SUMS.csv @@ -0,0 +1,10 @@ +path,bytes,sha256 +summary.json,4231,1311330a9f7b3278eb16dbb48d65b57b8337a695c6ad2026cac1efbff795b3c2 +thresholds.csv,870,2da2db3544398e84e124ae591ad3acac08d0ff9d716871067795fba630b9410f +calibration_audit.csv,5328,1333d0eae21afa6f87675db935d2ce69e026f0149d7d9c6707341a25bfdac444 +pure_evaluation.csv,1192,58b2d40de305d5f4c789b3b92c0110bdd74e98c8649928f8cea1c6810cded2d1 +null_evaluation.csv,279,782463aeb6919c0753ebe05212d15744a46e8df2e4d71a4a4d279b46741df074 +mixture_evaluation.csv,3154,37a864c5a8410b7ea91fcefee4a4ae2df11fc1a1d397fb4f7651520abbcf5a3e +out_of_span_evaluation.csv,500,181099f69267c8dc119d45fbd7b66a0a003cc05912f5dae459ef2742c5f07a9f +geometry.csv,1722,30bcf7c226d17f8a543cf08f58052ab33529b68d9e1a585c05da7ce3f4001d49 +artifact_manifest.json,3488,bf08f458a3f34339a62c8ae74fc0a1cafdf7f74f21e939d174df86b021bc21b7 diff --git a/results/design-mixture-diagnostic-n60/artifact_manifest.json b/results/design-mixture-diagnostic-n60/artifact_manifest.json new file mode 100644 index 0000000..8c68de8 --- /dev/null +++ b/results/design-mixture-diagnostic-n60/artifact_manifest.json @@ -0,0 +1,101 @@ +{ + "artifact": "post_failure_mixture_aware_design_diagnostic", + "configuration": { + "alpha": 0.05, + "budget": 60, + "calibration_audit_replicates": 200, + "calibration_audit_seed": 262147, + "calibration_replicates": 200, + "confidence_level": 0.95, + "design": "maximin_optimized", + "effect_size": 1.0, + "evaluation_replicates": 200, + "evaluation_seed": 324949, + "folds": 3, + "minimum_pure_retention_wilson_lower": 0.7, + "minimum_rejection_power_wilson_lower": 0.7, + "noise_std": 1.0, + "threshold_seed": 196613 + }, + "configuration_sha256": "338ac9eea51f37d09b46303bdec66d5acc8d5e6fb16740c9afb036de3063c5a7", + "files": [ + { + "bytes": 4231, + "path": "summary.json", + "sha256": "1311330a9f7b3278eb16dbb48d65b57b8337a695c6ad2026cac1efbff795b3c2" + }, + { + "bytes": 870, + "path": "thresholds.csv", + "sha256": "2da2db3544398e84e124ae591ad3acac08d0ff9d716871067795fba630b9410f" + }, + { + "bytes": 5328, + "path": "calibration_audit.csv", + "sha256": "1333d0eae21afa6f87675db935d2ce69e026f0149d7d9c6707341a25bfdac444" + }, + { + "bytes": 1192, + "path": "pure_evaluation.csv", + "sha256": "58b2d40de305d5f4c789b3b92c0110bdd74e98c8649928f8cea1c6810cded2d1" + }, + { + "bytes": 279, + "path": "null_evaluation.csv", + "sha256": "782463aeb6919c0753ebe05212d15744a46e8df2e4d71a4a4d279b46741df074" + }, + { + "bytes": 3154, + "path": "mixture_evaluation.csv", + "sha256": "37a864c5a8410b7ea91fcefee4a4ae2df11fc1a1d397fb4f7651520abbcf5a3e" + }, + { + "bytes": 500, + "path": "out_of_span_evaluation.csv", + "sha256": "181099f69267c8dc119d45fbd7b66a0a003cc05912f5dae459ef2742c5f07a9f" + }, + { + "bytes": 1722, + "path": "geometry.csv", + "sha256": "30bcf7c226d17f8a543cf08f58052ab33529b68d9e1a585c05da7ce3f4001d49" + } + ], + "inputs": [ + { + "checksum_table_sha256": "44a5188c43bda52e6fc9dc7007cf2de44a9671e9c5477ac88c3173c06cfdbd80", + "kind": "immutable_original_design_stress_failure", + "manifest_sha256": "d840a2ec34f5a386109c7f985033b53144fdcc1b1a0e0592b3274c0e38902b64", + "producer_commit": "c71695fda83ae93407599a909097962ee3fa9e0e", + "verified_payload_count": 9 + }, + { + "allocation_bytes": 14450, + "allocation_file": "optimal_design_allocation_seed7.csv", + "allocation_file_seed_field_present": false, + "allocation_seed": 7, + "allocation_seed_source": "explicit_cli_metadata", + "allocation_sha256": "a823be49faf6c6cbebf60b11d4b5ca895cf7734d6e9c577ee98f97a5907b69b2", + "construction_contract": { + "all_three_allocations_reconstructed": true, + "allocation_seed": 7, + "comparator_cap_semantics": "the maximin cap does not apply to the deterministic coupled-novelty or uniform-factorial constructors", + "maximin_max_point_fraction": 0.15, + "maximin_maximum_count_by_budget": { + "60": 9 + } + }, + "design_budgets": { + "coupled_novelty": 60, + "maximin_optimized": 60, + "uniform_factorial": 60 + }, + "kind": "chronologically_locked_primary_design_allocation", + "source_code_sha": "1b2028929ac6ebc1cce0882f0c22af9918044342", + "source_repository": "IPS-Stuttgart/Bayesian-ACh" + } + ], + "producer_commit": "564ab3cfe42f67867d70a5d5869c7c4401d94c8e", + "producer_git_dirty": false, + "repository": "IPS-Stuttgart/Bayesian-ACh", + "schema_version": 1 +} diff --git a/results/design-mixture-diagnostic-n60/calibration_audit.csv b/results/design-mixture-diagnostic-n60/calibration_audit.csv new file mode 100644 index 0000000..0e6d857 --- /dev/null +++ b/results/design-mixture-diagnostic-n60/calibration_audit.csv @@ -0,0 +1,24 @@ +scenario,candidate,audit_measure,replicates,successes,wrong_pure_calls,abstentions,rate,wilson_lower,wilson_upper,minimum_wilson_lower,contrast_enabled,reasons +matched_pure,innovation_l2,correct_pure_retention_rate,200,158,5,37,0.79,0.7283538314009755,0.84071587930021,0.7,True,"{""pairwise_composite_better"":1,""pure_ambiguity"":29,""pure_call"":163,""residual_lack_of_fit"":7}" +matched_pure,surprise,correct_pure_retention_rate,200,164,4,32,0.82,0.7608852497127511,0.8670537414057983,0.7,True,"{""pairwise_composite_better"":2,""pure_ambiguity"":25,""pure_call"":168,""residual_lack_of_fit"":5}" +matched_pure,gain,correct_pure_retention_rate,200,188,0,12,0.94,0.8980683070423372,0.9653478057456681,0.7,True,"{""pairwise_composite_better"":3,""pure_call"":188,""residual_lack_of_fit"":9}" +matched_pure,update_l2,correct_pure_retention_rate,200,181,0,19,0.905,0.8563983883518006,0.9383368972826135,0.7,True,"{""pairwise_composite_better"":4,""pure_ambiguity"":7,""pure_call"":181,""residual_lack_of_fit"":8}" +matched_pure,information_gain,correct_pure_retention_rate,200,188,0,12,0.94,0.8980683070423372,0.9653478057456681,0.7,True,"{""pairwise_composite_better"":3,""pure_ambiguity"":1,""pure_call"":188,""residual_lack_of_fit"":8}" +matched_pure,change_probability,correct_pure_retention_rate,200,183,0,17,0.915,0.8681041292843443,0.9462542498225244,0.7,True,"{""pairwise_composite_better"":7,""pure_call"":183,""residual_lack_of_fit"":10}" +null,null,correct_abstention_rate,200,197,3,197,0.985,0.9568342712073097,0.9948857622067417,0.7,True,"{""null_not_rejected"":196,""pure_ambiguity"":1,""pure_call"":3}" +fifty_fifty_mixture,innovation_l2+surprise,correct_abstention_rate,200,86,114,86,0.43,0.36334322333434416,0.4992951223584731,0.7,False,"{""null_not_rejected"":1,""pairwise_composite_better"":3,""pure_ambiguity"":80,""pure_call"":114,""residual_lack_of_fit"":2}" +fifty_fifty_mixture,innovation_l2+gain,correct_abstention_rate,200,168,32,168,0.84,0.7828592983793289,0.8843258796841297,0.7,True,"{""pairwise_composite_better"":103,""pure_ambiguity"":64,""pure_call"":32,""residual_lack_of_fit"":1}" +fifty_fifty_mixture,innovation_l2+update_l2,correct_abstention_rate,200,174,26,174,0.87,0.8163364812775159,0.9097179772033067,0.7,True,"{""pairwise_composite_better"":101,""pure_ambiguity"":73,""pure_call"":26}" +fifty_fifty_mixture,innovation_l2+information_gain,correct_abstention_rate,200,139,61,139,0.695,0.6280144790202163,0.7546358436926496,0.7,False,"{""pairwise_composite_better"":49,""pure_ambiguity"":89,""pure_call"":61,""residual_lack_of_fit"":1}" +fifty_fifty_mixture,innovation_l2+change_probability,correct_abstention_rate,200,166,34,166,0.83,0.7718411637168737,0.8757209208741302,0.7,True,"{""pairwise_composite_better"":98,""pure_ambiguity"":66,""pure_call"":34,""residual_lack_of_fit"":2}" +fifty_fifty_mixture,surprise+gain,correct_abstention_rate,200,187,13,187,0.935,0.8919809207009312,0.961623645350847,0.7,True,"{""pairwise_composite_better"":66,""pure_ambiguity"":121,""pure_call"":13}" +fifty_fifty_mixture,surprise+update_l2,correct_abstention_rate,200,163,37,163,0.815,0.7554293723884824,0.8626980719938395,0.7,True,"{""pairwise_composite_better"":74,""pure_ambiguity"":88,""pure_call"":37,""residual_lack_of_fit"":1}" +fifty_fifty_mixture,surprise+information_gain,correct_abstention_rate,200,123,77,123,0.615,0.5459986722024608,0.6796669027307678,0.7,False,"{""pairwise_composite_better"":43,""pure_ambiguity"":78,""pure_call"":77,""residual_lack_of_fit"":2}" +fifty_fifty_mixture,surprise+change_probability,correct_abstention_rate,200,161,39,161,0.805,0.7445595562538726,0.8539447946559949,0.7,True,"{""null_not_rejected"":1,""pairwise_composite_better"":109,""pure_ambiguity"":49,""pure_call"":39,""residual_lack_of_fit"":2}" +fifty_fifty_mixture,gain+update_l2,correct_abstention_rate,200,108,92,108,0.54,0.47082289188953336,0.6076694820002854,0.7,False,"{""pairwise_composite_better"":41,""pure_ambiguity"":64,""pure_call"":92,""residual_lack_of_fit"":3}" +fifty_fifty_mixture,gain+information_gain,correct_abstention_rate,200,143,57,143,0.715,0.6488465397039482,0.7730499699538271,0.7,False,"{""pairwise_composite_better"":60,""pure_ambiguity"":83,""pure_call"":57}" +fifty_fifty_mixture,gain+change_probability,correct_abstention_rate,200,200,0,200,1.0,0.9811546736227335,1.0,0.7,True,"{""pairwise_composite_better"":95,""pure_ambiguity"":105}" +fifty_fifty_mixture,update_l2+information_gain,correct_abstention_rate,200,94,106,94,0.47,0.40204754873265425,0.5390831708499817,0.7,False,"{""pairwise_composite_better"":25,""pure_ambiguity"":66,""pure_call"":106,""residual_lack_of_fit"":3}" +fifty_fifty_mixture,update_l2+change_probability,correct_abstention_rate,200,198,2,198,0.99,0.9642782382838231,0.9972533418664555,0.7,True,"{""pairwise_composite_better"":110,""pure_ambiguity"":88,""pure_call"":2}" +fifty_fifty_mixture,information_gain+change_probability,correct_abstention_rate,200,191,9,191,0.955,0.9167032959981803,0.976147456998507,0.7,True,"{""pairwise_composite_better"":133,""pure_ambiguity"":58,""pure_call"":9}" +out_of_span_probe,full_grid_orthogonalized_tanh_surprise,correct_abstention_rate,200,52,148,52,0.26,0.20413830063574687,0.3249074560253411,0.7,False,"{""null_not_rejected"":31,""pure_ambiguity"":18,""pure_call"":148,""residual_lack_of_fit"":3}" diff --git a/results/design-mixture-diagnostic-n60/geometry.csv b/results/design-mixture-diagnostic-n60/geometry.csv new file mode 100644 index 0000000..f1af3c7 --- /dev/null +++ b/results/design-mixture-diagnostic-n60/geometry.csv @@ -0,0 +1,16 @@ +first_candidate,second_candidate,best_pure_candidate,best_pure_affine_residual,true_pair_affine_residual,crossfit_oracle_log_score_gap_index,below_five_nat_power_index +innovation_l2,surprise,surprise,0.06476225526808417,2.5106422755032005e-31,1.8825461925239708,True +innovation_l2,gain,update_l2,0.2377395072664464,1.8957570419251692e-31,6.398602138782985,False +innovation_l2,update_l2,update_l2,0.2576905807957704,6.575124830138023e-32,6.878315004395995,False +innovation_l2,information_gain,information_gain,0.22866732483457436,2.3189189901515054e-31,6.177903187677524,False +innovation_l2,change_probability,change_probability,0.2715733105685233,1.2703433913178208e-31,7.2076488292956435,False +surprise,gain,update_l2,0.3334070092601475,3.315809387586691e-31,8.632119836108483,False +surprise,update_l2,update_l2,0.2882713806954706,6.993725703550589e-31,7.599039143577237,False +surprise,information_gain,information_gain,0.17513118479245662,1.4426499236756656e-31,4.841393639879367,True +surprise,change_probability,change_probability,0.24059459272699613,1.431864715987097e-31,6.467723248570308,False +gain,update_l2,gain,0.15154391801250797,2.865420764592673e-31,4.233107381393687,True +gain,information_gain,update_l2,0.24549715570701747,3.96330703488968e-31,6.586043161858336,False +gain,change_probability,gain,0.719444663119841,2.762008232078141e-31,16.260041052371072,False +update_l2,information_gain,update_l2,0.13734921104922596,1.9408238026238305e-31,3.861009040227646,True +update_l2,change_probability,update_l2,0.6095014120398538,1.0145670551702333e-30,14.277733472165602,False +information_gain,change_probability,change_probability,0.4585249642762335,3.867138932999944e-31,11.322768797515211,False diff --git a/results/design-mixture-diagnostic-n60/mixture_evaluation.csv b/results/design-mixture-diagnostic-n60/mixture_evaluation.csv new file mode 100644 index 0000000..aa0dc19 --- /dev/null +++ b/results/design-mixture-diagnostic-n60/mixture_evaluation.csv @@ -0,0 +1,16 @@ +first_candidate,second_candidate,replicates,false_pure_calls,abstentions,false_pure_call_rate,wilson_lower,wilson_upper,contrast_enabled_from_audit,claim_status,reasons +innovation_l2,surprise,200,112,88,0.56,0.4907167533465867,0.6270218074881414,False,mandatory_abstain_underpowered,"{""pairwise_composite_better"":3,""pure_ambiguity"":84,""pure_call"":112,""residual_lack_of_fit"":1}" +innovation_l2,gain,200,25,175,0.125,0.08611974475712646,0.17801425002582344,True,evaluated,"{""pairwise_composite_better"":121,""pure_ambiguity"":53,""pure_call"":25,""residual_lack_of_fit"":1}" +innovation_l2,update_l2,200,15,185,0.075,0.045974917184382,0.12004361023629455,True,evaluated,"{""pairwise_composite_better"":93,""pure_ambiguity"":89,""pure_call"":15,""residual_lack_of_fit"":3}" +innovation_l2,information_gain,200,53,147,0.265,0.20868154147109114,0.33017576192622416,False,mandatory_abstain_underpowered,"{""pairwise_composite_better"":39,""pure_ambiguity"":107,""pure_call"":53,""residual_lack_of_fit"":1}" +innovation_l2,change_probability,200,30,170,0.15,0.10713593562241996,0.20605579284166659,True,evaluated,"{""pairwise_composite_better"":87,""pure_ambiguity"":81,""pure_call"":30,""residual_lack_of_fit"":2}" +surprise,gain,200,13,187,0.065,0.03837635464915299,0.10801907929906893,True,evaluated,"{""pairwise_composite_better"":65,""pure_ambiguity"":121,""pure_call"":13,""residual_lack_of_fit"":1}" +surprise,update_l2,200,21,179,0.105,0.06970748792681017,0.15518031991123038,True,evaluated,"{""pairwise_composite_better"":54,""pure_ambiguity"":123,""pure_call"":21,""residual_lack_of_fit"":2}" +surprise,information_gain,200,98,102,0.49,0.4215627833121318,0.5588141232154135,False,mandatory_abstain_underpowered,"{""pairwise_composite_better"":41,""pure_ambiguity"":59,""pure_call"":98,""residual_lack_of_fit"":2}" +surprise,change_probability,200,39,161,0.195,0.14605520534400512,0.25544044374612745,True,evaluated,"{""pairwise_composite_better"":108,""pure_ambiguity"":50,""pure_call"":39,""residual_lack_of_fit"":3}" +gain,update_l2,200,93,107,0.465,0.3971856415700402,0.5341335312763685,False,mandatory_abstain_underpowered,"{""pairwise_composite_better"":40,""pure_ambiguity"":66,""pure_call"":93,""residual_lack_of_fit"":1}" +gain,information_gain,200,43,157,0.215,0.1637187625450758,0.2770230734899661,False,mandatory_abstain_underpowered,"{""pairwise_composite_better"":66,""pure_ambiguity"":91,""pure_call"":43}" +gain,change_probability,200,0,200,0.0,0.0,0.018845326377266564,True,evaluated,"{""pairwise_composite_better"":109,""pure_ambiguity"":91}" +update_l2,information_gain,200,100,100,0.5,0.43136085960389187,0.5686391403961081,False,mandatory_abstain_underpowered,"{""pairwise_composite_better"":31,""pure_ambiguity"":65,""pure_call"":100,""residual_lack_of_fit"":4}" +update_l2,change_probability,200,0,200,0.0,0.0,0.018845326377266564,True,evaluated,"{""pairwise_composite_better"":117,""pure_ambiguity"":83}" +information_gain,change_probability,200,9,191,0.045,0.023852543001492997,0.08329670400181957,True,evaluated,"{""pairwise_composite_better"":124,""pure_ambiguity"":65,""pure_call"":9,""residual_lack_of_fit"":2}" diff --git a/results/design-mixture-diagnostic-n60/null_evaluation.csv b/results/design-mixture-diagnostic-n60/null_evaluation.csv new file mode 100644 index 0000000..95c43bd --- /dev/null +++ b/results/design-mixture-diagnostic-n60/null_evaluation.csv @@ -0,0 +1,2 @@ +replicates,false_pure_calls,abstentions,false_pure_call_rate,wilson_lower,wilson_upper,contrast_enabled_from_audit,claim_status,reasons +200,1,199,0.005,0.0008831687156009762,0.027773704397892923,True,evaluated,"{""null_not_rejected"":196,""pure_ambiguity"":3,""pure_call"":1}" diff --git a/results/design-mixture-diagnostic-n60/out_of_span_evaluation.csv b/results/design-mixture-diagnostic-n60/out_of_span_evaluation.csv new file mode 100644 index 0000000..588194d --- /dev/null +++ b/results/design-mixture-diagnostic-n60/out_of_span_evaluation.csv @@ -0,0 +1,2 @@ +probe,replicates,false_pure_calls,abstentions,false_pure_call_rate,wilson_lower,wilson_upper,contrast_enabled_from_audit,claim_status,full_grid_prestandardization_residual_sd,full_grid_maximum_absolute_mean_inner_product,reasons +full_grid_orthogonalized_tanh_surprise,200,146,54,0.73,0.6645656480133157,0.7867655018531416,False,mandatory_abstain_underpowered,0.09057218165490002,4.3261690526226935e-14,"{""null_not_rejected"":29,""pure_ambiguity"":18,""pure_call"":146,""residual_lack_of_fit"":7}" diff --git a/results/design-mixture-diagnostic-n60/pure_evaluation.csv b/results/design-mixture-diagnostic-n60/pure_evaluation.csv new file mode 100644 index 0000000..9a19e99 --- /dev/null +++ b/results/design-mixture-diagnostic-n60/pure_evaluation.csv @@ -0,0 +1,7 @@ +candidate,replicates,correct_pure_calls,wrong_pure_calls,abstentions,correct_call_rate,wilson_lower,wilson_upper,candidate_enabled_from_audit,reasons +innovation_l2,200,160,3,37,0.8,0.7391448134346212,0.8495479907390189,True,"{""pairwise_composite_better"":2,""pure_ambiguity"":33,""pure_call"":163,""residual_lack_of_fit"":2}" +surprise,200,173,3,24,0.865,0.8107086162924745,0.9055342954521209,True,"{""pairwise_composite_better"":2,""pure_ambiguity"":17,""pure_call"":176,""residual_lack_of_fit"":5}" +gain,200,183,0,17,0.915,0.8681041292843443,0.9462542498225244,True,"{""pairwise_composite_better"":5,""pure_ambiguity"":2,""pure_call"":183,""residual_lack_of_fit"":10}" +update_l2,200,176,1,23,0.88,0.8276574790170326,0.9180200729362449,True,"{""pairwise_composite_better"":11,""pure_ambiguity"":6,""pure_call"":177,""residual_lack_of_fit"":6}" +information_gain,200,191,0,9,0.955,0.9167032959981803,0.976147456998507,True,"{""pairwise_composite_better"":1,""pure_ambiguity"":4,""pure_call"":191,""residual_lack_of_fit"":4}" +change_probability,200,188,0,12,0.94,0.8980683070423372,0.9653478057456681,True,"{""pairwise_composite_better"":5,""pure_call"":188,""residual_lack_of_fit"":7}" diff --git a/results/design-mixture-diagnostic-n60/summary.json b/results/design-mixture-diagnostic-n60/summary.json new file mode 100644 index 0000000..bdf9066 --- /dev/null +++ b/results/design-mixture-diagnostic-n60/summary.json @@ -0,0 +1,99 @@ +{ + "audit_power_definitions": { + "matched_pure": "correct pure-call retention rate", + "minimum_wilson_lower": 0.7, + "mixture_pair": "correct abstention rate for that exact pair", + "null": "correct abstention rate", + "out_of_span": "correct abstention rate for the fixed probe" + }, + "budget": 60, + "calibration_rule": { + "alpha": 0.05, + "composite_family": "maximum held-out score across all 15 nonnegative pairs, calibrated separately under each matched pure candidate", + "pure_over_null_family": "maximum over six pure candidates", + "rank": 191, + "residual_gof_family": "candidate-specific cross-fitted residual ratio", + "upper_conformal_quantile": "one-based order statistic ceil((n+1)*(1-alpha))", + "winner_runner_family": "best-versus-second-best pure gap" + }, + "candidate_power_enabled": { + "change_probability": true, + "gain": true, + "information_gain": true, + "innovation_l2": true, + "surprise": true, + "update_l2": true + }, + "chronology": "Method, thresholds, power gate, and fresh streams were committed after immutable baseline failure d1251ddd and before this evaluation.", + "design": "maximin_optimized", + "experiment": "post_failure_pairwise_cone_abstention_diagnostic", + "input_provenance": { + "allocation": { + "allocation_bytes": 14450, + "allocation_file": "optimal_design_allocation_seed7.csv", + "allocation_file_seed_field_present": false, + "allocation_seed": 7, + "allocation_seed_source": "explicit_cli_metadata", + "allocation_sha256": "a823be49faf6c6cbebf60b11d4b5ca895cf7734d6e9c577ee98f97a5907b69b2", + "construction_contract": { + "all_three_allocations_reconstructed": true, + "allocation_seed": 7, + "comparator_cap_semantics": "the maximin cap does not apply to the deterministic coupled-novelty or uniform-factorial constructors", + "maximin_max_point_fraction": 0.15, + "maximin_maximum_count_by_budget": { + "60": 9 + } + }, + "design_budgets": { + "coupled_novelty": 60, + "maximin_optimized": 60, + "uniform_factorial": 60 + }, + "kind": "chronologically_locked_primary_design_allocation", + "source_code_sha": "1b2028929ac6ebc1cce0882f0c22af9918044342", + "source_repository": "IPS-Stuttgart/Bayesian-ACh" + }, + "baseline": { + "checksum_table_sha256": "44a5188c43bda52e6fc9dc7007cf2de44a9671e9c5477ac88c3173c06cfdbd80", + "kind": "immutable_original_design_stress_failure", + "manifest_sha256": "d840a2ec34f5a386109c7f985033b53144fdcc1b1a0e0592b3274c0e38902b64", + "producer_commit": "c71695fda83ae93407599a909097962ee3fa9e0e", + "verified_payload_count": 9 + } + }, + "interpretation": "Separately configured sensitivity after the immutable original stress failure; not a main-paper open-set robustness claim.", + "maximum_enabled_mixture_false_pure_wilson_upper": 0.25544044374612745, + "maximum_mixture_false_pure_wilson_upper_descriptive_all": 0.6270218074881414, + "minimum_matched_pure_wilson_lower": 0.7391448134346212, + "mixture_contrasts_enabled_count": 9, + "null_false_pure_wilson_upper": 0.027773704397892923, + "null_power_enabled": true, + "out_of_span_false_pure_wilson_upper": 0.7867655018531416, + "out_of_span_power_enabled": false, + "pair_power_enabled": { + "gain+change_probability": true, + "gain+information_gain": false, + "gain+update_l2": false, + "information_gain+change_probability": true, + "innovation_l2+change_probability": true, + "innovation_l2+gain": true, + "innovation_l2+information_gain": false, + "innovation_l2+surprise": false, + "innovation_l2+update_l2": true, + "surprise+change_probability": true, + "surprise+gain": true, + "surprise+information_gain": false, + "surprise+update_l2": true, + "update_l2+change_probability": true, + "update_l2+information_gain": false + }, + "schema_version": 1, + "technical_gates": { + "all_fifteen_pairwise_composites": true, + "calibration_quantile_rank_fixed_before_evaluation": true, + "candidate_and_contrast_power_gates_applied": true, + "evaluation_not_used_for_thresholds": true, + "streams_disjoint": true, + "three_fold_cross_fitting": true + } +} diff --git a/results/design-mixture-diagnostic-n60/thresholds.csv b/results/design-mixture-diagnostic-n60/thresholds.csv new file mode 100644 index 0000000..a80b845 --- /dev/null +++ b/results/design-mixture-diagnostic-n60/thresholds.csv @@ -0,0 +1,7 @@ +candidate,alpha,calibration_replicates,upper_conformal_rank_one_based,pure_over_null_familywise,winner_over_runner_familywise,pairwise_composite_over_pure_familywise,residual_ratio_candidate_specific,candidate_enabled_from_audit +innovation_l2,0.05,200,191,3.4208193855608755,2.5286033861336534,2.5100567987347375,1.388053910062808,True +surprise,0.05,200,191,3.4208193855608755,2.5286033861336534,3.02308827824136,1.3696735645372042,True +gain,0.05,200,191,3.4208193855608755,2.5286033861336534,2.3614359361155977,1.3828665623933518,True +update_l2,0.05,200,191,3.4208193855608755,2.5286033861336534,1.8596029313160045,1.3459343250817886,True +information_gain,0.05,200,191,3.4208193855608755,2.5286033861336534,3.728345840374672,1.4137673381357303,True +change_probability,0.05,200,191,3.4208193855608755,2.5286033861336534,2.5054124365266546,1.3589679383289155,True