Add frontier experiment orchestration stack

.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: scientific-integrity-tests
+
+on:
+  push:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+        with:
+          version: "0.11.21"
+          enable-cache: true
+
+      - name: Synchronize locked environment
+        run: uv sync --locked --extra dev
+
+      - name: Compile Python sources
+        run: uv run python -m compileall -q cli data evaluation extraction features interventions probes scripts task_benchmark tasks tests
+
+      - name: Run integrity suite
+        run: uv run pytest

.gitignore

@@ -23,6 +23,11 @@ results/
 outputs/
 data/raw_sources/
 data/final/
+data/scenarios/
+data/rollouts/
+data/annotations/
+data/labeled/
+data/audits/
 phase4_real_main_results/
 phase5_neurips_main_results/
 phase6_honesty_auxiliary_results/

cli/aggregate_task_results.py

@@ -3,8 +3,6 @@
 import argparse
 from pathlib import Path
 
-import pandas as pd
-
 from evaluation.aggregation import collect_results
 from evaluation.task_aggregation import (
     compute_task_fsei,
@@ -13,7 +11,7 @@
     select_best_view_layer,
 )
 from evaluation.task_model_selection import rank_models
-from evaluation.task_statistics import add_ci_columns
+from evaluation.task_statistics import add_seed_summary_columns
 
 
 GROUP_COLS = [
@@ -31,24 +29,32 @@
     "eval_auroc",
     "eval_auprc",
     "eval_recall_at_1pct_fpr",
+    "eval_recall_at_frozen_fpr",
+    "eval_fpr_at_frozen_threshold",
     "eval_brier",
     "eval_ece",
     "test_auroc",
     "test_auprc",
     "test_recall_at_1pct_fpr",
+    "test_recall_at_frozen_fpr",
+    "test_fpr_at_frozen_threshold",
     "test_brier",
     "test_ece",
     "transfer_auroc",
     "transfer_auprc",
     "transfer_recall_at_1pct_fpr",
+    "transfer_recall_at_frozen_fpr",
+    "transfer_fpr_at_frozen_threshold",
     "transfer_brier",
     "transfer_ece",
     "wall_clock_s",
 ]
 
 
 def main() -> None:
-    parser = argparse.ArgumentParser(description="Aggregate structured task-benchmark results")
+    parser = argparse.ArgumentParser(
+        description="Aggregate structured task-benchmark results"
+    )
     parser.add_argument("--results_dir", required=True)
     parser.add_argument("--selection_metric", default="eval_recall_at_1pct_fpr_mean")
     parser.add_argument("--bootstrap_samples", type=int, default=2000)
@@ -59,11 +65,16 @@ def main() -> None:
         print("No results found.")
         return
 
-    summary = add_ci_columns(
+    if "status" in df.columns:
+        failed = df[df["status"] != "ok"]
+        if not failed.empty:
+            raise RuntimeError(
+                f"Refusing to aggregate {len(failed)} failed/partial runs; repair or explicitly remove them"
+            )
+    summary = add_seed_summary_columns(
         df,
         group_cols=GROUP_COLS,
         metric_cols=METRIC_COLS,
-        n_boot=args.bootstrap_samples,
     )
     best = select_best_view_layer(summary, selection_metric=args.selection_metric)
     view_layer = make_view_layer_table(best)