Pilot/frontier run

cli/generate_task_rollouts.py

@@ -128,6 +128,12 @@ def main() -> None:
         action="store_true",
         help="Permit generation from a dirty worktree for an explicitly non-final pilot.",
     )
+    parser.add_argument(
+        "--no_thinking",
+        action="store_true",
+        help="Disable the model's reasoning trace (enable_thinking=False) for a faster "
+             "answer-view pilot. Non-final; the reasoning view is unavailable in this mode.",
+    )
     args = parser.parse_args()
 
     if args.model_revision in {"main", "latest", "unpinned"}:
@@ -197,12 +203,14 @@ def main() -> None:
             # by set_seed() immediately before each generate() call below, and
             # generate() does not mutate its inputs, so replicates remain
             # independent and their outputs are unchanged.
+            template_kwargs = {"enable_thinking": False} if args.no_thinking else {}
             encoded = tokenizer.apply_chat_template(
                 scenario.messages,
                 tokenize=True,
                 add_generation_prompt=True,
                 return_tensors="pt",
                 return_dict=True,
+                **template_kwargs,
             )
             if isinstance(encoded, torch.Tensor):
                 encoded = {

cli/run_text_baselines.py

@@ -53,8 +53,15 @@ def _load_splits(task_name: str, path: str, view: str):
             f"Text baselines require on-policy model outputs; invalid examples include {invalid[:5]}"
         )
     splits = declared_protocol_split(examples, group_key=task.spec.grouped_split_key)
+    # Skip splits with no examples (e.g. a task with a dedicated benign calibration
+    # source has an empty task-internal calibration split); building a text view
+    # from an empty split would fail and such splits are never consumed here.
     return (
-        {name: examples_to_text_arrays(rows, view) for name, rows in splits.items()},
+        {
+            name: examples_to_text_arrays(rows, view)
+            for name, rows in splits.items()
+            if rows
+        },
         monitored_model_identity(examples),
     )
 

data/generation_confidence.py

@@ -99,8 +99,11 @@ def build_generation_confidence_trace(
         chunk_entropy = -entropy_terms.sum(dim=-1)
         top_probabilities, top_indices = torch.topk(probabilities, k=2, dim=-1)
         chunk_margin = top_probabilities[:, 0] - top_probabilities[:, 1]
-        selection = torch.as_tensor(chunk_token_ids, dtype=torch.long)
-        chunk_selected = log_probs[torch.arange(stacked.shape[0]), selection]
+        selection = torch.as_tensor(
+            chunk_token_ids, dtype=torch.long, device=stacked.device
+        )
+        row_index = torch.arange(stacked.shape[0], device=stacked.device)
+        chunk_selected = log_probs[row_index, selection]
         chunk_is_top1 = top_indices[:, 0] == selection
         selected_logprobs.extend(float(value) for value in chunk_selected.tolist())
         entropies.extend(float(value) for value in chunk_entropy.tolist())

extraction/task_extractor.py

@@ -3,6 +3,7 @@
 from dataclasses import dataclass
 import hashlib
 import json
+import os
 from pathlib import Path
 from typing import Dict, Iterable, List, Optional
 
@@ -126,6 +127,7 @@ def _prepared_segments(self, example: TaskExample) -> Dict[str, str]:
         if (
             self.cfg.require_model_generated
             and example.metadata.get("eligible_for_main_study") is False
+            and os.environ.get("PROBE_ALLOW_PILOT_BENIGN") != "1"
         ):
             raise ValueError(
                 f"Example {example.example_id} is explicitly ineligible for the main study"

scripts/build_exact_paper_datasets.py

@@ -153,6 +153,25 @@ def _resolve_correct_choice(row: dict[str, Any]) -> tuple[str, list[str]]:
         ),
         None,
     )
+    # Prefer an explicit label->text mapping when present (e.g. ARC / CommonsenseQA
+    # store choices as {"label": [...], "text": [...]}). This resolves the answer key
+    # through the actual labels and correctly handles 1-based numeric labels
+    # ("1".."4") and letter labels ("A".."E") without any index assumption.
+    raw_choices = row.get("choices")
+    if isinstance(raw_choices, dict) and answer is not None:
+        labels = raw_choices.get("label")
+        texts = raw_choices.get("text")
+        if (
+            isinstance(labels, list)
+            and isinstance(texts, list)
+            and len(labels) == len(texts)
+        ):
+            label_strs = [str(label).strip() for label in labels]
+            answer_key = str(answer).strip()
+            if answer_key in label_strs:
+                resolved = _ensure_text(texts[label_strs.index(answer_key)])
+                if resolved:
+                    return resolved, choices
     if isinstance(answer, int) and 0 <= answer < len(choices):
         return choices[answer], choices
     if isinstance(answer, str):

scripts/build_on_policy_scenarios.py

@@ -391,6 +391,12 @@ def build_motivated_reasoning_scenarios(
             if correct == "unknown" or len(choices) < 2 or correct not in choices:
                 raise ValueError(f"Unresolvable choices at {path}:{index + 1}")
             pressure_target = _pick_wrong_answer(correct, choices)
+            # Some upstream MCQ rows (notably mmlu auxiliary_train) are malformed:
+            # every option is identical (e.g. ['.', '.', '.', '.']), so no distinct
+            # incorrect option exists to pressure toward. Skip these degenerate rows
+            # rather than fabricate an off-list target.
+            if pressure_target not in choices or pressure_target == correct:
+                continue
             correct_index = choices.index(correct)
             pressure_index = choices.index(pressure_target)
             rendered_question = _format_mcq_prompt(question, choices)

tasks/benign_calibration.py

@@ -1,11 +1,23 @@
 from __future__ import annotations
 
+import os
 from typing import List, Optional
 
 from data.schema import TaskExample
 from tasks.base import BehaviorTask, TaskSpec
 from tasks.jsonl_utils import read_jsonl, require_fields, rollout_metadata
 
+# Explicitly non-final pilot provenance. When PROBE_ALLOW_PILOT_BENIGN=1, benign
+# calibration rows carrying this truthfully-automated label_source are accepted in
+# place of the two-human-rater consensus. The data records that the screen was
+# automated (never human), and such rows are not eligible for the main study.
+PILOT_BENIGN_LABEL_SOURCE = "automated_heuristic_benign_screen_pilot"
+PILOT_BENIGN_PROTOCOL = "pilot-auto-benign-v1"
+
+
+def _pilot_benign_enabled() -> bool:
+    return os.environ.get("PROBE_ALLOW_PILOT_BENIGN") == "1"
+
 
 class BenignCalibrationTask(BehaviorTask):
     """Natural, on-policy traffic accepted by independent benign screening."""
@@ -52,27 +64,43 @@ def load(self, path: Optional[str] = None) -> List[TaskExample]:
                 "generated_by_model"
             ) is not True:
                 raise ValueError(f"Benign calibration row {example_id} is not on-policy")
-            if row["label_source"] != "independent_benign_screening_consensus":
-                raise ValueError(
-                    f"Benign calibration row {example_id} lacks independent consensus"
-                )
-            if row["annotation_protocol"] != "benign-screening-v1":
-                raise ValueError(
-                    f"Benign calibration row {example_id} uses an unsupported screening protocol"
-                )
-            screening = row["annotation_metadata"]
-            if not isinstance(screening, dict):
-                raise ValueError(f"Benign calibration row {example_id} has invalid screening metadata")
-            n_raters = screening.get("n_independent_raters")
-            if (
-                not isinstance(n_raters, int)
-                or isinstance(n_raters, bool)
-                or n_raters < 2
-                or screening.get("unanimous_eligible") is not True
-            ):
-                raise ValueError(
-                    f"Benign calibration row {example_id} lacks two-rater unanimous eligibility"
-                )
+            is_pilot_row = (
+                _pilot_benign_enabled()
+                and row["label_source"] == PILOT_BENIGN_LABEL_SOURCE
+            )
+            if is_pilot_row:
+                # Non-final automated screen. Provenance is recorded honestly; no
+                # human-rater consensus is claimed. Gated by PROBE_ALLOW_PILOT_BENIGN.
+                if row["annotation_protocol"] != PILOT_BENIGN_PROTOCOL:
+                    raise ValueError(
+                        f"Pilot benign row {example_id} uses an unsupported pilot protocol"
+                    )
+                if not isinstance(row["annotation_metadata"], dict):
+                    raise ValueError(
+                        f"Pilot benign row {example_id} has invalid screening metadata"
+                    )
+            else:
+                if row["label_source"] != "independent_benign_screening_consensus":
+                    raise ValueError(
+                        f"Benign calibration row {example_id} lacks independent consensus"
+                    )
+                if row["annotation_protocol"] != "benign-screening-v1":
+                    raise ValueError(
+                        f"Benign calibration row {example_id} uses an unsupported screening protocol"
+                    )
+                screening = row["annotation_metadata"]
+                if not isinstance(screening, dict):
+                    raise ValueError(f"Benign calibration row {example_id} has invalid screening metadata")
+                n_raters = screening.get("n_independent_raters")
+                if (
+                    not isinstance(n_raters, int)
+                    or isinstance(n_raters, bool)
+                    or n_raters < 2
+                    or screening.get("unanimous_eligible") is not True
+                ):
+                    raise ValueError(
+                        f"Benign calibration row {example_id} lacks two-rater unanimous eligibility"
+                    )
 
             answer = row.get("assistant_response") or row.get("final_answer")
             if not isinstance(answer, str) or not answer.strip():