Complete Prime research claim evidence

Makefile

@@ -14,6 +14,7 @@ build build-env build-utils deploy update-version update-utils-version hub-valid
 eval eval-e1 eval-e2 report-network-logs e1 e2 e3 e4 e5 e6 \
 baseline-e1 baseline-e2 \
 lab-check lab-run-e1 lab-run-e1-judge lab-run-e2 env-eval-e1 env-eval-e2 config-validate svbench-v0.1-check suite-v1-check eval-e3 eval-e4 eval-e5 eval-e6 \
+prime-models prime-plan-research-claim \
 \
 data-e1 data-e1-ood data-e1-test data-e2 data-e2-local data-e2-test data-all data-test-all data-public-mini clone-e2-sources \
 hf-e1-meta hf-e2-meta hf-e1-push hf-e2-push hf-e1p-meta hf-e2p-meta hf-e1p-push hf-e2p-push hf-push-all \
@@ -31,6 +32,8 @@ export UV_PYTHON := python3.12
 
 # Team slug for Prime Intellect Hub deployment (can be overridden)
 TEAM ?= intertwine
+PRIME_TRAIN_FLAGS ?= --plain
+PRIME_SECRET_FLAGS ?=
 
 # Version bump type for deployments (patch, minor, major, or none)
 BUMP ?= patch
@@ -98,12 +101,14 @@ help:
 	@$(ECHO) "  make eval-e2 MODELS=... N=2 INCLUDE_TOOLS=true  - Reproducible E2 evals (config-verification)"
 	@$(ECHO) "  make baseline-e1 MODEL=... N=100                 - E1 baselines on public mini set"
 	@$(ECHO) "  make baseline-e2 MODEL=... N=100                 - E2 baselines on public mini set"
-	@$(ECHO) "  make lab-check                                    - Prime CLI v0.5+ compatibility gate"
-	@$(ECHO) "  make lab-run-e1                                   - Hosted RL training for E1 (prime rl run)"
-	@$(ECHO) "  make lab-run-e1-judge                             - Hosted RL training for E1 judge variant (WP3c)"
-	@$(ECHO) "  make lab-run-e2                                   - Hosted RL training for E2 (prime rl run)"
-	@$(ECHO) "  make env-eval-e1 MODEL=... TEAM=intertwine N=100  - Fallback prime env eval wrapper for E1"
-	@$(ECHO) "  make env-eval-e2 MODEL=... TEAM=intertwine N=50   - Fallback prime env eval wrapper for E2"
+	@$(ECHO) "  make lab-check                                    - Prime CLI compatibility gate"
+	@$(ECHO) "  make lab-run-e1                                   - Hosted training for E1 (prime train)"
+	@$(ECHO) "  make lab-run-e1-judge                             - Hosted training for E1 judge variant (WP3c)"
+	@$(ECHO) "  make lab-run-e2                                   - Hosted training for E2 (prime train)"
+	@$(ECHO) "  make env-eval-e1 MODEL=... TEAM=intertwine N=100  - Prime eval wrapper for E1"
+	@$(ECHO) "  make env-eval-e2 MODEL=... TEAM=intertwine N=50   - Prime eval wrapper for E2"
+	@$(ECHO) "  make prime-models                                 - List available hosted-training models"
+	@$(ECHO) "  make prime-plan-research-claim PROFILE=pilot      - Generate matched launch configs"
 	@$(ECHO) "  make config-validate                              - Validate eval/RL/ablation configs"
 	@$(ECHO) "  make svbench-v0.1-check                           - Validate SV-Bench v0.1 release package"
 	@$(ECHO) "  make suite-v1-check                               - Validate suite v1 completion checklist"
@@ -957,7 +962,7 @@ info:
 		$(ECHO) "Status: $(YELLOW)Run 'make setup' to get started$(NC)"; \
 	fi
 
-# Prime CLI compatibility gate (v0.5+)
+# Prime CLI compatibility gate
 lab-check: venv
 	@$(ECHO) "$(YELLOW)Running Prime CLI compatibility checks...$(NC)"
 	@$(ACTIVATE) && python scripts/prime_lab_check.py
@@ -967,7 +972,18 @@ config-validate: venv
 	@$(ACTIVATE) && uv run python scripts/validate_svbench_configs.py
 	@$(ECHO) "$(GREEN)✓ Config validation complete$(NC)"
 
-# Hosted RL training via Prime CLI v0.5+ (WP3a/WP3b)
+prime-models: venv
+	@$(ACTIVATE) && prime train models --output json --plain
+
+prime-plan-research-claim: venv
+	@PROFILE=$${PROFILE:-pilot}; \
+	MODEL_ARG=""; \
+	SECRET_ARG=""; \
+	if [ -n "$${MODEL:-}" ]; then MODEL_ARG="--model $$MODEL"; fi; \
+	if [ -n "$${SECRET_ENV_FILE:-}" ]; then SECRET_ARG="--secret-env-file $$SECRET_ENV_FILE"; fi; \
+	$(ACTIVATE) && uv run python scripts/prepare_prime_research_claim.py $$MODEL_ARG $$SECRET_ARG --profile $$PROFILE
+
+# Hosted training via Prime CLI (WP3a/WP3b)
 lab-run-e1: venv
 	@REWARD_SOURCE=$${REWARD_SOURCE:-executable}; \
 	case "$$REWARD_SOURCE" in \
@@ -977,12 +993,20 @@ lab-run-e1: venv
 		legacy) CONFIG=configs/rl/e1.toml ;; \
 		*) $(ECHO) "$(RED)Error: REWARD_SOURCE must be executable, llm_judge, hybrid, or legacy$(NC)"; exit 1 ;; \
 	esac; \
-	$(ECHO) "$(YELLOW)Launching hosted RL training for E1 with $$REWARD_SOURCE reward ($$CONFIG)...$(NC)"; \
-	$(ACTIVATE) && prime rl run "$$CONFIG"
+	SECRET_FLAGS="$(PRIME_SECRET_FLAGS)"; \
+	if [[ "$$REWARD_SOURCE" == "llm_judge" || "$$REWARD_SOURCE" == "judge" || "$$REWARD_SOURCE" == "hybrid" ]]; then \
+		if [ -z "$$SECRET_FLAGS" ] && grep -q '^OPENAI_API_KEY=' .env 2>/dev/null; then SECRET_FLAGS="--env-file .env"; fi; \
+		if [ -z "$$SECRET_FLAGS" ]; then $(ECHO) "$(RED)Error: $$REWARD_SOURCE requires OPENAI_API_KEY via PRIME_SECRET_FLAGS='--env-file .env' or local .env$(NC)"; exit 1; fi; \
+	fi; \
+	$(ECHO) "$(YELLOW)Launching hosted training for E1 with $$REWARD_SOURCE reward ($$CONFIG)...$(NC)"; \
+	$(ACTIVATE) && prime train "$$CONFIG" $(PRIME_TRAIN_FLAGS) $$SECRET_FLAGS
 
 lab-run-e1-judge: venv
-	@$(ECHO) "$(YELLOW)Launching hosted RL training for E1 judge variant (WP3c)...$(NC)"
-	@$(ACTIVATE) && prime rl run configs/rl/e1_judge.toml
+	@SECRET_FLAGS="$(PRIME_SECRET_FLAGS)"; \
+	if [ -z "$$SECRET_FLAGS" ] && grep -q '^OPENAI_API_KEY=' .env 2>/dev/null; then SECRET_FLAGS="--env-file .env"; fi; \
+	if [ -z "$$SECRET_FLAGS" ]; then $(ECHO) "$(RED)Error: E1 judge requires OPENAI_API_KEY via PRIME_SECRET_FLAGS='--env-file .env' or local .env$(NC)"; exit 1; fi; \
+	$(ECHO) "$(YELLOW)Launching hosted training for E1 judge variant (WP3c)...$(NC)"; \
+	$(ACTIVATE) && prime train configs/rl/e1_judge.toml $(PRIME_TRAIN_FLAGS) $$SECRET_FLAGS
 
 lab-run-e2: venv
 	@REWARD_SOURCE=$${REWARD_SOURCE:-executable}; \
@@ -993,19 +1017,24 @@ lab-run-e2: venv
 		legacy) CONFIG=configs/rl/e2.toml ;; \
 		*) $(ECHO) "$(RED)Error: REWARD_SOURCE must be executable, llm_judge, hybrid, or legacy$(NC)"; exit 1 ;; \
 	esac; \
-	$(ECHO) "$(YELLOW)Launching hosted RL training for E2 with $$REWARD_SOURCE reward ($$CONFIG)...$(NC)"; \
-	$(ACTIVATE) && prime rl run "$$CONFIG"
+	SECRET_FLAGS="$(PRIME_SECRET_FLAGS)"; \
+	if [[ "$$REWARD_SOURCE" == "llm_judge" || "$$REWARD_SOURCE" == "judge" || "$$REWARD_SOURCE" == "hybrid" ]]; then \
+		if [ -z "$$SECRET_FLAGS" ] && grep -q '^OPENAI_API_KEY=' .env 2>/dev/null; then SECRET_FLAGS="--env-file .env"; fi; \
+		if [ -z "$$SECRET_FLAGS" ]; then $(ECHO) "$(RED)Error: $$REWARD_SOURCE requires OPENAI_API_KEY via PRIME_SECRET_FLAGS='--env-file .env' or local .env$(NC)"; exit 1; fi; \
+	fi; \
+	$(ECHO) "$(YELLOW)Launching hosted training for E2 with $$REWARD_SOURCE reward ($$CONFIG)...$(NC)"; \
+	$(ACTIVATE) && prime train "$$CONFIG" $(PRIME_TRAIN_FLAGS) $$SECRET_FLAGS
 
-# Fallback hosted-style eval parity (WP2.5a)
+# Hosted/fallback eval parity (WP2.5a)
 env-eval-e1: venv
 	@MODEL=$${MODEL:-Qwen/Qwen3-4B-Instruct-2507}; \
 	N=$${N:-100}; \
-	$(ACTIVATE) && prime env eval $(TEAM)/sv-env-network-logs --model $$MODEL --num-examples $$N
+	$(ACTIVATE) && prime eval run $(TEAM)/sv-env-network-logs --model $$MODEL --num-examples $$N --save-results --disable-tui
 
 env-eval-e2: venv
 	@MODEL=$${MODEL:-Qwen/Qwen3-4B-Instruct-2507}; \
 	N=$${N:-50}; \
-	$(ACTIVATE) && prime env eval $(TEAM)/sv-env-config-verification --model $$MODEL --num-examples $$N
+	$(ACTIVATE) && prime eval run $(TEAM)/sv-env-config-verification --model $$MODEL --num-examples $$N --save-results --disable-tui
 
 eval-e3: venv
 	@N=$${N:-10}; $(ACTIVATE) && uv run python scripts/eval_beta_env.py --env e3 --num-examples $$N

bench/compare_rewards.py

@@ -26,6 +26,10 @@
     "tool_budget",
     "trainer",
 )
+HOSTED_IDENTITY_FIELDS = (
+    "prime_environment_id",
+    "environment_version",
+)
 
 
 def _load_json(path: Path) -> dict[str, Any]:
@@ -48,19 +52,26 @@ def _load_manifest(path: Path) -> dict[str, Any]:
     return manifest
 
 
-def _budget_mismatches(manifests: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:
+def _field_mismatches(
+    manifests: dict[str, dict[str, Any]],
+    fields: tuple[str, ...],
+) -> dict[str, dict[str, Any]]:
     baseline = manifests["executable"]
     mismatches: dict[str, dict[str, Any]] = {}
     for variant, manifest in manifests.items():
         if variant == "executable":
             continue
-        for field in BUDGET_FIELDS:
+        for field in fields:
             if manifest.get(field) != baseline.get(field):
                 mismatches.setdefault(field, {})["executable"] = baseline.get(field)
                 mismatches[field][variant] = manifest.get(field)
     return mismatches
 
 
+def _budget_mismatches(manifests: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:
+    return _field_mismatches(manifests, BUDGET_FIELDS)
+
+
 def _flatten_metrics(summary: dict[str, Any]) -> dict[str, float]:
     flat: dict[str, float] = {}
 
@@ -81,10 +92,19 @@ def visit(prefix: str, value: Any) -> None:
     return flat
 
 
-def _load_metrics(manifest: dict[str, Any]) -> dict[str, float]:
+def _load_metrics(manifest: dict[str, Any], *, allow_incomplete: bool) -> dict[str, float]:
     manifest_path = Path(str(manifest["_manifest_path"]))
     summary_path = _resolve(str(manifest["metrics_summary_path"]), manifest_path.parent)
-    return _flatten_metrics(_load_json(summary_path))
+    summary = _load_json(summary_path)
+    run_status = summary.get("run_status")
+    if not allow_incomplete and manifest.get("platform_mode") == "hosted" and run_status != "COMPLETED":
+        raise ValueError(
+            f"Run {manifest.get('run_id')} is not complete enough for claim comparison: {run_status}"
+        )
+    metrics = _flatten_metrics(summary)
+    if not allow_incomplete and not metrics:
+        raise ValueError(f"Run {manifest.get('run_id')} has no comparable metrics")
+    return metrics
 
 
 def _delta(left: float | None, right: float | None) -> float | None:
@@ -93,15 +113,29 @@ def _delta(left: float | None, right: float | None) -> float | None:
     return right - left
 
 
-def _build_summary(env: str, manifests: dict[str, dict[str, Any]], allow_unmatched: bool) -> dict[str, Any]:
+def _build_summary(
+    env: str,
+    manifests: dict[str, dict[str, Any]],
+    allow_unmatched: bool,
+    allow_incomplete: bool = False,
+) -> dict[str, Any]:
     mismatches = _budget_mismatches(manifests)
     if mismatches and not allow_unmatched:
         raise ValueError(
             "Budget parity failed. Use --allow-unmatched only for exploratory reports. "
             f"Mismatched fields: {', '.join(sorted(mismatches))}"
         )
 
-    metrics = {variant: _load_metrics(manifest) for variant, manifest in manifests.items()}
+    hosted_identity_mismatches = _field_mismatches(manifests, HOSTED_IDENTITY_FIELDS)
+    if hosted_identity_mismatches and not allow_unmatched:
+        raise ValueError(
+            "Hosted identity parity failed. Use --allow-unmatched only for exploratory reports. "
+            f"Mismatched fields: {', '.join(sorted(hosted_identity_mismatches))}"
+        )
+    metrics = {
+        variant: _load_metrics(manifest, allow_incomplete=allow_incomplete)
+        for variant, manifest in manifests.items()
+    }
     metric_names = sorted({name for variant_metrics in metrics.values() for name in variant_metrics})
     rows: list[dict[str, Any]] = []
     for name in metric_names:
@@ -122,8 +156,11 @@ def _build_summary(env: str, manifests: dict[str, dict[str, Any]], allow_unmatch
     return {
         "environment": env,
         "allow_unmatched": allow_unmatched,
+        "allow_incomplete": allow_incomplete,
         "budget_fields": list(BUDGET_FIELDS),
         "budget_mismatches": mismatches,
+        "hosted_identity_fields": list(HOSTED_IDENTITY_FIELDS),
+        "hosted_identity_mismatches": hosted_identity_mismatches,
         "runs": {
             variant: {
                 "run_id": manifest["run_id"],
@@ -160,6 +197,24 @@ def render_markdown(summary: dict[str, Any]) -> str:
         )
     else:
         lines.append("All required budget fields match across executable, judge, and hybrid manifests.")
+    lines.extend(
+        [
+            "",
+            "## Hosted Identity",
+            "",
+        ]
+    )
+    if summary["hosted_identity_mismatches"]:
+        lines.append(
+            "Hosted environment identity/version differs across variants and is reported separately."
+        )
+        lines.append("")
+        lines.extend(
+            f"- `{field}`: {values}"
+            for field, values in sorted(summary["hosted_identity_mismatches"].items())
+        )
+    else:
+        lines.append("Hosted environment IDs and versions match across variants, or were not reported.")
     lines.extend(
         [
             "",
@@ -185,7 +240,7 @@ def render_markdown(summary: dict[str, Any]) -> str:
             "",
             "## Failure-Mode Notes",
             "",
-            "- Add representative trace links after real hosted or fallback-hosted runs complete.",
+            "- Inspect `results.jsonl` or `rollouts_raw.json` before quoting representative traces.",
             "- Treat judge-only gains cautiously unless unsupported claims and invalid schemas remain low.",
             "",
         ]
@@ -202,14 +257,15 @@ def main() -> int:
     parser.add_argument("--out", required=True, help="Markdown output path")
     parser.add_argument("--json-out", default=None, help="JSON summary output path")
     parser.add_argument("--allow-unmatched", action="store_true", help="Allow unmatched budgets")
+    parser.add_argument("--allow-incomplete", action="store_true", help="Allow incomplete/empty metrics")
     args = parser.parse_args()
 
     manifests = {
         "executable": _load_manifest(Path(args.executable)),
         "judge": _load_manifest(Path(args.judge)),
         "hybrid": _load_manifest(Path(args.hybrid)),
     }
-    summary = _build_summary(args.env, manifests, args.allow_unmatched)
+    summary = _build_summary(args.env, manifests, args.allow_unmatched, args.allow_incomplete)
 
     out_path = Path(args.out)
     out_path.parent.mkdir(parents=True, exist_ok=True)

bench/leaderboard/v0.1.json

@@ -1,5 +1,107 @@
 {
   "version": "v0.1",
   "scope": "E1/E2 only",
-  "entries": []
+  "entries": [
+    {
+      "environment": "e1",
+      "model": "Qwen/Qwen3.5-2B",
+      "evidence_grade": "exploratory",
+      "matched_claim": false,
+      "exploratory_reason": "E1 judge run used a separate hosted environment identity from executable/hybrid runs.",
+      "reward_source": "executable",
+      "prime_run_id": "bp6qd19u06wzeqkdrf0h24xx",
+      "run_manifest": "results/runs/svbench-research-claim-20260515T141203Z/bp6qd19u06wzeqkdrf0h24xx/run_manifest.json",
+      "metrics": {
+        "avg_at_1": 1.785,
+        "failed_rollouts": 0.0,
+        "is_truncated_mean": 0.0,
+        "no_response_count": 0.0,
+        "step": 50
+      }
+    },
+    {
+      "environment": "e1",
+      "model": "Qwen/Qwen3.5-2B",
+      "evidence_grade": "exploratory",
+      "matched_claim": false,
+      "exploratory_reason": "E1 judge run used a separate hosted environment identity from executable/hybrid runs.",
+      "reward_source": "llm_judge",
+      "prime_run_id": "j1i1ys9rpdcluvtqneql2o4l",
+      "run_manifest": "results/runs/svbench-research-claim-20260515T141203Z/j1i1ys9rpdcluvtqneql2o4l/run_manifest.json",
+      "metrics": {
+        "avg_at_1": 0.44,
+        "pass_at_1": 0.44,
+        "failed_rollouts": 0.0,
+        "is_truncated_mean": 0.0,
+        "no_response_count": 0.0,
+        "step": 50
+      }
+    },
+    {
+      "environment": "e1",
+      "model": "Qwen/Qwen3.5-2B",
+      "evidence_grade": "exploratory",
+      "matched_claim": false,
+      "exploratory_reason": "E1 judge run used a separate hosted environment identity from executable/hybrid runs.",
+      "reward_source": "hybrid",
+      "prime_run_id": "iiy65ubvx5xwke4dfb9vos64",
+      "run_manifest": "results/runs/svbench-research-claim-20260515T141203Z/iiy65ubvx5xwke4dfb9vos64/run_manifest.json",
+      "metrics": {
+        "avg_at_1": 1.7900000000000005,
+        "failed_rollouts": 0.0,
+        "is_truncated_mean": 0.0,
+        "no_response_count": 0.0,
+        "step": 50
+      }
+    },
+    {
+      "environment": "e2",
+      "model": "Qwen/Qwen3.5-9B",
+      "evidence_grade": "claim-grade",
+      "matched_claim": true,
+      "reward_source": "executable",
+      "prime_run_id": "hu1j9b61il8dydv2bh1aj97d",
+      "run_manifest": "results/runs/svbench-research-claim-20260515T141203Z/hu1j9b61il8dydv2bh1aj97d/run_manifest.json",
+      "metrics": {
+        "avg_at_1": 0.15714285714285714,
+        "failed_rollouts": 0.0,
+        "is_truncated_mean": 0.0,
+        "no_response_count": 0.0,
+        "step": 1
+      }
+    },
+    {
+      "environment": "e2",
+      "model": "Qwen/Qwen3.5-9B",
+      "evidence_grade": "claim-grade",
+      "matched_claim": true,
+      "reward_source": "llm_judge",
+      "prime_run_id": "h4y9hb6fjom4xfbclitb2938",
+      "run_manifest": "results/runs/svbench-research-claim-20260515T141203Z/h4y9hb6fjom4xfbclitb2938/run_manifest.json",
+      "metrics": {
+        "avg_at_1": 0.5,
+        "pass_at_1": 0.5,
+        "failed_rollouts": 0.0,
+        "is_truncated_mean": 0.0,
+        "no_response_count": 0.0,
+        "step": 1
+      }
+    },
+    {
+      "environment": "e2",
+      "model": "Qwen/Qwen3.5-9B",
+      "evidence_grade": "claim-grade",
+      "matched_claim": true,
+      "reward_source": "hybrid",
+      "prime_run_id": "hzu95jfm266370x9kr9wwgkz",
+      "run_manifest": "results/runs/svbench-research-claim-20260515T141203Z/hzu95jfm266370x9kr9wwgkz/run_manifest.json",
+      "metrics": {
+        "avg_at_1": 1.157142857142857,
+        "failed_rollouts": 0.0,
+        "is_truncated_mean": 0.0,
+        "no_response_count": 0.0,
+        "step": 1
+      }
+    }
+  ]
 }