Complete SV-Bench v0.1 planning gates

Makefile

@@ -13,7 +13,7 @@ build build-env build-utils deploy update-version update-utils-version hub-valid
 \
 eval eval-e1 eval-e2 report-network-logs e1 e2 e3 e4 e5 e6 \
 baseline-e1 baseline-e2 \
-lab-check lab-run-e1 lab-run-e1-judge lab-run-e2 env-eval-e1 env-eval-e2 \
+lab-check lab-run-e1 lab-run-e1-judge lab-run-e2 env-eval-e1 env-eval-e2 config-validate svbench-v0.1-check suite-v1-check eval-e3 eval-e4 eval-e5 eval-e6 \
 \
 data-e1 data-e1-ood data-e1-test data-e2 data-e2-local data-e2-test data-all data-test-all data-public-mini clone-e2-sources \
 hf-e1-meta hf-e2-meta hf-e1-push hf-e2-push hf-e1p-meta hf-e2p-meta hf-e1p-push hf-e2p-push hf-push-all \
@@ -104,6 +104,9 @@ help:
 	@$(ECHO) "  make lab-run-e2                                   - Hosted RL training for E2 (prime rl run)"
 	@$(ECHO) "  make env-eval-e1 MODEL=... TEAM=intertwine N=100  - Fallback prime env eval wrapper for E1"
 	@$(ECHO) "  make env-eval-e2 MODEL=... TEAM=intertwine N=50   - Fallback prime env eval wrapper for E2"
+	@$(ECHO) "  make config-validate                              - Validate eval/RL/ablation configs"
+	@$(ECHO) "  make svbench-v0.1-check                           - Validate SV-Bench v0.1 release package"
+	@$(ECHO) "  make suite-v1-check                               - Validate suite v1 completion checklist"
 	@$(ECHO) "  make report-network-logs [RUN_IDS=\"id1 id2\"]    - Generate E1 metrics report (JSON)"
 	@$(ECHO) ""
 	@$(ECHO) "$(YELLOW)Hub Dataset Management:$(NC)"
@@ -959,18 +962,39 @@ lab-check: venv
 	@$(ECHO) "$(YELLOW)Running Prime CLI compatibility checks...$(NC)"
 	@$(ACTIVATE) && python scripts/prime_lab_check.py
 
+config-validate: venv
+	@$(ECHO) "$(YELLOW)Validating SV-Bench configs...$(NC)"
+	@$(ACTIVATE) && uv run python scripts/validate_svbench_configs.py
+	@$(ECHO) "$(GREEN)✓ Config validation complete$(NC)"
+
 # Hosted RL training via Prime CLI v0.5+ (WP3a/WP3b)
 lab-run-e1: venv
-	@$(ECHO) "$(YELLOW)Launching hosted RL training for E1 (network-logs)...$(NC)"
-	@$(ACTIVATE) && prime rl run configs/rl/e1.toml
+	@REWARD_SOURCE=$${REWARD_SOURCE:-executable}; \
+	case "$$REWARD_SOURCE" in \
+		executable) CONFIG=configs/rl/e1_executable_reward.toml ;; \
+		llm_judge|judge) CONFIG=configs/rl/e1_llm_judge_reward.toml ;; \
+		hybrid) CONFIG=configs/rl/e1_hybrid_reward.toml ;; \
+		legacy) CONFIG=configs/rl/e1.toml ;; \
+		*) $(ECHO) "$(RED)Error: REWARD_SOURCE must be executable, llm_judge, hybrid, or legacy$(NC)"; exit 1 ;; \
+	esac; \
+	$(ECHO) "$(YELLOW)Launching hosted RL training for E1 with $$REWARD_SOURCE reward ($$CONFIG)...$(NC)"; \
+	$(ACTIVATE) && prime rl run "$$CONFIG"
 
 lab-run-e1-judge: venv
 	@$(ECHO) "$(YELLOW)Launching hosted RL training for E1 judge variant (WP3c)...$(NC)"
 	@$(ACTIVATE) && prime rl run configs/rl/e1_judge.toml
 
 lab-run-e2: venv
-	@$(ECHO) "$(YELLOW)Launching hosted RL training for E2 (config-verification)...$(NC)"
-	@$(ACTIVATE) && prime rl run configs/rl/e2.toml
+	@REWARD_SOURCE=$${REWARD_SOURCE:-executable}; \
+	case "$$REWARD_SOURCE" in \
+		executable) CONFIG=configs/rl/e2_executable_reward.toml ;; \
+		llm_judge|judge) CONFIG=configs/rl/e2_llm_judge_reward.toml ;; \
+		hybrid) CONFIG=configs/rl/e2_hybrid_reward.toml ;; \
+		legacy) CONFIG=configs/rl/e2.toml ;; \
+		*) $(ECHO) "$(RED)Error: REWARD_SOURCE must be executable, llm_judge, hybrid, or legacy$(NC)"; exit 1 ;; \
+	esac; \
+	$(ECHO) "$(YELLOW)Launching hosted RL training for E2 with $$REWARD_SOURCE reward ($$CONFIG)...$(NC)"; \
+	$(ACTIVATE) && prime rl run "$$CONFIG"
 
 # Fallback hosted-style eval parity (WP2.5a)
 env-eval-e1: venv
@@ -982,3 +1006,26 @@ env-eval-e2: venv
 	@MODEL=$${MODEL:-Qwen/Qwen3-4B-Instruct-2507}; \
 	N=$${N:-50}; \
 	$(ACTIVATE) && prime env eval $(TEAM)/sv-env-config-verification --model $$MODEL --num-examples $$N
+
+eval-e3: venv
+	@N=$${N:-10}; $(ACTIVATE) && uv run python scripts/eval_beta_env.py --env e3 --num-examples $$N
+
+eval-e4: venv
+	@N=$${N:-10}; $(ACTIVATE) && uv run python scripts/eval_beta_env.py --env e4 --num-examples $$N
+
+eval-e5: venv
+	@N=$${N:-10}; $(ACTIVATE) && uv run python scripts/eval_beta_env.py --env e5 --num-examples $$N
+
+eval-e6: venv
+	@N=$${N:-10}; $(ACTIVATE) && uv run python scripts/eval_beta_env.py --env e6 --num-examples $$N
+
+svbench-v0.1-check: venv config-validate
+	@$(ECHO) "$(YELLOW)Checking SV-Bench v0.1 release package...$(NC)"
+	@$(ACTIVATE) && uv run svbench_manifest validate bench/fixtures/e1_run_manifest.json bench/fixtures/e2_run_manifest.json
+	@$(ACTIVATE) && uv run python scripts/check_svbench_v01.py --strict
+	@$(ECHO) "$(GREEN)✓ SV-Bench v0.1 checks passed$(NC)"
+
+suite-v1-check: venv config-validate
+	@$(ECHO) "$(YELLOW)Checking Security Verifiers Suite v1 gate...$(NC)"
+	@$(ACTIVATE) && uv run python scripts/check_suite_v1.py
+	@$(ECHO) "$(GREEN)✓ Suite v1 checks passed$(NC)"

README.md

@@ -16,6 +16,14 @@ Security Verifiers demonstrates how executable rewards can advance both security
 
 ## Environments
 
+## Current Status
+
+SV-Bench v0.1 is the near-term benchmark release and includes only E1/E2. E3-E6 are part of the broader Security Verifiers Suite roadmap and remain alpha/beta work until the v0.1 empirical proof is complete.
+
+- [SV-Bench status](SVBENCH_STATUS.md)
+- [SV-Bench guide](SVBENCH.md)
+- [2026 suite roadmap](plans/ROADMAP-2026-SUITE.md)
+
 | Environment                 | Type       | Task                                            | Status     |
 | --------------------------- | ---------- | ----------------------------------------------- | ---------- |
 | **E1: network-logs**        | SingleTurn | Anomaly detection with calibration & abstention | Production |
@@ -88,6 +96,9 @@ security-verifiers/
 | [Development Guide](docs/development.md)               | Contributing, testing, CI         |
 | [Hub Deployment](docs/hub-deployment.md)               | Deploy to Prime Intellect Hub     |
 | [Prime Lab Integration](docs/PRIME-LAB-INTEGRATION.md) | Hosted RL training and evaluation |
+| [SV-Bench Status](SVBENCH_STATUS.md)                   | Current release boundary and work packages |
+| [SV-Bench Guide](SVBENCH.md)                           | v0.1 commands, artifacts, and release scope |
+| [Suite Roadmap](plans/ROADMAP-2026-SUITE.md)           | v0.1 vs suite-v1 roadmap |
 | [Datasets Guide](docs/datasets.md)                     | Dataset access and management     |
 | [Logging Guide](docs/logging.md)                       | Weave tracing configuration       |
 | [CLAUDE.md](CLAUDE.md)                                 | Agent/LLM instructions            |

SUITE_V1_CHECKLIST.md

@@ -0,0 +1,18 @@
+# Security Verifiers Suite v1 Checklist
+
+| Environment | Status | Public Mini | Eval Config | Metrics Doc | Hub State |
+|---|---|---|---|---|---|
+| E1 network logs | Production | Yes | Yes | Yes | Ready |
+| E2 config verification | Production | Yes | Yes | Yes | Ready |
+| E3 code vulnerability | Beta slice | Yes | Yes | Yes | Planned |
+| E4 phishing detection | Beta slice | Yes | Yes | Yes | Planned |
+| E5 red-team attack | Sanitized beta slice | Yes | Yes | Yes | Planned |
+| E6 red-team defense | Sanitized beta slice | Yes | Yes | Yes | Planned |
+
+Completion gate:
+
+```bash
+make suite-v1-check
+```
+
+The suite gate distinguishes production, beta, gated, and restricted components. Passing smoke tests does not promote E3-E6 to production.

SVBENCH.md

@@ -0,0 +1,37 @@
+# SV-Bench
+
+SV-Bench is the benchmark release track for Security Verifiers. Version 0.1 measures whether executable security rewards produce reproducible, useful signals on two production environments:
+
+- E1 `sv-env-network-logs`
+- E2 `sv-env-config-verification`
+
+E3-E6 belong to the broader Security Verifiers Suite roadmap and are not part of SV-Bench v0.1.
+
+## Quick Commands
+
+```bash
+make setup && source .venv/bin/activate
+make baseline-e1 MODEL=gpt-5-mini N=10
+make baseline-e2 MODEL=gpt-5-mini INCLUDE_TOOLS=true N=10
+uv run svbench_manifest validate bench/fixtures/e1_run_manifest.json bench/fixtures/e2_run_manifest.json
+make svbench-v0.1-check
+```
+
+## Artifacts
+
+| Artifact | Purpose |
+|---|---|
+| `SVBENCH_STATUS.md` | Canonical current-state map. |
+| `bench/schemas/run_manifest.schema.json` | Reproducibility contract for run cards. |
+| `bench/scoreboards/` | Public mini-set scoreboard artifacts. |
+| `results/v0.1_baselines.md` | Baseline result summary. |
+| `results/v0.1_training.md` | Hosted-training result placeholder and artifact index. |
+| `reports/SVBENCH_v0.1_technical_report.md` | Narrow technical report template. |
+
+## Public and Gated Data
+
+Public mini sets exist for reproducible smoke tests and examples. Full benchmark splits are gated or held out where needed to reduce contamination. Red-team public data must remain sanitized and non-operational.
+
+## Reward-Source Claim
+
+The v0.1 research claim is only valid when executable, LLM-judge, and hybrid manifests match on budget fields. `svbench_compare_rewards` refuses unmatched budgets unless `--allow-unmatched` is explicitly passed for exploratory analysis.

SVBENCH_STATUS.md

@@ -0,0 +1,65 @@
+# SV-Bench Status
+
+This is the canonical contributor map for the current benchmark push.
+
+## Release Boundary
+
+SV-Bench v0.1 is scoped to E1 and E2 only:
+
+- E1 `sv-env-network-logs`: production benchmark environment.
+- E2 `sv-env-config-verification`: production benchmark environment.
+
+E3-E6 remain suite beta or preview work until the v0.1 empirical proof is complete. Do not expand E3-E6 public corpora or treat them as v0.1 release criteria.
+
+## Environment Status
+
+| ID | Environment | Status | v0.1 Included | Notes |
+|---|---|---|---|---|
+| E1 | `sv-env-network-logs` | Production | Yes | Calibrated network-log classification with abstention and asymmetric cost. |
+| E2 | `sv-env-config-verification` | Production | Yes | Tool-grounded configuration audit and patch verification. |
+| E3 | `sv-env-code-vulnerability` | Beta slice | No | Defensive toy repair tasks only; suite beta after v0.1. |
+| E4 | `sv-env-phishing-detection` | Beta slice | No | Calibrated email classification with safe evidence snippets. |
+| E5 | `sv-env-redteam-attack` | Preview/beta safety slice | No | Sanitized simulator only; no raw harmful corpus. |
+| E6 | `sv-env-redteam-defense` | Preview/beta safety slice | No | Safe helpfulness/harmlessness balance tests. |
+
+E5/E6 offensive or adversarial corpora must not be expanded or publicly released for SV-Bench v0.1. Public examples must remain sanitized, non-operational, and suitable for defensive evaluation.
+
+## Completed Work Packages
+
+| Package | Status | Evidence |
+|---|---|---|
+| WP0 | Complete | Benchmark integrity hardening and public/gated split foundations. |
+| WP1 | Complete | E1/E2 report generator, summary schema, and scoreboards. |
+| WP2 | Complete | Public mini sets and baseline wrappers. |
+| WP2.5 | Complete | Prime Lab integration scaffolding and hosted-run configs. |
+| WP2.5a | Complete | Hosted-style fallback normalization and Prime compatibility checks. |
+
+## Open Work Packages
+
+| Package | Status | Required Outcome |
+|---|---|---|
+| WP3a | Ready to run | Hosted RL proof for E1 executable reward. |
+| WP3b | Ready to run | Hosted RL proof for E2 executable reward. |
+| WP3c | Configured | Reward-source comparator for executable, LLM-judge, and hybrid rewards. |
+| WP4 | Planned | Multi-reward stability ablations. |
+| WP5 | In progress | SV-Bench v0.1 release package and technical report. |
+
+## Definition of Done for v0.1
+
+- E1/E2 public mini sets run through baseline commands.
+- E1/E2 run manifests validate with `uv run svbench_manifest validate`.
+- Reward-source comparison refuses unmatched budgets by default.
+- `make svbench-v0.1-check` passes in a clean checkout.
+- README, `SVBENCH.md`, status, roadmap, scoreboards, and technical report all state that v0.1 includes E1/E2 only.
+- E5/E6 unsafe or offensive corpora are absent from public release artifacts.
+
+## Known Blockers and Unknowns
+
+- Hosted training budget/results are not included until runs are actually launched and manifests are attached.
+- GDPO support is represented as a clearly named decoupled-normalization placeholder until runtime support is confirmed.
+- Prime Lab CLI naming can drift; `make lab-check` is the compatibility gate before live hosted runs.
+- E2 semantic patch preservation is the highest-risk scoring area and must be re-reviewed before using results in a paper claim.
+
+## Guardrail
+
+Do not expand E3-E6 before SV-Bench v0.1. E3-E6 work may add beta smoke tests, safety policies, public mini fixtures, docs, and suite-v1 checklists, but v0.1 claims and release artifacts must remain E1/E2-only.

bench/changelog.md

@@ -0,0 +1,8 @@
+# SV-Bench Changelog
+
+## v0.1
+
+- Defines SV-Bench v0.1 as E1/E2 only.
+- Adds run manifest validation.
+- Adds matched-budget reward-source comparator.
+- Adds release package and suite completion checks.