From 7236c29dcdd6a51e61afa898000b79a4d43dd1fe Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Mon, 13 Jul 2026 12:52:25 -0400 Subject: [PATCH] feat(cli): explicit validation modes --mode basic|strict|release (#52) loop doctor/validate/verify gain --mode: basic forces the structural fallback deterministically; strict/release require jsonschema and fail loud (ValidationModeError -> exit 2, stderr, no traceback) instead of silently downgrading; release == strict today (reserved, tripwire-tested). Reports gain requested_mode (auto|basic|strict|release) alongside the unchanged validation_mode key; VALIDATION_MODES re-exported; CI Quickstart smoke runs --mode release. 16 new tests honest in both env shapes (extras 544/14, pyyaml-only 526/32, zero regressions). Closes #52 Claude-Session: https://claude.ai/code/session_01EJ8zA8Cbi4o2amawpj8bZW --- .github/workflows/ci.yml | 2 +- README.md | 10 ++++ loop/__init__.py | 3 +- loop/__main__.py | 49 +++++++++++++++- loop/contract.py | 34 +++++++++-- scripts/test_ci_release_mode.py | 27 +++++++++ scripts/test_loop_cli.py | 93 ++++++++++++++++++++++++++++++ scripts/test_loop_contract_core.py | 67 +++++++++++++++++++++ 8 files changed, 276 insertions(+), 9 deletions(-) create mode 100644 scripts/test_ci_release_mode.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c0ac6d..75ed8c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: - name: Quickstart smoke test run: | - python -B -m loop doctor examples/coverage-repair + python -B -m loop doctor --mode release examples/coverage-repair python -B -m loop inspect examples/coverage-repair recipe-langgraph: diff --git a/README.md b/README.md index ae8bf75..fe04475 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ directory, and need no agent runtime to run. { "ok": true, "validation_mode": "structural-fallback", + "requested_mode": "auto", "schemas_checked": [ "loop-engineer/manifest@1", "loop-engineer/state@1", @@ -139,6 +140,15 @@ default, or real JSON-Schema validation against `schemas/*.json` when the optional `jsonschema` dependency is present (`pip install -e ".[schemas]"`), in which case it reads `"jsonschema"`. +For `doctor`, `validate`, and `verify`, `--mode basic|strict|release` makes the +choice explicit. With no flag, the current auto-detect behavior is unchanged +for backward compatibility with existing foreign and scoreboard callers. +`--mode basic` always uses the structural fallback, regardless of whether +`jsonschema` happens to be installed. `--mode strict` and `--mode release` +require real JSON-Schema validation and fail non-zero rather than silently +downgrade if the `jsonschema` extra is missing. `release` currently has the +same behavior as `strict`; it is reserved for a stricter future policy. + `inspect` is a static contract linter: it scores the loop contract's structure — what proof machinery is present and what is missing — without running the loop: diff --git a/loop/__init__.py b/loop/__init__.py index aaf16ed..5247905 100644 --- a/loop/__init__.py +++ b/loop/__init__.py @@ -6,11 +6,12 @@ """ from .paths import LoopPaths, resolve_loop_paths -from .contract import TERMINAL_STATES, doctor_report, validate_contract +from .contract import TERMINAL_STATES, VALIDATION_MODES, doctor_report, validate_contract __all__ = [ "LoopPaths", "TERMINAL_STATES", + "VALIDATION_MODES", "doctor_report", "resolve_loop_paths", "validate_contract", diff --git a/loop/__main__.py b/loop/__main__.py index 407028e..67bf991 100644 --- a/loop/__main__.py +++ b/loop/__main__.py @@ -5,7 +5,7 @@ import sys from pathlib import Path -from .contract import doctor_report +from .contract import VALIDATION_MODES, ValidationModeError, doctor_report _PROG = "python3 -m loop" @@ -21,10 +21,11 @@ {_USAGE} {_PROG} metrics [--baseline] + {_PROG} doctor|validate|verify [--mode basic|strict|release] commands: scaffold Write a fresh, doctor-clean loop contract into . - doctor Validate the contract objects (manifest, state, tasks, terminal). + doctor Validate the contract objects; --mode selects validation strength. validate Alias for doctor. verify Alias for doctor — check the contract's state. inspect Score an existing loop against the prime-directive checklist @@ -38,6 +39,9 @@ A workspace root or its .loop/ directory. options: + --mode {{basic,strict,release}} + (doctor/validate/verify only) basic forces structural checks; + strict/release require jsonschema. Default: auto-detect. --baseline (metrics only) write docs/metrics-baseline.json over a gate-backed run; exits non-zero and writes nothing otherwise. -h, --help Show this help and exit. @@ -75,6 +79,32 @@ def _print_json(report: dict) -> int: return 0 if report.get("ok") else 1 +def _extract_mode_flag(argv: list[str]) -> tuple[str | None, list[str]]: + """Extract the doctor-family validation mode without changing positional argv.""" + mode: str | None = None + remaining: list[str] = [] + index = 0 + while index < len(argv): + arg = argv[index] + if arg == "--mode": + if index + 1 >= len(argv): + raise ValueError("--mode requires a value") + value = argv[index + 1] + index += 2 + elif arg.startswith("--mode="): + value = arg.split("=", 1)[1] + index += 1 + else: + remaining.append(arg) + index += 1 + continue + if value not in VALIDATION_MODES: + valid = ", ".join(VALIDATION_MODES) + raise ValueError(f"invalid --mode value {value!r}; expected one of: {valid}") + mode = value + return mode, remaining + + def _run_metrics(argv: list[str]) -> int: """`metrics [--baseline] ` — parses its own flag, then delegates to scripts/metrics.py (resolved bundle-first, repo-relative fallback).""" @@ -124,6 +154,15 @@ def main(argv: list[str] | None = None) -> int: print(_USAGE, file=sys.stderr) return 2 + mode = None + if command in {"doctor", "validate", "verify"}: + try: + mode, argv = _extract_mode_flag(argv) + except ValueError as exc: + print(f"{command}: {exc}", file=sys.stderr) + print(_USAGE, file=sys.stderr) + return 2 + # metrics carries its own optional --baseline flag, so it parses its own args # before the generic single-target guards below. if command == "metrics": @@ -156,7 +195,11 @@ def main(argv: list[str] | None = None) -> int: return 0 if command in {"doctor", "validate", "verify"}: - return _print_json(doctor_report(target)) + try: + return _print_json(doctor_report(target, mode=mode)) + except ValidationModeError as exc: + print(f"{command}: {exc}", file=sys.stderr) + return 2 # command == "inspect": keep the historical inspector script as the scoring # UI over the same contract artifacts; import lazily to avoid making diff --git a/loop/contract.py b/loop/contract.py index c0fed3a..c41667a 100644 --- a/loop/contract.py +++ b/loop/contract.py @@ -23,6 +23,8 @@ "AbortedByHuman", ) +VALIDATION_MODES = ("basic", "strict", "release") + SCHEMA_IDS = ( "loop-engineer/manifest@1", "loop-engineer/state@1", @@ -31,6 +33,10 @@ ) +class ValidationModeError(RuntimeError): + """Raised when an explicit validation mode cannot be honored.""" + + class ContractIssue(dict): def __init__(self, code: str, message: str, path: Path | None = None): super().__init__(code=code, message=message) @@ -516,6 +522,24 @@ def _validation_mode() -> str: return "jsonschema" +def _resolve_requested_mode(mode: str | None) -> tuple[str, str]: + """Resolve the caller-facing mode to the internal validation branch.""" + if mode is None: + return "auto", _validation_mode() + if mode not in VALIDATION_MODES: + valid = ", ".join(VALIDATION_MODES) + raise ValidationModeError(f"unknown validation mode {mode!r}; expected one of: {valid}") + if mode == "basic": + return mode, "structural-fallback" + resolved = _validation_mode() + if resolved != "jsonschema": + raise ValidationModeError( + f"validation mode {mode!r} requires the jsonschema package; " + "install it or use --mode basic" + ) + return mode, resolved + + def _jsonschema_validate(data: dict[str, Any], name: str, path: Path, issues: list[dict]) -> None: import jsonschema # type: ignore @@ -664,7 +688,8 @@ def _derive_lifecycle(state: Any, terminal: Any, terminal_exists: bool) -> str: return "unknown" -def validate_contract(target: str | Path) -> dict[str, Any]: +def validate_contract(target: str | Path, *, mode: str | None = None) -> dict[str, Any]: + requested_mode, resolved_mode = _resolve_requested_mode(mode) paths = resolve_loop_paths(target) issues: list[dict] = [] manifest = read_manifest(paths.manifest) @@ -677,7 +702,7 @@ def validate_contract(target: str | Path) -> dict[str, Any]: else: terminal = _read_json(paths.terminal, issues) - mode = _validation_mode() + mode = resolved_mode if mode == "jsonschema": if manifest is None: issues.append(ContractIssue("missing_file", "missing manifest.yaml", paths.manifest)) @@ -717,11 +742,12 @@ def validate_contract(target: str | Path) -> dict[str, Any]: "ok": not issues, "paths": paths.to_json(), "validation_mode": mode, + "requested_mode": requested_mode, "schemas_checked": schemas_checked, "lifecycle": lifecycle, "issues": issues, } -def doctor_report(target: str | Path) -> dict[str, Any]: - return validate_contract(target) +def doctor_report(target: str | Path, *, mode: str | None = None) -> dict[str, Any]: + return validate_contract(target, mode=mode) diff --git a/scripts/test_ci_release_mode.py b/scripts/test_ci_release_mode.py new file mode 100644 index 0000000..4107b3c --- /dev/null +++ b/scripts/test_ci_release_mode.py @@ -0,0 +1,27 @@ +"""S2 CI release-mode wiring contract.""" + +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parent.parent + + +def _workflow() -> dict: + yaml = pytest.importorskip("yaml") + return yaml.safe_load((ROOT / ".github" / "workflows" / "ci.yml").read_text(encoding="utf-8")) + + +def _step(name_fragment: str) -> dict: + for step in _workflow()["jobs"]["gates"]["steps"]: + if name_fragment.lower() in str(step.get("name", "")).lower(): + return step + raise AssertionError(f"no gates step whose name contains {name_fragment!r}") + + +def test_quickstart_smoke_test_uses_release_mode_only_for_doctor(): + run = _step("Quickstart smoke test")["run"] + assert "loop doctor --mode release examples/coverage-repair" in run + assert "loop inspect --mode" not in run + assert "loop inspect examples/coverage-repair" in run diff --git a/scripts/test_loop_cli.py b/scripts/test_loop_cli.py index bacaeee..f83f3d9 100644 --- a/scripts/test_loop_cli.py +++ b/scripts/test_loop_cli.py @@ -147,3 +147,96 @@ def test_verify_is_a_wired_doctor_alias(tmp_path): def test_help_describes_verify_and_validate_as_aliases(): out = _run("--help").stdout.lower() assert "alias" in out, "help must reconcile verify/validate as doctor aliases" + + +# S2: explicit validation mode CLI ------------------------------------------------ + + +def test_help_documents_validation_modes(): + out = _run("--help").stdout + assert "--mode" in out + for value in ("basic", "strict", "release"): + assert value in out + + +def test_doctor_mode_release_and_basic_echo_requested_mode(): + import importlib.util + + strict = _run("doctor", "--mode=release", "examples/coverage-repair") + if importlib.util.find_spec("jsonschema") is not None: + assert strict.returncode == 0, strict.stderr + assert json.loads(strict.stdout)["requested_mode"] == "release" + assert json.loads(strict.stdout)["validation_mode"] == "jsonschema" + else: + assert strict.returncode == 2 + assert strict.stdout == "" + assert "jsonschema" in strict.stderr + assert "--mode basic" in strict.stderr + + basic = _run("doctor", "--mode", "basic", "examples/coverage-repair") + assert basic.returncode == 0, basic.stderr + assert json.loads(basic.stdout)["requested_mode"] == "basic" + assert json.loads(basic.stdout)["validation_mode"] == "structural-fallback" + + +def test_doctor_default_mode_reports_auto(): + result = _run("doctor", "examples/coverage-repair") + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["requested_mode"] == "auto" + + +def test_strict_mode_fails_loudly_without_jsonschema(tmp_path): + import os + + blocker = tmp_path / "blocker" + blocker.mkdir() + (blocker / "jsonschema.py").write_text("raise ImportError('blocked for test')\n", encoding="utf-8") + env = {**os.environ, "PYTHONPATH": os.pathsep.join((str(blocker), str(ROOT)))} + result = subprocess.run( + [sys.executable, "-m", "loop", "doctor", "--mode", "strict", "examples/coverage-repair"], + cwd=ROOT, + text=True, + capture_output=True, + env=env, + ) + assert result.returncode == 2 + assert result.stdout == "" + assert "jsonschema" in result.stderr + assert "Traceback" not in result.stderr + + +def test_invalid_or_missing_mode_is_a_usage_error(): + for args in (("doctor", "--mode", "bogus", "examples/coverage-repair"), ("doctor", "--mode")): + result = _run(*args) + assert result.returncode == 2 + assert result.stdout == "" + assert "usage" in result.stderr.lower() + assert "Traceback" not in result.stderr + assert all(value in _run("doctor", "--mode", "bogus", "examples/coverage-repair").stderr for value in ("basic", "strict", "release")) + + +def test_validate_and_verify_accept_mode_as_doctor_aliases(): + doctor = _run("doctor", "--mode", "basic", "examples/coverage-repair") + for command in ("validate", "verify"): + result = _run(command, "--mode", "basic", "examples/coverage-repair") + assert result.returncode == doctor.returncode + assert result.stdout == doctor.stdout + + +def test_mode_is_not_consumed_by_other_commands(tmp_path): + import os + + target = tmp_path / "target" + for command in ("inspect", "metrics"): + result = _run(command, "--mode", str(target)) + assert result.returncode != 0 + + result = subprocess.run( + [sys.executable, "-m", "loop", "scaffold", "--mode", str(target)], + cwd=tmp_path, + text=True, + capture_output=True, + env={**os.environ, "PYTHONPATH": str(ROOT)}, + ) + assert result.returncode == 0 + assert (tmp_path / "--mode").is_dir() diff --git a/scripts/test_loop_contract_core.py b/scripts/test_loop_contract_core.py index b6409c7..69fba0d 100644 --- a/scripts/test_loop_contract_core.py +++ b/scripts/test_loop_contract_core.py @@ -705,3 +705,70 @@ def test_loop_doctor_flags_stub_verify_scripts(tmp_path): report = json.loads(result.stdout) assert report["ok"] is False assert any(issue["code"] == "stub_verify_script" for issue in report["issues"]) + + +# S2: explicit public validation modes --------------------------------------- + + +def test_validation_mode_public_constants_and_default_report(tmp_path): + from loop import VALIDATION_MODES + from loop.contract import validate_contract + + workspace = _write_valid_loop(tmp_path) + report = validate_contract(workspace) + + assert VALIDATION_MODES == ("basic", "strict", "release") + assert report["requested_mode"] == "auto" + assert report["validation_mode"] in {"jsonschema", "structural-fallback"} + + +def test_basic_forces_structural_fallback_and_doctor_threads_mode(tmp_path): + from loop.contract import doctor_report, validate_contract + + workspace = _write_valid_loop(tmp_path) + assert validate_contract(workspace, mode="basic")["validation_mode"] == "structural-fallback" + report = doctor_report(workspace, mode="basic") + assert report["requested_mode"] == "basic" + assert report["validation_mode"] == "structural-fallback" + + +@pytest.mark.parametrize("requested", ["strict", "release"]) +def test_strict_and_release_require_jsonschema_before_reading_target(monkeypatch, tmp_path, requested): + import sys + + from loop.contract import ValidationModeError, validate_contract + + monkeypatch.setitem(sys.modules, "jsonschema", None) + with pytest.raises(ValidationModeError, match=r"jsonschema.*--mode basic"): + validate_contract(tmp_path / "missing-target", mode=requested) + + +@pytest.mark.parametrize("requested", ["strict", "release"]) +def test_strict_and_release_use_jsonschema_when_available(tmp_path, requested): + pytest.importorskip("jsonschema") + from loop.contract import doctor_report + + report = doctor_report(_write_valid_loop(tmp_path), mode=requested) + assert report["requested_mode"] == requested + assert report["validation_mode"] == "jsonschema" + + +def test_release_currently_equals_strict_reserved_semantics(tmp_path): + pytest.importorskip("jsonschema") + from loop.contract import validate_contract + + workspace = _write_valid_loop(tmp_path) + strict = validate_contract(workspace, mode="strict") + release = validate_contract(workspace, mode="release") + assert strict["requested_mode"] == "strict" + assert release["requested_mode"] == "release" + strict.pop("requested_mode") + release.pop("requested_mode") + assert release == strict + + +def test_unknown_public_validation_mode_raises_with_valid_values(tmp_path): + from loop.contract import ValidationModeError, validate_contract + + with pytest.raises(ValidationModeError, match=r"basic.*strict.*release"): + validate_contract(tmp_path, mode="turbo")