diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 6dbc9a9..fb996c9 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "loop-engineer", - "version": "0.6.1", + "version": "0.7.0", "description": "Design, launch, verify, repair, and improve agent loops. A Claude-Code-native architect+operator for long-running, verifiable, self-improving agentic-coding systems.", "author": { "name": "Sollan Systems", "url": "https://github.com/SollanSystems" }, "homepage": "https://github.com/SollanSystems/loop-engineer", diff --git a/CHANGELOG.md b/CHANGELOG.md index fa3e5fe..80a2d75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,60 @@ All notable changes to `loop-engineer` are documented here. `WORKFLOW.md` and `README.md` are reworded to describe the mechanism; the 0.3.4 history is left intact. -## Unreleased +## 0.7.0 — 2026-07-08 + +**ST2 — the portable standard.** The on-disk contract is now a documented, +versioned, tool-agnostic standard, not an implicit format one validator happens +to enforce. `reference/repo-os-contract.md` is promoted to the normative spec: +a stability note (§0 — `$id` majors of the form `loop-engineer/@`, +strictly additive within a major, breaking changes ship as a new major side by +side), an artifact/schema table across all 7 published schemas with required +keys read verbatim from `schemas/*.schema.json` (§11), the lifecycle vocabulary +and terminal-file-iff rule (§12), the repair-record vs rollout-record two-shape +clarification (§13), and a conformance checklist (§14, items A1–E1) any harness +can satisfy to claim it "emits a Loop-Engineer-conformant contract v1." + +### Added +- **`doctor` lifecycle line** — `validate_contract` (and so `loop doctor`) + reports `lifecycle: planned | running | terminated: | unknown`, + derived from `state.json` and the terminal file. Additive reporting only — + never an issue source — so an operator sees *why* no terminal file is + expected on an in-flight loop instead of being pushed to fabricate one. + DG-3 regression tests pin both directions in both validation modes: a + null-terminal loop without `terminal_state.json` is conformant; a non-null + `terminal_state` without the file still fails. +- **Round-trip template regression** (`scripts/test_template_roundtrip.py`) — + every `templates/*` artifact, filled with schema-valid values, passes + `validate_contract` with zero issues in both validation modes, for both an + in-flight and a terminated scaffold. The DG-class template↔validator↔schema + drift cannot silently return. +- **Runnable conformance checklist** (`scripts/test_conformance.py`) — executes + checklist items A1–E1 in CI against the flagship example + (`examples/coverage-repair`) and a fresh template scaffold, including + additive-key tolerance (D2) and lifecycle honesty (E1). A doc-parity test + pins every checklist ID to the normative doc so the checklist and its + documentation cannot drift apart. +- **README "A versioned, conformance-checkable standard"** — a pointer + subsection linking the promoted normative doc. + +### Fixed (external-review patch set, PRs #27–#30) +- **doctor evidence and surface fixes (#27)** — an empty-evidence `Succeeded` + now fails validation in both modes (G1 cross-check); ledger validation is + scoped to the canonical rollout/receipt files instead of force-validating + foreign `.loop/*.jsonl`, and fails closed on corrupt UTF-8; the fallback YAML + parser no longer strips `#` inside quoted strings; missing verify scripts and + dangling task file targets are surfaced as issues. +- **atomic terminate (#28)** — `loop.emit.terminate` writes the terminal record + exactly once, atomically; a second call raises `EmitError` instead of + silently overwriting the loop's end record. +- **strict-by-install gates (#29)** — the GitHub Action and the pre-commit hook + install the `[schemas]` extra so consumer repos gate in real JSON-Schema + mode, not the structural fallback; the Action's PR comment is sticky and its + score parsing robust. +- **inspector scores execution evidence (#30)** — `loop inspect` credits + verification gates on execution evidence rather than keyword presence, so a + keyword-stuffed contract can no longer buy a "strong" scorecard while the + gate-backed flagship example keeps its score. **B1 — the writer API.** `loop.emit` lets a foreign runtime (LangGraph, a plain script, any orchestrator) record an evidence-backed loop contract without diff --git a/README.md b/README.md index f4adf1e..49b00ac 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![CI](https://github.com/SollanSystems/loop-engineer/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/SollanSystems/loop-engineer/actions/workflows/ci.yml) [![Python 3.10–3.12](https://img.shields.io/badge/python-3.10%E2%80%933.12-blue)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE) -[![Release](https://img.shields.io/badge/release-0.6.1-blue)](https://github.com/SollanSystems/loop-engineer/tags) +[![Release](https://img.shields.io/badge/release-0.7.0-blue)](https://github.com/SollanSystems/loop-engineer/tags) Long-running agents commit **false completion**. After context compaction they forget what "done" meant, optimize to the visible test, patch in circles, and @@ -238,6 +238,14 @@ maps to a success criterion, its verifier passes, and evidence is recorded. See `reference/repo-os-contract.md` for the canonical artifact schemas. +### A versioned, conformance-checkable standard + +The on-disk contract is a **documented, versioned, tool-agnostic standard** — conformance is +defined by the published `schemas/*.schema.json` (`$id` `loop-engineer/@`, +additive within a major), and a runnable **conformance checklist** (A1–E1) lets any harness claim +*"emits a Loop-Engineer-conformant contract v1."* See +[`reference/repo-os-contract.md`](reference/repo-os-contract.md) §0 / §11 / §14. + --- ## Install @@ -389,8 +397,8 @@ license, and README differentiation. ## Status -- Version: `0.6.1` -- Release tag: `v0.6.1` (PyPI publish trigger; plugin tags through 0.6.0 used `loop-engineer--v`) +- Version: `0.7.0` +- Release tag: `v0.7.0` (PyPI publish trigger; plugin tags through 0.6.0 used `loop-engineer--v`) - License: MIT - Primary interface: Claude Code plugin - Portable core: Python CLI + JSON schemas diff --git a/loop/contract.py b/loop/contract.py index 9cbf821..9009389 100644 --- a/loop/contract.py +++ b/loop/contract.py @@ -485,6 +485,37 @@ def _validate_optional_records(paths: LoopPaths, mode: str, issues: list[dict]) return checked +def _derive_lifecycle(state: Any, terminal: Any, terminal_exists: bool) -> str: + """Report which lifecycle band a loop is in — additive reporting only, never + an issue source. Total and pure: never raises. + + Per the ratified rule: + 1. state parsed with a non-null terminal_state, OR terminal_state.json + present → ``terminated:`` where X is the terminal file's ``state`` + (dict + string) if available, else state.json's ``terminal_state`` if a + string, else ``unknown``. + 2. state parsed and iteration_id is 0 / "0" → ``planned``. + 3. state parsed → ``running``. + 4. else → ``unknown``. + """ + state_is_dict = isinstance(state, dict) + terminal_state_val = state.get("terminal_state") if state_is_dict else None + if (state_is_dict and terminal_state_val is not None) or terminal_exists: + if isinstance(terminal, dict) and isinstance(terminal.get("state"), str): + resolved = terminal["state"] + elif isinstance(terminal_state_val, str): + resolved = terminal_state_val + else: + resolved = "unknown" + return f"terminated:{resolved}" + if state_is_dict: + iteration_id = state.get("iteration_id") + if iteration_id == 0 or iteration_id == "0": + return "planned" + return "running" + return "unknown" + + def validate_contract(target: str | Path) -> dict[str, Any]: paths = resolve_loop_paths(target) issues: list[dict] = [] @@ -530,11 +561,14 @@ def validate_contract(target: str | Path) -> dict[str, Any]: schema_id for key, schema_id in _RECORD_SCHEMA_IDS if key in records_checked ] + lifecycle = _derive_lifecycle(state, terminal, paths.terminal.exists()) + return { "ok": not issues, "paths": paths.to_json(), "validation_mode": mode, "schemas_checked": schemas_checked, + "lifecycle": lifecycle, "issues": issues, } diff --git a/pyproject.toml b/pyproject.toml index 3b4f7b6..edac91d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "loop-engineer" -version = "0.6.1" +version = "0.7.0" description = "Portable Loop Contract Core: validate and inspect repo-native operating contracts for agent loops." readme = "README.md" requires-python = ">=3.10" diff --git a/reference/repo-os-contract.md b/reference/repo-os-contract.md index 2a35ac1..3efa46b 100644 --- a/reference/repo-os-contract.md +++ b/reference/repo-os-contract.md @@ -13,6 +13,36 @@ in `loop-patterns.md`; the safety/terminal semantics live in `safety-and-approva --- +## 0. The contract is a versioned, tool-agnostic standard + +This document is the **normative standard** for the repo-OS contract. It is not a +description of one tool's private file format: it is a **portable, tool-agnostic on-disk +standard**. Any surface that can read a repo, run a shell command, and write files can emit or +consume it — Loop Engineer is the *reference implementation*, not the only permitted producer. + +- **Conformance is defined by the published JSON Schemas** in `schemas/*.schema.json`, not by + any one validator's source code. Every schema-bearing artifact carries a `schema` key, and + every schema an `$id`, of the form **`loop-engineer/@`** + (e.g. `loop-engineer/state@1`). The major integer in that identifier is the version an + external emitter targets. +- **Within a major, changes are strictly additive and optional.** Every artifact schema sets + `"additionalProperties": true`, so a validator for major *N* accepts any artifact whose + required keys and types match major *N* and **ignores unknown keys** — a newer emitter's + extra fields never reject a valid v1 artifact. Adding an optional key, or a new optional + file, does not bump the major. +- **Breaking changes get a new major and a new `$id`.** Removing or renaming a required key, + changing a type, or tightening an enum ships as `loop-engineer/@2` with a new + `$id`. Both majors may be published and validated **side by side**. +- **Stability tiers.** The artifact table (§11) records each artifact's tier. For v1: + **manifest / state / tasks / terminal are `stable`**; **receipt / repair-record / + rollout-record are `provisional`** (the newest surfaces, whose additive shape may still be + refined within `@1`). + +A third-party harness whose output satisfies the §14 conformance checklist may claim it +**"emits a Loop-Engineer-conformant contract v1."** + +--- + ## 1. The full repo-OS tree ``` @@ -381,6 +411,134 @@ architecture decision record that `[[loop-architect]]` emits. --- +## 11. Artifact & schema reference + +Every schema-bearing artifact in the contract, its on-disk location, the schema that defines it, +its embedded `$id`, its **required keys** (read verbatim from `schemas/*.schema.json` — an +emitter MUST supply all of them), its lifecycle role, and its stability tier (§0). Required keys +are the floor; `additionalProperties: true` means an artifact may carry more. + +| Artifact | Contract path | Schema file | `$id` | Required keys | Lifecycle role | Tier | +|---|---|---|---|---|---|---| +| manifest | `.loop/manifest.yaml` | `schemas/manifest.schema.json` | `loop-engineer/manifest@1` | `schema`, `loop`, `policies`, `terminal_states` | The explicit, machine-readable operating contract for one loop (§10). | **stable** | +| state | `.loop/state.json` | `schemas/state.schema.json` | `loop-engineer/state@1` | `schema`, `iteration_id`, `state`, `plan_version`, `budget_remaining` | The live FSM cursor — the source of machine truth for resume (§7). | **stable** | +| tasks | `TASKS.json` *(workspace root)* | `schemas/tasks.schema.json` | `loop-engineer/tasks@1` | `schema`, `tasks`; each task: `id`, `title`, `status`, `criterion_ref`, `verify`, `depends_on`, `attempts`, `evidence` | The machine-readable task queue (§5). | **stable** | +| terminal | `.loop/terminal_state.json` | `schemas/terminal.schema.json` | `loop-engineer/terminal@1` | `schema`, `state`, `criteria_met`, `evidence`, `false_completion` | The single end record, written once at loop end (§8). | **stable** | +| receipt | `.loop/receipts/*.jsonl` | `schemas/receipt.schema.json` | `loop-engineer/receipt@1` | `schema`, `iteration_id`, `role`, `model`, `outcome` | Append-one-per-line dispatch/cost trail (role vs model, cost-per-success). | *provisional* | +| repair-record | `.loop/repair/.json` | `schemas/repair-record.schema.json` | `loop-engineer/repair@1` | `schema`, `iteration_id`, `attempt`, `failure_mode`, `hypothesis`, `repair_action`, `verification_before`, `verification_after`, `remaining_delta`, `productive` | One bounded repair pass (diagnosis shape); the canonical repair-productivity input (§13). | *provisional* | +| rollout-record | `.loop/rollout.jsonl` | `schemas/rollout-record.schema.json` | `loop-engineer/rollout@1` | `id`, `parent`, `verdict`, `score`, `score_delta`, `coherent_with_prior_winner`, `productive` | One candidate adjudication in a rollout / genetic-hardening ledger (§13). | *provisional* | + +The rollout-record's required set is the only one that does **not** require a `schema` envelope +key (the ledger writer today emits bare records); the schema permits one via +`additionalProperties`, but does not demand it. `doctor` validates receipts and repair/rollout +records **only when the files are present** (§14 C1–C3): an in-flight loop that has not yet +produced a trail still conforms. + +--- + +## 12. Lifecycle vocabulary + +The 7 terminal states (§8) are the **frozen** set of ways a loop *ends*. Before it ends, a loop +also holds non-terminal lifecycle values while it is *scaffolded but not started* or *running*. +These non-terminal values are **not** terminal states and never appear in the 7-member +`terminal_state` enum. Two rules make an in-flight loop a first-class, conformant state. + +### 12.1 The terminal-file-iff rule + +`terminal_state.json` is required **iff** `state.json`'s `terminal_state` is non-null. + +- While `state.json` reports `terminal_state: null`, the **absence** of `.loop/terminal_state.json` + is **conformant** — the loop is in-flight, not failing validation. (`validate_contract` gates + the terminal-file read on `state.terminal_state`; a null with no file is treated as an + in-flight loop, not a `missing_file` issue.) +- A non-null `terminal_state` **without** the terminal file is a `missing_file` failure. + +**Why the iff, not "always require a terminal file":** a gate that demands a terminal record from +a live loop pushes an operator to *write a terminal state onto a loop that has not terminated* — +a fabricated end record. That is exactly the false completion this contract exists to prevent. +The iff rule removes the incentive: an honest in-flight loop is green without inventing an ending. + +### 12.2 The `doctor` lifecycle line + +`doctor` (`validate_contract`) adds a `lifecycle` field to its report so an operator sees *why* +no terminal file is expected. It is derived (total and pure — never an issue source) as: + +1. **`terminated:`** — if `state.json` parsed with a non-null `terminal_state`, **or** + `.loop/terminal_state.json` exists. `` is the terminal file's `state` value when the file + parses to a dict with a string `state`; else `state.json`'s `terminal_state` when that is a + string; else `unknown`. +2. **`planned`** — else, if `state.json` parsed and its `iteration_id` is `0` (or `"0"`): + scaffolded, not yet run. +3. **`running`** — else, if `state.json` parsed: executing. +4. **`unknown`** — else (no parseable `state.json`). + +`planned`, `running`, and `unknown` are lifecycle-report values only; none is a terminal state, +and no terminal state ever surfaces as one of them. The `terminated:` form is the only overlap +point, and there `` is always drawn from the frozen 7 (or `unknown`). + +--- + +## 13. Two distinct record shapes — repair-record vs rollout-record + +The repair-record and the rollout-record are **different artifacts** that share only a +`productive` boolean; they must not be conflated (this section exists so no one conflates them +again). They differ in shape, location, and what `productive` measures: + +| | repair-record (`loop-engineer/repair@1`) | rollout-record (`loop-engineer/rollout@1`) | +|---|---|---| +| **Shape** | **Diagnosis** of one bounded repair pass | **Ledger** entry adjudicating one rollout candidate | +| **Location** | `.loop/repair/.json` (one JSON object per file) | `.loop/rollout.jsonl` (append one JSON object per line) | +| **Key fields** | `failure_mode`, `hypothesis`, `repair_action`, `verification_before`, `verification_after`, `remaining_delta`, `productive` | `id`, `parent`, `verdict`, `score`, `score_delta`, `coherent_with_prior_winner`, `productive` | +| **`productive` means** | repair-productivity: `verification_after.score > verification_before.score` | rollout-productivity: `score_delta` is not null and `> 0` | +| **Feeds** | the repair-productivity metric / baseline (`loop-repair`) | the flywheel's candidate-hardening view (`loop-flywheel`) | + +The repair-record is the diagnosis shape the repair skill prescribes and the eval structural +invariant pins; the rollout-record is genome/candidate bookkeeping. Publishing them as two `$id`s +resolves the historic "two 7-field shapes both called *the* repair record" ambiguity. + +--- + +## 14. Conformance checklist + +A harness that satisfies **every** item below may claim it **"emits a Loop-Engineer-conformant +contract v1."** Each item is a third-party-checkable statement against the published schemas. +Items **C1–C3 are checked-when-present** — an in-flight loop that has not yet emitted a receipt, +repair, or rollout trail still conforms. `scripts/test_conformance.py` executes this checklist in +CI against the flagship example ([`examples/coverage-repair`](../examples/coverage-repair)) and a +fresh template scaffold, so a drift between this doc, the schemas, and the shipped scaffold cannot +land silently. + +**A. Artifacts present & well-formed** +- **A1** — `.loop/manifest.yaml` validates against `loop-engineer/manifest@1` (including the + canonical 7 `terminal_states`, verbatim and in order). +- **A2** — `.loop/state.json` validates against `loop-engineer/state@1`. +- **A3** — `TASKS.json` validates against `loop-engineer/tasks@1`; no duplicate task ids; no task + marked `done` without `evidence`. +- **A4** — `RUNLOG.md` is present. + +**B. Lifecycle honesty** +- **B1** — Exactly one of: (`state.terminal_state` is null **and** no `terminal_state.json`) **or** + (`terminal_state` is one of the canonical 7 **and** `terminal_state.json` is present and valid). +- **B2** — `terminal_state.json`, when present, validates against `loop-engineer/terminal@1` with a + `criteria_met` object, an `evidence` list, and an explicit `false_completion` boolean; a + `Succeeded` terminal additionally has `false_completion=false`, at least one true criterion, and + non-empty `evidence`. + +**C. Evidentiary trail (checked when present)** +- **C1** — every `.loop/receipts/*.jsonl` line validates against `loop-engineer/receipt@1`. +- **C2** — every `.loop/repair/*.json` validates against `loop-engineer/repair@1`. +- **C3** — `.loop/rollout.jsonl`, when present, validates against `loop-engineer/rollout@1`. + +**D. Versioning** +- **D1** — every artifact's `schema` key names a published, current-major schema `$id`. +- **D2** — unknown keys are tolerated (additive fields never reject a v1 artifact). + +**E. Lifecycle report** +- **E1** — `doctor` reports a `lifecycle` value consistent with B1: `terminated:` iff the + terminal pair is present and valid; `planned` / `running` otherwise (§12.2). + +--- + Sources: "Designing a Loop Engineer Skill for Frontier Agent Workflows" (2026), synthesizing Anthropic guidance on long-running agent harnesses (anthropic.com, 2025), OpenAI Agents/Codex guidance, Google Conductor, and arXiv PreFlect (2602.07187), SWE-Marathon (2606.07682), Web Agents diff --git a/scripts/test_conformance.py b/scripts/test_conformance.py new file mode 100644 index 0000000..d53a9ad --- /dev/null +++ b/scripts/test_conformance.py @@ -0,0 +1,417 @@ +"""ST2 acceptance #8 — the runnable conformance checklist. + +One test per ratified checklist item ID (A1–E1); each runs against TWO fixtures: + + * ``terminated`` — the tracked flagship ``examples/coverage-repair`` (a real, + Succeeded contract that ships a terminal file and a repair record); + * ``inflight`` — a fresh scaffold built here in a tmp dir from ``templates/`` + (``terminal_state: null``, no terminal file — B1's first arm). + +C-items are checked-when-present: a fixture that genuinely ships no receipt / +rollout trail is skipped-with-reason for that item, and the trail the fixture +DOES ship is asserted on. Schema conformance is exercised in BOTH validation +modes where meaningful (the jsonschema path when the library is installed, and +the stdlib structural path — forced by hiding ``jsonschema``). + +The scaffold helper is intentionally local to this module. S2 writes a similar +helper in ``test_template_roundtrip.py``; duplication between the two modules is +accepted for this slice — this test does not import from S2's module. +""" + +from __future__ import annotations + +import json +import re +import shutil +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +import loop.contract as C # noqa: E402 +from loop.contract import TERMINAL_STATES, validate_contract # noqa: E402 +from loop.paths import resolve_loop_paths # noqa: E402 + +TEMPLATES = ROOT / "templates" +EXAMPLE = ROOT / "examples" / "coverage-repair" + +CHECKLIST_IDS = ("A1", "A2", "A3", "A4", "B1", "B2", "C1", "C2", "C3", "D1", "D2", "E1") + + +# --------------------------------------------------------------------------- # +# Local scaffold helper — fill templates/ into a fresh in-flight contract. +# --------------------------------------------------------------------------- # + +_STATE_FILL = { + "PROJECT_NAME": "conformance-inflight", + "ITERATION_ID": "0", # quoted in the template -> string "0" -> lifecycle "planned" + "PLAN_VERSION": "0", + "ACTIVE_TASK_ID": "T1", + "STATE": "Planned", + "BEST_SCORE": "null", + "FAILURE_MODE": "", + "PENDING_APPROVAL": "null", + "TIME_REMAINING": "30m", + "COST_REMAINING": "1.00usd", + "CHECKPOINT_PATH": ".loop/checkpoints/none", + "GOAL_DESCRIPTION": "In-flight conformance scaffold", + "CRITERION_1": "criterion one is proven by pytest -q", + "CONSTRAINT_1": "no external side effects", + "WORKSPACE_PATH": "./", + "ALLOWED_TOOL_1": "read", + "RISK_PROFILE": "low", + "TIME_BUDGET": "30m", + "COST_BUDGET": "1.00usd", + "APPROVAL_POLICY": "on_side_effects", + "REPAIR_ATTEMPTS": "0", + "REPAIR_CAP": "2", + "LAST_VERIFY_CMD": "pytest -q", + "LAST_VERIFY_OUTCOME": "PENDING", + "LAST_SCORE": "null", + "EVIDENCE_PATH": ".loop/artifacts/", + "SHORT_TERM_SUMMARY": "scaffolded, not yet run", + "LESSONS_PATH": ".loop/memory/lessons.md", +} + +_MANIFEST_FILL = { + "LOOP_NAME": "conformance-inflight", + "GOAL_DESCRIPTION": "In-flight conformance scaffold", + "CRITERION_1": "criterion one is proven by pytest -q", + "CONSTRAINT_1": "no external side effects", + "WORKSPACE_PATH": "./", + "ALLOWED_TOOLS": "read, workspace-write", + "RISK_PROFILE": "low", + "TIME_BUDGET": "30m", + "COST_BUDGET": "1.00usd", + "APPROVAL_POLICY": "on_side_effects", + "PERMISSION_1": "read-only", + "APPROVAL_GATE_1": "destructive_commands", + "REPAIR_CAP": "2", + "PLAN_THEN_EXECUTE": "false", +} + +_TASKS_FILL = { + "PROJECT_NAME": "conformance-inflight", + "TASK_ID": "T1", + "TASK_TITLE": "Do the bounded task", + "TASK_STATUS": "pending", + "TASK_CRITERION_REF": "1", + "TASK_VERIFY": "pytest -q", # a plain command: a verify surface, not a path. + "CREATED_AT": "2026-01-01T00:00:00Z", + "UPDATED_AT": "2026-01-01T00:00:00Z", +} + + +def _fill(template_name: str, mapping: dict[str, str]) -> str: + text = (TEMPLATES / template_name).read_text(encoding="utf-8") + for key, value in mapping.items(): + text = text.replace("{{" + key + "}}", value) + # `{{PLACEHOLDER}}` is a literal doc token in a manifest YAML comment, not a + # fillable field; every real placeholder must be substituted. + remaining = [p for p in re.findall(r"{{(\w+)}}", text) if p != "PLACEHOLDER"] + assert not remaining, f"unfilled placeholders in {template_name}: {remaining}" + return text + + +def _scaffold_inflight(target: Path) -> Path: + """Write a fresh, in-flight (terminal_state: null, no terminal file) contract + filled from the shipped templates/ into ``target``.""" + loop_dir = target / ".loop" + loop_dir.mkdir(parents=True) + + state_text = _fill("state.json.tmpl", _STATE_FILL) + tasks_text = _fill("TASKS.json.tmpl", _TASKS_FILL) + # Fail loudly here (not at validate time) if a fill produced invalid JSON. + json.loads(state_text) + json.loads(tasks_text) + + (loop_dir / "state.json").write_text(state_text, encoding="utf-8") + (loop_dir / "manifest.yaml").write_text(_fill("manifest.yaml.tmpl", _MANIFEST_FILL), encoding="utf-8") + (target / "TASKS.json").write_text(tasks_text, encoding="utf-8") + (target / "RUNLOG.md").write_text( + (TEMPLATES / "RUNLOG.md.tmpl").read_text(encoding="utf-8"), encoding="utf-8" + ) + return target + + +@pytest.fixture() +def contracts(tmp_path) -> dict[str, Path]: + """The two fixtures every checklist item is exercised against.""" + return { + "terminated": EXAMPLE, + "inflight": _scaffold_inflight(tmp_path / "inflight"), + } + + +# --------------------------------------------------------------------------- # +# Shared validation helpers — drive loop.contract's own validators, both modes. +# --------------------------------------------------------------------------- # + +def _has_jsonschema() -> bool: + try: + import jsonschema # noqa: F401 + return True + except Exception: + return False + + +_STRUCTURAL = { + "manifest": C._validate_manifest, + "state": C._validate_state, + "tasks": C._validate_tasks, + "terminal": C._validate_terminal, +} + + +def _artifact_issues_both_modes(name: str, data: dict, path: Path) -> None: + """Assert ``data`` validates against ``loop-engineer/@1`` in the stdlib + structural mode AND (when installed) the real jsonschema mode.""" + structural: list[dict] = [] + _STRUCTURAL[name](data, path, structural) + assert structural == [], f"{name} structural issues: {structural}" + if _has_jsonschema(): + js: list[dict] = [] + C._jsonschema_validate(data, name, path, js) + assert js == [], f"{name} jsonschema issues: {js}" + + +def _record_issues_both_modes(data: dict, schema_key: str, path: Path) -> None: + structural: list[dict] = [] + C._validate_record(data, schema_key, path, "structural-fallback", structural) + assert structural == [], f"{schema_key} structural issues: {structural}" + if _has_jsonschema(): + js: list[dict] = [] + C._validate_record(data, schema_key, path, "jsonschema", js) + assert js == [], f"{schema_key} jsonschema issues: {js}" + + +def _jsonl_issues_both_modes(path: Path, schema_key: str) -> None: + structural: list[dict] = [] + C._validate_jsonl(path, schema_key, "structural-fallback", structural) + assert structural == [], f"{path.name} structural issues: {structural}" + if _has_jsonschema(): + js: list[dict] = [] + C._validate_jsonl(path, schema_key, "jsonschema", js) + assert js == [], f"{path.name} jsonschema issues: {js}" + + +def _published_schema_ids() -> set[str]: + ids: set[str] = set() + for schema_file in sorted((ROOT / "schemas").glob("*.schema.json")): + ids.add(json.loads(schema_file.read_text(encoding="utf-8"))["$id"]) + return ids + + +# --------------------------------------------------------------------------- # +# A. Artifacts present & well-formed. +# --------------------------------------------------------------------------- # + +@pytest.mark.parametrize("kind", ["terminated", "inflight"]) +def test_a1_manifest_schema(contracts, kind): + paths = resolve_loop_paths(contracts[kind]) + data = C.read_manifest(paths.manifest) + assert isinstance(data, dict) and data, "manifest did not parse to a mapping" + assert data.get("schema") == "loop-engineer/manifest@1" + # The canonical 7 terminal_states, verbatim and in order. + assert list(data.get("terminal_states") or []) == list(TERMINAL_STATES) + _artifact_issues_both_modes("manifest", data, paths.manifest) + + +@pytest.mark.parametrize("kind", ["terminated", "inflight"]) +def test_a2_state_schema(contracts, kind): + paths = resolve_loop_paths(contracts[kind]) + data = json.loads(paths.state.read_text(encoding="utf-8")) + assert data.get("schema") == "loop-engineer/state@1" + _artifact_issues_both_modes("state", data, paths.state) + + +@pytest.mark.parametrize("kind", ["terminated", "inflight"]) +def test_a3_tasks_schema(contracts, kind): + paths = resolve_loop_paths(contracts[kind]) + data = json.loads(paths.tasks.read_text(encoding="utf-8")) + assert data.get("schema") == "loop-engineer/tasks@1" + _artifact_issues_both_modes("tasks", data, paths.tasks) + # Cross-task rules JSON Schema cannot express: id uniqueness, evidence-before-done. + semantics: list[dict] = [] + C._check_tasks_semantics(data, paths.tasks, semantics) + assert semantics == [], semantics + + +@pytest.mark.parametrize("kind", ["terminated", "inflight"]) +def test_a4_runlog_present(contracts, kind): + paths = resolve_loop_paths(contracts[kind]) + assert paths.runlog.name == "RUNLOG.md" + assert paths.runlog.is_file(), f"RUNLOG.md missing for {kind}" + + +# --------------------------------------------------------------------------- # +# B. Lifecycle honesty. +# --------------------------------------------------------------------------- # + +@pytest.mark.parametrize("kind", ["terminated", "inflight"]) +def test_b1_terminal_pair_exclusivity(contracts, kind): + paths = resolve_loop_paths(contracts[kind]) + state = json.loads(paths.state.read_text(encoding="utf-8")) + terminal_state = state.get("terminal_state") + terminal_present = paths.terminal.exists() + + arm_inflight = terminal_state is None and not terminal_present + terminal_valid = False + if terminal_present: + term_issues: list[dict] = [] + C._validate_terminal( + json.loads(paths.terminal.read_text(encoding="utf-8")), paths.terminal, term_issues + ) + terminal_valid = not term_issues + arm_terminated = terminal_state in TERMINAL_STATES and terminal_present and terminal_valid + + assert arm_inflight ^ arm_terminated, ( + f"B1 requires exactly one arm: inflight={arm_inflight} terminated={arm_terminated}" + ) + # And the contract as a whole must pass — no contradictory lifecycle issue. + assert validate_contract(contracts[kind])["ok"] is True + + +@pytest.mark.parametrize("kind", ["terminated", "inflight"]) +def test_b2_terminal_proof_surface(contracts, kind): + paths = resolve_loop_paths(contracts[kind]) + if not paths.terminal.exists(): + pytest.skip(f"{kind}: no terminal_state.json — B2 is checked-when-present") + data = json.loads(paths.terminal.read_text(encoding="utf-8")) + _artifact_issues_both_modes("terminal", data, paths.terminal) + assert isinstance(data.get("criteria_met"), dict) + assert isinstance(data.get("evidence"), list) + assert isinstance(data.get("false_completion"), bool) + if data.get("state") == "Succeeded": + assert data["false_completion"] is False + assert any(v is True for v in data["criteria_met"].values()) + assert data["evidence"], "Succeeded terminal must carry non-empty evidence" + + +# --------------------------------------------------------------------------- # +# C. Evidentiary trail (checked when present). +# --------------------------------------------------------------------------- # + +@pytest.mark.parametrize("kind", ["terminated", "inflight"]) +def test_c1_receipts_trail(contracts, kind): + paths = resolve_loop_paths(contracts[kind]) + receipts_dir = paths.loop_dir / "receipts" + receipts = sorted(receipts_dir.glob("*.jsonl")) if receipts_dir.is_dir() else [] + if not receipts: + pytest.skip(f"{kind}: no .loop/receipts/*.jsonl trail — C1 is checked-when-present") + for receipt in receipts: + _jsonl_issues_both_modes(receipt, "receipt") + + +@pytest.mark.parametrize("kind", ["terminated", "inflight"]) +def test_c2_repair_trail(contracts, kind): + paths = resolve_loop_paths(contracts[kind]) + repair_dir = paths.loop_dir / "repair" + records = sorted(repair_dir.glob("*.json")) if repair_dir.is_dir() else [] + if not records: + pytest.skip(f"{kind}: no .loop/repair/*.json trail — C2 is checked-when-present") + for record_path in records: + data = json.loads(record_path.read_text(encoding="utf-8")) + assert data.get("schema") == "loop-engineer/repair@1" + _record_issues_both_modes(data, "repair", record_path) + + +@pytest.mark.parametrize("kind", ["terminated", "inflight"]) +def test_c3_rollout_trail(contracts, kind): + paths = resolve_loop_paths(contracts[kind]) + rollout = paths.loop_dir / "rollout.jsonl" + if not rollout.is_file(): + pytest.skip(f"{kind}: no .loop/rollout.jsonl ledger — C3 is checked-when-present") + _jsonl_issues_both_modes(rollout, "rollout") + + +# --------------------------------------------------------------------------- # +# D. Versioning. +# --------------------------------------------------------------------------- # + +@pytest.mark.parametrize("kind", ["terminated", "inflight"]) +def test_d1_schema_ids_are_published(contracts, kind): + published = _published_schema_ids() + paths = resolve_loop_paths(contracts[kind]) + + manifest = C.read_manifest(paths.manifest) + assert manifest.get("schema") in published + + for path in (paths.state, paths.tasks): + data = json.loads(path.read_text(encoding="utf-8")) + assert data.get("schema") in published, (path.name, data.get("schema")) + + if paths.terminal.exists(): + terminal = json.loads(paths.terminal.read_text(encoding="utf-8")) + assert terminal.get("schema") in published + + repair_dir = paths.loop_dir / "repair" + if repair_dir.is_dir(): + for record_path in sorted(repair_dir.glob("*.json")): + record = json.loads(record_path.read_text(encoding="utf-8")) + assert record.get("schema") in published, (record_path.name, record.get("schema")) + + +@pytest.mark.parametrize("kind", ["terminated", "inflight"]) +def test_d2_additive_keys_are_tolerated(contracts, kind, tmp_path, monkeypatch): + # Copy the whole contract, inject an unknown additive key into every artifact, + # and assert validation still passes in BOTH modes (a v1 validator never + # rejects a newer emitter's additive fields). + dest = tmp_path / f"copy_{kind}" + shutil.copytree(contracts[kind], dest) + paths = resolve_loop_paths(dest) + + def _inject_json(path: Path, extra: dict) -> None: + data = json.loads(path.read_text(encoding="utf-8")) + data.update(extra) + path.write_text(json.dumps(data), encoding="utf-8") + + _inject_json(paths.state, {"x_unknown_additive": {"nested": [1, 2]}}) + tasks = json.loads(paths.tasks.read_text(encoding="utf-8")) + tasks["x_unknown_additive"] = "additive" + if tasks.get("tasks"): + tasks["tasks"][0]["x_unknown_task_key"] = "additive" + paths.tasks.write_text(json.dumps(tasks), encoding="utf-8") + if paths.terminal.exists(): + _inject_json(paths.terminal, {"x_unknown_additive": True}) + paths.manifest.write_text( + paths.manifest.read_text(encoding="utf-8") + "\nx_unknown_additive_key: additive\n", + encoding="utf-8", + ) + + # Pass 1: whatever mode is installed (jsonschema when present). + report_default = validate_contract(dest) + assert report_default["ok"] is True, report_default["issues"] + + # Pass 2: force the stdlib structural mode by hiding jsonschema. + monkeypatch.setitem(sys.modules, "jsonschema", None) + report_structural = validate_contract(dest) + assert report_structural["validation_mode"] == "structural-fallback" + assert report_structural["ok"] is True, report_structural["issues"] + + +# --------------------------------------------------------------------------- # +# E. Lifecycle report. +# --------------------------------------------------------------------------- # + +@pytest.mark.parametrize("kind", ["terminated", "inflight"]) +def test_e1_doctor_lifecycle_consistent_with_b1(contracts, kind): + report = validate_contract(contracts[kind]) + expected = "terminated:Succeeded" if kind == "terminated" else "planned" + assert report["lifecycle"] == expected + + +# --------------------------------------------------------------------------- # +# Doc-parity guard — the normative doc must publish every checklist ID. +# --------------------------------------------------------------------------- # + +def test_conformance_checklist_documented(): + doc = ROOT / "reference" / "repo-os-contract.md" + text = doc.read_text(encoding="utf-8") + assert "conformance checklist" in text.lower(), "normative doc lacks a conformance-checklist section" + for cid in CHECKLIST_IDS: + assert cid in text, f"checklist ID {cid} missing from normative doc" diff --git a/scripts/test_docs_version.py b/scripts/test_docs_version.py index 2552266..f09ea69 100644 --- a/scripts/test_docs_version.py +++ b/scripts/test_docs_version.py @@ -15,9 +15,9 @@ def test_readme_has_no_stale_seven_skills(): assert "all 9 skills" in readme -def test_plugin_version_is_0_6_1(): +def test_plugin_version_is_0_7_0(): plugin = json.loads(_read(".claude-plugin/plugin.json")) - assert plugin["version"] == "0.6.1" + assert plugin["version"] == "0.7.0" def test_pyproject_version_matches_plugin(): @@ -30,6 +30,7 @@ def test_pyproject_version_matches_plugin(): def test_changelog_has_current_and_historical_entries(): changelog = _read("CHANGELOG.md") + assert "## 0.7.0" in changelog assert "## 0.6.1" in changelog assert "## 0.6.0" in changelog assert "## 0.5.0" in changelog diff --git a/scripts/test_loop_contract_core.py b/scripts/test_loop_contract_core.py index 3acf9dd..4b94bf9 100644 --- a/scripts/test_loop_contract_core.py +++ b/scripts/test_loop_contract_core.py @@ -553,6 +553,145 @@ def test_f7_doctor_on_file_target_matches_dir_target(tmp_path): assert file_report["paths"] == dir_report["paths"] +def _force_structural_mode(monkeypatch): + """Make `_validation_mode` report the stdlib structural fallback even when + jsonschema is installed, so a test can pin lifecycle in BOTH modes cheaply. + """ + import loop.contract as contract + + monkeypatch.setattr(contract, "_validation_mode", lambda: "structural-fallback") + + +@pytest.mark.parametrize("mode", ["jsonschema", "structural-fallback"]) +def test_dg3_inflight_loop_with_null_terminal_is_conformant(tmp_path, monkeypatch, mode): + # DG-3 (a): a scaffolded loop with terminal_state:null and NO + # terminal_state.json is a first-class conformant state — it passes doctor + # clean and reports a non-terminal lifecycle, never a fabricated terminal. + if mode == "jsonschema": + pytest.importorskip("jsonschema") + else: + _force_structural_mode(monkeypatch) + from loop.contract import doctor_report + + target = _scaffold(tmp_path, "inflight") + # A fresh scaffold is iteration_id "0" / terminal_state null. + report = doctor_report(target) + assert report["ok"] is True, report["issues"] + assert report["validation_mode"] == mode + assert report["lifecycle"] == "planned" + + # Advance iteration past 0 and the same null-terminal loop reads "running". + state_path = target / ".loop" / "state.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + state["iteration_id"] = 3 + state["state"] = "execute" + state_path.write_text(json.dumps(state), encoding="utf-8") + running = doctor_report(target) + assert running["ok"] is True, running["issues"] + assert running["lifecycle"] == "running" + + +@pytest.mark.parametrize("mode", ["jsonschema", "structural-fallback"]) +def test_dg3_non_null_terminal_without_file_fails_but_reports_terminated(tmp_path, monkeypatch, mode): + # DG-3 (b): a state.json declaring a non-null terminal_state with NO + # terminal_state.json is a broken terminal pair — doctor fails with a + # missing_file issue, yet lifecycle still names the claimed terminal so the + # operator sees WHY it failed rather than a bare missing-file. + if mode == "jsonschema": + pytest.importorskip("jsonschema") + else: + _force_structural_mode(monkeypatch) + from loop.contract import doctor_report + + target = _scaffold(tmp_path, "orphan-terminal") + state_path = target / ".loop" / "state.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + state["iteration_id"] = 2 + state["terminal_state"] = "Succeeded" + state_path.write_text(json.dumps(state), encoding="utf-8") + # Ensure no terminal file exists anywhere the resolver would find it. + for candidate in (target / ".loop" / "terminal_state.json", target / "terminal_state.json"): + if candidate.exists(): + candidate.unlink() + + report = doctor_report(target) + assert report["ok"] is False + assert any(i["code"] == "missing_file" for i in report["issues"]), report["issues"] + assert report["lifecycle"] == "terminated:Succeeded" + + +@pytest.mark.parametrize("mode", ["jsonschema", "structural-fallback"]) +def test_dg3_terminated_contract_reports_state_from_terminal_file(tmp_path, monkeypatch, mode): + # DG-3 (c): a genuinely terminated contract (terminal file present) reports + # lifecycle "terminated:" taken from the terminal file. Exercised on + # the shipped examples/coverage-repair fixture (workspace-root terminal file). + if mode == "jsonschema": + pytest.importorskip("jsonschema") + else: + _force_structural_mode(monkeypatch) + from loop.contract import doctor_report + + example = ROOT / "examples" / "coverage-repair" + report = doctor_report(example) + assert report["lifecycle"] == "terminated:Succeeded" + assert report["validation_mode"] == mode + + +@pytest.mark.parametrize("mode", ["jsonschema", "structural-fallback"]) +def test_dg3_missing_state_json_reports_unknown_lifecycle(tmp_path, monkeypatch, mode): + # DG-3 (d): with no readable state.json and no terminal file, the lifecycle + # is "unknown" — the rule is total and never raises on an empty/garbage dir. + if mode == "jsonschema": + pytest.importorskip("jsonschema") + else: + _force_structural_mode(monkeypatch) + from loop.contract import doctor_report + + bare = tmp_path / "bare" + (bare / ".loop").mkdir(parents=True) + report = doctor_report(bare) + assert report["lifecycle"] == "unknown" + + +def test_dg3_unreadable_state_json_reports_unknown_lifecycle(tmp_path): + # DG-3 (d): a present-but-unparseable state.json still yields "unknown" + # (state parsed to None) rather than crashing the lifecycle derivation. + from loop.contract import doctor_report + + bare = tmp_path / "corrupt" + (bare / ".loop").mkdir(parents=True) + (bare / ".loop" / "state.json").write_text("{ not json", encoding="utf-8") + report = doctor_report(bare) + assert report["lifecycle"] == "unknown" + + +def test_dg3_lifecycle_derivation_is_total_and_pure(): + # E1 unit: the pure derivation covers each branch deterministically, never + # raising, independent of the filesystem. + from loop.contract import _derive_lifecycle + + assert _derive_lifecycle({"iteration_id": 0, "terminal_state": None}, None, False) == "planned" + assert _derive_lifecycle({"iteration_id": "0", "terminal_state": None}, None, False) == "planned" + assert _derive_lifecycle({"iteration_id": 5, "terminal_state": None}, None, False) == "running" + # terminal file dict state wins over state.json's terminal_state. + assert ( + _derive_lifecycle( + {"iteration_id": 2, "terminal_state": "FailedBudget"}, + {"state": "Succeeded"}, + True, + ) + == "terminated:Succeeded" + ) + # non-null terminal_state with no parseable terminal dict falls back to it. + assert ( + _derive_lifecycle({"iteration_id": 2, "terminal_state": "FailedBlocked"}, None, False) + == "terminated:FailedBlocked" + ) + # terminal file exists but neither source yields a string state. + assert _derive_lifecycle(None, None, True) == "terminated:unknown" + assert _derive_lifecycle(None, None, False) == "unknown" + + def test_loop_doctor_flags_stub_verify_scripts(tmp_path): workspace = _write_valid_loop(tmp_path) (workspace / "scripts" / "verify-fast").write_text( diff --git a/scripts/test_template_roundtrip.py b/scripts/test_template_roundtrip.py new file mode 100644 index 0000000..41cb84d --- /dev/null +++ b/scripts/test_template_roundtrip.py @@ -0,0 +1,251 @@ +"""ST2 acceptance #7/#9: every shipped ``templates/*`` artifact, filled with +schema-valid values and scaffolded into a real contract layout, passes +``validate_contract`` clean — in BOTH validation modes. + +This is the DG-drift guard: if a template, a schema, or the validator drifts out +of agreement (a placeholder the map can't fill, a fill value the schema rejects, +a validator rule the templates violate), one of these tests goes red. The class +cannot silently return. +""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from loop._resources import templates_dir # noqa: E402 +from loop.contract import _validation_mode, validate_contract # noqa: E402 + +_PLACEHOLDER_RE = re.compile(r"\{\{[A-Z0-9_]+\}\}") + + +def _fill_values() -> dict[str, str]: + """A schema-valid fill for every ``{{PLACEHOLDER}}`` the templates declare. + + Values that land in an unquoted JSON/YAML slot (integers, booleans, null, + canonical enums) are bare tokens; everything else is a plain string. These + are drawn from the published schemas — plan_version/iteration_id integers, + the terminal state from the canonical 7, task status from its enum — so the + round trip proves the SHIPPED templates + schemas + validator agree. + """ + + return { + # --- state.json / manifest / tasks / terminal: schema-typed slots --- + "PROJECT_NAME": "roundtrip-fixture", + "LOOP_NAME": "roundtrip-fixture", + "ITERATION_ID": "1", # quoted in state (string), bare in terminal (int) + "PLAN_VERSION": "1", # integer + "ACTIVE_TASK_ID": "T1", + "STATE": "running", + "BEST_SCORE": "null", # number | null + "FAILURE_MODE": "", + "PENDING_APPROVAL": "null", # object | null + "TIME_REMAINING": "30m", + "COST_REMAINING": "1.00usd", + "CHECKPOINT_PATH": "", + "GOAL_DESCRIPTION": "prove the template round trip validates", + "CRITERION_1": "the fast gate passes", + "CRITERION_2": "the full gate passes", + "CRITERION_3": "the safety gate passes", + "CONSTRAINT_1": "read-only workspace", + "CONSTRAINT_2": "no network", + "WORKSPACE_PATH": ".", + "ALLOWED_TOOLS": "Read", # -> YAML list [Read] + "ALLOWED_TOOL_1": "Read", # -> JSON string + "RISK_PROFILE": "low", + "TIME_BUDGET": "1h", + "COST_BUDGET": "5.00usd", + "APPROVAL_POLICY": "on_side_effects", + "REPAIR_ATTEMPTS": "0", # integer + "REPAIR_CAP": "2", # integer + "LAST_VERIFY_CMD": "scripts/verify-fast", + "LAST_VERIFY_OUTCOME": "PASS", + "LAST_SCORE": "null", # number | null + "EVIDENCE_PATH": ".loop/artifacts/", + "SHORT_TERM_SUMMARY": "in flight", + "LESSONS_PATH": ".loop/memory/lessons.md", + "PERMISSION_1": "read-only", + "APPROVAL_GATE_1": "on_side_effects", + "PLAN_THEN_EXECUTE": "true", # boolean + "TASK_ID": "T1", + "TASK_TITLE": "run the fast gate", + "TASK_STATUS": "pending", # enum member + "TASK_CRITERION_REF": "1", + "TASK_VERIFY": "scripts/verify-fast", + "CREATED_AT": "2026-06-30T00:00:00Z", + "UPDATED_AT": "2026-06-30T00:00:00Z", + # --- terminal_state.json: schema-typed slots --- + "TERMINAL_STATE": "Succeeded", # canonical 7 + "TERMINATED_AT": "2026-06-30T01:00:00Z", + "CRITERION_REF": "1", + "CRITERION_STATUS": "true", # boolean; >=1 true for a Succeeded terminal + "ARTIFACT_PATH": ".loop/artifacts/verify-T1.json", + "FALSE_COMPLETION": "false", # boolean + "TERMINAL_REASON": "fast gate passed with evidence", + # --- RUNLOG.md prose --- + "ITERATION_DATE": "2026-06-30", + "STATE_NAME": "running", + "ACTION_1": "ran the fast gate", + "ACTION_2": "recorded evidence", + "VERIFY_GATE": "verify-fast", + "VERIFY_OUTCOME": "PASS", + "VERIFY_CMD": "scripts/verify-fast", + "SCORE_BEFORE": "0.0", + "SCORE_AFTER": "1.0", + "ITERATION_OUTCOME": "task_passed", + "REPAIR_ACTION": "n/a", + "REPAIR_ATTEMPT": "0", + "IMPROVEMENT_FLAG": "n/a", + "ITERATION_NOTES": "clean first pass", + # --- SPEC / WORKFLOW / AGENTS / EVALS prose --- + "NON_GOAL_1": "no production writes", + "NON_GOAL_2": "no schema changes", + "EVIDENCE_1": ".loop/artifacts/verify-T1.json", + "EVIDENCE_2": ".loop/artifacts/verify-T2.json", + "VERIFY_CMD_1": "scripts/verify-fast", + "VERIFY_CMD_2": "scripts/verify-full", + "NETWORK_POLICY": "off", + "RUBRIC_TARGET": "9", + "W_CORRECTNESS": "0.2", + "W_COMPLETENESS": "0.15", + "W_VERIFICATION": "0.15", + "W_SAFETY": "0.15", + "W_REPAIR": "0.1", + "W_FC_RESISTANCE": "0.1", + "W_BREVITY": "0.05", + "W_EFFICIENCY": "0.05", + "W_LOOP_BEHAVIOR": "0.05", + "W_FLYWHEEL": "0.05", + # literal reference to "{{PLACEHOLDER}}" in the manifest header comment + "PLACEHOLDER": "example", + } + + +def _fill_all_templates(mapping: dict[str, str]) -> dict[str, str]: + """Fill every placeholder in every ``templates/*.tmpl`` file. + + Asserts (a) the map has a value for every token a template declares and + (b) no ``{{`` survives the fill — the two ways a template could drift past + an incomplete map. Returns ``{template_name: filled_text}``. + """ + + filled: dict[str, str] = {} + tmpl_files = sorted(templates_dir().glob("*.tmpl")) + assert tmpl_files, "no *.tmpl files found under templates/" + for tmpl in tmpl_files: + text = tmpl.read_text(encoding="utf-8") + + def repl(match: re.Match[str], _name: str = tmpl.name) -> str: + token = match.group(0)[2:-2] + assert token in mapping, f"{_name}: no fill value for {{{{{token}}}}}" + return mapping[token] + + out = _PLACEHOLDER_RE.sub(repl, text) + assert "{{" not in out, f"{tmpl.name}: unfilled placeholder remains:\n{out}" + filled[tmpl.name] = out + return filled + + +def _scaffold(root: Path, terminated: bool) -> Path: + """Write a filled contract into ``root`` in the layout resolve_loop_paths wants. + + In-flight: state.terminal_state stays null and no terminal_state.json exists. + Terminated: state.terminal_state becomes ``Succeeded`` and a matching, filled + terminal_state.json is written (the B1 pair). + """ + + filled = _fill_all_templates(_fill_values()) + + workspace = root + loop_dir = workspace / ".loop" + scripts = workspace / "scripts" + loop_dir.mkdir(parents=True) + scripts.mkdir() + + (workspace / "SPEC.md").write_text(filled["SPEC.md.tmpl"], encoding="utf-8") + (workspace / "WORKFLOW.md").write_text(filled["WORKFLOW.md.tmpl"], encoding="utf-8") + (workspace / "AGENTS.md").write_text(filled["AGENTS.md.tmpl"], encoding="utf-8") + (workspace / "TASKS.json").write_text(filled["TASKS.json.tmpl"], encoding="utf-8") + (workspace / "RUNLOG.md").write_text(filled["RUNLOG.md.tmpl"], encoding="utf-8") + (workspace / "EVALS-rubric.md").write_text(filled["EVALS-rubric.md.tmpl"], encoding="utf-8") + (loop_dir / "manifest.yaml").write_text(filled["manifest.yaml.tmpl"], encoding="utf-8") + + # Real, non-stub verify scripts (copied verbatim from the shipped templates, + # which ship without the stub markers) so the verify surface resolves honestly. + for src, dest in (("verify-fast.sh", "verify-fast"), ("verify-full.sh", "verify-full")): + (scripts / dest).write_text( + (templates_dir() / src).read_text(encoding="utf-8"), encoding="utf-8" + ) + + state = json.loads(filled["state.json.tmpl"]) + if terminated: + state["terminal_state"] = "Succeeded" + (loop_dir / "terminal_state.json").write_text( + filled["terminal_state.json.tmpl"], encoding="utf-8" + ) + (loop_dir / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + return workspace + + +def _assert_lifecycle(report: dict, expected: str) -> None: + """Assert the ratified lifecycle rule.""" + + assert report["lifecycle"] == expected + + +# (iteration_id "1", no terminal) -> running; (terminal Succeeded) -> terminated:Succeeded +_CASES = [ + pytest.param(False, "running", id="in-flight"), + pytest.param(True, "terminated:Succeeded", id="terminated"), +] + + +def test_every_template_fills_with_no_placeholder_remaining(): + # The core DG-drift guard: a template placeholder with no schema-valid fill + # (or a new placeholder nobody mapped) fails here, loudly, by name. + filled = _fill_all_templates(_fill_values()) + for name, text in filled.items(): + assert "{{" not in text, name + + +@pytest.mark.parametrize("terminated,expected_lifecycle", _CASES) +def test_roundtrip_validates_clean_jsonschema_mode(tmp_path, terminated, expected_lifecycle): + pytest.importorskip("jsonschema") + workspace = _scaffold(tmp_path / "ws", terminated) + + report = validate_contract(workspace) + + assert report["validation_mode"] == "jsonschema" + assert report["ok"] is True, report["issues"] + assert report["issues"] == [], report["issues"] + _assert_lifecycle(report, expected_lifecycle) + + +@pytest.mark.parametrize("terminated,expected_lifecycle", _CASES) +def test_roundtrip_validates_clean_structural_fallback_mode( + tmp_path, monkeypatch, terminated, expected_lifecycle +): + # Force structural-fallback by hiding jsonschema (import jsonschema -> None in + # sys.modules raises ImportError), mirroring the yaml-hiding pattern in + # test_loop_contract_core.py. This proves the stdlib hand checks agree with + # the shipped templates + schemas independently of the jsonschema library. + monkeypatch.setitem(sys.modules, "jsonschema", None) + assert _validation_mode() == "structural-fallback" + + workspace = _scaffold(tmp_path / "ws", terminated) + + report = validate_contract(workspace) + + assert report["validation_mode"] == "structural-fallback" + assert report["ok"] is True, report["issues"] + assert report["issues"] == [], report["issues"] + _assert_lifecycle(report, expected_lifecycle)