From 6a4f67da957639d097a9ed1c92a116ef4cdd3148 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:49:23 +0300 Subject: [PATCH 01/43] test: define single-workflow architecture boundary --- tests/test_architecture_consolidation.py | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tests/test_architecture_consolidation.py diff --git a/tests/test_architecture_consolidation.py b/tests/test_architecture_consolidation.py new file mode 100644 index 0000000..55230fa --- /dev/null +++ b/tests/test_architecture_consolidation.py @@ -0,0 +1,34 @@ +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +class ArchitectureConsolidationTests(unittest.TestCase): + def test_repository_contains_one_current_workflow_surface(self): + forbidden_runtime = [ + ROOT / "scripts" / "workflow_v2" / "migration_journal.py", + ROOT / "scripts" / "workflow_v2" / "migrations.py", + ROOT / "scripts" / "workflow_v2" / "migrations_cli.py", + ] + self.assertEqual( + [path.relative_to(ROOT).as_posix() for path in forbidden_runtime if path.exists()], + [], + ) + + historical_design_docs = sorted( + path.relative_to(ROOT).as_posix() + for path in (ROOT / "docs").glob("WORKFLOW_V2_*.md") + ) + self.assertEqual(historical_design_docs, []) + + def test_canonical_contract_does_not_offer_legacy_or_upgrade_runtime(self): + orchestration = (ROOT / "docs" / "ORCHESTRATION.md").read_text(encoding="utf-8") + self.assertNotIn("legacy contract", orchestration.lower()) + self.assertNotIn("workflow upgrade", orchestration.lower()) + self.assertNotIn("migration journal", orchestration.lower()) + + +if __name__ == "__main__": + unittest.main() From 3e8662f66b0cd76ae4a89d86731b45fa69bd9d3b Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:49:40 +0300 Subject: [PATCH 02/43] docs: define consolidated workflow architecture --- docs/ARCHITECTURE.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/ARCHITECTURE.md diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..93fe80c --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,29 @@ +# Architecture + +Book Translator has one supported workflow runtime. + +The current workflow is repository-authoritative and uses one durable state contract: schema-versioned metadata/progress, explicit source identity and corpus manifest, durable claims, machine review evidence, coordination, finalize/build state, and backend-neutral compare-and-swap storage. + +## Supported runtime + +- `scripts/book.py` is the primary CLI entrypoint. +- `scripts/corpus.py` owns source-corpus integrity and restore operations. +- `scripts/workflow_v2/` is the internal workflow package. The `v2` suffix is an implementation-era module name, not a second supported workflow version. +- `docs/ORCHESTRATION.md` and `docs/TRANSLATION.md` are the canonical execution contracts. +- Files under `books//` are authoritative durable book state. + +## State contract + +Every supported book workspace uses the current schema and must contain current workflow provenance, explicit source identity, a sealed source manifest, and machine review evidence. Missing schema versions, legacy lifecycle-only review state, unsealed legacy source state, and workflow migration journals are not supported runtime modes. + +An older workspace must be converted outside the production runtime before it is used. The current runtime does not auto-normalize legacy schemas and does not perform workflow-version migrations. + +## Runtime boundaries + +The orchestration core owns state transitions. Translator and Reviewer roles produce artifacts and decisions but do not race shared mutable state. Storage backends provide create-if-absent and compare-and-swap semantics; filesystem and GitHub API storage implement the same contract. + +Source integrity is independent from literary review. A valid source corpus does not imply a reviewed translation, and a Reviewer PASS does not replace structural or source-integrity validation. + +## Completion + +A book is complete only when all intended units have current machine-verifiable PASS evidence, durable lifecycle state is reviewed, structural and corpus checks pass, and requested output artifacts are built and verified. From a6a55d4223f150910179d4996e269337bfb8a692 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:54:14 +0300 Subject: [PATCH 03/43] refactor: remove legacy migration surface --- docs/WORKFLOW_V2_AGENT_PLAN.md | 146 -- docs/WORKFLOW_V2_CLAIMS_CAS_DESIGN.md | 324 ----- docs/WORKFLOW_V2_CLAIMS_CAS_PLAN.md | 384 ------ docs/WORKFLOW_V2_EPUB_OUTPUT_DESIGN.md | 213 --- docs/WORKFLOW_V2_EPUB_OUTPUT_PLAN.md | 108 -- docs/WORKFLOW_V2_FINALIZE_DESIGN.md | 286 ---- docs/WORKFLOW_V2_FINALIZE_PLAN.md | 402 ------ docs/WORKFLOW_V2_GITHUB_BACKEND_DESIGN.md | 366 ----- docs/WORKFLOW_V2_GITHUB_BACKEND_PLAN.md | 266 ---- docs/WORKFLOW_V2_MIGRATIONS_DESIGN.md | 272 ---- docs/WORKFLOW_V2_MIGRATIONS_PLAN.md | 219 --- docs/WORKFLOW_V2_PARALLEL_DESIGN.md | 57 - docs/WORKFLOW_V2_PRIVATE_SOURCE_DESIGN.md | 394 ------ docs/WORKFLOW_V2_PRIVATE_SOURCE_PLAN.md | 450 ------- docs/WORKFLOW_V2_RELIABILITY_PHASE1_DESIGN.md | 339 ----- docs/WORKFLOW_V2_RELIABILITY_PHASE1_PLAN.md | 318 ----- docs/WORKFLOW_V2_REVIEW_LEDGER_DESIGN.md | 520 -------- docs/WORKFLOW_V2_REVIEW_LEDGER_PLAN.md | 511 ------- docs/WORKFLOW_V2_REVIEW_REPORT_PLAN.md | 71 - docs/WORKFLOW_V2_SAFE_PATCH_DESIGN.md | 346 ----- docs/WORKFLOW_V2_SAFE_PATCH_PLAN.md | 286 ---- scripts/workflow_v2/migration_journal.py | 202 --- scripts/workflow_v2/migrations.py | 1186 ----------------- scripts/workflow_v2/migrations_cli.py | 198 --- tests/test_workflow_v2_migration_planner.py | 343 ----- .../test_workflow_v2_migration_reliability.py | 294 ---- .../test_workflow_v2_migration_transaction.py | 342 ----- .../test_workflow_v2_migration_visibility.py | 229 ---- tests/test_workflow_v2_migrations.py | 281 ---- tests/test_workflow_v2_migrations_cli.py | 230 ---- 30 files changed, 9583 deletions(-) delete mode 100644 docs/WORKFLOW_V2_AGENT_PLAN.md delete mode 100644 docs/WORKFLOW_V2_CLAIMS_CAS_DESIGN.md delete mode 100644 docs/WORKFLOW_V2_CLAIMS_CAS_PLAN.md delete mode 100644 docs/WORKFLOW_V2_EPUB_OUTPUT_DESIGN.md delete mode 100644 docs/WORKFLOW_V2_EPUB_OUTPUT_PLAN.md delete mode 100644 docs/WORKFLOW_V2_FINALIZE_DESIGN.md delete mode 100644 docs/WORKFLOW_V2_FINALIZE_PLAN.md delete mode 100644 docs/WORKFLOW_V2_GITHUB_BACKEND_DESIGN.md delete mode 100644 docs/WORKFLOW_V2_GITHUB_BACKEND_PLAN.md delete mode 100644 docs/WORKFLOW_V2_MIGRATIONS_DESIGN.md delete mode 100644 docs/WORKFLOW_V2_MIGRATIONS_PLAN.md delete mode 100644 docs/WORKFLOW_V2_PARALLEL_DESIGN.md delete mode 100644 docs/WORKFLOW_V2_PRIVATE_SOURCE_DESIGN.md delete mode 100644 docs/WORKFLOW_V2_PRIVATE_SOURCE_PLAN.md delete mode 100644 docs/WORKFLOW_V2_RELIABILITY_PHASE1_DESIGN.md delete mode 100644 docs/WORKFLOW_V2_RELIABILITY_PHASE1_PLAN.md delete mode 100644 docs/WORKFLOW_V2_REVIEW_LEDGER_DESIGN.md delete mode 100644 docs/WORKFLOW_V2_REVIEW_LEDGER_PLAN.md delete mode 100644 docs/WORKFLOW_V2_REVIEW_REPORT_PLAN.md delete mode 100644 docs/WORKFLOW_V2_SAFE_PATCH_DESIGN.md delete mode 100644 docs/WORKFLOW_V2_SAFE_PATCH_PLAN.md delete mode 100644 scripts/workflow_v2/migration_journal.py delete mode 100644 scripts/workflow_v2/migrations.py delete mode 100644 scripts/workflow_v2/migrations_cli.py delete mode 100644 tests/test_workflow_v2_migration_planner.py delete mode 100644 tests/test_workflow_v2_migration_reliability.py delete mode 100644 tests/test_workflow_v2_migration_transaction.py delete mode 100644 tests/test_workflow_v2_migration_visibility.py delete mode 100644 tests/test_workflow_v2_migrations.py delete mode 100644 tests/test_workflow_v2_migrations_cli.py diff --git a/docs/WORKFLOW_V2_AGENT_PLAN.md b/docs/WORKFLOW_V2_AGENT_PLAN.md deleted file mode 100644 index e336348..0000000 --- a/docs/WORKFLOW_V2_AGENT_PLAN.md +++ /dev/null @@ -1,146 +0,0 @@ -# Workflow v2 — Agent Execution Plan - -This file is a concise execution plan for AI agents working on Workflow v2. - -## Branch policy - -- Never develop Workflow v2 directly on `main`. -- Integration branch: `refactor/workflow-engine-v2`. -- Each issue uses its own feature/test/docs branch created from the current integration branch. -- Feature PRs target `refactor/workflow-engine-v2`. -- Final release is one squash PR from `refactor/workflow-engine-v2` to `main`. - -## Parallelism policy - -Target at most **3 implementation streams + 1 reliability/test stream**. - -Do not start a dependent task before its required state/API is stable in the integration branch. - -## Critical path - -```text -#7 state/storage - -> #8 claims/CAS - -> #9 review ledger - -> #10 status/resume - -> #12 finalize - -> integration dogfood - -> final squash PR -``` - -Keep this path moving first. - -## Execution waves - -### Wave 0 - -Parallel: - -- #7 State schemas and storage abstraction -- #13 Safe text patching -- #18 Test/fixture scaffolding - -### Wave 1 - -After #7 merges: - -- #8 Claims, leases and CAS -- #11 Source integrity and private-source mode -- #18 tests for #7/#8/#11 - -### Wave 2 - -After #8 merges: - -- #9 Review ledger, hashes and stale-review detection -- #17 GitHub API storage backend -- #18 concurrency/backend tests - -### Wave 3 - -After #9 merges: - -- #10 Deterministic status/resume -- #21 Generated review report -- #14 EPUB build/validation/output manifest -- #18 review/status/output tests - -### Wave 4 - -When #10 and #11 are ready: - -- #12 Atomic finalize -- #16 Workflow/schema migrations -- finish #14 integration with finalize -- #18 finalize/migration/failure tests - -### Wave 5 - -When #8 + #9 + #10 are ready: - -- #15 Explicit parallel mode and proposal reconciliation -- #18 race/failure-injection tests - -Wave 5 may overlap Wave 4 if workers are available. - -### Wave 6 — release hardening - -- #22 Commit discipline and audit-friendly boundaries -- #19 Remove core Actions dependency and align docs -- #18 full reliability gate -- full dogfood: resume -> work -> review -> restart -> resume -> finalize -> EPUB - -## Important dependency rules - -- #17 depends on #7 + #8; it does **not** need to wait for #9. -- #14 may start after #9; only final integration with `finalize` waits for #12. -- #12 consumes authoritative review ledger state from #9; it must not depend on generated Markdown reports from #21. -- #21 and #12 can run in parallel after #9, subject to #12 also having #10/#11 ready. -- #15 depends on #8 + #9 + #10; it does not need to wait for EPUB/finalize. -- #16 should wait until schemas from #9 and #11 are stable. -- #19 should be finalized late, after real CLI/domain behavior is stable. - -## Merge order target - -Preferred integration order, not necessarily task start order: - -1. #7 -2. #13 -3. #8 -4. #11 -5. #17 -6. #9 -7. #10 -8. #21 -9. #14 -10. #12 -11. #16 -12. #15 -13. #22 -14. #19 -15. close #18 only after the full release reliability gate - -## Agent rules - -- Read issue scope and acceptance criteria before coding. -- Branch from the latest `refactor/workflow-engine-v2`. -- Do not invent duplicate state/concurrency/review mechanisms in feature branches. -- Reuse the APIs introduced by upstream issues. -- Keep feature PRs focused; do not mix unrelated schema/workflow/content changes. -- Add or update tests with each implementation task; #18 is the cross-cutting release gate, not a substitute for task-local tests. -- Rebase/update from the integration branch before final PR review when upstream dependencies changed. -- Do not merge directly to `main`. - -## Release gate - -Do not open the final PR to `main` until all are true: - -- P0/P1 invariants are covered by tests. -- `refactor/workflow-engine-v2` is green. -- no mandatory GitHub Action is required for core translate/review/resume/finalize/build flow. -- review completion is derived from machine-readable ledger state. -- private-source mode works without committing copyrighted source binaries. -- filesystem and GitHub API execution paths satisfy the same domain invariants. -- full restart/resume/finalize/EPUB dogfood succeeds. - -Tracking epic: #20. diff --git a/docs/WORKFLOW_V2_CLAIMS_CAS_DESIGN.md b/docs/WORKFLOW_V2_CLAIMS_CAS_DESIGN.md deleted file mode 100644 index d0db4c3..0000000 --- a/docs/WORKFLOW_V2_CLAIMS_CAS_DESIGN.md +++ /dev/null @@ -1,324 +0,0 @@ -# Workflow v2 — Claims, leases and compare-and-swap concurrency - -Issue: #8 -Branch: `feature/workflow-v2-claims-cas` -Base while #7 is pending: `feature/workflow-v2-state-core` -Final PR target: `refactor/workflow-engine-v2` -Date: 2026-09-05 - -## Purpose - -Make multi-session coordination mechanically safe so the user is not required to schedule translator/reviewer work manually. This design adds durable per-unit claims, lease expiry, auditable release/cleanup, deterministic range-conflict handling, and stronger compare-and-swap semantics for shared mutable state. - -The repository remains authoritative. Claims are workflow state, not chat/session memory. - -## Scope - -In scope: - -- durable per-unit claims under `.workflow/claims/`; -- translator/reviewer roles; -- session identity, unit identity, base state revision/commit, workflow revision, timestamps, and expiry; -- atomic claim acquisition; -- deterministic overlapping-range rejection; -- release and expired-claim cleanup; -- append-only audit evidence for release/cleanup attempts and completions; -- version-checked deletion; -- stronger filesystem CAS semantics for shared mutable state; -- CLI operations for claim/list/release/cleanup; -- stable JSON output for later `status`/`resume` integration. - -Out of scope: - -- review-ledger/hash semantics (#9); -- deterministic resume/context selection (#10); -- private-source mode (#11); -- explicit parallel translation policy (#15); -- GitHub API backend (#17); -- migration commands (#16). - -## Canonical unit identity - -Issue #8 coordinates chapter units already defined by `progress.json`. The canonical unit ID is derived from the validated chapter number, not from a mutable human title or slug: - -```text -chapter-000001 -chapter-000002 -... -``` - -CLI selectors accept only a positive chapter number (`7`) or an inclusive numeric range (`7-12`) in #8. They are normalized against `progress.json` into canonical unit IDs before any mutation. Missing chapters, reversed ranges, duplicate numbers in progress state, and out-of-book selectors are rejected before claim acquisition starts. - -This keeps claim paths stable even when a chapter title/slug changes in a later migration. - -## Durable layout - -Active claims are one file per unit: - -```text -books//.workflow/claims/.json -``` - -A unit has one active claim path regardless of role. Therefore Translator and Reviewer cannot concurrently own the same unit. - -Claim lifecycle audit evidence is append-only: - -```text -books//.workflow/claim-events/-.json -``` - -Audit events are created through `create_if_absent`; they are never the source of current ownership. Current ownership is defined only by `.workflow/claims/`. - -## Claim schema - -The Workflow v2 `claim` schema contains these durable fields: - -- `schema_version`; -- `claim_id`: collision-resistant UUID hex identity for this specific acquisition; -- `unit_id` matching `chapter-[0-9]{6}`; -- `role`: `translator` or `reviewer`; -- `session_id`; -- `base_revision`: revision token of `progress.json` at acquisition time; -- `base_commit`: Git commit when available, otherwise `null`; -- `workflow_revision`: resolved workflow revision when available, otherwise the most specific recorded requested ref; -- `claimed_at`; -- `expires_at`. - -`claim_id` prevents an ABA lifecycle race: if a claim is deleted and later recreated for the same unit with otherwise identical content, the durable revision still changes, so a stale release/cleanup operation cannot delete the replacement claim. - -Times are UTC RFC 3339 timestamps. Expiry comparison is performed against an injected clock in domain logic so tests are deterministic. - -Unknown explicit schema versions remain fatal. No implicit claim migration is introduced. - -A claim cannot be created if the active book has no interpretable workflow revision in metadata; the coordination record must not fabricate provenance. - -## Claim acquisition - -Single-unit acquisition uses storage `create_if_absent` on the canonical unit claim path. Existing active content yields a conflict; it is never overwritten during acquisition. - -Expired claims are still conflicts until an explicit cleanup operation removes them. Acquisition does not silently steal expired claims because cleanup must remain auditable. - -### Range acquisition - -A requested range is normalized into a sorted unique list of canonical unit IDs before any write. - -Algorithm: - -1. validate the book/progress state and every requested unit; -2. inspect current claims and reject the first conflicting canonical unit in sorted order before writing when a conflict is already visible; -3. acquire unit claims in canonical unit order with `create_if_absent`; -4. if any acquisition conflicts due to a race, roll back claims already created by this operation using their exact returned revision tokens; -5. report the first conflicting unit in canonical order and create no successful range result unless the entire range is owned. - -Rollback failures are surfaced as blocking coordination errors and identify every claim path that could not be rolled back. They are never silently ignored. - -Another concurrent requester may briefly observe a partial in-progress range and receive a conflict; it must retry after the first request either completes or rolls back. No requester is told that a range was acquired until all units are durably owned. - -## Storage contract changes - -Add to `StorageBackend`: - -```python -def delete_if_version(path: str, expected_version: str) -> None: ... -``` - -`delete_if_version` must: - -- fail with `StorageNotFound` when the path is absent; -- fail with `StorageVersionConflict` when durable content no longer matches the expected revision; -- remove only the revision that was actually read/acquired. - -`WorkflowStateRepository` exposes the corresponding version-checked document deletion helper. Domain code does not bypass schema validation when interpreting a claim. - -## Filesystem CAS semantics - -Issue #7 introduced SHA-256 revision tokens and stale-write detection but left a small cross-process time-of-check/time-of-use window between the final revision check and `os.replace`. - -Issue #8 closes that gap for writers using the filesystem backend. - -The filesystem backend uses a short-lived advisory mutex keyed by the resolved storage root plus logical target path while executing `write_if_version` and `delete_if_version`. The mutex implementation uses Python standard-library platform adapters (`fcntl` on POSIX, `msvcrt` on Windows) behind one internal abstraction. - -Lock files live outside the logical repository state in the operating-system temporary directory and may persist as empty coordination files. Lock ownership is held by the OS file descriptor, so process termination releases ownership without requiring stale-lock cleanup. Lock files contain no workflow/lease state and are never returned by storage `list()`. - -Required properties: - -- the mutation critical section covers the version check and final replace/delete; -- two backend writers targeting the same logical path cannot both successfully commit from the same expected revision; -- stale writers fail with `StorageVersionConflict`; -- lock ownership is released on success and exception paths; -- logical path safety remains enforced before any mutation. - -Direct out-of-band filesystem edits do not participate in the mutex. They are still detected when they change content before the backend's final revision check, but #8 only guarantees linearization among writers using the storage backend. - -The future GitHub backend (#17) will implement the same storage contract using GitHub SHA/conditional mutation semantics rather than filesystem locks. - -## Release - -Release is ownership-sensitive. The caller supplies a selector and `session_id`. - -For each selected unit, domain logic: - -1. reads and validates the current claim and its exact revision; -2. verifies the claim belongs to the supplied session; -3. creates an immutable `release_requested` event containing the claim snapshot/revision and reason; -4. removes the active claim using `delete_if_version` with the exact revision that was read; -5. creates an immutable `released` completion event referencing the request event. - -A stale session cannot release a newer claim because ownership and version-checked deletion are both required. - -If deletion conflicts, the durable `release_requested` event remains as audit evidence of an unsuccessful attempt; no `released` completion event is created. If the final completion-event write fails after deletion, the operation returns an audit-persistence error and the request event remains durable, so the deletion is not invisible. - -For a multi-unit release selector, units are processed independently in canonical order. The command returns structured per-unit results and does not claim range-atomic release semantics. - -## Expired-claim cleanup - -Cleanup is explicit. It does not happen implicitly during acquisition. - -For each active claim in canonical order: - -1. read and validate the claim and exact revision; -2. compare `expires_at` with the injected current time; -3. skip live claims; -4. create an immutable `cleanup_requested` event containing the claim snapshot/revision, cleanup timestamp, and reason `lease_expired`; -5. delete the claim using `delete_if_version` with the inspected revision; -6. create an immutable `cleaned` completion event referencing the request event. - -If the claim changes between inspection and deletion, cleanup records only the request event and reports a conflict rather than deleting the replacement claim. - -Cleanup continues across independent units and returns deterministic per-unit results. A conflict on one unit cannot authorize deletion of another unit's live claim. - -## Claim event schema - -#8 adds a versioned `claim_event` document kind to the schema layer. - -Request events include: - -- `schema_version`; -- `event_id`; -- `action`: `release_requested` or `cleanup_requested`; -- `unit_id`; -- `claim_revision`; -- `claim`: the validated claim snapshot; -- `occurred_at`; -- `reason`; -- optional `detail`. - -Completion events include: - -- `schema_version`; -- `event_id`; -- `action`: `released` or `cleaned`; -- `unit_id`; -- `request_event_id`; -- `occurred_at`. - -Event filenames contain a sortable UTC timestamp plus unique ID; correctness does not depend on filename parsing. Event IDs are collision-resistant UUID hex values generated by the domain layer. - -## Domain component - -Add `scripts/workflow_v2/claims.py` containing claim-domain operations independent of CLI and filesystem details. - -Responsibilities: - -- map validated progress chapters to canonical unit IDs; -- normalize and validate single/range selectors; -- construct and validate claims; -- acquire one or many claims; -- list active claims; -- release owned claims; -- cleanup expired claims; -- create lifecycle audit events; -- return structured domain results/errors. - -Dependencies are `WorkflowStateRepository`/`StorageBackend` abstractions plus an injected clock and UUID factory. The module does not call GitHub, shell commands, or argparse. - -## CLI - -Extend `scripts/book.py` with these exact #8 surfaces: - -```text -book.py claim --role --session-id [--lease-seconds ] [--base-commit ] [--json] -book.py claims [--json] -book.py release --session-id [--json] -book.py cleanup-claims [--json] -``` - -`selector` is `N` or `N-M` with positive decimal chapter numbers. `--lease-seconds` defaults to `3600` and must be greater than zero. `--base-commit` is optional; when omitted, the helper may resolve the current Git commit when Git is available, otherwise it records `null` without fabricating a value. - -Fixed CLI invariants: - -- machine-readable JSON output uses stable keys and canonical unit ordering; -- acquisition reports success only after the whole requested selector is acquired; -- acquisition/storage/ownership conflicts return non-zero status; -- `claims` lists active claims in canonical unit order; -- cleanup removes only expired claims; -- release requires matching session ownership; -- CLI delegates coordination semantics to `claims.py` rather than duplicating them. - -## Error model - -Domain errors distinguish at least: - -- invalid selector/unit; -- claim conflict; -- ownership mismatch; -- storage version conflict; -- rollback failure; -- invalid claim/audit document; -- audit-persistence failure after a state mutation. - -Storage exceptions remain specific where their identity matters. CLI converts expected domain/storage failures to stable non-zero errors without stack traces. - -## Testing strategy - -Development is test-first. - -### Storage tests - -- `delete_if_version` deletes only the expected revision; -- stale version cannot delete replacement content; -- simultaneous stale writers cannot both commit successfully; -- mutation lock ownership releases on ordinary success/failure; -- path safety remains enforced. - -### Claim-domain tests - -- two sessions cannot acquire the same unit; -- Translator and Reviewer conflict on the same unit; -- expired claim is not silently stolen; -- overlapping ranges are rejected deterministically; -- partial range acquisition rolls back claims created by the failed attempt; -- foreign session cannot release a claim; -- stale release cannot delete a replacement claim; -- delete/recreate ABA cannot reuse the prior durable claim revision because `claim_id` is unique; -- cleanup removes expired but not live claims; -- cleanup records request/completion audit with reason and exact claim revision; -- injected clock makes expiry boundaries deterministic. - -### CLI tests - -- claim/list/release/cleanup happy paths; -- JSON output stability; -- non-zero conflicts; -- range behavior; -- existing extract/validate/build/corpus behavior remains green. - -Full suite runs on Python 3.10 and 3.12. - -## Acceptance mapping - -Issue #8 acceptance criteria map as follows: - -1. **Two sessions cannot claim the same unit concurrently** — one unit path + atomic `create_if_absent`, with concurrency/conflict tests. -2. **Overlapping ranges are rejected deterministically** — canonical sorted range acquisition plus rollback and conflict tests. -3. **Lost updates fail with conflict** — strengthened filesystem `write_if_version` mutation mutex and existing revision tokens. -4. **Expired claims can be cleaned up with an auditable reason** — explicit cleanup request/completion audit plus version-checked deletion. -5. **CLI supports claim/list/release/cleanup** — `book.py` subcommands backed by domain operations. - -## Dependency and PR strategy - -Until #23/#7 is merged, `feature/workflow-v2-claims-cas` is stacked on `feature/workflow-v2-state-core` because #8 directly depends on its schema/storage abstractions. - -The #8 implementation PR may initially target `feature/workflow-v2-state-core` so its diff contains only #8 work. After #23 merges into `refactor/workflow-engine-v2`, the branch will be rebased/updated as needed and the PR retargeted to `refactor/workflow-engine-v2`, preserving a clean integration diff. - -No merge into the integration branch or `main` is performed without a separate integration decision. diff --git a/docs/WORKFLOW_V2_CLAIMS_CAS_PLAN.md b/docs/WORKFLOW_V2_CLAIMS_CAS_PLAN.md deleted file mode 100644 index 87fd5a3..0000000 --- a/docs/WORKFLOW_V2_CLAIMS_CAS_PLAN.md +++ /dev/null @@ -1,384 +0,0 @@ -# Workflow v2 Claims and CAS Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Implement issue #8 so concurrent sessions coordinate through durable per-chapter claims, expired leases are auditable, and filesystem shared-state writes use real compare-and-swap critical sections. - -**Architecture:** Extend the #7 storage protocol with version-checked delete and serialize filesystem CAS mutations with OS advisory locks outside logical repository state. Add a backend-neutral claim domain service that derives canonical chapter unit IDs from validated progress state, acquires per-unit claim files atomically, rolls back failed ranges, and records append-only lifecycle audit events. Integrate the service through `scripts/book.py` while keeping GitHub/backend-specific behavior out of domain logic. - -**Tech Stack:** Python 3.10+, standard library only (`fcntl`/`msvcrt`, `uuid`, `datetime`, `json`, `argparse`, `unittest`). - -**Spec:** `docs/WORKFLOW_V2_CLAIMS_CAS_DESIGN.md` - -## Global Constraints - -- Repository state remains authoritative over chat history. -- No database, queue, backend service, mandatory GitHub Actions dependency, or new runtime package is introduced. -- Claims are one active file per canonical chapter unit under `.workflow/claims/`. -- Canonical unit IDs are `chapter-%06d`, derived from validated `progress.json` chapter numbers. -- Claim records include a collision-resistant `claim_id` UUID in addition to the approved spec fields; this prevents an ABA stale-release from deleting a newly recreated byte-identical claim. -- Expired claims remain conflicts until explicit cleanup records audit evidence. -- Range acquisition reports success only if every unit was acquired; partial acquisitions are rolled back with exact returned revisions. -- Filesystem CAS guarantees apply to writers using `StorageBackend`; direct out-of-band filesystem mutation is not serialized. -- Python 3.10 and 3.12 full suites must pass before the PR is review-ready. - ---- - -### Task 1: Strong filesystem CAS and versioned delete - -**Files:** -- Modify: `scripts/workflow_v2/storage.py` -- Modify: `scripts/workflow_v2/filesystem.py` -- Modify: `scripts/workflow_v2/repository.py` -- Modify: `scripts/workflow_v2/__init__.py` -- Test: `tests/test_workflow_v2_storage.py` -- Test: `tests/test_workflow_v2_repository.py` - -**Interfaces:** -- Consumes: existing `StoredValue`, `StorageVersionConflict`, SHA-256 content revisions, safe logical paths. -- Produces: `StorageBackend.delete_if_version(path: str, expected_version: str) -> None` and `WorkflowStateRepository.delete_if_version(path: str, schema: SchemaKind, expected_version: str) -> None`. - -- [ ] **Step 1: Write failing storage tests** - -Add tests that require version-checked deletion and a shared mutation mutex: - -```python -def test_delete_if_version_rejects_stale_revision(self): - storage = self.storage() - old = storage.create_if_absent("claim.json", b"old") - current = storage.write_if_version("claim.json", b"new", old) - with self.assertRaises(StorageVersionConflict): - storage.delete_if_version("claim.json", old) - self.assertEqual(storage.read("claim.json").version, current) - - -def test_delete_if_version_removes_matching_revision(self): - storage = self.storage() - version = storage.create_if_absent("claim.json", b"claim") - storage.delete_if_version("claim.json", version) - with self.assertRaises(StorageNotFound): - storage.read("claim.json") -``` - -Add a deterministic lock-contract test by injecting/patching the internal advisory-lock boundary and a multiprocessing stress test asserting that two backend writers starting from one expected revision never both report success. - -- [ ] **Step 2: Run the focused tests and verify RED** - -Run through CI/test runner: - -```text -python -m unittest tests.test_workflow_v2_storage tests.test_workflow_v2_repository -v -``` - -Expected: failures because `delete_if_version` and the mutation-lock implementation do not exist yet. - -- [ ] **Step 3: Implement the storage contract** - -Extend the protocol: - -```python -class StorageBackend(Protocol): - ... - def delete_if_version(self, path: str, expected_version: str) -> None: - ... -``` - -Implement an internal advisory mutex keyed by normalized resolved root + logical path. POSIX uses `fcntl.flock`; Windows uses `msvcrt.locking`. Lock files live under a deterministic directory in `tempfile.gettempdir()`, contain no workflow state, and are not visible to `StorageBackend.list()`. - -Wrap the complete read/check/replace and read/check/unlink critical sections: - -```python -with self._mutation_lock(path): - current = self.read(path) - if current.version != expected_version: - raise StorageVersionConflict(...) - # replace or unlink while still holding the OS lock -``` - -- [ ] **Step 4: Add repository version-checked delete** - -Validate the document before deletion and then delegate the exact revision to storage: - -```python -def delete_if_version(self, path, schema, expected_version): - loaded = self.read(path, schema) - if loaded.version != expected_version: - raise StorageVersionConflict(...) - self.storage.delete_if_version(path, expected_version) -``` - -- [ ] **Step 5: Run focused tests and verify GREEN** - -```text -python -m unittest tests.test_workflow_v2_storage tests.test_workflow_v2_repository -v -``` - -Expected: PASS. - -- [ ] **Step 6: Commit the independently reviewable storage change** - -```text -workflow: add strong filesystem CAS deletion -``` - ---- - -### Task 2: Claim and claim-event schemas plus selector model - -**Files:** -- Modify: `scripts/workflow_v2/schemas.py` -- Create: `scripts/workflow_v2/claims.py` -- Modify: `scripts/workflow_v2/__init__.py` -- Test: `tests/test_workflow_v2_schemas.py` -- Create: `tests/test_workflow_v2_claims.py` - -**Interfaces:** -- Consumes: `SchemaKind`, `WorkflowStateRepository`, `SCHEMA_VERSION`. -- Produces: `SchemaKind.CLAIM_EVENT`, `canonical_unit_id(number: int) -> str`, `resolve_selector(progress: Mapping[str, Any], selector: str) -> list[str]`, `ClaimManager` and claim-domain errors/results. - -- [ ] **Step 1: Write failing schema/selector tests** - -Require a claim UUID, canonical unit ID, nullable base commit, UTC timestamps, and action-specific claim events: - -```python -claim = { - "schema_version": 1, - "claim_id": "0123456789abcdef0123456789abcdef", - "unit_id": "chapter-000001", - "role": "translator", - "session_id": "session-a", - "base_revision": "progress-rev", - "base_commit": None, - "workflow_revision": "workflow-rev", - "claimed_at": "2026-09-05T12:00:00Z", - "expires_at": "2026-09-05T13:00:00Z", -} -self.assertEqual(parse_document(SchemaKind.CLAIM, claim).data, claim) -``` - -Selector tests cover `1`, `1-3`, reversed ranges, missing chapters, duplicate progress chapter numbers, zero/negative/non-numeric selectors, and canonical `chapter-%06d` ordering. - -- [ ] **Step 2: Run focused tests and verify RED** - -```text -python -m unittest tests.test_workflow_v2_schemas tests.test_workflow_v2_claims -v -``` - -Expected: missing `CLAIM_EVENT`, `claim_id`, and claims module/API failures. - -- [ ] **Step 3: Implement strict schema validation** - -Add `CLAIM_EVENT`; validate `claim_id` as 32 lowercase hex characters, `unit_id` as `chapter-[0-9]{6}`, `base_commit` as null or non-empty string, timestamps as timezone-aware UTC RFC3339 values, and `expires_at > claimed_at`. - -Request events accept `release_requested`/`cleanup_requested` and require exact claim snapshot/revision/reason. Completion events accept `released`/`cleaned` and require `request_event_id`. - -- [ ] **Step 4: Implement canonical selector resolution** - -```python -def canonical_unit_id(number: int) -> str: - if type(number) is not int or number < 1: - raise InvalidClaimSelector(...) - return f"chapter-{number:06d}" -``` - -`resolve_selector` builds a number->unit map from progress, rejects duplicate numbers before mutation, parses only positive `N` or `N-M`, and verifies every integer in an inclusive range exists. - -- [ ] **Step 5: Run schema/selector tests and verify GREEN** - -```text -python -m unittest tests.test_workflow_v2_schemas tests.test_workflow_v2_claims -v -``` - -Expected: PASS for schema and selector cases. - -- [ ] **Step 6: Commit the schema/domain foundation** - -```text -workflow: define durable claim identities and selectors -``` - ---- - -### Task 3: Claim acquisition, leases, rollback, release and cleanup audit - -**Files:** -- Modify: `scripts/workflow_v2/claims.py` -- Modify: `scripts/workflow_v2/__init__.py` -- Test: `tests/test_workflow_v2_claims.py` - -**Interfaces:** -- Consumes: repository create/read/delete, canonical unit IDs, `SchemaKind.CLAIM`, `SchemaKind.CLAIM_EVENT`. -- Produces: `ClaimManager.acquire(...)`, `list_active()`, `release(...)`, `cleanup_expired()`, structured lifecycle results, and errors `ClaimConflict`, `ClaimOwnershipError`, `ClaimRollbackError`, `ClaimAuditError`. - -- [ ] **Step 1: Write failing acquisition tests** - -Use an injected fixed UTC clock and deterministic UUID factory. Cover: - -```python -first = manager.acquire(progress, "1", role="translator", session_id="a", ...) -with self.assertRaises(ClaimConflict): - manager.acquire(progress, "1", role="reviewer", session_id="b", ...) -``` - -Also require expired-but-not-cleaned claims to remain conflicts, overlapping `1-3` vs `3-5` ranges to reject, and a forced `create_if_absent` race to roll back only revisions created by the failed batch. - -- [ ] **Step 2: Run claim tests and verify RED** - -```text -python -m unittest tests.test_workflow_v2_claims -v -``` - -Expected: missing lifecycle methods/results. - -- [ ] **Step 3: Implement acquisition/listing** - -Construct one claim document per unit with a unique `claim_id`, shared dispatch provenance, `claimed_at`, and `expires_at`. Preflight visible conflicts in canonical order, then call `repository.create` in canonical order. On `StorageAlreadyExists`, delete already-created batch members using their exact returned versions; if any rollback fails, raise `ClaimRollbackError` containing the unresolved paths. - -- [ ] **Step 4: Write failing release/cleanup tests** - -Require foreign-session release rejection, stale-version deletion protection, release request + completion audit, expired-only cleanup, live-claim preservation, cleanup request reason `lease_expired`, completion linkage, and deterministic unit ordering. - -Include an ABA regression: delete/recreate the same logical unit with the same session/timestamps but a different `claim_id`; a stale lifecycle operation must not delete the replacement. - -- [ ] **Step 5: Implement audited lifecycle operations** - -For release/cleanup, persist a request event first, then version-checked delete, then completion event. A delete conflict leaves only the request event. A completion-event persistence failure after deletion raises `ClaimAuditError` and preserves the request event as evidence that the state mutation occurred after a recorded attempt. - -Multi-unit release and cleanup process independent units in canonical order and return per-unit structured status; they do not claim range-atomic deletion semantics. - -- [ ] **Step 6: Run all claim-domain tests and verify GREEN** - -```text -python -m unittest tests.test_workflow_v2_claims tests.test_workflow_v2_schemas tests.test_workflow_v2_storage tests.test_workflow_v2_repository -v -``` - -Expected: PASS. - -- [ ] **Step 7: Commit lifecycle coordination** - -```text -workflow: add claims leases and lifecycle audit -``` - ---- - -### Task 4: `book.py` claim/list/release/cleanup CLI - -**Files:** -- Modify: `scripts/book.py` -- Modify: `tests/test_book_cli.py` - -**Interfaces:** -- Consumes: `ClaimManager`, validated metadata/progress documents plus exact progress revision. -- Produces: - - `book.py claim --role ... --session-id ... [--lease-seconds 3600] [--base-commit ...] [--json]` - - `book.py claims [--json]` - - `book.py release --session-id ... [--json]` - - `book.py cleanup-claims [--json]` - -- [ ] **Step 1: Write failing CLI tests** - -Create an isolated temporary book, then require: - -```text -claim 1 -> exit 0 -claim 1 from another session -> non-zero -claims --json -> stable canonical record list -release 1 wrong session -> non-zero and claim remains -release 1 owner -> exit 0 and audit files exist -cleanup-claims -> removes expired claims only -claim 1-3 -> reports success only after all three claims exist -``` - -Require invalid/missing workflow provenance to reject acquisition rather than fabricate `workflow_revision`. - -- [ ] **Step 2: Run CLI tests and verify RED** - -```text -python -m unittest tests.test_book_cli -v -``` - -Expected: argparse reports unknown claim commands / missing implementation. - -- [ ] **Step 3: Implement CLI integration** - -Add a helper that reads metadata/progress through `WorkflowStateRepository`, retaining `progress.version`. Resolve `workflow_revision` from `metadata.workflow.resolved_revision`, then `requested_ref`; otherwise raise `BookError`. - -Resolve `base_commit` from explicit `--base-commit` first. If omitted, best-effort `git rev-parse HEAD` is allowed in the CLI layer only; failure records `null`. - -Convert expected domain/storage/schema failures to `BookError`. JSON output uses `json.dumps(..., ensure_ascii=False, sort_keys=True)` and canonical unit ordering. - -- [ ] **Step 4: Run CLI tests and verify GREEN** - -```text -python -m unittest tests.test_book_cli tests.test_workflow_v2_claims -v -``` - -Expected: PASS. - -- [ ] **Step 5: Commit the CLI surface** - -```text -workflow: expose claim coordination commands -``` - ---- - -### Task 5: Execution-contract alignment and full regression verification - -**Files:** -- Modify: `docs/ORCHESTRATION.md` -- Modify: `tests/test_agent_contract.py` -- Verify: all tests under `tests/` - -**Interfaces:** -- Consumes: final claim CLI and lifecycle behavior. -- Produces: an orchestration contract that requires durable claim acquisition before dispatch and ownership-safe release/cleanup rather than manual scheduling. - -- [ ] **Step 1: Write the failing contract test** - -Require the orchestration contract to mention the executable claim gate and exact command surface without duplicating implementation details: - -```python -for phrase in ( - "python scripts/book.py claim", - "python scripts/book.py claims", - "python scripts/book.py release", - "python scripts/book.py cleanup-claims", -): - self.assertIn(phrase, text) -``` - -- [ ] **Step 2: Run the contract test and verify RED** - -```text -python -m unittest tests.test_agent_contract -v -``` - -Expected: missing claim-command contract phrases. - -- [ ] **Step 3: Update `docs/ORCHESTRATION.md`** - -Document that the orchestrator acquires the selected unit claim before literary dispatch, never treats expired ownership as free until cleanup, releases claims with matching session identity, and uses explicit cleanup for expired leases. Preserve the existing default sequential translation policy; #8 provides safe coordination primitives and does not enable #15 parallel mode. - -- [ ] **Step 4: Run the complete suite** - -```text -python -m unittest discover -s tests -v -``` - -Expected: all tests pass locally/CI with no regressions. - -- [ ] **Step 5: Inspect the feature diff against `feature/workflow-v2-state-core`** - -Confirm the diff contains only #8 design/plan, storage/CAS changes, claim domain/schema, CLI/tests, and targeted orchestration documentation. Verify no `docs/superpowers/`, database/queue/backend implementation, review ledger, status/resume, or parallel translation policy was introduced. - -- [ ] **Step 6: Commit final contract alignment** - -```text -docs: align orchestration with durable claims -``` - -- [ ] **Step 7: Open/update the stacked PR and verify CI on Python 3.10 and 3.12** - -Initially target `feature/workflow-v2-state-core` so the PR diff is #8-only while #23 is pending. After #23 merges, retarget to `refactor/workflow-engine-v2` and re-run CI before integration. diff --git a/docs/WORKFLOW_V2_EPUB_OUTPUT_DESIGN.md b/docs/WORKFLOW_V2_EPUB_OUTPUT_DESIGN.md deleted file mode 100644 index 71c8f47..0000000 --- a/docs/WORKFLOW_V2_EPUB_OUTPUT_DESIGN.md +++ /dev/null @@ -1,213 +0,0 @@ -# Workflow v2 EPUB Output Design (#14) - -## Goal - -Make EPUB a deterministic, validated Workflow v2 deliverable rather than a manual packaging step. Preserve the existing Markdown build path and keep generated artifacts subordinate to authoritative repository state. - -## Scope - -- Extend `book.py build ` with `--format markdown|epub`; default remains Markdown. -- EPUB builds use canonical `progress.json` chapter order. -- Normal builds require `reviewed` lifecycle plus current PASS evidence; `--allow-unreviewed` remains explicit preview mode and permits `translated`/`reviewed` units without claiming final-review completion. -- Generate a deterministic EPUB 3 package with metadata, language, nav/TOC, spine, CSS, chapter XHTML and optional cover. -- Validate every generated EPUB before reporting success. -- Write deterministic `output/manifest.json` describing the artifact and the exact build inputs. -- Add read-only `book.py build-status --format epub [--json]` returning `missing`, `current`, `stale` or `invalid`. -- Extend #18 reliability coverage for incomplete builds, interrupted/stale outputs and idempotent rebuilds. - -## Architecture - -### Domain module - -Add `scripts/workflow_v2/epub_output.py` with no argparse dependency. It owns: - -- safe path validation for output/cover paths; -- Markdown-to-XHTML rendering for the supported translation subset; -- deterministic EPUB assembly; -- EPUB structural validation; -- build-input snapshot/fingerprint construction; -- output manifest construction and validation; -- current/stale/invalid output resolution. - -The module uses only Python stdlib (`zipfile`, `xml.etree.ElementTree`, `html`, `hashlib`, `json`, `pathlib`, `io`). No third-party runtime dependency is introduced. - -`output/manifest.json` is intentionally **not** registered as an authoritative `SchemaKind`: it is a generated delivery projection. Its strict v1 shape is validated by `epub_output.validate_output_manifest()` before use. This keeps migration semantics for machine workflow state separate from generated output metadata. - -### CLI adapter and preflight - -Keep `book.py` as the user-facing parser and existing Markdown builder. Add a thin `workflow_v2/epub_cli.py` adapter that registers/executes the EPUB/build-status branch and translates expected failures into the existing concise CLI error surface. - -The existing `build` command remains backward-compatible: - -- no `--format` means `markdown`; -- `--output` remains supported for both formats; EPUB outputs must use `.epub`, while Markdown preserves its legacy filename semantics; -- `--allow-unreviewed` remains the explicit preview gate. - -EPUB build must not rely on raw `book.validate_book()` alone, because explicit `private_external` books intentionally have no source binary in `books//source/`. The CLI reuses the same normalized structural + corpus preflight path used by status/finalize (`status_cli.default_preflight` or an equivalent shared adapter): - -- structural errors after explicit-source normalization must be empty; -- corpus state must be `verified`; -- final build additionally requires every included unit to be `reviewed` with current PASS evidence; -- preview build requires each included unit to be `translated` or `reviewed`, but does not claim review completion. - -`book.py build-status` is read-only and never rewrites manifests or artifacts. - -## EPUB package contract - -The deterministic package contains: - -- `mimetype` as the first ZIP member, stored uncompressed, exact bytes `application/epub+zip`; -- `META-INF/container.xml`; -- `EPUB/package.opf`; -- `EPUB/nav.xhtml`; -- `EPUB/styles.css`; -- one XHTML document per included unit in canonical order; -- optional cover asset when `metadata.cover_path` is present, referenced from OPF with EPUB 3 `cover-image` semantics. - -`package.opf` uses EPUB 3.0 metadata. Required metadata: - -- title from `metadata.title`; -- creator when `metadata.author` is non-empty; -- language from `metadata.target_language`; -- deterministic identifier derived from book slug + build-input fingerprint; -- nav manifest entry, CSS, ordered chapter items, optional cover item. - -No wall-clock modified timestamp is used. ZIP entry timestamps are pinned to a constant DOS-compatible value so identical inputs produce identical EPUB bytes. - -## Markdown-to-XHTML subset - -Translations are already stored as Markdown. The builder converts a deterministic safe subset: - -- ATX headings `#` through `######`; -- paragraphs separated by blank lines; -- unordered list lines beginning `- ` or `* `; -- ordered list lines beginning `. `; -- fenced code blocks; -- all remaining text escaped as plain text. - -Inline Markdown interpretation is intentionally not added in #14. Raw HTML is escaped rather than executed. This keeps output deterministic and avoids introducing a Markdown dependency. - -Each chapter must render at least one non-empty body element or build fails. - -## Optional cover - -`metadata.cover_path` is optional and, when present: - -- must be a safe relative path inside the book workspace; -- must exist and be a regular file; -- supported extensions are `.jpg`, `.jpeg`, `.png`, `.gif`, `.svg`; -- its exact bytes participate in the build fingerprint; -- it is copied into the EPUB with the correct media type and referenced from OPF as the cover image. - -Existing books without `cover_path` need no migration. - -## Build-input snapshot and fingerprint - -The build-input snapshot is deterministic data derived only from relevant inputs: - -- build contract version (`epub-build-v1`); -- book slug; -- selected format and preview mode; -- metadata fields affecting EPUB bytes: title, author, target language, cover path; -- workflow resolved revision; -- ordered units: unit id/number/title/translation path/status + exact translation SHA-256; -- for final builds, normalized current review identity per unit: resolved state, exact source/translation hashes, workflow revision and review-contract revision; -- optional cover SHA-256; -- explicit build configuration version. - -Raw storage revision tokens are **not** part of the fingerprint. In particular, `review-ledger.json` revision is not a direct fingerprint input: appending semantically duplicate PASS evidence must not stale otherwise identical output. Final-build review identity is derived through the same current-resolution logic as #9/#21. - -The fingerprint is SHA-256 of canonical JSON for that snapshot. - -Repository/storage revisions are still recorded separately in the manifest as provenance. Repository HEAD is best-effort (`repository_commit`, nullable) and does not participate in staleness because unrelated commits must not invalidate an otherwise identical artifact. - -## Output manifest - -`books//output/manifest.json` is generated only after the candidate EPUB validates successfully. Strict domain-validated schema: - -```json -{ - "schema_version": 1, - "build_contract": "epub-build-v1", - "book_slug": "sample", - "format": "epub", - "preview": false, - "artifact_path": "output/sample.epub", - "artifact_sha256": "...", - "unit_count": 12, - "input_fingerprint": "...", - "repository_commit": "... or null", - "state_revisions": { - "metadata": "...", - "progress": "...", - "review_ledger": "..." - } -} -``` - -The manifest is a generated projection, not authoritative workflow state. - -If an existing manifest+artifact already resolve as `current`, an identical rebuild returns unchanged and preserves the existing manifest bytes. This prevents an unrelated new Git HEAD from causing provenance-only churn. - -## Staleness semantics - -`build-status` recomputes the current input snapshot/fingerprint and validates the stored manifest/artifact: - -- `missing`: manifest or artifact is absent; -- `current`: manifest validates, artifact exists and validates, artifact SHA matches, input fingerprint matches current relevant inputs; -- `stale`: manifest/artifact are structurally valid but the current relevant input fingerprint differs; -- `invalid`: manifest malformed/unsupported, path unsafe, artifact hash mismatch, EPUB invalid or manifest/artifact identity inconsistent. - -Relevant mutations that must produce `stale`: translation bytes, title/author/target language, chapter order/title/path/status, current review identity for final builds, cover path/bytes, workflow/build contract configuration. - -Changing an unrelated repository file/commit alone does not produce `stale`. - -## Write and failure semantics - -- Build performs read/preflight/render/assemble/validate entirely before replacing the final EPUB. -- Artifact is written with temp-file + atomic replace semantics through the filesystem adapter. -- Manifest is written only after the final artifact is present and revalidated. -- If candidate generation/validation fails, the prior valid artifact/manifest are left untouched. -- If process death occurs after artifact replacement but before manifest replacement, `build-status` returns `invalid` or `stale`, never `current`; rerun rebuilds deterministically. -- Identical successful rebuilds detect equal bytes and do not rewrite artifact/manifest. - -## Validation contract - -The validator rejects unless all are true: - -- ZIP opens successfully; -- `mimetype` exists, is first, stored, exact content; -- `META-INF/container.xml` parses and references an existing OPF; -- OPF parses as EPUB package document; -- title/language present; -- manifest contains nav, CSS and all expected chapter items; -- nav XHTML parses and links every chapter in order; -- spine count equals expected unit count and every `idref` resolves; -- every chapter XHTML parses and has non-empty body text/content; -- optional cover reference resolves and media type matches its supported extension. - -## Integration with #12 - -A normal final build after `book.py finalize` naturally sees all chapters as `reviewed` and current PASS evidence. #14 does not parse `STATE.md`, `FINAL_QUALITY_GATES.md` or `REVIEW_REPORT.md`; authoritative metadata/progress/review ledger and exact artifact bytes remain the source of truth. - -Preview builds may use translated-but-unreviewed units, are marked `preview: true`, and do not count as a final deliverable. - -## TDD slices - -1. Domain snapshot/fingerprint + strict generated-manifest validation + stale resolution. -2. Deterministic EPUB assembly + validator. -3. CLI `build --format epub`, private-source preflight, preview gate, output manifest and `build-status`. -4. #18 reliability: incomplete build, interrupted artifact/manifest window, stale relevant input, semantically duplicate review evidence, unrelated Git commit and identical rebuild idempotence. -5. Full matrix CI and final diff/review/ancestry audit. - -## Acceptance criteria - -- One command builds and validates a readable EPUB from a complete reviewed book. -- Existing Markdown build behavior remains compatible. -- `private_external` final builds work without persisting the private source binary. -- Output manifest identifies exact relevant workflow state and artifact hash. -- Relevant source/review state changes are reported stale; semantically duplicate review evidence and unrelated commits are not. -- Incomplete/unreviewed final build fails unless preview mode is explicit. -- Generated EPUB and manifest are byte-identical on unchanged successful rebuild. -- Failure/interruption never reports a mismatched output as current. -- Standard Python test suite covers all behavior without requiring GitHub Actions or external services. diff --git a/docs/WORKFLOW_V2_EPUB_OUTPUT_PLAN.md b/docs/WORKFLOW_V2_EPUB_OUTPUT_PLAN.md deleted file mode 100644 index 7817870..0000000 --- a/docs/WORKFLOW_V2_EPUB_OUTPUT_PLAN.md +++ /dev/null @@ -1,108 +0,0 @@ -# Workflow v2 EPUB Output Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add deterministic first-class EPUB build/validation, output manifest and stale-output detection while preserving Markdown build behavior. - -**Architecture:** `workflow_v2.epub_output` owns input fingerprints, EPUB assembly/validation and strict generated-manifest validation. `workflow_v2.epub_cli` integrates with the existing `book.py build` command and adds read-only `build-status`; authoritative workflow state remains metadata/progress/review ledger and exact artifact bytes. - -**Tech Stack:** Python 3.10+, stdlib only (`zipfile`, `xml.etree.ElementTree`, `html`, `hashlib`, `json`, `io`), existing Workflow v2 repository/storage/review APIs. - -**Spec:** `docs/WORKFLOW_V2_EPUB_OUTPUT_DESIGN.md` - -## Global Constraints - -- Target branch: `refactor/workflow-engine-v2`; never modify `main`. -- Feature branch: `feature/workflow-v2-epub-output`. -- No third-party runtime dependency. -- Existing Markdown build remains default/backward-compatible. -- EPUB and `output/manifest.json` are generated projections, not `SchemaKind` authoritative documents. -- `private_external` final builds work without persisted source binary. -- Final build requires current PASS evidence; preview mode is explicit. -- Every behavior slice follows RED -> minimal GREEN -> full matrix checkpoint. - ---- - -### Task 1: Build-input identity and generated-manifest status - -**Files:** -- Create: `scripts/workflow_v2/epub_output.py` -- Test: `tests/test_workflow_v2_epub_output.py` - -**Interfaces:** -- `EpubOutputError(RuntimeError)` -- `BUILD_CONTRACT = "epub-build-v1"` -- `OUTPUT_MANIFEST_PATH = "output/manifest.json"` -- `build_input_snapshot(metadata, progress, resolutions, artifact_reader, *, preview, cover_reader=None) -> dict[str, Any]` -- `input_fingerprint(snapshot) -> str` -- `validate_output_manifest(manifest) -> dict[str, Any]` -- `build_output_manifest(*, book_slug, preview, artifact_path, artifact_sha256, unit_count, input_fingerprint, repository_commit, state_revisions) -> dict[str, Any]` -- `resolve_output_status(manifest, *, artifact_bytes, current_fingerprint, expected_unit_count) -> dict[str, Any]` - -- [ ] Write RED tests proving relevant translation/metadata/order/cover/current-review changes alter the fingerprint, while review record IDs/commits, raw ledger revision and repository commit do not. -- [ ] Write RED tests proving preview snapshots omit review identity. -- [ ] Write RED tests proving strict generated-manifest validation rejects unsafe paths and malformed hashes, and status distinguishes `missing/current/stale/invalid`. -- [ ] Run `python -m unittest discover -s tests -v`; accept RED only if failures are the missing Task 1 APIs and baseline remains green. -- [ ] Implement only the Task 1 APIs. Fingerprints use SHA-256 of compact, sorted canonical JSON. Final review identity includes only resolved state, exact hashes, workflow revision and review-contract revision. -- [ ] Run full suite GREEN and commit `feat: add EPUB build input identity and manifest status`. - ---- - -### Task 2: Deterministic EPUB assembly and validator - -**Files:** -- Modify: `scripts/workflow_v2/epub_output.py` -- Test: `tests/test_workflow_v2_epub_output.py` - -**Interfaces:** -- `render_markdown_xhtml(title: str, markdown: str, *, language: str) -> bytes` -- `build_epub_bytes(*, book_slug, title, author, language, units, fingerprint, cover=None) -> bytes` -- `validate_epub_bytes(content: bytes, *, expected_unit_count: int) -> dict[str, Any]` - -- [ ] Write RED deterministic-package tests: identical inputs -> identical bytes; first ZIP member is stored `mimetype`; OPF/nav/spine order matches units; metadata/language/CSS and optional cover are present. -- [ ] Write RED corruption tests for mimetype/container/OPF/nav/spine/missing or malformed/empty chapters/bad cover reference. -- [ ] Run full suite and verify failures are only missing Task 2 APIs. -- [ ] Implement stdlib writer/validator with fixed ZIP timestamps `(1980, 1, 1, 0, 0, 0)` and escaped Markdown subset (headings, paragraphs, lists, fenced code; raw HTML escaped). -- [ ] Run full suite GREEN and commit `feat: build and validate deterministic EPUB bytes`. - ---- - -### Task 3: CLI build, persistence and build-status - -**Files:** -- Create: `scripts/workflow_v2/epub_cli.py` -- Modify: `scripts/book.py` -- Test: `tests/test_workflow_v2_epub_cli.py` -- Regression: `tests/test_book_cli.py` - -**Interfaces:** -- `epub_build_command(args, root: Path) -> int` -- `build_status_command(args, root: Path) -> int` -- `register_build_status_command(subparsers, root) -> None` - -- [ ] Write RED end-to-end tests for `build --format epub`, `build-status --format epub --json`, Markdown default compatibility, final reviewed/PASS gate, explicit preview, private-source build, output extension safety and deterministic rerun. -- [ ] Run full suite and verify RED is only absent CLI behavior. -- [ ] Add `--format markdown|epub` (default `markdown`), EPUB dispatch, `build-status`, and expected `EpubCliError` handling without changing existing Markdown semantics. -- [ ] Reuse normalized structural/corpus preflight from status/finalize. Final builds require all current PASS; preview requires translated/reviewed non-empty units. -- [ ] Assemble+validate before writes. If current output already matches, do not rewrite. Otherwise write artifact via existing filesystem CAS/atomic replace, revalidate/hash, then write canonical generated manifest. Repository HEAD is nullable provenance only. -- [ ] Run full suite GREEN and commit `feat: add first-class EPUB build CLI and output status`. - ---- - -### Task 4: #18 EPUB reliability and idempotence extension - -**Files:** -- Create: `tests/test_workflow_v2_epub_reliability.py` - -- [ ] Add scenarios: incomplete final build preserves prior output; simulated artifact-before-manifest crash is non-current and recoverable; translation/metadata/order/cover/current-review mutation is stale; semantically duplicate PASS remains current; unrelated Git commit remains current; identical successful rebuild preserves exact EPUB/manifest bytes and filesystem revision/inode where applicable. -- [ ] Run full suite. If a test exposes a real defect, retain RED evidence and fix only the owning implementation. -- [ ] Commit `test: cover EPUB build recovery and staleness`. - ---- - -### Task 5: Final verification and integration audit - -- [ ] Fresh exact-head GitHub matrix: Python 3.10 and 3.12 success; capture exact full-suite count from a job log. -- [ ] Requirement audit against #14: reviewed default, preview escape hatch, metadata/nav/spine/CSS/cover, validation, manifest provenance/hash, relevant staleness, private source, idempotence. -- [ ] PR audit: target integration; `behind_by=0`; merge-base matches integration base; only #14 docs/domain/CLI/tests changed; no unresolved comments/reviews/threads; PR mergeable; `main` unchanged. -- [ ] Only after clean guards, mark PR Ready and merge with expected head SHA into `refactor/workflow-engine-v2`. Preserve feature branch. Never merge to `main`. diff --git a/docs/WORKFLOW_V2_FINALIZE_DESIGN.md b/docs/WORKFLOW_V2_FINALIZE_DESIGN.md deleted file mode 100644 index 914ccb8..0000000 --- a/docs/WORKFLOW_V2_FINALIZE_DESIGN.md +++ /dev/null @@ -1,286 +0,0 @@ -# Workflow v2 Atomic Finalize — Design - -Issue: #12 -Branch: `feature/workflow-v2-finalize` -Base: `refactor/workflow-engine-v2` at `b5388b89108e37f63ec76869662a57632a0afa68` -Target: `refactor/workflow-engine-v2` - -## Goal - -Provide one idempotent `book.py finalize ` completion gate that mechanically proves the book is ready, promotes all eligible chapters to `reviewed` in one durable CAS, generates deterministic completion projections from authoritative state, and can recover safely after process interruption. - -## Non-goals - -- Do not build EPUB output (#14). -- Do not introduce schema migrations (#16), GitHub API transport (#17), or parallel proposal reconciliation (#15). -- Do not make generated Markdown authoritative. -- Do not require GitHub Actions. -- Do not merge or otherwise change `main`. - -## Architectural choice - -Use a transient durable finalization marker plus a short-lived book coordination mutex. Final lifecycle promotion is one CAS write of `progress.json`; generated reports are deterministic projections written only after promotion. - -Rejected alternatives: - -1. Sequentially call `accept_review()` for each chapter and roll back on failure. This can expose partially promoted lifecycle state and cannot guarantee rollback after a crash. -2. Introduce a permanent finalization state machine as a new authoritative database. This adds unnecessary durable state and complicates future migration/backend work. - -## Durable coordination - -### Coordination mutex - -Path: `.workflow/coordination-lock.json`. - -Purpose: serialize only the short admission transition between ordinary unit-claim acquisition and entering finalization. It is not held for the full finalization operation. - -Schema fields: - -- `schema_version`: 1 -- `lock_id`: 32 lowercase hex characters -- `operation`: `claim_admission` or `finalize_admission` -- `session_id`: non-empty string -- `acquired_at`: UTC timestamp -- `expires_at`: UTC timestamp, strictly later than `acquired_at` - -The mutex is obtained with `create_if_absent`. If present and expired, a contender may remove it only with `delete_if_version` and retry. A live mutex is a deterministic conflict. The default lease is short (60 seconds); tests use an injected clock. - -### Claim admission - -`ClaimManager.acquire()` changes as follows: - -1. Acquire the coordination mutex as `claim_admission`. -2. While holding it, reject acquisition if `.workflow/finalization.json` exists. -3. Run the existing deterministic unit-conflict preflight and create the requested unit claims. -4. Release the coordination mutex with version-checked delete in a `finally` path. -5. Existing range rollback semantics remain unchanged for ordinary create conflicts/errors. - -Because finalization admission uses the same mutex, a claim cannot appear after finalization has installed its marker. - -### Finalization marker - -Path: `.workflow/finalization.json`. - -Schema fields: - -- `schema_version`: 1 -- `lock_id`: 32 lowercase hex characters -- `book_slug`: exact progress `book_slug` -- `workflow_revision`: immutable metadata `workflow.resolved_revision` -- `base_progress_revision`: backend storage revision observed before promotion -- `candidate_progress_sha256`: SHA-256 of the deterministic serialized all-reviewed candidate progress bytes -- `phase`: `preparing` or `promoted` -- `promoted_progress_revision`: null in `preparing`; actual backend storage revision after successful progress CAS in `promoted` -- `session_id`: session that first created the marker (audit only; recovery is not owner-bound) -- `started_at`: UTC timestamp - -The marker is transient authoritative coordination state only while finalization is unfinished. It is removed after successful post-validation and report writes. - -`candidate_progress_sha256` is deliberately content-based rather than a predicted storage version. `StorageBackend` revision tokens are opaque and future backends (#17) need not use content hashes. - -## Deterministic document identity - -Expose one repository serialization helper used by both ordinary repository writes and finalize candidate hashing. Finalize must not duplicate JSON formatting rules or infer backend revision tokens. - -The helper validates through the declared schema and returns the exact canonical UTF-8 bytes that `create`/`write_if_version` would persist. Candidate SHA-256 is computed from those bytes. - -For recovery, the current raw `progress.json` storage bytes are hashed and compared with `candidate_progress_sha256` after schema validation. - -## Finalization state transition - -### Initial preflight - -Before acquiring locks, finalize computes a read-only candidate from current state and verifies: - -- structural validation succeeds; -- source corpus state is exactly `verified`; -- metadata uses immutable `workflow.resolved_revision` and ledger review evidence; -- every chapter has a non-empty translation artifact; -- every unit resolves to current `PASS` for exact source/translation/workflow/review-contract identity; -- no unsupported lifecycle state exists. - -The candidate progress document is a deterministic deep copy of current progress with every chapter status set to `reviewed`. No durable mutation occurs during initial preflight. - -Filesystem CLI pre/postflight reuses the same structural + corpus normalization path already used by `status`/`resume`, including explicit `private_external` source behavior. Finalize domain logic receives verified corpus/preflight data rather than reimplementing filesystem-specific corpus rules. - -### Admission and revalidation - -Finalize then: - -1. Acquires the coordination mutex as `finalize_admission`. -2. Creates `.workflow/finalization.json` with `create_if_absent`, or adopts an existing compatible marker for recovery. -3. While still holding the mutex, verifies that there are zero active unit claims. -4. Releases the coordination mutex. - -After the marker exists, new unit claims are rejected by claim admission. - -Finalize immediately re-reads metadata, progress, ledger, artifacts and corpus status. Any failed business precondition before progress promotion releases the finalization marker (version-checked) so translation/review work can continue. - -### One-CAS lifecycle promotion - -Recovery cases are resolved from marker phase, backend revision, and candidate content hash: - -- `phase=preparing` and current progress revision == `base_progress_revision`: revalidate exact PASS/artifact identities, then CAS-write the full candidate document once; -- `phase=preparing` and current raw progress SHA-256 == `candidate_progress_sha256`: a previous process committed progress but crashed before updating the marker; promote marker phase with CAS using the current actual backend revision; -- `phase=promoted` and current progress revision == `promoted_progress_revision` and content hash == `candidate_progress_sha256`: promotion is already complete; do not write progress again; -- any other progress revision/content combination: fail closed as a concurrent/unexpected state change. Do not claim completion. - -After a successful progress CAS, finalize CAS-updates the marker to `phase=promoted` and records the actual backend revision returned by storage. Crash between those two writes is covered by the content-hash recovery case above. - -The candidate write changes all remaining `translated` chapters to `reviewed` together. Already-`reviewed` chapters remain `reviewed`. No sequential per-chapter promotion is used. - -Immediately before the progress CAS, finalize rechecks zero active claims and current PASS identities while the finalization marker blocks new claim admission. - -## Crash and retry semantics - -- Crash before the finalization marker: no finalize mutation exists; retry starts normally. -- Crash after marker creation but before progress CAS: retry adopts the compatible `preparing` marker, sees current progress at `base_progress_revision`, revalidates, and performs the CAS. -- Crash after progress CAS but before marker phase update: retry matches candidate content SHA-256, records the actual current backend revision in the marker, and continues. -- Crash after marker phase update: retry verifies `promoted_progress_revision` + candidate content hash and continues without another lifecycle write. -- Crash while writing generated reports: progress may already be fully reviewed, but reports are non-authoritative; retry regenerates all reports from current authoritative state. -- Crash after all reports but before marker deletion: retry reproduces the same report bytes, verifies postconditions, then removes the marker. - -A successful rerun with unchanged state performs no substantive progress or report changes. - -## Orchestration visibility and mutation admission - -An unfinished finalization must be visible to a fresh session. - -`StatusResolver.status()` reads `.workflow/finalization.json` when present and exposes a deterministic `finalization` field. Inactive status is `{ "active": false }`. Active status includes `active=true`, marker `phase`, `workflow_revision`, and `started_at`. A malformed marker or workflow/book mismatch makes status invalid. - -`StatusResolver.resume()` gives an active, valid finalization marker priority over ordinary unit work and returns `operation="finalize"` with orchestrator context. This ensures a fresh session resumes completion instead of trying per-chapter `accept_review` after a crash. - -`ReviewLedgerManager.accept_review()` rejects lifecycle promotion while a finalization marker exists. Review recording already requires a live reviewer claim, and no new reviewer claim can be admitted once finalization starts. - -Out-of-band direct artifact edits are not made impossible by the storage abstraction; instead, exact source/translation hashes are revalidated immediately before the progress CAS and again during post-validation. Any such edit invalidates completion and prevents successful marker removal. - -## Completion snapshot - -Add backend-neutral `workflow_v2.finalize` logic that builds one completion snapshot from: - -- current metadata/progress revisions; -- verified corpus status; -- current claim count; -- `build_review_report_snapshot()` from #21; -- current lifecycle counts; -- exact workflow revision. - -Snapshot fields include: - -- schema identifier (`completion-report-v1`); -- `book_slug`; -- `workflow_revision`; -- state revisions (`metadata`, `progress`, `review_ledger`); -- lifecycle counts; -- corpus reproducibility state/mode; -- review summary and PASS coverage; -- quality-gate booleans. - -No wall-clock generation timestamp is included. - -## Generated reports - -All three files are projections of the same post-promotion authoritative state: - -1. `REVIEW_REPORT.md` — use #21 `render_review_report_markdown()` from the same review snapshot. -2. `STATE.md` — concise deterministic lifecycle/review/corpus/revision summary. -3. `FINAL_QUALITY_GATES.md` — deterministic checklist showing structural validity, verified corpus, zero claims, translation completeness, 100% current PASS review coverage, and all-reviewed lifecycle. - -Generated reports are written through existing versioned storage (`create_if_absent` or `write_if_version`). Identical bytes produce no rewrite. A concurrent report edit causes a deterministic conflict rather than blind overwrite. - -The report files may be temporarily incomplete as a set if the process crashes between writes. This is acceptable because they are non-authoritative; finalize does not report success or remove its marker until all three canonical files match the current snapshot. - -## Post-validation and completion - -After progress promotion and report writes, finalize performs a fresh post-validation: - -- structural validation remains clean; -- corpus remains verified; -- zero active unit claims; -- every chapter status is `reviewed`; -- every review resolution is current `pass`; -- generated report bytes equal rendering of the fresh snapshot. - -Only then is the finalization marker deleted with `delete_if_version` and the command returns success. - -## Error handling - -### Clean precondition failure - -Before lifecycle promotion, expected failures (missing/stale/corrections review, untranslated artifact, invalid corpus/structure, active claims) produce a concise CLI error and leave progress/reports unchanged. If a marker was created during admission, it is released before returning the business failure. - -### Conflict / uncertain mutation - -Storage CAS conflicts, incompatible recovery marker, unexpected progress revision/content hash, marker phase conflict, or report write conflict fail closed. If progress may already have been promoted, retain the finalization marker so a later `finalize` retry performs recovery rather than admitting new claims. - -### Stale coordination mutex - -An expired coordination mutex can be removed only by version-checked delete. A live mutex is never stolen. - -## CLI - -Add `book.py finalize ` via a focused `workflow_v2.finalize_cli` adapter. - -Optional flags: - -- `--session-id`: explicit non-empty session identifier for deterministic/auditable operation; default is a generated UUID when omitted. -- `--json`: emit deterministic completion snapshot/result after success. JSON does not change finalization semantics; finalize still performs the operation. - -Expected workflow errors are surfaced as `ERROR: ...` without traceback and exit code 1. - -## Schema changes - -Extend `SchemaKind` with: - -- `COORDINATION_LOCK` -- `FINALIZATION_LOCK` - -Both are strict version-1 durable transient documents. No existing metadata/progress schema version is bumped; #16 owns migrations. - -The existing `GENERATED_STATE` schema is not used as a new authoritative completion document. `STATE.md` remains a generated projection. - -## Testing strategy - -Strict TDD slices: - -1. Coordination RED/GREEN: - - claim admission blocked by finalization marker; - - claim/finalize admission race is serialized by coordination mutex; - - expired mutex cleanup is version-safe; - - existing claim rollback/ownership behavior remains green. -2. Finalization preflight RED/GREEN: - - rejects active claims, invalid/unsealed corpus, untranslated chapters, missing/stale/corrections reviews; - - no mutation on precondition failure. -3. Atomic promotion RED/GREEN: - - all chapters promoted by one progress CAS; - - stale progress conflict cannot partially promote; - - exact PASS identity rechecked immediately before CAS; - - backend-neutral candidate hash identity, including crash after progress CAS before marker phase update. -4. Crash recovery RED/GREEN: - - retry before CAS; - - retry after CAS; - - retry after partial report generation; - - successful rerun is idempotent; - - `status/resume` exposes and resumes active finalization; - - direct `accept_review` is blocked while finalization is active. -5. Reports/CLI RED/GREEN: - - deterministic `STATE.md`, `FINAL_QUALITY_GATES.md`, and regenerated `REVIEW_REPORT.md`; - - JSON success output; - - malformed state fails closed without traceback. -6. Extend #18 reliability coverage for interrupted finalize/idempotence in this same PR or a directly adjacent #18 slice before Phase 2 exit. - -Every production change requires full Python 3.10/3.12 CI. Final PR audit requires `behind_by=0`, no unresolved review threads, integration target unchanged, feature branch preserved, and `main` unchanged. - -## Acceptance mapping - -- `book.py finalize `: CLI adapter. -- Book-level lock and claim rejection: finalization marker + serialized coordination admission. -- No active claims / corpus / translation / PASS/hash verification: initial and immediate pre-CAS revalidation. -- Validate before and after promotion: explicit pre/post phases. -- Fresh-session recovery: active marker is visible in status and `resume` selects `finalize`. -- Competing lifecycle promotion: `accept_review` rejects while finalization is active. -- Generated `STATE.md`: deterministic completion snapshot projection. -- Generated final quality report: deterministic completion snapshot projection. -- No partial lifecycle promotion: one CAS of complete `progress.json` candidate. -- Idempotence: base revision + candidate content hash + promoted backend revision recovery, plus byte-identical report writes. -- Failure-path coverage: dedicated TDD and #18 reliability extension. diff --git a/docs/WORKFLOW_V2_FINALIZE_PLAN.md b/docs/WORKFLOW_V2_FINALIZE_PLAN.md deleted file mode 100644 index 5000c71..0000000 --- a/docs/WORKFLOW_V2_FINALIZE_PLAN.md +++ /dev/null @@ -1,402 +0,0 @@ -# Workflow v2 Atomic Finalize Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add an idempotent, crash-recoverable `book.py finalize ` that serializes finalization against claim admission, promotes all chapters with one progress CAS, and generates deterministic completion reports. - -**Architecture:** A short-lived coordination mutex serializes claim admission versus installation of a transient finalization marker. Finalization stores backend-neutral candidate content identity, performs one all-reviewed `progress.json` CAS, recovers across crash windows, and projects authoritative state into `REVIEW_REPORT.md`, `STATE.md`, and `FINAL_QUALITY_GATES.md`. - -**Tech Stack:** Python 3.10+, stdlib only, existing `WorkflowStateRepository`, `StorageBackend`, `ClaimManager`, `ReviewLedgerManager`, `StatusResolver`, `review_report` module, unittest/GitHub Actions matrix. - -**Spec:** `docs/WORKFLOW_V2_FINALIZE_DESIGN.md` - -## Global Constraints - -- Target only `refactor/workflow-engine-v2`; never change `main`. -- Preserve feature branch after merge. -- No schema-version bump for existing metadata/progress; #16 owns migrations. -- No EPUB build (#14), GitHub API backend (#17), or parallel reconciliation (#15). -- Generated Markdown is projection-only and contains no new wall-clock generation timestamp. -- Every production slice follows RED → minimal GREEN → full Python 3.10/3.12 matrix before the next behavioral slice. -- Final PR must be `behind_by=0`, mergeable, with no unresolved comments/reviews/threads. - ---- - -## File map - -- `scripts/workflow_v2/schemas.py` — strict transient coordination/finalization lock schemas. -- `scripts/workflow_v2/repository.py` — public canonical document serialization helper used by writes and candidate hashing. -- `scripts/workflow_v2/coordination.py` — short-lived book admission mutex. -- `scripts/workflow_v2/claims.py` — claim admission through the coordination mutex; reject active finalization. -- `scripts/workflow_v2/finalize.py` — backend-neutral preflight, marker recovery, one-CAS promotion, completion snapshot and Markdown renderers. -- `scripts/workflow_v2/status.py` — expose active finalization and route resume to finalize recovery. -- `scripts/workflow_v2/reviews.py` — reject direct `accept_review` while finalization is active. -- `scripts/workflow_v2/status_cli.py` — expose shared filesystem structural/corpus preflight for finalize CLI reuse. -- `scripts/workflow_v2/finalize_cli.py` — filesystem adapter, canonical report writes, `finalize` command. -- `scripts/book.py` — register finalize command and top-level expected error. -- `tests/test_workflow_v2_coordination.py` — mutex/claim admission races and expiry. -- `tests/test_workflow_v2_finalize.py` — preflight, atomic CAS, recovery and snapshot/rendering. -- `tests/test_workflow_v2_finalize_cli.py` — end-to-end command/report/idempotence/error behavior. -- `tests/test_workflow_v2_status.py`, `tests/test_workflow_v2_reviews.py` — recovery visibility and direct-promotion admission. -- `tests/test_workflow_v2_reliability.py` — #18 interrupted-finalize/idempotence scenarios. - ---- - -### Task 1: Canonical serialization and transient lock schemas - -**Files:** -- Modify: `scripts/workflow_v2/schemas.py` -- Modify: `scripts/workflow_v2/repository.py` -- Modify: `tests/test_workflow_v2_schemas.py` -- Modify: `tests/test_workflow_v2_repository.py` - -**Interfaces:** -- Produces: `SchemaKind.COORDINATION_LOCK`, `SchemaKind.FINALIZATION_LOCK`. -- Produces: `WorkflowStateRepository.serialize(path: str, schema: SchemaKind, data: Mapping[str, object]) -> bytes`. -- Existing `create()` and `write_if_version()` must call the same public serializer. - -- [ ] **Step 1: Add RED schema tests** - -Add tests that accept exactly: - -```python -coordination = { - "schema_version": 1, - "lock_id": "a" * 32, - "operation": "claim_admission", - "session_id": "session-a", - "acquired_at": "2026-09-06T20:00:00Z", - "expires_at": "2026-09-06T20:01:00Z", -} -finalization = { - "schema_version": 1, - "lock_id": "b" * 32, - "book_slug": "demo", - "workflow_revision": "workflow-rev", - "base_progress_revision": "base-rev", - "candidate_progress_sha256": "c" * 64, - "phase": "preparing", - "promoted_progress_revision": None, - "session_id": "session-a", - "started_at": "2026-09-06T20:00:00Z", -} -``` - -Reject invalid operation, invalid timestamp interval, invalid candidate SHA, `phase=preparing` with non-null promoted revision, and `phase=promoted` with null promoted revision. - -- [ ] **Step 2: Add RED repository serialization test** - -Assert `repository.serialize("progress.json", SchemaKind.PROGRESS, data)` returns the exact bytes later persisted by `create()` and that invalid schema data fails before any storage mutation. - -- [ ] **Step 3: Run full tests for RED witness** - -Run: `python -m unittest discover -s tests -v` -Expected: only the new lock-schema/serializer tests fail because the enum members/public serializer do not exist. - -- [ ] **Step 4: Implement minimal schema validators and public serializer** - -Use existing `_validate_hex_id`, `_parse_utc_timestamp`, `_validate_sha256`, and strict phase relationship validation. Rename/replace repository `_serialize` with public `serialize`; route create/write through it. - -- [ ] **Step 5: Run full matrix and commit** - -Expected: all tests GREEN on Python 3.10 and 3.12. -Commit: `feat: add finalize coordination schemas` - ---- - -### Task 2: Serialize claim admission against finalization - -**Files:** -- Create: `scripts/workflow_v2/coordination.py` -- Modify: `scripts/workflow_v2/claims.py` -- Create: `tests/test_workflow_v2_coordination.py` -- Modify: existing claim tests only where constructor injection is required. - -**Interfaces:** - -```python -@dataclass(frozen=True) -class CoordinationLease: - path: str - data: dict[str, Any] - version: str - -class CoordinationError(RuntimeError): ... -class CoordinationConflict(CoordinationError): ... - -class BookCoordinationManager: - def __init__(self, repository, *, now=None, id_factory=None): ... - def acquire(self, *, operation: str, session_id: str, lease_seconds: int = 60) -> CoordinationLease: ... - def release(self, lease: CoordinationLease) -> None: ... - def finalization_active(self) -> bool: ... -``` - -`ClaimManager.__init__` adds optional `coordination: BookCoordinationManager | None = None`; default coordinator shares the claim clock but uses its own UUID factory so existing deterministic claim-ID tests are not renumbered. - -- [ ] **Step 1: Write RED coordination tests** - -Cover live mutex conflict, exact-expiry cleanup, version-safe stale deletion, and `finalization_active()` schema validation. - -- [ ] **Step 2: Write RED claim admission tests** - -Cover: - -```python -# finalization marker already exists -> acquire raises ClaimConflict/coordination-derived claim error -# claim admission holds mutex while creating unit claims -# finalize admission cannot acquire same mutex concurrently -# after mutex release ordinary disjoint claims retain existing behavior -``` - -Use an instrumented storage in one race test to pause immediately after mutex acquisition and prove the competing admission cannot pass. - -- [ ] **Step 3: Run RED** - -Expected: failures only because `coordination.py` and claim integration do not exist. - -- [ ] **Step 4: Implement coordinator and minimal ClaimManager integration** - -Acquire mutex before existing range conflict/create logic, check `.workflow/finalization.json` while mutex is held, release in `finally`. Do not alter release/cleanup claim semantics. - -- [ ] **Step 5: Run full matrix and commit** - -Commit: `feat: serialize claim admission with finalization` - ---- - -### Task 3: Finalization preflight and one-CAS promotion - -**Files:** -- Create: `scripts/workflow_v2/finalize.py` -- Create: `tests/test_workflow_v2_finalize.py` - -**Interfaces:** - -```python -FINALIZATION_PATH = ".workflow/finalization.json" - -class FinalizationError(RuntimeError): ... -class FinalizationBlocked(FinalizationError): ... -class FinalizationConflict(FinalizationError): ... - -@dataclass(frozen=True) -class FinalizeResult: - snapshot: dict[str, Any] - progress_revision: str - promoted: bool - recovered: bool - -PreflightProvider = Callable[[], tuple[Sequence[str], Mapping[str, Any]]] - -class FinalizationManager: - def __init__( - self, - repository: WorkflowStateRepository, - *, - artifact_reader: Callable[[str], bytes], - preflight: PreflightProvider, - coordination: BookCoordinationManager | None = None, - now: Callable[[], datetime] | None = None, - id_factory: Callable[[], str] | None = None, - ): ... - - def finalize(self, *, session_id: str) -> FinalizeResult: ... -``` - -Internal pure helpers: - -```python -def build_reviewed_candidate(progress: Mapping[str, Any]) -> dict[str, Any]: ... -def sha256_bytes(content: bytes) -> str: ... -def build_completion_snapshot(...) -> dict[str, Any]: ... -``` - -- [ ] **Step 1: Write RED precondition tests** - -Each scenario records `progress.json`, report files and marker paths before/after and asserts zero mutation on clean failure: -- active claim; -- corpus `unsealed` or `invalid`; -- structural errors; -- untranslated/empty translation; -- review `missing`, `stale`, or `corrections_required`. - -- [ ] **Step 2: Run RED and implement read-only preflight** - -Use current metadata/progress + `ReviewLedgerManager.resolve_all()` + injected structural/corpus preflight. Do not mutate progress in this step. - -- [ ] **Step 3: Write RED atomic-promotion tests** - -Instrument repository/storage writes and assert a successful two-or-more-chapter finalize performs exactly one conditional write to `progress.json`, with every chapter `reviewed`. Inject a stale progress revision between admission and CAS; assert no partial promotion. - -- [ ] **Step 4: Implement marker admission and one progress CAS** - -While `finalize_admission` mutex is held: create/adopt marker and verify zero claims. Revalidate immediately before progress CAS. Serialize candidate with `repository.serialize`, hash bytes, and write whole progress once. - -- [ ] **Step 5: Write RED crash-recovery tests** - -Cover persisted states representing: -- `preparing` marker + base progress; -- `preparing` marker + candidate progress bytes (crash after CAS before marker update); -- `promoted` marker + exact promoted revision/content; -- incompatible marker/current progress combination. - -- [ ] **Step 6: Implement phase recovery** - -After progress CAS, update marker to `promoted` with actual returned backend revision. For the crash window, compare current raw progress SHA-256 with marker candidate hash before marker phase update. - -- [ ] **Step 7: Run full matrix and commit** - -Commit: `feat: add atomic recoverable finalize core` - ---- - -### Task 4: Recovery visibility and competing lifecycle mutation guard - -**Files:** -- Modify: `scripts/workflow_v2/status.py` -- Modify: `scripts/workflow_v2/reviews.py` -- Modify: `tests/test_workflow_v2_status.py` -- Modify: `tests/test_workflow_v2_reviews.py` - -**Interfaces:** - -Status adds: - -```json -{"finalization": {"active": false}} -``` - -or, when active: - -```json -{ - "finalization": { - "active": true, - "phase": "preparing", - "workflow_revision": "...", - "started_at": "..." - } -} -``` - -Resume returns `operation="finalize"` before ordinary unit selection when status is valid and finalization is active. - -- [ ] **Step 1: RED status/resume tests** - -Assert valid marker is visible and selected as next operation; malformed marker or book/workflow mismatch invalidates status. - -- [ ] **Step 2: RED direct-promotion test** - -Create a valid current PASS + finalization marker and assert `ReviewLedgerManager.accept_review()` refuses to mutate progress. - -- [ ] **Step 3: Implement minimal reads/guards** - -Read marker through `SchemaKind.FINALIZATION_LOCK`; expose bounded fields only. In `accept_review`, check marker before lifecycle CAS and raise `ReviewConflict`/`ReviewEvidenceError` with a stable message. - -- [ ] **Step 4: Full matrix and commit** - -Commit: `feat: expose finalize recovery in workflow status` - ---- - -### Task 5: Completion projections and CLI - -**Files:** -- Modify: `scripts/workflow_v2/finalize.py` -- Modify: `scripts/workflow_v2/status_cli.py` -- Create: `scripts/workflow_v2/finalize_cli.py` -- Modify: `scripts/book.py` -- Create: `tests/test_workflow_v2_finalize_cli.py` - -**Interfaces:** - -Expose shared filesystem preflight from status CLI: - -```python -def default_preflight(root: Path, slug: str) -> tuple[Sequence[str], Mapping[str, Any]]: ... -``` - -Keep existing status/resume behavior by routing both through this function. - -Finalize rendering: - -```python -def render_state_markdown(snapshot: Mapping[str, Any]) -> str: ... -def render_quality_gates_markdown(snapshot: Mapping[str, Any]) -> str: ... -``` - -CLI adapter: - -```python -class FinalizeCliError(RuntimeError): ... -def register_finalize_command(subparsers, root: Path) -> None: ... -``` - -- [ ] **Step 1: RED deterministic rendering tests** - -Assert repeated rendering produces identical bytes, includes book/workflow/revisions/lifecycle/review coverage/corpus mode, and contains no `generated_at`. - -- [ ] **Step 2: Implement completion snapshot/renderers** - -Build the snapshot only from post-promotion authoritative state and #21 review snapshot. `REVIEW_REPORT.md` uses the existing renderer unchanged. - -- [ ] **Step 3: RED CLI tests** - -End-to-end fixture must verify: -- `book.py finalize demo` promotes all chapters and creates all three reports; -- second run is idempotent and report bytes are identical; -- `--json` returns deterministic success payload after performing/confirming finalize; -- malformed ledger/corpus failure exits 1 without traceback or partial progress promotion; -- existing active claim blocks and remains intact; -- `private_external` verified corpus succeeds without source binary. - -- [ ] **Step 4: Implement filesystem adapter and report CAS writes** - -All report content is rendered before its write. Use `FilesystemStorage` create/update CAS; identical bytes are `unchanged`. After fresh postflight and byte verification, delete finalization marker with version guard. - -- [ ] **Step 5: Wire `book.py`** - -Register finalize command and add `FinalizeCliError` to the top-level expected exception tuple. - -- [ ] **Step 6: Full matrix and commit** - -Commit: `feat: add finalize command and completion reports` - ---- - -### Task 6: #18 interrupted-finalize reliability extension - -**Files:** -- Modify: `tests/test_workflow_v2_reliability.py` - -- [ ] **Step 1: Add fresh-process failure-injection tests** - -Cover: -- crash after finalization marker before progress CAS, then fresh `resume` selects finalize and retry completes; -- crash after progress CAS before marker phase update, then retry does not rewrite progress; -- crash after only one generated report, then retry regenerates the canonical set and removes marker; -- successful finalize rerun has identical progress/report bytes and no marker. - -Use actual temporary repository state and subprocess CLI where practical; do not add production fault-injection hooks solely for tests. - -- [ ] **Step 2: RED/GREEN only if a real defect is found** - -If tests expose a production defect, preserve the failing witness commit/run, diagnose root cause, apply the smallest owning-component fix, and run a fresh full matrix. - -- [ ] **Step 3: Commit** - -Commit: `test: cover interrupted workflow finalization` - ---- - -### Task 7: Completion audit and integration - -- [ ] Run final Python 3.10/3.12 matrix on the exact branch head and record test count/log evidence. -- [ ] Review full PR diff against `docs/WORKFLOW_V2_FINALIZE_DESIGN.md`; specifically recheck marker cleanup, report authority boundary, and no sequential lifecycle promotion. -- [ ] Confirm PR comments, submitted reviews and inline threads are empty/resolved. -- [ ] Confirm `feature/workflow-v2-finalize` is `behind_by=0` relative to `refactor/workflow-engine-v2`. -- [ ] Confirm `main` is unchanged. -- [ ] Update PR body with all RED/GREEN run IDs, changed files, recovery semantics and audit evidence. -- [ ] Mark Ready only after final GREEN/audit. -- [ ] Merge with expected-head guard only into `refactor/workflow-engine-v2`. -- [ ] Verify merge commit, issue #12 state, preserved feature branch and unchanged `main`. diff --git a/docs/WORKFLOW_V2_GITHUB_BACKEND_DESIGN.md b/docs/WORKFLOW_V2_GITHUB_BACKEND_DESIGN.md deleted file mode 100644 index 74e3fef..0000000 --- a/docs/WORKFLOW_V2_GITHUB_BACKEND_DESIGN.md +++ /dev/null @@ -1,366 +0,0 @@ -# Workflow v2 GitHub storage backend design - -## Status - -Approved by standing project authorization after self-review. This design covers issue #17 and targets `refactor/workflow-engine-v2` only. - -## Problem - -Workflow v2 domain operations already depend on the backend-neutral `StorageBackend` abstraction, but the only concrete durable backend is the local filesystem. ChatGPT Web can operate on GitHub repositories without a local checkout, so Workflow v2 needs a GitHub-backed storage implementation with the same read/list/create/CAS/delete semantics. - -The backend must not make GitHub Actions part of execution and must not leak GitHub-specific behavior into claims, review, status, finalize, migration, or literary contracts. - -## Goals - -1. Implement `StorageBackend` against GitHub repository APIs. -2. Use the GitHub blob SHA as the opaque storage revision returned by `StoredValue.version`. -3. Preserve filesystem-equivalent path safety, create-if-absent, compare-and-swap update/delete, and deterministic listing semantics. -4. Surface concurrent GitHub mutations as existing storage conflict types rather than hiding them behind retries. -5. Keep the GitHub transport injectable so ChatGPT-hosted GitHub capabilities and ordinary REST clients can share the same storage adapter. -6. Prove representative domain operations behave identically with filesystem and GitHub backends. -7. Document read/write capability requirements and fail-closed behavior when permissions or API guarantees are unavailable. - -## Non-goals - -- No database, queue, daemon, web service, or mandatory GitHub Action. -- No GitHub-specific changes to `docs/TRANSLATION.md` or literary role behavior. -- No automatic backend selection in every existing CLI in this issue. Core domain portability is the requirement; later integration/documentation work may provide environment-specific orchestration entrypoints. -- No automatic retry of mutating GitHub requests. -- No attempt to make several GitHub file commits atomically appear as one repository commit. Workflow v2's existing coordination, journal, rollback, and recovery protocols remain the cross-document safety mechanism. -- No source-book publication or permission broadening. - -## Architectural choice - -### Chosen: storage adapter over an injectable GitHub API client - -Introduce two layers: - -1. `github_api.py` defines GitHub file/tree value objects, an error model, a narrow client protocol, and an optional standard-library REST implementation. -2. `github_storage.py` implements the existing `StorageBackend` protocol using that client. - -Domain code continues to depend only on `StorageBackend` and `WorkflowStateRepository`. - -This is preferred over embedding Contents API calls directly in domain modules because it keeps concurrency and API-specific error classification at the storage boundary. It is preferred over implementing all writes through raw Git object assembly because file-level Contents API CAS already expresses the needed per-path expected-SHA semantics and is simpler to host from ChatGPT Web. - -### Rejected: GitHub calls directly in domain code - -This would duplicate error handling across claims/finalize/review/migrations and violate backend parity. - -### Rejected: raw Git object transaction backend as the primary interface - -Trees/commits can create multi-file commits, but they require a branch-head compare-and-swap protocol in addition to per-file identity and add complexity not required by #17. Workflow v2 already owns durable multi-step transaction/recovery semantics above storage. - -## Components - -### `github_api.py` - -Defines transport-facing types: - -```python -@dataclass(frozen=True) -class GitHubFile: - path: str - blob_sha: str - content: bytes - -@dataclass(frozen=True) -class GitHubTreeEntry: - path: str - type: str - sha: str - mode: str - -@dataclass(frozen=True) -class GitHubTree: - entries: tuple[GitHubTreeEntry, ...] - truncated: bool - -@dataclass(frozen=True) -class GitHubMutation: - blob_sha: str | None - commit_sha: str | None - -class GitHubApiError(RuntimeError): - status: int | None - message: str - -@runtime_checkable -class GitHubApiClient(Protocol): - def get_file(self, repository: str, path: str, ref: str) -> GitHubFile: ... - def get_tree(self, repository: str, ref: str) -> GitHubTree: ... - def create_file(self, repository: str, path: str, content: bytes, branch: str, message: str) -> GitHubMutation: ... - def update_file(self, repository: str, path: str, content: bytes, expected_blob_sha: str, branch: str, message: str) -> GitHubMutation: ... - def delete_file(self, repository: str, path: str, expected_blob_sha: str, branch: str, message: str) -> GitHubMutation: ... -``` - -The protocol is expressed in repository/file operations, not ChatGPT connector method names. A host adapter and a token-backed REST client can both implement it. - -### `GitHubRestClient` - -A concrete standard-library implementation supports environments that can make direct GitHub HTTPS requests. - -Constructor: - -```python -GitHubRestClient( - token: str | None, - *, - base_url: str = "https://api.github.com", - api_version: str = "2026-03-10", - opener=None, -) -``` - -Rules: - -- caller supplies credentials; the client does not read environment variables itself; -- requests use GitHub JSON media type and `X-GitHub-Api-Version: 2026-03-10`; -- URL path segments are safely quoted; -- write bodies encode bytes as Base64; -- file reads resolve the file blob SHA then fetch the Git blob bytes so reads are binary-safe and do not rely on small Contents inline payload limits; -- directories, symlinks, submodules, and other non-file entries are rejected as ordinary files; -- recursive tree responses expose `truncated` and are rejected by `GitHubStorage` when incomplete; -- HTTP/JSON failures become `GitHubApiError` with status and concise context; -- no mutation retry is performed. - -The REST client is tested with injected fake HTTP handling. CI never depends on live GitHub networking or credentials. - -### `GitHubStorage` - -Constructor: - -```python -GitHubStorage( - client: GitHubApiClient, - *, - repository: str, - branch: str, - root_prefix: str = "", - commit_prefix: str = "workflow-v2", -) -``` - -`repository` is `owner/name`. Mutations always target a branch, not a detached commit SHA. `root_prefix` maps a logical backend root to a repository directory such as `books/my-book`. - -## Logical path safety - -GitHubStorage applies the same logical path rules as filesystem storage before any API call: - -- path must be a string; -- empty path is allowed only for `list("")`; -- no absolute path; -- no backslash; -- no empty, `.` or `..` segments. - -`root_prefix` is validated once using equivalent relative-POSIX rules. Repository paths are formed only from validated components; callers cannot escape the configured backend root. - -`list(prefix)` returns logical paths relative to `root_prefix`, not repository-global paths. - -## Revision model - -`StoredValue.version` is the GitHub **blob SHA** for the exact file bytes. - -Rationale: - -- the abstraction promises a per-value revision, not a repository commit; -- Contents update/delete operations accept the current file/blob SHA as their CAS token; -- a branch may advance for unrelated files without invalidating a value revision; -- existing domain code treats revisions as opaque strings. - -Commit SHA remains secondary audit metadata inside transport results and never replaces the storage version. - -## Read semantics - -`read(path)`: - -1. validate/map the logical path; -2. call `client.get_file(repository, repo_path, branch)`; -3. map API 404 for path lookup to `StorageNotFound`; -4. return exact bytes and `blob_sha` as `StoredValue.version`; -5. map authorization, transport, malformed response, or invalid entry errors to `StorageError` with concise capability context. - -A private repository may intentionally return 404 for insufficient permission. For storage compatibility a path lookup 404 maps to `StorageNotFound`; capability documentation warns that callers needing permission diagnostics must establish repository access separately. - -## List semantics - -`list(prefix)`: - -1. validate prefix; -2. obtain the recursive Git tree for `branch`; -3. fail closed with `StorageError` if the response is truncated; -4. select ordinary blob entries beneath `root_prefix`; -5. retain paths matching the requested logical prefix: exact file returns itself, directory prefix returns descendants, missing prefix returns `[]`; -6. strip `root_prefix` and return sorted POSIX paths. - -Recursive Trees API is used instead of Contents directory enumeration because bounded directory responses could make large workspaces silently incomplete. - -## Create semantics - -`create_if_absent(path, content)`: - -1. validate path and require `bytes` content; -2. invoke create-file without an expected SHA; -3. classify a proven existing-path failure as `StorageAlreadyExists`; -4. do not retry a failed mutation; -5. read the path back from the branch; -6. require read-back bytes to equal intended content; -7. return the read-back blob SHA. - -Read-after-write is mandatory even when a transport returns a blob SHA. This keeps behavior compatible with hosted adapters that may return only commit identity and detects a concurrent overwrite after our mutation. - -If read-back contains different bytes, return `StorageVersionConflict`: our create may have committed, but another writer changed the path before confirmation, so reporting success is unsafe. - -## Update semantics - -`write_if_version(path, content, expected_version)`: - -1. validate inputs; -2. pre-read current file; -3. missing path -> `StorageNotFound`; -4. current SHA != expected -> `StorageVersionConflict` without mutation; -5. send one update with the expected blob SHA; -6. on GitHub 409/422, re-read only to classify: missing -> `StorageNotFound`; changed SHA -> `StorageVersionConflict`; unchanged expected SHA -> generic `StorageError`; -7. after apparent success, read back; -8. exact intended bytes -> return current blob SHA; -9. different bytes -> `StorageVersionConflict`. - -There is no blind retry. The domain operation must re-read/replan according to its existing conflict/recovery protocol. - -## Delete semantics - -`delete_if_version(path, expected_version)` mirrors update: - -1. pre-read and compare blob SHA; -2. send one delete with expected SHA; -3. classify 409/422 with a fresh read; -4. after apparent success, verify the path is absent; -5. if a file exists afterward, return `StorageVersionConflict` because another writer won after deletion. - -A missing path before deletion remains `StorageNotFound`. - -## Error classification - -| GitHub/API condition | Storage exception | -| --- | --- | -| Missing path lookup | `StorageNotFound` | -| Create proves path already exists | `StorageAlreadyExists` | -| Current/read-back SHA or bytes prove race | `StorageVersionConflict` | -| Unsafe logical path | `InvalidStoragePath` | -| 401/403, malformed response, truncated tree, network/transport failure | `StorageError` | -| 409/422 without evidence of changed version | `StorageError` | - -Permission denial or validation errors are never called concurrency unless a fresh read proves the expected version changed. - -## Commit semantics - -Each mutating storage operation produces at most one GitHub file commit. Commit messages are deterministic and bounded: - -- `workflow-v2: create ` -- `workflow-v2: update ` -- `workflow-v2: delete ` - -The backend does not use commit history as authoritative workflow state. Blob identity and durable files remain authoritative; Git history is secondary audit evidence. - -## Domain behavior - -No GitHub-specific changes are expected in `claims.py`, `reviews.py`, `status.py`, `finalize.py`, `migrations.py`, or `repository.py`. - -Representative parity tests instantiate those components with `WorkflowStateRepository(GitHubStorage(fake_client, ...))` and exercise coordination/CAS behavior. If a domain module needs GitHub-specific branching, the design has failed and should be corrected at the backend boundary instead. - -## Hosted ChatGPT execution - -The storage adapter itself does not import ChatGPT connector APIs. A host adapter may implement `GitHubApiClient` using the connected GitHub capability and pass it to `GitHubStorage`. - -This keeps Workflow v2 portable across ChatGPT Web connected GitHub execution, an ordinary Python process with direct GitHub REST access, and deterministic in-memory tests. No GitHub Action is needed for domain execution. - -## Permissions and capability failure - -Minimum remote capabilities: - -- repository/Contents read for `read` and `list`; -- repository/Contents write for create/update/delete; -- Git tree/blob read for complete listing and binary-safe reads. - -A read-only connection can still perform status-like reads but mutations fail with `StorageError` and a concise capability message. The backend never requests broader permissions, stores tokens in workflow state, or logs credentials. - -## Tests - -### Reusable backend contract - -Exercise identical assertions against FilesystemStorage and GitHubStorage: - -- create/read exact bytes and revision; -- duplicate create rejected without overwrite; -- current-version update succeeds; -- stale update preserves winner; -- current-version delete succeeds; -- stale delete preserves current file; -- missing read/update/delete semantics; -- nested deterministic listing, exact-file prefix, missing prefix; -- unsafe paths rejected before backend mutation. - -### GitHub-specific storage tests - -- blob SHA is returned as version; -- root-prefix mapping is exact; -- truncated recursive tree fails closed; -- authorization failure maps to `StorageError`; -- deterministic mutation messages; -- 409/422 is re-read and classified from durable state; -- apparent success followed by concurrent overwrite becomes conflict; -- mutations are never automatically retried. - -### REST client tests - -Using fake HTTP responses only: - -- headers/version/auth behavior; -- path quoting; -- file metadata + blob Base64 decode; -- tree parsing/truncation; -- Base64 mutation payloads and branch/expected-SHA fields; -- success parsing; -- 401/403/404/409/422 and malformed JSON error mapping; -- no network dependency. - -### Domain parity - -Run representative existing operations against GitHubStorage with a fake client: - -- claim acquire/conflict/release; -- book coordination lock lifecycle; -- status read-only resolution; -- one CAS-backed state transition/finalize admission path. - -No domain production changes are expected. - -### #18 reliability extension - -Inject races/failures around the GitHub client boundary: - -- stale writer between pre-read and update; -- concurrent overwrite between successful mutation and read-back; -- transient transport error does not cause duplicate commit retry; -- truncated list blocks operation rather than accepting partial state; -- claim conflict is recoverable by fresh read. - -Production changes are made only when a reliability test demonstrates a real defect. - -## Documentation - -Add GitHub backend execution/capability guidance to orchestration/setup documentation only. The literary translation contract stays backend-agnostic. Documentation states GitHub Actions remain optional CI and are not required for read/write workflow operations. - -## Release and audit gates - -Before merge to `refactor/workflow-engine-v2`: - -1. exact feature head passes full Python 3.10/3.12 suite; -2. backend contract passes for filesystem and GitHub implementations; -3. REST client tests are deterministic and network-free; -4. domain parity tests require no GitHub-specific domain branches; -5. #18 failure-injection coverage is green; -6. changed files remain limited to backend/runtime tests and orchestration/setup capability docs; -7. PR has no unresolved reviews/threads; -8. integration branch has not moved or feature is updated safely; -9. `main` remains unchanged; -10. merge is expected-head guarded into integration only and feature branch is preserved. diff --git a/docs/WORKFLOW_V2_GITHUB_BACKEND_PLAN.md b/docs/WORKFLOW_V2_GITHUB_BACKEND_PLAN.md deleted file mode 100644 index 785a3aa..0000000 --- a/docs/WORKFLOW_V2_GITHUB_BACKEND_PLAN.md +++ /dev/null @@ -1,266 +0,0 @@ -# Workflow v2 GitHub Storage Backend Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a GitHub API-backed `StorageBackend` that preserves Workflow v2 filesystem semantics and lets domain operations run against GitHub without a local checkout or mandatory GitHub Actions. - -**Architecture:** `github_api.py` owns transport-facing value types, errors, a narrow client protocol, and a standard-library REST client. `github_storage.py` adapts that client to the existing storage protocol using blob SHA revisions, strict path mapping, read-after-write verification, no mutation retries, and fail-closed error classification. Existing domain modules remain backend-neutral and are exercised by parity tests. - -**Tech Stack:** Python 3.10+, standard library (`urllib`, `json`, `base64`), existing `StorageBackend` / `WorkflowStateRepository`, `unittest`, GitHub REST API version `2026-03-10`. - -**Spec:** `docs/WORKFLOW_V2_GITHUB_BACKEND_DESIGN.md` - -## Global Constraints - -- Develop only on `feature/workflow-v2-github-backend`, based on integration commit `74cdfa9f5911bbff733907579da2a4b930c4090d`. -- PR targets `refactor/workflow-engine-v2`; never merge to `main` without explicit final user authorization. -- GitHub Actions are CI only; runtime behavior must not require an Action. -- No database, queue, daemon, mandatory SDK, or new external service. -- Domain code depends only on `StorageBackend`; no GitHub-specific branches in claims/reviews/status/finalize/migrations/repository. -- Blob SHA is an opaque storage revision; do not assume SHA length or algorithm. -- No blind retry of any mutating GitHub request. -- Recursive tree truncation is a hard failure, never partial success. -- Read-after-write verification is mandatory for create/update/delete. -- Literary contracts remain backend-agnostic. -- Every production slice follows test-only RED -> focused GREEN -> full-suite GREEN. - ---- - -### Task 1: Reusable storage contract and GitHub storage core - -**Files:** -- Create: `tests/storage_contract.py` -- Create: `tests/github_fake.py` -- Create: `tests/test_workflow_v2_storage_contract.py` -- Create: `tests/test_workflow_v2_github_storage.py` -- Create: `scripts/workflow_v2/github_api.py` -- Create: `scripts/workflow_v2/github_storage.py` -- Modify: `scripts/workflow_v2/__init__.py` - -**Interfaces:** - -```python -@dataclass(frozen=True) -class GitHubFile: - path: str - blob_sha: str - content: bytes - -@dataclass(frozen=True) -class GitHubTreeEntry: - path: str - type: str - sha: str - mode: str - -@dataclass(frozen=True) -class GitHubTree: - entries: tuple[GitHubTreeEntry, ...] - truncated: bool - -@dataclass(frozen=True) -class GitHubMutation: - blob_sha: str | None - commit_sha: str | None - -class GitHubApiError(RuntimeError): - def __init__(self, message: str, *, status: int | None = None): ... - -@runtime_checkable -class GitHubApiClient(Protocol): - def get_file(self, repository: str, path: str, ref: str) -> GitHubFile: ... - def get_tree(self, repository: str, ref: str) -> GitHubTree: ... - def create_file(self, repository: str, path: str, content: bytes, branch: str, message: str) -> GitHubMutation: ... - def update_file(self, repository: str, path: str, content: bytes, expected_blob_sha: str, branch: str, message: str) -> GitHubMutation: ... - def delete_file(self, repository: str, path: str, expected_blob_sha: str, branch: str, message: str) -> GitHubMutation: ... - -class GitHubStorage: - def __init__(self, client: GitHubApiClient, *, repository: str, branch: str, root_prefix: str = "", commit_prefix: str = "workflow-v2"): ... -``` - -- [ ] **Step 1 — RED contract helper:** create `tests/storage_contract.py` with `exercise_backend_contract(testcase, factory)`. It must assert exact binary create/read, duplicate create, current/stale update, current/stale delete, missing read/update/delete, nested sorted list, exact-file prefix, missing prefix, and unsafe paths. The stale cases must prove the winner bytes remain unchanged. - -```python -def exercise_backend_contract(testcase, factory): - storage = factory() - version = storage.create_if_absent("nested/state.bin", b"alpha\x00beta") - loaded = storage.read("nested/state.bin") - testcase.assertEqual(loaded.content, b"alpha\x00beta") - testcase.assertEqual(loaded.version, version) -``` - -- [ ] **Step 2 — RED filesystem contract class:** create `tests/test_workflow_v2_storage_contract.py` that runs the new contract against a fresh `FilesystemStorage` root. Do not modify existing filesystem storage tests. - -- [ ] **Step 3 — RED deterministic GitHub fake:** create `tests/github_fake.py` with `FakeGitHubApiClient`. It stores repository paths as exact bytes, derives deterministic opaque blob IDs from bytes, exposes a recursive tree, records mutation calls/messages, and supports one-shot before/after/error hooks used by later tasks. - -- [ ] **Step 4 — RED GitHub contract class:** create `tests/test_workflow_v2_github_storage.py`. Import the planned public GitHub API/storage surface through a `require_api()` helper so pre-implementation failures identify only the missing backend. Run the same `exercise_backend_contract` using `root_prefix="books/sample"`, and assert fake-client paths are repository-relative under that prefix. - -- [ ] **Step 5 — Run focused RED:** `python -m unittest tests.test_workflow_v2_storage_contract tests.test_workflow_v2_github_storage -v`. Expected: filesystem contract green; GitHub cases fail because `GitHubStorage`/GitHub API types are not implemented. - -- [ ] **Step 6 — Implement `github_api.py` transport data/protocol:** add the dataclasses, runtime-checkable protocol, and `GitHubApiError`. Require non-empty error messages; status must be `int` or `None`. Do not implement HTTP yet. - -- [ ] **Step 7 — Implement GitHubStorage construction/path mapping:** validate `repository` as exactly two non-empty `owner/name` components, non-empty `branch`, non-empty `commit_prefix`, and safe relative POSIX `root_prefix`. Implement path validation matching filesystem storage: empty only for `list("")`, no absolute paths, backslashes, empty segments, `.` or `..`. - -- [ ] **Step 8 — Implement read/list:** `read` maps client 404 to `StorageNotFound`, returns exact bytes and blob SHA, and maps all other API errors to `StorageError`. `list` calls recursive tree, rejects `truncated=True`, accepts only ordinary `blob` entries with mode `100644`/`100755`, strips `root_prefix`, applies exact-file-or-descendant prefix semantics, and sorts results. - -- [ ] **Step 9 — Implement create/update/delete happy paths:** update/delete pre-read and compare exact opaque blob SHA before mutation. Each operation performs exactly one client mutation with `workflow-v2: `, then verifies durable state by fresh read. Create/update return read-back blob SHA; delete requires the read-back to be missing. Do not trust mutation-return SHA as final proof. - -- [ ] **Step 10 — Export public types:** update `scripts/workflow_v2/__init__.py` to export `GitHubApiClient`, `GitHubApiError`, `GitHubFile`, `GitHubTree`, `GitHubTreeEntry`, `GitHubMutation`, and `GitHubStorage`. - -- [ ] **Step 11 — Focused GREEN:** `python -m unittest tests.test_workflow_v2_storage_contract tests.test_workflow_v2_storage tests.test_workflow_v2_github_storage -v`. - -- [ ] **Step 12 — Full GREEN:** run `python -m unittest discover -s tests -v` through the Python 3.10/3.12 CI matrix. - -- [ ] **Step 13 — Commit boundary:** preserve test-only RED separately from production. Suggested production commit: `feat: add GitHub storage backend core`. - ---- - -### Task 2: GitHub race and error classification - -**Files:** -- Modify: `tests/test_workflow_v2_github_storage.py` -- Modify: `scripts/workflow_v2/github_storage.py` - -**Interfaces:** consumes Task 1 `GitHubStorage` and `GitHubApiError(status=...)`. - -- [ ] **Step 1 — RED race tests:** use the deterministic fake hooks to assert: - - stale expected SHA is rejected before any mutation call; - - create 409/422 followed by an existing path => `StorageAlreadyExists`; - - update 409/422 followed by changed SHA => `StorageVersionConflict`; - - update 409/422 while expected SHA remains current => `StorageError`; - - delete 409/422 followed by changed SHA => `StorageVersionConflict`; - - successful create/update followed by immediate concurrent overwrite => `StorageVersionConflict`; - - successful delete followed by recreation => `StorageVersionConflict`; - - 401/403 => concise `StorageError` describing unavailable GitHub contents capability; - - mutation call count remains one; no automatic retry occurs. - -- [ ] **Step 2 — Run focused RED:** `python -m unittest tests.test_workflow_v2_github_storage -v`. Expected: only the newly specified race/classification behavior fails. - -- [ ] **Step 3 — Implement mutation-failure classification:** for client 409/422 perform one fresh read. Create classifies a now-existing path as `StorageAlreadyExists`. Update/delete classify missing/changed versions from durable state; if expected state is still current, raise generic `StorageError` rather than fabricating a race. Preserve the client exception as cause. - -- [ ] **Step 4 — Implement post-mutation race verification:** create/update compare exact intended bytes; delete verifies absence. Any different winner state becomes `StorageVersionConflict`. - -- [ ] **Step 5 — GREEN focused + full Python 3.10/3.12 matrix.** - -- [ ] **Step 6 — Commit:** `feat: classify GitHub storage races safely`. - ---- - -### Task 3: Standard-library GitHub REST client - -**Files:** -- Create: `tests/test_workflow_v2_github_api.py` -- Modify: `scripts/workflow_v2/github_api.py` -- Modify: `scripts/workflow_v2/__init__.py` - -**Interfaces:** - -```python -class GitHubRestClient: - def __init__(self, token: str | None, *, base_url: str = "https://api.github.com", api_version: str = "2026-03-10", opener=None): ... -``` - -- [ ] **Step 1 — RED HTTP harness:** create a fake opener/response in `tests/test_workflow_v2_github_api.py` that records every `urllib.request.Request` method, URL, headers, and body and returns queued response bytes without network access. - -- [ ] **Step 2 — RED read tests:** require `GET /repos/{owner}/{repo}/contents/{quoted-path}?ref={quoted-ref}`, validate `type == "file"` and non-empty `sha`, then require `GET /repos/{owner}/{repo}/git/blobs/{sha}`. Blob response must declare `encoding == "base64"`; strict Base64 decoding returns `GitHubFile` with exact bytes and the contents SHA. Non-file/malformed/invalid-base64 responses become `GitHubApiError`. - -- [ ] **Step 3 — RED tree tests:** require `GET /repos/{owner}/{repo}/git/trees/{quoted-ref}?recursive=1`, exact entry parsing, and preservation of the response `truncated` boolean. - -- [ ] **Step 4 — RED mutation tests:** create/update bodies contain Base64 `content`, `branch`, and `message`; update additionally contains `sha`. Delete contains `branch`, `message`, and `sha` but no content. Parse optional content SHA and commit SHA into `GitHubMutation`; delete accepts the documented success response without requiring content metadata. - -- [ ] **Step 5 — RED header/error tests:** every request has `Accept: application/vnd.github+json` and `X-GitHub-Api-Version: 2026-03-10`; `Authorization: Bearer ` exists only with a supplied token. HTTP 401/403/404/409/422, URL/transport failures, invalid UTF-8, and malformed JSON become `GitHubApiError`; token text never appears in messages. - -- [ ] **Step 6 — Run focused RED:** `python -m unittest tests.test_workflow_v2_github_api -v`. Expected: missing `GitHubRestClient`/HTTP methods only. - -- [ ] **Step 7 — Implement `_request_json`:** use the injected opener or `urllib.request.build_opener()`, deterministic JSON request bodies, UTF-8 JSON response decoding, `urllib.error.HTTPError`/`URLError` plus timeout/OSError handling, no retries, concise sanitized `GitHubApiError`. - -- [ ] **Step 8 — Implement client methods:** quote repository/path/ref pieces safely with `urllib.parse.quote`; validate JSON shapes; strict Base64 decode; construct Task 1 values. Do not read tokens from environment variables. - -- [ ] **Step 9 — Export `GitHubRestClient`; GREEN focused + full matrix.** - -- [ ] **Step 10 — Commit:** `feat: add GitHub REST storage transport`. - ---- - -### Task 4: Domain parity on GitHubStorage - -**Files:** -- Create: `tests/test_workflow_v2_github_backend_domain.py` -- Reuse: `tests/github_fake.py` -- Production: none unless a test proves a backend-boundary defect. - -- [ ] **Step 1 — Build canonical remote workspace:** use `WorkflowStateRepository(GitHubStorage(...))` to create schema-valid metadata/progress/ledger/source-manifest state under `books/sample`; do not seed JSON by bypassing repository validation. - -- [ ] **Step 2 — Claim parity:** acquire a translator claim, prove a second session conflicts, release by owner, and verify claim/audit durable state through GitHubStorage. - -- [ ] **Step 3 — Coordination parity:** acquire a coordination lease, prove concurrent live lease conflict, release, and reacquire. - -- [ ] **Step 4 — Status parity:** resolve status/resume for an extracted unit and assert the fake client recorded zero mutation calls during status resolution. - -- [ ] **Step 5 — CAS parity:** perform one repository/domain write, retain its old revision, commit a competing winner, then prove stale write fails and winner bytes remain authoritative. - -- [ ] **Step 6 — Run:** `python -m unittest tests.test_workflow_v2_github_backend_domain -v`. Immediate GREEN is valid test-only evidence. A genuine failure triggers `systematic-debugging`; fix only the backend boundary unless evidence proves an existing domain abstraction bug. - -- [ ] **Step 7 — Full matrix; commit parity tests separately from any production fix.** - ---- - -### Task 5: Capability and orchestration documentation - -**Files:** -- Modify: `tests/test_agent_contract.py` -- Modify: `docs/ORCHESTRATION.md` -- Modify: `docs/AGENT_SETUP.md` - -- [ ] **Step 1 — RED contract tests:** assert orchestration/setup docs explicitly state GitHub API storage is an allowed orchestrator execution substrate, core durable read/write execution does not require GitHub Actions, read-only access requires repository contents/tree/blob reads, mutations require contents write, CAS conflict requires re-read/replan instead of blind retry, credentials are never persisted in book state, and `docs/TRANSLATION.md` contains no GitHub client/API execution instructions. - -- [ ] **Step 2 — Run focused RED:** `python -m unittest tests.test_agent_contract -v`. - -- [ ] **Step 3 — Update orchestration docs:** add a compact GitHub-backed execution section near execution modes/single-writer rules. Keep transport mechanics out of Translator/Reviewer instructions. - -- [ ] **Step 4 — Update setup docs:** document repository/branch/root scoping, read vs write capabilities, explicit credential injection/non-persistence, fail-closed permission behavior, and that Actions remain optional CI only. - -- [ ] **Step 5 — GREEN agent contract + full matrix.** - -- [ ] **Step 6 — Commit:** `docs: document GitHub-backed Workflow v2 execution`. - ---- - -### Task 6: #18 GitHub backend reliability - -**Files:** -- Create: `tests/test_workflow_v2_github_backend_reliability.py` -- Reuse: `tests/github_fake.py` -- Production: only when failure injection demonstrates a real defect. - -- [ ] **Step 1 — Failure injection:** cover six independent cases: - 1. competing writer changes target after pre-read but before update -> conflict, winner preserved; - 2. competing writer overwrites immediately after successful update -> read-back conflict, winner preserved; - 3. transport failure during mutation -> exactly one mutation attempt, no hidden retry; - 4. truncated recursive tree -> list and domain discovery fail closed rather than accepting partial state; - 5. two claim create attempts race -> one owner wins, loser gets recoverable conflict behavior; - 6. a fresh read after conflict observes the winner and permits normal caller recovery. - -- [ ] **Step 2 — Run focused:** `python -m unittest tests.test_workflow_v2_github_backend_reliability -v`. Immediate GREEN is test-only evidence. For real RED, preserve the failing run before a minimum fix. - -- [ ] **Step 3 — Full matrix GREEN.** - -- [ ] **Step 4 — Commit reliability tests separately from any production fix.** - ---- - -### Task 7: Final verification and integration audit - -**Files:** no intended production changes. - -- [ ] **Step 1 — Exact-head full suite:** `python -m unittest discover -s tests -v` succeeds on Python 3.10 and 3.12 for the exact final feature head; record run id and exact test count. - -- [ ] **Step 2 — Acceptance audit:** map evidence to core coordination through GitHub without Actions, recoverable CAS conflicts, common filesystem/GitHub backend contract, concise capability failures, and absence of GitHub-specific literary/domain branching. - -- [ ] **Step 3 — Scope audit:** changed files are limited to design/plan, `github_api.py`, `github_storage.py`, package exports, fake/contract/backend/parity/reliability tests, `docs/ORCHESTRATION.md`, `docs/AGENT_SETUP.md`, and agent contract tests. Any domain production change requires explicit test-proven justification in the PR body. - -- [ ] **Step 4 — PR audit:** base `refactor/workflow-engine-v2`; behind by 0 or safely updated; merge base matches expected integration ancestry; no unresolved comments/reviews/threads; `main` unchanged. - -- [ ] **Step 5 — PR evidence:** include RED/GREEN run ids, exact final test count, backend contract/parity/reliability mapping, base/head SHAs, and explicit statement that GitHub Actions are not a runtime dependency. - -- [ ] **Step 6 — Ready + merge:** mark Ready only after all guards pass. Merge with `expected_head_sha` into `refactor/workflow-engine-v2` only. Preserve `feature/workflow-v2-github-backend`. Never merge `main`. diff --git a/docs/WORKFLOW_V2_MIGRATIONS_DESIGN.md b/docs/WORKFLOW_V2_MIGRATIONS_DESIGN.md deleted file mode 100644 index c2a31b4..0000000 --- a/docs/WORKFLOW_V2_MIGRATIONS_DESIGN.md +++ /dev/null @@ -1,272 +0,0 @@ -# Workflow v2 — migrations and compatibility design (#16) - -## Goal - -Allow an existing book workspace to adopt the currently installed Workflow v2 revision only through an explicit, recoverable command. Preserve pinned semantics before that command, never fabricate provenance/review/source identity, and never leave an interrupted upgrade in an unrecoverable mixed state. - -## Scope - -- Add `book.py workflow-upgrade --to `. -- Add explicit versioned migration planning for metadata, progress, review ledger, claims and source manifest JSON. -- Treat missing `schema_version` as logical schema v0 only inside the explicit upgrade path. -- Keep normal legacy reads/status/validate non-mutating. -- Validate legacy state, build all target state in memory, validate it, then apply through backend-neutral CAS. -- Add durable rollback/recovery for multi-document migration. -- Update metadata workflow provenance last. -- Make unfinished migration visible to fresh status/resume and block competing workflow transitions. -- Extend #18 with migration failure-injection/idempotence coverage. - -Out of scope: #17 GitHub backend, #15 parallel proposals, unknown future schema migrations, downloading/executing migration code from another revision, and rewriting generated reports/EPUB artifacts. - -## Existing compatibility boundary - -Current durable workflow schema version is v1. Outside `workflow-upgrade`, missing `schema_version` is accepted only for metadata/progress through existing `allow_legacy=True`, normalized in memory, and never written automatically. Explicit unsupported versions fail. #16 preserves that boundary. - -## Chosen architecture - -Use a pure migration registry + compatibility planner + transaction executor + transient durable migration journal. - -Rejected alternatives: - -1. One procedural v0→v1 CLI function: too coupled and weak for recovery/future migration steps. -2. Persist existing `allow_legacy=True` normalization: violates the no-silent-upgrade requirement. - -### Components - -- `workflow_v2/migrations.py`: pure version migration, raw discovery, compatibility planning, transaction executor. -- `workflow_v2/migration_journal.py`: isolated strict validation/loading/serialization for `.workflow/migration.json`. It depends only on storage-safe primitives and does not import review/finalize/status modules. -- `workflow_v2/migrations_cli.py`: installed-provenance resolution and CLI registration. -- `source_integrity.py`: private-external manifest reconstruction from already-recorded source identity + exact extracted bytes. -- `coordination.py`: allow `workflow_upgrade` operation and expose active migration marker. -- claims/finalize/review/status: block or route around active migration journal. - -The migration journal is transient orchestration state, not one of the normal versioned book-document schemas. It is therefore validated by `migration_journal.py`, not added to `SchemaKind`; this avoids coupling the central schema registry to recovery implementation details while still validating the journal strictly on every read/write. - -## Version model - -### Logical v0 - -For metadata, progress, review ledger, claim and source manifest, v0 means: - -- JSON object; -- no `schema_version` field; -- every other field needed by the v1 validator is already present and valid after adding `schema_version: 1`. - -Migration may not invent missing claim IDs, hashes, workflow revisions, review provenance or source identity. - -### v0 → v1 - -`migrate_document(kind, data)`: - -1. deep-copy mapping; -2. add `schema_version: 1`; -3. run the ordinary strict v1 validator; -4. return canonical target mapping. - -Incomplete or incompatible legacy data fails with a path/kind-specific compatibility error before durable mutation. - -Explicit v1 is strictly validated and not rewritten unless cross-document repair/provenance actually changes it. Explicit versions other than 1 are unsupported. - -## Runtime target and pinned provenance - -`workflow-upgrade --to ` is only allowed when `` exactly matches the installed `.book-translator-install.json` `resolved_revision`. - -Installed provenance must contain canonical repository + resolved revision. Existing metadata workflow repository, when present, must refer to the canonical repository. User-supplied text alone is never treated as proof that migration code for another revision is installed. - -An old book stays pinned until the explicit upgrade transaction succeeds. - -## Upgrade history and no-op - -Changed upgrade writes metadata last with: - -- installed repository; -- installed requested ref; -- installed resolved revision; -- `review_evidence: "review-ledger-v1"`; -- append-only deterministic `upgrade_history` entry containing `from_revision`, `to_revision` and migrated schema-family from/to versions. - -No wall-clock field is required. - -A true no-op requires all of: - -- no migration journal; -- targeted durable state already strict v1 and cross-document valid; -- no source/review repair required; -- metadata already pinned to requested installed revision with v1 review marker. - -A no-op rewrites nothing and appends no history. If schemas still need migration even when from/to revision strings match, it remains a changed schema upgrade. - -## Raw discovery - -Planner reads exact bytes/revisions directly through `StorageBackend` because legacy ledger/claim/manifest may not pass current repository parsing. - -Required/optional paths: - -- `metadata.json` required; -- `progress.json` required; -- `review-ledger.json` optional; -- `source-manifest.json` optional; -- `.workflow/claims/*.json` zero or more. - -UTF-8/JSON errors fail before writes. Original exact bytes and storage revisions are retained in the plan. - -## Review compatibility - -Machine review evidence is never fabricated. - -- Existing ledger must be v0-compatible or strict v1. -- Missing ledger produces an empty v1 candidate ledger for the same book slug. -- Every `reviewed` unit is re-resolved against exact current source/translation bytes and candidate ledger. -- Current PASS preserves `reviewed`. -- Missing/stale/unprovable PASS downgrades to `translated` if a non-empty translation exists. -- Missing/empty translation for a supposedly reviewed unit is a compatibility error. - -Legacy human review therefore preserves translation content but must pass the normal Reviewer flow before machine-reviewed lifecycle can be claimed again. - -## Source compatibility - -Source identity is never fabricated. - -Existing source manifest must migrate/validate and agree with metadata + actual source/extracted bytes. - -If absent: - -- embedded source: build v1 manifest from actual declared source file + all referenced extracted files; -- explicit `private_external`: original binary may remain absent only when metadata already contains complete source identity (`filename`, `size_bytes`, `sha256`, storage mode). Manifest source identity is copied from metadata; extracted hashes are computed from exact files; -- missing source identity or extracted artifact fails before writes. - -Candidate corpus is checked with the same source-integrity policy used by normal status/finalize. - -## Claims and admission - -Before mutation: - -- active finalization blocks upgrade; -- live claims block upgrade; -- expired claims may be schema-migrated but are not revived/extended; -- malformed legacy claims fail compatibility. - -Claim expiry uses injected UTC clock for deterministic tests. - -Upgrade acquires the existing coordination mutex with `operation="workflow_upgrade"`; #16 extends the coordination validator/manager allowed-operation set. After acquiring it, planner/executor rechecks captured revisions/admission state before writes. - -## Active migration visibility - -`.workflow/migration.json` remains authoritative after the short coordination lease expires. - -While it exists: - -- new claim acquisition rejects; -- finalize rejects; -- `accept_review` rejects; -- status exposes bounded migration recovery state; -- resume prioritizes `operation="workflow_upgrade"` rather than ordinary translate/review/finalize dispatch. - -Malformed journal causes fail-closed status/recovery. Read-only status/resume never mutate it. - -## Migration journal contract - -Path: `.workflow/migration.json`. - -`migration_journal.py` validates exactly: - -- `schema_version: 1`; -- `operation: "workflow_upgrade"`; -- non-empty book slug; -- `from_revision` null/non-empty string; -- non-empty `to_revision`; -- `phase: "prepared" | "applied"`; -- ordered `documents` array. - -Each entry: - -- safe relative path; -- known schema-family string; -- `original_exists` boolean; -- original storage revision/hash/base64 exact bytes when originally present, otherwise all three null; -- target SHA-256; -- resulting revision null while not known, non-empty string after write. - -For `phase="applied"`, every entry has a resulting revision. Base64 must decode strictly and hash to `original_sha256`. - -Journal stores enough data to restore exact original JSON bytes without reparsing/reserializing them. - -## Transaction order - -All target data is built/validated before journal creation. - -Apply order: - -1. source manifest; -2. review ledger; -3. claims sorted by path; -4. progress; -5. metadata last. - -Existing documents use captured CAS revisions; new documents use create-if-absent. After each target write, journal is CAS-updated with resulting revision. Crash after target write but before journal update remains recoverable via byte hashes. - -Metadata is the externally visible workflow pin and therefore writes last. - -## Crash/conflict recovery - -On command start, existing journal is recovered before a new plan. - -Recovery first acquires `workflow_upgrade` coordination. If a crashed process still holds an unexpired coordination lease, return deterministic conflict; normal expiry allows later recovery. - -For every journaled path classify current state: - -- `target`: current bytes hash to target; -- `original`: current bytes hash to original, or path remains absent when originally absent; -- `unknown`: anything else. - -Rules: - -- all target: strict full-state validation, delete journal with CAS, return `recovered`; -- known mixture of target/original: restore target-written paths to exact originals (delete paths originally absent), metadata first, remaining paths in reverse apply order; verify originals; delete journal; re-plan and execute; -- unknown: mutate nothing, preserve journal, fail closed. - -Normal CAS/apply failure uses the same rollback classifier. Rollback itself is crash-safe because the journal survives and every path remains classifiable as target/original. - -## Final validation - -Before metadata write and again after it: - -- metadata/progress structural validity; -- review ledger book slug/current reviewed-PASS safety; -- source manifest/corpus integrity; -- claims map to canonical existing units and preserve original lease/provenance. - -Journal is deleted only after post-write strict validation. - -## CLI - -`book.py workflow-upgrade --to [--json]` - -Result contains slug, from/to revision, `changed|unchanged|recovered`, migrated paths/families and reviewed→translated downgrade chapter numbers. Expected compatibility/conflict/recovery errors are concise and traceback-free. - -## TDD slices - -1. Pure migration registry + domain-local journal validation. -2. Compatibility planner: target provenance, review downgrade, source reconstruction, claim/finalization gates. -3. Coordination + journaled transaction/rollback/recovery. -4. Active migration visibility/admission guards. -5. CLI end-to-end + representative legacy fixtures/no-op. -6. #18 migration crash/CAS/unknown-mutation/idempotence reliability. -7. Full Python 3.10/3.12 CI + diff/review/ancestry audit. - -## Acceptance criteria - -- No silent upgrade from ordinary reads/status/validate. -- Old pinned workflow remains pinned until explicit successful command. -- Target must equal installed resolved revision. -- v0-compatible metadata/progress/ledger/claims/source manifest migrate deterministically to v1. -- Unknown/malformed legacy shapes fail precisely before mutation. -- Reviewed state without machine PASS becomes translated, never fabricated PASS. -- Source manifest reconstruction uses only provable source identity/exact corpus bytes. -- Live claims/finalization block upgrade; expired claims are not revived. -- Active migration blocks competing workflow mutations and is visible to fresh resume. -- Successful upgrade records old/new revision and true no-op does not duplicate history. -- Interrupted/failed migration restores exact prior known state; unknown concurrent bytes are never overwritten. -- Recovery is deterministic from a fresh process using repository state only. -- Standard Python suite covers migration behavior; GitHub Actions is optional CI, not runtime. -- Merge only to `refactor/workflow-engine-v2`; `main` unchanged. diff --git a/docs/WORKFLOW_V2_MIGRATIONS_PLAN.md b/docs/WORKFLOW_V2_MIGRATIONS_PLAN.md deleted file mode 100644 index b383e64..0000000 --- a/docs/WORKFLOW_V2_MIGRATIONS_PLAN.md +++ /dev/null @@ -1,219 +0,0 @@ -# Workflow v2 Migrations Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add explicit, crash-recoverable Workflow v2 upgrades from provable legacy v0/v1 state to the currently installed workflow revision without silent provenance or review/source fabrication. - -**Architecture:** `MigrationPlanner` builds a validated immutable plan; `MigrationExecutor` owns coordination, journaled CAS application, rollback and recovery. `migration_journal.py` strictly validates transient recovery state without extending the central book-document `SchemaKind` registry. CLI and existing workflow operations only adapt around these APIs. - -**Tech Stack:** Python 3.10+, stdlib (`json`, `base64`, `hashlib`, `datetime`, `copy`), existing storage/repository/review/source/coordination primitives, `unittest`. - -**Spec:** `docs/WORKFLOW_V2_MIGRATIONS_DESIGN.md` - -## Global Constraints - -- No silent durable migration outside explicit `workflow-upgrade`. -- Missing `schema_version` is logical v0 only in migration code. -- `--to` equals installed `resolved_revision` exactly. -- Never invent claim/review/source provenance. -- Metadata writes last. -- `.workflow/migration.json` is authoritative while present. -- Unknown concurrent bytes are never overwritten. -- No third-party runtime dependency. -- Merge only to `refactor/workflow-engine-v2`; `main` unchanged. - ---- - -### Task 1: Pure registry + journal validator - -**Files:** -- Create: `scripts/workflow_v2/migrations.py` -- Create: `scripts/workflow_v2/migration_journal.py` -- Modify: `scripts/workflow_v2/__init__.py` -- Test: `tests/test_workflow_v2_migrations.py` - -**Interfaces:** - -```python -class MigrationError(RuntimeError): ... -class MigrationCompatibilityError(MigrationError): ... -class MigrationConflict(MigrationError): ... - -@dataclass(frozen=True) -class MigratedDocument: - kind: SchemaKind - from_version: int - to_version: int - data: dict[str, Any] - changed: bool - -def detect_schema_version(data: Mapping[str, Any]) -> int: ... -def migrate_document(kind: SchemaKind, data: Mapping[str, Any]) -> MigratedDocument: ... -``` - -Journal API: - -```python -MIGRATION_PATH = ".workflow/migration.json" -class MigrationJournalError(RuntimeError): ... -def validate_migration_journal(data: Mapping[str, Any]) -> dict[str, Any]: ... -def serialize_migration_journal(data: Mapping[str, Any]) -> bytes: ... -def load_migration_journal(storage: StorageBackend) -> tuple[dict[str, Any], str]: ... -``` - -- [ ] **Step 1 — RED:** v0 supported documents add only schema_version and validate as v1; v1 unchanged; version 2/incomplete v0 fail precisely. Journal validator accepts valid prepared/applied forms and rejects phase/operation/path/hash/base64/original-null inconsistencies. -- [ ] **Step 2 — Run:** `python -m unittest tests.test_workflow_v2_migrations -v`; expected assertion failures for missing migration/journal APIs only. -- [ ] **Step 3 — Implement:** registry deep-copies and delegates to `parse_document`; journal validator is domain-local, strict, canonical JSON serializer, no review/finalize/status imports. -- [ ] **Step 4 — GREEN:** `python -m unittest tests.test_workflow_v2_migrations tests.test_workflow_v2_schemas -v`. -- [ ] **Step 5 — Commit:** `feat: add workflow migration registry and journal validation`. - ---- - -### Task 2: Compatibility planner - -**Files:** -- Modify: `scripts/workflow_v2/migrations.py` -- Modify: `scripts/workflow_v2/source_integrity.py` -- Test: `tests/test_workflow_v2_migration_planner.py` - -**Interfaces:** - -```python -@dataclass(frozen=True) -class PlannedWrite: - path: str - kind: SchemaKind - original_exists: bool - original_version: str | None - original_bytes: bytes | None - target_data: dict[str, Any] - target_bytes: bytes - from_version: int | None - to_version: int - -@dataclass(frozen=True) -class MigrationPlan: - book_slug: str - from_revision: str | None - to_revision: str - writes: tuple[PlannedWrite, ...] - lifecycle_downgrades: tuple[int, ...] - changed: bool - def write_for(self, path: str) -> PlannedWrite | None: ... - -class MigrationPlanner: - def __init__(self, repository, *, book_dir, artifact_reader, now): ... - def plan(self, *, slug: str, to_revision: str, installed: Mapping[str, Any]) -> MigrationPlan: ... -``` - -Source helper: - -```python -def build_private_source_manifest_from_identity(book_dir, metadata, progress) -> dict[str, Any]: ... -``` - -- [ ] **Step 1 — RED:** target mismatch, future/malformed schema, missing ledger, review downgrade/current PASS, missing translation, embedded/private manifest reconstruction, unprovable source, live/expired claim, finalization block, true no-op. -- [ ] **Step 2 — Run:** `python -m unittest tests.test_workflow_v2_migration_planner -v`. -- [ ] **Step 3 — Implement raw discovery:** strict UTF-8/JSON through storage, preserve exact bytes/revisions, discover sorted claims. -- [ ] **Step 4 — Implement candidate planning:** installed-target check; schema migrations; candidate ledger/manifest; reviewed lifecycle reconciliation; deterministic metadata workflow/history; strict target validation; canonical target bytes via repository serializer. No writes. -- [ ] **Step 5 — GREEN:** planner + private-source/corpus focused suites. -- [ ] **Step 6 — Commit:** `feat: plan explicit workflow upgrades`. - ---- - -### Task 3: Coordination + transaction executor - -**Files:** -- Modify: `scripts/workflow_v2/migrations.py` -- Modify: `scripts/workflow_v2/coordination.py` -- Modify: `scripts/workflow_v2/schemas.py` only to allow coordination operation `workflow_upgrade` -- Test: `tests/test_workflow_v2_migration_transaction.py` -- Test: `tests/test_workflow_v2_coordination.py` - -**Interfaces:** - -```python -@dataclass(frozen=True) -class MigrationResult: - book_slug: str - from_revision: str | None - to_revision: str - outcome: str # changed | unchanged | recovered - migrated_paths: tuple[str, ...] - lifecycle_downgrades: tuple[int, ...] - -class MigrationExecutor: - def __init__(self, repository, planner: MigrationPlanner, *, coordination): ... - def execute(self, plan: MigrationPlan, *, session_id: str) -> MigrationResult: ... - def recover(self, *, session_id: str, installed: Mapping[str, Any]) -> MigrationResult | None: ... -``` - -- [ ] **Step 1 — RED:** journal before target writes; deterministic order manifest→ledger→claims→progress→metadata; metadata last; captured CAS; journal revision updates; success deletes journal; stale write exact rollback; originally absent target deleted; coordination operation accepted. -- [ ] **Step 2 — Run focused RED.** -- [ ] **Step 3 — Implement:** raw target writes from canonical plan bytes; journal stores exact original base64/hash; on failure classify and rollback exact originals. -- [ ] **Step 4 — Recovery:** every path `original|target|unknown`; unknown fail closed/no mutation; known mixture rollback/delete/replan/execute; all target strict validate/delete journal/`recovered`. -- [ ] **Step 5 — GREEN focused transaction/coordination tests.** -- [ ] **Step 6 — Commit:** `feat: add crash-recoverable workflow upgrade transaction`. - ---- - -### Task 4: Active migration visibility/admission - -**Files:** -- Modify: `scripts/workflow_v2/coordination.py` -- Modify: `scripts/workflow_v2/claims.py` -- Modify: `scripts/workflow_v2/finalize.py` -- Modify: `scripts/workflow_v2/reviews.py` -- Modify: `scripts/workflow_v2/status.py` -- Test: `tests/test_workflow_v2_migration_visibility.py` - -**Interfaces:** `BookCoordinationManager.migration_active() -> bool` uses `load_migration_journal(storage)`. Status adds bounded `migration` section; resume prioritizes `operation="workflow_upgrade"`. - -- [ ] **Step 1 — RED:** active journal blocks `ClaimManager.acquire` (`ClaimError`), finalizer (`FinalizationError`), `accept_review` (`ReviewEvidenceError`); status exposes migration; resume selects workflow-upgrade; malformed journal invalidates status. -- [ ] **Step 2 — Run RED.** -- [ ] **Step 3 — Implement minimal guards/routing; no migration execution logic in these modules.** -- [ ] **Step 4 — GREEN visibility + existing claim/finalize/review/status regressions.** -- [ ] **Step 5 — Commit:** `feat: gate workflow operations during migration recovery`. - ---- - -### Task 5: CLI `workflow-upgrade` - -**Files:** -- Create: `scripts/workflow_v2/migrations_cli.py` -- Modify: `scripts/workflow_v2/review_cli.py` only to extend root registration chain -- Test: `tests/test_workflow_v2_migrations_cli.py` - -**Interfaces:** - -```python -def load_install_provenance(root: Path) -> dict[str, str | None]: ... -def register_migration_command(subparsers, root: Path, *, error_factory) -> None: ... -``` - -- [ ] **Step 1 — RED:** representative v0 upgrade records revisions; ordinary status/validate does not rewrite; target mismatch no-write; malformed fixture concise no-write; reviewed downgrade; private source no binary; second upgrade byte-idempotent unchanged; deterministic JSON. -- [ ] **Step 2 — Run:** parser should fail because command absent. -- [ ] **Step 3 — Implement:** build filesystem repository/artifact reader/planner/coordination/executor; `recover()` first, otherwise `plan()` then `execute()`; adapt errors into existing `ReviewCliError` factory; lazy register at end of `register_review_commands()` without modifying `book.py`. -- [ ] **Step 4 — GREEN CLI + book/finalize/EPUB regressions.** -- [ ] **Step 5 — Commit:** `feat: add explicit workflow upgrade command`. - ---- - -### Task 6: #18 migration reliability - -**Files:** -- Create: `tests/test_workflow_v2_migration_reliability.py` -- Production only if a reliability test demonstrates a real defect. - -- [ ] **Step 1:** durable boundaries: prepared journal/no target; crash after manifest/ledger; crash after metadata before journal deletion; CAS conflict rollback; unknown mutation preserves unknown bytes+journal; completed rerun byte-idempotent. -- [ ] **Step 2:** `python -m unittest tests.test_workflow_v2_migration_reliability -v`; preserve RED evidence for genuine defects before minimal fix. -- [ ] **Step 3:** commit test-only coverage separately from any production fix. - ---- - -### Task 7: Full verification/audit - -- [ ] **Step 1:** `python -m unittest discover -s tests -v`; exact final head GREEN Python 3.10 + 3.12. -- [ ] **Step 2:** acceptance audit maps tests to no-silent-upgrade, target proof, v0 migration, review downgrade, source reconstruction, admission visibility, metadata-last, rollback, unknown mutation, idempotence. -- [ ] **Step 3:** PR base integration; `behind_by=0`; merge-base equals branch-creation integration SHA; only #16 docs/migration/admission/tests; no unresolved comments/reviews; `main` unchanged. -- [ ] **Step 4:** update PR evidence; Ready only after clean guards; expected-head guarded merge only into integration; preserve feature branch; never merge `main`. diff --git a/docs/WORKFLOW_V2_PARALLEL_DESIGN.md b/docs/WORKFLOW_V2_PARALLEL_DESIGN.md deleted file mode 100644 index 2611231..0000000 --- a/docs/WORKFLOW_V2_PARALLEL_DESIGN.md +++ /dev/null @@ -1,57 +0,0 @@ -# Workflow v2 explicit parallel mode — minimal slice - -Status: approved architecture; implementation target is issue #15. - -## Decision - -Sequential orchestration remains the unconditional default. Parallel planning is enabled only for the current invocation with `resume --parallel N`, where `N > 1`. - -No durable per-book parallelism setting is introduced. - -## Durable shared-state snapshot - -`glossary.md` and `style-guide.md` remain shared orchestrator-owned state. A status snapshot records their backend versions as `state_revisions.glossary` and `state_revisions.style_guide`. - -A parallel worker claim persists the frozen shared-state revisions it received at dispatch time: - -```json -{ - "shared_state_revisions": { - "glossary": "", - "style_guide": "" - } -} -``` - -The field is optional for legacy/sequential claims so existing sequential behavior and stored claims remain valid. When present, both keys are required and non-empty. - -## Parallel planning - -`StatusResolver.resume(status)` keeps current sequential semantics. - -`StatusResolver.resume(status, parallel=N)` with `N > 1` returns a bounded parallel batch of at most `N` actionable units. It never emits the same unit twice and skips units that already have an active claim. Preflight, migration, and finalization continue to block/override worker dispatch exactly as in sequential mode. - -The returned worker context carries the same frozen `state_revisions`, including glossary/style versions, so the orchestrator can pass those exact revisions into `ClaimManager.acquire(..., shared_state_revisions=...)`. - -## Shared-state writes - -Workers do not write `glossary.md` or `style-guide.md` directly. Shared-state mutation remains an orchestrator/single-writer responsibility. Proposal reconciliation under `.workflow/proposals/` is a later slice of #15; this slice establishes the invocation boundary and durable shared-state snapshot required before proposal acceptance can be made stale-safe. - -## Safety invariants - -1. No `--parallel` means current sequential behavior. -2. Parallel batches contain only disjoint unclaimed units. -3. Every parallel worker context exposes frozen glossary/style revisions. -4. A durable claim can record the exact frozen glossary/style revisions used by that worker. -5. No new code path gives translators/reviewers direct shared-state write ownership. - -## TDD slice - -RED tests cover: -- glossary/style revisions in status/context; -- explicit `parallel=N` batch planning and skip-claimed behavior; -- sequential compatibility; -- durable claim persistence and schema validation of frozen shared-state revisions; -- CLI parsing for `resume --parallel N`. - -GREEN is limited to the minimum changes required for those tests. \ No newline at end of file diff --git a/docs/WORKFLOW_V2_PRIVATE_SOURCE_DESIGN.md b/docs/WORKFLOW_V2_PRIVATE_SOURCE_DESIGN.md deleted file mode 100644 index b16ac79..0000000 --- a/docs/WORKFLOW_V2_PRIVATE_SOURCE_DESIGN.md +++ /dev/null @@ -1,394 +0,0 @@ -# Workflow v2 — Source corpus integrity and explicit private-source mode - -Issue: #11 -Branch: `feature/workflow-v2-private-source` -Base: `refactor/workflow-engine-v2` at `196c3ae12a0f981f69bd5a0adb67fd41d6c09686` -Final PR target: `refactor/workflow-engine-v2` -Date: 2026-09-06 - -## Purpose - -Make source reproducibility machine-verifiable for every new Workflow v2 book without requiring copyrighted or private source binaries to be committed to Git. - -Repository state remains authoritative. The durable identity of the original source is recorded separately from the question of whether the original binary is stored in the workspace. A complete sealed extracted corpus is sufficient for literary work only when its manifest verifies exactly against current extracted artifacts and the declared source identity. - -## Scope - -In scope: - -- explicit source storage mode for new books: `embedded` or `private_external`; -- durable source filename, format, byte size, and SHA-256 identity; -- automatic sealed-corpus creation for new books after extraction; -- complete extracted-corpus verification before `resume` permits literary work; -- private-source validation and verification without a committed original binary; -- exact-source reattachment through the existing corpus restore flow; -- rejection of same-name/different-hash replacement sources; -- source reproducibility fields in Workflow v2 status output; -- backward compatibility for books created before the explicit source contract. - -Out of scope: - -- finalization behavior beyond exposing source reproducibility state to future #12; -- generated review reports (#21); -- GitHub storage/backend work (#17); -- workflow migration tooling (#16); -- parallel orchestration (#15); -- databases, queues, or mandatory external services; -- storing copyrighted/private source binaries in Git. - -## Core invariants - -1. Source identity is filename + format + byte size + exact SHA-256, not filename alone. -2. A new book is not ready for literary work until its extracted corpus is sealed and verifies completely. -3. `private_external` never requires the original binary to remain in the repository workspace after successful extraction/sealing. -4. A private source supplied later must match the recorded identity before any restore write occurs. -5. Same-name/different-hash source replacement is rejected. -6. Every extracted artifact listed by progress must have exactly one manifest entry with matching chapter identity, path, and exact SHA-256. -7. Partial, missing, malformed, or hash-mismatched extracted corpus blocks `resume`. -8. Source reproducibility is computed from authoritative metadata, manifest, and current files; no derived `status.json` is introduced. -9. Existing legacy books are not silently migrated to the new source contract. -10. The original source binary and the extracted corpus are different reproducibility layers: an embedded binary is additionally verified, while a private external binary may be absent after sealing. - -## Durable source contract - -New books created by this workflow include an explicit `source` object in `metadata.json`: - -```json -{ - "source_file": "book.epub", - "source_format": "epub", - "source": { - "storage_mode": "embedded", - "filename": "book.epub", - "size_bytes": 123456, - "sha256": "<64 lowercase hex>" - } -} -``` - -Rules: - -- `storage_mode` is exactly `embedded` or `private_external`; -- `filename` is a safe basename and must equal legacy-compatible `source_file`; -- `size_bytes` is a non-negative integer; -- `sha256` is lowercase SHA-256; -- `source_format` remains the existing top-level compatibility field; -- new-book creation always writes the explicit object; -- legacy metadata without `source` retains legacy behavior until explicit migration work in #16. - -The explicit `source` object is the enablement marker. No additional mutable mode file is introduced. - -## Source manifest contract - -`source-manifest.json` remains the authoritative sealed-corpus manifest and is extended for new explicit-source books: - -```json -{ - "schema_version": 1, - "source_file": "book.epub", - "source_format": "epub", - "source_storage_mode": "private_external", - "source_size_bytes": 123456, - "source_sha256": "<64 lowercase hex>", - "chapter_count": 2, - "extracted": [ - { - "number": 1, - "title": "Chapter One", - "path": "extracted/001-chapter-one.md", - "sha256": "<64 lowercase hex>" - } - ] -} -``` - -For new explicit-source books, metadata and manifest source identity must agree exactly on filename, format, storage mode, size, and hash. - -Existing legacy manifests that predate `source_storage_mode` and `source_size_bytes` remain parseable for legacy books. The stricter fields are required when metadata contains the explicit `source` object. - -## New-book extraction modes - -### Embedded - -Default extraction remains embedded: - -```bash -python scripts/book.py extract ... -``` - -The flow: - -1. validate and read the input source; -2. compute source filename, format, byte size, and SHA-256; -3. extract all chapters; -4. write metadata/progress/support files/review ledger; -5. copy the original source to `books//source/`; -6. construct `source-manifest.json` from the exact source identity and all extracted artifact hashes; -7. validate the manifest before completing initialization. - -A successfully created new embedded book therefore starts sealed, not `unsealed`. - -### Private external - -Explicit private mode: - -```bash -python scripts/book.py extract ... --private-source -``` - -The source is available during extraction but is not retained in the book workspace. - -The flow is identical through extraction and hashing, except no final copy of the original binary is placed under `books//source/`. Metadata and manifest retain only its filename, format, size, and SHA-256 identity. - -The extracted corpus and machine state remain suitable for Git storage. The private binary remains outside repository authority. - -## Atomic initialization boundary - -New-book initialization must not leave a valid-looking explicit-source workspace without a complete manifest. - -Implementation should build and validate metadata, progress, extracted artifacts, review ledger, and source manifest as one initialization flow. If construction fails before completion, the command fails rather than reporting a successfully initialized unsealed Workflow v2 book. - -This issue does not introduce a general transactional filesystem framework. It only preserves the existing extraction cleanup/error boundary while adding manifest creation to the required initialization set. - -## Verification semantics - -`corpus.verify_manifest()` remains the single corpus hash verifier reused by CLI/status. It is extended rather than duplicated. - -Common verification for both storage modes: - -1. parse metadata/progress/manifest; -2. require manifest source identity to match explicit metadata when present; -3. require manifest chapter count to match progress; -4. require one ordered manifest entry per progress chapter; -5. require number/title/path equality for each entry; -6. require every extracted artifact to exist; -7. hash every extracted artifact and require an exact match. - -Additional `embedded` verification: - -- `source/` must exist; -- byte size must match metadata/manifest; -- SHA-256 must match metadata/manifest. - -Additional `private_external` verification: - -- absence of `source/` is valid; -- if a source file is present at that canonical path, it must match recorded byte size and SHA-256 or verification fails; -- presence of a mismatched file is never ignored. - -The verifier returns structured source/corpus identity sufficient for status instead of requiring callers to reconstruct mode semantics. - -## Structural validation - -For explicit-source books, `book.py validate` requires: - -- the explicit metadata `source` object passes schema validation; -- `source_file`, `source_format`, and explicit source identity are internally consistent; -- `source-manifest.json` exists and passes schema validation; -- manifest source identity agrees with metadata; -- progress and manifest chapter counts/paths are structurally consistent; -- `embedded` requires the canonical source file to exist; -- `private_external` does not require the original source file to exist. - -Structural validation is not a second hashing implementation. Exact source/extracted hashes remain the responsibility of `corpus.verify_manifest()` and the status/resume preflight that calls it. - -Legacy books without the explicit `source` object retain current validation semantics and are not required to acquire a manifest automatically. - -## Status and resume integration - -For explicit-source books, absence of `source-manifest.json` is no longer reported as benign `unsealed`; it is `invalid` and blocks resume. - -For legacy books, the existing `unsealed` compatibility result remains available. - -Verified status exposes a normalized corpus/source summary: - -```json -{ - "corpus": { - "state": "verified", - "storage_mode": "private_external", - "source_file": "book.epub", - "source_sha256": "...", - "source_size_bytes": 123456, - "source_attached": false, - "chapter_count": 42 - } -} -``` - -`resume` remains read-only. It neither attaches sources nor seals/restores anything. If explicit-source verification fails, the existing #10 preflight path returns `blocked` before translator/reviewer work. - -No derived status document is written. - -## Restore and temporary reattachment - -The existing command remains the reattachment surface: - -```bash -python scripts/corpus.py restore [--expected-sha256 ...] -``` - -Before writes, restore verifies: - -- source format matches recorded format; -- supplied source basename matches recorded filename for explicit-source books; -- supplied byte size matches recorded size; -- supplied SHA-256 matches recorded SHA-256; -- any explicit `--expected-sha256` agrees with durable identity; -- re-extracted chapter count/titles and, when already sealed, artifact hashes agree with durable corpus identity. - -For `embedded`, successful restore may repopulate/update the canonical stored source as today, but only after exact identity validation. - -For `private_external`, successful restore uses the supplied source as a temporary reconstruction input and does not copy it into the book workspace. It atomically replaces/reconstructs extracted artifacts and updates the manifest only when the reconstructed corpus matches the durable source identity. - -Thus a private binary may be reattached for repair without becoming repository state. - -## Same-name replacement protection - -Filename is never sufficient provenance. - -If a supplied or canonical source has the expected name but a different size or hash, verification/restore fails before extracted artifacts, metadata, or manifest are modified. - -If size happens to match but SHA-256 differs, the hash mismatch still rejects it. - -No command automatically updates durable source identity from a replacement binary. Changing source identity is a migration/re-import operation outside #11. - -## Partial corpus behavior - -For explicit-source books, any of the following makes corpus state invalid: - -- missing manifest; -- missing manifest entry; -- extra manifest entry; -- chapter count mismatch; -- missing extracted file; -- path/number/title mismatch; -- extracted SHA mismatch; -- metadata/manifest source identity mismatch; -- required embedded source missing; -- present canonical source with incorrect identity. - -`status` reports the reason and `resume` returns `blocked` without claiming or mutating workflow state. - -## Compatibility boundary - -This issue intentionally avoids silent migration. - -Books without `metadata.source`: - -- retain current metadata/source validation; -- may retain a legacy source manifest; -- may still appear `unsealed` in #10 status if no manifest exists; -- are not assigned `embedded` merely because a source file exists; -- require future #16 migration to opt into explicit source semantics. - -Books with `metadata.source`: - -- are governed by #11 explicit source semantics; -- require a valid sealed manifest; -- fail closed when the explicit contract is incomplete or inconsistent. - -This distinction lets the current runtime preserve old pinned workspaces while making all newly created workspaces reproducible by construction. - -## Schema changes - -`scripts/workflow_v2/schemas.py` extends metadata validation for optional explicit `source` and source-manifest validation for the additive fields. - -Unknown additive fields continue to be preserved under the existing schema policy. - -Schema version remains `1` because these are additive fields with explicit enablement through `metadata.source`; legacy documents remain valid. A future incompatible representation would require a schema-version change or explicit migration. - -## CLI surface - -New user-visible option: - -```bash -python scripts/book.py extract ... --private-source -``` - -Existing corpus commands remain: - -```bash -python scripts/corpus.py seal -python scripts/corpus.py verify -python scripts/corpus.py restore -``` - -For new explicit-source books, manual `seal` is normally unnecessary because extraction seals automatically. `seal` remains useful for legacy workspaces and repair workflows, but it may not silently change explicit source identity or storage mode. - -No command is added solely to attach a private binary permanently. - -## Testing strategy - -TDD proceeds in small slices. - -### Slice 1 — explicit source schema and initialization - -RED tests first: - -- new embedded extraction writes explicit source identity and a sealed manifest; -- private extraction writes the same durable identity but leaves no original binary in the workspace; -- explicit-source metadata/manifest mismatches fail schema/domain validation. - -### Slice 2 — verification and resume gate - -RED tests first: - -- verified private corpus succeeds without original source binary; -- missing/partial/tampered extracted corpus fails verification; -- missing manifest for an explicit-source book is invalid, not unsealed; -- `book.py resume` returns `blocked` for those invalid states; -- legacy unsealed behavior remains unchanged. - -### Slice 3 — exact restore/reattachment - -RED tests first: - -- same-name/different-hash source is rejected before writes; -- wrong-size source is rejected before writes; -- exact private source reconstructs corpus without persisting the binary; -- embedded restore still persists the exact source as appropriate; -- failed restore leaves prior extracted/state/manifest content unchanged. - -### Slice 4 — status reporting - -RED tests first: - -- JSON status reports storage mode, source identity, attachment state, and verified chapter count deterministically; -- human status clearly identifies private versus embedded reproducibility; -- status remains read-only. - -After each RED/GREEN slice, run the focused tests and then the complete test suite on supported Python versions through the existing PR CI matrix. - -## Expected implementation files - -The intended minimal implementation surface is: - -```text -scripts/book.py -scripts/corpus.py -scripts/workflow_v2/schemas.py -scripts/workflow_v2/status_cli.py - -tests/test_workflow_v2_private_source.py -tests/test_corpus_cli.py # only where existing restore behavior is extended -tests/test_workflow_v2_status_cli.py # only for source status/resume integration -``` - -Additional files should be added only if a concrete test exposes a responsibility that cannot remain clear in these existing boundaries. - -## Relationship to later issues - -#11 provides #12 with verified machine-readable source reproducibility state but does not implement finalization. - -#16 may later migrate legacy workspaces into the explicit source contract. - -#17 may later transport the same durable metadata/manifest through a GitHub API backend without changing source-integrity semantics. - -The authoritative split remains: - -- metadata: declared original source identity and storage policy; -- source manifest: sealed extracted-corpus evidence bound to that identity; -- filesystem artifacts: current bytes to verify; -- status: read-only computed view; -- future generated reports: derived, non-authoritative presentation. diff --git a/docs/WORKFLOW_V2_PRIVATE_SOURCE_PLAN.md b/docs/WORKFLOW_V2_PRIVATE_SOURCE_PLAN.md deleted file mode 100644 index ba99efb..0000000 --- a/docs/WORKFLOW_V2_PRIVATE_SOURCE_PLAN.md +++ /dev/null @@ -1,450 +0,0 @@ -# Workflow v2 Private Source Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make every newly extracted Workflow v2 book start with a verified sealed corpus and explicit source identity, while allowing `private_external` sources to remain outside Git without blocking validation or resume. - -**Architecture:** Extend the existing additive Workflow v2 metadata/source-manifest schemas rather than creating a new state file. Keep `scripts/corpus.py` as the single hash/integrity authority, make `scripts/book.py extract` create metadata + extracted files + manifest consistently, and have #10 status consume structured corpus verification output through the existing read-only preflight path. - -**Tech Stack:** Python 3.10/3.12, stdlib `argparse`, `hashlib`, `json`, `pathlib`, `unittest`, existing Workflow v2 `FilesystemStorage` / `WorkflowStateRepository` / schemas. - -**Spec:** `docs/WORKFLOW_V2_PRIVATE_SOURCE_DESIGN.md` - -## Global Constraints - -- Source storage mode is exactly `embedded` or `private_external` for new explicit-source books. -- Source identity is filename + format + byte size + exact lowercase SHA-256. -- New books are sealed at successful extraction time; no successful new explicit-source workspace remains `unsealed`. -- `private_external` verifies without the original binary when the sealed extracted corpus is complete. -- `embedded` requires the canonical source binary and verifies its size/hash. -- Missing, partial, malformed, or hash-mismatched explicit corpus blocks `resume`. -- Existing books without `metadata.source` retain legacy behavior; no silent migration. -- No `status.json`, database, queue, external service, GitHub backend, finalization, or migration behavior is introduced. -- `main` is never modified. - ---- - -## File Structure - -- `scripts/workflow_v2/schemas.py`: optional explicit `metadata.source` and additive manifest identity validation. -- `scripts/book.py`: source identity during extract, `--private-source`, automatic seal, mode-aware structural validation. -- `scripts/corpus.py`: single source/extracted hash authority, structured verification, exact restore, private reattachment semantics. -- `scripts/workflow_v2/status_cli.py`: explicit-source fail-closed preflight and deterministic source reproducibility reporting. -- `tests/test_workflow_v2_private_source.py`: new-book explicit-source contracts. -- `tests/test_corpus_cli.py`: verify/restore regressions. -- `tests/test_workflow_v2_status_cli.py`: status/resume integration and compatibility. - -### Task 1: Explicit source schema and sealed initialization - -**Files:** -- Create: `tests/test_workflow_v2_private_source.py` -- Modify: `scripts/workflow_v2/schemas.py` -- Modify: `scripts/book.py` -- Modify: `scripts/corpus.py` - -**Interfaces:** -- Metadata: `source = {storage_mode, filename, size_bytes, sha256}`. -- Manifest adds `source_storage_mode` and `source_size_bytes` for explicit-source books. -- `corpus.build_manifest(book_dir, metadata, progress, source)` remains the shared manifest builder. - -- [ ] **Step 1: Write RED initialization tests** - -Use the temporary-repository pattern from `tests/test_corpus_cli.py` and add: - -```python -def test_new_embedded_book_records_identity_and_is_sealed(self): - source = self.repo / "sample.md" - source.write_text("# One\n\nAlpha.\n", encoding="utf-8") - source_bytes = source.read_bytes() - expected_sha = hashlib.sha256(source_bytes).hexdigest() - - self.run_book("extract", str(source), "--slug", "sample", "--target-language", "ru") - book = self.repo / "books" / "sample" - metadata = json.loads((book / "metadata.json").read_text(encoding="utf-8")) - manifest = json.loads((book / "source-manifest.json").read_text(encoding="utf-8")) - - self.assertEqual(metadata["source"], { - "storage_mode": "embedded", - "filename": "sample.md", - "size_bytes": len(source_bytes), - "sha256": expected_sha, - }) - self.assertEqual(manifest["source_storage_mode"], "embedded") - self.assertEqual(manifest["source_size_bytes"], len(source_bytes)) - self.assertEqual(manifest["source_sha256"], expected_sha) - self.assertTrue((book / "source" / "sample.md").is_file()) - self.run_corpus("verify", "sample") -``` - -```python -def test_new_private_book_is_sealed_without_source_binary(self): - source = self.repo / "private.md" - source.write_text("# One\n\nSecret.\n", encoding="utf-8") - source_bytes = source.read_bytes() - expected_sha = hashlib.sha256(source_bytes).hexdigest() - - self.run_book( - "extract", str(source), "--slug", "private-book", - "--target-language", "ru", "--private-source", - ) - book = self.repo / "books" / "private-book" - metadata = json.loads((book / "metadata.json").read_text(encoding="utf-8")) - manifest = json.loads((book / "source-manifest.json").read_text(encoding="utf-8")) - - self.assertEqual(metadata["source"]["storage_mode"], "private_external") - self.assertEqual(metadata["source"]["filename"], "private.md") - self.assertEqual(metadata["source"]["size_bytes"], len(source_bytes)) - self.assertEqual(metadata["source"]["sha256"], expected_sha) - self.assertEqual(manifest["source_storage_mode"], "private_external") - self.assertFalse((book / "source" / "private.md").exists()) - self.run_book("validate", "private-book") - self.run_corpus("verify", "private-book") -``` - -Add mutation cases that set `metadata["source"]["filename"] = "other.md"`, `storage_mode = "unknown"`, or replace one manifest identity field and assert validation/verification exits 1. - -- [ ] **Step 2: Commit RED tests only** - -```bash -git add tests/test_workflow_v2_private_source.py -git commit -m "test: define #11 explicit source initialization" -``` - -Expected RED causes: `metadata.source` absent, `--private-source` not recognized, automatic manifest absent. - -- [ ] **Step 3: Extend schema validation** - -In `_validate_metadata`, when `source` is present: - -```python -source = _require_mapping(data, "source", schema) -storage_mode = _require_nonempty_string(source, "storage_mode", schema, path="source.storage_mode") -if storage_mode not in {"embedded", "private_external"}: - raise _field(schema, "source.storage_mode", "must be embedded or private_external") -filename = _require_nonempty_string(source, "filename", schema, path="source.filename") -_validate_basename(filename, schema, "source.filename") -if filename != data["source_file"]: - raise _field(schema, "source.filename", "must equal source_file") -_require_int(source, "size_bytes", schema, minimum=0, path="source.size_bytes") -sha256 = _require_nonempty_string(source, "sha256", schema, path="source.sha256") -_validate_sha256(sha256, schema, "source.sha256") -``` - -In `_validate_source_manifest`, preserve legacy manifests. If either `source_storage_mode` or `source_size_bytes` exists, require both; validate mode and non-negative size. - -- [ ] **Step 4: Add explicit source identity to extract** - -Add `import hashlib`. Before workspace mutation: - -```python -source_bytes = source.read_bytes() -source_identity = { - "storage_mode": "private_external" if args.private_source else "embedded", - "filename": source.name, - "size_bytes": len(source_bytes), - "sha256": hashlib.sha256(source_bytes).hexdigest(), -} -``` - -Add `--private-source` to the existing extract parser. Store `source_identity` at `metadata["source"]`. For `private_external`, skip the canonical source copy; for embedded preserve it. - -- [ ] **Step 5: Seal during new-book initialization** - -Extend `corpus.build_manifest` so explicit metadata identity is copied into manifest and the supplied source bytes are checked against metadata identity. Reuse its existing extracted-file hashing. Call the shared builder/writer from `book.py extract`; do not duplicate extracted hashes in `book.py`. - -- [ ] **Step 6: Make structural validation mode-aware** - -Use exact messages: - -```python -source_contract = metadata.get("source") -canonical_source = book_dir / "source" / str(metadata.get("source_file")) -if isinstance(source_contract, dict): - if source_contract.get("storage_mode") == "embedded" and not canonical_source.is_file(): - errors.append(f"Embedded source file does not exist: source/{metadata.get('source_file')}") - if not (book_dir / "source-manifest.json").is_file(): - errors.append("Missing source-manifest.json for explicit-source book") -else: - if not canonical_source.is_file(): - errors.append(f"Source file declared in metadata.json does not exist: source/{metadata.get('source_file')}") -``` - -Do not hash in `validate_book`. - -- [ ] **Step 7: Verify GREEN and commit** - -```bash -python -m unittest tests.test_workflow_v2_private_source -v -python -m unittest discover -s tests -v -``` - -Require Python 3.10/3.12 CI success, then: - -```bash -git add scripts/book.py scripts/corpus.py scripts/workflow_v2/schemas.py tests/test_workflow_v2_private_source.py -git commit -m "feat: seal new books with explicit source identity" -``` - -### Task 2: Corpus verification and resume gate - -**Files:** -- Modify: `scripts/corpus.py` -- Modify: `scripts/workflow_v2/status_cli.py` -- Modify: `tests/test_workflow_v2_private_source.py` -- Modify: `tests/test_workflow_v2_status_cli.py` - -**Interfaces:** -- `verify_manifest(...) -> dict` with keys `state`, `storage_mode`, `source_file`, `source_sha256`, `source_size_bytes`, `source_attached`, `chapter_count`. - -- [ ] **Step 1: Write RED verification/status tests** - -Update fresh new-book status expectation to `verified`. Add: - -```python -def test_explicit_source_missing_manifest_blocks_resume(self): - book = self.initialize_book() - (book / "source-manifest.json").unlink() - result = self.run_book("resume", "sample", "--json", expect=1) - payload = self.canonical_json(result) - self.assertEqual(payload["operation"], "blocked") - self.assertEqual(payload["reason"], "preflight_failed") - self.assertTrue(any("source-manifest.json" in error for error in payload["errors"])) -``` - -For private mode assert: - -```python -self.assertEqual(status["corpus"]["storage_mode"], "private_external") -self.assertFalse(status["corpus"]["source_attached"]) -``` - -Tamper/delete one extracted artifact and assert `resume --json` exits 1 with `preflight_failed`. Create a legacy fixture by removing `metadata.source` and `source-manifest.json` while retaining the embedded source; assert `status["corpus"]["state"] == "unsealed"`. - -- [ ] **Step 2: Commit Task 2 RED tests** - -```bash -git add tests/test_workflow_v2_private_source.py tests/test_workflow_v2_status_cli.py -git commit -m "test: define #11 corpus verification gate" -``` - -- [ ] **Step 3: Return structured verification** - -For explicit metadata, require exact metadata↔manifest identity agreement. Always verify chapter count/order/path/title/hash. For embedded, require canonical source and exact size/hash. For private, allow canonical source absence but verify it if present. - -Return: - -```python -return { - "state": "verified", - "storage_mode": storage_mode, - "source_file": metadata["source_file"], - "source_sha256": expected_source_sha, - "source_size_bytes": expected_size, - "source_attached": source.is_file(), - "chapter_count": len(items), -} -``` - -For legacy metadata keep the current source-file requirement and return `storage_mode="legacy_embedded"`, `source_size_bytes=None`, `source_attached=True`. - -- [ ] **Step 4: Make #10 preflight fail closed for explicit source** - -```python -explicit_source = isinstance(metadata.get("source"), Mapping) -if not manifest_path.is_file(): - if explicit_source: - return structural_errors, { - "state": "invalid", - "storage_mode": metadata["source"].get("storage_mode"), - "error": "source-manifest.json is missing for explicit-source book", - } - return structural_errors, {"state": "unsealed"} -``` - -When verification succeeds, return its structured mapping directly. On verification failure return `state="invalid"`, preserve storage mode when available, and include `error`. - -- [ ] **Step 5: Keep human status concise** - -For explicit verified corpus produce: - -```text -claims=0 corpus=verified source=private_external attached=no -``` - -Use `yes/no` for attachment. Legacy `unsealed` output remains valid. - -- [ ] **Step 6: Verify GREEN and commit** - -```bash -python -m unittest tests.test_workflow_v2_private_source tests.test_workflow_v2_status_cli -v -python -m unittest discover -s tests -v -``` - -Then: - -```bash -git add scripts/corpus.py scripts/workflow_v2/status_cli.py tests/test_workflow_v2_private_source.py tests/test_workflow_v2_status_cli.py -git commit -m "feat: block resume on invalid explicit source corpus" -``` - -### Task 3: Exact restore and temporary private reattachment - -**Files:** -- Modify: `scripts/corpus.py` -- Modify: `tests/test_corpus_cli.py` - -**Interfaces:** -- Add `verify_supplied_source_identity(source, metadata, manifest, expected_sha256) -> dict` and call it before any restore mutation. - -- [ ] **Step 1: Write RED private restore tests** - -Exact-source success test: - -```python -def test_private_restore_rebuilds_without_persisting_binary(self): - source = self.repo / "sample.epub" - self.make_epub(source) - self.run_cli( - "book.py", "extract", str(source), "--slug", "sample", - "--target-language", "ru", "--private-source", - ) - book = self.repo / "books" / "sample" - progress = json.loads((book / "progress.json").read_text(encoding="utf-8")) - missing = book / progress["chapters"][0]["source_path"] - missing.unlink() - - self.run_cli("corpus.py", "restore", "sample", str(source)) - self.assertTrue(missing.is_file()) - self.assertFalse((book / "source" / "sample.epub").exists()) - self.run_cli("corpus.py", "verify", "sample") -``` - -Same-name/different-hash rejection test: - -```python -def test_private_restore_rejects_same_name_different_hash_before_writes(self): - source_dir = self.repo / "original" - source_dir.mkdir() - source = source_dir / "sample.epub" - self.make_epub(source) - self.run_cli( - "book.py", "extract", str(source), "--slug", "sample", - "--target-language", "ru", "--private-source", - ) - book = self.repo / "books" / "sample" - before_manifest = (book / "source-manifest.json").read_bytes() - before_extracted = { - path.relative_to(book).as_posix(): path.read_bytes() - for path in (book / "extracted").glob("*.md") - } - - replacement_dir = self.repo / "replacement" - replacement_dir.mkdir() - replacement = replacement_dir / "sample.epub" - self.make_epub(replacement, body_suffix=" changed") - result = self.run_cli("corpus.py", "restore", "sample", str(replacement), expect=1) - - self.assertIn("SHA-256 mismatch", result.stderr) - self.assertEqual((book / "source-manifest.json").read_bytes(), before_manifest) - self.assertEqual({ - path.relative_to(book).as_posix(): path.read_bytes() - for path in (book / "extracted").glob("*.md") - }, before_extracted) -``` - -Add a wrong-size explicit source case by appending bytes to a same-name copy and assert error contains `size mismatch` before state changes. - -- [ ] **Step 2: Commit Task 3 RED tests** - -```bash -git add tests/test_corpus_cli.py -git commit -m "test: define #11 exact private source restore" -``` - -- [ ] **Step 3: Implement pre-write identity validation** - -For explicit source require basename, detected format, byte size, and SHA-256 to match metadata. Require manifest identity to agree with metadata. Require any `--expected-sha256` to agree with durable identity. - -For legacy source retain current trusted-hash behavior from manifest or `--expected-sha256`. - -- [ ] **Step 4: Make restore commit path mode-aware** - -For embedded retain staged-source copy/replacement. For private do not stage or write a canonical source; atomically replace only reconstructed `extracted/` and the manifest after all checks pass. Preserve rollback behavior on write failure. - -- [ ] **Step 5: Verify GREEN and commit** - -```bash -python -m unittest tests.test_corpus_cli -v -python -m unittest discover -s tests -v -``` - -Then: - -```bash -git add scripts/corpus.py tests/test_corpus_cli.py -git commit -m "feat: restore private corpus without persisting source binary" -``` - -### Task 4: Final status coverage and verification - -**Files:** -- Modify: `tests/test_workflow_v2_status_cli.py` -- Modify: `tests/test_workflow_v2_private_source.py` -- Modify: `scripts/workflow_v2/status_cli.py` only if the RED presentation assertions require it. - -**Interfaces:** -- Explicit verified status corpus keys are exactly `state`, `storage_mode`, `source_file`, `source_sha256`, `source_size_bytes`, `source_attached`, `chapter_count`. -- `resume` remains read-only and bounded-context files do not change. - -- [ ] **Step 1: Add final RED/coverage assertions** - -For a private one-chapter book assert: - -```python -self.assertEqual(status["corpus"], { - "chapter_count": 1, - "source_attached": False, - "source_file": "sample.md", - "source_sha256": expected_sha, - "source_size_bytes": expected_size, - "state": "verified", - "storage_mode": "private_external", -}) -``` - -Hash `metadata.json`, `progress.json`, `review-ledger.json`, and `source-manifest.json` before and after `status` + `resume`; assert byte hashes unchanged and `status.json` absent. - -- [ ] **Step 2: Commit coverage tests** - -```bash -git add tests/test_workflow_v2_status_cli.py tests/test_workflow_v2_private_source.py -git commit -m "test: lock #11 source reproducibility status" -``` - -- [ ] **Step 3: Apply only production changes required by RED** - -Do not change `StatusResolver` operation selection or context descriptors. If necessary, adjust only source summary formatting in `status_cli.py`. - -- [ ] **Step 4: Run focused suites** - -```bash -python -m unittest tests.test_workflow_v2_private_source -v -python -m unittest tests.test_corpus_cli -v -python -m unittest tests.test_workflow_v2_status_cli -v -``` - -- [ ] **Step 5: Run full matrix suite** - -```bash -python -m unittest discover -s tests -v -``` - -Require GitHub Actions `unit-tests (3.10)` and `unit-tests (3.12)` both `success`. Inspect final logs for total count and individual #11 tests. - -- [ ] **Step 6: Self-review branch diff** - -Verify exact integration ancestry, `main` untouched, no branch deletions, no private binary in changes, no derived status state, and no #12/#16/#17/#15 scope. - -- [ ] **Step 7: Open PR to integration** - -Open a PR `feature/workflow-v2-private-source` → `refactor/workflow-engine-v2` with `Closes #11`, RED/GREEN run evidence, final matrix run IDs, compatibility notes, changed-file list, and explicit `main` untouched statement. Keep Draft until final matrix is green, then mark Ready for review. Do not merge without a separate integration authorization. diff --git a/docs/WORKFLOW_V2_RELIABILITY_PHASE1_DESIGN.md b/docs/WORKFLOW_V2_RELIABILITY_PHASE1_DESIGN.md deleted file mode 100644 index 32b3ddf..0000000 --- a/docs/WORKFLOW_V2_RELIABILITY_PHASE1_DESIGN.md +++ /dev/null @@ -1,339 +0,0 @@ -# Workflow v2 — Phase 1 reliability, failure injection and idempotence - -Issue: #18 -Branch: `test/workflow-v2-reliability` -Base: `refactor/workflow-engine-v2` at `b055e74c5694e000d047335820fd333f7bd74604` -PR target: `refactor/workflow-engine-v2` -Date: 2026-09-06 - -## Purpose - -Close the Phase 1 reliability gate for Workflow v2 by proving that the already integrated #7–#11 state, concurrency, review, resume and source-integrity primitives compose safely across fresh sessions and realistic interruption boundaries. - -The Phase 1 slice of #18 is intentionally a reliability test layer, not a new orchestration subsystem. It exercises real durable repository state through the public CLI and domain APIs, simulates interruption by stopping between durable writes, and verifies that a new session can determine the safe next action from repository state alone. - -Later #18 slices will extend this same reliability suite after finalize (#12), EPUB build (#14), migrations (#16), GitHub backend (#17) and explicit parallel mode (#15) exist. This Phase 1 slice does not pre-implement those features. - -## Scope - -In scope now: - -- fresh-session recovery after a durable claim is acquired and the worker disappears; -- deterministic blocking while a conflicting claim remains active; -- audited expired-claim cleanup and safe continuation afterward; -- recovery when translation bytes exist but lifecycle state was not advanced; -- recovery when lifecycle state is translated but review has not occurred; -- recovery when current PASS evidence exists but lifecycle promotion to `reviewed` did not occur; -- stale-review detection after reviewed translation bytes change; -- concurrent glossary CAS failure without lost update; -- fail-closed resume after an extracted corpus is removed or tampered; -- successful private-source resume after process restart without the original binary; -- explicit idempotence checks for read-only and safe retryable operations that already promise idempotent/repeat-safe behavior; -- standard Python test-suite execution, with no test depending on GitHub Actions as a runtime requirement. - -Out of scope for this slice: - -- interrupted/failed finalize and rollback (#12, then extend #18); -- build from incomplete state and output staleness (#14, then extend #18); -- workflow migration failure/rollback (#16, then extend #18); -- GitHub backend parity and API conflict injection (#17, then extend #18); -- shared-state proposal reconciliation in explicit parallel mode (#15, then extend #18); -- production-only fault-injection hooks introduced solely for tests; -- changes to `main`. - -## Design choice - -### Selected approach: end-to-end reliability harness over real repository state - -Add a focused integration-style test module that creates a temporary repository/workspace, invokes the real `scripts/book.py` / `scripts/corpus.py` CLI where user-visible orchestration behavior matters, and uses Workflow v2 repository/domain APIs only to construct precise interruption boundaries that the public CLI cannot naturally stop at. - -This gives each scenario two layers: - -1. **Fault setup:** create exactly the durable subset that would remain if a session died at the named boundary. -2. **Fresh-session observation:** invoke `book.py status` / `book.py resume` in a new subprocess and assert the deterministic safe continuation or block reason. - -This avoids duplicating unit tests already present for individual claims, CAS writes, review-ledger conflict resolution and corpus hashing. Existing component tests remain the proof that each primitive works in isolation; the new suite proves that the primitives compose correctly after interruption. - -### Rejected approach: expand only existing component test files - -Adding isolated cases to `test_workflow_v2_claims.py`, `test_workflow_v2_reviews.py` and `test_workflow_v2_status_cli.py` would increase coverage but would obscure the main #18 requirement: recovery behavior across durable boundaries and fresh sessions. Component files may receive a narrowly scoped regression case only if a RED reliability scenario exposes a bug owned by that component. - -### Rejected approach: production fault-injection API - -A generic crash/fault hook in production code is unnecessary for Phase 1. The relevant failure states can be constructed by stopping between existing durable operations. Test-only production hooks would widen the public/runtime surface without adding product value. - -## Test harness boundary - -Create `tests/test_workflow_v2_reliability.py` as the Phase 1 cross-cutting harness. - -The harness owns: - -- temporary repository creation; -- copying the current runtime scripts/package into that repository, following existing CLI test patterns; -- installation provenance fixture required by current book initialization; -- creation of a one- or two-chapter sealed book through real `book.py extract`; -- subprocess helpers for `book.py` and `corpus.py`; -- deterministic JSON parsing/assertion helpers; -- direct `WorkflowStateRepository` / `FilesystemStorage` access when a scenario must stop between two durable domain operations; -- deterministic claim/review fixtures with explicit timestamps and IDs when real-time CLI behavior would make a crash boundary nondeterministic; -- SHA snapshots used to prove read-only/idempotent behavior. - -The harness does not own alternative business logic. It must use current schemas, repository methods, storage primitives, claim manager, review manager and corpus verifier rather than reimplementing them. - -## Reliability invariants - -The Phase 1 suite treats these as release-gate invariants: - -1. Repository state, not the previous chat/process, determines recovery. -2. A live durable claim prevents a second session from dispatching conflicting work for that unit. -3. Claim expiry alone does not silently erase evidence; cleanup is explicit and auditable. -4. A crashed worker's claim remains the first recovery concern even if it already wrote translation/progress/review state. -5. Translation bytes without a lifecycle transition do not fabricate translated state. -6. Lifecycle `translated` without current PASS evidence resumes into review only after the crashed worker's claim is cleared. -7. Current PASS evidence without lifecycle promotion is recoverable through `accept_review` after the crashed reviewer claim is cleared; duplicate promotion does not corrupt state. -8. Changing source or translation bytes invalidates the PASS that was bound to the previous bytes. -9. A stale glossary writer cannot overwrite a newer glossary revision. -10. Invalid explicit source corpus state blocks literary work before dispatch. -11. `private_external` remains reproducible from a verified sealed extracted corpus after the source binary is absent and a new process starts. -12. Read-only status/resume never mutate durable state. -13. Retry-safe cleanup/release/promotion behavior must either be idempotent or return the current deterministic already-completed/not-owned result without corrupting state. - -## Failure scenarios - -### Scenario A — session death after claim - -Setup: - -1. initialize a sealed book with chapter 1 in `extracted`; -2. acquire a translator claim for chapter 1 using the public claim command; -3. simulate death by performing no translation or release; -4. start a fresh subprocess and call `resume`. - -Expected: - -- `resume` returns `operation=blocked` and `reason=unit_claimed` for chapter 1; -- claim identity/session/expiry are reported from durable state; -- no translation/progress/review files are mutated by the fresh resume. - -Recovery continuation is a separate deterministic fixture: - -1. create a schema-valid translator claim for chapter 1 directly through the repository with fixed historical `claimed_at` / `expires_at` timestamps that are unambiguously expired; -2. run public `cleanup-claims`; -3. verify exactly one cleanup request and one completion event exist with `reason=lease_expired` and matching request linkage; -4. call `cleanup-claims` again and assert `results=[]` with no additional audit records; -5. start a fresh subprocess and verify `resume` returns `translate` for chapter 1. - -No wall-clock sleeps are allowed. - -### Scenario B — death after translation bytes, before progress update - -Setup: - -1. initialize chapter 1 as `extracted`; -2. create a deterministic active translator claim for chapter 1; -3. create non-empty `translated/...md` bytes directly, representing the claimed worker writing its artifact; -4. stop before the progress CAS transition. - -Expected immediately after restart: - -- fresh `resume` is blocked with `reason=unit_claimed`; -- lifecycle remains `extracted` because progress is authoritative; -- the translation artifact does not fabricate translated lifecycle state. - -Recovery: - -1. replace the fixture claim with an expired deterministic claim for the same crash boundary and clean it through the normal cleanup path; -2. fresh `resume` selects `translate`, not `review`, even though orphan translation bytes exist; -3. status/resume do not rewrite or auto-promote those bytes. - -This documents the current recovery contract: orphaned translation bytes require the translator/orchestrator path to reconcile them explicitly; file presence alone is not lifecycle authority. - -### Scenario C — death after translated lifecycle transition, before review - -Setup: - -1. create valid translation bytes under an active translator claim; -2. advance chapter lifecycle to `translated` through a versioned repository write; -3. stop before claim release and before any review evidence is written. - -Expected immediately after restart: - -- fresh status reports lifecycle translated and review missing; -- fresh `resume` is blocked by the surviving translator claim rather than dispatching a reviewer concurrently. - -Recovery: - -1. clear an expired equivalent claim through audited cleanup; -2. fresh `resume` selects `review` for the translated chapter; -3. no review evidence is fabricated. - -### Scenario D — death after PASS record, before reviewed promotion - -Setup: - -1. create valid translated state and translation bytes; -2. create a matching reviewer claim; -3. record a current PASS in the ledger through the real review manager/CLI contract; -4. stop before lifecycle promotion and before claim release. - -Expected immediately after restart: - -- fresh status reports lifecycle translated + review pass + active reviewer claim; -- fresh `resume` is blocked by the surviving claim, preventing a second session from accepting/promoting while the reviewer still owns the unit. - -Recovery: - -1. clear an expired equivalent reviewer claim through audited cleanup; -2. fresh `resume` selects `accept_review`; -3. executing `accept-review` returns `changed=true` and promotes only the selected unit to `reviewed`; -4. executing `accept-review` again against unchanged current PASS returns `changed=false`, keeps the current progress revision/content stable, and does not append review-ledger records. - -If this exact retry contract fails, the RED test identifies a production reliability gap owned by the review-promotion layer. - -### Scenario E — translation changed after PASS - -Setup: - -1. reach a current PASS for a translated artifact; -2. ensure no active claim remains; -3. change exact translation bytes after PASS without adding new PASS evidence. - -Expected recoverable case: - -- with lifecycle still `translated`, status reports review stale and remains structurally valid; -- `resume` selects `review` for the stale artifact. - -Expected fail-closed case: - -1. restore original bytes, accept the PASS so lifecycle becomes `reviewed`, then modify translation bytes again; -2. status reports review stale and invalid because `reviewed` lacks current PASS evidence; -3. `resume` returns `blocked/preflight_failed`, never `complete`. - -The tests reuse current review resolution; they do not calculate independent replacement review state. - -### Scenario F — concurrent glossary changes through CAS - -Setup: - -1. initialize a book and read `glossary.md` through two independent `FilesystemStorage` clients, producing the same starting revision; -2. writer A changes glossary bytes and calls `write_if_version` with that observed revision; -3. writer B attempts a different glossary change using its stale starting revision. - -Expected: - -- writer A succeeds and returns a new revision; -- writer B receives `StorageVersionConflict`; -- final `glossary.md` bytes equal writer A’s complete content, with no partial/merged writer-B bytes; -- a third fresh storage client reads exactly the winning revision/content. - -This directly covers #18's concurrent-glossary failure injection using the same storage CAS primitive intended for shared mutable state. It does not introduce a special glossary schema or automatic merge policy. - -### Scenario G — corpus changes after a previously valid session - -Two subcases: - -- remove one extracted artifact from an explicit-source sealed book; -- modify one extracted artifact without updating its manifest hash. - -Expected for both: - -- fresh `status` reports invalid corpus/preflight evidence; -- fresh `resume` returns `blocked/preflight_failed`; -- no claim or workflow state mutation is created by resume. - -### Scenario H — private source survives process restart without binary - -Setup: - -1. initialize through `book.py extract --private-source`; -2. assert canonical source binary is absent; -3. use new subprocesses for status/resume so no initialization Python object is reused. - -Expected: - -- corpus is verified with `storage_mode=private_external` and `source_attached=false`; -- source filename/size/SHA identity matches metadata/manifest; -- `resume` selects normal literary work from the sealed extracted corpus; -- repeated status/resume leave durable state byte-identical. - -## Idempotence matrix - -The Phase 1 suite explicitly checks these current contracts: - -- `status`: two calls from unchanged state produce byte-for-byte equivalent canonical JSON and no durable writes; -- `resume`: two calls from unchanged state produce byte-for-byte equivalent canonical JSON and no durable writes; -- `cleanup-claims`: first cleanup of one expired claim produces the expected request/completion audit pair; second cleanup returns `results=[]` and creates no audit records; -- `release`: first owner release succeeds and creates one request/completion pair; second release for the now-absent claim fails deterministically with `ClaimConflict` / “no active claim” and creates no additional audit records; -- `accept-review`: first promotion with current PASS returns `changed=true`; the second call returns `changed=false`, does not alter progress content, and does not append ledger evidence. - -The suite does not redefine idempotence as “always return exit code 0”. Retrying after an uncertain client outcome is safe when it cannot corrupt or duplicate durable state and reports the already-completed/absent condition deterministically. - -## Production-change policy - -Start with tests only. - -For each reliability scenario: - -1. add the smallest test that expresses the durable recovery invariant; -2. run it against the current integration-derived branch; -3. if it passes, retain it as missing release-gate coverage and make no production change; -4. if it fails for the intended reliability reason, identify the owning component and add the minimum production fix; -5. run the focused test, neighboring component tests and the complete standard suite; -6. commit test evidence and any GREEN fix in audit-friendly boundaries. - -A failure caused only by a bad fixture/import/test assumption must be fixed in the test and re-run before any production change. A new test that already passes is valid reliability coverage; RED→GREEN is required only for behavior that is currently defective, not for adding tests around already correct behavior. - -Potential production owners if genuine gaps are exposed: - -- `scripts/workflow_v2/status.py` / `status_cli.py` — wrong resume/preflight composition; -- `scripts/workflow_v2/claims.py` / `claim_cli.py` — unsafe cleanup/retry semantics; -- `scripts/workflow_v2/reviews.py` / `review_cli.py` — PASS/promotion/idempotence gaps; -- `scripts/workflow_v2/repository.py` / filesystem storage implementation — CAS/lost-update gaps; -- `scripts/corpus.py` / source integrity helpers — fresh-session corpus verification gaps. - -No unrelated refactor is permitted. - -## Test determinism - -- Use `tempfile.TemporaryDirectory` and repository-local fixtures. -- Use explicit historical timestamps for expired claims and deterministic IDs for asserted audit records; no sleeping. -- Compare structured/canonical JSON rather than unstable prose unless the human CLI output itself is the contract under test. -- When proving read-only behavior, snapshot SHA-256 of authoritative files before and after the operation. -- Do not depend on network access, GitHub Actions, external services or a real private source. -- Tests run under the repository’s normal `python -m unittest discover -s tests -v` suite. - -## Expected implementation surface - -Primary new file: - -```text -tests/test_workflow_v2_reliability.py -``` - -Existing tests may receive only narrowly targeted regression assertions when a discovered defect belongs to an existing component boundary. - -Production files are modified only when a failing reliability scenario proves a real missing invariant. No production file is pre-authorized merely because it is listed as a possible owner above. - -Documentation: - -```text -docs/WORKFLOW_V2_RELIABILITY_PHASE1_DESIGN.md -``` - -The implementation plan will be added only after this written design is explicitly reviewed and approved. - -## Completion criteria for the Phase 1 slice - -The Phase 1 #18 slice is ready to merge into `refactor/workflow-engine-v2` when: - -- all scenarios A–H have deterministic automated coverage; -- the idempotence matrix is covered for the currently implemented operations; -- every discovered production reliability defect has its own demonstrated RED→GREEN cycle; -- full standard tests pass on all CI Python versions used by the repository; -- no test requires GitHub Actions to execute locally; -- the reliability PR contains no finalize/build/migration/backend/parallel implementation; -- branch remains based on/in sync with the current integration line; -- `main` is unchanged; -- review-thread/diff/CI audit is clean before Ready for review. - -After merge, #18 remains open for later incremental slices tied to #12, #14, #16, #17 and #15. Phase 1 of the epic can then be treated as reliability-covered, while full #18 closes only after the later slices are integrated. diff --git a/docs/WORKFLOW_V2_RELIABILITY_PHASE1_PLAN.md b/docs/WORKFLOW_V2_RELIABILITY_PHASE1_PLAN.md deleted file mode 100644 index a0c9f21..0000000 --- a/docs/WORKFLOW_V2_RELIABILITY_PHASE1_PLAN.md +++ /dev/null @@ -1,318 +0,0 @@ -# Workflow v2 Phase 1 Reliability Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add deterministic Phase 1 recovery, failure-injection and idempotence coverage for the already integrated Workflow v2 #7–#11 behavior, fixing production code only if a reliability scenario proves a real defect. - -**Architecture:** Add one cross-cutting integration harness, `tests/test_workflow_v2_reliability.py`, that creates a temporary repository, invokes the real `book.py`/`corpus.py` CLIs in fresh subprocesses, and uses the existing repository/domain APIs only to create exact crash-boundary durable states. Existing component tests remain unchanged unless a newly demonstrated defect requires a narrowly owned regression assertion. No production fault-injection API is introduced. - -**Tech Stack:** Python 3.10/3.12, `unittest`, `tempfile`, `subprocess`, `FilesystemStorage`, `WorkflowStateRepository`, `ClaimManager`, `ReviewLedgerManager`, existing Workflow v2 schemas/CLI. - -**Spec:** `docs/WORKFLOW_V2_RELIABILITY_PHASE1_DESIGN.md` - -## Global Constraints - -- Work only on `test/workflow-v2-reliability`, based on `refactor/workflow-engine-v2` at `b055e74c5694e000d047335820fd333f7bd74604`. -- Do not modify `main`. -- Do not delete branches. -- Phase 1 scope is only #7–#11 reliability; do not implement finalize (#12), build (#14), migrations (#16), GitHub backend (#17), or parallel mode (#15). -- Start with tests only. Production code changes are allowed only after a reliability test fails for the intended invariant rather than fixture/test error. -- No sleeps, network access, external services, real private source, or production-only fault hooks. -- Standard execution remains `python -m unittest discover -s tests -v`; GitHub Actions is CI evidence only, not a runtime dependency. -- Read-only `status`/`resume` must be proven non-mutating. -- Idempotence means safe retry without duplicate/corrupt durable state; deterministic already-done/not-found failure is acceptable where that is the existing contract. - ---- - -## File Map - -### Create - -- `tests/test_workflow_v2_reliability.py` - - temporary repository/CLI harness; - - deterministic crash-state fixtures; - - scenarios A–H from the approved spec; - - Phase 1 idempotence matrix. - -### Modify only if a RED proves a defect - -- `scripts/workflow_v2/status.py` or `status_cli.py` — incorrect fresh-session resume/preflight composition only. -- `scripts/workflow_v2/claims.py` or `claim_cli.py` — unsafe cleanup/retry behavior only. -- `scripts/workflow_v2/reviews.py` or `review_cli.py` — PASS promotion/idempotence defect only. -- `scripts/workflow_v2/repository.py` or `filesystem.py` — proven CAS/lost-update defect only. -- `scripts/corpus.py` or source-integrity helpers — proven fresh-session corpus defect only. - ---- - -### Task 1: Build the deterministic reliability harness and claim-crash recovery - -**Files:** -- Create: `tests/test_workflow_v2_reliability.py` - -**Interfaces:** -- Consumes public commands: `book.py extract`, `claim`, `cleanup-claims`, `status`, `resume`. -- Consumes domain APIs: `FilesystemStorage`, `WorkflowStateRepository`, `SchemaKind`. -- Produces helpers reused by Tasks 2–4: - - `run_book(*args, expect=0)` - - `run_corpus(*args, expect=0)` - - `initialize_book(private=False)` - - `canonical_json(result)` - - `book_storage()` / `book_repository()` - - `authoritative_snapshot()` - - `write_claim(...)` for fixed timestamps and IDs. - -- [ ] **Step 1: Create the harness skeleton and one live-claim crash test** - -Use the same runtime-copy pattern as `tests/test_workflow_v2_status_cli.py`: copy `scripts/book.py`, `scripts/corpus.py`, and the entire `scripts/workflow_v2/` package into a temporary repository; write `.book-translator-install.json` with resolved revision `0123456789abcdef`. - -The first test must initialize a one-chapter sealed Markdown book, acquire a translator claim through the public CLI, call `resume --json` in a fresh subprocess, and assert: - -```python -self.assertEqual(payload["operation"], "blocked") -self.assertEqual(payload["reason"], "unit_claimed") -self.assertEqual(payload["unit_id"], "chapter-000001") -self.assertEqual(payload["claim"]["session_id"], "translator-crashed") -``` - -Snapshot `metadata.json`, `progress.json`, `review-ledger.json`, `source-manifest.json`, and all current claim/audit paths before and after `resume`; assert no durable content changed. - -- [ ] **Step 2: Run the focused test against current code** - -Run through the PR CI harness by committing the test-only change and opening/using a draft PR to `refactor/workflow-engine-v2` if no push workflow exists. - -Expected outcome: PASS is acceptable because this task primarily adds missing reliability coverage. If it fails, inspect the exact failure before touching production code. - -- [ ] **Step 3: Add deterministic expired-claim cleanup recovery** - -Create a schema-valid claim directly through `WorkflowStateRepository.create()` with fixed values: - -```python -{ - "schema_version": 1, - "claim_id": "1" * 32, - "unit_id": "chapter-000001", - "role": "translator", - "session_id": "translator-crashed", - "base_revision": progress_revision, - "base_commit": None, - "workflow_revision": "0123456789abcdef", - "claimed_at": "2020-01-01T00:00:00Z", - "expires_at": "2020-01-01T00:01:00Z", -} -``` - -Then call public `cleanup-claims sample --json` and assert one result with `status=cleaned`. Read `.workflow/claim-events/*` through the repository and assert exactly one `cleanup_requested` record with `reason=lease_expired` and one `cleaned` completion linked by `request_event_id`. - -Call cleanup a second time and assert canonical JSON is exactly `{"results": []}` and the claim-event path set/content is unchanged. Fresh `resume` must return `translate` for chapter 1. - -- [ ] **Step 4: Add owner-release retry safety** - -Acquire a live claim through the CLI, release it through the CLI, snapshot `.workflow/claim-events`, then call release again with the same selector/session and expect exit code 1 with deterministic `no active claim`/claim-conflict semantics. Assert the second attempt creates no audit records and does not recreate the claim. - -- [ ] **Step 5: Commit Task 1 test coverage** - -Commit only `tests/test_workflow_v2_reliability.py` unless a genuine defect required a separately demonstrated production fix. - -Suggested message: `test: cover phase1 claim crash recovery` - ---- - -### Task 2: Cover translation and review crash boundaries - -**Files:** -- Modify: `tests/test_workflow_v2_reliability.py` -- Modify production review/status owner only if a RED proves a defect. - -**Interfaces:** -- Uses Task 1 harness. -- Consumes: `WorkflowStateRepository.read/write_if_version`, `ReviewLedgerManager`, `SchemaKind.PROGRESS`, `SchemaKind.CLAIM`. -- Produces recovery coverage for crash boundaries B–E. - -- [ ] **Step 1: Add crash after translation bytes but before progress CAS** - -Initialize chapter 1 as `extracted`. Create an active translator claim fixture. Write a non-empty translation artifact directly to the chapter’s declared `translation_path` without changing `progress.json`. - -Fresh `status --json` must still report lifecycle `extracted`. Fresh `resume --json` must first return `blocked/unit_claimed`. - -Replace the active fixture with an expired equivalent claim, run audited cleanup, then call fresh `resume --json`; assert `operation=translate`, never `review`. Snapshot progress/review ledger around status/resume to prove no auto-promotion. - -- [ ] **Step 2: Add crash after lifecycle translated but before review** - -Initialize the book, create translation bytes, create active translator claim, read `progress.json` with repository version, set chapter status to `translated`, and persist with `write_if_version()`. - -Fresh status must report `translated=1` and `reviews.missing=1`; fresh resume must be blocked by the surviving translator claim. After deterministic expired-claim cleanup, fresh resume must return `operation=review` and no review-ledger record may have been fabricated. - -- [ ] **Step 3: Add crash after PASS ledger write but before reviewed promotion** - -Build a translated chapter, create a matching reviewer claim, and record PASS using the real public `review-record` command with session `reviewer-crashed`. Stop before `accept-review` or claim release. - -Fresh status must show lifecycle `translated`, review `pass`, and the active reviewer claim. Fresh resume must return `blocked/unit_claimed`. - -For recovery, create the same durable PASS state with an expired reviewer claim (or replace only the claim while preserving ledger/progress), clean it, then assert fresh resume returns `accept_review`. - -Run `accept-review sample 1 --json` twice. Assert first result has `changed=true`; second has `changed=false`. Snapshot `review-ledger.json` after first acceptance and assert the second call does not change it. Assert the second call leaves `progress.json` content and revision stable. - -- [ ] **Step 4: Add stale-review recovery and fail-closed cases** - -Case 1: leave lifecycle `translated` with current PASS, remove any active claim, change translation bytes, then assert status review state is `stale` and fresh resume returns `review`. - -Case 2: restore original bytes, accept the PASS so lifecycle becomes `reviewed`, then modify translation bytes again. Assert status is invalid with an error containing `reviewed without current PASS evidence`, and resume returns `blocked/preflight_failed`, never `complete`. - -- [ ] **Step 5: Run focused + neighboring review/status tests** - -CI/test selection must include at minimum: - -```text -test_workflow_v2_reliability.py -test_workflow_v2_reviews.py -test_workflow_v2_review_cli.py -test_workflow_v2_status.py -test_workflow_v2_status_cli.py -``` - -If the new tests pass, make no production change. If an intended invariant fails, first capture the failing test/run as RED evidence, then apply the minimum owner fix and rerun these files before full suite. - -- [ ] **Step 6: Commit Task 2** - -Suggested message if tests only: `test: cover translation and review crash recovery` - -If a production defect is found, use two audit-friendly commits: one RED test commit and one minimum GREEN fix commit. - ---- - -### Task 3: Cover shared-state CAS, corpus failures, and private-source restart - -**Files:** -- Modify: `tests/test_workflow_v2_reliability.py` -- Modify production storage/corpus owner only if a RED proves a defect. - -**Interfaces:** -- Uses Task 1 harness. -- Consumes raw `FilesystemStorage.read/write_if_version` for `glossary.md` CAS. -- Consumes existing `status`/`resume` corpus preflight and `private_external` semantics. - -- [ ] **Step 1: Add concurrent glossary CAS test** - -Create two independent `FilesystemStorage(book_dir)` instances and read `glossary.md` from both, asserting they observe the same initial version. - -Writer A calls: - -```python -new_version = storage_a.write_if_version( - "glossary.md", - b"# Glossary\n\nalpha = A\n", - first_a.version, -) -``` - -Writer B calls `write_if_version()` with its stale `first_b.version` and different bytes, and must raise `StorageVersionConflict`. - -A third fresh storage instance must read exactly writer A’s bytes and `new_version`; no writer-B bytes may appear. - -- [ ] **Step 2: Add missing extracted artifact fail-closed test** - -Initialize an explicit-source book, verify baseline `status.valid == true`, delete the chapter’s extracted file, then call fresh status/resume. - -Assert status corpus state is `invalid`; resume returns `blocked/preflight_failed`; no `.workflow/claims/` entry is created by resume. - -- [ ] **Step 3: Add tampered extracted artifact fail-closed test** - -Initialize another explicit-source book, append deterministic bytes to the extracted artifact without changing `source-manifest.json`, then assert fresh status is invalid with a hash-mismatch error and resume is `blocked/preflight_failed`. Snapshot claims/progress/review ledger before/after resume to prove no mutation. - -- [ ] **Step 4: Add private-source process-restart test** - -Initialize with `book.py extract ... --private-source`. Assert the canonical source path under `books/sample/source/` is absent. Capture source identity from `metadata.json`. - -Use two entirely new subprocesses for `status --json` and `resume --json`. Assert: - -```python -self.assertEqual(status["corpus"]["state"], "verified") -self.assertEqual(status["corpus"]["storage_mode"], "private_external") -self.assertFalse(status["corpus"]["source_attached"]) -self.assertEqual(status["corpus"]["source_sha256"], metadata["source"]["sha256"]) -self.assertEqual(status["corpus"]["source_size_bytes"], metadata["source"]["size_bytes"]) -self.assertEqual(resume["operation"], "translate") -``` - -Run status and resume twice each from unchanged state and assert canonical JSON equality plus byte-identical authoritative snapshots. - -- [ ] **Step 5: Run focused + neighboring corpus/storage tests** - -Include at minimum: - -```text -test_workflow_v2_reliability.py -test_workflow_v2_repository.py -test_workflow_v2_storage.py -test_workflow_v2_private_source.py -test_corpus_cli.py -test_workflow_v2_status_cli.py -``` - -Capture a RED only for genuine invariant failures. Fixture mistakes must be corrected before production changes. - -- [ ] **Step 6: Commit Task 3** - -Suggested message if tests only: `test: cover cas corpus and private-source recovery` - ---- - -### Task 4: Full verification, PR audit, and Phase 1 release-gate evidence - -**Files:** -- Modify: PR body only; no repository files unless verification exposes a real issue. - -**Interfaces:** -- Consumes complete Task 1–3 suite. -- Produces merge-ready Phase 1 #18 reliability evidence while leaving #18 open for later slices. - -- [ ] **Step 1: Run complete standard suite on supported Python matrix** - -Require successful CI for the final head on Python 3.10 and 3.12 using the repository’s existing `python -m unittest discover -s tests -v` workflow. - -Record total test count and run ID(s). - -- [ ] **Step 2: Audit final branch diff** - -Compare `refactor/workflow-engine-v2` to `test/workflow-v2-reliability` and verify: - -- branch is not behind integration; -- new changes are limited to the design, plan, reliability tests, and any production file with a documented RED→GREEN defect; -- no finalize/build/migration/backend/parallel implementation entered the branch; -- no private/copyrighted source fixture was committed. - -- [ ] **Step 3: Audit PR discussion/reviews/threads** - -Fetch PR comments, submitted reviews and inline review threads. Resolve any blocking finding before Ready for review. Re-run CI after any code change. - -- [ ] **Step 4: Update PR body with evidence** - -Include: - -- base/head SHA; -- scenarios A–H coverage mapping; -- idempotence matrix results; -- any RED run and matching GREEN fix run; -- final full-suite matrix run/test count; -- changed files; -- statement that #18 remains open for later #12/#14/#16/#17/#15 extensions; -- statement that `main` was not changed and no branch was deleted. - -- [ ] **Step 5: Mark Ready for review only after all gates are green** - -Do not merge the PR without the project’s integration permission gate. Do not merge to `main`. - ---- - -## Self-Review Checklist - -Before execution, verify: - -- Every approved scenario A–H maps to an explicit task step. -- Idempotence covers status, resume, cleanup, release and accept-review. -- Crash scenarios B–D correctly preserve the worker/reviewer claim before cleanup. -- Concurrent shared-state injection uses `glossary.md`, not a surrogate state document. -- No step requires sleeping or wall-clock timing. -- No step pre-authorizes a production change without a genuine RED. -- Later #18 scopes remain explicitly deferred. -- All named APIs/commands exist in the current integration-derived branch. diff --git a/docs/WORKFLOW_V2_REVIEW_LEDGER_DESIGN.md b/docs/WORKFLOW_V2_REVIEW_LEDGER_DESIGN.md deleted file mode 100644 index c45d8c7..0000000 --- a/docs/WORKFLOW_V2_REVIEW_LEDGER_DESIGN.md +++ /dev/null @@ -1,520 +0,0 @@ -# Workflow v2 — Review ledger, hashes and stale-review detection - -Issue: #9 -Branch: `feature/workflow-v2-review-ledger` -Base while #8 is pending: `feature/workflow-v2-claims-cas` -Final PR target: `refactor/workflow-engine-v2` -Date: 2026-09-06 - -## Purpose - -Make literary review evidence machine-verifiable and bind every Reviewer outcome to the exact source artifact, translation artifact, workflow revision, and durable state context that were reviewed. - -Repository state remains authoritative. `progress.json` continues to represent lifecycle only; review evidence is stored separately and is resolved against current artifact hashes before a chapter may be treated as currently reviewed. - -## Scope - -In scope: - -- authoritative `review-ledger.json` machine state; -- immutable append-only review records inside the versioned ledger; -- SHA-256 identity for source and translation artifacts; -- workflow/review-contract provenance; -- `PASS` and `CORRECTIONS_REQUIRED` evidence; -- deterministic correction-round history; -- deterministic duplicate/supersession validation; -- current/stale/missing review resolution; -- compare-and-swap ledger updates using #8 storage primitives; -- a promotion gate that permits `progress.json.status=reviewed` only with current PASS evidence; -- structural validation of ledger-enabled reviewed state; -- stable domain and CLI surfaces for #10, #12, and #21. - -Out of scope: - -- generated Markdown review reports (#21); -- deterministic status/resume/context descriptors (#10); -- atomic book finalization (#12); -- private-source portability policy (#11); -- explicit parallel translation scheduling (#15); -- workflow migration tooling (#16); -- GitHub-specific storage/backend behavior (#17). - -## Core invariants - -1. A PASS applies only to the exact source and translation bytes that were reviewed. -2. Changing either artifact makes prior evidence stale without rewriting or deleting history. -3. `progress.json` never serves as proof that review occurred. -4. A chapter cannot be promoted to `reviewed` unless current PASS evidence exists at the moment of the CAS state transition. -5. Review records are append-only. Existing records are never edited in place. -6. One deterministic current outcome is resolvable for every unit/artifact version. -7. Correction history remains auditable across translation revisions. -8. Concurrent ledger writers cannot silently overwrite each other. -9. Ledger-enabled books fail closed when authoritative review evidence is missing or malformed. -10. Handwritten Markdown audit files are never authoritative completion evidence. - -## Durable layout - -The ledger is book-local shared workflow state: - -```text -books//review-ledger.json -``` - -The ledger is a single CAS-protected document rather than one mutable file per unit. The current workflow is sequential by default, so one document keeps ordering, duplicate detection, reporting, and finalization deterministic without introducing a second coordination index. - -The logical record history is append-only even though the JSON document itself is replaced through compare-and-swap when a record is appended. - -## Ledger enablement and backward compatibility - -New books created by the #9-capable workflow opt into authoritative review evidence explicitly in metadata: - -```json -{ - "workflow": { - "review_evidence": "review-ledger-v1" - } -} -``` - -For such books, `review-ledger.json` is created during initialization with an empty record set. - -This marker prevents the current runtime from projecting #9 requirements backward onto books pinned to workflow revisions that predate the review ledger. Existing books without `workflow.review_evidence` retain the review semantics of their recorded workflow revision until an explicit migration/upgrade establishes ledger evidence. - -For a ledger-enabled book, deleting `review-ledger.json` is a validation error; absence is not interpreted as legacy mode. - -## Ledger schema - -Version 1 ledger shape: - -```json -{ - "schema_version": 1, - "book_slug": "example-book", - "next_sequence": 4, - "records": [ - { - "record_id": "0123456789abcdef0123456789abcdef", - "sequence": 1, - "unit_id": "chapter-000001", - "outcome": "CORRECTIONS_REQUIRED", - "source_sha256": "<64 lowercase hex>", - "translation_sha256": "<64 lowercase hex>", - "workflow_revision": "", - "review_contract_revision": "docs/TRANSLATION.md@", - "reviewer_session_id": "review-session-a", - "reviewed_at": "2026-09-06T00:00:00Z", - "state_revision": "", - "review_commit": null, - "correction_round": 1, - "supersedes_record_id": null - } - ] -} -``` - -Top-level rules: - -- `schema_version` is required and must be supported explicitly; -- `book_slug` must match the active book; -- `next_sequence` is a positive integer; -- `records` is an array; -- record IDs are unique; -- sequence numbers are unique and strictly increasing in stored order; -- for an empty ledger, `next_sequence == 1`; -- otherwise `next_sequence == max(sequence) + 1`. - -Record rules: - -- `record_id`: 32 lowercase hexadecimal characters; -- `sequence`: positive integer assigned by the successful CAS append; -- `unit_id`: canonical `chapter-[0-9]{6}` identity from validated progress state; -- `outcome`: exactly `PASS` or `CORRECTIONS_REQUIRED`; -- source/translation hashes: lowercase SHA-256; -- `workflow_revision`: non-empty immutable workflow provenance for the book; -- `review_contract_revision`: deterministic identifier for the exact literary review contract selected by that workflow revision; -- `reviewer_session_id`: non-empty session identity associated with the reviewer claim; -- `reviewed_at`: UTC RFC 3339 timestamp; -- `state_revision`: exact `progress.json` storage revision observed when the review evidence was recorded; -- `review_commit`: non-empty string or null; -- `correction_round`: non-negative integer; -- `supersedes_record_id`: prior record ID for the same unit or null when no earlier record exists. - -Unknown additive fields remain preserved under the existing Workflow v2 schema policy. - -## Workflow and review-contract identity - -`workflow_revision` is the immutable workflow revision recorded for the book. Ledger-enabled recording refuses to fabricate it when the book lacks immutable resolved workflow provenance. - -For the current v3 literary contract, `review_contract_revision` is: - -```text -docs/TRANSLATION.md@ -``` - -The workflow commit makes this identifier content-addressable through Git history without requiring a second runtime hash protocol. A future manifest may select another literary contract path; the identifier then uses the actual selected contract path plus the same immutable workflow revision. - -The review contract identifier is evidence metadata. Current-review resolution requires it to match the contract expected by the book's recorded workflow revision. - -## Artifact identity - -The review domain computes hashes from canonical files; callers do not provide trusted artifact hashes. - -For one chapter, resolution uses the `source_path` and `translation_path` from validated `progress.json` and computes SHA-256 over exact file bytes. - -Rules: - -- missing source is a structural error; -- missing or empty translation cannot receive review evidence; -- source and translation paths must already satisfy Workflow v2 safe relative-path rules; -- text normalization, newline normalization, Unicode normalization, or Markdown parsing is not performed before hashing; -- any byte change produces a new artifact identity and invalidates prior current evidence. - -This deliberately makes stale-review detection mechanical rather than interpretive. - -## Reviewer claim boundary - -Recording a review outcome requires an active claim for the same unit with: - -- `role=reviewer`; -- matching `reviewer_session_id` / claim `session_id`; -- matching book workflow revision. - -The review writer reads and validates the claim immediately before appending evidence. An absent, expired-but-not-cleaned, foreign-session, wrong-role, or changed claim blocks recording. - -Lease expiry alone does not transfer ownership; #8 cleanup rules remain authoritative. - -The Orchestrator may persist the Reviewer result on behalf of the worker, but it must present the reviewer's session identity and current reviewer claim. This preserves the logical Reviewer/Orchestrator boundary without allowing unclaimed PASS fabrication. - -## Append and compare-and-swap flow - -`ReviewLedger` / `ReviewLedgerManager` uses `WorkflowStateRepository` and #8 CAS primitives. - -For each record append: - -1. load and validate metadata, progress, and canonical unit identity; -2. verify the active reviewer claim and session ownership; -3. read source/translation bytes and compute current hashes; -4. read `review-ledger.json` and retain its storage revision; -5. validate the complete ledger, including sequence/ID/supersession invariants; -6. derive the next record, correction round, and supersession link; -7. append exactly one immutable record in memory using `next_sequence`; -8. write the whole ledger using `write_if_version(expected_revision)`; -9. if the ledger changed concurrently, fail with a conflict; do not silently retry with the stale Reviewer result. - -For a correctly initialized ledger-enabled book, the ledger already exists. A legacy/migration path that creates a ledger is outside ordinary review recording and belongs to explicit upgrade/migration work. - -The caller may retry only after re-reading current repository state and re-validating that the Reviewer result still applies. - -## Deterministic supersession - -Each unit has one linear review history. - -For a newly appended record: - -- `supersedes_record_id` is null only when the unit has no prior record; -- otherwise it must point to the immediately preceding record for that same unit by sequence; -- the target must exist, belong to the same unit, and have a smaller sequence; -- records may not fork the supersession chain; -- a record may not supersede itself or a record from another unit. - -The ledger validator rejects duplicate record IDs, duplicate sequences, broken supersession targets, forks, non-monotonic stored ordering, and inconsistent `next_sequence`. - -Because the current record is always the highest-sequence record for the relevant artifact identity, duplicate/superseded history cannot yield two ambiguous current PASS states. - -## Current review resolution - -Resolution is computed; it is not stored as a second mutable summary. - -For each canonical unit, the resolver computes: - -- current source SHA-256; -- current translation SHA-256, or translation absence; -- expected workflow revision; -- expected review-contract revision; -- unit review history ordered by sequence. - -A record is an exact current-artifact match only when all of these match current state: - -- `unit_id`; -- `source_sha256`; -- `translation_sha256`; -- `workflow_revision`; -- `review_contract_revision`. - -Among exact matches, the highest `sequence` determines the current outcome. - -Resolution states: - -- `pass`: exact current-artifact record exists and latest exact outcome is `PASS`; -- `corrections_required`: exact current-artifact record exists and latest exact outcome is `CORRECTIONS_REQUIRED`; -- `stale`: the unit has review history but no exact record matches current source/translation/workflow/contract identity; -- `missing`: the unit has no review history; -- `untranslated`: no canonical non-empty translation artifact exists, so review coverage cannot exist. - -Older exact-match records remain history. A later `CORRECTIONS_REQUIRED` for the same artifact supersedes an earlier PASS and removes current PASS coverage. A later PASS restores coverage for that exact artifact. - -## Stale review semantics - -No watcher or state rewrite is required when an artifact changes. - -Example: - -1. translation hash `A` receives PASS; -2. `progress.json` may be promoted to `reviewed` after the PASS gate; -3. translation file changes to hash `B`; -4. resolver finds no exact PASS for `B`; -5. current review state is `stale`; -6. structural validation rejects `status=reviewed` until the new artifact is reviewed and promoted again. - -The historical PASS for hash `A` remains in the ledger as audit evidence but contributes zero current coverage. - -The same rule applies when the source hash or review contract identity changes. - -## Correction-round semantics - -`correction_round` counts correction cycles entered for one unit. - -Deterministic rules: - -- a first PASS with no prior `CORRECTIONS_REQUIRED` uses round `0`; -- the first `CORRECTIONS_REQUIRED` record uses round `1`; -- subsequent review records after that correction request remain round `1` until another `CORRECTIONS_REQUIRED` begins a new correction cycle; -- each later `CORRECTIONS_REQUIRED` increments the previous maximum correction round by one; -- a PASS never increments the round. - -Thus a typical sequence is: - -```text -CORRECTIONS_REQUIRED round=1 hash=A -PASS round=1 hash=B -CORRECTIONS_REQUIRED round=2 hash=B -PASS round=2 hash=C -``` - -All records remain linked through `supersedes_record_id`. - -## Review commit and state revision - -`state_revision` records the exact `progress.json` storage revision observed when evidence was appended. It is audit context, not by itself a validity key: unrelated progress changes must not make an otherwise exact source/translation PASS stale. - -`review_commit` records the relevant Git commit when available. CLI resolution is explicit `--review-commit` first, then best-effort repository `HEAD`; absence is stored as null. - -Artifact hashes and workflow/contract identity determine current review validity. State revision and commit provide traceability. - -## Promotion to `reviewed` - -`progress.json` remains lifecycle state only. - -The only supported #9 promotion path is a domain operation exposed by `book.py accept-review`. - -For one unit it: - -1. reads metadata, progress, ledger, source, and translation from current repository state; -2. requires the chapter lifecycle state to be `translated` or already `reviewed` for an idempotent no-op check; -3. resolves current review evidence; -4. requires resolution state `pass`; -5. verifies the PASS still matches current artifact/workflow/contract identity; -6. changes only that chapter status to `reviewed` in memory; -7. writes `progress.json` with `write_if_version` against the exact revision read in step 1; -8. on conflict, leaves current state unchanged and requires a fresh retry. - -The operation does not modify the ledger. - -If the chapter is already `reviewed` and current PASS remains valid, `accept-review` is idempotent. If it is `reviewed` but evidence is stale/missing, the command fails rather than claiming success. - -A `CORRECTIONS_REQUIRED` record never promotes lifecycle state. - -## Validation behavior - -For a book with `metadata.workflow.review_evidence == review-ledger-v1`, `book.py validate` additionally requires: - -- `review-ledger.json` exists and passes schema/domain validation; -- ledger `book_slug` matches metadata/progress book identity; -- every `status=reviewed` chapter resolves to current `pass` evidence; -- a stale, missing, corrections-required, untranslated, malformed, or ambiguous ledger state blocks structural validity; -- handwritten Markdown review/audit files are ignored for authoritative coverage. - -Validation does not automatically rewrite a stale reviewed chapter to `translated`. It reports the inconsistency and stops state advancement. The Orchestrator can then explicitly restore the correct lifecycle state and re-review as required. - -Books without the review-evidence marker continue to use the semantics of their pinned legacy workflow and are not silently migrated by current `book.py validate`. - -## Domain API - -A focused module is added under `scripts/workflow_v2/reviews.py`. - -Primary domain types/functions: - -```text -ReviewLedgerManager.record(...) -ReviewLedgerManager.resolve_unit(...) -ReviewLedgerManager.resolve_all(...) -ReviewLedgerManager.accept_review(...) -ReviewResolution -ReviewRecordResult -ReviewError -ReviewConflict -ReviewClaimError -ReviewEvidenceError -``` - -The manager depends on repository/storage abstractions and filesystem artifact reads supplied by the book/runtime adapter. It does not depend on GitHub APIs or chat state. - -`resolve_all()` is the stable machine-backed input for #10, #12, and #21. Consumers do not parse Markdown audit files or duplicate review-resolution rules. - -## CLI surface - -`book.py` exposes: - -```bash -python scripts/book.py review-record \ - --outcome PASS|CORRECTIONS_REQUIRED \ - --session-id \ - [--review-commit ] \ - [--json] - -python scripts/book.py reviews [--json] - -python scripts/book.py accept-review [--json] -``` - -`review-record`: - -- resolves exactly one chapter selector in #9; -- requires active matching reviewer claim; -- hashes canonical source/translation artifacts itself; -- appends one CAS-protected record; -- emits record identity, sequence, hashes, outcome, correction round, and ledger revision. - -`reviews`: - -- performs no mutation; -- returns deterministic canonical-unit order; -- reports current resolution plus sufficient record history/identifiers for generated reporting; -- JSON output is stable and machine-readable. - -`accept-review`: - -- performs the guarded lifecycle promotion described above; -- uses CAS on `progress.json`; -- returns the accepted unit and resulting progress revision; -- is idempotent only when existing `reviewed` state still has current PASS evidence. - -## Orchestration flow - -The normal sequential chapter path becomes: - -```text -translator claim - -> translation artifact - -> progress translated - -> release translator claim - -> reviewer claim - -> source-comparison review - -> review-record CORRECTIONS_REQUIRED - -> release reviewer claim - -> correction flow - -> reviewer claim again - -> fresh review - -> review-record PASS - -> accept-review - -> release reviewer claim - -> next chapter -``` - -The exact role transitions remain governed by `docs/ORCHESTRATION.md` and literary criteria by `docs/TRANSLATION.md`. - -A reviewer PASS returned in chat is not authoritative until it is recorded in the ledger and accepted through the promotion gate. - -## Error handling - -Fail closed on: - -- missing/malformed ledger for ledger-enabled books; -- missing immutable workflow provenance; -- invalid/missing artifact paths; -- empty/missing translation; -- foreign/missing/wrong-role reviewer claim; -- malformed review record/history; -- duplicate record IDs/sequences; -- broken/forked supersession chains; -- ledger CAS conflict; -- progress CAS conflict during promotion; -- stale/missing/current `CORRECTIONS_REQUIRED` evidence during promotion. - -No failure path silently rewrites another session's ledger/progress state. - -## Concurrency boundary - -The ledger is shared mutable state and uses the strong CAS semantics established by #8. - -Two writers may read the same ledger revision, but only one may commit a replacement for that revision. The loser receives `StorageVersionConflict` / a review-domain conflict and must re-read state before deciding whether the Reviewer result is still valid. - -No automatic blind retry is performed because a concurrent review record may change the authoritative current outcome or correction round. - -The single-ledger design intentionally serializes review-record commits. This is acceptable under the current sequential policy and remains correct if future parallel review is enabled; throughput optimization can later replace the storage layout behind the same resolver API if measurements justify it. - -## Initialization changes - -For books initialized by the #9-capable workflow: - -- metadata adds `workflow.review_evidence = review-ledger-v1`; -- `review-ledger.json` is created with: - -```json -{ - "schema_version": 1, - "book_slug": "", - "next_sequence": 1, - "records": [] -} -``` - -Initialization remains deterministic and does not create handwritten audit files. - -## Testing strategy - -TDD coverage must include: - -- strict ledger/record schema validation; -- unique IDs/sequences and `next_sequence` invariants; -- valid linear supersession and rejection of broken/forked chains; -- deterministic correction-round derivation; -- PASS for exact source/translation hashes; -- translation change -> stale; -- source change -> stale; -- workflow/review-contract change -> stale; -- later `CORRECTIONS_REQUIRED` supersedes an earlier PASS for the same artifact; -- later PASS restores current coverage; -- missing translation -> untranslated / no record allowed; -- foreign, missing, expired-but-not-cleaned, and wrong-role reviewer claims block recording; -- concurrent ledger writers from one expected revision -> exactly one succeeds; -- CAS conflict is surfaced without blind retry; -- missing/stale/mismatched PASS blocks `accept-review`; -- current PASS permits CAS promotion; -- progress CAS conflict leaves state unmodified; -- already-reviewed + current PASS is idempotent; -- already-reviewed + stale evidence fails; -- ledger-enabled `validate` rejects reviewed state without current PASS; -- legacy books without the marker preserve prior validation behavior; -- CLI record/list/accept JSON is deterministic; -- review coverage requires no Markdown audit parsing; -- full Python 3.10 and 3.12 regression suites remain green. - -## Acceptance mapping - -Issue #9 acceptance criteria map directly: - -- **Changing reviewed translation invalidates review:** exact-byte hash resolution becomes `stale`. -- **Coverage without Markdown:** `resolve_all()` computes machine coverage from ledger + current artifacts. -- **No reviewed promotion without PASS:** `accept-review` requires exact current PASS and CAS-updates progress. -- **Correction rounds auditable:** immutable records preserve round, hashes, sequence, state revision, commit, and supersession. -- **No ambiguous current PASS:** unique sequence plus linear supersession and highest-sequence exact-match resolution. -- **Sufficient for #21:** resolver exposes current/stale/missing outcome, hashes, workflow/contract provenance, review revision/commit, and history identifiers. - -## Branching and integration - -Development occurs on `feature/workflow-v2-review-ledger` stacked on `feature/workflow-v2-claims-cas` while #24/#8 is pending. - -The initial PR should target `feature/workflow-v2-claims-cas` so its diff is #9-only. After #23 and #24 are integrated into `refactor/workflow-engine-v2`, retarget #9 to the integration branch and rerun the complete CI matrix before merge. - -Do not merge #9 before its #7/#8 dependencies are integrated and the PR has been retargeted/reverified. \ No newline at end of file diff --git a/docs/WORKFLOW_V2_REVIEW_LEDGER_PLAN.md b/docs/WORKFLOW_V2_REVIEW_LEDGER_PLAN.md deleted file mode 100644 index d47a88e..0000000 --- a/docs/WORKFLOW_V2_REVIEW_LEDGER_PLAN.md +++ /dev/null @@ -1,511 +0,0 @@ -# Workflow v2 Review Ledger Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Implement issue #9 so Reviewer outcomes become immutable, hash-bound machine evidence, stale reviews are detected from current artifacts, and `progress.json` can reach `reviewed` only through a current PASS gate. - -**Architecture:** Keep one book-local `review-ledger.json` as a CAS-protected append-only logical history. Put review semantics in a backend-neutral `ReviewLedgerManager` that receives canonical progress/metadata documents plus an injected artifact reader, reuses #8 claim ownership and storage CAS primitives, computes exact SHA-256 identities itself, resolves current/stale/missing evidence deterministically, and exposes a guarded progress promotion operation. `book.py` remains an adapter for initialization, filesystem artifact reads, CLI parsing, validation, and best-effort Git commit provenance. - -**Tech Stack:** Python 3.10+, standard library only (`hashlib`, `datetime`, `uuid`, `json`, `argparse`, `unittest`), existing Workflow v2 schema/repository/storage/claim modules. - -**Spec:** `docs/WORKFLOW_V2_REVIEW_LEDGER_DESIGN.md` - -## Global Constraints - -- Repository state is authoritative over chat history. -- `progress.json` is lifecycle state only and never proves review coverage. -- New #9-capable books set `metadata.workflow.review_evidence = "review-ledger-v1"` and create `review-ledger.json` with `next_sequence = 1` and no records. -- Existing books without the marker retain prior workflow semantics and are not silently migrated. -- Ledger-enabled review recording requires immutable `metadata.workflow.resolved_revision`; requested refs are not accepted as immutable review provenance. -- Review records are append-only logical evidence; existing records are never rewritten individually. -- Source and translation hashes are SHA-256 over exact canonical file bytes; no normalization is permitted. -- Recording requires a live matching reviewer claim for the same canonical unit and session. Expired claims cannot record evidence and remain occupied until #8 cleanup. -- A ledger CAS conflict is surfaced and never blindly retried with the old Reviewer result. -- `status=reviewed` is valid for ledger-enabled books only when current exact PASS evidence exists. -- No generated report (#21), status/resume (#10), finalize (#12), migration (#16), GitHub backend (#17), or parallel scheduling (#15) is implemented here. -- Python 3.10 and 3.12 complete suites must pass before the stacked PR is review-ready. - ---- - -### Task 1: Strict review-ledger schema and new-book initialization - -**Files:** -- Modify: `scripts/workflow_v2/schemas.py` -- Modify: `scripts/book.py` -- Modify: `tests/test_workflow_v2_schemas.py` -- Modify: `tests/test_book_cli.py` - -**Interfaces:** -- Consumes: `SchemaKind.REVIEW_LEDGER`, `SCHEMA_VERSION`, existing metadata/progress repository writes. -- Produces: strict `REVIEW_LEDGER` validation and deterministic ledger initialization for new books. - -- [ ] **Step 1: Write failing schema tests** - -Require a complete ledger: - -```python -ledger = { - "schema_version": 1, - "book_slug": "sample", - "next_sequence": 2, - "records": [{ - "record_id": "00000000000000000000000000000001", - "sequence": 1, - "unit_id": "chapter-000001", - "outcome": "PASS", - "source_sha256": "a" * 64, - "translation_sha256": "b" * 64, - "workflow_revision": "0123456789abcdef", - "review_contract_revision": "docs/TRANSLATION.md@0123456789abcdef", - "reviewer_session_id": "reviewer-a", - "reviewed_at": "2026-09-06T00:00:00Z", - "state_revision": "progress-revision", - "review_commit": None, - "correction_round": 0, - "supersedes_record_id": None, - }], -} -self.assertEqual(parse_document(SchemaKind.REVIEW_LEDGER, ledger).data, ledger) -``` - -Add invalid cases for duplicate IDs, duplicate/non-increasing sequences, inconsistent `next_sequence`, invalid hashes/unit/outcome/timestamp, negative correction round, broken supersession target, cross-unit supersession, forked chains, and a first record that incorrectly supersedes another record. - -- [ ] **Step 2: Write failing extraction tests** - -Extend `test_extract_markdown_creates_complete_book_state` / provenance coverage to require: - -```python -metadata = json.loads((book / "metadata.json").read_text()) -self.assertEqual(metadata["workflow"]["review_evidence"], "review-ledger-v1") -ledger = json.loads((book / "review-ledger.json").read_text()) -self.assertEqual(ledger, { - "schema_version": 1, - "book_slug": "sample", - "next_sequence": 1, - "records": [], -}) -``` - -Extraction must still succeed when install provenance is absent; the marker/empty ledger are created, while later review recording will fail until immutable workflow provenance exists. - -- [ ] **Step 3: Run focused tests and verify RED** - -Run through CI: - -```text -python -m unittest tests.test_workflow_v2_schemas tests.test_book_cli -v -``` - -Expected: failures because the current ledger validator only checks `book_slug`/`records`, and extraction does not create the marker/ledger. - -- [ ] **Step 4: Implement strict ledger validation** - -In `_validate_review_ledger`, validate every record and then domain-shape invariants in stored order. Use existing `_validate_sha256`, `_validate_unit_id`, `_parse_utc_timestamp`, `_require_*` helpers. Track: - -```python -record_ids: set[str] = set() -sequences: set[int] = set() -last_by_unit: dict[str, str] = {} -superseded_by: dict[str, str] = {} -``` - -For each record require `supersedes_record_id == last_by_unit.get(unit_id)`. Reject any target already present in `superseded_by`, then update `last_by_unit[unit_id] = record_id`. Require `next_sequence == 1` for empty history and `next_sequence == records[-1]["sequence"] + 1` otherwise. - -- [ ] **Step 5: Initialize ledger-enabled books** - -In `extract_command`, extend the workflow dictionary before metadata serialization: - -```python -workflow = workflow_provenance() -workflow["review_evidence"] = "review-ledger-v1" -``` - -Create metadata, progress, and ledger through `WorkflowStateRepository`; ledger creation occurs before support files are reported as complete: - -```python -repository.create("review-ledger.json", SchemaKind.REVIEW_LEDGER, { - "schema_version": SCHEMA_VERSION, - "book_slug": slug, - "next_sequence": 1, - "records": [], -}) -``` - -- [ ] **Step 6: Run focused tests and verify GREEN** - -```text -python -m unittest tests.test_workflow_v2_schemas tests.test_book_cli -v -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -```text -workflow: define and initialize review ledger -``` - ---- - -### Task 2: Review recording and deterministic current-evidence resolution - -**Files:** -- Create: `scripts/workflow_v2/reviews.py` -- Modify: `scripts/workflow_v2/__init__.py` -- Create: `tests/test_workflow_v2_reviews.py` - -**Interfaces:** -- Consumes: `WorkflowStateRepository`, `SchemaKind.REVIEW_LEDGER`, #8 claim files, `canonical_unit_id`, exact progress revision, injected `artifact_reader(path: str) -> bytes`, injected UTC clock and ID factory. -- Produces: - - `ReviewLedgerManager.record(progress, progress_revision, metadata, chapter_number, *, outcome, reviewer_session_id, review_commit=None) -> ReviewRecordResult` - - `ReviewLedgerManager.resolve_unit(progress, metadata, chapter_number) -> ReviewResolution` - - `ReviewLedgerManager.resolve_all(progress, metadata) -> list[ReviewResolution]` - - errors `ReviewError`, `ReviewConflict`, `ReviewClaimError`, `ReviewEvidenceError`. - -- [ ] **Step 1: Write failing resolver tests** - -Use a temp repository, deterministic artifact reader, fixed clock/IDs, one reviewer claim, and initialized ledger. Require: - -```python -result = manager.record(... outcome="PASS", reviewer_session_id="reviewer-a") -resolution = manager.resolve_unit(progress, metadata, 1) -self.assertEqual(resolution.state, "pass") -self.assertEqual(resolution.source_sha256, sha256(source_bytes).hexdigest()) -self.assertEqual(resolution.translation_sha256, sha256(translation_bytes).hexdigest()) -``` - -Then mutate only translation bytes and require `state == "stale"`; restore exact prior bytes and require the exact prior PASS to become current again. Repeat with changed source bytes and changed expected review-contract/workflow identity. - -Also require `missing` when there is no unit history and `untranslated` when the canonical translation file is missing or empty. - -- [ ] **Step 2: Write failing record/claim tests** - -Cover: - -- caller-provided hashes do not exist in the API; -- missing `metadata.workflow.resolved_revision` raises `ReviewEvidenceError`; -- missing reviewer claim raises `ReviewClaimError`; -- translator claim raises `ReviewClaimError`; -- foreign reviewer session raises `ReviewClaimError`; -- expired reviewer claim raises `ReviewClaimError` without deleting the claim; -- claim workflow revision mismatch raises `ReviewClaimError`; -- missing/empty translation refuses record creation; -- first PASS uses correction round `0`; -- first `CORRECTIONS_REQUIRED` uses round `1`; -- PASS after correction retains round `1`; -- later `CORRECTIONS_REQUIRED` increments to round `2`; -- `supersedes_record_id` always links the immediately preceding unit record. - -- [ ] **Step 3: Run focused tests and verify RED** - -```text -python -m unittest tests.test_workflow_v2_reviews -v -``` - -Expected: import/API failures because `reviews.py` does not exist. - -- [ ] **Step 4: Implement focused review-domain types** - -Define immutable dataclasses: - -```python -@dataclass(frozen=True) -class ReviewRecordResult: - record: dict[str, Any] - ledger_revision: str - -@dataclass(frozen=True) -class ReviewResolution: - unit_id: str - chapter_number: int - state: str # pass|corrections_required|stale|missing|untranslated - source_sha256: str | None - translation_sha256: str | None - current_record: dict[str, Any] | None - history: tuple[dict[str, Any], ...] -``` - -Constructor: - -```python -ReviewLedgerManager( - repository, - *, - artifact_reader, - now=None, - id_factory=None, -) -``` - -The manager validates chapter identity from progress and reads canonical artifact paths only through `artifact_reader`. - -- [ ] **Step 5: Implement immutable provenance and claim validation** - -`_workflow_revision(metadata)` accepts only non-empty `metadata["workflow"]["resolved_revision"]`. Expected review contract is `docs/TRANSLATION.md@` for this v3 contract. - -`_require_reviewer_claim(unit_id, session_id, workflow_revision)` reads `.workflow/claims/.json`, requires role reviewer, matching session/workflow, and requires `expires_at > now`. Expired-but-present ownership remains a claim conflict for others under #8 but cannot authorize a new review record. - -- [ ] **Step 6: Implement artifact identity and resolution** - -Hash exact bytes: - -```python -def _sha256(content: bytes) -> str: - return hashlib.sha256(content).hexdigest() -``` - -Resolve all unit history sorted by sequence. Exact matches require unit/source hash/translation hash/workflow/review contract identity. Highest-sequence exact match determines `pass` vs `corrections_required`; history with no exact match is `stale`. Missing/empty translation is `untranslated`. - -- [ ] **Step 7: Implement CAS record append** - -Read and validate the ledger with its exact revision; derive the record ID, sequence, supersession, and correction round; append one record; call: - -```python -new_revision = repository.write_if_version( - "review-ledger.json", - SchemaKind.REVIEW_LEDGER, - new_ledger, - loaded.version, -) -``` - -Convert `StorageVersionConflict` to `ReviewConflict`. Never retry internally. - -- [ ] **Step 8: Add deterministic concurrent-writer test** - -Use a barrier around the first ledger read in two threads/processes so both writers start from one ledger revision. Require exactly one successful append and one `ReviewConflict`, with one durable new record. - -- [ ] **Step 9: Run focused tests and verify GREEN** - -```text -python -m unittest tests.test_workflow_v2_reviews tests.test_workflow_v2_claims tests.test_workflow_v2_storage -v -``` - -Expected: PASS. - -- [ ] **Step 10: Commit** - -```text -workflow: add hash-bound review evidence -``` - ---- - -### Task 3: Guarded reviewed promotion and ledger-aware structural validation - -**Files:** -- Modify: `scripts/workflow_v2/reviews.py` -- Modify: `scripts/book.py` -- Modify: `tests/test_workflow_v2_reviews.py` -- Modify: `tests/test_book_cli.py` - -**Interfaces:** -- Consumes: `ReviewLedgerManager.resolve_unit`, exact progress revision, `WorkflowStateRepository.write_if_version`. -- Produces: `ReviewLedgerManager.accept_review(...) -> AcceptReviewResult` and ledger-aware `book.py validate` behavior. - -- [ ] **Step 1: Write failing promotion tests** - -Require: - -```python -accepted = manager.accept_review(progress, progress_revision, metadata, 1) -self.assertEqual(accepted.unit_id, "chapter-000001") -self.assertEqual(accepted.status, "reviewed") -``` - -Cases: - -- no PASS -> `ReviewEvidenceError`; -- current `CORRECTIONS_REQUIRED` -> error; -- stale translation/source -> error; -- chapter not `translated`/`reviewed` -> error; -- current PASS promotes only selected chapter; -- stale progress revision -> `ReviewConflict`, no mutation; -- already reviewed + current PASS -> idempotent and returns existing progress revision without rewriting; -- already reviewed + stale evidence -> error. - -- [ ] **Step 2: Write failing validation tests** - -For a ledger-enabled book: - -- missing `review-ledger.json` is invalid; -- malformed ledger is invalid; -- `status=reviewed` with no current PASS is invalid; -- exact PASS + reviewed status validates; -- editing the reviewed translation makes `validate` fail as stale; -- a book without `workflow.review_evidence` retains legacy validation behavior and is not required to contain a ledger. - -- [ ] **Step 3: Run focused tests and verify RED** - -```text -python -m unittest tests.test_workflow_v2_reviews tests.test_book_cli -v -``` - -Expected: missing `accept_review` and ledger-aware validation behavior. - -- [ ] **Step 4: Implement `accept_review`** - -Define: - -```python -@dataclass(frozen=True) -class AcceptReviewResult: - unit_id: str - status: str - progress_revision: str - changed: bool -``` - -Require current exact `pass`, then deep-copy progress, set only selected chapter to `reviewed`, and write via exact CAS. Already-reviewed/current-PASS returns `changed=False` and does not rewrite. - -- [ ] **Step 5: Add a reusable ledger-validation adapter in `book.py`** - -For `workflow.get("review_evidence") == "review-ledger-v1"`, instantiate `ReviewLedgerManager` with a safe book-relative artifact reader. Read/validate ledger and call `resolve_all`. For every chapter already marked `reviewed`, require corresponding state `pass`; append deterministic validation errors otherwise. - -Do not require all translated chapters to already have PASS; only claimed `reviewed` lifecycle state must be backed by current evidence. Coverage completeness belongs to #12/#21. - -- [ ] **Step 6: Run focused tests and verify GREEN** - -```text -python -m unittest tests.test_workflow_v2_reviews tests.test_book_cli -v -``` - -Expected: PASS. - -- [ ] **Step 7: Commit** - -```text -workflow: gate reviewed state on current pass -``` - ---- - -### Task 4: Review CLI surface - -**Files:** -- Create: `scripts/workflow_v2/review_cli.py` -- Modify: `scripts/book.py` -- Modify: `scripts/workflow_v2/__init__.py` -- Create: `tests/test_workflow_v2_review_cli.py` - -**Interfaces:** -- Consumes: `ReviewLedgerManager.record`, `resolve_all`, `accept_review`, `FilesystemStorage`, exact book-relative artifact reader. -- Produces: - - `book.py review-record --outcome ... --session-id ... [--review-commit ...] [--json]` - - `book.py reviews [--json]` - - `book.py accept-review [--json]`. - -- [ ] **Step 1: Write failing end-to-end CLI tests** - -Initialize a book with immutable install provenance, create a translation, set chapter state to translated, acquire a reviewer claim through the existing claim CLI, then require: - -```text -review-record ... PASS --json -> exit 0, sequence/hash fields returned -reviews --json -> current state pass in canonical unit order -accept-review --json -> exit 0, progress becomes reviewed -edit translation -reviews --json -> state stale -accept-review --json -> non-zero and reviewed state is not falsely accepted -``` - -Also cover `CORRECTIONS_REQUIRED`, missing resolved workflow revision, foreign/non-reviewer claim, explicit `--review-commit`, and deterministic JSON key/unit/history ordering. - -- [ ] **Step 2: Run CLI tests and verify RED** - -```text -python -m unittest tests.test_workflow_v2_review_cli -v -``` - -Expected: argparse unknown commands / missing module. - -- [ ] **Step 3: Implement CLI adapter** - -Keep path/Git/argparse concerns out of `reviews.py`. `review_cli.py` loads metadata/progress with exact progress revision, creates an artifact reader rooted at the validated book directory, resolves best-effort `git rev-parse HEAD` only when `--review-commit` is omitted, maps review/schema/storage errors to `ReviewCliError`, and prints JSON with `sort_keys=True`. - -Only a single positive chapter number is accepted for `review-record` and `accept-review` in #9; ranges remain out of scope. - -- [ ] **Step 4: Register commands in `book.py`** - -Import `ReviewCliError, register_review_commands`, call `register_review_commands(subparsers, repo_root())`, and catch `ReviewCliError` alongside existing `BookError`/`ClaimCliError`. - -- [ ] **Step 5: Run CLI and regression tests and verify GREEN** - -```text -python -m unittest tests.test_workflow_v2_review_cli tests.test_workflow_v2_claim_cli tests.test_book_cli -v -``` - -Expected: PASS. - -- [ ] **Step 6: Commit** - -```text -workflow: expose machine review ledger commands -``` - ---- - -### Task 5: Execution-contract alignment, scope review, and full verification - -**Files:** -- Modify: `docs/ORCHESTRATION.md` -- Modify: `tests/test_agent_contract.py` -- Verify: all `tests/` - -**Interfaces:** -- Consumes: final #9 review CLI and promotion semantics. -- Produces: authoritative orchestration language requiring recorded current PASS before durable reviewed state. - -- [ ] **Step 1: Write failing contract test** - -Require exact executable surfaces and state boundary: - -```python -for phrase in ( - "python scripts/book.py review-record", - "python scripts/book.py reviews", - "python scripts/book.py accept-review", - "current pass", - "stale", -): - self.assertIn(phrase, orchestration.lower()) -``` - -Also assert the contract does not claim Markdown audit files as authoritative review coverage. - -- [ ] **Step 2: Run contract test and verify RED** - -```text -python -m unittest tests.test_agent_contract -v -``` - -Expected: missing #9 command/evidence contract phrases. - -- [ ] **Step 3: Update `docs/ORCHESTRATION.md`** - -Document the sequence reviewer claim -> Reviewer outcome -> `review-record` -> `accept-review` -> release. State explicitly that PASS in chat is not durable evidence, changed artifacts make prior evidence stale, and a ledger-enabled reviewed chapter must resolve to current exact PASS. Preserve literary criteria in `docs/TRANSLATION.md` and keep #10/#12/#21 behavior out of this contract change. - -- [ ] **Step 4: Run the complete suite** - -```text -python -m unittest discover -s tests -v -``` - -Expected: all tests pass on Python 3.10 and 3.12. - -- [ ] **Step 5: Inspect diff against `feature/workflow-v2-claims-cas`** - -Confirm only #9 spec/plan, review schema/domain/CLI, targeted book initialization/validation, tests, and orchestration documentation changed. Verify no generated review report, status/resume, finalize, migration, GitHub backend, database/queue, or parallel scheduling implementation was introduced. - -- [ ] **Step 6: Review production patches against acceptance criteria** - -Manually inspect schema validation, claim expiry/ownership checks, exact-byte artifact hashing, sequence/supersession/correction-round logic, CAS error conversion, already-reviewed idempotence, legacy marker behavior, and deterministic CLI output. Add regression tests for any uncovered correctness risk before declaring completion. - -- [ ] **Step 7: Open/update stacked PR and verify final CI** - -Initially target `feature/workflow-v2-claims-cas` so the diff remains #9-only. PR body must state that #23/#24 must integrate first; after dependencies land, retarget to `refactor/workflow-engine-v2` and rerun the full Python 3.10/3.12 matrix before merge. - -- [ ] **Step 8: Commit final contract alignment** - -```text -docs: bind orchestration to review ledger evidence -``` diff --git a/docs/WORKFLOW_V2_REVIEW_REPORT_PLAN.md b/docs/WORKFLOW_V2_REVIEW_REPORT_PLAN.md deleted file mode 100644 index e2cd3c9..0000000 --- a/docs/WORKFLOW_V2_REVIEW_REPORT_PLAN.md +++ /dev/null @@ -1,71 +0,0 @@ -# Workflow v2 Generated Review Report — Minimal Plan - -Issue: #21 -Branch: `feature/workflow-v2-review-report` -Base: `refactor/workflow-engine-v2` at `80c3fce90ac812c78aaf45a892ad4f648c7469ef` -Target: `refactor/workflow-engine-v2` - -## Goal - -Generate one deterministic `REVIEW_REPORT.md` from authoritative Workflow v2 review-ledger/progress/artifact state so review coverage no longer depends on handwritten `REVIEW_AUDIT_.md` files. - -## Design boundary - -- Add backend-neutral `workflow_v2.review_report` snapshot/rendering logic. -- Reuse `ReviewLedgerManager.resolve_all()` for current review state and the validated `review-ledger.json` history for audit details. -- Add `book.py review-report ` through the existing review CLI adapter. -- Default mode atomically updates `books//REVIEW_REPORT.md` and prints a concise summary. -- `--json` prints deterministic machine-readable snapshot data and does not write the Markdown report. -- `REVIEW_REPORT.md` is generated evidence, never authoritative state. Ledger, progress, current artifact bytes and workflow revision remain authoritative. -- Future #12 finalize must consume the same snapshot builder directly; #21 does not implement finalize. -- Handwritten `REVIEW_AUDIT_*` files are neither read nor required. - -## Record classification - -For each unit, history records are preserved in ledger sequence order. - -- `current`: the record selected by `ReviewLedgerManager.resolve_unit()` for current artifact/workflow identity. -- `stale`: record identity no longer matches current source/translation/workflow/contract identity. -- `superseded`: identity still matches but a later record for the same unit is current. -- `duplicate`: a later record repeats the same unit + source hash + translation hash + workflow revision + review-contract revision + outcome as an earlier record. It remains visible in history but does not add coverage. -- current `CORRECTIONS_REQUIRED` is reported as failed coverage (`corrections_required`). - -## Acceptance criteria - -1. `book.py review-report ` creates deterministic `books//REVIEW_REPORT.md` for unchanged state. -2. `book.py review-report --json` emits deterministic JSON and does not mutate the report file. -3. Summary contains total units and counts for `pass`, `corrections_required`, `missing`, `stale`, and `untranslated`, plus PASS coverage percentage/count. -4. 100% PASS coverage is computable entirely from ledger/progress/current artifact data; handwritten audit files are ignored. -5. Every unit row contains current state, source SHA-256, current translation SHA-256 when present, and current review revision/commit when a current record exists. -6. Stale records are visibly marked and never count toward PASS coverage. -7. Current `CORRECTIONS_REQUIRED` is visibly marked and never counts toward PASS coverage. -8. Superseded and duplicate records remain visible in per-unit history; duplicate records are explicitly linked to the earlier equivalent record. -9. Missing/untranslated units remain visible even with no review history. -10. Invalid/missing ledger or unreadable required artifacts fail closed with expected CLI error and no partial report replacement. -11. Generated Markdown contains no timestamps or nondeterministic data not already present in ledger state. -12. Snapshot/render functions are reusable by #12 finalize without parsing Markdown. -13. Full Python 3.10/3.12 CI matrix passes; branch remains `behind_by=0`; `main` remains unchanged; feature branch is preserved. - -## TDD slices - -### Slice 1 — snapshot semantics - -RED tests for mixed states (`pass`, `corrections_required`, `missing`, `stale`, `untranslated`), coverage counts, current record metadata, stale exclusion, superseded history and duplicate detection. - -GREEN: implement `review_report.py` snapshot builder using existing review resolution and validated ledger state. - -### Slice 2 — deterministic Markdown/JSON - -RED tests for stable Markdown bytes/order, explicit history classifications, no handwritten audit dependency, and deterministic JSON-serializable snapshot. - -GREEN: add deterministic renderer(s), no wall-clock fields. - -### Slice 3 — CLI generation - -RED tests for `book.py review-report`, canonical file path, idempotent identical rerun, `--json` read-only behavior, and fail-closed invalid ledger behavior. - -GREEN: wire the command through `review_cli.py` and safe generated-file replacement. - -### Verification - -Run full CI on Python 3.10/3.12, audit diff + PR comments/reviews/threads + branch ancestry + `main`, update PR evidence, mark Ready only after all checks, then merge only to `refactor/workflow-engine-v2` with expected-head guard. diff --git a/docs/WORKFLOW_V2_SAFE_PATCH_DESIGN.md b/docs/WORKFLOW_V2_SAFE_PATCH_DESIGN.md deleted file mode 100644 index dba8f77..0000000 --- a/docs/WORKFLOW_V2_SAFE_PATCH_DESIGN.md +++ /dev/null @@ -1,346 +0,0 @@ -# Workflow v2 — Safe text patching - -Issue: #13 -Branch: `feature/workflow-v2-safe-patch` -Base: `refactor/workflow-engine-v2` at `a288b6bddde65a064a38964d99b9d6c0f65749e6` -PR target: `refactor/workflow-engine-v2` -Date: 2026-09-06 - -## Purpose - -Add a deterministic, repository-safe alternative to rewriting an entire text artifact when ChatGPT/Codex needs to make a small terminology or wording correction. - -The patch operation must fail closed when the target bytes, match count or write revision differ from what the caller expected. It must preserve UTF-8 text and existing line endings outside the exact replaced spans, show the proposed diff, and support literal or explicitly requested regular-expression matching. - -## Scope - -In scope: - -- one generic repository-relative text patch operation; -- exact literal replacement by default; -- optional regular-expression replacement; -- mandatory `--expected-count` safety precondition; -- optional inclusive line scope; -- dry-run mode; -- concise unified diff output; -- strict UTF-8 input/output; -- optimistic-concurrency protection through the existing filesystem storage revision token; -- integration with `scripts/book.py` as a structural workflow command; -- unit and CLI tests for mismatch/no-write, Unicode, paragraph boundaries and line-ending preservation. - -Out of scope: - -- semantic/fuzzy matching; -- automatic conflict resolution or three-way merge; -- patching binary files; -- rewriting multiple files in one transaction; -- automatically updating lifecycle/review state after translation changes; -- GitHub API backend execution (#17 will reuse the backend-neutral patch domain operation later if appropriate); -- changes to `main`. - -## Selected architecture - -Create a backend-neutral patch domain module: - -```text -scripts/workflow_v2/text_patch.py -``` - -and a CLI adapter: - -```text -scripts/workflow_v2/patch_cli.py -``` - -`text_patch.py` owns matching, scoping, UTF-8 decoding/encoding, count validation, diff generation and the optimistic write. It accepts the existing `StorageBackend` protocol, so filesystem execution uses the already-tested `FilesystemStorage` CAS semantics rather than introducing a second atomic-write implementation. - -`patch_cli.py` owns argparse registration, repository-root path selection, user-visible errors and diff/summary printing. It defines `PatchCliError` in the same style as the existing claim/review adapters and does not import `book.py`, avoiding a CLI import cycle. `book.py` imports/registers the command and adds `PatchCliError` to its existing top-level expected-error catch. - -`workflow_v2.__init__` exports only the reusable domain API (`TextPatchError`, `TextPatchResult`, `patch_text`) alongside the other Workflow v2 domain/storage primitives. The argparse adapter is not part of that package-level API. - -This split keeps the patch behavior independently testable and leaves GitHub/backend-specific transport outside the literary/text logic. - -## CLI contract - -Primary command: - -```bash -python scripts/book.py patch \ - --old \ - --new \ - --expected-count \ - [--regex] \ - [--line-start ] \ - [--line-end ] \ - [--dry-run] -``` - -Examples: - -```bash -python scripts/book.py patch books/demo/translated/001.md \ - --old "old term" --new "preferred term" --expected-count 2 -``` - -```bash -python scripts/book.py patch books/demo/glossary.md \ - --old '^(Term):\s+(.+)$' --new '\1 — \2' --expected-count 1 --regex \ - --line-start 20 --line-end 40 --dry-run -``` - -Rules: - -- `` is relative to the repository root and is resolved by `FilesystemStorage(repo_root())`; absolute paths, `..`, `./`, backslash escapes, empty segments and other unsafe forms are rejected by the existing storage path-safety rules before mutation. -- `--old`, `--new` and `--expected-count` are required. -- `--expected-count` is an integer `>= 0`. -- literal matching is the default. -- `--regex` opts into Python regular-expression semantics for both pattern and replacement; replacement backreferences follow `re.sub` rules. -- `--line-start` and `--line-end` are 1-based inclusive line numbers. Either may be supplied independently; omitted start means line 1, omitted end means the final line. -- a requested line range must intersect valid file lines, use positive integers, and satisfy start <= end. -- matching/counting occurs only inside the selected line slice. -- a multi-line literal/regex can cross line boundaries inside the selected slice, including paragraph boundaries. -- matching never crosses outside the selected slice. -- `--dry-run` performs every read/validation/diff step but never calls the write primitive. - -## Domain API - -`text_patch.py` defines: - -```python -class TextPatchError(RuntimeError): - pass - -@dataclass(frozen=True) -class TextPatchResult: - path: str - match_count: int - changed: bool - dry_run: bool - original_version: str - new_version: str | None - diff: str - - -def patch_text( - storage: StorageBackend, - path: str, - *, - old: str, - new: str, - expected_count: int, - regex: bool = False, - line_start: int | None = None, - line_end: int | None = None, - dry_run: bool = False, -) -> TextPatchResult: - ... -``` - -The domain function reads exactly once for the baseline revision, computes the full replacement in memory, validates the observed count, prepares the diff, and writes only with `storage.write_if_version(path, updated_bytes, original.version)`. - -A stale concurrent writer therefore causes the existing `StorageVersionConflict`; the patch layer converts it into a `TextPatchError` that explicitly says the target changed before the patch could commit. It does not re-read and silently retry because that would invalidate the caller's observed-count assumption. - -`expected_count`, `line_start` and `line_end` use strict integer validation (`type(value) is int`) so booleans are not accepted accidentally through Python's `bool`/`int` relationship. - -## UTF-8 and line endings - -The storage backend returns bytes. Patch processing uses strict `bytes.decode("utf-8")` and `str.encode("utf-8")`. - -This deliberately avoids `Path.read_text()`/`write_text()` universal-newline behavior. Existing `LF`, `CRLF` or mixed newline bytes survive unchanged unless they are part of a replaced span. - -A UTF-8 BOM is preserved naturally: decoding as plain UTF-8 produces U+FEFF and encoding restores the same BOM bytes unless the patch explicitly targets that first character. - -Invalid UTF-8 raises `TextPatchError` before matching or writing. - -## Line scope algorithm - -Use `text.splitlines(keepends=True)` so physical line separators remain part of each line. - -For non-empty text: - -1. derive total physical line count from `splitlines(keepends=True)`; -2. normalize start to 1 when omitted; -3. normalize end to total line count when omitted; -4. reject start/end outside `1..total` or start > end; -5. split into `prefix`, `scope`, `suffix` by line indexes; -6. perform match count/replacement only on `scope`; -7. reconstruct `prefix + updated_scope + suffix` exactly. - -For an empty file, no explicit line range is valid. Without line scope, the whole empty string is the scope and normal expected-count logic applies. - -## Match-count semantics - -### Literal mode - -Use the same non-overlapping semantics as `str.replace`: - -```python -match_count = scope.count(old) -updated_scope = scope.replace(old, new) -``` - -An empty literal `old` is rejected. Python's implicit insertion behavior for empty patterns is too easy to misuse for a safe patch command. - -### Regex mode - -Compile `old` with `re.compile(old)`. Invalid patterns are reported as `TextPatchError` before writing. - -Use: - -```python -updated_scope, match_count = pattern.subn(new, scope) -``` - -Invalid replacement/backreference syntax is also reported before writing. - -Zero-width regular expressions are allowed only when the caller explicitly chose `--regex` and supplied the exact expected count; this is deterministic under `re.subn` and remains protected by count/CAS checks. - -## Expected-count gate - -Observed count must equal `expected_count` exactly. - -On mismatch: - -- return no partial result; -- do not call `write_if_version`; -- raise `TextPatchError` with both expected and observed counts; -- leave target bytes/revision unchanged. - -`expected_count=0` is valid and can be used as an assertion that the pattern is absent. The result is `changed=false`; no write occurs even outside dry-run mode because there are no matched spans. - -## No-op replacements - -If matches exist but replacement produces byte-identical content (for example literal old == new), report the observed match count and `changed=false` and do not rewrite the file. - -This prevents meaningless revision churn and keeps retry behavior deterministic. - -## Diff output - -Generate a standard unified diff with `difflib.unified_diff` from the original and updated full text using `splitlines(keepends=True)`. - -Labels are deterministic: - -```text ---- a/ -+++ b/ -``` - -The domain result stores the complete diff string. For unchanged content, `diff` is the empty string. The operation remains small by intent; callers are responsible for choosing a narrow pattern/scope and exact expected count rather than relying on output truncation. - -CLI behavior: - -- print the diff first when non-empty; -- then print one summary line: - -```text -patch : matches= changed= mode= -``` - -No ANSI color is emitted, so output is stable for ChatGPT/Codex consumption. - -## Concurrency and atomicity - -Filesystem writes reuse `FilesystemStorage.write_if_version`, which already: - -- validates safe logical paths; -- verifies the expected SHA-256 revision; -- rejects stale versions with `StorageVersionConflict`; -- atomically replaces successful writes; -- leaves no temporary file after success. - -The patch helper does not bypass this backend. - -A test storage can inject a concurrent update after the patch read but before its CAS write. The patch must surface a conflict and preserve the concurrent winner's complete content. - -## Error handling - -`TextPatchError` wraps expected patch-domain failures: - -- invalid expected-count API value; -- empty literal old text; -- invalid regex or replacement syntax; -- invalid UTF-8; -- invalid/out-of-range line scope; -- match-count mismatch; -- target missing/path unsafe/storage failure; -- stale write conflict. - -`patch_cli.py` catches `TextPatchError` and expected storage errors and raises `PatchCliError` with a concise message. `book.py` includes `PatchCliError` in its existing top-level expected-error tuple, so expected patch failures print one `ERROR: ...` line and exit 1 without a traceback. - -No CLI adapter imports `book.py`; repository root is injected into `register_patch_command(subparsers, root)` just as existing Workflow v2 CLI registration receives its root explicitly. - -## Interaction with workflow state - -The command is intentionally generic text mutation. It does **not** modify `progress.json`, review ledger, claims or source manifest automatically. - -If the caller patches a translated artifact that already has PASS evidence, existing hash-bound review resolution will naturally mark that PASS stale on the next status/review/finalize check. This preserves the current single source of truth instead of duplicating review invalidation logic in the patch helper. - -Likewise, patching generated Markdown or glossary/style text does not fabricate unrelated workflow state transitions. - -## Testing strategy - -### Domain tests — `tests/test_workflow_v2_text_patch.py` - -Tests-first coverage: - -1. literal expected-count mismatch raises and leaves bytes/version unchanged; -2. literal success replaces exactly the intended non-overlapping spans; -3. `expected_count=0` is a no-write assertion; -4. old == new is no-write despite observed matches; -5. dry-run returns the same diff/count as apply but leaves bytes/version unchanged; -6. Unicode/Cyrillic replacement remains exact UTF-8; -7. multi-line literal replacement can cross a paragraph boundary; -8. CRLF/mixed line endings outside replaced spans remain byte-identical; -9. inclusive line scope changes only selected lines and counts only inside them; -10. invalid/out-of-range scopes and boolean numeric inputs fail without writing; -11. regex substitution supports capture-group replacement and exact expected count; -12. invalid regex/replacement fails without writing; -13. invalid UTF-8 fails without writing; -14. injected stale CAS conflict preserves the concurrent winner and surfaces a deterministic patch error. - -### CLI tests — `tests/test_workflow_v2_patch_cli.py` - -Cover: - -- successful literal patch through `book.py patch`; -- mismatch exit 1/no mutation; -- dry-run prints diff but does not write; -- regex and line scope flags reach the domain behavior; -- unsafe repository-relative path is rejected; -- negative expected count is rejected by argparse/domain validation; -- output summary/diff is deterministic and traceback-free. - -### Regression suite - -After focused GREEN, run the complete standard suite on Python 3.10 and 3.12. No GitHub Action is required to execute the command itself; Actions remains only the CI harness in this development environment. - -## Expected implementation surface - -```text -scripts/book.py -scripts/workflow_v2/text_patch.py -scripts/workflow_v2/patch_cli.py -scripts/workflow_v2/__init__.py - -tests/test_workflow_v2_text_patch.py -tests/test_workflow_v2_patch_cli.py - -docs/WORKFLOW_V2_SAFE_PATCH_DESIGN.md -``` - -No other production file should change unless a test proves an existing shared primitive defect. - -## Completion criteria - -#13 is ready to integrate when: - -- every acceptance criterion has deterministic automated coverage; -- count mismatch/dry-run/conflict paths are proven no-write; -- Unicode, paragraph-boundary and line-ending tests pass; -- CLI can safely patch translation/glossary/generated repository text by repository-relative path; -- domain helper is callable through the package-level Workflow v2 API without importing argparse; -- full Python 3.10/3.12 matrix is green; -- diff/PR/review-thread audit is clean; -- branch is not behind integration; -- `main` is unchanged; -- no branch is deleted. diff --git a/docs/WORKFLOW_V2_SAFE_PATCH_PLAN.md b/docs/WORKFLOW_V2_SAFE_PATCH_PLAN.md deleted file mode 100644 index b5ee501..0000000 --- a/docs/WORKFLOW_V2_SAFE_PATCH_PLAN.md +++ /dev/null @@ -1,286 +0,0 @@ -# Workflow v2 Safe Text Patching Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a deterministic, UTF-8-safe, CAS-protected text patch helper and `book.py patch` command for exact or regex small corrections with expected-count, line-scope, dry-run and unified-diff safety gates. - -**Architecture:** Implement matching and mutation in backend-neutral `workflow_v2.text_patch`, using `StorageBackend.read()` plus `write_if_version()` for optimistic concurrency. Add a thin `patch_cli.py` adapter registered by `book.py`; export only the reusable domain API from `workflow_v2.__init__`. Test domain behavior before CLI wiring, then verify full regression matrix and PR boundaries. - -**Tech Stack:** Python 3.10/3.12, stdlib `re`, `difflib`, `dataclasses`, existing `StorageBackend`/`FilesystemStorage`, `argparse`, `unittest`. - -**Spec:** `docs/WORKFLOW_V2_SAFE_PATCH_DESIGN.md` - -## Global Constraints - -- Work only on `feature/workflow-v2-safe-patch`, based on `refactor/workflow-engine-v2` at `a288b6bddde65a064a38964d99b9d6c0f65749e6`. -- PR target is `refactor/workflow-engine-v2`; never modify or merge to `main`. -- Do not delete branches. -- Literal matching is default; regex requires explicit `--regex`. -- `--expected-count` is mandatory and must be a strict integer >= 0. -- Paths are repository-relative and resolved by `FilesystemStorage(repo_root())`; never bypass storage path-safety. -- Strict UTF-8 bytes in/out; do not normalize line endings through text-mode filesystem APIs. -- No fuzzy/semantic matching, multi-file transaction, automatic review/lifecycle mutation, or GitHub-backend implementation. -- No production code before a failing test demonstrates the missing behavior. - ---- - -## File Map - -### Create -- `scripts/workflow_v2/text_patch.py` — domain patch algorithm, validation, diff and CAS write. -- `scripts/workflow_v2/patch_cli.py` — argparse registration and stable CLI output/error adaptation. -- `tests/test_workflow_v2_text_patch.py` — domain RED/GREEN coverage. -- `tests/test_workflow_v2_patch_cli.py` — end-to-end CLI RED/GREEN coverage. - -### Modify -- `scripts/workflow_v2/__init__.py` — export `TextPatchError`, `TextPatchResult`, `patch_text`. -- `scripts/book.py` — import/register patch CLI and catch `PatchCliError`. - ---- - -### Task 1: Domain safety core — literal replacement, expected count, dry-run and UTF-8 - -**Files:** -- Create: `tests/test_workflow_v2_text_patch.py` -- Create after RED: `scripts/workflow_v2/text_patch.py` -- Modify after RED: `scripts/workflow_v2/__init__.py` - -**Interfaces:** -- Consumes: `StorageBackend`, `StoredValue`, `StorageError`, `StorageVersionConflict`. -- Produces: - -```python -class TextPatchError(RuntimeError): ... - -@dataclass(frozen=True) -class TextPatchResult: - path: str - match_count: int - changed: bool - dry_run: bool - original_version: str - new_version: str | None - diff: str - - -def patch_text( - storage: StorageBackend, - path: str, - *, - old: str, - new: str, - expected_count: int, - regex: bool = False, - line_start: int | None = None, - line_end: int | None = None, - dry_run: bool = False, -) -> TextPatchResult: ... -``` - -- [ ] **Step 1: Write RED tests for literal count/no-write semantics** - -Create a temp `FilesystemStorage` and assert: - -```python -with self.assertRaisesRegex(TextPatchError, "expected 2 match.*observed 1"): - patch_text(storage, "sample.md", old="alpha", new="beta", expected_count=2) -self.assertEqual(storage.read("sample.md"), before) -``` - -Also add: -- success replaces exactly two non-overlapping literal occurrences; -- `expected_count=0` returns `match_count=0`, `changed=False`, `new_version=None`, with identical stored version/content; -- `old == new` with observed matches returns `changed=False` and no rewrite; -- empty literal `old` raises before write; -- negative or boolean `expected_count` raises before write. - -- [ ] **Step 2: Verify RED in CI** - -Commit tests only and use a draft PR to `refactor/workflow-engine-v2` as CI harness. Expected failure: import/module/API missing, not fixture/syntax error. - -- [ ] **Step 3: Implement minimal literal domain behavior** - -In `text_patch.py`: -- validate strict integer count; -- `storage.read(path)` baseline once; -- strict UTF-8 decode; -- choose whole text as scope initially when no line bounds; -- literal `scope.count(old)` / `scope.replace(old,new)`; -- exact expected-count gate; -- if no byte change, return no-write result; -- otherwise generate unified diff and call `write_if_version` once with baseline version; -- translate expected storage/Unicode errors into `TextPatchError` with concise context. - -Export domain API from `workflow_v2.__init__`. - -- [ ] **Step 4: Verify GREEN for Task 1** - -Require the focused domain tests to pass on Python 3.10 and 3.12, plus existing storage tests. - -- [ ] **Step 5: Commit GREEN** - -Use an audit-friendly message such as `feat: add safe literal text patch domain`. - ---- - -### Task 2: Line scope, line-ending preservation, regex and CAS conflict - -**Files:** -- Modify: `tests/test_workflow_v2_text_patch.py` -- Modify after RED: `scripts/workflow_v2/text_patch.py` - -**Interfaces:** -- Extends the exact `patch_text()` API from Task 1; no new public API. - -- [ ] **Step 1: Write RED tests for line scope and raw-byte preservation** - -Add tests proving: -- Cyrillic/Unicode replacement encodes exact expected UTF-8; -- multi-line literal `"first\n\nsecond"` crosses a paragraph boundary; -- input containing `b"one\r\ntwo\nthree\r\n"` preserves every untouched newline byte after replacing only `two`; -- `line_start=2, line_end=3` counts/replaces only physical lines 2–3; -- omitted start/end expand to file boundaries; -- line 0, negative, boolean, start>end, end>line-count and explicit range on empty file all fail with no write. - -- [ ] **Step 2: Verify RED and implement line-slice algorithm** - -Use `splitlines(keepends=True)` and reconstruct `prefix + updated_scope + suffix`. No `Path.read_text/write_text` may be introduced. - -- [ ] **Step 3: Write RED tests for regex behavior** - -Cover: - -```python -result = patch_text( - storage, - "sample.md", - old=r"^(Term):\s+(.+)$", - new=r"\1 — \2", - expected_count=1, - regex=True, -) -``` - -with a pattern that actually matches the complete scope under the chosen flags. Add invalid regex and invalid replacement backreference tests; both must leave bytes/version unchanged. - -- [ ] **Step 4: Implement regex with `re.compile()` + `subn()`** - -Catch `re.error` for pattern and replacement execution and report `TextPatchError`. Do not silently fall back to literal mode. - -- [ ] **Step 5: Write RED CAS-conflict injection test** - -Create a `FilesystemStorage` subclass whose first `write_if_version()` call first commits a different winning payload via a second storage instance, then delegates the stale patch write. Assert: -- `patch_text()` raises a deterministic conflict `TextPatchError`; -- winner bytes remain complete; -- patch bytes are absent. - -- [ ] **Step 6: Implement minimal conflict adaptation and verify GREEN** - -Catch `StorageVersionConflict` separately and report that the target changed before patch commit. Run domain + storage tests on both Python versions. - -- [ ] **Step 7: Commit Task 2** - -Suggested message: `feat: add scoped regex safe patching`. - ---- - -### Task 3: `book.py patch` CLI integration - -**Files:** -- Create: `tests/test_workflow_v2_patch_cli.py` -- Create after RED: `scripts/workflow_v2/patch_cli.py` -- Modify after RED: `scripts/book.py` - -**Interfaces:** -- Consumes `patch_text()` from Task 1/2. -- Produces: - -```python -class PatchCliError(RuntimeError): ... - -def register_patch_command( - subparsers: argparse._SubParsersAction, - root: Path, -) -> None: ... -``` - -- [ ] **Step 1: Write RED CLI smoke test** - -Copy `book.py` + `workflow_v2/` into a temp repo, create `books/demo/translated/001.md`, and run: - -```text -book.py patch books/demo/translated/001.md --old alpha --new beta --expected-count 1 -``` - -Assert exit 0, exact target content, unified diff labels `a/...` and `b/...`, and summary: - -```text -patch books/demo/translated/001.md: matches=1 changed=yes mode=apply -``` - -Expected RED: parser reports unknown `patch` command. - -- [ ] **Step 2: Add RED error/no-write CLI cases** - -Cover: -- count mismatch -> exit 1, one `ERROR:` line, no traceback, no mutation; -- `--dry-run` -> diff + `mode=dry-run`, no mutation; -- unsafe `../outside.md` -> exit 1/no outside write; -- negative `--expected-count` -> exit 1; -- regex + line-start/end changes only intended scoped occurrence. - -- [ ] **Step 3: Implement `patch_cli.py`** - -Create strict non-negative integer argparse type for `--expected-count`, positive integer types for line bounds, register the command, instantiate `FilesystemStorage(root)`, call `patch_text`, print diff when non-empty then deterministic summary. Convert domain/storage expected failures to `PatchCliError`; never import `book.py`. - -- [ ] **Step 4: Wire `book.py`** - -Import `PatchCliError`, `register_patch_command`; register after build parser creation and before/alongside existing Workflow v2 commands; add `PatchCliError` to the top-level expected exception tuple. - -- [ ] **Step 5: Verify GREEN** - -Run CLI tests plus book CLI, storage, review validation and status tests to prove parser registration did not regress existing commands. - -- [ ] **Step 6: Commit Task 3** - -Suggested message: `feat: add safe patch cli`. - ---- - -### Task 4: Full verification, audit and integration - -**Files:** -- PR metadata only unless verification reveals a genuine defect. - -- [ ] **Step 1: Full matrix** - -Require fresh `python -m unittest discover -s tests -v` CI success on Python 3.10 and 3.12 for the final head. Record run ID and total tests. - -- [ ] **Step 2: Diff audit** - -Compare base/head and verify only the approved files changed; branch must be `behind_by=0`; no unrelated workflow, book content or private source file is present. - -- [ ] **Step 3: Review audit** - -Fetch PR comments, reviews and inline threads. Resolve any blocking finding and rerun CI after code changes. - -- [ ] **Step 4: Update PR evidence and Ready state** - -PR body must include RED/GREEN run IDs, acceptance coverage, changed files, final matrix count, base/head SHA, `main` unchanged, branch preserved. - -- [ ] **Step 5: Integrate only to `refactor/workflow-engine-v2`** - -Under the standing autonomous project directive, merge only if head/base/CI/review guards still match. Use expected-head guard. Preserve the feature branch. Never merge to `main`. - ---- - -## Self-Review Checklist - -- Literal, regex, count gate, dry-run, line scope, UTF-8, paragraph boundaries and line endings each map to explicit tests. -- Mismatch, invalid regex, invalid UTF-8, invalid scope and CAS conflict each prove no unintended write. -- Public signatures match the approved spec exactly. -- `PatchCliError` prevents import cycles. -- Repository-relative root is explicit and uses existing storage path safety. -- `workflow_v2.__init__` exports only domain API, not argparse adapter. -- No step mutates review/lifecycle state automatically. -- No GitHub backend or multi-file semantics leak into #13. diff --git a/scripts/workflow_v2/migration_journal.py b/scripts/workflow_v2/migration_journal.py deleted file mode 100644 index 8101480..0000000 --- a/scripts/workflow_v2/migration_journal.py +++ /dev/null @@ -1,202 +0,0 @@ -"""Strict validation and serialization for transient workflow migration recovery.""" - -from __future__ import annotations - -import base64 -import binascii -import copy -import hashlib -import json -import re -from collections.abc import Mapping -from pathlib import PurePosixPath -from typing import Any - -from .storage import StorageBackend - - -MIGRATION_PATH = ".workflow/migration.json" -_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") -_ALLOWED_KINDS = {"metadata", "progress", "review_ledger", "claim", "source_manifest"} -_TOP_LEVEL_KEYS = { - "schema_version", - "operation", - "book_slug", - "from_revision", - "to_revision", - "phase", - "documents", -} -_ENTRY_KEYS = { - "path", - "kind", - "original_exists", - "original_revision", - "original_sha256", - "original_bytes_base64", - "target_sha256", - "resulting_revision", -} - - -class MigrationJournalError(RuntimeError): - """Migration recovery journal is malformed or cannot be decoded safely.""" - - -def _nonempty_string(value: object, field: str) -> str: - if not isinstance(value, str) or not value.strip(): - raise MigrationJournalError(f"{field} must be a non-empty string") - return value - - -def _nullable_nonempty_string(value: object, field: str) -> str | None: - if value is None: - return None - return _nonempty_string(value, field) - - -def _sha256(value: object, field: str) -> str: - text = _nonempty_string(value, field) - if _SHA256_RE.fullmatch(text) is None: - raise MigrationJournalError( - f"{field} must be a 64-character lowercase hexadecimal SHA-256" - ) - return text - - -def _safe_path(value: object, field: str) -> str: - text = _nonempty_string(value, field) - if "\\" in text: - raise MigrationJournalError(f"{field} must be a safe relative POSIX path") - path = PurePosixPath(text) - if path.is_absolute() or not path.parts or any(part in {"", ".", ".."} for part in path.parts): - raise MigrationJournalError(f"{field} must be a safe relative POSIX path") - return text - - -def _require_exact_keys(data: Mapping[str, Any], expected: set[str], label: str) -> None: - actual = set(data) - missing = sorted(expected - actual) - extra = sorted(actual - expected) - if missing: - raise MigrationJournalError(f"{label} missing field(s): {', '.join(missing)}") - if extra: - raise MigrationJournalError(f"{label} has unsupported field(s): {', '.join(extra)}") - - -def validate_migration_journal(data: Mapping[str, Any]) -> dict[str, Any]: - """Return a validated deep copy of one migration recovery journal.""" - - if not isinstance(data, Mapping): - raise MigrationJournalError("migration journal must be a JSON object") - _require_exact_keys(data, _TOP_LEVEL_KEYS, "migration journal") - - if type(data["schema_version"]) is not int or data["schema_version"] != 1: - raise MigrationJournalError("schema_version must equal 1") - if data["operation"] != "workflow_upgrade": - raise MigrationJournalError("operation must equal workflow_upgrade") - _nonempty_string(data["book_slug"], "book_slug") - _nullable_nonempty_string(data["from_revision"], "from_revision") - _nonempty_string(data["to_revision"], "to_revision") - - phase = data["phase"] - if phase not in {"prepared", "applied"}: - raise MigrationJournalError("phase must be prepared or applied") - - documents = data["documents"] - if not isinstance(documents, list) or not documents: - raise MigrationJournalError("documents must be a non-empty array") - - seen_paths: set[str] = set() - for index, raw_entry in enumerate(documents): - label = f"documents[{index}]" - if not isinstance(raw_entry, Mapping): - raise MigrationJournalError(f"{label} must be an object") - _require_exact_keys(raw_entry, _ENTRY_KEYS, label) - - path = _safe_path(raw_entry["path"], f"{label}.path") - if path in seen_paths: - raise MigrationJournalError(f"{label}.path must be unique") - seen_paths.add(path) - - kind = _nonempty_string(raw_entry["kind"], f"{label}.kind") - if kind not in _ALLOWED_KINDS: - raise MigrationJournalError( - f"{label}.kind must be one of {', '.join(sorted(_ALLOWED_KINDS))}" - ) - - original_exists = raw_entry["original_exists"] - if type(original_exists) is not bool: - raise MigrationJournalError(f"{label}.original_exists must be a boolean") - - original_revision = raw_entry["original_revision"] - original_sha256 = raw_entry["original_sha256"] - original_base64 = raw_entry["original_bytes_base64"] - if original_exists: - _nonempty_string(original_revision, f"{label}.original_revision") - expected_hash = _sha256(original_sha256, f"{label}.original_sha256") - encoded = _nonempty_string(original_base64, f"{label}.original_bytes_base64") - try: - decoded = base64.b64decode(encoded.encode("ascii"), validate=True) - except (UnicodeEncodeError, binascii.Error, ValueError) as exc: - raise MigrationJournalError( - f"{label}.original_bytes_base64 must be strict base64" - ) from exc - actual_hash = hashlib.sha256(decoded).hexdigest() - if actual_hash != expected_hash: - raise MigrationJournalError( - f"{label}.original_bytes_base64 does not match original_sha256" - ) - elif any(value is not None for value in (original_revision, original_sha256, original_base64)): - raise MigrationJournalError( - f"{label} original revision/hash/bytes must all be null when original_exists is false" - ) - - _sha256(raw_entry["target_sha256"], f"{label}.target_sha256") - resulting = _nullable_nonempty_string( - raw_entry["resulting_revision"], f"{label}.resulting_revision" - ) - if phase == "applied" and resulting is None: - raise MigrationJournalError( - f"{label}.resulting_revision is required while phase is applied" - ) - - return copy.deepcopy(dict(data)) - - -def serialize_migration_journal(data: Mapping[str, Any]) -> bytes: - """Serialize a validated journal to deterministic UTF-8 JSON bytes.""" - - validated = validate_migration_journal(data) - try: - text = json.dumps( - validated, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) + "\n" - except (TypeError, ValueError) as exc: - raise MigrationJournalError(f"migration journal is not JSON-serializable: {exc}") from exc - return text.encode("utf-8") - - -def load_migration_journal(storage: StorageBackend) -> tuple[dict[str, Any], str]: - """Load and strictly validate the durable migration journal. - - StorageNotFound intentionally propagates so callers can distinguish an absent journal - from a malformed one. - """ - - stored = storage.read(MIGRATION_PATH) - try: - text = stored.content.decode("utf-8") - except UnicodeDecodeError as exc: - raise MigrationJournalError(f"{MIGRATION_PATH}: invalid UTF-8: {exc}") from exc - try: - raw = json.loads(text) - except json.JSONDecodeError as exc: - raise MigrationJournalError(f"{MIGRATION_PATH}: invalid JSON: {exc}") from exc - if not isinstance(raw, Mapping): - raise MigrationJournalError(f"{MIGRATION_PATH}: journal must be a JSON object") - return validate_migration_journal(raw), stored.version diff --git a/scripts/workflow_v2/migrations.py b/scripts/workflow_v2/migrations.py deleted file mode 100644 index b18b0b0..0000000 --- a/scripts/workflow_v2/migrations.py +++ /dev/null @@ -1,1186 +0,0 @@ -"""Explicit Workflow v2 schema migration, planning, and execution.""" - -from __future__ import annotations - -import base64 -import copy -import hashlib -import json -from collections.abc import Callable, Mapping -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from .coordination import CoordinationError -from .migration_journal import ( - MIGRATION_PATH, - MigrationJournalError, - load_migration_journal, - serialize_migration_journal, -) -from .repository import RepositoryError, WorkflowStateRepository -from .reviews import REVIEW_CONTRACT_PATH, REVIEW_EVIDENCE_VERSION -from .schemas import SchemaError, SchemaKind, parse_document -from .source_integrity import ( - SourceIntegrityError, - build_private_source_manifest_from_identity, - build_source_manifest, - sha256_path, -) -from .storage import ( - StorageAlreadyExists, - StorageError, - StorageNotFound, - StorageVersionConflict, -) - - -CANONICAL_REPOSITORY = "https://github.com/tim8es/book-translator" -FINALIZATION_PATH = ".workflow/finalization.json" -CLAIM_PREFIX = ".workflow/claims" - - -class MigrationError(RuntimeError): - """Base error for explicit Workflow v2 migration operations.""" - - -class MigrationCompatibilityError(MigrationError): - """Legacy durable state cannot be migrated without inventing data.""" - - -class MigrationConflict(MigrationError): - """Durable state changed across a migration coordination boundary.""" - - -@dataclass(frozen=True) -class MigratedDocument: - """One strictly validated document after a pure schema migration step.""" - - kind: SchemaKind - from_version: int - to_version: int - data: dict[str, Any] - changed: bool - - -@dataclass(frozen=True) -class PlannedWrite: - """One exact target write produced by a read-only migration plan.""" - - path: str - kind: SchemaKind - original_exists: bool - original_version: str | None - original_bytes: bytes | None - target_data: dict[str, Any] - target_bytes: bytes - from_version: int | None - to_version: int - - -@dataclass(frozen=True) -class MigrationPlan: - """A fully validated, immutable description of one explicit upgrade.""" - - book_slug: str - from_revision: str | None - to_revision: str - writes: tuple[PlannedWrite, ...] - lifecycle_downgrades: tuple[int, ...] - changed: bool - - def write_for(self, path: str) -> PlannedWrite | None: - for write in self.writes: - if write.path == path: - return write - return None - - -@dataclass(frozen=True) -class MigrationResult: - """Stable outcome returned by explicit workflow upgrade execution/recovery.""" - - book_slug: str - from_revision: str | None - to_revision: str - outcome: str - migrated_paths: tuple[str, ...] - lifecycle_downgrades: tuple[int, ...] - - -@dataclass(frozen=True) -class _RawDocument: - path: str - kind: SchemaKind - data: dict[str, Any] - content: bytes - version: str - migrated: MigratedDocument - - -def detect_schema_version(data: Mapping[str, Any]) -> int: - """Return logical schema version, treating an absent version as legacy v0.""" - - if not isinstance(data, Mapping): - raise MigrationCompatibilityError("document must be a JSON object") - if "schema_version" not in data: - return 0 - value = data["schema_version"] - if type(value) is not int: - raise MigrationCompatibilityError("schema_version must be an integer") - return value - - -def migrate_document(kind: SchemaKind, data: Mapping[str, Any]) -> MigratedDocument: - """Migrate one supported v0/v1 document to strict schema v1 in memory only.""" - - if not isinstance(kind, SchemaKind): - raise MigrationCompatibilityError("kind must be a SchemaKind") - version = detect_schema_version(data) - - if version == 1: - try: - parsed = parse_document(kind, data) - except SchemaError as exc: - raise MigrationCompatibilityError( - f"{kind.value} v1 is invalid: {exc}" - ) from exc - return MigratedDocument( - kind=kind, - from_version=1, - to_version=1, - data=copy.deepcopy(parsed.data), - changed=False, - ) - - if version != 0: - raise MigrationCompatibilityError( - f"{kind.value}: unsupported schema version {version}; expected 0 or 1" - ) - - candidate = copy.deepcopy(dict(data)) - candidate["schema_version"] = 1 - try: - parsed = parse_document(kind, candidate) - except SchemaError as exc: - raise MigrationCompatibilityError( - f"{kind.value} v0 is not v1-compatible: {exc}" - ) from exc - - return MigratedDocument( - kind=kind, - from_version=0, - to_version=1, - data=copy.deepcopy(parsed.data), - changed=True, - ) - - -class MigrationPlanner: - """Build a complete upgrade plan without mutating durable state.""" - - def __init__( - self, - repository: WorkflowStateRepository, - *, - book_dir: Path, - artifact_reader: Callable[[str], bytes], - now: Callable[[], datetime] | None = None, - ): - self.repository = repository - self.book_dir = Path(book_dir) - self._artifact_reader = artifact_reader - self._now_factory = now or (lambda: datetime.now(timezone.utc)) - - def _now(self) -> datetime: - value = self._now_factory() - if not isinstance(value, datetime) or value.tzinfo is None or value.utcoffset() is None: - raise MigrationCompatibilityError("migration clock must return a timezone-aware datetime") - return value.astimezone(timezone.utc) - - @staticmethod - def _parse_utc(value: object, label: str) -> datetime: - if not isinstance(value, str) or not value.strip(): - raise MigrationCompatibilityError(f"{label} must be a timestamp") - normalized = value[:-1] + "+00:00" if value.endswith("Z") else value - try: - parsed = datetime.fromisoformat(normalized) - except ValueError as exc: - raise MigrationCompatibilityError(f"{label} is not a valid timestamp") from exc - if parsed.tzinfo is None or parsed.utcoffset() is None: - raise MigrationCompatibilityError(f"{label} must be timezone-aware") - return parsed.astimezone(timezone.utc) - - def _read_raw(self, path: str, kind: SchemaKind, *, required: bool) -> _RawDocument | None: - try: - stored = self.repository.storage.read(path) - except StorageNotFound: - if required: - raise MigrationCompatibilityError(f"required migration document is missing: {path}") - return None - except StorageError as exc: - raise MigrationCompatibilityError(f"cannot read {path}: {exc}") from exc - - try: - text = stored.content.decode("utf-8") - except UnicodeDecodeError as exc: - raise MigrationCompatibilityError(f"{path}: invalid UTF-8: {exc}") from exc - try: - raw = json.loads(text) - except json.JSONDecodeError as exc: - raise MigrationCompatibilityError(f"{path}: invalid JSON: {exc}") from exc - if not isinstance(raw, Mapping): - raise MigrationCompatibilityError(f"{path}: document must be a JSON object") - migrated = migrate_document(kind, raw) - return _RawDocument( - path=path, - kind=kind, - data=copy.deepcopy(dict(raw)), - content=stored.content, - version=stored.version, - migrated=migrated, - ) - - def _serialize(self, path: str, kind: SchemaKind, data: Mapping[str, Any]) -> bytes: - try: - return self.repository.serialize(path, kind, data) - except (SchemaError, RepositoryError) as exc: - raise MigrationCompatibilityError(f"cannot serialize migration target {path}: {exc}") from exc - - def _write_if_changed( - self, - raw: _RawDocument | None, - *, - path: str, - kind: SchemaKind, - target: Mapping[str, Any], - absent_from_version: int | None = None, - ) -> PlannedWrite | None: - target_data = copy.deepcopy(dict(target)) - target_bytes = self._serialize(path, kind, target_data) - if raw is None: - return PlannedWrite( - path=path, - kind=kind, - original_exists=False, - original_version=None, - original_bytes=None, - target_data=target_data, - target_bytes=target_bytes, - from_version=absent_from_version, - to_version=1, - ) - if not raw.migrated.changed and target_data == raw.migrated.data: - return None - return PlannedWrite( - path=path, - kind=kind, - original_exists=True, - original_version=raw.version, - original_bytes=raw.content, - target_data=target_data, - target_bytes=target_bytes, - from_version=raw.migrated.from_version, - to_version=1, - ) - - @staticmethod - def _installed_target(installed: Mapping[str, Any], to_revision: str) -> tuple[str, str | None]: - if not isinstance(installed, Mapping): - raise MigrationCompatibilityError("installed workflow provenance is unavailable") - repository = installed.get("canonical_repository") - if repository != CANONICAL_REPOSITORY: - raise MigrationCompatibilityError( - f"installed canonical repository must be {CANONICAL_REPOSITORY!r}" - ) - resolved = installed.get("resolved_revision") - if not isinstance(resolved, str) or not resolved.strip(): - raise MigrationCompatibilityError("installed resolved_revision is unavailable") - if not isinstance(to_revision, str) or not to_revision.strip(): - raise MigrationCompatibilityError("target workflow revision must be non-empty") - if to_revision != resolved: - raise MigrationCompatibilityError( - f"requested target {to_revision!r} does not match installed resolved revision {resolved!r}" - ) - requested_ref = installed.get("requested_ref") - if requested_ref is not None and (not isinstance(requested_ref, str) or not requested_ref.strip()): - raise MigrationCompatibilityError("installed requested_ref must be null or a non-empty string") - return resolved, requested_ref - - def _ensure_no_finalization(self) -> None: - try: - self.repository.storage.read(FINALIZATION_PATH) - except StorageNotFound: - return - except StorageError as exc: - raise MigrationCompatibilityError(f"cannot inspect finalization state: {exc}") from exc - raise MigrationCompatibilityError("workflow upgrade is blocked while finalization is active") - - def _claims( - self, - valid_units: set[str], - ) -> list[_RawDocument]: - now = self._now() - claims: list[_RawDocument] = [] - try: - paths = self.repository.storage.list(CLAIM_PREFIX) - except StorageError as exc: - raise MigrationCompatibilityError(f"cannot list workflow claims: {exc}") from exc - for path in sorted(path for path in paths if path.endswith(".json")): - raw = self._read_raw(path, SchemaKind.CLAIM, required=True) - assert raw is not None - unit_id = raw.migrated.data.get("unit_id") - if unit_id not in valid_units: - raise MigrationCompatibilityError( - f"claim {path} references unknown unit {unit_id!r}" - ) - expires_at = self._parse_utc(raw.migrated.data.get("expires_at"), f"{path}.expires_at") - if expires_at > now: - raise MigrationCompatibilityError( - f"workflow upgrade is blocked by live claim {unit_id} until {raw.migrated.data['expires_at']}" - ) - claims.append(raw) - return claims - - def _read_artifact(self, relative_path: object, *, label: str) -> bytes: - if not isinstance(relative_path, str) or not relative_path.strip(): - raise MigrationCompatibilityError(f"{label} path is missing") - try: - content = self._artifact_reader(relative_path) - except (FileNotFoundError, OSError) as exc: - raise MigrationCompatibilityError(f"missing {label} artifact: {relative_path}") from exc - if not isinstance(content, bytes): - raise MigrationCompatibilityError(f"artifact reader must return bytes for {relative_path}") - return content - - def _reconcile_reviewed( - self, - progress: Mapping[str, Any], - metadata: Mapping[str, Any], - ledger: Mapping[str, Any], - ) -> tuple[dict[str, Any], tuple[int, ...]]: - updated = copy.deepcopy(dict(progress)) - chapters = updated.get("chapters") - assert isinstance(chapters, list) - workflow = metadata.get("workflow") - assert isinstance(workflow, Mapping) - revision = workflow.get("resolved_revision") - assert isinstance(revision, str) - contract = f"{REVIEW_CONTRACT_PATH}@{revision}" - records = ledger.get("records") - assert isinstance(records, list) - downgrades: list[int] = [] - - for chapter in chapters: - if not isinstance(chapter, dict) or chapter.get("status") != "reviewed": - continue - number = chapter.get("number") - if type(number) is not int: - raise MigrationCompatibilityError("reviewed chapter has invalid number") - source = self._read_artifact(chapter.get("source_path"), label="source") - translation = self._read_artifact(chapter.get("translation_path"), label="translation") - if not translation.strip(): - raise MigrationCompatibilityError( - f"chapter {number}: reviewed translation artifact is empty" - ) - source_sha = hashlib.sha256(source).hexdigest() - translation_sha = hashlib.sha256(translation).hexdigest() - unit_id = f"chapter-{number:06d}" - exact = [ - record - for record in records - if isinstance(record, Mapping) - and record.get("unit_id") == unit_id - and record.get("source_sha256") == source_sha - and record.get("translation_sha256") == translation_sha - and record.get("workflow_revision") == revision - and record.get("review_contract_revision") == contract - ] - if not exact or exact[-1].get("outcome") != "PASS": - chapter["status"] = "translated" - downgrades.append(number) - - try: - parsed = parse_document(SchemaKind.PROGRESS, updated) - except SchemaError as exc: - raise MigrationCompatibilityError(f"candidate progress is invalid: {exc}") from exc - return copy.deepcopy(parsed.data), tuple(downgrades) - - def _verify_manifest( - self, - metadata: Mapping[str, Any], - progress: Mapping[str, Any], - manifest: Mapping[str, Any], - ) -> None: - if manifest.get("source_file") != metadata.get("source_file"): - raise MigrationCompatibilityError("source manifest source_file does not match metadata") - if manifest.get("source_format") != metadata.get("source_format"): - raise MigrationCompatibilityError("source manifest source_format does not match metadata") - - chapters = progress.get("chapters") - extracted = manifest.get("extracted") - if not isinstance(chapters, list) or not isinstance(extracted, list): - raise MigrationCompatibilityError("source manifest/progress chapter arrays are invalid") - if manifest.get("chapter_count") != len(chapters) or len(extracted) != len(chapters): - raise MigrationCompatibilityError("source manifest chapter_count does not match progress") - - for index, (chapter, entry) in enumerate(zip(chapters, extracted), start=1): - if not isinstance(chapter, Mapping) or not isinstance(entry, Mapping): - raise MigrationCompatibilityError(f"source manifest chapter {index} is invalid") - expected = (chapter.get("number"), chapter.get("title"), chapter.get("source_path")) - actual = (entry.get("number"), entry.get("title"), entry.get("path")) - if actual != expected: - raise MigrationCompatibilityError( - f"source manifest chapter {index} identity does not match progress" - ) - relative = entry.get("path") - if not isinstance(relative, str): - raise MigrationCompatibilityError(f"source manifest chapter {index} path is invalid") - path = self.book_dir / relative - if not path.is_file(): - raise MigrationCompatibilityError(f"missing extracted source artifact: {relative}") - if sha256_path(path) != entry.get("sha256"): - raise MigrationCompatibilityError(f"extracted source hash mismatch: {relative}") - - explicit = metadata.get("source") if isinstance(metadata.get("source"), Mapping) else None - source_file = metadata.get("source_file") - source_path = self.book_dir / "source" / str(source_file) - if explicit is None: - if not source_path.is_file(): - raise MigrationCompatibilityError( - f"source identity cannot be proven; missing source/{source_file}" - ) - if sha256_path(source_path) != manifest.get("source_sha256"): - raise MigrationCompatibilityError("source manifest hash does not match embedded source") - return - - mode = explicit.get("storage_mode") - if manifest.get("source_storage_mode") != mode: - raise MigrationCompatibilityError("source manifest storage mode does not match metadata") - if manifest.get("source_size_bytes") != explicit.get("size_bytes"): - raise MigrationCompatibilityError("source manifest size does not match metadata source identity") - if manifest.get("source_sha256") != explicit.get("sha256"): - raise MigrationCompatibilityError("source manifest hash does not match metadata source identity") - if explicit.get("filename") != source_file: - raise MigrationCompatibilityError("metadata source filename identity does not match source_file") - if mode == "embedded" and not source_path.is_file(): - raise MigrationCompatibilityError(f"embedded source is missing: source/{source_file}") - if source_path.is_file(): - if source_path.stat().st_size != explicit.get("size_bytes"): - raise MigrationCompatibilityError("attached source size does not match metadata identity") - if sha256_path(source_path) != explicit.get("sha256"): - raise MigrationCompatibilityError("attached source hash does not match metadata identity") - - @staticmethod - def _schema_history( - metadata: MigratedDocument, - progress: MigratedDocument, - ledger: _RawDocument | None, - manifest: _RawDocument | None, - claims: list[_RawDocument], - ) -> dict[str, dict[str, int]]: - versions: dict[str, dict[str, int]] = {} - if metadata.from_version == 0: - versions["metadata"] = {"from": 0, "to": 1} - if progress.from_version == 0: - versions["progress"] = {"from": 0, "to": 1} - if ledger is None: - versions["review_ledger"] = {"from": 0, "to": 1} - elif ledger.migrated.from_version == 0: - versions["review_ledger"] = {"from": 0, "to": 1} - if manifest is None: - versions["source_manifest"] = {"from": 0, "to": 1} - elif manifest.migrated.from_version == 0: - versions["source_manifest"] = {"from": 0, "to": 1} - if any(claim.migrated.from_version == 0 for claim in claims): - versions["claims"] = {"from": 0, "to": 1} - return versions - - def plan( - self, - *, - slug: str, - to_revision: str, - installed: Mapping[str, Any], - ) -> MigrationPlan: - target_revision, requested_ref = self._installed_target(installed, to_revision) - if not isinstance(slug, str) or not slug.strip(): - raise MigrationCompatibilityError("book slug must be non-empty") - - metadata_raw = self._read_raw("metadata.json", SchemaKind.METADATA, required=True) - progress_raw = self._read_raw("progress.json", SchemaKind.PROGRESS, required=True) - assert metadata_raw is not None and progress_raw is not None - metadata = copy.deepcopy(metadata_raw.migrated.data) - progress = copy.deepcopy(progress_raw.migrated.data) - - if progress.get("book_slug") != slug: - raise MigrationCompatibilityError( - f"progress book_slug {progress.get('book_slug')!r} does not match requested book {slug!r}" - ) - chapters = progress.get("chapters") - if not isinstance(chapters, list) or metadata.get("chapter_count") != len(chapters): - raise MigrationCompatibilityError("metadata chapter_count does not match progress") - numbers = [chapter.get("number") for chapter in chapters if isinstance(chapter, Mapping)] - if len(numbers) != len(chapters) or len(numbers) != len(set(numbers)): - raise MigrationCompatibilityError("progress chapter numbers must be unique") - valid_units = {f"chapter-{number:06d}" for number in numbers if type(number) is int} - - workflow = metadata.get("workflow") - if workflow is not None and not isinstance(workflow, Mapping): - raise MigrationCompatibilityError("metadata workflow must be an object") - workflow_map = copy.deepcopy(dict(workflow)) if isinstance(workflow, Mapping) else {} - repository_name = workflow_map.get("repository") - if repository_name is not None and repository_name != CANONICAL_REPOSITORY: - raise MigrationCompatibilityError( - f"metadata workflow repository {repository_name!r} is incompatible" - ) - from_revision = workflow_map.get("resolved_revision") - if from_revision is not None and (not isinstance(from_revision, str) or not from_revision.strip()): - raise MigrationCompatibilityError("metadata workflow resolved_revision must be null or non-empty") - - self._ensure_no_finalization() - claims = self._claims(valid_units) - - ledger_raw = self._read_raw("review-ledger.json", SchemaKind.REVIEW_LEDGER, required=False) - if ledger_raw is None: - ledger = { - "schema_version": 1, - "book_slug": slug, - "next_sequence": 1, - "records": [], - } - try: - parse_document(SchemaKind.REVIEW_LEDGER, ledger) - except SchemaError as exc: - raise MigrationCompatibilityError(f"candidate review ledger is invalid: {exc}") from exc - else: - ledger = copy.deepcopy(ledger_raw.migrated.data) - if ledger.get("book_slug") != slug: - raise MigrationCompatibilityError("review ledger book_slug does not match progress") - - target_metadata = copy.deepcopy(metadata) - target_workflow = copy.deepcopy(workflow_map) - target_workflow["repository"] = CANONICAL_REPOSITORY - target_workflow["requested_ref"] = requested_ref - target_workflow["resolved_revision"] = target_revision - target_workflow["review_evidence"] = REVIEW_EVIDENCE_VERSION - target_metadata["workflow"] = target_workflow - - target_progress, downgrades = self._reconcile_reviewed( - progress, - target_metadata, - ledger, - ) - - manifest_raw = self._read_raw("source-manifest.json", SchemaKind.SOURCE_MANIFEST, required=False) - if manifest_raw is None: - explicit = target_metadata.get("source") - try: - if isinstance(explicit, Mapping) and explicit.get("storage_mode") == "private_external": - manifest = build_private_source_manifest_from_identity( - self.book_dir, - target_metadata, - target_progress, - ) - else: - source_file = target_metadata.get("source_file") - if not isinstance(source_file, str) or not source_file.strip(): - raise SourceIntegrityError("metadata source_file is unavailable") - manifest = build_source_manifest( - self.book_dir, - target_metadata, - target_progress, - self.book_dir / "source" / source_file, - ) - except SourceIntegrityError as exc: - raise MigrationCompatibilityError(f"source compatibility failed: {exc}") from exc - try: - manifest = parse_document(SchemaKind.SOURCE_MANIFEST, manifest).data - except SchemaError as exc: - raise MigrationCompatibilityError(f"candidate source manifest is invalid: {exc}") from exc - else: - manifest = copy.deepcopy(manifest_raw.migrated.data) - - self._verify_manifest(target_metadata, target_progress, manifest) - - non_metadata_writes: list[PlannedWrite] = [] - manifest_write = self._write_if_changed( - manifest_raw, - path="source-manifest.json", - kind=SchemaKind.SOURCE_MANIFEST, - target=manifest, - absent_from_version=0, - ) - if manifest_write is not None: - non_metadata_writes.append(manifest_write) - - ledger_write = self._write_if_changed( - ledger_raw, - path="review-ledger.json", - kind=SchemaKind.REVIEW_LEDGER, - target=ledger, - absent_from_version=0, - ) - if ledger_write is not None: - non_metadata_writes.append(ledger_write) - - for claim in claims: - write = self._write_if_changed( - claim, - path=claim.path, - kind=SchemaKind.CLAIM, - target=claim.migrated.data, - ) - if write is not None: - non_metadata_writes.append(write) - - progress_write = self._write_if_changed( - progress_raw, - path="progress.json", - kind=SchemaKind.PROGRESS, - target=target_progress, - ) - if progress_write is not None: - non_metadata_writes.append(progress_write) - - schema_versions = self._schema_history( - metadata_raw.migrated, - progress_raw.migrated, - ledger_raw, - manifest_raw, - claims, - ) - existing_history = target_workflow.get("upgrade_history", []) - if not isinstance(existing_history, list): - raise MigrationCompatibilityError("metadata workflow upgrade_history must be an array") - - workflow_without_new_history = copy.deepcopy(target_workflow) - target_metadata["workflow"] = workflow_without_new_history - - metadata_needs_change = ( - metadata_raw.migrated.changed - or target_metadata != metadata_raw.migrated.data - or bool(non_metadata_writes) - ) - if metadata_needs_change: - history_entry = { - "from_revision": from_revision, - "to_revision": target_revision, - "schema_versions": schema_versions, - } - workflow_with_history = copy.deepcopy(workflow_without_new_history) - workflow_with_history["upgrade_history"] = [*copy.deepcopy(existing_history), history_entry] - target_metadata["workflow"] = workflow_with_history - - try: - target_metadata = parse_document(SchemaKind.METADATA, target_metadata).data - target_progress = parse_document(SchemaKind.PROGRESS, target_progress).data - ledger = parse_document(SchemaKind.REVIEW_LEDGER, ledger).data - manifest = parse_document(SchemaKind.SOURCE_MANIFEST, manifest).data - except SchemaError as exc: - raise MigrationCompatibilityError(f"candidate workflow state is invalid: {exc}") from exc - - writes: list[PlannedWrite] = [] - for raw, path, kind, target, absent in ( - (manifest_raw, "source-manifest.json", SchemaKind.SOURCE_MANIFEST, manifest, 0), - (ledger_raw, "review-ledger.json", SchemaKind.REVIEW_LEDGER, ledger, 0), - ): - write = self._write_if_changed( - raw, - path=path, - kind=kind, - target=target, - absent_from_version=absent, - ) - if write is not None: - writes.append(write) - for claim in sorted(claims, key=lambda item: item.path): - write = self._write_if_changed( - claim, - path=claim.path, - kind=SchemaKind.CLAIM, - target=claim.migrated.data, - ) - if write is not None: - writes.append(write) - progress_write = self._write_if_changed( - progress_raw, - path="progress.json", - kind=SchemaKind.PROGRESS, - target=target_progress, - ) - if progress_write is not None: - writes.append(progress_write) - metadata_write = self._write_if_changed( - metadata_raw, - path="metadata.json", - kind=SchemaKind.METADATA, - target=target_metadata, - ) - if metadata_write is not None: - writes.append(metadata_write) - - return MigrationPlan( - book_slug=slug, - from_revision=from_revision, - to_revision=target_revision, - writes=tuple(writes), - lifecycle_downgrades=downgrades, - changed=bool(writes), - ) - - -class MigrationExecutor: - """Apply and recover one journaled multi-document workflow upgrade.""" - - def __init__( - self, - repository: WorkflowStateRepository, - planner: MigrationPlanner, - *, - coordination: Any, - ): - self.repository = repository - self.planner = planner - self.coordination = coordination - - @staticmethod - def _unchanged(plan: MigrationPlan) -> MigrationResult: - return MigrationResult( - book_slug=plan.book_slug, - from_revision=plan.from_revision, - to_revision=plan.to_revision, - outcome="unchanged", - migrated_paths=(), - lifecycle_downgrades=plan.lifecycle_downgrades, - ) - - @staticmethod - def _installed_from_plan(plan: MigrationPlan) -> dict[str, Any]: - metadata = plan.write_for("metadata.json") - if metadata is None: - raise MigrationCompatibilityError( - "changed workflow upgrade plan must include metadata provenance last" - ) - workflow = metadata.target_data.get("workflow") - if not isinstance(workflow, Mapping): - raise MigrationCompatibilityError("migration target metadata workflow is unavailable") - repository = workflow.get("repository") - resolved = workflow.get("resolved_revision") - if repository != CANONICAL_REPOSITORY or resolved != plan.to_revision: - raise MigrationCompatibilityError("migration target metadata provenance is inconsistent") - return { - "canonical_repository": repository, - "requested_ref": workflow.get("requested_ref"), - "resolved_revision": resolved, - } - - @staticmethod - def _journal_for(plan: MigrationPlan) -> dict[str, Any]: - documents: list[dict[str, Any]] = [] - for write in plan.writes: - original = write.original_bytes - documents.append( - { - "path": write.path, - "kind": write.kind.value, - "original_exists": write.original_exists, - "original_revision": write.original_version if write.original_exists else None, - "original_sha256": ( - hashlib.sha256(original).hexdigest() - if write.original_exists and original is not None - else None - ), - "original_bytes_base64": ( - base64.b64encode(original).decode("ascii") - if write.original_exists and original is not None - else None - ), - "target_sha256": hashlib.sha256(write.target_bytes).hexdigest(), - "resulting_revision": None, - } - ) - return { - "schema_version": 1, - "operation": "workflow_upgrade", - "book_slug": plan.book_slug, - "from_revision": plan.from_revision, - "to_revision": plan.to_revision, - "phase": "prepared", - "documents": documents, - } - - def _load_journal(self) -> tuple[dict[str, Any], str]: - try: - return load_migration_journal(self.repository.storage) - except MigrationJournalError as exc: - raise MigrationConflict(f"migration journal is invalid; recovery is blocked: {exc}") from exc - except StorageError: - raise - - def _acquire(self, session_id: str): - try: - return self.coordination.acquire( - operation="workflow_upgrade", - session_id=session_id, - ) - except CoordinationError as exc: - raise MigrationConflict(f"cannot acquire workflow upgrade coordination: {exc}") from exc - - def _release(self, lease) -> None: - try: - self.coordination.release(lease) - except CoordinationError as exc: - raise MigrationConflict(f"cannot release workflow upgrade coordination: {exc}") from exc - - def _preflight_plan(self, plan: MigrationPlan) -> Mapping[str, Any]: - installed = self._installed_from_plan(plan) - fresh = self.planner.plan( - slug=plan.book_slug, - to_revision=plan.to_revision, - installed=installed, - ) - if fresh != plan: - raise MigrationConflict("workflow upgrade plan changed before commit; re-plan required") - return installed - - @staticmethod - def _hash(content: bytes) -> str: - return hashlib.sha256(content).hexdigest() - - def _classify(self, entry: Mapping[str, Any]) -> str: - path = entry["path"] - try: - current = self.repository.storage.read(path) - except StorageNotFound: - return "original" if not entry["original_exists"] else "unknown" - except StorageError as exc: - raise MigrationConflict(f"cannot inspect migration recovery path {path}: {exc}") from exc - - current_hash = self._hash(current.content) - if current_hash == entry["target_sha256"]: - return "target" - if entry["original_exists"] and current_hash == entry["original_sha256"]: - return "original" - return "unknown" - - def _classifications(self, journal: Mapping[str, Any]) -> list[str]: - return [self._classify(entry) for entry in journal["documents"]] - - @staticmethod - def _decode_original(entry: Mapping[str, Any]) -> bytes: - encoded = entry.get("original_bytes_base64") - if not isinstance(encoded, str): - raise MigrationConflict(f"journal original bytes are unavailable for {entry.get('path')}") - return base64.b64decode(encoded.encode("ascii"), validate=True) - - def _delete_journal(self, version: str) -> None: - try: - self.repository.storage.delete_if_version(MIGRATION_PATH, version) - except (StorageNotFound, StorageVersionConflict) as exc: - raise MigrationConflict("migration journal changed before deletion") from exc - except StorageError as exc: - raise MigrationConflict(f"cannot delete migration journal: {exc}") from exc - - def _rollback_loaded( - self, - journal: Mapping[str, Any], - journal_version: str, - ) -> None: - states = self._classifications(journal) - unknown = [ - entry["path"] - for entry, state in zip(journal["documents"], states) - if state == "unknown" - ] - if unknown: - raise MigrationConflict( - "migration recovery found unknown concurrent state at " - + ", ".join(unknown) - + "; journal preserved" - ) - - for entry, state in reversed(list(zip(journal["documents"], states))): - if state != "target": - continue - path = entry["path"] - try: - current = self.repository.storage.read(path) - except StorageError as exc: - raise MigrationConflict( - f"migration recovery path {path} changed before rollback" - ) from exc - if self._hash(current.content) != entry["target_sha256"]: - raise MigrationConflict( - f"migration recovery path {path} changed before rollback; journal preserved" - ) - try: - if entry["original_exists"]: - self.repository.storage.write_if_version( - path, - self._decode_original(entry), - current.version, - ) - else: - self.repository.storage.delete_if_version(path, current.version) - except (StorageNotFound, StorageVersionConflict) as exc: - raise MigrationConflict( - f"migration recovery path {path} changed during rollback; journal preserved" - ) from exc - except StorageError as exc: - raise MigrationConflict( - f"cannot restore migration recovery path {path}; journal preserved: {exc}" - ) from exc - - after = self._classifications(journal) - not_original = [ - entry["path"] - for entry, state in zip(journal["documents"], after) - if state != "original" - ] - if not_original: - raise MigrationConflict( - "migration rollback could not prove original state at " - + ", ".join(not_original) - + "; journal preserved" - ) - self._delete_journal(journal_version) - - def _rollback_durable_journal(self) -> None: - try: - journal, version = self._load_journal() - except StorageNotFound: - raise MigrationConflict("migration failed but recovery journal is missing") - except StorageError as exc: - raise MigrationConflict(f"cannot read migration journal for rollback: {exc}") from exc - self._rollback_loaded(journal, version) - - def _post_validate(self, plan: MigrationPlan, installed: Mapping[str, Any]) -> None: - fresh = self.planner.plan( - slug=plan.book_slug, - to_revision=plan.to_revision, - installed=installed, - ) - if fresh.changed: - raise MigrationConflict( - "workflow upgrade target did not converge to a strict compatible no-op" - ) - - def _execute_locked(self, plan: MigrationPlan, *, session_id: str) -> MigrationResult: - if not plan.changed: - return self._unchanged(plan) - - try: - self.repository.storage.read(MIGRATION_PATH) - except StorageNotFound: - pass - except StorageError as exc: - raise MigrationConflict(f"cannot inspect migration journal: {exc}") from exc - else: - raise MigrationConflict("an unfinished migration journal already exists; recover it first") - - installed = self._preflight_plan(plan) - journal = self._journal_for(plan) - try: - journal_version = self.repository.storage.create_if_absent( - MIGRATION_PATH, - serialize_migration_journal(journal), - ) - except StorageAlreadyExists as exc: - raise MigrationConflict("migration journal appeared before commit") from exc - except (StorageError, MigrationJournalError) as exc: - raise MigrationConflict(f"cannot create migration journal: {exc}") from exc - - try: - for index, write in enumerate(plan.writes): - if write.original_exists: - if write.original_version is None: - raise MigrationCompatibilityError( - f"migration plan lacks original revision for {write.path}" - ) - resulting = self.repository.storage.write_if_version( - write.path, - write.target_bytes, - write.original_version, - ) - else: - resulting = self.repository.storage.create_if_absent( - write.path, - write.target_bytes, - ) - - journal["documents"][index]["resulting_revision"] = resulting - journal_version = self.repository.storage.write_if_version( - MIGRATION_PATH, - serialize_migration_journal(journal), - journal_version, - ) - - journal["phase"] = "applied" - journal_version = self.repository.storage.write_if_version( - MIGRATION_PATH, - serialize_migration_journal(journal), - journal_version, - ) - self._post_validate(plan, installed) - self._delete_journal(journal_version) - except Exception as exc: - try: - self._rollback_durable_journal() - except MigrationError as recovery_exc: - raise recovery_exc from exc - if isinstance(exc, MigrationError): - raise exc - raise MigrationConflict( - f"workflow upgrade failed and exact original state was restored: {exc}" - ) from exc - - return MigrationResult( - book_slug=plan.book_slug, - from_revision=plan.from_revision, - to_revision=plan.to_revision, - outcome="changed", - migrated_paths=tuple(write.path for write in plan.writes), - lifecycle_downgrades=plan.lifecycle_downgrades, - ) - - def execute(self, plan: MigrationPlan, *, session_id: str) -> MigrationResult: - if not isinstance(plan, MigrationPlan): - raise MigrationCompatibilityError("execute requires a MigrationPlan") - if not plan.changed: - return self._unchanged(plan) - lease = self._acquire(session_id) - try: - result = self._execute_locked(plan, session_id=session_id) - except Exception: - try: - self._release(lease) - except MigrationError: - pass - raise - self._release(lease) - return result - - def _journal_downgrades(self, journal: Mapping[str, Any]) -> tuple[int, ...]: - progress_entry = next( - (entry for entry in journal["documents"] if entry["kind"] == SchemaKind.PROGRESS.value), - None, - ) - if progress_entry is None or not progress_entry["original_exists"]: - return () - try: - original = json.loads(self._decode_original(progress_entry).decode("utf-8")) - current = json.loads(self.repository.storage.read(progress_entry["path"]).content.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError, StorageError): - return () - if not isinstance(original, Mapping) or not isinstance(current, Mapping): - return () - old_chapters = original.get("chapters") - new_chapters = current.get("chapters") - if not isinstance(old_chapters, list) or not isinstance(new_chapters, list): - return () - new_by_number = { - chapter.get("number"): chapter - for chapter in new_chapters - if isinstance(chapter, Mapping) and type(chapter.get("number")) is int - } - downgrades = [ - chapter["number"] - for chapter in old_chapters - if isinstance(chapter, Mapping) - and type(chapter.get("number")) is int - and chapter.get("status") == "reviewed" - and isinstance(new_by_number.get(chapter["number"]), Mapping) - and new_by_number[chapter["number"]].get("status") == "translated" - ] - return tuple(sorted(downgrades)) - - def _recover_locked( - self, - journal: Mapping[str, Any], - journal_version: str, - *, - session_id: str, - installed: Mapping[str, Any], - ) -> MigrationResult: - MigrationPlanner._installed_target(installed, journal["to_revision"]) - states = self._classifications(journal) - unknown = [ - entry["path"] - for entry, state in zip(journal["documents"], states) - if state == "unknown" - ] - if unknown: - raise MigrationConflict( - "migration recovery found unknown concurrent state at " - + ", ".join(unknown) - + "; journal preserved" - ) - - if states and all(state == "target" for state in states): - post = self.planner.plan( - slug=journal["book_slug"], - to_revision=journal["to_revision"], - installed=installed, - ) - if post.changed: - raise MigrationConflict( - "all journaled targets exist but full workflow state is not a strict no-op; journal preserved" - ) - downgrades = self._journal_downgrades(journal) - self._delete_journal(journal_version) - return MigrationResult( - book_slug=journal["book_slug"], - from_revision=journal["from_revision"], - to_revision=journal["to_revision"], - outcome="recovered", - migrated_paths=tuple(entry["path"] for entry in journal["documents"]), - lifecycle_downgrades=downgrades, - ) - - self._rollback_loaded(journal, journal_version) - fresh = self.planner.plan( - slug=journal["book_slug"], - to_revision=journal["to_revision"], - installed=installed, - ) - return self._execute_locked(fresh, session_id=session_id) - - def recover( - self, - *, - session_id: str, - installed: Mapping[str, Any], - ) -> MigrationResult | None: - try: - initial, initial_version = self._load_journal() - except StorageNotFound: - return None - except StorageError as exc: - raise MigrationConflict(f"cannot read migration journal: {exc}") from exc - - lease = self._acquire(session_id) - try: - try: - journal, version = self._load_journal() - except StorageNotFound as exc: - raise MigrationConflict("migration journal disappeared during recovery admission") from exc - except StorageError as exc: - raise MigrationConflict(f"cannot re-read migration journal: {exc}") from exc - if version != initial_version or journal != initial: - raise MigrationConflict("migration journal changed during recovery admission") - result = self._recover_locked( - journal, - version, - session_id=session_id, - installed=installed, - ) - except Exception: - try: - self._release(lease) - except MigrationError: - pass - raise - self._release(lease) - return result diff --git a/scripts/workflow_v2/migrations_cli.py b/scripts/workflow_v2/migrations_cli.py deleted file mode 100644 index 4e72872..0000000 --- a/scripts/workflow_v2/migrations_cli.py +++ /dev/null @@ -1,198 +0,0 @@ -"""CLI adapter for explicit Workflow v2 workspace upgrades.""" - -from __future__ import annotations - -import argparse -import json -from collections.abc import Callable, Mapping -from pathlib import Path -from typing import Any - -from .coordination import BookCoordinationManager -from .filesystem import FilesystemStorage -from .migrations import ( - CANONICAL_REPOSITORY, - MigrationError, - MigrationExecutor, - MigrationPlanner, - MigrationResult, -) -from .repository import WorkflowStateRepository -from .storage import StorageError - - -INSTALL_PROVENANCE_PATH = ".book-translator-install.json" - - -class MigrationCliError(RuntimeError): - """Installed provenance or CLI wiring is unusable for an explicit upgrade.""" - - -def load_install_provenance(root: Path) -> dict[str, str | None]: - """Load the workflow revision actually installed at this runtime root.""" - - path = Path(root) / INSTALL_PROVENANCE_PATH - try: - text = path.read_text(encoding="utf-8") - except (OSError, UnicodeError) as exc: - raise MigrationCliError(f"cannot read {INSTALL_PROVENANCE_PATH}: {exc}") from exc - try: - raw = json.loads(text) - except json.JSONDecodeError as exc: - raise MigrationCliError(f"invalid {INSTALL_PROVENANCE_PATH}: {exc}") from exc - if not isinstance(raw, Mapping): - raise MigrationCliError(f"{INSTALL_PROVENANCE_PATH} must be a JSON object") - if raw.get("schema_version") != 1: - raise MigrationCliError(f"{INSTALL_PROVENANCE_PATH} schema_version must equal 1") - if raw.get("canonical_repository") != CANONICAL_REPOSITORY: - raise MigrationCliError( - f"{INSTALL_PROVENANCE_PATH} canonical_repository must equal {CANONICAL_REPOSITORY!r}" - ) - - requested_ref = raw.get("requested_ref") - if requested_ref is not None and ( - not isinstance(requested_ref, str) or not requested_ref.strip() - ): - raise MigrationCliError( - f"{INSTALL_PROVENANCE_PATH} requested_ref must be null or a non-empty string" - ) - resolved_revision = raw.get("resolved_revision") - if not isinstance(resolved_revision, str) or not resolved_revision.strip(): - raise MigrationCliError( - f"{INSTALL_PROVENANCE_PATH} resolved_revision must be a non-empty string" - ) - install_root = raw.get("install_root") - if not isinstance(install_root, str) or not install_root.strip(): - raise MigrationCliError( - f"{INSTALL_PROVENANCE_PATH} install_root must be a non-empty string" - ) - - return { - "canonical_repository": CANONICAL_REPOSITORY, - "requested_ref": requested_ref, - "resolved_revision": resolved_revision, - } - - -def _book_directory(root: Path, slug: str) -> Path: - if ( - not isinstance(slug, str) - or not slug - or "/" in slug - or "\\" in slug - or slug in {".", ".."} - ): - raise MigrationCliError("book slug must be one directory name under books/") - books_root = (Path(root) / "books").resolve(strict=False) - book_dir = (books_root / slug).resolve(strict=False) - try: - book_dir.relative_to(books_root) - except ValueError as exc: - raise MigrationCliError("book slug escapes books/") from exc - if not book_dir.is_dir(): - raise MigrationCliError(f"Book directory does not exist: books/{slug}") - return book_dir - - -def _artifact_reader(book_dir: Path) -> Callable[[str], bytes]: - root = book_dir.resolve(strict=False) - - def read(relative_path: str) -> bytes: - if not isinstance(relative_path, str) or not relative_path.strip(): - raise OSError("artifact path must be a non-empty relative path") - target = (root / relative_path).resolve(strict=False) - try: - target.relative_to(root) - except ValueError as exc: - raise OSError(f"artifact path escapes book workspace: {relative_path}") from exc - return target.read_bytes() - - return read - - -def _payload(result: MigrationResult) -> dict[str, Any]: - return { - "book_slug": result.book_slug, - "from_revision": result.from_revision, - "lifecycle_downgrades": list(result.lifecycle_downgrades), - "migrated_paths": list(result.migrated_paths), - "outcome": result.outcome, - "to_revision": result.to_revision, - } - - -def workflow_upgrade_command( - args: argparse.Namespace, - root: Path, - *, - error_factory: Callable[[str], Exception], -) -> int: - try: - installed = load_install_provenance(root) - book_dir = _book_directory(root, args.slug) - repository = WorkflowStateRepository(FilesystemStorage(book_dir)) - planner = MigrationPlanner( - repository, - book_dir=book_dir, - artifact_reader=_artifact_reader(book_dir), - ) - coordination = BookCoordinationManager(repository) - executor = MigrationExecutor(repository, planner, coordination=coordination) - session_id = f"workflow-upgrade:{args.slug}" - - result = executor.recover(session_id=session_id, installed=installed) - if result is None: - plan = planner.plan( - slug=args.slug, - to_revision=args.to_revision, - installed=installed, - ) - result = executor.execute(plan, session_id=session_id) - except (MigrationCliError, MigrationError, StorageError, OSError, ValueError) as exc: - raise error_factory(str(exc)) from exc - - payload = _payload(result) - if args.json: - print(json.dumps(payload, ensure_ascii=False, sort_keys=True)) - else: - paths = ",".join(result.migrated_paths) if result.migrated_paths else "-" - downgrades = ",".join(str(item) for item in result.lifecycle_downgrades) or "-" - print( - f"workflow-upgrade {result.book_slug}: outcome={result.outcome} " - f"from={result.from_revision or '-'} to={result.to_revision} " - f"paths={paths} reviewed_to_translated={downgrades}" - ) - return 0 - - -def register_migration_command( - subparsers: argparse._SubParsersAction, - root: Path, - *, - error_factory: Callable[[str], Exception], -) -> None: - """Register the explicit, recoverable workflow-upgrade command.""" - - upgrade = subparsers.add_parser( - "workflow-upgrade", - help="Explicitly upgrade one book workspace to the installed Workflow v2 revision.", - ) - upgrade.add_argument("slug", help="Book slug under books/.") - upgrade.add_argument( - "--to", - dest="to_revision", - required=True, - help="Installed resolved workflow revision to adopt.", - ) - upgrade.add_argument( - "--json", - action="store_true", - help="Emit deterministic machine-readable JSON.", - ) - upgrade.set_defaults( - func=lambda args: workflow_upgrade_command( - args, - root, - error_factory=error_factory, - ) - ) diff --git a/tests/test_workflow_v2_migration_planner.py b/tests/test_workflow_v2_migration_planner.py deleted file mode 100644 index 8b373a5..0000000 --- a/tests/test_workflow_v2_migration_planner.py +++ /dev/null @@ -1,343 +0,0 @@ -import hashlib -import json -import sys -import tempfile -import unittest -from datetime import datetime, timezone -from pathlib import Path - - -PROJECT_ROOT = Path(__file__).resolve().parents[1] -SCRIPTS = PROJECT_ROOT / "scripts" -sys.path.insert(0, str(SCRIPTS)) - -from workflow_v2 import FilesystemStorage, WorkflowStateRepository -from workflow_v2.migrations import MigrationCompatibilityError - -try: - from workflow_v2.migrations import MigrationPlanner -except ImportError: - MigrationPlanner = None - - -CANONICAL = "https://github.com/tim8es/book-translator" -OLD = "old-revision" -NEW = "new-revision" -NOW = datetime(2026, 9, 7, 12, 0, tzinfo=timezone.utc) - - -class WorkflowV2MigrationPlannerTests(unittest.TestCase): - def setUp(self): - self.temp = tempfile.TemporaryDirectory() - self.book_dir = Path(self.temp.name) / "books" / "legacy" - for name in ("source", "extracted", "translated", "output", ".workflow/claims"): - (self.book_dir / name).mkdir(parents=True, exist_ok=True) - (self.book_dir / "source" / "legacy.md").write_bytes(b"source-book\n") - (self.book_dir / "extracted" / "001-one.md").write_bytes(b"source chapter\n") - (self.book_dir / "translated" / "001-one.md").write_bytes("перевод\n".encode("utf-8")) - self.storage = FilesystemStorage(self.book_dir) - self.repository = WorkflowStateRepository(self.storage) - - def tearDown(self): - self.temp.cleanup() - - def require_api(self): - self.assertIsNotNone(MigrationPlanner, "MigrationPlanner is not implemented") - - @staticmethod - def installed(revision=NEW): - return { - "canonical_repository": CANONICAL, - "requested_ref": "refactor/workflow-engine-v2", - "resolved_revision": revision, - } - - def metadata( - self, - *, - revision=OLD, - version=True, - review_marker=False, - private=False, - requested_ref="legacy-ref", - ): - data = { - "schema_version": 1, - "title": "Legacy", - "author": "Author", - "source_language": "en", - "target_language": "ru", - "source_format": "markdown", - "source_file": "legacy.md", - "chapter_count": 1, - "imported_at": "2026-09-01T10:00:00+00:00", - "workflow": { - "repository": CANONICAL, - "requested_ref": requested_ref, - "resolved_revision": revision, - }, - } - if review_marker: - data["workflow"]["review_evidence"] = "review-ledger-v1" - if private: - source_bytes = b"private-source-never-stored" - data["source"] = { - "storage_mode": "private_external", - "filename": "legacy.md", - "size_bytes": len(source_bytes), - "sha256": hashlib.sha256(source_bytes).hexdigest(), - } - if not version: - data.pop("schema_version") - return data - - def progress(self, *, status="reviewed", version=True): - data = { - "schema_version": 1, - "book_slug": "legacy", - "chapters": [ - { - "number": 1, - "title": "One", - "slug": "one", - "source_path": "extracted/001-one.md", - "translation_path": "translated/001-one.md", - "status": status, - } - ], - } - if not version: - data.pop("schema_version") - return data - - def ledger(self, *, revision=NEW, with_pass=False, version=True): - records = [] - if with_pass: - source_sha = hashlib.sha256((self.book_dir / "extracted/001-one.md").read_bytes()).hexdigest() - translation_sha = hashlib.sha256((self.book_dir / "translated/001-one.md").read_bytes()).hexdigest() - records.append( - { - "record_id": "a" * 32, - "sequence": 1, - "unit_id": "chapter-000001", - "outcome": "PASS", - "source_sha256": source_sha, - "translation_sha256": translation_sha, - "workflow_revision": revision, - "review_contract_revision": f"docs/TRANSLATION.md@{revision}", - "reviewer_session_id": "reviewer-1", - "reviewed_at": "2026-09-01T11:00:00Z", - "state_revision": "progress-old", - "review_commit": None, - "correction_round": 0, - "supersedes_record_id": None, - } - ) - data = { - "schema_version": 1, - "book_slug": "legacy", - "next_sequence": len(records) + 1, - "records": records, - } - if not version: - data.pop("schema_version") - return data - - def manifest(self, *, version=True): - source = self.book_dir / "source" / "legacy.md" - extracted = self.book_dir / "extracted" / "001-one.md" - data = { - "schema_version": 1, - "source_file": "legacy.md", - "source_format": "markdown", - "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), - "chapter_count": 1, - "extracted": [ - { - "number": 1, - "title": "One", - "path": "extracted/001-one.md", - "sha256": hashlib.sha256(extracted.read_bytes()).hexdigest(), - } - ], - } - if not version: - data.pop("schema_version") - return data - - def write_json(self, path, data): - target = self.book_dir / path - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - - def artifact_reader(self, relative_path): - return (self.book_dir / relative_path).read_bytes() - - def planner(self): - self.require_api() - return MigrationPlanner( - self.repository, - book_dir=self.book_dir, - artifact_reader=self.artifact_reader, - now=lambda: NOW, - ) - - def durable_snapshot(self): - return { - path.relative_to(self.book_dir).as_posix(): path.read_bytes() - for path in sorted(self.book_dir.rglob("*")) - if path.is_file() - } - - def test_legacy_embedded_workspace_plans_manifest_ledger_review_downgrade_and_metadata_pin(self): - self.write_json("metadata.json", self.metadata(version=False)) - self.write_json("progress.json", self.progress(version=False, status="reviewed")) - before = self.durable_snapshot() - - plan = self.planner().plan(slug="legacy", to_revision=NEW, installed=self.installed()) - - self.assertTrue(plan.changed) - self.assertEqual(plan.from_revision, OLD) - self.assertEqual(plan.to_revision, NEW) - self.assertEqual(plan.lifecycle_downgrades, (1,)) - self.assertEqual(self.durable_snapshot(), before, "planning must be read-only") - - metadata = plan.write_for("metadata.json") - progress = plan.write_for("progress.json") - ledger = plan.write_for("review-ledger.json") - manifest = plan.write_for("source-manifest.json") - self.assertIsNotNone(metadata) - self.assertIsNotNone(progress) - self.assertIsNotNone(ledger) - self.assertIsNotNone(manifest) - self.assertEqual(metadata.target_data["workflow"]["resolved_revision"], NEW) - self.assertEqual(metadata.target_data["workflow"]["review_evidence"], "review-ledger-v1") - self.assertEqual(progress.target_data["chapters"][0]["status"], "translated") - self.assertEqual(ledger.target_data["records"], []) - self.assertEqual(manifest.target_data["source_file"], "legacy.md") - - def test_target_must_equal_installed_revision_and_failure_is_read_only(self): - self.write_json("metadata.json", self.metadata(version=False)) - self.write_json("progress.json", self.progress(version=False, status="translated")) - before = self.durable_snapshot() - with self.assertRaises(MigrationCompatibilityError) as ctx: - self.planner().plan(slug="legacy", to_revision="other", installed=self.installed()) - self.assertIn("installed", str(ctx.exception).lower()) - self.assertEqual(self.durable_snapshot(), before) - - def test_current_pass_for_same_target_revision_preserves_reviewed_lifecycle(self): - self.write_json("metadata.json", self.metadata(revision=NEW, version=False, review_marker=True)) - self.write_json("progress.json", self.progress(version=False, status="reviewed")) - self.write_json("review-ledger.json", self.ledger(revision=NEW, with_pass=True, version=False)) - self.write_json("source-manifest.json", self.manifest(version=False)) - - plan = self.planner().plan(slug="legacy", to_revision=NEW, installed=self.installed()) - self.assertEqual(plan.lifecycle_downgrades, ()) - progress = plan.write_for("progress.json") - self.assertIsNotNone(progress, "v0 progress still requires schema migration") - self.assertEqual(progress.target_data["chapters"][0]["status"], "reviewed") - - def test_reviewed_unit_without_valid_translation_is_incompatible(self): - self.write_json("metadata.json", self.metadata(version=False)) - self.write_json("progress.json", self.progress(version=False, status="reviewed")) - (self.book_dir / "translated/001-one.md").write_bytes(b"") - with self.assertRaises(MigrationCompatibilityError) as ctx: - self.planner().plan(slug="legacy", to_revision=NEW, installed=self.installed()) - self.assertIn("translation", str(ctx.exception).lower()) - - def test_private_external_manifest_is_reconstructed_without_source_binary(self): - (self.book_dir / "source/legacy.md").unlink() - self.write_json("metadata.json", self.metadata(version=False, private=True)) - self.write_json("progress.json", self.progress(version=False, status="translated")) - - plan = self.planner().plan(slug="legacy", to_revision=NEW, installed=self.installed()) - manifest = plan.write_for("source-manifest.json") - self.assertIsNotNone(manifest) - self.assertEqual(manifest.target_data["source_storage_mode"], "private_external") - self.assertEqual(manifest.target_data["source_sha256"], self.metadata(private=True)["source"]["sha256"]) - self.assertFalse((self.book_dir / "source/legacy.md").exists()) - - def test_unprovable_source_identity_fails_before_mutation(self): - (self.book_dir / "source/legacy.md").unlink() - self.write_json("metadata.json", self.metadata(version=False)) - self.write_json("progress.json", self.progress(version=False, status="translated")) - before = self.durable_snapshot() - with self.assertRaises(MigrationCompatibilityError) as ctx: - self.planner().plan(slug="legacy", to_revision=NEW, installed=self.installed()) - self.assertIn("source", str(ctx.exception).lower()) - self.assertEqual(self.durable_snapshot(), before) - - def test_live_claim_and_finalization_block_but_expired_claim_can_be_migrated_without_extension(self): - self.write_json("metadata.json", self.metadata(version=False)) - self.write_json("progress.json", self.progress(version=False, status="translated")) - claim = { - "claim_id": "b" * 32, - "unit_id": "chapter-000001", - "role": "translator", - "session_id": "worker", - "base_revision": "base", - "base_commit": None, - "workflow_revision": OLD, - "claimed_at": "2026-09-07T11:30:00Z", - "expires_at": "2026-09-07T12:30:00Z", - } - self.write_json(".workflow/claims/chapter-000001.json", claim) - with self.assertRaises(MigrationCompatibilityError) as live: - self.planner().plan(slug="legacy", to_revision=NEW, installed=self.installed()) - self.assertIn("claim", str(live.exception).lower()) - - claim["claimed_at"] = "2026-09-07T10:00:00Z" - claim["expires_at"] = "2026-09-07T11:00:00Z" - self.write_json(".workflow/claims/chapter-000001.json", claim) - plan = self.planner().plan(slug="legacy", to_revision=NEW, installed=self.installed()) - migrated_claim = plan.write_for(".workflow/claims/chapter-000001.json") - self.assertIsNotNone(migrated_claim) - self.assertEqual(migrated_claim.target_data["expires_at"], "2026-09-07T11:00:00Z") - - (self.book_dir / ".workflow/claims/chapter-000001.json").unlink() - self.write_json(".workflow/finalization.json", {"anything": "present"}) - with self.assertRaises(MigrationCompatibilityError) as finalizing: - self.planner().plan(slug="legacy", to_revision=NEW, installed=self.installed()) - self.assertIn("final", str(finalizing.exception).lower()) - - def test_requested_ref_drift_requires_provenance_write(self): - self.write_json( - "metadata.json", - self.metadata(revision=NEW, review_marker=True, requested_ref="legacy-ref"), - ) - self.write_json("progress.json", self.progress(status="translated")) - self.write_json("review-ledger.json", self.ledger(revision=NEW)) - self.write_json("source-manifest.json", self.manifest()) - - plan = self.planner().plan(slug="legacy", to_revision=NEW, installed=self.installed()) - self.assertTrue(plan.changed) - metadata = plan.write_for("metadata.json") - self.assertIsNotNone(metadata) - self.assertEqual( - metadata.target_data["workflow"]["requested_ref"], - "refactor/workflow-engine-v2", - ) - - def test_fully_current_workspace_is_true_noop(self): - self.write_json( - "metadata.json", - self.metadata( - revision=NEW, - review_marker=True, - requested_ref="refactor/workflow-engine-v2", - ), - ) - self.write_json("progress.json", self.progress(status="translated")) - self.write_json("review-ledger.json", self.ledger(revision=NEW)) - self.write_json("source-manifest.json", self.manifest()) - before = self.durable_snapshot() - - plan = self.planner().plan(slug="legacy", to_revision=NEW, installed=self.installed()) - self.assertFalse(plan.changed) - self.assertEqual(plan.writes, ()) - self.assertEqual(plan.lifecycle_downgrades, ()) - self.assertEqual(self.durable_snapshot(), before) - - -if __name__ == "__main__": - unittest.main() \ No newline at end of file diff --git a/tests/test_workflow_v2_migration_reliability.py b/tests/test_workflow_v2_migration_reliability.py deleted file mode 100644 index d53c0f7..0000000 --- a/tests/test_workflow_v2_migration_reliability.py +++ /dev/null @@ -1,294 +0,0 @@ -import json -import sys -import tempfile -import unittest -from datetime import datetime, timedelta, timezone -from pathlib import Path - - -PROJECT_ROOT = Path(__file__).resolve().parents[1] -SCRIPTS = PROJECT_ROOT / "scripts" -sys.path.insert(0, str(SCRIPTS)) - -from workflow_v2 import FilesystemStorage, WorkflowStateRepository -from workflow_v2.coordination import BookCoordinationManager -from workflow_v2.migration_journal import MIGRATION_PATH, load_migration_journal -from workflow_v2.migrations import MigrationError, MigrationExecutor, MigrationPlanner -from workflow_v2.storage import StorageNotFound, StorageVersionConflict - - -CANONICAL = "https://github.com/tim8es/book-translator" -OLD = "old-revision" -NEW = "new-revision" -REQUESTED = "refactor/workflow-engine-v2" -NOW = datetime(2026, 9, 7, 12, 0, tzinfo=timezone.utc) -RECOVERY_NOW = NOW + timedelta(minutes=2) - - -class SimulatedProcessCrash(BaseException): - """Bypass executor Exception handlers to model abrupt process loss.""" - - -class FaultStorage: - def __init__(self, inner): - self.inner = inner - self.crash = None - self.conflict = None - self._crashed = False - self._conflicted = False - - def read(self, path): - return self.inner.read(path) - - def _before(self, operation, path): - if self.crash == (operation, path) and not self._crashed: - self._crashed = True - raise SimulatedProcessCrash(f"crash at {operation}:{path}") - if self.conflict == (operation, path) and not self._conflicted: - self._conflicted = True - raise StorageVersionConflict(f"injected conflict at {operation}:{path}") - - def write_if_version(self, path, content, expected_version): - self._before("write", path) - return self.inner.write_if_version(path, content, expected_version) - - def create_if_absent(self, path, content): - self._before("create", path) - return self.inner.create_if_absent(path, content) - - def delete_if_version(self, path, expected_version): - self._before("delete", path) - return self.inner.delete_if_version(path, expected_version) - - def list(self, prefix=""): - return self.inner.list(prefix) - - -class WorkflowV2MigrationReliabilityTests(unittest.TestCase): - def setUp(self): - self.temp = tempfile.TemporaryDirectory() - self.book_dir = Path(self.temp.name) / "books" / "legacy" - for name in ("source", "extracted", "translated", "output", ".workflow/claims"): - (self.book_dir / name).mkdir(parents=True, exist_ok=True) - (self.book_dir / "source/legacy.md").write_bytes(b"source-book\n") - (self.book_dir / "extracted/001-one.md").write_bytes(b"source chapter\n") - (self.book_dir / "translated/001-one.md").write_bytes("перевод\n".encode("utf-8")) - self._write_json("metadata.json", self._metadata()) - self._write_json("progress.json", self._progress()) - self.base_storage = FilesystemStorage(self.book_dir) - - def tearDown(self): - self.temp.cleanup() - - @staticmethod - def installed(): - return { - "canonical_repository": CANONICAL, - "requested_ref": REQUESTED, - "resolved_revision": NEW, - } - - @staticmethod - def _metadata(): - return { - "title": "Legacy", - "author": "Author", - "source_language": "en", - "target_language": "ru", - "source_format": "markdown", - "source_file": "legacy.md", - "chapter_count": 1, - "imported_at": "2026-09-01T10:00:00+00:00", - "workflow": { - "repository": CANONICAL, - "requested_ref": "legacy-ref", - "resolved_revision": OLD, - }, - } - - @staticmethod - def _progress(): - return { - "book_slug": "legacy", - "chapters": [ - { - "number": 1, - "title": "One", - "slug": "one", - "source_path": "extracted/001-one.md", - "translation_path": "translated/001-one.md", - "status": "translated", - } - ], - } - - def _write_json(self, path, data): - target = self.book_dir / path - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - - def _runtime(self, storage, *, now=NOW): - repository = WorkflowStateRepository(storage) - planner = MigrationPlanner( - repository, - book_dir=self.book_dir, - artifact_reader=lambda path: (self.book_dir / path).read_bytes(), - now=lambda: now, - ) - coordination = BookCoordinationManager( - repository, - now=lambda: now, - id_factory=lambda: "d" * 32, - ) - executor = MigrationExecutor(repository, planner, coordination=coordination) - return repository, planner, executor - - def _plan(self, storage=None): - _, planner, _ = self._runtime(storage or self.base_storage) - return planner.plan(slug="legacy", to_revision=NEW, installed=self.installed()) - - def _recover_fresh(self): - _, planner, executor = self._runtime(self.base_storage, now=RECOVERY_NOW) - result = executor.recover(session_id="recovery-session", installed=self.installed()) - return planner, result - - def _snapshot(self, *, include_transient=False): - result = {} - for path in sorted(self.book_dir.rglob("*")): - if not path.is_file(): - continue - relative = path.relative_to(self.book_dir).as_posix() - if not include_transient and relative in { - MIGRATION_PATH, - ".workflow/coordination-lock.json", - }: - continue - result[relative] = path.read_bytes() - return result - - def assert_converged(self, planner): - final = planner.plan(slug="legacy", to_revision=NEW, installed=self.installed()) - self.assertFalse(final.changed) - with self.assertRaises(StorageNotFound): - self.base_storage.read(MIGRATION_PATH) - with self.assertRaises(StorageNotFound): - self.base_storage.read(".workflow/coordination-lock.json") - - def test_prepared_journal_with_no_target_writes_recovers_from_fresh_process(self): - faults = FaultStorage(self.base_storage) - faults.crash = ("create", "source-manifest.json") - _, planner, executor = self._runtime(faults) - plan = planner.plan(slug="legacy", to_revision=NEW, installed=self.installed()) - before = self._snapshot() - - with self.assertRaises(SimulatedProcessCrash): - executor.execute(plan, session_id="crashed-session") - - journal, _ = load_migration_journal(self.base_storage) - self.assertEqual(journal["phase"], "prepared") - self.assertTrue(all(item["resulting_revision"] is None for item in journal["documents"])) - self.assertEqual(self._snapshot(), before) - - fresh_planner, result = self._recover_fresh() - self.assertIsNotNone(result) - self.assertEqual(result.outcome, "changed") - self.assert_converged(fresh_planner) - - def test_crash_after_manifest_and_ledger_rolls_back_replans_and_converges(self): - faults = FaultStorage(self.base_storage) - faults.crash = ("write", "progress.json") - _, planner, executor = self._runtime(faults) - plan = planner.plan(slug="legacy", to_revision=NEW, installed=self.installed()) - - with self.assertRaises(SimulatedProcessCrash): - executor.execute(plan, session_id="crashed-session") - - self.assertIsNotNone(self.base_storage.read("source-manifest.json")) - self.assertIsNotNone(self.base_storage.read("review-ledger.json")) - journal, _ = load_migration_journal(self.base_storage) - states = {entry["path"]: entry["resulting_revision"] for entry in journal["documents"]} - self.assertIsNotNone(states["source-manifest.json"]) - self.assertIsNotNone(states["review-ledger.json"]) - self.assertIsNone(states["progress.json"]) - self.assertIsNone(states["metadata.json"]) - - fresh_planner, result = self._recover_fresh() - self.assertIsNotNone(result) - self.assertEqual(result.outcome, "changed") - self.assert_converged(fresh_planner) - - def test_crash_after_metadata_before_journal_deletion_recovers_as_completed(self): - faults = FaultStorage(self.base_storage) - faults.crash = ("delete", MIGRATION_PATH) - _, planner, executor = self._runtime(faults) - plan = planner.plan(slug="legacy", to_revision=NEW, installed=self.installed()) - - with self.assertRaises(SimulatedProcessCrash): - executor.execute(plan, session_id="crashed-session") - - journal, _ = load_migration_journal(self.base_storage) - self.assertEqual(journal["phase"], "applied") - self.assertTrue(all(item["resulting_revision"] for item in journal["documents"])) - - fresh_planner, result = self._recover_fresh() - self.assertIsNotNone(result) - self.assertEqual(result.outcome, "recovered") - self.assert_converged(fresh_planner) - - def test_cas_conflict_during_apply_restores_exact_original_state(self): - faults = FaultStorage(self.base_storage) - faults.conflict = ("write", "progress.json") - _, planner, executor = self._runtime(faults) - plan = planner.plan(slug="legacy", to_revision=NEW, installed=self.installed()) - before = self._snapshot() - - with self.assertRaises(MigrationError): - executor.execute(plan, session_id="conflict-session") - - self.assertEqual(self._snapshot(), before) - with self.assertRaises(StorageNotFound): - self.base_storage.read(MIGRATION_PATH) - with self.assertRaises(StorageNotFound): - self.base_storage.read("source-manifest.json") - with self.assertRaises(StorageNotFound): - self.base_storage.read("review-ledger.json") - - def test_unknown_concurrent_mutation_after_crash_preserves_bytes_and_journal(self): - faults = FaultStorage(self.base_storage) - faults.crash = ("write", "progress.json") - _, planner, executor = self._runtime(faults) - plan = planner.plan(slug="legacy", to_revision=NEW, installed=self.installed()) - - with self.assertRaises(SimulatedProcessCrash): - executor.execute(plan, session_id="crashed-session") - - foreign = b"foreign concurrent metadata bytes\n" - current = self.base_storage.read("metadata.json") - self.base_storage.write_if_version("metadata.json", foreign, current.version) - journal_before = self.base_storage.read(MIGRATION_PATH).content - - with self.assertRaises(MigrationError): - self._recover_fresh() - - self.assertEqual(self.base_storage.read("metadata.json").content, foreign) - self.assertEqual(self.base_storage.read(MIGRATION_PATH).content, journal_before) - - def test_completed_upgrade_rerun_is_byte_and_write_idempotent(self): - _, planner, executor = self._runtime(self.base_storage) - plan = planner.plan(slug="legacy", to_revision=NEW, installed=self.installed()) - first = executor.execute(plan, session_id="first-session") - self.assertEqual(first.outcome, "changed") - before = self._snapshot(include_transient=True) - - _, fresh_planner, fresh_executor = self._runtime(self.base_storage, now=RECOVERY_NOW) - second_plan = fresh_planner.plan(slug="legacy", to_revision=NEW, installed=self.installed()) - self.assertFalse(second_plan.changed) - second = fresh_executor.execute(second_plan, session_id="second-session") - - self.assertEqual(second.outcome, "unchanged") - self.assertEqual(second.migrated_paths, ()) - self.assertEqual(self._snapshot(include_transient=True), before) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_workflow_v2_migration_transaction.py b/tests/test_workflow_v2_migration_transaction.py deleted file mode 100644 index 19e595c..0000000 --- a/tests/test_workflow_v2_migration_transaction.py +++ /dev/null @@ -1,342 +0,0 @@ -import base64 -import hashlib -import json -import sys -import tempfile -import unittest -from datetime import datetime, timezone -from pathlib import Path - - -PROJECT_ROOT = Path(__file__).resolve().parents[1] -SCRIPTS = PROJECT_ROOT / "scripts" -sys.path.insert(0, str(SCRIPTS)) - -from workflow_v2 import FilesystemStorage, WorkflowStateRepository -from workflow_v2.coordination import BookCoordinationManager -from workflow_v2.migration_journal import MIGRATION_PATH, serialize_migration_journal -from workflow_v2.migrations import MigrationError, MigrationPlanner -from workflow_v2.storage import StorageError, StorageNotFound - -try: - from workflow_v2.migrations import MigrationExecutor, MigrationResult -except ImportError: - MigrationExecutor = None - MigrationResult = None - - -CANONICAL = "https://github.com/tim8es/book-translator" -OLD = "old-revision" -NEW = "new-revision" -REQUESTED = "refactor/workflow-engine-v2" -NOW = datetime(2026, 9, 7, 12, 0, tzinfo=timezone.utc) - - -class RecordingStorage: - def __init__(self, inner): - self.inner = inner - self.events = [] - self.fail_once = None - self._failed = False - - def _maybe_fail(self, operation, path): - if self.fail_once == (operation, path) and not self._failed: - self._failed = True - raise StorageError(f"injected {operation} failure for {path}") - - def read(self, path): - return self.inner.read(path) - - def write_if_version(self, path, content, expected_version): - self.events.append(("write", path)) - self._maybe_fail("write", path) - return self.inner.write_if_version(path, content, expected_version) - - def create_if_absent(self, path, content): - self.events.append(("create", path)) - self._maybe_fail("create", path) - return self.inner.create_if_absent(path, content) - - def delete_if_version(self, path, expected_version): - self.events.append(("delete", path)) - self._maybe_fail("delete", path) - return self.inner.delete_if_version(path, expected_version) - - def list(self, prefix=""): - return self.inner.list(prefix) - - -class WorkflowV2MigrationTransactionTests(unittest.TestCase): - def setUp(self): - self.temp = tempfile.TemporaryDirectory() - self.book_dir = Path(self.temp.name) / "books" / "legacy" - for name in ("source", "extracted", "translated", "output", ".workflow/claims"): - (self.book_dir / name).mkdir(parents=True, exist_ok=True) - (self.book_dir / "source/legacy.md").write_bytes(b"source-book\n") - (self.book_dir / "extracted/001-one.md").write_bytes(b"source chapter\n") - (self.book_dir / "translated/001-one.md").write_bytes("перевод\n".encode("utf-8")) - self._write_json("metadata.json", self._metadata()) - self._write_json("progress.json", self._progress()) - - self.base_storage = FilesystemStorage(self.book_dir) - self.storage = RecordingStorage(self.base_storage) - self.repository = WorkflowStateRepository(self.storage) - self.planner = MigrationPlanner( - self.repository, - book_dir=self.book_dir, - artifact_reader=lambda path: (self.book_dir / path).read_bytes(), - now=lambda: NOW, - ) - self.coordination = BookCoordinationManager( - self.repository, - now=lambda: NOW, - id_factory=lambda: "c" * 32, - ) - - def tearDown(self): - self.temp.cleanup() - - def require_executor(self): - self.assertIsNotNone(MigrationExecutor, "MigrationExecutor is not implemented") - self.assertIsNotNone(MigrationResult, "MigrationResult is not implemented") - - @staticmethod - def installed(): - return { - "canonical_repository": CANONICAL, - "requested_ref": REQUESTED, - "resolved_revision": NEW, - } - - @staticmethod - def _metadata(): - return { - "title": "Legacy", - "author": "Author", - "source_language": "en", - "target_language": "ru", - "source_format": "markdown", - "source_file": "legacy.md", - "chapter_count": 1, - "imported_at": "2026-09-01T10:00:00+00:00", - "workflow": { - "repository": CANONICAL, - "requested_ref": "legacy-ref", - "resolved_revision": OLD, - }, - } - - @staticmethod - def _progress(): - return { - "book_slug": "legacy", - "chapters": [ - { - "number": 1, - "title": "One", - "slug": "one", - "source_path": "extracted/001-one.md", - "translation_path": "translated/001-one.md", - "status": "translated", - } - ], - } - - def _write_json(self, path, data): - target = self.book_dir / path - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - - def _plan(self): - return self.planner.plan(slug="legacy", to_revision=NEW, installed=self.installed()) - - def _executor(self): - self.require_executor() - return MigrationExecutor( - self.repository, - self.planner, - coordination=self.coordination, - ) - - def _snapshot(self, *, transient=False): - result = {} - for path in sorted(self.book_dir.rglob("*")): - if not path.is_file(): - continue - relative = path.relative_to(self.book_dir).as_posix() - if not transient and relative in { - MIGRATION_PATH, - ".workflow/coordination-lock.json", - }: - continue - result[relative] = path.read_bytes() - return result - - @staticmethod - def _journal_for(plan, *, phase="prepared", resulting=None): - resulting = resulting or {} - documents = [] - for write in plan.writes: - original = write.original_bytes - documents.append( - { - "path": write.path, - "kind": write.kind.value, - "original_exists": write.original_exists, - "original_revision": write.original_version if write.original_exists else None, - "original_sha256": ( - hashlib.sha256(original).hexdigest() - if write.original_exists and original is not None - else None - ), - "original_bytes_base64": ( - base64.b64encode(original).decode("ascii") - if write.original_exists and original is not None - else None - ), - "target_sha256": hashlib.sha256(write.target_bytes).hexdigest(), - "resulting_revision": resulting.get(write.path), - } - ) - return { - "schema_version": 1, - "operation": "workflow_upgrade", - "book_slug": plan.book_slug, - "from_revision": plan.from_revision, - "to_revision": plan.to_revision, - "phase": phase, - "documents": documents, - } - - def _create_journal(self, data): - return self.base_storage.create_if_absent(MIGRATION_PATH, serialize_migration_journal(data)) - - def _apply_write_directly(self, write): - if write.original_exists: - return self.base_storage.write_if_version( - write.path, - write.target_bytes, - write.original_version, - ) - return self.base_storage.create_if_absent(write.path, write.target_bytes) - - def test_coordination_accepts_workflow_upgrade_operation(self): - lease = self.coordination.acquire(operation="workflow_upgrade", session_id="upgrade-1") - try: - self.assertEqual(lease.data["operation"], "workflow_upgrade") - finally: - self.coordination.release(lease) - - def test_execute_journals_before_targets_updates_journal_and_writes_metadata_last(self): - plan = self._plan() - expected_paths = tuple(write.path for write in plan.writes) - self.assertEqual( - expected_paths, - ("source-manifest.json", "review-ledger.json", "progress.json", "metadata.json"), - ) - - result = self._executor().execute(plan, session_id="upgrade-1") - - self.assertEqual(result.outcome, "changed") - self.assertEqual(result.migrated_paths, expected_paths) - with self.assertRaises(StorageNotFound): - self.base_storage.read(MIGRATION_PATH) - with self.assertRaises(StorageNotFound): - self.base_storage.read(".workflow/coordination-lock.json") - - target_events = [ - path - for operation, path in self.storage.events - if path in expected_paths and operation in {"create", "write"} - ] - self.assertEqual(target_events, list(expected_paths)) - journal_events = [ - (index, operation) - for index, (operation, path) in enumerate(self.storage.events) - if path == MIGRATION_PATH - ] - self.assertTrue(journal_events) - first_target = next( - index for index, (_, path) in enumerate(self.storage.events) if path == expected_paths[0] - ) - self.assertEqual(journal_events[0][1], "create") - self.assertLess(journal_events[0][0], first_target) - self.assertGreaterEqual( - sum(1 for operation, path in self.storage.events if path == MIGRATION_PATH and operation == "write"), - len(expected_paths) + 1, - "journal must record every resulting revision and the final applied phase", - ) - self.assertLess( - next(index for index, event in enumerate(self.storage.events) if event[1] == "progress.json"), - next(index for index, event in enumerate(self.storage.events) if event[1] == "metadata.json"), - ) - - def test_apply_failure_rolls_back_exact_originals_and_deletes_new_targets(self): - plan = self._plan() - before = self._snapshot() - self.storage.fail_once = ("write", "progress.json") - - with self.assertRaises(MigrationError): - self._executor().execute(plan, session_id="upgrade-1") - - self.assertEqual(self._snapshot(), before) - with self.assertRaises(StorageNotFound): - self.base_storage.read(MIGRATION_PATH) - with self.assertRaises(StorageNotFound): - self.base_storage.read("source-manifest.json") - with self.assertRaises(StorageNotFound): - self.base_storage.read("review-ledger.json") - - def test_recover_all_target_state_validates_and_only_removes_journal(self): - plan = self._plan() - resulting = {} - for write in plan.writes: - resulting[write.path] = self._apply_write_directly(write) - self._create_journal(self._journal_for(plan, phase="applied", resulting=resulting)) - before = self._snapshot() - - result = self._executor().recover(session_id="upgrade-2", installed=self.installed()) - - self.assertIsNotNone(result) - self.assertEqual(result.outcome, "recovered") - self.assertEqual(result.migrated_paths, tuple(write.path for write in plan.writes)) - self.assertEqual(self._snapshot(), before) - with self.assertRaises(StorageNotFound): - self.base_storage.read(MIGRATION_PATH) - - def test_recover_known_mixture_rolls_back_then_replans_and_executes(self): - plan = self._plan() - self._create_journal(self._journal_for(plan)) - self._apply_write_directly(plan.writes[0]) - - result = self._executor().recover(session_id="upgrade-2", installed=self.installed()) - - self.assertIsNotNone(result) - self.assertEqual(result.outcome, "changed") - self.assertEqual(result.migrated_paths, tuple(write.path for write in plan.writes)) - with self.assertRaises(StorageNotFound): - self.base_storage.read(MIGRATION_PATH) - final_plan = self.planner.plan(slug="legacy", to_revision=NEW, installed=self.installed()) - self.assertFalse(final_plan.changed) - - def test_recover_unknown_state_preserves_unknown_bytes_and_journal(self): - plan = self._plan() - self._create_journal(self._journal_for(plan)) - foreign = b"foreign concurrent bytes\n" - metadata = self.base_storage.read("metadata.json") - self.base_storage.write_if_version("metadata.json", foreign, metadata.version) - journal_before = self.base_storage.read(MIGRATION_PATH) - - with self.assertRaises(MigrationError): - self._executor().recover(session_id="upgrade-2", installed=self.installed()) - - self.assertEqual(self.base_storage.read("metadata.json").content, foreign) - self.assertEqual(self.base_storage.read(MIGRATION_PATH).content, journal_before.content) - with self.assertRaises(StorageNotFound): - self.base_storage.read("source-manifest.json") - with self.assertRaises(StorageNotFound): - self.base_storage.read("review-ledger.json") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_workflow_v2_migration_visibility.py b/tests/test_workflow_v2_migration_visibility.py deleted file mode 100644 index 9fb5113..0000000 --- a/tests/test_workflow_v2_migration_visibility.py +++ /dev/null @@ -1,229 +0,0 @@ -import base64 -import hashlib -import sys -import tempfile -import unittest -from datetime import datetime, timezone -from pathlib import Path - - -PROJECT_ROOT = Path(__file__).resolve().parents[1] -SCRIPTS = PROJECT_ROOT / "scripts" -sys.path.insert(0, str(SCRIPTS)) - -from workflow_v2 import FilesystemStorage, WorkflowStateRepository -from workflow_v2.claims import ClaimError, ClaimManager -from workflow_v2.coordination import BookCoordinationManager -from workflow_v2.finalize import FinalizationError, FinalizationManager -from workflow_v2.migration_journal import MIGRATION_PATH, serialize_migration_journal -from workflow_v2.reviews import ReviewEvidenceError, ReviewLedgerManager -from workflow_v2.schemas import SchemaKind -from workflow_v2.status import StatusResolver -from workflow_v2.storage import StorageNotFound - - -NOW = datetime(2026, 9, 7, 12, 0, tzinfo=timezone.utc) -OLD = "old-revision" -NEW = "new-revision" - - -class WorkflowV2MigrationVisibilityTests(unittest.TestCase): - def setUp(self): - self.temp = tempfile.TemporaryDirectory() - self.book_dir = Path(self.temp.name) - (self.book_dir / ".workflow/claims").mkdir(parents=True) - (self.book_dir / "extracted").mkdir() - (self.book_dir / "translated").mkdir() - (self.book_dir / "extracted/001-one.md").write_bytes(b"source chapter\n") - (self.book_dir / "translated/001-one.md").write_bytes(b"translation\n") - self.storage = FilesystemStorage(self.book_dir) - self.repository = WorkflowStateRepository(self.storage) - self.metadata = { - "schema_version": 1, - "title": "Book", - "target_language": "ru", - "source_format": "markdown", - "source_file": "book.md", - "chapter_count": 1, - "workflow": { - "repository": "https://github.com/tim8es/book-translator", - "requested_ref": "legacy-ref", - "resolved_revision": OLD, - }, - } - self.progress = { - "schema_version": 1, - "book_slug": "book", - "chapters": [ - { - "number": 1, - "title": "One", - "slug": "one", - "source_path": "extracted/001-one.md", - "translation_path": "translated/001-one.md", - "status": "translated", - } - ], - } - self.metadata_revision = self.repository.create( - "metadata.json", SchemaKind.METADATA, self.metadata - ) - self.progress_revision = self.repository.create( - "progress.json", SchemaKind.PROGRESS, self.progress - ) - self._create_valid_journal() - - def tearDown(self): - self.temp.cleanup() - - @staticmethod - def _journal(): - original = b"original metadata bytes\n" - target = b"target metadata bytes\n" - return { - "schema_version": 1, - "operation": "workflow_upgrade", - "book_slug": "book", - "from_revision": OLD, - "to_revision": NEW, - "phase": "prepared", - "documents": [ - { - "path": "metadata.json", - "kind": "metadata", - "original_exists": True, - "original_revision": "original-revision", - "original_sha256": hashlib.sha256(original).hexdigest(), - "original_bytes_base64": base64.b64encode(original).decode("ascii"), - "target_sha256": hashlib.sha256(target).hexdigest(), - "resulting_revision": None, - } - ], - } - - def _create_valid_journal(self): - self.storage.create_if_absent( - MIGRATION_PATH, - serialize_migration_journal(self._journal()), - ) - - def coordination(self): - return BookCoordinationManager( - self.repository, - now=lambda: NOW, - id_factory=lambda: "c" * 32, - ) - - def test_active_migration_blocks_claim_admission_without_mutating_journal(self): - before = self.storage.read(MIGRATION_PATH) - manager = ClaimManager( - self.repository, - now=lambda: NOW, - id_factory=lambda: "d" * 32, - coordination=self.coordination(), - ) - - with self.assertRaises(ClaimError) as ctx: - manager.acquire( - self.progress, - "1", - role="translator", - session_id="translator-1", - base_revision=self.progress_revision, - base_commit=None, - workflow_revision=OLD, - ) - - self.assertIn("migration", str(ctx.exception).lower()) - self.assertEqual(self.storage.read(MIGRATION_PATH), before) - self.assertEqual(self.storage.list(".workflow/claims"), []) - - def test_active_migration_blocks_finalization_admission_before_marker_creation(self): - manager = FinalizationManager( - self.repository, - artifact_reader=lambda path: (self.book_dir / path).read_bytes(), - preflight=lambda: ((), {"state": "verified"}), - coordination=self.coordination(), - now=lambda: NOW, - id_factory=lambda: "e" * 32, - ) - - with self.assertRaises(FinalizationError) as ctx: - manager._admit( - session_id="finalizer-1", - progress_revision=self.progress_revision, - book_slug="book", - workflow_revision=OLD, - candidate_hash="a" * 64, - ) - - self.assertIn("migration", str(ctx.exception).lower()) - with self.assertRaises(StorageNotFound): - self.storage.read(".workflow/finalization.json") - - def test_active_migration_blocks_accept_review_for_the_migration_reason(self): - manager = ReviewLedgerManager( - self.repository, - artifact_reader=lambda path: (self.book_dir / path).read_bytes(), - ) - - with self.assertRaises(ReviewEvidenceError) as ctx: - manager.accept_review( - self.progress, - self.progress_revision, - self.metadata, - 1, - ) - - self.assertIn("migration", str(ctx.exception).lower()) - self.assertEqual( - self.repository.read("progress.json", SchemaKind.PROGRESS).version, - self.progress_revision, - ) - - def test_status_exposes_bounded_migration_and_resume_prioritizes_workflow_upgrade(self): - before = self.storage.read(MIGRATION_PATH) - resolver = StatusResolver( - self.repository, - artifact_reader=lambda path: (self.book_dir / path).read_bytes(), - ) - - status = resolver.status(corpus={"state": "unsealed"}) - self.assertTrue(status["valid"]) - self.assertEqual( - status["migration"], - { - "active": True, - "phase": "prepared", - "from_revision": OLD, - "to_revision": NEW, - "document_count": 1, - }, - ) - resume = resolver.resume(status) - self.assertEqual(resume["operation"], "workflow_upgrade") - self.assertIn(MIGRATION_PATH, resume["context"]["files"]) - self.assertEqual(self.storage.read(MIGRATION_PATH), before) - - def test_malformed_migration_journal_invalidates_status_without_mutation(self): - current = self.storage.read(MIGRATION_PATH) - malformed = b"{not-json\n" - self.storage.write_if_version(MIGRATION_PATH, malformed, current.version) - before = self.storage.read(MIGRATION_PATH) - resolver = StatusResolver( - self.repository, - artifact_reader=lambda path: (self.book_dir / path).read_bytes(), - ) - - status = resolver.status(corpus={"state": "unsealed"}) - self.assertFalse(status["valid"]) - self.assertEqual(status["migration"], {"active": False}) - self.assertTrue(any("migration" in error.lower() for error in status["errors"])) - resume = resolver.resume(status) - self.assertEqual(resume["operation"], "blocked") - self.assertEqual(resume["reason"], "preflight_failed") - self.assertEqual(self.storage.read(MIGRATION_PATH), before) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_workflow_v2_migrations.py b/tests/test_workflow_v2_migrations.py deleted file mode 100644 index baa9c09..0000000 --- a/tests/test_workflow_v2_migrations.py +++ /dev/null @@ -1,281 +0,0 @@ -import copy -import sys -import unittest -from pathlib import Path - - -PROJECT_ROOT = Path(__file__).resolve().parents[1] -SCRIPTS = PROJECT_ROOT / "scripts" -sys.path.insert(0, str(SCRIPTS)) - -try: - from workflow_v2.schemas import SchemaKind, parse_document -except ModuleNotFoundError: - SchemaKind = None - parse_document = None - -try: - from workflow_v2.migrations import ( - MigrationCompatibilityError, - detect_schema_version, - migrate_document, - ) -except ModuleNotFoundError: - MigrationCompatibilityError = None - detect_schema_version = None - migrate_document = None - -try: - from workflow_v2.migration_journal import ( - MigrationJournalError, - serialize_migration_journal, - validate_migration_journal, - ) -except ModuleNotFoundError: - MigrationJournalError = None - serialize_migration_journal = None - validate_migration_journal = None - - -class WorkflowV2MigrationRegistryTests(unittest.TestCase): - def require_api(self): - self.assertIsNotNone(migrate_document, "workflow_v2.migrations is not implemented") - self.assertIsNotNone(detect_schema_version, "detect_schema_version is not implemented") - self.assertIsNotNone(MigrationCompatibilityError, "migration compatibility error is not implemented") - - @staticmethod - def metadata(): - return { - "schema_version": 1, - "title": "Legacy Example", - "author": "Author", - "source_language": "en", - "target_language": "ru", - "source_format": "markdown", - "source_file": "legacy.md", - "chapter_count": 1, - "imported_at": "2026-09-01T10:00:00+00:00", - "workflow": { - "repository": "https://github.com/tim8es/book-translator", - "requested_ref": "legacy-ref", - "resolved_revision": "legacy-revision", - }, - } - - @staticmethod - def progress(): - return { - "schema_version": 1, - "book_slug": "legacy", - "chapters": [ - { - "number": 1, - "title": "One", - "slug": "one", - "source_path": "extracted/001-one.md", - "translation_path": "translated/001-one.md", - "status": "translated", - } - ], - } - - @staticmethod - def ledger(): - return { - "schema_version": 1, - "book_slug": "legacy", - "next_sequence": 1, - "records": [], - } - - @staticmethod - def claim(): - return { - "schema_version": 1, - "claim_id": "0123456789abcdef0123456789abcdef", - "unit_id": "chapter-000001", - "role": "translator", - "session_id": "session-1", - "base_revision": "progress-revision", - "base_commit": None, - "workflow_revision": "legacy-revision", - "claimed_at": "2026-09-01T10:00:00Z", - "expires_at": "2026-09-01T11:00:00Z", - } - - @staticmethod - def source_manifest(): - return { - "schema_version": 1, - "source_file": "legacy.md", - "source_format": "markdown", - "source_sha256": "a" * 64, - "chapter_count": 1, - "extracted": [ - { - "number": 1, - "title": "One", - "path": "extracted/001-one.md", - "sha256": "b" * 64, - } - ], - } - - def test_detect_schema_version_treats_missing_as_zero_and_rejects_non_integer(self): - self.require_api() - self.assertEqual(detect_schema_version({"title": "legacy"}), 0) - self.assertEqual(detect_schema_version({"schema_version": 1}), 1) - with self.assertRaises(MigrationCompatibilityError): - detect_schema_version({"schema_version": "1"}) - - def test_v0_supported_documents_add_only_schema_version_and_validate_as_v1(self): - self.require_api() - documents = { - SchemaKind.METADATA: self.metadata(), - SchemaKind.PROGRESS: self.progress(), - SchemaKind.REVIEW_LEDGER: self.ledger(), - SchemaKind.CLAIM: self.claim(), - SchemaKind.SOURCE_MANIFEST: self.source_manifest(), - } - for kind, current in documents.items(): - with self.subTest(kind=kind): - legacy = copy.deepcopy(current) - del legacy["schema_version"] - original = copy.deepcopy(legacy) - result = migrate_document(kind, legacy) - self.assertEqual(result.kind, kind) - self.assertEqual(result.from_version, 0) - self.assertEqual(result.to_version, 1) - self.assertTrue(result.changed) - self.assertEqual(result.data, current) - self.assertEqual(legacy, original) - self.assertEqual(parse_document(kind, result.data).data, current) - - def test_explicit_v1_is_validated_and_returned_unchanged(self): - self.require_api() - current = self.progress() - result = migrate_document(SchemaKind.PROGRESS, current) - self.assertEqual(result.from_version, 1) - self.assertEqual(result.to_version, 1) - self.assertFalse(result.changed) - self.assertEqual(result.data, current) - self.assertIsNot(result.data, current) - - def test_future_version_and_incomplete_v0_fail_precisely(self): - self.require_api() - future = self.metadata() - future["schema_version"] = 2 - with self.assertRaises(MigrationCompatibilityError) as future_error: - migrate_document(SchemaKind.METADATA, future) - self.assertIn("unsupported", str(future_error.exception).lower()) - self.assertIn("2", str(future_error.exception)) - - incomplete = {"claim_id": "0123456789abcdef0123456789abcdef"} - with self.assertRaises(MigrationCompatibilityError) as legacy_error: - migrate_document(SchemaKind.CLAIM, incomplete) - self.assertIn("claim", str(legacy_error.exception).lower()) - self.assertIn("v0", str(legacy_error.exception).lower()) - - -class WorkflowV2MigrationJournalTests(unittest.TestCase): - def require_api(self): - self.assertIsNotNone(validate_migration_journal, "migration journal validator is not implemented") - self.assertIsNotNone(serialize_migration_journal, "migration journal serializer is not implemented") - self.assertIsNotNone(MigrationJournalError, "migration journal error is not implemented") - - @staticmethod - def valid_journal(): - import base64 - import hashlib - - original = b'{"legacy": true}\n' - return { - "schema_version": 1, - "operation": "workflow_upgrade", - "book_slug": "legacy", - "from_revision": "old-revision", - "to_revision": "new-revision", - "phase": "prepared", - "documents": [ - { - "path": "metadata.json", - "kind": "metadata", - "original_exists": True, - "original_revision": "old-storage-revision", - "original_sha256": hashlib.sha256(original).hexdigest(), - "original_bytes_base64": base64.b64encode(original).decode("ascii"), - "target_sha256": "c" * 64, - "resulting_revision": None, - }, - { - "path": "review-ledger.json", - "kind": "review_ledger", - "original_exists": False, - "original_revision": None, - "original_sha256": None, - "original_bytes_base64": None, - "target_sha256": "d" * 64, - "resulting_revision": None, - }, - ], - } - - def test_valid_prepared_journal_validates_and_serializes_canonically(self): - self.require_api() - journal = self.valid_journal() - validated = validate_migration_journal(journal) - self.assertEqual(validated, journal) - payload = serialize_migration_journal(journal) - self.assertTrue(payload.endswith(b"\n")) - self.assertEqual(payload, serialize_migration_journal(copy.deepcopy(journal))) - - def test_journal_rejects_invalid_phase_operation_and_unsafe_path(self): - self.require_api() - for field, value in (("phase", "rolling-back"), ("operation", "finalize")): - with self.subTest(field=field): - invalid = self.valid_journal() - invalid[field] = value - with self.assertRaises(MigrationJournalError): - validate_migration_journal(invalid) - - invalid = self.valid_journal() - invalid["documents"][0]["path"] = "../metadata.json" - with self.assertRaises(MigrationJournalError): - validate_migration_journal(invalid) - - def test_journal_rejects_bad_hash_base64_and_original_identity_combinations(self): - self.require_api() - bad_hash = self.valid_journal() - bad_hash["documents"][0]["target_sha256"] = "not-a-hash" - with self.assertRaises(MigrationJournalError): - validate_migration_journal(bad_hash) - - bad_base64 = self.valid_journal() - bad_base64["documents"][0]["original_bytes_base64"] = "***not-base64***" - with self.assertRaises(MigrationJournalError): - validate_migration_journal(bad_base64) - - inconsistent_missing = self.valid_journal() - inconsistent_missing["documents"][1]["original_revision"] = "should-be-null" - with self.assertRaises(MigrationJournalError): - validate_migration_journal(inconsistent_missing) - - inconsistent_existing = self.valid_journal() - inconsistent_existing["documents"][0]["original_sha256"] = None - with self.assertRaises(MigrationJournalError): - validate_migration_journal(inconsistent_existing) - - def test_applied_journal_requires_resulting_revision_for_every_document(self): - self.require_api() - applied = self.valid_journal() - applied["phase"] = "applied" - with self.assertRaises(MigrationJournalError): - validate_migration_journal(applied) - - for entry in applied["documents"]: - entry["resulting_revision"] = "new-storage-revision" - self.assertEqual(validate_migration_journal(applied)["phase"], "applied") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_workflow_v2_migrations_cli.py b/tests/test_workflow_v2_migrations_cli.py deleted file mode 100644 index 5b23460..0000000 --- a/tests/test_workflow_v2_migrations_cli.py +++ /dev/null @@ -1,230 +0,0 @@ -import json -import shutil -import subprocess -import sys -import tempfile -import unittest -from pathlib import Path - - -PROJECT_ROOT = Path(__file__).resolve().parents[1] -BOOK_SCRIPT = PROJECT_ROOT / "scripts" / "book.py" -CORPUS_SCRIPT = PROJECT_ROOT / "scripts" / "corpus.py" -WORKFLOW_V2 = PROJECT_ROOT / "scripts" / "workflow_v2" -CANONICAL = "https://github.com/tim8es/book-translator" -OLD = "legacy-revision" -NEW = "0123456789abcdef" -OTHER = "fedcba9876543210" - - -class WorkflowV2MigrationsCliTests(unittest.TestCase): - def setUp(self): - self.tmp = tempfile.TemporaryDirectory() - self.repo = Path(self.tmp.name) / "repo" - (self.repo / "scripts").mkdir(parents=True) - shutil.copy2(BOOK_SCRIPT, self.repo / "scripts" / "book.py") - shutil.copy2(CORPUS_SCRIPT, self.repo / "scripts" / "corpus.py") - shutil.copytree(WORKFLOW_V2, self.repo / "scripts" / "workflow_v2") - self.install_path = self.repo / ".book-translator-install.json" - self.install_path.write_text( - json.dumps( - { - "schema_version": 1, - "canonical_repository": CANONICAL, - "requested_ref": "refactor/workflow-engine-v2", - "resolved_revision": NEW, - "install_root": ".", - }, - sort_keys=True, - ) - + "\n", - encoding="utf-8", - ) - - def tearDown(self): - self.tmp.cleanup() - - def run_book(self, *args, expect=0): - result = subprocess.run( - [sys.executable, str(self.repo / "scripts" / "book.py"), *args], - cwd=self.repo, - capture_output=True, - text=True, - ) - self.assertEqual( - result.returncode, - expect, - msg=f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}", - ) - return result - - def initialize_book(self, *, private=False): - source = self.repo / "sample.md" - source.write_text("# One\n\nAlpha.\n", encoding="utf-8") - args = ["extract", str(source), "--slug", "sample", "--target-language", "ru"] - if private: - args.append("--private-source") - self.run_book(*args) - return self.repo / "books" / "sample" - - @staticmethod - def _write_json(path: Path, payload): - path.write_text( - json.dumps(payload, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) - - def make_legacy(self, book: Path, *, reviewed=False): - metadata_path = book / "metadata.json" - metadata = json.loads(metadata_path.read_text(encoding="utf-8")) - metadata.pop("schema_version", None) - metadata["workflow"] = { - "repository": CANONICAL, - "requested_ref": "legacy-ref", - "resolved_revision": OLD, - } - self._write_json(metadata_path, metadata) - - progress_path = book / "progress.json" - progress = json.loads(progress_path.read_text(encoding="utf-8")) - progress.pop("schema_version", None) - if reviewed: - chapter = progress["chapters"][0] - translation_path = book / chapter["translation_path"] - translation_path.parent.mkdir(parents=True, exist_ok=True) - translation_path.write_text("Перевод.\n", encoding="utf-8") - chapter["status"] = "reviewed" - self._write_json(progress_path, progress) - - for name in ("review-ledger.json", "source-manifest.json"): - path = book / name - payload = json.loads(path.read_text(encoding="utf-8")) - payload.pop("schema_version", None) - self._write_json(path, payload) - - @staticmethod - def snapshot(book: Path): - return { - path.relative_to(book).as_posix(): path.read_bytes() - for path in sorted(book.rglob("*")) - if path.is_file() - } - - @staticmethod - def canonical_json(result): - payload = json.loads(result.stdout) - if result.returncode == 0: - assert result.stdout == json.dumps(payload, ensure_ascii=False, sort_keys=True) + "\n" - return payload - - def test_legacy_v0_upgrade_records_revisions_downgrade_and_deterministic_json(self): - book = self.initialize_book() - self.make_legacy(book, reviewed=True) - - result = self.run_book("workflow-upgrade", "sample", "--to", NEW, "--json") - payload = self.canonical_json(result) - - self.assertEqual(payload["book_slug"], "sample") - self.assertEqual(payload["from_revision"], OLD) - self.assertEqual(payload["to_revision"], NEW) - self.assertEqual(payload["outcome"], "changed") - self.assertEqual(payload["lifecycle_downgrades"], [1]) - self.assertEqual( - payload["migrated_paths"], - ["source-manifest.json", "review-ledger.json", "progress.json", "metadata.json"], - ) - - metadata = json.loads((book / "metadata.json").read_text(encoding="utf-8")) - progress = json.loads((book / "progress.json").read_text(encoding="utf-8")) - ledger = json.loads((book / "review-ledger.json").read_text(encoding="utf-8")) - manifest = json.loads((book / "source-manifest.json").read_text(encoding="utf-8")) - self.assertEqual(metadata["schema_version"], 1) - self.assertEqual(progress["schema_version"], 1) - self.assertEqual(ledger["schema_version"], 1) - self.assertEqual(manifest["schema_version"], 1) - self.assertEqual(metadata["workflow"]["resolved_revision"], NEW) - self.assertEqual(metadata["workflow"]["requested_ref"], "refactor/workflow-engine-v2") - self.assertEqual(metadata["workflow"]["review_evidence"], "review-ledger-v1") - self.assertEqual(metadata["workflow"]["upgrade_history"][-1]["from_revision"], OLD) - self.assertEqual(metadata["workflow"]["upgrade_history"][-1]["to_revision"], NEW) - self.assertEqual(progress["chapters"][0]["status"], "translated") - self.assertFalse((book / ".workflow" / "migration.json").exists()) - - def test_target_mismatch_is_concise_and_read_only(self): - book = self.initialize_book() - self.make_legacy(book) - before = self.snapshot(book) - - result = self.run_book("workflow-upgrade", "sample", "--to", OTHER, expect=1) - - self.assertIn("installed", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - self.assertEqual(self.snapshot(book), before) - - def test_malformed_legacy_fixture_fails_concisely_without_mutation(self): - book = self.initialize_book() - self.make_legacy(book) - progress_path = book / "progress.json" - progress = json.loads(progress_path.read_text(encoding="utf-8")) - progress["chapters"][0].pop("title") - self._write_json(progress_path, progress) - before = self.snapshot(book) - - result = self.run_book("workflow-upgrade", "sample", "--to", NEW, expect=1) - - self.assertIn("title", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - self.assertEqual(self.snapshot(book), before) - - def test_private_source_upgrade_never_restores_source_binary(self): - book = self.initialize_book(private=True) - self.make_legacy(book) - metadata = json.loads((book / "metadata.json").read_text(encoding="utf-8")) - source_path = book / "source" / metadata["source_file"] - self.assertFalse(source_path.exists()) - - payload = self.canonical_json( - self.run_book("workflow-upgrade", "sample", "--to", NEW, "--json") - ) - - self.assertEqual(payload["outcome"], "changed") - self.assertFalse(source_path.exists()) - manifest = json.loads((book / "source-manifest.json").read_text(encoding="utf-8")) - self.assertEqual(manifest["source_storage_mode"], "private_external") - self.assertEqual(manifest["source_sha256"], metadata["source"]["sha256"]) - - def test_ordinary_validate_does_not_silently_upgrade_legacy_state(self): - book = self.initialize_book() - self.make_legacy(book) - before = self.snapshot(book) - - result = self.run_book("validate", "sample", expect=1) - - self.assertIn("source-manifest.json", result.stderr) - self.assertEqual(self.snapshot(book), before) - metadata = json.loads((book / "metadata.json").read_text(encoding="utf-8")) - progress = json.loads((book / "progress.json").read_text(encoding="utf-8")) - self.assertNotIn("schema_version", metadata) - self.assertNotIn("schema_version", progress) - self.assertEqual(metadata["workflow"]["resolved_revision"], OLD) - - def test_second_upgrade_is_byte_idempotent_unchanged(self): - book = self.initialize_book() - self.make_legacy(book) - first = self.canonical_json( - self.run_book("workflow-upgrade", "sample", "--to", NEW, "--json") - ) - self.assertEqual(first["outcome"], "changed") - before = self.snapshot(book) - - second_result = self.run_book("workflow-upgrade", "sample", "--to", NEW, "--json") - second = self.canonical_json(second_result) - - self.assertEqual(second["outcome"], "unchanged") - self.assertEqual(second["migrated_paths"], []) - self.assertEqual(second["lifecycle_downgrades"], []) - self.assertEqual(self.snapshot(book), before) - - -if __name__ == "__main__": - unittest.main() From b2bb005c33597c5c24333f102a42ec9824fcbdd1 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:55:13 +0300 Subject: [PATCH 04/43] refactor: drop migration API exports --- scripts/workflow_v2/__init__.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/scripts/workflow_v2/__init__.py b/scripts/workflow_v2/__init__.py index 11c2964..ed029bd 100644 --- a/scripts/workflow_v2/__init__.py +++ b/scripts/workflow_v2/__init__.py @@ -1,4 +1,4 @@ -"""Public internal API for Workflow v2 state infrastructure.""" +"""Public internal API for the Book Translator workflow runtime.""" from .claims import ( ActiveClaim, @@ -24,21 +24,6 @@ GitHubTreeEntry, ) from .github_storage import GitHubStorage -from .migration_journal import ( - MIGRATION_PATH, - MigrationJournalError, - load_migration_journal, - serialize_migration_journal, - validate_migration_journal, -) -from .migrations import ( - MigratedDocument, - MigrationCompatibilityError, - MigrationConflict, - MigrationError, - detect_schema_version, - migrate_document, -) from .parallel_schema import install_parallel_schema_extensions from .repository import LoadedDocument, RepositoryError, WorkflowStateRepository from .reviews import ( @@ -95,12 +80,6 @@ "InvalidClaimSelector", "InvalidStoragePath", "LoadedDocument", - "MIGRATION_PATH", - "MigratedDocument", - "MigrationCompatibilityError", - "MigrationConflict", - "MigrationError", - "MigrationJournalError", "ParsedDocument", "RepositoryError", "ReviewClaimError", @@ -123,12 +102,7 @@ "UnsupportedSchemaVersion", "WorkflowStateRepository", "canonical_unit_id", - "detect_schema_version", - "load_migration_journal", - "migrate_document", "parse_document", "patch_text", "resolve_selector", - "serialize_migration_journal", - "validate_migration_journal", ] From 764c5b644dc302c48bfb8cfac716287971a07ebe Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:55:40 +0300 Subject: [PATCH 05/43] refactor: remove migration admission path --- scripts/workflow_v2/coordination.py | 31 +++-------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/scripts/workflow_v2/coordination.py b/scripts/workflow_v2/coordination.py index 658fbfa..fd8f054 100644 --- a/scripts/workflow_v2/coordination.py +++ b/scripts/workflow_v2/coordination.py @@ -1,4 +1,4 @@ -"""Short-lived book admission coordination for Workflow v2.""" +"""Short-lived book admission coordination for the current workflow.""" from __future__ import annotations @@ -9,7 +9,6 @@ from typing import Any from uuid import uuid4 -from .migration_journal import MigrationJournalError, load_migration_journal from .repository import RepositoryError, WorkflowStateRepository from .schemas import SCHEMA_VERSION, SchemaError, SchemaKind from .storage import ( @@ -51,7 +50,7 @@ def _format_utc(value: datetime) -> str: class BookCoordinationManager: - """Serialize short claim/finalize/upgrade/acceptance/reconciliation transitions.""" + """Serialize short claim/finalize/acceptance/reconciliation transitions.""" def __init__( self, @@ -82,19 +81,6 @@ def _read(self) -> CoordinationLease: loaded = self.repository.read(COORDINATION_PATH, SchemaKind.COORDINATION_LOCK) return CoordinationLease(COORDINATION_PATH, loaded.data, loaded.version) - def migration_active(self) -> bool: - """Return whether a strict durable migration journal requires recovery.""" - - try: - load_migration_journal(self.repository.storage) - except StorageNotFound: - return False - except MigrationJournalError as exc: - raise CoordinationError(f"migration journal is invalid: {exc}") from exc - except StorageError as exc: - raise CoordinationError(f"migration journal is unavailable: {exc}") from exc - return True - def acquire( self, *, @@ -105,28 +91,17 @@ def acquire( allowed = { "claim_admission", "finalize_admission", - "workflow_upgrade", "proposal_reconcile", "translation_acceptance", } if operation not in allowed: raise CoordinationError( - "operation must be claim_admission, finalize_admission, workflow_upgrade, proposal_reconcile, or translation_acceptance" + "operation must be claim_admission, finalize_admission, proposal_reconcile, or translation_acceptance" ) if not isinstance(session_id, str) or not session_id.strip(): raise CoordinationError("session_id must be a non-empty string") if type(lease_seconds) is not int or lease_seconds <= 0: raise CoordinationError("lease_seconds must be a positive integer") - if operation in { - "claim_admission", - "finalize_admission", - "proposal_reconcile", - "translation_acceptance", - }: - if self.migration_active(): - raise CoordinationConflict( - "admission is blocked while workflow migration recovery is active" - ) now = self._now() document = { From f90e68365e7e71f5cf957c4171a9f5d4fdf88384 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:56:51 +0300 Subject: [PATCH 06/43] refactor: make status current-workflow only --- scripts/workflow_v2/status.py | 93 +++++++++-------------------------- 1 file changed, 23 insertions(+), 70 deletions(-) diff --git a/scripts/workflow_v2/status.py b/scripts/workflow_v2/status.py index 77e0911..e5e879f 100644 --- a/scripts/workflow_v2/status.py +++ b/scripts/workflow_v2/status.py @@ -1,4 +1,4 @@ -"""Read-only Workflow v2 status and resume resolution.""" +"""Read-only status and resume resolution for the current workflow.""" from __future__ import annotations @@ -7,7 +7,6 @@ from .claims import ClaimManager, canonical_unit_id from .coordination import FINALIZATION_PATH -from .migration_journal import MIGRATION_PATH, MigrationJournalError, load_migration_journal from .repository import RepositoryError, WorkflowStateRepository from .reviews import REVIEW_EVIDENCE_VERSION, ReviewEvidenceError, ReviewLedgerManager from .schemas import SchemaError, SchemaKind @@ -47,16 +46,8 @@ def status( structural_errors: Sequence[str] = (), corpus: Mapping[str, Any] | None = None, ) -> dict[str, Any]: - metadata_doc = self.repository.read( - "metadata.json", - SchemaKind.METADATA, - allow_legacy=True, - ) - progress_doc = self.repository.read( - "progress.json", - SchemaKind.PROGRESS, - allow_legacy=True, - ) + metadata_doc = self.repository.read("metadata.json", SchemaKind.METADATA) + progress_doc = self.repository.read("progress.json", SchemaKind.PROGRESS) metadata = metadata_doc.data progress = progress_doc.data @@ -73,33 +64,19 @@ def status( errors = [str(error) for error in structural_errors if str(error)] if workflow_revision is None: errors.append("workflow revision is unavailable") + if not isinstance(workflow, Mapping) or workflow.get("review_evidence") != REVIEW_EVIDENCE_VERSION: + errors.append( + f"workflow review evidence must be {REVIEW_EVIDENCE_VERSION!r}" + ) - corpus_data = dict(corpus or {"state": "unsealed"}) + corpus_data = dict(corpus or {"state": "invalid", "error": "source corpus preflight was not provided"}) corpus_state = corpus_data.get("state") - if corpus_state not in {"verified", "unsealed", "invalid"}: + if corpus_state not in {"verified", "invalid"}: errors.append(f"unsupported corpus state: {corpus_state!r}") elif corpus_state == "invalid": detail = corpus_data.get("error") errors.append(str(detail) if detail else "source corpus integrity is invalid") - migration: dict[str, Any] = {"active": False} - try: - journal, _ = load_migration_journal(self.repository.storage) - except StorageNotFound: - pass - except (MigrationJournalError, StorageError) as exc: - errors.append(f"migration state is unavailable or invalid: {exc}") - else: - migration = { - "active": True, - "phase": journal["phase"], - "from_revision": journal["from_revision"], - "to_revision": journal["to_revision"], - "document_count": len(journal["documents"]), - } - if journal["book_slug"] != progress.get("book_slug"): - errors.append("migration journal book_slug does not match progress") - finalization: dict[str, Any] = {"active": False} try: marker = self.repository.read(FINALIZATION_PATH, SchemaKind.FINALIZATION_LOCK).data @@ -133,32 +110,19 @@ def status( review_states_by_number: dict[int, str] = {} review_ledger_revision: str | None = None - review_mode = "legacy_lifecycle" - if isinstance(workflow, Mapping) and workflow.get("review_evidence") == REVIEW_EVIDENCE_VERSION: - review_mode = REVIEW_EVIDENCE_VERSION - try: - review_ledger_revision = self.repository.read( - "review-ledger.json", - SchemaKind.REVIEW_LEDGER, - ).version - review_manager = ReviewLedgerManager( - self.repository, - artifact_reader=self._artifact_reader, - ) - for resolution in review_manager.resolve_all(progress, metadata): - review_states_by_number[resolution.chapter_number] = resolution.state - except (ReviewEvidenceError, StorageError, StorageNotFound) as exc: - errors.append(f"review evidence is unavailable or invalid: {exc}") - else: - for chapter in chapters: - number = chapter.get("number") - lifecycle_state = chapter.get("status") - if lifecycle_state == "reviewed": - review_states_by_number[number] = "pass" - elif lifecycle_state == "translated": - review_states_by_number[number] = "missing" - else: - review_states_by_number[number] = "untranslated" + try: + review_ledger_revision = self.repository.read( + "review-ledger.json", + SchemaKind.REVIEW_LEDGER, + ).version + review_manager = ReviewLedgerManager( + self.repository, + artifact_reader=self._artifact_reader, + ) + for resolution in review_manager.resolve_all(progress, metadata): + review_states_by_number[resolution.chapter_number] = resolution.state + except (ReviewEvidenceError, RepositoryError, SchemaError, StorageError, StorageNotFound) as exc: + errors.append(f"review evidence is unavailable or invalid: {exc}") reviews = {state: 0 for state in _REVIEW_STATES} units: list[dict[str, Any]] = [] @@ -228,14 +192,13 @@ def status( "schema_version": STATUS_SCHEMA_VERSION, "book_slug": progress.get("book_slug"), "workflow_revision": workflow_revision, - "review_mode": review_mode, + "review_mode": REVIEW_EVIDENCE_VERSION, "valid": not errors, "errors": errors, "lifecycle": lifecycle, "reviews": reviews, "claims": claims, "corpus": corpus_data, - "migration": migration, "finalization": finalization, "units": units, "state_revisions": revisions, @@ -262,14 +225,6 @@ def resume( "context": self._context("blocked", None, status), } - migration = status.get("migration") - if isinstance(migration, Mapping) and migration.get("active") is True: - return { - "schema_version": STATUS_SCHEMA_VERSION, - "operation": "workflow_upgrade", - "context": self._context("workflow_upgrade", None, status), - } - finalization = status.get("finalization") if isinstance(finalization, Mapping) and finalization.get("active") is True: return { @@ -438,8 +393,6 @@ def _context( files.append("review-ledger.json") if operation == "finalize": files.append(FINALIZATION_PATH) - if operation == "workflow_upgrade": - files.append(MIGRATION_PATH) return { "role": role, From bd07add7a01932c491c74459c314f6049fb09c5f Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:58:24 +0300 Subject: [PATCH 07/43] refactor: require current source identity schema --- scripts/workflow_v2/source_schema.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/scripts/workflow_v2/source_schema.py b/scripts/workflow_v2/source_schema.py index 805a606..f53af65 100644 --- a/scripts/workflow_v2/source_schema.py +++ b/scripts/workflow_v2/source_schema.py @@ -1,4 +1,4 @@ -"""Package-level Workflow v2 explicit-source schema extensions.""" +"""Current-workflow explicit-source schema requirements.""" from __future__ import annotations @@ -11,7 +11,7 @@ def _validate_metadata_source(data: Mapping[str, Any], schema: SchemaKind) -> None: if "source" not in data: - return + raise schemas._field(schema, "source", "is required") source = data.get("source") if not isinstance(source, Mapping): raise schemas._field(schema, "source", "must be an object") @@ -42,13 +42,10 @@ def _validate_metadata_source(data: Mapping[str, Any], schema: SchemaKind) -> No def _validate_manifest_source_extension(data: Mapping[str, Any], schema: SchemaKind) -> None: - has_mode = "source_storage_mode" in data - has_size = "source_size_bytes" in data - if has_mode != has_size: - missing = "source_size_bytes" if has_mode else "source_storage_mode" - raise schemas._field(schema, missing, "is required with explicit source manifest identity") - if not has_mode: - return + if "source_storage_mode" not in data: + raise schemas._field(schema, "source_storage_mode", "is required") + if "source_size_bytes" not in data: + raise schemas._field(schema, "source_size_bytes", "is required") mode = data.get("source_storage_mode") if mode not in {"embedded", "private_external"}: @@ -65,7 +62,7 @@ def _validate_manifest_source_extension(data: Mapping[str, Any], schema: SchemaK def install_source_schema_extensions() -> None: - """Install explicit-source validators exactly once at package import time.""" + """Install current explicit-source validators exactly once at package import time.""" if getattr(schemas, "_explicit_source_v1_installed", False): return From 2cfcd93bc5659832a7ab143c4158c2a7a8559494 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:59:04 +0300 Subject: [PATCH 08/43] refactor: remove migration CLI registration --- scripts/workflow_v2/review_cli.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/workflow_v2/review_cli.py b/scripts/workflow_v2/review_cli.py index 2043a96..7b9ec80 100644 --- a/scripts/workflow_v2/review_cli.py +++ b/scripts/workflow_v2/review_cli.py @@ -1,4 +1,4 @@ -"""Argparse integration for Workflow v2 machine review evidence.""" +"""Argparse integration for current machine review evidence.""" from __future__ import annotations @@ -62,7 +62,7 @@ def _repository(root: Path, slug: str) -> tuple[Path, WorkflowStateRepository]: def _load_progress(repository: WorkflowStateRepository) -> tuple[dict[str, Any], str]: try: - loaded = repository.read("progress.json", SchemaKind.PROGRESS, allow_legacy=True) + loaded = repository.read("progress.json", SchemaKind.PROGRESS) except (SchemaError, RepositoryError, StorageError) as exc: raise ReviewCliError(f"Invalid progress.json: {exc}") from exc return loaded.data, loaded.version @@ -70,7 +70,7 @@ def _load_progress(repository: WorkflowStateRepository) -> tuple[dict[str, Any], def _load_metadata(repository: WorkflowStateRepository) -> dict[str, Any]: try: - return repository.read("metadata.json", SchemaKind.METADATA, allow_legacy=True).data + return repository.read("metadata.json", SchemaKind.METADATA).data except (SchemaError, RepositoryError, StorageError) as exc: raise ReviewCliError(f"Invalid metadata.json: {exc}") from exc @@ -383,9 +383,6 @@ def register_review_commands(subparsers: argparse._SubParsersAction, root: Path) register_status_commands(subparsers, root, error_factory=ReviewCliError) - # Lazy imports avoid registration cycles while extending the existing top-level parser. from .epub_cli import register_epub_commands - from .migrations_cli import register_migration_command register_epub_commands(subparsers, root, error_factory=ReviewCliError) - register_migration_command(subparsers, root, error_factory=ReviewCliError) From ca14fdee01e9309891034aae96e68c93aca3f2c1 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:59:39 +0300 Subject: [PATCH 09/43] refactor: enforce current workspace at preflight boundary --- scripts/workflow_v2/source_schema.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/workflow_v2/source_schema.py b/scripts/workflow_v2/source_schema.py index f53af65..805a606 100644 --- a/scripts/workflow_v2/source_schema.py +++ b/scripts/workflow_v2/source_schema.py @@ -1,4 +1,4 @@ -"""Current-workflow explicit-source schema requirements.""" +"""Package-level Workflow v2 explicit-source schema extensions.""" from __future__ import annotations @@ -11,7 +11,7 @@ def _validate_metadata_source(data: Mapping[str, Any], schema: SchemaKind) -> None: if "source" not in data: - raise schemas._field(schema, "source", "is required") + return source = data.get("source") if not isinstance(source, Mapping): raise schemas._field(schema, "source", "must be an object") @@ -42,10 +42,13 @@ def _validate_metadata_source(data: Mapping[str, Any], schema: SchemaKind) -> No def _validate_manifest_source_extension(data: Mapping[str, Any], schema: SchemaKind) -> None: - if "source_storage_mode" not in data: - raise schemas._field(schema, "source_storage_mode", "is required") - if "source_size_bytes" not in data: - raise schemas._field(schema, "source_size_bytes", "is required") + has_mode = "source_storage_mode" in data + has_size = "source_size_bytes" in data + if has_mode != has_size: + missing = "source_size_bytes" if has_mode else "source_storage_mode" + raise schemas._field(schema, missing, "is required with explicit source manifest identity") + if not has_mode: + return mode = data.get("source_storage_mode") if mode not in {"embedded", "private_external"}: @@ -62,7 +65,7 @@ def _validate_manifest_source_extension(data: Mapping[str, Any], schema: SchemaK def install_source_schema_extensions() -> None: - """Install current explicit-source validators exactly once at package import time.""" + """Install explicit-source validators exactly once at package import time.""" if getattr(schemas, "_explicit_source_v1_installed", False): return From 5bf406f6c5eaf6a2e9792b465f28eb93abbf4a3e Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:00:22 +0300 Subject: [PATCH 10/43] refactor: reject unsealed legacy workspaces --- scripts/workflow_v2/status_cli.py | 51 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/scripts/workflow_v2/status_cli.py b/scripts/workflow_v2/status_cli.py index d8ac7a0..68acdcc 100644 --- a/scripts/workflow_v2/status_cli.py +++ b/scripts/workflow_v2/status_cli.py @@ -1,4 +1,4 @@ -"""Argparse integration for read-only Workflow v2 status and resume.""" +"""Argparse integration for read-only current-workflow status and resume.""" from __future__ import annotations @@ -63,7 +63,7 @@ def _resolver(root: Path, slug: str) -> StatusResolver: def default_preflight(root: Path, slug: str) -> tuple[Sequence[str], Mapping[str, Any]]: - """Reuse existing structural and corpus validators without copying hash logic.""" + """Run structural and sealed-corpus validation for a supported workspace.""" try: book_module = importlib.import_module("book") @@ -73,42 +73,42 @@ def default_preflight(root: Path, slug: str) -> tuple[Sequence[str], Mapping[str except Exception as exc: raise StatusCliError(f"cannot run structural preflight: {exc}") from exc - manifest_path = book_dir / "source-manifest.json" mode = source_storage_mode(metadata) + if mode is None: + return structural_errors, { + "state": "invalid", + "error": "metadata.json source identity is required by the current workflow", + } + + manifest_path = book_dir / "source-manifest.json" if not manifest_path.is_file(): - if mode is not None: - return structural_errors, { - "state": "invalid", - "storage_mode": mode, - "error": "source-manifest.json is missing for explicit-source book", - } - return structural_errors, {"state": "unsealed"} + return structural_errors, { + "state": "invalid", + "storage_mode": mode, + "error": "source-manifest.json is required by the current workflow", + } try: corpus_module = importlib.import_module("corpus") manifest = corpus_module.load_source_manifest(book_dir) if manifest is None: - if mode is not None: - return structural_errors, { - "state": "invalid", - "storage_mode": mode, - "error": "source-manifest.json is missing for explicit-source book", - } - return structural_errors, {"state": "unsealed"} + return structural_errors, { + "state": "invalid", + "storage_mode": mode, + "error": "source-manifest.json is required by the current workflow", + } verified = corpus_module.verify_manifest(book_dir, metadata, progress, manifest) except Exception as exc: - payload: dict[str, Any] = {"state": "invalid", "error": str(exc)} - if mode is not None: - payload["storage_mode"] = mode + payload: dict[str, Any] = { + "state": "invalid", + "storage_mode": mode, + "error": str(exc), + } return structural_errors, payload return structural_errors, dict(verified) -# Private alias retained for callers that imported the pre-#12 helper directly. -_default_preflight = default_preflight - - def _snapshot( args: argparse.Namespace, root: Path, @@ -297,7 +297,6 @@ def register_status_commands( ) ) - # Imported lazily to keep finalize CLI independent of status CLI internals. from .finalize_cli import register_finalize_command register_finalize_command( @@ -305,4 +304,4 @@ def register_status_commands( root, preflight=resolved_preflight, error_factory=error_factory, - ) \ No newline at end of file + ) From f91cadc083b7d3ef546091e81aad9697308ce116 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:01:07 +0300 Subject: [PATCH 11/43] refactor: disable legacy document normalization --- scripts/workflow_v2/repository.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/workflow_v2/repository.py b/scripts/workflow_v2/repository.py index d68e0fc..9459aac 100644 --- a/scripts/workflow_v2/repository.py +++ b/scripts/workflow_v2/repository.py @@ -1,4 +1,4 @@ -"""Schema-aware JSON repository for Workflow v2 durable state.""" +"""Schema-aware JSON repository for the current durable workflow state.""" from __future__ import annotations @@ -127,6 +127,12 @@ def read( *, allow_legacy: bool = False, ) -> LoadedDocument: + """Read a current-schema document. + + ``allow_legacy`` remains only as a temporary call-site compatibility parameter; + it no longer enables normalization or acceptance of legacy documents. + """ + stored = self.storage.read(path) try: text = stored.content.decode("utf-8") @@ -137,13 +143,13 @@ def read( except json.JSONDecodeError as exc: raise RepositoryError(f"{path}: invalid JSON: {exc}") from exc - parsed = parse_document(schema, raw, allow_legacy=allow_legacy) + parsed = parse_document(schema, raw) if schema == SchemaKind.PROGRESS: self._verify_translation_acceptance_integrity(parsed.data) return LoadedDocument( data=parsed.data, version=stored.version, - legacy=parsed.legacy, + legacy=False, ) def create( From 38401ae8fe9a84e145a6a00327f7a0e499b78d1d Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:02:44 +0300 Subject: [PATCH 12/43] docs: collapse orchestration to one current workflow --- docs/ORCHESTRATION.md | 471 ++++++++---------------------------------- 1 file changed, 83 insertions(+), 388 deletions(-) diff --git a/docs/ORCHESTRATION.md b/docs/ORCHESTRATION.md index 23ecb37..5ec68a9 100644 --- a/docs/ORCHESTRATION.md +++ b/docs/ORCHESTRATION.md @@ -1,107 +1,32 @@ # Orchestration protocol -This file is authoritative for Book Translator execution topology, book initialization/resume, bounded role context, chapter-state transitions, single-writer persistence, failure handling, validation ordering, source-corpus integrity, and output/completion sequencing. +This file is authoritative for Book Translator execution topology, durable state transitions, claims, review evidence, source integrity, resume behavior, finalization, and output sequencing. The repository supports one current workflow contract; older workspace formats must be converted outside the production runtime before use. -It is loaded by the `orchestrator` context profile together with `AGENTS.md`. +The Orchestrator loads the current `agent-manifest.json`, follows its `context_profiles`, and uses `metadata.json.workflow` as durable provenance. `contract_read_order` is not a supported routing mechanism. Do not silently reinterpret an incompatible workspace as current state. -Literary translation and review quality belong exclusively to `docs/TRANSLATION.md`. This file consumes Translator artifacts and Reviewer outcomes without restating the literary checklist. +Literary translation and review quality belong to `docs/TRANSLATION.md`. Commit boundaries and recovery-friendly history are defined in `docs/COMMIT_DISCIPLINE.md`. -## Orchestrator responsibility +## Execution topology -The Orchestrator coordinates durable book work without carrying an ever-growing copy of the book or setup instructions in conversational context. - -At the start of a run: - -1. identify the Book Translator installation root; -2. identify the active book or determine that a new book must be initialized; -3. determine the workflow revision associated with that book; -4. read `agent-manifest.json` from that revision; -5. select the execution contract using the schema-aware routing rules below; -6. read only the durable book state needed to choose the next operation. - -For an existing book, `metadata.json.workflow` is the provenance source for its execution contract. - -## Workflow-contract compatibility - -Never project the currently installed manifest schema backward onto a book pinned to an older workflow revision. - -After reading `agent-manifest.json` from the book's recorded workflow revision: - -1. If that manifest contains a valid `context_profiles` mapping, use the role-specific profile declared by that same revision. -2. If `context_profiles` is absent but the manifest contains a legacy `contract_read_order`, follow that recorded revision's legacy contract mechanism exactly. Do not require v3 `context_profiles`, and do not mix current v3 contract files into the legacy run. -3. If neither routing mechanism can be interpreted safely, stop before state-changing work and require an explicit compatible workflow upgrade or report the exact reproducibility limitation. - -This compatibility rule applies to Orchestrator, Translator, and Reviewer contract selection. A legacy book may therefore continue under its recorded pre-v3 contract without being silently upgraded merely because the installed Book Translator revision now uses schema v3. - -## Execution modes - -### Preferred: `isolated_workers` - -Use independent worker sessions/subagents when the active environment supports them. - -For each chapter: +Preferred execution uses isolated workers: ```text Orchestrator - -> translator role (fresh bounded context) - -> reviewer role (fresh independent bounded context) - -> Orchestrator accepts/rejects proposals and persists valid state + -> translator role + -> durable translation acceptance + -> reviewer role + -> durable review evidence + -> orchestrator promotion -> next chapter ``` -The reviewer must not inherit hidden reasoning from the translator. It receives the source, translation artifact, and required durable literary context. - -### Fallback: `single_agent_bounded_context` - -When isolated workers are unavailable, preserve the same logical role boundaries in one physical agent session: - -1. build and load only the translator context pack; -2. complete the translation role and persist/return its artifact; -3. end that role context; -4. rebuild the reviewer context pack from durable files; -5. independently perform review under `docs/TRANSLATION.md` when the active workflow revision uses the v3 translation contract, or under the equivalent literary rules from the selected legacy contract; -6. return the Reviewer outcome to the Orchestrator role; -7. perform the state transition only from the Orchestrator role. - -Do not collapse translation and review into one pass merely because only one physical agent is available. +When isolated workers are unavailable, use `single_agent_bounded_context`: construct the Translator context, finish that role, then reconstruct an independent Reviewer context from durable files. Do not pass hidden Translator reasoning to the Reviewer and do not collapse translation and review into one pass. -## Single-writer rule +Only the orchestrator may update global mutable state such as `progress.json`, `glossary.md`, `style-guide.md`, source integrity state, workflow provenance, and other book-wide decisions. Workers may return artifacts and proposals; shared-state proposals are reconciled through the central compare-and-swap path. -Only the orchestrator may update global mutable state during an orchestrated run. +## Supported workspace -This strict rule applies to: - -- `progress.json`; -- `glossary.md`; -- `style-guide.md`; -- `source-manifest.json`; -- book-level metadata; -- workflow provenance; -- other shared book-wide decisions. - -Translator and Reviewer roles may return artifacts, findings, warnings, corrections, and proposed glossary/style decisions. They do not independently race to persist shared state. - -When a worker has a frozen shared-state snapshot, glossary/style suggestions are persisted as immutable `.workflow/proposals/.json` records tied to the worker claim, `base_commit`, workflow revision, and the exact glossary/style revisions used by that worker. Workers do not directly mutate `glossary.md` or `style-guide.md`. The Orchestrator reconciles each proposal through the central single-writer path: acquire the book coordination mutex, re-check the frozen shared-state revisions, apply an accepted replacement with compare-and-swap, and persist the immutable `.workflow/proposals/.resolution.json`. A stale proposal is resolved as stale without overwriting newer shared state. - -A translation artifact may be written directly by a worker only when the active environment provides a non-conflicting target. The Orchestrator remains responsible for accepting that artifact as canonical book state. - -## Book selection - -On a new session: - -1. enumerate valid book workspaces under the active installation root; -2. if the user explicitly identifies a book, select it; -3. otherwise, if exactly one incomplete book exists, select it automatically; -4. if multiple incomplete books exist and user intent does not identify one, ask only which book to resume; -5. never choose arbitrarily between multiple plausible active books. - -## New-book initialization - -If the source book has not yet been initialized, create durable book state before translation begins. - -Prefer `scripts/book.py extract` when Python is available and the source format is supported by the helper. Otherwise reproduce the same state directly and conservatively. - -A valid initialized book has: +A supported book lives at `books//` and has current-schema durable state: ```text books// @@ -117,221 +42,118 @@ books// └── style-guide.md ``` -Initialization requirements: - -- preserve the supplied source unchanged under `source/`; -- preserve real reading order during extraction; -- use stable, unique chapter numbering/slugs and aligned source/translation paths; -- create per-book metadata and progress state; -- copy the selected workflow repository/requested ref/resolved revision into `metadata.json.workflow`; -- for a ledger-enabled workflow, record `metadata.json.workflow.review_evidence` and initialize the matching empty `review-ledger.json` rather than fabricating historical review evidence; -- initialize glossary and style-guide durable memory; -- perform structural validation before sustained translation; -- seal the verified source corpus after successful extraction. +The current workflow requires explicit workflow provenance, source identity, a sealed source manifest, and machine review evidence. Missing schema versions, lifecycle-only review state, and unsealed historical workspace shapes are unsupported rather than alternate execution modes. -When Python is available, seal the initialized corpus once: +For a new book, prefer: ```bash +python scripts/book.py extract --slug --target-language python scripts/corpus.py seal ``` -`source-manifest.json` records the SHA-256 identity of the preserved source and every extracted source artifact. Treat it as integrity/provenance state, not literary state. It must not be regenerated from an unverified replacement source merely to make validation look clean. - -For EPUB manual extraction, use the package/spine reading order rather than treating every XHTML resource as a chapter. For other structured text sources, respect explicit document structure and preserve a larger unit when boundaries are ambiguous. - -## Source corpus preflight - -Before dispatching literary work for an existing book, perform a corpus preflight once per resumed run or whenever repository/source state may have changed. - -The preflight is book-wide, not chapter-by-chapter: +Initialization preserves the supplied source, real reading order, stable chapter paths, `metadata.json.workflow`, an empty review ledger, glossary/style state, and source identity. -1. read `metadata.json` and `progress.json`; -2. compare `metadata.chapter_count` with the number of progress entries; -3. verify that every `source_path` required by `progress.json` exists, not merely the next chapter; -4. verify the preserved source declared by metadata is available when the active workspace is expected to be self-contained; -5. when `source-manifest.json` exists, verify the preserved source and every extracted artifact against its recorded SHA-256 values; -6. run structural validation before selecting the next literary task. +## Corpus preflight -When Python is available, run structural validation and, for a sealed workspace, integrity verification: +Run a book-wide corpus preflight before dispatching literary work and again whenever source/repository state may have changed: ```bash python scripts/book.py validate python scripts/corpus.py verify ``` -Run `corpus.py verify` only when `source-manifest.json` exists. A hash mismatch is a blocking integrity failure: do not continue literary work and do not regenerate the manifest merely to accept the changed files. +The preflight verifies chapter counts and paths, preserved source identity, every extracted artifact, and the SHA-256 values in `source-manifest.json`. A mismatch is blocking; never regenerate a manifest merely to bless changed bytes. -A workspace with 205 progress entries and only 13 extracted artifacts is not a partially valid 13-chapter source corpus. It is an incomplete corpus and must be repaired before translation/review continues. - -If the original source is available but the extracted tree is incomplete or integrity verification fails, restore the complete source corpus in one batch. Do not repair missing extracted chapters one at a time. - -For a sealed workspace: +If the exact trusted source is available but the extracted tree is damaged, restore the complete source corpus in one batch: ```bash python scripts/corpus.py restore ``` -For a legacy workspace without `source-manifest.json`, recovery requires a trusted SHA-256 from durable provenance or the user: - -```bash -python scripts/corpus.py restore --expected-sha256 -``` - -The restore operation must verify source identity and complete extraction before replacing canonical source artifacts. It must preserve `progress.json`, translation files, review states, glossary, and style guide. After recovery, run both structural validation and sealed-corpus verification before dispatching literary work: - -```bash -python scripts/book.py validate -python scripts/corpus.py verify -``` - -Do not substitute a later online edition, archive export, or same-named file when the recorded SHA-256 does not match. If no trusted source identity is available, report the source-reproducibility block instead of guessing. +Do not repair missing extracted chapters one at a time. Do not substitute a later edition or same-named source whose identity differs. After restore, run structural validation and `python scripts/corpus.py verify ` before dispatching literary work. -A checkpoint that intentionally omits a private/copyrighted source binary may still preserve translation work, but it is not self-contained for source-dependent review. Record that limitation explicitly. Once the exact private source is reattached, recover the whole corpus in one pass rather than repeatedly asking for individual chapters. +## Lifecycle -## Chapter states - -The workflow uses the states declared by `agent-manifest.json.chapter_states`: +The current lifecycle is: ```text pending -> extracted -> translated -> reviewed ``` -Operational meanings: - -- `pending`: the chapter is known but its usable extracted source is not yet ready; -- `extracted`: the source chapter artifact exists and is ready for translation; -- `translated`: a complete translation artifact exists, but the required independent literary review has not yet produced an accepted PASS for that artifact; -- `reviewed`: the current canonical translation artifact has passed review under the literary contract associated with the book's workflow revision and the Orchestrator has completed the required state acceptance/validation transition. - -Never use `reviewed` as a convenience label for a translation that merely looks fluent or complete. - -## Sequential chapter policy +- `pending`: usable source unit is not ready. +- `extracted`: source unit exists and may be translated. +- `translated`: canonical translation has passed machine translation acceptance but lacks accepted current review. +- `reviewed`: the exact canonical translation has current PASS evidence and has been promoted by the Orchestrator. -Translate chapters sequentially by default: +Do not translate multiple chapters concurrently by default. Sequential execution preserves terminology, voice, ambiguity, and continuity decisions: ```text -T1 -> R1 -> state commit -> T2 -> R2 -> state commit -> T3 ... +T1 -> R1 -> durable state -> T2 -> R2 -> durable state ``` -Do not translate multiple chapters concurrently by default. +Explicit parallel execution is invocation-scoped only through `resume --parallel N` with `N > 1`. -Later chapters can depend on terminology, character voice, ambiguity, and continuity decisions established during earlier reviewed chapters. Explicit parallel execution is therefore opt-in per invocation only: `resume --parallel N` with `N > 1`. Without that flag, scheduling remains sequential and unchanged. +## Durable claims -In explicit parallel mode, the Orchestrator may plan up to `N` disjoint, currently unclaimed worker assignments. Every dispatched parallel claim must record a non-empty `base_commit` plus the frozen `glossary.md` and `style-guide.md` storage revisions from the same planning snapshot. If those revisions or the base commit cannot be recorded, do not dispatch the unit in parallel. Shared-state suggestions from parallel workers use the proposal/reconciliation path above; they never authorize direct concurrent glossary/style writes. - -## Selecting the next chapter - -Unless the user explicitly requests another scope: - -1. complete the source corpus preflight for the active book; -2. inspect `progress.json` in chapter order; -3. choose the first chapter whose state is not `reviewed`; -4. verify the source artifact referenced for that chapter exists; -5. repair invalid extraction/state before dispatching literary work. - -Do not retranslate a reviewed chapter without a concrete reason. If a reviewed translation changes materially, move it back to the appropriate non-reviewed state until the changed artifact passes review again. - -## Durable claim gate - -Before dispatching literary work, the Orchestrator must acquire a durable claim for the exact chapter or validated range and role being dispatched. Claim acquisition is an execution gate: if it conflicts, do not start the worker and do not ask the user to manually schedule competing sessions. - -When Python is available, acquire the claim with the active session identity: +A worker may not start literary work without owning a durable claim for the exact unit and role: ```bash -python scripts/book.py claim --role --session-id -``` - -For explicit parallel dispatch, first obtain the fixed planning snapshot from `resume --parallel N`; use `--json` for machine consumption. Pass the exact returned values into `claim` without recomputing Git HEAD or shared-state revisions: - -```bash -python scripts/book.py claim \ +python scripts/book.py claim \ --role \ - --session-id \ - --base-commit \ - --glossary-revision \ - --style-guide-revision + --session-id ``` -Human `resume --parallel N` output emits the same three values as a copyable `claim-snapshot` flag line. If any planning value is unavailable or stale by claim admission time, do not dispatch that parallel worker. - -Inspect current ownership when needed with: +Inspect ownership with: ```bash python scripts/book.py claims ``` -A lease timestamp is not automatic permission to reuse a unit. An expired claim remains occupied until explicit cleanup removes it and records the auditable `lease_expired` lifecycle evidence. When stale claims need reclamation, run: +Expired claims remain occupied until auditable cleanup: ```bash python scripts/book.py cleanup-claims ``` -Release a claim only from the owning session, after the Orchestrator has accepted the role result or explicitly abandoned that unit. Use the same session identity that acquired the claim: +Release only from the owning session after its result is accepted or explicitly abandoned: ```bash python scripts/book.py release --session-id ``` -For the normal chapter pipeline, acquire a translator claim before translator dispatch, release it after the translation result is durably accepted or abandoned, then acquire the reviewer claim before reviewer dispatch and release it after the review result is processed. Do not infer ownership from chat history; durable claim state is authoritative. - -Range claims provide safe coordination for an explicitly requested bounded range, but they do not enable parallel translation by themselves. Explicit parallel scheduling must still return disjoint units, and each parallel worker claim must bind the exact unit/role to the planning snapshot's `base_commit`, glossary revision, and style-guide revision before work starts. - -## Translator context pack +### Parallel snapshot binding -To dispatch the `translator` role: +For explicit parallel work, obtain the planning snapshot from `resume --parallel N` and pass the exact returned values into claim acquisition: -1. read `agent-manifest.json` from the book's workflow revision; -2. select the Translator contract using the workflow-contract compatibility rules above: use the `translator` `context_profiles` entry for v3 manifests, or the recorded legacy `contract_read_order` mechanism for pre-v3 manifests; -3. add the task-specific durable inputs: - - metadata relevant to language/book identity and workflow provenance; - - current `glossary.md`; - - current `style-guide.md`; - - current chapter source; - - the smallest prior excerpt/context needed for continuity; - - expected translation artifact/path; - - exact task/output request. - -Do not include `docs/AGENT_SETUP.md` or this orchestration contract in the Translator context merely because the Orchestrator has them loaded when the active v3 profile does not require them. For a legacy workflow, follow that revision's own contract loading rules rather than inventing a v3 subset. +```bash +python scripts/book.py claim \ + --role \ + --session-id \ + --base-commit \ + --glossary-revision \ + --style-guide-revision +``` -If the chapter directly continues a scene and prior text is materially necessary, include the necessary bounded context. Otherwise prefer durable glossary/style decisions plus a small continuity excerpt over entire prior chapters. +Human output exposes the same `--base-commit`, `--glossary-revision`, and `--style-guide-revision` values. If the frozen shared-state snapshot drifts, reject stale work rather than overwriting newer state. -## Accepting a Translator result +## Translator boundary -The Translator returns a complete chapter artifact plus any proposals/warnings defined by the active book workflow's literary contract. A Translator result is not durably accepted merely because the translation file exists. +Translator context contains only the current Translator `context_profiles` contracts plus bounded durable inputs: metadata, glossary, style guide, source unit, necessary continuity context, and target artifact path. -For the first `extracted -> translated` transition, acceptance must run while the matching Translator claim is still live: +A written translation file is not a durable state transition. While the matching Translator claim is live, accept the canonical artifact with: ```bash python scripts/book.py accept-translation \ --session-id ``` -`accept-translation` verifies the exact chapter and workflow revision, the owning live Translator claim, a non-empty canonical translation artifact, and the current source/translation SHA-256 identities. When the claim contains frozen glossary/style revisions, the command re-checks that shared-state snapshot under the same book coordination mutex used by proposal reconciliation; drift before acceptance rejects the result as stale without advancing `progress.json`. - -On success, one compare-and-swap of `progress.json` performs both effects atomically: it changes the chapter state to `translated` and stores `translation_acceptance` evidence on that chapter. The evidence binds the accepted source and translation hashes to the exact claim id/revision, Translator session, claim base progress revision, `base_commit`, workflow revision, and frozen shared-state revisions (or `null` for a legacy/sequential claim without a snapshot). There is no separate evidence-write window in which lifecycle state can advance without its acceptance provenance. - -Only after `accept-translation` succeeds may the Orchestrator release the Translator claim. A retry after a successful progress CAS is idempotent when the current canonical artifacts still match the stored `translation_acceptance`, including after the original claim has been released. If the artifact identity or stored evidence no longer matches, fail closed rather than fabricating acceptance. - -Proposed global decisions are handled explicitly rather than silently committed by the worker. Accepted glossary/style proposals are reconciled centrally through the single-writer proposal path; the worker result never directly races those shared files. +`accept-translation` verifies the current workflow revision, owning live Translator claim, canonical source/translation bytes, artifact SHA-256 identity, and any frozen shared-state snapshot. Only a successful compare-and-swap may advance `extracted -> translated`. Release the Translator claim afterward. -## Reviewer context pack +## Reviewer boundary -To dispatch the `reviewer` role: +Reviewer context contains the exact source, canonical translation, current glossary/style decisions, bounded continuity context, and `docs/TRANSLATION.md`. It does not receive hidden Translator reasoning. -1. select the Reviewer contract from the same book workflow revision using the compatibility rules above: use the `reviewer` `context_profiles` entry for v3 manifests, or the recorded legacy `contract_read_order` mechanism for pre-v3 manifests; -2. include the current source chapter; -3. include the current canonical translation artifact; -4. include current glossary and style-guide decisions; -5. include only bounded continuity context required to judge the passage; -6. request the review outcome defined by the active workflow revision. - -Do not pass the Translator's hidden reasoning or justification. - -## Review/state boundary - -For the v3 literary contract, the Reviewer returns either `PASS` or `CORRECTIONS_REQUIRED` under `docs/TRANSLATION.md`. A legacy workflow follows the equivalent review/state semantics defined by that recorded revision. - -For a ledger-enabled book, a Reviewer result in chat or worker output is not durable review evidence by itself. The Orchestrator must record the outcome while the matching reviewer claim is still live: +The Reviewer outcome is `PASS` or `CORRECTIONS_REQUIRED`. Chat output alone is not authoritative review coverage. Record the outcome while the Reviewer claim is live: ```bash python scripts/book.py review-record \ @@ -339,186 +161,59 @@ python scripts/book.py review-record \ --session-id ``` -The command hashes the current canonical source and translation bytes and binds the record to the book's immutable workflow/review-contract revision. Handwritten Markdown audit or review files may be useful notes, but they are not authoritative review coverage. - -Inspect machine-resolved review state when needed with: +Inspect machine review state with: ```bash python scripts/book.py reviews ``` -A ledger-enabled chapter has current PASS coverage only when the highest-sequence exact record matches the current source hash, translation hash, workflow revision, and review-contract revision and has outcome `PASS`. If either artifact changes, the old record remains audit history but current review resolution becomes `stale`; no chat statement or Markdown note restores coverage. - -The normal ledger-enabled state boundary is: +A current PASS is bound to the exact source hash, translation hash, workflow revision, and review contract revision. Changing the artifact makes prior evidence stale. Markdown review notes may be useful context, but they are not authoritative review coverage. -```text -translated artifact - -> acquire reviewer claim - -> Reviewer under the active book workflow revision - -> record Reviewer outcome with review-record while claim is live - -> CORRECTIONS_REQUIRED: remain translated - -> release reviewer claim - -> apply/obtain corrections through the translator boundary - -> acquire a fresh reviewer claim - -> review corrected artifact again - -> PASS: verify current PASS with machine review state - -> structural/integrity validation while still translated - -> promote through accept-review - -> validate the resulting reviewed state - -> release reviewer claim -``` - -For a current PASS, promote lifecycle state only through: +`CORRECTIONS_REQUIRED` keeps the unit translated. Apply corrections through the Translator boundary and run an independent review again. A current PASS is necessary but does not itself mutate lifecycle state. Promote only through: ```bash python scripts/book.py accept-review ``` -`accept-review` re-resolves current evidence and uses compare-and-swap on `progress.json`; a missing, stale, mismatched, or current `CORRECTIONS_REQUIRED` record cannot promote the chapter. For snapshot-backed parallel review evidence, promotion also re-checks the stored glossary/style revisions used by the reviewer, so a shared-state change between `review-record` and `accept-review` blocks promotion as stale. If a concurrent state change occurs, re-read repository state rather than treating the old PASS result as reusable authority. - -If the outcome is `CORRECTIONS_REQUIRED`, do not mark the chapter reviewed. Apply or obtain the corrections through the appropriate role boundary and re-run independent review on the corrected artifact until a Reviewer returns `PASS`, record each outcome, and only then attempt `accept-review`. - -A `PASS` is necessary but not sufficient for the durable state transition. Before promotion, the Orchestrator must also ensure the reviewed artifact is the canonical artifact, required files exist, accepted global decisions have been applied consistently, and structural/integrity state validates. For ledger-enabled books, `progress.json.status=reviewed` is valid only while the current exact review resolution remains `pass`; if later artifact changes make that evidence stale, validation must fail until lifecycle state and review evidence are reconciled explicitly. - -## Context freshness - -A worker result is valid only for the durable state it was given. - -When hashes/revisions are available, associate a dispatch with: - -- `metadata.json.workflow.resolved_revision`; -- current chapter/progress state; -- current glossary state; -- current style-guide state; -- the current source/translation artifacts relevant to that role. - -If shared state changes materially before a result is accepted, rebuild or re-check the result against the new state rather than accepting it blindly. - -## Resume behavior - -Repository state is more authoritative than chat history. - -For an existing book: - -1. select the installation root and book deterministically; -2. read the book's `metadata.json.workflow` before selecting the execution contract; -3. use its `resolved_revision` when available, otherwise the most specific recorded requested ref; -4. read `agent-manifest.json` from that recorded workflow revision; -5. select the Orchestrator contract using the workflow-contract compatibility rules above: v3 `context_profiles` when present, otherwise the recorded legacy `contract_read_order` mechanism; -6. read `progress.json`, `glossary.md`, `style-guide.md`, and `source-manifest.json` when present; -7. run the source corpus preflight and repair the complete corpus if required; -8. inspect only the bounded source/translation context required for the next operation; -9. continue from the first non-`reviewed` chapter unless the user explicitly requests another scope. - -Do not use a successful lookup of the next chapter as a substitute for corpus preflight. A later missing or hash-mismatched source artifact is a repository-integrity defect even when the immediate chapter happens to exist. - -`/.book-translator-install.json` describes the workflow currently installed at that root, but it does not override an existing book's recorded workflow provenance. - -If the installed revision differs from the book revision, do not silently rewrite the book's provenance or execute it under the newer contract. Load and interpret the recorded workflow revision when possible; otherwise state the exact reproducibility limitation before making state-changing claims. - -Different books in one workspace may legitimately retain different workflow revisions. - -## Explicit workflow upgrade - -A workflow upgrade for an existing book is an explicit state transition, never an incidental side effect of installation changes. - -Before upgrading: - -1. record the current book provenance and validate current structure/integrity; -2. resolve the requested new workflow revision; -3. inspect compatibility relevant to the book's durable schema/state; -4. update the book provenance only as part of the explicit upgrade; -5. select the Orchestrator contract using the new revision's routing mechanism; -6. validate the book again before continuing chapter work. - -If compatibility cannot be established safely, keep the existing book revision instead of guessing. - -## Structural validation - -Structural validation verifies repository consistency, not literary fidelity. - -When Python is available, prefer: - -```bash -python scripts/book.py validate -``` - -Use the equivalent helper path for a namespaced installation. +`accept-review` re-resolves current PASS evidence and compare-and-swaps `progress.json`; missing, stale, mismatched, or `CORRECTIONS_REQUIRED` evidence cannot produce `reviewed`. -At minimum, structural validation must protect these invariants: +## Resume -- the preserved source declared by metadata exists when the workspace is expected to be self-contained; -- chapter numbers/slugs are unique and ordered; -- chapter count matches progress entries; -- every extracted-or-later chapter has its source artifact; -- the actual extracted corpus is complete relative to `progress.json`, not just complete through the next chapter; -- translated-or-reviewed chapters have non-empty translation artifacts; -- glossary and style guide exist for active books; -- workflow provenance is present for new books; -- source identity/integrity state is retained when `source-manifest.json` exists; -- no translation artifact replaced the preserved source; -- when `translation_acceptance` evidence is present, its recorded source and translation hashes still match the current canonical artifacts; -- for ledger-enabled books, `review-ledger.json` exists, validates, and every chapter marked `reviewed` resolves to current exact PASS evidence. +Repository state is authoritative, not chat history. For every resumed run: -When `source-manifest.json` exists, structural validation is not enough: `python scripts/corpus.py verify ` must also confirm the preserved source and extracted SHA-256 values before literary work resumes. +1. identify the book deterministically; +2. read current-schema `metadata.json` and `progress.json`; +3. run corpus preflight; +4. inspect claims and machine review evidence; +5. continue from the first valid non-reviewed operation unless the user requested another bounded scope. -Neither structural nor integrity validation can substitute for a Reviewer `PASS` under the active literary contract. +Do not silently infer an old workspace into a supported shape. Invalid or incomplete durable state blocks mutation and must be converted or repaired explicitly outside the runtime. -## Failure handling +## Failure and recovery -- If translation fails, is incomplete, or cannot pass `accept-translation`, do not advance the chapter to `translated`. -- If stored `translation_acceptance` no longer matches the canonical source or translation artifact, treat the translated lifecycle state as invalid until explicitly reconciled. -- If review fails to run, errors, returns `CORRECTIONS_REQUIRED`, or cannot be durably recorded, keep the chapter `translated`. -- If review evidence is missing, malformed, mismatched, or stale, do not promote or continue treating the lifecycle state as validly reviewed. -- If corpus preflight, hash verification, or structural validation fails, stop state advancement, repair durable state in one batch when possible, and validate again before starting the next chapter. -- If the exact source is unavailable, do not silently use a same-title/same-name replacement. -- If the required workflow revision cannot be loaded or its routing mechanism cannot be interpreted, do not silently substitute another revision and claim exact reproducibility. -- If isolated workers are unavailable, use `single_agent_bounded_context` rather than asking the user to manually orchestrate roles. +- Translation failure or failed `accept-translation` leaves the unit unadvanced. +- `CORRECTIONS_REQUIRED` or missing/stale review evidence leaves the unit `translated`. +- Corpus or structural failure blocks literary work. +- A source identity mismatch is never repaired by substituting different bytes. +- A rejected compare-and-swap is replanned from a fresh read; never blindly retry a mutation. +- Finalization and build operations must be idempotent or fail closed around ambiguous partial state. -## Building output +## Finalization and output -Build output only from canonical translation artifacts in chapter order. +Before declaring a book complete, verify every intended unit is present in reading order, every unit is `reviewed` with current PASS evidence, structural validation succeeds, corpus SHA-256 verification succeeds, shared literary decisions are consistent, and requested output is actually built and checked. -Markdown is the transparent default. When Python is available: +Default Markdown build: ```bash python scripts/book.py build ``` -The default build path requires chapters to be `reviewed`. A preview containing merely translated chapters should be produced only when the user explicitly requests an unreviewed preview and the output is clearly identified as such. - -Only claim EPUB, DOCX, PDF, or another deliverable when the active environment actually created and checked that artifact. - -## Book completion +An unreviewed preview is allowed only when explicitly requested and clearly identified as such. EPUB/final output must be built only from canonical durable state and verified before claiming completion. -Before declaring a book complete, the Orchestrator verifies that: - -1. every intended chapter is present and in real reading order; -2. every intended chapter is `reviewed` through the review/state boundary above; -3. for ledger-enabled books, every intended chapter resolves to current exact PASS review evidence from `review-ledger.json`; -4. structural validation succeeds; -5. when `source-manifest.json` exists, sealed-corpus SHA-256 verification succeeds; -6. glossary and style-guide decisions are consistent across the book; -7. selected difficult, ambiguous, emotionally important, or plot-critical passages are re-checked under the active literary contract when a book-level consistency check warrants it; -8. requested output artifacts are ordered/checked if output was requested; -9. the preserved source remains unchanged; -10. workflow provenance remains intact; -11. source-corpus integrity/provenance remains reproducible or any intentional private-source limitation is explicitly recorded. - -A built output file alone is not evidence that the book is complete. +`STATE.md`, `FINAL_QUALITY_GATES.md`, and `REVIEW_REPORT.md` are generated projections; authoritative state remains the machine-readable records and current artifact bytes. ## GitHub API storage -GitHub API storage is a supported durable orchestration substrate for environments such as ChatGPT Web that can call GitHub APIs but do not have a local checkout. GitHub Actions are not required for runtime orchestration. - -The backend must preserve the same `StorageBackend` semantics as filesystem execution: reads and listings are observational; create is create-if-absent; update/delete are compare-and-swap against the last observed blob revision. A rejected or ambiguous mutation must not be blindly retried. Re-read authoritative GitHub state, classify the durable result, and replan from that state. - -Use GitHub-specific mechanics only at the storage/transport boundary. Literary Translator and Reviewer contracts remain backend-agnostic. - -## Authoritative machine state and generated projections - -Authoritative execution and review state lives in versioned machine-readable records such as `metadata.json`, `progress.json`, `review-ledger.json`, `source-manifest.json`, durable claims/coordination records, proposal/resolution records, and output manifests. Current artifact bytes and their recorded revisions/hashes are part of that authority. - -`STATE.md`, `FINAL_QUALITY_GATES.md`, and `REVIEW_REPORT.md` are generated projections of authoritative machine state. They are deterministic human-readable evidence, not lifecycle or review authority. Handwritten range audits or edits to generated Markdown must never be used to manufacture PASS coverage or completion. When authoritative state changes, regenerate the projections through the normal review-report/finalize paths. +GitHub API storage is a supported durable backend for environments without a local checkout. It preserves the same create-if-absent and compare-and-swap semantics as filesystem storage. GitHub Actions are not required for runtime orchestration. -Repository contributors and Git-backed book workflows should follow `docs/COMMIT_DISCIPLINE.md` for audit-friendly commit boundaries, provenance, and revert/recovery behavior. That document governs Git history as a secondary audit trail; it does not replace the machine-state acceptance rules in this orchestration contract. +Transport failures or ambiguous mutations must not be blindly retried. Re-read authoritative GitHub state, classify what actually happened, and replan. GitHub-specific mechanics remain at the storage boundary; Translator and Reviewer contracts stay backend-agnostic. From 4b4ffac0d899878b6a8bf3d939797db267ae4a23 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:08:22 +0300 Subject: [PATCH 13/43] refactor: remove migration gate from review promotion --- scripts/workflow_v2/reviews.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/scripts/workflow_v2/reviews.py b/scripts/workflow_v2/reviews.py index 0054995..91e28ab 100644 --- a/scripts/workflow_v2/reviews.py +++ b/scripts/workflow_v2/reviews.py @@ -1,4 +1,4 @@ -"""Machine-verifiable Workflow v2 review evidence.""" +"""Machine-verifiable review evidence for the current workflow.""" from __future__ import annotations @@ -13,7 +13,6 @@ from .claims import canonical_unit_id from .coordination import FINALIZATION_PATH -from .migration_journal import MigrationJournalError, load_migration_journal from .repository import LoadedDocument, RepositoryError, WorkflowStateRepository from .schemas import SCHEMA_VERSION, SchemaError, SchemaKind, parse_document from .shared_state import SharedStateError, SharedStateStale, require_current_shared_state @@ -435,19 +434,6 @@ def accept_review( if not isinstance(progress_revision, str) or not progress_revision.strip(): raise ReviewEvidenceError("progress revision must be a non-empty string") - try: - load_migration_journal(self.repository.storage) - except StorageNotFound: - pass - except (MigrationJournalError, StorageError) as exc: - raise ReviewEvidenceError( - f"cannot verify migration admission state: {exc}" - ) from exc - else: - raise ReviewEvidenceError( - "review promotion is blocked while workflow migration recovery is active" - ) - try: self.repository.read(FINALIZATION_PATH, SchemaKind.FINALIZATION_LOCK) except StorageNotFound: From c6759514af654f53ebaeb667c8ab2e2411fedad4 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:28:48 +0300 Subject: [PATCH 14/43] refactor: require current workspace metadata contract --- scripts/workflow_v2/source_schema.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/scripts/workflow_v2/source_schema.py b/scripts/workflow_v2/source_schema.py index 805a606..4e15b5d 100644 --- a/scripts/workflow_v2/source_schema.py +++ b/scripts/workflow_v2/source_schema.py @@ -1,4 +1,4 @@ -"""Package-level Workflow v2 explicit-source schema extensions.""" +"""Package-level current-workspace schema extensions.""" from __future__ import annotations @@ -9,12 +9,13 @@ from .schemas import SchemaKind +_CURRENT_REVIEW_EVIDENCE = "review-ledger-v1" + + def _validate_metadata_source(data: Mapping[str, Any], schema: SchemaKind) -> None: - if "source" not in data: - return source = data.get("source") if not isinstance(source, Mapping): - raise schemas._field(schema, "source", "must be an object") + raise schemas._field(schema, "source", "is required and must be an object") mode = source.get("storage_mode") if not isinstance(mode, str) or not mode.strip(): @@ -41,6 +42,18 @@ def _validate_metadata_source(data: Mapping[str, Any], schema: SchemaKind) -> No schemas._validate_sha256(sha256, schema, "source.sha256") +def _validate_current_workflow(data: Mapping[str, Any], schema: SchemaKind) -> None: + workflow = data.get("workflow") + if not isinstance(workflow, Mapping): + raise schemas._field(schema, "workflow", "is required and must be an object") + if workflow.get("review_evidence") != _CURRENT_REVIEW_EVIDENCE: + raise schemas._field( + schema, + "workflow.review_evidence", + f"must equal {_CURRENT_REVIEW_EVIDENCE!r}", + ) + + def _validate_manifest_source_extension(data: Mapping[str, Any], schema: SchemaKind) -> None: has_mode = "source_storage_mode" in data has_size = "source_size_bytes" in data @@ -65,7 +78,7 @@ def _validate_manifest_source_extension(data: Mapping[str, Any], schema: SchemaK def install_source_schema_extensions() -> None: - """Install explicit-source validators exactly once at package import time.""" + """Install current-workspace validators exactly once at package import time.""" if getattr(schemas, "_explicit_source_v1_installed", False): return @@ -76,6 +89,7 @@ def install_source_schema_extensions() -> None: def validate_metadata(data: Mapping[str, Any], schema: SchemaKind) -> None: metadata_validator(data, schema) _validate_metadata_source(data, schema) + _validate_current_workflow(data, schema) def validate_manifest(data: Mapping[str, Any], schema: SchemaKind) -> None: manifest_validator(data, schema) From edc65fa15b0ea5f1a534abbc5580e0078cc42db2 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:32:16 +0300 Subject: [PATCH 15/43] fix: enforce current workspace after extraction --- scripts/workflow_v2/source_schema.py | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/scripts/workflow_v2/source_schema.py b/scripts/workflow_v2/source_schema.py index 4e15b5d..805a606 100644 --- a/scripts/workflow_v2/source_schema.py +++ b/scripts/workflow_v2/source_schema.py @@ -1,4 +1,4 @@ -"""Package-level current-workspace schema extensions.""" +"""Package-level Workflow v2 explicit-source schema extensions.""" from __future__ import annotations @@ -9,13 +9,12 @@ from .schemas import SchemaKind -_CURRENT_REVIEW_EVIDENCE = "review-ledger-v1" - - def _validate_metadata_source(data: Mapping[str, Any], schema: SchemaKind) -> None: + if "source" not in data: + return source = data.get("source") if not isinstance(source, Mapping): - raise schemas._field(schema, "source", "is required and must be an object") + raise schemas._field(schema, "source", "must be an object") mode = source.get("storage_mode") if not isinstance(mode, str) or not mode.strip(): @@ -42,18 +41,6 @@ def _validate_metadata_source(data: Mapping[str, Any], schema: SchemaKind) -> No schemas._validate_sha256(sha256, schema, "source.sha256") -def _validate_current_workflow(data: Mapping[str, Any], schema: SchemaKind) -> None: - workflow = data.get("workflow") - if not isinstance(workflow, Mapping): - raise schemas._field(schema, "workflow", "is required and must be an object") - if workflow.get("review_evidence") != _CURRENT_REVIEW_EVIDENCE: - raise schemas._field( - schema, - "workflow.review_evidence", - f"must equal {_CURRENT_REVIEW_EVIDENCE!r}", - ) - - def _validate_manifest_source_extension(data: Mapping[str, Any], schema: SchemaKind) -> None: has_mode = "source_storage_mode" in data has_size = "source_size_bytes" in data @@ -78,7 +65,7 @@ def _validate_manifest_source_extension(data: Mapping[str, Any], schema: SchemaK def install_source_schema_extensions() -> None: - """Install current-workspace validators exactly once at package import time.""" + """Install explicit-source validators exactly once at package import time.""" if getattr(schemas, "_explicit_source_v1_installed", False): return @@ -89,7 +76,6 @@ def install_source_schema_extensions() -> None: def validate_metadata(data: Mapping[str, Any], schema: SchemaKind) -> None: metadata_validator(data, schema) _validate_metadata_source(data, schema) - _validate_current_workflow(data, schema) def validate_manifest(data: Mapping[str, Any], schema: SchemaKind) -> None: manifest_validator(data, schema) From 7ea8d7ebe0415aee3db023c2b78279b55f96e43d Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:36:36 +0300 Subject: [PATCH 16/43] refactor: fail closed on incomplete workflow workspaces --- scripts/workflow_v2/source_cli.py | 101 ++++++++++++++++++++++-------- 1 file changed, 74 insertions(+), 27 deletions(-) diff --git a/scripts/workflow_v2/source_cli.py b/scripts/workflow_v2/source_cli.py index a5f528b..c75d540 100644 --- a/scripts/workflow_v2/source_cli.py +++ b/scripts/workflow_v2/source_cli.py @@ -1,4 +1,4 @@ -"""Explicit Workflow v2 source identity and book CLI integration.""" +"""Explicit source identity and current-workspace CLI integration.""" from __future__ import annotations @@ -18,6 +18,9 @@ from .storage import StorageError +CURRENT_REVIEW_EVIDENCE = "review-ledger-v1" + + class SourceCliError(RuntimeError): """Expected explicit-source workflow error suitable for CLI output.""" @@ -43,22 +46,34 @@ def normalize_structural_errors( metadata: Mapping[str, Any], errors: Sequence[str], ) -> list[str]: - """Apply explicit-source structural policy without duplicating hash verification.""" + """Apply the current workspace contract without duplicating hash verification.""" result = [str(error) for error in errors] source = _explicit_source(metadata) if source is None: - return result - - source_file = metadata.get("source_file") - legacy_missing = f"Source file declared in metadata.json does not exist: source/{source_file}" - if source.get("storage_mode") == "private_external": - result = [error for error in result if error != legacy_missing] - - if not (book_dir / "source-manifest.json").is_file(): - message = "Missing source-manifest.json for explicit-source book" + message = "metadata.json source identity is required for the current workflow" + if message not in result: + result.append(message) + else: + source_file = metadata.get("source_file") + missing_source = f"Source file declared in metadata.json does not exist: source/{source_file}" + if source.get("storage_mode") == "private_external": + result = [error for error in result if error != missing_source] + + if not (book_dir / "source-manifest.json").is_file(): + message = "Missing source-manifest.json for current workflow book" + if message not in result: + result.append(message) + + workflow = metadata.get("workflow") + if not isinstance(workflow, Mapping) or workflow.get("review_evidence") != CURRENT_REVIEW_EVIDENCE: + message = ( + "metadata.json workflow.review_evidence must equal " + f"{CURRENT_REVIEW_EVIDENCE!r} for the current workflow" + ) if message not in result: result.append(message) + return result @@ -108,6 +123,24 @@ def _source_identity(source: Path, *, private: bool) -> dict[str, Any]: } +def _current_workspace_errors(book_module: Any, slug: str) -> list[str]: + try: + book_dir, metadata, _ = book_module.load_book(slug) + except Exception as exc: + return [str(exc)] + + errors, _ = book_module.validate_book(slug) + errors = normalize_structural_errors(book_dir, metadata, errors) + errors.extend( + manifest_structure_errors( + book_dir, + metadata, + book_module.state_repository(book_dir), + ) + ) + return errors + + def source_extract_command( args: argparse.Namespace, root: Path, @@ -147,12 +180,10 @@ def source_extract_command( if identity["storage_mode"] == "private_external" and stored_source.is_file(): stored_source.unlink() - errors, _ = book_module.validate_book(slug) - errors = normalize_structural_errors(book_dir, metadata, errors) - errors.extend(manifest_structure_errors(book_dir, metadata, repository)) + errors = _current_workspace_errors(book_module, slug) if errors: raise SourceCliError( - "Explicit-source initialization failed validation:\n- " + "\n- ".join(errors) + "Current workflow initialization failed validation:\n- " + "\n- ".join(errors) ) except (SourceIntegrityError, SchemaError, RepositoryError, StorageError) as exc: if not existed_before and book_dir.exists(): @@ -171,17 +202,11 @@ def source_extract_command( def source_validate_command(args: argparse.Namespace, root: Path) -> int: book_module = _active_book_module() - errors, warnings = book_module.validate_book(args.slug) + errors = _current_workspace_errors(book_module, args.slug) try: - book_dir, metadata, _ = book_module.load_book(args.slug) + _, warnings = book_module.validate_book(args.slug) except Exception: - book_dir = None - metadata = None - if book_dir is not None and isinstance(metadata, Mapping): - errors = normalize_structural_errors(book_dir, metadata, errors) - errors.extend( - manifest_structure_errors(book_dir, metadata, book_module.state_repository(book_dir)) - ) + warnings = [] for warning in warnings: print(f"WARNING: {warning}") @@ -193,6 +218,20 @@ def source_validate_command(args: argparse.Namespace, root: Path) -> int: return 0 +def source_build_command( + args: argparse.Namespace, + root: Path, + original: Callable[[argparse.Namespace], int], +) -> int: + book_module = _active_book_module() + errors = _current_workspace_errors(book_module, args.slug) + if errors: + raise SourceCliError( + "Book does not satisfy the current workflow contract:\n- " + "\n- ".join(errors) + ) + return original(args) + + def _adapt_errors(command: Callable[[argparse.Namespace], int], error_factory: ErrorFactory): def run(args: argparse.Namespace) -> int: try: @@ -209,16 +248,18 @@ def register_source_overrides( *, error_factory: ErrorFactory, ) -> None: - """Extend existing book.py extract/validate parsers without duplicating them.""" + """Extend extract/validate/build with the current workspace contract.""" extract = subparsers.choices.get("extract") validate = subparsers.choices.get("validate") - if extract is None or validate is None: - raise SourceCliError("book.py extract/validate parsers are unavailable") + build = subparsers.choices.get("build") + if extract is None or validate is None or build is None: + raise SourceCliError("book.py extract/validate/build parsers are unavailable") if getattr(extract, "_explicit_source_v1_registered", False): return original_extract = extract.get_default("func") + original_build = build.get_default("func") extract.add_argument( "--private-source", action="store_true", @@ -238,3 +279,9 @@ def register_source_overrides( error_factory, ) ) + build.set_defaults( + func=_adapt_errors( + lambda args: source_build_command(args, root, original_build), + error_factory, + ) + ) From b5c7533688da54367937c38ceb5ab2ea4b87f9b3 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:37:54 +0300 Subject: [PATCH 17/43] test: replace legacy book CLI expectations --- tests/test_book_cli.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/tests/test_book_cli.py b/tests/test_book_cli.py index 7371e39..a16ff38 100644 --- a/tests/test_book_cli.py +++ b/tests/test_book_cli.py @@ -132,19 +132,11 @@ def test_build_requires_reviewed_by_default(self): self.run_cli("build", "sample-book", expect=1) self.run_cli("build", "sample-book", "--allow-unreviewed") - # This smoke test predates machine review evidence and only verifies the - # build command's lifecycle-state filter, so keep its final state legacy. - metadata_path = book / "metadata.json" - metadata = json.loads(metadata_path.read_text(encoding="utf-8")) - metadata["workflow"].pop("review_evidence", None) - metadata_path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - (book / "review-ledger.json").unlink() - for chapter in progress["chapters"]: chapter["status"] = "reviewed" progress_path.write_text(json.dumps(progress, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - self.run_cli("build", "sample-book") - self.assertTrue((book / "output" / "sample-book.md").is_file()) + result = self.run_cli("build", "sample-book", expect=1) + self.assertIn("current PASS review evidence", result.stderr) def test_validate_requires_style_guide(self): source = self.repo / "sample.md" @@ -167,7 +159,7 @@ def test_validate_rejects_unsupported_explicit_schema_version(self): result = self.run_cli("validate", "sample", expect=1) self.assertIn("unsupported version 2", result.stderr) - def test_validate_accepts_legacy_state_without_rewriting(self): + def test_validate_rejects_unversioned_state_without_rewriting(self): source = self.repo / "sample.md" source.write_text("# A\n\nOne.\n\n# B\n\nTwo.\n", encoding="utf-8") self.run_cli("extract", str(source), "--slug", "sample", "--target-language", "ru") @@ -182,7 +174,8 @@ def test_validate_accepts_legacy_state_without_rewriting(self): path.write_bytes(content) original_bytes[path.name] = content - self.run_cli("validate", "sample") + result = self.run_cli("validate", "sample", expect=1) + self.assertIn("schema_version", result.stderr) for path in paths: self.assertEqual(path.read_bytes(), original_bytes[path.name]) @@ -240,4 +233,4 @@ def test_extract_minimal_epub_uses_spine_order_and_metadata(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() From 6940ef1361beab0ea062566104f97160558ecfed Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:38:23 +0300 Subject: [PATCH 18/43] test: require versioned repository state --- tests/test_workflow_v2_repository.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/test_workflow_v2_repository.py b/tests/test_workflow_v2_repository.py index 681e6ff..69f977b 100644 --- a/tests/test_workflow_v2_repository.py +++ b/tests/test_workflow_v2_repository.py @@ -70,19 +70,16 @@ def test_invalid_create_is_rejected_before_storage_mutation(self): self.assertEqual(self.storage.list(), []) - def test_read_returns_revision_and_legacy_flag_without_rewriting(self): + def test_read_rejects_unversioned_state_even_with_former_compatibility_flag(self): repo = self.repo() - legacy = self.metadata() - del legacy["schema_version"] - original = (json.dumps(legacy, ensure_ascii=False, indent=2) + "\n").encode("utf-8") - version = self.storage.create_if_absent("metadata.json", original) + unversioned = self.metadata() + del unversioned["schema_version"] + original = (json.dumps(unversioned, ensure_ascii=False, indent=2) + "\n").encode("utf-8") + self.storage.create_if_absent("metadata.json", original) - loaded = repo.read("metadata.json", SchemaKind.METADATA, allow_legacy=True) + with self.assertRaises(SchemaError): + repo.read("metadata.json", SchemaKind.METADATA, allow_legacy=True) - self.assertIsInstance(loaded, LoadedDocument) - self.assertTrue(loaded.legacy) - self.assertEqual(loaded.version, version) - self.assertEqual(loaded.data["schema_version"], 1) self.assertEqual(self.storage.read("metadata.json").content, original) def test_stale_repository_write_propagates_conflict_without_mutation(self): From 5390b70a858ff8eb8f4412196bfcb0646f0faebb Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:38:59 +0300 Subject: [PATCH 19/43] test: reject unsealed workflow workspaces --- tests/test_workflow_v2_status_cli.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_workflow_v2_status_cli.py b/tests/test_workflow_v2_status_cli.py index 1f4bb55..cff5f24 100644 --- a/tests/test_workflow_v2_status_cli.py +++ b/tests/test_workflow_v2_status_cli.py @@ -156,7 +156,7 @@ def test_explicit_source_missing_manifest_blocks_resume(self): self.assertEqual(payload["reason"], "preflight_failed") self.assertTrue(any("source-manifest.json" in error for error in payload["errors"])) - def test_legacy_book_without_explicit_source_can_remain_unsealed(self): + def test_missing_explicit_source_is_invalid_current_state(self): book = self.initialize_book() metadata_path = book / "metadata.json" metadata = json.loads(metadata_path.read_text(encoding="utf-8")) @@ -164,9 +164,13 @@ def test_legacy_book_without_explicit_source_can_remain_unsealed(self): metadata_path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") (book / "source-manifest.json").unlink() - status = self.canonical_json(self.run_book("status", "sample", "--json")) - self.assertTrue(status["valid"]) - self.assertEqual(status["corpus"]["state"], "unsealed") + status = self.canonical_json(self.run_book("status", "sample", "--json", expect=1)) + self.assertFalse(status["valid"]) + self.assertEqual(status["corpus"]["state"], "invalid") + self.assertTrue(any("source" in error.lower() for error in status["errors"])) + + resume = self.canonical_json(self.run_book("resume", "sample", "--json", expect=1)) + self.assertEqual(resume["operation"], "blocked") def test_resume_blocks_on_real_corpus_hash_mismatch(self): book = self.initialize_book() From c9333ee29a8fb5539e0634344104d21e472c4ce5 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:39:28 +0300 Subject: [PATCH 20/43] test: require review evidence marker --- tests/test_workflow_v2_review_validation.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_workflow_v2_review_validation.py b/tests/test_workflow_v2_review_validation.py index 1188f65..68388b8 100644 --- a/tests/test_workflow_v2_review_validation.py +++ b/tests/test_workflow_v2_review_validation.py @@ -158,7 +158,7 @@ def test_editing_reviewed_translation_makes_validation_stale(self): result = self.run_cli("validate", "sample", expect=1) self.assertIn("stale", result.stderr.lower()) - def test_book_without_review_evidence_marker_keeps_legacy_validation_behavior(self): + def test_missing_review_evidence_marker_is_invalid_current_state(self): metadata_path = self.book / "metadata.json" metadata = json.loads(metadata_path.read_text(encoding="utf-8")) del metadata["workflow"]["review_evidence"] @@ -167,7 +167,8 @@ def test_book_without_review_evidence_marker_keeps_legacy_validation_behavior(se self.make_translation() self.mark_status("reviewed") - self.run_cli("validate", "sample") + result = self.run_cli("validate", "sample", expect=1) + self.assertIn("review_evidence", result.stderr) if __name__ == "__main__": From 1c0a520cd0693a2835e26399e88dd55bb2c32393 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:40:09 +0300 Subject: [PATCH 21/43] test: use current workflow fixtures on GitHub backend --- tests/test_workflow_v2_github_backend_domain.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_workflow_v2_github_backend_domain.py b/tests/test_workflow_v2_github_backend_domain.py index 22a1f45..6138309 100644 --- a/tests/test_workflow_v2_github_backend_domain.py +++ b/tests/test_workflow_v2_github_backend_domain.py @@ -35,6 +35,7 @@ def setUp(self): "repository": "https://github.com/tim8es/book-translator", "requested_ref": "refactor/workflow-engine-v2", "resolved_revision": "rev-1", + "review_evidence": "review-ledger-v1", }, } self.progress = { @@ -57,6 +58,17 @@ def setUp(self): self.progress_revision = self.repository.create( "progress.json", SchemaKind.PROGRESS, self.progress ) + self.repository.create( + "review-ledger.json", + SchemaKind.REVIEW_LEDGER, + { + "schema_version": 1, + "book_slug": "sample", + "next_sequence": 1, + "records": [], + }, + ) + self.storage.create_if_absent("chapters/chapter-001.md", b"# One\n\nAlpha.\n") def claim_manager(self, session_seed=1): ids = iter([f"{session_seed + index:032x}" for index in range(20)]) @@ -118,7 +130,7 @@ def test_status_and_resume_are_read_only_on_github_storage(self): self.client.mutations.clear() resolver = StatusResolver( self.repository, - artifact_reader=lambda path: (_ for _ in ()).throw(FileNotFoundError(path)), + artifact_reader=lambda path: self.storage.read(path).content, ) status = resolver.status(corpus={"state": "verified", "storage_mode": "embedded"}) From a62a1e0fe0fec430e1054f9f1ffbcdae0965a4a0 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:40:47 +0300 Subject: [PATCH 22/43] test: use current parallel workflow fixtures --- tests/test_workflow_v2_parallel.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_workflow_v2_parallel.py b/tests/test_workflow_v2_parallel.py index 7cba1e2..8211c36 100644 --- a/tests/test_workflow_v2_parallel.py +++ b/tests/test_workflow_v2_parallel.py @@ -35,6 +35,7 @@ def setUp(self): "workflow": { "requested_ref": "refactor/workflow-engine-v2", "resolved_revision": "workflow-revision", + "review_evidence": "review-ledger-v1", }, } self.progress = { @@ -56,6 +57,21 @@ def setUp(self): self.progress_revision = self.repository.create( "progress.json", SchemaKind.PROGRESS, self.progress ) + self.repository.create( + "review-ledger.json", + SchemaKind.REVIEW_LEDGER, + { + "schema_version": SCHEMA_VERSION, + "book_slug": "demo", + "next_sequence": 1, + "records": [], + }, + ) + for number in (1, 2, 3): + self.storage.create_if_absent( + f"extracted/chapter-{number:04d}.md", + f"# Chapter {number}\n\nSource {number}.\n".encode(), + ) self.glossary_revision = self.storage.create_if_absent( "glossary.md", b"# Glossary\n" ) @@ -217,6 +233,7 @@ def test_resume_cli_accepts_invocation_scoped_parallel_flag(self): subparsers = parser.add_subparsers(dest="command") subparsers.add_parser("extract") subparsers.add_parser("validate") + subparsers.add_parser("build") register_status_commands( subparsers, self.root, From 3d314c96e04a962a2ed2dbc1a4b2ac5f85f8d06f Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:42:13 +0300 Subject: [PATCH 23/43] docs: name translation acceptance evidence --- docs/ORCHESTRATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ORCHESTRATION.md b/docs/ORCHESTRATION.md index 5ec68a9..4032923 100644 --- a/docs/ORCHESTRATION.md +++ b/docs/ORCHESTRATION.md @@ -147,7 +147,7 @@ python scripts/book.py accept-translation \ --session-id ``` -`accept-translation` verifies the current workflow revision, owning live Translator claim, canonical source/translation bytes, artifact SHA-256 identity, and any frozen shared-state snapshot. Only a successful compare-and-swap may advance `extracted -> translated`. Release the Translator claim afterward. +`accept-translation` verifies the current workflow revision, owning live Translator claim, canonical source/translation bytes, artifact SHA-256 identity, and any frozen shared-state snapshot. A successful acceptance persists exact `translation_acceptance` evidence in `progress.json`; only its successful compare-and-swap may advance `extracted -> translated`. Release the Translator claim afterward. ## Reviewer boundary From 072e38aac40d21d20811717ba0f7aaeab4058d20 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:45:56 +0300 Subject: [PATCH 24/43] test: assert current build admission failure --- tests/test_book_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_book_cli.py b/tests/test_book_cli.py index a16ff38..96e0bef 100644 --- a/tests/test_book_cli.py +++ b/tests/test_book_cli.py @@ -136,7 +136,7 @@ def test_build_requires_reviewed_by_default(self): chapter["status"] = "reviewed" progress_path.write_text(json.dumps(progress, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") result = self.run_cli("build", "sample-book", expect=1) - self.assertIn("current PASS review evidence", result.stderr) + self.assertIn("review-ledger validation failed", result.stderr) def test_validate_requires_style_guide(self): source = self.repo / "sample.md" From abd8cb16d451da043ea12868aad6d102a2d9e885 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:23:46 +0300 Subject: [PATCH 25/43] test: align invalid status reporting exit semantics --- tests/test_workflow_v2_status_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_workflow_v2_status_cli.py b/tests/test_workflow_v2_status_cli.py index cff5f24..70aa2c9 100644 --- a/tests/test_workflow_v2_status_cli.py +++ b/tests/test_workflow_v2_status_cli.py @@ -164,7 +164,7 @@ def test_missing_explicit_source_is_invalid_current_state(self): metadata_path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") (book / "source-manifest.json").unlink() - status = self.canonical_json(self.run_book("status", "sample", "--json", expect=1)) + status = self.canonical_json(self.run_book("status", "sample", "--json")) self.assertFalse(status["valid"]) self.assertEqual(status["corpus"]["state"], "invalid") self.assertTrue(any("source" in error.lower() for error in status["errors"])) From adf5585b7f246702edaedc0cb1a958f641ec79dc Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:24:29 +0300 Subject: [PATCH 26/43] test: keep corpus restore fixture on current workflow --- tests/test_corpus_cli.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/test_corpus_cli.py b/tests/test_corpus_cli.py index 0f131f7..8c38254 100644 --- a/tests/test_corpus_cli.py +++ b/tests/test_corpus_cli.py @@ -119,20 +119,12 @@ def test_restore_rebuilds_complete_corpus_without_mutating_translation_state(sel self.run_cli("book.py", "extract", str(source), "--slug", "sample", "--target-language", "ru") book = self.repo / "books" / "sample" - # Model a legacy book whose lifecycle predates explicit source/review evidence. - metadata_path = book / "metadata.json" - metadata = json.loads(metadata_path.read_text(encoding="utf-8")) - metadata.pop("source", None) - metadata["workflow"].pop("review_evidence", None) - metadata_path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - (book / "review-ledger.json").unlink() - progress_path = book / "progress.json" progress = json.loads(progress_path.read_text(encoding="utf-8")) translation_path = book / progress["chapters"][0]["translation_path"] translation_path.parent.mkdir(parents=True, exist_ok=True) translation_path.write_text("# Первая\n\nПеревод.\n", encoding="utf-8") - progress["chapters"][0]["status"] = "reviewed" + progress["chapters"][0]["status"] = "translated" progress_path.write_text(json.dumps(progress, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") shutil.rmtree(book / "extracted") From 84638290f1b676e2628934792aaf1e5415a2d63c Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:24:55 +0300 Subject: [PATCH 27/43] test: map missing GitHub artifacts to domain semantics --- tests/test_workflow_v2_github_backend_domain.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_workflow_v2_github_backend_domain.py b/tests/test_workflow_v2_github_backend_domain.py index 6138309..bf20465 100644 --- a/tests/test_workflow_v2_github_backend_domain.py +++ b/tests/test_workflow_v2_github_backend_domain.py @@ -8,7 +8,7 @@ from workflow_v2.repository import WorkflowStateRepository from workflow_v2.schemas import SchemaKind from workflow_v2.status import StatusResolver -from workflow_v2.storage import StorageVersionConflict +from workflow_v2.storage import StorageNotFound, StorageVersionConflict NOW = datetime(2026, 9, 7, 12, 0, 0, tzinfo=timezone.utc) @@ -83,6 +83,12 @@ def claim_manager(self, session_seed=1): ), ) + def read_artifact(self, path): + try: + return self.storage.read(path).content + except StorageNotFound as exc: + raise FileNotFoundError(path) from exc + def test_claim_conflict_release_and_audit_match_backend_neutral_domain_behavior(self): manager = self.claim_manager(1) claims = manager.acquire( @@ -130,7 +136,7 @@ def test_status_and_resume_are_read_only_on_github_storage(self): self.client.mutations.clear() resolver = StatusResolver( self.repository, - artifact_reader=lambda path: self.storage.read(path).content, + artifact_reader=self.read_artifact, ) status = resolver.status(corpus={"state": "verified", "storage_mode": "embedded"}) From 30afe190990d68efa04b043da4c6d6a4dbc9dc21 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:42:12 +0300 Subject: [PATCH 28/43] fix: close current workflow build admission gaps --- scripts/workflow_v2/source_cli.py | 131 ++++++++++++++++++++++++++++-- 1 file changed, 123 insertions(+), 8 deletions(-) diff --git a/scripts/workflow_v2/source_cli.py b/scripts/workflow_v2/source_cli.py index c75d540..b270069 100644 --- a/scripts/workflow_v2/source_cli.py +++ b/scripts/workflow_v2/source_cli.py @@ -107,6 +107,124 @@ def manifest_structure_errors( return errors +def manifest_integrity_errors( + book_dir: Path, + metadata: Mapping[str, Any], + progress: Mapping[str, Any], + repository: Any, +) -> list[str]: + """Verify the sealed current corpus before validate/build admission.""" + + source = _explicit_source(metadata) + if source is None or not (book_dir / "source-manifest.json").is_file(): + return [] + try: + manifest = repository.read("source-manifest.json", SchemaKind.SOURCE_MANIFEST).data + except (SchemaError, RepositoryError, StorageError) as exc: + return [f"Invalid source-manifest.json: {exc}"] + + errors: list[str] = [] + source_file = metadata.get("source_file") + source_path = book_dir / "source" / str(source_file) + storage_mode = source.get("storage_mode") + if source_path.is_file(): + expected_size = source.get("size_bytes") + if source_path.stat().st_size != expected_size: + errors.append( + f"Preserved source size mismatch: expected {expected_size}, got {source_path.stat().st_size}" + ) + actual_source_sha = sha256_path(source_path) + expected_source_sha = manifest.get("source_sha256") + if actual_source_sha != expected_source_sha: + errors.append( + f"Preserved source hash mismatch: expected {expected_source_sha}, got {actual_source_sha}" + ) + elif storage_mode == "embedded": + errors.append(f"Preserved source is missing: source/{source_file}") + + chapters = progress.get("chapters") + items = manifest.get("extracted") + if not isinstance(chapters, list) or not isinstance(items, list): + return errors + if manifest.get("chapter_count") != len(chapters) or len(items) != len(chapters): + errors.append("source-manifest.json chapter_count/extracted entries disagree with progress.json") + return errors + + for chapter, item in zip(chapters, items): + if not isinstance(chapter, Mapping) or not isinstance(item, Mapping): + errors.append("source-manifest.json extracted entries must match progress chapter objects") + continue + source_rel = chapter.get("source_path") + if item.get("path") != source_rel: + errors.append( + f"Manifest path mismatch for chapter {chapter.get('number')}: expected {source_rel}, got {item.get('path')!r}" + ) + continue + if item.get("number") != chapter.get("number"): + errors.append(f"Manifest chapter number mismatch for {source_rel}") + if item.get("title") != chapter.get("title"): + errors.append(f"Manifest chapter title mismatch for {source_rel}") + if not isinstance(source_rel, str): + errors.append(f"Invalid extracted source path in progress.json: {source_rel!r}") + continue + rel = Path(source_rel) + if rel.is_absolute() or ".." in rel.parts: + errors.append(f"Source path escapes book workspace: {source_rel}") + continue + path = book_dir / rel + if not path.is_file(): + errors.append(f"Extracted artifact is missing: {source_rel}") + continue + actual_sha = sha256_path(path) + expected_sha = item.get("sha256") + if actual_sha != expected_sha: + errors.append( + f"Extracted artifact hash mismatch for {source_rel}: expected {expected_sha}, got {actual_sha}" + ) + return errors + + +def translation_acceptance_errors( + metadata: Mapping[str, Any], + progress: Mapping[str, Any], +) -> list[str]: + """Reject translated-or-later state that bypassed Translator acceptance.""" + + workflow = metadata.get("workflow") + workflow_revision = None + if isinstance(workflow, Mapping): + for key in ("resolved_revision", "requested_ref"): + value = workflow.get(key) + if isinstance(value, str) and value.strip(): + workflow_revision = value + break + + errors: list[str] = [] + chapters = progress.get("chapters") + if not isinstance(chapters, list): + return errors + for chapter in chapters: + if not isinstance(chapter, Mapping) or chapter.get("status") not in {"translated", "reviewed"}: + continue + number = chapter.get("number") + evidence = chapter.get("translation_acceptance") + if not isinstance(evidence, Mapping): + errors.append( + f"Chapter {number}: status={chapter.get('status')} requires current translation_acceptance evidence" + ) + continue + expected_unit = f"chapter-{int(number):06d}" if type(number) is int and number > 0 else None + if expected_unit is not None and evidence.get("unit_id") != expected_unit: + errors.append(f"Chapter {number}: translation_acceptance unit identity is invalid") + if evidence.get("role") != "translator": + errors.append(f"Chapter {number}: translation_acceptance role must be translator") + if workflow_revision is not None and evidence.get("workflow_revision") != workflow_revision: + errors.append( + f"Chapter {number}: translation_acceptance uses another workflow revision" + ) + return errors + + def _active_book_module(): main = sys.modules.get("__main__") if main is not None and hasattr(main, "slugify") and hasattr(main, "state_repository"): @@ -125,19 +243,16 @@ def _source_identity(source: Path, *, private: bool) -> dict[str, Any]: def _current_workspace_errors(book_module: Any, slug: str) -> list[str]: try: - book_dir, metadata, _ = book_module.load_book(slug) + book_dir, metadata, progress = book_module.load_book(slug) except Exception as exc: return [str(exc)] errors, _ = book_module.validate_book(slug) errors = normalize_structural_errors(book_dir, metadata, errors) - errors.extend( - manifest_structure_errors( - book_dir, - metadata, - book_module.state_repository(book_dir), - ) - ) + repository = book_module.state_repository(book_dir) + errors.extend(manifest_structure_errors(book_dir, metadata, repository)) + errors.extend(manifest_integrity_errors(book_dir, metadata, progress, repository)) + errors.extend(translation_acceptance_errors(metadata, progress)) return errors From a7f31564876911a8eb819394cf94d23e5645301e Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:42:58 +0300 Subject: [PATCH 29/43] fix: require translator acceptance before review --- scripts/workflow_v2/review_cli.py | 52 +++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/scripts/workflow_v2/review_cli.py b/scripts/workflow_v2/review_cli.py index 7b9ec80..26e1c83 100644 --- a/scripts/workflow_v2/review_cli.py +++ b/scripts/workflow_v2/review_cli.py @@ -121,6 +121,56 @@ def _translation_manager( ) +def _require_current_translation_acceptance( + progress: Mapping[str, Any], + metadata: Mapping[str, Any], + chapter_number: int, +) -> None: + """Keep Reviewer operations behind the durable Translator acceptance gate.""" + + chapters = progress.get("chapters") + if not isinstance(chapters, list): + raise ReviewCliError("progress state must contain a chapters array") + chapter = next( + ( + item + for item in chapters + if isinstance(item, Mapping) and item.get("number") == chapter_number + ), + None, + ) + if chapter is None: + raise ReviewCliError(f"progress does not contain chapter {chapter_number}") + evidence = chapter.get("translation_acceptance") + if not isinstance(evidence, Mapping): + raise ReviewCliError( + f"chapter {chapter_number} requires current translation_acceptance evidence before review" + ) + expected_unit = f"chapter-{chapter_number:06d}" + if evidence.get("unit_id") != expected_unit: + raise ReviewCliError( + f"chapter {chapter_number} translation_acceptance unit identity is invalid" + ) + if evidence.get("role") != "translator": + raise ReviewCliError( + f"chapter {chapter_number} translation_acceptance role must be translator" + ) + workflow = metadata.get("workflow") + workflow_revision = None + if isinstance(workflow, Mapping): + for key in ("resolved_revision", "requested_ref"): + value = workflow.get(key) + if isinstance(value, str) and value.strip(): + workflow_revision = value + break + if workflow_revision is None: + raise ReviewCliError("metadata workflow revision is unavailable") + if evidence.get("workflow_revision") != workflow_revision: + raise ReviewCliError( + f"chapter {chapter_number} translation_acceptance uses another workflow revision" + ) + + def _print_json(payload: Mapping[str, Any]) -> None: print(json.dumps(payload, ensure_ascii=False, sort_keys=True)) @@ -193,6 +243,7 @@ def review_record_command(args: argparse.Namespace, root: Path) -> int: book_dir, repository = _repository(root, args.slug) progress, progress_revision = _load_progress(repository) metadata = _load_metadata(repository) + _require_current_translation_acceptance(progress, metadata, args.chapter) manager = _manager(book_dir, repository) review_commit = args.review_commit if args.review_commit is not None else _git_head(root) try: @@ -298,6 +349,7 @@ def accept_review_command(args: argparse.Namespace, root: Path) -> int: book_dir, repository = _repository(root, args.slug) progress, progress_revision = _load_progress(repository) metadata = _load_metadata(repository) + _require_current_translation_acceptance(progress, metadata, args.chapter) manager = _manager(book_dir, repository) try: result = manager.accept_review(progress, progress_revision, metadata, args.chapter) From 3267fc9aed604172987e2e68cc0d20d0d40a12cf Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:43:55 +0300 Subject: [PATCH 30/43] test: keep review fixtures behind translator acceptance --- tests/test_workflow_v2_review_cli.py | 44 ++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/tests/test_workflow_v2_review_cli.py b/tests/test_workflow_v2_review_cli.py index 52a28cb..80c0f0b 100644 --- a/tests/test_workflow_v2_review_cli.py +++ b/tests/test_workflow_v2_review_cli.py @@ -61,8 +61,25 @@ def initialize_book(self, slug="sample"): translation = book / progress["chapters"][0]["translation_path"] translation.parent.mkdir(parents=True, exist_ok=True) translation.write_text("# Один\n\nАльфа.\n", encoding="utf-8") - progress["chapters"][0]["status"] = "translated" - progress_path.write_text(json.dumps(progress, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + self.run_cli( + "claim", + slug, + "1", + "--role", + "translator", + "--session-id", + "translator-a", + "--json", + ) + self.run_cli( + "accept-translation", + slug, + "1", + "--session-id", + "translator-a", + "--json", + ) + self.run_cli("release", slug, "1", "--session-id", "translator-a") return book, translation def claim_reviewer(self, slug="sample", *, session_id="reviewer-a", role="reviewer"): @@ -191,7 +208,28 @@ def test_review_record_refuses_missing_resolved_revision(self): "--json", expect=1, ) - self.assertIn("resolved_revision", result.stderr) + self.assertIn("translation_acceptance", result.stderr) + + def test_review_record_rejects_missing_translation_acceptance(self): + book, _ = self.initialize_book() + progress_path = book / "progress.json" + progress = json.loads(progress_path.read_text(encoding="utf-8")) + progress["chapters"][0].pop("translation_acceptance", None) + progress_path.write_text(json.dumps(progress, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + self.claim_reviewer() + + result = self.run_cli( + "review-record", + "sample", + "1", + "--outcome", + "PASS", + "--session-id", + "reviewer-a", + "--json", + expect=1, + ) + self.assertIn("translation_acceptance", result.stderr) if __name__ == "__main__": From 9d2510b18f11c680def298762e8bc419fa0f5c0e Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:44:27 +0300 Subject: [PATCH 31/43] fix: normalize storage not-found semantics --- scripts/workflow_v2/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/workflow_v2/storage.py b/scripts/workflow_v2/storage.py index 8cff7a3..7286fe6 100644 --- a/scripts/workflow_v2/storage.py +++ b/scripts/workflow_v2/storage.py @@ -10,7 +10,7 @@ class StorageError(RuntimeError): """Base error for storage backend failures.""" -class StorageNotFound(StorageError): +class StorageNotFound(StorageError, FileNotFoundError): """The requested logical path does not exist.""" From 11cd5364c9a447184f79eb8cee4e93dd3639bd33 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:44:58 +0300 Subject: [PATCH 32/43] test: exercise native GitHub not-found semantics --- tests/test_workflow_v2_github_backend_domain.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/test_workflow_v2_github_backend_domain.py b/tests/test_workflow_v2_github_backend_domain.py index bf20465..6138309 100644 --- a/tests/test_workflow_v2_github_backend_domain.py +++ b/tests/test_workflow_v2_github_backend_domain.py @@ -8,7 +8,7 @@ from workflow_v2.repository import WorkflowStateRepository from workflow_v2.schemas import SchemaKind from workflow_v2.status import StatusResolver -from workflow_v2.storage import StorageNotFound, StorageVersionConflict +from workflow_v2.storage import StorageVersionConflict NOW = datetime(2026, 9, 7, 12, 0, 0, tzinfo=timezone.utc) @@ -83,12 +83,6 @@ def claim_manager(self, session_seed=1): ), ) - def read_artifact(self, path): - try: - return self.storage.read(path).content - except StorageNotFound as exc: - raise FileNotFoundError(path) from exc - def test_claim_conflict_release_and_audit_match_backend_neutral_domain_behavior(self): manager = self.claim_manager(1) claims = manager.acquire( @@ -136,7 +130,7 @@ def test_status_and_resume_are_read_only_on_github_storage(self): self.client.mutations.clear() resolver = StatusResolver( self.repository, - artifact_reader=self.read_artifact, + artifact_reader=lambda path: self.storage.read(path).content, ) status = resolver.status(corpus={"state": "verified", "storage_mode": "embedded"}) From 1bf0e96ade65b588bf5d178e25e7af52b22d6315 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:45:58 +0300 Subject: [PATCH 33/43] test: cover current build admission gates --- tests/test_book_cli.py | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/tests/test_book_cli.py b/tests/test_book_cli.py index 96e0bef..5e77117 100644 --- a/tests/test_book_cli.py +++ b/tests/test_book_cli.py @@ -101,7 +101,7 @@ def test_extract_records_install_provenance_in_book_metadata(self): }, ) - def test_build_requires_reviewed_by_default(self): + def test_build_rejects_translated_or_reviewed_state_without_translation_acceptance(self): source = self.repo / "sample.txt" source.write_text( "Chapter 1\n\nOriginal one.\n\nChapter 2\n\nOriginal two.\n", @@ -128,15 +128,43 @@ def test_build_requires_reviewed_by_default(self): chapter["status"] = "translated" progress_path.write_text(json.dumps(progress, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - self.run_cli("validate", "sample-book") - self.run_cli("build", "sample-book", expect=1) - self.run_cli("build", "sample-book", "--allow-unreviewed") + validate = self.run_cli("validate", "sample-book", expect=1) + self.assertIn("translation_acceptance", validate.stderr) + preview = self.run_cli("build", "sample-book", "--allow-unreviewed", expect=1) + self.assertIn("translation_acceptance", preview.stderr) for chapter in progress["chapters"]: chapter["status"] = "reviewed" progress_path.write_text(json.dumps(progress, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - result = self.run_cli("build", "sample-book", expect=1) - self.assertIn("review-ledger validation failed", result.stderr) + final = self.run_cli("build", "sample-book", expect=1) + self.assertIn("translation_acceptance", final.stderr) + + def test_validate_and_build_reject_tampered_preserved_source(self): + source = self.repo / "sample.md" + source.write_text("# A\n\nOne.\n\n# B\n\nTwo.\n", encoding="utf-8") + self.run_cli("extract", str(source), "--slug", "sample", "--target-language", "ru") + + preserved = self.repo / "books" / "sample" / "source" / "sample.md" + preserved.write_text(preserved.read_text(encoding="utf-8") + "\nTAMPERED\n", encoding="utf-8") + + validate = self.run_cli("validate", "sample", expect=1) + self.assertIn("source hash mismatch", validate.stderr.lower()) + build = self.run_cli("build", "sample", "--allow-unreviewed", expect=1) + self.assertIn("source hash mismatch", build.stderr.lower()) + + def test_validate_and_build_reject_tampered_extracted_artifact(self): + source = self.repo / "sample.md" + source.write_text("# A\n\nOne.\n\n# B\n\nTwo.\n", encoding="utf-8") + self.run_cli("extract", str(source), "--slug", "sample", "--target-language", "ru") + book = self.repo / "books" / "sample" + progress = json.loads((book / "progress.json").read_text(encoding="utf-8")) + extracted = book / progress["chapters"][0]["source_path"] + extracted.write_text(extracted.read_text(encoding="utf-8") + "\nTAMPERED\n", encoding="utf-8") + + validate = self.run_cli("validate", "sample", expect=1) + self.assertIn("extracted artifact hash mismatch", validate.stderr.lower()) + build = self.run_cli("build", "sample", "--allow-unreviewed", expect=1) + self.assertIn("extracted artifact hash mismatch", build.stderr.lower()) def test_validate_requires_style_guide(self): source = self.repo / "sample.md" From 5d4cd0beb94c95b21a62d218a4c49ef732622144 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:48:15 +0300 Subject: [PATCH 34/43] test: accept translations in EPUB fixtures --- tests/test_workflow_v2_epub_cli.py | 33 ++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/test_workflow_v2_epub_cli.py b/tests/test_workflow_v2_epub_cli.py index 0da6a23..de4ac0c 100644 --- a/tests/test_workflow_v2_epub_cli.py +++ b/tests/test_workflow_v2_epub_cli.py @@ -85,15 +85,36 @@ def initialize_book(self, slug="sample", *, private=False): return self.repo / "books" / slug def mark_translated(self, book): - progress_path = book / "progress.json" - progress = json.loads(progress_path.read_text(encoding="utf-8")) + progress = json.loads((book / "progress.json").read_text(encoding="utf-8")) translation = book / progress["chapters"][0]["translation_path"] translation.parent.mkdir(parents=True, exist_ok=True) translation.write_text("# Один\n\nАльфа.\n", encoding="utf-8") - progress["chapters"][0]["status"] = "translated" - progress_path.write_text( - json.dumps(progress, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", + slug = book.name + self.run_cli( + "claim", + slug, + "1", + "--role", + "translator", + "--session-id", + "translator-a", + "--json", + ) + self.run_cli( + "accept-translation", + slug, + "1", + "--session-id", + "translator-a", + "--json", + ) + self.run_cli( + "release", + slug, + "1", + "--session-id", + "translator-a", + "--json", ) def initialize_final_book(self, slug="sample", *, private=False): From 47f9a50fcfaccdc670132dc91eb5d229baa9a9de Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:51:58 +0300 Subject: [PATCH 35/43] test: preserve accepted translation across corpus restore --- tests/test_corpus_cli.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/test_corpus_cli.py b/tests/test_corpus_cli.py index 8c38254..51f4932 100644 --- a/tests/test_corpus_cli.py +++ b/tests/test_corpus_cli.py @@ -124,8 +124,19 @@ def test_restore_rebuilds_complete_corpus_without_mutating_translation_state(sel translation_path = book / progress["chapters"][0]["translation_path"] translation_path.parent.mkdir(parents=True, exist_ok=True) translation_path.write_text("# Первая\n\nПеревод.\n", encoding="utf-8") - progress["chapters"][0]["status"] = "translated" - progress_path.write_text(json.dumps(progress, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + self.run_cli( + "book.py", "claim", "sample", "1", + "--role", "translator", "--session-id", "translator-a", "--json" + ) + self.run_cli( + "book.py", "accept-translation", "sample", "1", + "--session-id", "translator-a", "--json" + ) + self.run_cli( + "book.py", "release", "sample", "1", + "--session-id", "translator-a", "--json" + ) + progress = json.loads(progress_path.read_text(encoding="utf-8")) shutil.rmtree(book / "extracted") (book / "extracted").mkdir() From 256c87a92c8989e3e967c2901a1905403a6cc7a1 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:52:34 +0300 Subject: [PATCH 36/43] test: accept translations in finalize fixtures --- tests/test_workflow_v2_finalize_cli.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/test_workflow_v2_finalize_cli.py b/tests/test_workflow_v2_finalize_cli.py index 8167672..6794a43 100644 --- a/tests/test_workflow_v2_finalize_cli.py +++ b/tests/test_workflow_v2_finalize_cli.py @@ -144,15 +144,19 @@ def initialize_ready_book(self, slug="sample", *, private_source=False, keep_cla self.run_cli(*extract_args) book = self.repo / "books" / slug - progress_path = book / "progress.json" - progress = json.loads(progress_path.read_text(encoding="utf-8")) + progress = json.loads((book / "progress.json").read_text(encoding="utf-8")) translation = book / progress["chapters"][0]["translation_path"] translation.parent.mkdir(parents=True, exist_ok=True) translation.write_text("# Один\n\nАльфа.\n", encoding="utf-8") - progress["chapters"][0]["status"] = "translated" - progress_path.write_text( - json.dumps(progress, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", + self.run_cli( + "claim", slug, "1", "--role", "translator", + "--session-id", "translator-a", "--json" + ) + self.run_cli( + "accept-translation", slug, "1", "--session-id", "translator-a", "--json" + ) + self.run_cli( + "release", slug, "1", "--session-id", "translator-a", "--json" ) self.run_cli( From 72de21d8302c213230d91e400a840235b4456fd5 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:53:20 +0300 Subject: [PATCH 37/43] test: accept translations in finalize recovery fixtures --- .../test_workflow_v2_finalize_reliability.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/test_workflow_v2_finalize_reliability.py b/tests/test_workflow_v2_finalize_reliability.py index a5d453b..3568eb8 100644 --- a/tests/test_workflow_v2_finalize_reliability.py +++ b/tests/test_workflow_v2_finalize_reliability.py @@ -88,16 +88,20 @@ def initialize_ready_book(self): "ru", ) book = self.repo / "books" / "sample" - repository = self.repository(book) - progress = repository.read("progress.json", SchemaKind.PROGRESS) - translated = dict(progress.data) - translated["chapters"] = [dict(item) for item in progress.data["chapters"]] - translation_path = book / translated["chapters"][0]["translation_path"] + progress = self.repository(book).read("progress.json", SchemaKind.PROGRESS).data + translation_path = book / progress["chapters"][0]["translation_path"] translation_path.parent.mkdir(parents=True, exist_ok=True) translation_path.write_text("# Один\n\nАльфа.\n", encoding="utf-8") - translated["chapters"][0]["status"] = "translated" - repository.write_if_version( - "progress.json", SchemaKind.PROGRESS, translated, progress.version + self.run_book( + "claim", "sample", "1", "--role", "translator", + "--session-id", "translator-a", "--json" + ) + self.run_book( + "accept-translation", "sample", "1", + "--session-id", "translator-a", "--json" + ) + self.run_book( + "release", "sample", "1", "--session-id", "translator-a", "--json" ) self.run_book( From d0a90eecfe251db489b0e2c44a15b96ecf94d738 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:54:11 +0300 Subject: [PATCH 38/43] test: accept translations in EPUB reliability fixtures --- tests/test_workflow_v2_epub_reliability.py | 26 +++++++++++++--------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tests/test_workflow_v2_epub_reliability.py b/tests/test_workflow_v2_epub_reliability.py index 1073ce4..6d15a8f 100644 --- a/tests/test_workflow_v2_epub_reliability.py +++ b/tests/test_workflow_v2_epub_reliability.py @@ -79,21 +79,27 @@ def initialize_book(self, slug="sample", *, chapters=1): "ru", ) book = self.repo / "books" / slug - progress_path = book / "progress.json" - progress = json.loads(progress_path.read_text(encoding="utf-8")) + progress = json.loads((book / "progress.json").read_text(encoding="utf-8")) self.assertEqual(len(progress["chapters"]), chapters) for chapter in progress["chapters"]: + number = chapter["number"] translation = book / chapter["translation_path"] translation.parent.mkdir(parents=True, exist_ok=True) translation.write_text( - f"# Глава {chapter['number']}\n\nПеревод {chapter['number']}.\n", + f"# Глава {number}\n\nПеревод {number}.\n", encoding="utf-8", ) - chapter["status"] = "translated" - progress_path.write_text( - json.dumps(progress, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) + session = f"translator-{number}" + self.run_cli( + "claim", slug, str(number), "--role", "translator", + "--session-id", session, "--json" + ) + self.run_cli( + "accept-translation", slug, str(number), "--session-id", session, "--json" + ) + self.run_cli( + "release", slug, str(number), "--session-id", session, "--json" + ) return book def initialize_final_book(self, slug="sample", *, chapters=1): @@ -261,8 +267,8 @@ def test_relevant_metadata_order_cover_translation_and_review_changes_are_stale( ) self.run_cli("release", slug, "1", "--session-id", "reviewer-change", "--json") - changed = self.status(slug) - self.assertEqual(changed["state"], "stale", changed) + changed = self.status(slug, expect=1 if case == "translation" else 0) + self.assertEqual(changed["state"], "invalid" if case == "translation" else "stale", changed) def test_semantically_duplicate_pass_evidence_does_not_make_output_stale(self): self.initialize_final_book() From fb19b205030098412befc73eebe04a75f1c354e4 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:55:08 +0300 Subject: [PATCH 39/43] test: validate reviews on accepted translations --- tests/test_workflow_v2_review_validation.py | 29 +++++++++++++++------ 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/tests/test_workflow_v2_review_validation.py b/tests/test_workflow_v2_review_validation.py index 68388b8..4c2836a 100644 --- a/tests/test_workflow_v2_review_validation.py +++ b/tests/test_workflow_v2_review_validation.py @@ -72,6 +72,20 @@ def make_translation(self, content="Перевод.\n"): path.write_text(content, encoding="utf-8") return path + def accept_translation(self, content="Перевод.\n"): + path = self.make_translation(content) + self.run_cli( + "claim", "sample", "1", "--role", "translator", + "--session-id", "translator-a", "--json" + ) + self.run_cli( + "accept-translation", "sample", "1", "--session-id", "translator-a", "--json" + ) + self.run_cli( + "release", "sample", "1", "--session-id", "translator-a", "--json" + ) + return path + def mark_status(self, status): repository = self.state() loaded = repository.read("progress.json", SchemaKind.PROGRESS) @@ -136,35 +150,34 @@ def test_ledger_enabled_book_rejects_malformed_ledger(self): self.assertIn("review-ledger", result.stderr.lower()) def test_reviewed_status_without_current_pass_is_invalid(self): - self.make_translation() + self.accept_translation() self.mark_status("reviewed") result = self.run_cli("validate", "sample", expect=1) self.assertIn("current pass", result.stderr.lower()) def test_exact_current_pass_allows_reviewed_status(self): - self.make_translation() - self.mark_status("translated") + self.accept_translation() self.record_pass() self.mark_status("reviewed") self.run_cli("validate", "sample") - def test_editing_reviewed_translation_makes_validation_stale(self): - translation = self.make_translation() - self.mark_status("translated") + def test_editing_reviewed_translation_fails_acceptance_integrity(self): + translation = self.accept_translation() self.record_pass() self.mark_status("reviewed") translation.write_text("Изменённый перевод.\n", encoding="utf-8") result = self.run_cli("validate", "sample", expect=1) - self.assertIn("stale", result.stderr.lower()) + self.assertIn("translation_acceptance", result.stderr) + self.assertIn("sha256 mismatch", result.stderr) def test_missing_review_evidence_marker_is_invalid_current_state(self): + self.accept_translation() metadata_path = self.book / "metadata.json" metadata = json.loads(metadata_path.read_text(encoding="utf-8")) del metadata["workflow"]["review_evidence"] metadata_path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") (self.book / "review-ledger.json").unlink() - self.make_translation() self.mark_status("reviewed") result = self.run_cli("validate", "sample", expect=1) From f406b2ee3828ef8813647b8f715345194a0ff543 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:56:11 +0300 Subject: [PATCH 40/43] test: drive reliability fixtures through translator acceptance --- tests/test_workflow_v2_reliability.py | 59 ++++++++++++++++----------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/tests/test_workflow_v2_reliability.py b/tests/test_workflow_v2_reliability.py index e997be6..8d1dcfd 100644 --- a/tests/test_workflow_v2_reliability.py +++ b/tests/test_workflow_v2_reliability.py @@ -182,18 +182,25 @@ def write_translation(self, book, text="# Один\n\nАльфа.\n"): def mark_translated(self, book, text="# Один\n\nАльфа.\n"): translation = self.write_translation(book, text) - repository = self.book_repository(book) - loaded = repository.read("progress.json", SchemaKind.PROGRESS) - updated = dict(loaded.data) - updated["chapters"] = [dict(chapter) for chapter in loaded.data["chapters"]] - updated["chapters"][0]["status"] = "translated" - revision = repository.write_if_version( - "progress.json", - SchemaKind.PROGRESS, - updated, - loaded.version, - ) - return translation, revision + claim_path = book / ".workflow" / "claims" / "chapter-000001.json" + created_claim = not claim_path.is_file() + if created_claim: + session_id = "translator-helper" + self.run_book( + "claim", "sample", "1", "--role", "translator", + "--session-id", session_id, "--json" + ) + else: + claim = json.loads(claim_path.read_text(encoding="utf-8")) + session_id = claim["session_id"] + accepted = self.canonical_json( + self.run_book( + "accept-translation", "sample", "1", "--session-id", session_id, "--json" + ) + ) + if created_claim: + self.release_claim(session_id) + return translation, accepted["progress_revision"] def claim_reviewer(self, session_id="reviewer-crashed"): return self.run_book( @@ -474,14 +481,24 @@ def test_stale_pass_on_translated_unit_resumes_review(self): self.record_pass("reviewer-a") self.release_claim("reviewer-a") + self.run_book( + "claim", "sample", "1", "--role", "translator", + "--session-id", "translator-correction", "--json" + ) translation.write_text("# Один\n\nИзменённая Альфа.\n", encoding="utf-8") + self.run_book( + "accept-translation", "sample", "1", + "--session-id", "translator-correction", "--json" + ) + self.release_claim("translator-correction") + status = self.canonical_json(self.run_book("status", "sample", "--json")) self.assertTrue(status["valid"]) self.assertEqual(status["reviews"]["stale"], 1) resumed = self.canonical_json(self.run_book("resume", "sample", "--json")) self.assertEqual(resumed["operation"], "review") - def test_stale_pass_on_reviewed_unit_fails_closed(self): + def test_tampered_reviewed_translation_fails_closed(self): book = self.initialize_book() translation, _ = self.mark_translated(book) self.claim_reviewer("reviewer-a") @@ -493,17 +510,11 @@ def test_stale_pass_on_reviewed_unit_fails_closed(self): self.assertTrue(accepted["changed"]) translation.write_text("# Один\n\nИзменённая Альфа.\n", encoding="utf-8") - status = self.canonical_json(self.run_book("status", "sample", "--json")) - self.assertFalse(status["valid"]) - self.assertEqual(status["reviews"]["stale"], 1) - self.assertTrue( - any("reviewed without current PASS evidence" in error for error in status["errors"]) - ) - resumed = self.canonical_json( - self.run_book("resume", "sample", "--json", expect=1) - ) - self.assertEqual(resumed["operation"], "blocked") - self.assertEqual(resumed["reason"], "preflight_failed") + status = self.run_book("status", "sample", "--json", expect=1) + self.assertIn("translation_acceptance", status.stderr) + self.assertIn("sha256 mismatch", status.stderr) + resumed = self.run_book("resume", "sample", "--json", expect=1) + self.assertIn("translation_acceptance", resumed.stderr) def test_concurrent_glossary_cas_rejects_stale_writer_without_lost_update(self): book = self.initialize_book() From 2a426b402b06503228e5d5f48db419dfb643fa5c Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:57:07 +0300 Subject: [PATCH 41/43] test: classify reviewed translation tamper as invalid acceptance --- tests/test_workflow_v2_review_cli.py | 92 ++++++---------------------- 1 file changed, 19 insertions(+), 73 deletions(-) diff --git a/tests/test_workflow_v2_review_cli.py b/tests/test_workflow_v2_review_cli.py index 80c0f0b..f6e54d0 100644 --- a/tests/test_workflow_v2_review_cli.py +++ b/tests/test_workflow_v2_review_cli.py @@ -62,38 +62,19 @@ def initialize_book(self, slug="sample"): translation.parent.mkdir(parents=True, exist_ok=True) translation.write_text("# Один\n\nАльфа.\n", encoding="utf-8") self.run_cli( - "claim", - slug, - "1", - "--role", - "translator", - "--session-id", - "translator-a", - "--json", + "claim", slug, "1", "--role", "translator", + "--session-id", "translator-a", "--json", ) self.run_cli( - "accept-translation", - slug, - "1", - "--session-id", - "translator-a", - "--json", + "accept-translation", slug, "1", "--session-id", "translator-a", "--json", ) self.run_cli("release", slug, "1", "--session-id", "translator-a") return book, translation def claim_reviewer(self, slug="sample", *, session_id="reviewer-a", role="reviewer"): return self.run_cli( - "claim", - slug, - "1", - "--role", - role, - "--session-id", - session_id, - "--base-commit", - "dispatch-commit", - "--json", + "claim", slug, "1", "--role", role, "--session-id", session_id, + "--base-commit", "dispatch-commit", "--json", ) def assert_canonical_json(self, result): @@ -104,22 +85,14 @@ def assert_canonical_json(self, result): ) return payload - def test_pass_record_list_accept_and_stale_detection_end_to_end(self): + def test_pass_record_list_accept_and_tamper_detection_end_to_end(self): book, translation = self.initialize_book() self.claim_reviewer() recorded = self.assert_canonical_json( self.run_cli( - "review-record", - "sample", - "1", - "--outcome", - "PASS", - "--session-id", - "reviewer-a", - "--review-commit", - "review-commit-a", - "--json", + "review-record", "sample", "1", "--outcome", "PASS", + "--session-id", "reviewer-a", "--review-commit", "review-commit-a", "--json", ) ) record = recorded["record"] @@ -147,9 +120,9 @@ def test_pass_record_list_accept_and_stale_detection_end_to_end(self): self.assertEqual(progress["chapters"][0]["status"], "reviewed") translation.write_text("# Один\n\nИзменённая Альфа.\n", encoding="utf-8") - stale = self.assert_canonical_json(self.run_cli("reviews", "sample", "--json")) - self.assertEqual(stale["reviews"][0]["state"], "stale") - self.assertIsNone(stale["reviews"][0]["current_record"]) + tampered = self.run_cli("reviews", "sample", "--json", expect=1) + self.assertIn("translation_acceptance", tampered.stderr) + self.assertIn("sha256 mismatch", tampered.stderr) self.run_cli("accept-review", "sample", "1", "--json", expect=1) def test_corrections_required_is_recorded_and_blocks_acceptance(self): @@ -158,14 +131,8 @@ def test_corrections_required_is_recorded_and_blocks_acceptance(self): recorded = self.assert_canonical_json( self.run_cli( - "review-record", - "sample", - "1", - "--outcome", - "CORRECTIONS_REQUIRED", - "--session-id", - "reviewer-a", - "--json", + "review-record", "sample", "1", "--outcome", "CORRECTIONS_REQUIRED", + "--session-id", "reviewer-a", "--json", ) ) self.assertEqual(recorded["record"]["correction_round"], 1) @@ -177,15 +144,8 @@ def test_review_record_requires_matching_reviewer_claim(self): self.initialize_book() self.claim_reviewer(session_id="someone-else") foreign = self.run_cli( - "review-record", - "sample", - "1", - "--outcome", - "PASS", - "--session-id", - "reviewer-a", - "--json", - expect=1, + "review-record", "sample", "1", "--outcome", "PASS", + "--session-id", "reviewer-a", "--json", expect=1, ) self.assertIn("someone-else", foreign.stderr) @@ -198,15 +158,8 @@ def test_review_record_refuses_missing_resolved_revision(self): self.claim_reviewer() result = self.run_cli( - "review-record", - "sample", - "1", - "--outcome", - "PASS", - "--session-id", - "reviewer-a", - "--json", - expect=1, + "review-record", "sample", "1", "--outcome", "PASS", + "--session-id", "reviewer-a", "--json", expect=1, ) self.assertIn("translation_acceptance", result.stderr) @@ -219,15 +172,8 @@ def test_review_record_rejects_missing_translation_acceptance(self): self.claim_reviewer() result = self.run_cli( - "review-record", - "sample", - "1", - "--outcome", - "PASS", - "--session-id", - "reviewer-a", - "--json", - expect=1, + "review-record", "sample", "1", "--outcome", "PASS", + "--session-id", "reviewer-a", "--json", expect=1, ) self.assertIn("translation_acceptance", result.stderr) From f02266db42ffddab06f96b9a59408ec23aa1beb6 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:42:57 +0300 Subject: [PATCH 42/43] test: give corpus restore current workflow provenance --- tests/test_corpus_cli.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_corpus_cli.py b/tests/test_corpus_cli.py index 51f4932..e2d4fe9 100644 --- a/tests/test_corpus_cli.py +++ b/tests/test_corpus_cli.py @@ -14,6 +14,7 @@ CORPUS_SCRIPT = PROJECT_ROOT / "scripts" / "corpus.py" WORKFLOW_V2 = PROJECT_ROOT / "scripts" / "workflow_v2" TEMPLATES = PROJECT_ROOT / "docs" / "templates" +REVISION = "0123456789abcdef" class CorpusCliTests(unittest.TestCase): @@ -27,6 +28,19 @@ def setUp(self): shutil.copytree(WORKFLOW_V2, self.repo / "scripts" / "workflow_v2") if TEMPLATES.exists(): shutil.copytree(TEMPLATES, self.repo / "docs" / "templates") + (self.repo / ".book-translator-install.json").write_text( + json.dumps( + { + "schema_version": 1, + "canonical_repository": "https://github.com/tim8es/book-translator", + "requested_ref": "refactor/workflow-engine-v2", + "resolved_revision": REVISION, + "install_root": ".", + } + ) + + "\n", + encoding="utf-8", + ) def tearDown(self): self.tmp.cleanup() From d78c35db0559dff1dd2f072e6d9f1385d715fef2 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:47:46 +0300 Subject: [PATCH 43/43] fix: allow source corpus recovery before acceptance recheck --- scripts/workflow_v2/repository.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/workflow_v2/repository.py b/scripts/workflow_v2/repository.py index 9459aac..abfcfce 100644 --- a/scripts/workflow_v2/repository.py +++ b/scripts/workflow_v2/repository.py @@ -103,6 +103,11 @@ def _verify_translation_acceptance_integrity(self, progress: Mapping[str, Any]) try: artifact = self.storage.read(artifact_path) except StorageNotFound: + if path_key == "source_path": + # Missing source corpus is a recoverable corpus-integrity condition. + # Structural/corpus preflight still fails closed, while corpus restore + # must be able to read schema-valid progress before recreating source bytes. + continue mismatches.append(f"{hash_key} cannot be verified because {artifact_path} is missing") continue actual = hashlib.sha256(artifact.content).hexdigest()