diff --git a/.gitignore b/.gitignore index 147cfe2e..57b4fc19 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ __pycache__/ *.pyo .DS_Store .firecrawl/ +.smoke/ dist/ *.tgz skills/bmad-story-automator/build/ diff --git a/docs/changelog/260602.md b/docs/changelog/260602.md new file mode 100644 index 00000000..c4d46560 --- /dev/null +++ b/docs/changelog/260602.md @@ -0,0 +1,152 @@ +# Changelog - 260602 + +## 260602-20:36:00 - Deterministic gate integration + +### Summary +Promoted fast deterministic smoke gates into `verify` and added an explicit full deterministic release smoke wrapper. + +### Added +- Added `smoke:deterministic-full` for prepared-repo reset, create, dev-loop, and finish-loop smoke coverage. + +### Changed +- Updated `verify` to run version, package, CLI, helper contract, mode, Python, and shell smoke gates. +- Updated Phase 06 gate map, implementation notes, TODO status, and handoff evidence. + +### Files +- `docs/plans/automator-deterministic-smoke-coverage/TODO/phase-06.md` +- `docs/plans/automator-deterministic-smoke-coverage/gate-map.md` +- `docs/plans/automator-deterministic-smoke-coverage/handoff-log.md` +- `docs/plans/automator-deterministic-smoke-coverage/implementation-notes.md` +- `package.json` + +### QA Notes +N/A + +## 260602-20:25:00 - Finish loop smoke coverage + +### Summary +Added Phase 05 deterministic finish-loop coverage for automate, review, finalize, retrospective, wrapup, and host commit isolation. + +### Added +- Added `smoke:finish-loop` and a temp git-backed finish-loop smoke runner. +- Added assertions for automate done/skip, incomplete review diagnostics, review completion, smoke-repo commits, source-of-truth finalization, multi-epic retrospective skip semantics, wrapup marker removal, and host repo isolation. + +### Changed +- Updated Phase 05 coverage baseline, gate map, implementation notes, TODO status, and handoff evidence. + +### Files +- `docs/plans/automator-deterministic-smoke-coverage/TODO/phase-05.md` +- `docs/plans/automator-deterministic-smoke-coverage/coverage-baseline.md` +- `docs/plans/automator-deterministic-smoke-coverage/gate-map.md` +- `docs/plans/automator-deterministic-smoke-coverage/handoff-log.md` +- `docs/plans/automator-deterministic-smoke-coverage/implementation-notes.md` +- `package.json` +- `scripts/run-smoke-finish-loop.py` + +### QA Notes +N/A + +## 260602-20:08:14 - Mode smoke coverage + +### Summary +Added Phase 04 deterministic mode coverage for create startup, resume, validate, edit, and marker lifecycle helpers. + +### Added +- Added `smoke:modes` and a temp-fixture mode smoke runner. +- Added mode assertions for stop-hook states, existing-state discovery, workflow-derived route menus/hints, marker JSON/heartbeat lifecycle, validation/source-of-truth checks, and edit helper routes. + +### Changed +- Updated Phase 04 coverage baseline, gate map, implementation notes, TODO status, and handoff evidence. + +### Files +- `docs/plans/automator-deterministic-smoke-coverage/04-create-dev-resume-validate-edit-coverage.md` +- `docs/plans/automator-deterministic-smoke-coverage/TODO/phase-04.md` +- `docs/plans/automator-deterministic-smoke-coverage/coverage-baseline.md` +- `docs/plans/automator-deterministic-smoke-coverage/gate-map.md` +- `docs/plans/automator-deterministic-smoke-coverage/handoff-log.md` +- `docs/plans/automator-deterministic-smoke-coverage/implementation-notes.md` +- `package.json` +- `scripts/run-smoke-modes.py` + +### QA Notes +N/A + +## 260602-19:50:10 - Runtime helper contract smokes + +### Summary +Added Phase 03 runtime helper contract smoke coverage and a named `smoke:contracts` gate. + +### Added +- Added a skip-fail `smoke:contracts` runner for parser, monitor, runner, build-cmd, state-update, runtime-policy, marker/root, and success-verifier contract suites. +- Added focused runtime helper contract tests for parser subprocess failure payloads, monitor terminal JSON, runner edge states, and `tmux-wrapper spawn` runner mode. + +### Changed +- Updated Phase 03 coverage baseline, gate map, implementation notes, TODO status, and handoff evidence. + +### Files +- `docs/plans/automator-deterministic-smoke-coverage/TODO/phase-03.md` +- `docs/plans/automator-deterministic-smoke-coverage/coverage-baseline.md` +- `docs/plans/automator-deterministic-smoke-coverage/gate-map.md` +- `docs/plans/automator-deterministic-smoke-coverage/handoff-log.md` +- `docs/plans/automator-deterministic-smoke-coverage/implementation-notes.md` +- `package.json` +- `scripts/run-smoke-contracts.py` +- `tests/test_runtime_helper_contracts.py` + +### QA Notes +N/A + +## 260602-19:41:15 - Package identity smoke contracts + +### Summary +Added Phase 02 package tarball and prepared-repo install identity contracts. + +### Added +- Added package contract assertions for npm pack contents, executable modes, forbidden generated files, tarball identity, and selected installed-file checksums. +- Added smoke prep manifests for current package identity and installed automator manifests. + +### Changed +- Updated smoke prepare to install the just-packed current tarball and verify prepared `.claude/skills` files against that tarball. +- Updated Phase 02 plan evidence and package/install gate mapping. + +### Files +- `docs/plans/automator-deterministic-smoke-coverage/TODO/phase-02.md` +- `docs/plans/automator-deterministic-smoke-coverage/coverage-baseline.md` +- `docs/plans/automator-deterministic-smoke-coverage/gate-map.md` +- `docs/plans/automator-deterministic-smoke-coverage/handoff-log.md` +- `docs/plans/automator-deterministic-smoke-coverage/implementation-notes.md` +- `package.json` +- `scripts/assert-package-contracts.py` +- `scripts/smoke_prep/automator.py` +- `scripts/smoke_prep/cli.py` +- `scripts/smoke_prep/inputs.py` +- `scripts/smoke_prep/package_contracts.py` +- `scripts/smoke_prep/report.py` + +### QA Notes +N/A + +## 260602-19:24:42 - Baseline deterministic smoke inputs + +### Summary +Added Phase 01 deterministic smoke baseline, version alignment checks, and resolved smoke input recording. + +### Changed +- Added version alignment and smoke input determinism scripts. +- Aligned story automator workflow metadata to the current release version. +- Updated smoke prep to record BMAD Method npm identity and install the resolved version. +- Added Phase 01 coverage baseline, gate-map updates, implementation notes, and handoff evidence. + +### Files +- `docs/plans/automator-deterministic-smoke-coverage/coverage-baseline.md` +- `docs/plans/automator-deterministic-smoke-coverage/gate-map.md` +- `docs/plans/automator-deterministic-smoke-coverage/handoff-log.md` +- `docs/versioning.md` +- `package.json` +- `scripts/check-smoke-inputs.py` +- `scripts/check-version-alignment.py` +- `scripts/smoke_prep/inputs.py` +- `skills/bmad-story-automator/workflow.md` + +### QA Notes +N/A diff --git a/docs/changelog/260618.md b/docs/changelog/260618.md new file mode 100644 index 00000000..3e48b02f --- /dev/null +++ b/docs/changelog/260618.md @@ -0,0 +1,46 @@ +# Changelog - 260618 + +## 260618-05:42:21 - Review remediation for deterministic smoke PR + +### Summary +Resolved review findings around story parsing, deterministic smoke subprocess safety, package/prep error handling, marker cleanup, release docs, and changelog traceability. + +### Added +- Added regression coverage for hyphenated explicit story IDs and bare numeric subheading parsing. +- Added smoke-script contract tests for malformed npm/package JSON, `--help` behavior, subprocess timeouts, skip-install reports, and host env isolation. + +### Changed +- Hardened smoke prep JSON/package assertions to fail with `SmokeError` instead of raw exceptions. +- Split mode-smoke reporting out of the main runner and kept the runner under the repo LOC guard. +- Updated release docs and plan evidence to distinguish fast `verify` gates from `smoke:deterministic-full`. + +### Files +- `docs/changelog/260602.md` +- `docs/changelog/260618.md` +- `docs/development.md` +- `docs/plans/automator-deterministic-smoke-coverage/README.md` +- `docs/plans/automator-deterministic-smoke-coverage/gate-map.md` +- `docs/plans/automator-deterministic-smoke-coverage/handoff-log.md` +- `docs/versioning.md` +- `scripts/run-smoke-automator.py` +- `scripts/run-smoke-dev-loop.py` +- `scripts/run-smoke-finish-loop.py` +- `scripts/run-smoke-modes.py` +- `scripts/smoke_prep/cli.py` +- `scripts/smoke_prep/inputs.py` +- `scripts/smoke_prep/mode_report.py` +- `scripts/smoke_prep/package_contracts.py` +- `scripts/smoke_prep/process.py` +- `scripts/smoke_prep/report.py` +- `skills/bmad-story-automator/src/story_automator/commands/orchestrator_epic_agents.py` +- `skills/bmad-story-automator/src/story_automator/core/epic_parser.py` +- `tests/test_cli_contracts.py` +- `tests/test_smoke_script_contracts.py` + +### QA Notes +- Focused regression suite: `PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=skills/bmad-story-automator/src python3 -m unittest tests.test_smoke_script_contracts tests.test_orchestrator_epic_agents tests.test_cli_contracts tests.test_epic_parser` pass. +- Changelog timestamp assertion: pass for `docs/changelog/260602.md`. +- `npm run pack:assert`: pass after package-surface correction. +- `npm run verify`: pass after review-loop fix batch. +- `git diff --check`: pass after review-loop fix batch. +- Review-loop: parser reviewer had no actionable findings; runtime, docs, and contract findings fixed and reverified. diff --git a/docs/development.md b/docs/development.md index 71dcdd77..b7d506eb 100644 --- a/docs/development.md +++ b/docs/development.md @@ -14,9 +14,17 @@ PYTHONPATH=skills/bmad-story-automator/src python3 -m story_automator --help `npm run verify` expands to: - `npm run test:python` -- `npm run pack:dry-run` +- `npm run version:check` +- `npm run pack:assert` +- `npm run test:cli` +- `npm run smoke:contracts` +- `npm run smoke:modes` - `npm run test:smoke` +Run `npm run smoke:deterministic-full` separately before release when prepared +repo reset, install identity, create, dev-loop, and finish-loop coverage is +needed. + ## Smoke Test Coverage The smoke suite validates: @@ -29,14 +37,30 @@ The smoke suite validates: - installed runtime policy, prompt templates, and parse contracts - prompt-building behavior for Claude and Codex child sessions +## External Automator Smoke Prep + +Prepare the pinned `bma-d/gunz` smoke project in the ignored `.smoke/` +workspace: + +```bash +npm run smoke:prepare -- --reset +``` + +This clones only the `bmad-smoke-test` branch, checks out +`fca6470d329668019dace305b5f0f3c9b62cb113`, installs BMAD core and BMM with +`bmad-method@next`, then installs the project-local automator into that target +project. The script leaves `.smoke/SMOKE_NEXT_STEPS.md` with the exact manual +Claude Code entrypoint for the semi-automated smoke run. + ## Repo Verification Flow ```mermaid flowchart TD A["Edit installer, skills, or runtime"] --> B["Run npm run test:python"] - B --> C["Run npm run pack:dry-run"] - C --> D["Run npm run test:smoke"] + B --> C["Run npm run pack:assert"] + C --> D["Run npm run smoke:contracts + smoke:modes"] D --> E["Run npm run verify"] + E --> F["Run npm run smoke:deterministic-full before release"] ``` ## Packaging Surface @@ -96,10 +120,11 @@ Publish steps: Recommended release checklist: 1. `npm run verify` -2. use `secrets` skill for npm auth material; search exact key names, then `secrets load ` into the publish shell; never print token values -3. inspect the package dry-run output -4. confirm README and docs match shipped behavior -5. publish +2. `npm run smoke:deterministic-full` +3. use `secrets` skill for npm auth material; search exact key names, then `secrets load ` into the publish shell; never print token values +4. inspect the package dry-run output +5. confirm README and docs match shipped behavior +6. publish For BMAD Method stable tags, preview tags, registry `next`, and npm dist-tags, use [Versioning And Release Channels](./versioning.md). diff --git a/docs/plans/automator-deterministic-smoke-coverage/01-baseline-and-version-determinism.md b/docs/plans/automator-deterministic-smoke-coverage/01-baseline-and-version-determinism.md new file mode 100644 index 00000000..5e88601d --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/01-baseline-and-version-determinism.md @@ -0,0 +1,53 @@ +# Phase 01 - Baseline And Version Determinism + +## Clean Context Start + +Before doing this phase, read [README.md](./README.md), this phase file, [TODO/phase-01.md](./TODO/phase-01.md), [gate-map.md](./gate-map.md), [implementation-notes.md](./implementation-notes.md), and relevant prior entries in [handoff-log.md](./handoff-log.md). + +Do not read later phase files or later TODO files as acceptance criteria for this phase. + +## Goal + +Create a source-of-truth coverage baseline for the automator workflow and add deterministic version/input checks so later smoke phases are not built on moving or stale package metadata. + +## Inputs + +- [skills/bmad-story-automator/workflow.md](../../../skills/bmad-story-automator/workflow.md) +- [skills/bmad-story-automator/data/orchestration-policy.json](../../../skills/bmad-story-automator/data/orchestration-policy.json) +- [package.json](../../../package.json) +- [skills/module.yaml](../../../skills/module.yaml) +- [skills/bmad-story-automator/pyproject.toml](../../../skills/bmad-story-automator/pyproject.toml) +- [skills/bmad-story-automator/src/story_automator/__init__.py](../../../skills/bmad-story-automator/src/story_automator/__init__.py) +- [.claude-plugin/plugin.json](../../../.claude-plugin/plugin.json) +- [.claude-plugin/marketplace.json](../../../.claude-plugin/marketplace.json) + +## Implementation Steps + +1. Build a coverage inventory table that maps each automator mode and policy step to a deterministic gate status: `fact`, `gap`, `blocked`, `stale`, or `spec-only`. +2. Add or update a repo-local deterministic metadata check that asserts version alignment across package, plugin, module, Python package, runtime `__init__`, and workflow frontmatter. +3. Decide whether `bmad-method@next` should be pinned for deterministic prep or recorded/asserted as an explicit smoke input. +4. Update [gate-map.md](./gate-map.md) with the metadata/version gate and any blocked input-pin gate. +5. Record any stale metadata findings in [implementation-notes.md](./implementation-notes.md). + +## Verification + +- Run the new or updated metadata/version command. +- Run the new or updated smoke input command. +- Run `npm run test:cli`. +- Run `git diff --check`. +- Confirm [gate-map.md](./gate-map.md) has entries for version alignment and smoke input determinism. + +## Exit Criteria + +- Coverage baseline exists and classifies create, resume, validate, edit, create-story, dev-story, automate, review, commit/finalize, retrospective, wrapup, and package/install surfaces. +- Stale version metadata either fixed or explicitly marked `stale` with a follow-up owner. +- Moving external inputs are pinned or explicitly asserted. +- Phase 01 handoff entry appended. + +## Implementation Notes Requirements + +Keep [implementation-notes.md](./implementation-notes.md) current while implementing. Record stale metadata, input pinning tradeoffs, and any reason a version surface cannot be aligned immediately. + +## Handoff Requirements + +Append a Phase 01 entry to [handoff-log.md](./handoff-log.md) with commands run, version surfaces checked, facts classified, blockers, and next recommended command. diff --git a/docs/plans/automator-deterministic-smoke-coverage/02-package-and-prepared-repo-contracts.md b/docs/plans/automator-deterministic-smoke-coverage/02-package-and-prepared-repo-contracts.md new file mode 100644 index 00000000..ef7c338f --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/02-package-and-prepared-repo-contracts.md @@ -0,0 +1,59 @@ +# Phase 02 - Package And Prepared Repo Contracts + +## Clean Context Start + +Before doing this phase, read [README.md](./README.md), this phase file, [TODO/phase-02.md](./TODO/phase-02.md), [gate-map.md](./gate-map.md), [implementation-notes.md](./implementation-notes.md), and relevant earlier entries in [handoff-log.md](./handoff-log.md). + +Do not read later phase files or later TODO files as acceptance criteria for this phase. + +## Goal + +Harden the package and prepared-repo smoke contract so packed tarballs, installed files, dependency skill roots, and prepared `gunz` layout fail deterministically when installer drift occurs. + +This phase is release-blocking. A prepared repo smoke result is not trustworthy until it proves that `.smoke/gunz` installed the current packed tarball rather than a stale same-shape install. + +## Inputs + +- [scripts/prepare-smoke-test.py](../../../scripts/prepare-smoke-test.py) +- [scripts/smoke_prep/](../../../scripts/smoke_prep) +- [scripts/smoke-test.sh](../../../scripts/smoke-test.sh) +- [scripts/run-smoke-automator.py](../../../scripts/run-smoke-automator.py) +- [scripts/run-smoke-dev-loop.py](../../../scripts/run-smoke-dev-loop.py) +- [package.json](../../../package.json) +- [install.sh](../../../install.sh) + +## Implementation Steps + +1. Add a deterministic `npm run pack:assert` gate for required package files and forbidden generated/cache files using `npm pack --dry-run --json`. +2. Capture and assert packed tarball identity in the same `npm run pack:assert` gate, or a separately named `npm run pack:identity` gate if implementation size warrants it, using `npm pack --json --pack-destination `: package name, version, filename, integrity or shasum, generated tarball path, and selected checksums. +3. Create or reuse a shared installed-file manifest for the real packed install into `.smoke/gunz`. +4. Assert `.smoke/gunz` installed the current packed tarball, not a stale same-shape install: installed version surfaces match tarball metadata, selected installed files have checksums matching the tarball contents, and prep report records the tarball identity. +5. Compare narrow installed-file checksums against the extracted tarball for `SKILL.md`, `scripts/story-automator`, policy JSON, parse contracts, prompt templates, `pyproject.toml`, review `contract.json`, and version surfaces. +6. Extend `smoke:prepare` layout verification beyond helper `--help` to cover runtime source, policy JSON, parse/prompt files, templates, review skill contract, module metadata, and dependency skill entrypoints. +7. Add a deterministic installed-root check for supported runtime roots when feasible: `.claude`, `.agents`, and `.codex`. If external BMAD install cannot prepare all roots, mark missing roots as `blocked` or `spec-only` in the coverage baseline. +8. Use stable JSON and checksum assertions; do not accept terminal-output-only `npm pack` success as package proof. +9. Update [gate-map.md](./gate-map.md) with package-content, installed-identity, installed-manifest, and prepared-repo gates. + +## Verification + +- Run the package content assertion. +- Run `npm run smoke:prepare -- --reset`. +- Run `npm run smoke:run`. +- Run `npm run smoke:dev-loop`. +- Run `git diff --check`. + +## Exit Criteria + +- The prepared external repo verifies the installed automator package identity and checksums, not just local source fixtures or same-shape installed files. +- Package content assertions catch missing required files and unexpected generated files. +- Unsupported or unavailable install roots are explicitly classified instead of silently ignored. +- `npm run pack:assert` is fast enough for the future default `npm run verify`; prepared repo install checks remain explicit unless Phase 06 proves CI/runtime budget. +- Phase 02 handoff entry appended. + +## Implementation Notes Requirements + +Record any installer-root limitations, BMAD Method pinning decisions, and package manifest tradeoffs in [implementation-notes.md](./implementation-notes.md). + +## Handoff Requirements + +Append a Phase 02 entry to [handoff-log.md](./handoff-log.md) with exact smoke prep command output summary, tarball/package details, installed manifest path, and next recommended command. diff --git a/docs/plans/automator-deterministic-smoke-coverage/03-runtime-helper-contract-smokes.md b/docs/plans/automator-deterministic-smoke-coverage/03-runtime-helper-contract-smokes.md new file mode 100644 index 00000000..4eaa1122 --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/03-runtime-helper-contract-smokes.md @@ -0,0 +1,63 @@ +# Phase 03 - Runtime Helper Contract Smokes + +## Clean Context Start + +Before doing this phase, read [README.md](./README.md), this phase file, [TODO/phase-03.md](./TODO/phase-03.md), [gate-map.md](./gate-map.md), [implementation-notes.md](./implementation-notes.md), and relevant earlier entries in [handoff-log.md](./handoff-log.md). + +Do not read later phase files or later TODO files as acceptance criteria for this phase. + +## Goal + +Add fast deterministic contract smokes for the helper layer that the automator relies on during live orchestration: parser subprocess, monitor JSON, runner lifecycle, build-command safety flags, state-update failures, runtime policy snapshots, and success verifiers. + +This phase is release-blocking. Higher-level lifecycle smokes must not hide helper drift behind broad success output; they should depend on helper JSON contracts that are already proven here. + +## Inputs + +- [skills/bmad-story-automator/src/story_automator/commands/orchestrator_parse.py](../../../skills/bmad-story-automator/src/story_automator/commands/orchestrator_parse.py) +- [skills/bmad-story-automator/src/story_automator/commands/tmux.py](../../../skills/bmad-story-automator/src/story_automator/commands/tmux.py) +- [skills/bmad-story-automator/src/story_automator/core/tmux_runtime.py](../../../skills/bmad-story-automator/src/story_automator/core/tmux_runtime.py) +- [skills/bmad-story-automator/src/story_automator/core/success_verifiers.py](../../../skills/bmad-story-automator/src/story_automator/core/success_verifiers.py) +- [skills/bmad-story-automator/src/story_automator/core/state_validation.py](../../../skills/bmad-story-automator/src/story_automator/core/state_validation.py) +- [skills/bmad-story-automator/src/story_automator/core/runtime_policy.py](../../../skills/bmad-story-automator/src/story_automator/core/runtime_policy.py) +- [skills/bmad-story-automator/data/orchestration-policy.json](../../../skills/bmad-story-automator/data/orchestration-policy.json) + +## Implementation Steps + +1. Add a fast `smoke:contracts` runner or equivalent Python/unit-test gate. +2. Exercise `orchestrator-helper parse-output` with a fake `claude` on `PATH` and a parser matrix covering: success JSON, missing/empty output file, missing `--state-file` value, invalid runtime policy, unknown step contract, parse-contract load failure, subprocess timeout, subprocess nonzero, no JSON object, JSON decode failure, and schema-invalid JSON. Assert exit code plus JSON `status`, `reason`, and `structuredIssues` where present. +3. Exercise `monitor-session --json` terminal states and diagnostics covering: `not_found`, `timeout`, `completed`, `incomplete`, `crashed`, `stuck`, completed-but-unverified output, invalid option handling, and invalid persisted session-state diagnostics. Assert `final_state`, `exit_reason`, `output_verified`, and `structuredIssues` shape rather than relying on exit code. +4. Exercise `SA_TMUX_RUNTIME=runner` success and crash paths through `tmux-wrapper spawn`. +5. Assert runner edge states where deterministic: `spawn_error`, interrupted output/state, and launch-never-succeeded/stuck mapping. +6. Assert `build-cmd` launch branches: default Codex safety flags, `AI_COMMAND` override behavior, non-Codex `--agent claude`, quoted `--model`, unknown step, invalid state file, and invalid/missing state policy. +7. Assert invalid `state-update` transitions and malformed `--set` fail without mutating the state file. +8. Assert success verifier happy and failure paths: missing/duplicate create artifacts, incomplete review, story-file fallback, epic incomplete, source mismatch diagnostics, and `sprint_status_not_updated` notes where story file says done but sprint status does not. +9. Assert marker path and runtime-root resolution helpers for `.agents`, `.codex`, and `.claude` layouts so higher-level smokes do not hard-code `.claude`. +10. Assert status/source-of-truth helper behavior for story-file status, sprint-status status, and mismatch surfacing. +11. Assert runtime policy snapshot creation at state-doc build time, state/reference binding to the snapshot path/hash, and later helper reads from the pinned snapshot rather than mutable source policy. +12. Assert runtime policy snapshot drift or missing snapshots fail closed. +13. Keep this runner local and fast enough for `npm run verify`; prefer temp fixtures, fake subprocesses, and `SA_TMUX_RUNTIME=runner` over live tmux/provider behavior. +14. Update [gate-map.md](./gate-map.md) with named rows or stable coverage IDs for every helper contract family and edge matrix, not a single generic helper row. + +## Verification + +- Run the new `smoke:contracts` command. +- Run `npm run test:python`. +- Run `npm run test:cli`. +- Run `git diff --check`. + +## Exit Criteria + +- Parser, monitor, runner, build-command, state-update, marker/root resolution, status/source-of-truth helpers, runtime-policy snapshot creation/failure, and success-verifier contracts are tested without live LLM dependence. +- Gate-map rows are split by helper contract family so future implementation cannot hide missing edge matrices behind one broad gate. +- Contract failures produce structured diagnostics where the code claims they should. +- `smoke:contracts` is ready to be wired into the future default `npm run verify`. +- Phase 03 handoff entry appended. + +## Implementation Notes Requirements + +Record any helper behavior that differs from docs, especially exit-code versus JSON-contract behavior, in [implementation-notes.md](./implementation-notes.md). + +## Handoff Requirements + +Append a Phase 03 entry to [handoff-log.md](./handoff-log.md) with commands run, fake parser setup, runner runtime settings, failure classes tested, and next recommended command. diff --git a/docs/plans/automator-deterministic-smoke-coverage/04-create-dev-resume-validate-edit-coverage.md b/docs/plans/automator-deterministic-smoke-coverage/04-create-dev-resume-validate-edit-coverage.md new file mode 100644 index 00000000..e5731c52 --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/04-create-dev-resume-validate-edit-coverage.md @@ -0,0 +1,65 @@ +# Phase 04 - Create Dev Resume Validate Edit Coverage + +## Clean Context Start + +Before doing this phase, read [README.md](./README.md), this phase file, [TODO/phase-04.md](./TODO/phase-04.md), [gate-map.md](./gate-map.md), [implementation-notes.md](./implementation-notes.md), and relevant earlier entries in [handoff-log.md](./handoff-log.md). + +Do not read later phase files or later TODO files as acceptance criteria for this phase. + +## Goal + +Expand deterministic smoke coverage across create/dev plus the non-create automator modes: resume, validate, and edit. This phase should prove state discovery, sprint comparison, marker lifecycle, validation reports, edit summaries, and mode routing contracts. + +Most mode coverage should run against narrow temp BMAD-style fixtures, not the prepared `.smoke/gunz` repo. Prepared-repo coverage remains useful for realistic flow checks, but default verification should not depend on external reset/network surfaces. + +## Inputs + +- [scripts/run-smoke-automator.py](../../../scripts/run-smoke-automator.py) +- [scripts/run-smoke-dev-loop.py](../../../scripts/run-smoke-dev-loop.py) +- [skills/bmad-story-automator/steps-c/step-01-init.md](../../../skills/bmad-story-automator/steps-c/step-01-init.md) +- [skills/bmad-story-automator/steps-c/step-01b-continue.md](../../../skills/bmad-story-automator/steps-c/step-01b-continue.md) +- [skills/bmad-story-automator/steps-c/step-02-preflight.md](../../../skills/bmad-story-automator/steps-c/step-02-preflight.md) +- [skills/bmad-story-automator/steps-c/step-02a-preflight-config.md](../../../skills/bmad-story-automator/steps-c/step-02a-preflight-config.md) +- [skills/bmad-story-automator/steps-c/step-02b-preflight-finalize.md](../../../skills/bmad-story-automator/steps-c/step-02b-preflight-finalize.md) +- [skills/bmad-story-automator/steps-v/](../../../skills/bmad-story-automator/steps-v) +- [skills/bmad-story-automator/steps-e/](../../../skills/bmad-story-automator/steps-e) + +## Implementation Steps + +1. Extend current create/dev deterministic smokes or add a new runner for create startup, resume, validate, and edit mode contracts. Prefer a fast `smoke:modes` temp-fixture runner for default verify, with prepared-repo checks kept explicit. +2. Cover preflight breadth: multi-story selection, explicit IDs, invalid ranges, complexity matrix persistence, and at least one richer agent-config variant. +3. Cover create startup guard cases from `step-01-init`: stop-hook ok, changed, pending-trust, and failure payloads; existing-state detection; sprint-status present; sprint-status missing precondition/abort where helper-backed; and init log creation. +4. Cover resume by seeding incomplete state files and asserting explicit path handling, no-path latest incomplete discovery, no-incomplete fallback to fresh create, `state-summary`, `sprint-compare`, route selection by `currentStep`, and marker creation only after resume. +5. Cover resume menu branches deterministically where helper-backed: View/inspect, Modify/edit route, Start Over/fresh route, Abort/no mutation, and Resume/continue route. +6. Cover marker lifecycle using dynamic marker path from `orchestrator-helper marker path`: gitignore entry, marker JSON shape, heartbeat update, stop-hook block/allow cases, and marker removal. Do not hard-code `.claude`. +7. Assert state file fields and artifact outputs directly: frontmatter, `status`, `currentStory`, `currentStep`, `agentsFile`, `complexityFile`, policy snapshot path/hash, progress rows, action log deltas, reports, state docs, complexity JSON, agents file, dev logs, marker JSON, and `.gitignore` entries. +8. Cover source-of-truth mismatch cases for story-file status versus `sprint-status.yaml`; surface mismatches rather than treating marker absence or command exit status as completion proof. +9. Cover validate mode helpers: `validate-state --help`, `list-sessions --help`, `derive-project-slug --help`, structure issue reporting, progress-row consistency, and compact report output. +10. Cover edit mode helpers and branches where helper-backed: status, range, text context, AI command, artifact path updates, save, discard, edit-more, post-edit resume, post-edit validate, and exit route hints without requiring interactive input. Keep fully interactive prompt behavior workflow-only unless a deterministic helper exists. +11. Update [gate-map.md](./gate-map.md) with create-startup/create/dev/resume/validate/edit gates. +12. When fixture setup writes story files or `sprint-status.yaml`, label it as simulated child workflow output. Do not make tests imply the orchestrator owns sprint-status mutation. + +## Verification + +- Run `npm run smoke:run`. +- Run `npm run smoke:dev-loop`. +- Run the new resume/validate/edit smoke command. +- Run `npm run test:python`. +- Run `git diff --check`. + +## Exit Criteria + +- Deterministic smokes cover all public automator modes: create startup, create/preflight/dev, resume, validate, and edit. +- Create startup guard cannot pass if stop-hook setup, existing-state handling, or required sprint-status detection regresses. +- Marker lifecycle uses the helper-resolved path and does not hard-code `.claude`. +- Validate/edit coverage exercises helper contracts without interactive prompts. +- Fast mode coverage is suitable for `npm run verify`; heavier prepared-repo checks remain explicit unless Phase 06 promotes them. +- Phase 04 handoff entry appended. + +## Implementation Notes Requirements + +Record any behavior that remains interactive-only, any mode that cannot be fully deterministic, and any marker-root tradeoff in [implementation-notes.md](./implementation-notes.md). + +## Handoff Requirements + +Append a Phase 04 entry to [handoff-log.md](./handoff-log.md) with commands run, seeded state files, marker paths, validation/edit gaps, and next recommended command. diff --git a/docs/plans/automator-deterministic-smoke-coverage/05-automate-review-finish-retro-coverage.md b/docs/plans/automator-deterministic-smoke-coverage/05-automate-review-finish-retro-coverage.md new file mode 100644 index 00000000..e4445ad6 --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/05-automate-review-finish-retro-coverage.md @@ -0,0 +1,66 @@ +# Phase 05 - Automate Review Finish Retro Coverage + +## Clean Context Start + +Before doing this phase, read [README.md](./README.md), this phase file, [TODO/phase-05.md](./TODO/phase-05.md), [gate-map.md](./gate-map.md), [implementation-notes.md](./implementation-notes.md), and relevant earlier entries in [handoff-log.md](./handoff-log.md). + +Do not read later phase files or later TODO files as acceptance criteria for this phase. + +## Goal + +Add deterministic coverage for the second half of the automator lifecycle: automate, review loop, commit/finalize, epic completion, retrospective routing, execution complete, and wrapup. + +This phase is release-blocking. Existing deterministic smokes stop before the review-to-done and finish-loop gates that are most likely to create false-positive smoke readiness. + +## Inputs + +- [skills/bmad-story-automator/steps-c/step-03a-execute-review.md](../../../skills/bmad-story-automator/steps-c/step-03a-execute-review.md) +- [skills/bmad-story-automator/steps-c/step-03b-execute-finish.md](../../../skills/bmad-story-automator/steps-c/step-03b-execute-finish.md) +- [skills/bmad-story-automator/steps-c/step-03c-execute-complete.md](../../../skills/bmad-story-automator/steps-c/step-03c-execute-complete.md) +- [skills/bmad-story-automator/steps-c/step-04-wrapup.md](../../../skills/bmad-story-automator/steps-c/step-04-wrapup.md) +- [skills/bmad-story-automator/data/code-review-loop.md](../../../skills/bmad-story-automator/data/code-review-loop.md) +- [skills/bmad-story-automator/data/retrospective-automation.md](../../../skills/bmad-story-automator/data/retrospective-automation.md) +- [skills/bmad-story-automator/data/orchestration-policy.json](../../../skills/bmad-story-automator/data/orchestration-policy.json) + +## Implementation Steps + +1. Add a deterministic finish-loop smoke for two stories that begins after dev completion and drives automate/review/finalize transitions. +2. Cover automate success and non-blocking failure/skip rows. +3. Cover review completion through `verify-code-review` or `verify-step review`, including incomplete review retry/escalation payloads where deterministic. +4. Cover commit/finalize without depending on real product implementation: create a controlled change, run `commit-story`, assert JSON, commit metadata, and progress-row `git-commit=done`. +5. Add host-repo commit isolation sentinel before running finish-loop: record host HEAD/status, create or identify a host-only uncommitted sentinel, run finish-loop against `.smoke/gunz` or an explicit temp smoke repo, then assert host HEAD/status and sentinel state are unchanged while only the smoke repo receives the controlled commit. +6. Add a target-repo safety guard: the finish-loop runner should refuse `commit-story --repo` unless the repo path is under the configured smoke workspace, except behind an explicit unsafe override reserved for manual debugging. +7. Cover sprint-status and story-file fallback rules for finalization. +8. Cover epic completion detection using `check-epic-complete`, `get-epic-stories`, and `sprint-status check-epic`. +9. Cover retrospective agent resolution and `build-cmd retro`; use runner/fake monitor output so retrospective failure is recorded as skipped and non-blocking. +10. Add a seeded multi-epic deterministic fixture and bind it to `npm run smoke:finish-loop -- --scenario multi-epic`, or make `npm run smoke:finish-loop` always include this scenario. Selected stories must cross at least two epics; Epic 1 completes before Epic 2; Epic 1 retrospective triggers inside the execution loop; Epic 1 retrospective failure records `skipped` and does not block Epic 2; Epic 2 continues to completion; state records separate `retrospectives.epic-*`; wrapup occurs only after all selected stories complete. +11. Assert artifact and source-of-truth outputs: final state fields, progress rows, action log deltas, review fallback payloads, incomplete review diagnostics, commit SHA, smoke repo `git log`, story-file status, sprint-status status, retro state entries, summary report, marker removal, and host `.gitignore`/status invariants. +12. Cover `step-03c` and wrapup state transitions: `EXECUTION_COMPLETE`, `COMPLETE`, summary metrics, and marker removal. +13. Update [gate-map.md](./gate-map.md) with automate/review/finalize/commit-isolation/single-epic-retro/multi-epic-retro/wrapup gates. + +## Verification + +- Run the new finish-loop smoke command. +- Run the multi-epic scenario command if it is split: `npm run smoke:finish-loop -- --scenario multi-epic`. +- Run `npm run smoke:dev-loop`. +- Run `npm run test:python`. +- Run `git log --oneline -3` inside the controlled smoke repo if commit-story creates commits there. +- Run `git diff --check`. + +## Exit Criteria + +- Deterministic smoke covers the full create-to-wrapup lifecycle except live LLM code quality. +- Review incomplete, single-epic retrospective failure, and multi-epic non-blocking retrospective failure are proven non-happy-path contracts. +- Multi-epic lifecycle proves per-epic retrospective timing and independent continuation to later epics. +- The multi-epic scenario is wired into `smoke:finish-loop` or has a concrete split command that Phase 06 includes in `smoke:deterministic-full`. +- Commit behavior is isolated to the smoke repo and proven with a host HEAD/status sentinel so it cannot commit host repo changes by accident. +- Finish-loop commit/finalize refuses unsafe repo targets unless an explicit manual override is supplied. +- Phase 05 handoff entry appended. + +## Implementation Notes Requirements + +Record any commit isolation constraints, review-loop limitations, retrospective skip semantics, and source-of-truth fallback decisions in [implementation-notes.md](./implementation-notes.md). + +## Handoff Requirements + +Append a Phase 05 entry to [handoff-log.md](./handoff-log.md) with commands run, smoke repo SHAs, state files, review/retro outcomes, blockers, and next recommended command. diff --git a/docs/plans/automator-deterministic-smoke-coverage/06-gate-integration-and-readiness-review.md b/docs/plans/automator-deterministic-smoke-coverage/06-gate-integration-and-readiness-review.md new file mode 100644 index 00000000..a4e808af --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/06-gate-integration-and-readiness-review.md @@ -0,0 +1,60 @@ +# Phase 06 - Gate Integration And Readiness Review + +## Clean Context Start + +Before doing this phase, read [README.md](./README.md), this phase file, [TODO/phase-06.md](./TODO/phase-06.md), [implementation-notes.md](./implementation-notes.md), [gate-map.md](./gate-map.md), and relevant earlier entries in [handoff-log.md](./handoff-log.md). + +Do not read later phase files or later TODO files as acceptance criteria for this phase. + +## Goal + +Wire the deterministic smoke suite into the right repo gates, then run a clean-context coverage review to verify the plan and implementation cover the automator's full functionality. + +Default verification should stay fast, local, and deterministic. Prepared-repo reset/network gates should be explicit pre-release or nightly checks unless CI proves they are stable enough for every `npm run verify`. + +## Inputs + +- [package.json](../../../package.json) +- [gate-map.md](./gate-map.md) +- All phase handoff entries +- Existing test commands: `test:python`, `test:cli`, `pack:dry-run`, `test:smoke`, `smoke:run`, `smoke:dev-loop`, and new smoke commands from earlier phases. + +## Implementation Steps + +1. Decide which deterministic smokes belong in `npm run verify` and which remain heavier manual/pre-live gates. Target default verify shape: `test:python && version:check && pack:assert && test:cli && smoke:contracts && smoke:modes && test:smoke`. +2. Update `package.json` scripts so fast deterministic contract gates are always run by `verify`; keep external repo reset-heavy gates explicit if they are too slow for default verify. +3. Add or confirm an explicit full prepared-repo wrapper target, for example `smoke:deterministic-full`, that runs `smoke:prepare -- --reset`, `smoke:run`, `smoke:dev-loop`, and `smoke:finish-loop`. +4. Ensure every gate in [gate-map.md](./gate-map.md) has owner/location, command, reset/cache policy, CI status, pass/fail signal, failure diagnostic, and risk note. +5. Document live/manual smoke boundaries separately from deterministic gates: provider auth, rate limits, trust prompts, provider outages, semantic quality of generated stories/code/reviews/retrospectives, moving registry/git inputs without pin/assert, and conversational UX beyond helper-backed effects. +6. Run the full local verification set selected by the gate map. +7. Run `general-subagent-review-loop` against the completed implementation and gate map with at least: + - workflow lifecycle reviewer + - runtime/helper contract reviewer + - package/install determinism reviewer + - validate/edit/resume reviewer +8. Triage reviewer findings; fix credible P0/P1 gaps or explicitly document blockers. +9. Update [implementation-notes.md](./implementation-notes.md), [handoff-log.md](./handoff-log.md), and [gate-map.md](./gate-map.md) with final readiness. + +## Verification + +- Run `npm run verify`. +- Run every additional deterministic smoke command marked required in [gate-map.md](./gate-map.md). +- Run `git diff --check`. +- Run clean-context sub-agent review and record results in [handoff-log.md](./handoff-log.md). + +## Exit Criteria + +- Gate map is complete and no required deterministic gate is missing a command or pass/fail signal. +- `verify` includes the intended fast deterministic gates. +- `smoke:deterministic-full` or equivalent explicit prepared-repo release gate exists and is documented separately from default verify. +- Manual/live boundaries are documented so deterministic smoke readiness is not misrepresented as LLM quality or provider readiness. +- Clean-context review is P0/P1 clean or blockers are explicitly documented. +- Phase 06 handoff entry appended. + +## Implementation Notes Requirements + +Record gate inclusion tradeoffs, slow-gate exclusions, CI assumptions, and final coverage risks in [implementation-notes.md](./implementation-notes.md). + +## Handoff Requirements + +Append a Phase 06 entry to [handoff-log.md](./handoff-log.md) with commands run, reviewer roles/results, unresolved risks, and final recommended merge/readiness action. diff --git a/docs/plans/automator-deterministic-smoke-coverage/README.md b/docs/plans/automator-deterministic-smoke-coverage/README.md new file mode 100644 index 00000000..a14ef1cd --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/README.md @@ -0,0 +1,108 @@ +# Automator Deterministic Smoke Coverage Plan + +## Purpose + +Build a deterministic smoke coverage suite that verifies the Story Automator control plane before any live LLM smoke run. The suite should prove package/install determinism, helper contracts, create/dev/review/finalize lifecycle behavior, resume/validate/edit modes, marker safety, and gate wiring. + +## Historical Baseline Findings + +These findings describe the 2026-06-02 baseline that launched this plan. See the gate map and handoff log for current implementation status. + +- Current deterministic smokes cover prepared `gunz`, epic/story parsing, state creation, story artifact creation, and a two-story dev status transition. +- Current smokes do not yet cover automate, review, commit/finalize, single-epic or multi-epic retrospective behavior, wrapup, validate mode, edit mode, resume routing, monitor terminal states, parser subprocess contracts, or package version alignment. +- `smoke:prepare` pins the `gunz` repo commit but installs `bmad-method@next`; that is a moving input unless pinned or recorded and asserted. +- Workflow metadata already has a likely stale version surface: package/runtime surfaces report `1.15.0`, while `skills/bmad-story-automator/workflow.md` reports `1.12.0`. +- At baseline, `npm run verify` did not include the new deterministic external smokes. +- Oracle review on 2026-06-02 confirmed the plan shape is sound but release readiness must block on Phase 02 package/prep identity, Phase 03 helper JSON contracts, and Phase 05 review/finish-loop coverage. Existing smoke runners alone are not enough to call the repo smoke-ready. + +## Oracle-Applied Architecture + +Use a layered deterministic control-plane suite instead of one giant automator smoke: + +1. Fast in-repo contract gates: fake subprocesses, runner mode, temp fixtures, helper JSON assertions, state/policy/success-verifier matrices. +2. Package/install identity gates: `npm pack --dry-run --json`, tarball identity, selected installed-file checksums, version surface alignment, and forbidden generated files. +3. Local no-network mode fixtures: resume, validate, edit, create startup, marker lifecycle, and source-of-truth mismatch checks using temp BMAD-style projects. +4. Explicit release smokes: `smoke:prepare`, `smoke:run`, and `smoke:dev-loop` against prepared `.smoke/gunz`; `smoke:finish-loop` against a temp git-backed smoke repo for host commit isolation; `smoke:deterministic-full` runs both sets. + +Default `npm run verify` should eventually run only fast deterministic gates. Prepared-repo reset/network gates remain explicit unless CI provides stable cache and time budget. + +## Release-Blocking Priorities + +- Phase 02 is release-blocking because a prepared repo can pass with a stale same-shape install unless tarball identity and installed checksums are asserted. +- Phase 03 is release-blocking because higher-level runners rely on helper contracts that can exit with structured failure payloads rather than simple process failure. +- Phase 05 is release-blocking because review completion cannot be inferred from a child process exiting; it must be verified through sprint status or story-file fallback, then finalize/retro/wrapup must be proven. +- Host mutation isolation is release-blocking. Any finish-loop runner must prove host HEAD/status are unchanged and should refuse commit/finalize operations unless the target repo is under the smoke workspace, except behind an explicit unsafe override. + +## Assertion Contract + +Every deterministic smoke should assert structured outputs and selected file contents, not only command exit status. + +- State assertions: frontmatter fields, `status`, `currentStory`, `currentStep`, `agentsFile`, `complexityFile`, policy snapshot path/hash, progress rows, and action log deltas. +- Source-of-truth assertions: story-file status, sprint-status status, source mismatch diagnostics, review fallback behavior, incomplete review payloads, and `sprint_status_not_updated` notes where applicable. +- Artifact assertions: reports, state docs, complexity JSON, agents file, dev logs, commit SHA, marker JSON, `.gitignore` entries, and selected checksums. Keep these narrow; do not introduce broad whole-repo snapshots. + +## Manual And Live Boundaries + +Keep these out of deterministic `verify`: + +- Live provider/auth behavior: Claude/Codex auth, rate limits, trust prompts, outages, provider-specific reasoning, and whether a real LLM follows every BMAD step. +- Live implementation quality: whether generated stories, implementation patches, reviews, and retrospectives are semantically good. +- Moving network surfaces unless pinned or recorded/asserted: `bmad-method@next`, registry `next`, external git refs, npm registry state. +- Interactive conversational UX beyond helper-backed branch effects, state mutations, route hints, and file outputs. + +## Recommended Implementation Order + +1. Version/input determinism. +2. Package identity and installed manifest checks. +3. Helper contracts. +4. Existing smoke marker path fixes. +5. Resume/validate/edit temp-fixture smokes. +6. Create/dev breadth. +7. Finish-loop smoke. +8. Multi-epic fixture. +9. Gate integration and clean-context review. + +## Assumptions + +- Target repo: the repository checkout containing this plan packet. +- Plan root: `docs/plans/automator-deterministic-smoke-coverage/`. +- External target remains the prepared `.smoke/gunz` workspace unless a later phase intentionally broadens it. +- Live LLM implementation quality is out of scope for deterministic smoke; deterministic gates verify the automator control plane and source-of-truth transitions. +- Use this fact status taxonomy when classifying coverage: `fact`, `gap`, `blocked`, `stale`, `spec-only`. + +## Phase Files + +- [Phase 01 - Baseline And Version Determinism](./01-baseline-and-version-determinism.md) +- [Phase 02 - Package And Prepared Repo Contracts](./02-package-and-prepared-repo-contracts.md) +- [Phase 03 - Runtime Helper Contract Smokes](./03-runtime-helper-contract-smokes.md) +- [Phase 04 - Create Dev Resume Validate Edit Coverage](./04-create-dev-resume-validate-edit-coverage.md) +- [Phase 05 - Automate Review Finish Retro Coverage](./05-automate-review-finish-retro-coverage.md) +- [Phase 06 - Gate Integration And Readiness Review](./06-gate-integration-and-readiness-review.md) + +## Supporting Files + +- [Coverage baseline](./coverage-baseline.md) +- [TODO index](./TODO.md) +- [Gate map](./gate-map.md) +- [Implementation notes](./implementation-notes.md) +- [Handoff log](./handoff-log.md) + +## Clean Context Agent Protocol + +Every phase agent must read this `README.md`, [gate-map.md](./gate-map.md), its assigned phase file, only its assigned phase TODO file, [implementation-notes.md](./implementation-notes.md), and relevant earlier entries in [handoff-log.md](./handoff-log.md) before starting. Append a new handoff entry before ending. + +Do not read later phase files or later TODO files as acceptance criteria for the current phase. + +## Implementation Notes Protocol + +Every phase agent must keep [implementation-notes.md](./implementation-notes.md) current with user-facing decisions, spec gaps, required changes, tradeoffs, deviations, and notable risks. Use [handoff-log.md](./handoff-log.md) only for next-agent continuity. + +## Gate Map Protocol + +Every phase that creates, changes, or promotes deterministic gates must update [gate-map.md](./gate-map.md). Final review and smoke readiness must consume the gate map instead of rediscovering commands from scattered notes. + +## Smoke Implementation Discipline + +Keep new smoke runners small. Extract shared fixture creation, helper invocation, JSON assertions, package identity checks, fake subprocesses, state seeding, marker resolution, and git sentinels into focused modules under `scripts/smoke_prep/` or a new `scripts/smoke_lib/` package instead of duplicating that logic across runners. + +When deterministic fixtures write story files or `sprint-status.yaml`, label that as fixture setup that simulates child workflow output. The automator control plane must not claim ownership of sprint-status writes. diff --git a/docs/plans/automator-deterministic-smoke-coverage/TODO.md b/docs/plans/automator-deterministic-smoke-coverage/TODO.md new file mode 100644 index 00000000..ffe5825d --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/TODO.md @@ -0,0 +1,14 @@ +# Automator Deterministic Smoke Coverage TODO + +## Purpose + +This file is an index for phase-scoped TODO files. Clean-context agents must read only their assigned phase TODO file, not later TODO files. + +## Phase TODOs + +- [Phase 01 - Baseline And Version Determinism](./TODO/phase-01.md) +- [Phase 02 - Package And Prepared Repo Contracts](./TODO/phase-02.md) +- [Phase 03 - Runtime Helper Contract Smokes](./TODO/phase-03.md) +- [Phase 04 - Create Dev Resume Validate Edit Coverage](./TODO/phase-04.md) +- [Phase 05 - Automate Review Finish Retro Coverage](./TODO/phase-05.md) +- [Phase 06 - Gate Integration And Readiness Review](./TODO/phase-06.md) diff --git a/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-01.md b/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-01.md new file mode 100644 index 00000000..5b4b2ad9 --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-01.md @@ -0,0 +1,15 @@ +# Phase 01 TODO - Baseline And Version Determinism + +## Scope + +Use this checklist only for Phase 01. Do not use later phase TODO files as acceptance criteria. + +## Checklist + +- [x] Read [README.md](../README.md), [01-baseline-and-version-determinism.md](../01-baseline-and-version-determinism.md), this TODO file, [gate-map.md](../gate-map.md), and relevant earlier entries in [handoff-log.md](../handoff-log.md). +- [x] Keep [implementation-notes.md](../implementation-notes.md) current while implementing. +- [x] Create the automator coverage baseline and classify facts with `fact`, `gap`, `blocked`, `stale`, or `spec-only`. +- [x] Add or update deterministic version/input checks. +- [x] Update [gate-map.md](../gate-map.md) for Phase 01 gates. +- [x] Run the phase verification checks. +- [x] Append the Phase 01 handoff entry before ending. diff --git a/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-02.md b/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-02.md new file mode 100644 index 00000000..fe134519 --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-02.md @@ -0,0 +1,20 @@ +# Phase 02 TODO - Package And Prepared Repo Contracts + +## Scope + +Use this checklist only for Phase 02. Do not use later phase TODO files as acceptance criteria. + +## Checklist + +- [x] Read [README.md](../README.md), [02-package-and-prepared-repo-contracts.md](../02-package-and-prepared-repo-contracts.md), this TODO file, [gate-map.md](../gate-map.md), and relevant earlier entries in [handoff-log.md](../handoff-log.md). +- [x] Keep [implementation-notes.md](../implementation-notes.md) current while implementing. +- [x] Add package dry-run JSON assertions. +- [x] Add packed tarball identity assertions for name, version, filename, integrity or shasum. +- [x] Add prepared-repo installed manifest verification. +- [x] Add installed-version and installed-file checksum checks against the current tarball. +- [x] Ensure package checks use `npm pack --dry-run --json` and `npm pack --json` data, not terminal-output-only assertions. +- [x] Make `pack:assert` suitable for the future default `npm run verify`. +- [x] Classify unsupported install roots explicitly. +- [x] Update [gate-map.md](../gate-map.md) for Phase 02 gates. +- [x] Run the phase verification checks. +- [x] Append the Phase 02 handoff entry before ending. diff --git a/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-03.md b/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-03.md new file mode 100644 index 00000000..4a2500b1 --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-03.md @@ -0,0 +1,21 @@ +# Phase 03 TODO - Runtime Helper Contract Smokes + +## Scope + +Use this checklist only for Phase 03. Do not use later phase TODO files as acceptance criteria. + +## Checklist + +- [x] Read [README.md](../README.md), [03-runtime-helper-contract-smokes.md](../03-runtime-helper-contract-smokes.md), this TODO file, [gate-map.md](../gate-map.md), and relevant earlier entries in [handoff-log.md](../handoff-log.md). +- [x] Keep [implementation-notes.md](../implementation-notes.md) current while implementing. +- [x] Add deterministic parse-output success and fail-closed matrix checks. +- [x] Add monitor-session terminal state and diagnostics matrix checks. +- [x] Add runner lifecycle and edge-state checks. +- [x] Add build-cmd branch, safety flag, model, override, and negative checks. +- [x] Add state-update, runtime-policy snapshot creation/failure, and success-verifier edge checks. +- [x] Add marker/root resolution helper checks for `.agents`, `.codex`, and `.claude`; prove smokes do not hard-code `.claude`. +- [x] Add status/source-of-truth helper checks for story-file status, sprint-status status, mismatch diagnostics, story-file fallback, and `sprint_status_not_updated`. +- [x] Keep `smoke:contracts` local/fast with fake subprocesses and `SA_TMUX_RUNTIME=runner` so it can run in default `npm run verify`. +- [x] Update [gate-map.md](../gate-map.md) with named rows or stable IDs for each Phase 03 helper contract family. +- [x] Run the phase verification checks. +- [x] Append the Phase 03 handoff entry before ending. diff --git a/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-04.md b/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-04.md new file mode 100644 index 00000000..39cafc7d --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-04.md @@ -0,0 +1,24 @@ +# Phase 04 TODO - Create Dev Resume Validate Edit Coverage + +## Scope + +Use this checklist only for Phase 04. Do not use later phase TODO files as acceptance criteria. + +## Checklist + +- [x] Read [README.md](../README.md), [04-create-dev-resume-validate-edit-coverage.md](../04-create-dev-resume-validate-edit-coverage.md), this TODO file, [gate-map.md](../gate-map.md), and relevant earlier entries in [handoff-log.md](../handoff-log.md). +- [x] Keep [implementation-notes.md](../implementation-notes.md) current while implementing. +- [x] Add create startup guard checks for stop-hook states, existing-state detection, sprint-status present, and sprint-status missing precondition. +- [x] Prefer temp BMAD-style fixtures for `smoke:modes`; keep prepared `.smoke/gunz` mode checks explicit unless Phase 06 promotes them. +- [x] Extend create/dev smoke coverage for preflight breadth and agent config variants. +- [x] Add resume state discovery, sprint comparison, menu branch, and route checks. +- [x] Add marker lifecycle checks. +- [x] Resolve marker path dynamically through helper output; do not hard-code `.claude`. +- [x] Assert direct state/artifact fields: frontmatter, status, current story/step, agents/complexity files, policy snapshot path/hash, progress rows, action log deltas, reports, state docs, marker JSON, `.gitignore`, and selected artifacts. +- [x] Add source-of-truth mismatch cases for story-file status versus `sprint-status.yaml`; surface mismatches instead of accepting marker absence or exit status as completion proof. +- [x] Label fixture writes to story files or `sprint-status.yaml` as simulated child workflow output. +- [x] Add validate mode helper and report checks. +- [x] Add edit mode helper, menu branch, save/discard/edit-more, and route checks. +- [x] Update [gate-map.md](../gate-map.md) for Phase 04 gates. +- [x] Run the phase verification checks. +- [x] Append the Phase 04 handoff entry before ending. diff --git a/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-05.md b/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-05.md new file mode 100644 index 00000000..59e7c5c6 --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-05.md @@ -0,0 +1,20 @@ +# Phase 05 TODO - Automate Review Finish Retro Coverage + +## Scope + +Use this checklist only for Phase 05. Do not use later phase TODO files as acceptance criteria. + +## Checklist + +- [x] Read [README.md](../README.md), [05-automate-review-finish-retro-coverage.md](../05-automate-review-finish-retro-coverage.md), this TODO file, [gate-map.md](../gate-map.md), and relevant earlier entries in [handoff-log.md](../handoff-log.md). +- [x] Keep [implementation-notes.md](../implementation-notes.md) current while implementing. +- [x] Add deterministic automate success and skip coverage. +- [x] Add review completion and incomplete-review coverage. +- [x] Add commit/finalize coverage in the smoke repo only, including host HEAD/status sentinel proof. +- [x] Add a repo-target safety guard so finish-loop commit/finalize refuses host repo paths unless an explicit unsafe override is supplied. +- [x] Add single-epic completion and retrospective route coverage. +- [x] Add multi-epic retrospective timing, skipped failure, and continuation coverage. +- [x] Add execution-complete and wrapup coverage. +- [x] Update [gate-map.md](../gate-map.md) for Phase 05 gates. +- [x] Run the phase verification checks. +- [x] Append the Phase 05 handoff entry before ending. diff --git a/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-06.md b/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-06.md new file mode 100644 index 00000000..37f496c5 --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/TODO/phase-06.md @@ -0,0 +1,18 @@ +# Phase 06 TODO - Gate Integration And Readiness Review + +## Scope + +Use this checklist only for Phase 06. Do not use later phase TODO files as acceptance criteria. + +## Checklist + +- [x] Read [README.md](../README.md), [06-gate-integration-and-readiness-review.md](../06-gate-integration-and-readiness-review.md), this TODO file, [gate-map.md](../gate-map.md), and relevant earlier entries in [handoff-log.md](../handoff-log.md). +- [x] Keep [implementation-notes.md](../implementation-notes.md) current while implementing. +- [x] Decide default `verify` gates versus heavier explicit smoke gates. +- [x] Target default `verify`: `test:python`, `version:check`, `pack:assert`, `test:cli`, `smoke:contracts`, `smoke:modes`, and `test:smoke`. +- [x] Wire selected fast deterministic gates into `verify`. +- [x] Add or confirm explicit prepared-repo release wrapper, such as `smoke:deterministic-full`, separate from default `verify`. +- [x] Complete all gate-map fields. +- [x] Run final verification commands. +- [x] Run clean-context sub-agent review and triage findings. +- [x] Append the Phase 06 handoff entry before ending. diff --git a/docs/plans/automator-deterministic-smoke-coverage/coverage-baseline.md b/docs/plans/automator-deterministic-smoke-coverage/coverage-baseline.md new file mode 100644 index 00000000..fb497b75 --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/coverage-baseline.md @@ -0,0 +1,39 @@ +# Automator Coverage Baseline + +## Scope + +Phase 01 source-of-truth baseline for deterministic smoke coverage. Status values: +`fact`, `gap`, `blocked`, `stale`, and `spec-only`. + +## Baseline + +| Surface | Source | Current deterministic gate | Status | Notes | +| --- | --- | --- | --- | --- | +| create mode route | `workflow.md`, `steps-c/` | `npm run smoke:run`; `npm run smoke:modes` | fact | Prepared `.smoke/gunz` create smoke verifies story `1.1`; Phase 04 mode smoke adds multi-story temp state setup, sprint-status present/missing checks, stop-hook guard states, and init-log proof. | +| resume mode route | `workflow.md`, `steps-c/step-01b-continue.md` | `npm run smoke:modes` | fact | Phase 04 temp fixture covers explicit path summary, latest incomplete discovery, no-incomplete fresh-create fallback, sprint compare, workflow-derived menu branch names/route hints, view action-log summary, start-over backup simulation, abort state update, and marker creation timing. | +| validate mode route | `workflow.md`, `steps-v/` | `npm run smoke:modes` | fact | Phase 04 covers helper help contracts, happy-path state validation, structure issue reporting, session inventory, sprint compare, progress-row metrics, and compact mode report output. | +| edit mode route | `workflow.md`, `steps-e/` | `npm run smoke:modes` | fact | Phase 04 covers workflow-derived menu names/post-edit route hints plus helper-backed status/range/current-story/AI-command/artifact-path/text saves, discard rollback, and edit-more state update. Interactive prompts remain workflow-only. | +| create-story policy step | `orchestration-policy.json` step `create` | `npm run smoke:run` | fact | Existing smoke asserts generated story artifact for one story. | +| dev-story policy step | `orchestration-policy.json` step `dev` | `npm run smoke:dev-loop`; `npm run smoke:modes` | fact | Dev-loop smoke covers two-story transition with parsed fixture output; Phase 04 mode smoke adds complexity-file persistence and richer per-task agent-config state coverage. | +| automate policy step | `orchestration-policy.json` step `auto` | `npm run smoke:finish-loop` | fact | Phase 05 temp git-backed finish-loop smoke covers automate `done` and non-blocking `skip` progress rows. | +| review policy step | `orchestration-policy.json` step `review` | `npm run smoke:contracts`; `npm run smoke:finish-loop` | fact | Unit tests cover verifier pieces; Phase 05 smoke covers incomplete review diagnostics and verified review completion from story/sprint source truth. | +| commit/finalize | `steps-c/step-03b-finalize.md` | `npm run smoke:finish-loop` | fact | Phase 05 smoke commits controlled story changes inside a temp smoke repo, records commit SHAs, and proves host HEAD/status isolation plus unsafe-host target rejection. | +| retrospective policy step | `orchestration-policy.json` step `retro` | `npm run smoke:finish-loop` | fact | Phase 05 multi-epic fixture resolves retro agent/build command and records per-epic skipped retrospectives as non-blocking before later epic continuation. | +| wrapup | `steps-c/step-04-wrapup.md`, `data/wrapup-templates.md` | `npm run smoke:finish-loop` | fact | Phase 05 smoke covers `EXECUTION_COMPLETE`, `COMPLETE`, summary metrics, learnings file creation, final state validation, and marker removal. | +| parser subprocess contracts | `data/parse/*.json`, helper CLI | `npm run smoke:contracts` | fact | Phase 03 covers success, missing/empty output, state flag/policy/contract failures, subprocess timeout/nonzero/no-json, and schema-invalid JSON. | +| monitor terminal states | helper CLI, tmux runtime | `npm run smoke:contracts` | fact | Phase 03 covers completed, incomplete, crashed, stuck, timeout, invalid persisted session-state diagnostics on not_found, invalid options, and runner success/crash/edge mapping. | +| build-cmd helper branches | `tmux-wrapper build-cmd` | `npm run smoke:contracts` | fact | Phase 03 covers Codex safety flags, `AI_COMMAND`, Claude/model quoting, unknown step, state-file, and invalid policy branches. | +| marker/root resolution | runtime layout helper | `npm run test:python` | fact | Existing unit tests cover runtime-layout helper behavior. | +| package version surfaces | package/plugin/module/Python/workflow metadata | `npm run version:check` | fact | Workflow frontmatter was stale at `1.12.0`; Phase 01 aligned it to `1.15.0`. | +| smoke repo input | `scripts/smoke_prep/config.py`, `gunz.py` | `npm run smoke:input-check` | fact | `gunz` repo is pinned by full SHA; command resolves `bmad-method@next` version/integrity. | +| package/install identity | package tarball and prepared repo install | `npm run pack:assert`; `.smoke/PACKAGE_IDENTITY.json`; `.smoke/INSTALLED_AUTOMATOR_MANIFEST.json` | fact | Phase 02 replaced terminal-output-only dry run with JSON identity, tarball SHA256, and installed-file checksum assertions. | +| installed BMAD Method input | `scripts/smoke_prep/inputs.py` | `.smoke/SMOKE_INPUTS.json` from `smoke:prepare` | fact | Moving npm dist-tag remains explicit and recorded with resolved version/integrity per prep run. | +| prepared `.claude/skills` install root | `install.sh`, `.smoke/gunz/.claude/skills` | `npm run smoke:prepare -- --reset` | fact | Phase 02 installed manifest verifies dependency entrypoints, package identity, and selected copied-file checksums. | +| prepared `.agents/skills` install root | `install.sh`, `.smoke/gunz/.agents/skills` | `npm run smoke:prepare -- --reset` | spec-only | Installer supports the root, but BMAD prep with `--tools claude-code` does not create `.agents/skills` dependency entrypoints; installed manifest reports this as `unsupported`. | +| prepared `.codex/skills` install root | `install.sh`, `.smoke/gunz/.codex/skills` | `npm run smoke:prepare -- --reset` | spec-only | Installer supports the root, but BMAD prep with `--tools claude-code` does not create `.codex/skills` dependency entrypoints; installed manifest reports this as `unsupported`. | +| live LLM quality | provider/auth/runtime behavior | none | blocked | Explicitly outside deterministic smoke gates. | + +## Deferred Work + +No Phase 01 deferred-work items. Remaining `gap` and `blocked` rows are already +owned by Phase 02-06 TODOs and will be validated or implemented in those phases. diff --git a/docs/plans/automator-deterministic-smoke-coverage/gate-map.md b/docs/plans/automator-deterministic-smoke-coverage/gate-map.md new file mode 100644 index 00000000..c8e58c3f --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/gate-map.md @@ -0,0 +1,31 @@ +# Automator Deterministic Smoke Coverage Gate Map + +| Gate | Owner/location | Local command | Env/reset/cache policy | CI status | Pass/fail signal | Failure diagnostic | Blocked/risk note | +| --- | --- | --- | --- | --- | --- | --- | --- | +| Version and metadata alignment | Phase 01 / `scripts/check-version-alignment.py` | `npm run version:check` | No external reset. Reads repo metadata only. | Not CI-backed yet | Fails on mismatched package/plugin/module/Python/workflow versions | Command output names mismatched files and values | Workflow version aligned to `1.15.0` in Phase 01. | +| Smoke input determinism | Phase 01 / `scripts/check-smoke-inputs.py`, `scripts/smoke_prep/inputs.py`, and prep report | `npm run smoke:input-check`; `npm run smoke:prepare -- --reset` writes `.smoke/SMOKE_INPUTS.json` | No workspace reset for check; npm registry lookup required for `bmad-method@next` identity. Prep installs the resolved `bmad-method@` from the manifest. | Not CI-backed yet | Fails when repo pin is malformed, BMAD Method spec changes without contract update, or npm identity lacks version/integrity | Check output and `.smoke/SMOKE_INPUTS.json` | `bmad-method@next` remains moving, but each prep records resolved version/integrity and installs the resolved version. | +| Package content assertion | Phase 02 / `scripts/assert-package-contracts.py`, `scripts/smoke_prep/package_contracts.py` | `npm run pack:assert` from `npm pack --dry-run --json` | No workspace reset. Creates temp tarball only. | Not CI-backed yet | Required files present, executable modes preserved, forbidden generated paths absent | JSON package file list diff | Should run in `verify`. | +| Packed tarball identity | Phase 02 / `scripts/smoke_prep/package_contracts.py` | `npm run pack:assert`; `npm run smoke:prepare -- --reset` writes `.smoke/PACKAGE_IDENTITY.json` | No workspace reset for `pack:assert`; prep records generated tarball in `.smoke/packages`. | Not CI-backed yet | Name, version, filename, integrity, shasum, tarball SHA256, generated tarball path, and selected checksums match current package metadata | Pack JSON, tarball checksum output, `.smoke/PACKAGE_IDENTITY.json` | Prevents stale same-shape install from passing. | +| Prepared repo installed manifest | Phase 02 / `scripts/smoke_prep/package_contracts.py` | `npm run smoke:prepare -- --reset` writes `.smoke/INSTALLED_AUTOMATOR_MANIFEST.json` | Deletes and recreates `.smoke`; network needed unless cache present. Verifies only supported roots with dependency entrypoints. | Not CI-backed yet | Installed tarball identity, installed version surfaces, installed-file checksums, and dependency skill entrypoints match current tarball | `.smoke` prep logs, `PACKAGE_IDENTITY.json`, `INSTALLED_AUTOMATOR_MANIFEST.json`, checksum diff | `.claude/skills` is fact for prepared gunz; `.agents/skills` and `.codex/skills` are spec-only unless prep tools change. | +| Planning/create smoke | Existing / `scripts/run-smoke-automator.py`; Phase 04 / `scripts/run-smoke-modes.py` | `npm run smoke:run`; `npm run smoke:modes` | Prepared create uses `.smoke/gunz`; mode smoke uses temp BMAD-style fixture. | Not CI-backed yet | Prepared story `1.1` create passes; mode fixture covers multi-story and explicit-ID selection, invalid range empty-selection behavior, stop-hook states, sprint-status present/missing, and init log | `.smoke/AUTOMATED_SMOKE_REPORT.md`; `.smoke/MODE_SMOKE_REPORT.json` | Prepared create remains story `1.1`; breadth comes from the temp mode fixture. | +| Two-story dev-loop smoke | Existing / `scripts/run-smoke-dev-loop.py`; Phase 04 / `scripts/run-smoke-modes.py` | `npm run smoke:dev-loop`; `npm run smoke:modes` | Prepared dev loop uses `.smoke/gunz`; mode smoke uses temp BMAD-style fixture. | Not CI-backed yet | Two prepared stories transition to done; mode fixture asserts complexity-file persistence and richer per-task agent config in state | `.smoke/AUTOMATED_DEV_LOOP_SMOKE_REPORT.md`; `.smoke/MODE_SMOKE_REPORT.json` | Prepared dev-loop still uses a fixed parsed-dev fixture by design. | +| Parser contract matrix | Phase 03 / `scripts/run-smoke-contracts.py`, `tests/test_runtime_helper_contracts.py`, `tests/test_orchestrator_parse.py` | `npm run smoke:contracts` | Temp files and patched fake parser subprocess; no live LLM. | Not CI-backed yet | Success and fail-closed parse-output cases assert exit code plus JSON payload; skipped tests fail the gate | JSON `status`, `reason`, and `structuredIssues` assertions | JSON-decode branch is defensive because `extract_json_line` filters invalid JSON before parse. | +| Monitor terminal-state matrix | Phase 03 / `scripts/run-smoke-contracts.py`, `tests/test_runtime_helper_contracts.py`, `tests/test_success_verifiers.py` | `npm run smoke:contracts` | Patched session-status fixtures; no live LLM. | Not CI-backed yet | Completed, incomplete, crashed, stuck, timeout, not_found, invalid persisted session-state diagnostics, and invalid option cases assert JSON fields; skipped tests fail the gate | JSON `final_state`, `exit_reason`, `output_verified`, and `structuredIssues` where emitted | Monitor exits 0 for terminal failure states; assert JSON, not exit code. | +| Runner lifecycle matrix | Phase 03 / `scripts/run-smoke-contracts.py`, `tests/test_runtime_helper_contracts.py`, `tests/test_tmux_runtime.py` | `npm run smoke:contracts` | `SA_TMUX_RUNTIME=runner`; temp dirs; tmux is preferred for full coverage, but expected `tmux not available` skips do not fail this gate. | Not CI-backed yet | Wrapper success/crash plus spawn_error, interrupted, and launch-never/stuck mapping cases pass | Runner state/output file assertions | Uses runner mode and temp artifacts; no live provider. | +| Build-cmd branch matrix | Phase 03 / `tests/test_runtime_helper_contracts.py`, `tests/test_state_policy_metadata.py`, `tests/test_agent_config_model.py` | `npm run smoke:contracts` | Temp state/policy fixtures; no live LLM. | Not CI-backed yet | Codex flags, `AI_COMMAND`, Claude branch, model quoting, unknown step, invalid policy cases pass | Command string assertions and stderr | `AI_COMMAND` owns the command branch when `AI_AGENT` is unset. | +| State policy verifier matrix | Phase 03 / `tests/test_state_validation.py`, `tests/test_runtime_policy.py`, `tests/test_state_policy_metadata.py`, `tests/test_success_verifiers.py` | `npm run smoke:contracts` | Temp state/artifact fixtures. | Not CI-backed yet | State-update no-mutation failures, policy snapshot creation/failure, pinned snapshot reads, and success verifier edge cases pass | JSON payloads, frontmatter no-mutation assertions, and PolicyError assertions | Existing focused suites are included in `smoke:contracts`. | +| Marker/root resolution helper matrix | Phase 03 / `tests/test_runtime_layout.py` | `npm run smoke:contracts` | Temp runtime-layout fixtures for `.agents`, `.codex`, and `.claude`; no live LLM. | Not CI-backed yet | Helper-resolved marker paths and runtime roots match each fixture and no hard-coded `.claude` path is accepted | Path assertions and marker command JSON | Prevents marker drift across supported runtime roots. | +| Status/source-of-truth helper matrix | Phase 03 / `tests/test_success_verifiers.py` | `npm run smoke:contracts` | Temp story-file and sprint-status fixtures. | Not CI-backed yet | Story-file status, sprint-status status, mismatch diagnostics, story-file fallback, and `sprint_status_not_updated` cases pass | Verifier payload and fixture file assertions | Prevents completion inference from marker absence or process exit alone. | +| Create startup guard smoke | Phase 04 / `scripts/run-smoke-modes.py` | `npm run smoke:modes` | Temp BMAD-style `.agents` fixture; prepared `.smoke/gunz` only for explicit `smoke:run`/`smoke:dev-loop`. | Not CI-backed yet | Stop-hook configured/pending-trust/invalid-json states, existing-state detection, sprint-status present/missing precondition, and init log checks pass | `.smoke/MODE_SMOKE_REPORT.json` and helper JSON payloads | Fixture setup simulates child workflow outputs; orchestrator does not own sprint-status mutation. | +| Resume/validate/edit smoke | Phase 04 / `scripts/run-smoke-modes.py` | `npm run smoke:modes` | Seeded temp state files; no live LLM or prepared repo required for default gate. | Not CI-backed yet | Explicit path summary, latest incomplete discovery, no-incomplete fallback, workflow-derived menu labels/route hints, view/start-over/abort branch-equivalent helper checks, marker, validate structure issue reporting, and edit branch helper contracts pass | Mode smoke JSON report and seeded state diff | Interactive edit prompts remain workflow-only; helper-backed save/discard/edit-more contracts are deterministic. | +| Mode state/artifact assertion matrix | Phase 04 / `scripts/run-smoke-modes.py` | `npm run smoke:modes` | Seeded temp BMAD-style fixtures; no prepared repo needed for default gate. | Not CI-backed yet | Frontmatter, status, current story/step, agents/complexity files, rendered agent config, policy snapshot path/hash, progress rows/metrics, action log, init log, state docs, parsed marker JSON, heartbeat mutation, `.gitignore`, dev log, compact report, and selected artifacts match expectations | Mode smoke JSON report, state file assertions, and marker JSON | Prevents route-only smoke success. | +| Mode source-mismatch matrix | Phase 04 / `scripts/run-smoke-modes.py` | `npm run smoke:modes` | Seeded story-file and sprint-status fixtures. | Not CI-backed yet | Story-file versus sprint-status mismatches are surfaced through the shared review verifier `sprint_status_not_updated` note | Mode smoke JSON report and verifier payload assertions | Required source-of-truth behavior from Oracle review. | +| Finish-loop smoke | Phase 05 / `scripts/run-smoke-finish-loop.py` | `npm run smoke:finish-loop` | Temp git-backed BMAD fixture; controlled commits only in temp smoke repo; no live LLM. | Not CI-backed yet | Automate done/skip rows, incomplete review diagnostics, review completion, finalize source-truth checks, execution-complete, wrapup, and final validation pass | `.smoke/FINISH_LOOP_SMOKE_REPORT.json`, `.smoke/finish-loop-diagnostics/` state copy, and persisted smoke repo `git-log.txt` | Commit isolation is critical. | +| Host commit isolation sentinel | Phase 05 / `scripts/run-smoke-finish-loop.py` | `npm run smoke:finish-loop`; `python3 scripts/run-smoke-finish-loop.py --target-repo ` must fail unless `--allow-unsafe-repo` is set | Records host HEAD/status before and after finish-loop; refuses host repo target through runner guard before `commit-story`. | Not CI-backed yet | Host HEAD/status unchanged, unsafe host target rejected, and temp smoke repo has expected story commits | Host `git status`, host HEAD, persisted `git-log.txt`, JSON report | Prevents accidental `commit-story --repo` targeting host repo. | +| Multi-epic retro fixture | Phase 05 / `scripts/run-smoke-finish-loop.py` | `npm run smoke:finish-loop` | Seeded two-epic state/status fixtures; no live LLM. | Not CI-backed yet | Epic 1 retro triggers before Epic 2 completion, retro build-cmd is generated, retrospective failure is recorded as skipped/non-blocking, Epic 2 continues to completion, separate `retrospectives.epic-*` entries exist, wrapup waits for all stories | Finish-loop smoke report and state document | Scenario is always included in `smoke:finish-loop`. | +| Full Python tests | Existing / `package.json` | `npm run test:python` | No external reset. | Not CI-backed yet | unittest reports OK | unittest output | None known. | +| CLI smoke | Existing / `package.json` | `npm run test:cli` | No external reset. | Not CI-backed yet | Both CLI help commands exit 0 | command stderr/stdout | None known. | +| Package dry run | Existing / `package.json` | `npm run pack:dry-run` | No external reset. | Not CI-backed yet | npm exits 0 | npm tarball file list | Superseded for release confidence by `pack:assert`; retained as a manual npm packaging inspection command. | +| Shell smoke fixture | Existing / `scripts/smoke-test.sh` | `npm run test:smoke` | Uses repo fixture behavior. | Not CI-backed yet | script exits 0 | script output | Does not cover external prepared repo. | +| Final verify | Phase 06 / `package.json` | `npm run verify`: `test:python && version:check && pack:assert && test:cli && smoke:contracts && smoke:modes && test:smoke` | No prepared-repo reset or network by default. | Not CI-backed yet | All selected fast deterministic gates pass | command output plus `.smoke/MODE_SMOKE_REPORT.json` when mode smoke runs | Prepared repo reset/network gates stay explicit. | +| Full deterministic release smoke | Phase 06 / `package.json` | `npm run smoke:deterministic-full`: `smoke:prepare -- --reset && smoke:run && smoke:dev-loop && smoke:finish-loop` | Prepared `.smoke/gunz` for create/dev plus temp finish-loop repo; reset/network/cache-heavy; explicit pre-release/nightly gate. | Not CI-backed yet | Prepared create, dev-loop, finish-loop, package identity, installed manifest, and host isolation pass | Combined smoke reports under `.smoke/`, including `.smoke/FINISH_LOOP_SMOKE_REPORT.json` and `.smoke/finish-loop-diagnostics/` | Not suitable for default verify until CI cache and runtime budget are proven. | diff --git a/docs/plans/automator-deterministic-smoke-coverage/handoff-log.md b/docs/plans/automator-deterministic-smoke-coverage/handoff-log.md new file mode 100644 index 00000000..24fa3b01 --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/handoff-log.md @@ -0,0 +1,473 @@ +# Automator Deterministic Smoke Coverage Handoff Log + +## Purpose + +This file carries implementation context between clean-context agents. Each phase agent must read all earlier entries before starting and append a new entry before ending. + +Do not rely on conversation history for phase continuity. Put next-agent continuity facts here. + +For user-facing decisions, spec gaps, required changes, tradeoffs, deviations, and notable risks, update [implementation-notes.md](./implementation-notes.md). + +## Entry Template + +```md +## Phase NN - YYYY-MM-DD - agent/session + +### Summary + +- What changed or was verified. + +### Commands Run + +```bash +exact command +``` + +### Results + +- Pass/fail. +- Important SHAs, tags, paths, versions. + +### Decisions And Assumptions + +- Decision made and why. +- Assumptions the next phase should preserve or re-check. + +### Blockers Or Risks + +- Blocker, owner, next action. +- Or `None`. + +### Next Phase Notes + +- Read these files. +- Run this command next. +- Watch for this failure mode. +``` + +## Phase Entries + +## Plan Creation - 2026-06-02 - Codex + +### Summary + +- Created the phased plan packet for deterministic Story Automator smoke coverage. +- Source inputs were prior deterministic smoke work plus three read-only sub-agent audits: workflow coverage, runtime/helper contracts, and package/prep integration. + +### Commands Run + +```bash +git status --short --branch +find docs/plans -maxdepth 2 -type f +sed -n '1,220p' skills/bmad-story-automator/workflow.md +sed -n '1,220p' skills/bmad-story-automator/data/orchestration-policy.json +sed -n '1,220p' skills/bmad-story-automator/steps-v/step-v-01-check.md +sed -n '1,220p' skills/bmad-story-automator/steps-v/step-v-02-report.md +sed -n '1,260p' skills/bmad-story-automator/steps-e/step-e-01-load.md +``` + +### Results + +- Plan packet created under `docs/plans/automator-deterministic-smoke-coverage/`. +- No implementation code changed in this plan creation step. + +### Decisions And Assumptions + +- Plan slug: `automator-deterministic-smoke-coverage`. +- The plan must cover the full policy sequence `create`, `dev`, `auto`, `review`, `retro` plus public modes `create`, `resume`, `validate`, and `edit`. +- Deterministic smokes should avoid live LLM dependence by using fake parser subprocesses, runner mode, seeded state files, and controlled smoke repo commits. + +### Blockers Or Risks + +- `bmad-method@next` is a moving smoke-prep input until pinned or asserted. +- Workflow version metadata may be stale relative to package version. + +### Next Phase Notes + +- Start Phase 01 with [01-baseline-and-version-determinism.md](./01-baseline-and-version-determinism.md). +- Update [gate-map.md](./gate-map.md) as soon as concrete commands are added. + +## Oracle Review Applied - 2026-06-02 - Codex + +### Summary + +- Applied Oracle's deterministic smoke architecture review to the plan packet. +- Strengthened release-blocking language for package/prep identity, helper JSON contracts, and finish-loop review/finalize coverage. +- Split intended gates into fast default `verify` targets versus explicit prepared-repo release/nightly smoke targets. + +### Commands Run + +```bash +sed -n '1,260p' /Users/joon/.codex/attachments/74a1085b-7972-4524-9413-63f6757a1af1/pasted-text.txt +git status --short --branch +find docs/plans/automator-deterministic-smoke-coverage -maxdepth 2 -type f | sort +sed -n '1,260p' docs/plans/automator-deterministic-smoke-coverage/README.md +sed -n '1,260p' docs/plans/automator-deterministic-smoke-coverage/gate-map.md +``` + +### Results + +- Updated `README.md`, `gate-map.md`, phase files 02-06, phase TODO files 02-06, and `implementation-notes.md`. +- No implementation code changed in this application step. + +### Decisions And Assumptions + +- `npm run verify` should eventually run fast local deterministic gates only. +- Prepared `.smoke/gunz` reset/network gates should remain explicit unless Phase 06 proves CI/runtime stability. +- `smoke:finish-loop` must refuse unsafe host repo commit targets by default. +- `smoke:modes` should use temp BMAD-style fixtures for default verification. + +### Blockers Or Risks + +- The gates are still planned, not implemented. +- Phase 02, Phase 03, and Phase 05 remain release-blocking before smoke readiness can be claimed. + +### Next Phase Notes + +- Start implementation with Phase 01, then prioritize Phase 02 and Phase 03 before adding broader lifecycle runners. +- Preserve the Oracle-applied gate split in [gate-map.md](./gate-map.md). + +## Oracle Application Review Loop - 2026-06-02 - Codex + +### Summary + +- Ran `general-subagent-review-loop` against the Oracle-applied plan packet. +- Used three read-only reviewer slices: requirements coverage, gate/command mapping, and phase/TODO consistency. +- Fixed all credible P2 findings, then ran a targeted final Oracle-application review pass. + +### Commands Run + +```bash +sed -n '1,220p' /Users/joon/.agents/skills/general-subagent-review-loop/steps/01-scope-review-pass.md +sed -n '1,240p' /Users/joon/.agents/skills/general-subagent-review-loop/steps/02-build-review-packets.md +sed -n '1,220p' /Users/joon/.agents/skills/general-subagent-review-loop/steps/03-triage-findings.md +sed -n '1,240p' /Users/joon/.agents/skills/general-subagent-review-loop/steps/04-fix-and-verify.md +sed -n '1,240p' /Users/joon/.agents/skills/general-subagent-review-loop/steps/05-stop-or-loop.md +rg -n '[[:blank:]]$' docs/plans/automator-deterministic-smoke-coverage || true +wc -l docs/plans/automator-deterministic-smoke-coverage/*.md docs/plans/automator-deterministic-smoke-coverage/TODO/*.md +rg -n "pack:identity|pack:assert|pack --json --pack-destination|multi-epic|marker/root|Status/source|Mode state/artifact|Mode source-mismatch|Manual And Live Boundaries|sprint_status_not_updated|frontmatter|commit SHA|smoke:deterministic-full" docs/plans/automator-deterministic-smoke-coverage +git status --short --branch +``` + +### Results + +- Requirements reviewer: no actionable findings. +- Gate/command reviewer: two P2 findings fixed. +- Phase/TODO consistency reviewer: two P2 findings fixed. +- Targeted final Oracle-application auditor: no actionable findings. +- Static verification: no trailing whitespace; all plan files remain under 500 LOC. + +### Decisions And Assumptions + +- Tarball identity is now bound to `npm run pack:assert`. +- Multi-epic retrospective coverage is now bound to `npm run smoke:finish-loop -- --scenario multi-epic`, or to `npm run smoke:finish-loop` if that command always includes the scenario. +- Phase 03 now has explicit marker/root and status/source-of-truth helper matrix rows and TODOs. +- Phase 04 now has explicit state/artifact and source-mismatch matrix rows and TODOs. + +### Blockers Or Risks + +- None for Oracle application completeness. +- Implementation gates are still not built; Phase 02, Phase 03, and Phase 05 remain release-blocking before smoke readiness. + +### Next Phase Notes + +- Start Phase 01 implementation. +- Preserve the P0/P1/P2-clean review baseline when later phases change the gate map. + +## Phase 01 - 2026-06-02 - Codex + +### Summary + +- Added the Phase 01 coverage baseline at `coverage-baseline.md`. +- Added `npm run version:check` for package/plugin/module/Python/runtime/workflow version alignment. +- Fixed stale `skills/bmad-story-automator/workflow.md` frontmatter from `1.12.0` to `1.15.0`. +- Added `npm run smoke:input-check` and `.smoke/SMOKE_INPUTS.json` manifest recording for deterministic smoke inputs. +- Changed `smoke:prepare` to resolve `bmad-method@next` once, record version/integrity, and install the resolved `bmad-method@` package. +- Updated `gate-map.md`, `implementation-notes.md`, `docs/versioning.md`, and Phase 01 TODO status. +- Ran two clean-context review agents; fixed all P2 findings. + +### Commands Run + +```bash +npm run version:check +npm run smoke:input-check +npm run test:cli +npm run smoke:prepare -- --skip-bmad-install --skip-automator-install +npm run test:python +git diff --check +wc -l scripts/check-version-alignment.py scripts/check-smoke-inputs.py scripts/smoke_prep/*.py docs/plans/automator-deterministic-smoke-coverage/*.md docs/plans/automator-deterministic-smoke-coverage/TODO/*.md docs/versioning.md +``` + +### Results + +- `npm run version:check`: pass; all checked surfaces report `1.15.0`. +- `npm run smoke:input-check`: pass; resolved `bmad-method@next` to `6.8.1-next.0` with integrity `sha512-r8lDToLh57N0BiNsBOcD5wV+JWrR87rvdU2oKm3bhOGykHiCkj3f6BB96ymgftuTDdeK5OMr3AiQNKAsk6/I0A==`. +- `npm run test:cli`: pass. +- `npm run smoke:prepare -- --skip-bmad-install --skip-automator-install`: pass; wrote ignored `.smoke/SMOKE_INPUTS.json`; reused prepared `.smoke/gunz` at commit `fca6470d329668019dace305b5f0f3c9b62cb113`. +- `npm run test:python`: pass, 537 tests. +- `git diff --check`: pass. +- File size check: all touched files remain below 500 LOC. + +### Decisions And Assumptions + +- `package.json.version` remains the canonical stable release version for metadata alignment. +- Alias names are intentionally different by channel: npm `bmad-story-automator`, plugin `bmad-automator`, BMAD module `automator`, Python/workflow `story-automator`. +- `bmad-method@next` remains the requested installer input for now, but prep installs the resolved version recorded in `SMOKE_INPUTS.json`, avoiding a second dist-tag resolution. +- No Phase 01 deferred-work items remain. Remaining `gap` and `blocked` baseline rows are already owned by Phase 02-06 TODOs. + +### Blockers Or Risks + +- No Phase 01 blocker. +- Phase 02 still needs package tarball identity and prepared-repo installed manifest assertions. +- The prepared `.smoke/gunz` checkout was already dirty in ignored smoke artifacts during prep; this did not affect host repo status. + +### Next Phase Notes + +- Start Phase 02 with `02-package-and-prepared-repo-contracts.md` and `TODO/phase-02.md`. +- Preserve the resolved-install behavior when adding package identity checks. +- Recommended next command: `npm pack --dry-run --json` as the first input to `pack:assert`. + +## Phase 02 - 2026-06-02 - Codex + +### Summary + +- Added `pack:assert` with JSON package content, executable mode, forbidden generated-file, tarball identity, SHA256, and selected checksum assertions. +- Added shared package contract helpers in `scripts/smoke_prep/package_contracts.py`. +- Updated `smoke:prepare` to write `.smoke/PACKAGE_IDENTITY.json` and `.smoke/INSTALLED_AUTOMATOR_MANIFEST.json`. +- Installed-file verification now compares selected copied skill files in prepared `.smoke/gunz` against the current packed tarball. +- Classified prepared root support: `.claude/skills` is verified fact; `.agents/skills` and `.codex/skills` are spec-only for the current `--tools claude-code` smoke prep. + +### Commands Run + +```bash +npm run pack:assert +npm run smoke:prepare -- --reset +npm run smoke:run +npm run smoke:dev-loop +npm run smoke:dev-loop +npm run version:check +npm run smoke:input-check +npm run test:cli +npm run test:python +git diff --check +``` + +### Results + +- `npm run pack:assert`: pass. Tarball `bmad-story-automator-1.15.0.tgz`; integrity `sha512-rlnPSIrZqXA76GLR7GHWKsKKW1bVHfGSVSQhdjBPwbAiufHynFtADAC51p9if6r6JBlA1PmPdZSc7ez1G9RAkw==`; shasum `8883e0199744c914f941de51af26a4b690a1c048`; SHA256 `b4c228d8441cebcea7d041f89bb46d5498953111c6c3a1441ee5c1dbd821b5ae`; entry count `195`; selected checksum count `115`. +- `npm run smoke:prepare -- --reset`: pass. Reinstalled current tarball and wrote package/installed manifests. +- Installed workflow proof: `.smoke/gunz/.claude/skills/bmad-story-automator/workflow.md` reports `1.15.0`. +- `npm run smoke:run`: pass; created story `1.1` smoke state and report. +- `npm run smoke:dev-loop`: ran twice. First run failed on an `agents_file_not_found` state left by the just-created partial dev-loop attempt; second run reset artifacts and passed both stories. +- `npm run version:check`: pass. +- `npm run smoke:input-check`: pass; BMAD Method resolved to `6.8.1-next.0`. +- `npm run test:cli`: pass. +- `npm run test:python`: pass, 537 tests. +- `git diff --check`: pass. + +### Decisions And Assumptions + +- `pack:assert` is fast and temp-dir only, so it is suitable for future default `verify`. +- Prepared repo install checks remain tied to `smoke:prepare -- --reset` because they require network/cache-heavy BMAD prep. +- The installed manifest reports roots without required dependency skill entrypoints as `unsupported`; the plan taxonomy records these prepared-gunz roots as `spec-only`. + +### Blockers Or Risks + +- No Phase 02 blocker. +- Phase 03 should reuse `package_contracts.py` style: narrow fixtures, JSON assertions, no terminal-output-only success. + +### Next Phase Notes + +- Start Phase 03 with `03-runtime-helper-contract-smokes.md` and `TODO/phase-03.md`. +- Recommended first command: inspect existing unit coverage around `parse-output`, `monitor-session`, `tmux-wrapper build-cmd`, and runtime policy helpers before designing `smoke:contracts`. + +## Phase 03 - 2026-06-02 - Codex + +### Summary + +- Added `npm run smoke:contracts` as the Phase 03 fast helper-contract gate, backed by `scripts/run-smoke-contracts.py`. +- Added `tests/test_runtime_helper_contracts.py` for missing parser subprocess, monitor terminal-state, build-cmd, runner edge-state, and `tmux-wrapper spawn` runner-mode assertions. +- Reused existing focused suites for state-update no-mutation, runtime-policy snapshots, state metadata, marker/root resolution, success verifiers, and status/source-of-truth helper behavior. +- Updated `coverage-baseline.md`, `gate-map.md`, `implementation-notes.md`, and Phase 03 TODO status. + +### Commands Run + +```bash +npm run smoke:contracts +npm run test:python +npm run test:cli +git diff --check +wc -l tests/test_runtime_helper_contracts.py scripts/smoke_prep/package_contracts.py +``` + +### Results + +- `npm run smoke:contracts`: pass, 296 tests. +- `npm run test:python`: pass, 544 tests. +- `npm run test:cli`: pass. +- `git diff --check`: pass. +- File size check: `tests/test_runtime_helper_contracts.py` is 280 LOC; Phase 02 `package_contracts.py` remains 329 LOC. + +### Decisions And Assumptions + +- `smoke:contracts` intentionally composes focused unittest modules instead of adding a large shell runner. Unexpected skipped tests fail the gate; expected `tmux not available` skips pass and reduce full runner coverage. +- `monitor-session` remains a JSON-contract command: crashed, timeout, incomplete, and not_found states can return process exit code `0`. +- `parse-output` JSON-decode handling is defensive because `extract_json_line` returns only JSON-valid candidate lines; no-json and schema-invalid payloads are the deterministic failure cases. +- `tmux-wrapper spawn` success/crash coverage runs in `SA_TMUX_RUNTIME=runner`; tmux is preferred for complete coverage, but expected `tmux not available` skips do not fail the gate. + +### Blockers Or Risks + +- No Phase 03 blocker. +- `smoke:contracts` is not wired into `npm run verify` yet; Phase 06 owns verify promotion after Phase 04 creates `smoke:modes`. + +### Next Phase Notes + +- Start Phase 04 with `04-create-dev-resume-validate-edit-coverage.md` and `TODO/phase-04.md`. +- Preserve the split: Phase 04 should use temp BMAD-style fixtures for resume/validate/edit mode smokes and keep prepared `.smoke/gunz` for realistic external flow only. + +## Phase 04 - 2026-06-02 - Codex + +### Summary + +- Added `npm run smoke:modes` backed by `scripts/run-smoke-modes.py`. +- The new mode smoke uses a temp BMAD-style `.agents` fixture and asserts create startup guards, preflight selection breadth, resume discovery/routes/fallback, marker JSON lifecycle, validation/source mismatch, artifact outputs, and edit helper branch contracts. +- Prepared `.smoke/gunz` create/dev checks remain explicit through `smoke:run` and `smoke:dev-loop`. +- Updated `coverage-baseline.md`, `gate-map.md`, `implementation-notes.md`, and Phase 04 TODO status. + +### Commands Run + +```bash +npm run smoke:modes +npm run smoke:run +npm run smoke:dev-loop +npm run test:python +git diff --check +wc -l scripts/run-smoke-modes.py +``` + +### Results + +- `npm run smoke:modes`: pass; wrote `.smoke/MODE_SMOKE_REPORT.json`. +- `npm run smoke:run`: pass; created story `1.1` smoke state and report in prepared `.smoke/gunz`. +- `npm run smoke:dev-loop`: pass; simulated dev completion for stories `1.1` and `1.2` in prepared `.smoke/gunz`. +- `npm run test:python`: pass, 544 tests. +- `git diff --check`: pass. +- `scripts/run-smoke-modes.py` is under the 500 LOC repo limit. +- Seeded temp fixture state fields: `storyRange=["1.1","1.2"]`, `status=IN_PROGRESS`, `currentStory=1.1`, `currentStep=step-03-execute`, `complexityFile=_bmad-output/story-automator/complexity-smoke.json`, `agentsFile=_bmad-output/story-automator/agents-smoke.md`, and a policy snapshot path/hash. +- Seeded simulated child outputs: `_bmad-output/implementation-artifacts/sprint-status.yaml`, `_bmad-output/implementation-artifacts/1-1-first.md`, `_bmad-output/story-automator/dev-log-smoke.md`, and `_bmad-output/story-automator/mode-report-smoke.json`. +- Preflight proof: multi-story selection, explicit story IDs, reversed numeric ranges, invalid range empty-selection behavior, and rendered `review=claude` agent config. +- Resume proof: explicit state summary, latest incomplete discovery, no-incomplete fresh-create fallback, workflow-derived menu labels/route hint, view action-log extraction, start-over backup simulation, and abort state update. +- Marker path proof: helper resolved `.agents/.story-automator-active`; `.gitignore` entry was added dynamically from helper output; marker JSON shape and heartbeat mutation were parsed from the marker file. +- Source mismatch proof: shared review verifier returned `note=sprint_status_not_updated` when story-file status was `done` but sprint status remained `ready-for-dev`. +- Startup precondition proof: helper checked sprint-status present and missing states; there is no standalone startup CLI, so abort wording remains a workflow precondition rather than an executed branch. +- Validation/edit boundary: validation helper contracts include happy path, structure issue reporting, progress-row metrics, exact-ID done branch, and compact report output; edit menu prompts remain interactive-only, while workflow-derived menu labels/route hints plus status/range/current-story/AI-command/artifact-path/text save, discard rollback, and edit-more state update are covered through asserted state helper mutations. + +### Decisions And Assumptions + +- Mode fixtures write story files and `sprint-status.yaml` as simulated child workflow output, not as orchestrator-owned mutation. +- Edit mode remains interactive in the workflow. Phase 04 asserts deterministic helper-backed save/discard/edit-more route contracts and post-edit route hints. +- Marker assertions resolve the active marker path through helper output and verify `.agents/.story-automator-active`; no hard-coded `.claude` path is used. + +### Blockers Or Risks + +- No Phase 04 blocker. +- Phase 06 should decide whether to promote `smoke:modes` into `npm run verify`; prepared `.smoke/gunz` checks remain explicit. + +### Next Phase Notes + +- Start Phase 05 with `05-automate-review-finish-retro-coverage.md` and `TODO/phase-05.md`. +- Preserve the host mutation isolation requirement for finish-loop work. +- Recommended next command: inspect commit/finalize helpers and host HEAD/status sentinel surfaces before implementing `smoke:finish-loop`. + +## Phase 05 - 2026-06-02 - Codex + +### Summary + +- Added `npm run smoke:finish-loop` backed by `scripts/run-smoke-finish-loop.py`. +- The new finish-loop smoke uses a temp git-backed BMAD-style fixture and asserts automate done/skip, review incomplete/completed, commit/finalize, multi-epic retrospective skip, execution-complete, wrapup, marker removal, and host commit isolation. +- Updated `coverage-baseline.md`, `gate-map.md`, `implementation-notes.md`, and Phase 05 TODO status. + +### Commands Run + +```bash +npm run smoke:finish-loop +PYTHONPATH=skills/bmad-story-automator/src python3 scripts/run-smoke-finish-loop.py --target-repo /Users/joon/.codex/worktrees/9b27/bmad-story-automator +npm run smoke:dev-loop +npm run test:python +git diff --check +wc -l scripts/run-smoke-finish-loop.py +``` + +### Results + +- `npm run smoke:finish-loop`: pass; wrote `.smoke/FINISH_LOOP_SMOKE_REPORT.json`. +- Unsafe host-target guard command: failed as expected with `unsafe commit repo outside smoke workspace`. +- `npm run smoke:dev-loop`: pass; prepared `.smoke/gunz` dev-loop remains green. +- `npm run test:python`: pass, 544 tests. +- `git diff --check`: pass. +- `scripts/run-smoke-finish-loop.py` is under the 500 LOC repo limit. +- Smoke repo commits: controlled commits were created for stories `1.1`, `1.2`, and `2.1` inside the temp smoke repo only; `git log` was persisted to `.smoke/finish-loop-diagnostics/git-log.txt`. +- Host isolation proof: host HEAD/status was captured before and after finish-loop and remained unchanged; runner target guard rejected the host repo before `commit-story`. +- Review proof: story `1.2` first produced `workflow_not_complete`, then all three stories verified complete through review/source-of-truth helpers. +- Retrospective proof: Epic 1 retrospective was recorded as `skipped` before Epic 2 completion; Epic 2 continued to completion and recorded its own skipped retro entry; state copy persisted these entries under `retrospectives.epic-*`. +- Wrapup proof: state transitioned through `EXECUTION_COMPLETE` to `COMPLETE`; metrics reported 3/3 stories complete; learnings file was written; marker was removed. + +### Decisions And Assumptions + +- The finish-loop smoke uses a temp git repo instead of prepared `.smoke/gunz` so commit isolation can be proven without network/reset dependency; durable diagnostics are copied to `.smoke/finish-loop-diagnostics/` before temp cleanup. +- Retrospective agent execution is simulated as skipped deterministic output; the smoke asserts retro-agent resolution and retro build-cmd generation, then verifies non-blocking continuation. +- Finish-loop target safety is enforced in the runner before invoking `commit-story`; `commit-story` itself remains unchanged. An explicit `--allow-unsafe-repo` override exists only for manual debugging. + +### Blockers Or Risks + +- No Phase 05 blocker. +- Phase 06 still owns promotion of `smoke:finish-loop` into `verify` or `smoke:deterministic-full`. + +### Next Phase Notes + +- Start Phase 06 with `06-gate-integration-and-readiness-review.md` and `TODO/phase-06.md`. +- Recommended first command: run all fast deterministic gates together and update `npm run verify` only after the command set is stable. + +## Phase 06 - 2026-06-02 - Codex + +### Summary + +- Updated `npm run verify` to run fast local deterministic gates. +- Added `npm run smoke:deterministic-full` as the explicit prepared/reset pre-release smoke wrapper. +- Updated `gate-map.md`, `implementation-notes.md`, and Phase 06 TODO status. + +### Commands Run + +```bash +npm run verify +npm run smoke:deterministic-full +git diff --check +``` + +### Results + +- `npm run verify`: pass. Ran Python tests, version alignment, package assertions, CLI smoke, helper contracts, mode smoke, and shell smoke. +- `npm run smoke:deterministic-full`: pass. Ran smoke prepare reset, prepared create smoke, prepared dev-loop smoke, and finish-loop smoke. +- `git diff --check`: pass. + +### Decisions And Assumptions + +- Default `verify` stays reset-free and network-free by excluding `smoke:prepare`, prepared create/dev-loop, and full finish-loop wrapper. +- `smoke:deterministic-full` remains explicit because it reclones/resets `.smoke/gunz` and depends on the npm/git/network/cache surfaces used by `smoke:prepare`. +- Live provider/auth, semantic output quality, and interactive conversational UX remain outside deterministic readiness. + +### Reviewer Results + +- Workflow lifecycle/gate truthfulness reviewer: no actionable findings. Verified `npm run verify`, `npm run smoke:finish-loop`, and unsafe host-target failure. +- Package/install determinism and runtime/helper boundary reviewer: no actionable findings. Verified `npm run pack:assert` and `npm run smoke:contracts`. +- Validate/edit/resume reviewer: no actionable findings. Verified Phase 04 mode coverage through `npm run smoke:modes`, including explicit resume paths, latest incomplete discovery, validation structure issues, edit save/discard/edit-more helpers, and helper-backed post-edit route hints. + +### Blockers Or Risks + +- No Phase 06 blocker. +- `smoke:deterministic-full` remains reset/network/cache-heavy and should stay explicit until CI proves stable runtime and cache budget. +- Deterministic readiness does not cover live provider auth/outages, trust prompts, rate limits, semantic generated output quality, or interactive UX beyond helper-backed contracts. + +### Final Recommendation + +- Ready for local deterministic smoke readiness review. Use `npm run verify` as the default fast gate and `npm run smoke:deterministic-full` as the explicit pre-release gate. diff --git a/docs/plans/automator-deterministic-smoke-coverage/implementation-notes.md b/docs/plans/automator-deterministic-smoke-coverage/implementation-notes.md new file mode 100644 index 00000000..b39e937e --- /dev/null +++ b/docs/plans/automator-deterministic-smoke-coverage/implementation-notes.md @@ -0,0 +1,178 @@ +# Automator Deterministic Smoke Coverage Implementation Notes + +## Purpose + +This file is the running user-facing implementation record. Keep decisions, spec gaps, required changes, tradeoffs, deviations, risks, and user-relevant context here. + +This is separate from [handoff-log.md](./handoff-log.md). Use the handoff log for next-agent continuity: what to read, exact commands, blockers, and next recommended actions. + +## Note Template + +```md +## YYYY-MM-DD - phase/session + +### Context + +- What part of the spec or implementation this note concerns. + +### Decision, Change, Or Tradeoff + +- What was decided or changed. +- Why it was necessary. + +### User Impact + +- What the user should know. +- Follow-up needed, or `None`. +``` + +## Notes + +## 2026-06-02 - phase 06 gate integration + +### Context + +- Phase 06 required final gate wiring, explicit heavy-smoke separation, and readiness review. + +### Decision, Change, Or Tradeoff + +- Updated `npm run verify` to run the fast deterministic local gate set: `test:python`, `version:check`, `pack:assert`, `test:cli`, `smoke:contracts`, `smoke:modes`, and `test:smoke`. +- Added `npm run smoke:deterministic-full` as the explicit reset/network-heavy pre-release gate: `smoke:prepare -- --reset`, `smoke:run`, `smoke:dev-loop`, and `smoke:finish-loop`. +- Kept `smoke:finish-loop` out of default `verify` even though it is local, because Phase 06 target verify was already broad and the release-wrapper gate captures finish-loop readiness explicitly. +- Deterministic readiness still excludes live provider/auth behavior, rate limits, trust prompts, outages, semantic quality of generated implementation/reviews/retrospectives, and interactive UX beyond helper-backed effects. + +### User Impact + +- `npm run verify` is now the default fast confidence gate. +- `npm run smoke:deterministic-full` is the fuller pre-release smoke with prepared repo reset and package/install identity proof. + +## 2026-06-02 - phase 05 finish-loop coverage + +### Context + +- Phase 05 required deterministic coverage for automate, review, commit/finalize, retrospective, execution-complete, wrapup, and host commit isolation. + +### Decision, Change, Or Tradeoff + +- Added `npm run smoke:finish-loop` using `scripts/run-smoke-finish-loop.py` and a temp git-backed BMAD-style fixture. +- The runner seeds a three-story, two-epic state and proves automate `done` plus non-blocking `skip`, incomplete review diagnostics, review completion, smoke-repo-only commits, sprint/story source-of-truth finalization, epic completion helpers, retro-agent and retro build-cmd coverage, state-recorded skipped retrospective semantics, continuation into a later epic, execution-complete/wrapup transitions, final metrics, learnings output, and marker removal. +- Commit isolation is enforced by a runner target guard that rejects the host repo before `commit-story`, then host HEAD/status is compared before and after the smoke. The runner also supports an explicit `--allow-unsafe-repo` manual override for debugging. +- The report keeps durable diagnostics under `.smoke/finish-loop-diagnostics/`, including a state document copy and temp smoke repo `git-log.txt`, because the working temp repo is deleted after command exit. +- Retrospective execution remains simulated as skipped deterministic output; no live retrospective agent is spawned. This matches the deterministic boundary while proving the non-blocking state/log contract. + +### User Impact + +- Finish-loop readiness now has a fast local gate that can prove commit safety without mutating the host checkout. +- Phase 06 can promote `smoke:finish-loop` into the deterministic full smoke or verify gates. + +## 2026-06-02 - phase 04 mode coverage + +### Context + +- Phase 04 required deterministic coverage for create startup, resume, validate, edit, marker lifecycle, and direct state/artifact assertions. + +### Decision, Change, Or Tradeoff + +- Added `npm run smoke:modes` using `scripts/run-smoke-modes.py` and a temp BMAD-style `.agents` fixture. +- The mode smoke asserts invalid range empty-selection behavior, stop-hook configured/pending-trust/failure states, sprint-status present/missing preconditions, state discovery, explicit path resume summary, no-incomplete fresh-create fallback, workflow-derived resume/edit menus and route hints, resume branch-equivalent helper checks, marker path/heartbeat/block/allow behavior, state validation, structure issue reporting, source-of-truth mismatch surfacing through the shared review verifier, and helper-backed edit save/discard/edit-more contracts. +- State/artifact assertions include rendered agent config, progress row metrics, action-log deltas, complexity/agents artifact paths, edit-time artifact path updates, simulated child dev log, compact mode report, parsed marker JSON, heartbeat mutation, and dynamic `.gitignore` entries. +- Fixture writes to story files and `sprint-status.yaml` are simulated child workflow output for source-of-truth checks; they are not treated as orchestrator-owned mutations. +- Fully interactive edit menu prompts and docs-path prompt behavior remain workflow-only because no deterministic helper exists for them yet. Phase 04 covers deterministic helper-backed branches and records route hints instead of trying to automate conversational waits. + +### User Impact + +- Resume/validate/edit now have fast local deterministic coverage suitable for future `verify` promotion. +- Prepared `.smoke/gunz` create/dev checks remain explicit external-flow gates and are not required by `smoke:modes`. + +## 2026-06-02 - phase 03 runtime helper contracts + +### Context + +- Phase 03 required deterministic helper contract coverage before broader lifecycle smokes. + +### Decision, Change, Or Tradeoff + +- Added `npm run smoke:contracts` as a focused unittest gate over parser, monitor, runner, build-cmd, state-update, runtime-policy, state metadata, marker/root, and success-verifier contract suites. +- Added `tests/test_runtime_helper_contracts.py` for missing parser subprocess, monitor terminal-state, build-cmd branch, runner edge-state, and `tmux-wrapper spawn` runner-mode coverage. +- Kept production helper code unchanged; the phase exposed assertion wording drift only (`state file unreadable` for missing state files). +- `parse-output`'s JSON decode branch is defensive because `extract_json_line` already filters invalid JSON before returning a candidate line; the enforced contract is no-json or schema-invalid failure payloads. + +### User Impact + +- Helper drift now fails through a named fast gate instead of being hidden behind broad smoke success. +- `monitor-session` terminal failures still usually exit `0`; callers must read JSON `final_state`, `exit_reason`, and `output_verified`. + +## 2026-06-02 - phase 02 package and prepared repo contracts + +### Context + +- Phase 02 required package tarball identity and prepared-repo installed-file proof. + +### Decision, Change, Or Tradeoff + +- Added `npm run pack:assert` using `npm pack --dry-run --json` plus `npm pack --json --pack-destination `. +- `pack:assert` now checks required package files, executable modes, forbidden generated/cache files, package identity, tarball SHA256, and selected tarball member checksums. +- `smoke:prepare` now writes `.smoke/PACKAGE_IDENTITY.json` and `.smoke/INSTALLED_AUTOMATOR_MANIFEST.json`. +- Prepared `.smoke/gunz` install verification compares selected installed `.claude/skills` files against the current tarball checksums. +- `.agents/skills` and `.codex/skills` are classified as `spec-only` for prepared gunz because BMAD prep uses `--tools claude-code`, which only creates complete `.claude/skills` dependency entrypoints. + +### User Impact + +- Prepared smoke runs now fail if `.smoke/gunz` is still using a stale same-version tarball. +- The Phase 02 verification caught and replaced a stale installed workflow (`1.12.0`) with the current `1.15.0` tarball install. + +## 2026-06-02 - phase 01 baseline and version inputs + +### Context + +- Phase 01 required a coverage baseline plus deterministic metadata and smoke input checks. + +### Decision, Change, Or Tradeoff + +- Added `coverage-baseline.md` as the source-of-truth Phase 01 inventory for current deterministic smoke facts and gaps. +- Fixed stale `skills/bmad-story-automator/workflow.md` frontmatter from `1.12.0` to `1.15.0`. +- Added `npm run version:check` to compare package, plugin, marketplace, module, Python, runtime, and workflow versions. +- Kept `bmad-method@next` as the BMAD installer input for now, but made it explicit through `npm run smoke:input-check` and `.smoke/SMOKE_INPUTS.json` recording during `smoke:prepare`. +- `smoke:prepare` now installs the resolved `bmad-method@` from that manifest instead of resolving the moving dist-tag twice. + +### User Impact + +- Release metadata drift is now caught before smoke runs. +- Prepared smoke runs still start from the moving BMAD Method npm `next` dist-tag, but each run records the resolved version/integrity and installs that resolved version. + Phase 02 can decide whether to replace that with a pinned installer version. + +## 2026-06-02 - plan creation + +### Context + +- The deterministic smoke suite currently covers planning/create and two-story dev-loop plumbing. + +### Decision, Change, Or Tradeoff + +- The plan separates live LLM quality from deterministic control-plane verification. Deterministic gates should prove helper contracts, source-of-truth transitions, state updates, package/install determinism, and mode routing. +- The plan treats automator validate and edit modes as public surfaces that require deterministic coverage, not optional documentation-only flows. + +### User Impact + +- The next implementation work should not stop at create/dev. It needs to verify review, finalize, retrospective, wrapup, resume, validate, edit, marker, parser, monitor, and packaging behavior. +- `bmad-method@next` and stale workflow version metadata are known risks to resolve early. + +## 2026-06-02 - oracle review application + +### Context + +- Oracle reviewed the deterministic smoke plan and attached critical source paths from the 2026-06-02 bundle. + +### Decision, Change, Or Tradeoff + +- The plan now treats Phase 02 package/prep identity, Phase 03 helper JSON contracts, and Phase 05 review/finish-loop coverage as release-blocking. +- Default `npm run verify` should target fast local deterministic gates: `test:python`, `version:check`, `pack:assert`, `test:cli`, `smoke:contracts`, `smoke:modes`, and `test:smoke`. +- Prepared-repo reset/network checks should remain explicit through `smoke:prepare`, `smoke:run`, `smoke:dev-loop`, `smoke:finish-loop`, and a wrapper such as `smoke:deterministic-full`. +- Phase 04 should prefer temp BMAD-style fixtures for resume/validate/edit and only use `.smoke/gunz` for realistic external flow coverage. +- Finish-loop commit/finalize needs a hard target-repo safety guard, not only an after-the-fact host HEAD/status sentinel. + +### User Impact + +- The plan is stricter now: smoke readiness cannot be claimed after only create/dev smokes. +- Implementers should extract shared smoke utilities instead of growing large repeated runners. +- Fixture writes to story files or `sprint-status.yaml` must be described as simulated child workflow output, not automator-owned mutation. diff --git a/docs/versioning.md b/docs/versioning.md index 13ebd809..618eac24 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -148,6 +148,7 @@ For a release version `X.Y.Z`, update these files: - `skills/module.yaml`: `module_version` - `skills/bmad-story-automator/pyproject.toml`: Python package version - `skills/bmad-story-automator/src/story_automator/__init__.py`: runtime version +- `skills/bmad-story-automator/workflow.md`: workflow frontmatter version - docs or changelog entries that mention the shipped version For preview versions, use semver in Node/plugin/module metadata and PEP 440 in @@ -200,6 +201,7 @@ v1.16.0-next.0 ```bash npm run verify npm pack --dry-run +npm run smoke:deterministic-full ``` 4. Commit the preview. @@ -207,7 +209,9 @@ npm pack --dry-run ```bash git add package.json .claude-plugin/plugin.json .claude-plugin/marketplace.json \ skills/module.yaml skills/bmad-story-automator/pyproject.toml \ - skills/bmad-story-automator/src/story_automator/__init__.py + skills/bmad-story-automator/src/story_automator/__init__.py \ + skills/bmad-story-automator/workflow.md \ + git commit -m "chore: prepare next preview" ``` @@ -299,6 +303,7 @@ Update every file listed in [Files To Bump](#files-to-bump). ```bash npm run verify npm pack --dry-run +npm run smoke:deterministic-full ``` 5. Commit the stable bump if it was not already part of the merge. @@ -306,7 +311,9 @@ npm pack --dry-run ```bash git add package.json .claude-plugin/plugin.json .claude-plugin/marketplace.json \ skills/module.yaml skills/bmad-story-automator/pyproject.toml \ - skills/bmad-story-automator/src/story_automator/__init__.py + skills/bmad-story-automator/src/story_automator/__init__.py \ + skills/bmad-story-automator/workflow.md \ + git commit -m "chore: release v1.16.0" ``` @@ -446,6 +453,7 @@ For each preview or stable release, record: - changed version files - `npm run verify` result - `npm pack --dry-run` result +- `npm run smoke:deterministic-full` result for release candidates - install smoke commands and manifest excerpts - npm publish status, including skipped publishes - rollback note and known installer caveats diff --git a/package.json b/package.json index d668c037..31c4a25c 100644 --- a/package.json +++ b/package.json @@ -21,11 +21,23 @@ ], "scripts": { "build:python": "python3 -m build skills/bmad-story-automator", + "version:check": "python3 scripts/check-version-alignment.py", + "smoke:input-check": "PYTHONPATH=scripts python3 scripts/check-smoke-inputs.py", "test:cli": "PYTHONPATH=skills/bmad-story-automator/src python3 -m story_automator --help >/dev/null && skills/bmad-story-automator/scripts/story-automator --help >/dev/null", + "pack:assert": "PYTHONPATH=scripts python3 scripts/assert-package-contracts.py", + "smoke:contracts": "PYTHONPATH=skills/bmad-story-automator/src python3 scripts/run-smoke-contracts.py", + "smoke:modes": "PYTHONPATH=skills/bmad-story-automator/src python3 scripts/run-smoke-modes.py", + "smoke:finish-loop": "PYTHONPATH=skills/bmad-story-automator/src python3 scripts/run-smoke-finish-loop.py", "pack:dry-run": "npm pack --dry-run", "test:python": "PYTHONPATH=skills/bmad-story-automator/src python3 -m unittest discover -s tests", "test:smoke": "bash scripts/smoke-test.sh", - "verify": "npm run test:python && npm run pack:dry-run && npm run test:cli && npm run test:smoke" + "smoke:prepare": "python3 scripts/prepare-smoke-test.py", + "smoke:run": "python3 scripts/run-smoke-automator.py", + "smoke:dev-loop": "python3 scripts/run-smoke-dev-loop.py", + "smoke:deterministic-full": "npm run smoke:prepare -- --reset && npm run smoke:run && npm run smoke:dev-loop && npm run smoke:finish-loop", + "smoke:auto": "npm run smoke:prepare -- --reset && npm run smoke:run", + "smoke:dev-auto": "npm run smoke:prepare -- --reset && npm run smoke:dev-loop", + "verify": "npm run test:python && npm run version:check && npm run pack:assert && npm run test:cli && npm run smoke:contracts && npm run smoke:modes && npm run test:smoke" }, "engines": { "node": ">=18" diff --git a/scripts/assert-package-contracts.py b/scripts/assert-package-contracts.py new file mode 100644 index 00000000..577768b2 --- /dev/null +++ b/scripts/assert-package-contracts.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from smoke_prep.config import repo_root +from smoke_prep.package_contracts import assert_package_contract +from smoke_prep.process import SmokeError + + +def main() -> int: + try: + identity = assert_package_contract(repo_root()) + except (OSError, subprocess.CalledProcessError, ValueError, SmokeError) as exc: + print(f"package contract failed: {exc}", file=sys.stderr) + return 1 + + print("package contract ok") + print(json.dumps(identity, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check-smoke-inputs.py b/scripts/check-smoke-inputs.py new file mode 100644 index 00000000..e81de7f5 --- /dev/null +++ b/scripts/check-smoke-inputs.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import sys +from subprocess import CalledProcessError + +from smoke_prep.inputs import smoke_inputs +from smoke_prep.process import SmokeError + + +def main() -> int: + try: + inputs = smoke_inputs() + except (CalledProcessError, OSError, ValueError, SmokeError) as exc: + print(f"smoke input determinism failed: {exc}", file=sys.stderr) + return 1 + + try: + gunz = inputs["gunz"] + bmad = inputs["bmadMethod"] + repo = gunz["repo"] + branch = gunz["branch"] + commit = gunz["commit"] + spec = bmad["spec"] + resolved_version = bmad["resolvedVersion"] + install_spec = bmad["installSpec"] + integrity = bmad["integrity"] + except (KeyError, TypeError) as exc: + print(f"smoke input determinism failed: malformed payload: {exc}", file=sys.stderr) + return 1 + + print("smoke input determinism ok") + print(f"- repo: {repo}") + print(f"- branch: {branch}") + print(f"- commit: {commit}") + print(f"- bmad method npm spec: {spec}") + print(f"- bmad method resolved version: {resolved_version}") + print(f"- bmad method install spec: {install_spec}") + print(f"- bmad method integrity: {integrity}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check-version-alignment.py b/scripts/check-version-alignment.py new file mode 100644 index 00000000..e92a4e45 --- /dev/null +++ b/scripts/check-version-alignment.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import re +import sys +import tomllib +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def read_json(path: str) -> dict: + return json.loads((ROOT / path).read_text(encoding="utf-8")) + + +def read_text(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def yaml_scalar(text: str, key: str) -> str: + match = re.search(rf"^{re.escape(key)}:\s*[\"']?([^\"'\n]+)[\"']?\s*$", text, re.M) + if not match: + raise ValueError(f"missing YAML scalar: {key}") + return match.group(1) + + +def workflow_frontmatter_version(text: str) -> str: + match = re.match(r"---\n(.*?)\n---", text, re.S) + if not match: + raise ValueError("missing workflow frontmatter") + return yaml_scalar(match.group(1), "version") + + +def python_version(text: str, path: str) -> str: + match = re.search(r'__version__\s*=\s*"([^"]+)"', text) + if not match: + raise ValueError(f"missing Python __version__ assignment: {path}") + return match.group(1) + + +def marketplace_plugin_version(marketplace: dict, plugin: dict) -> str: + plugin_name = plugin.get("name") + if not isinstance(plugin_name, str) or not plugin_name: + raise ValueError(".claude-plugin/plugin.json missing non-empty name") + + entries = marketplace.get("plugins") + if not isinstance(entries, list): + raise ValueError(".claude-plugin/marketplace.json missing plugins list") + + matches = [entry for entry in entries if isinstance(entry, dict) and entry.get("name") == plugin_name] + if not matches: + raise ValueError(f".claude-plugin/marketplace.json missing plugin: {plugin_name}") + if len(matches) > 1: + raise ValueError(f".claude-plugin/marketplace.json has duplicate plugin: {plugin_name}") + + version = matches[0].get("version") + if not isinstance(version, str) or not version: + raise ValueError(f".claude-plugin/marketplace.json plugin {plugin_name} missing non-empty version") + return version + + +def main() -> int: + package = read_json("package.json") + plugin = read_json(".claude-plugin/plugin.json") + marketplace = read_json(".claude-plugin/marketplace.json") + pyproject = tomllib.loads(read_text("skills/bmad-story-automator/pyproject.toml")) + init_text = read_text("skills/bmad-story-automator/src/story_automator/__init__.py") + + surfaces = { + "package.json": package["version"], + ".claude-plugin/plugin.json": plugin["version"], + ".claude-plugin/marketplace.json": marketplace_plugin_version(marketplace, plugin), + "skills/module.yaml": yaml_scalar(read_text("skills/module.yaml"), "module_version"), + "skills/bmad-story-automator/pyproject.toml": pyproject["project"]["version"], + "skills/bmad-story-automator/src/story_automator/__init__.py": python_version( + init_text, + "skills/bmad-story-automator/src/story_automator/__init__.py", + ), + "skills/bmad-story-automator/workflow.md": workflow_frontmatter_version( + read_text("skills/bmad-story-automator/workflow.md") + ), + } + + expected = package["version"] + mismatches = { + path: version for path, version in surfaces.items() if version != expected + } + if mismatches: + print(f"version alignment failed; expected {expected}", file=sys.stderr) + for path, version in mismatches.items(): + print(f"- {path}: {version}", file=sys.stderr) + return 1 + + print(f"version alignment ok: {expected}") + for path in sorted(surfaces): + print(f"- {path}: {surfaces[path]}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (KeyError, ValueError, json.JSONDecodeError, tomllib.TOMLDecodeError) as exc: + print(f"version alignment failed: {exc}", file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/prepare-smoke-test.py b/scripts/prepare-smoke-test.py new file mode 100644 index 00000000..158c4f21 --- /dev/null +++ b/scripts/prepare-smoke-test.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +"""Prepare a pinned external BMAD project for Story Automator smoke testing.""" + +from smoke_prep.cli import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run-smoke-automator.py b/scripts/run-smoke-automator.py new file mode 100644 index 00000000..09fd5842 --- /dev/null +++ b/scripts/run-smoke-automator.py @@ -0,0 +1,487 @@ +#!/usr/bin/env python3 +"""Run the pinned gunz Story Automator smoke deterministically.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from smoke_prep.config import repo_root +from smoke_prep.process import SmokeError, deterministic_smoke_env, ensure_tool +from smoke_prep.workspace import resolve_workspace + + +STORY_ID = "1.1" +OUTPUT_FOLDER = Path("_bmad-output/story-automator") +IMPLEMENTATION_FOLDER = Path("_bmad-output/implementation-artifacts") +EPIC_FILE = Path("_bmad-output/planning-artifacts/epics.md") +SPRINT_STATUS = IMPLEMENTATION_FOLDER / "sprint-status.yaml" +SKILL_ROOT = Path(".claude/skills/bmad-story-automator") +HELPER = SKILL_ROOT / "scripts/story-automator" +RULES = SKILL_ROOT / "data/complexity-rules.json" +STATE_TEMPLATE = SKILL_ROOT / "templates/state-document.md" +AGENT_CONFIG = {"defaultPrimary": "codex", "defaultFallback": False} +RUN_TIMEOUT_SECONDS = 900 + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run deterministic Story Automator smoke checks against a prepared gunz repo.", + ) + parser.add_argument( + "--workspace", + default=".smoke", + help="Repo-relative smoke workspace produced by smoke:prepare.", + ) + parser.add_argument( + "--story", + default=STORY_ID, + choices=[STORY_ID], + help="Story ID to exercise. Defaults to 1.1.", + ) + parser.add_argument( + "--keep-artifacts", + action="store_true", + help="Do not clear prior smoke-generated story/state artifacts before running.", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(sys.argv[1:] if argv is None else argv) + root = repo_root() + try: + ensure_tool("git") + workspace = resolve_workspace(root, args.workspace) + project = workspace / "gunz" + if not project.is_dir(): + raise SmokeError(f"prepared project missing: {project}. Run smoke:prepare first.") + runner = SmokeRunner(root=root, workspace=workspace, project=project, story_id=args.story) + summary = runner.run(reset_artifacts=not args.keep_artifacts) + except (OSError, subprocess.CalledProcessError, SmokeError, ValueError) as exc: + print(f"smoke run failed: {exc}", file=sys.stderr) + return 1 + + print("") + print("smoke run ok") + print(json.dumps(summary, indent=2)) + return 0 + + +class SmokeRunner: + def __init__(self, *, root: Path, workspace: Path, project: Path, story_id: str) -> None: + self.root = root + self.workspace = workspace + self.project = project + self.story_id = story_id + self.run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + self.env = deterministic_smoke_env(project) + self.helper = project / HELPER + + def close(self) -> None: + return None + + + def run(self, *, reset_artifacts: bool) -> dict[str, Any]: + self._assert_prepared() + if reset_artifacts: + self._reset_generated_artifacts() + + hook = self._helper_json( + "ensure-stop-hook", + "--settings", + ".claude/settings.json", + "--command", + ".claude/skills/bmad-story-automator/scripts/story-automator", + "stop-hook", + "--timeout", + "10", + ) + self._write_init_log(hook) + + epic = self._helper_json("parse-epic", "--file", str(EPIC_FILE)) + stories = epic.get("stories") + if not isinstance(stories, list) or not stories: + raise SmokeError("parse-epic returned no stories") + ids_csv = ",".join(str(story.get("storyId")) for story in stories if isinstance(story, dict)) + story_count = int(epic.get("count") or len(stories)) + selected = self._helper_json("parse-story-range", "--input", self.story_id, "--total", str(story_count), "--ids", ids_csv) + selected_ids = selected.get("storyIds") + if selected_ids != [self.story_id]: + raise SmokeError(f"story range did not select only {self.story_id}: {selected}") + story = self._helper_json("parse-story", "--epic", str(EPIC_FILE), "--story", self.story_id, "--rules", str(RULES)) + complexity = story.get("complexity") + if not isinstance(complexity, dict) or not complexity.get("level"): + raise SmokeError("parse-story did not return complexity") + + before_count = self._story_count() + preflight_path = self._write_preflight(epic, selected, story) + state_path = self._build_state(epic, selected) + complexity_path = self._write_complexity(state_path, story) + agents_path = self._build_agents(state_path, complexity_path) + self._finalize_state(state_path, agents_path, complexity_path) + self._create_marker(state_path) + story_path = self._write_story_artifact(story) + self._update_sprint_status(story_path) + after_count = self._story_count() + + state_validation = self._helper_json("validate-state", "--state", str(state_path)) + if state_validation.get("ok") is not True or state_validation.get("issueCount") != 0: + raise SmokeError(f"state validation failed: {state_validation}") + story_validation = self._helper_json( + "validate-story-creation", + "check", + self.story_id, + "--before", + str(before_count), + "--after", + str(after_count), + "--state-file", + str(state_path), + ) + if story_validation.get("verified") is not True: + raise SmokeError(f"story validation failed: {story_validation}") + sprint_status = self._helper_json("orchestrator-helper", "sprint-status", "get", self.story_id) + if sprint_status.get("status") != "ready-for-dev": + raise SmokeError(f"sprint status not ready-for-dev: {sprint_status}") + + report_path = self._write_report( + state_path=state_path, + preflight_path=preflight_path, + complexity_path=complexity_path, + agents_path=agents_path, + story_path=story_path, + story=story, + state_validation=state_validation, + story_validation=story_validation, + sprint_status=sprint_status, + ) + self._remove_marker() + return { + "project": str(self.project), + "report": str(report_path), + "story": self.story_id, + "story_file": str(story_path), + "state_file": str(state_path), + "complexity": complexity, + "sprint_status": sprint_status, + } + + def _assert_prepared(self) -> None: + for rel in (HELPER, EPIC_FILE, RULES, STATE_TEMPLATE, SPRINT_STATUS): + path = self.project / rel + if not path.is_file(): + raise SmokeError(f"prepared smoke file missing: {path}") + self._run(str(self.helper), "--help") + + def _reset_generated_artifacts(self) -> None: + shutil.rmtree(self.project / OUTPUT_FOLDER, ignore_errors=True) + marker_info = self._marker_path_info() + marker = Path(str(marker_info["file"])) + marker.unlink(missing_ok=True) + for path in (self.project / IMPLEMENTATION_FOLDER).glob(f"{self._story_prefix()}-*.md"): + path.unlink() + sprint = self.project / SPRINT_STATUS + text = sprint.read_text(encoding="utf-8") + text = re.sub(r"(?m)^(\s*epic-1:\s*).*$", r"\1backlog", text) + text = re.sub(rf"(?m)^(\s*{re.escape(self._story_slug())}:\s*).*$", r"\1backlog", text) + sprint.write_text(text, encoding="utf-8") + + def _write_init_log(self, hook: dict[str, Any]) -> None: + folder = self.project / OUTPUT_FOLDER + folder.mkdir(parents=True, exist_ok=True) + changed = str(hook.get("changed", "")).lower() + (folder / f"init-log-{self.run_id}.md").write_text( + f"[{self._iso_now()}] init: stop-hook-changed={changed} existing_state=\n", + encoding="utf-8", + ) + + def _write_preflight(self, epic: dict[str, Any], selected: dict[str, Any], story: dict[str, Any]) -> Path: + path = self.project / OUTPUT_FOLDER / f"preflight-1-{self.run_id}.md" + complexity = story["complexity"] + content = [ + "# Preflight Snapshot", + "", + f"- Timestamp: {self.run_id}", + f"- Epic path: {EPIC_FILE}", + f"- Epic name: {epic.get('epicTitle')}", + f"- Story count: {epic.get('count')}", + f"- Selected count: {selected.get('count')}", + f"- Selected IDs: {', '.join(selected.get('storyIds', []))}", + "- Custom instructions:", + "", + "## Complexity Summary", + f"- {self.story_id} | {complexity.get('level')} | score={complexity.get('score')}", + "", + "## Stories JSON", + "```json", + json.dumps([_story_summary(story)], indent=2), + "```", + "", + ] + path.write_text("\n".join(content), encoding="utf-8") + return path.relative_to(self.project) + + def _build_state(self, epic: dict[str, Any], selected: dict[str, Any]) -> Path: + config = { + "epic": self.story_id.split(".", 1)[0], + "epicName": epic.get("epicTitle", ""), + "storyRange": selected.get("storyIds", []), + "status": "READY", + "currentStory": None, + "currentStep": "preflight", + "aiCommand": "codex exec --full-auto", + "customInstructions": "", + "overrides": {"skipAutomate": True, "maxParallel": 1}, + "agentConfig": AGENT_CONFIG, + } + result = self._helper_json( + "build-state-doc", + "--template", + str(STATE_TEMPLATE), + "--output-folder", + str(OUTPUT_FOLDER), + "--config-json", + json.dumps(config), + ) + path = str(result.get("path") or "") + if not path: + raise SmokeError(f"build-state-doc did not return a path: {result}") + return Path(path) + + def _write_complexity(self, state_path: Path, story: dict[str, Any]) -> Path: + path = OUTPUT_FOLDER / f"complexity-{state_path.stem}.json" + (self.project / path).write_text( + json.dumps({"stories": [_story_summary(story)]}, indent=2) + "\n", + encoding="utf-8", + ) + return path + + def _build_agents(self, state_path: Path, complexity_path: Path) -> Path: + path = OUTPUT_FOLDER / "agents" / f"agents-{state_path.stem}.md" + result = self._helper_json( + "orchestrator-helper", + "agents-build", + "--state-file", + str(state_path), + "--complexity-file", + str(complexity_path), + "--output", + str(path), + "--config-json", + json.dumps(AGENT_CONFIG), + ) + if result.get("ok") is not True: + raise SmokeError(f"agents-build failed: {result}") + return path + + def _finalize_state(self, state_path: Path, agents_path: Path, complexity_path: Path) -> None: + for key, value in ( + ("agentsFile", str(agents_path)), + ("complexityFile", str(complexity_path)), + ("status", "IN_PROGRESS"), + ("currentStory", self.story_id), + ("currentStep", "step-03-execute"), + ("lastUpdated", self._iso_now()), + ): + result = self._helper_json("orchestrator-helper", "state-update", str(state_path), "--set", f"{key}={value}") + if result.get("ok") is not True: + raise SmokeError(f"state-update failed for {key}: {result}") + + def _create_marker(self, state_path: Path) -> None: + marker_info = self._marker_path_info() + self._helper_json("ensure-marker-gitignore", "--gitignore", ".gitignore", "--entry", str(marker_info["entry"])) + slug = self._helper_json("derive-project-slug").get("slug") or "gunz" + result = self._run( + str(self.helper), + "orchestrator-helper", + "marker", + "create", + "--epic", + "1", + "--story", + self.story_id, + "--remaining", + "1", + "--state-file", + str(state_path), + "--project-slug", + str(slug), + "--pid", + str(os.getpid()), + "--heartbeat", + self._iso_now(), + ) + if "Marker created:" not in result.stdout: + raise SmokeError(f"marker create failed: {result.stdout}") + marker = Path(str(marker_info["file"])) + if not marker.is_file(): + raise SmokeError(f"marker was not created at active path: {marker}") + + def _remove_marker(self) -> None: + result = self._run(str(self.helper), "orchestrator-helper", "marker", "remove") + marker = Path(str(self._marker_path_info()["file"])) + if result.returncode != 0 or marker.exists(): + raise SmokeError(f"marker remove failed: {result.stdout}") + + def _marker_path_info(self) -> dict[str, Any]: + marker_info = self._helper_json("orchestrator-helper", "marker", "path") + if not marker_info.get("file") or not marker_info.get("entry"): + raise SmokeError(f"marker path helper returned incomplete payload: {marker_info}") + return marker_info + + def _write_story_artifact(self, story: dict[str, Any]) -> Path: + folder = self.project / IMPLEMENTATION_FOLDER + folder.mkdir(parents=True, exist_ok=True) + title = str(story.get("title") or "Story") + criteria = story.get("acceptanceCriteria") + ac_lines = [str(item) for item in criteria] if isinstance(criteria, list) else [] + path = folder / f"{self._story_slug()}.md" + content = [ + f"# Story {self.story_id}: {title}", + "", + "Status: ready-for-dev", + "", + "", + "", + "## Story", + "", + str(story.get("description") or ""), + "", + "## Acceptance Criteria", + "", + *[f"- {line}" for line in ac_lines[:12]], + "", + "## Dev Agent Record", + "", + "### Completion Notes List", + "", + "- Deterministic smoke created this artifact without invoking a live LLM.", + "", + "### File List", + "", + f"- `{IMPLEMENTATION_FOLDER / path.name}`", + "", + ] + path.write_text("\n".join(content), encoding="utf-8") + return path.relative_to(self.project) + + def _update_sprint_status(self, story_path: Path) -> None: + sprint = self.project / SPRINT_STATUS + text = sprint.read_text(encoding="utf-8") + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + text = re.sub(r"(?m)^# last_updated:.*$", f"# last_updated: {today}", text) + text = re.sub(r"(?m)^last_updated:.*$", f"last_updated: {today}", text) + text = re.sub(r"(?m)^(\s*epic-1:\s*).*$", r"\1in-progress", text) + text = re.sub(rf"(?m)^(\s*{re.escape(story_path.stem)}:\s*).*$", r"\1ready-for-dev", text) + sprint.write_text(text, encoding="utf-8") + + def _write_report( + self, + *, + state_path: Path, + preflight_path: Path, + complexity_path: Path, + agents_path: Path, + story_path: Path, + story: dict[str, Any], + state_validation: dict[str, Any], + story_validation: dict[str, Any], + sprint_status: dict[str, Any], + ) -> Path: + report = self.workspace / "AUTOMATED_SMOKE_REPORT.md" + complexity = story["complexity"] + lines = [ + "# Automated Story Automator Smoke", + "", + f"- Timestamp: {self._iso_now()}", + f"- Project: `{self.project}`", + f"- Story: `{self.story_id}`", + f"- Story title: `{story.get('title')}`", + f"- Complexity: `{complexity.get('level')}` score `{complexity.get('score')}`", + "", + "## Artifacts", + "", + f"- `{preflight_path}`", + f"- `{state_path}`", + f"- `{complexity_path}`", + f"- `{agents_path}`", + f"- `{story_path}`", + f"- `{SPRINT_STATUS}`", + "", + "## Verification", + "", + f"- State: `{json.dumps(state_validation, separators=(',', ':'))}`", + f"- Story creation: `{json.dumps(story_validation, separators=(',', ':'))}`", + f"- Sprint status: `{json.dumps(sprint_status, separators=(',', ':'))}`", + "", + ] + report.write_text("\n".join(lines), encoding="utf-8") + return report + + def _story_count(self) -> int: + result = self._run(str(self.helper), "validate-story-creation", "count", self.story_id) + return int(result.stdout.strip()) + + def _story_prefix(self) -> str: + return self.story_id.replace(".", "-") + + def _story_slug(self) -> str: + status = self._helper_json("orchestrator-helper", "sprint-status", "get", self.story_id) + story_key = str(status.get("story") or "") + if status.get("found") is True and story_key: + return story_key + parsed = self._helper_json("parse-story", "--epic", str(EPIC_FILE), "--story", self.story_id, "--rules", str(RULES)) + return f"{self._story_prefix()}-{_slugify(str(parsed.get('title') or 'story'))}" + + def _helper_json(self, *args: str) -> dict[str, Any]: + result = self._run(str(self.helper), *args) + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise SmokeError(f"helper did not return json for {' '.join(args)}: {result.stdout}") from exc + if payload.get("ok") is False: + raise SmokeError(f"helper failed for {' '.join(args)}: {payload}") + return payload + + def _run(self, *args: str) -> subprocess.CompletedProcess[str]: + try: + return subprocess.run( + list(args), + cwd=self.project, + env=self.env, + text=True, + capture_output=True, + check=True, + timeout=RUN_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired as exc: + raise SmokeError(f"command timed out after {RUN_TIMEOUT_SECONDS}s: {' '.join(args)}") from exc + + @staticmethod + def _iso_now() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _story_summary(story: dict[str, Any]) -> dict[str, Any]: + return { + "storyId": story.get("storyId"), + "title": story.get("title"), + "complexity": story.get("complexity"), + } + + +def _slugify(value: str) -> str: + return "-".join(part for part in re.split(r"[^A-Za-z0-9]+", value.lower()) if part) or "story" + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run-smoke-contracts.py b/scripts/run-smoke-contracts.py new file mode 100644 index 00000000..74a7bcb4 --- /dev/null +++ b/scripts/run-smoke-contracts.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + + +TEST_MODULES = [ + "tests.test_runtime_helper_contracts", + "tests.test_orchestrator_parse", + "tests.test_success_verifiers", + "tests.test_tmux_runtime", + "tests.test_state_validation", + "tests.test_runtime_policy", + "tests.test_state_policy_metadata", + "tests.test_runtime_layout", + "tests.test_agent_config_model", +] + +ENVIRONMENT_SKIP_REASONS = {"tmux not available"} + + +def unexpected_skips(skipped: list[tuple[object, str]]) -> list[tuple[object, str]]: + return [(test, reason) for test, reason in skipped if str(reason) not in ENVIRONMENT_SKIP_REASONS] + + +def main() -> int: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + suite = unittest.defaultTestLoader.loadTestsFromNames(TEST_MODULES) + result = unittest.TextTestRunner(verbosity=1).run(suite) + if result.skipped: + unexpected = unexpected_skips(result.skipped) + if unexpected: + print(f"smoke:contracts got {len(unexpected)} unexpected skipped tests", file=sys.stderr) + for test, reason in unexpected: + print(f"- {test}: {reason}", file=sys.stderr) + return 1 + print(f"smoke:contracts skipped {len(result.skipped)} allowed environment-dependent tests", file=sys.stderr) + for test, reason in result.skipped: + print(f"- {test}: {reason}", file=sys.stderr) + return 0 if result.wasSuccessful() else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run-smoke-dev-loop.py b/scripts/run-smoke-dev-loop.py new file mode 100644 index 00000000..86bac387 --- /dev/null +++ b/scripts/run-smoke-dev-loop.py @@ -0,0 +1,456 @@ +#!/usr/bin/env python3 +"""Run deterministic two-story Story Automator dev-loop smoke checks.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from smoke_prep.config import repo_root +from smoke_prep.process import SmokeError, deterministic_smoke_env, ensure_tool +from smoke_prep.workspace import resolve_workspace + + +STORY_IDS = ("1.1", "1.2") +OUTPUT_FOLDER = Path("_bmad-output/story-automator") +IMPLEMENTATION_FOLDER = Path("_bmad-output/implementation-artifacts") +DEV_LOOP_FOLDER = OUTPUT_FOLDER / "dev-loop" +EPIC_FILE = Path("_bmad-output/planning-artifacts/epics.md") +SPRINT_STATUS = IMPLEMENTATION_FOLDER / "sprint-status.yaml" +SKILL_ROOT = Path(".claude/skills/bmad-story-automator") +HELPER = SKILL_ROOT / "scripts/story-automator" +RULES = SKILL_ROOT / "data/complexity-rules.json" +STATE_TEMPLATE = SKILL_ROOT / "templates/state-document.md" +AGENT_CONFIG = {"defaultPrimary": "codex", "defaultFallback": False} +RUN_TIMEOUT_SECONDS = 900 +PARSED_DEV = { + "status": "SUCCESS", + "tests_passed": True, + "build_passed": True, + "summary": "Deterministic smoke simulated a successful dev-story session.", + "next_action": "proceed", +} + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run deterministic two-story dev-loop smoke checks against prepared gunz.", + ) + parser.add_argument("--workspace", default=".smoke", help="Repo-relative workspace from smoke:prepare.") + parser.add_argument( + "--stories", + default=",".join(STORY_IDS), + choices=[",".join(STORY_IDS)], + help="Story IDs to exercise. Fixed to 1.1,1.2 for this smoke.", + ) + parser.add_argument("--keep-artifacts", action="store_true", help="Do not clear prior generated dev-loop artifacts.") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(sys.argv[1:] if argv is None else argv) + root = repo_root() + try: + ensure_tool("git") + workspace = resolve_workspace(root, args.workspace) + project = workspace / "gunz" + if not project.is_dir(): + raise SmokeError(f"prepared project missing: {project}. Run smoke:prepare first.") + runner = DevLoopSmokeRunner(root=root, workspace=workspace, project=project, story_ids=list(STORY_IDS)) + summary = runner.run(reset_artifacts=not args.keep_artifacts) + except (OSError, subprocess.CalledProcessError, SmokeError, ValueError) as exc: + print(f"dev-loop smoke failed: {exc}", file=sys.stderr) + return 1 + + print("") + print("dev-loop smoke ok") + print(json.dumps(summary, indent=2)) + return 0 + + +class DevLoopSmokeRunner: + def __init__(self, *, root: Path, workspace: Path, project: Path, story_ids: list[str]) -> None: + self.root = root + self.workspace = workspace + self.project = project + self.story_ids = story_ids + self.run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + self.env = deterministic_smoke_env(project) + self.helper = project / HELPER + + def close(self) -> None: + return None + + + def run(self, *, reset_artifacts: bool) -> dict[str, Any]: + self._assert_prepared() + if reset_artifacts: + self._reset_generated_artifacts() + + epic = self._helper_json("parse-epic", "--file", str(EPIC_FILE)) + story_count = int(epic.get("count") or 0) + ids_csv = ",".join(str(story.get("storyId")) for story in epic.get("stories", []) if isinstance(story, dict)) + selected = self._helper_json("parse-story-range", "--input", ",".join(self.story_ids), "--total", str(story_count), "--ids", ids_csv) + if selected.get("storyIds") != self.story_ids: + raise SmokeError(f"story range did not select exactly {self.story_ids}: {selected}") + + stories = [self._parse_story(story_id) for story_id in self.story_ids] + state_path = self._build_state(epic, selected) + complexity_path = self._write_complexity(state_path, stories) + agents_path = self._build_agents(state_path, complexity_path) + self._set_state_fields( + state_path, + agentsFile=str(agents_path), + complexityFile=str(complexity_path), + status="IN_PROGRESS", + currentStory=self.story_ids[0], + currentStep="step-03-execute", + lastUpdated=self._iso_now(), + ) + + results = [] + for index, story in enumerate(stories, start=1): + results.append(self._run_story_dev_loop(index, len(stories), state_path, story)) + + self._set_state_fields(state_path, currentStep="step-03a-execute-review", lastUpdated=self._iso_now()) + self._append_state_log(state_path, "Dev loop complete, proceeding to review phase") + + state_validation = self._helper_json("validate-state", "--state", str(state_path)) + if state_validation.get("ok") is not True or state_validation.get("issueCount") != 0: + raise SmokeError(f"state validation failed: {state_validation}") + report_path = self._write_report(state_path, complexity_path, agents_path, results, state_validation) + return {"project": str(self.project), "report": str(report_path), "stories": results, "state_file": str(state_path)} + + def _assert_prepared(self) -> None: + for rel in (HELPER, EPIC_FILE, RULES, STATE_TEMPLATE, SPRINT_STATUS): + path = self.project / rel + if not path.is_file(): + raise SmokeError(f"prepared smoke file missing: {path}") + self._run(str(self.helper), "--help") + self._run(str(self.helper), "tmux-wrapper", "build-cmd", "--help") + self._run(str(self.helper), "orchestrator-helper", "--help") + + def _reset_generated_artifacts(self) -> None: + shutil.rmtree(self.project / OUTPUT_FOLDER, ignore_errors=True) + (self.project / DEV_LOOP_FOLDER).mkdir(parents=True, exist_ok=True) + for story_id in self.story_ids: + for path in (self.project / IMPLEMENTATION_FOLDER).glob(f"{self._story_prefix(story_id)}-*.md"): + path.unlink() + sprint = self.project / SPRINT_STATUS + text = sprint.read_text(encoding="utf-8") + text = re.sub(r"(?m)^(\s*epic-1:\s*).*$", r"\1backlog", text) + for story_id in self.story_ids: + text = re.sub(rf"(?m)^(\s*{re.escape(self._story_slug(story_id))}:\s*).*$", r"\1backlog", text) + sprint.write_text(text, encoding="utf-8") + + def _parse_story(self, story_id: str) -> dict[str, Any]: + story = self._helper_json("parse-story", "--epic", str(EPIC_FILE), "--story", story_id, "--rules", str(RULES)) + if not isinstance(story.get("complexity"), dict): + raise SmokeError(f"parse-story did not return complexity for {story_id}") + return story + + def _build_state(self, epic: dict[str, Any], selected: dict[str, Any]) -> Path: + config = { + "epic": "1", + "epicName": epic.get("epicTitle", ""), + "storyRange": selected.get("storyIds", []), + "status": "READY", + "currentStory": None, + "currentStep": "preflight", + "aiCommand": "codex exec --full-auto", + "customInstructions": "Deterministic smoke for two-story dev loop.", + "overrides": {"skipAutomate": True, "maxParallel": 1}, + "agentConfig": AGENT_CONFIG, + } + result = self._helper_json( + "build-state-doc", + "--template", + str(STATE_TEMPLATE), + "--output-folder", + str(OUTPUT_FOLDER), + "--config-json", + json.dumps(config), + ) + path = str(result.get("path") or "") + if not path: + raise SmokeError(f"build-state-doc did not return a path: {result}") + return Path(path) + + def _write_complexity(self, state_path: Path, stories: list[dict[str, Any]]) -> Path: + path = OUTPUT_FOLDER / f"complexity-dev-loop-{state_path.stem}.json" + summaries = [{"storyId": item["storyId"], "title": item["title"], "complexity": item["complexity"]} for item in stories] + (self.project / path).write_text(json.dumps({"stories": summaries}, indent=2) + "\n", encoding="utf-8") + return path + + def _build_agents(self, state_path: Path, complexity_path: Path) -> Path: + path = OUTPUT_FOLDER / "agents" / f"agents-dev-loop-{state_path.stem}.md" + result = self._helper_json( + "orchestrator-helper", + "agents-build", + "--state-file", + str(state_path), + "--complexity-file", + str(complexity_path), + "--output", + str(path), + "--config-json", + json.dumps(AGENT_CONFIG), + ) + if result.get("ok") is not True: + raise SmokeError(f"agents-build failed: {result}") + return path + + def _run_story_dev_loop(self, index: int, total: int, state_path: Path, story: dict[str, Any]) -> dict[str, Any]: + story_id = str(story["storyId"]) + self._set_state_fields(state_path, currentStory=story_id, currentStep="step-03-execute", lastUpdated=self._iso_now()) + self._append_progress_row(state_path, story_id) + self._append_state_log(state_path, f"Starting story {story_id}") + + create_agent = self._helper_json("orchestrator-helper", "agents-resolve", "--state-file", str(state_path), "--story", story_id, "--task", "create") + dev_agent = self._helper_json("orchestrator-helper", "agents-resolve", "--state-file", str(state_path), "--story", story_id, "--task", "dev") + dev_cmd = self._run( + str(self.helper), + "tmux-wrapper", + "build-cmd", + "dev", + story_id, + "--agent", + str(dev_agent.get("primary") or "codex"), + "--state-file", + str(state_path), + ).stdout.strip() + if "codex exec" not in dev_cmd and "claude" not in dev_cmd: + raise SmokeError(f"dev build-cmd did not produce an agent command for {story_id}") + + story_path = self._write_story_artifact(story, "ready-for-dev") + self._update_sprint_status(story_id, "ready-for-dev") + create_validation = self._helper_json("orchestrator-helper", "verify-step", "create", story_id, "--state-file", str(state_path)) + if create_validation.get("verified") is not True: + raise SmokeError(f"create verifier failed for {story_id}: {create_validation}") + self._replace_progress_row(state_path, story_id, "done", "-", "-", "-", "-", "in-progress") + + dev_log = self._write_dev_log(story_id, dev_cmd) + parsed_dev = dict(PARSED_DEV) + if parsed_dev["next_action"] != "proceed": + raise SmokeError(f"dev parser fixture did not proceed for {story_id}") + story_path = self._write_story_artifact(story, "done") + self._write_story_dev_record(story_path, dev_log) + self._update_sprint_status(story_id, "done") + self._replace_progress_row(state_path, story_id, "done", "done", "-", "-", "-", "in-progress") + + file_status = self._helper_json("orchestrator-helper", "story-file-status", story_id) + sprint_status = self._helper_json("orchestrator-helper", "sprint-status", "get", story_id) + if file_status.get("status") != "done" or sprint_status.get("status") != "done": + raise SmokeError(f"dev status transition failed for {story_id}: {file_status} {sprint_status}") + return { + "story": story_id, + "index": index, + "total": total, + "story_file": str(story_path), + "dev_log": str(dev_log), + "create_agent": create_agent, + "dev_agent": dev_agent, + "parsed_dev": parsed_dev, + "file_status": file_status, + "sprint_status": sprint_status, + } + + def _write_story_artifact(self, story: dict[str, Any], status: str) -> Path: + folder = self.project / IMPLEMENTATION_FOLDER + folder.mkdir(parents=True, exist_ok=True) + story_id = str(story["storyId"]) + title = str(story.get("title") or "Story") + path = folder / f"{self._story_slug(story_id)}.md" + criteria = story.get("acceptanceCriteria") + ac_lines = [str(item) for item in criteria] if isinstance(criteria, list) else [] + content = [ + "---", + f"Title: Story {story_id}: {title}", + f"Status: {status}", + "---", + "", + f"# Story {story_id}: {title}", + "", + f"Status: {status}", + "", + "", + "", + "## Story", + "", + str(story.get("description") or ""), + "", + "## Acceptance Criteria", + "", + *[f"- {line}" for line in ac_lines[:12]], + "", + "## Dev Agent Record", + "", + "### Completion Notes List", + "", + f"- Deterministic dev-loop smoke marked story {status}.", + "", + "### File List", + "", + f"- `{IMPLEMENTATION_FOLDER / path.name}`", + "", + ] + path.write_text("\n".join(content), encoding="utf-8") + return path.relative_to(self.project) + + def _write_dev_log(self, story_id: str, dev_cmd: str) -> Path: + path = DEV_LOOP_FOLDER / f"dev-{self._story_prefix(story_id)}-{self.run_id}.log" + (self.project / path).parent.mkdir(parents=True, exist_ok=True) + payload = dict(PARSED_DEV) + lines = [ + f"[{self._iso_now()}] dev-story {story_id}", + f"COMMAND={dev_cmd}", + f"SUCCESS story={story_id} tests=true build=true", + json.dumps(payload, separators=(",", ":")), + "", + ] + (self.project / path).write_text("\n".join(lines), encoding="utf-8") + return path + + def _write_story_dev_record(self, story_path: Path, dev_log: Path) -> None: + path = self.project / story_path + text = path.read_text(encoding="utf-8") + text += f"\n### Debug Log References\n\n- `{dev_log}`\n" + path.write_text(text, encoding="utf-8") + + def _update_sprint_status(self, story_id: str, status: str) -> None: + sprint = self.project / SPRINT_STATUS + text = sprint.read_text(encoding="utf-8") + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + text = re.sub(r"(?m)^# last_updated:.*$", f"# last_updated: {today}", text) + text = re.sub(r"(?m)^last_updated:.*$", f"last_updated: {today}", text) + text = re.sub(r"(?m)^(\s*epic-1:\s*).*$", r"\1in-progress", text) + text = re.sub(rf"(?m)^(\s*{re.escape(self._story_slug(story_id))}:\s*).*$", rf"\1{status}", text) + sprint.write_text(text, encoding="utf-8") + + def _append_progress_row(self, state_path: Path, story_id: str) -> None: + state = self.project / state_path + text = state.read_text(encoding="utf-8") + if re.search(rf"(?m)^\| {re.escape(story_id)} \|", text): + return + row = f"| {story_id} | - | - | - | - | - | in-progress |" + text = text.replace("", f"\n{row}") + state.write_text(text, encoding="utf-8") + + def _replace_progress_row(self, state_path: Path, story_id: str, *cells: str) -> None: + state = self.project / state_path + row = f"| {story_id} | " + " | ".join(cells) + " |" + text = state.read_text(encoding="utf-8") + text = re.sub(rf"(?m)^\| {re.escape(story_id)} \|.*$", row, text) + state.write_text(text, encoding="utf-8") + + def _set_state_fields(self, state_path: Path, **fields: object) -> None: + for key, value in fields.items(): + result = self._helper_json("orchestrator-helper", "state-update", str(state_path), "--set", f"{key}={value}") + if result.get("ok") is not True: + raise SmokeError(f"state-update failed for {key}: {result}") + + def _append_state_log(self, state_path: Path, message: str) -> None: + state = self.project / state_path + text = state.read_text(encoding="utf-8") + text += f"\n- **[{self._iso_now()}]** {message}\n" + state.write_text(text, encoding="utf-8") + + def _write_report( + self, + state_path: Path, + complexity_path: Path, + agents_path: Path, + results: list[dict[str, Any]], + state_validation: dict[str, Any], + ) -> Path: + report = self.workspace / "AUTOMATED_DEV_LOOP_SMOKE_REPORT.md" + lines = [ + "# Automated Story Automator Dev Loop Smoke", + "", + f"- Timestamp: {self._iso_now()}", + f"- Project: `{self.project}`", + f"- Stories: `{', '.join(self.story_ids)}`", + f"- Scope: deterministic dev-loop plumbing; live implementation quality is not asserted.", + "", + "## Artifacts", + "", + f"- `{state_path}`", + f"- `{complexity_path}`", + f"- `{agents_path}`", + f"- `{DEV_LOOP_FOLDER}`", + "", + "## Verification", + "", + f"- State: `{json.dumps(state_validation, separators=(',', ':'))}`", + ] + for result in results: + lines.extend( + [ + "", + f"### Story {result['story']}", + "", + f"- Story file: `{result['story_file']}`", + f"- Dev log: `{result['dev_log']}`", + f"- Dev parser fixture: `{json.dumps(result['parsed_dev'], separators=(',', ':'))}`", + f"- Story file status: `{json.dumps(result['file_status'], separators=(',', ':'))}`", + f"- Sprint status: `{json.dumps(result['sprint_status'], separators=(',', ':'))}`", + ] + ) + report.write_text("\n".join(lines) + "\n", encoding="utf-8") + return report + + def _helper_json(self, *args: str) -> dict[str, Any]: + result = self._run(str(self.helper), *args) + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise SmokeError(f"helper did not return json for {' '.join(args)}: {result.stdout}") from exc + if payload.get("ok") is False: + raise SmokeError(f"helper failed for {' '.join(args)}: {payload}") + return payload + + def _run(self, *args: str) -> subprocess.CompletedProcess[str]: + try: + return subprocess.run( + list(args), + cwd=self.project, + env=self.env, + text=True, + capture_output=True, + check=True, + timeout=RUN_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired as exc: + raise SmokeError(f"command timed out after {RUN_TIMEOUT_SECONDS}s: {' '.join(args)}") from exc + + @staticmethod + def _story_prefix(story_id: str) -> str: + return story_id.replace(".", "-") + + def _story_slug(self, story_id: str) -> str: + status = self._helper_json("orchestrator-helper", "sprint-status", "get", story_id) + story_key = str(status.get("story") or "") + if status.get("found") is True and story_key: + return story_key + parsed = self._helper_json("parse-story", "--epic", str(EPIC_FILE), "--story", story_id, "--rules", str(RULES)) + return f"{self._story_prefix(story_id)}-{_slugify(str(parsed.get('title') or 'story'))}" + + @staticmethod + def _iso_now() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _slugify(value: str) -> str: + return "-".join(part for part in re.split(r"[^A-Za-z0-9]+", value.lower()) if part) or "story" + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run-smoke-finish-loop.py b/scripts/run-smoke-finish-loop.py new file mode 100644 index 00000000..00093b61 --- /dev/null +++ b/scripts/run-smoke-finish-loop.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import io +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import argparse +from contextlib import redirect_stdout +from datetime import datetime, timezone +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "skills" / "bmad-story-automator" / "src")) + +from story_automator.commands.basic import cmd_commit_story # noqa: E402 +from story_automator.commands.orchestrator import cmd_orchestrator_helper # noqa: E402 +from story_automator.commands.state import cmd_build_state_doc, cmd_state_metrics, cmd_validate_state # noqa: E402 +from story_automator.commands.tmux import cmd_tmux_wrapper # noqa: E402 +from smoke_prep.process import deterministic_smoke_env # noqa: E402 + + +RUN_TIMEOUT_SECONDS = 900 + + +class FinishSmokeError(Exception): + pass + + +class FinishLoopSmokeRunner: + def __init__(self, *, target_repo: Path | None = None, allow_unsafe_repo: bool = False) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.project = Path(self.tmp.name) / "finish-smoke" + self.output = self.project / "_bmad-output" / "story-automator" + self.artifacts = self.project / "_bmad-output" / "implementation-artifacts" + self.target_repo = target_repo + self.allow_unsafe_repo = allow_unsafe_repo + self.results: dict[str, object] = {} + + def close(self) -> None: + self.tmp.cleanup() + + def run(self) -> dict[str, object]: + host = self._host_sentinel() + self._install_fixture() + self._init_git() + state_file = self._build_state() + commit_repo = self._resolve_commit_repo() + marker = self._create_marker(state_file) + commits = [] + for story_id in ("1.1", "1.2", "2.1"): + commits.append(self._finish_story(state_file, story_id, commit_repo)) + self._maybe_run_retro(state_file, story_id) + self._complete_and_wrap(state_file, marker) + self._assert_host_unchanged(host) + return self._write_report(state_file, commits, commit_repo) + + def _install_fixture(self) -> None: + skills = self.project / ".agents" / "skills" + skills.mkdir(parents=True) + shutil.copytree(REPO_ROOT / "skills" / "bmad-story-automator", skills / "bmad-story-automator") + shutil.copytree(REPO_ROOT / "skills" / "bmad-story-automator-review", skills / "bmad-story-automator-review") + for name in ("bmad-create-story", "bmad-dev-story", "bmad-retrospective", "bmad-qa-generate-e2e-tests"): + folder = skills / name + folder.mkdir() + (folder / "SKILL.md").write_text(f"# {name}\n", encoding="utf-8") + (folder / "workflow.md").write_text(f"# {name}\n", encoding="utf-8") + (folder / "checklist.md").write_text("# checklist\n", encoding="utf-8") + self.artifacts.mkdir(parents=True) + self._write_sprint({"1.1": "done", "1.2": "ready-for-dev", "2.1": "ready-for-dev"}) + for story_id in ("1.1", "1.2", "2.1"): + self._write_story(story_id, "done" if story_id == "1.1" else "ready-for-dev") + self._write_epic_file("1", ["1.1", "1.2"]) + self._write_epic_file("2", ["2.1"]) + + def _init_git(self) -> None: + self._git("init") + self._git("config", "user.email", "smoke@example.invalid") + self._git("config", "user.name", "Finish Smoke") + (self.project / "README.md").write_text("# Finish smoke repo\n", encoding="utf-8") + self._git("add", "README.md") + self._git("commit", "-m", "chore: seed smoke repo") + + def _build_state(self) -> Path: + config = { + "epic": "multi", + "epicName": "Finish Loop Smoke", + "storyRange": ["1.1", "1.2", "2.1"], + "status": "IN_PROGRESS", + "currentStory": "1.1", + "currentStep": "step-03a-execute-review", + "aiCommand": "codex exec", + "customInstructions": "Finish-loop deterministic smoke.", + "overrides": {"skipAutomate": False, "maxParallel": 1}, + "agentConfig": {"defaultPrimary": "codex", "defaultFallback": False, "perTask": {"retro": {"primary": "claude", "fallback": False}}}, + } + template = self.project / ".agents" / "skills" / "bmad-story-automator" / "templates" / "state-document.md" + payload = self._json(*self._call(cmd_build_state_doc, ["--template", str(template), "--output-folder", str(self.output), "--config-json", json.dumps(config)])) + self._expect(payload["ok"] is True, f"build state failed: {payload}") + state = Path(str(payload["path"])) + for story_id in ("1.1", "1.2", "2.1"): + self._replace_progress(state, story_id, "done", "done", "-", "-", "-", "in-progress") + return state + + def _finish_story(self, state: Path, story_id: str, commit_repo: Path) -> dict[str, object]: + self._state_update(state, currentStory=story_id, currentStep="step-03a-execute-review") + automate = "done" if story_id != "1.2" else "skip" + self._replace_progress(state, story_id, "done", "done", automate, "-", "-", "in-progress") + incomplete = None + if story_id == "1.2": + self._write_story(story_id, "in-progress") + incomplete = self._json(*self._call(cmd_orchestrator_helper, ["verify-code-review", story_id])) + self._expect(incomplete["verified"] is False and incomplete["reason"] == "workflow_not_complete", f"incomplete review not surfaced: {incomplete}") + self._write_story(story_id, "done") + review = self._json(*self._call(cmd_orchestrator_helper, ["verify-step", "review", story_id, "--state-file", str(state)])) + self._expect(review["verified"] is True, f"review verification failed: {review}") + self._replace_progress(state, story_id, "done", "done", automate, "done", "-", "in-progress") + change = commit_repo / f"story-{story_id.replace('.', '-')}.txt" + change.write_text(f"implemented {story_id} at {self._iso_now()}\n", encoding="utf-8") + commit = self._json(*self._call(cmd_commit_story, ["--repo", str(commit_repo), "--story", story_id, "--title", f"Finish smoke {story_id}"])) + self._expect(commit["ok"] is True and self._git("rev-parse", "HEAD", cwd=commit_repo).stdout.strip() == commit["commit"], f"commit failed: {commit}") + self._write_sprint_status(story_id, "done") + final = self._json(*self._call(cmd_orchestrator_helper, ["story-file-status", story_id])) + sprint = self._json(*self._call(cmd_orchestrator_helper, ["sprint-status", "get", story_id])) + self._expect(final["status"] == "done" and sprint["done"] is True, f"final source check failed: {final} {sprint}") + self._replace_progress(state, story_id, "done", "done", automate, "done", "done", "done") + self._append_log(state, f"Story {story_id}: complete (commit + sprint-status verified)") + return {"story": story_id, "commit": commit["commit"], "automate": automate, "review": review, "incompleteReview": incomplete} + + def _maybe_run_retro(self, state: Path, story_id: str) -> None: + epic = story_id.split(".", 1)[0] + last = self._json(*self._call(cmd_orchestrator_helper, ["check-epic-complete", epic, story_id, "--state-file", str(state)])) + stories = self._json(*self._call(cmd_orchestrator_helper, ["get-epic-stories", epic, "--state-file", str(state)])) + status = self._json(*self._call(cmd_orchestrator_helper, ["sprint-status", "check-epic", epic])) + if not (last.get("isLastStory") and status.get("allStoriesDone")): + return + retro_agent = self._json(*self._call(cmd_orchestrator_helper, ["retro-agent", "--state-file", str(state)])) + code, retro_cmd = self._call(cmd_tmux_wrapper, ["build-cmd", "retro", epic, "--agent", str(retro_agent["primary"])]) + self._expect(code == 0 and "retrospective" in retro_cmd.lower(), f"retro build-cmd failed: {retro_cmd}") + self._append_log(state, f"Epic {epic} retrospective: skipped (reason: deterministic_smoke_runner)") + self._upsert_retro_state(state, epic, "skipped", "deterministic_smoke_runner") + self.results.setdefault("retrospectives", {})[f"epic-{epic}"] = {"status": "skipped", "reason": "deterministic_smoke_runner", "stories": stories["stories"]} + + def _complete_and_wrap(self, state: Path, marker: Path) -> None: + self._state_update(state, status="EXECUTION_COMPLETE", currentStep="step-04-wrapup") + self._append_log(state, "All stories complete - execution finished") + metrics = self._json(*self._call(cmd_state_metrics, ["--state", str(state)])) + self._expect(metrics["total"] == 3 and metrics["storiesCompleted"] == 3, f"metrics failed: {metrics}") + learnings = self.output / "learnings.md" + learnings.write_text(f"## Run: {self._iso_now()}\n\n- Finish-loop smoke completed.\n", encoding="utf-8") + self._state_update(state, status="COMPLETE") + self._append_log(state, "State document finalized") + code, _ = self._call(cmd_orchestrator_helper, ["marker", "remove"]) + self._expect(code == 0 and not marker.exists(), "marker not removed on wrapup") + validation = self._json(*self._call(cmd_validate_state, ["--state", str(state)])) + self._expect(validation["issueCount"] == 0, f"final state invalid: {validation}") + self.results["wrapup"] = {"metrics": metrics, "learnings": str(learnings.relative_to(self.project)), "markerRemoved": True} + + def _resolve_commit_repo(self) -> Path: + target = (self.target_repo or self.project).resolve() + if self._repo_allowed(target): + self.results["targetGuard"] = { + "unsafeHostRejected": self._guard_rejects(REPO_ROOT), + "target": self._repo_descriptor(target), + } + return target + if self.allow_unsafe_repo: + self.results["targetGuard"] = {"unsafeOverrideUsed": True, "target": self._repo_descriptor(target)} + return target + raise FinishSmokeError(f"unsafe commit repo outside smoke workspace: {target}") + + def _guard_rejects(self, repo: Path) -> bool: + return not self._repo_allowed(repo.resolve()) + + def _repo_allowed(self, repo: Path) -> bool: + smoke_root = self.project.resolve() + return repo == smoke_root or smoke_root in repo.parents + + def _create_marker(self, state: Path) -> Path: + marker_info = self._json(*self._call(cmd_orchestrator_helper, ["marker", "path"])) + marker = Path(str(marker_info["file"])) + self._call(cmd_orchestrator_helper, ["marker", "create", "--epic", "multi", "--story", "1.1", "--remaining", "3", "--state-file", str(state), "--project-slug", "finish-smoke", "--pid", "456"]) + self._expect(marker.exists(), "marker create failed") + return marker + + def _write_report(self, state: Path, commits: list[dict[str, object]], commit_repo: Path) -> dict[str, object]: + persisted = self._persist_diagnostics(state, commit_repo) + report = REPO_ROOT / ".smoke" / "FINISH_LOOP_SMOKE_REPORT.json" + report.parent.mkdir(parents=True, exist_ok=True) + payload = { + "createdAt": self._iso_now(), + "project": self._ephemeral_project_descriptor(), + "commitRepo": self._repo_descriptor(commit_repo), + "stateFile": persisted["stateFile"], + "diagnostics": persisted, + "commits": commits, + **self.results, + "report": str(report), + } + report.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + self.results["diagnostics"] = persisted + self.results["report"] = str(report) + return payload + + def _persist_diagnostics(self, state: Path, commit_repo: Path) -> dict[str, object]: + dest = REPO_ROOT / ".smoke" / "finish-loop-diagnostics" + shutil.rmtree(dest, ignore_errors=True) + dest.mkdir(parents=True) + state_dest = dest / state.name + state_dest.write_text(state.read_text(encoding="utf-8"), encoding="utf-8") + log = self._git("log", "--oneline", "-5", cwd=commit_repo).stdout + (dest / "git-log.txt").write_text(log, encoding="utf-8") + learnings = self.output / "learnings.md" + persisted_learnings = None + if learnings.exists(): + persisted_learnings = dest / learnings.name + persisted_learnings.write_text(learnings.read_text(encoding="utf-8"), encoding="utf-8") + if "wrapup" in self.results: + self.results["wrapup"]["learnings"] = str(persisted_learnings) + return { + "folder": str(dest), + "stateFile": str(state_dest), + "gitLog": str(dest / "git-log.txt"), + "gitLogRepo": self._repo_descriptor(commit_repo), + **({"learnings": str(persisted_learnings)} if persisted_learnings else {}), + } + + def _ephemeral_project_descriptor(self) -> dict[str, object]: + return { + "kind": "ephemeral", + "name": "finish-loop smoke fixture", + "retained": False, + } + + def _repo_descriptor(self, repo: Path) -> dict[str, object]: + resolved = repo.resolve() + if self._repo_allowed(resolved): + return { + "kind": "ephemeral", + "name": "finish-loop commit repo", + "retained": False, + } + return {"kind": "external", "path": str(resolved)} + + def _host_sentinel(self) -> dict[str, str]: + return { + "head": self._run(["git", "rev-parse", "HEAD"], cwd=REPO_ROOT).stdout.strip(), + "status": self._run(["git", "status", "--porcelain"], cwd=REPO_ROOT).stdout, + } + + def _assert_host_unchanged(self, before: dict[str, str]) -> None: + after = self._host_sentinel() + self._expect(after == before, f"host repo changed: before={before} after={after}") + self.results["hostIsolation"] = {"headUnchanged": True, "statusUnchanged": True} + + def _write_sprint(self, statuses: dict[str, str]) -> None: + rows = "\n".join(f"{story}: {status}" for story, status in statuses.items()) + (self.artifacts / "sprint-status.yaml").write_text(rows + "\n", encoding="utf-8") + + def _write_sprint_status(self, story_id: str, status: str) -> None: + sprint = self.artifacts / "sprint-status.yaml" + text = sprint.read_text(encoding="utf-8") + text = text.replace(f"{story_id}: ready-for-dev", f"{story_id}: {status}").replace(f"{story_id}: in-progress", f"{story_id}: {status}") + sprint.write_text(text, encoding="utf-8") + + def _write_story(self, story_id: str, status: str) -> None: + path = self.artifacts / f"{story_id.replace('.', '-')}-finish-smoke.md" + path.write_text(f"---\nTitle: Story {story_id}\nStatus: {status}\n---\n\n# Story {story_id}\n", encoding="utf-8") + + def _write_epic_file(self, epic: str, stories: list[str]) -> None: + lines = [f"# Epic {epic}", ""] + for story in stories: + lines.append(f"### Story {story}: Finish smoke {story}") + (self.artifacts / f"epic-{epic}.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + + def _replace_progress(self, state: Path, story_id: str, *cells: str) -> None: + row = f"| {story_id} | " + " | ".join(cells) + " |" + text = state.read_text(encoding="utf-8") + + state.write_text(re.sub(rf"(?m)^\| {re.escape(story_id)} \|.*$", row, text), encoding="utf-8") + + def _append_log(self, state: Path, message: str) -> None: + state.write_text(state.read_text(encoding="utf-8") + f"\n- **[{self._iso_now()}]** {message}\n", encoding="utf-8") + + def _state_update(self, state: Path, **fields: object) -> None: + for key, value in fields.items(): + payload = self._json(*self._call(cmd_orchestrator_helper, ["state-update", str(state), "--set", f"{key}={value}"])) + self._expect(payload["ok"] is True, f"state update failed {key}: {payload}") + + def _upsert_retro_state(self, state: Path, epic: str, status: str, reason: str) -> None: + text = state.read_text(encoding="utf-8") + block = ( + f"\nretrospectives.epic-{epic}:\n" + f" status: {status}\n" + f" reason: {reason}\n" + f" timestamp: {self._iso_now()}\n" + ) + marker = "---" + parts = text.split(marker, 2) + self._expect(len(parts) == 3, "state frontmatter missing for retro update") + front = re.sub(rf"\nretrospectives\.epic-{re.escape(epic)}:\n(?: .*\n)*", "\n", parts[1]) + state.write_text(f"{marker}{front.rstrip()}{block}{marker}{parts[2]}", encoding="utf-8") + + def _git(self, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: + return self._run(["git", *args], cwd=cwd or self.project) + + def _run(self, args: list[str], *, cwd: Path) -> subprocess.CompletedProcess[str]: + try: + return subprocess.run(args, cwd=cwd, text=True, capture_output=True, check=True, timeout=RUN_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired as exc: + raise FinishSmokeError(f"command timed out after {RUN_TIMEOUT_SECONDS}s: {' '.join(args)}") from exc + + def _call(self, fn, args: list[str]) -> tuple[int, str]: + old_env = os.environ.copy() + env = deterministic_smoke_env(self.project, {"BMAD_RUNTIME_PROVIDER": "codex"}) + out = io.StringIO() + try: + os.environ.clear() + os.environ.update(env) + with redirect_stdout(out): + code = fn(args) + return code, out.getvalue() + finally: + os.environ.clear() + os.environ.update(old_env) + + def _json(self, code: int, raw: str) -> dict[str, object]: + self._expect(code in {0, 1}, f"unexpected exit {code}: {raw}") + try: + payload = json.loads(raw) + except json.JSONDecodeError as exc: + raise FinishSmokeError(f"helper returned invalid JSON: {raw}") from exc + if not isinstance(payload, dict): + raise FinishSmokeError(f"helper returned non-object JSON: {raw}") + return payload + + def _expect(self, condition: bool, message: str) -> None: + if not condition: + raise FinishSmokeError(message) + + def _iso_now(self) -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run deterministic finish-loop smoke.") + parser.add_argument("--target-repo", default="", help="Commit target repo. Defaults to the temp smoke repo.") + parser.add_argument("--allow-unsafe-repo", action="store_true", help="Allow committing outside the smoke workspace for manual debugging.") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(sys.argv[1:] if argv is None else argv) + runner = FinishLoopSmokeRunner( + target_repo=Path(args.target_repo) if args.target_repo else None, + allow_unsafe_repo=bool(args.allow_unsafe_repo), + ) + try: + summary = runner.run() + except (FinishSmokeError, OSError, subprocess.CalledProcessError, ValueError, json.JSONDecodeError) as exc: + print(f"finish-loop smoke failed: {exc}", file=sys.stderr) + return 1 + finally: + runner.close() + print("finish-loop smoke ok") + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run-smoke-modes.py b/scripts/run-smoke-modes.py new file mode 100644 index 00000000..0fe4465f --- /dev/null +++ b/scripts/run-smoke-modes.py @@ -0,0 +1,497 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse, io, json, os, shutil, sys, tempfile +from contextlib import redirect_stdout +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "skills" / "bmad-story-automator" / "src")) +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from story_automator.commands.basic import ( # noqa: E402 + cmd_derive_project_slug, + cmd_ensure_marker_gitignore, + cmd_ensure_stop_hook, + cmd_list_sessions, + cmd_stop_hook, +) +from story_automator.commands.orchestrator import cmd_orchestrator_helper # noqa: E402 +from story_automator.commands.state import cmd_build_state_doc, cmd_sprint_compare, cmd_state_metrics, cmd_validate_state # noqa: E402 +from story_automator.core.agent_config import load_agent_config_from_state # noqa: E402 +from story_automator.core.epic_parser import parse_story_range # noqa: E402 +from smoke_prep.process import deterministic_smoke_env # noqa: E402 +from smoke_prep.mode_report import write_mode_report # noqa: E402 + +class SmokeModesError(Exception): + pass + +class ModeSmokeRunner: + def __init__(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.project = Path(self.tmp.name) + self.output = self.project / "_bmad-output" / "story-automator" + self.artifacts = self.project / "_bmad-output" / "implementation-artifacts" + self.results: dict[str, object] = {} + + def close(self) -> None: + self.tmp.cleanup() + + def run(self) -> dict[str, object]: + self._install_fixture() + self._assert_validate_helpers() + self._assert_preflight_selection_contracts() + self._assert_create_startup_guards() + state_file = self._build_state() + self._assert_state_and_resume_contracts(state_file) + self._assert_marker_lifecycle(state_file) + self._assert_validate_and_source_truth(state_file) + self._assert_edit_route_contracts(state_file) + return {"project": str(self.project), **self.results} + + def write_report(self, summary: dict[str, object]) -> tuple[Path, dict[str, object]]: + try: + return write_mode_report(REPO_ROOT, summary) + except ValueError as exc: + raise SmokeModesError(str(exc)) from exc + + def _install_fixture(self) -> None: + skills = self.project / ".agents" / "skills" + skills.mkdir(parents=True) + shutil.copytree(REPO_ROOT / "skills" / "bmad-story-automator", skills / "bmad-story-automator") + shutil.copytree(REPO_ROOT / "skills" / "bmad-story-automator-review", skills / "bmad-story-automator-review") + for name, extras in { + "bmad-create-story": ["discover-inputs.md", "checklist.md", "template.md"], + "bmad-dev-story": ["checklist.md"], + "bmad-retrospective": [], + "bmad-qa-generate-e2e-tests": ["checklist.md"], + }.items(): + folder = skills / name + folder.mkdir() + (folder / "SKILL.md").write_text(f"# {name}\n", encoding="utf-8") + (folder / "workflow.md").write_text(f"# {name}\n", encoding="utf-8") + for extra in extras: + (folder / extra).write_text(f"# {extra}\n", encoding="utf-8") + self.artifacts.mkdir(parents=True) + (self.artifacts / "sprint-status.yaml").write_text("1-1-first: ready-for-dev\n1-2-second: backlog\n", encoding="utf-8") + + def _assert_validate_helpers(self) -> None: + for fn, args, text in ( + (cmd_validate_state, ["--help"], "validate-state"), + (cmd_list_sessions, ["--help"], "list-sessions"), + (cmd_derive_project_slug, ["--help"], "derive-project-slug"), + ): + code, output = self._call(fn, args) + self._expect(code == 0 and text in output, f"helper help failed: {text}") + + def _assert_preflight_selection_contracts(self) -> None: + ids = "1.1,1.2" + multi = parse_story_range("1-2", 2, ids) + explicit = parse_story_range("1.1,1.2", 2, ids) + reversed_range = parse_story_range("2-1", 2, ids) + invalid = parse_story_range("99", 2, ids) + self._expect(multi["storyIds"] == ["1.1", "1.2"], f"multi-story range failed: {multi}") + self._expect(explicit["storyIds"] == ["1.1", "1.2"], f"explicit ID range failed: {explicit}") + self._expect(reversed_range["indices"] == [1, 2], f"reversed numeric range failed: {reversed_range}") + self._expect(invalid["ok"] is True and invalid["count"] == 0, f"invalid range contract changed: {invalid}") + self.results["preflight"] = { + "multiStory": multi["storyIds"], + "explicitIds": explicit["storyIds"], + "invalidRange": "empty-selection", + } + + def _assert_create_startup_guards(self) -> None: + code, raw = self._call(cmd_ensure_stop_hook, ["--command", "story-automator", "stop-hook", "--timeout", "10"]) + first = self._json(code, raw) + self._expect(first["ok"] is True and first["changed"] is True, f"stop-hook configure failed: {first}") + code, raw = self._call(cmd_ensure_stop_hook, ["--command", "story-automator", "stop-hook", "--timeout", "10"]) + second = self._json(code, raw) + self._expect(second["ok"] is True and second["reason"] in {"already_configured", "pending_trust"}, f"stop-hook verify failed: {second}") + hooks = self.project / ".codex" / "hooks.json" + hooks.write_text("{bad json", encoding="utf-8") + code, raw = self._call(cmd_ensure_stop_hook, ["--command", "story-automator", "stop-hook"]) + invalid = self._json(code, raw) + self._expect(code == 1 and invalid["error"] == "invalid_json", f"stop-hook invalid json not surfaced: {invalid}") + hooks.unlink() + self.output.mkdir(parents=True, exist_ok=True) + (self.output / "init-log-smoke.md").write_text("[smoke] init: stop-hook checked existing_state=\n", encoding="utf-8") + code, raw = self._call(cmd_orchestrator_helper, ["sprint-status", "exists"]) + self._expect(code == 0 and raw.strip() == "true", "sprint-status present check failed") + (self.artifacts / "sprint-status.yaml").unlink() + code, raw = self._call(cmd_orchestrator_helper, ["sprint-status", "exists"]) + self._expect(code == 0 and raw.strip() == "false", "sprint-status missing check failed") + (self.artifacts / "sprint-status.yaml").write_text("1-1-first: ready-for-dev\n1-2-second: backlog\n", encoding="utf-8") + self.results["createStartup"] = { + "stopHookFirst": first["reason"], + "stopHookSecond": second["reason"], + "invalidHook": invalid["error"], + "sprintStatusPrecondition": "present-and-missing-checked", + } + + def _build_state(self) -> Path: + config = { + "epic": "1", + "epicName": "Smoke Epic", + "storyRange": ["1.1", "1.2"], + "status": "IN_PROGRESS", + "currentStory": "1.1", + "currentStep": "step-03-execute", + "aiCommand": "codex exec", + "customInstructions": "Mode smoke fixture.", + "overrides": {"skipAutomate": True, "maxParallel": 2}, + "agentConfig": { + "defaultPrimary": "codex", + "defaultFallback": False, + "perTask": {"review": {"primary": "claude", "fallback": False}}, + }, + } + template = self.project / ".agents" / "skills" / "bmad-story-automator" / "templates" / "state-document.md" + code, raw = self._call( + cmd_build_state_doc, + ["--template", str(template), "--output-folder", str(self.output), "--config-json", json.dumps(config)], + ) + payload = self._json(code, raw) + self._expect(payload["ok"] is True, f"build-state-doc failed: {payload}") + state_file = Path(payload["path"]) + text = state_file.read_text(encoding="utf-8") + self._expect("policySnapshotFile:" in text and "| 1.1 |" in text and "| 1.2 |" in text, "state artifact missing required fields") + complexity_file = self.output / "complexity-smoke.json" + agents_file = self.output / "agents-smoke.md" + dev_log = self.output / "dev-log-smoke.md" + mode_report = self.output / "mode-report-smoke.json" + complexity_file.write_text('{"stories":[{"storyId":"1.1","complexity":{"level":"medium"}}]}\n', encoding="utf-8") + agents_file.write_text("# Agents\n\n- 1.1 create codex\n- 1.1 review claude\n", encoding="utf-8") + dev_log.write_text("# Dev Log\n\n- Simulated child dev workflow completed for 1.1.\n", encoding="utf-8") + mode_report.write_text('{"mode":"create-dev","status":"simulated-child-output"}\n', encoding="utf-8") + code, raw = self._call(cmd_orchestrator_helper, ["state-update", str(state_file), "--set", "complexityFile=_bmad-output/story-automator/complexity-smoke.json"]) + self._expect(self._json(code, raw)["ok"] is True, "complexityFile state update failed") + code, raw = self._call(cmd_orchestrator_helper, ["state-update", str(state_file), "--set", "agentsFile=_bmad-output/story-automator/agents-smoke.md"]) + self._expect(self._json(code, raw)["ok"] is True, "agentsFile state update failed") + self._append_action_log(state_file, "smoke create/dev simulated child output recorded") + updated_text = state_file.read_text(encoding="utf-8") + self._expect("complexity-smoke.json" in updated_text and "agents-smoke.md" in updated_text, "artifact paths not saved in state") + self._expect(complexity_file.exists() and agents_file.exists() and dev_log.exists() and mode_report.exists(), "selected artifacts not written") + self.results["preflight"]["complexityMatrix"] = "medium" + self.results["preflight"]["agentConfigVariant"] = "review=claude" + self.results["artifacts"] = { + "actionLog": "smoke create/dev simulated child output recorded", + "complexity": str(complexity_file.relative_to(self.project)), + "agents": str(agents_file.relative_to(self.project)), + "devLog": str(dev_log.relative_to(self.project)), + "modeReport": str(mode_report.relative_to(self.project)), + } + return state_file + + def _assert_state_and_resume_contracts(self, state_file: Path) -> None: + code, raw = self._call(cmd_orchestrator_helper, ["state-list", str(self.output)]) + listing = self._json(code, raw) + self._expect(listing["ok"] is True and len(listing["files"]) == 1, f"state-list failed: {listing}") + code, raw = self._call(cmd_orchestrator_helper, ["state-latest-incomplete", str(self.output)]) + latest = self._json(code, raw) + self._expect(latest["ok"] is True and latest["path"] == str(state_file), f"latest incomplete failed: {latest}") + code, raw = self._call(cmd_orchestrator_helper, ["state-summary", str(state_file)]) + summary = self._json(code, raw) + self._expect( + summary["currentStep"] == "step-03-execute" + and summary["policySnapshotHash"] + and summary["lastAction"] == "smoke create/dev simulated child output recorded", + f"state summary failed: {summary}", + ) + complete_dir = self.output / "complete-only" + complete_dir.mkdir() + complete_state = complete_dir / state_file.name + complete_state.write_text(state_file.read_text(encoding="utf-8").replace('status: "IN_PROGRESS"', 'status: "COMPLETE"'), encoding="utf-8") + code, raw = self._call(cmd_orchestrator_helper, ["state-latest-incomplete", str(complete_dir)]) + no_incomplete = self._json(code, raw) + self._expect(no_incomplete["ok"] is False and no_incomplete["error"] == "no_incomplete_state", f"fresh-create fallback failed: {no_incomplete}") + resume_contract = self._assert_resume_menu_branch_contracts(state_file) + self.results.setdefault("createStartup", {})["existingStateDetected"] = True + self.results["resume"] = { + "explicitPathSummary": summary["currentStep"], + "latestIncomplete": latest["path"], + "routeHint": resume_contract["routeHint"], + "freshCreateFallback": no_incomplete["error"], + "menuBranches": resume_contract["menuBranches"], + } + + def _assert_marker_lifecycle(self, state_file: Path) -> None: + code, raw = self._call(cmd_orchestrator_helper, ["marker", "path"]) + marker_path = self._json(code, raw) + entry = marker_path["entry"] + self._expect(entry == ".agents/.story-automator-active", f"marker entry not dynamic .agents path: {marker_path}") + code, raw = self._call(cmd_ensure_marker_gitignore, ["--gitignore", str(self.project / ".gitignore"), "--entry", entry]) + self._expect(self._json(code, raw)["changed"] is True, "marker gitignore not updated") + code, _ = self._call( + cmd_orchestrator_helper, + [ + "marker", + "create", + "--epic", + "1", + "--story", + "1.1", + "--remaining", + "2", + "--state-file", + str(state_file), + "--project-slug", + "smoke", + "--pid", + "123", + "--heartbeat", + "2000-01-01T00:00:00Z", + ], + ) + self._expect(code == 0, "marker create failed") + marker_file = Path(str(marker_path["file"])) + marker_payload = json.loads(marker_file.read_text(encoding="utf-8")) + self._expect( + marker_payload["epic"] == "1" + and marker_payload["currentStory"] == "1.1" + and marker_payload["storiesRemaining"] == 2 + and marker_payload["stateFile"] == str(state_file) + and marker_payload["projectSlug"] == "smoke" + and marker_payload["heartbeat"] == "2000-01-01T00:00:00Z", + f"marker JSON shape failed: {marker_payload}", + ) + code, raw = self._call(cmd_orchestrator_helper, ["marker", "check"]) + marker_check, checked_marker = self._json_objects(code, raw) + self._expect( + marker_check["exists"] is True and checked_marker["storiesRemaining"] == 2, + f"marker check failed: {raw}", + ) + blocked_code, blocked = self._call(cmd_stop_hook, []) + blocked_payload = self._json(blocked_code, blocked) + self._expect(blocked_payload["decision"] == "block", f"stop-hook did not block active marker: {blocked_payload}") + old_heartbeat = marker_payload["heartbeat"] + code, _ = self._call(cmd_orchestrator_helper, ["marker", "heartbeat"]) + self._expect(code == 0, "marker heartbeat failed") + heartbeat_payload = json.loads(marker_file.read_text(encoding="utf-8")) + self._expect(heartbeat_payload["heartbeat"] != old_heartbeat, f"marker heartbeat did not change: {heartbeat_payload}") + code, _ = self._call(cmd_orchestrator_helper, ["marker", "remove"]) + self._expect(code == 0 and not marker_file.exists(), "marker remove failed") + code, raw = self._call(cmd_stop_hook, []) + self._expect(code == 0 and raw == "", "stop-hook did not allow after marker removal") + self.results["marker"] = {"entry": entry, "blocked": True, "gitignore": True, "heartbeatChanged": True} + + def _assert_validate_and_source_truth(self, state_file: Path) -> None: + code, raw = self._call(cmd_validate_state, ["--state", str(state_file)]) + validation = self._json(code, raw) + self._expect(validation["ok"] is True and validation["issueCount"] == 0, f"validate-state failed: {validation}") + code, raw = self._call(cmd_list_sessions, ["--slug", "smoke"]) + sessions = self._json(code, raw) + self._expect("sessions" in sessions, f"list-sessions failed: {sessions}") + code, raw = self._call(cmd_sprint_compare, ["--state", str(state_file), "--sprint", str(self.artifacts / "sprint-status.yaml")]) + compare = self._json(code, raw) + self._expect(compare["ok"] is True, f"sprint-compare failed: {compare}") + compare_state = self.output / "compare-progress.md" + compare_state.write_text(state_file.read_text(encoding="utf-8").replace('currentStory: "1.1"', 'currentStory: "1.2"'), encoding="utf-8") + code, raw = self._call(cmd_sprint_compare, ["--state", str(compare_state), "--sprint", str(self.artifacts / "sprint-status.yaml")]) + progress_compare = self._json(code, raw) + self._expect(progress_compare["checked"] == ["1.1"] and progress_compare["incomplete"] == ["1.1"], f"progress compare did not inspect prior story: {progress_compare}") + done_sprint = self.artifacts / "sprint-status-exact-done.yaml" + done_sprint.write_text("1.1: done\n1.2: backlog\n", encoding="utf-8") + code, raw = self._call(cmd_sprint_compare, ["--state", str(compare_state), "--sprint", str(done_sprint)]) + done_compare = self._json(code, raw) + self._expect(done_compare["checked"] == ["1.1"] and done_compare["incomplete"] == [], f"sprint done branch failed: {done_compare}") + code, raw = self._call(cmd_state_metrics, ["--state", str(state_file)]) + metrics = self._json(code, raw) + self._expect(metrics["ok"] is True and metrics["total"] == 2, f"progress row metrics failed: {metrics}") + broken_state = self.output / "invalid-structure.md" + broken_state.write_text(state_file.read_text(encoding="utf-8").replace('status: "IN_PROGRESS"', 'status: ""'), encoding="utf-8") + code, raw = self._call(cmd_validate_state, ["--state", str(broken_state)]) + broken = self._json(code, raw) + self._expect(broken["ok"] is True and broken["issueCount"] > 0 and broken["structure"] == "issues", f"structure issues not reported: {broken}") + story = self.artifacts / "1-1-first.md" + story.write_text('---\nTitle: "Story 1.1"\nStatus: done\n---\n', encoding="utf-8") + code, raw = self._call(cmd_orchestrator_helper, ["story-file-status", "1.1"]) + file_status = self._json(code, raw) + code, raw = self._call(cmd_orchestrator_helper, ["sprint-status", "get", "1.1"]) + sprint_status = self._json(code, raw) + self._expect(file_status["status"] == "done" and sprint_status["status"] == "ready-for-dev", "source mismatch not surfaced") + code, raw = self._call(cmd_orchestrator_helper, ["verify-step", "review", "1.1", "--state-file", str(state_file)]) + review = self._json(code, raw) + self._expect( + review.get("verified") is True + and review.get("source") == "story-file" + and review.get("note") == "sprint_status_not_updated", + f"review verifier did not surface sprint/story mismatch: {review}", + ) + self.results["validate"] = { + "state": "ok", + "sessions": sessions.get("count", 0), + "structureIssues": broken["issueCount"], + "progressRows": metrics["total"], + "progressChecked": progress_compare["checked"], + "progressDoneBranch": done_compare["checked"], + "sourceMismatch": "sprint_status_not_updated", + } + + def _assert_edit_route_contracts(self, state_file: Path) -> None: + menu = self._assert_edit_menu_contracts(state_file) + config = load_agent_config_from_state(state_file) + review = config.per_task.get("review") + self._expect(review is not None and review.primary == "claude" and review.fallback is False, f"agent config variant not rendered: {config}") + code, raw = self._call(cmd_orchestrator_helper, ["state-update", str(state_file), "--set", "status=PAUSED"]) + self._expect(self._json(code, raw)["ok"] is True, "edit status save failed") + code, raw = self._call(cmd_orchestrator_helper, ["state-update", str(state_file), "--set", 'storyRange=["1.2"]']) + self._expect(self._json(code, raw)["ok"] is True, "edit range save failed") + code, raw = self._call(cmd_orchestrator_helper, ["state-update", str(state_file), "--set", "currentStory=1.2"]) + self._expect(self._json(code, raw)["ok"] is True, "edit current story save failed") + code, raw = self._call(cmd_orchestrator_helper, ["state-update", str(state_file), "--set", "aiCommand=claude --resume"]) + self._expect(self._json(code, raw)["ok"] is True, "edit AI command save failed") + (self.output / "complexity-edited.json").write_text('{"stories":[{"storyId":"1.2","complexity":{"level":"low"}}]}\n', encoding="utf-8") + (self.output / "agents-edited.md").write_text("# Agents\n\n- 1.2 dev codex\n", encoding="utf-8") + code, raw = self._call(cmd_orchestrator_helper, ["state-update", str(state_file), "--set", "complexityFile=_bmad-output/story-automator/complexity-edited.json"]) + self._expect(self._json(code, raw)["ok"] is True, "edit complexity path save failed") + code, raw = self._call(cmd_orchestrator_helper, ["state-update", str(state_file), "--set", "agentsFile=_bmad-output/story-automator/agents-edited.md"]) + self._expect(self._json(code, raw)["ok"] is True, "edit agents path save failed") + code, raw = self._call(cmd_orchestrator_helper, ["state-update", str(state_file), "--set", "customInstructions=Edited context"]) + self._expect(self._json(code, raw)["ok"] is True, "edit text save failed") + edited = state_file.read_text(encoding="utf-8") + code, raw = self._call(cmd_orchestrator_helper, ["state-summary", str(state_file)]) + summary = self._json(code, raw) + self._expect(summary["currentStep"] == "step-03-execute" and summary["currentStory"] == "1.2", f"post-edit route source drifted: {summary}") + self.results["edit"] = { + "saved": ( + "customInstructions: Edited context" in edited + and 'storyRange: ["1.2"]' in edited + and "aiCommand: claude --resume" in edited + and "complexity-edited.json" in edited + and "agents-edited.md" in edited + and summary["status"] == "PAUSED" + ), + **menu, + } + self._expect(self.results["edit"]["saved"] is True, "edit save assertions failed") + + def _assert_resume_menu_branch_contracts(self, state_file: Path) -> dict[str, object]: + step = (REPO_ROOT / "skills" / "bmad-story-automator" / "steps-c" / "step-01b-continue.md").read_text(encoding="utf-8") + for token in ("[R]esume", "[V]iew", "[M]odify", "[S]tart Over", "[X]Abort"): + self._expect(token in step, f"resume menu token missing: {token}") + summary = self._json(*self._call(cmd_orchestrator_helper, ["state-summary", str(state_file)])) + self._expect(summary["lastAction"] == "smoke create/dev simulated child output recorded", f"view branch action log missing: {summary}") + start_over = self.output / "orchestration-start-over.md" + start_over.write_text(state_file.read_text(encoding="utf-8"), encoding="utf-8") + backup = start_over.with_name(f"{start_over.name}.backup-smoke") + start_over.rename(backup) + self._expect(backup.exists() and not start_over.exists(), "start-over backup simulation failed") + abort_state = self.output / "orchestration-abort.md" + abort_state.write_text(state_file.read_text(encoding="utf-8"), encoding="utf-8") + code, raw = self._call(cmd_orchestrator_helper, ["state-update", str(abort_state), "--set", "status=ABORTED"]) + abort = self._json(code, raw) + self._expect(abort["ok"] is True and "status" in abort["updated"], f"abort state update failed: {abort}") + return { + "routeHint": self._route_hint(step, "step-03-execute"), + "menuBranches": ["view-action-log", "modify-route", "start-over-backup", "abort-state", "resume-marker-route"], + } + + def _assert_edit_menu_contracts(self, state_file: Path) -> dict[str, object]: + step = (REPO_ROOT / "skills" / "bmad-story-automator" / "steps-e" / "step-e-01-load.md").read_text(encoding="utf-8") + for token in ("[S]tatus", "[R]ange", "[O]verrides", "[T]ext Context", "[I] Command", "[D]ocs", "[X]Exit", "[S]ave", "[D]iscard", "[E]dit more", "[R]esume", "[V]alidate"): + self._expect(token in step, f"edit menu token missing: {token}") + discard_path = self.output / "discard-copy.md" + before = state_file.read_text(encoding="utf-8") + discard_path.write_text(before, encoding="utf-8") + staged = before.replace("Mode smoke fixture.", "Discard candidate") + self._expect(staged != before, "discard fixture did not stage a change") + discard_path.write_text(staged, encoding="utf-8") + discard_path.write_text(before, encoding="utf-8") + self._expect(discard_path.read_text(encoding="utf-8") == before, "discard branch should restore original state") + edit_more = self.output / "edit-more-copy.md" + edit_more.write_text(before, encoding="utf-8") + code, raw = self._call(cmd_orchestrator_helper, ["state-update", str(edit_more), "--set", "currentStep=step-e-01-load"]) + payload = self._json(code, raw) + self._expect(payload["ok"] is True and "currentStep" in payload["updated"], f"edit-more route state update failed: {payload}") + post_edit_routes = { + "resume": self._route_hint(step, "Route based on `currentStep`"), + "validate": self._route_hint(step, "Load `{validateStep}`"), + "exit": self._route_hint(step, 'Display "Edit complete." and end'), + } + return { + "discarded": discard_path.read_text(encoding="utf-8") == before, + "editMore": "currentStep=step-e-01-load", + "postEditRouteHints": post_edit_routes, + "workflowMenuDerived": True, + } + + def _route_hint(self, workflow_text: str, text: str) -> str: + self._expect(text in workflow_text, f"route hint missing from workflow: {text}") + return text + + def _append_action_log(self, state_file: Path, entry: str) -> None: + text = state_file.read_text(encoding="utf-8") + line = f"* {entry}" + marker = "" + self._expect(marker in text, "action log marker missing") + state_file.write_text(text.replace(marker, f"{line}\n{marker}", 1), encoding="utf-8") + + def _call(self, fn, args: list[str]) -> tuple[int, str]: + old_env = os.environ.copy() + old_stdin = sys.stdin + env = deterministic_smoke_env(self.project) + stdout = io.StringIO() + try: + os.environ.clear() + os.environ.update(env) + sys.stdin = io.StringIO("") + with redirect_stdout(stdout): + code = fn(args) + return code, stdout.getvalue() + finally: + sys.stdin = old_stdin + os.environ.clear() + os.environ.update(old_env) + + def _json(self, code: int, raw: str) -> dict[str, object]: + self._expect(code in {0, 1}, f"unexpected exit code {code}: {raw}") + try: + return json.loads(raw) + except json.JSONDecodeError as exc: + raise SmokeModesError(f"expected JSON, got {raw!r}") from exc + + def _json_objects(self, code: int, raw: str) -> list[dict[str, object]]: + self._expect(code in {0, 1}, f"unexpected exit code {code}: {raw}") + decoder = json.JSONDecoder() + objects: list[dict[str, object]] = [] + index = 0 + while index < len(raw): + while index < len(raw) and raw[index].isspace(): + index += 1 + if index >= len(raw): + break + try: + payload, index = decoder.raw_decode(raw, index) + except json.JSONDecodeError as exc: + raise SmokeModesError(f"expected JSON object stream, got {raw!r}") from exc + self._expect(isinstance(payload, dict), f"expected JSON object in output: {raw}") + objects.append(payload) + self._expect(objects, f"no JSON objects in output: {raw}") + return objects + + def _expect(self, condition: bool, message: str) -> None: + if not condition: + raise SmokeModesError(message) + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run deterministic in-process smoke mode contracts.") + return parser.parse_args(argv) + +def main(argv: list[str] | None = None) -> int: + parse_args(sys.argv[1:] if argv is None else argv) + runner = ModeSmokeRunner() + try: + summary = runner.run() + report, payload = runner.write_report(summary) + except (OSError, SmokeModesError, ValueError) as exc: + print(f"smoke:modes failed: {exc}", file=sys.stderr) + return 1 + finally: + runner.close() + print("mode smoke ok") + print(json.dumps({"report": str(report), **payload}, indent=2)) + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/smoke_prep/__init__.py b/scripts/smoke_prep/__init__.py new file mode 100644 index 00000000..f491cf40 --- /dev/null +++ b/scripts/smoke_prep/__init__.py @@ -0,0 +1 @@ +"""Smoke prep helpers for Story Automator.""" diff --git a/scripts/smoke_prep/automator.py b/scripts/smoke_prep/automator.py new file mode 100644 index 00000000..85e8c03d --- /dev/null +++ b/scripts/smoke_prep/automator.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import os +from pathlib import Path + +from .package_contracts import pack_project +from .process import SmokeError, run, scrub_host_overrides, step + + +def install_bmad(gunz_dir: Path, env: dict[str, str], bmad_method_spec: str) -> None: + step("Install BMAD core and BMM") + run( + [ + "npx", + "--yes", + bmad_method_spec, + "install", + "--tools", + "claude-code", + "--action", + "install", + "--modules", + "core,bmm", + "--yes", + "--directory", + str(gunz_dir), + ], + cwd=gunz_dir, + env=env, + ) + + +def pack_project_automator(root: Path, workspace: Path, env: dict[str, str]) -> dict: + step("Pack project-local automator") + pack_dir = workspace / "packages" + return pack_project(root, pack_dir, env) + + +def install_project_automator( + gunz_dir: Path, + package_identity: dict, + env: dict[str, str], +) -> None: + step("Install project-local automator into smoke project") + run( + [ + "npx", + "--yes", + "--package", + f"file:{package_identity['tarball']}", + "bmad-story-automator", + str(gunz_dir), + ], + cwd=gunz_dir, + env=env, + ) + + +def verify_layout(gunz_dir: Path) -> None: + step("Verify smoke project layout") + helper = ( + gunz_dir + / ".claude" + / "skills" + / "bmad-story-automator" + / "scripts" + / "story-automator" + ) + required = [ + gunz_dir / "_bmad" / "_config" / "manifest.yaml", + gunz_dir / ".claude" / "skills" / "bmad-create-story" / "SKILL.md", + gunz_dir / ".claude" / "skills" / "bmad-dev-story" / "SKILL.md", + gunz_dir / ".claude" / "skills" / "bmad-retrospective" / "SKILL.md", + gunz_dir / ".claude" / "skills" / "bmad-story-automator" / "SKILL.md", + gunz_dir / ".claude" / "skills" / "bmad-story-automator-review" / "SKILL.md", + helper, + ] + missing = [str(path) for path in required if not path.exists()] + if missing: + raise SmokeError("missing expected smoke files:\n" + "\n".join(missing)) + + run([str(helper), "--help"], cwd=gunz_dir) + + +def smoke_env(workspace: Path) -> dict[str, str]: + env = scrub_host_overrides(os.environ.copy()) + home = workspace / "home" + npm_cache = workspace / "npm-cache" + home.mkdir(parents=True, exist_ok=True) + npm_cache.mkdir(parents=True, exist_ok=True) + env["HOME"] = str(home) + env["NPM_CONFIG_CACHE"] = str(npm_cache) + return env diff --git a/scripts/smoke_prep/cli.py b/scripts/smoke_prep/cli.py new file mode 100644 index 00000000..263d008d --- /dev/null +++ b/scripts/smoke_prep/cli.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import argparse +import json +import subprocess +import sys + +from .automator import ( + install_bmad, + install_project_automator, + pack_project_automator, + smoke_env, + verify_layout, +) +from .config import repo_root +from .gunz import prepare_gunz +from .inputs import smoke_inputs, write_smoke_inputs +from .package_contracts import verify_installed_package, write_package_identity +from .process import SmokeError, ensure_tool +from .report import write_next_steps +from .workspace import reset_dir, resolve_workspace + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Prepare the pinned gunz BMAD project for Story Automator smoke testing.", + ) + parser.add_argument( + "--workspace", + default=".smoke", + help="Repo-relative ignored workspace for clone, npm cache, and reports.", + ) + parser.add_argument( + "--reset", + action="store_true", + help="Delete the smoke workspace before preparing it.", + ) + parser.add_argument( + "--skip-bmad-install", + action="store_true", + help="Skip BMAD core/BMM install; useful after a previous successful run.", + ) + parser.add_argument( + "--skip-automator-install", + action="store_true", + help="Skip installing the project-local automator into the smoke repo.", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(sys.argv[1:] if argv is None else argv) + root = repo_root() + + try: + ensure_tool("git") + workspace = resolve_workspace(root, args.workspace) + gunz_dir = workspace / "gunz" + ensure_tool("node") + ensure_tool("npm") + ensure_tool("npx") + + if args.reset: + reset_dir(workspace) + workspace.mkdir(parents=True, exist_ok=True) + + env = smoke_env(workspace) + prepare_gunz(workspace, gunz_dir) + inputs = smoke_inputs(env) + write_smoke_inputs(workspace, inputs) + if not args.skip_bmad_install: + install_bmad(gunz_dir, env, inputs["bmadMethod"]["installSpec"]) + if not args.skip_automator_install: + package_identity = pack_project_automator(root, workspace, env) + write_package_identity(workspace, package_identity) + install_project_automator(gunz_dir, package_identity, env) + verify_layout(gunz_dir) + if not args.skip_automator_install: + verify_installed_package(gunz_dir, package_identity, workspace) + next_steps = write_next_steps(workspace, gunz_dir, automator_installed=not args.skip_automator_install) + + except (OSError, subprocess.CalledProcessError, SmokeError, ValueError, json.JSONDecodeError) as exc: + print(f"smoke prep failed: {exc}", file=sys.stderr) + return 1 + + print("") + print("smoke prep ok") + print(f"workspace: {workspace}") + print(f"project: {gunz_dir}") + print(f"next steps: {next_steps}") + return 0 diff --git a/scripts/smoke_prep/config.py b/scripts/smoke_prep/config.py new file mode 100644 index 00000000..efafa678 --- /dev/null +++ b/scripts/smoke_prep/config.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from pathlib import Path + + +REPO_URL = "https://github.com/bma-d/gunz.git" +BRANCH = "bmad-smoke-test" +PINNED_COMMIT = "fca6470d329668019dace305b5f0f3c9b62cb113" +BMAD_METHOD_NPM_SPEC = "bmad-method@next" + + +def repo_root() -> Path: + return Path(__file__).resolve().parents[2] diff --git a/scripts/smoke_prep/gunz.py b/scripts/smoke_prep/gunz.py new file mode 100644 index 00000000..956dc5fb --- /dev/null +++ b/scripts/smoke_prep/gunz.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from pathlib import Path + +from .config import BRANCH, PINNED_COMMIT, REPO_URL +from .process import SmokeError, run, step + + +def prepare_gunz(workspace: Path, gunz_dir: Path) -> None: + step("Prepare pinned gunz smoke repo") + if gunz_dir.exists(): + print(f"reuse existing clone: {gunz_dir}") + origin = run(["git", "remote", "get-url", "origin"], cwd=gunz_dir, capture=True).stdout.strip() + if origin != REPO_URL: + raise SmokeError(f"existing clone origin mismatch: expected {REPO_URL}, got {origin}") + else: + run( + [ + "git", + "clone", + "--single-branch", + "--branch", + BRANCH, + REPO_URL, + str(gunz_dir), + ], + cwd=workspace, + ) + + run( + ["git", "fetch", "origin", f"refs/heads/{BRANCH}:refs/remotes/origin/{BRANCH}"], + cwd=gunz_dir, + ) + ancestry = run( + ["git", "merge-base", "--is-ancestor", PINNED_COMMIT, f"origin/{BRANCH}"], + cwd=gunz_dir, + check=False, + ) + if ancestry.returncode != 0: + raise SmokeError(f"pinned commit {PINNED_COMMIT} is not on origin/{BRANCH}") + + run(["git", "checkout", PINNED_COMMIT], cwd=gunz_dir) + actual = run(["git", "rev-parse", "HEAD"], cwd=gunz_dir, capture=True) + if actual.stdout.strip() != PINNED_COMMIT: + raise SmokeError( + f"gunz HEAD mismatch: expected {PINNED_COMMIT}, got {actual.stdout.strip()}" + ) diff --git a/scripts/smoke_prep/inputs.py b/scripts/smoke_prep/inputs.py new file mode 100644 index 00000000..0ff324bb --- /dev/null +++ b/scripts/smoke_prep/inputs.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import json +import re +import subprocess +from pathlib import Path + +from .config import BMAD_METHOD_NPM_SPEC, BRANCH, PINNED_COMMIT, REPO_URL +from .process import SmokeError + + +FULL_SHA_RE = re.compile(r"^[0-9a-f]{40}$") + + +def _resolve_bmad_method(env: dict[str, str] | None = None) -> dict[str, str]: + try: + result = subprocess.run( + [ + "npm", + "view", + BMAD_METHOD_NPM_SPEC, + "version", + "dist.integrity", + "--json", + ], + env=env, + text=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=60, + ) + except subprocess.TimeoutExpired as exc: + raise SmokeError(f"npm view timed out for {BMAD_METHOD_NPM_SPEC}") from exc + try: + metadata = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise SmokeError( + f"unexpected npm identity for {BMAD_METHOD_NPM_SPEC}: {result.stdout.strip()}" + ) from exc + if not isinstance(metadata, dict): + raise SmokeError( + f"unexpected npm identity for {BMAD_METHOD_NPM_SPEC}: {result.stdout.strip()}" + ) + dist = metadata.get("dist", {}) + if dist is None: + dist = {} + if not isinstance(dist, dict): + raise SmokeError( + f"unexpected npm identity for {BMAD_METHOD_NPM_SPEC}: {result.stdout.strip()}" + ) + version = metadata.get("version") + integrity = dist.get("integrity") or metadata.get("dist.integrity") + if not isinstance(version, str) or not version or not isinstance(integrity, str) or not integrity: + raise SmokeError( + f"missing npm identity for {BMAD_METHOD_NPM_SPEC}: {result.stdout.strip()}" + ) + return { + "spec": BMAD_METHOD_NPM_SPEC, + "resolvedVersion": version, + "installSpec": f"bmad-method@{version}", + "integrity": integrity, + } + + +def smoke_inputs(env: dict[str, str] | None = None) -> dict: + if not REPO_URL.startswith("https://github.com/"): + raise SmokeError(f"unexpected smoke repo URL: {REPO_URL}") + if not BRANCH: + raise SmokeError("missing smoke repo branch") + if not FULL_SHA_RE.match(PINNED_COMMIT): + raise SmokeError(f"smoke repo commit is not a full SHA: {PINNED_COMMIT}") + if BMAD_METHOD_NPM_SPEC != "bmad-method@next": + raise SmokeError( + "BMAD Method installer input changed; update Phase 01 input contract " + f"before accepting: {BMAD_METHOD_NPM_SPEC}" + ) + return { + "gunz": { + "repo": REPO_URL, + "branch": BRANCH, + "commit": PINNED_COMMIT, + }, + "bmadMethod": _resolve_bmad_method(env), + } + + +def write_smoke_inputs(workspace: Path, inputs: dict) -> Path: + path = workspace / "SMOKE_INPUTS.json" + path.write_text( + json.dumps(inputs, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return path diff --git a/scripts/smoke_prep/mode_report.py b/scripts/smoke_prep/mode_report.py new file mode 100644 index 00000000..7afc6c6b --- /dev/null +++ b/scripts/smoke_prep/mode_report.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import json +import shutil +from datetime import datetime, timezone +from pathlib import Path + + +def write_mode_report(repo_root: Path, summary: dict[str, object]) -> tuple[Path, dict[str, object]]: + report = repo_root / ".smoke" / "MODE_SMOKE_REPORT.json" + report.parent.mkdir(parents=True, exist_ok=True) + payload = _report_payload(repo_root, summary) + report.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + return report, payload + + +def _report_payload(repo_root: Path, summary: dict[str, object]) -> dict[str, object]: + payload = dict(summary) + payload["project"] = { + "kind": "ephemeral", + "name": "mode smoke fixture", + "retained": False, + } + payload["createdAt"] = datetime.now(timezone.utc).isoformat() + diagnostics = _persist_diagnostics(repo_root, payload) + if diagnostics: + payload["diagnostics"] = diagnostics + resume = payload.get("resume") + if isinstance(resume, dict) and isinstance(resume.get("latestIncomplete"), str): + if "latestIncomplete" not in diagnostics: + raise ValueError(f"failed to persist latest incomplete state: {resume['latestIncomplete']}") + payload["resume"] = {**resume, "latestIncomplete": diagnostics["latestIncomplete"]} + return payload + + +def _persist_diagnostics(repo_root: Path, payload: dict[str, object]) -> dict[str, str]: + dest = repo_root / ".smoke" / "mode-diagnostics" + shutil.rmtree(dest, ignore_errors=True) + dest.mkdir(parents=True) + diagnostics: dict[str, str] = {"folder": str(dest)} + resume = payload.get("resume") + latest = resume.get("latestIncomplete") if isinstance(resume, dict) else None + if isinstance(latest, str): + latest_path = Path(latest) + if latest_path.exists(): + latest_dest = dest / latest_path.name + latest_dest.write_text(latest_path.read_text(encoding="utf-8"), encoding="utf-8") + diagnostics["latestIncomplete"] = str(latest_dest) + return diagnostics diff --git a/scripts/smoke_prep/package_contracts.py b/scripts/smoke_prep/package_contracts.py new file mode 100644 index 00000000..ba411713 --- /dev/null +++ b/scripts/smoke_prep/package_contracts.py @@ -0,0 +1,361 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +import tarfile +import tempfile +from pathlib import Path + +from .process import SmokeError + + +REQUIRED_PACKAGE_FILES = [ + "package.json", + "README.md", + "LICENSE", + "ref.png", + "install.sh", + "bin/bmad-story-automator", + ".claude-plugin/plugin.json", + ".claude-plugin/marketplace.json", + "skills/module.yaml", + "skills/module-help.csv", + "skills/bmad-story-automator/SKILL.md", + "skills/bmad-story-automator/README.md", + "skills/bmad-story-automator/LICENSE", + "skills/bmad-story-automator/workflow.md", + "skills/bmad-story-automator/pyproject.toml", + "skills/bmad-story-automator/scripts/story-automator", + "skills/bmad-story-automator/data/orchestration-policy.json", + "skills/bmad-story-automator/data/complexity-rules.json", + "skills/bmad-story-automator/data/agent-config-presets.json", + "skills/bmad-story-automator/data/parse/auto.json", + "skills/bmad-story-automator/data/parse/create.json", + "skills/bmad-story-automator/data/parse/dev.json", + "skills/bmad-story-automator/data/parse/retro.json", + "skills/bmad-story-automator/data/parse/review.json", + "skills/bmad-story-automator/data/prompts/auto.md", + "skills/bmad-story-automator/data/prompts/create.md", + "skills/bmad-story-automator/data/prompts/dev.md", + "skills/bmad-story-automator/data/prompts/retro.md", + "skills/bmad-story-automator/data/prompts/review.md", + "skills/bmad-story-automator/templates/state-document.md", + "skills/bmad-story-automator/steps-c/step-01-init.md", + "skills/bmad-story-automator/steps-c/step-01b-continue.md", + "skills/bmad-story-automator/steps-c/step-02-preflight.md", + "skills/bmad-story-automator/steps-c/step-02a-preflight-config.md", + "skills/bmad-story-automator/steps-c/step-02b-preflight-finalize.md", + "skills/bmad-story-automator/steps-c/step-03-execute.md", + "skills/bmad-story-automator/steps-c/step-03a-execute-review.md", + "skills/bmad-story-automator/steps-c/step-03b-execute-finish.md", + "skills/bmad-story-automator/steps-c/step-03c-execute-complete.md", + "skills/bmad-story-automator/steps-c/step-04-wrapup.md", + "skills/bmad-story-automator/steps-e/step-e-01-load.md", + "skills/bmad-story-automator/steps-v/step-v-01-check.md", + "skills/bmad-story-automator/steps-v/step-v-02-report.md", + "skills/bmad-story-automator/src/story_automator/__init__.py", + "skills/bmad-story-automator/src/story_automator/__main__.py", + "skills/bmad-story-automator/src/story_automator/cli.py", + "skills/bmad-story-automator/src/story_automator/adapters/tmux.py", + "skills/bmad-story-automator/src/story_automator/commands/__init__.py", + "skills/bmad-story-automator/src/story_automator/commands/agent_config_cmd.py", + "skills/bmad-story-automator/src/story_automator/commands/basic.py", + "skills/bmad-story-automator/src/story_automator/commands/orchestrator.py", + "skills/bmad-story-automator/src/story_automator/commands/orchestrator_epic_agents.py", + "skills/bmad-story-automator/src/story_automator/commands/orchestrator_parse.py", + "skills/bmad-story-automator/src/story_automator/commands/orchestrator_state.py", + "skills/bmad-story-automator/src/story_automator/commands/state.py", + "skills/bmad-story-automator/src/story_automator/commands/tmux.py", + "skills/bmad-story-automator/src/story_automator/commands/tmux_monitor.py", + "skills/bmad-story-automator/src/story_automator/commands/validate_story_creation.py", + "skills/bmad-story-automator/src/story_automator/core/agent_config.py", + "skills/bmad-story-automator/src/story_automator/core/agent_config_frontmatter.py", + "skills/bmad-story-automator/src/story_automator/core/agent_plan.py", + "skills/bmad-story-automator/src/story_automator/core/common.py", + "skills/bmad-story-automator/src/story_automator/core/diagnostics.py", + "skills/bmad-story-automator/src/story_automator/core/epic_parser.py", + "skills/bmad-story-automator/src/story_automator/core/frontmatter.py", + "skills/bmad-story-automator/src/story_automator/core/monitoring.py", + "skills/bmad-story-automator/src/story_automator/core/orchestration_events.py", + "skills/bmad-story-automator/src/story_automator/core/parse_contracts.py", + "skills/bmad-story-automator/src/story_automator/core/review_verify.py", + "skills/bmad-story-automator/src/story_automator/core/runtime_layout.py", + "skills/bmad-story-automator/src/story_automator/core/runtime_policy.py", + "skills/bmad-story-automator/src/story_automator/core/session_state.py", + "skills/bmad-story-automator/src/story_automator/core/sprint.py", + "skills/bmad-story-automator/src/story_automator/core/state_validation.py", + "skills/bmad-story-automator/src/story_automator/core/stop_hooks.py", + "skills/bmad-story-automator/src/story_automator/core/story_keys.py", + "skills/bmad-story-automator/src/story_automator/core/success_verifiers.py", + "skills/bmad-story-automator/src/story_automator/core/tmux_runtime.py", + "skills/bmad-story-automator/src/story_automator/core/utils.py", + "skills/bmad-story-automator/src/story_automator/core/workflow_paths.py", + "skills/bmad-story-automator-review/SKILL.md", + "skills/bmad-story-automator-review/checklist.md", + "skills/bmad-story-automator-review/contract.json", + "skills/bmad-story-automator-review/instructions.xml", + "skills/bmad-story-automator-review/workflow.yaml", +] + +EXECUTABLE_PACKAGE_FILES = { + "bin/bmad-story-automator", + "install.sh", + "skills/bmad-story-automator/scripts/story-automator", +} + +SUPPORTED_SKILL_ROOTS = [".agents/skills", ".claude/skills", ".codex/skills"] +DEPENDENCY_SKILLS = ["bmad-create-story", "bmad-dev-story", "bmad-retrospective"] + + +def assert_package_contract(root: Path, env: dict[str, str] | None = None) -> dict: + package = _load_package_json(root) + dry_run = _npm_pack_json(root, ["--dry-run", "--json"], env) + _assert_pack_metadata(dry_run, package) + _assert_content(dry_run) + + with tempfile.TemporaryDirectory(prefix="bmad-pack-assert-") as tmp: + identity = pack_project(root, Path(tmp), env) + _assert_tarball_checksums(Path(identity["tarball"]), identity["selectedChecksums"]) + return identity + + +def pack_project(root: Path, pack_dir: Path, env: dict[str, str] | None) -> dict: + pack_dir.mkdir(parents=True, exist_ok=True) + for tarball in pack_dir.glob("*.tgz"): + tarball.unlink() + + package = _load_package_json(root) + packed = _npm_pack_json(root, ["--json", "--pack-destination", str(pack_dir)], env) + _assert_pack_metadata(packed, package) + _assert_content(packed) + tarball = pack_dir / packed["filename"] + if not tarball.is_file(): + raise SmokeError(f"missing packed tarball: {tarball}") + return _identity_from_pack(root, packed, tarball) + + +def _load_package_json(root: Path) -> dict: + path = root / "package.json" + try: + package = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise SmokeError(f"invalid package.json: {path}: {exc}") from exc + if not isinstance(package, dict): + raise SmokeError(f"unexpected package.json shape: {package!r}") + return package + + +def write_package_identity(workspace: Path, identity: dict) -> Path: + path = workspace / "PACKAGE_IDENTITY.json" + path.write_text(json.dumps(identity, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return path + + +def verify_installed_package(gunz_dir: Path, identity: dict, workspace: Path) -> Path: + manifest = { + "package": { + key: identity[key] + for key in ("name", "version", "filename", "integrity", "shasum", "tarballSha256") + if key in identity + }, + "roots": [], + } + failures: list[str] = [] + for rel_root in SUPPORTED_SKILL_ROOTS: + root = gunz_dir / rel_root + deps_present = all((root / dep / "SKILL.md").is_file() for dep in DEPENDENCY_SKILLS) + story = root / "bmad-story-automator" + review = root / "bmad-story-automator-review" + if not deps_present: + manifest["roots"].append( + { + "root": rel_root, + "status": "unsupported", + "reason": "missing required dependency skill entrypoints", + } + ) + continue + if not story.is_dir() or not review.is_dir(): + failures.append(f"{rel_root}: automator skills not installed") + continue + + root_result = {"root": rel_root, "status": "installed", "checksums": {}} + for package_path, expected in identity["selectedChecksums"].items(): + if not package_path.startswith("skills/"): + continue + installed_rel = package_path.removeprefix("skills/") + installed_path = root / installed_rel + if not installed_path.is_file(): + failures.append(f"{rel_root}: missing installed file {installed_rel}") + continue + actual = _sha256(installed_path) + root_result["checksums"][installed_rel] = actual + if actual != expected: + failures.append( + f"{rel_root}: checksum mismatch for {installed_rel}: {actual} != {expected}" + ) + manifest["roots"].append(root_result) + + if not any(root["status"] == "installed" for root in manifest["roots"]): + failures.append("no supported skill root installed automator") + if failures: + raise SmokeError("installed package verification failed:\n" + "\n".join(failures)) + + path = workspace / "INSTALLED_AUTOMATOR_MANIFEST.json" + path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return path + + +def _npm_pack_json(root: Path, args: list[str], env: dict[str, str] | None) -> dict: + try: + result = subprocess.run( + ["npm", "pack", *args], + cwd=root, + env=env, + text=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=900, + ) + except subprocess.TimeoutExpired as exc: + raise SmokeError(f"npm pack timed out after 900s: {' '.join(args)}") from exc + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise SmokeError( + "failed to parse npm pack JSON" + f"\nstdout:\n{result.stdout.strip()}" + f"\nstderr:\n{result.stderr.strip()}" + ) from exc + if not isinstance(payload, list) or len(payload) != 1: + raise SmokeError(f"unexpected npm pack JSON: {result.stdout.strip()}") + return payload[0] + + +def _assert_pack_metadata(metadata: dict, package: dict) -> None: + if not isinstance(metadata, dict): + raise SmokeError(f"unexpected npm pack entry shape: {metadata!r}") + if not isinstance(package, dict): + raise SmokeError(f"unexpected package.json shape: {package!r}") + missing_package = [key for key in ("name", "version") if not package.get(key)] + if missing_package: + raise SmokeError("package identity failed:\nmissing package fields: " + ", ".join(missing_package)) + expected_filename = f"{package['name']}-{package['version']}.tgz" + failures = [] + for key in ("name", "version"): + if metadata.get(key) != package[key]: + failures.append(f"{key}: {metadata.get(key)} != {package[key]}") + if metadata.get("filename") != expected_filename: + failures.append(f"filename: {metadata.get('filename')} != {expected_filename}") + if not (metadata.get("integrity") or metadata.get("shasum")): + failures.append("missing integrity or shasum") + if failures: + raise SmokeError("package identity failed:\n" + "\n".join(failures)) + + +def _assert_content(metadata: dict) -> None: + if not isinstance(metadata, dict): + raise SmokeError(f"unexpected npm pack entry shape: {metadata!r}") + files = metadata.get("files") + if not isinstance(files, list): + raise SmokeError("package content failed:\nmissing files list") + entries: dict[str, dict] = {} + failures = [] + for entry in files: + if not isinstance(entry, dict) or not isinstance(entry.get("path"), str): + failures.append(f"malformed file entry: {entry!r}") + continue + entries[entry["path"]] = entry + missing = sorted(set(REQUIRED_PACKAGE_FILES) - set(entries)) + if missing: + failures.append("missing required files:\n" + "\n".join(missing)) + for path in EXECUTABLE_PACKAGE_FILES: + if entries.get(path, {}).get("mode") != 493: + failures.append(f"expected executable mode 493 for {path}") + forbidden = sorted(path for path in entries if _is_forbidden(path)) + if forbidden: + failures.append("forbidden generated files:\n" + "\n".join(forbidden)) + if failures: + raise SmokeError("package content failed:\n" + "\n\n".join(failures)) + + +def _is_forbidden(path: str) -> bool: + parts = path.split("/") + return ( + "__pycache__" in parts + or ".pytest_cache" in parts + or "node_modules" in parts + or "dist" in parts + or path.endswith((".pyc", ".pyo", ".tgz", ".DS_Store")) + or path.startswith(".firecrawl/") + or path.startswith(".smoke/") + or path.startswith("skills/bmad-story-automator/build/") + or ".egg-info" in parts + ) + + +def _identity_from_pack(root: Path, metadata: dict, tarball: Path) -> dict: + return { + "name": metadata["name"], + "version": metadata["version"], + "filename": metadata["filename"], + "integrity": metadata.get("integrity"), + "shasum": metadata.get("shasum"), + "tarball": str(tarball), + "tarballSha256": _sha256(tarball), + "entryCount": metadata.get("entryCount"), + "selectedChecksums": _selected_checksums_from_tarball(tarball), + "sourcePackageJson": str(root / "package.json"), + } + + +def _selected_checksums_from_tarball(tarball: Path) -> dict[str, str]: + checksums: dict[str, str] = {} + with tarfile.open(tarball, "r:gz") as archive: + for member_info in archive.getmembers(): + if not member_info.isfile(): + continue + package_path = _installed_skill_package_path(member_info.name) + if package_path is None: + continue + member = archive.extractfile(member_info) + if member is None: + raise SmokeError(f"missing tarball member: {member_info.name}") + checksums[package_path] = hashlib.sha256(member.read()).hexdigest() + return checksums + + +def _installed_skill_package_path(tar_member_name: str) -> str | None: + prefix = "package/" + if not tar_member_name.startswith(prefix): + return None + package_path = tar_member_name.removeprefix(prefix) + if package_path.startswith( + ( + "skills/bmad-story-automator/", + "skills/bmad-story-automator-review/", + ) + ): + return package_path + return None + + +def _assert_tarball_checksums(tarball: Path, checksums: dict[str, str]) -> None: + with tarfile.open(tarball, "r:gz") as archive: + for package_path, expected in checksums.items(): + member = archive.extractfile(f"package/{package_path}") + if member is None: + raise SmokeError(f"missing tarball member: {package_path}") + actual = hashlib.sha256(member.read()).hexdigest() + if actual != expected: + raise SmokeError(f"tarball checksum mismatch: {package_path}") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/scripts/smoke_prep/process.py b/scripts/smoke_prep/process.py new file mode 100644 index 00000000..ab90544a --- /dev/null +++ b/scripts/smoke_prep/process.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + + +class SmokeError(RuntimeError): + pass + + +MARKER_OVERRIDE_ENV = ( + "BMAD_STORY_AUTOMATOR_ACTIVE_MARKER", + "STORY_AUTOMATOR_ACTIVE_MARKER", +) +HOST_OVERRIDE_ENV = ( + "BMAD_SKILLS_ROOT", + "BMAD_RUNTIME_PROVIDER", + "STORY_AUTOMATOR_RUNTIME_PROVIDER", +) + + +def scrub_host_overrides(env: dict[str, str]) -> dict[str, str]: + for name in (*MARKER_OVERRIDE_ENV, *HOST_OVERRIDE_ENV): + env.pop(name, None) + return env + + +def deterministic_smoke_env(project: Path, extra: dict[str, str] | None = None) -> dict[str, str]: + env = scrub_host_overrides(os.environ.copy()) + env["PROJECT_ROOT"] = str(project) + env.update(extra or {}) + return env + + +def step(name: str) -> None: + print(f"\n==> {name}", flush=True) + + +def run( + args: list[str], + *, + cwd: Path, + env: dict[str, str] | None = None, + capture: bool = False, + check: bool = True, + timeout_seconds: float | None = 900, +) -> subprocess.CompletedProcess[str]: + print(f"+ ({cwd}) {' '.join(args)}", flush=True) + try: + return subprocess.run( + args, + cwd=cwd, + env=env, + text=True, + check=check, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.STDOUT if capture else None, + timeout=timeout_seconds, + ) + except subprocess.TimeoutExpired as exc: + raise SmokeError(f"command timed out after {timeout_seconds}s: {' '.join(args)}") from exc + + +def ensure_tool(name: str) -> None: + if shutil.which(name) is None: + raise SmokeError(f"missing required tool: {name}") diff --git a/scripts/smoke_prep/report.py b/scripts/smoke_prep/report.py new file mode 100644 index 00000000..43d935b1 --- /dev/null +++ b/scripts/smoke_prep/report.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import shlex +from pathlib import Path + +from .config import BMAD_METHOD_NPM_SPEC, BRANCH, PINNED_COMMIT, REPO_URL + + +def write_next_steps(workspace: Path, gunz_dir: Path, *, automator_installed: bool = True) -> Path: + next_steps = workspace / "SMOKE_NEXT_STEPS.md" + helper = ( + gunz_dir + / ".claude" + / "skills" + / "bmad-story-automator" + / "scripts" + / "story-automator" + ) + quoted_gunz_dir = shlex.quote(str(gunz_dir)) + quoted_helper = shlex.quote(str(helper)) + helper_lines = ( + [ + "Helper sanity check:", + "", + "```bash", + f"{quoted_helper} --help", + "```", + ] + if automator_installed + else [ + "Helper sanity check:", + "", + "- skipped by `--skip-automator-install`", + ] + ) + next_steps.write_text( + "\n".join( + [ + "# Story Automator Smoke", + "", + "Prepared project:", + "", + f"```text\n{gunz_dir}\n```", + "", + "Pinned source:", + "", + f"- repo: `{REPO_URL}`", + f"- branch: `{BRANCH}`", + f"- commit: `{PINNED_COMMIT}`", + f"- deterministic input manifest: `{workspace / 'SMOKE_INPUTS.json'}`", + f"- package identity: `{workspace / 'PACKAGE_IDENTITY.json'}`" if automator_installed else "- package identity: skipped by `--skip-automator-install`", + f"- installed manifest: `{workspace / 'INSTALLED_AUTOMATOR_MANIFEST.json'}`" if automator_installed else "- installed manifest: skipped by `--skip-automator-install`", + "", + "Installed pieces:", + "", + f"- BMAD core and BMM via `{BMAD_METHOD_NPM_SPEC}`", + "- project-local `bmad-story-automator` packed from this checkout" if automator_installed else "- project-local automator install skipped", + "", + "Manual smoke start:", + "", + "```bash", + f"cd {quoted_gunz_dir}", + "claude", + "```", + "", + "Then ask Claude Code:", + "", + "```text", + "Use the bmad-story-automator skill. Run the smoke test in this repo.", + "```", + "", + *helper_lines, + "", + ] + ), + encoding="utf-8", + ) + return next_steps diff --git a/scripts/smoke_prep/workspace.py b/scripts/smoke_prep/workspace.py new file mode 100644 index 00000000..77cdda9c --- /dev/null +++ b/scripts/smoke_prep/workspace.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +from .process import SmokeError + + +def reset_dir(path: Path) -> None: + if path.exists(): + shutil.rmtree(path) + + +def resolve_workspace(root: Path, workspace_arg: str) -> Path: + requested = Path(workspace_arg) + if requested.is_absolute(): + raise SmokeError("--workspace must be a repo-relative ignored path") + + workspace = (root / requested).resolve() + try: + relative = workspace.relative_to(root) + except ValueError as exc: + raise SmokeError("--workspace must stay inside this repo") from exc + + if relative == Path("."): + raise SmokeError("--workspace cannot be the repo root") + + check = subprocess.run( + ["git", "check-ignore", "-q", "--", relative.as_posix()], + cwd=root, + check=False, + ) + if check.returncode != 0: + raise SmokeError( + f"--workspace must be gitignored before use: {relative.as_posix()}" + ) + + return workspace diff --git a/skills/bmad-story-automator/src/story_automator/commands/orchestrator_epic_agents.py b/skills/bmad-story-automator/src/story_automator/commands/orchestrator_epic_agents.py index 8339250a..a145fadf 100644 --- a/skills/bmad-story-automator/src/story_automator/commands/orchestrator_epic_agents.py +++ b/skills/bmad-story-automator/src/story_automator/commands/orchestrator_epic_agents.py @@ -8,12 +8,12 @@ from story_automator.core.agent_config import AgentConfigResolved, load_agent_config_from_state, parse_agent_config_json, resolve_agent_for_task from story_automator.core.agent_plan import AgentPlanInputError, agent_plan_error, build_agents_file, load_agents_plan_for_resolution, load_complexity_payload, resolve_agents_payload from story_automator.core.diagnostics import issues_from_exception +from story_automator.core.epic_parser import EPIC_HEADER_RE, STORY_HEADER_RE from story_automator.core.frontmatter import find_frontmatter_value, parse_frontmatter from story_automator.core.sprint import sprint_status_epic from story_automator.core.story_keys import StoryKey, normalize_story_key, normalize_story_key_for_epic from story_automator.core.utils import file_exists, get_project_root, print_json, read_text, trim_lines - def check_epic_complete_action(args: list[str]) -> int: try: if len(args) < 2: @@ -98,18 +98,27 @@ def check_blocking_action(args: list[str]) -> int: if norm is None: print_json({"ok": False, "error": "could_not_normalize_key", "input": args[0]}) return 1 - epic = norm.id.split(".", 1)[0] + epic = norm.id.rsplit(".", 1)[0] epic_file = find_epic_file(epic) if not epic_file: print_json({"ok": True, "blocking": True, "story": norm.id, "epic": epic, "dependents": [], "reason": "epic_file_not_found", "source": "unknown"}) return 0 dependents: list[str] = [] current_story = "" + current_epic = "" for line in trim_lines(read_text(epic_file)): - match = re.match(r"^###\s+Story\s+([^:]+):", line) + epic_match = EPIC_HEADER_RE.match(line) + if epic_match: + current_epic = epic_match.group(1).strip() + current_story = "" + continue + match = STORY_HEADER_RE.match(line) if match: - candidate_story = match.group(1).strip() - current_story = candidate_story if _story_matches_epic(project_root, epic, candidate_story) else "" + candidate_story = (match.group(1) or match.group(2) or "").strip() + if _story_matches_epic(project_root, epic, candidate_story): + current_story = candidate_story + elif not current_epic or current_epic != epic: + current_story = "" continue if current_story and re.search(r"(?i)Dependencies:|\*\*Dependencies\*\*:", line): if _line_references_story(project_root, epic, norm, args[0], line): @@ -224,10 +233,9 @@ def find_epic_file(epic: str) -> str: def _epic_file_has_story(epic_file: Path, epic: str, *, project_root: str) -> bool: - story_re = re.compile(r"^###\s+Story\s+([^:]+):") for line in trim_lines(read_text(epic_file)): - match = story_re.match(line) - if match and _story_matches_epic(project_root, epic, match.group(1).strip()): + match = STORY_HEADER_RE.match(line) + if match and _story_matches_epic(project_root, epic, (match.group(1) or match.group(2) or "").strip()): return True return False @@ -237,15 +245,14 @@ def _epic_json_value(epic: str) -> int | str: def _story_ids_from_epic_file(epic_file: str, epic: str) -> list[str]: - story_re = re.compile(r"^###\s+Story\s+([^:]+):") stories: list[str] = [] seen_ids: set[str] = set() project_root = get_project_root() for line in trim_lines(read_text(epic_file)): - match = story_re.match(line) + match = STORY_HEADER_RE.match(line) if not match: continue - story = match.group(1).strip() + story = (match.group(1) or match.group(2) or "").strip() norm = normalize_story_key_for_epic(project_root, epic, story) if norm is None or norm.id.rsplit(".", 1)[0] != epic or norm.id in seen_ids: continue @@ -303,7 +310,7 @@ def _story_key_rank(story: str, norm: StoryKey | None) -> int: def _line_references_story(project_root: str, epic: str, target: StoryKey, requested_story: str, line: str) -> bool: requested_full_key = _is_explicit_full_key(requested_story, target) - for match in re.finditer(r"\b(?:\d+\.\d+|\d+-\d+(?:-[\w]+)*|[A-Za-z][\w-]*(?:\.\d+|-\d+(?:-[\w]+)*))\b", line): + for match in re.finditer(r"\b(?:\d+(?:\.\d+)+|\d+-\d+(?:-[\w]+)*|[A-Za-z][\w-]*(?:\.\d+|-\d+(?:-[\w]+)*))\b", line): token = match.group(0) norm = normalize_story_key_for_epic(project_root, epic, token) if norm is not None and norm.id == target.id: diff --git a/skills/bmad-story-automator/src/story_automator/core/epic_parser.py b/skills/bmad-story-automator/src/story_automator/core/epic_parser.py index 19d35e9b..c7318dd9 100644 --- a/skills/bmad-story-automator/src/story_automator/core/epic_parser.py +++ b/skills/bmad-story-automator/src/story_automator/core/epic_parser.py @@ -9,6 +9,10 @@ from .story_keys import normalize_story_key, normalize_story_key_for_epic +STORY_HEADER_RE = re.compile(r"^###\s+(?:(?:Story\s+)?(\d+(?:\.\d+)+)|Story\s+([^:]+)):\s*(.*)$", re.IGNORECASE) +EPIC_HEADER_RE = re.compile(r"^##\s+Epic\s+([A-Za-z][\w-]*|\d+(?:\.\d+)*):\s*(.*)$", re.IGNORECASE) + + def parse_epic_file(epic_file: str | Path) -> dict[str, Any]: content = read_text(epic_file) lines = trim_lines(content) @@ -18,20 +22,19 @@ def parse_epic_file(epic_file: str | Path) -> dict[str, Any]: if line.startswith("# "): epic_title = line.removeprefix("# ").strip() break - story_re = re.compile(r"^###\s+Story\s+([^:]+):\s*(.*)$") - epic_re = re.compile(r"^##\s+Epic\s+([A-Za-z][\w-]*|\d+):\s*(.*)$") current_epic = "" current_epic_title = "" stories: list[dict[str, str]] = [] for line in lines: - epic_match = epic_re.match(line) + epic_match = EPIC_HEADER_RE.match(line) if epic_match: current_epic = epic_match.group(1).strip() current_epic_title = epic_match.group(2).strip() continue - story_match = story_re.match(line) + story_match = STORY_HEADER_RE.match(line) if story_match: - raw_story, title = story_match.groups() + numeric_story, named_story, title = story_match.groups() + raw_story = numeric_story or named_story or "" story_key = _normalize_header_story(project_root, current_epic, raw_story.strip()) if story_key is None: continue @@ -55,24 +58,27 @@ def parse_story(epic_file: str | Path, story_id: str, rules_file: str | Path) -> content = read_text(epic_file) lines = trim_lines(content) project_root = _project_root_for_epic_file(epic_file) - epic_re = re.compile(r"^##\s+Epic\s+([A-Za-z][\w-]*|\d+):") - header_re = re.compile(r"^###\s+Story\s+([^:]+):\s*(.*)$") target_id = story_id start_index = -1 title = "" current_epic = "" for index, line in enumerate(lines): - epic_match = epic_re.match(line) + epic_match = EPIC_HEADER_RE.match(line) if epic_match: current_epic = epic_match.group(1).strip() continue - match = header_re.match(line) + match = STORY_HEADER_RE.match(line) if match: - raw_story, raw_title = match.groups() + numeric_story, named_story, raw_title = match.groups() + raw_story = numeric_story or named_story or "" story_key = _normalize_header_story(project_root, current_epic, raw_story.strip()) if story_key is None: continue - if target_id not in {raw_story.strip(), story_key.id, story_key.prefix, story_key.key}: + target_aliases = {raw_story.strip(), story_key.id, story_key.prefix, story_key.key} + title_slug = _slugify_title(raw_title) + if title_slug and not _is_explicit_header_story(raw_story.strip(), story_key.id): + target_aliases.add(f"{story_key.prefix}-{title_slug}") + if target_id not in target_aliases: continue start_index = index target_id = story_key.id @@ -84,9 +90,15 @@ def parse_story(epic_file: str | Path, story_id: str, rules_file: str | Path) -> acceptance_criteria: list[str] = [] dependencies = "" in_ac = False + content_epic = current_epic for line in lines[start_index + 1 :]: - if line.startswith("### Story ") or line.startswith("## Epic "): + if EPIC_HEADER_RE.match(line): break + boundary_match = STORY_HEADER_RE.match(line) + if boundary_match: + raw_story = (boundary_match.group(1) or boundary_match.group(2) or "").strip() + if _normalize_header_story(project_root, content_epic, raw_story) is not None: + break if "Acceptance Criteria" in line: in_ac = True continue @@ -157,11 +169,14 @@ def _project_root_for_epic_file(epic_file: str | Path) -> str: def _normalize_header_story(project_root: str, current_epic: str, raw_story: str): - if current_epic: - story_key = normalize_story_key_for_epic(project_root, current_epic, raw_story) - if story_key is not None: - return story_key - return normalize_story_key(project_root, raw_story) + if not current_epic: + return normalize_story_key(project_root, raw_story) + story_key = normalize_story_key_for_epic(project_root, current_epic, raw_story) + if story_key is None: + return None + if story_key.id != current_epic and not story_key.id.startswith(f"{current_epic}."): + return None + return story_key def parse_story_range(user_input: str, total: int, ids_csv: str = "") -> dict[str, Any]: @@ -170,19 +185,29 @@ def parse_story_range(user_input: str, total: int, ids_csv: str = "") -> dict[st ids = [part.strip() for part in ids_csv.split(",")] if ids_csv else [] selected: set[int] = set() normalized = user_input.lower().replace(" ", "") + id_index = {story_id.lower(): index + 1 for index, story_id in enumerate(ids)} if normalized == "all": selected = set(range(1, total + 1)) else: for part in normalized.split(","): if not part: continue - if "-" in part: + if part in id_index: + selected.add(id_index[part]) + elif "-" in part: start_raw, end_raw = part.split("-", 1) if start_raw.isdigit() and end_raw.isdigit(): start = int(start_raw) end = int(end_raw) low, high = sorted((start, end)) selected.update(range(low, high + 1)) + elif start_raw in id_index and end_raw in id_index: + start = id_index[start_raw] + end = id_index[end_raw] + low, high = sorted((start, end)) + selected.update(range(low, high + 1)) + else: + raise ValueError(f"invalid_story_range:{part}") elif part.isdigit(): selected.add(int(part)) indices = sorted(index for index in selected if 1 <= index <= total) @@ -219,8 +244,7 @@ def _story_aliases(story: dict[str, str]) -> set[str]: if story_key: aliases.add(story_key) if not _is_explicit_header_story(header_story, story_id): - epic, _, story_num = story_id.rpartition(".") - prefix = f"{epic}-{story_num}" + prefix = story_id.replace(".", "-") aliases.add(prefix) title_slug = _slugify_title(story.get("title", "")) if title_slug: @@ -231,16 +255,15 @@ def _story_aliases(story: dict[str, str]) -> set[str]: def _is_explicit_header_story(header_story: str, story_id: str) -> bool: if not header_story: return False - epic, _, story_num = story_id.rpartition(".") - return header_story not in {story_id, f"{epic}-{story_num}"} + return header_story not in {story_id, story_id.replace(".", "-")} def _slugify_title(title: str) -> str: return "-".join(part for part in re.split(r"[^A-Za-z0-9]+", title.lower()) if part) -def _story_sort_key(value: str) -> tuple[int, int, str, int, str]: +def _story_sort_key(value: str) -> tuple[int, tuple[int, ...], str, int, str]: epic, _, story_num = value.rpartition(".") - if epic.isdigit(): - return (0, int(epic), "", int(story_num) if story_num.isdigit() else 0, value) - return (1, 0, epic, int(story_num) if story_num.isdigit() else 0, value) + if all(part.isdigit() for part in value.split(".")): + return (0, tuple(int(part) for part in value.split(".")), "", 0, value) + return (1, (), epic, int(story_num) if story_num.isdigit() else 0, value) diff --git a/skills/bmad-story-automator/src/story_automator/core/story_keys.py b/skills/bmad-story-automator/src/story_automator/core/story_keys.py index 2a9783e7..f2549313 100644 --- a/skills/bmad-story-automator/src/story_automator/core/story_keys.py +++ b/skills/bmad-story-automator/src/story_automator/core/story_keys.py @@ -20,17 +20,17 @@ def sprint_status_file(project_root: str) -> str: def normalize_story_key(project_root: str, value: str) -> StoryKey | None: - if re.fullmatch(r"\d+\.\d+", value): + if re.fullmatch(r"\d+(?:\.\d+)+", value): story_id = value prefix = value.replace(".", "-") key = "" - elif re.fullmatch(r"\d+-\d+", value): + elif re.fullmatch(r"\d+(?:-\d+)+", value): prefix = value story_id = value.replace("-", ".") key = "" - elif re.fullmatch(r"\d+-\d+-.+", value): + elif re.fullmatch(r"\d+(?:-\d+)+-.+", value): key = value - prefix = "-".join(value.split("-", 2)[:2]) + prefix = _numeric_full_key_prefix(value) story_id = prefix.replace("-", ".") elif re.fullmatch(r"[A-Za-z][\w-]*\.\d+", value): story_id = value @@ -63,6 +63,11 @@ def normalize_story_key_for_epic(project_root: str, epic: str, value: str) -> St return None return norm + nested_numeric = _nested_numeric_story_for_epic(epic, value) + if nested_numeric is not None: + story_id, prefix, key = nested_numeric + return _complete_story_key(project_root, story_id, prefix, key) + dotted = re.fullmatch(rf"{re.escape(epic)}\.(\d+)", value) if dotted: story_num = dotted.group(1) @@ -139,6 +144,36 @@ def _status_keys(content: str) -> list[str]: return keys +def _numeric_full_key_prefix(value: str) -> str: + parts = value.split("-") + if len(parts) >= 4 and all(part.isdigit() for part in parts[:3]): + nested_prefix = "-".join(parts[:3]) + if not _numeric_title_segment(parts): + return nested_prefix + return "-".join(parts[:2]) + + +def _nested_numeric_story_for_epic(epic: str, value: str) -> tuple[str, str, str] | None: + if not all(part.isdigit() for part in epic.split(".")): + return None + epic_prefix = epic.replace(".", "-") + match = re.fullmatch(rf"{re.escape(epic_prefix)}-(\d+)(?:-.+)?", value) + if not match: + return None + parts = value.split("-") + if len(parts) >= 4 and _numeric_title_segment(parts): + return None + prefix = f"{epic_prefix}-{match.group(1)}" + key = value if value != prefix else "" + return f"{epic}.{match.group(1)}", prefix, key + + +def _numeric_title_segment(parts: list[str]) -> bool: + if len(parts[2]) == 4: + return True + return len(parts[2]) == 2 and len(parts) >= 4 and parts[3] in {"release"} + + def _full_key_matches_story(project_root: str, key: str, story_id: str, *, allow_ambiguous_same_id: bool) -> bool: norm = normalize_story_key(project_root, key) if norm is not None: diff --git a/skills/bmad-story-automator/src/story_automator/core/success_verifiers.py b/skills/bmad-story-automator/src/story_automator/core/success_verifiers.py index 79f860e3..08c1dbb4 100644 --- a/skills/bmad-story-automator/src/story_automator/core/success_verifiers.py +++ b/skills/bmad-story-automator/src/story_automator/core/success_verifiers.py @@ -289,7 +289,7 @@ def _epic_identifier(project_root: str, story_key: str) -> str: return story_key norm = normalize_story_key(project_root, story_key) if norm is not None: - return norm.id.split(".", 1)[0] + return norm.id.rsplit(".", 1)[0] if re.fullmatch(r"[A-Za-z][\w-]*", story_key) and sprint_status_epic(project_root, story_key)[0]: return story_key return "" diff --git a/skills/bmad-story-automator/workflow.md b/skills/bmad-story-automator/workflow.md index 9dacea3b..fd52f225 100644 --- a/skills/bmad-story-automator/workflow.md +++ b/skills/bmad-story-automator/workflow.md @@ -1,6 +1,6 @@ --- name: story-automator -version: "1.12.0" +version: "1.15.0" description: "Automate the build cycle for stories in an epic using T-Mux sessions with full resumability, smart parallelism, decision escalation, and automated retrospectives (tri-modal: create, validate, edit)" web_bundle: true configPath: '{project-root}/_bmad/bmm/config.yaml' diff --git a/tests/test_cli_contracts.py b/tests/test_cli_contracts.py index d525fa86..a98bc20e 100644 --- a/tests/test_cli_contracts.py +++ b/tests/test_cli_contracts.py @@ -69,6 +69,82 @@ def test_parse_story_success_scores_story(self) -> None: self.assertEqual(payload["complexity"]["score"], 3) self.assertEqual(payload["complexity"]["level"], "Medium") + def test_parse_story_accepts_bmad_numbered_headers(self) -> None: + epic = self.root / "epic.md" + epic.write_text( + "# Product Epic\n\n## EPIC 1: Platform\n\n### 1.1: Add database sync\nDescription line.\n\nAcceptance Criteria\n- Works reliably\n\n### 1.2: Next story\n", + encoding="utf-8", + ) + rules = self.root / "rules.json" + rules.write_text( + json.dumps({"rules": [{"pattern": "database", "score": 3, "label": "Touches DB"}], "thresholds": {"low_max": 1, "medium_max": 3}}), + encoding="utf-8", + ) + + code, payload = self._main_json(["parse-story", "--epic", str(epic), "--story", "1.1", "--rules", str(rules)]) + + self.assertEqual(code, 0) + self.assertTrue(payload["ok"]) + self.assertEqual(payload["storyId"], "1.1") + self.assertEqual(payload["title"], "Add database sync") + self.assertEqual(payload["complexity"]["level"], "Medium") + + def test_parse_epic_accepts_bmad_numbered_headers(self) -> None: + epic = self.root / "epic.md" + epic.write_text( + "# Product Epic\n\n## EPIC 1: Platform\n\n### 1.1: Add database sync\n\n## Epic 2: Browse\n\n### Story 2.1: Search listings\n", + encoding="utf-8", + ) + + code, payload = self._main_json(["parse-epic", "--file", str(epic)]) + + self.assertEqual(code, 0) + self.assertEqual(payload["count"], 2) + self.assertEqual([story["storyId"] for story in payload["stories"]], ["1.1", "2.1"]) + self.assertEqual(payload["stories"][0]["epicTitle"], "Platform") + + def test_parse_story_range_accepts_explicit_story_ids(self) -> None: + code, payload = self._main_json(["parse-story-range", "--input", "1.1", "--total", "3", "--ids", "1.1,1.2,2.1"]) + + self.assertEqual(code, 0) + self.assertEqual(payload["indices"], [1]) + self.assertEqual(payload["storyIds"], ["1.1"]) + + def test_parse_story_range_accepts_hyphenated_explicit_story_ids(self) -> None: + code, payload = self._main_json(["parse-story-range", "--input", "multi-leg-3", "--total", "2", "--ids", "multi-leg-3,multi-leg-4-next"]) + + self.assertEqual(code, 0) + self.assertEqual(payload["indices"], [1]) + self.assertEqual(payload["storyIds"], ["multi-leg-3"]) + + def test_parse_story_ignores_bare_numeric_subheading_from_other_epic(self) -> None: + epic = self.root / "epic.md" + epic.write_text( + "# Product Epic\n\n" + "## Epic 1: Platform\n\n" + "### 1.1: Add database sync\n" + "Intro line.\n" + "### 2.0: Non-story subheading\n" + "Later detail with database.\n\n" + "Acceptance Criteria\n" + "- Works reliably\n", + encoding="utf-8", + ) + rules = self.root / "rules.json" + rules.write_text( + json.dumps({"rules": [{"pattern": "database", "score": 3, "label": "Touches DB"}], "thresholds": {"low_max": 1, "medium_max": 3}}), + encoding="utf-8", + ) + + code, epic_payload = self._main_json(["parse-epic", "--file", str(epic)]) + self.assertEqual(code, 0) + self.assertEqual([story["storyId"] for story in epic_payload["stories"]], ["1.1"]) + + code, story_payload = self._main_json(["parse-story", "--epic", str(epic), "--story", "1.1", "--rules", str(rules)]) + self.assertEqual(code, 0) + self.assertIn("Later detail", story_payload["description"]) + self.assertEqual(story_payload["acceptanceCriteria"], ["- Works reliably"]) + def test_parse_story_read_failure_returns_json_error(self) -> None: epic = self._epic_file() rules = self.root / "rules.json" diff --git a/tests/test_epic_parser.py b/tests/test_epic_parser.py index fac1b585..30b3c8cf 100644 --- a/tests/test_epic_parser.py +++ b/tests/test_epic_parser.py @@ -4,7 +4,7 @@ import unittest from pathlib import Path -from story_automator.core.epic_parser import epic_complete, parse_epic_file, parse_story +from story_automator.core.epic_parser import epic_complete, parse_epic_file, parse_story, parse_story_range class EpicParserTests(unittest.TestCase): @@ -77,6 +77,19 @@ def test_parse_story_accepts_canonical_id_for_full_key_header(self) -> None: self.assertEqual(payload["storyId"], "multi-leg.3") self.assertEqual(payload["title"], "Old") + def test_parse_story_rejects_missing_explicit_full_key_sibling(self) -> None: + self.epic_file.write_text( + """# Epic Numeric +## Epic 1.2: Nested Epic +### Story 1-2-3-old: New +Acceptance Criteria +- Works +""", + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "story_not_found"): + parse_story(self.epic_file, "1-2-3-new", self.rules_file) + def test_epic_complete_accepts_non_numeric_story_ids(self) -> None: self.epic_file.write_text( """# Epic Multi Leg @@ -144,6 +157,87 @@ def test_epic_complete_sorts_numeric_epics_numerically(self) -> None: self.assertFalse(payload["epicComplete"]) self.assertEqual(payload["maxEpicStory"], "10.1") + def test_parse_epic_file_accepts_three_level_numeric_story_ids(self) -> None: + self.epic_file.write_text( + """# Epic Numeric +### 1.1.1: Nested +""", + encoding="utf-8", + ) + payload = parse_epic_file(self.epic_file) + self.assertTrue(payload["ok"]) + self.assertEqual(payload["stories"][0]["storyId"], "1.1.1") + self.assertEqual(payload["stories"][0]["epicNum"], "1.1") + self.assertEqual(payload["stories"][0]["storyNum"], "1") + + def test_parse_epic_file_accepts_nested_numeric_story_under_numeric_epic(self) -> None: + self.epic_file.write_text( + """# Epic Numeric +## Epic 1.1: Nested Epic +### 1.1.1: Nested +""", + encoding="utf-8", + ) + payload = parse_epic_file(self.epic_file) + self.assertTrue(payload["ok"]) + self.assertEqual(payload["stories"][0]["storyId"], "1.1.1") + self.assertEqual(payload["stories"][0]["epicTitle"], "Nested Epic") + + def test_epic_complete_sorts_nested_numeric_story_ids_numerically(self) -> None: + self.epic_file.write_text( + """# Epic Numeric +### 1.9.1: Old +### 1.10.1: New +""", + encoding="utf-8", + ) + payload = epic_complete(self.epic_file, "1.9.1") + self.assertTrue(payload["ok"]) + self.assertFalse(payload["epicComplete"]) + self.assertEqual(payload["maxEpicStory"], "1.10.1") + + def test_nested_numeric_story_accepts_canonical_dashed_full_key_alias(self) -> None: + self.epic_file.write_text( + """# Epic Numeric +## Epic 1.2: Nested Epic +### 1.2.3: Nested +Acceptance Criteria +- Works +""", + encoding="utf-8", + ) + story = parse_story(self.epic_file, "1-2-3-nested", self.rules_file) + self.assertEqual(story["storyId"], "1.2.3") + complete = epic_complete(self.epic_file, "1-2-3-nested") + self.assertTrue(complete["ok"]) + self.assertTrue(complete["epicComplete"]) + + def test_nested_numeric_story_accepts_multi_digit_dashed_full_key_under_epic(self) -> None: + self.epic_file.write_text( + """# Epic Numeric +## Epic 1.2: Nested Epic +### Story 1-2-21-nested: Nested +""", + encoding="utf-8", + ) + payload = parse_epic_file(self.epic_file) + self.assertEqual(payload["stories"][0]["storyId"], "1.2.21") + + def test_nested_numeric_epic_preserves_title_like_numeric_segments(self) -> None: + self.epic_file.write_text( + """# Epic Numeric +## Epic 1.2: Nested Epic +### Story 1-2-2026-release: Release +""", + encoding="utf-8", + ) + payload = parse_epic_file(self.epic_file) + self.assertEqual(payload["stories"][0]["storyId"], "1.2") + + def test_parse_story_range_rejects_unknown_hyphenated_story_id(self) -> None: + with self.assertRaisesRegex(ValueError, "invalid_story_range:multi-leg-3x"): + parse_story_range("multi-leg-3x", 2, "multi-leg-3,multi-leg-4") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_normalize_story_key.py b/tests/test_normalize_story_key.py index 9f3610fa..4d605586 100644 --- a/tests/test_normalize_story_key.py +++ b/tests/test_normalize_story_key.py @@ -44,6 +44,52 @@ def test_numeric_full_key(self) -> None: self.assertEqual(result.prefix, "1-2") self.assertEqual(result.key, "1-2-user-authentication") + def test_numeric_full_key_with_numeric_title_segment(self) -> None: + result = normalize_story_key(str(self.project_root), "1-2-2026-release") + assert result is not None + self.assertEqual(result.id, "1.2") + self.assertEqual(result.prefix, "1-2") + self.assertEqual(result.key, "1-2-2026-release") + + def test_numeric_full_key_with_two_digit_numeric_title_segment(self) -> None: + result = normalize_story_key(str(self.project_root), "1-2-42-release") + assert result is not None + self.assertEqual(result.id, "1.2") + self.assertEqual(result.prefix, "1-2") + self.assertEqual(result.key, "1-2-42-release") + + def test_nested_numeric_full_key(self) -> None: + result = normalize_story_key(str(self.project_root), "1-2-3-nested") + assert result is not None + self.assertEqual(result.id, "1.2.3") + self.assertEqual(result.prefix, "1-2-3") + self.assertEqual(result.key, "1-2-3-nested") + + def test_nested_numeric_full_key_with_multi_digit_story_number(self) -> None: + result = normalize_story_key(str(self.project_root), "1-2-10-nested") + assert result is not None + self.assertEqual(result.id, "1.2.10") + self.assertEqual(result.prefix, "1-2-10") + self.assertEqual(result.key, "1-2-10-nested") + + def test_nested_numeric_full_key_above_twenty(self) -> None: + result = normalize_story_key(str(self.project_root), "1-2-21-nested") + assert result is not None + self.assertEqual(result.id, "1.2.21") + self.assertEqual(result.prefix, "1-2-21") + self.assertEqual(result.key, "1-2-21-nested") + + def test_epic_hint_preserves_numeric_title_segments(self) -> None: + year = normalize_story_key_for_epic(str(self.project_root), "1.2", "1-2-2026-release") + assert year is not None + self.assertEqual(year.id, "1.2") + self.assertEqual(year.prefix, "1-2") + + release = normalize_story_key_for_epic(str(self.project_root), "1.2", "1-2-42-release") + assert release is not None + self.assertEqual(release.id, "1.2") + self.assertEqual(release.prefix, "1-2") + # --- Non-numeric epic keys (the regression this patch restores) --- def test_non_numeric_dotted_id(self) -> None: diff --git a/tests/test_orchestrator_epic_agents.py b/tests/test_orchestrator_epic_agents.py index 20944a43..33bfd91b 100644 --- a/tests/test_orchestrator_epic_agents.py +++ b/tests/test_orchestrator_epic_agents.py @@ -179,6 +179,25 @@ def test_check_blocking_does_not_treat_longer_dotted_epic_reference_as_current_e self.assertFalse(payload["blocking"]) self.assertEqual(payload["dependents"], []) + def test_check_blocking_accepts_nested_numeric_story_dependencies(self) -> None: + path = self.project_root / "_bmad-output" / "implementation-artifacts" / "epic-1.2.md" + path.write_text( + textwrap.dedent( + """ + ## Epic 1.2: Nested + ### 1.2.4: Later + Dependencies: 1.2.3 + """ + ), + encoding="utf-8", + ) + exit_code, payload = self._run_action(check_blocking_action, ["1.2.3"]) + self.assertEqual(exit_code, 0) + self.assertTrue(payload["ok"]) + self.assertTrue(payload["blocking"]) + self.assertEqual(payload["epic"], "1.2") + self.assertEqual(payload["dependents"], ["1.2.4"]) + def test_get_epic_stories_state_file_accepts_non_numeric_full_keys(self) -> None: state_file = self._write_state( """ @@ -281,6 +300,24 @@ def test_get_epic_stories_epic_file_accepts_full_key_headers(self) -> None: self.assertEqual(payload["stories"], ["multi-leg-3-old", "multi-leg-4-next"]) self.assertEqual(payload["count"], 2) + def test_get_epic_stories_epic_file_accepts_bmad_numbered_headers(self) -> None: + path = self.project_root / "_bmad-output" / "implementation-artifacts" / "epic-1.md" + path.write_text( + textwrap.dedent( + """ + ## Epic 1: Platform + ### 1.3: Quantity + ### 1.4: Next + """ + ), + encoding="utf-8", + ) + exit_code, payload = self._run_action(get_epic_stories_action, ["1"]) + self.assertEqual(exit_code, 0) + self.assertTrue(payload["ok"]) + self.assertEqual(payload["stories"], ["1.3", "1.4"]) + self.assertEqual(payload["count"], 2) + def test_get_epic_stories_epic_file_ignores_other_epic_full_key_headers(self) -> None: path = self.project_root / "_bmad-output" / "implementation-artifacts" / "epic-multi-leg.md" path.write_text( @@ -358,6 +395,45 @@ def test_check_blocking_accepts_full_key_header_in_exact_epic_file(self) -> None self.assertEqual(payload["dependents"], ["multi-leg-4-next"]) self.assertEqual(payload["source"], "epic_file") + def test_check_blocking_accepts_bmad_numbered_headers(self) -> None: + path = self.project_root / "_bmad-output" / "implementation-artifacts" / "epic-1.md" + path.write_text( + textwrap.dedent( + """ + ## Epic 1: Platform + ### 1.4: Later + Dependencies: 1.3 + """ + ), + encoding="utf-8", + ) + exit_code, payload = self._run_action(check_blocking_action, ["1.3"]) + self.assertEqual(exit_code, 0) + self.assertTrue(payload["ok"]) + self.assertTrue(payload["blocking"]) + self.assertEqual(payload["dependents"], ["1.4"]) + self.assertEqual(payload["source"], "epic_file") + + def test_check_blocking_ignores_nonmatching_subheading_inside_explicit_epic(self) -> None: + path = self.project_root / "_bmad-output" / "implementation-artifacts" / "epic-1.md" + path.write_text( + textwrap.dedent( + """ + ## Epic 1: Platform + ### 1.2: Later + Details before a story-like subheading. + ### 2.0: Non-story subheading + Dependencies: 1.1 + """ + ), + encoding="utf-8", + ) + exit_code, payload = self._run_action(check_blocking_action, ["1.1"]) + self.assertEqual(exit_code, 0) + self.assertTrue(payload["ok"]) + self.assertTrue(payload["blocking"]) + self.assertEqual(payload["dependents"], ["1.2"]) + def test_check_blocking_accepts_full_key_header_in_suffixed_epic_file(self) -> None: self._write_epic_file( """ diff --git a/tests/test_runtime_helper_contracts.py b/tests/test_runtime_helper_contracts.py new file mode 100644 index 00000000..51126480 --- /dev/null +++ b/tests/test_runtime_helper_contracts.py @@ -0,0 +1,280 @@ +from __future__ import annotations + +import io +import json +import shutil +import subprocess +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from unittest.mock import patch + +from story_automator.commands.orchestrator_parse import parse_output_action +from story_automator.commands.tmux import _build_cmd, cmd_monitor_session, cmd_tmux_wrapper +from story_automator.core.tmux_runtime import _terminal_runner_status, save_session_state, session_paths +from story_automator.core.tmux_runtime import cleanup_runtime_artifacts, command_exists, session_status, tmux_kill_session +from story_automator.core.utils import COMMAND_TIMEOUT_EXIT, CommandResult + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +class RuntimeHelperContractsTests(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.project_root = Path(self.tmp.name) + self._install_bundle() + self._install_required_skills() + self.output_file = self.project_root / "session-output.txt" + self.output_file.write_text("session transcript\n", encoding="utf-8") + + def tearDown(self) -> None: + self.tmp.cleanup() + + def test_parse_output_fail_closed_subprocess_matrix(self) -> None: + cases = [ + ( + "timeout", + CommandResult("", COMMAND_TIMEOUT_EXIT, subprocess.TimeoutExpired(["claude"], 1)), + "sub-agent call timed out", + "sub_agent", + ), + ( + "nonzero", + CommandResult("boom", 42, RuntimeError("boom")), + "sub-agent call failed", + "sub_agent", + ), + ( + "no_json", + CommandResult("plain text only", 0), + "sub-agent returned invalid json", + "payload", + ), + ( + "schema_invalid", + CommandResult('{"status":"SUCCESS"}', 0), + "sub-agent returned invalid json", + "story_created", + ), + ] + for label, result, reason, field in cases: + with self.subTest(label=label): + stdout = io.StringIO() + with self._env(), patch("story_automator.commands.orchestrator_parse.run_cmd", return_value=result), redirect_stdout(stdout): + code = parse_output_action([str(self.output_file), "create"]) + self.assertEqual(code, 1) + payload = json.loads(stdout.getvalue()) + self.assertEqual(payload["status"], "error") + self.assertEqual(payload["reason"], reason) + self.assertEqual(payload["structuredIssues"][0]["field"], field) + + def test_parse_output_rejects_missing_and_empty_outputs_before_subprocess(self) -> None: + missing = self.project_root / "missing.txt" + empty = self.project_root / "empty.txt" + empty.write_text("", encoding="utf-8") + for path in (missing, empty): + with self.subTest(path=path.name): + stdout = io.StringIO() + with self._env(), patch("story_automator.commands.orchestrator_parse.run_cmd") as mock_run, redirect_stdout(stdout): + code = parse_output_action([str(path), "create"]) + self.assertEqual(code, 1) + mock_run.assert_not_called() + payload = json.loads(stdout.getvalue()) + self.assertEqual(payload["reason"], "output file not found or empty") + self.assertEqual(payload["structuredIssues"][0]["field"], "output_file") + + def test_build_cmd_covers_agent_safety_and_override_branches(self) -> None: + stdout = io.StringIO() + with self._env(extra={"AI_AGENT": "codex", "AI_COMMAND": ""}), redirect_stdout(stdout): + code = _build_cmd(["review", "1.1", "--model", "gpt-5.5"]) + rendered = stdout.getvalue() + self.assertEqual(code, 0) + self.assertIn("codex exec -s workspace-write", rendered) + self.assertIn("approval_policy=\"never\"", rendered) + self.assertIn("--disable plugins --disable sqlite --disable shell_snapshot", rendered) + self.assertIn("--model gpt-5.5", rendered) + + stdout = io.StringIO() + with self._env(extra={"AI_AGENT": "", "AI_COMMAND": "custom-ai --json"}), redirect_stdout(stdout): + code = _build_cmd(["review", "1.1"]) + self.assertEqual(code, 0) + self.assertTrue(stdout.getvalue().startswith("unset CLAUDECODE && custom-ai --json ")) + + stdout = io.StringIO() + with self._env(extra={"AI_AGENT": "", "AI_COMMAND": ""}), redirect_stdout(stdout): + code = _build_cmd(["review", "1.1", "--agent", "claude", "--model", "claude sonnet"]) + self.assertEqual(code, 0) + self.assertIn("claude --dangerously-skip-permissions --model 'claude sonnet'", stdout.getvalue()) + + def test_build_cmd_negative_contracts_fail_without_prompt_text_leakage(self) -> None: + for args, expected in ( + (["ship", "1.1"], "unknown step: ship"), + (["review", "1.1", "--state-file"], "--state-file requires a value"), + (["review", "1.1", "--state-file", str(self.project_root / "missing.md")], "state file unreadable"), + ): + with self.subTest(args=args): + stderr = io.StringIO() + with self._env(), redirect_stderr(stderr): + code = _build_cmd(args) + self.assertEqual(code, 1) + self.assertIn(expected, stderr.getvalue()) + + override_dir = self.project_root / "_bmad" / "bmm" + override_dir.mkdir(parents=True, exist_ok=True) + (override_dir / "story-automator.policy.json").write_text(json.dumps({"runtime": {"parser": {"provider": "bad"}}}), encoding="utf-8") + stderr = io.StringIO() + with self._env(), redirect_stderr(stderr): + code = _build_cmd(["review", "1.1"]) + self.assertEqual(code, 1) + self.assertIn("runtime.parser.provider", stderr.getvalue()) + + def test_monitor_session_json_terminal_state_matrix(self) -> None: + cases = [ + ( + "completed", + [ + {"todos_done": 2, "todos_total": 2, "session_state": "completed", "wait_estimate": 0, "active_task": ""}, + {"active_task": "/tmp/session.txt", "todos_done": 2, "todos_total": 2, "session_state": "completed", "wait_estimate": 0}, + ], + {"final_state": "completed", "exit_reason": "normal_completion", "output_verified": True}, + ), + ( + "crashed", + [ + {"todos_done": 1, "todos_total": 2, "session_state": "crashed", "wait_estimate": 0, "active_task": ""}, + {"active_task": "/tmp/crash.txt", "todos_done": 1, "todos_total": 2, "session_state": "crashed", "wait_estimate": 7}, + ], + {"final_state": "crashed", "exit_reason": "exit_code_7", "output_verified": False}, + ), + ( + "stuck", + [ + {"todos_done": 0, "todos_total": 0, "session_state": "stuck", "wait_estimate": 0, "active_task": ""}, + {"active_task": "/tmp/stuck.txt", "todos_done": 0, "todos_total": 0, "session_state": "stuck", "wait_estimate": 0}, + ], + {"final_state": "stuck", "exit_reason": "never_active", "output_verified": False}, + ), + ( + "not_found", + [{"todos_done": 0, "todos_total": 0, "session_state": "not_found", "wait_estimate": 0, "active_task": ""}], + {"final_state": "not_found", "exit_reason": "session_gone", "output_verified": False}, + ), + ] + for label, statuses, expected in cases: + with self.subTest(label=label): + stdout = io.StringIO() + with self._env(), patch("story_automator.commands.tmux.time.sleep"), patch( + "story_automator.commands.tmux.session_status", side_effect=statuses + ), redirect_stdout(stdout): + code = cmd_monitor_session(["sa-test", "--json", "--max-polls", "1", "--initial-wait", "0", "--workflow", "dev"]) + self.assertEqual(code, 0) + payload = json.loads(stdout.getvalue()) + for key, value in expected.items(): + self.assertEqual(payload[key], value) + + def test_runner_terminal_status_maps_edge_results(self) -> None: + cases = [ + ("spawn-error", "spawn_error", "runner_exec_failed", 127, "crashed"), + ("interrupted", "interrupted", "signal_terminated", 130, "crashed"), + ("launch-never", "unknown", "launch_never_succeeded", "", "stuck"), + ] + for suffix, result, reason, exit_code, expected_state in cases: + with self.subTest(suffix=suffix): + session = f"sa-contract-{suffix}" + paths = session_paths(session, str(self.project_root)) + paths.output.write_text("runner output\n", encoding="utf-8") + save_session_state( + paths.state, + { + "schemaVersion": 1, + "session": session, + "projectRoot": str(self.project_root), + "lifecycle": "finished", + "result": result, + "failureReason": reason, + "exitCode": exit_code, + }, + ) + status = _terminal_runner_status(session, {"result": result, "failureReason": reason, "exitCode": exit_code}, full=True, project_root=str(self.project_root)) + self.assertEqual(status["session_state"], expected_state) + self.assertTrue(str(status["active_task"]).endswith(f"output-{session}.txt")) + + def _install_bundle(self) -> None: + target_root = self.project_root / ".claude" / "skills" + target_root.mkdir(parents=True, exist_ok=True) + shutil.copytree(REPO_ROOT / "skills" / "bmad-story-automator", target_root / "bmad-story-automator") + shutil.copytree(REPO_ROOT / "skills" / "bmad-story-automator-review", target_root / "bmad-story-automator-review") + + def _install_required_skills(self) -> None: + extras = { + "bmad-create-story": {"discover-inputs.md": "# discover\n", "checklist.md": "# checklist\n", "template.md": "# template\n"}, + "bmad-dev-story": {"checklist.md": "# checklist\n"}, + "bmad-retrospective": {}, + "bmad-qa-generate-e2e-tests": {"checklist.md": "# checklist\n"}, + } + for name, files in extras.items(): + skill_dir = self.project_root / ".claude" / "skills" / name + skill_dir.mkdir(parents=True, exist_ok=True) + (skill_dir / "SKILL.md").write_text(f"# {name}\n", encoding="utf-8") + (skill_dir / "workflow.md").write_text(f"# {name}\n", encoding="utf-8") + for rel, content in files.items(): + (skill_dir / rel).write_text(content, encoding="utf-8") + + def _env(self, extra: dict[str, str] | None = None): + env = {"PROJECT_ROOT": str(self.project_root), "AI_AGENT": "", "AI_COMMAND": ""} + env.update(extra or {}) + return patch.dict("os.environ", env, clear=False) + + +@unittest.skipUnless(command_exists("tmux"), "tmux not available") +class TmuxWrapperRunnerContractsTests(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.project_root = Path(self.tmp.name) + self.sessions: list[str] = [] + + def tearDown(self) -> None: + try: + for session in self.sessions: + try: + tmux_kill_session(session, str(self.project_root)) + except Exception: + pass + cleanup_runtime_artifacts(session, str(self.project_root)) + finally: + self.tmp.cleanup() + + def test_tmux_wrapper_spawn_runner_success_and_crash(self) -> None: + cases = [ + ("success", "printf wrapper-ok", "completed", 0), + ("crash", "printf wrapper-boom && exit 7", "crashed", 7), + ] + for label, command, expected_state, expected_wait in cases: + with self.subTest(label=label): + stdout = io.StringIO() + with patch.dict("os.environ", {"PROJECT_ROOT": str(self.project_root), "SA_TMUX_RUNTIME": "runner"}, clear=False), redirect_stdout(stdout): + code = cmd_tmux_wrapper(["spawn", "dev", "1", f"1.{len(self.sessions) + 1}", "--command", command, "--agent", "codex"]) + self.assertEqual(code, 0) + session = stdout.getvalue().strip() + self.sessions.append(session) + status = self._wait_for_terminal(session) + self.assertEqual(status["session_state"], expected_state) + self.assertEqual(status["wait_estimate"], expected_wait) + + def _wait_for_terminal(self, session: str) -> dict[str, str | int]: + import time + + deadline = time.time() + 30 + last: dict[str, str | int] = {} + while time.time() < deadline: + last = session_status(session, full=False, codex=True, project_root=str(self.project_root), mode="runner") + if str(last["session_state"]) in {"completed", "crashed", "stuck"}: + return last + time.sleep(0.1) + self.fail(f"session did not reach terminal state: {last}") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_smoke_script_contracts.py b/tests/test_smoke_script_contracts.py new file mode 100644 index 00000000..1d76bae4 --- /dev/null +++ b/tests/test_smoke_script_contracts.py @@ -0,0 +1,575 @@ +from __future__ import annotations + +import importlib.util +import io +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from contextlib import redirect_stderr +from pathlib import Path +from unittest.mock import patch + + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = REPO_ROOT / "scripts" + + +def load_script_module(name: str, path: Path): + with patch.object(sys, "path", [str(SCRIPTS), *sys.path]): + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load script module: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def import_script_package(name: str): + with patch.object(sys, "path", [str(SCRIPTS), *sys.path]): + return __import__(name, fromlist=["*"]) + + +class VersionAlignmentScriptTests(unittest.TestCase): + def test_python_version_missing_raises_targeted_error(self) -> None: + module = load_script_module("check_version_alignment", SCRIPTS / "check-version-alignment.py") + + with self.assertRaisesRegex(ValueError, "missing Python __version__ assignment"): + module.python_version("# no version here\n", "pkg/__init__.py") + + def test_marketplace_plugin_version_uses_stable_plugin_name(self) -> None: + module = load_script_module("check_version_alignment", SCRIPTS / "check-version-alignment.py") + + version = module.marketplace_plugin_version( + { + "plugins": [ + {"name": "other-plugin", "version": "9.9.9"}, + {"name": "bmad-automator", "version": "1.15.0"}, + ] + }, + {"name": "bmad-automator"}, + ) + + self.assertEqual(version, "1.15.0") + + def test_marketplace_plugin_version_requires_match(self) -> None: + module = load_script_module("check_version_alignment", SCRIPTS / "check-version-alignment.py") + + with self.assertRaisesRegex(ValueError, "missing plugin: bmad-automator"): + module.marketplace_plugin_version({"plugins": []}, {"name": "bmad-automator"}) + + +class SmokeContractsScriptTests(unittest.TestCase): + def test_allowed_environment_skips_do_not_fail_default_contract_gate(self) -> None: + module = load_script_module("run_smoke_contracts", SCRIPTS / "run-smoke-contracts.py") + stderr = io.StringIO() + + class Result: + skipped = [("tmux test", "tmux not available")] + + def wasSuccessful(self) -> bool: + return True + + class Runner: + def __init__(self, *, verbosity: int) -> None: + self.verbosity = verbosity + + def run(self, suite): + return Result() + + with ( + patch.object(module.unittest.defaultTestLoader, "loadTestsFromNames", return_value=object()) as load_tests, + patch.object(module.unittest, "TextTestRunner", Runner), + redirect_stderr(stderr), + ): + code = module.main() + + self.assertEqual(code, 0) + self.assertIn("smoke:contracts skipped 1 allowed environment-dependent tests", stderr.getvalue()) + load_tests.assert_called_once_with(module.TEST_MODULES) + + def test_unexpected_skips_fail_default_contract_gate(self) -> None: + module = load_script_module("run_smoke_contracts", SCRIPTS / "run-smoke-contracts.py") + stderr = io.StringIO() + + class Result: + skipped = [("feature test", "temporarily disabled")] + + def wasSuccessful(self) -> bool: + return True + + class Runner: + def __init__(self, *, verbosity: int) -> None: + self.verbosity = verbosity + + def run(self, suite): + return Result() + + with ( + patch.object(module.unittest.defaultTestLoader, "loadTestsFromNames", return_value=object()), + patch.object(module.unittest, "TextTestRunner", Runner), + redirect_stderr(stderr), + ): + code = module.main() + + self.assertEqual(code, 1) + self.assertIn("smoke:contracts got 1 unexpected skipped tests", stderr.getvalue()) + + +class DeterministicSmokeEnvTests(unittest.TestCase): + def test_subprocess_runners_clear_marker_override_env(self) -> None: + automator = load_script_module("run_smoke_automator", SCRIPTS / "run-smoke-automator.py") + dev_loop = load_script_module("run_smoke_dev_loop", SCRIPTS / "run-smoke-dev-loop.py") + + with patch.dict( + os.environ, + { + "BMAD_STORY_AUTOMATOR_ACTIVE_MARKER": "/tmp/outside-a", + "STORY_AUTOMATOR_ACTIVE_MARKER": "/tmp/outside-b", + "BMAD_SKILLS_ROOT": "/tmp/outside-skills", + "BMAD_RUNTIME_PROVIDER": "codex", + "STORY_AUTOMATOR_RUNTIME_PROVIDER": "codex", + }, + clear=False, + ): + runner = automator.SmokeRunner( + root=REPO_ROOT, + workspace=REPO_ROOT / ".smoke", + project=REPO_ROOT / ".smoke" / "gunz", + story_id="1.1", + ) + dev = dev_loop.DevLoopSmokeRunner( + root=REPO_ROOT, + workspace=REPO_ROOT / ".smoke", + project=REPO_ROOT / ".smoke" / "gunz", + story_ids=["1.1"], + ) + + for env in (runner.env, dev.env): + self.assertNotIn("BMAD_STORY_AUTOMATOR_ACTIVE_MARKER", env) + self.assertNotIn("STORY_AUTOMATOR_ACTIVE_MARKER", env) + self.assertNotIn("BMAD_SKILLS_ROOT", env) + self.assertNotIn("BMAD_RUNTIME_PROVIDER", env) + self.assertNotIn("STORY_AUTOMATOR_RUNTIME_PROVIDER", env) + + def test_smoke_prep_env_clears_host_overrides(self) -> None: + automator = import_script_package("smoke_prep.automator") + + with tempfile.TemporaryDirectory() as tmp: + with patch.dict( + os.environ, + { + "BMAD_STORY_AUTOMATOR_ACTIVE_MARKER": "/tmp/outside-a", + "STORY_AUTOMATOR_ACTIVE_MARKER": "/tmp/outside-b", + "BMAD_SKILLS_ROOT": "/tmp/outside-skills", + "BMAD_RUNTIME_PROVIDER": "codex", + "STORY_AUTOMATOR_RUNTIME_PROVIDER": "codex", + }, + clear=False, + ): + env = automator.smoke_env(Path(tmp)) + + self.assertNotIn("BMAD_STORY_AUTOMATOR_ACTIVE_MARKER", env) + self.assertNotIn("STORY_AUTOMATOR_ACTIVE_MARKER", env) + self.assertNotIn("BMAD_SKILLS_ROOT", env) + self.assertNotIn("BMAD_RUNTIME_PROVIDER", env) + self.assertNotIn("STORY_AUTOMATOR_RUNTIME_PROVIDER", env) + self.assertTrue(env["HOME"].startswith(tmp)) + self.assertTrue(env["NPM_CONFIG_CACHE"].startswith(tmp)) + + def test_in_process_runners_clear_host_override_env_during_calls(self) -> None: + modes = load_script_module("run_smoke_modes", SCRIPTS / "run-smoke-modes.py") + finish = load_script_module("run_smoke_finish_loop", SCRIPTS / "run-smoke-finish-loop.py") + + def assert_clean_env(args): + self.assertNotIn("BMAD_STORY_AUTOMATOR_ACTIVE_MARKER", os.environ) + self.assertNotIn("STORY_AUTOMATOR_ACTIVE_MARKER", os.environ) + self.assertNotIn("BMAD_SKILLS_ROOT", os.environ) + self.assertNotIn("STORY_AUTOMATOR_RUNTIME_PROVIDER", os.environ) + return 0 + + def assert_mode_env(args): + self.assertNotIn("BMAD_RUNTIME_PROVIDER", os.environ) + return assert_clean_env(args) + + with patch.dict( + os.environ, + { + "BMAD_STORY_AUTOMATOR_ACTIVE_MARKER": "/tmp/outside-a", + "STORY_AUTOMATOR_ACTIVE_MARKER": "/tmp/outside-b", + "BMAD_SKILLS_ROOT": "/tmp/outside-skills", + "BMAD_RUNTIME_PROVIDER": "codex", + "STORY_AUTOMATOR_RUNTIME_PROVIDER": "codex", + }, + clear=False, + ): + mode_runner = modes.ModeSmokeRunner() + finish_runner = finish.FinishLoopSmokeRunner() + try: + self.assertEqual(mode_runner._call(assert_mode_env, [])[0], 0) + self.assertEqual(finish_runner._call(assert_clean_env, [])[0], 0) + finally: + mode_runner.close() + finish_runner.close() + + self.assertEqual(os.environ["BMAD_STORY_AUTOMATOR_ACTIVE_MARKER"], "/tmp/outside-a") + self.assertEqual(os.environ["STORY_AUTOMATOR_ACTIVE_MARKER"], "/tmp/outside-b") + self.assertEqual(os.environ["BMAD_SKILLS_ROOT"], "/tmp/outside-skills") + self.assertEqual(os.environ["BMAD_RUNTIME_PROVIDER"], "codex") + self.assertEqual(os.environ["STORY_AUTOMATOR_RUNTIME_PROVIDER"], "codex") + + +class SmokePrepCliTests(unittest.TestCase): + def test_smoke_input_check_malformed_payload_returns_clean_failure(self) -> None: + module = load_script_module("check_smoke_inputs", SCRIPTS / "check-smoke-inputs.py") + stderr = io.StringIO() + + with patch.object(module, "smoke_inputs", return_value={"gunz": {}}), redirect_stderr(stderr): + code = module.main() + + self.assertEqual(code, 1) + self.assertIn("smoke input determinism failed: malformed payload:", stderr.getvalue()) + self.assertNotIn("Traceback", stderr.getvalue()) + + def test_value_error_returns_clean_failure(self) -> None: + cli = import_script_package("smoke_prep.cli") + + with tempfile.TemporaryDirectory() as tmp: + stderr = io.StringIO() + with ( + patch.object(cli, "repo_root", return_value=REPO_ROOT), + patch.object(cli, "ensure_tool"), + patch.object(cli, "resolve_workspace", return_value=Path(tmp)), + patch.object(cli, "prepare_gunz"), + patch.object(cli, "smoke_env", return_value={}), + patch.object(cli, "smoke_inputs", side_effect=ValueError("bad smoke input")), + redirect_stderr(stderr), + ): + code = cli.main([]) + + self.assertEqual(code, 1) + self.assertIn("smoke prep failed: bad smoke input", stderr.getvalue()) + self.assertNotIn("Traceback", stderr.getvalue()) + + def test_skip_automator_install_report_marks_artifacts_skipped(self) -> None: + report = import_script_package("smoke_prep.report") + + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) + gunz = workspace / "gunz" + gunz.mkdir() + next_steps = report.write_next_steps(workspace, gunz, automator_installed=False) + + text = next_steps.read_text(encoding="utf-8") + self.assertIn("package identity: skipped", text) + self.assertIn("installed manifest: skipped", text) + self.assertIn("project-local automator install skipped", text) + self.assertNotIn("PACKAGE_IDENTITY.json`", text) + self.assertNotIn("INSTALLED_AUTOMATOR_MANIFEST.json`", text) + self.assertNotIn("bmad-story-automator/scripts/story-automator", text) + + def test_shared_process_run_times_out_cleanly(self) -> None: + process = import_script_package("smoke_prep.process") + + with patch.object( + process.subprocess, + "run", + side_effect=process.subprocess.TimeoutExpired(["git", "fetch"], 900), + ): + with self.assertRaisesRegex(process.SmokeError, "command timed out after 900s: git fetch"): + process.run(["git", "fetch"], cwd=REPO_ROOT) + + def test_npm_view_timeout_becomes_smoke_error(self) -> None: + inputs = import_script_package("smoke_prep.inputs") + + with patch.object( + inputs.subprocess, + "run", + side_effect=inputs.subprocess.TimeoutExpired(["npm", "view"], 60), + ): + with self.assertRaisesRegex(inputs.SmokeError, "npm view timed out"): + inputs._resolve_bmad_method({}) + + def test_npm_view_non_object_metadata_becomes_smoke_error(self) -> None: + inputs = import_script_package("smoke_prep.inputs") + + class Result: + stdout = "[]" + + with patch.object(inputs.subprocess, "run", return_value=Result()): + with self.assertRaisesRegex(inputs.SmokeError, "unexpected npm identity"): + inputs._resolve_bmad_method({}) + + def test_npm_view_malformed_metadata_becomes_smoke_error(self) -> None: + inputs = import_script_package("smoke_prep.inputs") + + class BadJson: + stdout = "{bad json" + + class BadDist: + stdout = '{"version":"1.2.3","dist":[]}' + + class BadVersion: + stdout = '{"version":["1.2.3"],"dist":{"integrity":"sha512-good"}}' + + class BadIntegrity: + stdout = '{"version":"1.2.3","dist":{"integrity":["sha512-bad"]}}' + + with patch.object(inputs.subprocess, "run", return_value=BadJson()): + with self.assertRaisesRegex(inputs.SmokeError, "unexpected npm identity"): + inputs._resolve_bmad_method({}) + with patch.object(inputs.subprocess, "run", return_value=BadDist()): + with self.assertRaisesRegex(inputs.SmokeError, "unexpected npm identity"): + inputs._resolve_bmad_method({}) + for result in (BadVersion(), BadIntegrity()): + with patch.object(inputs.subprocess, "run", return_value=result): + with self.assertRaisesRegex(inputs.SmokeError, "missing npm identity"): + inputs._resolve_bmad_method({}) + + def test_npm_pack_timeout_becomes_smoke_error(self) -> None: + package_contracts = import_script_package("smoke_prep.package_contracts") + + with patch.object( + package_contracts.subprocess, + "run", + side_effect=package_contracts.subprocess.TimeoutExpired(["npm", "pack"], 900), + ): + with self.assertRaisesRegex(package_contracts.SmokeError, "npm pack timed out after 900s"): + package_contracts._npm_pack_json(REPO_ROOT, ["--json"], {}) + + def test_npm_pack_malformed_metadata_becomes_smoke_error(self) -> None: + package_contracts = import_script_package("smoke_prep.package_contracts") + + with self.assertRaisesRegex(package_contracts.SmokeError, "missing files list"): + package_contracts._assert_content({}) + with self.assertRaisesRegex(package_contracts.SmokeError, "missing package fields"): + package_contracts._assert_pack_metadata({}, {}) + with self.assertRaisesRegex(package_contracts.SmokeError, "malformed file entry"): + package_contracts._assert_content({"files": [{}]}) + + def test_package_json_decode_failure_becomes_smoke_error(self) -> None: + package_contracts = import_script_package("smoke_prep.package_contracts") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "package.json").write_text("{bad json", encoding="utf-8") + with self.assertRaisesRegex(package_contracts.SmokeError, "invalid package.json"): + package_contracts.assert_package_contract(root, {}) + with self.assertRaisesRegex(package_contracts.SmokeError, "invalid package.json"): + package_contracts.pack_project(root, root / "dist", {}) + + def test_workspace_check_ignore_uses_path_delimiter(self) -> None: + workspace = import_script_package("smoke_prep.workspace") + + with patch.object(workspace.subprocess, "run") as run: + run.return_value.returncode = 0 + resolved = workspace.resolve_workspace(REPO_ROOT, ".smoke") + + self.assertEqual(resolved, (REPO_ROOT / ".smoke").resolve()) + self.assertEqual(run.call_args.args[0], ["git", "check-ignore", "-q", "--", ".smoke"]) + + +class SmokeModesScriptTests(unittest.TestCase): + def test_help_exits_without_running_smoke(self) -> None: + module = load_script_module("run_smoke_modes", SCRIPTS / "run-smoke-modes.py") + + with patch.object(module.ModeSmokeRunner, "run", side_effect=AssertionError("should not run")): + with self.assertRaises(SystemExit) as caught: + module.main(["--help"]) + + self.assertEqual(caught.exception.code, 0) + + def test_runner_local_subprocesses_have_timeouts(self) -> None: + automator = load_script_module("run_smoke_automator", SCRIPTS / "run-smoke-automator.py") + dev_loop = load_script_module("run_smoke_dev_loop", SCRIPTS / "run-smoke-dev-loop.py") + finish = load_script_module("run_smoke_finish_loop", SCRIPTS / "run-smoke-finish-loop.py") + calls: list[dict[str, object]] = [] + + def capture_run(*args, **kwargs): + calls.append(kwargs) + return subprocess.CompletedProcess(args[0], 0, stdout="{}", stderr="") + + with patch.object(automator.subprocess, "run", side_effect=capture_run): + runner = automator.SmokeRunner(root=REPO_ROOT, workspace=REPO_ROOT / ".smoke", project=REPO_ROOT, story_id="1.1") + runner._run("echo", "ok") + with patch.object(dev_loop.subprocess, "run", side_effect=capture_run): + runner = dev_loop.DevLoopSmokeRunner(root=REPO_ROOT, workspace=REPO_ROOT / ".smoke", project=REPO_ROOT, story_ids=["1.1"]) + runner._run("echo", "ok") + with patch.object(finish.subprocess, "run", side_effect=capture_run): + runner = finish.FinishLoopSmokeRunner() + try: + runner._run(["git", "status"], cwd=REPO_ROOT) + finally: + runner.close() + + self.assertEqual([call.get("timeout") for call in calls], [900, 900, 900]) + + def test_json_objects_parses_concatenated_marker_output(self) -> None: + module = load_script_module("run_smoke_modes", SCRIPTS / "run-smoke-modes.py") + runner = module.ModeSmokeRunner() + try: + payloads = runner._json_objects(0, '{"exists":true}\n{"storiesRemaining":2}\n') + finally: + runner.close() + + self.assertEqual(payloads[0]["exists"], True) + self.assertEqual(payloads[1]["storiesRemaining"], 2) + + def test_report_payload_persists_latest_incomplete_state(self) -> None: + module = load_script_module("run_smoke_modes", SCRIPTS / "run-smoke-modes.py") + runner = module.ModeSmokeRunner() + state_file = runner.output / "orchestration-smoke.md" + try: + state_file.parent.mkdir(parents=True) + state_file.write_text('status: "IN_PROGRESS"\n', encoding="utf-8") + + report, payload = runner.write_report( + { + "project": str(runner.project), + "resume": {"latestIncomplete": str(state_file)}, + } + ) + persisted = Path(payload["resume"]["latestIncomplete"]) + finally: + runner.close() + + self.assertEqual(payload["project"]["kind"], "ephemeral") + self.assertNotIn("path", payload["project"]) + self.assertTrue(persisted.exists()) + self.assertEqual(persisted.read_text(encoding="utf-8"), 'status: "IN_PROGRESS"\n') + self.assertEqual(json.loads(report.read_text(encoding="utf-8")), payload) + + def test_report_payload_fails_closed_when_latest_incomplete_cannot_be_persisted(self) -> None: + module = load_script_module("run_smoke_modes", SCRIPTS / "run-smoke-modes.py") + runner = module.ModeSmokeRunner() + try: + missing = runner.output / "missing-state.md" + with self.assertRaisesRegex(module.SmokeModesError, "failed to persist latest incomplete state"): + runner.write_report( + { + "project": str(runner.project), + "resume": {"latestIncomplete": str(missing)}, + } + ) + finally: + runner.close() + + +class SmokeStorySlugTests(unittest.TestCase): + def test_automator_story_slug_ignores_unfound_sprint_status_story_echo(self) -> None: + module = load_script_module("run_smoke_automator", SCRIPTS / "run-smoke-automator.py") + runner = module.SmokeRunner( + root=REPO_ROOT, + workspace=REPO_ROOT / ".smoke", + project=REPO_ROOT / ".smoke" / "gunz", + story_id="1.1", + ) + try: + with patch.object( + runner, + "_helper_json", + side_effect=[ + {"found": False, "story": "1.1", "status": "not_found"}, + {"title": "First Story"}, + ], + ): + slug = runner._story_slug() + finally: + runner.close() + + self.assertEqual(slug, "1-1-first-story") + + def test_dev_loop_story_slug_ignores_unfound_sprint_status_story_echo(self) -> None: + module = load_script_module("run_smoke_dev_loop", SCRIPTS / "run-smoke-dev-loop.py") + runner = module.DevLoopSmokeRunner( + root=REPO_ROOT, + workspace=REPO_ROOT / ".smoke", + project=REPO_ROOT / ".smoke" / "gunz", + story_ids=["1.1"], + ) + try: + with patch.object( + runner, + "_helper_json", + side_effect=[ + {"found": False, "story": "1.1", "status": "not_found"}, + {"title": "First Story"}, + ], + ): + slug = runner._story_slug("1.1") + finally: + runner.close() + + self.assertEqual(slug, "1-1-first-story") + + +class FinishLoopSmokeScriptTests(unittest.TestCase): + def test_ephemeral_descriptors_do_not_expose_cleaned_paths(self) -> None: + module = load_script_module("run_smoke_finish_loop", SCRIPTS / "run-smoke-finish-loop.py") + runner = module.FinishLoopSmokeRunner() + try: + project_descriptor = runner._ephemeral_project_descriptor() + repo_descriptor = runner._repo_descriptor(runner.project) + finally: + runner.close() + + self.assertEqual(project_descriptor["kind"], "ephemeral") + self.assertFalse(project_descriptor["retained"]) + self.assertNotIn("path", project_descriptor) + self.assertEqual(repo_descriptor["kind"], "ephemeral") + self.assertFalse(repo_descriptor["retained"]) + self.assertNotIn("path", repo_descriptor) + + def test_write_report_returns_persisted_payload_without_temp_paths(self) -> None: + if shutil.which("git") is None: + self.skipTest("git not available") + module = load_script_module("run_smoke_finish_loop", SCRIPTS / "run-smoke-finish-loop.py") + runner = module.FinishLoopSmokeRunner() + try: + runner.project.mkdir(parents=True) + runner._init_git() + state = runner.project / "orchestration-smoke.md" + state.write_text('status: "COMPLETE"\n', encoding="utf-8") + learnings = runner.output / "learnings.md" + learnings.parent.mkdir(parents=True) + learnings.write_text("## Learnings\n", encoding="utf-8") + runner.results["wrapup"] = {"learnings": str(learnings.relative_to(runner.project))} + payload = runner._write_report(state, [{"story": "1.1", "commit": "abc123"}], runner.project) + report = Path(payload["report"]) + temp_root = runner.tmp.name + finally: + runner.close() + + persisted = json.loads(report.read_text(encoding="utf-8")) + self.assertEqual(persisted, payload) + self.assertTrue(Path(payload["diagnostics"]["stateFile"]).exists()) + self.assertTrue(Path(payload["diagnostics"]["gitLog"]).exists()) + self.assertTrue(Path(payload["diagnostics"]["learnings"]).exists()) + self.assertEqual(payload["wrapup"]["learnings"], payload["diagnostics"]["learnings"]) + self.assert_no_temp_path(payload, temp_root) + + def test_json_rejects_non_object_payload(self) -> None: + module = load_script_module("run_smoke_finish_loop", SCRIPTS / "run-smoke-finish-loop.py") + runner = module.FinishLoopSmokeRunner() + try: + with self.assertRaisesRegex(module.FinishSmokeError, "helper returned non-object JSON"): + runner._json(0, "[]") + finally: + runner.close() + + def assert_no_temp_path(self, value: object, temp_root: str) -> None: + if isinstance(value, dict): + for child in value.values(): + self.assert_no_temp_path(child, temp_root) + elif isinstance(value, list): + for child in value: + self.assert_no_temp_path(child, temp_root) + elif isinstance(value, str): + self.assertNotIn(temp_root, value) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_success_verifiers.py b/tests/test_success_verifiers.py index 81918ef5..0172b99a 100644 --- a/tests/test_success_verifiers.py +++ b/tests/test_success_verifiers.py @@ -745,6 +745,13 @@ def test_epic_complete_checks_sprint_status(self) -> None: self.assertTrue(payload["verified"]) self.assertEqual(payload["doneStories"], 2) + def test_epic_complete_uses_nested_numeric_epic_identifier(self) -> None: + self._write_sprint_status("1-2-3-story-three: done\n1-2-4-story-four: done\n") + payload = epic_complete(project_root=str(self.project_root), story_key="1.2.3") + self.assertTrue(payload["verified"]) + self.assertEqual(payload["epic"], "1.2") + self.assertEqual(payload["doneStories"], 2) + def test_epic_complete_accepts_bare_epic_id(self) -> None: self._write_sprint_status("1-1-story-one: done\n1-2-story-two: done\n") payload = epic_complete(project_root=str(self.project_root), story_key="1")