From fb1e7866e14c68332401907dbdbc621740a023a0 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sat, 12 Sep 2026 04:54:11 +0200 Subject: [PATCH 1/2] chore: archive stale material and typecheck maintenance tooling --- .github/workflows/publish.yml | 2 +- .github/workflows/runtime-images.yml | 2 +- .github/workflows/test.yml | 6 +- AGENTS.md | 2 + CONTRIBUTING.md | 10 +- README.md | 5 +- .../DAIMON_RUNTIME_MIGRATION_PLAN.md | 0 DISTRIBUTION.md => archive/DISTRIBUTION.md | 0 archive/README.md | 15 + .../daimon-spawnfile-current-blockers.html | 0 .../fixture-owned-composition-boundary.html | 0 .../diagrams/spawnfile-architecture.html | 0 .../diagrams/spawnfile-boundary-audit.md | 0 .../diagrams/spawnfile-public-api.html | 0 .../diagrams/spawnfile-yaml-spec.html | 0 .../scripts}/bootstrap-worktree.mjs | 0 .../scripts}/bootstrap-worktree.test.mjs | 0 .../scripts}/build-closure.mjs | 0 .../scripts}/build-closure.test.mjs | 0 .../scripts}/loop-verify.mjs | 0 .../scripts}/loop-verify.test.mjs | 0 .../scripts}/tap-self-test.mjs | 0 .../scripts}/tap-self-test.test.mjs | 0 .../scripts}/worktree-bootstrap/AGENTS.md | 0 .../scripts}/worktree-bootstrap/CLAUDE.md | 0 .../scripts}/worktree-bootstrap/create.mjs | 0 .../scripts}/worktree-bootstrap/install.mjs | 0 .../worktree-bootstrap/install.test.mjs | 0 .../scripts}/worktree-bootstrap/receipt.mjs | 0 .../scripts}/worktree-bootstrap/repos.mjs | 0 .../worktree-bootstrap/test-helpers.mjs | 0 .../scripts}/worktree-bootstrap/verify.mjs | 0 .../worktree-bootstrap/verify.test.mjs | 0 blueprints/README.md | 19 +- package.json | 32 +- runtime-images/daimon/SourceBundle.Dockerfile | 7 +- scripts/AGENTS.md | 103 ++--- scripts/README.md | 35 ++ ...mjs => build-local-daimon-runtime.test.ts} | 57 ++- ...time.mjs => build-local-daimon-runtime.ts} | 142 +++++-- ...t.test.mjs => build-local-moltnet.test.ts} | 9 +- ...cal-moltnet.mjs => build-local-moltnet.ts} | 65 ++- scripts/compile-explicit-test-mcp.mjs | 22 - scripts/compile-explicit-test-mcp.test.mjs | 5 - scripts/compile-explicit-test-mcp.test.ts | 54 +++ scripts/compile-explicit-test-mcp.ts | 192 +++++++++ ... create-linux-amd64-dependency-closure.ts} | 90 ++-- ...e.mjs => create-linux-amd64-go-closure.ts} | 11 +- ...mjs => create-source-provenance-bundle.ts} | 7 +- ...net-source-provenance.integration.test.ts} | 0 scripts/native-helper-artifacts.mjs | 16 - ...st.mjs => native-helper-artifacts.test.ts} | 2 +- scripts/native-helper-artifacts.ts | 71 ++++ ....mjs => native-helper-integration.test.ts} | 16 +- ...st.mjs => native-helper-workflows.test.ts} | 8 +- ... product-state-volume-integration.test.ts} | 2 +- ...rce-provenance-bundle.integration.test.ts} | 18 +- ...bundle.mjs => source-provenance-bundle.ts} | 163 +++++-- scripts/tsconfig.json | 17 + scripts/typescript-policy.test.ts | 26 ++ scripts/verify-package-closure.mjs | 285 ------------- scripts/verify-package-closure.ts | 398 ++++++++++++++++++ specs/research/WORKSPACE-PACKAGING.md | 2 +- src/compiler/containerArtifacts.test.ts | 2 +- src/compiler/containerArtifacts.ts | 2 +- src/compiler/moltnetReleaseAuthority.ts | 2 +- src/deployment/native/AGENTS.md | 2 +- src/deployment/native/{build.mjs => build.ts} | 2 +- .../{copyArtifacts.mjs => copyArtifacts.ts} | 2 +- src/distribution/AGENTS.md | 2 +- src/e2e/localMoltnetRelease.ts | 2 +- src/evidenceExportHelper/AGENTS.md | 2 +- .../{copyAssets.mjs => copyAssets.ts} | 0 ...affoldAssets.mjs => copyScaffoldAssets.ts} | 11 +- tsconfig.build.json | 6 +- tsconfig.json | 6 + 76 files changed, 1353 insertions(+), 604 deletions(-) rename DAIMON_RUNTIME_MIGRATION_PLAN.md => archive/DAIMON_RUNTIME_MIGRATION_PLAN.md (100%) rename DISTRIBUTION.md => archive/DISTRIBUTION.md (100%) create mode 100644 archive/README.md rename {docs => archive}/diagrams/daimon-spawnfile-current-blockers.html (100%) rename {docs => archive}/diagrams/fixture-owned-composition-boundary.html (100%) rename {docs => archive}/diagrams/spawnfile-architecture.html (100%) rename {docs => archive}/diagrams/spawnfile-boundary-audit.md (100%) rename {docs => archive}/diagrams/spawnfile-public-api.html (100%) rename {docs => archive}/diagrams/spawnfile-yaml-spec.html (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/bootstrap-worktree.mjs (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/bootstrap-worktree.test.mjs (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/build-closure.mjs (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/build-closure.test.mjs (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/loop-verify.mjs (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/loop-verify.test.mjs (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/tap-self-test.mjs (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/tap-self-test.test.mjs (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/worktree-bootstrap/AGENTS.md (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/worktree-bootstrap/CLAUDE.md (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/worktree-bootstrap/create.mjs (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/worktree-bootstrap/install.mjs (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/worktree-bootstrap/install.test.mjs (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/worktree-bootstrap/receipt.mjs (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/worktree-bootstrap/repos.mjs (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/worktree-bootstrap/test-helpers.mjs (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/worktree-bootstrap/verify.mjs (100%) rename {scripts => archive/legacy-worktree-tools/scripts}/worktree-bootstrap/verify.test.mjs (100%) create mode 100644 scripts/README.md rename scripts/{build-local-daimon-runtime.test.mjs => build-local-daimon-runtime.test.ts} (89%) rename scripts/{build-local-daimon-runtime.mjs => build-local-daimon-runtime.ts} (75%) rename scripts/{build-local-moltnet.test.mjs => build-local-moltnet.test.ts} (87%) rename scripts/{build-local-moltnet.mjs => build-local-moltnet.ts} (81%) delete mode 100644 scripts/compile-explicit-test-mcp.mjs delete mode 100644 scripts/compile-explicit-test-mcp.test.mjs create mode 100644 scripts/compile-explicit-test-mcp.test.ts create mode 100644 scripts/compile-explicit-test-mcp.ts rename scripts/{create-linux-amd64-dependency-closure.mjs => create-linux-amd64-dependency-closure.ts} (52%) rename scripts/{create-linux-amd64-go-closure.mjs => create-linux-amd64-go-closure.ts} (81%) rename scripts/{create-source-provenance-bundle.mjs => create-source-provenance-bundle.ts} (71%) rename scripts/{moltnet-source-provenance.integration.test.mjs => moltnet-source-provenance.integration.test.ts} (100%) delete mode 100644 scripts/native-helper-artifacts.mjs rename scripts/{native-helper-artifacts.test.mjs => native-helper-artifacts.test.ts} (98%) create mode 100644 scripts/native-helper-artifacts.ts rename scripts/{native-helper-integration.test.mjs => native-helper-integration.test.ts} (74%) rename scripts/{native-helper-workflows.test.mjs => native-helper-workflows.test.ts} (77%) rename scripts/{product-state-volume-integration.test.mjs => product-state-volume-integration.test.ts} (98%) rename scripts/{source-provenance-bundle.integration.test.mjs => source-provenance-bundle.integration.test.ts} (93%) rename scripts/{source-provenance-bundle.mjs => source-provenance-bundle.ts} (67%) create mode 100644 scripts/tsconfig.json create mode 100644 scripts/typescript-policy.test.ts delete mode 100644 scripts/verify-package-closure.mjs create mode 100644 scripts/verify-package-closure.ts rename src/deployment/native/{build.mjs => build.ts} (98%) rename src/deployment/native/{copyArtifacts.mjs => copyArtifacts.ts} (95%) rename src/evidenceExportHelper/{copyAssets.mjs => copyAssets.ts} (100%) rename src/runtime/{copyScaffoldAssets.mjs => copyScaffoldAssets.ts} (77%) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3f767909..e00e054a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -93,7 +93,7 @@ jobs: run: rm -rf -- moltnet-fixture - name: Verify native helper syscalls - run: node --test scripts/native-helper-artifacts.test.mjs scripts/native-helper-integration.test.mjs + run: node --experimental-strip-types --test scripts/native-helper-artifacts.test.ts scripts/native-helper-integration.test.ts - name: Verify package contents run: npm run verify:package-closure diff --git a/.github/workflows/runtime-images.yml b/.github/workflows/runtime-images.yml index 89d68ade..89cd9e4f 100644 --- a/.github/workflows/runtime-images.yml +++ b/.github/workflows/runtime-images.yml @@ -70,7 +70,7 @@ jobs: # args the Dockerfile does not declare — so it had never produced an # image, only a red check. # - # `npm run build:local-daimon` (scripts/build-local-daimon-runtime.mjs) + # `npm run build:local-daimon` (scripts/build-local-daimon-runtime.ts) # assembles all of that and is the supported path today. Publishing # from CI means teaching this workflow to call that script with a # non-loopback registry target, which `resolveLocalImageTag` currently diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aa71e83e..559e0398 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,10 +45,10 @@ jobs: run: npm run build - name: Verify native helper syscalls - run: node --test scripts/native-helper-artifacts.test.mjs scripts/native-helper-integration.test.mjs + run: node --experimental-strip-types --test scripts/native-helper-artifacts.test.ts scripts/native-helper-integration.test.ts - - name: Verify Daimon image layering - run: node --test scripts/build-local-daimon-runtime.test.mjs + - name: Verify maintained TypeScript scripts + run: npm run test:scripts # Preseed reads the candidate volume through its Docker Mountpoint and # relies on host rename/fsync/hardlink semantics a container cannot give diff --git a/AGENTS.md b/AGENTS.md index 1f2a33f5..f999b7ce 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,6 +10,7 @@ This repository is the reference implementation of the Spawnfile v0.1 compiler. ├── runtimes.yaml # Runtime registry — pinned versions and status ├── moltnet-releases.json # Pinned, checksum-verified Moltnet release authority ├── blueprints/ # Frozen reference layouts per runtime at pinned version +├── archive/ # Historical plans, diagrams, and retired tooling ├── specs/ # Specs, architecture docs, and runtime research ├── scripts/ # Bootstrap and repo helper scripts ├── src/ # CLI, compiler, adapters, manifests, reports, target contracts @@ -33,6 +34,7 @@ This repository is the reference implementation of the Spawnfile v0.1 compiler. - Prefer small, composable modules with explicit responsibilities. - Do not allow source files to grow past 400 lines. Split early when files start creeping up. - Use named exports only. Do not introduce default exports. +- Write maintained repository scripts in TypeScript and include them in strict typechecking. Keep historical plans and audit diagrams in `archive/`, not at the root or in active documentation. - Use barrel exports for folder entry points. - Keep test files next to the files they cover: `file.ts` and `file.test.ts`. - Aim for 90% or better coverage. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5532bb56..eac35b36 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ ## Prerequisites -- Node 22+ +- Node 22.19+ (including native TypeScript script execution) - Docker if you want to run the `spawnfile build` end-to-end test against a real image ## Local Workflow @@ -28,7 +28,7 @@ npm run typecheck Run the CLI on a fixture without a global link: ```bash -npm run dev -- validate test/fixtures/single-agent +npm run dev -- validate fixtures/single-agent ``` ## Website @@ -41,13 +41,13 @@ npm run build ## Testing -- `npm test` — unit tests via Vitest. +- `npm test` — compiler tests via Vitest, runtime node:test cases, and maintained script unit tests. - `npm run coverage` — unit tests with coverage output. - `npm run test:e2e:docker-auth` — end-to-end against a real compiled container. Needs Docker. ## Adding a runtime -Runtime adapters live in `src/adapters/`. Each adapter lowers the resolved graph into runtime-native output and reports per-capability support. +Runtime adapters live in `src/runtime/`. Each adapter lowers the resolved graph into runtime-native output and reports per-capability support. - Start from [`specs/SPEC.md`](specs/SPEC.md) to understand the source shape. - Read [`specs/COMPILER.md`](specs/COMPILER.md) for the adapter contract. @@ -60,6 +60,8 @@ When adding a runtime, also update `runtimes.yaml` with the pinned version and s - Keep `README.md` focused on getting started. - Put detailed specification material in `specs/`. +- Keep historical plans, reviews, and audit diagrams in `archive/`. +- Write repository scripts in TypeScript. Their purpose, callers, and checks belong in [`scripts/README.md`](scripts/README.md). - Keep package `AGENTS.md` guides in present tense and aligned with current code, with `CLAUDE.md` symlinks for compatibility. ## Commit style diff --git a/README.md b/README.md index 26a83090..ad1fc7cb 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,7 @@ v0.1 targets autonomous agent runtimes that share a markdown workspace identity |-----------|---------------|---------|-----------------------------------------------| | OpenClaw | active | ✅ | Discord, Telegram, WhatsApp, Slack | | PicoClaw | active | | Discord, Telegram, Slack (WhatsApp blocked) | +| Daimon | active | | Subscription CLI engines, Moltnet | | Pi | active | | Embedded org app, Moltnet client config | | OpenFang | exploratory | | No active adapter yet | | Hermes Agent | exploratory | | No active adapter yet | @@ -217,7 +218,9 @@ The source-of-truth specs live in this repo: - [`specs/STATUS.md`](specs/STATUS.md) — static and live operational status - [`specs/DISTRIBUTION.md`](specs/DISTRIBUTION.md) — image distribution, publish, and sourceless run - [`specs/research/WORKSPACE-PACKAGING.md`](specs/research/WORKSPACE-PACKAGING.md) — planned source-directory and toolset authoring; not implemented -- [`test/fixtures/`](test/fixtures/) — canonical example projects +- [`fixtures/`](fixtures/) — test-only projects; [`examples/`](examples/) — user-facing examples +- [`scripts/README.md`](scripts/README.md) — maintained tooling and its callers +- [`archive/`](archive/) — historical plans, diagrams, and retired tooling ## From source diff --git a/DAIMON_RUNTIME_MIGRATION_PLAN.md b/archive/DAIMON_RUNTIME_MIGRATION_PLAN.md similarity index 100% rename from DAIMON_RUNTIME_MIGRATION_PLAN.md rename to archive/DAIMON_RUNTIME_MIGRATION_PLAN.md diff --git a/DISTRIBUTION.md b/archive/DISTRIBUTION.md similarity index 100% rename from DISTRIBUTION.md rename to archive/DISTRIBUTION.md diff --git a/archive/README.md b/archive/README.md new file mode 100644 index 00000000..62de0045 --- /dev/null +++ b/archive/README.md @@ -0,0 +1,15 @@ +# Archive + +Historical material retained for context, not current instructions or executable +tooling. File contents describe the repository at the time they were written. + +| Material | Why archived | Current reference | +| --- | --- | --- | +| [Daimon migration plan](DAIMON_RUNTIME_MIGRATION_PLAN.md) | Plan for replacing the former Pi alias; predates the standalone adapter | [Daimon adapter](../src/runtime/daimon/AGENTS.md), [runtime contracts](../specs/RUNTIMES.md) | +| [Distribution design](DISTRIBUTION.md) | Original phased design, superseded as implementation authority | [Distribution specification](../specs/DISTRIBUTION.md) | +| [Diagrams and boundary audit](diagrams/) | Point-in-time illustrations and migration blockers; not maintained against current code | [Compiler specification](../specs/COMPILER.md), [ecosystem boundaries](../specs/ECOSYSTEM_RUNTIME_BOUNDARIES.md) | +| [Legacy worktree tools](legacy-worktree-tools/) | Assume sibling projects live inside `ecosystem/`; that layout no longer exists here. No package command or CI job uses them | [Contributing](../CONTRIBUTING.md), [maintained scripts](../scripts/README.md) | + +Archived JavaScript is preserved as historical source. It is excluded from the +maintained TypeScript scripts, package contents, and automated test discovery. +Do not run archived tools or copy their configuration into a current deployment. diff --git a/docs/diagrams/daimon-spawnfile-current-blockers.html b/archive/diagrams/daimon-spawnfile-current-blockers.html similarity index 100% rename from docs/diagrams/daimon-spawnfile-current-blockers.html rename to archive/diagrams/daimon-spawnfile-current-blockers.html diff --git a/docs/diagrams/fixture-owned-composition-boundary.html b/archive/diagrams/fixture-owned-composition-boundary.html similarity index 100% rename from docs/diagrams/fixture-owned-composition-boundary.html rename to archive/diagrams/fixture-owned-composition-boundary.html diff --git a/docs/diagrams/spawnfile-architecture.html b/archive/diagrams/spawnfile-architecture.html similarity index 100% rename from docs/diagrams/spawnfile-architecture.html rename to archive/diagrams/spawnfile-architecture.html diff --git a/docs/diagrams/spawnfile-boundary-audit.md b/archive/diagrams/spawnfile-boundary-audit.md similarity index 100% rename from docs/diagrams/spawnfile-boundary-audit.md rename to archive/diagrams/spawnfile-boundary-audit.md diff --git a/docs/diagrams/spawnfile-public-api.html b/archive/diagrams/spawnfile-public-api.html similarity index 100% rename from docs/diagrams/spawnfile-public-api.html rename to archive/diagrams/spawnfile-public-api.html diff --git a/docs/diagrams/spawnfile-yaml-spec.html b/archive/diagrams/spawnfile-yaml-spec.html similarity index 100% rename from docs/diagrams/spawnfile-yaml-spec.html rename to archive/diagrams/spawnfile-yaml-spec.html diff --git a/scripts/bootstrap-worktree.mjs b/archive/legacy-worktree-tools/scripts/bootstrap-worktree.mjs similarity index 100% rename from scripts/bootstrap-worktree.mjs rename to archive/legacy-worktree-tools/scripts/bootstrap-worktree.mjs diff --git a/scripts/bootstrap-worktree.test.mjs b/archive/legacy-worktree-tools/scripts/bootstrap-worktree.test.mjs similarity index 100% rename from scripts/bootstrap-worktree.test.mjs rename to archive/legacy-worktree-tools/scripts/bootstrap-worktree.test.mjs diff --git a/scripts/build-closure.mjs b/archive/legacy-worktree-tools/scripts/build-closure.mjs similarity index 100% rename from scripts/build-closure.mjs rename to archive/legacy-worktree-tools/scripts/build-closure.mjs diff --git a/scripts/build-closure.test.mjs b/archive/legacy-worktree-tools/scripts/build-closure.test.mjs similarity index 100% rename from scripts/build-closure.test.mjs rename to archive/legacy-worktree-tools/scripts/build-closure.test.mjs diff --git a/scripts/loop-verify.mjs b/archive/legacy-worktree-tools/scripts/loop-verify.mjs similarity index 100% rename from scripts/loop-verify.mjs rename to archive/legacy-worktree-tools/scripts/loop-verify.mjs diff --git a/scripts/loop-verify.test.mjs b/archive/legacy-worktree-tools/scripts/loop-verify.test.mjs similarity index 100% rename from scripts/loop-verify.test.mjs rename to archive/legacy-worktree-tools/scripts/loop-verify.test.mjs diff --git a/scripts/tap-self-test.mjs b/archive/legacy-worktree-tools/scripts/tap-self-test.mjs similarity index 100% rename from scripts/tap-self-test.mjs rename to archive/legacy-worktree-tools/scripts/tap-self-test.mjs diff --git a/scripts/tap-self-test.test.mjs b/archive/legacy-worktree-tools/scripts/tap-self-test.test.mjs similarity index 100% rename from scripts/tap-self-test.test.mjs rename to archive/legacy-worktree-tools/scripts/tap-self-test.test.mjs diff --git a/scripts/worktree-bootstrap/AGENTS.md b/archive/legacy-worktree-tools/scripts/worktree-bootstrap/AGENTS.md similarity index 100% rename from scripts/worktree-bootstrap/AGENTS.md rename to archive/legacy-worktree-tools/scripts/worktree-bootstrap/AGENTS.md diff --git a/scripts/worktree-bootstrap/CLAUDE.md b/archive/legacy-worktree-tools/scripts/worktree-bootstrap/CLAUDE.md similarity index 100% rename from scripts/worktree-bootstrap/CLAUDE.md rename to archive/legacy-worktree-tools/scripts/worktree-bootstrap/CLAUDE.md diff --git a/scripts/worktree-bootstrap/create.mjs b/archive/legacy-worktree-tools/scripts/worktree-bootstrap/create.mjs similarity index 100% rename from scripts/worktree-bootstrap/create.mjs rename to archive/legacy-worktree-tools/scripts/worktree-bootstrap/create.mjs diff --git a/scripts/worktree-bootstrap/install.mjs b/archive/legacy-worktree-tools/scripts/worktree-bootstrap/install.mjs similarity index 100% rename from scripts/worktree-bootstrap/install.mjs rename to archive/legacy-worktree-tools/scripts/worktree-bootstrap/install.mjs diff --git a/scripts/worktree-bootstrap/install.test.mjs b/archive/legacy-worktree-tools/scripts/worktree-bootstrap/install.test.mjs similarity index 100% rename from scripts/worktree-bootstrap/install.test.mjs rename to archive/legacy-worktree-tools/scripts/worktree-bootstrap/install.test.mjs diff --git a/scripts/worktree-bootstrap/receipt.mjs b/archive/legacy-worktree-tools/scripts/worktree-bootstrap/receipt.mjs similarity index 100% rename from scripts/worktree-bootstrap/receipt.mjs rename to archive/legacy-worktree-tools/scripts/worktree-bootstrap/receipt.mjs diff --git a/scripts/worktree-bootstrap/repos.mjs b/archive/legacy-worktree-tools/scripts/worktree-bootstrap/repos.mjs similarity index 100% rename from scripts/worktree-bootstrap/repos.mjs rename to archive/legacy-worktree-tools/scripts/worktree-bootstrap/repos.mjs diff --git a/scripts/worktree-bootstrap/test-helpers.mjs b/archive/legacy-worktree-tools/scripts/worktree-bootstrap/test-helpers.mjs similarity index 100% rename from scripts/worktree-bootstrap/test-helpers.mjs rename to archive/legacy-worktree-tools/scripts/worktree-bootstrap/test-helpers.mjs diff --git a/scripts/worktree-bootstrap/verify.mjs b/archive/legacy-worktree-tools/scripts/worktree-bootstrap/verify.mjs similarity index 100% rename from scripts/worktree-bootstrap/verify.mjs rename to archive/legacy-worktree-tools/scripts/worktree-bootstrap/verify.mjs diff --git a/scripts/worktree-bootstrap/verify.test.mjs b/archive/legacy-worktree-tools/scripts/worktree-bootstrap/verify.test.mjs similarity index 100% rename from scripts/worktree-bootstrap/verify.test.mjs rename to archive/legacy-worktree-tools/scripts/worktree-bootstrap/verify.test.mjs diff --git a/blueprints/README.md b/blueprints/README.md index 83f0d359..610b7d12 100644 --- a/blueprints/README.md +++ b/blueprints/README.md @@ -1,6 +1,8 @@ # Blueprints -Frozen reference layouts for each runtime at the version pinned in `runtimes.yaml`. +Frozen upstream reference layouts for OpenClaw and PicoClaw at the versions +pinned in [`runtimes.yaml`](../runtimes.yaml). These files document upstream +defaults; the compiler does not read this directory during a build. Each blueprint shows exactly what a runtime expects when you set up a bot — config files, workspace structure, doc locations, skill directories. @@ -18,13 +20,26 @@ same review. | OpenClaw | npm | `openclaw.json` | AGENTS, BOOTSTRAP, HEARTBEAT, IDENTITY, SOUL, TOOLS, USER | `openclaw onboard` | | PicoClaw | Go | `config.json` | AGENTS, SOUL, USER, IDENTITY, HEARTBEAT, memory/MEMORY | `config.example.json` | +## Compiler-generated runtimes + +Daimon has no upstream onboarding scaffold to freeze here. Spawnfile's +[Daimon adapter](../src/runtime/daimon/AGENTS.md) generates +`daimon-organization-runtime.json` and each agent's workspace from the resolved +organization. Its [configuration tests](../src/runtime/daimon/config.test.ts) +and [adapter tests](../src/runtime/daimon/adapter.test.ts) verify the versioned +runtime contract, workspace documents, and launch artifacts. A separate +hand-maintained blueprint is not a compile input or a missing runtime dependency. + +The legacy Pi adapter likewise generates its application and configuration; +see its [working guide](../src/runtime/pi/AGENTS.md). + ## Incompatible Runtimes These were evaluated but are fundamentally incompatible with Spawnfile's config + markdown workspace model: - **IronClaw** — env-vars-only orchestrator/worker system, no agent config files - **NanoClaw** — code-driven via Claude Code skills, no declarative config surface -See `specs/research/RUNTIME-NOTES.md` for the full research on each. +See [runtime research](../specs/research/RUNTIME-NOTES.md) for the full research on each. ## Adapter Candidates diff --git a/package.json b/package.json index e9af15b0..ea2122f4 100644 --- a/package.json +++ b/package.json @@ -17,9 +17,9 @@ "node": ">=22.19.0" }, "scripts": { - "compile:explicit-test-mcp": "node scripts/compile-explicit-test-mcp.mjs", - "build": "rm -rf dist && tsc --project tsconfig.build.json && chmod +x dist/cli/index.js && node ./src/evidenceExportHelper/copyAssets.mjs && node ./src/runtime/copyScaffoldAssets.mjs && node ./src/deployment/native/copyArtifacts.mjs", - "build:native": "node ./src/deployment/native/build.mjs", + "compile:explicit-test-mcp": "node --experimental-strip-types scripts/compile-explicit-test-mcp.ts", + "build": "rm -rf dist && tsc --project tsconfig.build.json && chmod +x dist/cli/index.js && node --experimental-strip-types ./src/evidenceExportHelper/copyAssets.ts && node --experimental-strip-types ./src/runtime/copyScaffoldAssets.ts && node --experimental-strip-types ./src/deployment/native/copyArtifacts.ts", + "build:native": "node --experimental-strip-types ./src/deployment/native/build.ts", "clean": "rm -rf coverage dist", "coverage": "vitest run --coverage", "dev": "tsx src/cli/index.ts", @@ -33,14 +33,14 @@ "runtime:picoclaw-image": "docker build -f runtime-images/picoclaw/Dockerfile -t noopolis/spawnfile-runtime-picoclaw:0.3.1-local runtime-images/picoclaw", "runtime:daimon-image": "npm run build:local-daimon", "audit:generate": "tsx src/audit/auditCli.ts", - "build:local-moltnet": "node ./scripts/build-local-moltnet.mjs", - "build:local-daimon": "node ./scripts/build-local-daimon-runtime.mjs", - "bundle:source-provenance": "node ./scripts/create-source-provenance-bundle.mjs", - "prepare:linux-amd64-closure": "node ./scripts/create-linux-amd64-dependency-closure.mjs", - "prepare:linux-amd64-go-closure": "node ./scripts/create-linux-amd64-go-closure.mjs", - "test:source-provenance-docker": "node --test scripts/source-provenance-bundle.integration.test.mjs", - "test:moltnet-source-provenance-docker": "node --test scripts/moltnet-source-provenance.integration.test.mjs", - "verify:package-closure": "node ./scripts/verify-package-closure.mjs", + "build:local-moltnet": "node --experimental-strip-types ./scripts/build-local-moltnet.ts", + "build:local-daimon": "node --experimental-strip-types ./scripts/build-local-daimon-runtime.ts", + "bundle:source-provenance": "node --experimental-strip-types ./scripts/create-source-provenance-bundle.ts", + "prepare:linux-amd64-closure": "node --experimental-strip-types ./scripts/create-linux-amd64-dependency-closure.ts", + "prepare:linux-amd64-go-closure": "node --experimental-strip-types ./scripts/create-linux-amd64-go-closure.ts", + "test:source-provenance-docker": "node --experimental-strip-types --test scripts/source-provenance-bundle.integration.test.ts", + "test:moltnet-source-provenance-docker": "node --experimental-strip-types --test scripts/moltnet-source-provenance.integration.test.ts", + "verify:package-closure": "node --experimental-strip-types ./scripts/verify-package-closure.ts", "test:e2e:docker-auth": "tsx src/e2e/cli.ts", "test:e2e:daimon-memory-recall": "tsx src/e2e/cli.ts daimon-memory-recall", "test:e2e:jungian-self-org": "tsx src/e2e/cli.ts jungian-self-org", @@ -51,19 +51,21 @@ "test:e2e:ollama-embeddings-probe": "tsx src/e2e/cli.ts ollama-embeddings-probe", "test:e2e:operational-smoke": "tsx src/e2e/cli.ts operational-smoke", "test:e2e:lifecycle-smoke": "tsx src/e2e/cli.ts lifecycle-smoke", - "test": "npm run test:vitest; vitest_status=$?; npm run test:verdict; verdict_status=$?; npm run test:coverage-verdict; coverage_status=$?; npm run test:node; node_status=$?; failed_lanes=''; if [ \"$verdict_status\" -eq 2 ]; then failed_lanes='vitest test failures'; fi; if [ \"$node_status\" -eq 2 ]; then if [ -n \"$failed_lanes\" ]; then failed_lanes=\"$failed_lanes; node:test suite\"; else failed_lanes='node:test suite'; fi; fi; result_status=0; if [ -n \"$failed_lanes\" ]; then echo \"FAIL(tests): $failed_lanes\"; result_status=1; fi; if [ \"$verdict_status\" -ne 0 ] && [ \"$verdict_status\" -ne 2 ]; then echo 'FAIL(vitest-verdict): could not determine the vitest result'; result_status=1; fi; if [ \"$node_status\" -ne 0 ] && [ \"$node_status\" -ne 2 ]; then echo 'FAIL(node-lane): could not determine the node:test result'; result_status=1; fi; if [ \"$result_status\" -ne 0 ]; then exit 1; elif [ \"$vitest_status\" -ne 0 ]; then echo \"FAIL(vitest): exited $vitest_status with no test failure\"; exit 1; elif [ \"$coverage_status\" -eq 2 ]; then echo 'FAIL(coverage): thresholds not met (tests all passed)'; exit 2; elif [ \"$coverage_status\" -ne 0 ]; then echo 'FAIL(coverage-verdict): could not evaluate coverage (tests all passed)'; exit 1; else echo 'PASS'; fi", "test:vitest": "rm -f coverage/vitest-results.json coverage/coverage-summary.json && vitest run --coverage.thresholds.branches=0 --coverage.thresholds.functions=0 --coverage.thresholds.lines=0 --coverage.thresholds.statements=0", "test:verdict": "node -e \"const fs=require('fs'); const file='coverage/vitest-results.json'; if (!fs.existsSync(file)) { console.error('VERDICT: results file missing'); process.exit(1); } let result; try { result=JSON.parse(fs.readFileSync(file, 'utf8')); } catch { console.error('VERDICT: results file unparseable'); process.exit(1); } const total=result.numTotalTests; const totalSuites=result.numTotalTestSuites; const passedTests=result.numPassedTests; const failedTests=result.numFailedTests; const pendingTests=result.numPendingTests; const todoTests=result.numTodoTests; const failedSuites=result.numFailedTestSuites; if (typeof result.success !== 'boolean' || !Number.isInteger(passedTests) || passedTests < 0 || !Number.isInteger(failedTests) || failedTests < 0 || !Number.isInteger(pendingTests) || pendingTests < 0 || !Number.isInteger(todoTests) || todoTests < 0 || !Number.isInteger(failedSuites) || failedSuites < 0 || !Number.isInteger(totalSuites) || totalSuites < 0) { console.error('VERDICT: results schema unrecognised'); process.exit(1); } if (!Number.isInteger(total) || total <= 0) { console.error('VERDICT: no tests found'); process.exit(1); } if (passedTests + failedTests + pendingTests + todoTests !== total) { console.error('VERDICT: results counters do not reconcile (tests='+total+', passed='+passedTests+', failed='+failedTests+', skipped='+pendingTests+', todo='+todoTests+')'); process.exit(1); } const failed=result.success === false || failedTests > 0 || failedSuites > 0; if (failed) { console.error('VERDICT: test failures found (tests='+failedTests+', suites='+failedSuites+', success='+result.success+')'); process.exit(2); } if (passedTests === 0) { console.error('VERDICT: no tests actually ran (skipped='+pendingTests+', todo='+todoTests+')'); process.exit(1); } console.log('VERDICT: tests passed (tests='+total+', suites='+totalSuites+', passed='+passedTests+', skipped='+pendingTests+', todo='+todoTests+')');\"", "test:coverage-verdict": "node --import tsx -e \"import fs from 'node:fs'; const file='coverage/coverage-summary.json'; const fail=(message)=>{ console.error('COVERAGE VERDICT: '+message); process.exit(1); }; let summary; try { summary=JSON.parse(fs.readFileSync(file, 'utf8')); } catch (error) { fail(error && error.code === 'ENOENT' ? 'summary file missing' : 'summary file unparseable'); } import('./vitest.config.ts').then(({default:config})=>{ const thresholds=config.test && config.test.coverage && config.test.coverage.thresholds; if (!thresholds || typeof thresholds !== 'object' || !summary || typeof summary !== 'object' || !summary.total || typeof summary.total !== 'object') fail('summary or threshold schema unrecognised'); const metricNames=['branches','functions','lines','statements']; const metrics=metricNames.filter((metric)=>Object.prototype.hasOwnProperty.call(thresholds, metric)); if (metrics.length === 0) fail('no threshold metrics configured'); let shortfall=false; for (const metric of metrics) { const threshold=thresholds[metric]; const pct=summary.total[metric] && summary.total[metric].pct; if (typeof threshold !== 'number' || !Number.isFinite(threshold) || typeof pct !== 'number' || !Number.isFinite(pct)) fail('missing or non-numeric value for '+metric); console.log('COVERAGE: '+metric+' '+pct+'% (threshold '+threshold+'%)'); if (pct < threshold) shortfall=true; } if (shortfall) { console.error('COVERAGE VERDICT: thresholds not met'); process.exit(2); } console.log('COVERAGE VERDICT: thresholds met'); }).catch((error)=>fail('could not load vitest config: '+error.message));\"", "test:node": "mkdir -p coverage || exit 1; rm -f coverage/node-test.tap || exit 1; node --import tsx --test --test-reporter=tap src/runtime/pi/appControlDeliveryMetadata.test.ts > coverage/node-test.tap 2>&1; node_status=$?; cat coverage/node-test.tap; cat_status=$?; npm run test:node-verdict; verdict_status=$?; if [ \"$verdict_status\" -eq 2 ]; then exit 2; fi; if [ \"$verdict_status\" -ne 0 ]; then exit 1; fi; if [ \"$cat_status\" -ne 0 ]; then echo 'NODE VERDICT: could not read TAP output' >&2; exit 1; fi; if [ \"$node_status\" -ne 0 ]; then echo \"NODE VERDICT: node:test runner exited $node_status despite a passing TAP summary\" >&2; exit 1; fi; exit 0", "test:node-verdict": "node -e \"const fs=require('fs'); const text=fs.readFileSync('coverage/node-test.tap', 'utf8'); const tests=text.match(/^# tests ([0-9]+)\\r?$/m); const suites=text.match(/^# suites ([0-9]+)\\r?$/m); const passed=text.match(/^# pass ([0-9]+)\\r?$/m); const failed=text.match(/^# fail ([0-9]+)\\r?$/m); const skipped=text.match(/^# skipped ([0-9]+)\\r?$/m); const todo=text.match(/^# todo ([0-9]+)\\r?$/m); const cancelled=text.match(/^# cancelled ([0-9]+)\\r?$/m); if (!tests || !suites || !passed || !failed || !skipped || !todo || !cancelled) { console.error('NODE VERDICT: TAP summary unrecognised'); process.exit(1); } const testCount=Number(tests[1]); const suiteCount=Number(suites[1]); const passCount=Number(passed[1]); const failCount=Number(failed[1]); const skippedCount=Number(skipped[1]); const todoCount=Number(todo[1]); const cancelledCount=Number(cancelled[1]); if (failCount > 0) { console.error('NODE VERDICT: test failures found (tests='+testCount+', failed='+failCount+')'); process.exit(2); } if (cancelledCount > 0) { console.error('NODE VERDICT: incomplete run (cancelled='+cancelledCount+')'); process.exit(1); } if (passCount + skippedCount + todoCount + cancelledCount !== testCount) { console.error('NODE VERDICT: TAP counts do not reconcile (tests='+testCount+', passed='+passCount+', skipped='+skippedCount+', todo='+todoCount+', cancelled='+cancelledCount+')'); process.exit(1); } if (testCount < 10) { console.error('NODE VERDICT: expected at least 10 node:test cases, found '+testCount+' — cases were removed or the file was gutted'); process.exit(1); } if (passCount < 10) { console.error('NODE VERDICT: expected at least 10 passing node:test cases, found '+passCount+' (skipped='+skippedCount+', todo='+todoCount+')'); process.exit(1); } if (suiteCount < 1) { console.error('NODE VERDICT: expected at least 1 node:test suite, found '+suiteCount); process.exit(1); } console.log('NODE VERDICT: tests passed (tests='+testCount+', suites='+suiteCount+', passed='+passCount+', skipped='+skippedCount+', todo='+todoCount+', failed=0)');\"", - "test:product-state-volume": "node scripts/product-state-volume-integration.test.mjs", + "test:product-state-volume": "node --experimental-strip-types scripts/product-state-volume-integration.test.ts", "test:boundaries": "vitest run --coverage.enabled=false src/ownership/rootOwnershipBoundary.test.ts src/deployment/providerRuntimeBoundary.test.ts src/ownership/mnemePublicImportBoundary.test.ts src/target/containerBundleArchive.test.ts", "test:causal-conformance": "tsx src/ledger/causalConformanceCli.ts", - "typecheck": "tsc --project tsconfig.json --noEmit", + "typecheck": "tsc --project tsconfig.json --noEmit && tsc --project scripts/tsconfig.json", "test:e2e:distribution-image": "tsx src/e2e/cli.ts distribution-image", "test:e2e:distribution-roundtrip": "tsx src/e2e/cli.ts distribution-roundtrip", - "test:e2e:daimon-org": "tsx src/e2e/cli.ts daimon-org" + "test:e2e:daimon-org": "tsx src/e2e/cli.ts daimon-org", + "test:scripts": "node --experimental-strip-types --test scripts/build-local-daimon-runtime.test.ts scripts/build-local-moltnet.test.ts scripts/compile-explicit-test-mcp.test.ts scripts/native-helper-workflows.test.ts scripts/typescript-policy.test.ts", + "test:unit": "npm run test:vitest; vitest_status=$?; npm run test:verdict; verdict_status=$?; npm run test:coverage-verdict; coverage_status=$?; npm run test:node; node_status=$?; failed_lanes=''; if [ \"$verdict_status\" -eq 2 ]; then failed_lanes='vitest test failures'; fi; if [ \"$node_status\" -eq 2 ]; then if [ -n \"$failed_lanes\" ]; then failed_lanes=\"$failed_lanes; node:test suite\"; else failed_lanes='node:test suite'; fi; fi; result_status=0; if [ -n \"$failed_lanes\" ]; then echo \"FAIL(tests): $failed_lanes\"; result_status=1; fi; if [ \"$verdict_status\" -ne 0 ] && [ \"$verdict_status\" -ne 2 ]; then echo 'FAIL(vitest-verdict): could not determine the vitest result'; result_status=1; fi; if [ \"$node_status\" -ne 0 ] && [ \"$node_status\" -ne 2 ]; then echo 'FAIL(node-lane): could not determine the node:test result'; result_status=1; fi; if [ \"$result_status\" -ne 0 ]; then exit 1; elif [ \"$vitest_status\" -ne 0 ]; then echo \"FAIL(vitest): exited $vitest_status with no test failure\"; exit 1; elif [ \"$coverage_status\" -eq 2 ]; then echo 'FAIL(coverage): thresholds not met (tests all passed)'; exit 2; elif [ \"$coverage_status\" -ne 0 ]; then echo 'FAIL(coverage-verdict): could not evaluate coverage (tests all passed)'; exit 1; else echo 'PASS'; fi", + "test": "npm run test:unit && npm run test:scripts" }, "dependencies": { "@noopolis/stele": "0.0.2", diff --git a/runtime-images/daimon/SourceBundle.Dockerfile b/runtime-images/daimon/SourceBundle.Dockerfile index 3071b947..032982c1 100644 --- a/runtime-images/daimon/SourceBundle.Dockerfile +++ b/runtime-images/daimon/SourceBundle.Dockerfile @@ -22,13 +22,10 @@ RUN test "$(sha256sum /tmp/source.tar | awk '{print "sha256:" $1}')" = "${SOURCE && npm ls --all \ && cp -a /closure/node_modules /src/node_modules \ && cd /src \ - && rm -rf dist \ - && node node_modules/typescript/bin/tsc --project tsconfig.build.json \ - && node --input-type=module -e 'import fs from "node:fs"; import crypto from "node:crypto"; import { RUNTIME_CONTRACT_MANIFEST as m } from "./dist/contracts/runtimeContractManifest.js"; const c=v=>Array.isArray(v)?"["+v.map(c).join(",")+"]":v&&typeof v==="object"?"{"+Object.keys(v).sort().map(k=>JSON.stringify(k)+":"+c(v[k])).join(",")+"}":JSON.stringify(v); const bytes=Buffer.from(c(m)+"\n"); fs.writeFileSync("dist/runtime/contract-manifest.json",bytes); fs.writeFileSync("dist/runtime/contract-manifest.sha256","sha256:"+crypto.createHash("sha256").update(bytes).digest("hex")+"\n")' \ && node -e 'const expected={amd64:"x64",arm64:"arm64"}[process.argv[1]]; if (!expected || process.platform!=="linux" || process.arch!==expected) process.exit(1)' "${TARGETARCH}" \ - && DAIMON_REQUIRE_ENGINE_BROKER=1 node src/runtime/native/copyArtifact.mjs \ + && DAIMON_REQUIRE_ENGINE_BROKER=1 npm run build \ && test -x dist/runtime/native/daimon-engine-broker \ - && node src/runtime/native/verifyArtifacts.mjs \ + && npm run verify:native \ && npm pack --ignore-scripts --offline --pack-destination /out \ && package="$(find /out -maxdepth 1 -type f -name '*.tgz' -print)" \ && test -n "${package}" \ diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index 069b959d..61f5a172 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -1,73 +1,40 @@ # Scripts Guide -## Structure +Maintained repository tooling lives here. [README.md](README.md) records each +entrypoint, its callers, and prerequisites. Historical worktree and burnlist +tools live in `../archive/legacy-worktree-tools/` and are not active helpers. -~~~text -scripts/ -├── AGENTS.md # Structure and local constraints for repository scripts -├── CLAUDE.md # Compatibility symlink to AGENTS.md -├── bootstrap-worktree.mjs # Dependency-free create, check, list, and smoke CLI -├── bootstrap-worktree.test.mjs # Bare node:test coverage and bootstrap self-test gate -├── build-closure.mjs # Builds a selected repository dependency closure in order -├── build-closure.test.mjs # Injected-registry and injected-run closure tests -├── build-local-daimon-runtime.mjs # Builds/pushes a receipt-bound Daimon image to an explicit loopback registry -├── build-local-daimon-runtime.test.mjs # AGY archive, provenance, redaction, and image-authority tests -├── create-source-provenance-bundle.mjs # Deterministic dirty-tree-safe all-input archive creator -├── create-linux-amd64-dependency-closure.mjs # Pinned-container lock/cache closure preparation -├── create-linux-amd64-go-closure.mjs # Pinned Go module-cache preparation for offline amd64 builds -├── source-provenance-bundle.mjs # Strict manifest, exclusion, ustar, digest, and drift checks -├── build-local-moltnet.mjs # Builds and stamps a local Moltnet release through Go -├── compile-explicit-test-mcp.mjs # Lowers bounded test-only MCP declarations against a compiled Daimon report -├── loop-verify.mjs # Runs mechanical loop gates and summarizes suite failures -├── loop-verify.test.mjs # Tests loop verification freshness and TAP parsing helpers -├── tap-self-test.mjs # Shared TAP parsing, test discovery, and case assertions -└── worktree-bootstrap/ - ├── AGENTS.md # Structure and constraints for bootstrap implementation modules - ├── CLAUDE.md # Compatibility symlink to AGENTS.md - ├── create.mjs # Creates nested, branch, and detached git worktrees - ├── install.mjs # Makes source hooks effective, clone-verifies dependencies, and inherits inputs - ├── receipt.mjs # Writes and validates bootstrap provenance receipts - ├── repos.mjs # Six-repository registry, install targets, and smoke commands - └── verify.mjs # Non-vacuous environment and npm-install verification -~~~ +## Rules -## Local design constraints +- Write TypeScript with named exports, strict types, and erasable syntax. Keep + each source file below 400 lines. Do not bypass checking with broad `any` or + `@ts-nocheck`. +- Run source scripts with Node 22.19+ and `--experimental-strip-types`. + `npm run typecheck` checks `scripts/tsconfig.json` and product source. +- Use Node builtins and local modules. Runtime JavaScript generated for a + container remains an output artifact, not another script implementation. +- Keep tests next to their source. Add ordinary tests to `test:scripts`; + real Docker or provider operations must remain explicit opt-in commands. +- Every entrypoint needs a documented caller and purpose. One-off audit dumps + and task-specific automation do not belong here. +- Scripts must not read, print, or embed provider credential contents. -- Keep scripts on plain Node.js 22 ESM with zero third-party imports; use only node builtins and sibling modules. -- Use named exports only and keep every source file below 400 lines. -- The local Daimon builder accepts the official AGY archive only through explicit - version, credential-free URL, SHA-512 archive, and SHA-256 extracted-executable - pins. It pushes only to the fixed loopback development repository and emits an - ignored immutable manifest/receipt identity; it currently fails closed outside - `linux/amd64` and never edits the runtime registry. -- Its public artifact inputs are `AGY_CLI_VERSION`, `AGY_CLI_URL`, - `AGY_CLI_SHA512`, `AGY_CLI_SHA256`, `GROK_CLI_VERSION`, `GROK_CLI_URL`, - `GROK_CLI_SHA256`, and `CODEX_CLI_SHA256`; URLs containing credentials, - queries, or fragments are rejected before Docker is invoked. -- The unchanged default Daimon source mode requires clean Git. Explicit archive - mode requires both `SPAWNFILE_DAIMON_SOURCE_BUNDLE` and - `SPAWNFILE_DAIMON_DEPENDENCY_BUNDLE`; each is a strict deterministic ustar - created by `npm run bundle:source-provenance -- ` and - `npm run bundle:source-provenance -- --dependencies `. The first - root is the reviewed source tree; the dependency root contains exactly its - reviewed `package.json`, package-lock v3 graph, local package archives, and - npm cache prepared directly from the reviewed Daimon package/lock by `npm run - prepare:linux-amd64-closure -- - `. Docker runs `npm ci --offline`, checks every installed - package against the lock, then ships the pruned production closure. The target is always - `linux/amd64`, including from an arm64 host; arm64 output is not supported. - Docker verifies and consumes those exact bytes without Git metadata or npm - registry dependency resolution. Run `npm run test:source-provenance-docker` - for the mandatory network-disabled real-Docker archive gate. -- The unchanged local Moltnet default also requires clean Git. Dirty-tree archive - mode requires strict `--build-source` and `--go-dependencies` provenance - archives in `SPAWNFILE_MOLTNET_SOURCE_BUNDLE` and - `SPAWNFILE_MOLTNET_GO_DEPENDENCY_BUNDLE`. Prepare the latter with - `npm run prepare:linux-amd64-go-closure -- `. - The pinned Go container verifies the module graph twice, the final build uses - `GOPROXY=off` and Docker `--network=none`, and the local stamp binds both - archive identities and the toolchain digest. -- Clone node_modules with APFS clonefile, verify it against the worktree lock, and fall back to npm ci on any defect; never symlink it. -- Preserve absolute paths and the literal WORKTREE prefix in bootstrap diagnostics. -- Hook-bearing source repositories are discovered from their source `.githooks/` directory. Bootstrap copies missing hook directories as regular files, records the action, and verifies the target worktree's existing `core.hooksPath` without mutating shared git config. -- bootstrap-worktree.test.mjs must run under bare node --test. Nothing else globs tests under scripts, so bootstrap-worktree.mjs runs it as its own non-vacuous self-test gate. +## Local runtime builds + +- Daimon builds accept only explicit versions, credential-free HTTPS URLs, and + executable/archive digest pins. Reject URLs with credentials, queries, or + fragments before Docker runs. +- The local builder pushes only to the fixed loopback development repository. + Its generated immutable manifest/receipt identity is ignored and never + edits `runtimes.yaml`. Clean-source builds select the native Docker + architecture unless an explicit supported architecture is supplied. +- Clean Git remains the default source mode. Explicit Daimon archive mode + requires `SPAWNFILE_DAIMON_SOURCE_BUNDLE` and + `SPAWNFILE_DAIMON_DEPENDENCY_BUNDLE`. Strict deterministic USTAR archives bind + source, lockfile, package archives, and npm cache. Dependency preparation is + pinned to `linux/amd64`; Docker verifies those bytes and installs offline. +- Moltnet archive mode requires source and Go dependency provenance archives. + Its pinned Go build verifies the graph and runs with `GOPROXY=off` and + Docker `--network=none`. +- Run the relevant network-disabled Docker provenance gate when changing + archive content, Docker staging, dependency installation, or builder behavior. diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..cc91c3eb --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,35 @@ +# Repository scripts + +These are maintained build, verification, and explicit development tools. +The compiler and installed CLI live in `src/`; this directory is not shipped +in the npm package. Scripts run as native TypeScript on Node 22.19+. + +| Entrypoint | Caller | Purpose / prerequisites | +| --- | --- | --- | +| `build-local-daimon-runtime.ts` | `npm run build:local-daimon` | Build a locally sourced runtime; explicit artifact pins, Docker, and loopback registry required | +| `build-local-moltnet.ts` | `npm run build:local-moltnet` | Build and stamp release binaries from an explicit source checkout | +| `create-source-provenance-bundle.ts` | `npm run bundle:source-provenance` | Create deterministic archives with manifests and credential exclusions | +| `create-linux-amd64-dependency-closure.ts` | `npm run prepare:linux-amd64-closure` | Prepare reviewed npm dependencies/cache in the pinned build container | +| `create-linux-amd64-go-closure.ts` | `npm run prepare:linux-amd64-go-closure` | Prepare the pinned Go module-cache closure | +| `compile-explicit-test-mcp.ts` | `npm run compile:explicit-test-mcp` | Lower bounded test MCP declarations against a compiled report; build the CLI first | +| `verify-package-closure.ts` | `npm run verify:package-closure` | Verify the packed CLI and runtime closure | +| `product-state-volume-integration.test.ts` | `npm run test:product-state-volume`, CI | Test real volume preseed; Docker and host volume access required | +| `source-provenance-bundle.integration.test.ts` | `npm run test:source-provenance-docker` | Test the real offline Daimon archive build | +| `moltnet-source-provenance.integration.test.ts` | `npm run test:moltnet-source-provenance-docker` | Test the real offline Moltnet archive build | + +`source-provenance-bundle.ts` and `native-helper-artifacts.ts` are shared +modules imported by these entrypoints and the native artifact copy step. +Adjacent unit tests run through `npm run test:scripts`; native artifact and +syscall integration tests also run explicitly in CI. + +`npm run typecheck` checks every maintained script with strict TypeScript. +`npm test` includes script unit tests. Docker integration tests have separate +commands so unit runs never launch deployments or consume provider tokens. + +The root registries are production package inputs: +[`runtimes.yaml`](../runtimes.yaml) pins runtime versions/images and +[`moltnet-releases.json`](../moltnet-releases.json) pins architecture-specific +Moltnet release assets and checksums. Their loaders and npm packaging expect +them at the package root. + +Retired worktree/burnlist tooling is preserved in the [archive](../archive/). diff --git a/scripts/build-local-daimon-runtime.test.mjs b/scripts/build-local-daimon-runtime.test.ts similarity index 89% rename from scripts/build-local-daimon-runtime.test.mjs rename to scripts/build-local-daimon-runtime.test.ts index 61fc0da0..0762af47 100644 --- a/scripts/build-local-daimon-runtime.test.mjs +++ b/scripts/build-local-daimon-runtime.test.ts @@ -11,11 +11,16 @@ import { resolveLocalBuildArchitecture, resolveLocalImageTag, resolvePushedImageReference -} from "./build-local-daimon-runtime.mjs"; -import { collectSourceManifest, createSourceBundle, validateSourceBundle } from "./source-provenance-bundle.mjs"; +} from "./build-local-daimon-runtime.ts"; +import { collectSourceManifest, createSourceBundle, validateSourceBundle } from "./source-provenance-bundle.ts"; -const digest = (character, length = 64) => character.repeat(length); -const artifactEnvironment = () => ({ +type DockerfileStage = { + body: string; + parent: string; +}; + +const digest = (character: string, length = 64): string => character.repeat(length); +const artifactEnvironment = (): Record => ({ AGY_CLI_SHA256: digest("a"), AGY_CLI_SHA512: digest("b", 128), AGY_CLI_URL: "https://example.invalid/agy/antigravity-linux-amd64.tar.gz", @@ -64,6 +69,7 @@ test("artifact pins reject credential-bearing URLs without disclosing credential let message = ""; assert.throws(() => readDaimonCliArtifactPins(env), (error) => { + if (!(error instanceof Error)) return false; message = error.message; return /AGY_CLI_URL/u.test(message); }); @@ -199,8 +205,10 @@ test("dependency bundle binds its amd64 lock closure and rejects a hostile tar", mkdirSync(path.join(root, "npm-cache")); writeFileSync(path.join(root, "npm-cache", "content"), "cache\n"); writeFileSync(path.join(root, "package.json"), '{"name":"closure"}\n'); writeFileSync(path.join(root, "package-lock.json"), JSON.stringify({ lockfileVersion: 3, name: "closure", packages: { "": {}, "node_modules/@openai/codex": { version: "1.0.0", integrity: `sha512-${"A".repeat(86)}==` }, "node_modules/typescript": { version: "5.0.0", integrity: `sha512-${"B".repeat(86)}==` } } })); const verified = validateSourceBundle(readFileSync((createSourceBundle(root, archive, "dependencies"), archive))); - assert.equal(verified.manifest.dependency_lock.target, "linux/amd64"); - assert.equal(verified.manifest.dependency_lock.packages.length, 2); + const dependencyLock = verified.manifest.dependency_lock; + assert.ok(dependencyLock && "packages" in dependencyLock); + assert.equal(dependencyLock.target, "linux/amd64"); + assert.equal(dependencyLock.packages.length, 2); const hostile = readFileSync(archive); hostile.write("../escape", 0, "utf8"); assert.throws(() => validateSourceBundle(hostile), /checksum|unsafe/u); } finally { rmSync(root, { force: true, recursive: true }); rmSync(archive, { force: true }); } @@ -269,29 +277,48 @@ test("Daimon Dockerfile verifies the AGY archive before extracting antigravity a assert.doesNotMatch(dockerfile, /npm cache clean/u); }); +test("Daimon source-bundle Dockerfile uses public package scripts for native artifacts", () => { + const dockerfile = readFileSync(new URL("../runtime-images/daimon/SourceBundle.Dockerfile", import.meta.url), "utf8"); + const archAssertion = dockerfile.indexOf("const expected={amd64:\"x64\",arm64:\"arm64\"}"); + const packageBuild = dockerfile.indexOf("DAIMON_REQUIRE_ENGINE_BROKER=1 npm run build"); + const executableCheck = dockerfile.indexOf("test -x dist/runtime/native/daimon-engine-broker"); + const verifyNative = dockerfile.indexOf("npm run verify:native"); + const pack = dockerfile.indexOf("npm pack --ignore-scripts --offline --pack-destination /out"); + + assert.ok(archAssertion >= 0); + assert.ok(archAssertion < packageBuild); + assert.ok(packageBuild < executableCheck); + assert.ok(executableCheck < verifyNative); + assert.ok(verifyNative < pack); + assert.doesNotMatch(dockerfile, /src\/runtime\/native\/(?:copyArtifact|verifyArtifacts)\.mjs/u); +}); + test("Daimon Dockerfile stage graph preserves cache and offline-network boundaries", () => { const dockerfile = readFileSync(new URL("../runtime-images/daimon/Dockerfile", import.meta.url), "utf8"); - const stages = new Map(); + const stages = new Map(); const fromPattern = /^FROM\s+(\S+)\s+AS\s+(\S+)\s*$/gimu; const declarations = [...dockerfile.matchAll(fromPattern)]; for (const [index, declaration] of declarations.entries()) { const [, rawParent, name] = declaration; + assert.ok(rawParent && name); const parent = rawParent.replaceAll("${DAIMON_DEPENDENCY_MODE}", "registry"); const bodyStart = declaration.index + declaration[0].length; const bodyEnd = declarations[index + 1]?.index ?? dockerfile.length; stages.set(name, { body: dockerfile.slice(bodyStart, bodyEnd), parent }); } - const ancestry = (graph, target) => { - const chain = []; - const visited = new Set(); + const ancestry = (graph: Map, target: string): string[] => { + const chain: string[] = []; + const visited = new Set(); let current = target; while (graph.has(current)) { assert.ok(!visited.has(current), `stage ancestry must not contain a cycle at ${current}`); visited.add(current); chain.push(current); - current = graph.get(current).parent; + const stage = graph.get(current); + assert.ok(stage); + current = stage.parent; } chain.push(current); return chain; @@ -311,15 +338,17 @@ test("Daimon Dockerfile stage graph preserves cache and offline-network boundari "grok_cli", "grok_source_offline-bundle", "base", "base_offline-bundle", "${NODE_BASE_IMAGE}" ]); - const assertAncestorsExcludeDaimonInputs = (graph, target) => { + const assertAncestorsExcludeDaimonInputs = (graph: Map, target: string): void => { for (const ancestor of ancestry(graph, target).slice(1, -1)) { - assert.doesNotMatch(graph.get(ancestor).body, /daimon\.tgz|source-inputs\.json/u, `${ancestor} must not depend on Daimon package inputs`); + const stage = graph.get(ancestor); + assert.ok(stage); + assert.doesNotMatch(stage.body, /daimon\.tgz|source-inputs\.json/u, `${ancestor} must not depend on Daimon package inputs`); } }; assertAncestorsExcludeDaimonInputs(stages, "daimon_registry"); assertAncestorsExcludeDaimonInputs(offlineStages, "daimon_offline-bundle"); - const stagesContaining = (pattern) => [...stages] + const stagesContaining = (pattern: RegExp): string[] => [...stages] .filter(([, stage]) => pattern.test(stage.body)) .map(([name]) => name) .sort(); diff --git a/scripts/build-local-daimon-runtime.mjs b/scripts/build-local-daimon-runtime.ts similarity index 75% rename from scripts/build-local-daimon-runtime.mjs rename to scripts/build-local-daimon-runtime.ts index d7636fa2..ad520aff 100644 --- a/scripts/build-local-daimon-runtime.mjs +++ b/scripts/build-local-daimon-runtime.ts @@ -9,15 +9,84 @@ import os from "node:os"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -import { hashTrackedSourceEntries } from "./build-local-moltnet.mjs"; -import { validateSourceBundle } from "./source-provenance-bundle.mjs"; +import { hashTrackedSourceEntries } from "./build-local-moltnet.ts"; +import type { TrackedSourceEntry } from "./build-local-moltnet.ts"; +import { validateSourceBundle } from "./source-provenance-bundle.ts"; +import type { NodeDependencyLock, SourceBundleProfile, SourceBundleReceipt } from "./source-provenance-bundle.ts"; + +type DaimonBuildArchitecture = "amd64" | "arm64"; +type SupportedHostArchitecture = "arm64" | "x64"; +type DigestAlgorithm = "sha256" | "sha512"; + +type DaimonCliArtifacts = { + agy: { + archive_sha512: string; + executable_sha256: string; + url: string; + version: string; + }; + codex: { + executable_sha256: string; + }; + grok: { + executable_sha256: string; + url: string; + version: string; + }; +}; + +type DaimonSourceInputs = { + dependencies: { + archive_sha256: string; + manifest_sha256: string; + package_lock_sha256: string; + runtime_archive_sha256: string; + }; + mode: "source-bundle"; + source: { + archive_sha256: string; + manifest_sha256: string; + }; + version: "spawnfile.daimon-source-inputs.v1"; +}; + +type SourceBundleWithPath = SourceBundleReceipt & { + path: string; +}; + +type BundleBuiltDaimon = { + packagePath: string; + sourceInputs: DaimonSourceInputs; + sourceSha256: string; +}; + +type BuildSourceIdentity = { + dependencies: { + archive_sha256: string; + manifest_sha256: string; + }; + source: { + archive_sha256: string; + manifest_sha256: string; + }; + target: string; +}; + +type CapabilityReceiptOptions = { + architecture: DaimonBuildArchitecture; + artifacts: DaimonCliArtifacts; + manifestSha256: string; + packageSha256: string; + sourceInputs?: DaimonSourceInputs; + sourceSha256: string; +}; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const configuredDaimonSource = process.env.SPAWNFILE_DAIMON_SOURCE_DIR?.trim(); const daimonDir = configuredDaimonSource ? path.resolve(configuredDaimonSource) : path.resolve(repoRoot, "..", "daimon"); -const sha256 = (value) => `sha256:${createHash("sha256").update(value).digest("hex")}`; +const sha256 = (value: Buffer | string) => `sha256:${createHash("sha256").update(value).digest("hex")}`; const LOCAL_REPOSITORY_PATH = "noopolis/spawnfile-runtime-daimon"; const sha256Digest = /^[a-f0-9]{64}$/u; const sha512Digest = /^[a-f0-9]{128}$/u; @@ -29,14 +98,14 @@ const LOCAL_DEVELOPMENT_PROVENANCE = Object.freeze({ unpublished: true }); -const requiredDigest = (env, name, algorithm, pattern) => { +const requiredDigest = (env: Record, name: string, algorithm: DigestAlgorithm, pattern: RegExp): string => { const value = env[name]?.trim(); const bare = value?.replace(new RegExp(`^${algorithm}:`, "u"), ""); if (!bare || !pattern.test(bare)) throw new Error(`${name} must be a ${algorithm.toUpperCase()} digest`); return `${algorithm}:${bare}`; }; -const requiredUrl = (env, name) => { +const requiredUrl = (env: Record, name: string): string => { const value = env[name]?.trim(); let parsed; try { @@ -50,16 +119,17 @@ const requiredUrl = (env, name) => { ) { throw new Error(`${name} must be a credential-free HTTPS URL without query or fragment`); } + if (!value) throw new Error(`${name} must be a credential-free HTTPS URL without query or fragment`); return value; }; -const requiredVersion = (env, name) => { +const requiredVersion = (env: Record, name: string): string => { const value = env[name]?.trim(); if (!value || !version.test(value)) throw new Error(`${name} must be an explicit artifact version`); return value; }; -export const readDaimonCliArtifactPins = (env = process.env) => ({ +export const readDaimonCliArtifactPins = (env: Record = process.env): DaimonCliArtifacts => ({ agy: { archive_sha512: requiredDigest(env, "AGY_CLI_SHA512", "sha512", sha512Digest), executable_sha256: requiredDigest(env, "AGY_CLI_SHA256", "sha256", sha256Digest), @@ -76,19 +146,19 @@ export const readDaimonCliArtifactPins = (env = process.env) => ({ } }); -export const resolveLocalImageTag = (value) => { +export const resolveLocalImageTag = (value: string | undefined): string => { const tag = value?.trim(); const match = tag?.match(/^127\.0\.0\.1:((?:[1-9]\d{0,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5]))\/noopolis\/spawnfile-runtime-daimon:([A-Za-z0-9_][A-Za-z0-9_.-]{0,127})$/u); const port = Number(match?.[1]); const label = match?.[2] ?? ""; const repository = match ? `127.0.0.1:${port}/${LOCAL_REPOSITORY_PATH}` : ""; - if (!Number.isInteger(port) || port < 1 || port > 65_535) throw new Error("SPAWNFILE_DAIMON_LOCAL_IMAGE_TAG must use an explicit 127.0.0.1 loopback registry port"); + if (!tag || !Number.isInteger(port) || port < 1 || port > 65_535) throw new Error("SPAWNFILE_DAIMON_LOCAL_IMAGE_TAG must use an explicit 127.0.0.1 loopback registry port"); if (!label || label === "latest" || !/^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/u.test(label)) { throw new Error(`SPAWNFILE_DAIMON_LOCAL_IMAGE_TAG must be an explicit non-latest tag under ${repository}`); } return tag; }; -export const resolveLocalBuildArchitecture = (hostArchitecture, requested = process.env.SPAWNFILE_DAIMON_TARGET_ARCH) => { +export const resolveLocalBuildArchitecture = (hostArchitecture: string, requested = process.env.SPAWNFILE_DAIMON_TARGET_ARCH): DaimonBuildArchitecture => { if (hostArchitecture !== "x64" && hostArchitecture !== "arm64") { throw new Error("Local Daimon builds require an x64 or arm64 Docker host"); } @@ -100,33 +170,35 @@ export const resolveLocalBuildArchitecture = (hostArchitecture, requested = proc return requested ?? (hostArchitecture === "arm64" ? "arm64" : "amd64"); }; -export const resolvePushedImageReference = (imageTag, repoDigests) => { +export const resolvePushedImageReference = (imageTag: string, repoDigests: string[]): string => { const repository = imageTag.slice(0, imageTag.lastIndexOf(":")); resolveLocalImageTag(imageTag); const expectedPrefix = `${repository}@`; const matches = repoDigests.filter((value) => value.startsWith(expectedPrefix)); - if (matches.length !== 1 || !new RegExp(`^${expectedPrefix.replaceAll(".", "\\.")}sha256:[a-f0-9]{64}$`, "u").test(matches[0])) { + const match = matches[0]; + if (matches.length !== 1 || !match || !new RegExp(`^${expectedPrefix.replaceAll(".", "\\.")}sha256:[a-f0-9]{64}$`, "u").test(match)) { throw new Error("Docker did not return one immutable local Daimon image manifest digest"); } if (imageTag.slice(0, imageTag.lastIndexOf(":")) !== repository) { throw new Error("Local Daimon image tag and pushed repository disagree"); } - return matches[0]; + return match; }; -const trackedEntries = (root) => execFileSync("git", ["-C", root, "ls-files", "-s", "-z"], { encoding: "utf8" }) +const trackedEntries = (root: string): TrackedSourceEntry[] => execFileSync("git", ["-C", root, "ls-files", "-s", "-z"], { encoding: "utf8" }) .split("\0").filter(Boolean).map((entry) => { const tab = entry.indexOf("\t"); const [mode] = entry.slice(0, tab).split(" "); + if (!mode || tab < 0) throw new Error("Unable to parse tracked source entry"); return { mode, path: entry.slice(tab + 1) }; }); -const assertClean = (root) => { +const assertClean = (root: string): void => { const status = execFileSync("git", ["-C", root, "status", "--porcelain=v1", "--untracked-files=all"], { encoding: "utf8" }); if (status.trim()) throw new Error("Local Daimon image build requires a clean source tree"); }; -const stagePackagedDaimon = (directory) => { +const stagePackagedDaimon = (directory: string): string => { const source = process.env.SPAWNFILE_DAIMON_PACKAGE_TARBALL?.trim(); if (!source || !path.isAbsolute(source)) throw new Error("SPAWNFILE_DAIMON_PACKAGE_TARBALL must be an absolute packaged Daimon tarball"); const entry = lstatSync(source); @@ -139,7 +211,7 @@ const stagePackagedDaimon = (directory) => { return staged; }; -const requiredBundle = (envName, stagedName, directory, expectedProfile) => { +const requiredBundle = (envName: string, stagedName: string, directory: string, expectedProfile: SourceBundleProfile): SourceBundleWithPath => { const source = process.env[envName]?.trim(); if (!source || !path.isAbsolute(source)) throw new Error(`${envName} must be an absolute deterministic source-provenance tar`); const entry = lstatSync(source); if (!entry.isFile() || entry.isSymbolicLink() || entry.size === 0) throw new Error(`${envName} must be a nonempty regular file`); @@ -148,15 +220,15 @@ const requiredBundle = (envName, stagedName, directory, expectedProfile) => { copyFileSync(source, staged); return { ...provenance, path: staged }; }; -export const resolveDaimonSourceMode = (env = process.env) => { +export const resolveDaimonSourceMode = (env: Record = process.env): "clean-git" | "source-bundle" => { const source = env.SPAWNFILE_DAIMON_SOURCE_BUNDLE?.trim(), dependencies = env.SPAWNFILE_DAIMON_DEPENDENCY_BUNDLE?.trim(); if (!source && !dependencies) return "clean-git"; if (!source || !dependencies) throw new Error("Archive provenance requires both SPAWNFILE_DAIMON_SOURCE_BUNDLE and SPAWNFILE_DAIMON_DEPENDENCY_BUNDLE"); return "source-bundle"; }; -const stageOfflineCliAssets = (directory, artifacts) => { - for (const [envName, destination, expected, algorithm] of [["SPAWNFILE_AGY_CLI_ARCHIVE", "agy.tar.gz", artifacts.agy.archive_sha512, "sha512"], ["SPAWNFILE_GROK_CLI_FILE", "grok", artifacts.grok.executable_sha256, "sha256"]]) { +const stageOfflineCliAssets = (directory: string, artifacts: DaimonCliArtifacts): void => { + for (const [envName, destination, expected, algorithm] of [["SPAWNFILE_AGY_CLI_ARCHIVE", "agy.tar.gz", artifacts.agy.archive_sha512, "sha512"], ["SPAWNFILE_GROK_CLI_FILE", "grok", artifacts.grok.executable_sha256, "sha256"]] as const) { const source = process.env[envName]?.trim(); if (!source || !path.isAbsolute(source)) throw new Error(`${envName} must be an absolute pinned offline CLI asset`); const item = lstatSync(source); if (!item.isFile() || item.isSymbolicLink() || item.size === 0) throw new Error(`${envName} must be a nonempty regular file`); const actual = `${algorithm}:${createHash(algorithm).update(readFileSync(source)).digest("hex")}`; if (actual !== expected) throw new Error(`${envName} checksum disagrees with its artifact pin`); @@ -164,7 +236,12 @@ const stageOfflineCliAssets = (directory, artifacts) => { } }; -const stageBundleBuiltDaimon = (directory, artifacts, architecture) => { +const nodeDependencyLock = (lock: SourceBundleReceipt["manifest"]["dependency_lock"]): NodeDependencyLock => { + if (!lock || !("package_lock_sha256" in lock)) throw new Error("Dependency bundle lacks package-lock identity"); + return lock; +}; + +const stageBundleBuiltDaimon = (directory: string, artifacts: DaimonCliArtifacts, architecture: DaimonBuildArchitecture): BundleBuiltDaimon => { const sourceDirectory = path.join(directory, "source_bundle"), dependencyDirectory = path.join(directory, "dependency_bundle"); mkdirSync(sourceDirectory, { recursive: true }); mkdirSync(dependencyDirectory, { recursive: true }); const source = requiredBundle("SPAWNFILE_DAIMON_SOURCE_BUNDLE", "source.tar", sourceDirectory, "source"); @@ -181,11 +258,12 @@ const stageBundleBuiltDaimon = (directory, artifacts, architecture) => { const runtimeDependencies = path.join(output, "runtime-dependencies.tar"); if (!existsSync(runtimeDependencies)) throw new Error("Remote bundle build did not produce its runtime dependency closure"); const runtimeArchiveSha256 = sha256(readFileSync(runtimeDependencies)); copyFileSync(runtimeDependencies, path.join(directory, "dependencies.tar")); + const dependencyLock = nodeDependencyLock(dependencies.manifest.dependency_lock); const sourceInputs = { dependencies: { archive_sha256: dependencies.archive_sha256, manifest_sha256: dependencies.manifest_sha256, - package_lock_sha256: dependencies.manifest.dependency_lock.package_lock_sha256, runtime_archive_sha256: runtimeArchiveSha256 }, + package_lock_sha256: dependencyLock.package_lock_sha256, runtime_archive_sha256: runtimeArchiveSha256 }, mode: "source-bundle", source: { archive_sha256: source.archive_sha256, manifest_sha256: source.manifest_sha256 }, - version: "spawnfile.daimon-source-inputs.v1" }; - const buildIdentity = JSON.parse(readFileSync(path.join(output, "source-inputs.json"), "utf8")); + version: "spawnfile.daimon-source-inputs.v1" } satisfies DaimonSourceInputs; + const buildIdentity = JSON.parse(readFileSync(path.join(output, "source-inputs.json"), "utf8")) as BuildSourceIdentity; if (buildIdentity.target !== `linux/${architecture}` || buildIdentity.source.archive_sha256 !== source.archive_sha256 || buildIdentity.source.manifest_sha256 !== source.manifest_sha256 || buildIdentity.dependencies.archive_sha256 !== dependencies.archive_sha256 || buildIdentity.dependencies.manifest_sha256 !== dependencies.manifest_sha256) { throw new Error("Remote bundle build source identity does not match its attested inputs"); @@ -195,7 +273,7 @@ const stageBundleBuiltDaimon = (directory, artifacts, architecture) => { return { packagePath, sourceInputs, sourceSha256: sha256(Buffer.from(JSON.stringify(sourceInputs))) }; }; -export const createLocalDaimonCapabilityReceipt = ({ architecture, artifacts, manifestSha256, packageSha256, sourceInputs, sourceSha256 }) => ({ +export const createLocalDaimonCapabilityReceipt = ({ architecture, artifacts, manifestSha256, packageSha256, sourceInputs, sourceSha256 }: CapabilityReceiptOptions) => ({ architecture, daimon: { package_sha256: packageSha256, source_sha256: sourceSha256, ...(sourceInputs ? { source_inputs: sourceInputs } : {}) }, engines: { @@ -225,7 +303,7 @@ export const createLocalDaimonCapabilityReceipt = ({ architecture, artifacts, ma version: "spawnfile.daimon-runtime-capability-receipt.v1" }); -const main = () => { +const main = (): void => { if (configuredDaimonSource && !path.isAbsolute(configuredDaimonSource)) { throw new Error("SPAWNFILE_DAIMON_SOURCE_DIR must be absolute"); } @@ -242,14 +320,15 @@ const main = () => { const bundled = sourceMode === "source-bundle" ? stageBundleBuiltDaimon(packageDirectory, artifacts, architecture) : null; const packagePath = bundled?.packagePath ?? stagePackagedDaimon(packageDirectory); const manifestBytes = execFileSync("tar", ["-xOf", packagePath, "package/dist/runtime/contract-manifest.json"]); - const receipt = createLocalDaimonCapabilityReceipt({ + const receiptOptions: CapabilityReceiptOptions = { architecture, artifacts, manifestSha256: sha256(manifestBytes), packageSha256: sha256(readFileSync(packagePath)), - sourceInputs: bundled?.sourceInputs, sourceSha256: bundled?.sourceSha256 ?? hashTrackedSourceEntries(daimonDir, trackedEntries(daimonDir)) - }); + }; + if (bundled) receiptOptions.sourceInputs = bundled.sourceInputs; + const receipt = createLocalDaimonCapabilityReceipt(receiptOptions); const receiptBytes = Buffer.from(`${JSON.stringify(receipt)}\n`); execFileSync("docker", ["build", ...(bundled ? ["--network=none"] : []), "--platform", `linux/${architecture}`, "--build-context", `daimon_package=${packageDirectory}`, "-f", path.join(repoRoot, "runtime-images", "daimon", "Dockerfile"), "-t", imageTag, @@ -273,9 +352,10 @@ const main = () => { const [imageConfigDigest, imageArchitecture, repoDigestsJson] = execFileSync( "docker", ["image", "inspect", "--format", "{{.Id}}\n{{.Architecture}}\n{{json .RepoDigests}}", imageTag], { encoding: "utf8" } ).trim().split("\n"); - if (!/^sha256:[a-f0-9]{64}$/u.test(imageConfigDigest)) throw new Error("Docker did not return an immutable image config digest"); + if (!imageConfigDigest || !/^sha256:[a-f0-9]{64}$/u.test(imageConfigDigest)) throw new Error("Docker did not return an immutable image config digest"); if (imageArchitecture !== architecture) throw new Error("Docker image architecture does not match the selected local Daimon inputs"); - const imageReference = resolvePushedImageReference(imageTag, JSON.parse(repoDigestsJson)); + if (!repoDigestsJson) throw new Error("Docker did not return local Daimon repository digests"); + const imageReference = resolvePushedImageReference(imageTag, JSON.parse(repoDigestsJson) as string[]); const imageManifestDigest = imageReference.slice(imageReference.indexOf("@") + 1); const registryAuthority = imageTag.slice(0, imageTag.indexOf("/")); writeFileSync(path.join(repoRoot, ".local-daimon-runtime-identity.json"), `${JSON.stringify({ diff --git a/scripts/build-local-moltnet.test.mjs b/scripts/build-local-moltnet.test.ts similarity index 87% rename from scripts/build-local-moltnet.test.mjs rename to scripts/build-local-moltnet.test.ts index d967db69..cb755e70 100644 --- a/scripts/build-local-moltnet.test.mjs +++ b/scripts/build-local-moltnet.test.ts @@ -4,18 +4,21 @@ import os from "node:os"; import path from "node:path"; import test from "node:test"; -import { createCapabilityProbeConfig, hashTrackedSourceEntries } from "./build-local-moltnet.mjs"; +import { createCapabilityProbeConfig, hashTrackedSourceEntries } from "./build-local-moltnet.ts"; test("Daimon capability probe represents the complete required runtime contract", () => { const receiptStorePath = "/var/lib/spawnfile/moltnet/networks/capability/daimon-receipts/daimon-agent.json"; const config = createCapabilityProbeConfig("daimon", receiptStorePath); - assert.deepEqual(config.attachments[0].runtime, { + const attachment = config.attachments[0]; + assert.ok(attachment); + assert.deepEqual(attachment.runtime, { kind: "daimon", control_url: "http://127.0.0.1:19700", receipt_store_path: receiptStorePath, token_env: "SPAWNFILE_DAIMON_CONTROL_TOKEN" }); - assert.equal(path.isAbsolute(config.attachments[0].runtime.receipt_store_path), true); + assert.equal(attachment.runtime.kind, "daimon"); + assert.equal(path.isAbsolute(attachment.runtime.receipt_store_path), true); }); test("source hashing accepts contained CLAUDE symlinks deterministically", () => { diff --git a/scripts/build-local-moltnet.mjs b/scripts/build-local-moltnet.ts similarity index 81% rename from scripts/build-local-moltnet.mjs rename to scripts/build-local-moltnet.ts index 68115351..fbdda891 100644 --- a/scripts/build-local-moltnet.mjs +++ b/scripts/build-local-moltnet.ts @@ -8,7 +8,21 @@ import { existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readlinkSy import os from "node:os"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -import { validateSourceBundle } from "./source-provenance-bundle.mjs"; +import { validateSourceBundle } from "./source-provenance-bundle.ts"; +import type { SourceBundleProfile, SourceBundleReceipt } from "./source-provenance-bundle.ts"; + +export type GoArchitecture = "amd64" | "arm64"; + +export type TrackedSourceEntry = { + mode: string; + path: string; +}; + +type CapabilityKind = "daimon" | "pi"; + +type ProvenanceArchive = SourceBundleReceipt & { + path: string; +}; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const configuredMoltnetSource = process.env.SPAWNFILE_MOLTNET_SOURCE_DIR?.trim(); @@ -17,22 +31,22 @@ const moltnetDir = configuredMoltnetSource : path.resolve(repoRoot, "..", "moltnet"); const configuredReleaseOutput = process.env.SPAWNFILE_MOLTNET_LOCAL_RELEASE_OUTPUT?.trim(); const releaseDir = configuredReleaseOutput ? path.resolve(configuredReleaseOutput) : path.join(moltnetDir, "dist", "spawnfile-local-release"); -const sha256 = (value) => createHash("sha256").update(value).digest("hex"); +const sha256 = (value: Buffer | string): string => createHash("sha256").update(value).digest("hex"); -export const goarchForHost = () => { +export const goarchForHost = (): GoArchitecture => { if (process.arch === "arm64") return "arm64"; if (process.arch === "x64") return "amd64"; throw new Error(`Unsupported host architecture for local Moltnet build: ${process.arch}`); }; -export const goarchForTarget = () => { +export const goarchForTarget = (): GoArchitecture => { const requested = process.env.MOLTNET_TARGET_GOARCH; if (requested === undefined) return goarchForHost(); if (requested !== "amd64" && requested !== "arm64") throw new Error(`Unsupported MOLTNET_TARGET_GOARCH: ${requested}`); return requested; }; -const normalizedTrackedPath = (root, relativePath) => { +const normalizedTrackedPath = (root: string, relativePath: string): { relative: string; resolved: string } => { if (!relativePath || path.isAbsolute(relativePath)) throw new Error("Tracked source path must be relative"); const resolved = path.resolve(root, relativePath); const relative = path.relative(root, resolved); @@ -42,7 +56,7 @@ const normalizedTrackedPath = (root, relativePath) => { return { relative: relative.split(path.sep).join("/"), resolved }; }; -const containedSymlink = (root, linkPath) => { +const containedSymlink = (root: string, linkPath: string): { link: string; target: string } => { const link = readlinkSync(linkPath); if (!link || path.isAbsolute(link)) throw new Error(`Tracked symlink must be nonempty and relative: ${linkPath}`); const target = path.resolve(path.dirname(linkPath), link); @@ -56,7 +70,7 @@ const containedSymlink = (root, linkPath) => { /** Hash Git-tracked entries without dereferencing links. The link text and * in-tree target identity are both bound, so tracked CLAUDE.md symlinks are * accepted deterministically but cannot introduce an out-of-tree read. */ -export const hashTrackedSourceEntries = (root, entries) => { +export const hashTrackedSourceEntries = (root: string, entries: TrackedSourceEntry[]): string => { const digest = createHash("sha256"); const ordered = [...entries].sort((left, right) => left.path.localeCompare(right.path)); const trackedPaths = new Set(ordered.map((entry) => normalizedTrackedPath(root, entry.path).relative)); @@ -82,7 +96,7 @@ export const hashTrackedSourceEntries = (root, entries) => { return `sha256:${digest.digest("hex")}`; }; -const readTrackedEntries = (root) => execFileSync("git", ["-C", root, "ls-files", "-s", "-z"], { encoding: "utf8" }) +const readTrackedEntries = (root: string): TrackedSourceEntry[] => execFileSync("git", ["-C", root, "ls-files", "-s", "-z"], { encoding: "utf8" }) .split("\0").filter(Boolean).map((entry) => { const tab = entry.indexOf("\t"); const [mode] = entry.slice(0, tab).split(" "); @@ -90,12 +104,12 @@ const readTrackedEntries = (root) => execFileSync("git", ["-C", root, "ls-files" return { mode, path: entry.slice(tab + 1) }; }); -const assertCleanSource = (root) => { +const assertCleanSource = (root: string): void => { const status = execFileSync("git", ["-C", root, "status", "--porcelain=v1", "--untracked-files=all"], { encoding: "utf8" }); if (status.trim()) throw new Error("Local Moltnet build requires a clean source tree"); }; -export const createCapabilityProbeConfig = (kind, receiptStorePath) => ({ +export const createCapabilityProbeConfig = (kind: CapabilityKind, receiptStorePath: string) => ({ version: "moltnet.node.v1", moltnet: { base_url: "http://127.0.0.1:9", network_id: "capability" }, attachments: [{ agent: { id: `${kind}-agent`, name: `${kind} agent` }, runtime: kind === "daimon" @@ -103,10 +117,10 @@ export const createCapabilityProbeConfig = (kind, receiptStorePath) => ({ : { kind, control_url: "http://127.0.0.1:19690/agents/pi-agent/wake" } }] }); -const assertBuiltBinaryCapabilities = (binaryPath) => { +const assertBuiltBinaryCapabilities = (binaryPath: string): void => { const temporaryDirectory = mkdtempSync(path.join(os.tmpdir(), "spawnfile-moltnet-capability-")); try { - for (const kind of ["pi", "daimon"]) { + for (const kind of ["pi", "daimon"] as const) { const configPath = path.join(temporaryDirectory, `${kind}.json`); const receiptDirectory = path.join(temporaryDirectory, "daimon-receipts"); mkdirSync(receiptDirectory, { mode: 0o700, recursive: true }); @@ -118,7 +132,7 @@ const assertBuiltBinaryCapabilities = (binaryPath) => { encoding: "utf8", env: { ...process.env, SPAWNFILE_DAIMON_CONTROL_TOKEN: "local-capability-probe" }, timeout: 1_000 }); const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`; - if (result.error?.code === "ETIMEDOUT") continue; // parser accepted; the endpoint is deliberately unreachable. + if ((result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT") continue; // parser accepted; the endpoint is deliberately unreachable. if (/unsupported|only supported|required|invalid/i.test(output)) { throw new Error(`Built Moltnet binary does not accept ${kind}-bridge: ${output.trim()}`); } @@ -130,8 +144,8 @@ const assertBuiltBinaryCapabilities = (binaryPath) => { } }; -const assertDockerBinaryCapabilities = (binaryPath, arch) => { - for (const kind of ["pi", "daimon"]) { +const assertDockerBinaryCapabilities = (binaryPath: string, arch: GoArchitecture): void => { + for (const kind of ["pi", "daimon"] as const) { const temporaryDirectory = mkdtempSync(path.join(os.tmpdir(), "spawnfile-moltnet-probe-")), configPath = path.join(temporaryDirectory, "config.json"), receiptDirectory = path.join(temporaryDirectory, "receipts"); mkdirSync(receiptDirectory); writeFileSync(configPath, JSON.stringify(createCapabilityProbeConfig(kind, "/receipts/agent.json"))); const id = execFileSync("docker", ["create", "--platform", `linux/${arch}`, "--env", "SPAWNFILE_DAIMON_CONTROL_TOKEN=probe", "node:24-bookworm-slim@sha256:a9f5f7c91a432850b2a8a7797adf5eadb6c733ceed61167806cee7ea7fbc29df", "timeout", "2", "/moltnet", "node", "/config.json"], { encoding: "utf8" }).trim(); @@ -140,13 +154,18 @@ const assertDockerBinaryCapabilities = (binaryPath, arch) => { } }; -const requiredBundle = (name, profile) => { +const requiredArchiveEnvPath = (name: string): string => { const value = process.env[name]?.trim(); if (!value || !path.isAbsolute(value)) throw new Error(`${name} must be an absolute provenance archive`); + return value; +}; + +const requiredBundle = (name: string, profile: SourceBundleProfile): ProvenanceArchive => { + const value = requiredArchiveEnvPath(name); const item = lstatSync(value); if (!item.isFile() || item.isSymbolicLink() || !item.size) throw new Error(`${name} must be a nonempty regular file`); const receipt = validateSourceBundle(readFileSync(value)); if (receipt.manifest.exclude_policy.profile !== profile) throw new Error(`${name} has the wrong provenance profile`); return { ...receipt, path: value }; }; -const main = () => { +const main = (): void => { if (configuredMoltnetSource && !path.isAbsolute(configuredMoltnetSource)) { throw new Error("SPAWNFILE_MOLTNET_SOURCE_DIR must be absolute"); } @@ -156,6 +175,8 @@ const main = () => { if (!archiveMode) assertCleanSource(moltnetDir); const arch = goarchForTarget(); if (!archiveMode && arch !== goarchForHost()) throw new Error("Cross-compiled local archives cannot prove their binary capabilities on this host"); + const sourceBundlePath = archiveMode ? requiredArchiveEnvPath("SPAWNFILE_MOLTNET_SOURCE_BUNDLE") : undefined; + const goDependencyBundlePath = archiveMode ? requiredArchiveEnvPath("SPAWNFILE_MOLTNET_GO_DEPENDENCY_BUNDLE") : undefined; const workDirectory = mkdtempSync(path.join(os.tmpdir(), "spawnfile-moltnet-build-")); const binaryPath = path.join(workDirectory, "moltnet"); const asset = `moltnet_linux_${arch}.tar.gz`; @@ -183,12 +204,18 @@ const main = () => { } finally { rmSync(workDirectory, { force: true, recursive: true }); } + const sourceSha256 = archiveMode && sourceBundlePath + ? validateSourceBundle(readFileSync(sourceBundlePath)).archive_sha256 + : hashTrackedSourceEntries(moltnetDir, readTrackedEntries(moltnetDir)); + const sourceInputs = archiveMode && sourceBundlePath && goDependencyBundlePath + ? { dependencies_sha256: validateSourceBundle(readFileSync(goDependencyBundlePath)).archive_sha256, mode: "source-bundle", source_sha256: validateSourceBundle(readFileSync(sourceBundlePath)).archive_sha256, toolchain: "golang:1.24-bookworm@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac" } + : undefined; const stamp = { arch, asset, capabilities: ["daimon-bridge", "pi-bridge"], development: { mode: "local-development", non_production: true, unsigned: true, unpublished: true }, sha256: sha256(readFileSync(assetPath)), - source_sha256: archiveMode ? validateSourceBundle(readFileSync(process.env.SPAWNFILE_MOLTNET_SOURCE_BUNDLE)).archive_sha256 : hashTrackedSourceEntries(moltnetDir, readTrackedEntries(moltnetDir)), - ...(archiveMode ? { source_inputs: { dependencies_sha256: validateSourceBundle(readFileSync(process.env.SPAWNFILE_MOLTNET_GO_DEPENDENCY_BUNDLE)).archive_sha256, mode: "source-bundle", source_sha256: validateSourceBundle(readFileSync(process.env.SPAWNFILE_MOLTNET_SOURCE_BUNDLE)).archive_sha256, toolchain: "golang:1.24-bookworm@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac" } } : {}), + source_sha256: sourceSha256, + ...(sourceInputs ? { source_inputs: sourceInputs } : {}), stamp_version: "spawnfile.local-moltnet-release-stamp.v1" }; writeFileSync(path.join(releaseDir, `local_moltnet_release_stamp_${arch}.json`), `${JSON.stringify(stamp)}\n`); diff --git a/scripts/compile-explicit-test-mcp.mjs b/scripts/compile-explicit-test-mcp.mjs deleted file mode 100644 index 4f585c12..00000000 --- a/scripts/compile-explicit-test-mcp.mjs +++ /dev/null @@ -1,22 +0,0 @@ -import { createHash } from "node:crypto"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; -import path from "node:path"; - -const args = Object.fromEntries(process.argv.slice(2).reduce((rows, item, index, all) => item.startsWith("--") ? [...rows, [item.slice(2), all[index + 1]]] : rows, [])); -if (!args.declaration || !args.report || !args.out) throw new Error("usage: --declaration --report --out "); -const declarationBytes = await readFile(path.resolve(args.declaration)); const declaration = JSON.parse(declarationBytes); -const report = JSON.parse(await readFile(path.resolve(args.report), "utf8")); -if (!report || !/^sf1:[a-f0-9]{12}$/u.test(report.compile_fingerprint)) throw new Error("explicit-test MCP lowering requires a compiled Spawnfile report"); -const compiledAgents = new Set((report.container?.runtime_instances ?? []).filter((instance) => instance.runtime === "daimon").flatMap((instance) => instance.node_ids ?? [])); -exact(declaration, ["version", "servers"]); if (declaration.version !== "spawnfile.explicit-test-mcp-declaration.v1" || !Array.isArray(declaration.servers) || declaration.servers.length > 8) throw new Error("invalid explicit-test MCP declaration"); -const servers = declaration.servers.map((value) => { exact(value, ["id", "agent_id", "command", "args", "tools", "env_names"]); if (![value.id, value.agent_id].every(identifier) || !absolute(value.command) || !strings(value.args, 16, absolute) || !strings(value.tools, 16, identifier) || !strings(value.env_names, 16, identifier)) throw new Error("invalid explicit-test MCP server declaration"); return value; }).sort((left, right) => left.id.localeCompare(right.id)); -if (servers.some(({ agent_id }) => !compiledAgents.has(agent_id))) throw new Error("explicit-test MCP server agent is absent from compiled Daimon instances"); -if (new Set(servers.map(({ id }) => id)).size !== servers.length) throw new Error("duplicate explicit-test MCP server id"); -const artifact = { version: "spawnfile.explicit-test-mcp.v1", compile_fingerprint: report.compile_fingerprint, servers }; -const artifactBytes = Buffer.from(`${JSON.stringify(artifact)}\n`); const digest = value => `sha256:${createHash("sha256").update(value).digest("hex")}`; -const receipt = { version: "spawnfile.explicit-test-mcp-receipt.v1", compile_fingerprint: report.compile_fingerprint, declaration_sha256: digest(declarationBytes), artifact_sha256: digest(artifactBytes), servers: servers.map(({ id, agent_id, tools }) => ({ id, agent_id, tools })) }; -await mkdir(path.resolve(args.out), { recursive: true }); await writeFile(path.resolve(args.out, "explicit-test-mcp.json"), artifactBytes, { mode: 0o600 }); await writeFile(path.resolve(args.out, "explicit-test-mcp-receipt.json"), `${JSON.stringify(receipt)}\n`, { mode: 0o600 }); -function exact(value, keys) { if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).sort().join() !== [...keys].sort().join()) throw new Error("unexpected explicit-test MCP field"); } -function identifier(value) { return typeof value === "string" && /^[A-Za-z_][A-Za-z0-9_.:-]{0,127}$/u.test(value); } -function absolute(value) { return typeof value === "string" && value.startsWith("/") && value.length <= 1024; } -function strings(value, limit, validator) { return Array.isArray(value) && value.length <= limit && value.every(validator) && new Set(value).size === value.length; } diff --git a/scripts/compile-explicit-test-mcp.test.mjs b/scripts/compile-explicit-test-mcp.test.mjs deleted file mode 100644 index 2b122dc6..00000000 --- a/scripts/compile-explicit-test-mcp.test.mjs +++ /dev/null @@ -1,5 +0,0 @@ -import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import os from "node:os"; import path from "node:path"; import test from "node:test"; -test("explicit-test MCP lowering binds compiled identity and declared server tools", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "spawnfile-test-mcp-")); try { const declaration = path.join(root, "declaration.json"), report = path.join(root, "report.json"), out = path.join(root, "out"); await writeFile(declaration, JSON.stringify({ version: "spawnfile.explicit-test-mcp-declaration.v1", servers: [{ id: "fixture", agent_id: "agent:alpha", command: "/usr/local/bin/node", args: ["/fixture/server.mjs"], tools: ["checkpoint"], env_names: [] }] })); await writeFile(report, JSON.stringify({ compile_fingerprint: "sf1:0123456789ab", container: { runtime_instances: [{ runtime: "daimon", node_ids: ["agent:alpha"] }] } })); execFileSync(process.execPath, ["scripts/compile-explicit-test-mcp.mjs", "--declaration", declaration, "--report", report, "--out", out]); const artifact = JSON.parse(await readFile(path.join(out, "explicit-test-mcp.json"))); const receipt = JSON.parse(await readFile(path.join(out, "explicit-test-mcp-receipt.json"))); assert.equal(artifact.compile_fingerprint, "sf1:0123456789ab"); assert.deepEqual(receipt.servers, [{ id: "fixture", agent_id: "agent:alpha", tools: ["checkpoint"] }]); } finally { await rm(root, { recursive: true, force: true }); } }); diff --git a/scripts/compile-explicit-test-mcp.test.ts b/scripts/compile-explicit-test-mcp.test.ts new file mode 100644 index 00000000..8b3d3174 --- /dev/null +++ b/scripts/compile-explicit-test-mcp.test.ts @@ -0,0 +1,54 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +type JsonObject = Record; + +const parseObject = async (filePath: string): Promise => { + const value = JSON.parse(await readFile(filePath, "utf8")) as unknown; + assert.ok(value !== null && typeof value === "object" && !Array.isArray(value)); + return value as JsonObject; +}; + +test("explicit-test MCP lowering binds compiled identity and declared server tools", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "spawnfile-test-mcp-")); + try { + const declaration = path.join(root, "declaration.json"); + const report = path.join(root, "report.json"); + const out = path.join(root, "out"); + await writeFile(declaration, JSON.stringify({ + servers: [{ + agent_id: "agent:alpha", + args: ["/fixture/server.mjs"], + command: "/usr/local/bin/node", + env_names: [], + id: "fixture", + tools: ["checkpoint"], + }], + version: "spawnfile.explicit-test-mcp-declaration.v1", + })); + await writeFile(report, JSON.stringify({ + compile_fingerprint: "sf1:0123456789ab", + container: { runtime_instances: [{ node_ids: ["agent:alpha"], runtime: "daimon" }] }, + })); + execFileSync(process.execPath, [ + "--experimental-strip-types", + "scripts/compile-explicit-test-mcp.ts", + "--declaration", + declaration, + "--report", + report, + "--out", + out, + ]); + const artifact = await parseObject(path.join(out, "explicit-test-mcp.json")); + const receipt = await parseObject(path.join(out, "explicit-test-mcp-receipt.json")); + assert.equal(artifact.compile_fingerprint, "sf1:0123456789ab"); + assert.deepEqual(receipt.servers, [{ agent_id: "agent:alpha", id: "fixture", tools: ["checkpoint"] }]); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/scripts/compile-explicit-test-mcp.ts b/scripts/compile-explicit-test-mcp.ts new file mode 100644 index 00000000..82dba5a5 --- /dev/null +++ b/scripts/compile-explicit-test-mcp.ts @@ -0,0 +1,192 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +type JsonObject = Record; + +type ExplicitTestMcpServer = { + agent_id: string; + args: string[]; + command: string; + env_names: string[]; + id: string; + tools: string[]; +}; + +type ExplicitTestMcpDeclaration = { + servers: ExplicitTestMcpServer[]; + version: "spawnfile.explicit-test-mcp-declaration.v1"; +}; + +type CompileReport = { + compile_fingerprint: string; + container?: { + runtime_instances?: RuntimeInstance[]; + }; +}; + +type RuntimeInstance = { + node_ids?: string[]; + runtime?: string; +}; + +type RequiredArgs = { + declaration: string; + out: string; + report: string; +}; + +const digest = (value: Uint8Array): string => ( + `sha256:${createHash("sha256").update(value).digest("hex")}` +); + +const parseArgs = (argv: string[]): RequiredArgs => { + const args = new Map(); + for (let index = 0; index < argv.length; index += 1) { + const item = argv[index]; + const next = argv[index + 1]; + if (item?.startsWith("--") && next !== undefined) { + args.set(item.slice(2), next); + index += 1; + } + } + const declaration = args.get("declaration"); + const report = args.get("report"); + const out = args.get("out"); + if (!declaration || !report || !out) { + throw new Error("usage: --declaration --report --out "); + } + return { declaration, out, report }; +}; + +const asJsonObject = (value: unknown): JsonObject | null => ( + value !== null && typeof value === "object" && !Array.isArray(value) + ? value as JsonObject + : null +); + +const exact = (value: unknown, keys: string[]): JsonObject => { + const object = asJsonObject(value); + if (!object || Object.keys(object).sort().join() !== [...keys].sort().join()) { + throw new Error("unexpected explicit-test MCP field"); + } + return object; +}; + +const identifier = (value: unknown): value is string => ( + typeof value === "string" && /^[A-Za-z_][A-Za-z0-9_.:-]{0,127}$/u.test(value) +); + +const absolute = (value: unknown): value is string => ( + typeof value === "string" && value.startsWith("/") && value.length <= 1024 +); + +const strings = ( + value: unknown, + limit: number, + validator: (candidate: unknown) => candidate is string, +): value is string[] => ( + Array.isArray(value) + && value.length <= limit + && value.every(validator) + && new Set(value).size === value.length +); + +const parseServer = (value: unknown): ExplicitTestMcpServer => { + const server = exact(value, ["id", "agent_id", "command", "args", "tools", "env_names"]); + if (!identifier(server.id) + || !identifier(server.agent_id) + || !absolute(server.command) + || !strings(server.args, 16, absolute) + || !strings(server.tools, 16, identifier) + || !strings(server.env_names, 16, identifier)) { + throw new Error("invalid explicit-test MCP server declaration"); + } + return { + agent_id: server.agent_id, + args: server.args, + command: server.command, + env_names: server.env_names, + id: server.id, + tools: server.tools, + }; +}; + +const parseDeclaration = (value: unknown): ExplicitTestMcpDeclaration => { + const declaration = exact(value, ["version", "servers"]); + if (declaration.version !== "spawnfile.explicit-test-mcp-declaration.v1" + || !Array.isArray(declaration.servers) + || declaration.servers.length > 8) { + throw new Error("invalid explicit-test MCP declaration"); + } + return { + servers: declaration.servers.map(parseServer), + version: declaration.version, + }; +}; + +const parseRuntimeInstance = (value: unknown): RuntimeInstance | null => { + const object = asJsonObject(value); + if (!object) return null; + const nodeIds = object.node_ids; + return { + node_ids: Array.isArray(nodeIds) && nodeIds.every((nodeId) => typeof nodeId === "string") + ? nodeIds + : undefined, + runtime: typeof object.runtime === "string" ? object.runtime : undefined, + }; +}; + +const parseCompileReport = (value: unknown): CompileReport => { + const report = asJsonObject(value); + if (!report + || typeof report.compile_fingerprint !== "string" + || !/^sf1:[a-f0-9]{12}$/u.test(report.compile_fingerprint)) { + throw new Error("explicit-test MCP lowering requires a compiled Spawnfile report"); + } + const container = asJsonObject(report.container); + const runtimeInstances = Array.isArray(container?.runtime_instances) + ? container.runtime_instances.map(parseRuntimeInstance).filter((instance) => instance !== null) + : undefined; + return { + compile_fingerprint: report.compile_fingerprint, + container: runtimeInstances ? { runtime_instances: runtimeInstances } : undefined, + }; +}; + +const args = parseArgs(process.argv.slice(2)); +const declarationBytes = await readFile(path.resolve(args.declaration)); +const declaration = parseDeclaration(JSON.parse(declarationBytes.toString("utf8")) as unknown); +const report = parseCompileReport(JSON.parse(await readFile(path.resolve(args.report), "utf8")) as unknown); +const compiledAgents = new Set( + (report.container?.runtime_instances ?? []) + .filter((instance) => instance.runtime === "daimon") + .flatMap((instance) => instance.node_ids ?? []), +); +const servers = [...declaration.servers].sort((left, right) => left.id.localeCompare(right.id)); +if (servers.some(({ agent_id }) => !compiledAgents.has(agent_id))) { + throw new Error("explicit-test MCP server agent is absent from compiled Daimon instances"); +} +if (new Set(servers.map(({ id }) => id)).size !== servers.length) { + throw new Error("duplicate explicit-test MCP server id"); +} +const artifact = { + compile_fingerprint: report.compile_fingerprint, + servers, + version: "spawnfile.explicit-test-mcp.v1", +}; +const artifactBytes = Buffer.from(`${JSON.stringify(artifact)}\n`); +const receipt = { + artifact_sha256: digest(artifactBytes), + compile_fingerprint: report.compile_fingerprint, + declaration_sha256: digest(declarationBytes), + servers: servers.map(({ id, agent_id, tools }) => ({ agent_id, id, tools })), + version: "spawnfile.explicit-test-mcp-receipt.v1", +}; +await mkdir(path.resolve(args.out), { recursive: true }); +await writeFile(path.resolve(args.out, "explicit-test-mcp.json"), artifactBytes, { mode: 0o600 }); +await writeFile( + path.resolve(args.out, "explicit-test-mcp-receipt.json"), + `${JSON.stringify(receipt)}\n`, + { mode: 0o600 }, +); diff --git a/scripts/create-linux-amd64-dependency-closure.mjs b/scripts/create-linux-amd64-dependency-closure.ts similarity index 52% rename from scripts/create-linux-amd64-dependency-closure.mjs rename to scripts/create-linux-amd64-dependency-closure.ts index 3f69cca7..9755530e 100644 --- a/scripts/create-linux-amd64-dependency-closure.mjs +++ b/scripts/create-linux-amd64-dependency-closure.ts @@ -7,29 +7,47 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; const NODE_IMAGE = "node:24-bookworm-slim@sha256:a9f5f7c91a432850b2a8a7797adf5eadb6c733ceed61167806cee7ea7fbc29df"; -const fail = (message) => { throw new Error(message); }; -const absolute = (value, label) => { - if (!value || !path.isAbsolute(value) || path.resolve(value) !== value) fail(`${label} must be a normalized absolute path`); - return value; +type PackageLockPackage = { + integrity?: string; + resolved?: string; + version?: string; +}; +type PackageLock = { + lockfileVersion?: number; + packages?: Record; +}; +type CacheIndexEntry = { + integrity?: string; + key?: string; + time?: number; +}; + +const fail = (message: string): never => { throw new Error(message); }; +const absolute = (value: string | undefined, label: string): string => { + if (typeof value === "string" && value && path.isAbsolute(value) && path.resolve(value) === value) return value; + throw new Error(`${label} must be a normalized absolute path`); }; -export const validateClosureProject = (root) => { - const packageJson = JSON.parse(readFileSync(path.join(root, "package.json"), "utf8")); - const lock = JSON.parse(readFileSync(path.join(root, "package-lock.json"), "utf8")); - if (lock.lockfileVersion !== 3 || !lock.packages?.[""]) fail("dependency closure requires package-lock v3"); - const codex = lock.packages["node_modules/@openai/codex"]; - if (!codex || typeof codex.version !== "string" || !/^sha512-[A-Za-z0-9+/]+={0,2}$/u.test(codex.integrity ?? "")) fail("dependency closure requires integrity-pinned @openai/codex"); - if (!lock.packages["node_modules/typescript"]) fail("dependency closure requires locked TypeScript"); +export const validateClosureProject = (root: string): { codex_version: string } => { + const packageJson = JSON.parse(readFileSync(path.join(root, "package.json"), "utf8")) as { dependencies?: Record; devDependencies?: Record }; + const lock = JSON.parse(readFileSync(path.join(root, "package-lock.json"), "utf8")) as PackageLock; + const packages = lock.packages; + if (lock.lockfileVersion !== 3 || !packages || !packages[""]) throw new Error("dependency closure requires package-lock v3"); + const codex = packages["node_modules/@openai/codex"]; + if (!codex) throw new Error("dependency closure requires integrity-pinned @openai/codex"); + if (typeof codex.version !== "string" || !/^sha512-[A-Za-z0-9+/]+={0,2}$/u.test(codex.integrity ?? "")) throw new Error("dependency closure requires integrity-pinned @openai/codex"); + if (!packages["node_modules/typescript"]) throw new Error("dependency closure requires locked TypeScript"); const declared = { ...packageJson.dependencies, ...packageJson.devDependencies }["@openai/codex"]; - if (!declared || /^(?:\^|~|>|<|\*|latest)/u.test(declared)) fail("@openai/codex must be exactly pinned"); - return { codex_version: codex.version }; + if (!declared || /^(?:\^|~|>|<|\*|latest)/u.test(declared)) throw new Error("@openai/codex must be exactly pinned"); + const codexVersion = codex.version; + return { codex_version: codexVersion }; }; -const normalizeCacheIndex = (root) => { - const visit = (directory) => { for (const entry of readdirSync(directory, { withFileTypes: true })) { +const normalizeCacheIndex = (root: string): void => { + const visit = (directory: string): void => { for (const entry of readdirSync(directory, { withFileTypes: true })) { const target = path.join(directory, entry.name); if (entry.isDirectory()) visit(target); else if (entry.isFile()) { const lines = readFileSync(target, "utf8").trim().split("\n").filter(Boolean).map((line) => { - const tab = line.indexOf("\t"), value = JSON.parse(line.slice(tab + 1)); value.time = 0; + const tab = line.indexOf("\t"), value = JSON.parse(line.slice(tab + 1)) as CacheIndexEntry; value.time = 0; const json = JSON.stringify(value); return `${createHash("sha1").update(json).digest("hex")}\t${json}`; }); writeFileSync(target, `${lines.join("\n")}\n`); } @@ -37,43 +55,45 @@ const normalizeCacheIndex = (root) => { visit(root); }; -const hydrateLockIntegrities = (root) => { - const byUrl = new Map(); const visit = (directory) => { for (const entry of readdirSync(directory, { withFileTypes: true })) { +const hydrateLockIntegrities = (root: string): void => { + const byUrl = new Map(); const visit = (directory: string): void => { for (const entry of readdirSync(directory, { withFileTypes: true })) { const target = path.join(directory, entry.name); if (entry.isDirectory()) visit(target); else if (entry.isFile()) for (const line of readFileSync(target, "utf8").trim().split("\n").filter(Boolean)) { - const value = JSON.parse(line.slice(line.indexOf("\t") + 1)); if (typeof value.key === "string" && typeof value.integrity === "string") byUrl.set(value.key.replace(/^make-fetch-happen:request-cache:/u, ""), value.integrity); + const value = JSON.parse(line.slice(line.indexOf("\t") + 1)) as CacheIndexEntry; if (typeof value.key === "string" && typeof value.integrity === "string") byUrl.set(value.key.replace(/^make-fetch-happen:request-cache:/u, ""), value.integrity); } } }; visit(path.join(root, "npm-cache", "_cacache", "index-v5")); - const lockPath = path.join(root, "package-lock.json"), lock = JSON.parse(readFileSync(lockPath, "utf8")); + const lockPath = path.join(root, "package-lock.json"), lock = JSON.parse(readFileSync(lockPath, "utf8")) as PackageLock; for (const [key, entry] of Object.entries(lock.packages ?? {})) if (key.startsWith("node_modules/") && !entry.integrity) { - const integrity = byUrl.get(entry.resolved); if (!integrity) fail(`npm cache lacks immutable content identity for ${key.slice(13)}`); entry.integrity = integrity; + const resolved = entry.resolved; + if (typeof resolved !== "string") throw new Error(`npm cache lacks immutable content identity for ${key.slice(13)}`); + const integrity = byUrl.get(resolved); if (!integrity) throw new Error(`npm cache lacks immutable content identity for ${key.slice(13)}`); entry.integrity = integrity; } writeFileSync(lockPath, `${JSON.stringify(lock, null, 2)}\n`); }; -export const createLinuxAmd64Closure = (input, output, codexVersion) => { - const source = absolute(input, "input"); absolute(output, "output"); +export const createLinuxAmd64Closure = (input: string | undefined, output: string | undefined, codexVersion: string | undefined): { codex_version: string; image: string; target: "linux/amd64" } => { + const source = absolute(input, "input"); const target = absolute(output, "output"); if (!lstatSync(source).isDirectory() || lstatSync(source).isSymbolicLink()) fail("input must be a real directory"); if (!/^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?$/u.test(codexVersion ?? "")) fail("Codex version must be an exact semver"); - if (existsSync(output) && readdirSync(output).length) fail("output must be absent or empty"); - const sourceLock = JSON.parse(readFileSync(path.join(source, "package-lock.json"), "utf8")); + if (existsSync(target) && readdirSync(target).length) fail("output must be absent or empty"); + const sourceLock = JSON.parse(readFileSync(path.join(source, "package-lock.json"), "utf8")) as PackageLock; if (sourceLock.lockfileVersion !== 3 || !sourceLock.packages?.["node_modules/typescript"]) fail("source requires a package-lock v3 graph with TypeScript"); - mkdirSync(output, { recursive: true }); const staging = mkdtempSync(path.join(os.tmpdir(), "spawnfile-closure-project-")); + mkdirSync(target, { recursive: true }); const staging = mkdtempSync(path.join(os.tmpdir(), "spawnfile-closure-project-")); copyFileSync(path.join(source, "package.json"), path.join(staging, "package.json")); copyFileSync(path.join(source, "package-lock.json"), path.join(staging, "package-lock.json")); for (const name of readdirSync(source).filter((entry) => entry.endsWith(".tgz")).sort()) copyFileSync(path.join(source, name), path.join(staging, name)); - const run = (network, inputDirectory, command, copyBack) => { + const run = (network: "none" | undefined, inputDirectory: string, command: string, copyBack = false): void => { const id = execFileSync("docker", ["create", "--platform", "linux/amd64", ...(network ? ["--network", network] : []), NODE_IMAGE, "sh", "-ceu", command], { encoding: "utf8" }).trim(); - try { execFileSync("docker", ["cp", `${inputDirectory}/.`, `${id}:/closure`]); execFileSync("docker", ["start", "--attach", id], { stdio: "inherit" }); if (copyBack) execFileSync("docker", ["cp", `${id}:/closure/.`, output]); } + try { execFileSync("docker", ["cp", `${inputDirectory}/.`, `${id}:/closure`]); execFileSync("docker", ["start", "--attach", id], { stdio: "inherit" }); if (copyBack) execFileSync("docker", ["cp", `${id}:/closure/.`, target]); } finally { execFileSync("docker", ["rm", "--force", id], { stdio: "ignore" }); } }; try { run(undefined, staging, `cd /closure; npm install --package-lock-only --ignore-scripts --save-exact @openai/codex@${codexVersion}; npm ci --ignore-scripts --cache /closure/npm-cache; npm ls --all; test \"$(node -p \"process.platform+'/'+process.arch\")\" = linux/x64; rm -rf node_modules`, true); } finally { rmSync(staging, { force: true, recursive: true }); } - hydrateLockIntegrities(output); - const identity = validateClosureProject(output); if (identity.codex_version !== codexVersion) fail("prepared Codex version disagrees with the requested pin"); - rmSync(path.join(output, "npm-cache", "_logs"), { force: true, recursive: true }); rmSync(path.join(output, "npm-cache", "_update-notifier-last-checked"), { force: true }); - normalizeCacheIndex(path.join(output, "npm-cache", "_cacache", "index-v5")); - run("none", output, "cd /closure; npm ci --offline --ignore-scripts --cache /closure/npm-cache; npm ls --all; rm -rf node_modules", true); - rmSync(path.join(output, "npm-cache", "_logs"), { force: true, recursive: true }); rmSync(path.join(output, "npm-cache", "_update-notifier-last-checked"), { force: true }); - normalizeCacheIndex(path.join(output, "npm-cache", "_cacache", "index-v5")); + hydrateLockIntegrities(target); + const identity = validateClosureProject(target); if (identity.codex_version !== codexVersion) fail("prepared Codex version disagrees with the requested pin"); + rmSync(path.join(target, "npm-cache", "_logs"), { force: true, recursive: true }); rmSync(path.join(target, "npm-cache", "_update-notifier-last-checked"), { force: true }); + normalizeCacheIndex(path.join(target, "npm-cache", "_cacache", "index-v5")); + run("none", target, "cd /closure; npm ci --offline --ignore-scripts --cache /closure/npm-cache; npm ls --all; rm -rf node_modules", true); + rmSync(path.join(target, "npm-cache", "_logs"), { force: true, recursive: true }); rmSync(path.join(target, "npm-cache", "_update-notifier-last-checked"), { force: true }); + normalizeCacheIndex(path.join(target, "npm-cache", "_cacache", "index-v5")); return { ...identity, image: NODE_IMAGE, target: "linux/amd64" }; }; diff --git a/scripts/create-linux-amd64-go-closure.mjs b/scripts/create-linux-amd64-go-closure.ts similarity index 81% rename from scripts/create-linux-amd64-go-closure.mjs rename to scripts/create-linux-amd64-go-closure.ts index fb29c796..988775e4 100644 --- a/scripts/create-linux-amd64-go-closure.mjs +++ b/scripts/create-linux-amd64-go-closure.ts @@ -6,16 +6,19 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; export const GO_IMAGE = "golang:1.24-bookworm@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac"; -const fail = (message) => { throw new Error(message); }; -const exact = (value, label) => { if (!value || !path.isAbsolute(value) || path.resolve(value) !== value) fail(`${label} must be normalized absolute`); return value; }; +const fail = (message: string): never => { throw new Error(message); }; +const exact = (value: string | undefined, label: string): string => { + if (typeof value === "string" && value && path.isAbsolute(value) && path.resolve(value) === value) return value; + throw new Error(`${label} must be normalized absolute`); +}; -const run = (input, network, copyBack) => { +const run = (input: string, network?: "none", copyBack?: string): void => { const id = execFileSync("docker", ["create", "--platform", "linux/amd64", ...(network ? ["--network", network] : []), "--env", "GOMODCACHE=/closure/gomodcache", "--env", "GOCACHE=/closure/gobuildcache", GO_IMAGE, "sh", "-ceu", "cd /closure; go mod download; go mod verify; test \"$(go env GOOS)/$(go env GOARCH)\" = linux/amd64; tar -cf /tmp/closure.tar ."], { encoding: "utf8" }).trim(); try { execFileSync("docker", ["cp", `${input}/.`, `${id}:/closure`]); execFileSync("docker", ["start", "--attach", id], { stdio: "inherit" }); if (copyBack) { const archive = path.join(copyBack, ".closure-transfer.tar"); execFileSync("docker", ["cp", `${id}:/tmp/closure.tar`, archive]); execFileSync("tar", ["-xf", archive, "-C", copyBack]); rmSync(archive); execFileSync("chmod", ["-R", "u+rwX", copyBack]); } } finally { execFileSync("docker", ["rm", "--force", id], { stdio: "ignore" }); } }; -export const createLinuxAmd64GoClosure = (sourcePath, outputPath) => { +export const createLinuxAmd64GoClosure = (sourcePath: string | undefined, outputPath: string | undefined): { image: string; target: "linux/amd64" } => { const source = exact(sourcePath, "source"), output = exact(outputPath, "output"); if (!lstatSync(source).isDirectory() || lstatSync(source).isSymbolicLink()) fail("source must be a real directory"); if (existsSync(output) && readdirSync(output).length) fail("output must be absent or empty"); mkdirSync(output, { recursive: true }); diff --git a/scripts/create-source-provenance-bundle.mjs b/scripts/create-source-provenance-bundle.ts similarity index 71% rename from scripts/create-source-provenance-bundle.mjs rename to scripts/create-source-provenance-bundle.ts index 00e66baf..47e7f7b0 100644 --- a/scripts/create-source-provenance-bundle.mjs +++ b/scripts/create-source-provenance-bundle.ts @@ -1,12 +1,13 @@ #!/usr/bin/env node import path from "node:path"; import { pathToFileURL } from "node:url"; -import { createSourceBundle } from "./source-provenance-bundle.mjs"; +import { createSourceBundle } from "./source-provenance-bundle.ts"; +import type { SourceBundleProfile } from "./source-provenance-bundle.ts"; -export const main = (argv = process.argv.slice(2)) => { +export const main = (argv = process.argv.slice(2)): void => { const profile = argv[0] === "--dependencies" ? "dependencies" : argv[0] === "--go-dependencies" ? "go-dependencies" : argv[0] === "--build-source" ? "build-source" : "source", values = profile === "source" ? argv : argv.slice(1); if (values.length !== 2 || !values.every(path.isAbsolute)) throw new Error("usage: create-source-provenance-bundle [--build-source|--dependencies|--go-dependencies] "); - const receipt = createSourceBundle(values[0], values[1], profile); + const receipt = createSourceBundle(values[0] ?? "", values[1] ?? "", profile as SourceBundleProfile); process.stdout.write(`${JSON.stringify({ ...receipt, manifest: undefined, source_archive: values[1], version: "spawnfile.source-provenance-bundle-receipt.v1" })}\n`); }; diff --git a/scripts/moltnet-source-provenance.integration.test.mjs b/scripts/moltnet-source-provenance.integration.test.ts similarity index 100% rename from scripts/moltnet-source-provenance.integration.test.mjs rename to scripts/moltnet-source-provenance.integration.test.ts diff --git a/scripts/native-helper-artifacts.mjs b/scripts/native-helper-artifacts.mjs deleted file mode 100644 index 5e19d89f..00000000 --- a/scripts/native-helper-artifacts.mjs +++ /dev/null @@ -1,16 +0,0 @@ -import { createHash } from "node:crypto"; -import { lstat, readFile } from "node:fs/promises"; -import path from "node:path"; - -const expectedMachine = { x64: 62, arm64: 183 }; -export const verifyNativeHelperArtifacts = async (root) => { - for (const architecture of Object.keys(expectedMachine)) { - const binaryPath = path.join(root, `rename-noreplace-${architecture}`); const provenancePath = `${binaryPath}.provenance.json`; - let binary; let metadata; - try { [binary, metadata] = await Promise.all([readFile(binaryPath), lstat(binaryPath)]); } catch { throw new Error(`Missing Linux ${architecture} rename-noreplace helper`); } - if (!metadata.isFile() || metadata.isSymbolicLink() || !(metadata.mode & 0o111) || binary.subarray(0, 4).toString("hex") !== "7f454c46" || binary.readUInt16LE(18) !== expectedMachine[architecture]) throw new Error(`Wrong-architecture or unsafe Linux ${architecture} rename-noreplace helper`); - let provenance; try { provenance = JSON.parse(await readFile(provenancePath, "utf8")); } catch { throw new Error(`Missing Linux ${architecture} rename-noreplace provenance`); } - const digest = `sha256:${createHash("sha256").update(binary).digest("hex")}`; - if (provenance?.version !== "spawnfile.rename-noreplace-build.v1" || provenance.architecture !== architecture || provenance.target !== `linux/${architecture === "x64" ? "amd64" : "arm64"}` || provenance.binary_sha256 !== digest || provenance.builder_image !== "gcc:14.2.0@sha256:b99b86a28812b1e6453a231a947dc43d76fe192788a12f344a9b568bf9f5d24c" || provenance.compiler !== "gcc:14.2.0" || !/^sha256:[a-f0-9]{64}$/u.test(provenance.source_sha256)) throw new Error(`Invalid Linux ${architecture} rename-noreplace provenance`); - } -}; diff --git a/scripts/native-helper-artifacts.test.mjs b/scripts/native-helper-artifacts.test.ts similarity index 98% rename from scripts/native-helper-artifacts.test.mjs rename to scripts/native-helper-artifacts.test.ts index 1581bdef..c833f395 100644 --- a/scripts/native-helper-artifacts.test.mjs +++ b/scripts/native-helper-artifacts.test.ts @@ -4,7 +4,7 @@ import os from "node:os"; import path from "node:path"; import test from "node:test"; -import { verifyNativeHelperArtifacts } from "./native-helper-artifacts.mjs"; +import { verifyNativeHelperArtifacts } from "./native-helper-artifacts.ts"; test("native helper closure rejects missing and wrong-architecture artifacts", async () => { await verifyNativeHelperArtifacts(path.resolve("src/deployment/native/artifacts")); diff --git a/scripts/native-helper-artifacts.ts b/scripts/native-helper-artifacts.ts new file mode 100644 index 00000000..b8663688 --- /dev/null +++ b/scripts/native-helper-artifacts.ts @@ -0,0 +1,71 @@ +import { createHash } from "node:crypto"; +import { lstat, readFile } from "node:fs/promises"; +import path from "node:path"; + +const expectedMachine = { x64: 62, arm64: 183 } as const; +type NativeArchitecture = keyof typeof expectedMachine; + +type NativeHelperProvenance = { + architecture: NativeArchitecture; + binary_sha256: string; + builder_image: string; + compiler: string; + source_sha256: string; + target: string; + version: "spawnfile.rename-noreplace-build.v1"; +}; + +const nativeArchitectures = Object.keys(expectedMachine) as NativeArchitecture[]; + +const isNativeHelperProvenance = ( + value: unknown, +): value is NativeHelperProvenance => ( + value !== null + && typeof value === "object" + && !Array.isArray(value) + && typeof (value as { architecture?: unknown }).architecture === "string" + && typeof (value as { binary_sha256?: unknown }).binary_sha256 === "string" + && typeof (value as { builder_image?: unknown }).builder_image === "string" + && typeof (value as { compiler?: unknown }).compiler === "string" + && typeof (value as { source_sha256?: unknown }).source_sha256 === "string" + && typeof (value as { target?: unknown }).target === "string" + && (value as { version?: unknown }).version === "spawnfile.rename-noreplace-build.v1" +); + +export const verifyNativeHelperArtifacts = async (root: string): Promise => { + for (const architecture of nativeArchitectures) { + const binaryPath = path.join(root, `rename-noreplace-${architecture}`); + const provenancePath = `${binaryPath}.provenance.json`; + let binary: Buffer; + let metadata: Awaited>; + try { + [binary, metadata] = await Promise.all([readFile(binaryPath), lstat(binaryPath)]); + } catch { + throw new Error(`Missing Linux ${architecture} rename-noreplace helper`); + } + if (!metadata.isFile() + || metadata.isSymbolicLink() + || !(metadata.mode & 0o111) + || binary.subarray(0, 4).toString("hex") !== "7f454c46" + || binary.readUInt16LE(18) !== expectedMachine[architecture]) { + throw new Error(`Wrong-architecture or unsafe Linux ${architecture} rename-noreplace helper`); + } + let provenance: unknown; + try { + provenance = JSON.parse(await readFile(provenancePath, "utf8")) as unknown; + } catch { + throw new Error(`Missing Linux ${architecture} rename-noreplace provenance`); + } + const digest = `sha256:${createHash("sha256").update(binary).digest("hex")}`; + if (!isNativeHelperProvenance(provenance) + || provenance.architecture !== architecture + || provenance.target !== `linux/${architecture === "x64" ? "amd64" : "arm64"}` + || provenance.binary_sha256 !== digest + || provenance.builder_image + !== "gcc:14.2.0@sha256:b99b86a28812b1e6453a231a947dc43d76fe192788a12f344a9b568bf9f5d24c" + || provenance.compiler !== "gcc:14.2.0" + || !/^sha256:[a-f0-9]{64}$/u.test(provenance.source_sha256)) { + throw new Error(`Invalid Linux ${architecture} rename-noreplace provenance`); + } + } +}; diff --git a/scripts/native-helper-integration.test.mjs b/scripts/native-helper-integration.test.ts similarity index 74% rename from scripts/native-helper-integration.test.mjs rename to scripts/native-helper-integration.test.ts index f96d3f29..191602e2 100644 --- a/scripts/native-helper-integration.test.mjs +++ b/scripts/native-helper-integration.test.ts @@ -6,9 +6,16 @@ import { promisify } from "node:util"; const execFile = promisify(execFileCallback); const helpers = path.resolve("dist/deployment/native"); -const docker = async (args) => await execFile("docker", args); +const docker = async (args: string[]) => await execFile("docker", args); -for (const architecture of ["x64", "arm64"]) test(`real Linux ${architecture} helper atomically activates and preserves EEXIST destinations`, async () => { +const isExecErrorWithStdout = (error: unknown, stdout: string): boolean => ( + error !== null + && typeof error === "object" + && "stdout" in error + && error.stdout === stdout +); + +for (const architecture of ["x64", "arm64"] as const) test(`real Linux ${architecture} helper atomically activates and preserves EEXIST destinations`, async () => { const platform = architecture === "x64" ? "amd64" : "arm64"; const container = `spawnfile-helper-test-${architecture}-${process.pid}`; try { await docker(["create", "--name", container, "--platform", `linux/${platform}`, "alpine:3.22", "sleep", "300"]); await docker(["start", container]); @@ -20,7 +27,10 @@ for (const architecture of ["x64", "arm64"]) test(`real Linux ${architecture} he const suffix = nonempty ? "nonempty" : "empty"; await docker(["exec", container, "mkdir", `/work/source-${suffix}`, `/work/destination-${suffix}`]); if (nonempty) await docker(["exec", container, "sh", "-c", `printf 'keep\\n' > /work/destination-${suffix}/keep`]); const inode = (await docker(["exec", container, "stat", "-c", "%i", `/work/destination-${suffix}`])).stdout; - await assert.rejects(docker(["exec", container, "/rename-noreplace", "/work", `source-${suffix}`, `destination-${suffix}`]), (error) => error.stdout === '{"ok":false,"error":"EEXIST","errno":17}\n'); + await assert.rejects( + docker(["exec", container, "/rename-noreplace", "/work", `source-${suffix}`, `destination-${suffix}`]), + (error) => isExecErrorWithStdout(error, '{"ok":false,"error":"EEXIST","errno":17}\n'), + ); assert.equal((await docker(["exec", container, "stat", "-c", "%i", `/work/destination-${suffix}`])).stdout, inode); if (nonempty) assert.equal((await docker(["exec", container, "cat", `/work/destination-${suffix}/keep`])).stdout, "keep\n"); } diff --git a/scripts/native-helper-workflows.test.mjs b/scripts/native-helper-workflows.test.ts similarity index 77% rename from scripts/native-helper-workflows.test.mjs rename to scripts/native-helper-workflows.test.ts index 0a774a12..dc6e5dd8 100644 --- a/scripts/native-helper-workflows.test.mjs +++ b/scripts/native-helper-workflows.test.ts @@ -2,7 +2,7 @@ import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; import test from "node:test"; -const gate = "node --test scripts/native-helper-artifacts.test.mjs scripts/native-helper-integration.test.mjs"; +const gate = "node --experimental-strip-types --test scripts/native-helper-artifacts.test.ts scripts/native-helper-integration.test.ts"; test("PR/main and publish workflows explicitly configure QEMU and run native syscall gates", async () => { for (const workflow of [".github/workflows/test.yml", ".github/workflows/publish.yml"]) { @@ -18,7 +18,7 @@ test("native helper build uses only the pinned compiler image", async () => { test("normal package build copies shipped helpers without invoking the native rebuild", async () => { const packageJson = JSON.parse(await readFile("package.json", "utf8")); - assert.match(packageJson.scripts.build, /copyArtifacts\.mjs/u); - assert.doesNotMatch(packageJson.scripts.build, /native\/build\.mjs|docker/u); - assert.equal(packageJson.scripts["build:native"], "node ./src/deployment/native/build.mjs"); + assert.match(packageJson.scripts.build, /copyArtifacts\.ts/u); + assert.doesNotMatch(packageJson.scripts.build, /native\/build\.ts|docker/u); + assert.equal(packageJson.scripts["build:native"], "node --experimental-strip-types ./src/deployment/native/build.ts"); }); diff --git a/scripts/product-state-volume-integration.test.mjs b/scripts/product-state-volume-integration.test.ts similarity index 98% rename from scripts/product-state-volume-integration.test.mjs rename to scripts/product-state-volume-integration.test.ts index 277d1a8e..543570d7 100644 --- a/scripts/product-state-volume-integration.test.mjs +++ b/scripts/product-state-volume-integration.test.ts @@ -9,7 +9,7 @@ const volume = `spawnfile-product-preseed-${process.pid}`; const rollbackVolume = `${volume}-rollback`; const cleanupVolume = `${volume}-cleanup`; const container = `spawnfile-product-preseed-${process.pid}`; -const run = (args) => execFileSync("docker", args, { encoding: "utf8", stdio: "pipe" }); +const run = (args: string[]): string => execFileSync("docker", args, { encoding: "utf8", stdio: "pipe" }); const runCliAuthorityClone = () => { if (process.platform !== "linux") return; const sourceVolume = `${volume}-authority-source`, candidateVolume = `${volume}-authority-candidate`, sourceContainer = `${container}-authority`, authority = path.join(source, "authority.json"), proof = path.join(source, "proof.json"), authorityRequest = path.join(source, "authority-request.json"), cloneRequest = path.join(source, "clone-request.json"), cloneReceipt = path.join(source, "clone-receipt.json"); diff --git a/scripts/source-provenance-bundle.integration.test.mjs b/scripts/source-provenance-bundle.integration.test.ts similarity index 93% rename from scripts/source-provenance-bundle.integration.test.mjs rename to scripts/source-provenance-bundle.integration.test.ts index 2cb25b83..0cf3c5e4 100644 --- a/scripts/source-provenance-bundle.integration.test.mjs +++ b/scripts/source-provenance-bundle.integration.test.ts @@ -5,17 +5,17 @@ import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, write import path from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; -import { createSourceBundle, validateSourceBundle } from "./source-provenance-bundle.mjs"; +import { createSourceBundle, validateSourceBundle } from "./source-provenance-bundle.ts"; import { renderRuntimeLinkMaterializer } from "../dist/compiler/containerRuntimeLinkMaterializer.js"; const repository = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -const digest = (file) => `sha256:${execFileSync("shasum", ["-a", "256", file], { encoding: "utf8" }).split(" ")[0]}`; -const sha512 = (file) => `sha512:${createHash("sha512").update(readFileSync(file)).digest("hex")}`; +const digest = (file: string): string => `sha256:${execFileSync("shasum", ["-a", "256", file], { encoding: "utf8" }).split(" ")[0] ?? ""}`; +const sha512 = (file: string): string => `sha512:${createHash("sha512").update(readFileSync(file)).digest("hex")}`; test("actual Daimon lock produces a real offline linux/amd64 shipped artifact and rejects tampering", { timeout: 360_000 }, () => { execFileSync("docker", ["version"], { stdio: "ignore" }); const temporary = mkdtempSync(path.join(repository, ".spawnfile-source-docker-")); - let registry; + let registry: string | undefined; try { const closure = path.join(temporary, "closure"), // CI has no sibling checkout, so it points this at a fetched fixture the @@ -53,13 +53,17 @@ test("actual Daimon lock produces a real offline linux/amd64 shipped artifact an writeFileSync(grok, "#!/bin/sh\nexit 0\n", { mode: 0o755 }); mkdirSync(agyTree); writeFileSync(agy, "#!/bin/sh\nexit 0\n", { mode: 0o755 }); execFileSync("tar", ["-czf", agyTar, "-C", agyTree, "antigravity"]); const runtimeArchive = digest(path.join(output, "runtime-dependencies.tar")); - const sourceInputs = { dependencies: { archive_sha256: dependencyReceipt.archive_sha256, manifest_sha256: dependencyReceipt.manifest_sha256, package_lock_sha256: dependencyReceipt.manifest.dependency_lock.package_lock_sha256, runtime_archive_sha256: runtimeArchive }, mode: "source-bundle", source: { archive_sha256: sourceReceipt.archive_sha256, manifest_sha256: sourceReceipt.manifest_sha256 }, version: "spawnfile.daimon-source-inputs.v1" }; + const dependencyLock = dependencyReceipt.manifest.dependency_lock; + assert.ok(dependencyLock && "package_lock_sha256" in dependencyLock); + const sourceInputs = { dependencies: { archive_sha256: dependencyReceipt.archive_sha256, manifest_sha256: dependencyReceipt.manifest_sha256, package_lock_sha256: dependencyLock.package_lock_sha256, runtime_archive_sha256: runtimeArchive }, mode: "source-bundle", source: { archive_sha256: sourceReceipt.archive_sha256, manifest_sha256: sourceReceipt.manifest_sha256 }, version: "spawnfile.daimon-source-inputs.v1" }; writeFileSync(path.join(packageContext, "source-inputs.json"), `${JSON.stringify(sourceInputs)}\n`); const manifestBytes = execFileSync("tar", ["-xOf", path.join(output, "daimon.tgz"), "package/dist/runtime/contract-manifest.json"]), manifestSha = `sha256:${createHash("sha256").update(manifestBytes).digest("hex")}`; const installed = path.join(temporary, "installed"); mkdirSync(installed); execFileSync("tar", ["-xf", path.join(output, "runtime-dependencies.tar"), "-C", installed]); const codexSha = digest(path.join(installed, "@openai", "codex", "bin", "codex.js")), grokSha = digest(grok), agySha = digest(agy), packageSha = digest(path.join(output, "daimon.tgz")); registry = execFileSync("docker", ["run", "--detach", "--publish", "127.0.0.1::5000", "registry:2"], { encoding: "utf8" }).trim(); - const mapped = execFileSync("docker", ["port", registry, "5000/tcp"], { encoding: "utf8" }).trim().split("\n")[0], port = mapped.slice(mapped.lastIndexOf(":") + 1); + const mapped = execFileSync("docker", ["port", registry, "5000/tcp"], { encoding: "utf8" }).trim().split("\n")[0]; + assert.ok(mapped); + const port = mapped.slice(mapped.lastIndexOf(":") + 1); const identityPath = path.join(repository, ".local-daimon-runtime-identity.json"), priorIdentity = existsSync(identityPath) ? readFileSync(identityPath) : null; try { execFileSync("npm", ["run", "--silent", "build:local-daimon"], { cwd: repository, env: { ...process.env, AGY_CLI_SHA256: agySha, AGY_CLI_SHA512: sha512(agyTar), AGY_CLI_URL: "https://invalid.example/agy", AGY_CLI_VERSION: "fixture", CODEX_CLI_SHA256: codexSha, GROK_CLI_SHA256: grokSha, GROK_CLI_URL: "https://invalid.example/grok", GROK_CLI_VERSION: "fixture", SPAWNFILE_AGY_CLI_ARCHIVE: agyTar, SPAWNFILE_DAIMON_DEPENDENCY_BUNDLE: dependencyTar, SPAWNFILE_DAIMON_LOCAL_IMAGE_TAG: `127.0.0.1:${port}/noopolis/spawnfile-runtime-daimon:archive-wrapper`, SPAWNFILE_DAIMON_SOURCE_BUNDLE: sourceTar, SPAWNFILE_GROK_CLI_FILE: grok }, stdio: "ignore" }); @@ -123,7 +127,7 @@ test("actual Daimon lock produces a real offline linux/amd64 shipped artifact an mkdirSync(badRoot);execFileSync("tar",["-xf",sourceTar,"-C",badRoot]);rmSync(path.join(badRoot,".spawnfile-source-manifest.json"),{force:true});const artifact=path.join(badRoot,"src","runtime","native","artifacts","daimon-engine-broker-x64");if(fault==="missing")rmSync(artifact);else writeFileSync(artifact,"wrong-native-artifact\n",{mode:0o755}); const badReceipt=createSourceBundle(badRoot,badTar);mkdirSync(badContext);mkdirSync(badOutput);cpSync(badTar,path.join(badContext,"source.tar"));const badArgs=args.map((value)=>value===`source_bundle=${sourceContext}`?`source_bundle=${badContext}`:value===`SOURCE_ARCHIVE_SHA256=${sourceReceipt.archive_sha256}`?`SOURCE_ARCHIVE_SHA256=${badReceipt.archive_sha256}`:value===`SOURCE_MANIFEST_SHA256=${sourceReceipt.manifest_sha256}`?`SOURCE_MANIFEST_SHA256=${badReceipt.manifest_sha256}`:value===`type=local,dest=${output}`?`type=local,dest=${badOutput}`:value);assert.notEqual(spawnSync("docker",badArgs,{stdio:"ignore"}).status,0); } - const tampered = readFileSync(path.join(dependencyContext, "dependencies.tar")); tampered[tampered.length - 1025] ^= 1; writeFileSync(path.join(dependencyContext, "dependencies.tar"), tampered); + const tampered = readFileSync(path.join(dependencyContext, "dependencies.tar")); const tamperOffset = tampered.length - 1025; tampered[tamperOffset] = (tampered[tamperOffset] ?? 0) ^ 1; writeFileSync(path.join(dependencyContext, "dependencies.tar"), tampered); assert.notEqual(spawnSync("docker", args, { stdio: "ignore" }).status, 0); } finally { if (registry) spawnSync("docker", ["rm", "--force", registry], { stdio: "ignore" }); rmSync(temporary, { force: true, recursive: true }); } }); diff --git a/scripts/source-provenance-bundle.mjs b/scripts/source-provenance-bundle.ts similarity index 67% rename from scripts/source-provenance-bundle.mjs rename to scripts/source-provenance-bundle.ts index 623627bf..ba47625c 100644 --- a/scripts/source-provenance-bundle.mjs +++ b/scripts/source-provenance-bundle.ts @@ -2,7 +2,68 @@ import { createHash } from "node:crypto"; import { lstatSync, readFileSync, readdirSync, readlinkSync, realpathSync, writeFileSync } from "node:fs"; import path from "node:path"; -const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex"); +export type SourceBundleProfile = "source" | "build-source" | "dependencies" | "go-dependencies"; + +export type SourceManifestEntry = + | { mode: number; path: string; sha256: string; size: number; type: "file" } + | { mode: number; path: string; type: "directory" } + | { link: string; mode: number; path: string; type: "symlink" }; + +export type NodeDependencyLock = { + package_lock_sha256: string; + packages: { integrity: string; path: string; version: string }[]; + required: string[]; + target: "linux/amd64"; +}; + +export type GoDependencyLock = { + go_mod_sha256: string; + go_sum_sha256: string; + required: string[]; + target: "linux/amd64"; +}; + +export type SourceManifest = { + dependency_lock?: GoDependencyLock | NodeDependencyLock; + entries: SourceManifestEntry[]; + exclude_policy: { + credential_content: string; + credential_directories: string[]; + credential_files: string; + editor_backups: true; + names: string[]; + profile: SourceBundleProfile; + secret_names: string; + }; + root: "."; + version: "spawnfile.source-input-manifest.v1"; +}; + +export type SourceBundleReceipt = { + archive_sha256: string; + manifest: SourceManifest; + manifest_sha256: string; +}; + +type PackageLockEntry = { + integrity?: unknown; + resolved?: unknown; + version?: unknown; +}; + +type PackageLock = { + lockfileVersion?: unknown; + packages?: Record; +}; + +type TarEntry = { + content: Buffer; + link: string; + type: "0" | "2" | "5"; +}; + +const sourceBundleProfiles = ["source", "build-source", "dependencies", "go-dependencies"] as const satisfies readonly SourceBundleProfile[]; +const sha256 = (bytes: Buffer | string) => createHash("sha256").update(bytes).digest("hex"); const sourceExcludedNames = new Set([".git", ".hg", ".svn", "node_modules", "dist", "dist-test-runtime", "coverage", ".cache", ".npm", ".runtime", ".spawn", ".spawn-dev"]); const buildSourceExcludedNames = new Set([...sourceExcludedNames].filter((name) => name !== "dist")); const dependencyExcludedNames = new Set([".git", ".hg", ".svn", ".cache", ".npm"]); @@ -11,24 +72,41 @@ const secretName = new RegExp(String.raw`^(?:\.env(?:\..+)?|.*(?:credential|cred const credentialFile = new RegExp(String.raw`^(?:\.npmrc|\.netrc|\.yarnrc(?:\.yml)?|(?:auth|cookies?|keyrings?|sessions?)(?:\.${credentialStoreExtension})?|id_(?:rsa|dsa|ecdsa|ed25519)(?:\..*)?|.*\.(?:pem|key))$`, "iu"); const credentialDirectory = new Set([".aws", ".codex", ".config", ".docker", ".gcloud", ".grok", ".ssh", "cookie", "cookies", "credential", "credentials", "gcloud", "keyring", "keyrings", "secrets", "sessions", "tokens"]); const credentialContent = /(?:-----BEGIN ((?:RSA |DSA |EC |OPENSSH )?PRIVATE KEY)-----\s+[A-Za-z0-9+/=\r\n]{80,}\s+-----END \1-----|AKIA(?!IOSFODNN7EXAMPLE)[0-9A-Z]{16}|ASIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35}|gh[pousr]_[0-9A-Za-z]{30,255}|github_pat_[0-9A-Za-z_]{40,255}|xox[baprs]-[0-9A-Za-z-]{20,255}|_authToken\s*[=:]\s*[0-9A-Za-z._~+\/-]{16,}|authorization\s*[=:]\s*["']Bearer\s+(?!should-not-survive)[0-9A-Za-z._~+\/-]{16,})/u; -const safeRelative = (value) => value && !path.isAbsolute(value) && !value.includes("\\") && value.split("/").every((part) => part && part !== "." && part !== ".."); +const safeRelative = (value: string | undefined): value is string => { + if (!value) return false; + return !path.isAbsolute(value) && !value.includes("\\") && value.split("/").every((part) => part && part !== "." && part !== ".."); +}; + +const isTarEntryType = (value: string): value is TarEntry["type"] => value === "0" || value === "2" || value === "5"; -const excluded = (relative, profile, isDirectory = false) => { +const parsePackageLock = (bytes: Buffer): PackageLock => { + try { + const value = JSON.parse(bytes.toString("utf8")) as unknown; + if (!value || typeof value !== "object") throw new Error("not an object"); + return value as PackageLock; + } catch { + throw new Error("Dependency closure package-lock.json is invalid JSON"); + } +}; + +const excluded = (relative: string, profile: SourceBundleProfile, isDirectory = false): boolean => { const parts = relative.split("/"); if (profile === "go-dependencies" && relative.startsWith("gomodcache/")) return false; const names = profile === "dependencies" || profile === "go-dependencies" ? dependencyExcludedNames : profile === "build-source" ? buildSourceExcludedNames : sourceExcludedNames; - return parts.some((part) => names.has(part) || credentialDirectory.has(part)) || (!isDirectory && (secretName.test(parts.at(-1)) || credentialFile.test(parts.at(-1)))) || parts.some((part) => part.endsWith("~")); + const basename = parts.at(-1) ?? ""; + return parts.some((part) => names.has(part) || credentialDirectory.has(part)) || (!isDirectory && (secretName.test(basename) || credentialFile.test(basename))) || parts.some((part) => part.endsWith("~")); }; -const assertNoCredentialContent = (relative, bytes) => { +const assertNoCredentialContent = (relative: string, bytes: Buffer): void => { if (credentialContent.test(bytes.toString("utf8"))) throw new Error(`Source input contains credential-shaped content: ${relative}`); }; -const assertSymlinkGraph = (entries) => { +const assertSymlinkGraph = (entries: SourceManifestEntry[]): void => { const byPath = new Map(entries.map((entry) => [entry.path, entry])); for (const origin of entries) if (origin.type === "symlink") { - let current = origin; const seen = new Set([origin.path]); + let current: SourceManifestEntry | undefined = origin; const seen = new Set([origin.path]); for (let depth = 0; depth < 40; depth += 1) { + if (current.type !== "symlink") break; const target = path.posix.normalize(path.posix.join(path.posix.dirname(current.path), current.link)); if (!safeRelative(target) || seen.has(target)) throw new Error(`Source symlink chain is cyclic or escapes its root: ${origin.path}`); const next = byPath.get(target); if (!next) throw new Error(`Source symlink target is not an included input: ${origin.path}`); @@ -39,11 +117,11 @@ const assertSymlinkGraph = (entries) => { } }; -export const collectSourceManifest = (root, profile = "source") => { - if (!["source", "build-source", "dependencies", "go-dependencies"].includes(profile)) throw new Error("Unknown provenance bundle profile"); +export const collectSourceManifest = (root: string, profile: SourceBundleProfile = "source"): SourceManifest => { + if (!sourceBundleProfiles.includes(profile)) throw new Error("Unknown provenance bundle profile"); const canonicalRoot = realpathSync(root); - const entries = []; - const visit = (relative) => { + const entries: SourceManifestEntry[] = []; + const visit = (relative: string): void => { const absolute = path.join(canonicalRoot, relative); for (const name of readdirSync(absolute).sort()) { const child = relative ? `${relative}/${name}` : name; @@ -57,7 +135,7 @@ export const collectSourceManifest = (root, profile = "source") => { const target = path.resolve(path.dirname(path.join(canonicalRoot, child)), link); const targetRelative = path.relative(canonicalRoot, target); if (!safeRelative(targetRelative.split(path.sep).join("/"))) throw new Error(`Source symlink escapes its root: ${child}`); - let targetItem; try { targetItem = lstatSync(target); } catch { throw new Error(`Source symlink target is not an included input: ${child}`); } + let targetItem: ReturnType; try { targetItem = lstatSync(target); } catch { throw new Error(`Source symlink target is not an included input: ${child}`); } if (excluded(targetRelative, profile, targetItem.isDirectory())) throw new Error(`Source symlink target is not an included input: ${child}`); entries.push({ path: child, link, mode: item.mode & 0o777, type: "symlink" }); continue; @@ -72,17 +150,17 @@ export const collectSourceManifest = (root, profile = "source") => { if (!entries.length) throw new Error("Source bundle is empty"); assertSymlinkGraph(entries); const names = profile === "dependencies" || profile === "go-dependencies" ? dependencyExcludedNames : profile === "build-source" ? buildSourceExcludedNames : sourceExcludedNames; - let dependency_lock; + let dependency_lock: GoDependencyLock | NodeDependencyLock | undefined; if (profile === "dependencies") { const lockPath = path.join(canonicalRoot, "package-lock.json"), lock = readFileSync(lockPath); assertNoCredentialContent("package-lock.json", lock); - let parsedLock; try { parsedLock = JSON.parse(lock.toString("utf8")); } catch { throw new Error("Dependency closure package-lock.json is invalid JSON"); } + const parsedLock = parsePackageLock(lock); if (parsedLock.lockfileVersion !== 3 || !parsedLock.packages || typeof parsedLock.packages !== "object") throw new Error("Dependency closure requires package-lock v3 package graph truth"); const required = ["npm-cache", "package-lock.json", "package.json"]; const included = new Set(entries.map((entry) => entry.path)); if (required.some((entry) => !included.has(entry))) throw new Error("Dependency bundle lacks the required lock-backed amd64 build/runtime closure"); const packages = Object.entries(parsedLock.packages).filter(([key]) => key.startsWith("node_modules/")).map(([key, lockEntry]) => { - if (typeof lockEntry.version !== "string" || !/^sha512-[A-Za-z0-9+/]+={0,2}$/u.test(lockEntry.integrity ?? "")) throw new Error(`Package-lock dependency lacks immutable version/integrity: ${key.slice(13)}`); + if (typeof lockEntry.version !== "string" || typeof lockEntry.integrity !== "string" || !/^sha512-[A-Za-z0-9+/]+={0,2}$/u.test(lockEntry.integrity)) throw new Error(`Package-lock dependency lacks immutable version/integrity: ${key.slice(13)}`); return { integrity: lockEntry.integrity, path: key.slice(13), version: lockEntry.version }; }).sort((left, right) => left.path.localeCompare(right.path)); if (!["@openai/codex", "typescript"].every((name) => packages.some((entry) => entry.path === name))) throw new Error("Dependency lock lacks pinned Codex or TypeScript"); @@ -96,21 +174,21 @@ export const collectSourceManifest = (root, profile = "source") => { return { entries, ...(dependency_lock ? { dependency_lock } : {}), exclude_policy: { credential_content: credentialContent.source, credential_directories: [...credentialDirectory].sort(), credential_files: credentialFile.source, names: [...names].sort(), secret_names: secretName.source, editor_backups: true, profile }, root: ".", version: "spawnfile.source-input-manifest.v1" }; }; -export const canonicalManifestBytes = (manifest) => Buffer.from(`${JSON.stringify(manifest)}\n`); -export const sourceManifestDigest = (manifest) => `sha256:${sha256(canonicalManifestBytes(manifest))}`; +export const canonicalManifestBytes = (manifest: SourceManifest): Buffer => Buffer.from(`${JSON.stringify(manifest)}\n`); +export const sourceManifestDigest = (manifest: SourceManifest): string => `sha256:${sha256(canonicalManifestBytes(manifest))}`; -export const assertManifestStable = (root, expected) => { +export const assertManifestStable = (root: string, expected: SourceManifest): void => { const actual = canonicalManifestBytes(collectSourceManifest(root)); if (!actual.equals(canonicalManifestBytes(expected))) throw new Error("Source inputs drifted while the provenance bundle was created"); }; -const octal = (value, width) => `${value.toString(8).padStart(width - 1, "0")}\0`; -const tarHeader = (name, mode, size, type, link = "") => { +const octal = (value: number, width: number): string => `${value.toString(8).padStart(width - 1, "0")}\0`; +const tarHeader = (name: string, mode: number, size: number, type: string, link = ""): Buffer => { const header = Buffer.alloc(512); - const put = (value, offset, length) => header.write(value, offset, Math.min(length, Buffer.byteLength(value)), "utf8"); + const put = (value: string, offset: number, length: number): number => header.write(value, offset, Math.min(length, Buffer.byteLength(value)), "utf8"); let basename = name, prefix = ""; if (Buffer.byteLength(name) > 100) { - const candidates = [...name.matchAll(/\//gu)].map((match) => match.index).reverse(); + const candidates = [...name.matchAll(/\//gu)].map((match) => match.index).filter((index): index is number => index !== undefined).reverse(); const split = candidates.find((index) => Buffer.byteLength(name.slice(0, index)) <= 155 && Buffer.byteLength(name.slice(index + 1)) <= 100); if (split === undefined) throw new Error(`Source bundle path exceeds the deterministic ustar bound: ${name}`); prefix = name.slice(0, split); basename = name.slice(split + 1); @@ -123,26 +201,27 @@ const tarHeader = (name, mode, size, type, link = "") => { return header; }; -const paxPathRecord = (name) => { +const paxPathRecord = (name: string): Buffer => { const body = `path=${name}\n`; let length = Buffer.byteLength(body) + 3; for (;;) { const next = Buffer.byteLength(`${length} ${body}`); if (next === length) return Buffer.from(`${length} ${body}`); length = next; } }; -export const createSourceBundle = (root, outputPath, profile = "source", hooks = {}) => { +export const createSourceBundle = (root: string, outputPath: string, profile: SourceBundleProfile = "source", hooks: { afterRead?: () => void } = {}): SourceBundleReceipt => { const relativeOutput = path.relative(realpathSync(root), path.resolve(outputPath)); if (relativeOutput && relativeOutput !== ".." && !relativeOutput.startsWith(`..${path.sep}`) && !path.isAbsolute(relativeOutput)) { throw new Error("Source bundle output must be outside its input root"); } - const manifest = collectSourceManifest(root, profile), chunks = []; - const headerName = (name) => { + const manifest = collectSourceManifest(root, profile), chunks: Buffer[] = []; + const headerName = (name: string): string => { try { tarHeader(name, 0o644, 0, "0"); return name; } catch (error) { + if (!(error instanceof Error)) throw error; if (!/path exceeds/u.test(error.message)) throw error; const pax = paxPathRecord(name), identity = sha256(Buffer.from(name)).slice(0, 32); chunks.push(tarHeader(`PaxHeaders/${identity}`, 0o644, pax.length, "x"), pax, Buffer.alloc((512 - pax.length % 512) % 512)); return `entry-${identity}`; } }; - const append = (name, bytes, mode = 0o644) => { + const append = (name: string, bytes: Buffer, mode = 0o644): void => { chunks.push(tarHeader(headerName(name), mode, bytes.length, "0"), bytes, Buffer.alloc((512 - bytes.length % 512) % 512)); }; append(".spawnfile-source-manifest.json", canonicalManifestBytes(manifest)); @@ -159,18 +238,26 @@ export const createSourceBundle = (root, outputPath, profile = "source", hooks = return { archive_sha256: `sha256:${sha256(bytes)}`, manifest, manifest_sha256: sourceManifestDigest(manifest) }; }; -const tarText = (field) => { const end = field.indexOf(0); return field.subarray(0, end < 0 ? field.length : end).toString("utf8"); }; -const tarNumber = (field) => { const text = field.toString("ascii").replace(/\0.*$/u, "").trim(); if (!/^[0-7]+$/u.test(text)) throw new Error("Source bundle has an invalid numeric field"); return Number.parseInt(text, 8); }; +const tarText = (field: Buffer): string => { const end = field.indexOf(0); return field.subarray(0, end < 0 ? field.length : end).toString("utf8"); }; +const tarNumber = (field: Buffer): number => { const text = field.toString("ascii").replace(/\0.*$/u, "").trim(); if (!/^[0-7]+$/u.test(text)) throw new Error("Source bundle has an invalid numeric field"); return Number.parseInt(text, 8); }; + +const parseManifest = (bytes: Buffer): SourceManifest => { + try { + return JSON.parse(bytes.toString("utf8")) as SourceManifest; + } catch { + throw new Error("Source bundle manifest is invalid JSON"); + } +}; -export const validateSourceBundle = (bytes) => { +export const validateSourceBundle = (bytes: Buffer): SourceBundleReceipt => { if (bytes.length < 1024 || bytes.length % 512) throw new Error("Source bundle is truncated"); - const files = new Map(); let offset = 0, pendingPath, terminated = false; + const files = new Map(); let offset = 0, pendingPath: string | undefined, terminated = false; while (offset + 512 <= bytes.length) { const header = bytes.subarray(offset, offset + 512); if (header.every((byte) => byte === 0)) { if (!bytes.subarray(offset).every((byte) => byte === 0)) throw new Error("Source bundle has trailing data"); terminated = true; break; } if (tarText(header.subarray(257, 263)) !== "ustar") throw new Error("Source bundle is not strict ustar"); const expected = tarNumber(header.subarray(148, 156)); let sum = 0; - for (let index = 0; index < 512; index += 1) sum += index >= 148 && index < 156 ? 32 : header[index]; + for (let index = 0; index < 512; index += 1) sum += index >= 148 && index < 156 ? 32 : header[index] ?? 0; if (sum !== expected) throw new Error("Source bundle checksum mismatch"); const basename = tarText(header.subarray(0, 100)), prefix = tarText(header.subarray(345, 500)); const headerPath = prefix ? `${prefix}/${basename}` : basename, size = tarNumber(header.subarray(124, 136)), type = String.fromCharCode(header[156] || 48); @@ -182,19 +269,19 @@ export const validateSourceBundle = (bytes) => { pendingPath = match[2]; offset += 512 + Math.ceil(size / 512) * 512; continue; } const rawName = pendingPath ?? headerPath; pendingPath = undefined; const name = type === "5" && rawName.endsWith("/") ? rawName.slice(0, -1) : rawName; - if (!safeRelative(name) || files.has(name) || !["0", "2", "5"].includes(type) || (type !== "0" && size !== 0)) throw new Error("Source bundle contains an unsafe entry"); + if (!safeRelative(name) || files.has(name) || !isTarEntryType(type) || (type !== "0" && size !== 0)) throw new Error("Source bundle contains an unsafe entry"); files.set(name, { content, link: tarText(header.subarray(157, 257)), type }); offset += 512 + Math.ceil(size / 512) * 512; } if (!terminated || pendingPath) throw new Error("Source bundle lacks exact termination"); const manifestFile = files.get(".spawnfile-source-manifest.json"); if (!manifestFile || manifestFile.type !== "0") throw new Error("Source bundle lacks its input manifest"); - let manifest; try { manifest = JSON.parse(manifestFile.content.toString("utf8")); } catch { throw new Error("Source bundle manifest is invalid JSON"); } + const manifest = parseManifest(manifestFile.content); const profile = manifest.exclude_policy?.profile, names = profile === "dependencies" || profile === "go-dependencies" ? dependencyExcludedNames : profile === "build-source" ? buildSourceExcludedNames : sourceExcludedNames; - const dependencyLockValid = profile !== "dependencies" || (manifest.dependency_lock?.target === "linux/amd64" && /^sha256:[a-f0-9]{64}$/u.test(manifest.dependency_lock?.package_lock_sha256) && - Array.isArray(manifest.dependency_lock?.packages) && manifest.dependency_lock.packages.every((entry) => /^sha512-[A-Za-z0-9+/]+={0,2}$/u.test(entry.integrity) && typeof entry.path === "string" && typeof entry.version === "string") && + const dependencyLockValid = profile !== "dependencies" || (manifest.dependency_lock?.target === "linux/amd64" && "package_lock_sha256" in manifest.dependency_lock && /^sha256:[a-f0-9]{64}$/u.test(manifest.dependency_lock.package_lock_sha256) && + "packages" in manifest.dependency_lock && Array.isArray(manifest.dependency_lock.packages) && manifest.dependency_lock.packages.every((entry) => /^sha512-[A-Za-z0-9+/]+={0,2}$/u.test(entry.integrity) && typeof entry.path === "string" && typeof entry.version === "string") && JSON.stringify(manifest.dependency_lock?.required) === JSON.stringify(["npm-cache", "package-lock.json", "package.json"])); - const goDependencyLockValid = profile !== "go-dependencies" || (manifest.dependency_lock?.target === "linux/amd64" && /^sha256:[a-f0-9]{64}$/u.test(manifest.dependency_lock?.go_mod_sha256) && /^sha256:[a-f0-9]{64}$/u.test(manifest.dependency_lock?.go_sum_sha256) && JSON.stringify(manifest.dependency_lock?.required) === JSON.stringify(["go.mod", "go.sum", "gomodcache"])); - if (!["source", "build-source", "dependencies", "go-dependencies"].includes(profile) || manifest.version !== "spawnfile.source-input-manifest.v1" || !Array.isArray(manifest.entries) || manifest.root !== "." || + const goDependencyLockValid = profile !== "go-dependencies" || (manifest.dependency_lock?.target === "linux/amd64" && "go_mod_sha256" in manifest.dependency_lock && "go_sum_sha256" in manifest.dependency_lock && /^sha256:[a-f0-9]{64}$/u.test(manifest.dependency_lock.go_mod_sha256) && /^sha256:[a-f0-9]{64}$/u.test(manifest.dependency_lock.go_sum_sha256) && JSON.stringify(manifest.dependency_lock?.required) === JSON.stringify(["go.mod", "go.sum", "gomodcache"])); + if (!sourceBundleProfiles.includes(profile) || manifest.version !== "spawnfile.source-input-manifest.v1" || !Array.isArray(manifest.entries) || manifest.root !== "." || !dependencyLockValid || !goDependencyLockValid || JSON.stringify(manifest.exclude_policy) !== JSON.stringify({ credential_content: credentialContent.source, credential_directories: [...credentialDirectory].sort(), credential_files: credentialFile.source, names: [...names].sort(), secret_names: secretName.source, editor_backups: true, profile }) || diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json new file mode 100644 index 00000000..4ba8d277 --- /dev/null +++ b/scripts/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "allowImportingTsExtensions": true, + "erasableSyntaxOnly": true, + "noEmit": true, + "types": ["node"] + }, + "include": [ + "./**/*.ts", + "../src/deployment/native/build.ts", + "../src/deployment/native/copyArtifacts.ts", + "../src/evidenceExportHelper/copyAssets.ts", + "../src/runtime/copyScaffoldAssets.ts" + ], + "exclude": [] +} diff --git a/scripts/typescript-policy.test.ts b/scripts/typescript-policy.test.ts new file mode 100644 index 00000000..864cca20 --- /dev/null +++ b/scripts/typescript-policy.test.ts @@ -0,0 +1,26 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; + +const javascriptSources = (directory: string): string[] => readdirSync(directory, { withFileTypes: true }) + .flatMap((entry) => entry.isDirectory() + ? javascriptSources(path.join(directory, entry.name)) + : /\.(?:[cm]?js)$/.test(entry.name) ? [path.join(directory, entry.name)] : []); + +test("maintained scripts contain no JavaScript source files", () => { + assert.deepEqual(javascriptSources(fileURLToPath(new URL(".", import.meta.url))), []); +}); + +test("script language check detects a nested JavaScript regression", () => { + const root = mkdtempSync(path.join(os.tmpdir(), "spawnfile-script-policy-")); + try { + const filename = path.join(root, "regression.mjs"); + writeFileSync(filename, "export const value = 1;\n"); + assert.deepEqual(javascriptSources(root), [filename]); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/scripts/verify-package-closure.mjs b/scripts/verify-package-closure.mjs deleted file mode 100644 index 9fb6682c..00000000 --- a/scripts/verify-package-closure.mjs +++ /dev/null @@ -1,285 +0,0 @@ -#!/usr/bin/env node - -import { spawn } from "node:child_process"; -import { createHash } from "node:crypto"; -import { lstat, mkdir, mkdtemp, readFile, realpath, rm, stat, writeFile } from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { pathToFileURL, fileURLToPath } from "node:url"; - -import { verifyNativeHelperArtifacts } from "./native-helper-artifacts.mjs"; - -const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)); -const packageRoot = path.resolve(scriptDirectory, ".."); -const packageManifestPath = path.join(packageRoot, "package.json"); -const lockPath = path.join(packageRoot, "package-lock.json"); -const STELE = "@noopolis/stele"; -const STELE_VERSION = "0.0.2"; - -const fail = (message) => { - throw new Error(message); -}; - -const readJson = async (filePath) => JSON.parse(await readFile(filePath, "utf8")); - -const run = (command, args, cwd, env = process.env) => new Promise((resolve, reject) => { - const child = spawn(command, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); - let stdout = ""; - let stderr = ""; - child.stdout.setEncoding("utf8"); - child.stderr.setEncoding("utf8"); - child.stdout.on("data", (chunk) => { stdout += chunk; }); - child.stderr.on("data", (chunk) => { stderr += chunk; }); - child.on("error", reject); - child.on("close", (code) => { - if (code !== 0) { - reject(new Error(`${command} ${args.join(" ")} failed (${code})\n${stderr}`)); - return; - } - resolve({ stderr, stdout }); - }); -}); - -const parseSinglePack = (stdout) => { - let parsed; - for (let index = stdout.lastIndexOf("["); index >= 0; index = stdout.lastIndexOf("[", index - 1)) { - try { - const candidate = JSON.parse(stdout.slice(index)); - if (Array.isArray(candidate)) { - parsed = candidate; - break; - } - } catch { - // Lifecycle scripts may write to stdout before npm's final JSON array. - } - } - if (!Array.isArray(parsed) || parsed.length !== 1) fail("npm pack must report exactly one tarball"); - const [result] = parsed; - if (!result || typeof result.filename !== "string" || !Array.isArray(result.files)) { - fail("npm pack returned an invalid manifest"); - } - return result; -}; - -const assertSourceClosure = async (manifest, lock) => { - if (manifest.dependencies?.[STELE] !== STELE_VERSION) { - fail(`${STELE} must use the published ${STELE_VERSION} release coordinate`); - } - const bundled = manifest.bundledDependencies ?? manifest.bundleDependencies; - if (bundled !== undefined) fail("published registry dependencies must not be bundled"); - const locked = lock.packages?.[`node_modules/${STELE}`]; - if (!locked || locked.link === true || locked.version !== STELE_VERSION) { - fail(`${STELE} lock entry must be a physical, version-matched registry package`); - } - const expectedTarball = `https://registry.npmjs.org/@noopolis/stele/-/stele-${STELE_VERSION}.tgz`; - if (locked.resolved !== expectedTarball) { - fail(`${STELE} must resolve from the npm registry, got ${String(locked.resolved)}`); - } - if (typeof locked.integrity !== "string" || !locked.integrity.startsWith("sha512-")) { - fail(`${STELE} registry lock is missing sha512 integrity`); - } - const installed = path.join(packageRoot, "node_modules", STELE); - if ((await lstat(installed)).isSymbolicLink()) { - fail(`${STELE} must be physically installed before packing; source-checkout links are rejected`); - } - const installedManifest = await readJson(path.join(installed, "package.json")); - if (installedManifest.version !== STELE_VERSION) fail(`${STELE} installed version drifted`); - return locked.resolved; -}; - -const assertPackedManifest = (manifest) => { - if (typeof manifest.name !== "string" || typeof manifest.version !== "string") { - fail("packed manifest lacks an exact package identity"); - } - if (manifest.dependencies?.[STELE] !== STELE_VERSION) { - fail(`packed ${STELE} coordinate drifted from ${STELE_VERSION}`); - } - if (manifest.bundledDependencies !== undefined || manifest.bundleDependencies !== undefined) { - fail("packed manifest unexpectedly bundles registry dependencies"); - } - for (const [name, value] of Object.entries(manifest.dependencies ?? {})) { - if (typeof value === "string" && value.startsWith("file:")) { - fail(`packed runtime dependency ${name} retains a checkout-relative file coordinate`); - } - } -}; - -const dependencyRoot = async (installRoot, installedRoot) => { - const candidates = [ - path.join(installedRoot, "node_modules", STELE), - path.join(installRoot, "node_modules", STELE), - ]; - for (const candidate of candidates) { - try { - await lstat(candidate); - return candidate; - } catch (error) { - if (error?.code !== "ENOENT") throw error; - } - } - fail(`${STELE} was not installed from the packed Spawnfile dependency graph`); -}; - -const assertInstalledClosure = async (installRoot, manifest, tarballPath) => { - await writeFile(path.join(installRoot, "package.json"), "{\"private\":true}\n", "utf8"); - await run("npm", [ - "install", "--ignore-scripts", "--no-audit", "--no-fund", "--no-package-lock", - "--registry=https://registry.npmjs.org", tarballPath, - ], installRoot); - const installedRoot = path.join(installRoot, "node_modules", manifest.name); - if ((await lstat(installedRoot)).isSymbolicLink()) fail(`${manifest.name} installed as a source link`); - const installedManifest = await readJson(path.join(installedRoot, "package.json")); - if (installedManifest.name !== manifest.name || installedManifest.version !== manifest.version) { - fail("isolated install package identity drifted from the packed manifest"); - } - const helperProgram = path.join(installedRoot, "dist", "evidenceExportHelper", "helperProgram.mjs"); - const helperMetadata = await lstat(helperProgram); - if (!helperMetadata.isFile() || helperMetadata.isSymbolicLink() || helperMetadata.size < 1) { - fail("packed evidence helper asset is missing or unsafe"); - } - const helperRecipe = await import(pathToFileURL(path.join( - installedRoot, "dist", "evidenceExportHelper", "recipe.js", - )).href); - const helper = await helperRecipe.loadLocalEvidenceHelperRecipe(); - await verifyNativeHelperArtifacts(path.join(installedRoot, "dist", "deployment", "native")); - const helperSource = await readFile(helperProgram, "utf8"); - if (!helperSource.startsWith("#!/usr/local/bin/node") - || !(helper.context instanceof Uint8Array) - || !/^sha256:[a-f0-9]{64}$/u.test(helper.recipeDigest)) { - fail("packed evidence helper asset cannot be imported into its recipe"); - } - const moltnetBinaries = await import(pathToFileURL(path.join( - installedRoot, - "dist/compiler/moltnetBinaries.js", - )).href); - const expectedMoltnetExports = [ - "MOLTNET_ALLOW_LOCAL_E2E_ENV", "MOLTNET_BINARY_NAMES", "MOLTNET_BIN_DIRECTORY", - "MOLTNET_LOCAL_RELEASE_DIR_ENV", "MOLTNET_RELEASE_DIR_ENV", - "MOLTNET_RELEASE_IDENTITY_VERSION", "MOLTNET_RELEASE_STAMP_VERSION", - "resolveMoltnetCliCommand", "stageMoltnetBinaries", - ]; - const actualMoltnetExports = Object.keys(moltnetBinaries).sort(); - if (JSON.stringify(actualMoltnetExports) !== JSON.stringify(expectedMoltnetExports)) { - fail(`packed compiler Moltnet staging exports drifted: ${actualMoltnetExports.join(", ")}`); - } - const moltnetReleases = await Promise.all(["amd64", "arm64"].map(async (architecture) => { - const outputDirectory = path.join(installRoot, `moltnet-${architecture}`); - const identity = await moltnetBinaries.stageMoltnetBinaries(outputDirectory, { architecture }); - const binaryPath = path.join(outputDirectory, "moltnet-bin", "moltnet"); - const [binary, metadata] = await Promise.all([readFile(binaryPath), stat(binaryPath)]); - if (binary.subarray(0, 4).toString("hex") !== "7f454c46" || !(metadata.mode & 0o111)) { - fail(`packed compiler did not stage an executable Linux ${architecture} Moltnet binary`); - } - return identity; - })); - const steleRoot = await dependencyRoot(installRoot, installedRoot); - if ((await lstat(steleRoot)).isSymbolicLink()) fail(`${STELE} installed as a source link`); - const steleManifest = await readJson(path.join(steleRoot, "package.json")); - if (steleManifest.version !== STELE_VERSION) fail(`${STELE} installed version drifted`); - const steleImport = steleManifest.exports?.["."]?.import; - if (typeof steleImport !== "string" || !steleImport.startsWith("./")) { - fail(`${STELE} does not expose a package-relative ESM entrypoint`); - } - const installRealRoot = await realpath(installRoot); - const steleRealPath = await realpath(path.resolve(steleRoot, steleImport)); - if (!steleRealPath.startsWith(`${installRealRoot}${path.sep}`)) { - fail(`${STELE} resolved outside the isolated install`); - } - const stele = await import(pathToFileURL(steleRealPath).href); - if (typeof stele.parseCausalJsonl !== "function") fail(`${STELE} runtime import is incomplete`); - const executable = path.join(installRoot, "node_modules", ".bin", Object.keys(manifest.bin ?? {})[0]); - const executableEnvironment = { - ...process.env, - PATH: `${path.dirname(executable)}${path.delimiter}${process.env.PATH ?? ""}`, - }; - await run(executable, ["--help"], installRoot, executableEnvironment); - const capabilities = JSON.parse((await run( - executable, ["capabilities", "--json"], installRoot, executableEnvironment, - )).stdout); - if (capabilities?.version !== "spawnfile.capabilities.v1" - || capabilities?.implementation?.package !== manifest.name - || capabilities?.implementation?.version !== manifest.version - || capabilities?.capabilities?.composed_lifecycle?.complete !== true - || capabilities?.capabilities?.composed_lifecycle?.command_set_version - !== "spawnfile.composed-lifecycle-contract-set.v1") { - fail("packed CLI capability contract or package identity drifted"); - } - return { - capabilitiesVersion: capabilities.version, - commandSetVersion: capabilities.capabilities.composed_lifecycle.command_set_version, - moltnetReleases, - steleResolved: path.relative(installRealRoot, steleRealPath), - }; -}; - -const main = async () => { - const manifest = await readJson(packageManifestPath); - const lock = await readJson(lockPath); - const steleRegistryTarball = await assertSourceClosure(manifest, lock); - const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), `${manifest.name.replaceAll("/", "-")}-closure-`)); - try { - const packDirectory = path.join(temporaryRoot, "pack"); - const installRoot = path.join(temporaryRoot, "install"); - await Promise.all([ - mkdir(packDirectory, { recursive: true }), - mkdir(installRoot, { recursive: true }), - ]); - const packed = parseSinglePack((await run( - "npm", ["pack", "--json", "--pack-destination", packDirectory], packageRoot, - )).stdout); - const tarballPath = path.join(packDirectory, packed.filename); - const packedBytes = await readFile(tarballPath); - const packedIntegrity = `sha512-${createHash("sha512").update(packedBytes).digest("base64")}`; - const packedShasum = createHash("sha1").update(packedBytes).digest("hex"); - if (packed.integrity !== packedIntegrity || packed.shasum !== packedShasum) { - fail("npm pack manifest integrity does not match the inspected tarball bytes"); - } - const entries = new Set(packed.files.map((entry) => entry.path)); - if (!entries.has("dist/evidenceExportHelper/helperProgram.mjs") - || !entries.has("dist/evidenceExportHelper/recipe.js")) { - fail("packed tarball omits the evidence helper runtime assets"); - } - for (const architecture of ["x64", "arm64"]) { - if (!entries.has(`dist/deployment/native/rename-noreplace-${architecture}`) - || !entries.has(`dist/deployment/native/rename-noreplace-${architecture}.provenance.json`)) { - fail(`packed tarball omits the Linux ${architecture} rename-noreplace helper or provenance`); - } - } - if ([...entries].some((entry) => /\.test-helper\.(?:js|d\.ts)$/u.test(entry))) { - fail("packed tarball leaked a test helper"); - } - if ([...entries].some((entry) => /\/[^/]*RunOperatorInputs\.(?:js|d\.ts)$/u.test(entry))) { - fail("packed tarball leaked an unbound operator-input contract"); - } - if ((packed.bundled?.length ?? 0) !== 0) fail("npm pack unexpectedly bundled dependencies"); - if ([...entries].some((entry) => entry.startsWith("node_modules/") - || entry.includes("ecosystem/") || /vendor\/.*\.tgz$/u.test(entry))) { - fail("packed tarball leaked dependencies, a source checkout, or a vendor archive"); - } - const packedManifest = JSON.parse((await run( - "tar", ["-xOf", tarballPath, "package/package.json"], packageRoot, - )).stdout); - assertPackedManifest(packedManifest); - if (packedManifest.name !== manifest.name || packedManifest.version !== manifest.version) { - fail("packed manifest identity drifted from the source manifest"); - } - const installed = await assertInstalledClosure(installRoot, manifest, tarballPath); - process.stdout.write(`${JSON.stringify({ - bundled: packed.bundled ?? [], - capabilities_version: installed.capabilitiesVersion, - command_set_version: installed.commandSetVersion, - entries: packed.entryCount, - integrity: packed.integrity, - package: packed.id, - packed_file: packed.filename, - moltnet_releases: installed.moltnetReleases, - runtime_dependencies: packedManifest.dependencies, - stele_registry_tarball: steleRegistryTarball, - stele_resolved_inside_install: installed.steleResolved, - }, null, 2)}\n`); - } finally { - await rm(temporaryRoot, { force: true, recursive: true }); - } -}; - -await main(); diff --git a/scripts/verify-package-closure.ts b/scripts/verify-package-closure.ts new file mode 100644 index 00000000..15145242 --- /dev/null +++ b/scripts/verify-package-closure.ts @@ -0,0 +1,398 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { lstat, mkdir, mkdtemp, readFile, realpath, rm, stat, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL, fileURLToPath } from "node:url"; + +import { verifyNativeHelperArtifacts } from "./native-helper-artifacts.ts"; + +const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)); +const packageRoot = path.resolve(scriptDirectory, ".."); +const packageManifestPath = path.join(packageRoot, "package.json"); +const lockPath = path.join(packageRoot, "package-lock.json"); +const STELE = "@noopolis/stele"; +const STELE_VERSION = "0.0.2"; + +type JsonObject = Record; +type PackageManifest = { bin?: Record; bundleDependencies?: unknown; bundledDependencies?: unknown; dependencies?: Record; name: string; version: string }; +type PackageLock = { packages?: Record }; +type NpmPackResult = { bundled?: unknown[]; entryCount?: number; files: { path: string }[]; filename: string; id?: string; integrity?: string; shasum?: string }; +type RunResult = { stderr: string; stdout: string }; +type InstalledClosure = { capabilitiesVersion: string; commandSetVersion: string; moltnetReleases: unknown[]; steleResolved: string }; +type EvidenceHelperRecipe = { context: unknown; recipeDigest: unknown }; +type EvidenceHelperRecipeModule = { loadLocalEvidenceHelperRecipe: () => Promise }; +type MoltnetBinariesModule = { + stageMoltnetBinaries: (outputDirectory: string, options: { architecture: "amd64" | "arm64" }) => Promise; +} & Record; +type Capabilities = { + capabilities: { composed_lifecycle: { command_set_version: string; complete: true } }; + implementation: { package: string; version: string }; + version: "spawnfile.capabilities.v1"; +}; + +const fail = (message: string): never => { + throw new Error(message); +}; + +const isJsonObject = (value: unknown): value is JsonObject => ( + value !== null && typeof value === "object" && !Array.isArray(value) +); + +const readJson = async (filePath: string): Promise => ( + JSON.parse(await readFile(filePath, "utf8")) as unknown +); + +const isErrorWithCode = (error: unknown, code: string): boolean => ( + isJsonObject(error) && error.code === code +); + +const asEvidenceHelperRecipeModule = (value: unknown): EvidenceHelperRecipeModule => { + if (!isJsonObject(value) || typeof value.loadLocalEvidenceHelperRecipe !== "function") { + fail("packed evidence helper recipe module is incomplete"); + } + return value as EvidenceHelperRecipeModule; +}; + +const asMoltnetBinariesModule = (value: unknown): MoltnetBinariesModule => { + if (!isJsonObject(value) || typeof value.stageMoltnetBinaries !== "function") { + fail("packed compiler Moltnet staging module is incomplete"); + } + return value as MoltnetBinariesModule; +}; + +const parseCapabilities = (value: unknown): Capabilities => { + const message = "packed CLI capability contract or package identity drifted"; + const capabilities = isJsonObject(value) + ? value + : fail(message); + if (capabilities.version !== "spawnfile.capabilities.v1") fail(message); + const implementation = isJsonObject(capabilities.implementation) + ? capabilities.implementation + : fail(message); + const capabilitySet = isJsonObject(capabilities.capabilities) + ? capabilities.capabilities + : fail(message); + const composedLifecycle = isJsonObject(capabilitySet.composed_lifecycle) + ? capabilitySet.composed_lifecycle + : fail(message); + const packageName = typeof implementation.package === "string" + ? implementation.package + : fail(message); + const packageVersion = typeof implementation.version === "string" + ? implementation.version + : fail(message); + const commandSetVersion = composedLifecycle.command_set_version + === "spawnfile.composed-lifecycle-contract-set.v1" + ? composedLifecycle.command_set_version + : fail(message); + if (composedLifecycle.complete !== true) fail(message); + return { + capabilities: { composed_lifecycle: { command_set_version: commandSetVersion, complete: true } }, + implementation: { package: packageName, version: packageVersion }, + version: "spawnfile.capabilities.v1", + }; +}; + +const resolvePackageImport = (manifest: unknown, packageName: string): string => { + const object = isJsonObject(manifest) ? manifest : null; + const exportsField = isJsonObject(object?.exports) ? object.exports : null; + const rootExport = isJsonObject(exportsField?.["."]) ? exportsField["."] : null; + const importPath = rootExport?.import; + if (typeof importPath !== "string" || !importPath.startsWith("./")) { + fail(`${packageName} does not expose a package-relative ESM entrypoint`); + } + return typeof importPath === "string" ? importPath : fail(`${packageName} import path is invalid`); +}; + +const run = ( + command: string, + args: string[], + cwd: string, + env: NodeJS.ProcessEnv = process.env, +): Promise => new Promise((resolve, reject) => { + const child = spawn(command, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { stdout += chunk; }); + child.stderr.on("data", (chunk: string) => { stderr += chunk; }); + child.on("error", reject); + child.on("close", (code) => { + if (code !== 0) { + reject(new Error(`${command} ${args.join(" ")} failed (${code})\n${stderr}`)); + return; + } + resolve({ stderr, stdout }); + }); +}); + +const assertPackageManifest = (value: unknown): PackageManifest => { + const object = isJsonObject(value) ? value : fail("manifest lacks an exact package identity"); + const name = typeof object.name === "string" ? object.name : fail("manifest lacks an exact package identity"); + const version = typeof object.version === "string" ? object.version : fail("manifest lacks an exact package identity"); + const dependencies = object.dependencies; + const bin = object.bin; + if (dependencies !== undefined + && (!isJsonObject(dependencies) + || Object.values(dependencies).some((dependency) => typeof dependency !== "string"))) { + fail("manifest dependencies must be string coordinates"); + } + if (bin !== undefined + && (!isJsonObject(bin) || Object.values(bin).some((entry) => typeof entry !== "string"))) { + fail("manifest bin entries must be string paths"); + } + return { + bin: bin as Record | undefined, + bundleDependencies: object.bundleDependencies, + bundledDependencies: object.bundledDependencies, + dependencies: dependencies as Record | undefined, + name, + version, + }; +}; + +const assertPackageLock = (value: unknown): PackageLock => { + const object = isJsonObject(value) ? value : fail("lockfile must be a JSON object"); + const packages = object.packages; + if (packages !== undefined && !isJsonObject(packages)) fail("lockfile packages must be an object"); + const packageEntries = packages ?? {}; + const typedPackages: PackageLock["packages"] = {}; + for (const [name, entryValue] of Object.entries(packageEntries)) { + const entry = isJsonObject(entryValue) ? entryValue : fail(`lock entry ${name} must be an object`); + typedPackages[name] = { + integrity: typeof entry.integrity === "string" ? entry.integrity : undefined, + link: typeof entry.link === "boolean" ? entry.link : undefined, + resolved: typeof entry.resolved === "string" ? entry.resolved : undefined, + version: typeof entry.version === "string" ? entry.version : undefined, + }; + } + return { packages: typedPackages }; +}; +const parsePackResult = (value: unknown): NpmPackResult | null => { + if (!isJsonObject(value) + || typeof value.filename !== "string" + || !Array.isArray(value.files) + || value.files.some((entry) => !isJsonObject(entry) || typeof entry.path !== "string")) { + return null; + } + const files = value.files.map((entry) => { + if (!isJsonObject(entry) || typeof entry.path !== "string") { + fail("npm pack returned an invalid file entry"); + } + return { path: entry.path }; + }); + return { + bundled: Array.isArray(value.bundled) ? value.bundled : undefined, + entryCount: typeof value.entryCount === "number" ? value.entryCount : undefined, + files, + filename: value.filename, + id: typeof value.id === "string" ? value.id : undefined, + integrity: typeof value.integrity === "string" ? value.integrity : undefined, + shasum: typeof value.shasum === "string" ? value.shasum : undefined, + }; +}; + +const parseSinglePack = (stdout: string): NpmPackResult => { + let parsed: unknown; + for (let index = stdout.lastIndexOf("["); index >= 0; index = stdout.lastIndexOf("[", index - 1)) { + try { + const candidate = JSON.parse(stdout.slice(index)) as unknown; + if (Array.isArray(candidate)) { + parsed = candidate; + break; + } + } catch { + // Lifecycle scripts may write to stdout before npm's final JSON array. + } + } + const parsedArray = Array.isArray(parsed) ? parsed : fail("npm pack must report exactly one tarball"); + if (parsedArray.length !== 1) fail("npm pack must report exactly one tarball"); + const result = parsedArray[0] as unknown; + return parsePackResult(result) ?? fail("npm pack returned an invalid manifest"); +}; + +const assertSourceClosure = async (manifest: PackageManifest, lock: PackageLock): Promise => { + if (manifest.dependencies?.[STELE] !== STELE_VERSION) { + fail(`${STELE} must use the published ${STELE_VERSION} release coordinate`); + } + const bundled = manifest.bundledDependencies ?? manifest.bundleDependencies; + if (bundled !== undefined) fail("published registry dependencies must not be bundled"); + const locked = lock.packages?.[`node_modules/${STELE}`] + ?? fail(`${STELE} lock entry must be a physical, version-matched registry package`); + if (locked.link === true || locked.version !== STELE_VERSION) { + fail(`${STELE} lock entry must be a physical, version-matched registry package`); + } + const expectedTarball = `https://registry.npmjs.org/@noopolis/stele/-/stele-${STELE_VERSION}.tgz`; + const lockedResolved = locked.resolved; + if (lockedResolved !== expectedTarball) { + fail(`${STELE} must resolve from the npm registry, got ${String(locked.resolved)}`); + } + if (typeof locked.integrity !== "string" || !locked.integrity.startsWith("sha512-")) { + fail(`${STELE} registry lock is missing sha512 integrity`); + } + const installed = path.join(packageRoot, "node_modules", STELE); + if ((await lstat(installed)).isSymbolicLink()) { + fail(`${STELE} must be physically installed before packing; source-checkout links are rejected`); + } + const installedManifest = assertPackageManifest(await readJson(path.join(installed, "package.json"))); + if (installedManifest.version !== STELE_VERSION) fail(`${STELE} installed version drifted`); + return typeof lockedResolved === "string" ? lockedResolved : fail(`${STELE} lock resolved entry is invalid`); +}; + +const assertPackedManifest = (manifest: PackageManifest): void => { + if (manifest.dependencies?.[STELE] !== STELE_VERSION) fail(`packed ${STELE} coordinate drifted from ${STELE_VERSION}`); + if (manifest.bundledDependencies !== undefined || manifest.bundleDependencies !== undefined) fail("packed manifest unexpectedly bundles registry dependencies"); + for (const [name, value] of Object.entries(manifest.dependencies ?? {})) { + if (typeof value === "string" && value.startsWith("file:")) fail(`packed runtime dependency ${name} retains a checkout-relative file coordinate`); + } +}; + +const dependencyRoot = async (installRoot: string, installedRoot: string): Promise => { + const candidates = [path.join(installedRoot, "node_modules", STELE), path.join(installRoot, "node_modules", STELE)]; + for (const candidate of candidates) { + try { + await lstat(candidate); + return candidate; + } catch (error) { + if (!isErrorWithCode(error, "ENOENT")) throw error; + } + } + return fail(`${STELE} was not installed from the packed Spawnfile dependency graph`); +}; + +const assertInstalledClosure = async ( + installRoot: string, + manifest: PackageManifest, + tarballPath: string, +): Promise => { + await writeFile(path.join(installRoot, "package.json"), "{\"private\":true}\n", "utf8"); + await run("npm", ["install", "--ignore-scripts", "--no-audit", "--no-fund", "--no-package-lock", "--registry=https://registry.npmjs.org", tarballPath], installRoot); + const installedRoot = path.join(installRoot, "node_modules", manifest.name); + if ((await lstat(installedRoot)).isSymbolicLink()) fail(`${manifest.name} installed as a source link`); + const installedManifest = assertPackageManifest(await readJson(path.join(installedRoot, "package.json"))); + if (installedManifest.name !== manifest.name || installedManifest.version !== manifest.version) fail("isolated install package identity drifted from the packed manifest"); + const helperProgram = path.join(installedRoot, "dist", "evidenceExportHelper", "helperProgram.mjs"); + const helperMetadata = await lstat(helperProgram); + if (!helperMetadata.isFile() || helperMetadata.isSymbolicLink() || helperMetadata.size < 1) fail("packed evidence helper asset is missing or unsafe"); + const helperRecipe = asEvidenceHelperRecipeModule(await import( + pathToFileURL(path.join(installedRoot, "dist", "evidenceExportHelper", "recipe.js")).href, + ) as unknown); + const helper = await helperRecipe.loadLocalEvidenceHelperRecipe(); + await verifyNativeHelperArtifacts(path.join(installedRoot, "dist", "deployment", "native")); + const helperSource = await readFile(helperProgram, "utf8"); + if (!helperSource.startsWith("#!/usr/local/bin/node") + || !(helper.context instanceof Uint8Array) + || typeof helper.recipeDigest !== "string" + || !/^sha256:[a-f0-9]{64}$/u.test(helper.recipeDigest)) { + fail("packed evidence helper asset cannot be imported into its recipe"); + } + const moltnetBinaries = asMoltnetBinariesModule(await import( + pathToFileURL(path.join(installedRoot, "dist/compiler/moltnetBinaries.js")).href, + ) as unknown); + const expectedMoltnetExports = ["MOLTNET_ALLOW_LOCAL_E2E_ENV", "MOLTNET_BINARY_NAMES", "MOLTNET_BIN_DIRECTORY", "MOLTNET_LOCAL_RELEASE_DIR_ENV", "MOLTNET_RELEASE_DIR_ENV", "MOLTNET_RELEASE_IDENTITY_VERSION", "MOLTNET_RELEASE_STAMP_VERSION", "resolveMoltnetCliCommand", "stageMoltnetBinaries"]; + const actualMoltnetExports = Object.keys(moltnetBinaries).sort(); + if (JSON.stringify(actualMoltnetExports) !== JSON.stringify(expectedMoltnetExports)) { + fail(`packed compiler Moltnet staging exports drifted: ${actualMoltnetExports.join(", ")}`); + } + const moltnetReleases = await Promise.all((["amd64", "arm64"] as const).map(async (architecture) => { + const outputDirectory = path.join(installRoot, `moltnet-${architecture}`); + const identity = await moltnetBinaries.stageMoltnetBinaries(outputDirectory, { architecture }); + const binaryPath = path.join(outputDirectory, "moltnet-bin", "moltnet"); + const [binary, metadata] = await Promise.all([readFile(binaryPath), stat(binaryPath)]); + if (binary.subarray(0, 4).toString("hex") !== "7f454c46" || !(metadata.mode & 0o111)) { + fail(`packed compiler did not stage an executable Linux ${architecture} Moltnet binary`); + } + return identity; + })); + const steleRoot = await dependencyRoot(installRoot, installedRoot); + if ((await lstat(steleRoot)).isSymbolicLink()) fail(`${STELE} installed as a source link`); + const steleManifest = assertPackageManifest(await readJson(path.join(steleRoot, "package.json"))); + if (steleManifest.version !== STELE_VERSION) fail(`${STELE} installed version drifted`); + const steleImport = resolvePackageImport(await readJson(path.join(steleRoot, "package.json")), STELE); + const installRealRoot = await realpath(installRoot); + const steleRealPath = await realpath(path.resolve(steleRoot, steleImport)); + if (!steleRealPath.startsWith(`${installRealRoot}${path.sep}`)) { + fail(`${STELE} resolved outside the isolated install`); + } + const stele = await import(pathToFileURL(steleRealPath).href) as unknown; + if (!isJsonObject(stele) || typeof stele.parseCausalJsonl !== "function") { + fail(`${STELE} runtime import is incomplete`); + } + const executableName = Object.keys(manifest.bin ?? {})[0]; + if (!executableName) fail(`${manifest.name} packed manifest lacks a CLI bin`); + const executable = path.join(installRoot, "node_modules", ".bin", executableName); + const executableEnvironment = { ...process.env, PATH: `${path.dirname(executable)}${path.delimiter}${process.env.PATH ?? ""}` }; + await run(executable, ["--help"], installRoot, executableEnvironment); + const capabilities = parseCapabilities( + JSON.parse((await run(executable, ["capabilities", "--json"], installRoot, executableEnvironment)).stdout) as unknown, + ); + if (capabilities.implementation.package !== manifest.name + || capabilities.implementation.version !== manifest.version) { + fail("packed CLI capability contract or package identity drifted"); + } + return { + capabilitiesVersion: capabilities.version, + commandSetVersion: capabilities.capabilities.composed_lifecycle.command_set_version, + moltnetReleases, + steleResolved: path.relative(installRealRoot, steleRealPath), + }; +}; + +const main = async () => { + const manifest = assertPackageManifest(await readJson(packageManifestPath)); + const lock = assertPackageLock(await readJson(lockPath)); + const steleRegistryTarball = await assertSourceClosure(manifest, lock); + const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), `${manifest.name.replaceAll("/", "-")}-closure-`)); + try { + const packDirectory = path.join(temporaryRoot, "pack"); + const installRoot = path.join(temporaryRoot, "install"); + await Promise.all([mkdir(packDirectory, { recursive: true }), mkdir(installRoot, { recursive: true })]); + const packed = parseSinglePack((await run("npm", ["pack", "--json", "--pack-destination", packDirectory], packageRoot)).stdout); + const tarballPath = path.join(packDirectory, packed.filename); + const packedBytes = await readFile(tarballPath); + const packedIntegrity = `sha512-${createHash("sha512").update(packedBytes).digest("base64")}`; + const packedShasum = createHash("sha1").update(packedBytes).digest("hex"); + if (packed.integrity !== packedIntegrity || packed.shasum !== packedShasum) fail("npm pack manifest integrity does not match the inspected tarball bytes"); + const entries = new Set(packed.files.map((entry) => entry.path)); + if (!entries.has("dist/evidenceExportHelper/helperProgram.mjs") + || !entries.has("dist/evidenceExportHelper/recipe.js")) { + fail("packed tarball omits the evidence helper runtime assets"); + } + for (const architecture of ["x64", "arm64"]) { + if (!entries.has(`dist/deployment/native/rename-noreplace-${architecture}`) + || !entries.has(`dist/deployment/native/rename-noreplace-${architecture}.provenance.json`)) { + fail(`packed tarball omits the Linux ${architecture} rename-noreplace helper or provenance`); + } + } + if ([...entries].some((entry) => /\.test-helper\.(?:js|d\.ts)$/u.test(entry))) { + fail("packed tarball leaked a test helper"); + } + if ([...entries].some((entry) => /\/[^/]*RunOperatorInputs\.(?:js|d\.ts)$/u.test(entry))) { + fail("packed tarball leaked an unbound operator-input contract"); + } + if ((packed.bundled?.length ?? 0) !== 0) fail("npm pack unexpectedly bundled dependencies"); + if ([...entries].some((entry) => entry.startsWith("node_modules/") + || entry.includes("ecosystem/") || /vendor\/.*\.tgz$/u.test(entry))) { + fail("packed tarball leaked dependencies, a source checkout, or a vendor archive"); + } + const packedManifest = assertPackageManifest(JSON.parse((await run("tar", ["-xOf", tarballPath, "package/package.json"], packageRoot)).stdout) as unknown); + assertPackedManifest(packedManifest); + if (packedManifest.name !== manifest.name || packedManifest.version !== manifest.version) fail("packed manifest identity drifted from the source manifest"); + const installed = await assertInstalledClosure(installRoot, manifest, tarballPath); + process.stdout.write(`${JSON.stringify({ + bundled: packed.bundled ?? [], capabilities_version: installed.capabilitiesVersion, + command_set_version: installed.commandSetVersion, entries: packed.entryCount, + integrity: packed.integrity, package: packed.id, packed_file: packed.filename, + moltnet_releases: installed.moltnetReleases, runtime_dependencies: packedManifest.dependencies, + stele_registry_tarball: steleRegistryTarball, stele_resolved_inside_install: installed.steleResolved, + }, null, 2)}\n`); + } finally { + await rm(temporaryRoot, { force: true, recursive: true }); + } +}; + +await main(); diff --git a/specs/research/WORKSPACE-PACKAGING.md b/specs/research/WORKSPACE-PACKAGING.md index b36a37e8..4a8291f1 100644 --- a/specs/research/WORKSPACE-PACKAGING.md +++ b/specs/research/WORKSPACE-PACKAGING.md @@ -34,7 +34,7 @@ Implementation references: [workspace schema](../../src/manifest/workspaceSchema [resource inheritance](../../src/compiler/workspaceResources.ts), [bundle validation](../../src/compiler/workspaceBundleArtifacts.ts), [MCP schema](../../src/manifest/mcpSchemas.ts) and -[source provenance helper](../../scripts/source-provenance-bundle.mjs). +[source provenance helper](../../scripts/source-provenance-bundle.ts). See [offline workspace bundles](../CONTAINERS.md#offline-workspace-bundles). ## Proposed authoring experience diff --git a/src/compiler/containerArtifacts.test.ts b/src/compiler/containerArtifacts.test.ts index 89f30000..9312ca0d 100644 --- a/src/compiler/containerArtifacts.test.ts +++ b/src/compiler/containerArtifacts.test.ts @@ -1303,7 +1303,7 @@ describe("createContainerArtifacts distribution contract", () => { expect(message).toContain("daimon-bridge"); expect(message).toContain("daimon_lab"); expect(message).toContain("exit at boot"); - expect(message).toContain("build-local-moltnet.mjs"); + expect(message).toContain("build-local-moltnet.ts"); }); it("fails closed when a daimon Moltnet attachment has no staged release identity at all", async () => { diff --git a/src/compiler/containerArtifacts.ts b/src/compiler/containerArtifacts.ts index 92e6ead4..dd30314d 100644 --- a/src/compiler/containerArtifacts.ts +++ b/src/compiler/containerArtifacts.ts @@ -79,7 +79,7 @@ const assertMoltnetDaimonBridgeCapability = ( `Moltnet release ${release?.version ?? "(unstaged)"} does not advertise the daimon-bridge capability, but ` + `${daimonNetworks.length} network attachment(s) lower a daimon runtime bridge (${daimonNetworks.join(", ")}). ` + "The published release implements pi-bridge only and rejects a daimon node config at strict JSON decode, so " - + "the container would exit at boot. Build Moltnet locally (scripts/build-local-moltnet.mjs, then " + + "the container would exit at boot. Build Moltnet locally (scripts/build-local-moltnet.ts, then " + "SPAWNFILE_LOCAL_MOLTNET_RELEASE_DIR with SPAWNFILE_ALLOW_LOCAL_E2E=1) to stage a daimon-bridge release, or " + "remove the Moltnet attachment from the daimon agent(s)." ); diff --git a/src/compiler/moltnetReleaseAuthority.ts b/src/compiler/moltnetReleaseAuthority.ts index 3d5b0ec7..641a9cde 100644 --- a/src/compiler/moltnetReleaseAuthority.ts +++ b/src/compiler/moltnetReleaseAuthority.ts @@ -13,7 +13,7 @@ export type MoltnetTargetArchitecture = "amd64" | "arm64"; * compare this list by exact equality (`upReceipt.ts`, `localMoltnetAuthority.ts`) * rather than as a set, so a differently ordered but equivalent list is * rejected as a different release. The local builder - * (`scripts/build-local-moltnet.mjs`) already writes this order; every producer + * (`scripts/build-local-moltnet.ts`) already writes this order; every producer * must match it, and `moltnetReleaseAuthority.test.ts` asserts it. * * This is a UNION, never a replacement: an older pi-only release must keep diff --git a/src/deployment/native/AGENTS.md b/src/deployment/native/AGENTS.md index a1250aba..b46705fe 100644 --- a/src/deployment/native/AGENTS.md +++ b/src/deployment/native/AGENTS.md @@ -5,7 +5,7 @@ syscalls Node does not expose. Helpers accept validated single-component names, emit one bounded JSON result, and fail closed. `artifacts/` contains the verified Linux x64 and arm64 package inputs so the -normal Node build stays offline and Docker-free. `copyArtifacts.mjs` verifies +normal Node build stays offline and Docker-free. `copyArtifacts.ts` verifies and copies them into `dist`. Only the explicit maintainer `build:native` path may replace them using the digest-pinned compiler image; CI rebuilds and syscall-tests both architectures before publication. diff --git a/src/deployment/native/build.mjs b/src/deployment/native/build.ts similarity index 98% rename from src/deployment/native/build.mjs rename to src/deployment/native/build.ts index 60768ac6..d6ef0e80 100644 --- a/src/deployment/native/build.mjs +++ b/src/deployment/native/build.ts @@ -10,7 +10,7 @@ mkdirSync(outputDirectory, { recursive: true }); const sourceSha256 = `sha256:${createHash("sha256").update(readFileSync(path.join(sourceDirectory, "renameNoreplace.c"))).digest("hex")}`; const builderImage = "gcc:14.2.0@sha256:b99b86a28812b1e6453a231a947dc43d76fe192788a12f344a9b568bf9f5d24c"; const compiler = "gcc:14.2.0"; -for (const [nodeArchitecture, dockerArchitecture] of [["x64", "amd64"], ["arm64", "arm64"]]) { +for (const [nodeArchitecture, dockerArchitecture] of [["x64", "amd64"], ["arm64", "arm64"]] as const) { const nonce = randomUUID(); const image = `spawnfile-rename-noreplace:${nonce}`; const container = `spawnfile-rename-noreplace-${nonce}`; try { execFileSync("docker", ["build", "--platform", `linux/${dockerArchitecture}`, "--tag", image, sourceDirectory], { stdio: "inherit" }); diff --git a/src/deployment/native/copyArtifacts.mjs b/src/deployment/native/copyArtifacts.ts similarity index 95% rename from src/deployment/native/copyArtifacts.mjs rename to src/deployment/native/copyArtifacts.ts index 8dd0767c..cd75d70a 100644 --- a/src/deployment/native/copyArtifacts.mjs +++ b/src/deployment/native/copyArtifacts.ts @@ -2,7 +2,7 @@ import { cp, mkdir } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { verifyNativeHelperArtifacts } from "../../../scripts/native-helper-artifacts.mjs"; +import { verifyNativeHelperArtifacts } from "../../../scripts/native-helper-artifacts.ts"; const source = fileURLToPath(new URL("./artifacts", import.meta.url)); const destination = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../dist/deployment/native"); diff --git a/src/distribution/AGENTS.md b/src/distribution/AGENTS.md index b51fe358..0baf4b20 100644 --- a/src/distribution/AGENTS.md +++ b/src/distribution/AGENTS.md @@ -1,6 +1,6 @@ # Distribution Guide -This folder owns the distribution report and image-contract artifacts for published Spawnfile images, per `DISTRIBUTION.md` and `specs/DISTRIBUTION.md`. +This folder owns the distribution report and image-contract artifacts for published Spawnfile images, per [`specs/DISTRIBUTION.md`](../../specs/DISTRIBUTION.md). ## Structure diff --git a/src/e2e/localMoltnetRelease.ts b/src/e2e/localMoltnetRelease.ts index f9809dc2..f3915818 100644 --- a/src/e2e/localMoltnetRelease.ts +++ b/src/e2e/localMoltnetRelease.ts @@ -43,7 +43,7 @@ export const localMoltnetReleaseAssetName = (): string => `moltnet_linux_${localMoltnetArch()}.tar.gz`; /** Companion capability stamp written next to the tarball by - * build-local-moltnet.mjs; asserted below so a stale pre-pi-bridge same-arch + * build-local-moltnet.ts; asserted below so a stale pre-pi-bridge same-arch * build cannot silently pass a mere existence check. */ export const localMoltnetReleaseStampName = (): string => `moltnet_release_stamp_${localMoltnetArch()}.json`; diff --git a/src/evidenceExportHelper/AGENTS.md b/src/evidenceExportHelper/AGENTS.md index 93fb7bc2..0ac44ff2 100644 --- a/src/evidenceExportHelper/AGENTS.md +++ b/src/evidenceExportHelper/AGENTS.md @@ -7,7 +7,7 @@ helper. It is a target setup facility, not a target-resource operation. - `helperProgram.mjs` is the image entrypoint source. It reads only the fixed `/spawnfile/evidence` mount and emits strict canonical USTAR to stdout. -- `copyAssets.mjs` copies that source beside the compiled modules for npm +- `copyAssets.ts` copies that source beside the compiled modules for npm packaging. - `recipe.ts` loads the shipped source, creates the fixed Dockerfile and canonical build context, and derives source identities. diff --git a/src/evidenceExportHelper/copyAssets.mjs b/src/evidenceExportHelper/copyAssets.ts similarity index 100% rename from src/evidenceExportHelper/copyAssets.mjs rename to src/evidenceExportHelper/copyAssets.ts diff --git a/src/runtime/copyScaffoldAssets.mjs b/src/runtime/copyScaffoldAssets.ts similarity index 77% rename from src/runtime/copyScaffoldAssets.mjs rename to src/runtime/copyScaffoldAssets.ts index a260509a..a2880a1c 100644 --- a/src/runtime/copyScaffoldAssets.mjs +++ b/src/runtime/copyScaffoldAssets.ts @@ -8,6 +8,13 @@ const destinationRoot = path.join(repoRoot, "dist", "runtime"); const entries = await readdir(sourceRoot, { withFileTypes: true }); +const isErrorWithCode = (error: unknown, code: string): boolean => ( + error !== null + && typeof error === "object" + && "code" in error + && error.code === code +); + for (const entry of entries) { if (!entry.isDirectory()) { continue; @@ -20,10 +27,10 @@ for (const entry of entries) { await cp(sourcePath, destinationPath, { filter: (source) => !["AGENTS.md", "CLAUDE.md"].includes(path.basename(source)), force: true, - recursive: true + recursive: true, }); } catch (error) { - if (!(error && typeof error === "object" && "code" in error && error.code === "ENOENT")) { + if (!isErrorWithCode(error, "ENOENT")) { throw error; } } diff --git a/tsconfig.build.json b/tsconfig.build.json index aab5fb69..44a3dc0b 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -13,6 +13,10 @@ "exclude": [ "src/**/*.test.ts", "src/**/*.test-helper.ts", - "src/e2e/**/*.ts" + "src/e2e/**/*.ts", + "src/deployment/native/build.ts", + "src/deployment/native/copyArtifacts.ts", + "src/evidenceExportHelper/copyAssets.ts", + "src/runtime/copyScaffoldAssets.ts" ] } diff --git a/tsconfig.json b/tsconfig.json index 19c646ec..4cdd75da 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -19,5 +19,11 @@ "include": [ "src/**/*.ts", "vitest.config.ts" + ], + "exclude": [ + "src/deployment/native/build.ts", + "src/deployment/native/copyArtifacts.ts", + "src/evidenceExportHelper/copyAssets.ts", + "src/runtime/copyScaffoldAssets.ts" ] } From 55d72434e29a84ef21cfd01b128699af55f9e9b2 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sat, 12 Sep 2026 05:02:25 +0200 Subject: [PATCH 2/2] fix: make provenance checks work before build and export archives reliably --- .../source-provenance-bundle.integration.test.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/source-provenance-bundle.integration.test.ts b/scripts/source-provenance-bundle.integration.test.ts index 0cf3c5e4..4ed0741f 100644 --- a/scripts/source-provenance-bundle.integration.test.ts +++ b/scripts/source-provenance-bundle.integration.test.ts @@ -6,7 +6,12 @@ import path from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; import { createSourceBundle, validateSourceBundle } from "./source-provenance-bundle.ts"; -import { renderRuntimeLinkMaterializer } from "../dist/compiler/containerRuntimeLinkMaterializer.js"; + +type RuntimeLinkMaterializerModule = typeof import("../src/compiler/containerRuntimeLinkMaterializer.js"); + +const { renderRuntimeLinkMaterializer } = await import( + new URL("../dist/compiler/containerRuntimeLinkMaterializer.js", import.meta.url).href +) as RuntimeLinkMaterializerModule; const repository = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const digest = (file: string): string => `sha256:${execFileSync("shasum", ["-a", "256", file], { encoding: "utf8" }).split(" ")[0] ?? ""}`; @@ -70,9 +75,10 @@ test("actual Daimon lock produces a real offline linux/amd64 shipped artifact an const wrapperIdentity = JSON.parse(readFileSync(identityPath, "utf8")); assert.equal(wrapperIdentity.image_architecture, "amd64"); assert.match(wrapperIdentity.image_reference, new RegExp(`^127\\.0\\.0\\.1:${port}/noopolis/spawnfile-runtime-daimon@sha256:[a-f0-9]{64}$`, "u")); } finally { if (priorIdentity) writeFileSync(identityPath, priorIdentity); else rmSync(identityPath, { force: true }); } const receipt = { architecture: "amd64", daimon: { package_sha256: packageSha, source_inputs: sourceInputs, source_sha256: digest(path.join(packageContext, "source-inputs.json")) }, engines: { agy: { executable_sha256: agySha }, codex: { executable_sha256: codexSha }, grok: { executable_sha256: grokSha } }, manifest_sha256: manifestSha, provenance: { agy: { archive: { format: "tar.gz", sha512: sha512(agyTar), url: "https://invalid.example/agy", version: "fixture" } }, grok: { executable: { sha256: grokSha, url: "https://invalid.example/grok", version: "fixture" } } }, version: "spawnfile.daimon-runtime-capability-receipt.v1" }; - const shipped = path.join(temporary, "shipped"); mkdirSync(shipped); - execFileSync("docker", ["build", "--network=none", "--platform", "linux/amd64", "--build-context", `daimon_package=${packageContext}`, "--output", `type=local,dest=${shipped}`, - "--build-arg", `DAIMON_CAPABILITY_RECEIPT_BASE64=${Buffer.from(`${JSON.stringify(receipt)}\n`).toString("base64")}`, "--build-arg", `DAIMON_MANIFEST_SHA256=${manifestSha}`, "--build-arg", `DAIMON_PACKAGE_SHA256=${packageSha}`, "--build-arg", `DAIMON_SOURCE_SHA256=${receipt.daimon.source_sha256}`, "--build-arg", "DAIMON_DEPENDENCY_MODE=offline-bundle", "--build-arg", `DAIMON_DEPENDENCY_ARCHIVE_SHA256=${runtimeArchive}`, "--build-arg", `CODEX_CLI_SHA256=${codexSha}`, "--build-arg", "GROK_CLI_VERSION=fixture", "--build-arg", "GROK_CLI_URL=https://invalid.example/grok", "--build-arg", `GROK_CLI_SHA256=${grokSha.slice(7)}`, "--build-arg", "AGY_CLI_VERSION=fixture", "--build-arg", "AGY_CLI_URL=https://invalid.example/agy", "--build-arg", `AGY_CLI_SHA512=${sha512(agyTar).slice(7)}`, "--build-arg", `AGY_CLI_SHA256=${agySha.slice(7)}`, "-f", path.join(repository, "runtime-images", "daimon", "Dockerfile"), repository], { stdio: "ignore" }); + const shipped = path.join(temporary, "shipped"), shippedTar = path.join(temporary, "shipped.tar"); mkdirSync(shipped); + execFileSync("docker", ["build", "--network=none", "--platform", "linux/amd64", "--build-context", `daimon_package=${packageContext}`, "--output", `type=tar,dest=${shippedTar}`, + "--build-arg", `DAIMON_CAPABILITY_RECEIPT_BASE64=${Buffer.from(`${JSON.stringify(receipt)}\n`).toString("base64")}`, "--build-arg", `DAIMON_MANIFEST_SHA256=${manifestSha}`, "--build-arg", `DAIMON_PACKAGE_SHA256=${packageSha}`, "--build-arg", `DAIMON_SOURCE_SHA256=${receipt.daimon.source_sha256}`, "--build-arg", "DAIMON_DEPENDENCY_MODE=offline-bundle", "--build-arg", `DAIMON_DEPENDENCY_ARCHIVE_SHA256=${runtimeArchive}`, "--build-arg", `CODEX_CLI_SHA256=${codexSha}`, "--build-arg", "GROK_CLI_VERSION=fixture", "--build-arg", "GROK_CLI_URL=https://invalid.example/grok", "--build-arg", `GROK_CLI_SHA256=${grokSha.slice(7)}`, "--build-arg", "AGY_CLI_VERSION=fixture", "--build-arg", "AGY_CLI_URL=https://invalid.example/agy", "--build-arg", `AGY_CLI_SHA512=${sha512(agyTar).slice(7)}`, "--build-arg", `AGY_CLI_SHA256=${agySha.slice(7)}`, "-f", path.join(repository, "runtime-images", "daimon", "Dockerfile"), repository], { stdio: "inherit" }); + execFileSync("tar", ["-xf", shippedTar, "-C", shipped], { stdio: "inherit" }); assert.deepEqual(JSON.parse(readFileSync(path.join(shipped, "opt", "spawnfile", "runtime-installs", "daimon", "source-inputs.json"), "utf8")), receipt.daimon.source_inputs); const shippedRoot=path.join(shipped,"opt","spawnfile","runtime-installs","daimon"); assert.equal(digest(path.join(shippedRoot,"bin","daimon-engine-broker")),"sha256:e3fe2738fc8a979861085b4003bf2d5d7c284874897cb6ec2e2e2383211768bd");