From 6f3404dd46c10e1b228cd1a9f1c962634061fc66 Mon Sep 17 00:00:00 2001 From: Jarel Remick Date: Tue, 29 Sep 2026 14:34:36 +1000 Subject: [PATCH 1/4] Add portable CI and release checks with isolated evidence --- CONTRIBUTING.md | 2 + docs/LOCAL_CI.md | 140 ++ scripts/lib/secret-patterns.mjs | 8 + scripts/local-ci.mjs | 1181 +++++++++++++++++ scripts/local-ci.sh | 22 + scripts/run-fullstack-e2e.mjs | 45 +- scripts/scan-secrets.mjs | 8 +- scripts/test/fullstack-e2e-isolation.test.mjs | 83 ++ scripts/test/local-ci.test.mjs | 720 ++++++++++ 9 files changed, 2199 insertions(+), 10 deletions(-) create mode 100644 docs/LOCAL_CI.md create mode 100644 scripts/lib/secret-patterns.mjs create mode 100644 scripts/local-ci.mjs create mode 100755 scripts/local-ci.sh create mode 100644 scripts/test/fullstack-e2e-isolation.test.mjs create mode 100644 scripts/test/local-ci.test.mjs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea5c8ec7..c65c641a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,6 +45,8 @@ TEST_DATABASE_URL=postgres://myskills_test:myskills_test@localhost:5432/myskills `TEST_DATABASE_URL` must point at a disposable database whose name includes `test` or `ci`; the test resets that schema. +To run the full CI, release or CodeQL gates on a Linux host with Docker, use `scripts/local-ci.sh`; see [Local CI](docs/LOCAL_CI.md). + ## Pull Request Expectations Every PR should include: diff --git a/docs/LOCAL_CI.md b/docs/LOCAL_CI.md new file mode 100644 index 00000000..fdddec79 --- /dev/null +++ b/docs/LOCAL_CI.md @@ -0,0 +1,140 @@ +# Local CI And Release Checks + +`scripts/local-ci.sh` runs the same gates as the GitHub Actions workflows on a Linux host with +Docker. Contributors can use it before opening a pull request. An external runner can call it +after checkout and read its evidence. The GitHub workflows remain in place until a replacement +has shown equivalent results; this entrypoint does not report statuses or publish anything. + +## Requirements + +- Linux on amd64. The E2E Compose file pins MinIO to `linux/amd64`, so other architectures need + emulation and do not give equivalent evidence. +- Docker Engine with Compose v2 and BuildKit, reachable through the local socket. +- Git, Bash and network access to the npm registry, Docker Hub, GitHub release assets (MinIO + binaries) and the Playwright browser download host. +- Node.js 22 (at least 22.13) and Node.js 24, each with the npm version in `package.json` + `packageManager`. The script checks versions; it never installs npm globally. +- Chromium system libraries for Playwright. CI installs them with `--with-deps`; locally install + them once with `sudo npx playwright install-deps chromium`. + +## Usage + +Commit first. The script tests the checked-out commit in fresh clones and refuses a dirty tree. + +```bash +export LOCAL_CI_RUN_ID="pr-123-$(date +%s)" +export LOCAL_CI_EVIDENCE_DIR="$HOME/local-ci-evidence/$LOCAL_CI_RUN_ID" +export LOCAL_CI_NODE22_BIN=/path/to/node-22/bin +export LOCAL_CI_NODE24_BIN=/path/to/node-24/bin +export LOCAL_CI_SOURCE_SHA="$(git rev-parse HEAD)" +scripts/local-ci.sh verify +scripts/local-ci.sh verify --job check-node22 # one job; the result is marked non-gating +``` + +Release verification needs the release tag at `HEAD` and a main ref that contains it: + +```bash +git fetch origin main --tags +LOCAL_CI_RELEASE_TAG="v$(node -p 'require("./package.json").version')" scripts/local-ci.sh release-check +``` + +CodeQL needs an official CodeQL CLI bundle: + +```bash +LOCAL_CI_CODEQL_BIN=/path/to/codeql/codeql scripts/local-ci.sh codeql +``` + +| Mode | Jobs | +|---|---| +| `verify` | `check-node22`, `postgres-node22`, `web-e2e-node22`, `check-node24`, `postgres-node24`, `web-e2e-node24`, `railway-images` | +| `release-check` | `release` | +| `codeql` | `codeql-javascript-typescript` | + +## Inputs + +| Variable | Rule | +|---|---| +| `LOCAL_CI_RUN_ID` | Required. 1-48 lowercase letters, digits or hyphens. It names every container, Compose project, image tag and the workspace. | +| `LOCAL_CI_EVIDENCE_DIR` | Required. Absolute, outside the source tree, and without an earlier `result.json`. | +| `LOCAL_CI_SOURCE_SHA` | Optional. Must equal `HEAD`. Without it the result is not gating. | +| `LOCAL_CI_NODE22_BIN`, `LOCAL_CI_NODE24_BIN` | Directories containing `node`, `npm` and `npx`. When unset, `node` on `PATH` is used only for its own major version. | +| `LOCAL_CI_WORK_DIR` | Optional parent for the per-run workspace. Defaults to the OS temporary directory. | +| `LOCAL_CI_RELEASE_TAG`, `LOCAL_CI_MAIN_REF` | `release-check` only. The tag must be `v` and point at `HEAD`. `HEAD` must be an ancestor of the main ref (default `refs/remotes/origin/main`). The script does not fetch. | +| `LOCAL_CI_CODEQL_BIN`, `LOCAL_CI_CODEQL_CATEGORY` | `codeql` only. The category defaults to `/language:javascript-typescript`. | +| `MYSKILLS_E2E_PORT`, `MYSKILLS_E2E_WEB_PORT`, `MYSKILLS_E2E_MAILPIT_PORT` | Optional loopback ports. Free ports are chosen when unset. | + +Jobs receive an allowlisted environment (paths, locale, Docker endpoint, proxy and CA settings, +browser and npm caches) with `CI=true`. Tokens such as `GITHUB_TOKEN` or `NPM_TOKEN` are not +passed to jobs. Do not keep publishing credentials in the account that runs jobs: dependency +scripts and tests run with that account's files and its Docker access. + +## Results + +Exit status 0 means passed, 1 failed, 2 rejected before any work, and 128 plus the signal number +means cancelled. Read `result.json` rather than relying on the exit status. It is written +atomically and contains: + +- `status`: `passed`, `failed`, `rejected` or `cancelled`. `passed` also requires complete + cleanup and clean evidence. +- `gating` and `gatingBlockers`: only a complete job set with a verified `LOCAL_CI_SOURCE_SHA` can + gate a commit. +- `contexts`: for a complete `verify` run, the protected-branch contexts `check`, `web-e2e` and + `postgres-integration`. Partial runs report `null`. +- `jobs`: status, reason, steps, exit codes, timings and a hashed log for each job. +- `cleanup` and `artifacts`: cleanup outcome and the SHA-256 of every evidence file. + +Other evidence: `logs/.log`, `resources.json`, `environment.json`, +`browser-evidence//` (the reviewed summaries and screenshots from +`scripts/collect-browser-evidence.mjs`), `release/` (verified artifacts and +`verification.json`) and `codeql/` (SARIF and a summary). + +Credential-shaped output is replaced with `[redacted]` in logs and fails the run. The full-stack +runner also redacts its generated credentials. Raw Playwright reports, traces and videos stay in +the job clone and are deleted with it. + +## Isolation And Cleanup + +Each job runs in its own clone of the pinned commit inside +`/myskills-local-ci-`. The workspace is created exclusively, so two runs cannot +share a run ID on one host. The script records each container, Compose project and image in +`resources.json` before or as it creates it, and removes only those exact names. Compose cleanup +matches the exact `com.docker.compose.project` label. A container whose creation failed, for +example because of a name conflict, is never removed. The script never prunes and never matches +name prefixes. Shared npm, Playwright and Docker build caches are kept. + +Each step runs in its own process group. `SIGTERM` stops the current step, cleans up and writes a +cancelled result; allow about 60 seconds. After `SIGKILL`, use `resources.json` to remove the +listed resources. + +## Mapping From GitHub Actions + +| Workflow gate | Local equivalent | Difference | +|---|---|---| +| CI `Check / Node 22.x`, `Check / Node 24.x`: `npm ci`, `npm run check` | `check-node22`, `check-node24` | Node patch versions come from the supplied toolchains, not the latest `22.x`/`24.x`. The npm version is checked, not installed. | +| CI `check` aggregate | `contexts.check` (all seven `verify` jobs) | None. | +| CI `Web E2E / Node 22.x`, `Web E2E / Node 24.x` (15-minute timeout): browser install, workspace build, mocked browser run, evidence collection, full-stack run, evidence collection, evidence upload | `web-e2e-node22`, `web-e2e-node24` with the same steps, conditions and 15-minute limit | Browser system libraries come from host setup. Evidence is exported to the evidence directory instead of a 7-day artifact. A job without exported evidence fails, as with `if-no-files-found: error`. | +| CI `web-e2e` aggregate | `contexts["web-e2e"]` | None. | +| CI `Railway images`: `Dockerfile.api`, `Dockerfile.web`, `Dockerfile.backup` and two credential-free `--network none` smoke runs | `railway-images` | Builds use `--pull` and run-scoped tags. Image IDs are recorded and the images are removed afterwards. | +| CI `Postgres / Node 22.x`, `Postgres / Node 24.x` with a `postgres:17-alpine` service | `postgres-node22`, `postgres-node24` | Same image, credentials and health check on a random loopback port. | +| CI `postgres-integration` aggregate | `contexts["postgres-integration"]` | None. | +| Release `Verify tag and main ancestry` | `release-check` input validation | The runner supplies full history, the tag and a current main ref; the script does not fetch. | +| Release `postgres:17` service, `npm ci`, browser install, `npm run release:verify` with tag enforcement | `release` job steps | Same commands and environment. | +| Release image builds: root `Dockerfile` `api`, `mcp-http`, `web`; Railway API and web; backup image and smoke runs | `release` job `build-*` and `smoke-*` steps | Run-scoped tags; nothing is pushed. | +| Release `Upload release artifacts` | `verify-release-artifacts` step and `release/artifacts/` | Also checks that the artifact set is exact, that `SHA256SUMS` and the metadata match, and that the source archive rebuilds byte for byte from the pinned commit. | +| CodeQL `Analyze JavaScript and TypeScript`: `.github/codeql/codeql-config.yml` with `security-extended` | `codeql` mode | Uses the repository config plus `queries: - uses: security-extended`, and fails if an excluded query still reports. The number of findings does not gate; GitHub alert state, including dismissals, stays authoritative after upload. | + +## Not Covered Here + +These GitHub functions stay with GitHub, or with an external runner, until a separate cutover +replaces them: + +- Pull request, push, tag and weekly CodeQL triggers, and release concurrency. +- Reporting the `check`, `web-e2e` and `postgres-integration` statuses, and branch protection. +- SARIF upload and the code scanning merge rule. +- Artifact retention. +- Dependabot, which is not an Actions workflow. + +`scripts/check-prerelease.mjs`, `scripts/check-structure.mjs` and the release documentation +still describe the workflow files; update them when the workflows are retired. The full-stack +Compose run builds from cached base images without `--pull`, so the host cache can differ from a +fresh GitHub runner until it is refreshed. diff --git a/scripts/lib/secret-patterns.mjs b/scripts/lib/secret-patterns.mjs new file mode 100644 index 00000000..a54c8041 --- /dev/null +++ b/scripts/lib/secret-patterns.mjs @@ -0,0 +1,8 @@ +// Shared by the repository secret scan and local CI evidence redaction. +export const secretPatterns = [ + { name: "Vendor API token", pattern: /\bATATT[0-9A-Za-z_-]{20,}\b/ }, + { name: "GitHub token", pattern: /\b(?:ghp|gho|ghu|ghs|ghr)_[0-9A-Za-z_]{30,}\b/ }, + { name: "OpenAI API key", pattern: /\bsk-[A-Za-z0-9_-]{32,}\b/ }, + { name: "Private key block", pattern: /-----BEGIN (?:RSA |EC |OPENSSH |)PRIVATE KEY-----/ }, + { name: "AWS access key", pattern: /\bAKIA[0-9A-Z]{16}\b/ }, +]; diff --git a/scripts/local-ci.mjs b/scripts/local-ci.mjs new file mode 100644 index 00000000..6eca278e --- /dev/null +++ b/scripts/local-ci.mjs @@ -0,0 +1,1181 @@ +#!/usr/bin/env node + +// Portable equivalent of the GitHub Actions CI, release and CodeQL gates. Invoke it through +// scripts/local-ci.sh; docs/LOCAL_CI.md describes the inputs, jobs and result contract. + +import { spawn, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + accessSync, + closeSync, + constants, + copyFileSync, + existsSync, + lstatSync, + mkdirSync, + openSync, + readFileSync, + readdirSync, + realpathSync, + renameSync, + rmSync, + statSync, + unlinkSync, + writeFileSync, + writeSync, +} from "node:fs"; +import { createServer } from "node:net"; +import { arch, platform, release as osRelease, tmpdir } from "node:os"; +import { basename, delimiter, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; +import { secretPatterns } from "./lib/secret-patterns.mjs"; + +const sourceRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const jobCatalog = { + verify: ["check-node22", "postgres-node22", "web-e2e-node22", "check-node24", "postgres-node24", "web-e2e-node24", "railway-images"], + "release-check": ["release"], + codeql: ["codeql-javascript-typescript"], +}; +const contextJobs = { + check: jobCatalog.verify, + "web-e2e": ["web-e2e-node22", "web-e2e-node24"], + "postgres-integration": ["postgres-node22", "postgres-node24"], +}; +const usage = `Usage: scripts/local-ci.sh [--job ]... + +Jobs: + verify ${jobCatalog.verify.join(", ")} + release-check ${jobCatalog["release-check"].join(", ")} + codeql ${jobCatalog.codeql.join(", ")} + +Required environment: LOCAL_CI_RUN_ID, LOCAL_CI_EVIDENCE_DIR. See docs/LOCAL_CI.md.`; +const runIdPattern = /^[a-z0-9](?:[a-z0-9-]{0,46}[a-z0-9])?$/; +const webE2eTimeoutMs = 15 * 60_000; +const alwaysStepTimeoutMs = 60_000; +const killGraceMs = 10_000; +const maxLogBytes = 64 * 1024 * 1024; +const maxPendingLine = 1024 * 1024; +const codeqlSuiteLine = "\nqueries:\n - uses: security-extended\n"; +const defaultCodeqlCategory = "/language:javascript-typescript"; +const composeServiceImages = ["api", "web", "minio", "minio-init"]; +// Only these variables reach job processes; runner tokens stay with the runner. +const passthroughEnv = [ + "HOME", "USER", "LOGNAME", "SHELL", "LANG", "LANGUAGE", "LC_ALL", "LC_CTYPE", "TZ", "TERM", "TMPDIR", + "XDG_CACHE_HOME", "XDG_CONFIG_HOME", "XDG_RUNTIME_DIR", + "DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_CONFIG", "DOCKER_CERT_PATH", "DOCKER_TLS_VERIFY", "BUILDKIT_PROGRESS", + "PLAYWRIGHT_BROWSERS_PATH", "npm_config_cache", "NPM_CONFIG_CACHE", + "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy", + "SSL_CERT_FILE", "SSL_CERT_DIR", "NODE_EXTRA_CA_CERTS", "MYSKILLS_E2E_BROWSER_EXECUTABLE", +]; +const compiledSecretPatterns = secretPatterns.map(({ pattern }) => new RegExp(pattern.source, `${pattern.flags.replace("g", "")}g`)); + +class Rejection extends Error { + constructor(reason, message) { + super(message); + this.reason = reason; + } +} + +const state = { + cancelled: false, + signal: null, + activeSteps: new Set(), +}; + +async function main() { + let options; + try { + options = parseArguments(process.argv.slice(2)); + } catch (error) { + console.error(error.message); + return 2; + } + if (options.help) { + console.log(usage); + return 0; + } + const runId = process.env.LOCAL_CI_RUN_ID ?? ""; + if (!runIdPattern.test(runId)) { + console.error("LOCAL_CI_RUN_ID must be 1-48 lowercase letters, digits or hyphens, starting and ending with a letter or digit."); + return 2; + } + let evidence; + try { + evidence = validateEvidenceDirectory(process.env.LOCAL_CI_EVIDENCE_DIR); + } catch (error) { + console.error(error.message); + return 2; + } + mkdirSync(evidence, { recursive: true }); + + const startedAt = new Date().toISOString(); + const base = { schemaVersion: 1, app: "myskills", mode: options.mode, runId, selectedJobs: options.jobs, startedAt }; + let run; + try { + run = prepareRun(options, runId, evidence); + } catch (error) { + if (!(error instanceof Rejection)) throw error; + console.error(error.message); + writeJsonAtomic(join(evidence, "result.json"), { + ...base, status: "rejected", reason: error.reason, message: error.message, gating: false, + jobs: [], contexts: null, artifacts: [], finishedAt: new Date().toISOString(), + }); + return 2; + } + + for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"]) { + process.on(signal, () => cancel(signal)); + } + return executeRun(run, base); +} + +function parseArguments(argv) { + const [mode, ...rest] = argv; + if (mode === "--help" || mode === "-h") return { help: true }; + if (!Object.hasOwn(jobCatalog, mode ?? "")) throw new Error(usage); + const requested = []; + for (let index = 0; index < rest.length; index += 1) { + const arg = rest[index]; + let value; + if (arg === "--job") value = rest[++index]; + else if (arg.startsWith("--job=")) value = arg.slice("--job=".length); + else throw new Error(usage); + if (!jobCatalog[mode].includes(value)) { + throw new Error(`Unknown job for ${mode}. Choose from: ${jobCatalog[mode].join(", ")}.`); + } + requested.push(value); + } + const jobs = requested.length > 0 ? jobCatalog[mode].filter((id) => requested.includes(id)) : [...jobCatalog[mode]]; + return { mode, jobs, complete: jobs.length === jobCatalog[mode].length }; +} + +function validateEvidenceDirectory(value) { + if (!value) throw new Error("LOCAL_CI_EVIDENCE_DIR is required and must be absolute."); + if (!isAbsolute(value)) throw new Error("LOCAL_CI_EVIDENCE_DIR must be absolute."); + const target = resolve(value); + const real = realPathAllowingMissing(target); + const source = realpathSync(sourceRoot); + if (isWithin(real, source) || isWithin(source, real)) { + throw new Error("LOCAL_CI_EVIDENCE_DIR must be outside the source tree and must not contain it."); + } + if (existsSync(target)) { + if (!statSync(target).isDirectory()) throw new Error("LOCAL_CI_EVIDENCE_DIR must be a directory."); + if (existsSync(join(target, "result.json"))) { + throw new Error("LOCAL_CI_EVIDENCE_DIR already contains result.json; use a new directory for each run."); + } + } + return real; +} + +function prepareRun(options, runId, evidence) { + const top = git(["rev-parse", "--show-toplevel"]); + if (top.status !== 0 || realpathSync(top.stdout.trim()) !== realpathSync(sourceRoot)) { + throw new Rejection("source-not-repository", "The entrypoint must run from its own git checkout."); + } + const head = git(["rev-parse", "HEAD"]).stdout.trim(); + const expectedSha = process.env.LOCAL_CI_SOURCE_SHA || null; + if (expectedSha !== null) { + if (!/^[0-9a-f]{40}(?:[0-9a-f]{24})?$/.test(expectedSha)) { + throw new Rejection("invalid-source-sha", "LOCAL_CI_SOURCE_SHA must be a full lowercase commit SHA."); + } + if (expectedSha !== head) throw new Rejection("source-sha-mismatch", "LOCAL_CI_SOURCE_SHA does not match HEAD."); + } + const status = git(["status", "--porcelain", "--untracked-files=all"]); + if (status.status !== 0 || status.stdout.trim() !== "") { + throw new Rejection("dirty-source", "The checkout has uncommitted or untracked files. Local CI tests commits only; commit the change first."); + } + const rootPackage = JSON.parse(readFileSync(join(sourceRoot, "package.json"), "utf8")); + const npmVersion = /^npm@(\d+\.\d+\.\d+)$/.exec(String(rootPackage.packageManager ?? ""))?.[1]; + if (!npmVersion) throw new Rejection("invalid-package-manager", "package.json packageManager must pin an exact npm version."); + + const run = { + ...options, + runId, + evidence, + head, + expectedSha, + rootPackage, + npmVersion, + gatingBlockers: [ + ...(options.complete ? [] : ["partial-job-selection"]), + ...(expectedSha ? [] : ["source-sha-not-supplied"]), + ], + }; + if (options.mode === "release-check") run.release = validateRelease(head, rootPackage); + if (options.mode === "codeql") run.codeql = validateCodeql(); + run.workspace = reserveWorkspace(runId, evidence); + return run; +} + +function validateRelease(head, rootPackage) { + const tag = process.env.LOCAL_CI_RELEASE_TAG ?? ""; + if (!tag) throw new Rejection("release-tag-required", "LOCAL_CI_RELEASE_TAG is required for release-check."); + if (!/^v\d+\.\d+\.\d+(?:-[0-9A-Za-z.]+)?$/.test(tag)) throw new Rejection("invalid-release-tag", "LOCAL_CI_RELEASE_TAG is not a release tag."); + if (tag !== `v${rootPackage.version}`) { + throw new Rejection("release-tag-version-mismatch", `LOCAL_CI_RELEASE_TAG must equal v${rootPackage.version}.`); + } + const tagCommit = git(["rev-parse", "--verify", "--quiet", `refs/tags/${tag}^{commit}`]); + if (tagCommit.status !== 0 || tagCommit.stdout.trim() !== head) { + throw new Rejection("release-tag-not-at-head", "The release tag must exist and point at HEAD."); + } + const mainRef = process.env.LOCAL_CI_MAIN_REF || "refs/remotes/origin/main"; + if (!mainRef.startsWith("refs/") || git(["check-ref-format", mainRef]).status !== 0) { + throw new Rejection("invalid-main-ref", "LOCAL_CI_MAIN_REF must be a full ref name such as refs/remotes/origin/main."); + } + const main = git(["rev-parse", "--verify", "--quiet", `${mainRef}^{commit}`]); + if (main.status !== 0) throw new Rejection("main-ref-unavailable", "LOCAL_CI_MAIN_REF does not resolve to a commit in this checkout."); + const mainSha = main.stdout.trim(); + const ancestry = git(["merge-base", "--is-ancestor", head, mainSha]); + if (ancestry.status === 1) throw new Rejection("not-on-main", "The tagged commit is not an ancestor of the main ref."); + if (ancestry.status !== 0) throw new Rejection("main-ancestry-unverified", "Main ancestry could not be verified; the checkout may be shallow."); + return { tag, mainRef, mainSha }; +} + +function validateCodeql() { + const bin = process.env.LOCAL_CI_CODEQL_BIN ?? ""; + if (!isAbsolute(bin) || !isExecutable(bin)) { + throw new Rejection("codeql-cli-unavailable", "LOCAL_CI_CODEQL_BIN must be the absolute path of a CodeQL CLI bundle executable."); + } + const category = process.env.LOCAL_CI_CODEQL_CATEGORY || defaultCodeqlCategory; + if (!/^[A-Za-z0-9/:._-]{1,200}$/.test(category)) throw new Rejection("invalid-codeql-category", "LOCAL_CI_CODEQL_CATEGORY contains unsupported characters."); + const configText = readFileSync(join(sourceRoot, ".github/codeql/codeql-config.yml"), "utf8"); + if (/^queries\s*:/m.test(configText)) { + throw new Rejection("codeql-config-unsupported", "The CodeQL config already declares queries; the security-extended suite cannot be appended safely."); + } + const excludedRules = [...configText.matchAll(/-\s*exclude:\s*\n\s+id:\s*([^\s#]+)/g)].map((match) => match[1]); + return { bin, category, excludedRules }; +} + +function reserveWorkspace(runId, evidence) { + const input = process.env.LOCAL_CI_WORK_DIR || tmpdir(); + if (!isAbsolute(input) || !existsSync(input) || !statSync(input).isDirectory()) { + throw new Rejection("invalid-work-dir", "LOCAL_CI_WORK_DIR must be an existing absolute directory."); + } + const workRoot = realpathSync(input); + if (isWithin(workRoot, realpathSync(sourceRoot)) || isWithin(workRoot, evidence)) { + throw new Rejection("invalid-work-dir", "LOCAL_CI_WORK_DIR must be outside the source tree and the evidence directory."); + } + const workspace = join(workRoot, `myskills-local-ci-${runId}`); + try { + // Exclusive creation doubles as the per-host lock for this run ID. + mkdirSync(workspace); + } catch (error) { + if (error?.code === "EEXIST") { + throw new Rejection("run-workspace-exists", "A workspace for this LOCAL_CI_RUN_ID already exists; use a new run ID or clean up the earlier run."); + } + throw error; + } + return workspace; +} + +async function executeRun(run, base) { + const ledger = new ResourceLedger(join(run.evidence, "resources.json"), run.runId); + ledger.track("workspace", run.workspace, null, "created"); + mkdirSync(join(run.evidence, "logs"), { recursive: true }); + const environment = collectEnvironment(run); + const jobs = []; + const failureReasons = []; + const sinks = []; + console.log(`[local-ci] ${run.mode} ${run.runId}: ${run.jobs.join(", ")}`); + try { + for (const id of run.jobs) { + if (state.cancelled) { + jobs.push({ id, status: "not-run", reason: "cancelled", steps: [] }); + continue; + } + const job = new JobContext(id, run, ledger, environment); + sinks.push({ id, sink: job.sink }); + jobs.push(await job.execute()); + } + } catch (error) { + failureReasons.push(`internal-error: ${error instanceof Error ? error.message : String(error)}`); + } + + const cleanupFailures = ledger.cleanup(new LogSink(null)); + try { + rmSync(run.workspace, { recursive: true, force: true }); + ledger.mark(ledger.find("workspace", run.workspace), "removed"); + } catch (error) { + ledger.mark(ledger.find("workspace", run.workspace), "remove-failed"); + cleanupFailures.push(`workspace: ${error.code ?? error.message}`); + } + writeJsonAtomic(join(run.evidence, "environment.json"), environment); + + for (const job of jobs) { + if (job.status !== "passed" && job.status !== "not-run") failureReasons.push(`job-${job.status}:${job.id}`); + } + for (const { id, sink } of sinks) { + if (sink.redactions > 0) failureReasons.push(`secret-pattern-redacted:${id}`); + } + for (const path of scanEvidenceForSecrets(run.evidence)) failureReasons.push(`secret-pattern-in-evidence:${path}`); + if (cleanupFailures.length > 0) failureReasons.push("cleanup-failed"); + + const allPassed = jobs.length === run.jobs.length && jobs.every(({ status }) => status === "passed"); + const status = state.cancelled ? "cancelled" : allPassed && failureReasons.length === 0 ? "passed" : "failed"; + // Redaction, quarantine, cleanup and internal failures cannot be attributed safely, so they fail every context. + const runLevelFailure = failureReasons.some((reason) => !reason.startsWith("job-")); + const contexts = run.mode === "verify" && run.complete ? Object.fromEntries(Object.entries(contextJobs).map(([name, ids]) => [ + name, + status === "cancelled" ? "cancelled" + : !runLevelFailure && ids.every((id) => jobs.find((job) => job.id === id)?.status === "passed") ? "passed" : "failed", + ])) : null; + const result = { + ...base, + status, + gating: run.gatingBlockers.length === 0, + gatingBlockers: run.gatingBlockers, + complete: run.complete, + source: { sha: run.head, expectedSha: run.expectedSha, packageVersion: run.rootPackage.version }, + ...(run.release ? { release: run.release } : {}), + jobs, + contexts, + failureReasons, + cleanup: { status: cleanupFailures.length === 0 ? "complete" : "failed", failures: cleanupFailures }, + artifacts: manifest(run.evidence), + finishedAt: new Date().toISOString(), + }; + writeJsonAtomic(join(run.evidence, "result.json"), result); + console.log(`[local-ci] ${status}: ${run.evidence}/result.json`); + if (status === "cancelled") return 128 + ({ SIGHUP: 1, SIGINT: 2, SIGTERM: 15 }[state.signal] ?? 15); + return status === "passed" ? 0 : 1; +} + +class JobContext { + constructor(id, run, ledger, environment) { + this.id = id; + this.run = run; + this.ledger = ledger; + this.environment = environment; + this.sink = new LogSink(join(run.evidence, "logs", `${id}.log`)); + this.steps = []; + this.reason = null; + this.timedOut = false; + this.cancelled = false; + this.toolchain = null; + this.clone = null; + this.deadline = id.startsWith("web-e2e-") ? Date.now() + webE2eTimeoutMs : null; + this.details = {}; + } + + get stopped() { + return this.cancelled || state.cancelled; + } + + fail(reason) { + this.reason ??= reason; + return false; + } + + async execute() { + const startedAt = Date.now(); + this.sink.note(`job ${this.id} started`); + try { + await jobRunners[this.id.replace(/-node2[24]$/, "")](this, this.id.match(/node(2[24])$/)?.[1]); + } catch (error) { + this.sink.note(`internal error: ${error instanceof Error ? error.message : String(error)}`); + this.fail("internal-error"); + } + if (this.clone) { + try { + rmSync(dirname(this.clone), { recursive: true, force: true }); + } catch (error) { + this.sink.note(`workspace cleanup deferred: ${error.code ?? error.message}`); + } + } + const cleanupFailures = this.ledger.cleanup(this.sink, this.id); + if (cleanupFailures.length > 0) this.fail("cleanup-failed"); + const status = this.stopped ? "cancelled" : this.timedOut ? "timed-out" : this.reason ? "failed" : "passed"; + this.sink.note(`job ${this.id} ${status}${this.reason ? ` (${this.reason})` : ""}`); + this.sink.close(); + const logPath = `logs/${this.id}.log`; + const logBytes = readFileSync(join(this.run.evidence, logPath)); + return { + id: this.id, + status, + reason: this.stopped ? "cancelled" : this.timedOut ? "timeout" : this.reason, + startedAt: new Date(startedAt).toISOString(), + finishedAt: new Date().toISOString(), + durationMs: Date.now() - startedAt, + steps: this.steps, + log: { path: logPath, sha256: sha256(logBytes), bytes: logBytes.length, truncated: this.sink.truncated }, + ...this.details, + }; + } + + useToolchain(line) { + const toolchain = resolveToolchain(line, this.run.npmVersion); + this.environment.toolchains[`node${line}`] = toolchain.ok + ? { node: toolchain.nodeVersion, npm: toolchain.npmVersion } + : { unavailable: toolchain.reason }; + if (!toolchain.ok) { + this.sink.note(`Node ${line} toolchain rejected: ${toolchain.detail}`); + return this.fail(toolchain.reason); + } + this.toolchain = toolchain; + return true; + } + + async checkout() { + const directory = join(this.run.workspace, this.id, "source"); + mkdirSync(dirname(directory), { recursive: true }); + const cloned = await this.step("checkout", "git", ["-c", "core.hooksPath=/dev/null", "clone", "--quiet", "--no-checkout", "--shared", sourceRoot, directory], { cwd: this.run.workspace }); + this.clone = directory; + if (!cloned) return false; + if (!await this.step("checkout-commit", "git", ["-c", "core.hooksPath=/dev/null", "checkout", "--quiet", "--detach", this.run.head])) return false; + const head = git(["rev-parse", "HEAD"], directory).stdout.trim(); + if (head !== this.run.head) return this.fail("checkout-mismatch"); + return true; + } + + env(extra = {}) { + return jobEnvironment(this.toolchain?.dir ?? null, extra); + } + + skip(name) { + this.steps.push({ name, status: "skipped" }); + return false; + } + + ran(name) { + return this.steps.some((step) => step.name === name && step.status !== "skipped"); + } + + canRun() { + return !this.stopped && !this.timedOut && this.reason === null; + } + + async step(name, command, args, options = {}) { + if (this.stopped || (!options.always && (this.timedOut || this.reason !== null))) return this.skip(name); + let timeoutMs = options.always ? alwaysStepTimeoutMs : null; + if (!options.always && this.deadline !== null) { + timeoutMs = this.deadline - Date.now(); + if (timeoutMs <= 0) { + this.timedOut = true; + return this.skip(name); + } + } + const startedAt = Date.now(); + this.sink.note(`step ${name}: ${[command, ...args].join(" ")}`); + const outcome = await spawnStep(command, args, { + cwd: options.cwd ?? this.clone ?? this.run.workspace, + env: options.env ?? this.env(options.extraEnv), + timeoutMs, + sink: this.sink, + }); + let status = outcome.exitCode === 0 ? "passed" : "failed"; + if (outcome.cancelled) { + status = "cancelled"; + this.cancelled = true; + } else if (outcome.timedOut) { + status = "timed-out"; + if (!options.always) this.timedOut = true; + } + this.steps.push({ name, status, exitCode: outcome.exitCode, signal: outcome.signal, durationMs: Date.now() - startedAt }); + this.sink.note(`step ${name} ${status} (exit ${outcome.exitCode ?? outcome.signal ?? outcome.error})`); + if (status === "failed" || (status === "timed-out" && options.always)) this.fail(options.reason ?? "step-failed"); + return status === "passed"; + } + + async postgres(image, health) { + const name = containerName(this.run.runId, this.id, "postgres"); + if (!await this.step("postgres-pull", "docker", ["pull", image], { reason: "service-unavailable" })) return null; + const entry = this.ledger.track("container", name, this.id, "creating"); + const created = await this.step("postgres-service", "docker", [ + "run", "-d", "--name", name, "--label", `io.myskills.local-ci.run-id=${this.run.runId}`, + "-e", "POSTGRES_USER=myskills_test", "-e", "POSTGRES_PASSWORD=myskills_test", "-e", "POSTGRES_DB=myskills_test", + "-p", "127.0.0.1::5432", + "--health-cmd", "pg_isready -U myskills_test -d myskills_test", + "--health-interval", health.interval, "--health-timeout", health.timeout, "--health-retries", health.retries, + image, + ], { reason: "service-unavailable" }); + // A failed create may mean the name belongs to someone else; never remove it. + this.ledger.mark(entry, created ? "created" : "not-created"); + if (!created) return null; + const deadline = Date.now() + 180_000; + let healthStatus = ""; + while (!this.stopped && Date.now() < deadline) { + healthStatus = docker(["inspect", "--format", "{{.State.Health.Status}}", name]).stdout.trim(); + if (healthStatus === "healthy" || healthStatus === "unhealthy") break; + await delay(1000); + } + this.sink.note(`postgres service health: ${healthStatus || "unknown"}`); + if (healthStatus !== "healthy") return this.fail("service-unavailable") || null; + const port = /^127\.0\.0\.1:(\d+)$/m.exec(docker(["port", name, "5432/tcp"]).stdout)?.[1]; + if (!port) return this.fail("service-unavailable") || null; + return `postgres://myskills_test:myskills_test@127.0.0.1:${port}/myskills_test`; + } + + async build(name, args, tag) { + const entry = this.ledger.track("image", tag, this.id, "creating"); + const built = await this.step(name, "docker", ["build", "--pull", ...args, "--tag", tag, "."]); + this.ledger.mark(entry, this.steps.at(-1).status === "skipped" ? "not-created" : "created"); + if (built) { + this.details.images ??= {}; + this.details.images[tag] = docker(["image", "inspect", "--format", "{{.Id}}", tag]).stdout.trim() || null; + } + return built; + } + + async smokeBackup(image) { + for (const [name, script] of [["smoke-backup-run", "run-registry-backup.mjs"], ["smoke-backup-restore", "restore-registry-backup.mjs"]]) { + const container = containerName(this.run.runId, this.id, name); + const entry = this.ledger.track("container", container, this.id, "creating"); + await this.step(name, "docker", [ + "run", "--rm", "--name", container, "--label", `io.myskills.local-ci.run-id=${this.run.runId}`, + "--network", "none", image, "node", `scripts/${script}`, "--help", + ]); + // --rm removes the container when the CLI exits normally; only an interrupted run needs cleanup. + const last = this.steps.at(-1); + this.ledger.mark(entry, last.status === "passed" || last.status === "failed" ? "removed" : last.status === "skipped" ? "not-created" : "created"); + } + } + + trackComposeProject(suffix) { + const project = containerName(this.run.runId, this.id, suffix); + this.ledger.track("compose-project", project, this.id, "created"); + return project; + } +} + +const jobRunners = { + async check(job, line) { + if (!job.useToolchain(line) || !await job.checkout()) return; + await job.step("install", "npm", ["ci"]); + await job.step("check", "npm", ["run", "check"]); + }, + + async postgres(job, line) { + if (!job.useToolchain(line) || !await job.checkout()) return; + const databaseUrl = await job.postgres("postgres:17-alpine", { interval: "5s", timeout: "5s", retries: "10" }); + await job.step("install", "npm", ["ci"]); + await job.step("test-postgres", "npm", ["run", "test:postgres"], { extraEnv: { TEST_DATABASE_URL: databaseUrl ?? "" } }); + }, + + async "web-e2e"(job, line) { + if (!job.useToolchain(line) || !await job.checkout()) return; + const ports = await e2ePorts(); + job.details.ports = ports; + await job.step("install", "npm", ["ci"]); + await job.step("playwright-browser", "npx", ["playwright", "install", "chromium"]); + await job.step("build", "npm", ["run", "build", "-w", "@myskills-app/core", "-w", "@myskills-app/auth", "-w", "@myskills-app/skill-package", "-w", "@myskills-app/api"]); + await job.step("mocked-browser", "npm", ["run", "test:e2e", "-w", "@myskills-app/web", "--", "--reporter=line,json"], { + extraEnv: { ...ports, PLAYWRIGHT_JSON_OUTPUT_FILE: "test-results/mocked-report.json" }, + }); + await collectBrowserEvidence(job, "mocked-browser", "collect-mocked-evidence", "mocked-report.json", "mocked"); + const project = job.canRun() ? job.trackComposeProject("fullstack") : null; + await job.step("fullstack-browser", "npm", ["run", "test:e2e:fullstack"], { extraEnv: { ...ports, MYSKILLS_E2E_COMPOSE_PROJECT: project ?? "" } }); + await collectBrowserEvidence(job, "fullstack-browser", "collect-fullstack-evidence", "fullstack-report.json", "fullstack"); + exportBrowserEvidence(job); + }, + + async "railway-images"(job) { + if (!await job.checkout()) return; + const tag = (repository) => `${repository}:local-ci-${job.run.runId}`; + await job.build("build-railway-api", ["--file", "Dockerfile.api"], tag("myskills-app-api")); + await job.build("build-railway-web", ["--file", "Dockerfile.web", "--build-arg", "VITE_API_BASE_URL=/api"], tag("myskills-app-web")); + await job.build("build-backup", ["--file", "Dockerfile.backup"], tag("myskills-registry-backup")); + await job.smokeBackup(tag("myskills-registry-backup")); + }, + + async release(job) { + const { tag } = job.run.release; + if (!job.useToolchain("22") || !await job.checkout()) return; + const tags = git(["tag", "--points-at", "HEAD"], job.clone).stdout.split(/\r?\n/); + if (!tags.includes(tag)) return job.fail("release-tag-missing-in-clone"); + const databaseUrl = await job.postgres("postgres:17", { interval: "10s", timeout: "5s", retries: "5" }); + const ports = await e2ePorts(); + job.details.ports = ports; + await job.step("install", "npm", ["ci"]); + await job.step("playwright-browser", "npx", ["playwright", "install", "chromium"]); + const project = job.canRun() ? job.trackComposeProject("release") : null; + await job.step("release-verify", "npm", ["run", "release:verify"], { + extraEnv: { + ...ports, + TEST_DATABASE_URL: databaseUrl ?? "", + RELEASE_REQUIRE_TAG: "true", + RELEASE_EXPECTED_TAG: tag, + MYSKILLS_E2E_COMPOSE_PROJECT: project ?? "", + }, + }); + const image = (repository) => `${repository}:local-ci-${job.run.runId}`; + await job.build("build-api", ["--file", "Dockerfile", "--target", "api"], image("myskills-app-api")); + await job.build("build-mcp-http", ["--file", "Dockerfile", "--target", "mcp-http"], image("myskills-app-mcp-http")); + await job.build("build-web", ["--file", "Dockerfile", "--target", "web", "--build-arg", "VITE_API_BASE_URL=/api"], image("myskills-app-web")); + await job.build("build-railway-api", ["--file", "Dockerfile.api"], image("myskills-app-railway-api")); + await job.build("build-railway-web", ["--file", "Dockerfile.web", "--build-arg", "VITE_API_BASE_URL=/api"], image("myskills-app-railway-web")); + await job.build("build-backup", ["--file", "Dockerfile.backup"], image("myskills-registry-backup")); + await job.smokeBackup(image("myskills-registry-backup")); + if (!job.canRun()) return job.skip("verify-release-artifacts"); + const verification = verifyReleaseArtifacts(job.clone, job.run, tag); + const target = join(job.run.evidence, "release"); + mkdirSync(target, { recursive: true }); + writeJsonAtomic(join(target, "verification.json"), verification.record); + job.steps.push({ name: "verify-release-artifacts", status: verification.ok ? "passed" : "failed" }); + if (!verification.ok) { + job.sink.note(`release artifact verification failed: ${verification.record.failures.join("; ")}`); + return job.fail("artifact-verification-failed"); + } + mkdirSync(join(target, "artifacts")); + for (const file of verification.files) copyFileSync(join(verification.directory, file), join(target, "artifacts", file)); + job.details.releaseArtifacts = verification.record.artifacts; + }, + + async "codeql-javascript-typescript"(job) { + const { bin, category, excludedRules } = job.run.codeql; + if (!await job.checkout()) return; + const work = dirname(job.clone); + const config = join(work, "codeql-config.yml"); + // The verified external route: repository filters verbatim plus the suite the workflow requests. + writeFileSync(config, `${readFileSync(join(job.clone, ".github/codeql/codeql-config.yml"), "utf8")}${codeqlSuiteLine}`); + const nodeDir = codeqlNodeDirectory(); + job.toolchain = nodeDir ? { dir: nodeDir } : null; + const version = spawnSync(bin, ["version", "--format=json"], { encoding: "utf8", env: job.env() }); + job.environment.codeql = safeJson(version.stdout)?.version ?? null; + const output = join(job.run.evidence, "codeql", "javascript-typescript.sarif"); + mkdirSync(dirname(output), { recursive: true }); + await job.step("database-create", bin, [ + "database", "create", join(work, "database"), "--language=javascript-typescript", "--build-mode=none", + `--source-root=${job.clone}`, `--codescanning-config=${config}`, "--threads=0", + ]); + await job.step("database-analyze", bin, [ + "database", "analyze", join(work, "database"), "--format=sarifv2.1.0", `--output=${output}`, + `--sarif-category=${category}`, "--threads=0", + ]); + if (!job.canRun()) return; + const summary = summarizeSarif(output, category, excludedRules); + job.details.codeql = { version: job.environment.codeql, category, excludedRules, results: summary.results, byRule: summary.byRule }; + writeJsonAtomic(join(job.run.evidence, "codeql", "summary.json"), { ...job.details.codeql, locations: summary.locations, checks: summary.checks }); + if (summary.failure) return job.fail(summary.failure); + }, +}; + +async function collectBrowserEvidence(job, browserStep, name, report, phase) { + if (!job.ran(browserStep)) return job.skip(name); + const results = "apps/web/test-results"; + return job.step(name, "node", ["scripts/collect-browser-evidence.mjs", `${results}/${report}`, results, `dist/browser-evidence/${phase}`], { always: true }); +} + +function exportBrowserEvidence(job) { + if (job.stopped) return job.skip("export-browser-evidence"); + const source = join(job.clone, "dist", "browser-evidence"); + const target = join(job.run.evidence, "browser-evidence", job.id); + const copied = existsSync(source) ? copyRegularFiles(source, target) : 0; + job.details.browserEvidence = `browser-evidence/${job.id}`; + // Mirrors upload-artifact's if-no-files-found: error. + job.steps.push({ name: "export-browser-evidence", status: copied > 0 ? "passed" : "failed", files: copied }); + if (copied === 0) job.fail("evidence-missing"); +} + +function verifyReleaseArtifacts(clone, run, tag) { + const failures = []; + const record = { tag, commitSha: run.head, failures, artifacts: [] }; + const dist = join(clone, "dist"); + const outputs = existsSync(dist) ? readdirSync(dist).filter((name) => name.startsWith("release-verify-")) : []; + if (outputs.length !== 1) { + failures.push(`expected one dist/release-verify-* output, found ${outputs.length}`); + return { ok: false, record }; + } + const directory = join(dist, outputs[0], "artifacts"); + const { name, version, packageManager, engines } = run.rootPackage; + const archive = `${name}-${version}-source.tar`; + const files = [archive, "release-metadata.json", "SHA256SUMS"].sort(); + const present = existsSync(directory) ? readdirSync(directory, { withFileTypes: true }) : []; + if (present.some((entry) => !entry.isFile()) || JSON.stringify(present.map((entry) => entry.name).sort()) !== JSON.stringify(files)) { + failures.push("artifact set must be exactly the source archive, release-metadata.json and SHA256SUMS"); + return { ok: false, record }; + } + const digests = Object.fromEntries(files.map((file) => [file, sha256(readFileSync(join(directory, file)))])); + const sums = new Map(); + for (const line of readFileSync(join(directory, "SHA256SUMS"), "utf8").split("\n").filter(Boolean)) { + const match = /^([0-9a-f]{64}) {2}(\S+)$/.exec(line); + if (!match) failures.push("SHA256SUMS contains a malformed line"); + else sums.set(match[2], match[1]); + } + if (JSON.stringify([...sums.keys()].sort()) !== JSON.stringify([archive, "release-metadata.json"].sort())) { + failures.push("SHA256SUMS must list the source archive and metadata only"); + } + for (const [file, digest] of sums) { + if (digests[file] !== digest) failures.push(`SHA256SUMS mismatch for ${file}`); + } + const metadata = safeJson(readFileSync(join(directory, "release-metadata.json"), "utf8")); + const archiveBytes = statSync(join(directory, archive)).size; + const expectations = [ + ["name", metadata?.name === name], + ["version", metadata?.version === version], + ["expectedTag", metadata?.expectedTag === tag], + ["tags", Array.isArray(metadata?.tags) && metadata.tags.includes(tag)], + ["commitSha", metadata?.commitSha === run.head], + ["dirty", metadata?.dirty === false], + ["packageManager", metadata?.packageManager === packageManager], + ["nodeEngine", metadata?.nodeEngine === engines?.node], + ["artifacts", JSON.stringify(metadata?.artifacts) === JSON.stringify([{ file: archive, byteSize: archiveBytes, sha256: digests[archive] }])], + ]; + for (const [field, ok] of expectations) if (!ok) failures.push(`release-metadata.json ${field} does not match the verified source`); + // Rebuild the archive from the pinned commit; equal bytes bind the artifact to that source. + const rebuilt = join(dirname(clone), "reproduced-source.tar"); + const archived = git(["archive", "--format=tar", `--prefix=${name}-${version}/`, "-o", rebuilt, run.head], clone); + if (archived.status !== 0 || sha256(readFileSync(rebuilt)) !== digests[archive]) { + failures.push("source archive does not reproduce from the pinned commit"); + } + record.artifacts = files.map((file) => ({ file, sha256: digests[file] })); + return { ok: failures.length === 0, record, directory, files }; +} + +function summarizeSarif(path, category, excludedRules) { + const checks = {}; + let failure = null; + const sarif = existsSync(path) ? safeJson(readFileSync(path, "utf8")) : null; + const runs = Array.isArray(sarif?.runs) ? sarif.runs : []; + const rules = new Set(); + const byRule = {}; + const locations = []; + let results = 0; + for (const sarifRun of runs) { + for (const component of [sarifRun.tool?.driver, ...(sarifRun.tool?.extensions ?? [])]) { + for (const rule of component?.rules ?? []) rules.add(rule.id); + } + for (const result of sarifRun.results ?? []) { + results += 1; + byRule[result.ruleId] = (byRule[result.ruleId] ?? 0) + 1; + const location = result.locations?.[0]?.physicalLocation; + locations.push({ ruleId: result.ruleId, uri: location?.artifactLocation?.uri ?? null, line: location?.region?.startLine ?? null }); + } + } + checks.sarifPresent = runs.length > 0; + checks.queriesRan = rules.size > 0; + checks.filtersApplied = excludedRules.every((id) => !rules.has(id)); + checks.category = runs.length > 0 && runs.every((sarifRun) => String(sarifRun.automationDetails?.id ?? "").startsWith(category)); + if (!checks.sarifPresent) failure = "codeql-sarif-missing"; + else if (!checks.queriesRan) failure = "codeql-no-queries"; + else if (!checks.filtersApplied) failure = "codeql-filter-not-applied"; + else if (!checks.category) failure = "codeql-category-mismatch"; + // Result counts are reported, not gated: GitHub alert dismissals remain the authority. + return { checks, failure, results, byRule, locations }; +} + +class ResourceLedger { + constructor(path, runId) { + this.path = path; + this.runId = runId; + this.resources = []; + this.save(); + } + + track(kind, name, job, stateName) { + const entry = { kind, name, job, state: stateName }; + this.resources.push(entry); + this.save(); + return entry; + } + + find(kind, name) { + return this.resources.find((entry) => entry.kind === kind && entry.name === name); + } + + mark(entry, stateName) { + entry.state = stateName; + this.save(); + } + + save() { + writeJsonAtomic(this.path, { runId: this.runId, resources: this.resources }); + } + + cleanup(sink, job = undefined) { + const failures = []; + for (const entry of [...this.resources].reverse()) { + if ((job !== undefined && entry.job !== job) || entry.kind === "workspace") continue; + if (!["created", "creating"].includes(entry.state)) continue; + if (entry.kind === "container" && entry.state === "creating") continue; + const removed = removeResource(entry, sink); + this.mark(entry, removed ? "removed" : "remove-failed"); + if (!removed) failures.push(`${entry.kind} ${entry.name}`); + } + return failures; + } +} + +function removeResource(entry, sink) { + if (entry.kind === "container") return removeContainers([entry.name], sink); + if (entry.kind === "image") return removeImage(entry.name, sink); + if (entry.kind === "compose-project") { + // Exact Compose project label match; never name prefixes or prune. + const filter = `label=com.docker.compose.project=${entry.name}`; + const listed = (args) => docker(args).stdout.split(/\s+/).filter(Boolean); + let ok = removeContainers(listed(["ps", "-aq", "--filter", filter]), sink); + const networks = listed(["network", "ls", "-q", "--filter", filter]); + if (networks.length > 0) ok = dockerLogged(["network", "rm", ...networks], sink) && ok; + const volumes = listed(["volume", "ls", "-q", "--filter", filter]); + if (volumes.length > 0) ok = dockerLogged(["volume", "rm", ...volumes], sink) && ok; + for (const service of composeServiceImages) ok = removeImage(`${entry.name}-${service}`, sink) && ok; + return ok; + } + return false; +} + +function removeContainers(names, sink) { + if (names.length === 0) return true; + const result = docker(["rm", "-f", "-v", ...names]); + sink.note(`docker rm -f -v ${names.join(" ")}: exit ${result.status}`); + return result.status === 0 || /No such container/i.test(result.stderr); +} + +function removeImage(reference, sink) { + if (docker(["image", "inspect", "--format", "{{.Id}}", reference]).status !== 0) return true; + return dockerLogged(["image", "rm", reference], sink); +} + +function dockerLogged(args, sink) { + const result = docker(args); + sink.note(`docker ${args.join(" ")}: exit ${result.status}`); + return result.status === 0; +} + +class LogSink { + constructor(path) { + this.fd = path ? openSync(path, "a") : null; + this.bytes = 0; + this.truncated = false; + this.redactions = 0; + this.pending = new Map(); + } + + write(stream, text) { + const lines = `${this.pending.get(stream) ?? ""}${text}`.split("\n"); + let rest = lines.pop(); + for (const line of lines) this.emit(`${line}\n`); + if (rest.length > maxPendingLine) { + this.emit(`${rest}\n`); + rest = ""; + } + this.pending.set(stream, rest); + } + + flush() { + for (const [stream, rest] of this.pending) { + if (rest) this.emit(`${rest}\n`); + this.pending.set(stream, ""); + } + } + + note(message) { + this.emit(`[local-ci] ${message}\n`); + } + + emit(text) { + let clean = text; + for (const pattern of compiledSecretPatterns) { + clean = clean.replace(pattern, () => { + this.redactions += 1; + return "[redacted]"; + }); + } + process.stdout.write(clean); + if (this.fd === null || this.truncated) return; + const size = Buffer.byteLength(clean); + if (this.bytes + size > maxLogBytes) { + writeSync(this.fd, "[local-ci] log truncated at the size limit\n"); + this.truncated = true; + return; + } + writeSync(this.fd, clean); + this.bytes += size; + } + + close() { + this.flush(); + if (this.fd !== null) closeSync(this.fd); + this.fd = null; + } +} + +function spawnStep(command, args, { cwd, env, timeoutMs, sink }) { + return new Promise((resolvePromise) => { + let settled = false; + let timedOut = false; + let timer = null; + let closeTimer = null; + let child; + const handle = { stop: () => terminate(child) }; + const finish = (outcome) => { + if (settled) return; + settled = true; + clearTimeout(timer); + clearTimeout(closeTimer); + state.activeSteps.delete(handle); + sink.flush(); + resolvePromise({ ...outcome, timedOut, cancelled: state.cancelled }); + }; + try { + // Each step leads its own process group so timeouts and cancellation reach every descendant. + child = spawn(command, args, { cwd, env, detached: true, stdio: ["ignore", "pipe", "pipe"] }); + } catch (error) { + sink.note(`could not start ${command}: ${error.code ?? error.message}`); + finish({ exitCode: null, signal: null, error: error.code ?? "spawn-failed" }); + return; + } + state.activeSteps.add(handle); + if (state.cancelled) terminate(child); + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (text) => sink.write("stdout", text)); + child.stderr.on("data", (text) => sink.write("stderr", text)); + if (timeoutMs !== null) { + timer = setTimeout(() => { + timedOut = true; + sink.note(`timed out after ${Math.round(timeoutMs / 1000)} s`); + terminate(child); + }, timeoutMs); + } + child.once("error", (error) => { + sink.note(`could not start ${command}: ${error.code ?? error.message}`); + finish({ exitCode: null, signal: null, error: error.code ?? "spawn-failed" }); + }); + child.once("exit", (code, signal) => { + // Stop leftovers in the step's own group, then wait briefly for output to drain. + signalGroup(child, "SIGTERM"); + closeTimer = setTimeout(() => { + signalGroup(child, "SIGKILL"); + finish({ exitCode: code, signal }); + }, killGraceMs); + }); + child.once("close", (code, signal) => finish({ exitCode: code, signal })); + }); +} + +function terminate(child) { + if (!child?.pid) return; + signalGroup(child, "SIGTERM"); + setTimeout(() => signalGroup(child, "SIGKILL"), killGraceMs).unref(); +} + +function signalGroup(child, signal) { + try { + process.kill(-child.pid, signal); + } catch { + // The group has already exited. + } +} + +function cancel(signal) { + if (state.cancelled) return; + state.cancelled = true; + state.signal = signal; + console.error(`[local-ci] ${signal} received; stopping the current step and cleaning up this run's resources.`); + for (const handle of state.activeSteps) handle.stop(); +} + +function resolveToolchain(line, npmVersion) { + const configured = process.env[`LOCAL_CI_NODE${line}_BIN`]; + const unavailable = (detail) => ({ ok: false, reason: "toolchain-unavailable", detail }); + let dir; + if (configured) { + if (!isAbsolute(configured)) return unavailable(`LOCAL_CI_NODE${line}_BIN must be absolute`); + dir = configured; + } else { + const node = findOnPath("node"); + if (!node) return unavailable(`set LOCAL_CI_NODE${line}_BIN`); + dir = dirname(node); + } + for (const tool of ["node", "npm", "npx"]) { + if (!isExecutable(join(dir, tool))) return unavailable(`${tool} is missing from the Node ${line} directory`); + } + const nodeVersion = spawnSync(join(dir, "node"), ["--version"], { encoding: "utf8", env: jobEnvironment(dir) }).stdout.trim(); + const [major, minor] = (/^v(\d+)\.(\d+)\.\d+$/.exec(nodeVersion) ?? []).slice(1).map(Number); + if (major !== Number(line)) { + const detail = `found ${nodeVersion || "no version"}, need Node ${line}`; + return configured ? { ok: false, reason: "toolchain-mismatch", detail } : unavailable(detail); + } + if (line === "22" && minor < 13) return { ok: false, reason: "toolchain-mismatch", detail: `${nodeVersion} is below the 22.13 engine floor` }; + const foundNpm = spawnSync(join(dir, "npm"), ["--version"], { encoding: "utf8", env: jobEnvironment(dir) }).stdout.trim(); + if (foundNpm !== npmVersion) { + return { ok: false, reason: "toolchain-mismatch", detail: `npm ${foundNpm || "unknown"} does not match packageManager npm@${npmVersion}` }; + } + return { ok: true, dir, nodeVersion, npmVersion: foundNpm }; +} + +function codeqlNodeDirectory() { + for (const line of ["24", "22"]) { + const dir = process.env[`LOCAL_CI_NODE${line}_BIN`]; + if (dir && isAbsolute(dir) && isExecutable(join(dir, "node"))) return dir; + } + return null; +} + +function jobEnvironment(toolchainDir, extra = {}) { + const env = {}; + for (const name of passthroughEnv) { + if (process.env[name] !== undefined) env[name] = process.env[name]; + } + env.PATH = toolchainDir ? `${toolchainDir}${delimiter}${process.env.PATH ?? ""}` : process.env.PATH ?? ""; + // GitHub-hosted runners set CI=true; Playwright retries and server reuse depend on it. + env.CI = "true"; + return { ...env, ...extra }; +} + +function collectEnvironment(run) { + const environment = { + platform: platform(), + arch: arch(), + osRelease: osRelease(), + git: git(["--version"]).stdout.trim(), + toolchains: {}, + docker: null, + }; + if (run.jobs.some((id) => !id.startsWith("check-") && !id.startsWith("codeql-"))) { + const version = docker(["version", "--format", "{{.Server.Version}} {{.Server.Os}}/{{.Server.Arch}}"]); + environment.docker = version.status === 0 ? version.stdout.trim() : "unavailable"; + } + return environment; +} + +async function e2ePorts() { + const ports = {}; + for (const name of ["MYSKILLS_E2E_PORT", "MYSKILLS_E2E_WEB_PORT", "MYSKILLS_E2E_MAILPIT_PORT"]) { + const supplied = process.env[name]; + ports[name] = supplied && /^\d+$/.test(supplied) && Number(supplied) >= 1024 && Number(supplied) <= 65535 + ? supplied + : String(await freeLoopbackPort()); + } + return ports; +} + +function freeLoopbackPort() { + return new Promise((resolvePromise, rejectPromise) => { + const server = createServer(); + server.unref(); + server.once("error", rejectPromise); + server.listen(0, "127.0.0.1", () => { + const { port } = server.address(); + server.close(() => resolvePromise(port)); + }); + }); +} + +function scanEvidenceForSecrets(evidence) { + const findings = []; + for (const path of listEvidenceFiles(evidence)) { + if (path === "result.json" || /\.(png|jpe?g|gif|webp|zip)$/i.test(path)) continue; + const text = readFileSync(join(evidence, path), "latin1"); + if (compiledSecretPatterns.some((pattern) => { + pattern.lastIndex = 0; + return pattern.test(text); + })) { + // Quarantine by removal: the run fails and the file is never exported. + unlinkSync(join(evidence, path)); + findings.push(path); + } + } + return findings; +} + +function manifest(evidence) { + return listEvidenceFiles(evidence) + .filter((path) => path !== "result.json") + .map((path) => { + const bytes = readFileSync(join(evidence, path)); + return { path, sha256: sha256(bytes), bytes: bytes.length }; + }); +} + +function listEvidenceFiles(directory, prefix = "") { + const files = []; + for (const entry of readdirSync(directory, { withFileTypes: true }).sort((left, right) => left.name.localeCompare(right.name))) { + const path = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) files.push(...listEvidenceFiles(join(directory, entry.name), path)); + else if (entry.isFile() && !entry.name.includes(".tmp-")) files.push(path); + } + return files; +} + +function copyRegularFiles(source, target) { + let copied = 0; + for (const entry of readdirSync(source)) { + const from = join(source, entry); + const stats = lstatSync(from); + if (stats.isDirectory()) copied += copyRegularFiles(from, join(target, entry)); + else if (stats.isFile()) { + mkdirSync(target, { recursive: true }); + copyFileSync(from, join(target, entry)); + copied += 1; + } + } + return copied; +} + +function containerName(runId, jobId, suffix) { + return `myskills-ci-${runId}-${jobId}-${suffix}`; +} + +function git(args, cwd = sourceRoot) { + return spawnSync("git", ["-c", "core.hooksPath=/dev/null", ...args], { cwd, encoding: "utf8", env: jobEnvironment(null) }); +} + +function docker(args) { + return spawnSync("docker", args, { encoding: "utf8", env: jobEnvironment(null), timeout: 120_000 }); +} + +function findOnPath(name) { + for (const directory of (process.env.PATH ?? "").split(delimiter)) { + if (directory && isExecutable(join(directory, name))) return join(directory, name); + } + return null; +} + +function isExecutable(path) { + try { + accessSync(path, constants.X_OK); + return statSync(path).isFile(); + } catch { + return false; + } +} + +function realPathAllowingMissing(path) { + const missing = []; + let current = path; + while (!existsSync(current)) { + missing.unshift(basename(current)); + const parent = dirname(current); + if (parent === current) break; + current = parent; + } + return join(realpathSync(current), ...missing); +} + +function isWithin(child, parent) { + const path = relative(parent, child); + return path === "" || (!isAbsolute(path) && path !== ".." && !path.startsWith(`..${sep}`)); +} + +function writeJsonAtomic(path, value) { + const temporary = `${path}.tmp-${process.pid}`; + writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`); + renameSync(temporary, path); +} + +function safeJson(text) { + try { + return JSON.parse(text); + } catch { + return null; + } +} + +function sha256(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function delay(ms) { + return new Promise((resolvePromise) => setTimeout(resolvePromise, ms)); +} + +// Invoked last so every class and job table above is initialized first. +main().then((code) => { + // Let piped output drain; the unref'd timer only bounds a stuck handle. + process.exitCode = code; + setTimeout(() => process.exit(code), 5_000).unref(); +}, (error) => { + console.error(`local-ci: internal error: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; +}); diff --git a/scripts/local-ci.sh b/scripts/local-ci.sh new file mode 100755 index 00000000..17ba3fbe --- /dev/null +++ b/scripts/local-ci.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Portable CI and release-check entrypoint. The contract is in docs/LOCAL_CI.md. +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +node_bin="" +for candidate in "${LOCAL_CI_NODE22_BIN:-}" "${LOCAL_CI_NODE24_BIN:-}"; do + if [ -n "$candidate" ] && [ -x "$candidate/node" ]; then + node_bin="$candidate/node" + break + fi +done +if [ -z "$node_bin" ]; then + node_bin="$(command -v node || true)" +fi +if [ -z "$node_bin" ]; then + echo "local-ci: node was not found. Set LOCAL_CI_NODE22_BIN or LOCAL_CI_NODE24_BIN, or put node on PATH." >&2 + exit 2 +fi + +# exec keeps the orchestrator as the signalled process so cancellation cleans up. +exec "$node_bin" "$script_dir/local-ci.mjs" "$@" diff --git a/scripts/run-fullstack-e2e.mjs b/scripts/run-fullstack-e2e.mjs index 07171f5e..61b28fb0 100644 --- a/scripts/run-fullstack-e2e.mjs +++ b/scripts/run-fullstack-e2e.mjs @@ -5,7 +5,12 @@ import { fileURLToPath } from "node:url"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const composeFile = resolve(root, "docker-compose.e2e.yml"); -const projectName = `myskills-beta2-e2e-${process.pid}-${randomBytes(4).toString("hex")}`; +// CI runners pass an exact per-run project so they can clean up after a hard stop. +const projectName = process.env.MYSKILLS_E2E_COMPOSE_PROJECT ?? `myskills-beta2-e2e-${process.pid}-${randomBytes(4).toString("hex")}`; +if (!/^[a-z0-9][a-z0-9_-]{0,62}$/.test(projectName)) { + console.error("MYSKILLS_E2E_COMPOSE_PROJECT must be a lowercase Docker Compose project name of at most 63 characters."); + process.exit(1); +} const webPort = process.env.MYSKILLS_E2E_WEB_PORT ?? "43100"; const mailpitPort = process.env.MYSKILLS_E2E_MAILPIT_PORT ?? "43101"; const baseURL = `http://127.0.0.1:${webPort}`; @@ -25,6 +30,15 @@ const environment = { MYSKILLS_E2E_POSTGRES_PASSWORD: randomCredential(24), MYSKILLS_E2E_WEB_PORT: webPort, }; +// Container logs can echo these generated values; keep them out of CI output. +const generatedSecrets = [ + environment.MYSKILLS_E2E_AUTH_SECRET, + environment.MYSKILLS_E2E_INVITEE_PASSWORD, + environment.MYSKILLS_E2E_MINIO_ROOT_PASSWORD, + environment.MYSKILLS_E2E_MINIO_ROOT_USER, + environment.MYSKILLS_E2E_OWNER_PASSWORD, + environment.MYSKILLS_E2E_POSTGRES_PASSWORD, +]; let teardownStarted = false; @@ -41,6 +55,7 @@ try { const owner = await prepareOwnerMfa(); environment.MYSKILLS_E2E_OWNER_RECOVERY_CODES = JSON.stringify(owner.recoveryCodes); environment.MYSKILLS_ACCEPTANCE_OWNER_TOKEN = owner.sessionToken; + generatedSecrets.push(owner.sessionToken, ...owner.recoveryCodes); await run(resolve(root, "node_modules/.bin/playwright"), [ "test", "--config", @@ -183,10 +198,12 @@ function run(command, args, options = {}) { const child = spawn(command, args, { cwd: root, env: environment, - stdio: "inherit", + stdio: ["inherit", "pipe", "pipe"], }); + forwardRedacted(child.stdout, process.stdout); + forwardRedacted(child.stderr, process.stderr); child.once("error", rejectPromise); - child.once("exit", (code, signal) => { + child.once("close", (code, signal) => { if (code === 0 || options.allowFailure) { resolvePromise(); return; @@ -195,3 +212,25 @@ function run(command, args, options = {}) { }); }); } + +function forwardRedacted(input, output) { + // Line buffering keeps a value split across chunks from escaping redaction. + let pending = ""; + input.setEncoding("utf8"); + input.on("data", (chunk) => { + const lines = (pending + chunk).split("\n"); + pending = lines.pop(); + for (const line of lines) output.write(`${redact(line)}\n`); + }); + input.on("end", () => { + if (pending) output.write(redact(pending)); + }); +} + +function redact(text) { + let redacted = text; + for (const secret of generatedSecrets) { + if (secret) redacted = redacted.split(secret).join("[redacted]"); + } + return redacted; +} diff --git a/scripts/scan-secrets.mjs b/scripts/scan-secrets.mjs index 2ff1c02f..e8162926 100644 --- a/scripts/scan-secrets.mjs +++ b/scripts/scan-secrets.mjs @@ -3,15 +3,9 @@ import { execFileSync } from "node:child_process"; import { closeSync, constants, fstatSync, openSync, readFileSync } from "node:fs"; import { join } from "node:path"; +import { secretPatterns as patterns } from "./lib/secret-patterns.mjs"; const root = repoRoot(); -const patterns = [ - { name: "Vendor API token", pattern: /\bATATT[0-9A-Za-z_-]{20,}\b/ }, - { name: "GitHub token", pattern: /\b(?:ghp|gho|ghu|ghs|ghr)_[0-9A-Za-z_]{30,}\b/ }, - { name: "OpenAI API key", pattern: /\bsk-[A-Za-z0-9_-]{32,}\b/ }, - { name: "Private key block", pattern: /-----BEGIN (?:RSA |EC |OPENSSH |)PRIVATE KEY-----/ }, - { name: "AWS access key", pattern: /\bAKIA[0-9A-Z]{16}\b/ }, -]; const findings = []; for (const file of scanCandidates()) { diff --git a/scripts/test/fullstack-e2e-isolation.test.mjs b/scripts/test/fullstack-e2e-isolation.test.mjs new file mode 100644 index 00000000..153fdca9 --- /dev/null +++ b/scripts/test/fullstack-e2e-isolation.test.mjs @@ -0,0 +1,83 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; + +// Failure cases: generated full-stack credentials echoed by Docker or Compose output reach +// CI logs; a CI runner cannot name, and so cannot exactly clean up, the Compose project; an +// unsafe project name reaches the Docker CLI. +const credentialNames = [ + "MYSKILLS_E2E_AUTH_SECRET", + "MYSKILLS_E2E_INVITEE_PASSWORD", + "MYSKILLS_E2E_MINIO_ROOT_PASSWORD", + "MYSKILLS_E2E_MINIO_ROOT_USER", + "MYSKILLS_E2E_OWNER_PASSWORD", + "MYSKILLS_E2E_POSTGRES_PASSWORD", +]; + +test("full-stack E2E redacts generated credentials and uses the supplied Compose project", (t) => { + const fixture = fakeDocker(t); + const project = "myskills-ci-fixture-web-e2e-node22"; + const result = runFullstack(fixture, { MYSKILLS_E2E_COMPOSE_PROJECT: project }); + assert.notEqual(result.status, 0, "the fake Compose config step fails, so the run must fail"); + + const records = fixture.records(); + assert.ok(records.length >= 3, "config, diagnostics and teardown should all reach Docker"); + const credentials = credentialNames.map((name) => records[0].env[name]); + assert.ok(credentials.every((value) => typeof value === "string" && value.length >= 6)); + const output = `${result.stdout}\n${result.stderr}`; + for (const value of credentials) assert.equal(output.includes(value), false); + assert.match(output, /\[redacted\]/); + + for (const record of records) { + assert.deepEqual(record.args.slice(0, 3), ["compose", "--project-name", project]); + } + assert.ok(records.some(({ args }) => args.includes("down") && args.includes("--volumes") && args.includes("--remove-orphans"))); +}); + +test("full-stack E2E rejects an unsafe Compose project name before invoking Docker", (t) => { + const fixture = fakeDocker(t); + const result = runFullstack(fixture, { MYSKILLS_E2E_COMPOSE_PROJECT: "Bad Name;touch pwned" }); + assert.notEqual(result.status, 0); + assert.deepEqual(fixture.records(), []); + assert.doesNotMatch(`${result.stdout}\n${result.stderr}`, /touch pwned/); +}); + +function fakeDocker(t) { + const root = mkdtempSync(join(tmpdir(), "myskills-fullstack-isolation-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + const bin = join(root, "bin"); + mkdirSync(bin); + const recordPath = join(root, "record.jsonl"); + const script = join(root, "fake-docker.mjs"); + writeFileSync(script, ` +import { appendFileSync } from "node:fs"; +const args = process.argv.slice(2); +const names = ${JSON.stringify(credentialNames)}; +const env = Object.fromEntries(names.map((name) => [name, process.env[name]])); +appendFileSync(${JSON.stringify(recordPath)}, JSON.stringify({ args, env }) + "\\n"); +for (const name of names) { + process.stdout.write(name + " startup banner: " + process.env[name] + "\\n"); + process.stderr.write("diagnostic " + process.env[name] + "\\n"); +} +process.exit(args.includes("config") ? 1 : 0); +`); + writeFileSync(join(bin, "docker"), `#!/bin/sh\nexec '${process.execPath}' '${script}' "$@"\n`, { mode: 0o755 }); + return { + bin, + records: () => existsSync(recordPath) + ? readFileSync(recordPath, "utf8").trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)) + : [], + }; +} + +function runFullstack(fixture, env) { + return spawnSync(process.execPath, [resolve("scripts/run-fullstack-e2e.mjs")], { + cwd: resolve("."), + encoding: "utf8", + timeout: 60_000, + env: { PATH: `${fixture.bin}:${process.env.PATH}`, HOME: process.env.HOME ?? tmpdir(), ...env }, + }); +} diff --git a/scripts/test/local-ci.test.mjs b/scripts/test/local-ci.test.mjs new file mode 100644 index 00000000..e2fd054a --- /dev/null +++ b/scripts/test/local-ci.test.mjs @@ -0,0 +1,720 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawn, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, relative, resolve, sep } from "node:path"; +import test from "node:test"; + +// Failure cases (written before scripts/local-ci.mjs existed). The real entrypoint runs +// against a temporary git fixture; only docker, npm, npx and codeql are fakes that record +// calls and simulate tool outcomes. +// - Unsafe run IDs, evidence paths inside or around the source, stale or malformed source SHAs, +// dirty trees, unknown modes or jobs, and reused evidence directories start work or resources. +// - A job failure hides later jobs, or a failed, partial or unpinned run reports required contexts. +// - A missing Node line or wrong npm version is silently replaced by another toolchain. +// - Web E2E exit 0 without a browser report passes, or full-stack runs after a failed step. +// - Cleanup misses a created container, Compose project or image, removes a container it did not +// create after a name conflict, or matches by prune or name prefix. +// - Cancellation leaves containers, step processes or the per-run workspace behind. +// - Runner credentials reach job processes, or credential-shaped output reaches exported evidence. +// - release-check accepts a wrong tag, a tag not at HEAD, or a commit outside main. +// - Tampered release artifacts pass, or release-check publishes images or packages. +// - CodeQL output that ignores the repository query filters passes. + +const runId = "fixture-run"; +const rootPackage = JSON.parse(readFileSync(resolve("package.json"), "utf8")); +const releaseTag = `v${rootPackage.version}`; +const verifyJobs = ["check-node22", "check-node24", "web-e2e-node22", "web-e2e-node24", "postgres-node22", "postgres-node24", "railway-images"]; +const fixtureFiles = [ + "package.json", + ".gitignore", + ".github/codeql/codeql-config.yml", + "scripts/local-ci.sh", + "scripts/local-ci.mjs", + "scripts/lib/secret-patterns.mjs", + "scripts/collect-browser-evidence.mjs", + "scripts/create-release-artifacts.mjs", + "scripts/verify-release.mjs", +]; +const webBuild = "run build -w @myskills-app/core -w @myskills-app/auth -w @myskills-app/skill-package -w @myskills-app/api"; +const mockedBrowser = "run test:e2e -w @myskills-app/web -- --reporter=line,json"; + +test("unsafe or stale inputs are rejected before any tool, container or workspace is used", (t) => { + const fixture = makeFixture(t); + const secretLookingId = "$(touch pwned)"; + const rows = [ + { name: "missing run ID", env: { LOCAL_CI_RUN_ID: undefined }, stderr: /LOCAL_CI_RUN_ID/ }, + { name: "path run ID", env: { LOCAL_CI_RUN_ID: "../escape" }, stderr: /LOCAL_CI_RUN_ID/ }, + { name: "uppercase run ID", env: { LOCAL_CI_RUN_ID: "Fixture-Run" }, stderr: /LOCAL_CI_RUN_ID/ }, + { name: "shell run ID", env: { LOCAL_CI_RUN_ID: secretLookingId }, stderr: /LOCAL_CI_RUN_ID/ }, + { name: "relative evidence", evidence: "evidence", stderr: /LOCAL_CI_EVIDENCE_DIR must be absolute/ }, + { name: "evidence inside source", evidence: join(fixture.source, "evidence"), stderr: /outside the source/ }, + { name: "evidence through symlink", evidence: join(fixture.root, "source-link", "evidence"), stderr: /outside the source/ }, + { name: "evidence containing source", evidence: fixture.root, stderr: /outside the source/ }, + { name: "unknown mode", args: ["deploy"], stderr: /Usage/ }, + { name: "unknown job", args: ["verify", "--job", "nope"], stderr: /Unknown job/ }, + { name: "malformed SHA", env: { LOCAL_CI_SOURCE_SHA: "abc123" }, reason: "invalid-source-sha" }, + { name: "stale SHA", env: { LOCAL_CI_SOURCE_SHA: "0".repeat(40) }, reason: "source-sha-mismatch" }, + { name: "CodeQL without CLI", args: ["codeql"], reason: "codeql-cli-unavailable" }, + ]; + symlinkSync(fixture.source, join(fixture.root, "source-link")); + for (const row of rows) { + const run = runLocalCi(fixture, row.args ?? ["verify"], { env: row.env, evidence: row.evidence }); + assert.equal(run.status, 2, `${row.name}: ${run.output}`); + if (row.stderr) assert.match(run.stderr, row.stderr, row.name); + if (row.reason) { + assert.equal(run.result?.status, "rejected", row.name); + assert.equal(run.result.reason, row.reason, row.name); + } else if (row.evidence !== fixture.root) { + assert.equal(existsSync(join(resolve(fixture.root, row.evidence ?? run.evidence), "result.json")), false, row.name); + } + assert.doesNotMatch(run.output, /touch pwned/, row.name); + assert.deepEqual(fixture.records(), [], `${row.name} must not invoke tools`); + assert.equal(existsSync(fixture.runWorkspace), false, row.name); + } + assert.equal(existsSync(join(fixture.source, "evidence")), false); + + const reused = join(fixture.root, "reused"); + mkdirSync(reused); + writeFileSync(join(reused, "result.json"), "sentinel\n"); + const reuse = runLocalCi(fixture, ["verify"], { evidence: reused }); + assert.equal(reuse.status, 2); + assert.match(reuse.stderr, /already contains result\.json/); + assert.equal(readFileSync(join(reused, "result.json"), "utf8"), "sentinel\n"); + + writeFileSync(join(fixture.source, "untracked.txt"), "dirty\n"); + const dirty = runLocalCi(fixture, ["verify"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha } }); + rmSync(join(fixture.source, "untracked.txt")); + assert.equal(dirty.status, 2); + assert.equal(dirty.result?.reason, "dirty-source"); + assert.deepEqual(fixture.records(), []); +}); + +test("verify runs every required job on both Node lines and reports gating contexts", (t) => { + const fixture = makeFixture(t); + const canary = `runner-credential-${createHash("sha256").update(fixture.root).digest("hex").slice(0, 16)}`; + fixture.configure({ canary }); + const run = runLocalCi(fixture, ["verify"], { + env: { LOCAL_CI_SOURCE_SHA: fixture.sha, GITHUB_TOKEN: canary, GH_TOKEN: canary, NPM_TOKEN: canary }, + }); + assert.equal(run.status, 0, run.output); + const { result } = run; + assert.equal(result.status, "passed"); + assert.equal(result.gating, true); + assert.deepEqual(result.gatingBlockers, []); + assert.equal(result.complete, true); + assert.equal(result.source.sha, fixture.sha); + assert.deepEqual(result.jobs.map(({ id }) => id).sort(), [...verifyJobs].sort()); + assert.ok(result.jobs.every(({ status }) => status === "passed")); + assert.deepEqual(result.contexts, { check: "passed", "web-e2e": "passed", "postgres-integration": "passed" }); + + const records = fixture.records(); + const npm = records.filter(({ tool, args }) => tool === "npm" && args[0] !== "--version"); + for (const line of ["22", "24"]) { + const commands = new Set(npm.filter((record) => record.line === line).map(({ args }) => args.join(" "))); + for (const expected of ["ci", "run check", webBuild, mockedBrowser, "run test:e2e:fullstack", "run test:postgres"]) { + assert.ok(commands.has(expected), `Node ${line} must run ${expected}`); + } + } + for (const record of npm) { + assert.equal(record.head, fixture.sha, "every job runs in a clone of the pinned commit"); + assert.equal(isInside(record.cwd, fixture.source), false, "jobs must not run in the caller's checkout"); + assert.equal(record.env.CI, "true"); + } + for (const record of npm.filter(({ args }) => args.join(" ") === "run test:postgres")) { + assert.match(record.env.TEST_DATABASE_URL, /^postgres:\/\/myskills_test:myskills_test@127\.0\.0\.1:55432\/myskills_test$/); + } + const projects = npm.filter(({ args }) => args.join(" ") === "run test:e2e:fullstack").map(({ env }) => env.MYSKILLS_E2E_COMPOSE_PROJECT); + assert.equal(new Set(projects).size, 2); + assert.ok(projects.every((project) => project.includes(runId))); + assert.equal(records.some(({ canarySeen }) => canarySeen), false, "runner credentials must not reach jobs"); + + const docker = records.filter(({ tool }) => tool === "docker"); + assertExactCleanup(docker, projects); + assert.ok(docker.some(({ args }) => args[0] === "run" && args.includes("postgres:17-alpine"))); + const builds = docker.filter(({ args }) => args[0] === "build").map(({ args }) => args.join(" ")); + assert.ok(builds.some((build) => build.includes("--file Dockerfile.api"))); + assert.ok(builds.some((build) => build.includes("--file Dockerfile.web") && build.includes("--build-arg VITE_API_BASE_URL=/api"))); + assert.ok(builds.some((build) => build.includes("--file Dockerfile.backup"))); + assert.equal(docker.filter(({ args }) => args[0] === "run" && args.includes("--rm") && args.includes("--network") && args.includes("none")).length, 2); + assertNoPublication(records); + + for (const line of ["22", "24"]) { + for (const phase of ["mocked", "fullstack"]) { + const summary = JSON.parse(readFileSync(join(run.evidence, "browser-evidence", `web-e2e-node${line}`, phase, "summary.json"), "utf8")); + assert.equal(summary.reportStatus, "available"); + } + } + assertEvidenceManifest(run.evidence, result); + for (const job of result.jobs) { + assert.equal(sha256(readFileSync(join(run.evidence, job.log.path))), job.log.sha256); + } + const resources = JSON.parse(readFileSync(join(run.evidence, "resources.json"), "utf8")); + assert.ok(resources.resources.length > 0); + assert.ok(resources.resources.every(({ state }) => state === "removed"), JSON.stringify(resources)); + assert.equal(existsSync(fixture.runWorkspace), false); + assert.equal(listFiles(run.evidence).some((file) => readFileSync(join(run.evidence, file)).includes(canary)), false); + assert.equal(run.output.includes(canary), false); +}); + +test("job failures are isolated, fail the gating contexts and never remove another run's container", (t) => { + const fixture = makeFixture(t); + const conflicting = `myskills-ci-${runId}-postgres-node22-postgres`; + fixture.configure({ + rules: [{ tool: "npm", line: "24", prefix: "run check", exit: 1 }], + omitMockedReport: ["22"], + dockerConflicts: [conflicting], + }); + const run = runLocalCi(fixture, ["verify"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha } }); + assert.equal(run.status, 1, run.output); + const { result } = run; + assert.equal(result.status, "failed"); + assert.equal(result.gating, true, "a pinned complete run is authoritative even when it fails"); + const job = (id) => result.jobs.find((candidate) => candidate.id === id); + assert.equal(job("check-node22").status, "passed"); + assert.equal(job("check-node24").status, "failed"); + assert.equal(job("web-e2e-node22").status, "failed"); + assert.equal(job("web-e2e-node22").steps.find(({ name }) => name === "fullstack-browser").status, "skipped"); + assert.equal(job("web-e2e-node24").status, "passed"); + assert.equal(job("postgres-node22").status, "failed"); + assert.equal(job("postgres-node22").reason, "service-unavailable"); + assert.equal(job("postgres-node24").status, "passed"); + assert.equal(job("railway-images").status, "passed"); + assert.deepEqual(result.contexts, { check: "failed", "web-e2e": "failed", "postgres-integration": "failed" }); + + const records = fixture.records(); + const npmCommands = (line) => records.filter((record) => record.tool === "npm" && record.line === line).map(({ args }) => args.join(" ")); + assert.equal(npmCommands("22").includes("run test:e2e:fullstack"), false); + assert.equal(npmCommands("22").includes("run test:postgres"), false); + assert.ok(npmCommands("24").includes("run test:e2e:fullstack")); + const mocked = JSON.parse(readFileSync(join(run.evidence, "browser-evidence", "web-e2e-node22", "mocked", "summary.json"), "utf8")); + assert.equal(mocked.reportStatus, "unavailable"); + const docker = records.filter(({ tool }) => tool === "docker"); + assert.ok(docker.some(({ args }) => args[0] === "run" && args.includes(conflicting)), "the conflicting create must have been attempted"); + const projects = records.filter(({ env }) => env.MYSKILLS_E2E_COMPOSE_PROJECT).map(({ env }) => env.MYSKILLS_E2E_COMPOSE_PROJECT); + assertExactCleanup(docker, projects, { conflicts: [conflicting] }); + assertEvidenceManifest(run.evidence, result); + assert.equal(existsSync(fixture.runWorkspace), false); +}); + +test("partial or unpinned runs never report the required contexts", (t) => { + const fixture = makeFixture(t); + const run = runLocalCi(fixture, ["verify", "--job", "check-node22"]); + assert.equal(run.status, 0, run.output); + assert.equal(run.result.status, "passed"); + assert.equal(run.result.complete, false); + assert.equal(run.result.gating, false); + assert.deepEqual([...run.result.gatingBlockers].sort(), ["partial-job-selection", "source-sha-not-supplied"]); + assert.equal(run.result.contexts, null); + const records = fixture.records(); + assert.ok(records.every(({ tool, line }) => tool === "npm" && line === "22")); +}); + +test("a missing or mismatched toolchain fails its jobs without substituting another version", (t) => { + const fixture = makeFixture(t); + fixture.configure({ npmVersion: { 24: "11.0.0" } }); + const run = runLocalCi(fixture, ["verify", "--job", "check-node22", "--job", "check-node24"], { + env: { + LOCAL_CI_NODE22_BIN: undefined, + PATH: `${fixture.bin}:${join(fixture.root, "oldnode")}:${process.env.PATH}`, + }, + }); + assert.equal(run.status, 1, run.output); + const job = (id) => run.result.jobs.find((candidate) => candidate.id === id); + assert.equal(job("check-node22").status, "failed"); + assert.equal(job("check-node22").reason, "toolchain-unavailable"); + assert.equal(job("check-node24").status, "failed"); + assert.equal(job("check-node24").reason, "toolchain-mismatch"); + assert.deepEqual(fixture.records().filter(({ args }) => args[0] !== "--version"), []); +}); + +test("cancellation stops the running step and removes exactly the run's resources", async (t) => { + const fixture = makeFixture(t); + fixture.configure({ rules: [{ tool: "npm", prefix: "run test:postgres", sleepMs: 60_000, exit: 0 }] }); + const evidence = fixture.newEvidence(); + const child = spawn("bash", [join(fixture.source, "scripts/local-ci.sh"), "verify", "--job", "postgres-node22"], { + cwd: fixture.source, + env: fixture.env({ LOCAL_CI_EVIDENCE_DIR: evidence, LOCAL_CI_SOURCE_SHA: fixture.sha }), + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + child.stdout.on("data", (chunk) => { output += chunk; }); + child.stderr.on("data", (chunk) => { output += chunk; }); + const exited = new Promise((resolvePromise) => child.once("close", (code, signal) => resolvePromise({ code, signal }))); + const step = await waitFor(() => fixture.records().find(({ args }) => args.join(" ") === "run test:postgres")); + child.kill("SIGTERM"); + const { code } = await exited; + assert.equal(code, 143, output); + const result = JSON.parse(readFileSync(join(evidence, "result.json"), "utf8")); + assert.equal(result.status, "cancelled"); + assert.equal(isAlive(step.pid), false, "the step process group must be stopped"); + const container = `myskills-ci-${runId}-postgres-node22-postgres`; + assert.ok(fixture.records().some(({ tool, args }) => tool === "docker" && args[0] === "rm" && args.includes(container))); + const resources = JSON.parse(readFileSync(join(evidence, "resources.json"), "utf8")); + assert.ok(resources.resources.every(({ state }) => state === "removed"), JSON.stringify(resources)); + assert.equal(existsSync(fixture.runWorkspace), false); +}); + +test("credential-shaped output is redacted from exported evidence and fails the run and its contexts", (t) => { + const fixture = makeFixture(t); + const token = ["gh", "p_", "Fixture0123456789abcdefghijklmnopqrstuv"].join(""); + fixture.configure({ rules: [{ tool: "npm", line: "22", prefix: "run check", stdout: `leaked ${token}` }] }); + const run = runLocalCi(fixture, ["verify"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha } }); + assert.equal(run.status, 1, run.output); + assert.equal(run.result.status, "failed"); + assert.ok(run.result.failureReasons.some((reason) => reason.startsWith("secret-pattern-redacted"))); + // Every job exited 0; a run-level failure must still never surface as a passing context. + assert.ok(run.result.jobs.every(({ status }) => status === "passed")); + assert.deepEqual(run.result.contexts, { check: "failed", "web-e2e": "failed", "postgres-integration": "failed" }); + assert.equal(run.output.includes(token), false); + for (const file of listFiles(run.evidence)) { + assert.equal(readFileSync(join(run.evidence, file), "utf8").includes(token), false, file); + } + assert.match(readFileSync(join(run.evidence, "logs", "check-node22.log"), "utf8"), /\[redacted\]/); +}); + +test("release-check rejects wrong tags, moved tags and commits outside main before any work", (t) => { + const fixture = makeFixture(t, { tag: true }); + const pinned = { LOCAL_CI_SOURCE_SHA: fixture.sha }; + const rows = [ + { name: "tag missing", env: pinned, reason: "release-tag-required" }, + { name: "tag differs from version", env: { ...pinned, LOCAL_CI_RELEASE_TAG: "v0.0.1" }, reason: "release-tag-version-mismatch" }, + { name: "main ref missing", env: { ...pinned, LOCAL_CI_RELEASE_TAG: releaseTag, LOCAL_CI_MAIN_REF: "refs/remotes/origin/absent" }, reason: "main-ref-unavailable" }, + ]; + for (const row of rows) { + const run = runLocalCi(fixture, ["release-check"], { env: row.env }); + assert.equal(run.status, 2, `${row.name}: ${run.output}`); + assert.equal(run.result.reason, row.reason, row.name); + } + + const orphan = git(fixture.source, "commit-tree", `${fixture.sha}^{tree}`, "-m", "unrelated main"); + git(fixture.source, "update-ref", "refs/remotes/origin/main", orphan); + const outside = runLocalCi(fixture, ["release-check"], { env: { ...pinned, LOCAL_CI_RELEASE_TAG: releaseTag } }); + assert.equal(outside.status, 2, outside.output); + assert.equal(outside.result.reason, "not-on-main"); + + git(fixture.source, "commit", "-q", "--allow-empty", "-m", "after tag"); + const moved = git(fixture.source, "rev-parse", "HEAD"); + git(fixture.source, "update-ref", "refs/remotes/origin/main", moved); + const notAtHead = runLocalCi(fixture, ["release-check"], { env: { LOCAL_CI_SOURCE_SHA: moved, LOCAL_CI_RELEASE_TAG: releaseTag } }); + assert.equal(notAtHead.status, 2, notAtHead.output); + assert.equal(notAtHead.result.reason, "release-tag-not-at-head"); + assert.deepEqual(fixture.records(), []); +}); + +test("release-check verifies tagged artifacts and release images without publishing", (t) => { + const fixture = makeFixture(t, { tag: true }); + const run = runLocalCi(fixture, ["release-check"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha, LOCAL_CI_RELEASE_TAG: releaseTag } }); + assert.equal(run.status, 0, run.output); + assert.equal(run.result.status, "passed"); + assert.equal(run.result.gating, true); + assert.equal(run.result.release.tag, releaseTag); + assert.equal(run.result.release.mainSha, fixture.sha); + + const records = fixture.records(); + const verify = records.find(({ tool, args }) => tool === "npm" && args.join(" ") === "run release:verify"); + assert.equal(verify.line, "22"); + assert.equal(verify.env.RELEASE_REQUIRE_TAG, "true"); + assert.equal(verify.env.RELEASE_EXPECTED_TAG, releaseTag); + assert.match(verify.env.TEST_DATABASE_URL, /@127\.0\.0\.1:55432\/myskills_test$/); + const docker = records.filter(({ tool }) => tool === "docker"); + assert.ok(docker.some(({ args }) => args[0] === "run" && args.includes("postgres:17"))); + const builds = docker.filter(({ args }) => args[0] === "build").map(({ args }) => args.join(" ")); + for (const target of ["api", "mcp-http"]) assert.ok(builds.some((build) => build.includes(`--target ${target} `)), target); + assert.ok(builds.some((build) => build.includes("--target web ") && build.includes("--build-arg VITE_API_BASE_URL=/api"))); + for (const file of ["Dockerfile.api", "Dockerfile.web", "Dockerfile.backup"]) assert.ok(builds.some((build) => build.includes(`--file ${file} `)), file); + assert.equal(docker.filter(({ args }) => args[0] === "run" && args.includes("--rm") && args.includes("none")).length, 2); + assertExactCleanup(docker, [verify.env.MYSKILLS_E2E_COMPOSE_PROJECT]); + assertNoPublication(records); + + const artifacts = join(run.evidence, "release", "artifacts"); + const archive = `${rootPackage.name}-${rootPackage.version}-source.tar`; + assert.deepEqual(readdirSync(artifacts).sort(), [archive, "SHA256SUMS", "release-metadata.json"].sort()); + const sums = readFileSync(join(artifacts, "SHA256SUMS"), "utf8").trim().split("\n"); + for (const line of sums) { + const [digest, file] = line.split(/\s+/); + assert.equal(sha256(readFileSync(join(artifacts, file))), digest, file); + } + const metadata = JSON.parse(readFileSync(join(artifacts, "release-metadata.json"), "utf8")); + assert.equal(metadata.commitSha, fixture.sha); + assert.equal(metadata.dirty, false); + assert.ok(metadata.tags.includes(releaseTag)); + assertEvidenceManifest(run.evidence, run.result); + assert.equal(existsSync(fixture.runWorkspace), false); +}); + +test("release-check rejects tampered release artifacts", (t) => { + const fixture = makeFixture(t, { tag: true }); + fixture.configure({ tamperReleaseArchive: true }); + const run = runLocalCi(fixture, ["release-check"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha, LOCAL_CI_RELEASE_TAG: releaseTag } }); + assert.equal(run.status, 1, run.output); + const job = run.result.jobs.find(({ id }) => id === "release"); + assert.equal(job.status, "failed"); + assert.equal(job.reason, "artifact-verification-failed"); + assert.equal(existsSync(join(run.evidence, "release", "artifacts")), false); + const resources = JSON.parse(readFileSync(join(run.evidence, "resources.json"), "utf8")); + assert.ok(resources.resources.every(({ state }) => state === "removed"), JSON.stringify(resources)); +}); + +test("CodeQL applies the repository query filters and fails closed when they are ignored", (t) => { + const fixture = makeFixture(t); + const codeql = join(fixture.bin, "codeql"); + // Two findings stand in for alerts that GitHub dismissed; local counts must not gate. + fixture.configure({ codeqlFindings: 2 }); + const passed = runLocalCi(fixture, ["codeql"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha, LOCAL_CI_CODEQL_BIN: codeql } }); + assert.equal(passed.status, 0, passed.output); + const repositoryConfig = readFileSync(join(fixture.source, ".github/codeql/codeql-config.yml"), "utf8"); + const derived = readFileSync(join(fixture.root, "codeql-config.yml"), "utf8"); + assert.ok(derived.startsWith(repositoryConfig), "repository query filters are preserved verbatim"); + assert.match(derived.slice(repositoryConfig.length), /^\s*queries:\n\s+- uses: security-extended\n$/); + const records = fixture.records().filter(({ tool }) => tool === "codeql"); + const create = records.find(({ args }) => args[1] === "create"); + assert.ok(create.args.includes("--language=javascript-typescript") && create.args.includes("--build-mode=none")); + const analyze = records.find(({ args }) => args[1] === "analyze"); + assert.ok(analyze.args.includes("--sarif-category=/language:javascript-typescript")); + assert.equal(analyze.args.some((arg) => arg.endsWith(".qls")), false, "queries come from the derived config"); + assert.ok(listFiles(passed.evidence).some((file) => file.endsWith(".sarif"))); + assert.equal(passed.result.jobs[0].codeql.results, 2); + + fixture.configure({ codeqlRules: ["js/sql-injection", "js/missing-rate-limiting"] }); + const ignored = runLocalCi(fixture, ["codeql"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha, LOCAL_CI_CODEQL_BIN: codeql } }); + assert.equal(ignored.status, 1, ignored.output); + assert.equal(ignored.result.jobs[0].reason, "codeql-filter-not-applied"); +}); + +function makeFixture(t, { tag = false } = {}) { + const root = realpathSync(mkdtempSync(join(tmpdir(), "myskills-local-ci-"))); + t.after(() => rmSync(root, { recursive: true, force: true })); + const source = join(root, "source"); + for (const file of fixtureFiles) { + mkdirSync(dirname(join(source, file)), { recursive: true }); + copyFileSync(resolve(file), join(source, file)); + } + chmodSync(join(source, "scripts/local-ci.sh"), 0o755); + git(source, "init", "-q", "-b", "main"); + git(source, "add", "-A"); + git(source, "commit", "-q", "-m", "fixture"); + const sha = git(source, "rev-parse", "HEAD"); + git(source, "update-ref", "refs/remotes/origin/main", sha); + if (tag) git(source, "tag", releaseTag); + + mkdirSync(join(root, "tools")); + writeFileSync(join(root, "tools", "fake-tool.mjs"), `(${fakeToolMain.toString()})();\n`); + const bin = join(root, "bin"); + writeShim(join(bin, "docker"), fakeInvocation(root, "docker")); + writeShim(join(bin, "codeql"), fakeInvocation(root, "codeql")); + for (const line of ["22", "24"]) { + writeShim(join(root, `node${line}`, "node"), nodeShim(`v${line}.99.0`)); + for (const tool of ["npm", "npx"]) writeShim(join(root, `node${line}`, tool), fakeInvocation(root, tool, line)); + } + writeShim(join(root, "oldnode", "node"), nodeShim("v20.0.0")); + const work = join(root, "work"); + mkdirSync(work); + let evidenceCount = 0; + const recordPath = join(root, "record.jsonl"); + const fixture = { + root, + source, + sha, + bin, + runWorkspace: join(work, `myskills-local-ci-${runId}`), + configure: (config) => { + writeFileSync(join(root, "fake-config.json"), JSON.stringify(config)); + rmSync(recordPath, { force: true }); + }, + records: () => existsSync(recordPath) + ? readFileSync(recordPath, "utf8").trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)) + : [], + newEvidence: () => join(root, `evidence-${++evidenceCount}`), + env: (overrides = {}) => { + const env = { + PATH: `${bin}:${process.env.PATH}`, + HOME: process.env.HOME ?? root, + LOCAL_CI_RUN_ID: runId, + LOCAL_CI_WORK_DIR: work, + LOCAL_CI_NODE22_BIN: join(root, "node22"), + LOCAL_CI_NODE24_BIN: join(root, "node24"), + }; + if (process.env.TMPDIR) env.TMPDIR = process.env.TMPDIR; + for (const [name, value] of Object.entries(overrides)) { + if (value === undefined) delete env[name]; + else env[name] = value; + } + return env; + }, + }; + fixture.configure({}); + return fixture; +} + +function runLocalCi(fixture, args, { env = {}, evidence } = {}) { + const evidenceDir = evidence ?? fixture.newEvidence(); + const run = spawnSync("bash", [join(fixture.source, "scripts/local-ci.sh"), ...args], { + cwd: fixture.source, + encoding: "utf8", + timeout: 120_000, + env: fixture.env({ LOCAL_CI_EVIDENCE_DIR: evidenceDir, ...env }), + }); + const result = readJson(join(resolve(fixture.root, evidenceDir), "result.json")); + return { ...run, evidence: evidenceDir, result, output: `${run.stdout}\n${run.stderr}` }; +} + +function readJson(path) { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + // Missing, or a sentinel that the entrypoint must leave untouched. + return null; + } +} + +function assertExactCleanup(docker, projects, { conflicts = [] } = {}) { + const attempted = docker.filter(({ args }) => args[0] === "run" && args.includes("-d")).map(({ args }) => args[args.indexOf("--name") + 1]); + assert.ok(attempted.length > 0); + for (const name of attempted) { + const removed = docker.some(({ args }) => args[0] === "rm" && args.includes("-f") && args.includes(name)); + assert.equal(removed, !conflicts.includes(name), `container ${name}`); + } + const tags = docker.filter(({ args }) => args[0] === "build").map(({ args }) => args[args.indexOf("--tag") + 1]); + assert.ok(tags.length > 0); + for (const tag of tags) { + assert.ok(docker.some(({ args }) => args[0] === "image" && args[1] === "rm" && args.includes(tag)), `image ${tag} must be removed`); + } + for (const project of projects) { + assert.ok(docker.some(({ args }) => args[0] === "ps" && args.includes(`label=com.docker.compose.project=${project}`)), project); + for (const service of ["api", "web"]) { + assert.ok(docker.some(({ args }) => args[0] === "image" && args[1] === "rm" && args.includes(`${project}-${service}`)), `${project}-${service}`); + } + } + for (const { args } of docker) { + assert.equal(args.some((arg) => /prune/.test(arg)), false, args.join(" ")); + assert.notEqual(args[0], "system"); + args.forEach((arg, index) => { + if (args[index - 1] !== "--filter") return; + assert.ok(projects.some((project) => arg === `label=com.docker.compose.project=${project}`), `filter ${arg} must be exact`); + }); + } +} + +function assertNoPublication(records) { + for (const { tool, args } of records) { + assert.equal(tool === "docker" && ["push", "login"].includes(args[0]), false, args.join(" ")); + assert.equal(tool === "npm" && ["publish", "login"].includes(args[0]), false, args.join(" ")); + } +} + +function assertEvidenceManifest(evidence, result) { + const files = listFiles(evidence).filter((file) => file !== "result.json"); + assert.deepEqual(result.artifacts.map(({ path }) => path).sort(), files.sort()); + for (const artifact of result.artifacts) { + assert.equal(sha256(readFileSync(join(evidence, artifact.path))), artifact.sha256, artifact.path); + } +} + +function listFiles(directory, prefix = "") { + const files = []; + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const path = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) files.push(...listFiles(join(directory, entry.name), path)); + else if (entry.isFile()) files.push(path); + } + return files; +} + +function isInside(child, parent) { + // Clone directories are gone after the run; the fixture root is already a real path. + const path = relative(realpathSync(parent), child); + return path === "" || (!path.startsWith("..") && !path.startsWith(sep) && !path.includes(`..${sep}`)); +} + +function isAlive(pid) { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function waitFor(probe, timeoutMs = 30_000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const value = probe(); + if (value) return value; + await new Promise((resolvePromise) => setTimeout(resolvePromise, 50)); + } + throw new Error("Timed out waiting for the fixture step to start."); +} + +function sha256(buffer) { + return createHash("sha256").update(buffer).digest("hex"); +} + +function git(cwd, ...args) { + return execFileSync("git", [ + "-c", "user.name=Fixture", + "-c", "user.email=fixture@example.test", + "-c", "commit.gpgsign=false", + "-c", "tag.gpgsign=false", + "-c", "core.hooksPath=/dev/null", + ...args, + ], { cwd, encoding: "utf8" }).trim(); +} + +function writeShim(path, body) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `#!/bin/sh\n${body}\n`, { mode: 0o755 }); +} + +function fakeInvocation(root, tool, line = "") { + return [ + `FAKE_TOOL_ROOT=${shellQuote(root)}`, + `FAKE_LINE=${line}`, + "export FAKE_TOOL_ROOT FAKE_LINE", + `exec ${shellQuote(process.execPath)} ${shellQuote(join(root, "tools", "fake-tool.mjs"))} ${tool} "$@"`, + ].join("\n"); +} + +function nodeShim(version) { + return `if [ "$1" = "--version" ]; then echo ${version}; exit 0; fi\nexec ${shellQuote(process.execPath)} "$@"`; +} + +function shellQuote(value) { + return `'${value.replaceAll("'", "'\\''")}'`; +} + +// Serialized into each fixture. It records every call and simulates tool outcomes only; +// the entrypoint's decisions (ordering, statuses, cleanup, evidence) are what the tests assert. +async function fakeToolMain() { + const { spawnSync: spawnTool } = await import("node:child_process"); + const { createHash: hash } = await import("node:crypto"); + const fs = await import("node:fs"); + const path = await import("node:path"); + const [tool, ...args] = process.argv.slice(2); + const root = process.env.FAKE_TOOL_ROOT; + const line = process.env.FAKE_LINE || null; + const config = JSON.parse(fs.readFileSync(path.join(root, "fake-config.json"), "utf8")); + const key = args.join(" "); + const env = {}; + for (const name of ["CI", "TEST_DATABASE_URL", "MYSKILLS_E2E_COMPOSE_PROJECT", "RELEASE_REQUIRE_TAG", "RELEASE_EXPECTED_TAG", "PLAYWRIGHT_JSON_OUTPUT_FILE"]) { + if (process.env[name] !== undefined) env[name] = process.env[name]; + } + const canarySeen = Boolean(config.canary) && Object.values(process.env).some((value) => String(value).includes(config.canary)); + const head = tool === "npm" && fs.existsSync(".git") + ? spawnTool("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).stdout.trim() + : undefined; + fs.appendFileSync(path.join(root, "record.jsonl"), `${JSON.stringify({ tool, args, line, cwd: process.cwd(), pid: process.pid, env, canarySeen, head })}\n`); + + const rule = (config.rules ?? []).find((candidate) => candidate.tool === tool + && (!candidate.line || candidate.line === line) && key.startsWith(candidate.prefix)); + if (rule?.stdout) process.stdout.write(`${rule.stdout}\n`); + if (rule?.sleepMs) await new Promise((resolvePromise) => setTimeout(resolvePromise, rule.sleepMs)); + if (rule && rule.exit !== undefined) process.exit(rule.exit); + process.exit(simulate()); + + function simulate() { + if (tool === "npm") { + if (key === "--version") return print((config.npmVersion ?? {})[line] ?? "11.12.1"); + if (key === "run test:e2e -w @myskills-app/web -- --reporter=line,json") { + if (!(config.omitMockedReport ?? []).includes(line)) writeReport("apps/web/test-results/mocked-report.json"); + return 0; + } + if (key === "run test:e2e:fullstack") { + writeReport("apps/web/test-results/fullstack-report.json"); + return 0; + } + if (key === "run release:verify") return real(["scripts/verify-release.mjs"]); + if (key.startsWith("run release:artifacts -- ")) { + const status = real(["scripts/create-release-artifacts.mjs", ...args.slice(3)]); + if (status === 0 && config.tamperReleaseArchive) { + const output = valueAfter("--out"); + const archive = fs.readdirSync(output).find((name) => name.endsWith(".tar")); + fs.appendFileSync(path.join(output, archive), "x"); + } + return status; + } + return 0; + } + if (tool === "docker") { + if (args[0] === "version") return print("28.3.0"); + if (args[0] === "run" && args.includes("-d")) { + if ((config.dockerConflicts ?? []).includes(valueAfter("--name"))) { + process.stderr.write("Conflict. The container name is already in use.\n"); + return 125; + } + return print("c".repeat(64)); + } + if (args[0] === "inspect") return print("healthy"); + if (args[0] === "port") return print("127.0.0.1:55432"); + if (args[0] === "image" && args[1] === "inspect") return print(`sha256:${hash("sha256").update(args.at(-1)).digest("hex")}`); + return 0; + } + if (tool === "codeql") { + if (args[0] === "version") return print(JSON.stringify({ version: "2.99.0" })); + if (args[0] === "database" && args[1] === "create") { + fs.copyFileSync(option("--codescanning-config"), path.join(root, "codeql-config.yml")); + fs.mkdirSync(args[2], { recursive: true }); + return 0; + } + if (args[0] === "database" && args[1] === "analyze") { + const rules = (config.codeqlRules ?? ["js/sql-injection"]).map((id) => ({ id })); + const results = Array.from({ length: config.codeqlFindings ?? 0 }, (_, index) => ({ + ruleId: "js/insufficient-password-hash", + locations: [{ physicalLocation: { artifactLocation: { uri: "packages/auth/src/session-token.ts" }, region: { startLine: 13 + index } } }], + })); + const sarif = { + version: "2.1.0", + runs: [{ + tool: { driver: { name: "CodeQL", rules: [] }, extensions: [{ name: "codeql/javascript-queries", rules }] }, + automationDetails: { id: `${option("--sarif-category")}/` }, + results, + }], + }; + fs.writeFileSync(option("--output"), JSON.stringify(sarif)); + return 0; + } + } + return 0; + } + + function print(text) { + process.stdout.write(`${text}\n`); + return 0; + } + + function writeReport(file) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify({ + errors: [], + suites: [{ title: "fixture", specs: [{ title: "journey", file: "fixture.spec.ts", line: 1, column: 1, tests: [{ + projectName: "chromium", expectedStatus: "passed", status: "expected", results: [{ status: "passed", retry: 0, duration: 1 }], + }] }] }], + })); + } + + function real(scriptArgs) { + return spawnTool(process.execPath, scriptArgs, { stdio: "inherit" }).status ?? 1; + } + + function valueAfter(flag) { + return args[args.indexOf(flag) + 1]; + } + + function option(name) { + const inline = args.find((arg) => arg.startsWith(`${name}=`)); + return inline ? inline.slice(name.length + 1) : valueAfter(name); + } +} From 1ddb88b95696b021c21b50f8bba19d0e6a405e89 Mon Sep 17 00:00:00 2001 From: Jarel Remick Date: Tue, 29 Sep 2026 15:06:53 +1000 Subject: [PATCH 2/4] Use local CI evidence in release and operator guidance --- AGENTS.md | 3 ++- README.md | 1 + SUPPORT.md | 2 +- docs/BUSINESS_SAFE_RELEASE_GOAL.md | 4 ++-- docs/CODEX_CLOUD.md | 12 +++++----- docs/LOCAL_CI.md | 35 ++++++++++++++++++++++-------- docs/RAILWAY_DEPLOYMENT.md | 6 ++--- docs/RELEASE.md | 34 +++++++++++++++++++---------- docs/THREAT_MODEL.md | 8 ++++--- scripts/check-prerelease.mjs | 7 ++++++ scripts/check-structure.mjs | 3 +++ 11 files changed, 78 insertions(+), 37 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4e20eb7c..00ce38f1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,7 @@ # MySkills Agent Instructions Version: 1.0.0 -Last updated: 2026-06-19 +Last updated: 2026-09-29 ## Source Of Truth @@ -42,6 +42,7 @@ Run the narrowest check that proves the change, then broaden when touching share - Disposable Postgres integration gate: `TEST_DATABASE_URL=postgres://myskills_test:myskills_test@localhost:5432/myskills_test npm run test:postgres` - Release artifact gate: `npm run release:artifacts` - Production env preflight: `npm run check:prod-env -- --env-file .env.production` +- Full CI, release or CodeQL gate on Linux with Docker: `scripts/local-ci.sh verify|release-check|codeql` (see `docs/LOCAL_CI.md`) `npm run test:postgres` must use a disposable database whose name includes `test` or `ci`; it resets that schema. diff --git a/README.md b/README.md index e2f02f7a..359809cf 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,7 @@ The [MCP guide](apps/mcp/README.md) covers the source-based stdio and HTTP serve - [Libraries](docs/LIBRARIES.md) — save sources, review changes and curate shared skills. - [Architecture](docs/ARCHITECTURE.md) — how the registry, API and clients fit together. - [Contributing](CONTRIBUTING.md) — development setup and pull request guidance. +- [Local CI](docs/LOCAL_CI.md) — run the CI, release and CodeQL gates with `scripts/local-ci.sh`; maintainers dispatch the same entrypoint through their `local-ci` controller. - [Support](SUPPORT.md) · [GitHub issues](https://github.com/jremick/myskills/issues) — questions, bugs and feature requests. - [Security policy](SECURITY.md) — report vulnerabilities privately. - [Changelog](CHANGELOG.md) — user-facing changes and upgrade notes. diff --git a/SUPPORT.md b/SUPPORT.md index aa9f4945..7e8a07c6 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -16,7 +16,7 @@ Supported for beta feedback: - Fresh-clone setup and local self-hosting experiments. - API, web, CLI, and MCP behavior documented in this repository. - The example skill package under `examples/skills`. -- Release artifacts generated by the documented release workflow. +- Release artifacts generated by the documented release verification. Not supported yet: diff --git a/docs/BUSINESS_SAFE_RELEASE_GOAL.md b/docs/BUSINESS_SAFE_RELEASE_GOAL.md index e32b0fd2..dd42e74a 100644 --- a/docs/BUSINESS_SAFE_RELEASE_GOAL.md +++ b/docs/BUSINESS_SAFE_RELEASE_GOAL.md @@ -1,7 +1,7 @@ # Business-Safe Production Release Goal Version: 0.1.0-beta.4 -Last updated: 2026-06-30 +Last updated: 2026-09-29 ## Goal @@ -74,4 +74,4 @@ Turn the public beta into a business-safe, production-ready open-source release - Fresh clone and production-like deploy rehearsals pass. - Security review and threat model are refreshed after the production hardening work. - All public docs describe the supported and unsupported production posture without stale alpha-only caveats. -- A release candidate tag is cut and the release workflow succeeds. +- A release candidate tag is cut and tagged release verification (`scripts/local-ci.sh release-check`) succeeds. diff --git a/docs/CODEX_CLOUD.md b/docs/CODEX_CLOUD.md index fc0bcc81..a77405e1 100644 --- a/docs/CODEX_CLOUD.md +++ b/docs/CODEX_CLOUD.md @@ -1,20 +1,20 @@ # Codex Cloud Setup Version: 0.1.0-beta.17 -Last updated: 2026-07-13 +Last updated: 2026-09-29 This runbook makes MySkills ready for subscription-based Codex cloud/web tasks while keeping implementation work on GitHub pull requests and avoiding API-billed GitHub Actions agents for now. ## Current Repo Contract -Codex cloud should mirror the existing GitHub CI contract: +Codex cloud should mirror the existing CI contract, defined portably by `scripts/local-ci.sh` (see [Local CI](LOCAL_CI.md)): - CI installs dependencies with `npm ci`. - CI runs `npm run check` for the general gate. - CI runs `npm run test:postgres` in a separate job with disposable Postgres. -- Release verification runs the canonical `npm run release:verify` gate, then builds production Docker targets in the tag workflow. +- Release verification runs the canonical `npm run release:verify` gate, then builds production Docker targets (`scripts/local-ci.sh release-check`). -Do not add a GitHub Actions workflow that invokes a coding agent yet. Use Codex cloud/web to create branches and pull requests, then let the existing CI and human review gates decide whether to merge. +The full entrypoint needs Linux with Docker and is not expected to run inside a Codex cloud task. Do not add a GitHub Actions workflow that invokes a coding agent yet. Use Codex cloud/web to create branches and pull requests, then let the required checks and human review decide whether to merge. ## Codex Environment @@ -45,10 +45,10 @@ Inspect the MySkills repository instructions and CI. Do not change runtime behav Expected behavior: -- The agent reads `AGENTS.md`, `README.md`, `package.json`, and `.github/workflows/ci.yml`. +- The agent reads `AGENTS.md`, `README.md`, `package.json`, and `docs/LOCAL_CI.md`. - The diff is documentation-only. - No secrets, deployment variables, GitHub Actions agent workflows, or production deploy changes are added. -- The PR waits for existing GitHub CI and human approval before merge. +- The PR waits for the required checks and human approval before merge. ## Verification Commands For Agents diff --git a/docs/LOCAL_CI.md b/docs/LOCAL_CI.md index fdddec79..3ddfd2e5 100644 --- a/docs/LOCAL_CI.md +++ b/docs/LOCAL_CI.md @@ -2,8 +2,9 @@ `scripts/local-ci.sh` runs the same gates as the GitHub Actions workflows on a Linux host with Docker. Contributors can use it before opening a pull request. An external runner can call it -after checkout and read its evidence. The GitHub workflows remain in place until a replacement -has shown equivalent results; this entrypoint does not report statuses or publish anything. +after checkout and read its evidence. Release and merge instructions use its results. The GitHub +workflows stay in the repository as a parity reference during the migration. This entrypoint +does not report statuses or publish anything. ## Requirements @@ -106,6 +107,22 @@ Each step runs in its own process group. `SIGTERM` stops the current step, clean cancelled result; allow about 60 seconds. After `SIGKILL`, use `resources.json` to remove the listed resources. +## Maintainer Controller + +Maintainers dispatch this entrypoint to a trusted Linux host with a private `local-ci` controller. +The controller snapshots the exact commit, runs the entrypoint, collects `result.json` and the +evidence, and reports the protected contexts. Contributors do not need it. Dispatch only trusted +changes. The syntax below is current; final paths and configuration may still change. + +```bash +local-ci submit --app myskills --job verify --commit +local-ci submit --app myskills --job release-check --commit --tag v +local-ci submit --app myskills --job codeql --commit +local-ci wait +local-ci report --app myskills --commit # dry run; --execute posts statuses +local-ci sarif upload # dry run; --execute uploads SARIF +``` + ## Mapping From GitHub Actions | Workflow gate | Local equivalent | Difference | @@ -125,16 +142,16 @@ listed resources. ## Not Covered Here -These GitHub functions stay with GitHub, or with an external runner, until a separate cutover -replaces them: +The entrypoint does not provide these functions. Until cutover, GitHub Actions and repository +settings provide them; afterwards the maintainers' controller and GitHub settings do: - Pull request, push, tag and weekly CodeQL triggers, and release concurrency. -- Reporting the `check`, `web-e2e` and `postgres-integration` statuses, and branch protection. +- Reporting `local-ci/check`, `local-ci/web-e2e` and `local-ci/postgres-integration` after the matching branch-protection cutover. These distinct names replace the Actions contexts `check`, `web-e2e` and `postgres-integration`. - SARIF upload and the code scanning merge rule. - Artifact retention. - Dependabot, which is not an Actions workflow. -`scripts/check-prerelease.mjs`, `scripts/check-structure.mjs` and the release documentation -still describe the workflow files; update them when the workflows are retired. The full-stack -Compose run builds from cached base images without `--pull`, so the host cache can differ from a -fresh GitHub runner until it is refreshed. +The workflow files stay as the parity reference, so `scripts/check-structure.mjs` still requires +them and `scripts/check-prerelease.mjs` still checks their static contract. Remove those checks +only together with the files. The full-stack Compose run builds from cached base images without +`--pull`, so the host cache can differ from a fresh GitHub runner until it is refreshed. diff --git a/docs/RAILWAY_DEPLOYMENT.md b/docs/RAILWAY_DEPLOYMENT.md index 2a601d7c..053a61a1 100644 --- a/docs/RAILWAY_DEPLOYMENT.md +++ b/docs/RAILWAY_DEPLOYMENT.md @@ -500,11 +500,11 @@ curl --doh-url https://cloudflare-dns.com/dns-query https://api.myskills.sh/read The current live project is intentionally manual but can be made easier without changing hosting providers: -1. Keep feature work on a branch and require GitHub CI to pass. -2. Merge or fast-forward the Railway-connected branch after the rendered checks pass. Verify required CI for the exact merged source before promotion, and capture a current database-and-artifact recovery point. +1. Keep feature work on a branch and require the protected checks (`local-ci/check`, `local-ci/web-e2e` and `local-ci/postgres-integration` after cutover) to pass; see [Local CI](LOCAL_CI.md). +2. Merge or fast-forward the Railway-connected branch after the rendered checks pass. Verify the required checks for the exact merged source before promotion, and capture a current database-and-artifact recovery point. 3. When changing artifact publication or cleanup coordination, remove incompatible API writers and cleanup workers before starting the replacement. For Libraries beta.8, drain beta.7 API instances and workers before accepting library writes; older code does not enforce private-attestation and library-binding guards. Follow the [Libraries rollback boundary](RELEASE.md#libraries-beta8-compatibility-boundary). Account for the resulting API interruption in the rollout plan. 4. Deploy `api` from the approved commit and wait for Railway success and direct `/ready` before uploading `web` from the same commit. The web proxy must start after the healthy API so it does not retain an address for a retiring private instance. 5. Compare direct API, web, and proxy `/version.json` with the approved source. Verify web health and same-origin `/api/health` and `/api/ready`. 6. Complete staging's real browser/CLI journey before production. After production promotion, verify HTML revalidation in an existing browser cache, existing-session auth, authorized private package delivery, anonymous denial, rendered package text and navigation, and recent logs. Use a fresh context for anonymous checks and preserve existing user sessions during verification. Use read requests for production checks; package access still writes its normal audit events. -The release workflow is intentionally verification-only and does not deploy Railway. Follow the staging, production approval, and rollback boundary in [Release Process](RELEASE.md). Any future deploy automation must use scoped project credentials, preserve a separate staging/user-test step, require explicit production approval, deploy API and web from the same commit in API-ready-then-web order, and report resulting deployment IDs plus direct and same-origin health/browser readback. +Release verification (`scripts/local-ci.sh release-check`, and the tag workflow while it remains) is intentionally verification-only and does not deploy Railway. Follow the staging, production approval, and rollback boundary in [Release Process](RELEASE.md). Any future deploy automation must use scoped project credentials, preserve a separate staging/user-test step, require explicit production approval, deploy API and web from the same commit in API-ready-then-web order, and report resulting deployment IDs plus direct and same-origin health/browser readback. diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 484eb11f..4f9b8515 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -139,7 +139,7 @@ From a clean candidate checkout, with a disposable database whose name includes TEST_DATABASE_URL=postgres://myskills_test:myskills_test@localhost:5432/myskills_test npm run release:verify ``` -This single command runs repo quality/security checks, the exact public CLI pack/install smoke, route-mocked browser E2E, a production-like Docker Compose API/web/MinIO/Postgres browser journey, Postgres integration, and release artifact creation. Use `npm run check:prerelease`, `npm run smoke:cli-package`, or individual test commands only to diagnose a failure; do not substitute a collection of partial runs for the canonical gate. +This single command runs repo quality/security checks, the exact public CLI pack/install smoke, route-mocked browser E2E, a production-like Docker Compose API/web/MinIO/Postgres browser journey, Postgres integration, and release artifact creation. On a Linux host with Docker, `scripts/local-ci.sh verify` runs the full CI matrix for the same commit in fresh clones, and `scripts/local-ci.sh release-check` runs this gate for a tag; see [Local CI](LOCAL_CI.md). Use `npm run check:prerelease`, `npm run smoke:cli-package`, or individual test commands only to diagnose a failure; do not substitute a collection of partial runs for the canonical gate. When using the Windows PC for container testing, run the verifier and PostgreSQL on that host so lease checks use the same clock. The verifier needs a clean Git checkout of the candidate, including `.git` and `.github`; the production Docker build context excludes those paths and is not a complete release-verification checkout. Match the Playwright container version to the repository's installed Playwright version and retain the reports and artifacts before removing the disposable runner. @@ -154,7 +154,7 @@ Final artifact generation refuses a dirty worktree. `--allow-dirty` exists only ## Staging And User Test 1. Select one immutable candidate commit on a branch. Record the full SHA and intended version. -2. Require the GitHub CI jobs for that commit to pass on Node 22 and 24 LTS, web E2E, and disposable Postgres. +2. Require a gating `scripts/local-ci.sh verify` result for that commit: complete, pinned with `LOCAL_CI_SOURCE_SHA`, and `passed` for `check`, `web-e2e` and `postgres-integration`. These cover Node 22 and 24 LTS, web E2E, disposable Postgres and the Railway images. While the GitHub workflows still run during the migration, treat their results as parity evidence and resolve any disagreement before approval. 3. Exercise the same commit through a dedicated staging environment. If no dedicated Railway staging environment is configured, the documented production Compose stack may serve as beta staging, but record that limitation; do not use Railway production as the first test environment. 4. Record user-test evidence for first-run setup, login/MFA, owner invitation and invitee registration through a captured or staging-only email, public browse/detail, author submission/withdrawal, maintainer artifact inspection and hash-attested review/publication, CLI validate/scan/search/export/install/rollback, and MCP read-only discovery. 5. Re-run the canonical gate after any candidate change. Evidence from an earlier SHA is stale. @@ -166,24 +166,34 @@ Staging deployment is not release approval. User-test acceptance is a maintainer The owner approves each external action separately and in order: 1. **Tag approval**: authorize creation/push of `v` only after the acceptance ledger, clean canonical gate, current GitHub controls, and staging/user-test evidence are reviewed. -2. **Package/release approval**: after the verification-only tag workflow passes, separately authorize any npm `beta` publish, GitHub Release creation, container registry push, or public announcement. The current workflow performs none of these actions. +2. **Package/release approval**: after the pushed tag passes `scripts/local-ci.sh release-check` with a gating result for the tagged commit, separately authorize any npm `beta` publish, GitHub Release creation, container registry push, or public announcement. Release verification performs none of these actions. 3. **Production approval**: separately authorize Railway production migration/deploy. API and web must use the same commit; the migration plan, backup/restore readiness, smoke owner, and rollback target must be named. -A green workflow is evidence, not an approval signal. Never reuse or move an existing tag to repair a failed release. +A passing check is evidence, not an approval signal. Never reuse or move an existing tag to repair a failed release. -## Tag And Workflow Protection +## Tag Verification And Protection -The release workflow triggers on `v*.*.*` tags and: +Verify a pushed tag from a clean checkout of that tag, with full history and a current main ref: -- checks out full history; -- requires the tag to equal `v`; -- resolves the tag commit and requires it to be an ancestor of `origin/main`; -- runs the canonical release gate with tag enforcement; +```bash +VERSION=$(node -p "require('./package.json').version") +git fetch origin main --tags +git checkout --detach "v${VERSION}" +LOCAL_CI_RELEASE_TAG="v${VERSION}" LOCAL_CI_SOURCE_SHA="$(git rev-parse HEAD)" scripts/local-ci.sh release-check +``` + +Set `LOCAL_CI_RUN_ID`, `LOCAL_CI_EVIDENCE_DIR` and the Node toolchains as described in [Local CI](LOCAL_CI.md). Release-check: + +- requires the tag to equal `v` and to point at the checked-out commit; +- requires that commit to be an ancestor of `refs/remotes/origin/main` (or `LOCAL_CI_MAIN_REF`); +- runs the canonical release gate with tag enforcement against disposable `postgres:17`; - builds the root Dockerfile `api`, `web`, and `mcp-http` targets plus the exact `Dockerfile.api` and `Dockerfile.web` used by Railway; - builds `Dockerfile.backup` and checks both command entrypoints without credentials or network access; -- uploads verification artifacts only. +- verifies the artifact set, checksums and metadata, rebuilds the source archive from the tagged commit, and exports the verified artifacts as evidence only. + +The `v*.*.*` tag workflow runs the same steps and remains a parity reference until the migration in [Local CI](LOCAL_CI.md) is complete. -Configure a GitHub ruleset for the release-tag pattern (for example `v*`) that restricts tag creation, update, and deletion to the release maintainer role. Protect `main` with the aggregate `check` context (which requires both Node matrix jobs), web E2E, and Postgres integration; require current branches and choose administrator bypass deliberately. Read the live ruleset/protection state immediately before release; workflow YAML cannot prove that repository settings are applied. +Configure a GitHub ruleset for the release-tag pattern (for example `v*`) that restricts tag creation, update, and deletion to the release maintainer role. Protect `main` with the aggregate `check` context (which requires both Node lines, web E2E, Postgres and the Railway images), `web-e2e`, and `postgres-integration`. GitHub Actions reports these contexts until cutover. At cutover, migrate the requirements to the distinct controller contexts `local-ci/check`, `local-ci/web-e2e` and `local-ci/postgres-integration`. The controller reports them from complete, SHA-pinned `verify` results. Require current branches and choose administrator bypass deliberately. Read the live ruleset/protection state, including which app may report each context, immediately before release; neither workflow YAML nor a local result proves that repository settings are applied. ## Tagging diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index e52906a8..668b044d 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -1,7 +1,7 @@ # Threat Model Version: 0.2.0-draft -Last updated: 2026-09-01 +Last updated: 2026-09-29 ## Scope @@ -92,7 +92,8 @@ future authorization. - CLI auth, registry, sharing, team, and read-only architecture commands. - MCP registry and read-only architecture projection tools. - Package directory and `.zip` parsing, Docker/production configuration, and - GitHub release workflow. + release verification (`scripts/local-ci.sh` and the GitHub workflows it is + replacing). Pattern migration has a local authenticated preview/create route. There is no public sync-run route. The fixture planner in the consolidated preview is @@ -120,7 +121,8 @@ explicit and request-scoped; a target is never inferred. | Artifact tampering/direct object exposure | Modified or unreviewed content | Opaque storage keys, API-owned writes, byte-size/SHA-256 verification, fail-closed mismatch handling | Signed/direct delivery with authorization and audit | | MCP bearer misuse | Unauthorized metadata or unsafe action | Scoped API token before protocol handling, read-only architecture tools, no bundle payloads, host/origin restrictions | Per-tool authoritative audit and future role-gated tools | | Audit/error leakage | Token/package/private-data exposure | Sanitized audit details, generic auth responses, no raw target state, bounded fields | Structured audit export hardening | -| Release pipeline/provenance weakness | Harder source-to-artifact proof | Reproducible source archive, checksums, tag/version checks, release workflow | Pinned actions/images, SBOM, signatures, protected release tags | +| Release pipeline/provenance weakness | Harder source-to-artifact proof | Reproducible source archive rebuilt from the tagged commit, checksums, tag/version/main-ancestry checks, hashed release-check evidence | Pinned actions/images, SBOM, signatures, protected release tags; local evidence has no hosted-runner identity until a signed runner identity exists | +| Local CI host trust | Test and dependency code runs with the runner account's files and Docker access | Trusted changes only, allowlisted job environment without tokens, no publishing credentials in the job account, run-scoped cleanup, redacted and hashed evidence | Untrusted pull requests need a separately verified disposable executor | ## Audit findings at this branch review diff --git a/scripts/check-prerelease.mjs b/scripts/check-prerelease.mjs index adbb2208..edb5ea0d 100644 --- a/scripts/check-prerelease.mjs +++ b/scripts/check-prerelease.mjs @@ -65,6 +65,7 @@ const policyFiles = [ "docs/CODEX_CLOUD.md", "docs/COMPATIBILITY.md", "docs/DEPLOYMENT.md", + "docs/LOCAL_CI.md", "docs/API_MCP_CLI_PLAN.md", "docs/ARCHITECTURE.md", "docs/DATA_MODEL.md", @@ -87,6 +88,10 @@ const forbiddenPolicyPhrases = [ /current alpha repository/i, /first public private-development launch/i, /private-development deployment currently/i, + // Release approval uses local CI results, not a green GitHub Actions run. + /Require the GitHub CI jobs/i, + /after the verification-only tag workflow passes/i, + /require GitHub CI to pass/i, /^## Public Alpha Install$/im, ...[ ["check:alpha", "-release"], @@ -237,6 +242,8 @@ function checkReleaseMarkers() { assertContains("docs/BETA_RELEASE_GOAL.md", `Target release: \`${expectedTag}\`.`); assertContains("docs/BETA_RELEASE_GOAL.md", "npm run release:verify"); assertContains("docs/RELEASE.md", "npm run release:verify"); + assertContains("docs/RELEASE.md", "scripts/local-ci.sh verify"); + assertContains("docs/RELEASE.md", "scripts/local-ci.sh release-check"); assertContains("docs/RELEASE.md", "## Approval Boundary"); assertContains("docs/RELEASE.md", "## Rollback"); diff --git a/scripts/check-structure.mjs b/scripts/check-structure.mjs index 74ba65be..3affee29 100644 --- a/scripts/check-structure.mjs +++ b/scripts/check-structure.mjs @@ -88,6 +88,9 @@ const requiredPaths = [ "scripts/check-prerelease.mjs", "scripts/production-compose-policy.mjs", "scripts/create-release-artifacts.mjs", + "scripts/local-ci.sh", + "scripts/local-ci.mjs", + "docs/LOCAL_CI.md", "scripts/run-fullstack-e2e.mjs", "scripts/smoke-cli-package.mjs", "scripts/verify-release.mjs", From d256442c0ddb0ed64b971ebc16874296b28e0f7c Mon Sep 17 00:00:00 2001 From: Jarel Remick <3012014+jremick@users.noreply.github.com> Date: Tue, 29 Sep 2026 18:54:59 +1000 Subject: [PATCH 3/4] fix(ci): isolate run reservations and protect evidence --- docs/LOCAL_CI.md | 21 ++-- scripts/local-ci.mjs | 110 ++++++++++++++++++-- scripts/test/local-ci.test.mjs | 182 +++++++++++++++++++++++++++++++-- 3 files changed, 291 insertions(+), 22 deletions(-) diff --git a/docs/LOCAL_CI.md b/docs/LOCAL_CI.md index 3ddfd2e5..fee6ba89 100644 --- a/docs/LOCAL_CI.md +++ b/docs/LOCAL_CI.md @@ -56,10 +56,11 @@ LOCAL_CI_CODEQL_BIN=/path/to/codeql/codeql scripts/local-ci.sh codeql | Variable | Rule | |---|---| | `LOCAL_CI_RUN_ID` | Required. 1-48 lowercase letters, digits or hyphens. It names every container, Compose project, image tag and the workspace. | -| `LOCAL_CI_EVIDENCE_DIR` | Required. Absolute, outside the source tree, and without an earlier `result.json`. | +| `LOCAL_CI_EVIDENCE_DIR` | Required. Absolute, outside the source tree, not a symbolic link, and absent or empty. Every file in it is scanned and exported, so a populated directory is refused. | | `LOCAL_CI_SOURCE_SHA` | Optional. Must equal `HEAD`. Without it the result is not gating. | | `LOCAL_CI_NODE22_BIN`, `LOCAL_CI_NODE24_BIN` | Directories containing `node`, `npm` and `npx`. When unset, `node` on `PATH` is used only for its own major version. | | `LOCAL_CI_WORK_DIR` | Optional parent for the per-run workspace. Defaults to the OS temporary directory. | +| `DOCKER_HOST`, `DOCKER_CONTEXT` | Jobs that use Docker need a local `unix://` endpoint; a remote endpoint is refused. Set at most one of the two, because `DOCKER_CONTEXT` overrides `DOCKER_HOST`. | | `LOCAL_CI_RELEASE_TAG`, `LOCAL_CI_MAIN_REF` | `release-check` only. The tag must be `v` and point at `HEAD`. `HEAD` must be an ancestor of the main ref (default `refs/remotes/origin/main`). The script does not fetch. | | `LOCAL_CI_CODEQL_BIN`, `LOCAL_CI_CODEQL_CATEGORY` | `codeql` only. The category defaults to `/language:javascript-typescript`. | | `MYSKILLS_E2E_PORT`, `MYSKILLS_E2E_WEB_PORT`, `MYSKILLS_E2E_MAILPIT_PORT` | Optional loopback ports. Free ports are chosen when unset. | @@ -77,8 +78,8 @@ atomically and contains: - `status`: `passed`, `failed`, `rejected` or `cancelled`. `passed` also requires complete cleanup and clean evidence. -- `gating` and `gatingBlockers`: only a complete job set with a verified `LOCAL_CI_SOURCE_SHA` can - gate a commit. +- `gating` and `gatingBlockers`: only a complete job set with a verified `LOCAL_CI_SOURCE_SHA` on a + Linux/amd64 host can gate a commit. Other hosts report `unsupported-host-platform`. - `contexts`: for a complete `verify` run, the protected-branch contexts `check`, `web-e2e` and `postgres-integration`. Partial runs report `null`. - `jobs`: status, reason, steps, exit codes, timings and a hashed log for each job. @@ -96,16 +97,22 @@ the job clone and are deleted with it. ## Isolation And Cleanup Each job runs in its own clone of the pinned commit inside -`/myskills-local-ci-`. The workspace is created exclusively, so two runs cannot -share a run ID on one host. The script records each container, Compose project and image in +`/myskills-local-ci-`. Before that, the run ID is reserved by creating +`/var/tmp/myskills-local-ci-locks/` exclusively, whatever `TMPDIR` or the work directory is, +so two runs on one host cannot share a run ID. The lock directory must be owned by the running user +and not writable by others. The reservation covers only this host's Docker daemon, which is why +remote Docker endpoints are refused. A reservation is never taken over. The script records each container, Compose project and image in `resources.json` before or as it creates it, and removes only those exact names. Compose cleanup matches the exact `com.docker.compose.project` label. A container whose creation failed, for example because of a name conflict, is never removed. The script never prunes and never matches name prefixes. Shared npm, Playwright and Docker build caches are kept. Each step runs in its own process group. `SIGTERM` stops the current step, cleans up and writes a -cancelled result; allow about 60 seconds. After `SIGKILL`, use `resources.json` to remove the -listed resources. +cancelled result; allow about 60 seconds. The reservation is released only after complete cleanup. +If cleanup fails, or after `SIGKILL`, the reservation stays and the run ID is refused. To recover, +remove the resources listed in that run's `resources.json`. Then remove +`/var/tmp/myskills-local-ci-locks/` only if its `owner.json` `owner` equals the +`run-id-reservation` entry's `owner` in the same `resources.json`. ## Maintainer Controller diff --git a/scripts/local-ci.mjs b/scripts/local-ci.mjs index 6eca278e..0e8dbd2b 100644 --- a/scripts/local-ci.mjs +++ b/scripts/local-ci.mjs @@ -4,7 +4,7 @@ // scripts/local-ci.sh; docs/LOCAL_CI.md describes the inputs, jobs and result contract. import { spawn, spawnSync } from "node:child_process"; -import { createHash } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import { accessSync, closeSync, @@ -58,6 +58,10 @@ const maxPendingLine = 1024 * 1024; const codeqlSuiteLine = "\nqueries:\n - uses: security-extended\n"; const defaultCodeqlCategory = "/language:javascript-typescript"; const composeServiceImages = ["api", "web", "minio", "minio-init"]; +// Run IDs are reserved here, independent of TMPDIR and LOCAL_CI_WORK_DIR: exclusive mkdir on the local +// filesystem is atomic, which Docker resource names are not. It covers only a local Docker daemon. +const runIdLockRoot = "/var/tmp/myskills-local-ci-locks"; +const dockerJobPattern = /^(?:(?:postgres|web-e2e)-node2[24]|railway-images|release)$/; // Only these variables reach job processes; runner tokens stay with the runner. const passthroughEnv = [ "HOME", "USER", "LOGNAME", "SHELL", "LANG", "LANGUAGE", "LC_ALL", "LC_CTYPE", "TZ", "TERM", "TMPDIR", @@ -153,17 +157,35 @@ function validateEvidenceDirectory(value) { if (!value) throw new Error("LOCAL_CI_EVIDENCE_DIR is required and must be absolute."); if (!isAbsolute(value)) throw new Error("LOCAL_CI_EVIDENCE_DIR must be absolute."); const target = resolve(value); + let entry = null; + try { + entry = lstatSync(target); + } catch { + // Absent: created exclusively below. + } + if (entry?.isSymbolicLink()) throw new Error("LOCAL_CI_EVIDENCE_DIR must not be a symbolic link; pass the real directory path."); const real = realPathAllowingMissing(target); const source = realpathSync(sourceRoot); if (isWithin(real, source) || isWithin(source, real)) { throw new Error("LOCAL_CI_EVIDENCE_DIR must be outside the source tree and must not contain it."); } - if (existsSync(target)) { - if (!statSync(target).isDirectory()) throw new Error("LOCAL_CI_EVIDENCE_DIR must be a directory."); - if (existsSync(join(target, "result.json"))) { - throw new Error("LOCAL_CI_EVIDENCE_DIR already contains result.json; use a new directory for each run."); + if (!entry) { + mkdirSync(dirname(real), { recursive: true }); + try { + mkdirSync(real); + return real; + } catch (error) { + if (error?.code !== "EEXIST") throw error; + entry = lstatSync(real); + if (entry.isSymbolicLink()) throw new Error("LOCAL_CI_EVIDENCE_DIR must not be a symbolic link; pass the real directory path."); } } + if (!entry.isDirectory()) throw new Error("LOCAL_CI_EVIDENCE_DIR must be a directory."); + if (existsSync(join(real, "result.json"))) { + throw new Error("LOCAL_CI_EVIDENCE_DIR already contains result.json; use a new directory for each run."); + } + // Every file below the evidence directory is scanned, may be removed as quarantine, and is exported. + if (readdirSync(real).length > 0) throw new Error("LOCAL_CI_EVIDENCE_DIR must be empty or absent; it contains files that the run did not write."); return real; } @@ -199,14 +221,72 @@ function prepareRun(options, runId, evidence) { gatingBlockers: [ ...(options.complete ? [] : ["partial-job-selection"]), ...(expectedSha ? [] : ["source-sha-not-supplied"]), + ...(platform() === "linux" && arch() === "x64" ? [] : ["unsupported-host-platform"]), ], }; if (options.mode === "release-check") run.release = validateRelease(head, rootPackage); if (options.mode === "codeql") run.codeql = validateCodeql(); - run.workspace = reserveWorkspace(runId, evidence); + if (options.jobs.some((id) => dockerJobPattern.test(id))) requireLocalDocker(); + run.runIdLock = reserveRunId(runId); + try { + run.workspace = reserveWorkspace(runId, evidence); + } catch (error) { + releaseRunId(run.runIdLock); + throw error; + } return run; } +// The run-ID reservation is host-local, so Docker jobs must use this host's daemon. The Docker CLI lets +// DOCKER_CONTEXT override DOCKER_HOST; both together are refused rather than guessed. +function requireLocalDocker() { + const { DOCKER_HOST: host, DOCKER_CONTEXT: context } = process.env; + if (host && context) { + throw new Rejection("docker-endpoint-ambiguous", "Set only one of DOCKER_HOST and DOCKER_CONTEXT for Docker jobs; DOCKER_CONTEXT would override DOCKER_HOST."); + } + const format = ["--format", "{{.Endpoints.docker.Host}}"]; + const endpoint = context ? docker(["context", "inspect", context, ...format]).stdout.trim() + : host || docker(["context", "inspect", ...format]).stdout.trim(); + if (!endpoint.startsWith("unix://")) { + throw new Rejection("docker-endpoint-not-local", "Docker jobs need a local unix:// Docker endpoint; the run-ID reservation is host-local and cannot cover a remote daemon."); + } +} + +function reserveRunId(runId) { + try { + mkdirSync(runIdLockRoot, { mode: 0o700 }); + } catch (error) { + if (error?.code !== "EEXIST") throw new Rejection("run-id-lock-unavailable", `Cannot create ${runIdLockRoot}: ${error.code ?? error.message}.`); + } + const root = lstatSync(runIdLockRoot); + if (root.isSymbolicLink() || !root.isDirectory() || root.uid !== process.getuid() || (root.mode & 0o022) !== 0) { + throw new Rejection("run-id-lock-unavailable", `${runIdLockRoot} must be a real directory owned by this user and not writable by group or others.`); + } + const path = join(runIdLockRoot, runId); + try { + mkdirSync(path, { mode: 0o700 }); + } catch (error) { + if (error?.code === "EEXIST") { + throw new Rejection("run-id-in-use", `LOCAL_CI_RUN_ID is already reserved on this host (${path}). Use a new run ID. After a killed run, remove the resources in its resources.json and then this reservation; it is never taken over.`); + } + throw new Rejection("run-id-lock-unavailable", `Cannot reserve the run ID: ${error.code ?? error.message}.`); + } + const owner = randomBytes(16).toString("hex"); + writeFileSync(join(path, "owner.json"), `${JSON.stringify({ owner, pid: process.pid, startedAt: new Date().toISOString() })}\n`, { mode: 0o600, flag: "wx" }); + return { path, owner }; +} + +// Removes the reservation only when it still carries this run's owner token. +function releaseRunId(lock) { + try { + if (JSON.parse(readFileSync(join(lock.path, "owner.json"), "utf8")).owner !== lock.owner) return false; + rmSync(lock.path, { recursive: true }); + return true; + } catch { + return false; + } +} + function validateRelease(head, rootPackage) { const tag = process.env.LOCAL_CI_RELEASE_TAG ?? ""; if (!tag) throw new Rejection("release-tag-required", "LOCAL_CI_RELEASE_TAG is required for release-check."); @@ -270,6 +350,8 @@ function reserveWorkspace(runId, evidence) { async function executeRun(run, base) { const ledger = new ResourceLedger(join(run.evidence, "resources.json"), run.runId); + const reservation = ledger.track("run-id-reservation", run.runIdLock.path, null, "created"); + reservation.owner = run.runIdLock.owner; ledger.track("workspace", run.workspace, null, "created"); mkdirSync(join(run.evidence, "logs"), { recursive: true }); const environment = collectEnvironment(run); @@ -299,6 +381,20 @@ async function executeRun(run, base) { ledger.mark(ledger.find("workspace", run.workspace), "remove-failed"); cleanupFailures.push(`workspace: ${error.code ?? error.message}`); } + // Removals that failed during job cleanup count too; they are not retried here. + for (const entry of ledger.resources) { + const failure = `${entry.kind} ${entry.name}`; + if (entry.state === "remove-failed" && !cleanupFailures.includes(failure) && !failure.startsWith("workspace ")) cleanupFailures.push(failure); + } + // While known resources remain, keep the run ID reserved so another run cannot reuse their names. + if (cleanupFailures.length > 0) { + ledger.mark(reservation, "retained"); + } else if (releaseRunId(run.runIdLock)) { + ledger.mark(reservation, "removed"); + } else { + ledger.mark(reservation, "remove-failed"); + cleanupFailures.push("run-id-reservation"); + } writeJsonAtomic(join(run.evidence, "environment.json"), environment); for (const job of jobs) { @@ -784,7 +880,7 @@ class ResourceLedger { cleanup(sink, job = undefined) { const failures = []; for (const entry of [...this.resources].reverse()) { - if ((job !== undefined && entry.job !== job) || entry.kind === "workspace") continue; + if ((job !== undefined && entry.job !== job) || entry.kind === "workspace" || entry.kind === "run-id-reservation") continue; if (!["created", "creating"].includes(entry.state)) continue; if (entry.kind === "container" && entry.state === "creating") continue; const removed = removeResource(entry, sink); diff --git a/scripts/test/local-ci.test.mjs b/scripts/test/local-ci.test.mjs index e2fd054a..cf07520d 100644 --- a/scripts/test/local-ci.test.mjs +++ b/scripts/test/local-ci.test.mjs @@ -5,6 +5,7 @@ import { chmodSync, copyFileSync, existsSync, + lstatSync, mkdirSync, mkdtempSync, readFileSync, @@ -33,8 +34,19 @@ import test from "node:test"; // - release-check accepts a wrong tag, a tag not at HEAD, or a commit outside main. // - Tampered release artifacts pass, or release-check publishes images or packages. // - CodeQL output that ignores the repository query filters passes. - -const runId = "fixture-run"; +// Review follow-up (PR 120), also written before the fix: +// - A complete pinned run on a host other than Linux/amd64 reports gating Linux/amd64 evidence. +// - Runs that start together with one run ID but different work or temporary directories all reserve +// it, so their container, Compose-project and image names collide on the Docker host; or a stale +// or foreign reservation is taken over or removed. +// - Docker jobs run against a remote daemon that a host-local reservation cannot cover, including a +// remote DOCKER_CONTEXT that overrides a local DOCKER_HOST. +// - A run whose cleanup failed releases its reservation, so the run ID can be reused while its +// resources remain. +// - A populated or symlinked evidence destination is accepted, so the evidence scan can delete or the +// manifest can export files that the run did not write. + +const runId = `fixture-run-${process.pid}`; const rootPackage = JSON.parse(readFileSync(resolve("package.json"), "utf8")); const releaseTag = `v${rootPackage.version}`; const verifyJobs = ["check-node22", "check-node24", "web-e2e-node22", "web-e2e-node24", "postgres-node22", "postgres-node24", "railway-images"]; @@ -51,6 +63,9 @@ const fixtureFiles = [ ]; const webBuild = "run build -w @myskills-app/core -w @myskills-app/auth -w @myskills-app/skill-package -w @myskills-app/api"; const mockedBrowser = "run test:e2e -w @myskills-app/web -- --reporter=line,json"; +// The gate is defined for Linux/amd64 hosts; elsewhere a complete pinned run must stay non-gating. +const hostBlockers = process.platform === "linux" && process.arch === "x64" ? [] : ["unsupported-host-platform"]; +const runIdReservation = join("/var/tmp/myskills-local-ci-locks", runId); test("unsafe or stale inputs are rejected before any tool, container or workspace is used", (t) => { const fixture = makeFixture(t); @@ -113,8 +128,8 @@ test("verify runs every required job on both Node lines and reports gating conte assert.equal(run.status, 0, run.output); const { result } = run; assert.equal(result.status, "passed"); - assert.equal(result.gating, true); - assert.deepEqual(result.gatingBlockers, []); + assert.equal(result.gating, hostBlockers.length === 0); + assert.deepEqual(result.gatingBlockers, hostBlockers); assert.equal(result.complete, true); assert.equal(result.source.sha, fixture.sha); assert.deepEqual(result.jobs.map(({ id }) => id).sort(), [...verifyJobs].sort()); @@ -182,7 +197,7 @@ test("job failures are isolated, fail the gating contexts and never remove anoth assert.equal(run.status, 1, run.output); const { result } = run; assert.equal(result.status, "failed"); - assert.equal(result.gating, true, "a pinned complete run is authoritative even when it fails"); + assert.equal(result.gating, hostBlockers.length === 0, "a pinned complete run on a Linux/amd64 host is authoritative even when it fails"); const job = (id) => result.jobs.find((candidate) => candidate.id === id); assert.equal(job("check-node22").status, "passed"); assert.equal(job("check-node24").status, "failed"); @@ -217,7 +232,7 @@ test("partial or unpinned runs never report the required contexts", (t) => { assert.equal(run.result.status, "passed"); assert.equal(run.result.complete, false); assert.equal(run.result.gating, false); - assert.deepEqual([...run.result.gatingBlockers].sort(), ["partial-job-selection", "source-sha-not-supplied"]); + assert.deepEqual([...run.result.gatingBlockers].sort(), ["partial-job-selection", "source-sha-not-supplied", ...hostBlockers].sort()); assert.equal(run.result.contexts, null); const records = fixture.records(); assert.ok(records.every(({ tool, line }) => tool === "npm" && line === "22")); @@ -266,6 +281,7 @@ test("cancellation stops the running step and removes exactly the run's resource const resources = JSON.parse(readFileSync(join(evidence, "resources.json"), "utf8")); assert.ok(resources.resources.every(({ state }) => state === "removed"), JSON.stringify(resources)); assert.equal(existsSync(fixture.runWorkspace), false); + assert.equal(existsSync(runIdReservation), false, "cancellation releases the run-ID reservation"); }); test("credential-shaped output is redacted from exported evidence and fails the run and its contexts", (t) => { @@ -320,7 +336,7 @@ test("release-check verifies tagged artifacts and release images without publish const run = runLocalCi(fixture, ["release-check"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha, LOCAL_CI_RELEASE_TAG: releaseTag } }); assert.equal(run.status, 0, run.output); assert.equal(run.result.status, "passed"); - assert.equal(run.result.gating, true); + assert.equal(run.result.gating, hostBlockers.length === 0); assert.equal(run.result.release.tag, releaseTag); assert.equal(run.result.release.mainSha, fixture.sha); @@ -395,6 +411,152 @@ test("CodeQL applies the repository query filters and fails closed when they are assert.equal(ignored.result.jobs[0].reason, "codeql-filter-not-applied"); }); +test("a complete pinned run on a host other than Linux/amd64 never reports gating evidence", (t) => { + const fixture = makeFixture(t); + const run = runLocalCi(fixture, ["codeql"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha, LOCAL_CI_CODEQL_BIN: join(fixture.bin, "codeql") } }); + assert.equal(run.status, 0, run.output); + const environment = readJson(join(run.evidence, "environment.json")); + const supported = environment.platform === "linux" && environment.arch === "x64"; + assert.equal(run.result.status, "passed", "the run itself still completes for development use"); + assert.equal(run.result.complete, true); + assert.equal(run.result.gating, supported, `${environment.platform}/${environment.arch}`); + assert.equal(run.result.gatingBlockers.includes("unsupported-host-platform"), !supported); +}); + +test("runs that start together with one run ID reserve it once on the host, whatever their work and temporary directories", async (t) => { + const fixture = makeFixture(t); + t.after(() => rmSync(runIdReservation, { recursive: true, force: true })); + fixture.configure({ rules: [{ tool: "npm", line: "22", prefix: "ci", sleepMs: 4000 }] }); + const callers = ["a", "b", "c", "d"].map((name) => { + const caller = { name, work: join(fixture.root, `work-${name}`), tmp: join(fixture.root, `tmp-${name}`), evidence: fixture.newEvidence() }; + mkdirSync(caller.work); + mkdirSync(caller.tmp); + return caller; + }); + const exits = await Promise.all(callers.map((caller) => new Promise((resolvePromise) => { + const child = spawn("bash", [join(fixture.source, "scripts/local-ci.sh"), "verify", "--job", "postgres-node22"], { + cwd: fixture.source, + env: fixture.env({ LOCAL_CI_EVIDENCE_DIR: caller.evidence, LOCAL_CI_SOURCE_SHA: fixture.sha, LOCAL_CI_WORK_DIR: caller.work, TMPDIR: caller.tmp }), + stdio: "ignore", + }); + child.once("close", (code) => resolvePromise(code)); + }))); + const outcomes = callers.map((caller, index) => ({ ...caller, code: exits[index], result: readJson(join(caller.evidence, "result.json")) })); + const winners = outcomes.filter(({ code }) => code === 0); + const refused = outcomes.filter(({ code }) => code === 2); + assert.equal(winners.length, 1, JSON.stringify(outcomes.map(({ name, code }) => [name, code]))); + assert.equal(refused.length, callers.length - 1); + assert.equal(winners[0].result.status, "passed"); + for (const caller of refused) { + assert.equal(caller.result?.status, "rejected", caller.name); + assert.equal(caller.result.reason, "run-id-in-use", caller.name); + assert.deepEqual(readdirSync(caller.work), [], `${caller.name} must not keep a workspace`); + const docker = fixture.records().filter(({ tool, env }) => tool === "docker" && env.TMPDIR === caller.tmp); + assert.ok(docker.every(({ args }) => args[0] === "context"), `${caller.name} may only read its Docker endpoint: ${JSON.stringify(docker.map(({ args }) => args))}`); + } + assert.equal(existsSync(runIdReservation), false, "the finished run releases its reservation"); +}); + +test("a stale or foreign run-ID reservation is never taken over or removed", (t) => { + const fixture = makeFixture(t); + t.after(() => rmSync(runIdReservation, { recursive: true, force: true })); + mkdirSync(runIdReservation, { recursive: true }); + const owner = join(runIdReservation, "owner.json"); + writeFileSync(owner, JSON.stringify({ owner: "another-run", pid: 1 })); + const before = readFileSync(owner); + const run = runLocalCi(fixture, ["verify", "--job", "postgres-node22"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha } }); + assert.equal(run.status, 2, run.output); + assert.equal(run.result?.reason, "run-id-in-use"); + assert.deepEqual(readFileSync(owner), before); + assert.equal(existsSync(fixture.runWorkspace), false); + assert.ok(fixture.records().every(({ tool, args }) => tool === "docker" && args[0] === "context"), JSON.stringify(fixture.records())); +}); + +test("Docker jobs refuse a remote Docker endpoint that a host-local reservation cannot cover", (t) => { + const fixture = makeFixture(t); + for (const endpoint of ["tcp://127.0.0.1:2375", "ssh://ci@docker.example.invalid"]) { + const run = runLocalCi(fixture, ["verify", "--job", "postgres-node22"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha, DOCKER_HOST: endpoint } }); + assert.equal(run.status, 2, `${endpoint}: ${run.output}`); + assert.equal(run.result?.reason, "docker-endpoint-not-local", endpoint); + assert.deepEqual(fixture.records(), [], endpoint); + assert.equal(existsSync(runIdReservation), false, endpoint); + } + fixture.configure({ dockerEndpoint: "tcp://10.0.0.5:2376" }); + const context = runLocalCi(fixture, ["verify", "--job", "postgres-node22"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha } }); + assert.equal(context.status, 2, context.output); + assert.equal(context.result?.reason, "docker-endpoint-not-local", "a remote current Docker context is refused too"); + + // DOCKER_CONTEXT overrides DOCKER_HOST in the Docker CLI, so a local DOCKER_HOST proves nothing then. + fixture.configure({ dockerContexts: { "remote-ci": "tcp://10.0.0.5:2376", "wsl-local": "unix:///var/run/docker.sock" } }); + const combined = runLocalCi(fixture, ["verify", "--job", "postgres-node22"], { + env: { LOCAL_CI_SOURCE_SHA: fixture.sha, DOCKER_HOST: "unix:///var/run/docker.sock", DOCKER_CONTEXT: "remote-ci" }, + }); + assert.equal(combined.status, 2, combined.output); + assert.equal(combined.result?.reason, "docker-endpoint-ambiguous"); + assert.equal(existsSync(runIdReservation), false); + const namedRemote = runLocalCi(fixture, ["verify", "--job", "postgres-node22"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha, DOCKER_CONTEXT: "remote-ci" } }); + assert.equal(namedRemote.status, 2, namedRemote.output); + assert.equal(namedRemote.result?.reason, "docker-endpoint-not-local"); + const namedLocal = runLocalCi(fixture, ["verify", "--job", "postgres-node22"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha, DOCKER_CONTEXT: "wsl-local" } }); + assert.equal(namedLocal.status, 0, namedLocal.output); + assert.ok(fixture.records().some(({ tool, args }) => tool === "docker" && args.slice(0, 3).join(" ") === "context inspect wsl-local")); +}); + +test("a run whose cleanup failed keeps its reservation so the run ID cannot be reused", (t) => { + const fixture = makeFixture(t); + t.after(() => rmSync(runIdReservation, { recursive: true, force: true })); + fixture.configure({ rules: [{ tool: "docker", prefix: "rm -f -v", exit: 1 }] }); + const run = runLocalCi(fixture, ["verify", "--job", "postgres-node22"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha } }); + assert.equal(run.status, 1, run.output); + assert.equal(run.result.status, "failed"); + assert.equal(run.result.cleanup.status, "failed"); + const owner = readFileSync(join(runIdReservation, "owner.json")); + const reservation = readJson(join(run.evidence, "resources.json")).resources.find(({ kind }) => kind === "run-id-reservation"); + assert.equal(reservation.state, "retained"); + assert.equal(reservation.owner, JSON.parse(owner).owner, "resources.json identifies the exact owner for manual recovery"); + + fixture.configure({}); + const second = runLocalCi(fixture, ["verify", "--job", "postgres-node22"], { env: { LOCAL_CI_SOURCE_SHA: fixture.sha } }); + assert.equal(second.status, 2, second.output); + assert.equal(second.result?.reason, "run-id-in-use"); + assert.deepEqual(readFileSync(join(runIdReservation, "owner.json")), owner); +}); + +test("populated or symlinked evidence destinations are refused before anything is scanned, moved or written", (t) => { + const fixture = makeFixture(t); + const secret = ["gh", "p_", "Q".repeat(36)].join(""); + const populated = join(fixture.root, "populated"); + mkdirSync(join(populated, "nested"), { recursive: true }); + writeFileSync(join(populated, "notes.txt"), `keep this ${secret}\n`); + writeFileSync(join(populated, "nested", "data.bin"), Buffer.from([0, 1, 2, 255])); + const emptyTarget = join(fixture.root, "empty-target"); + mkdirSync(emptyTarget); + const emptyLink = join(fixture.root, "evidence-link"); + symlinkSync(emptyTarget, emptyLink); + const populatedLink = join(fixture.root, "populated-link"); + symlinkSync(populated, populatedLink); + const before = snapshot(populated); + for (const [name, evidence, message] of [ + ["populated directory", populated, /must be empty/], + ["symlink to an empty directory", emptyLink, /symbolic link/], + ["symlink to a populated directory", populatedLink, /symbolic link/], + ]) { + const run = runLocalCi(fixture, ["verify"], { evidence, env: { LOCAL_CI_SOURCE_SHA: fixture.sha } }); + assert.equal(run.status, 2, `${name}: ${run.output}`); + assert.match(run.stderr, message, name); + assert.deepEqual(fixture.records(), [], `${name} must not invoke tools`); + assert.equal(existsSync(fixture.runWorkspace), false, name); + assert.equal(existsSync(runIdReservation), false, name); + } + assert.deepEqual(snapshot(populated), before, "pre-existing files, including credential-shaped bytes, are preserved exactly"); + assert.deepEqual(readdirSync(emptyTarget), []); + assert.equal(lstatSync(emptyLink).isSymbolicLink(), true); +}); + +function snapshot(directory) { + return Object.fromEntries(listFiles(directory).map((file) => [file, sha256(readFileSync(join(directory, file)))])); +} + function makeFixture(t, { tag = false } = {}) { const root = realpathSync(mkdtempSync(join(tmpdir(), "myskills-local-ci-"))); t.after(() => rmSync(root, { recursive: true, force: true })); @@ -609,7 +771,7 @@ async function fakeToolMain() { const config = JSON.parse(fs.readFileSync(path.join(root, "fake-config.json"), "utf8")); const key = args.join(" "); const env = {}; - for (const name of ["CI", "TEST_DATABASE_URL", "MYSKILLS_E2E_COMPOSE_PROJECT", "RELEASE_REQUIRE_TAG", "RELEASE_EXPECTED_TAG", "PLAYWRIGHT_JSON_OUTPUT_FILE"]) { + for (const name of ["CI", "TEST_DATABASE_URL", "MYSKILLS_E2E_COMPOSE_PROJECT", "RELEASE_REQUIRE_TAG", "RELEASE_EXPECTED_TAG", "PLAYWRIGHT_JSON_OUTPUT_FILE", "TMPDIR"]) { if (process.env[name] !== undefined) env[name] = process.env[name]; } const canarySeen = Boolean(config.canary) && Object.values(process.env).some((value) => String(value).includes(config.canary)); @@ -650,6 +812,10 @@ async function fakeToolMain() { } if (tool === "docker") { if (args[0] === "version") return print("28.3.0"); + if (args[0] === "context" && args[1] === "inspect") { + const name = args[2] === "--format" ? null : args[2]; + return print((name ? config.dockerContexts?.[name] : config.dockerEndpoint) ?? "unix:///var/run/docker.sock"); + } if (args[0] === "run" && args.includes("-d")) { if ((config.dockerConflicts ?? []).includes(valueAfter("--name"))) { process.stderr.write("Conflict. The container name is already in use.\n"); From d034e483811b9ed846091dcd0f2daaea0b164448 Mon Sep 17 00:00:00 2001 From: Jarel Remick <3012014+jremick@users.noreply.github.com> Date: Tue, 29 Sep 2026 19:04:12 +1000 Subject: [PATCH 4/4] fix(ci): retain evidence directory validation error cause --- scripts/local-ci.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/local-ci.mjs b/scripts/local-ci.mjs index 0e8dbd2b..98db1af6 100644 --- a/scripts/local-ci.mjs +++ b/scripts/local-ci.mjs @@ -177,7 +177,7 @@ function validateEvidenceDirectory(value) { } catch (error) { if (error?.code !== "EEXIST") throw error; entry = lstatSync(real); - if (entry.isSymbolicLink()) throw new Error("LOCAL_CI_EVIDENCE_DIR must not be a symbolic link; pass the real directory path."); + if (entry.isSymbolicLink()) throw new Error("LOCAL_CI_EVIDENCE_DIR must not be a symbolic link; pass the real directory path.", { cause: error }); } } if (!entry.isDirectory()) throw new Error("LOCAL_CI_EVIDENCE_DIR must be a directory.");