diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 81d9f7c..5751321 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -17,6 +17,16 @@ (neutron#6, teploy-cli#18/#19/#20 — links above/below). - The integration-check test-quoting defect (spaced checkout paths) was fixed in PR #56. +- **S08 PR-body/webhook surfacing built (PR #58, this wave):** the detector's + findings now reach the pull request's Verification section and the + run-completion webhook behind `SHIP_TEST_INTEGRITY_SURFACING=off|shadow|on` + (default off; invalid values are off with a log). Advisory only — not a + gate, not in the delivery record, and no step is added to the durable log + in any mode. The one step-sequence effect, under `on` only: a run whose + only verification part is an integrity finding now records the existing + `verification` step. Clean diffs surface nothing (pinned by test). Not one + live run; soak `shadow` on the real worker and read its logs before any + `on` flip, the same rule as the other shadows. ## 2026-10-04 — wave 4 (S07, S13, S15/S17, S19, S21, S22, S24, S26, S27) @@ -123,7 +133,8 @@ detail is in the programme's wave-2 status table. Items that matter for audit: - **S05 stack-detect (PR #17):** unwired; readiness still keys on project config only, not manifest, lockfile or recipe digest. - **S08 test-integrity (PRs #18, #31):** advisory-only on the run page; not a - gate; not in the PR body or webhook; false-positive rate unmeasured. + gate; PR-body/webhook surfacing landed in wave 5 (PR #58) behind a + default-off flag; false-positive rate unmeasured. - **S16 schedules (PR #24):** the overlap policy is inert in the worker until `sweepWorkflowSchedules` is given `isRunning`. - **S25 policy (PR #25):** tested pure core, no enforcement; every entry point diff --git a/NEXT_SESSION.md b/NEXT_SESSION.md index c431330..1eab1e0 100644 --- a/NEXT_SESSION.md +++ b/NEXT_SESSION.md @@ -15,8 +15,8 @@ Wave 4's state ("implemented and checked by automated tests only") is now partly ## Next work, in order 1. **Shadow-log review** after soak (placement/policy/budget/tool-manifest/knowledge-provenance JSONL + reports); set `SHIP_MODEL_ROUTING_POLICY` so the routing shadow has something to record. Only then discuss any `on` flips. -2. **S08 wiring** (approved): findings into PR body + webhook behind a default-off flag; worker-path change, so shadow-first per the standing rule. -3. **Live proofs still open**: S01 credential proofs on a real sandbox with a private repo; S27 real teploy-adapter run against a scratch target; S19 doctor probes against the real store/clock; human observation of dashboard users. (S03's storage migration is done: migration 008 rehearsed on the restored `pre-80c9187` copy, receipt in `evals/receipts/2026-10-04-s03-requirements-migration.json`; the store stays unwired — a write path is separate future work.) +2. **S08 wiring** (approved): BUILT in wave 5 (this PR) — findings in the PR body + run-completion webhook behind `SHIP_TEST_INTEGRITY_SURFACING=off|shadow|on`, default off. Still to do: soak `shadow` on the real worker, then decide the `on` flip. +3. **Live proofs still open**: S01 credential proofs on a real sandbox with a private repo; S27 real teploy-adapter run against a scratch target; S19 doctor probes against the real store/clock; human observation of dashboard users. (S03's storage migration is done: migration 008 rehearsed on the restored `pre-80c9187` copy, receipt in `evals/receipts/2026-10-04-s03-requirements-migration.json`; the store stays unwired — a write path is separate future work. S07's plan-park wiring landed in #59.) 4. **Code still unfinished**: S04 and S12 barely started; most of S07 beyond the grounding check; S10 follow-ups (offline/error states, other roles, screen readers); wiring the inert modules (S15/S17 into delivery+incidents, S19 snapshot producer + restore-check, S18 worker tree provisioning, S07 into the plan-park point). 5. **Re-grade retained runs where possible**: only transcripts were preserved for pre-batch runs, so regrades are limited to transcript+fixture-verifiable scenarios; write supplementary `regrade-*.json` beside the records, never replacing originals. 6. **Deferred**: configuration B of the S02 comparison (needs real API spend, owner-gated). diff --git a/docs/SHIP_RELEASE_PROGRAMME_2026-09-21.md b/docs/SHIP_RELEASE_PROGRAMME_2026-09-21.md index cb74721..8fa8b13 100644 --- a/docs/SHIP_RELEASE_PROGRAMME_2026-09-21.md +++ b/docs/SHIP_RELEASE_PROGRAMME_2026-09-21.md @@ -505,7 +505,7 @@ This section supersedes the earlier table where rows overlap (S13, S28, S05, S06 | S09 | #15 | `src/finding-continuity.ts` classifier (still-open, resolved, regressed, new, moved, superseded, unconfirmed-disappearance) | 38 tests, line-only negative controls, 4 mutations; synthetic fixtures | Not wired; thresholds unvalidated on real PRs; forge reads carry no inline comments or thread state | | S03 | #16 | Read-only "Task state" panel on the run page from `taskRecord()` | 8 view-model tests + negative controls; web 144; build; Chromium render at 390 and 1440 on seeded data | No real run observed; no storage or write path for criteria, evidence or waivers; runs list unchanged | | S05 | #17 | `src/stack-detect.ts` cited stack and service proposals, readiness digest | 32 tests + negative controls (two attempted controls did not fail; redundant by design) | Not wired; no real-repo corpus; regex TOML/YAML parsing; nothing proposed was executed | -| S08 | #18, #31 | `src/test-integrity.ts` detector; advisory read-time panel in the run Verification view (`SHIP_ORACLE_PATHS` read in the web layer only) | 47 detector tests + 14 mutation controls; 9 projection tests + negative control (a missing diff never renders clean); web 153; build | Not a gate; not in the PR body, webhook or delivery record (would change the worker path, needs a decision); precision unmeasured; component not browser-rendered | +| S08 | #18, #31 | `src/test-integrity.ts` detector; advisory read-time panel in the run Verification view (`SHIP_ORACLE_PATHS` read in the web layer only) | 47 detector tests + 14 mutation controls; 9 projection tests + negative control (a missing diff never renders clean); web 153; build | Not a gate; PR-body/webhook surfacing approved 2026-10-04 and landed in wave 5 (#58) behind `SHIP_TEST_INTEGRITY_SURFACING`; still not in the delivery record; precision unmeasured; component not browser-rendered | | S22 | #19 | `src/mission.ts` mission graph validation, readiness by accepted deliverable at an exact revision, acceptance aggregation | 24 tests + 8 mutation controls | Not wired or persisted; no launching, replanning or cancel; write-grant conflicts checked as strings only | | S20 | #20 | `docs/claims/ledger.json` -> `docs/CLAIM_LEDGER.md`, `scripts/claim-ledger.mjs --check` | 14 tests; 7 guards disabled to confirm | Not in CI; 12 seeded claims, none verified; tests are cited by name only | | S23 | #21 | `src/model-routing.ts` versioned routing and fallback policy with gates | 16 tests + 6 negative controls | Not wired; failure classification, policy store and the fixed-vs-routed comparison not built | @@ -558,6 +558,14 @@ Upstream feature requests for the Teploy CLI, **not yet filed**: a deploy dry-ru Also merged this wave: #9 plan and inventory, #10 devalue 5.9.3 override. S10 and S02 grader sensitivity landed after this table was first written and are the first two rows after S13/S06. +#### Wave 5 (2026-10-04): S08 surfacing + +Same rules as waves 3 and 4: off by default, default-off equivalence proven by tests. This row exists because the owner approved the direction on 2026-10-04 (see the wave-5 notes at the top of `AUDIT_OPEN.md`). + +| Package | PR | Flag (default) | Implemented | Checked | Still open (specific) | +| --- | --- | --- | --- | --- | --- | +| S08 | #58 | `SHIP_TEST_INTEGRITY_SURFACING=off\|shadow\|on` (off) | The detector's findings on the pull request body (a "Test integrity" part inside the existing Verification section, marker-spliced and idempotent like the rest) and as an additive `test_integrity` field on the run-completion webhook payload. No new step, no recorded step result changes, delivery record untouched; clean adds nothing anywhere. Shadow logs what `on` would surface, per the standing worker-path rule | 13 wiring tests: default-off byte-equivalence (section and payload), real-path flag-on tests through `verificationSection`/`spliceVerification`/`runWebhookPayload` and the worker's own terminal composition, negative controls (clean diff, no/`empty`/non-string diff, garbage diff, invalid flag value), cap and middle-cut honesty | Not one live run (no forge, worker or receiver reachable); shadow not soaked on a worker; detector precision still unmeasured (wave-2 open item); the `fix` command's own PR path (`fix-evidence.ts`) does not surface findings — worker paths only | + ### Evidence that was not available - Private execution receipts (`COMPLETION_2026-09-25/`), the claim ledger and the competitive review are not in this repository or this environment, so the plan's statements that rest on them (local `42af7fb`, last live deployment `dd5b13e`, the 21/13/2 product baseline) were **not re-verified** here. The 21/13/2 figure was later recounted against the retained records under `evals/product-journeys/results/` (2026-10-03): it is not reproducible (28 attempts, not 36); the origin of 36/21/13 is in the private receipts, which remain unavailable. diff --git a/src/durable.ts b/src/durable.ts index cf1e241..41fc596 100644 --- a/src/durable.ts +++ b/src/durable.ts @@ -63,6 +63,7 @@ import { import { buildIfDeclared, flowIfPresent, observeIfDeclared, recordLadder, smokeIfDeclared, visualIfDeclared, type AssetSink, type LadderHooks } from "./ladder-steps.js"; import { compareAroundNow, effectiveTelemetryTarget, telemetryAppliesTo, telemetryRegression, type TelemetryTarget, type TelemetryVerdict } from "./observe.js"; import { spliceVerification, verificationSection, type Evidence } from "./verification.js"; +import { surfacedTestIntegrity } from "./test-integrity-surface.js"; import { shortHash, warmClient, warmSlugOf, type WarmState } from "./warm.js"; import { mergeFact, verificationSummary, type VerificationFacts } from "./verification-summary.js"; import { preExisting, runTests, testComment, testTargetFromInput, testsFailedNudge, type TestOutcome, type TestTarget } from "./tests.js"; @@ -2766,6 +2767,13 @@ async function publishIfRepoRun( if (followUpPreview !== undefined && input.supersedePreviews === true) await supersedeOlderPreviews(ctx, config, input, followUpPreview); const legs = await runLadderLegs(ctx, executor, config, input, followUpPreview, co.branch, { baseline, build, tests }, assetSink(ref, token, input.pr, config.artifacts)); const followUpProof = proofLinks(legs); + // S08 surfacing: the detector over the diff this follow-up just published, + // rendered into the same Verification section as the rest of the evidence. + const followUpIntegrity = surfacedTestIntegrity({ + diff: push.kind === "pushed" ? push.diff : undefined, + where: "the pull request body", + runId: ctx.runId, + }); const followUp: Evidence = { ...(tests !== undefined ? { tests } : {}), ...(baseline !== undefined ? { testsBaseline: baseline } : {}), @@ -2773,6 +2781,7 @@ async function publishIfRepoRun( ...(input.telemetry === true ? { telemetry: await telemetryIfAsked(ctx, config, input) } : {}), ...(legs.rungs !== undefined ? { rungs: legs.rungs } : {}), ...(followUpProof.length > 0 ? { proof: followUpProof } : {}), + ...(followUpIntegrity !== undefined ? { testIntegrity: followUpIntegrity } : {}), }; await publishVerification(ctx, ref, token, input.pr, followUp); // NO auto-merge and NO rollback watch on this path, deliberately. A review @@ -2857,6 +2866,11 @@ async function publishIfRepoRun( if (legs.flow !== undefined) facts.flow = legs.flow; if (legs.observe !== undefined) facts.observeWindow = legs.observe; const proof = proofLinks(legs); + // S08 surfacing: the detector over the diff this run published (the recorded + // repo-push diff — the same bytes the run page's panel analyses), rendered + // into the Verification section publishVerification amends onto the body. + // Nothing is recorded; off and shadow write nothing at all. + const integrity = surfacedTestIntegrity({ diff: push.diff, where: "the pull request body", runId: ctx.runId }); await publishVerification(ctx, ref, token, pr.number, { ...(tests !== undefined ? { tests } : {}), ...(baseline !== undefined ? { testsBaseline: baseline } : {}), @@ -2867,6 +2881,7 @@ async function publishIfRepoRun( // So a root-level suite over a change confined to one subtree is called // out on the pull request rather than read as a green gate (tests.ts). ...(changedList.length > 0 ? { changedPaths: changedList.map((f) => f.path) } : {}), + ...(integrity !== undefined ? { testIntegrity: integrity } : {}), }); // Computed ONCE, outside any step, and fed to both gates below. It is a pure // function of the `telemetry-check` step's recorded verdict (see diff --git a/src/notify.ts b/src/notify.ts index 0f67269..63e2b97 100644 --- a/src/notify.ts +++ b/src/notify.ts @@ -1,6 +1,7 @@ import type { ScanFinding } from "./findings.js"; import type { ChangeClass } from "./change-class.js"; import { rungsForWire, type Rung } from "./ladder.js"; +import type { TestIntegrityRecord } from "./test-integrity-surface.js"; import { canonicalJson, EVENT_SCHEMA_VERSION } from "./tool-manifest.js"; import type { EventEnvelope } from "./tool-manifest.js"; import { createHash, createHmac } from "node:crypto"; @@ -77,6 +78,13 @@ export interface RunNotification { merged?: boolean; /** The agent's own result status on a terminal run (see RunWebhookPayload.outcome). */ outcome?: string; + /** + * S08 surfacing (advisory, behind SHIP_TEST_INTEGRITY_SURFACING=on): the + * test-integrity detector's findings over the run's published diff. Absent + * when the flag is off or shadow, or when nothing was found — a consumer + * treats absent as "nothing surfaced", never as "the tests are sound". + */ + testIntegrity?: TestIntegrityRecord; } /** @@ -303,6 +311,21 @@ export interface RunWebhookPayload { * wire; this is what says it did not finish. Additive: absent on parks. */ outcome?: string; + /** + * S08 surfacing (additive): present only when SHIP_TEST_INTEGRITY_SURFACING + * is on AND the detector found something in the published diff. Advisory — + * a receiver shows it, never gates on it; the same caveat the run page's + * panel carries (it flags patterns, it does not prove intent). + */ + test_integrity?: { + verdict: "suspicious" | "tampered"; + total: number; + omitted?: number; + findings: Array<{ kind: string; file: string; line: number; severity: string; confidence: string; evidence: string }>; + /** False when the published diff was recorded with a middle cut, so the read is partial. */ + diff_complete: boolean; + oracle_configured: boolean; + }; } export function runWebhookPayload(event: RunNotification, publicUrl?: string): RunWebhookPayload { @@ -338,6 +361,18 @@ export function runWebhookPayload(event: RunNotification, publicUrl?: string): R ? { verification: { rungs: rungsForWire(event.verification.rungs), summary: event.verification.summary } } : {}), ...(event.merged !== undefined ? { merged: event.merged } : {}), + ...(event.testIntegrity !== undefined + ? { + test_integrity: { + verdict: event.testIntegrity.verdict, + total: event.testIntegrity.total, + ...(event.testIntegrity.omitted !== undefined ? { omitted: event.testIntegrity.omitted } : {}), + findings: event.testIntegrity.findings.map((f) => ({ ...f })), + diff_complete: event.testIntegrity.diffComplete, + oracle_configured: event.testIntegrity.oracleConfigured, + }, + } + : {}), }; } diff --git a/src/test-integrity-surface.test.ts b/src/test-integrity-surface.test.ts new file mode 100644 index 0000000..405aa11 --- /dev/null +++ b/src/test-integrity-surface.test.ts @@ -0,0 +1,219 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { runWebhookPayload } from "./notify.js"; +import type { RunNotification } from "./notify.js"; +import { pushDiffFromEvents, surfacedTestIntegrity, testIntegrityRecord, testIntegritySection, testIntegritySurfacingMode } from "./test-integrity-surface.js"; +import type { IntegrityLogEvent } from "./test-integrity-surface.js"; +import { spliceVerification, verificationSection } from "./verification.js"; +import type { Evidence } from "./verification.js"; + +// S08 surfacing (programme wave 5): the detector's findings on the pull +// request body and the run webhook, behind SHIP_TEST_INTEGRITY_SURFACING. +// The repo's wiring discipline applies to every test below: default-off +// byte-equivalence, a real-path test with the flag on through the actual +// composition code, and a negative control per rule. + +// Diff builder, same convention as test-integrity.test.ts: lines carry their +// own +/-/space prefix and the hunk counts are computed. +function edit(path: string, body: string[]): string { + const o = body.filter((l) => l[0] === "-" || l[0] === " ").length; + const n = body.filter((l) => l[0] === "+" || l[0] === " ").length; + return [`diff --git a/${path} b/${path}`, `--- a/${path}`, `+++ b/${path}`, `@@ -10,${o} +10,${n} @@`, ...body, ""].join("\n"); +} + +const SKIP_DIFF = edit("src/billing.test.ts", [" ctx", '+it.skip("bills correctly", () => {', "+ expect(bill(2)).toBe(4);", "+});"]); +const CLEAN_DIFF = edit("src/billing.ts", [" x", "-return 2 * n;", "+return n + n;"]); + +const ev = (type: string, data?: unknown, name?: string): IntegrityLogEvent => ({ type, ...(name !== undefined ? { name } : {}), ...(data !== undefined ? { data } : {}) }); +const pushStep = (result: unknown): IntegrityLogEvent[] => [ev("step-completed", { result }, "repo-push")]; + +// --------------------------------------------------------------------------- +// mode parsing + +test("the flag is off unless explicitly shadow or on; junk is off and logged", () => { + assert.equal(testIntegritySurfacingMode({}).mode, "off"); + assert.equal(testIntegritySurfacingMode({ SHIP_TEST_INTEGRITY_SURFACING: "" }).mode, "off"); + assert.equal(testIntegritySurfacingMode({ SHIP_TEST_INTEGRITY_SURFACING: "off" }).mode, "off"); + assert.equal(testIntegritySurfacingMode({ SHIP_TEST_INTEGRITY_SURFACING: "shadow" }).mode, "shadow"); + assert.equal(testIntegritySurfacingMode({ SHIP_TEST_INTEGRITY_SURFACING: "on" }).mode, "on"); + assert.deepEqual(testIntegritySurfacingMode({ SHIP_TEST_INTEGRITY_SURFACING: "yes" }), { mode: "off", invalid: "yes" }); +}); + +// --------------------------------------------------------------------------- +// (a) default-off byte-equivalence — PR body and webhook payload + +test("flag unset: the PR body's Verification section is byte-identical, no Test integrity part", () => { + const evidence: Evidence = { tests: { kind: "passed", command: "pytest", durationMs: 10 } }; + const withFindingsAvailable = verificationSection({ ...evidence }, "run-1"); + assert.ok(withFindingsAvailable !== null); + // The pre-change build had no testIntegrity field at all; the same input + // must render to the same bytes. + assert.equal(verificationSection({ ...evidence, testIntegrity: undefined }, "run-1"), withFindingsAvailable); + assert.doesNotMatch(withFindingsAvailable, /Test integrity/); + // And the wiring hands the worker nothing to spread in. + for (const env of [{}, { SHIP_TEST_INTEGRITY_SURFACING: "" }, { SHIP_TEST_INTEGRITY_SURFACING: "off" }]) { + assert.equal(surfacedTestIntegrity({ diff: SKIP_DIFF, env, where: "test", log: () => {} }), undefined, JSON.stringify(env)); + } +}); + +test("flag unset: the webhook payload is byte-identical to the pre-change shape", () => { + // Pinned literal: this is exactly what a worker on the default flag sends. + const notification: RunNotification = { runId: "run-9", status: "completed", pr: "http://f/o/r/pulls/9", repo: "o/r", task: "t", outcome: "finished" }; + assert.equal( + JSON.stringify(runWebhookPayload(notification)), + '{"run_id":"run-9","status":"completed","pr":"http://f/o/r/pulls/9","repo":"o/r","task":"t","outcome":"finished"}', + ); + // Through the worker's own composition spread, with findings available: + const integrity = surfacedTestIntegrity({ diff: SKIP_DIFF, env: {}, where: "test", log: () => {} }); + assert.equal(integrity, undefined); + const owed = { ...notification, ...(integrity !== undefined ? { testIntegrity: integrity } : {}) }; + assert.equal(JSON.stringify(runWebhookPayload(owed)), JSON.stringify(runWebhookPayload(notification))); +}); + +// --------------------------------------------------------------------------- +// (b) real path with the flag on — through verificationSection and runWebhookPayload + +test("flag on: findings land in the PR body's Verification section, between the markers", () => { + const integrity = surfacedTestIntegrity({ diff: SKIP_DIFF, env: { SHIP_TEST_INTEGRITY_SURFACING: "on" }, where: "test", log: () => {} }); + assert.ok(integrity !== undefined); + const section = verificationSection({ testIntegrity: integrity }, "run-1"); + assert.ok(section !== null); + assert.match(section, //); + assert.match(section, /\*\*Test integrity\*\* — tampered \(1 finding\)/); + assert.match(section, /`src\/billing\.test\.ts:11` skipped-test \[high\/high\]/); + assert.match(section, /it\.skip\("bills correctly"/); + assert.match(section, /flags patterns, it does not prove intent/); + // The section splices into a body the way every other part does, and a + // second run replaces its span rather than stacking a second copy. + const first = spliceVerification("The agent's summary.", section!); + assert.ok(first.includes("The agent's summary.")); + assert.equal(spliceVerification(first, section!).indexOf("Test integrity"), first.indexOf("Test integrity")); +}); + +test("flag on: the webhook payload gains the additive test_integrity field, snake_case on the wire", () => { + const integrity = surfacedTestIntegrity({ + diff: SKIP_DIFF, + env: { SHIP_TEST_INTEGRITY_SURFACING: "on", SHIP_ORACLE_PATHS: "acceptance/**" }, + where: "test", + log: () => {}, + }); + assert.ok(integrity !== undefined); + assert.equal(integrity.oracleConfigured, true); + const payload = runWebhookPayload({ runId: "run-9", status: "completed", testIntegrity: integrity }); + assert.equal(payload.test_integrity?.verdict, "tampered"); + assert.equal(payload.test_integrity?.total, 1); + assert.equal(payload.test_integrity?.diff_complete, true); + assert.equal(payload.test_integrity?.oracle_configured, true); + assert.deepEqual( + Object.keys(payload.test_integrity!.findings[0]!), + ["kind", "file", "line", "severity", "confidence", "evidence"], + ); + assert.ok(Buffer.byteLength(JSON.stringify(payload)) < 64 * 1024); +}); + +test("flag on: the worker's own composition carries the field off the recorded repo-push diff", () => { + const events = [ + ...pushStep({ kind: "pushed", sha: "feed0000", diff: SKIP_DIFF }), + ev("run-completed", { output: { status: "finished", pr: "http://f/o/r/pulls/9" } }), + ]; + // Exactly the call the terminal branch makes, then exactly its spread. + const integrity = surfacedTestIntegrity({ + diff: pushDiffFromEvents(events), + env: { SHIP_TEST_INTEGRITY_SURFACING: "on" }, + where: "the run webhook payload", + runId: "run-9", + log: () => {}, + }); + const owed: RunNotification = { runId: "run-9", status: "completed", ...(integrity !== undefined ? { testIntegrity: integrity } : {}) }; + const payload = runWebhookPayload(owed); + assert.equal(payload.test_integrity?.findings[0]?.kind, "skipped-test"); + assert.equal(payload.test_integrity?.findings[0]?.file, "src/billing.test.ts"); +}); + +test("an oracle edit surfaces as tampered, not suspicious", () => { + const oracleEdit = edit("acceptance/oracle.test.ts", [" ctx", "+// touched", "+x = 2;"]); + const integrity = surfacedTestIntegrity({ + diff: oracleEdit, + env: { SHIP_TEST_INTEGRITY_SURFACING: "on", SHIP_ORACLE_PATHS: "acceptance/**" }, + where: "test", + log: () => {}, + }); + assert.ok(integrity !== undefined); + assert.equal(integrity.verdict, "tampered"); + assert.ok(integrity.findings.some((f) => f.kind === "oracle-edit")); +}); + +// --------------------------------------------------------------------------- +// (c) negative controls + +test("flag on but no findings: no section, no field — clean is silence, not a clean bill", () => { + const integrity = surfacedTestIntegrity({ diff: CLEAN_DIFF, env: { SHIP_TEST_INTEGRITY_SURFACING: "on" }, where: "test", log: () => {} }); + assert.equal(integrity, undefined, "a clean diff surfaces nothing"); + assert.equal(verificationSection({ testIntegrity: undefined }, "run-1"), null, "no other evidence and no findings means no section at all"); + const payload = runWebhookPayload({ runId: "r", status: "completed" }); + assert.ok(!("test_integrity" in payload)); +}); + +test("no usable diff never crashes and never surfaces", () => { + const on = { SHIP_TEST_INTEGRITY_SURFACING: "on" } as NodeJS.ProcessEnv; + // No repo-push step, an unpushed result, a missing or non-string diff. + assert.equal(surfacedTestIntegrity({ diff: pushDiffFromEvents([]), env: on, where: "test", log: () => {} }), undefined); + assert.equal(surfacedTestIntegrity({ diff: pushDiffFromEvents(pushStep({ kind: "empty" })), env: on, where: "test", log: () => {} }), undefined); + assert.equal(surfacedTestIntegrity({ diff: pushDiffFromEvents(pushStep({ kind: "pushed", sha: "x" })), env: on, where: "test", log: () => {} }), undefined); + assert.equal(surfacedTestIntegrity({ diff: pushDiffFromEvents(pushStep({ kind: "pushed", diff: 42 })), env: on, where: "test", log: () => {} }), undefined); + // Garbage text is not a diff: the parser yields nothing, verdict clean. + assert.equal(testIntegrityRecord("not a diff at all", []), undefined); + assert.equal(surfacedTestIntegrity({ diff: "not a diff at all", env: on, where: "test", log: () => {} }), undefined); +}); + +test("shadow computes and logs what on would surface, and writes nothing", () => { + const lines: string[] = []; + const integrity = surfacedTestIntegrity({ + diff: SKIP_DIFF, + env: { SHIP_TEST_INTEGRITY_SURFACING: "shadow" }, + where: "the pull request body", + runId: "run-4", + log: (l) => lines.push(l), + }); + assert.equal(integrity, undefined, "shadow never hands back a record"); + assert.deepEqual(lines, ["[test-integrity] shadow: run run-4: would surface tampered (1 finding) on the pull request body; nothing was written"]); + // A clean diff in shadow is silence: there is nothing that on would add. + const quiet: string[] = []; + surfacedTestIntegrity({ diff: CLEAN_DIFF, env: { SHIP_TEST_INTEGRITY_SURFACING: "shadow" }, where: "test", log: (l) => quiet.push(l) }); + assert.deepEqual(quiet, []); +}); + +test("an unreadable flag value is treated as off, with one log line", () => { + const lines: string[] = []; + const integrity = surfacedTestIntegrity({ + diff: SKIP_DIFF, + env: { SHIP_TEST_INTEGRITY_SURFACING: "yes" }, + where: "test", + log: (l) => lines.push(l), + }); + assert.equal(integrity, undefined); + assert.deepEqual(lines, ['[test-integrity] SHIP_TEST_INTEGRITY_SURFACING="yes" is not off|shadow|on; treated as off']); +}); + +// --------------------------------------------------------------------------- +// bounds honesty + +test("many findings are capped and the remainder is counted, never silently dropped", () => { + const many = Array.from({ length: 25 }, (_, i) => edit(`src/t${i}.test.ts`, [" ctx", `+it.skip("s${i}", () => {});`])).join(""); + const integrity = testIntegrityRecord(many, []); + assert.ok(integrity !== undefined); + assert.equal(integrity.total, 25); + assert.equal(integrity.findings.length, 20); + assert.equal(integrity.omitted, 5); + assert.match(testIntegritySection(integrity), /- … and 5 more\./); +}); + +test("a middle-cut diff says the read is incomplete, in the section and on the wire", () => { + const cut = `... [1234 chars omitted from the middle of this diff] ...\n${SKIP_DIFF}`; + const integrity = testIntegrityRecord(cut, []); + assert.ok(integrity !== undefined); + assert.equal(integrity.diffComplete, false); + assert.match(testIntegritySection(integrity), /Part of the published diff was omitted, so this read is incomplete\./); + assert.equal(runWebhookPayload({ runId: "r", status: "completed", testIntegrity: integrity }).test_integrity?.diff_complete, false); +}); diff --git a/src/test-integrity-surface.ts b/src/test-integrity-surface.ts new file mode 100644 index 0000000..743dfe9 --- /dev/null +++ b/src/test-integrity-surface.ts @@ -0,0 +1,183 @@ +/** + * Wiring for surfacing the S08 test-integrity detector (src/test-integrity.ts) + * where a reviewer or a consumer actually looks: the pull request body and the + * run-completion webhook. Approved by the owner 2026-10-04 (programme wave-5 + * notes); before that the detector's only surface was the run page's read-time + * panel (web/src/lib/test-integrity-view.ts), which nothing outside the + * dashboard reads. + * + * MODES (`SHIP_TEST_INTEGRITY_SURFACING`), the standing shadow-first rule for + * a worker-path change: + * + * unset / "off" default. Nothing is computed or written; the pull request + * body and the webhook payload are byte-identical to a build + * without this module (pinned by a test). + * "shadow" the detector runs at the same points and what it WOULD + * surface is logged; nothing is written. Run this on a real + * worker and read the logs before turning "on". + * "on" findings are surfaced: a "Test integrity" part inside the + * pull request's existing Verification section + * (verification.ts) and an additive `test_integrity` field + * on the run webhook payload (notify.ts). + * + * An unreadable value is treated as "off" and logged, the same contract as + * SHIP_BUDGET_RESERVATION (budget-gate.ts): an operator typo must not turn + * surfacing on by accident. + * + * WHAT IS NOT CHANGED. No workflow step is added and no recorded step result + * changes, in any mode: the analysis runs beside the existing publish points + * and nothing is written to the durable log, so a replay reads back exactly + * what it recorded. The delivery record (delivery.ts) is untouched — the + * approval was pull request body and webhook only. The one step-sequence + * effect, under "on" only: a run whose ONLY verification part is an integrity + * finding now records the existing `verification` step where it previously + * had nothing to publish. ADVISORY ONLY, like the panel: nothing here feeds + * approval, merge, classification or the delivery record. + * + * CLEAN MEANS NOTHING TO SAY. With the flag on and no findings, no section is + * added and no field is sent. Printing "no test-tampering patterns found" on + * every clean pull request would train a reviewer to skip the section that + * sometimes carries the real thing (the same rule fix-evidence.ts states), and + * on the wire an absent field is the honest shape: a consumer treats absent + * as "nothing surfaced", never as "the tests are sound". + */ +import { analyzeTestIntegrity } from "./test-integrity.js"; +import type { TestIntegrityFinding } from "./test-integrity.js"; + +export type TestIntegritySurfacingMode = "off" | "shadow" | "on"; + +export const TEST_INTEGRITY_SURFACING_FLAG = "SHIP_TEST_INTEGRITY_SURFACING"; + +export function testIntegritySurfacingMode(env: NodeJS.ProcessEnv = process.env): { mode: TestIntegritySurfacingMode; invalid?: string } { + const raw = (env[TEST_INTEGRITY_SURFACING_FLAG] ?? "").trim().toLowerCase(); + if (raw === "" || raw === "off") return { mode: "off" }; + if (raw === "shadow" || raw === "on") return { mode: raw }; + return { mode: "off", invalid: env[TEST_INTEGRITY_SURFACING_FLAG] as string }; +} + +/** `SHIP_ORACLE_PATHS`: comma-separated globs, blanks dropped; unset is no oracle. Same contract as the web panel's parser. */ +export function parseOraclePaths(raw: string | undefined): string[] { + return (raw ?? "").split(",").map((s) => s.trim()).filter((s) => s !== ""); +} + +/** How many findings travel; the rest are counted in `omitted`. The detector clips each evidence line itself. */ +export const SURFACE_FINDINGS_CAP = 20; + +/** The middle-cut marker git.ts truncateMiddle writes (web/src/lib/test-integrity-view.ts reads the same shape). */ +const DIFF_CUT = /\.\.\. \[(?:\d+ chars )?omitted from the middle of this diff\] \.\.\./; + +export type SurfaceFinding = Pick; + +export interface TestIntegrityRecord { + /** Findings exist whenever a record exists, so this is never "clean". */ + verdict: "suspicious" | "tampered"; + total: number; + findings: SurfaceFinding[]; + /** Findings past SURFACE_FINDINGS_CAP, still counted in `total`. */ + omitted?: number; + /** False when the published diff had a middle section cut out, so the read is partial. */ + diffComplete: boolean; + oracleConfigured: boolean; +} + +/** + * The detector over a published diff, bounded for both surfaces. Undefined — + * never a throw — when there is no diff, nothing parseable, or no finding: + * every one of those adds nothing, by the rule in the module header. + */ +export function testIntegrityRecord(diff: string | undefined, oraclePaths: string[]): TestIntegrityRecord | undefined { + if (typeof diff !== "string" || !diff.includes("diff --git ")) return undefined; + let analysed: ReturnType; + try { + analysed = analyzeTestIntegrity(diff, { oraclePaths }); + } catch { + // Advisory: a detector fault must not fail a run or a delivery. + return undefined; + } + const findings = analysed.findings; + if (findings.length === 0) return undefined; + const shown = findings.slice(0, SURFACE_FINDINGS_CAP).map(({ kind, file, line, severity, confidence, evidence }) => ({ + kind, + file, + line, + severity, + confidence, + evidence, + })); + return { + verdict: analysed.verdict === "tampered" ? "tampered" : "suspicious", + total: findings.length, + findings: shown, + ...(findings.length > shown.length ? { omitted: findings.length - shown.length } : {}), + diffComplete: !DIFF_CUT.test(diff), + oracleConfigured: oraclePaths.length > 0, + }; +} + +/** + * The pull request's Test integrity part. Wording follows the run page's + * panel (web/src/views/workspace.tsx TestIntegrityLine) on purpose: the two + * surfaces should read as the same statement, and the panel's wording is + * already pinned by its own tests. + */ +export function testIntegritySection(record: TestIntegrityRecord): string { + const lines = [ + `**Test integrity** — ${record.verdict} (${record.total} finding${record.total === 1 ? "" : "s"}). A person should look; this flags patterns, it does not prove intent.`, + ...record.findings.map((f) => `- \`${f.file}${f.line > 0 ? `:${f.line}` : ""}\` ${f.kind} [${f.severity}/${f.confidence}] ${f.evidence}`), + ]; + if (record.omitted !== undefined && record.omitted > 0) lines.push(`- … and ${record.omitted} more.`); + if (!record.diffComplete) lines.push("Part of the published diff was omitted, so this read is incomplete."); + return lines.join("\n"); +} + +export interface IntegrityLogEvent { + type: string; + name?: string; + data?: unknown; +} + +/** + * The diff a run published, off its recorded `repo-push` step — the same read + * the web panel does, so the two surfaces cannot analyse different bytes. + * Undefined when no push was recorded, nothing was pushed, or the diff was + * not recorded. + */ +export function pushDiffFromEvents(events: readonly IntegrityLogEvent[]): string | undefined { + const push = events.filter((e) => e.type === "step-completed" && e.name === "repo-push").at(-1); + const result = push?.data as { result?: { kind?: unknown; diff?: unknown } } | undefined; + if (result?.result?.kind !== "pushed") return undefined; + return typeof result.result.diff === "string" ? result.result.diff : undefined; +} + +/** + * What both surface points call: read the mode, run the detector over the + * diff, and either hand back the record ("on"), log and hand back nothing + * ("shadow"), or do nothing at all ("off"). Returns undefined whenever + * nothing should be surfaced, so every caller is one conditional spread. + */ +export function surfacedTestIntegrity(args: { + diff: string | undefined; + /** Read for the mode AND the oracle paths, so a test can pass a fake env. */ + env?: NodeJS.ProcessEnv; + log?: (line: string) => void; + /** What the shadow log names as the destination, e.g. "the run webhook payload". */ + where: string; + runId?: string; +}): TestIntegrityRecord | undefined { + const env = args.env ?? process.env; + const log = args.log ?? ((line: string) => console.warn(line)); + const { mode, invalid } = testIntegritySurfacingMode(env); + if (invalid !== undefined) { + log(`[test-integrity] ${TEST_INTEGRITY_SURFACING_FLAG}=${JSON.stringify(invalid)} is not off|shadow|on; treated as off`); + } + if (mode === "off") return undefined; + const record = testIntegrityRecord(args.diff, parseOraclePaths(env.SHIP_ORACLE_PATHS)); + if (record === undefined) return undefined; + if (mode === "shadow") { + log( + `[test-integrity] shadow: ${args.runId !== undefined ? `run ${args.runId}: ` : ""}would surface ${record.verdict} (${record.total} finding${record.total === 1 ? "" : "s"}) on ${args.where}; nothing was written`, + ); + return undefined; + } + return record; +} diff --git a/src/verification.ts b/src/verification.ts index 34b4163..ce54ba1 100644 --- a/src/verification.ts +++ b/src/verification.ts @@ -24,6 +24,7 @@ import { telemetryComment } from "./observe.js"; import type { TestOutcome } from "./tests.js"; import { testComment, testScopeNote } from "./tests.js"; import type { Rung } from "./ladder.js"; +import { testIntegritySection, type TestIntegrityRecord } from "./test-integrity-surface.js"; export const VERIFICATION_START = ""; export const VERIFICATION_END = ""; @@ -56,6 +57,14 @@ export interface Evidence { * proofLinks): the visual pair and the flow's shots, in reading order. */ proof?: Array<{ name: string; url: string }>; + /** + * S08 surfacing (advisory, behind SHIP_TEST_INTEGRITY_SURFACING=on): the + * detector's findings over the published diff, rendered by + * test-integrity-surface.ts next to the test line they qualify. Present + * only when the flag is on AND something was found; absent is + * byte-identical to a build without the field. + */ + testIntegrity?: TestIntegrityRecord; } /** @@ -75,6 +84,11 @@ export function verificationSection(evidence: Evidence, runId: string): string | const scope = testScopeNote(tests.command, evidence.changedPaths ?? []); if (scope !== undefined) parts.push(`**Scope:** ${scope}`); } + // Beside the test line it qualifies, the same placement the run page's + // Verification panel uses for its advisory line. + if (evidence.testIntegrity !== undefined) { + parts.push(testIntegritySection(evidence.testIntegrity)); + } const preview = evidence.preview; if (preview !== undefined && preview.kind !== "skipped") { parts.push(previewComment(preview, runId)); diff --git a/src/worker.ts b/src/worker.ts index 100a053..48addbe 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -66,6 +66,7 @@ import type { Windows } from "./governance.js"; import { makeObserveEmitter } from "./observe.js"; import { multiNotifier, projectNotifier, scanReport, slackNotifier, webhookNotifier } from "./notify.js"; import type { RunNotification, RunOrigin } from "./notify.js"; +import { pushDiffFromEvents, surfacedTestIntegrity } from "./test-integrity-surface.js"; import { NucleusAkirooReceipts } from "./akiroo-receipts.js"; import { ladderRungsFromEvents, type Rung } from "./ladder.js"; import { runVerificationSummary, verificationFactsFromEvents } from "./verification-summary.js"; @@ -1118,7 +1119,23 @@ export function startWorker(options: WorkerOptions): { } // Terminal: include the PR link when the run opened one, and the // findings when it was a scan. - await owe({ runId, status: outcome.status, ...context, ...terminalContext(events), ...verification }); + // S08 surfacing (flag-gated, advisory): the detector over the + // published diff, as an additive field. Off is no field and no + // computation worth speaking of; shadow logs what on would send. + const integrity = surfacedTestIntegrity({ + diff: pushDiffFromEvents(events), + log, + where: "the run webhook payload", + runId, + }); + await owe({ + runId, + status: outcome.status, + ...context, + ...terminalContext(events), + ...verification, + ...(integrity !== undefined ? { testIntegrity: integrity } : {}), + }); await flush(); })()); }