diff --git a/notebook-result-replay-ledger/README.md b/notebook-result-replay-ledger/README.md new file mode 100644 index 0000000..82cf78d --- /dev/null +++ b/notebook-result-replay-ledger/README.md @@ -0,0 +1,73 @@ +# Notebook Result Replay Ledger + +This module adds a focused computation-aware reproducibility slice for +SCIBASE project repositories. It models notebook cells, data/code/environment +dependencies, result artifacts, and release gates so a repository version can +answer one practical question before publication: + +> Which scientific outputs were generated from the current data, code, and +> runtime evidence, and which outputs must be replayed before this version is +> cited or released? + +The implementation is dependency-free and can run as a local validation step, +pre-release check, or future API service. + +## What It Covers + +- Hashes notebook inputs, code, runtime specs, and result artifacts into a + stable replay digest. +- Detects stale result outputs after upstream data, code, notebook, or + environment changes. +- Separates release-blocking failures from advisory replay work. +- Builds rollback-ready replay packets for reviewers and maintainers. +- Emits citation/version impact notes when a tagged repository version should + not advertise a clean "cite this project" badge. +- Includes tests, sample scientific repository data, a CLI demo, a requirement + map, and a short demo video. + +## Quick Start + +```bash +npm run check +npm test +npm run demo +``` + +Expected demo summary: + +```text +Release decision: block-release +Replay findings: 2 +Release-blocking artifacts: 2 +``` + +## Repository Layout + +```text +notebook-result-replay-ledger/ + data/sample-repository.json + docs/demo.svg + docs/demo.mp4 + docs/requirement-map.md + scripts/demo.js + src/replay-ledger.js + test/replay-ledger.test.js +``` + +## Design Notes + +The ledger intentionally stays narrower than a full project repository backend. +It assumes SCIBASE already has repository components such as `data/`, `code/`, +`notebooks/`, `results/`, `protocols/`, and `metadata.json`. This package adds +the reproducibility layer that ties those components together at publication +time: + +1. A notebook cell declares its data, code, notebook, and environment inputs. +2. A result artifact stores the dependency hashes used when it was last + replayed. +3. The ledger recomputes a current replay digest from the repository manifest. +4. Any mismatch, failed run, or late dependency change becomes a finding. +5. Findings are classified into release-blocking or advisory outcomes. + +That makes the result suitable for a pull request check, a release checklist, or +a reviewer-facing reproducibility panel. diff --git a/notebook-result-replay-ledger/data/sample-repository.json b/notebook-result-replay-ledger/data/sample-repository.json new file mode 100644 index 0000000..262e704 --- /dev/null +++ b/notebook-result-replay-ledger/data/sample-repository.json @@ -0,0 +1,176 @@ +{ + "schemaVersion": "replay-ledger.v1", + "repository": { + "id": "scibase-neuro-organoid-atlas", + "name": "Neuro Organoid Response Atlas", + "releaseCandidate": "preprint-v2.2-rc1", + "lastStableTag": "preprint-v2.1", + "doi": "10.5555/scibase.neuro-organoid-atlas.v2", + "citationBadge": "cite-this-project" + }, + "components": [ + { + "id": "data.raw-counts", + "kind": "data", + "path": "data/raw/organoid-expression.csv", + "hash": "sha256:data-raw-counts-v4", + "previousHash": "sha256:data-raw-counts-v3", + "changedAt": "2026-05-12T10:00:00Z", + "lastStableTag": "preprint-v2.1", + "largeFile": true + }, + { + "id": "data.qc-manifest", + "kind": "data", + "path": "data/processed/qc-manifest.json", + "hash": "sha256:data-qc-manifest-v2", + "previousHash": "sha256:data-qc-manifest-v1", + "changedAt": "2026-05-12T10:35:00Z", + "lastStableTag": "preprint-v2.1" + }, + { + "id": "code.normalizer", + "kind": "code", + "path": "code/analysis/normalize_counts.py", + "hash": "sha256:normalizer-v5", + "previousHash": "sha256:normalizer-v4", + "changedAt": "2026-05-13T08:25:00Z", + "lastStableTag": "preprint-v2.1" + }, + { + "id": "code.plotter", + "kind": "code", + "path": "code/analysis/build_figures.py", + "hash": "sha256:plotter-v2", + "previousHash": "sha256:plotter-v2", + "changedAt": "2026-05-10T12:00:00Z", + "lastStableTag": "preprint-v2.1" + }, + { + "id": "notebook.run-analysis", + "kind": "notebook", + "path": "notebooks/run_analysis.ipynb", + "hash": "sha256:run-analysis-v7", + "previousHash": "sha256:run-analysis-v6", + "changedAt": "2026-05-13T09:10:00Z", + "lastStableTag": "preprint-v2.1" + }, + { + "id": "notebook.qc-review", + "kind": "notebook", + "path": "notebooks/qc_review.ipynb", + "hash": "sha256:qc-review-v3", + "previousHash": "sha256:qc-review-v3", + "changedAt": "2026-05-11T14:00:00Z", + "lastStableTag": "preprint-v2.1" + }, + { + "id": "env.analysis", + "kind": "environment", + "path": "environment.yml", + "hash": "sha256:conda-analysis-2026-05-14", + "previousHash": "sha256:conda-analysis-2026-05-08", + "changedAt": "2026-05-14T07:45:00Z", + "lastStableTag": "preprint-v2.1" + }, + { + "id": "result.figure-1a", + "kind": "result", + "path": "results/figure-1a.png", + "hash": "sha256:figure-1a-before-data-update", + "lastReplayedAt": "2026-05-11T16:20:00Z", + "status": "passed", + "releaseCritical": true, + "citationCritical": true, + "recordedDependencyHashes": { + "data.raw-counts": "sha256:data-raw-counts-v3", + "code.normalizer": "sha256:normalizer-v4", + "code.plotter": "sha256:plotter-v2", + "notebook.run-analysis": "sha256:run-analysis-v6", + "env.analysis": "sha256:conda-analysis-2026-05-08" + } + }, + { + "id": "result.model-metrics", + "kind": "result", + "path": "results/model-metrics.json", + "hash": "sha256:model-metrics-current", + "lastReplayedAt": "2026-05-14T09:15:00Z", + "status": "passed", + "releaseCritical": true, + "citationCritical": true, + "recordedDependencyHashes": { + "data.raw-counts": "sha256:data-raw-counts-v4", + "code.normalizer": "sha256:normalizer-v5", + "notebook.run-analysis": "sha256:run-analysis-v7", + "env.analysis": "sha256:conda-analysis-2026-05-14" + } + }, + { + "id": "result.qc-table", + "kind": "result", + "path": "results/qc-table.csv", + "hash": "sha256:qc-table-failed-replay", + "lastReplayedAt": "2026-05-14T08:05:00Z", + "status": "failed", + "releaseCritical": true, + "citationCritical": false, + "failureReason": "Notebook replay stopped after container image changed.", + "recordedDependencyHashes": { + "data.qc-manifest": "sha256:data-qc-manifest-v2", + "notebook.qc-review": "sha256:qc-review-v3", + "env.analysis": "sha256:conda-analysis-2026-05-14" + } + }, + { + "id": "protocol.analysis-plan", + "kind": "protocol", + "path": "protocols/analysis-plan.md", + "hash": "sha256:analysis-plan-v2", + "changedAt": "2026-05-09T10:00:00Z", + "lastStableTag": "preprint-v2.1" + }, + { + "id": "metadata.project", + "kind": "metadata", + "path": "metadata.json", + "hash": "sha256:metadata-v4", + "changedAt": "2026-05-14T11:00:00Z", + "lastStableTag": "preprint-v2.1" + } + ], + "notebookCells": [ + { + "id": "cell.figure-1a", + "notebookId": "notebook.run-analysis", + "label": "Generate Figure 1A response heatmap", + "inputComponentIds": ["data.raw-counts"], + "codeComponentIds": ["code.normalizer", "code.plotter"], + "environmentComponentId": "env.analysis", + "outputResultIds": ["result.figure-1a"] + }, + { + "id": "cell.model-metrics", + "notebookId": "notebook.run-analysis", + "label": "Recompute classifier metrics", + "inputComponentIds": ["data.raw-counts"], + "codeComponentIds": ["code.normalizer"], + "environmentComponentId": "env.analysis", + "outputResultIds": ["result.model-metrics"] + }, + { + "id": "cell.qc-table", + "notebookId": "notebook.qc-review", + "label": "Validate sample quality table", + "inputComponentIds": ["data.qc-manifest"], + "codeComponentIds": [], + "environmentComponentId": "env.analysis", + "outputResultIds": ["result.qc-table"] + } + ], + "releaseRules": { + "blockOnFailedReplay": true, + "blockOnCriticalStaleOutputs": true, + "warnOnCitationCriticalStaleOutputs": true + } +} diff --git a/notebook-result-replay-ledger/docs/demo.mp4 b/notebook-result-replay-ledger/docs/demo.mp4 new file mode 100644 index 0000000..ce71d54 Binary files /dev/null and b/notebook-result-replay-ledger/docs/demo.mp4 differ diff --git a/notebook-result-replay-ledger/docs/demo.svg b/notebook-result-replay-ledger/docs/demo.svg new file mode 100644 index 0000000..fd68f25 --- /dev/null +++ b/notebook-result-replay-ledger/docs/demo.svg @@ -0,0 +1,52 @@ + + Notebook result replay ledger demo + Dashboard showing release blocked by stale scientific notebook results and failed replay evidence. + + + Notebook Result Replay Ledger + SCIBASE project repository release gate for preprint-v2.2-rc1 + + BLOCK RELEASE + + + Replay summary + 2 + findings across notebook outputs + 2 release-critical artifacts blocked + 1 citation badge impact + + + Current digest + sha256:dc54...2c76 + Current data, code, notebook, and env hashes. + + + Recorded digest + sha256:6168...8361 + Stored before upstream data/code changes. + + + Release-blocking evidence + + + + results/figure-1a.png + Data, code, notebook, and environment hashes changed after replay. + + needs replay + + + results/qc-table.csv + Notebook replay failed after the container image changed. + + failed replay + + + results/model-metrics.json + Current replay digest matches the release candidate manifest. + + clean + + + Rollback packet: preprint-v2.1:results/figure-1a.png | replay command: scibase replay cell.figure-1a --candidate preprint-v2.2-rc1 + diff --git a/notebook-result-replay-ledger/docs/requirement-map.md b/notebook-result-replay-ledger/docs/requirement-map.md new file mode 100644 index 0000000..6b9a61a --- /dev/null +++ b/notebook-result-replay-ledger/docs/requirement-map.md @@ -0,0 +1,48 @@ +# Requirement Map + +Issue: SCIBASE-AI/SCIBASE.AI#10, Project Repository & Version Control. + +## Repository Structure & Components + +The sample manifest models scientific repository components across `data/`, +`code/`, `notebooks/`, `results/`, `protocols/`, and `metadata.json`. The +ledger validates that each component has a stable id, path, kind, content hash, +and timestamp where relevant. + +## File & Metadata Versioning + +Every component carries a `sha256:` content hash plus previous hash and stable +tag metadata where useful. Result artifacts store the dependency hashes used at +the last replay, allowing the ledger to compare recorded evidence against the +current release-candidate manifest. + +## Computation-Aware Reproducibility + +Notebook cells declare their data inputs, code inputs, notebook file, runtime +spec, and result artifacts. The ledger derives a replay digest from that +evidence, detects stale result outputs after dependency changes, and classifies +failed notebook replays as release blockers. + +## Rollback And Release Gating + +Each finding creates a rollback packet with the last stable tag, result path, +dependency paths, and replay command. Release-critical stale artifacts return +`block-release`; clean artifacts stay out of the findings list. + +## Repository Identifiers & Citation + +Citation-critical results generate impact notes when the release candidate +should not present a clean citation badge. This keeps DOI/version metadata from +overstating reproducibility before a result is replayed. + +## Programmatic Access & Export + +The module exports pure functions (`createReplayLedger`, +`validateRepositoryManifest`, `createReplaySnapshot`) that can back a REST +endpoint, CLI command, pull request check, or export-bundle validator. + +## Scope Boundary + +This is not another broad repository implementation. It is a focused +reproducibility layer for notebook/result replay evidence inside the project +repository model described by the bounty. diff --git a/notebook-result-replay-ledger/package.json b/notebook-result-replay-ledger/package.json new file mode 100644 index 0000000..cb72958 --- /dev/null +++ b/notebook-result-replay-ledger/package.json @@ -0,0 +1,15 @@ +{ + "name": "notebook-result-replay-ledger", + "version": "0.1.0", + "description": "Dependency-free replay ledger for scientific notebook outputs and release gates.", + "private": true, + "scripts": { + "check": "node scripts/demo.js --json > /dev/null", + "demo": "node scripts/demo.js", + "test": "node --test test/*.test.js" + }, + "engines": { + "node": ">=18" + }, + "license": "MIT" +} diff --git a/notebook-result-replay-ledger/scripts/demo.js b/notebook-result-replay-ledger/scripts/demo.js new file mode 100644 index 0000000..1cc2394 --- /dev/null +++ b/notebook-result-replay-ledger/scripts/demo.js @@ -0,0 +1,17 @@ +const fs = require("node:fs"); +const path = require("node:path"); +const { createReplayLedger, formatLedgerReport } = require("../src/replay-ledger"); + +const manifestPath = path.join(__dirname, "..", "data", "sample-repository.json"); +const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); +const ledger = createReplayLedger(manifest); + +if (process.argv.includes("--json")) { + process.stdout.write(`${JSON.stringify(ledger, null, 2)}\n`); +} else { + process.stdout.write(`${formatLedgerReport(ledger)}\n`); +} + +if (ledger.releaseDecision === "invalid-manifest") { + process.exitCode = 1; +} diff --git a/notebook-result-replay-ledger/src/replay-ledger.js b/notebook-result-replay-ledger/src/replay-ledger.js new file mode 100644 index 0000000..6e8052d --- /dev/null +++ b/notebook-result-replay-ledger/src/replay-ledger.js @@ -0,0 +1,371 @@ +const crypto = require("node:crypto"); + +const REQUIRED_COMPONENT_KINDS = new Set([ + "data", + "code", + "notebook", + "result", + "environment", + "protocol", + "metadata" +]); + +function stableStringify(value) { + if (Array.isArray(value)) { + return `[${value.map((entry) => stableStringify(entry)).join(",")}]`; + } + + if (value && typeof value === "object") { + const keys = Object.keys(value).sort(); + return `{${keys + .map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`) + .join(",")}}`; + } + + return JSON.stringify(value); +} + +function digest(value) { + return `sha256:${crypto.createHash("sha256").update(stableStringify(value)).digest("hex")}`; +} + +function assertIsoDate(value, label, errors) { + if (!value || Number.isNaN(Date.parse(value))) { + errors.push(`${label} must be an ISO timestamp`); + } +} + +function buildComponentIndex(manifest) { + return new Map((manifest.components || []).map((component) => [component.id, component])); +} + +function requireComponent(index, id, label) { + const component = index.get(id); + if (!component) { + throw new Error(`Missing ${label} component: ${id}`); + } + return component; +} + +function componentReference(component, hashOverride) { + return { + id: component.id, + kind: component.kind, + path: component.path, + hash: hashOverride || component.hash + }; +} + +function createReplaySnapshot(manifest, cell, recordedHashes = {}) { + const index = buildComponentIndex(manifest); + const notebook = requireComponent(index, cell.notebookId, "notebook"); + const environment = requireComponent(index, cell.environmentComponentId, "environment"); + const inputComponents = cell.inputComponentIds.map((id) => requireComponent(index, id, "input")); + const codeComponents = cell.codeComponentIds.map((id) => requireComponent(index, id, "code")); + + return { + repositoryId: manifest.repository.id, + releaseCandidate: manifest.repository.releaseCandidate, + cellId: cell.id, + cellLabel: cell.label, + notebook: componentReference(notebook, recordedHashes[notebook.id]), + inputs: inputComponents.map((component) => componentReference(component, recordedHashes[component.id])), + code: codeComponents.map((component) => componentReference(component, recordedHashes[component.id])), + environment: componentReference(environment, recordedHashes[environment.id]) + }; +} + +function listCellDependencyIds(cell) { + return [ + cell.notebookId, + ...cell.inputComponentIds, + ...cell.codeComponentIds, + cell.environmentComponentId + ]; +} + +function changedAfterReplay(component, result) { + if (!component.changedAt || !result.lastReplayedAt) { + return false; + } + + return Date.parse(component.changedAt) > Date.parse(result.lastReplayedAt); +} + +function buildReplayFinding(manifest, cell, result) { + const index = buildComponentIndex(manifest); + const dependencyIds = listCellDependencyIds(cell); + const currentSnapshot = createReplaySnapshot(manifest, cell); + const recordedSnapshot = createReplaySnapshot(manifest, cell, result.recordedDependencyHashes || {}); + const currentDigest = digest(currentSnapshot); + const recordedDigest = digest(recordedSnapshot); + const staleDependencies = dependencyIds + .map((id) => requireComponent(index, id, "dependency")) + .filter((component) => { + const recordedHash = result.recordedDependencyHashes?.[component.id]; + return recordedHash && recordedHash !== component.hash; + }) + .map((component) => ({ + id: component.id, + path: component.path, + kind: component.kind, + recordedHash: result.recordedDependencyHashes[component.id], + currentHash: component.hash, + changedAt: component.changedAt + })); + + const lateChanges = dependencyIds + .map((id) => requireComponent(index, id, "dependency")) + .filter((component) => changedAfterReplay(component, result)) + .map((component) => ({ + id: component.id, + path: component.path, + kind: component.kind, + changedAt: component.changedAt + })); + + const problems = []; + if (result.status === "failed") { + problems.push({ + code: "failed-replay", + message: result.failureReason || "Notebook replay did not complete." + }); + } + + if (staleDependencies.length > 0) { + problems.push({ + code: "stale-dependency-hash", + message: "Recorded replay dependencies no longer match the repository manifest." + }); + } + + if (lateChanges.length > 0) { + problems.push({ + code: "dependency-changed-after-replay", + message: "One or more dependencies changed after the result was last replayed." + }); + } + + const severity = result.releaseCritical && problems.length > 0 ? "block-release" : "advisory"; + + return { + resultId: result.id, + resultPath: result.path, + cellId: cell.id, + cellLabel: cell.label, + releaseCritical: Boolean(result.releaseCritical), + citationCritical: Boolean(result.citationCritical), + lastReplayedAt: result.lastReplayedAt, + resultHash: result.hash, + currentReplayDigest: currentDigest, + recordedReplayDigest: recordedDigest, + staleDependencies, + lateChanges, + problems, + severity + }; +} + +function validateRepositoryManifest(manifest) { + const errors = []; + + if (manifest.schemaVersion !== "replay-ledger.v1") { + errors.push("schemaVersion must be replay-ledger.v1"); + } + + if (!manifest.repository?.id) { + errors.push("repository.id is required"); + } + + if (!manifest.repository?.releaseCandidate) { + errors.push("repository.releaseCandidate is required"); + } + + const seenIds = new Set(); + const kinds = new Set(); + for (const component of manifest.components || []) { + if (!component.id) { + errors.push("every component requires an id"); + } + if (seenIds.has(component.id)) { + errors.push(`duplicate component id: ${component.id}`); + } + seenIds.add(component.id); + + if (!component.kind || !REQUIRED_COMPONENT_KINDS.has(component.kind)) { + errors.push(`component ${component.id || ""} has unsupported kind`); + } else { + kinds.add(component.kind); + } + + if (!component.path) { + errors.push(`component ${component.id || ""} requires a path`); + } + if (!component.hash?.startsWith("sha256:")) { + errors.push(`component ${component.id || ""} requires a sha256 hash`); + } + if (component.changedAt) { + assertIsoDate(component.changedAt, `${component.id}.changedAt`, errors); + } + if (component.lastReplayedAt) { + assertIsoDate(component.lastReplayedAt, `${component.id}.lastReplayedAt`, errors); + } + } + + for (const kind of REQUIRED_COMPONENT_KINDS) { + if (!kinds.has(kind)) { + errors.push(`repository manifest should include at least one ${kind} component`); + } + } + + const index = buildComponentIndex(manifest); + for (const cell of manifest.notebookCells || []) { + if (!cell.id) { + errors.push("every notebook cell requires an id"); + } + if (!index.has(cell.notebookId)) { + errors.push(`cell ${cell.id} references missing notebook ${cell.notebookId}`); + } + if (!index.has(cell.environmentComponentId)) { + errors.push(`cell ${cell.id} references missing environment ${cell.environmentComponentId}`); + } + for (const id of [...(cell.inputComponentIds || []), ...(cell.codeComponentIds || []), ...(cell.outputResultIds || [])]) { + if (!index.has(id)) { + errors.push(`cell ${cell.id} references missing component ${id}`); + } + } + } + + return errors; +} + +function buildRollbackPacket(manifest, finding) { + const stalePaths = finding.staleDependencies.map((dependency) => dependency.path); + const replayCommand = `scibase replay ${finding.cellId} --candidate ${manifest.repository.releaseCandidate}`; + + return { + resultId: finding.resultId, + resultPath: finding.resultPath, + lastKnownGoodTag: manifest.repository.lastStableTag, + rollbackTarget: `${manifest.repository.lastStableTag}:${finding.resultPath}`, + replayCommand, + dependencyPaths: stalePaths, + reviewerNote: + finding.severity === "block-release" + ? "Hold the release until this notebook cell is replayed with the current manifest." + : "Replay recommended before advertising this output as current." + }; +} + +function buildCitationImpact(manifest, findings) { + return findings + .filter((finding) => finding.citationCritical && finding.problems.length > 0) + .map((finding) => ({ + resultId: finding.resultId, + badge: manifest.repository.citationBadge, + releaseCandidate: manifest.repository.releaseCandidate, + note: `${manifest.repository.releaseCandidate} should not show a clean citation badge until ${finding.resultPath} is replayed.` + })); +} + +function createReplayLedger(manifest) { + const validationErrors = validateRepositoryManifest(manifest); + if (validationErrors.length > 0) { + return { + valid: false, + validationErrors, + releaseDecision: "invalid-manifest", + findings: [], + rollbackPackets: [], + citationImpacts: [], + summary: { + totalResults: 0, + replayFindings: 0, + releaseBlockingArtifacts: 0, + citationImpacts: 0 + } + }; + } + + const index = buildComponentIndex(manifest); + const allFindings = []; + for (const cell of manifest.notebookCells) { + for (const resultId of cell.outputResultIds) { + const result = requireComponent(index, resultId, "result"); + const finding = buildReplayFinding(manifest, cell, result); + if (finding.problems.length > 0) { + allFindings.push(finding); + } + } + } + + const releaseBlockingFindings = allFindings.filter((finding) => finding.severity === "block-release"); + const releaseDecision = releaseBlockingFindings.length > 0 ? "block-release" : "ready"; + const rollbackPackets = allFindings.map((finding) => buildRollbackPacket(manifest, finding)); + const citationImpacts = buildCitationImpact(manifest, allFindings); + const summary = { + repositoryId: manifest.repository.id, + releaseCandidate: manifest.repository.releaseCandidate, + totalResults: manifest.components.filter((component) => component.kind === "result").length, + replayFindings: allFindings.length, + releaseBlockingArtifacts: releaseBlockingFindings.length, + citationImpacts: citationImpacts.length + }; + + return { + valid: true, + validationErrors: [], + releaseDecision, + findings: allFindings, + rollbackPackets, + citationImpacts, + summary, + auditDigest: digest({ + summary, + findings: allFindings.map((finding) => ({ + resultId: finding.resultId, + severity: finding.severity, + currentReplayDigest: finding.currentReplayDigest, + recordedReplayDigest: finding.recordedReplayDigest + })) + }) + }; +} + +function formatLedgerReport(ledger) { + if (!ledger.valid) { + return [`Manifest invalid:`, ...ledger.validationErrors.map((error) => `- ${error}`)].join("\n"); + } + + const lines = [ + `Repository: ${ledger.summary.repositoryId}`, + `Release candidate: ${ledger.summary.releaseCandidate}`, + `Release decision: ${ledger.releaseDecision}`, + `Replay findings: ${ledger.summary.replayFindings}`, + `Release-blocking artifacts: ${ledger.summary.releaseBlockingArtifacts}`, + `Citation impacts: ${ledger.summary.citationImpacts}`, + `Audit digest: ${ledger.auditDigest}`, + "" + ]; + + for (const finding of ledger.findings) { + lines.push(`${finding.severity.toUpperCase()} ${finding.resultPath}`); + lines.push(` cell: ${finding.cellLabel}`); + lines.push(` current digest: ${finding.currentReplayDigest}`); + lines.push(` recorded digest: ${finding.recordedReplayDigest}`); + for (const problem of finding.problems) { + lines.push(` - ${problem.code}: ${problem.message}`); + } + } + + return lines.join("\n"); +} + +module.exports = { + createReplayLedger, + createReplaySnapshot, + digest, + formatLedgerReport, + stableStringify, + validateRepositoryManifest +}; diff --git a/notebook-result-replay-ledger/test/replay-ledger.test.js b/notebook-result-replay-ledger/test/replay-ledger.test.js new file mode 100644 index 0000000..bfd74bb --- /dev/null +++ b/notebook-result-replay-ledger/test/replay-ledger.test.js @@ -0,0 +1,106 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); +const test = require("node:test"); +const { + createReplayLedger, + digest, + stableStringify, + validateRepositoryManifest +} = require("../src/replay-ledger"); + +const samplePath = path.join(__dirname, "..", "data", "sample-repository.json"); + +function loadSample() { + return JSON.parse(fs.readFileSync(samplePath, "utf8")); +} + +function findComponent(manifest, id) { + return manifest.components.find((component) => component.id === id); +} + +test("detects stale scientific outputs after data, code, notebook, and environment changes", () => { + const ledger = createReplayLedger(loadSample()); + const figureFinding = ledger.findings.find((finding) => finding.resultId === "result.figure-1a"); + + assert.equal(ledger.releaseDecision, "block-release"); + assert.equal(figureFinding.severity, "block-release"); + assert.deepEqual( + figureFinding.staleDependencies.map((dependency) => dependency.id).sort(), + ["code.normalizer", "data.raw-counts", "env.analysis", "notebook.run-analysis"] + ); + assert.match(figureFinding.currentReplayDigest, /^sha256:[a-f0-9]{64}$/); + assert.notEqual(figureFinding.currentReplayDigest, figureFinding.recordedReplayDigest); +}); + +test("flags failed replay runs even when dependency hashes match", () => { + const ledger = createReplayLedger(loadSample()); + const qcFinding = ledger.findings.find((finding) => finding.resultId === "result.qc-table"); + + assert.equal(qcFinding.severity, "block-release"); + assert.equal(qcFinding.staleDependencies.length, 0); + assert.equal(qcFinding.problems.some((problem) => problem.code === "failed-replay"), true); +}); + +test("keeps current replayed outputs out of the findings list", () => { + const ledger = createReplayLedger(loadSample()); + + assert.equal( + ledger.findings.some((finding) => finding.resultId === "result.model-metrics"), + false + ); +}); + +test("builds rollback packets and citation impacts for reviewer workflow", () => { + const ledger = createReplayLedger(loadSample()); + + assert.equal(ledger.rollbackPackets.length, ledger.findings.length); + assert.equal( + ledger.rollbackPackets.some((packet) => packet.rollbackTarget.endsWith(":results/figure-1a.png")), + true + ); + assert.deepEqual( + ledger.citationImpacts.map((impact) => impact.resultId), + ["result.figure-1a"] + ); +}); + +test("reports ready when all release-critical results are current and replayed", () => { + const manifest = loadSample(); + const result = findComponent(manifest, "result.figure-1a"); + result.lastReplayedAt = "2026-05-14T10:20:00Z"; + result.recordedDependencyHashes = { + "data.raw-counts": "sha256:data-raw-counts-v4", + "code.normalizer": "sha256:normalizer-v5", + "code.plotter": "sha256:plotter-v2", + "notebook.run-analysis": "sha256:run-analysis-v7", + "env.analysis": "sha256:conda-analysis-2026-05-14" + }; + + const qcResult = findComponent(manifest, "result.qc-table"); + qcResult.status = "passed"; + delete qcResult.failureReason; + + const ledger = createReplayLedger(manifest); + + assert.equal(ledger.releaseDecision, "ready"); + assert.equal(ledger.summary.releaseBlockingArtifacts, 0); +}); + +test("validates repository manifest shape", () => { + const manifest = loadSample(); + manifest.components = manifest.components.filter((component) => component.kind !== "metadata"); + + const errors = validateRepositoryManifest(manifest); + + assert.equal(errors.some((error) => error.includes("metadata component")), true); +}); + +test("uses deterministic stable digests for audit records", () => { + const left = { b: 2, a: { d: 4, c: 3 } }; + const right = { a: { c: 3, d: 4 }, b: 2 }; + + assert.equal(stableStringify(left), stableStringify(right)); + assert.equal(digest(left), digest(right)); + assert.match(digest(left), /^sha256:[a-f0-9]{64}$/); +});