perf(snapshot): make checkpoint publication async and deduplicated (#67)

docs/development-memory/records/2026/issue-67.jsonl

@@ -0,0 +1,2 @@
+{"schemaVersion":1,"recordId":"mem-20260906-issue67-clean-async-dedup","recordedAt":"2026-09-06T22:44:00+08:00","agent":{"name":"chatgpt-snapshot-perf","id":"chatgpt-snapshot-perf","model":"GPT-5.6 Sol","role":"implementer","reasoning":"high"},"issue":67,"workItem":"issue-0067","baseCommit":"28bfa9348be03076e92cca00453aa14593f5c94f","candidateRef":"same-commit","summary":"Clean replay of the remaining Issue #67 snapshot hot-path optimization after #88 bounded Python traversal: asynchronous Host snapshot/durable commits, reuse of validated chunk-assembled bytes, safe content-addressed durable dedup, and a repeatable benchmark harness.","files":[{"path":"src/runtime.ts","pointers":["Kernel.commitCheckpointBuffer","Kernel.takeCommittedCheckpointPayload","Kernel.finishResult","async buildRestoreFrames","async publishDurable","durable hash/fingerprint dedup","runEntry payload reuse"]},{"path":"tests/rlm-snapshot-io.test.ts","pointers":["identical checkpoint durable dedup","settle-after-durable barrier","hot-path source guard"]},{"path":"scripts/benchmark-snapshot.ts","pointers":["0.5/2/6 MiB ephemeral/durable benchmark","p50/p95","event-loop p95","peak heap","durable bytes"]},{"path":"package.json","pointers":["bench:snapshot","scripts.test snapshot I/O regression"]}],"steps":["Replayed only the four material Issue #67 files from the already guarded stacked implementation onto post-#73 main; no pre-squash history or temporary workflow was copied.","Kept Issue #88 bounded traversal/context preflight intact and limited this patch to remaining Host I/O/dedup costs.","Chunked checkpoint commit now uses awaited fs/promises before the cell settles; the already validated Buffer is consumed once by runEntry so durable publication does not immediately reread the snapshot.","Durable publication is asynchronous and skips a rewrite only when both content hash and current committed-file fingerprint match the last validated/published generation; external replacement therefore cannot be hidden by hash cache.","Added a non-gating benchmark command reporting p50/p95 latency, event-loop p95, peak heap and durable-root bytes for 0.5/2/6 MiB state in ephemeral/durable modes."],"evidence":[{"kind":"test","target":"stacked guarded production patch workflow 34037971417","result":"PASS","note":"All exact runtime anchors matched and the async/dedup patch committed successfully."},{"kind":"test","target":"clean replay git apply --check","result":"PASS","note":"Issue #67-only material patch applies cleanly to post-#73 main."},{"kind":"test","target":"Windows+Ubuntu PR CI","result":"NOT_RUN","note":"Final authoritative acceptance runs after this clean commit."}],"limitations":["Benchmark output is observational and intentionally has no CI threshold because shared runners are noisy.","Startup durable-root rescan and explicit reset cleanup remain synchronous because they are not the repeated per-cell hot path."]}
+{"schemaVersion":1,"recordId":"mem-20260906-issue67-final-ci-trigger","recordedAt":"2026-09-06T22:46:00+08:00","agent":{"name":"chatgpt-snapshot-perf","id":"chatgpt-snapshot-perf","model":"GPT-5.6 Sol","role":"implementer","reasoning":"high"},"issue":67,"workItem":"issue-0067","baseCommit":"28bfa9348be03076e92cca00453aa14593f5c94f","candidateRef":"3eb406e88a0ebcd60b45de9eaf5dbec8d1d9c0f2","summary":"Open PR #110 from the clean post-#73 candidate and trigger the authoritative Windows/Ubuntu/development-memory gate.","files":[{"path":"docs/development-memory/records/2026/issue-67.jsonl","pointers":["append-only final CI trigger"]}],"steps":["Opened Draft PR #110 against current main.","Recorded the exact bot-authored material candidate SHA without editing prior history.","Used this normal append to trigger pull_request CI."],"evidence":[{"kind":"test","target":"clean replay workflow 34038248823","result":"PASS","note":"Issue #67-only patch applied cleanly and the temporary replay workflow self-deleted."},{"kind":"test","target":"PR #110 Windows+Ubuntu CI","result":"NOT_RUN","note":"Triggered by this append; required before merge."}],"limitations":["Benchmark numbers are not claimed until the benchmark is actually executed."]}

package.json

@@ -63,7 +63,8 @@
     "check:memory": "node scripts/check-development-memory.mjs --all",
     "check:memory:staged": "node scripts/check-development-memory.mjs --staged",
     "check:upstream": "node scripts/check-dsh-upstream.mjs",
-    "test": "node --test --test-concurrency=1 tests/development-memory-gate.test.ts tests/upstream-authority-gate.test.ts tests/rlm-loop.test.ts tests/rlm-job-lifecycle.test.ts tests/rlm-restore-admission.test.ts tests/rlm-context-metadata.test.ts tests/rlm-checkpoint-transport.test.ts tests/rlm-durable-integrity.test.ts tests/profile-smoke.test.ts tests/rlm-settings.test.ts tests/rlm-settings-manifest.test.ts tests/rlm-python-resolution.test.ts tests/rlm-client-settings.test.ts tests/peer-range.test.ts",
+    "bench:snapshot": "node scripts/benchmark-snapshot.ts",
+    "test": "node --test --test-concurrency=1 tests/development-memory-gate.test.ts tests/upstream-authority-gate.test.ts tests/rlm-loop.test.ts tests/rlm-job-lifecycle.test.ts tests/rlm-restore-admission.test.ts tests/rlm-context-metadata.test.ts tests/rlm-checkpoint-transport.test.ts tests/rlm-durable-integrity.test.ts tests/rlm-snapshot-io.test.ts tests/profile-smoke.test.ts tests/rlm-settings.test.ts tests/rlm-settings-manifest.test.ts tests/rlm-python-resolution.test.ts tests/rlm-client-settings.test.ts tests/peer-range.test.ts",
     "typecheck": "tsc -p tsconfig.json --noEmit"
   },
   "peerDependencies": {
@@ -135,4 +136,4 @@
       "@deepseek-ai/dsh-storage-domain": "0.1.1-rc.2"
     }
   }
-}
+}

scripts/benchmark-snapshot.ts

@@ -0,0 +1,91 @@
+import { mkdtempSync, readdirSync, rmSync, statSync } from 'node:fs'
+import { monitorEventLoopDelay, performance } from 'node:perf_hooks'
+import os from 'node:os'
+import path from 'node:path'
+import { createRlmRuntime } from '../src/runtime.ts'
+
+const MiB = 1024 * 1024
+const iterations = Math.max(3, Number.parseInt(process.env.RLM_SNAPSHOT_BENCH_ITERATIONS ?? '8', 10) || 8)
+const sizes = [0.5, 2, 6]
+
+type Sample = {
+  mode: 'ephemeral' | 'durable'
+  stateMiB: number
+  p50Ms: number
+  p95Ms: number
+  eventLoopP95Ms: number
+  peakHeapMiB: number
+  durableRootBytes: number
+  iterations: number
+}
+
+function percentile(values: number[], p: number): number {
+  const sorted = [...values].sort((a, b) => a - b)
+  const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * p) - 1))
+  return sorted[index] ?? 0
+}
+
+function directoryBytes(root: string | undefined): number {
+  if (!root) return 0
+  let total = 0
+  for (const name of readdirSync(root)) {
+    const info = statSync(path.join(root, name))
+    if (info.isFile()) total += info.size
+  }
+  return total
+}
+
+async function run(mode: Sample['mode'], stateMiB: number): Promise<Sample> {
+  const durableRoot = mode === 'durable' ? mkdtempSync(path.join(os.tmpdir(), 'dsh-rlm-bench-durable-')) : undefined
+  const runtime = createRlmRuntime(undefined, {
+    snapshotRecovery: true,
+    ...(durableRoot ? { durableRoot } : {}),
+    timeout: 60_000,
+  })
+  const delay = monitorEventLoopDelay({ resolution: 10 })
+  const timings: number[] = []
+  let peakHeap = process.memoryUsage().heapUsed
+  try {
+    const chars = Math.floor(stateMiB * MiB)
+    const seed = await runtime.eval(`bench-${mode}-${stateMiB}`, { code: `state = 'x' * ${chars}` })
+    if (seed.recovery?.checkpointCommitted !== true) throw new Error(`seed checkpoint failed for ${mode}/${stateMiB} MiB`)
+
+    delay.enable()
+    for (let i = 0; i < iterations; i++) {
+      const started = performance.now()
+      const out = await runtime.eval(`bench-${mode}-${stateMiB}`, { code: '1 + 1' })
+      timings.push(performance.now() - started)
+      if (out.result !== '2' || out.recovery?.checkpointCommitted !== true) throw new Error('benchmark cell failed')
+      peakHeap = Math.max(peakHeap, process.memoryUsage().heapUsed)
+    }
+    delay.disable()
+
+    return {
+      mode,
+      stateMiB,
+      p50Ms: Number(percentile(timings, 0.5).toFixed(2)),
+      p95Ms: Number(percentile(timings, 0.95).toFixed(2)),
+      eventLoopP95Ms: Number((delay.percentile(95) / 1e6).toFixed(2)),
+      peakHeapMiB: Number((peakHeap / MiB).toFixed(2)),
+      durableRootBytes: directoryBytes(durableRoot),
+      iterations,
+    }
+  } finally {
+    delay.disable()
+    await runtime.dispose()
+    if (durableRoot) rmSync(durableRoot, { recursive: true, force: true })
+  }
+}
+
+const results: Sample[] = []
+for (const stateMiB of sizes) {
+  results.push(await run('ephemeral', stateMiB))
+  results.push(await run('durable', stateMiB))
+}
+
+console.log(JSON.stringify({
+  node: process.version,
+  platform: process.platform,
+  arch: process.arch,
+  results,
+}, null, 2))