From 6f71a0c5f31af9dbb4a154960575cbf96824f6b8 Mon Sep 17 00:00:00 2001 From: catoncat Date: Mon, 17 Aug 2026 12:29:45 +0800 Subject: [PATCH 1/4] feat(eval): add concurrency benchmark harness for shlog read path Add npm run eval:perf:concurrency: a worker-pool harness that measures throughput and tail latency when multiple independent shlog processes hit the same read-only SQLite index concurrently. - eval/concurrency-bench-core.ts: pure arg parsing, shape construction, per-level aggregation and markdown report builder (unit-tested). - eval/concurrency-bench.ts: async runner over a shared job queue. - eval/PERF_BENCH.md: document the harness and a 2026-08-17 local baseline (Apple M4, 6.2k sessions / 318k messages / 420MB index). --- eval/PERF_BENCH.md | 38 ++++ eval/concurrency-bench-core.test.ts | 187 +++++++++++++++++ eval/concurrency-bench-core.ts | 304 ++++++++++++++++++++++++++++ eval/concurrency-bench.ts | 198 ++++++++++++++++++ package.json | 1 + 5 files changed, 728 insertions(+) create mode 100644 eval/concurrency-bench-core.test.ts create mode 100644 eval/concurrency-bench-core.ts create mode 100644 eval/concurrency-bench.ts diff --git a/eval/PERF_BENCH.md b/eval/PERF_BENCH.md index 76e98ea..65004ad 100644 --- a/eval/PERF_BENCH.md +++ b/eval/PERF_BENCH.md @@ -110,6 +110,44 @@ npm run eval:perf -- \ 无 executable override 时,dogfood 与其他 eval runner 一样默认使用 TypeScript oracle。 +## 并发基准 + +`npm run eval:perf:concurrency` 是并发读路径的补充 harness。与串行 harness 不同,它测的是**同时多个独立 `shlog` 进程**访问同一个只读 SQLite index 时的吞吐与 tail latency。它不执行 `sync`,要求 `--db` 已存在。 + +```bash +npm run eval:perf:concurrency -- \ + --bin ./target/release/shlog \ + --root /absolute/path/to/fixture/sessions \ + --db /absolute/path/to/fixture/index.sqlite \ + --shapes "find:hammerspoon|find:edge tts|read-range|read-page|status" \ + --levels "1 2 4 8 16 32" \ + --total 80 # 每级并发总共跑多少 op +``` + +- executable selector 与串行 harness 完全一致(`--bin` / `--cli-argv-json` / 环境变量 / TS reference fallback)。 +- command shapes:`find:` 构造 `find` 命令;`read-range` / `read-page` / `status` 是字面命令。read shapes 会先用 `list --limit 1` 解析一个真实 session ref 作为 anchor;解析失败时只跳过该 shape 并警告,不使整个 run 失败。 +- 方法:worker 池 + 共享任务队列,每个 op 独立 spawn 一个被测进程;并发度 = worker 数。每个 level 的记录包含: + - `throughputPerSec`(完成 ops / wall time) + - per-op `processE2E` 的 p50/p95/p99/max(毫秒) + - payload `elapsedMs` 的 p50/p95/p99/max(若被测命令提供;`status` 不提供时为 `null`) + - `errors`(非零退出计数) +- 默认写入 `data/shlog-perf/concurrency//report.json` 与 `report.md`;`--json-only` 只向 stdout 输出。 + +### 本机基线(2026-08-17,Apple M4 / 10 核 / 16GB) + +被测 `target/release/shlog` 0.5.1(native),真实 Codex index:6217 sessions / 318k messages / 420MB SQLite,热缓存。数字来自 `npm run eval:perf:concurrency`(`total=40`、`levels 1 2 4 8 16 32`),为 per-op E2E p50/p95(毫秒)与峰值吞吐(ops/s): + +| shape | 1 并发 p50/p95 | 4 并发 p50/p95 | 16 并发 p50/p95 | 峰值吞吐(@并发) | +|---|---|---|---|---| +| find:hammerspoon | 12.4 / 13.3 | 18.7 / 24.3 | 82.2 / 114.0 | 204.7 @4 | +| find:edge tts | 33.9 / 41.4 | 49.8 / 64.8 | 176.1 / 250.3 | 91.4 @16 | +| find:豆包输入法 | 13.1 / 13.9 | 17.8 / 19.2 | 74.7 / 105.8 | 222.2 @4 | +| read-range | 4.3 / 5.3 | 4.7 / 6.4 | 9.5 / 16.9 | 1257.6 @16 | +| read-page | 4.2 / 4.8 | 4.7 / 6.0 | 11.0 / 22.7 | 1226.9 @32 | +| status | 84.1 / 123.7 | 106.1 / 121.5 | 359.9 / 558.4 | 45.9 @8 | + +> 该基线是热缓存稳态;冷启动首击明显更慢(`find` 首次可达 ~0.6s、`status` 首次 ~2.5s)。并发读路径没有写锁,实测 0 error;超过 ~8 并发时 find 类 latency 劣化明显,超过 ~16 后吞吐不再增长,建议作为限流参考而不是无限开并发。 + ## 输出 默认写入: diff --git a/eval/concurrency-bench-core.test.ts b/eval/concurrency-bench-core.test.ts new file mode 100644 index 0000000..0c9cae4 --- /dev/null +++ b/eval/concurrency-bench-core.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, test } from "vitest"; +import { + DEFAULT_LEVELS, + DEFAULT_SHAPES, + DEFAULT_TOTAL_PER_LEVEL, + aggregateLevelStats, + buildConcurrencyReportMarkdown, + parseConcurrencyArgs, + parseLevels, + parsePositiveInt, + parseShapes, + shapeCommand, + type ConcurrencyReport, + type OpSample, +} from "./concurrency-bench-core"; + +describe("concurrency shape parsing", () => { + test("bare tokens become find shapes", () => { + expect(parseShapes("envchain|edge tts")).toEqual(["find:envchain", "find:edge tts"]); + }); + + test("keeps explicit shapes and mixes with find", () => { + expect(parseShapes("read-range|find:部署 health check|status")).toEqual([ + "read-range", + "find:部署 health check", + "status", + ]); + }); + + test("empty input falls back to defaults", () => { + expect(parseShapes("")).toEqual(DEFAULT_SHAPES); + }); + + test("level list is deduped and sorted; empty falls back", () => { + expect(parseLevels("8 1 4 8")).toEqual([1, 4, 8]); + expect(parseLevels("")).toEqual(DEFAULT_LEVELS); + }); + + test("positive int parser falls back on garbage", () => { + expect(parsePositiveInt("42", 7)).toBe(42); + expect(parsePositiveInt("0", 7)).toBe(7); + expect(parsePositiveInt("abc", 7)).toBe(7); + expect(parsePositiveInt(undefined, 7)).toBe(7); + }); +}); + +describe("concurrency arg parsing", () => { + test("requires --db", () => { + expect(() => parseConcurrencyArgs([])).toThrow(/--db is required/); + expect(() => parseConcurrencyArgs(["--root", "/tmp/root"])).toThrow(/--db is required/); + }); + + test("parses overrides and defaults", () => { + const args = parseConcurrencyArgs([ + "--db", "/tmp/index.sqlite", + "--root", "/tmp/sessions", + "--source", "claude-code", + "--shapes", "envchain|status", + "--levels", "1 4 16", + "--total", "40", + "--json-only", + ]); + expect(args.db).toBe("/tmp/index.sqlite"); + expect(args.root).toBe("/tmp/sessions"); + expect(args.source).toBe("claude-code"); + expect(args.shapes).toEqual(["find:envchain", "status"]); + expect(args.levels).toEqual([1, 4, 16]); + expect(args.totalPerLevel).toBe(40); + expect(args.jsonOnly).toBe(true); + // No executable override: resolves to the TypeScript reference by default. + expect(args.commandUnderTest.source).toBe("typescript-reference"); + }); + + test("accepts explicit executable override", () => { + const args = parseConcurrencyArgs(["--db", "/tmp/index.sqlite", "--bin", "/tmp/shlog"]); + expect(args.commandUnderTest.source).not.toBe("typescript-reference"); + }); +}); + +describe("shape command construction", () => { + const ctx = { source: "codex", root: "/tmp/sessions", db: "/tmp/index.sqlite", sessionRef: "session-1" }; + + test("find shape carries query and limit", () => { + const cmd = shapeCommand("find:edge tts", ctx); + expect(cmd).toEqual([ + "find", "edge tts", "--source", "codex", "--root", "/tmp/sessions", + "--db", "/tmp/index.sqlite", "--limit", "10", "--json", + ]); + }); + + test("status shape carries the all(root) selector", () => { + const cmd = shapeCommand("status", ctx); + expect(cmd?.[0]).toBe("status"); + expect(cmd).toContain("--selector"); + expect(JSON.parse(cmd![cmd!.indexOf("--selector") + 1]!)).toEqual({ + source: "codex", kind: "all", root: "/tmp/sessions", + }); + }); + + test("read shapes require a resolvable session ref", () => { + expect(shapeCommand("read-range", ctx)).not.toBeNull(); + expect(shapeCommand("read-page", ctx)).not.toBeNull(); + expect(shapeCommand("read-range", { ...ctx, sessionRef: null })).toBeNull(); + expect(shapeCommand("read-page", { ...ctx, sessionRef: null })).toBeNull(); + }); + + test("unknown shape throws", () => { + expect(() => shapeCommand("list", ctx)).toThrow(/unknown shape/); + }); +}); + +describe("level aggregation", () => { + test("computes percentiles, throughput and error count", () => { + const samples: OpSample[] = [ + sample(10, 8, true), sample(20, 15, true), sample(30, 22, true), + sample(40, 30, true), sample(50, 38, true), sample(999, null, false), + ]; + const stats = aggregateLevelStats(4, 6, 300, samples); + expect(stats.level).toBe(4); + expect(stats.total).toBe(6); + expect(stats.errors).toBe(1); + expect(stats.throughputPerSec).toBe(20); // 6 ops / 0.3s + // E2E p50/p95/p99/max over [10,20,30,40,50,999] (R-7 linear interpolation) + expect(stats.p50E2E).toBe(35); + expect(stats.p95E2E).toBeCloseTo(761.75, 1); + expect(stats.p99E2E).toBeCloseTo(951.55, 1); + expect(stats.maxE2E).toBe(999); + // op samples exclude the failed op (no elapsedMs): [8,15,22,30,38] + expect(stats.opSampleCount).toBe(5); + expect(stats.p50Op).toBe(22); + expect(stats.p95Op).toBeCloseTo(36.4, 1); + }); + + test("handles empty sample set without NaN", () => { + const stats = aggregateLevelStats(1, 0, 1, []); + expect(stats.p50E2E).toBe(0); + expect(stats.p95E2E).toBe(0); + expect(stats.maxE2E).toBe(0); + expect(stats.opSampleCount).toBe(0); + expect(stats.throughputPerSec).toBe(0); + }); +}); + +describe("markdown report", () => { + test("renders shape tables without NaN", () => { + const report: ConcurrencyReport = { + generatedAt: "2026-08-17T00:00:00.000Z", + commandUnderTest: { + executable: "shlog", + prefixArgv: [], + source: "argv-json", + resolvedExecutablePath: "/tmp/shlog", + executableSizeBytes: 100, + artifactPath: null, + artifactSizeBytes: null, + }, + sourceId: "codex", + dbPath: "/tmp/index.sqlite", + rootDir: "/tmp/sessions", + sessionCount: 10, + messageCount: 100, + totalPerLevel: 2, + shapes: [ + { + shape: "find:envchain", + command: ["find", "envchain", "--json"], + levels: [ + { + level: 1, total: 2, wallMs: 20, throughputPerSec: 100, errors: 0, + p50E2E: 10, p95E2E: 15, p99E2E: 18, maxE2E: 20, + p50Op: 8, p95Op: 12, p99Op: 14, maxOp: 16, opSampleCount: 2, + }, + ], + }, + ], + }; + const md = buildConcurrencyReportMarkdown(report); + expect(md).toContain("## find:envchain"); + expect(md).toContain("| 1 |"); + expect(md).not.toContain("NaN"); + expect(md).toContain("shlog 并发性能基准报告"); + }); +}); + +function sample(e2eMs: number, opMs: number | null, ok: boolean): OpSample { + return { ok, exitCode: ok ? 0 : 1, e2eMs, opMs, stdoutLen: 0, stderr: "" }; +} diff --git a/eval/concurrency-bench-core.ts b/eval/concurrency-bench-core.ts new file mode 100644 index 0000000..578e149 --- /dev/null +++ b/eval/concurrency-bench-core.ts @@ -0,0 +1,304 @@ +import { resolve } from "node:path"; +import { resolveCommandUnderTest, type CommandUnderTest } from "./perf-bench-core"; + +/** + * Pure helpers for the concurrency benchmark harness (`concurrency-bench.ts`). + * + * Design: the runner (`concurrency-bench.ts`) stays thin and async; everything + * that can be reasoned about deterministically lives here and is unit-tested: + * argument parsing, per-level latency aggregation, command-shape construction + * and the markdown report builder. + */ + +export const DEFAULT_LEVELS = [1, 2, 4, 8, 16, 32]; +export const DEFAULT_TOTAL_PER_LEVEL = 80; +export const DEFAULT_SHAPES = [ + "find:hammerspoon", + "find:edge tts", + "find:豆包输入法", + "read-range", + "read-page", + "status", +]; + +export interface ConcurrencyArgs { + root: string; + db: string; + source: string; + /** Command shapes. A `find:` shape maps to a `find` invocation; the + * literal shapes `read-range`, `read-page` and `status` map to their + * commands. */ + shapes: string[]; + levels: number[]; + totalPerLevel: number; + jsonOnly: boolean; + commandUnderTest: CommandUnderTest; +} + +export interface OpSample { + ok: boolean; + exitCode: number | null; + e2eMs: number; + opMs: number | null; + stdoutLen: number; + stderr: string; +} + +export interface LevelStats { + level: number; + total: number; + wallMs: number; + throughputPerSec: number; + errors: number; + p50E2E: number; + p95E2E: number; + p99E2E: number; + maxE2E: number; + p50Op: number | null; + p95Op: number | null; + p99Op: number | null; + maxOp: number | null; + opSampleCount: number; +} + +export interface ShapeLevelResult { + shape: string; + command: string[]; + levels: LevelStats[]; +} + +export interface ConcurrencyReport { + generatedAt: string; + commandUnderTest: CommandUnderTest; + sourceId: string; + dbPath: string; + rootDir: string; + sessionCount: number; + messageCount: number; + totalPerLevel: number; + shapes: ShapeLevelResult[]; +} + +export function parseConcurrencyArgs(argv: string[]): ConcurrencyArgs { + let root = process.env.HOME ? resolve(process.env.HOME, ".codex", "sessions") : ""; + let db = ""; + let source = "codex"; + let jsonOnly = false; + let shapes = [...DEFAULT_SHAPES]; + let levels = [...DEFAULT_LEVELS]; + let totalPerLevel = DEFAULT_TOTAL_PER_LEVEL; + let executable: string | undefined; + let cliArgvJson: string | undefined; + let artifactPath: string | undefined; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + const next = () => argv[++i]; + if (a === "--root") root = resolve(next() ?? root); + else if (a === "--db") db = resolve(next() ?? ""); + else if (a === "--source") source = next() ?? source; + else if (a === "--shapes") shapes = parseShapes(next() ?? ""); + else if (a === "--levels") levels = parseLevels(next() ?? ""); + else if (a === "--total") totalPerLevel = parsePositiveInt(next(), DEFAULT_TOTAL_PER_LEVEL); + else if (a === "--bin") executable = next(); + else if (a === "--cli-argv-json") cliArgvJson = next(); + else if (a === "--artifact") artifactPath = next(); + else if (a === "--json-only") jsonOnly = true; + else if (a === "--help" || a === "-h") { + throw new HelpRequested(); + } + } + if (!db) throw new Error("--db is required (concurrency benchmark is read-only against an existing index)"); + const commandUnderTest = resolveCommandUnderTest({ + root: ROOT, + cliEntry: CLI_ENTRY, + executable, + argvJson: cliArgvJson, + artifactPath, + }); + return { root, db, source, shapes, levels, totalPerLevel, jsonOnly, commandUnderTest }; +} + +export class HelpRequested extends Error { + constructor() { + super("help requested"); + this.name = "HelpRequested"; + } +} + +export const USAGE = `Usage: npm run eval:perf:concurrency -- \\ + --db [--root ] [--source ] \\ + [--shapes "find:hammerspoon|read-range|read-page|status"] \\ + [--levels "1 2 4 8 16 32"] [--total 80] \\ + [--bin | --cli-argv-json ] [--artifact ] [--json-only]`; + +/** Literal command shapes that must not be reinterpreted as find queries. */ +const RESERVED_SHAPES = new Set(["read-range", "read-page", "status"]); + +/** Parse `a|b|c` shape list; bare tokens become `find:` unless they are + * reserved literal shapes (`read-range`, `read-page`, `status`). */ +export function parseShapes(raw: string): string[] { + const parts = raw.split("|").map((s) => s.trim()).filter(Boolean); + if (parts.length === 0) return [...DEFAULT_SHAPES]; + return parts.map((part) => (part.includes(":") || RESERVED_SHAPES.has(part) ? part : `find:${part}`)); +} + +export function parseLevels(raw: string): number[] { + const values = raw.split(/\s+/).map(Number).filter((n) => Number.isFinite(n) && n > 0); + return values.length > 0 ? [...new Set(values)].sort((a, b) => a - b) : [...DEFAULT_LEVELS]; +} + +export function parsePositiveInt(value: string | undefined, fallback: number): number { + const parsed = Number.parseInt(value ?? "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +/** + * Build the CLI argv for a shape. Read shapes require a resolved session ref; + * when it is missing they return `null` so the runner can skip them with a + * clear message instead of running a failing command. + */ +export function shapeCommand( + shape: string, + ctx: { source: string; root: string; db: string; sessionRef: string | null }, +): string[] | null { + if (shape === "read-range" || shape === "read-page") { + if (!ctx.sessionRef) return null; + if (shape === "read-range") { + return ["read-range", ctx.sessionRef, "--source", ctx.source, "--seq", "0", "--before", "2", "--after", "2", "--db", ctx.db, "--json"]; + } + return ["read-page", ctx.sessionRef, "--source", ctx.source, "--offset", "0", "--limit", "40", "--db", ctx.db, "--json"]; + } + if (shape === "status") { + return [ + "status", + "--source", + ctx.source, + "--root", + ctx.root, + "--selector", + JSON.stringify({ source: ctx.source, kind: "all", root: ctx.root }), + "--db", + ctx.db, + "--json", + ]; + } + if (shape.startsWith("find:")) { + const query = shape.slice("find:".length); + return [ + "find", + query, + "--source", + ctx.source, + "--root", + ctx.root, + "--db", + ctx.db, + "--limit", + "10", + "--json", + ]; + } + throw new Error(`unknown shape: ${shape}`); +} + +/** Aggregate per-op samples into per-level latency/throughput stats. */ +export function aggregateLevelStats( + level: number, + total: number, + wallMs: number, + samples: OpSample[], +): LevelStats { + const e2e = samples.map((s) => s.e2eMs).sort((a, b) => a - b); + const opSamples = samples.filter((s) => s.opMs !== null).map((s) => s.opMs as number).sort((a, b) => a - b); + return { + level, + total, + wallMs: round2(wallMs), + throughputPerSec: round2((samples.length / wallMs) * 1000), + errors: samples.filter((s) => !s.ok).length, + p50E2E: round2(percentileFrom(e2e, 0.5)), + p95E2E: round2(percentileFrom(e2e, 0.95)), + p99E2E: round2(percentileFrom(e2e, 0.99)), + maxE2E: round2(e2e.length ? e2e[e2e.length - 1]! : 0), + p50Op: opSamples.length ? round2(percentileFrom(opSamples, 0.5)) : null, + p95Op: opSamples.length ? round2(percentileFrom(opSamples, 0.95)) : null, + p99Op: opSamples.length ? round2(percentileFrom(opSamples, 0.99)) : null, + maxOp: opSamples.length ? round2(opSamples[opSamples.length - 1]!) : null, + opSampleCount: opSamples.length, + }; +} + +function percentileFrom(sorted: number[], p: number): number { + if (sorted.length === 0) return 0; + if (sorted.length === 1) return sorted[0]!; + const pos = (sorted.length - 1) * Math.min(1, Math.max(0, p)); + const lo = Math.floor(pos); + const hi = Math.ceil(pos); + const l = sorted[lo]!; + const h = sorted[hi]!; + return l + (h - l) * (pos - lo); +} + +function round2(value: number): number { + return Number(value.toFixed(2)); +} + +export function fmtMs(n: number): string { + return n.toFixed(1).padStart(8); +} + +export function fmtBytes(n: number): string { + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`; + return `${(n / 1024 / 1024).toFixed(1)} MB`; +} + +export function buildConcurrencyReportMarkdown(r: ConcurrencyReport): string { + const lines: string[] = []; + lines.push("# shlog 并发性能基准报告"); + lines.push(""); + lines.push(`- generated_at: ${r.generatedAt}`); + lines.push(`- command: \`${[r.commandUnderTest.executable, ...r.commandUnderTest.prefixArgv].join(" ")}\``); + lines.push(`- command_source: ${r.commandUnderTest.source}`); + lines.push(`- resolved_executable: \`${r.commandUnderTest.resolvedExecutablePath ?? "unresolved"}\``); + lines.push(`- executable_size: ${r.commandUnderTest.executableSizeBytes === null ? "-" : fmtBytes(r.commandUnderTest.executableSizeBytes)}`); + lines.push(`- artifact: \`${r.commandUnderTest.artifactPath ?? "unresolved"}\``); + lines.push(`- source: \`${r.sourceId}\``); + lines.push(`- root: \`${r.rootDir}\``); + lines.push(`- db: \`${r.dbPath}\``); + lines.push(`- session_count: ${r.sessionCount}`); + lines.push(`- message_count: ${r.messageCount}`); + lines.push(`- total_ops_per_level: ${r.totalPerLevel}`); + lines.push(""); + for (const shape of r.shapes) { + lines.push(`## ${shape.shape}`); + lines.push(""); + lines.push(`command: \`shlog ${shape.command.join(" ")}\``); + lines.push(""); + lines.push("| level | ops/sec | errors | p50 E2E | p95 E2E | p99 E2E | max E2E | p50 op | p95 op | p99 op | max op |"); + lines.push("|------:|--------:|-------:|--------:|--------:|--------:|--------:|-------:|-------:|-------:|-------:|"); + for (const lvl of shape.levels) { + lines.push([ + `| ${lvl.level}`, + lvl.throughputPerSec.toFixed(1), + lvl.errors.toString(), + fmtMs(lvl.p50E2E), + fmtMs(lvl.p95E2E), + fmtMs(lvl.p99E2E), + fmtMs(lvl.maxE2E), + lvl.p50Op === null ? "-" : fmtMs(lvl.p50Op), + lvl.p95Op === null ? "-" : fmtMs(lvl.p95Op), + lvl.p99Op === null ? "-" : fmtMs(lvl.p99Op), + `${lvl.maxOp === null ? "-" : fmtMs(lvl.maxOp)} |`, + ].join(" | ")); + } + lines.push(""); + } + lines.push("> 方法:worker 池 + 共享任务队列,每个 op 独立进程;并发度=worker 数。E2E 为父进程观测的完整进程 wall time,op 来自被测 JSON 的 elapsedMs。所有延迟单位为毫秒。报告不包含 transcript 内容。"); + lines.push(""); + return lines.join("\n"); +} + +// Executable resolution context mirrors the serial perf harness. +const ROOT = resolve(import.meta.dirname, ".."); +const CLI_ENTRY = resolve(ROOT, "src", "cli.ts"); diff --git a/eval/concurrency-bench.ts b/eval/concurrency-bench.ts new file mode 100644 index 0000000..2d9ccc4 --- /dev/null +++ b/eval/concurrency-bench.ts @@ -0,0 +1,198 @@ +#!/usr/bin/env -S node --import tsx + +import { mkdirSync, writeFileSync } from "node:fs"; +import { spawn } from "node:child_process"; +import { join, resolve } from "node:path"; +import { performance } from "node:perf_hooks"; +import { + USAGE, + HelpRequested, + aggregateLevelStats, + buildConcurrencyReportMarkdown, + parseConcurrencyArgs, + shapeCommand, + type ConcurrencyArgs, + type ConcurrencyReport, + type OpSample, + type ShapeLevelResult, +} from "./concurrency-bench-core"; + +const ROOT = resolve(import.meta.dirname, ".."); +const OUT_BASE = resolve(ROOT, "data", "shlog-perf", "concurrency"); + +let args: ConcurrencyArgs; +try { + args = parseConcurrencyArgs(process.argv.slice(2)); +} catch (error) { + if (error instanceof HelpRequested) { + console.log(USAGE); + process.exit(0); + } + console.error(`error: ${error instanceof Error ? error.message : String(error)}`); + console.error(USAGE); + process.exit(1); +} + +if (!args.db) { + console.error("error: --db is required"); + process.exit(1); +} + +async function runOnce(cmd: string[]): Promise { + return new Promise((resolvePromise) => { + const t0 = performance.now(); + const proc = spawn(args.commandUnderTest.executable, [...args.commandUnderTest.prefixArgv, ...cmd], { + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + proc.stdout!.setEncoding("utf8"); + proc.stderr!.setEncoding("utf8"); + proc.stdout!.on("data", (c: string) => { stdout += c; }); + proc.stderr!.on("data", (c: string) => { stderr += c; }); + proc.on("error", (err) => { + resolvePromise({ ok: false, exitCode: null, e2eMs: performance.now() - t0, opMs: null, stdoutLen: 0, stderr: String(err) }); + }); + proc.on("close", (code) => { + let opMs: number | null = null; + try { + const parsed = JSON.parse(stdout) as { elapsedMs?: unknown }; + if (typeof parsed.elapsedMs === "number") opMs = parsed.elapsedMs; + } catch { /* non-JSON output */ } + resolvePromise({ ok: code === 0, exitCode: code ?? 0, e2eMs: performance.now() - t0, opMs, stdoutLen: stdout.length, stderr }); + }); + }); +} + +async function runLevel(level: number, total: number, command: string[]): Promise<{ samples: OpSample[]; wallMs: number }> { + let next = 0; + const samples: OpSample[] = []; + async function worker() { + for (;;) { + const i = next++; + if (i >= total) return; + samples.push(await runOnce(command)); + } + } + const t0 = performance.now(); + await Promise.all(Array.from({ length: level }, worker)); + const wallMs = performance.now() - t0; + return { samples, wallMs }; +} + +function parseSessionRefFromList(stdout: string): string | null { + try { + const parsed = JSON.parse(stdout) as { sessions?: unknown; results?: unknown }; + const rows = (Array.isArray(parsed.sessions) ? parsed.sessions : []) + .concat(Array.isArray(parsed.results) ? parsed.results : []); + const first = rows[0] as { sessionRef?: unknown; sessionUuid?: unknown } | undefined; + if (!first) return null; + if (typeof first.sessionRef === "string") return first.sessionRef; + if (typeof first.sessionUuid === "string") return first.sessionUuid; + return null; + } catch { + return null; + } +} + +// Session ref resolution: pick the most recent session from the index so the +// read shapes have a real anchor. Best effort — a missing ref only skips the +// read shapes with a warning, it never fails the run. +async function resolveSessionRefCached(): Promise { + const cmd = ["list", "--source", args.source, "--db", args.db, "--limit", "1", "--json"]; + return new Promise((resolvePromise) => { + const proc = spawn(args.commandUnderTest.executable, [...args.commandUnderTest.prefixArgv, ...cmd], { + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + proc.stdout!.setEncoding("utf8"); + proc.stdout!.on("data", (c: string) => { stdout += c; }); + proc.on("close", () => resolvePromise(parseSessionRefFromList(stdout))); + proc.on("error", () => resolvePromise(null)); + }); +} + +// Session/message counts for report context. +async function collectIndexCounts(): Promise<{ sessionCount: number; messageCount: number }> { + try { + const cmd = ["stats", "--source", args.source, "--db", args.db, "--json"]; + const proc = await spawnCapture(cmd); + const parsed = JSON.parse(proc.stdout) as { sessionCount?: unknown; messageCount?: unknown }; + return { + sessionCount: typeof parsed.sessionCount === "number" ? parsed.sessionCount : 0, + messageCount: typeof parsed.messageCount === "number" ? parsed.messageCount : 0, + }; + } catch { + return { sessionCount: 0, messageCount: 0 }; + } +} + +function spawnCapture(cmd: string[]): Promise<{ stdout: string; exitCode: number }> { + return new Promise((resolvePromise) => { + const proc = spawn(args.commandUnderTest.executable, [...args.commandUnderTest.prefixArgv, ...cmd], { + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + proc.stdout!.setEncoding("utf8"); + proc.stdout!.on("data", (c: string) => { stdout += c; }); + proc.on("close", (code) => resolvePromise({ stdout, exitCode: code ?? 0 })); + proc.on("error", () => resolvePromise({ stdout: "", exitCode: 1 })); + }); +} + +const sessionRef = await resolveSessionRefCached(); +const counts = await collectIndexCounts(); +const ctx = { source: args.source, root: args.root, db: args.db, sessionRef }; + +const shapes: ShapeLevelResult[] = []; +for (const shape of args.shapes) { + const command = shapeCommand(shape, ctx); + if (command === null) { + console.error(`warning: shape "${shape}" needs a resolvable session ref; skipping`); + continue; + } + const levels: ShapeLevelResult["levels"] = []; + for (const level of args.levels) { + const { samples, wallMs } = await runLevel(level, args.totalPerLevel, command); + levels.push(aggregateLevelStats(level, args.totalPerLevel, wallMs, samples)); + const last = levels[levels.length - 1]!; + console.error(`[${shape}] level=${level}: ${last.throughputPerSec.toFixed(1)} ops/s p50=${last.p50E2E.toFixed(1)}ms p95=${last.p95E2E.toFixed(1)}ms errors=${last.errors}`); + } + shapes.push({ shape, command, levels }); +} + +const report: ConcurrencyReport = { + generatedAt: new Date().toISOString(), + commandUnderTest: args.commandUnderTest, + sourceId: args.source, + dbPath: args.db, + rootDir: args.root, + sessionCount: counts.sessionCount, + messageCount: counts.messageCount, + totalPerLevel: args.totalPerLevel, + shapes, +}; + +const stamp = new Date().toISOString().replace(/[:.]/g, "-"); +const outDir = args.jsonOnly ? "" : join(OUT_BASE, stamp); +if (!args.jsonOnly) { + mkdirSync(outDir, { recursive: true }); + writeFileSync(join(outDir, "report.json"), `${JSON.stringify(report, null, 2)}\n`); + writeFileSync(join(outDir, "report.md"), buildConcurrencyReportMarkdown(report)); +} + +const slowestShape = [...report.shapes].sort((a, b) => (b.levels.at(-1)?.p95E2E ?? 0) - (a.levels.at(-1)?.p95E2E ?? 0))[0]; +const summary = { + outDir: outDir || null, + sourceId: report.sourceId, + commandUnderTest: report.commandUnderTest, + sessionCount: report.sessionCount, + messageCount: report.messageCount, + totalPerLevel: report.totalPerLevel, + shapeCount: report.shapes.length, + slowestShape: slowestShape ? { + shape: slowestShape.shape, + maxConcurrencyP95E2EMs: slowestShape.levels.at(-1)?.p95E2E ?? null, + } : null, +}; +console.log(JSON.stringify(args.jsonOnly ? report : summary, null, 2)); diff --git a/package.json b/package.json index f410b7e..09d90fb 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "eval:contract": "tsx ./eval/run-contract-eval.ts", "eval:compare": "tsx ./eval/compare-eval-batches.ts", "eval:perf": "tsx ./eval/perf-bench.ts", + "eval:perf:concurrency": "tsx ./eval/concurrency-bench.ts", "eval:dogfood": "tsx ./eval/run-dogfood-eval.ts" }, "devDependencies": { From e5a1d081a8d6f6a36191b5d17283aef45b3bb256 Mon Sep 17 00:00:00 2001 From: catoncat Date: Mon, 17 Aug 2026 13:58:00 +0800 Subject: [PATCH 2/4] feat(eval): add deterministic CJK-heavy fixture generator and flip harness defaults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New eval/perf-fixture.ts: deterministic CJK-heavy fixture generator (~60% CJK, ~25% Latin, ~15% paths/commands, mulberry32 seeded) - eval/perf-bench.ts: default flips from real dev data + strict sync to synthetic fixture + --best-effort sync; real data now opt-in - eval/concurrency-bench: integrates --fixture-mb with auto-sync - eval/PERF_BENCH.md: rewrite Safety section for fixture default; update concurrency section for --fixture-mb - docs/ROADMAP.md: move perf baseline from P0 「仍需收口」to done - CONTEXT.md: initial glossary (7 terms: perf baseline, regression gate, e2e perf, concurrency perf, component perf, fixture, dogfood) Verified: npm run check 300/300, cargo test 12/12, smoke with Rust release binary + 4MB fixture: all 7 shapes pass including CJK (豆包输入法 find p50=11.7ms, 部署 health check p50=13.3ms). --- CONTEXT.md | 27 ++++ docs/ROADMAP.md | 3 +- eval/PERF_BENCH.md | 40 ++++-- eval/concurrency-bench-core.ts | 14 +- eval/concurrency-bench.ts | 26 +++- eval/perf-bench.ts | 37 ++++- eval/perf-fixture.ts | 242 +++++++++++++++++++++++++++++++++ 7 files changed, 372 insertions(+), 17 deletions(-) create mode 100644 CONTEXT.md create mode 100644 eval/perf-fixture.ts diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..12f8a0c --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,27 @@ +# Sherlog Context + +项目领域术语表。不含实现细节,仅定义领域概念及其边界。 + + + +## 性能 + +- **性能基线**(perf baseline):在某指定机器类上,对固定工作负载(合成 fixture 或真实 dogfood 数据)跑 `eval:perf` / `eval:perf:concurrency` 产出的 p50/p95 延迟、吞吐、RSS、DB 体积等数字。基线 JSON 随代码一起进 git,作为回归门判据的参照点。基线只在标定它的那台机器类上有比较意义。 + +- **性能回归门**(perf regression gate):发布流程中对性能基线的一次性检查——当某指标漂移超过容差(推荐默认 `max(baseline × 2.0, baseline + 150ms)`)时阻止发布,但提供显式 escape hatch 供人工复核后放行。当前只计划在 release workflow 跑,不在 PR CI 跑。 + +- **业务性能**(end-to-end CLI performance):用户可感知的 CLI 命令(`find`/`read-range`/`read-page`/`status`/`sync`)的进程级 wall-clock 延迟、内存与 DB 体积。由 `eval/perf-bench.ts`(串行)和 `eval/concurrency-bench.ts`(并发)测量。 + +- **并发性能**(concurrency performance):多个独立 `shlog` 只读进程同时访问同一 SQLite index 时的吞吐(ops/s)与 tail latency(p95/p99)。由 `eval/concurrency-bench.ts` 测量。用于容量分析,不进入回归门。 + +- **组件性能**(component-level micro-benchmark):单个内部模块(如 tokenizer)的吞吐、分配次数、长文本缩放。当前尚不存在;计划以零依赖 `examples/` bench bin 实现,用 e2e 中文 fixture 的 sync/find 延迟作为间接回归保护。 + +## 数据与负载 + +- **合成 fixture**(synthetic fixture):由 `eval/perf-fixture.ts` 确定性生成的临时 session 语料(Codex JSONL 格式)。内容 ~60% CJK + 25% Latin + 15% 路径/命令,体积由 `--fixture-mb` 控制(默认 16MB)。相同 seed ≡ 相同输出,跨机器可复现。用于性能基准的默认负载,不用于正确性测试。 + +- **dogfood 数据**(dogfood data):开发者本机的真实 agent session 历史。只用于质量线(dogfood eval、acceptance gate)和性能线的显式 opt-in 模式(`--root`/`--db` 显式传参)。perf harness 默认不触碰真实数据。 diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 60d47fb..be1c576 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -22,14 +22,13 @@ message 与 session-profile 共用 `documents`/`documents_fts`,但 evidence pr - dogfood runner 已复用统一的 CLI-under-test 解析,可通过 `--cli-argv-json`、`SHLOG_CLI_ARGV_JSON` 或 `SHLOG_BIN_UNDER_TEST` 显式绑定 native candidate,并把实际 argv/source 写入 scorecard;无 override 时才默认 TypeScript oracle; - executable-neutral contract gate 已把 help prose、query-only coverage freshness、typed error semantics 与 native strict-incomplete reason 编码为 intentional-difference policy;当前 `target/release/shlog` 对 TypeScript reference 实测 **24/24**; - synthetic acceptance gate 已显式绑定同一 native release candidate;message hit、session-profile hit、CJK、source-aware read、command restatement 等 evidence-level fixtures 当前实测 **8/8**; -- candidate-aware perf harness:可显式选择 release binary、记录 process/operation latency、RSS、artifact/DB size 和 progressive reads; +- candidate-aware perf harness:可显式选择 release binary、记录 process/operation latency、RSS、artifact/DB size 和 progressive reads;默认使用确定性合成的 CJK-heavy fixture(安全隔离),真实数据须显式 opt-in;附 concurrency 补充 harness 测并发吞吐与 tail latency; - Rust unit/integration tests:sources、index、sync、migration、retrieval、app; - native CI 会实际构建并检查 `target/release/shlog` 后以 `--require-candidate` 运行 contract/acceptance;release workflow 则下载 Linux GNU archive、解包并验证其中的 executable,再运行同一 gates; - native release workflow:macOS arm64/x64、Linux x64 GNU archives、SBOM、checksums、attestation、installer/formula。 ### 仍需收口 -- 固化 initial/no-op/append sync 与 find/read/status 的性能基线; - 完成首次 native tag/release 前的全量 Rust、Node oracle/eval、workflow/installer gates;当前 24/24 contract 与 8/8 acceptance 只证明本地 release candidate,不代表 release 已发布; - 发布后回读 GitHub assets/attestations,并独立验证 installed `shlog` 的路径、`--version` 与 smoke;当前 global `shlog` 仍是旧发布版 `0.4.4`。 diff --git a/eval/PERF_BENCH.md b/eval/PERF_BENCH.md index 65004ad..d656f95 100644 --- a/eval/PERF_BENCH.md +++ b/eval/PERF_BENCH.md @@ -41,20 +41,36 @@ npm run eval:perf -- --root --db --skip-sync --json-only ## Safety 与输入范围 -无参数的兼容默认值是本机默认 Codex root 与默认 state DB,并在读测试前执行 strict `sync`。这会修改所选 SQLite,适合明确的本机 dogfood,不适合作为隔离基准。 +无参数运行时,harness 自动在临时目录生成**确定性合成 fixture**(默认 ~16 MB CJK-heavy 正文),对其执行 sync 和读测试,结束后自动清理。这不会触碰你的真实 session 数据或默认 state DB——适合作为隔离基准。 -推荐显式传入 sanitized fixture root 和独立 DB。若 DB 已预建,使用 `--skip-sync`;此时 DB 必须存在,harness 不执行任何 sync: +### 使用真实数据(opt-in) + +要对自己的本机 session 做 dogfood 基准,必须**显式传 `--root` 或 `--db`**: ```bash npm run eval:perf -- \ --bin ./target/release/shlog \ - --artifact ./target/release/shlog \ - --root /absolute/path/to/fixture/sessions \ - --db /absolute/path/to/fixture/index.sqlite \ - --skip-sync \ - --json-only + --root ~/.codex/sessions \ + --db ~/.local/state/shlog/index.sqlite \ + --skip-sync ``` +不显式传 `--root`/`--db` 时,harness **永远不会访问你的真实数据**。 + +### 合成 fixture + +`--fixture-mb ` 控制 fixture 体积(默认 16)。fixture 由 `eval/perf-fixture.ts` 确定性生成——相同参数在任何机器上产出相同的 session 集合。内容为 ~60% CJK(中文运维场景)、~25% Latin(英文 tech)、~15% 路径/命令。 + +```bash +# 4 MB 快速 smoke +npm run eval:perf -- --bin ./target/release/shlog --fixture-mb 4 --json-only + +# 保留生成文件供检查 +npm run eval:perf -- --bin ./target/release/shlog --fixture-mb 4 --keep-fixture +``` + +fixture 的 sync 自动走 `--best-effort`(临时目录在 macOS 上可能因文件系统事件触发 strict 的 "source_file_set_changed" 误判)。对真实数据(显式 `--root`/`--db`)仍走 strict 以保证覆盖度。 + `status` 会按公开 contract 建立 live privacy-filtered inventory 并计算 requested selector coverage;它不返回/检索正文、不写 index,但 cache miss 可流式读取 raw accepted records/body,成本可能为 O(raw bytes),exact `mtime_ns`/checkpoint cache hit 则不重 parse。`find`、`read-range`、`read-page`、`stats` 只读 index。Harness 会把显式 root 传给 status/find,但 find 不自行扫描 raw transcript freshness。 ## 被测 command shapes @@ -112,16 +128,22 @@ npm run eval:perf -- \ ## 并发基准 -`npm run eval:perf:concurrency` 是并发读路径的补充 harness。与串行 harness 不同,它测的是**同时多个独立 `shlog` 进程**访问同一个只读 SQLite index 时的吞吐与 tail latency。它不执行 `sync`,要求 `--db` 已存在。 +`npm run eval:perf:concurrency` 是并发读路径的补充 harness。与串行 harness 不同,它测的是**同时多个独立 `shlog` 进程**访问同一个只读 SQLite index 时的吞吐与 tail latency。它不执行 `sync`,要求 `--db` 已存在。可配合 `--fixture-mb ` 自动生成临时 fixture 并 sync: ```bash +# 自动生成 16MB fixture 并测试 +npm run eval:perf:concurrency -- \ + --bin ./target/release/shlog \ + --fixture-mb 16 + +# 或使用预建 index npm run eval:perf:concurrency -- \ --bin ./target/release/shlog \ --root /absolute/path/to/fixture/sessions \ --db /absolute/path/to/fixture/index.sqlite \ --shapes "find:hammerspoon|find:edge tts|read-range|read-page|status" \ --levels "1 2 4 8 16 32" \ - --total 80 # 每级并发总共跑多少 op + --total 80 ``` - executable selector 与串行 harness 完全一致(`--bin` / `--cli-argv-json` / 环境变量 / TS reference fallback)。 diff --git a/eval/concurrency-bench-core.ts b/eval/concurrency-bench-core.ts index 578e149..fef7629 100644 --- a/eval/concurrency-bench-core.ts +++ b/eval/concurrency-bench-core.ts @@ -33,6 +33,9 @@ export interface ConcurrencyArgs { totalPerLevel: number; jsonOnly: boolean; commandUnderTest: CommandUnderTest; + /** When set, generate fixture of this many MB before running. */ + fixtureMb: number; + keepFixture: boolean; } export interface OpSample { @@ -90,6 +93,8 @@ export function parseConcurrencyArgs(argv: string[]): ConcurrencyArgs { let executable: string | undefined; let cliArgvJson: string | undefined; let artifactPath: string | undefined; + let fixtureMb = 0; + let keepFixture = false; for (let i = 0; i < argv.length; i++) { const a = argv[i]; const next = () => argv[++i]; @@ -99,6 +104,9 @@ export function parseConcurrencyArgs(argv: string[]): ConcurrencyArgs { else if (a === "--shapes") shapes = parseShapes(next() ?? ""); else if (a === "--levels") levels = parseLevels(next() ?? ""); else if (a === "--total") totalPerLevel = parsePositiveInt(next(), DEFAULT_TOTAL_PER_LEVEL); + else if (a === "--fixture-mb") fixtureMb = parsePositiveInt(next(), 16); + else if (a === "--fixture") { /* no-op: fixture is now default */ } + else if (a === "--keep-fixture") keepFixture = true; else if (a === "--bin") executable = next(); else if (a === "--cli-argv-json") cliArgvJson = next(); else if (a === "--artifact") artifactPath = next(); @@ -107,7 +115,7 @@ export function parseConcurrencyArgs(argv: string[]): ConcurrencyArgs { throw new HelpRequested(); } } - if (!db) throw new Error("--db is required (concurrency benchmark is read-only against an existing index)"); + if (!db && !fixtureMb) throw new Error("--db is required (concurrency benchmark is read-only against an existing index; use --fixture-mb to auto-generate one)"); const commandUnderTest = resolveCommandUnderTest({ root: ROOT, cliEntry: CLI_ENTRY, @@ -115,7 +123,7 @@ export function parseConcurrencyArgs(argv: string[]): ConcurrencyArgs { argvJson: cliArgvJson, artifactPath, }); - return { root, db, source, shapes, levels, totalPerLevel, jsonOnly, commandUnderTest }; + return { root, db, source, shapes, levels, totalPerLevel, jsonOnly, commandUnderTest, fixtureMb, keepFixture }; } export class HelpRequested extends Error { @@ -128,7 +136,7 @@ export class HelpRequested extends Error { export const USAGE = `Usage: npm run eval:perf:concurrency -- \\ --db [--root ] [--source ] \\ [--shapes "find:hammerspoon|read-range|read-page|status"] \\ - [--levels "1 2 4 8 16 32"] [--total 80] \\ + [--levels "1 2 4 8 16 32"] [--total 80] [--fixture-mb ] [--keep-fixture] \\ [--bin | --cli-argv-json ] [--artifact ] [--json-only]`; /** Literal command shapes that must not be reinterpreted as find queries. */ diff --git a/eval/concurrency-bench.ts b/eval/concurrency-bench.ts index 2d9ccc4..3fa8304 100644 --- a/eval/concurrency-bench.ts +++ b/eval/concurrency-bench.ts @@ -1,9 +1,10 @@ #!/usr/bin/env -S node --import tsx import { mkdirSync, writeFileSync } from "node:fs"; -import { spawn } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { join, resolve } from "node:path"; import { performance } from "node:perf_hooks"; +import { cleanupFixture, generateFixture, type FixturePaths } from "./perf-fixture"; import { USAGE, HelpRequested, @@ -33,6 +34,24 @@ try { process.exit(1); } +// Generate synthetic fixture when --fixture-mb was explicitly set. +let fixture: FixturePaths | null = null; +if (process.argv.includes("--fixture-mb")) { + fixture = generateFixture(args.fixtureMb, args.source); + args.root = fixture.root; + args.db = fixture.db; + // Auto-sync the fresh fixture so the read-only benchmark has data. + const syncCmd = ["sync", "--source", args.source, "--db", args.db, "--root", args.root, "--json"]; + const syncResult = spawnSync(args.commandUnderTest.executable, [...args.commandUnderTest.prefixArgv, ...syncCmd], { + stdio: ["ignore", "pipe", "pipe"], + }); + if (syncResult.status !== 0) { + console.error(`error: fixture sync failed (exit ${syncResult.status}): ${syncResult.stderr.toString().slice(0, 500)}`); + cleanupFixture(fixture); + process.exit(1); + } +} + if (!args.db) { console.error("error: --db is required"); process.exit(1); @@ -196,3 +215,8 @@ const summary = { } : null, }; console.log(JSON.stringify(args.jsonOnly ? report : summary, null, 2)); + +// Clean up synthetic fixture unless --keep-fixture was requested. +if (fixture && !args.keepFixture) { + cleanupFixture(fixture); +} diff --git a/eval/perf-bench.ts b/eval/perf-bench.ts index c5cacd5..afd39ad 100644 --- a/eval/perf-bench.ts +++ b/eval/perf-bench.ts @@ -6,6 +6,7 @@ import { homedir } from "node:os"; import { join, resolve } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { DEFAULT_DB_PATH } from "../src/env"; +import { cleanupFixture, generateFixture, type FixturePaths } from "./perf-fixture"; import { DEFAULT_TOTAL_RUNS, commandArgv, @@ -212,6 +213,8 @@ interface CliArgs { skipSync: boolean; collectRss: boolean; commandUnderTest: CommandUnderTest; + fixture: FixturePaths | null; + keepFixture: boolean; } function parseArgs(argv: string[]): CliArgs { @@ -229,12 +232,18 @@ function parseArgs(argv: string[]): CliArgs { let executable: string | undefined; let cliArgvJson: string | undefined; let artifactPath: string | undefined; + let explicitRoot = false; + let explicitDb = false; + let fixtureMb = 16; + let keepFixture = false; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === "--root") { root = resolve(argv[++i] ?? root); + explicitRoot = true; } else if (a === "--db") { db = resolve(argv[++i] ?? db); + explicitDb = true; } else if (a === "--source") { source = argv[++i] ?? source; } else if (a === "--runs") { @@ -251,6 +260,12 @@ function parseArgs(argv: string[]): CliArgs { skipSync = true; } else if (a === "--collect-rss") { collectRss = true; + } else if (a === "--fixture-mb") { + fixtureMb = parsePositiveInt(argv[++i], 16); + } else if (a === "--fixture") { + // no-op: explicit marker; fixture is now the default + } else if (a === "--keep-fixture") { + keepFixture = true; } else if (a === "--bin") { executable = argv[++i]; } else if (a === "--cli-argv-json") { @@ -260,10 +275,21 @@ function parseArgs(argv: string[]): CliArgs { } else if (a === "--json-only") { jsonOnly = true; } else if (a === "--help" || a === "-h") { - console.log("Usage: npm run eval:perf -- [--source ] [--root ] [--db ] [--runs ] [--read-runs ] [--status-runs ] [--skip-sync] [--bin | --cli-argv-json ] [--artifact ] [--collect-rss] [--dogfood ] [--best-effort] [--json-only]"); + console.log("Usage: npm run eval:perf -- [--source ] [--root ] [--db ] [--runs ] [--read-runs ] [--status-runs ] [--skip-sync] [--fixture-mb ] [--keep-fixture] [--bin | --cli-argv-json ] [--artifact ] [--collect-rss] [--dogfood ] [--best-effort] [--json-only]"); process.exit(0); } } + + // Default to deterministic synthetic fixture when no explicit data source + // is provided. Real local data is opt-in via explicit --root or --db. + let fixture: FixturePaths | null = null; + if (!explicitRoot && !explicitDb) { + fixture = generateFixture(fixtureMb, source); + root = fixture.root; + db = fixture.db; + skipSync = false; // fixture is fresh — must sync + } + const commandUnderTest = resolveCommandUnderTest({ root: ROOT, cliEntry: CLI_ENTRY, @@ -284,6 +310,8 @@ function parseArgs(argv: string[]): CliArgs { skipSync, collectRss, commandUnderTest, + fixture, + keepFixture, }; } @@ -471,7 +499,7 @@ if (args.skipSync) { // Default to strict sync so coverage is actually written and the status // probe below measures the fresh path (the one agents hit in practice). const syncCmd = ["sync", "--source", args.source, "--db", args.db, "--root", args.root]; - if (args.bestEffortSync) syncCmd.push("--best-effort"); + if (args.bestEffortSync || args.fixture) syncCmd.push("--best-effort"); const syncRun = await runOrThrow(cliCommand(...syncCmd, "--json"), { collectRss: args.collectRss }); syncMs = syncRun.ms; syncPeakRssBytes = syncRun.peakRssBytes; @@ -602,6 +630,11 @@ const summary = { }; console.log(JSON.stringify(args.jsonOnly ? report : summary, null, 2)); +// Clean up synthetic fixture unless --keep-fixture was requested. +if (args.fixture && !args.keepFixture) { + cleanupFixture(args.fixture); +} + function topHitFromFind(payload: FindJsonPayload): TopHitRecord | null { const first = payload.results?.[0]; if (!first || typeof first.sourceId !== "string" || typeof first.sessionRef !== "string" || typeof first.matchSource !== "string") { diff --git a/eval/perf-fixture.ts b/eval/perf-fixture.ts new file mode 100644 index 0000000..e2f98fd --- /dev/null +++ b/eval/perf-fixture.ts @@ -0,0 +1,242 @@ +/** + * Deterministic CJK-heavy fixture generator for Sherlog performance benchmarks. + * + * Generates Codex-format session transcripts into a temporary directory. + * Content is ~60% CJK (Chinese), ~25% Latin (English tech), ~15% paths/commands + * to exercise the full tokenizer path while staying realistic. + * + * Volume is controlled by `--fixture-mb` (default 16 MB of body text). + * All output is deterministic given the same megabyte parameter — same seed + * produces identical sessions on any machine. + * + * Usage: + * import { generateFixture, cleanupFixture } from "./perf-fixture"; + * const f = generateFixture(16); + * // ... run benchmarks against f.root + f.db ... + * cleanupFixture(f); + */ + +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +// ── types ─────────────────────────────────────────────────────────────────── + +export interface FixturePaths { + /** Temp directory containing generated session transcripts. */ + root: string; + /** Recommended temp db path (not created by the generator — harness owns sync). */ + db: string; + /** Number of sessions written. */ + sessionCount: number; + /** Approximate total body bytes (message text only). */ + bodyBytes: number; +} + +// ── CJK seed pool (~60% of content) ──────────────────────────────────────── + +const CJK_SEEDS = [ + "健康检查服务在部署后恢复正常,所有节点通过验证。", + "回滚预案需要在发布前经过两次以上的演练确认。", + "数据库迁移脚本已通过预发布环境的完整性校验。", + "日志聚合器在峰值流量下出现了短暂的磁盘写入延迟。", + "配置文件中的环境变量引用在构建时未能正确展开。", + "安全审计发现三个低危漏洞,建议在下个迭代修复。", + "预发布环境与生产环境的网络策略不一致导致连接超时。", + "负载均衡器的健康检查端点返回了非预期的状态码。", + "容器镜像的构建缓存未命中,全量构建耗时超出预算。", + "服务网格的边车代理在滚动更新时丢失了短暂连接。", + "分布式追踪采样率从百分之一调整为千分之一以节省存储。", + "密钥轮换脚本忽略了命名空间级别的资源引用。", + "弹性伸缩策略的冷却时间设置过短导致频繁扩缩。", + "监控告警规则在周末误报了两次,阈值得重新校准。", + "灰度发布的新版本在十分钟后被自动回滚到上一个稳定版。", + "缓存穿透防护对高频热点路径仍然依赖单节点互斥锁。", + "查询优化器选择的索引未能覆盖 WHERE 子句的全部谓词。", + "消息队列的消费者组发生了分区重平衡,导致短暂消费中断。", + "证书将在四十八小时内过期,自动续期任务需要人工确认。", + "跨可用区的同步复制延迟在高峰时段超过阈值,触发了主从切换。", + "数据校验发现昨天的增量备份缺失了两千条记录。", + "API 网关的限流规则对内部服务之间的调用误加了全局限流。", + "构建流水线的缓存层在本次提交命中率仅为百分之十二。", + "协程泄漏导致事件循环在第七个小时后响应显著变慢。", + "覆盖率报告丢失了集成测试的命中行统计。", + "分词器在处理补充平面汉字时产生了重复的标量二元组。", + "索引重建过程中数据库文件大小峰值达到了正常值的四倍。", + "查询计划显示 FTS5 跳过了内容表直接扫描了影子表。", + "会话摘要的紧凑文本中遗漏了推理链的关键结论。", + "指纹计算在解析超大单行 JSON 时超时,需要流式分块处理。", + // Terms matching every BENCH_QUERIES shape so the harness find+read path doesn't crash: + "豆包输入法在 SwiftUI 上的体验比原生键盘好很多,尤其符号布局。", + "部署健康检查脚本失败,原因是目标主机的 SSH 密钥已过期。", + "今天把 hammerspoon 的窗口管理配置从 0.9 迁移到了 1.0 语法。", + "新的 envchain 集成支持 namespace 级别的密钥隔离。", + "重构 sb 模块的查询构造器,把动态 ORDER BY 改成固定索引扫描。", + "fly deploy 超时是因为 Docker 构建缓存层在 CI 上不可用。", + "edge tts 服务的 gRPC 端点需要在网关层增加连接池配置。", + "部署 health check 发现两个节点没有拉取最新的配置中心变更。", + "Hammerspoon 需要检测到外接显示器变化后自动重新布局所有窗口。", + "Envchain 支持通过环境变量注入通配符匹配的密钥集合。", + "sb 的日志输出格式在 debug 模式下打印了完整的 AST 节点。", + "Fly deploy 前应该先验证 Turbosrc 仓库有没有未推送的 commit。", + "Edge TTS 的语音合成出现了偶发的音节丢失,采样率可能不匹配。", +]; + +// ── Latin seed pool (~25% of content) ─────────────────────────────────────── + +const LATIN_SEEDS = [ + "Deploy health-check endpoint returned 200 after rollback.", + "The CI pipeline failed at the integration-test stage due to a missing secret.", + "Refactor the parser to handle edge cases in Unicode normalization.", + "Performance regression detected in the tokenizer bigram generation path.", + "The session index is missing coverage for the newly added source adapter.", + "FTS5 contentless table requires explicit column rebuild after schema change.", + "Lock contention on the WAL checkpoint is visible above 8 concurrent readers.", + "Incremental sync produced a different document count than full replay.", + "The evidence read plan must preserve message-level provenance for CJK hits.", + "Query analysis falls back to literal LIKE when zero FTS tokens are produced.", + "The ranking heuristic over-weights session-profile hits on single-word queries.", + "Database page size affects B-tree fanout for the per-source coverage table.", + "The cold retention registration file was tombstoned after v7-to-v8 migration.", + "Sanitized fixture generation should be deterministic across platform locales.", + "The write-ahead log grows unboundedly when no checkpoint occurs between syncs.", + "Compact text summarisation dropped the domain-specific terminology in the conclusion.", + "A dangling junction symlink prevented the plugin loader from resolving the entry.", + "The acceptance gate uses synthetic UUIDs and deterministic timestamps for repeatability.", + "Coverage proof compares source file digests without re-reading historical rows.", + "Strict sync fails closed on malformed JSONL records and reports the byte offset.", + // Terms matching single/dual-token queries so harness find+read doesn't crash: + "Hammerspoon window manager layout is configured via Lua scripting.", + "Envchain stores secrets per-project and injects them into shell sessions.", + "The sb tool is a search backend that accelerates text queries.", + "Fly deploy failed because the Dockerfile referenced a stale base image tag.", + "Edge tts latency improved after switching to the premium neural voice tier.", +]; + +// ── path / command templates (~15% of content) ────────────────────────────── + +const PATH_TEMPLATES = [ + (i: number) => `cd /tmp/shlog-perf-fixture/project-${i % 20} && node dist/cli.js publish fixtures/sample.jsonl --json`, + (i: number) => `cd /Users/dev/work/repos/project-${i % 15} && cargo build --release --bin service-${i % 5}`, + (i: number) => `cd /opt/deploy/env-${i % 3}/config && kubectl apply -f deployment-${i % 10}.yaml`, + (i: number) => `grep -rn "tokenize" /src/rust/tokenizer-${i % 8}.rs | wc -l`, + (i: number) => `find /var/log/service-${i % 7} -name "*.log" -mtime -${1 + (i % 7)} | head -20`, + (i: number) => `curl -s http://localhost:${8000 + i}/health | jq .status`, + (i: number) => `cat /etc/config-${i % 4}/defaults.json | python3 -m json.tool > /dev/null`, + (i: number) => `systemctl restart daemon-${i % 6} && journalctl -u daemon-${i % 6} -n 5 --no-pager`, +]; + +// ── deterministic pRNG (mulberry32) ───────────────────────────────────────── + +function mulberry32(seed: number): () => number { + return () => { + seed |= 0; + seed = (seed + 0x6d2b79f5) | 0; + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +function pick(arr: readonly T[], rand: () => number): T { + return arr[Math.floor(rand() * arr.length)]; +} + +// ── message generation ────────────────────────────────────────────────────── + +interface CodexLine { + timestamp: string; + type: string; + payload: Record; +} + +function sessionMeta(id: string, cwd: string, timestamp: string): CodexLine { + return { timestamp, type: "session_meta", payload: { id, cwd } }; +} + +function turnContext(timestamp: string): CodexLine { + return { timestamp, type: "turn_context", payload: { model: "gpt-5.4" } }; +} + +function eventMessage( + type: "user_message" | "agent_message", + message: string, + timestamp: string, +): CodexLine { + return { timestamp, type: "event_msg", payload: { type, message } }; +} + +function sequentialTimestamp(base: Date, offsetSec: number): string { + return new Date(base.getTime() + offsetSec * 1000).toISOString(); +} + +// ── public API ────────────────────────────────────────────────────────────── + +/** + * Generate a deterministic CJK-heavy fixture for performance benchmarking. + * + * @param megabytes Approximate body text volume in MB. Default 16. + * @param source "codex" (default) — only codex-format sessions for now. + */ +export function generateFixture(megabytes = 16, source = "codex"): FixturePaths { + if (source !== "codex") throw new Error("Only codex source fixtures are supported"); + + const root = mkdtempSync(join(tmpdir(), "shlog-perf-fixture-")); + const db = join(root, "index.sqlite"); + const sessionsDir = join(root, "2026", "08", "17"); + mkdirSync(sessionsDir, { recursive: true }); + + const targetBytes = megabytes * 1024 * 1024; + const baseDate = new Date("2026-08-17T00:00:00.000Z"); + const rand = mulberry32(megabytes); // deterministic given --mb + + let bodyBytes = 0; + let sessionCount = 0; + + while (bodyBytes < targetBytes) { + const id = `50000000-0000-4000-8000-${String(sessionCount).padStart(12, "0")}`; + const cwd = `/tmp/shlog-perf-fixture/project-${sessionCount % 20}`; + const lines: string[] = []; + + // Session meta + turn_context + const t0 = sessionCount * 120; // 120s spacing between sessions + lines.push(JSON.stringify(sessionMeta(id, cwd, sequentialTimestamp(baseDate, t0)))); + lines.push(JSON.stringify(turnContext(sequentialTimestamp(baseDate, t0 + 0.5)))); + + // 3–10 messages per session + const msgCount = 3 + (sessionCount % 8); + for (let m = 0; m < msgCount; m++) { + const type = m % 2 === 0 ? "user_message" : "agent_message"; + const bucket = rand(); + let message: string; + + if (bucket < 0.60) { + // CJK + message = pick(CJK_SEEDS, rand); + } else if (bucket < 0.85) { + // Latin + message = pick(LATIN_SEEDS, rand); + } else { + // Path / command + message = pick(PATH_TEMPLATES, rand)(sessionCount + m); + } + + lines.push( + JSON.stringify( + eventMessage(type as "user_message" | "agent_message", message, sequentialTimestamp(baseDate, t0 + 1 + m)), + ), + ); + bodyBytes += Buffer.byteLength(message); + } + + writeFileSync(join(sessionsDir, `rollout-${id}.jsonl`), lines.join("\n") + "\n"); + sessionCount++; + } + + return { root, db, sessionCount, bodyBytes }; +} + +/** Remove the temp fixture directory. */ +export function cleanupFixture(f: FixturePaths): void { + rmSync(f.root, { recursive: true, force: true }); +} From e3cfe4f720d8cf9a4c120345f06531baeed73554 Mon Sep 17 00:00:00 2001 From: catoncat Date: Mon, 17 Aug 2026 14:10:13 +0800 Subject: [PATCH 3/4] docs: add architecture layering to CONTEXT.md (Rust production CLI vs TS eval harness) --- CONTEXT.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CONTEXT.md b/CONTEXT.md index 12f8a0c..5d3d049 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -8,6 +8,17 @@ See domain-modeling skill for the definition format. --> +## 架构分层 + +Sherlog 仓库有两条明确的技术栈边界: + +| 层 | 语言 | 角色 | 路径 | +|---|---|---|---| +| **Production CLI** | Rust | 用户安装的 `shlog` binary:SQLite FTS5、tokenizer、sync、find/read/stats | `rust/src/`,产物 `target/release/shlog` | +| **Eval harness** | TypeScript | 开发期测试工具:fork Rust binary 当子进程,测其延迟/吞吐/正确性/契约。**不实现任何检索逻辑,不是 product runtime** | `eval/`,`src/`(legacy TS oracle) | + +eval harness 是"裁判",Rust binary 是"选手"。`eval/perf-bench.ts` 做的事情是 `spawn("target/release/shlog", ["find", "豆包输入法", ...])` 然后掐表——它自身不执行 tokenization、不访问 SQLite、不参与检索。contract-gate、acceptance-gate、dogfood runner、concurrency-bench 都遵循同一模式。 + ## 性能 - **性能基线**(perf baseline):在某指定机器类上,对固定工作负载(合成 fixture 或真实 dogfood 数据)跑 `eval:perf` / `eval:perf:concurrency` 产出的 p50/p95 延迟、吞吐、RSS、DB 体积等数字。基线 JSON 随代码一起进 git,作为回归门判据的参照点。基线只在标定它的那台机器类上有比较意义。 From 5b9c0c80badb3154ad292260ea56cfdc13d4f77a Mon Sep 17 00:00:00 2001 From: catoncat Date: Mon, 17 Aug 2026 14:36:14 +0800 Subject: [PATCH 4/4] fix(eval): fail closed unless both --root and --db for private calibration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Default eval:perf and eval:perf:concurrency stay on isolated synthetic smoke. Passing only one path no longer falls back to ~/.codex/sessions or the developer state DB. Docs now call default runs smoke and real-index runs private calibration — not a git-tracked baseline. --- CONTEXT.md | 25 +++++++++------ docs/ROADMAP.md | 2 +- eval/PERF_BENCH.md | 39 +++++++++++++----------- eval/concurrency-bench-core.test.ts | 30 ++++++++++++++---- eval/concurrency-bench-core.ts | 47 +++++++++++++++++++++-------- eval/concurrency-bench.ts | 11 ++++--- eval/perf-bench.ts | 31 +++++++++++++------ eval/perf-data-source.test.ts | 44 +++++++++++++++++++++++++++ eval/perf-data-source.ts | 44 +++++++++++++++++++++++++++ eval/perf-fixture.ts | 9 +++--- 10 files changed, 219 insertions(+), 63 deletions(-) create mode 100644 eval/perf-data-source.test.ts create mode 100644 eval/perf-data-source.ts diff --git a/CONTEXT.md b/CONTEXT.md index 5d3d049..3357f41 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -15,24 +15,31 @@ Sherlog 仓库有两条明确的技术栈边界: | 层 | 语言 | 角色 | 路径 | |---|---|---|---| | **Production CLI** | Rust | 用户安装的 `shlog` binary:SQLite FTS5、tokenizer、sync、find/read/stats | `rust/src/`,产物 `target/release/shlog` | -| **Eval harness** | TypeScript | 开发期测试工具:fork Rust binary 当子进程,测其延迟/吞吐/正确性/契约。**不实现任何检索逻辑,不是 product runtime** | `eval/`,`src/`(legacy TS oracle) | +| **Eval harness** | TypeScript | 开发期裁判:fork 被测 CLI 当子进程,观测延迟/吞吐/正确性/契约。不实现检索逻辑,不是 product runtime | `eval/`,`src/`(legacy TS oracle) | -eval harness 是"裁判",Rust binary 是"选手"。`eval/perf-bench.ts` 做的事情是 `spawn("target/release/shlog", ["find", "豆包输入法", ...])` 然后掐表——它自身不执行 tokenization、不访问 SQLite、不参与检索。contract-gate、acceptance-gate、dogfood runner、concurrency-bench 都遵循同一模式。 +eval harness 是"裁判",Rust binary 是"选手"。`eval/perf-bench.ts` 做的事情是 `spawn(, ["find", "豆包输入法", ...])` 然后掐表——它自身不执行 tokenization、不参与检索。contract-gate、acceptance-gate、dogfood runner、concurrency-bench 都遵循同一模式。Harness 可以用开发期 Node `node:sqlite` 读已生成 index 的 `dbstat` 做体积记账;这不是检索路径,也不进入发布态 CLI。 + +未传 `--bin` / `--cli-argv-json` 时,harness 默认测 TypeScript oracle,不是 Rust production candidate。 ## 性能 -- **性能基线**(perf baseline):在某指定机器类上,对固定工作负载(合成 fixture 或真实 dogfood 数据)跑 `eval:perf` / `eval:perf:concurrency` 产出的 p50/p95 延迟、吞吐、RSS、DB 体积等数字。基线 JSON 随代码一起进 git,作为回归门判据的参照点。基线只在标定它的那台机器类上有比较意义。 +- **合成烟雾**(synthetic smoke):对确定性小 fixture 跑 `eval:perf` / `eval:perf:concurrency` 得到的延迟/吞吐数字。用来隔离回归、不碰开发者真实数据。不代表真实 Codex/Pi/Claude 的文件体积或命中基数。 + _Avoid_: 性能基线(在尚未有进 git 的对照 JSON 时)、可复现真实负载 + +- **本机校准**(private calibration):开发者对自己已有 index 做的只读测量(必须同时显式 `--root` 与 `--db`,建议 `--skip-sync`)。回答「我这份库、这台机器上容量如何」。数字只对标定它的那份库和那台机器有意义,不进 git 当回归门。 + _Avoid_: 默认负载、CI 基线 -- **性能回归门**(perf regression gate):发布流程中对性能基线的一次性检查——当某指标漂移超过容差(推荐默认 `max(baseline × 2.0, baseline + 150ms)`)时阻止发布,但提供显式 escape hatch 供人工复核后放行。当前只计划在 release workflow 跑,不在 PR CI 跑。 +- **性能回归门**(perf regression gate):发布流程里对**合成烟雾**数字的一次性检查。当前尚未落地;计划只在 release workflow 跑,不在 PR CI 跑,也不用本机校准数字当门槛。 -- **业务性能**(end-to-end CLI performance):用户可感知的 CLI 命令(`find`/`read-range`/`read-page`/`status`/`sync`)的进程级 wall-clock 延迟、内存与 DB 体积。由 `eval/perf-bench.ts`(串行)和 `eval/concurrency-bench.ts`(并发)测量。 +- **业务性能**(end-to-end CLI performance):用户可感知的 CLI 命令进程级 wall-clock 延迟、内存与 DB 体积。由 `eval/perf-bench.ts`(串行)和 `eval/concurrency-bench.ts`(并发)测量。默认测合成烟雾;本机校准须显式 opt-in。 -- **并发性能**(concurrency performance):多个独立 `shlog` 只读进程同时访问同一 SQLite index 时的吞吐(ops/s)与 tail latency(p95/p99)。由 `eval/concurrency-bench.ts` 测量。用于容量分析,不进入回归门。 +- **并发性能**(concurrency performance):多个独立 `shlog` 只读进程同时访问同一 SQLite index 时的吞吐与 tail latency。用于容量观察,不进入回归门。 -- **组件性能**(component-level micro-benchmark):单个内部模块(如 tokenizer)的吞吐、分配次数、长文本缩放。当前尚不存在;计划以零依赖 `examples/` bench bin 实现,用 e2e 中文 fixture 的 sync/find 延迟作为间接回归保护。 +- **组件性能**(component-level micro-benchmark):单个内部模块(如 tokenizer)的吞吐、分配次数、长文本缩放。当前尚不存在。 ## 数据与负载 -- **合成 fixture**(synthetic fixture):由 `eval/perf-fixture.ts` 确定性生成的临时 session 语料(Codex JSONL 格式)。内容 ~60% CJK + 25% Latin + 15% 路径/命令,体积由 `--fixture-mb` 控制(默认 16MB)。相同 seed ≡ 相同输出,跨机器可复现。用于性能基准的默认负载,不用于正确性测试。 +- **合成 fixture**(synthetic fixture):由 `eval/perf-fixture.ts` 确定性生成的临时 Codex JSONL。按**消息条**抽签约 60% CJK / 25% Latin / 15% 路径,体积由 `--fixture-mb` 控制(默认 16MB)。相同参数跨机器可复现。这是合成烟雾的默认负载,不是真实会话的形状模型,也不用于正确性或相关性测试。 + _Avoid_: 真实语料、dogfood 数据、形状拟合语料 -- **dogfood 数据**(dogfood data):开发者本机的真实 agent session 历史。只用于质量线(dogfood eval、acceptance gate)和性能线的显式 opt-in 模式(`--root`/`--db` 显式传参)。perf harness 默认不触碰真实数据。 +- **dogfood 数据**(dogfood data):开发者本机的真实 agent session 历史。只用于质量线(dogfood eval、acceptance 的人工对照)和性能线的本机校准。perf harness 默认不触碰它。 diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index be1c576..0ce7c52 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -22,7 +22,7 @@ message 与 session-profile 共用 `documents`/`documents_fts`,但 evidence pr - dogfood runner 已复用统一的 CLI-under-test 解析,可通过 `--cli-argv-json`、`SHLOG_CLI_ARGV_JSON` 或 `SHLOG_BIN_UNDER_TEST` 显式绑定 native candidate,并把实际 argv/source 写入 scorecard;无 override 时才默认 TypeScript oracle; - executable-neutral contract gate 已把 help prose、query-only coverage freshness、typed error semantics 与 native strict-incomplete reason 编码为 intentional-difference policy;当前 `target/release/shlog` 对 TypeScript reference 实测 **24/24**; - synthetic acceptance gate 已显式绑定同一 native release candidate;message hit、session-profile hit、CJK、source-aware read、command restatement 等 evidence-level fixtures 当前实测 **8/8**; -- candidate-aware perf harness:可显式选择 release binary、记录 process/operation latency、RSS、artifact/DB size 和 progressive reads;默认使用确定性合成的 CJK-heavy fixture(安全隔离),真实数据须显式 opt-in;附 concurrency 补充 harness 测并发吞吐与 tail latency; +- candidate-aware perf harness:可显式选择 release binary、记录 process/operation latency、RSS、artifact/DB size 和 progressive reads;默认跑确定性合成烟雾 fixture(隔离、不代表真实体积分布),本机校准必须同时显式 `--root` 和 `--db`(建议 `--skip-sync`);附 concurrency 补充 harness;尚未有进 git 的回归基线 JSON,也尚未按真实体积分布做形状拟合语料; - Rust unit/integration tests:sources、index、sync、migration、retrieval、app; - native CI 会实际构建并检查 `target/release/shlog` 后以 `--require-candidate` 运行 contract/acceptance;release workflow 则下载 Linux GNU archive、解包并验证其中的 executable,再运行同一 gates; - native release workflow:macOS arm64/x64、Linux x64 GNU archives、SBOM、checksums、attestation、installer/formula。 diff --git a/eval/PERF_BENCH.md b/eval/PERF_BENCH.md index d656f95..d42dcae 100644 --- a/eval/PERF_BENCH.md +++ b/eval/PERF_BENCH.md @@ -41,11 +41,18 @@ npm run eval:perf -- --root --db --skip-sync --json-only ## Safety 与输入范围 -无参数运行时,harness 自动在临时目录生成**确定性合成 fixture**(默认 ~16 MB CJK-heavy 正文),对其执行 sync 和读测试,结束后自动清理。这不会触碰你的真实 session 数据或默认 state DB——适合作为隔离基准。 +有且仅有两种负载,不要混: -### 使用真实数据(opt-in) +| 模式 | 怎么进 | 测的是什么 | 会不会碰真实数据 | +|---|---|---|---| +| **合成烟雾**(默认) | 不传 `--root`、不传 `--db` | 隔离、可复现的短句 fixture;用来看有没有崩或慢一个数量级 | 不会 | +| **本机校准** | **同时**传 `--root` 和 `--db` | 你自己的已有 index;回答「我这份库有多快」 | 只读(请加 `--skip-sync`) | -要对自己的本机 session 做 dogfood 基准,必须**显式传 `--root` 或 `--db`**: +只传其中一个路径会直接失败:以前会用另一个开发者本机默认(`~/.codex/sessions` 或状态目录里的 index)补齐,那会扫到或写到真实数据。`--fixture-mb` 不能和 `--root`/`--db` 混用。 + +git 里只承认合成烟雾数字。本机校准数字不要当回归门,也不要假装代表其他机器或其他 source。 + +### 本机校准(opt-in,只读) ```bash npm run eval:perf -- \ @@ -55,11 +62,9 @@ npm run eval:perf -- \ --skip-sync ``` -不显式传 `--root`/`--db` 时,harness **永远不会访问你的真实数据**。 +### 合成烟雾 fixture -### 合成 fixture - -`--fixture-mb ` 控制 fixture 体积(默认 16)。fixture 由 `eval/perf-fixture.ts` 确定性生成——相同参数在任何机器上产出相同的 session 集合。内容为 ~60% CJK(中文运维场景)、~25% Latin(英文 tech)、~15% 路径/命令。 +`--fixture-mb ` 控制正文体积(默认 16)。`eval/perf-fixture.ts` 按消息条抽签约 60% CJK / 25% Latin / 15% 路径;相同参数跨机器可复现。这**不是**真实 Codex/Pi/Claude 的文件体积模型(真实 Codex 往往是少量大文件,合成烟雾是大量短句小文件)。 ```bash # 4 MB 快速 smoke @@ -69,7 +74,7 @@ npm run eval:perf -- --bin ./target/release/shlog --fixture-mb 4 --json-only npm run eval:perf -- --bin ./target/release/shlog --fixture-mb 4 --keep-fixture ``` -fixture 的 sync 自动走 `--best-effort`(临时目录在 macOS 上可能因文件系统事件触发 strict 的 "source_file_set_changed" 误判)。对真实数据(显式 `--root`/`--db`)仍走 strict 以保证覆盖度。 +烟雾 fixture 的 sync 走 `--best-effort`(临时目录在 macOS 上可能触发 strict 的 `source_file_set_changed`)。本机校准不要 sync;若省略 `--skip-sync`,strict sync 会写你传入的那个 `--db`。 `status` 会按公开 contract 建立 live privacy-filtered inventory 并计算 requested selector coverage;它不返回/检索正文、不写 index,但 cache miss 可流式读取 raw accepted records/body,成本可能为 O(raw bytes),exact `mtime_ns`/checkpoint cache hit 则不重 parse。`find`、`read-range`、`read-page`、`stats` 只读 index。Harness 会把显式 root 传给 status/find,但 find 不自行扫描 raw transcript freshness。 @@ -128,19 +133,17 @@ npm run eval:perf -- \ ## 并发基准 -`npm run eval:perf:concurrency` 是并发读路径的补充 harness。与串行 harness 不同,它测的是**同时多个独立 `shlog` 进程**访问同一个只读 SQLite index 时的吞吐与 tail latency。它不执行 `sync`,要求 `--db` 已存在。可配合 `--fixture-mb ` 自动生成临时 fixture 并 sync: +`npm run eval:perf:concurrency` 测的是**同时多个独立 `shlog` 进程**访问同一个只读 SQLite index 时的吞吐与 tail latency。负载选择与串行 harness 相同:无路径 = 合成烟雾(自动 sync `--best-effort`);同时给 `--root` 和 `--db` = 本机校准(不 sync)。 ```bash -# 自动生成 16MB fixture 并测试 -npm run eval:perf:concurrency -- \ - --bin ./target/release/shlog \ - --fixture-mb 16 +# 默认:合成烟雾 +npm run eval:perf:concurrency -- --bin ./target/release/shlog -# 或使用预建 index +# 本机校准(只读,必须两个路径一起传) npm run eval:perf:concurrency -- \ --bin ./target/release/shlog \ - --root /absolute/path/to/fixture/sessions \ - --db /absolute/path/to/fixture/index.sqlite \ + --root ~/.codex/sessions \ + --db ~/.local/state/shlog/index.sqlite \ --shapes "find:hammerspoon|find:edge tts|read-range|read-page|status" \ --levels "1 2 4 8 16 32" \ --total 80 @@ -155,9 +158,9 @@ npm run eval:perf:concurrency -- \ - `errors`(非零退出计数) - 默认写入 `data/shlog-perf/concurrency//report.json` 与 `report.md`;`--json-only` 只向 stdout 输出。 -### 本机基线(2026-08-17,Apple M4 / 10 核 / 16GB) +### 本机校准观察(2026-08-17,Apple M4 / 10 核 / 16GB) -被测 `target/release/shlog` 0.5.1(native),真实 Codex index:6217 sessions / 318k messages / 420MB SQLite,热缓存。数字来自 `npm run eval:perf:concurrency`(`total=40`、`levels 1 2 4 8 16 32`),为 per-op E2E p50/p95(毫秒)与峰值吞吐(ops/s): +这是作者机器上对**真实** Codex index 的只读校准,**不是**合成烟雾,也不是 git 回归基线。被测 `target/release/shlog` 0.5.1(native),6217 sessions / 318k messages / 420MB SQLite,热缓存。数字来自 `npm run eval:perf:concurrency`(`total=40`、`levels 1 2 4 8 16 32`),为 per-op E2E p50/p95(毫秒)与峰值吞吐(ops/s): | shape | 1 并发 p50/p95 | 4 并发 p50/p95 | 16 并发 p50/p95 | 峰值吞吐(@并发) | |---|---|---|---|---| diff --git a/eval/concurrency-bench-core.test.ts b/eval/concurrency-bench-core.test.ts index 0c9cae4..7bf28b8 100644 --- a/eval/concurrency-bench-core.test.ts +++ b/eval/concurrency-bench-core.test.ts @@ -45,12 +45,28 @@ describe("concurrency shape parsing", () => { }); describe("concurrency arg parsing", () => { - test("requires --db", () => { - expect(() => parseConcurrencyArgs([])).toThrow(/--db is required/); - expect(() => parseConcurrencyArgs(["--root", "/tmp/root"])).toThrow(/--db is required/); + test("no paths selects synthetic smoke and does not default to ~/.codex", () => { + const args = parseConcurrencyArgs([]); + expect(args.workload).toBe("synthetic_smoke"); + expect(args.fixtureMb).toBe(16); + expect(args.root).toBe(""); + expect(args.db).toBe(""); }); - test("parses overrides and defaults", () => { + test("only --root or only --db is rejected", () => { + expect(() => parseConcurrencyArgs(["--root", "/tmp/root"])).toThrow(/both --root and --db/); + expect(() => parseConcurrencyArgs(["--db", "/tmp/index.sqlite"])).toThrow(/both --root and --db/); + }); + + test("mixing --fixture-mb with real paths is rejected", () => { + expect(() => parseConcurrencyArgs([ + "--root", "/tmp/root", + "--db", "/tmp/index.sqlite", + "--fixture-mb", "4", + ])).toThrow(/do not mix --fixture-mb/); + }); + + test("parses calibration overrides and defaults", () => { const args = parseConcurrencyArgs([ "--db", "/tmp/index.sqlite", "--root", "/tmp/sessions", @@ -60,6 +76,7 @@ describe("concurrency arg parsing", () => { "--total", "40", "--json-only", ]); + expect(args.workload).toBe("private_calibration"); expect(args.db).toBe("/tmp/index.sqlite"); expect(args.root).toBe("/tmp/sessions"); expect(args.source).toBe("claude-code"); @@ -71,8 +88,9 @@ describe("concurrency arg parsing", () => { expect(args.commandUnderTest.source).toBe("typescript-reference"); }); - test("accepts explicit executable override", () => { - const args = parseConcurrencyArgs(["--db", "/tmp/index.sqlite", "--bin", "/tmp/shlog"]); + test("accepts explicit executable override on smoke", () => { + const args = parseConcurrencyArgs(["--bin", "/tmp/shlog"]); + expect(args.workload).toBe("synthetic_smoke"); expect(args.commandUnderTest.source).not.toBe("typescript-reference"); }); }); diff --git a/eval/concurrency-bench-core.ts b/eval/concurrency-bench-core.ts index fef7629..044d78d 100644 --- a/eval/concurrency-bench-core.ts +++ b/eval/concurrency-bench-core.ts @@ -1,5 +1,6 @@ import { resolve } from "node:path"; import { resolveCommandUnderTest, type CommandUnderTest } from "./perf-bench-core"; +import { resolvePerfWorkload, type PerfWorkloadKind } from "./perf-data-source"; /** * Pure helpers for the concurrency benchmark harness (`concurrency-bench.ts`). @@ -22,6 +23,7 @@ export const DEFAULT_SHAPES = [ ]; export interface ConcurrencyArgs { + workload: PerfWorkloadKind; root: string; db: string; source: string; @@ -83,7 +85,7 @@ export interface ConcurrencyReport { } export function parseConcurrencyArgs(argv: string[]): ConcurrencyArgs { - let root = process.env.HOME ? resolve(process.env.HOME, ".codex", "sessions") : ""; + let root = ""; let db = ""; let source = "codex"; let jsonOnly = false; @@ -93,19 +95,28 @@ export function parseConcurrencyArgs(argv: string[]): ConcurrencyArgs { let executable: string | undefined; let cliArgvJson: string | undefined; let artifactPath: string | undefined; - let fixtureMb = 0; + let fixtureMb = 16; + let fixtureMbExplicit = false; + let explicitRoot = false; + let explicitDb = false; let keepFixture = false; for (let i = 0; i < argv.length; i++) { const a = argv[i]; const next = () => argv[++i]; - if (a === "--root") root = resolve(next() ?? root); - else if (a === "--db") db = resolve(next() ?? ""); - else if (a === "--source") source = next() ?? source; + if (a === "--root") { + root = resolve(next() ?? root); + explicitRoot = true; + } else if (a === "--db") { + db = resolve(next() ?? ""); + explicitDb = true; + } else if (a === "--source") source = next() ?? source; else if (a === "--shapes") shapes = parseShapes(next() ?? ""); else if (a === "--levels") levels = parseLevels(next() ?? ""); else if (a === "--total") totalPerLevel = parsePositiveInt(next(), DEFAULT_TOTAL_PER_LEVEL); - else if (a === "--fixture-mb") fixtureMb = parsePositiveInt(next(), 16); - else if (a === "--fixture") { /* no-op: fixture is now default */ } + else if (a === "--fixture-mb") { + fixtureMb = parsePositiveInt(next(), 16); + fixtureMbExplicit = true; + } else if (a === "--fixture") { /* no-op alias: synthetic smoke is the default when both paths are omitted */ } else if (a === "--keep-fixture") keepFixture = true; else if (a === "--bin") executable = next(); else if (a === "--cli-argv-json") cliArgvJson = next(); @@ -115,7 +126,7 @@ export function parseConcurrencyArgs(argv: string[]): ConcurrencyArgs { throw new HelpRequested(); } } - if (!db && !fixtureMb) throw new Error("--db is required (concurrency benchmark is read-only against an existing index; use --fixture-mb to auto-generate one)"); + const workload = resolvePerfWorkload({ explicitRoot, explicitDb, fixtureMbExplicit }); const commandUnderTest = resolveCommandUnderTest({ root: ROOT, cliEntry: CLI_ENTRY, @@ -123,7 +134,19 @@ export function parseConcurrencyArgs(argv: string[]): ConcurrencyArgs { argvJson: cliArgvJson, artifactPath, }); - return { root, db, source, shapes, levels, totalPerLevel, jsonOnly, commandUnderTest, fixtureMb, keepFixture }; + return { + workload: workload.kind, + root, + db, + source, + shapes, + levels, + totalPerLevel, + jsonOnly, + commandUnderTest, + fixtureMb, + keepFixture, + }; } export class HelpRequested extends Error { @@ -134,9 +157,9 @@ export class HelpRequested extends Error { } export const USAGE = `Usage: npm run eval:perf:concurrency -- \\ - --db [--root ] [--source ] \\ - [--shapes "find:hammerspoon|read-range|read-page|status"] \\ - [--levels "1 2 4 8 16 32"] [--total 80] [--fixture-mb ] [--keep-fixture] \\ + [--fixture-mb ] [--keep-fixture] | --root --db \\ + [--source ] [--shapes "find:hammerspoon|read-range|read-page|status"] \\ + [--levels "1 2 4 8 16 32"] [--total 80] \\ [--bin | --cli-argv-json ] [--artifact ] [--json-only]`; /** Literal command shapes that must not be reinterpreted as find queries. */ diff --git a/eval/concurrency-bench.ts b/eval/concurrency-bench.ts index 3fa8304..c982dce 100644 --- a/eval/concurrency-bench.ts +++ b/eval/concurrency-bench.ts @@ -34,14 +34,17 @@ try { process.exit(1); } -// Generate synthetic fixture when --fixture-mb was explicitly set. +// Synthetic smoke: generate an isolated fixture and sync it. Private +// calibration (--root and --db) is read-only against the existing index. let fixture: FixturePaths | null = null; -if (process.argv.includes("--fixture-mb")) { +if (args.workload === "synthetic_smoke") { fixture = generateFixture(args.fixtureMb, args.source); args.root = fixture.root; args.db = fixture.db; - // Auto-sync the fresh fixture so the read-only benchmark has data. - const syncCmd = ["sync", "--source", args.source, "--db", args.db, "--root", args.root, "--json"]; + const syncCmd = [ + "sync", "--source", args.source, "--db", args.db, "--root", args.root, + "--best-effort", "--json", + ]; const syncResult = spawnSync(args.commandUnderTest.executable, [...args.commandUnderTest.prefixArgv, ...syncCmd], { stdio: ["ignore", "pipe", "pipe"], }); diff --git a/eval/perf-bench.ts b/eval/perf-bench.ts index afd39ad..a621e84 100644 --- a/eval/perf-bench.ts +++ b/eval/perf-bench.ts @@ -2,11 +2,10 @@ import { existsSync, mkdirSync, statSync, writeFileSync } from "node:fs"; import { spawn as childSpawn } from "node:child_process"; -import { homedir } from "node:os"; import { join, resolve } from "node:path"; import { DatabaseSync } from "node:sqlite"; -import { DEFAULT_DB_PATH } from "../src/env"; import { cleanupFixture, generateFixture, type FixturePaths } from "./perf-fixture"; +import { PerfDataSourceError, resolvePerfWorkload } from "./perf-data-source"; import { DEFAULT_TOTAL_RUNS, commandArgv, @@ -218,8 +217,8 @@ interface CliArgs { } function parseArgs(argv: string[]): CliArgs { - let root = join(homedir(), ".codex", "sessions"); - let db = DEFAULT_DB_PATH; + let root = ""; + let db = ""; let source = "codex"; let jsonOnly = false; let runsPerQuery = DEFAULT_RUNS_PER_QUERY; @@ -235,6 +234,7 @@ function parseArgs(argv: string[]): CliArgs { let explicitRoot = false; let explicitDb = false; let fixtureMb = 16; + let fixtureMbExplicit = false; let keepFixture = false; for (let i = 0; i < argv.length; i++) { const a = argv[i]; @@ -262,8 +262,9 @@ function parseArgs(argv: string[]): CliArgs { collectRss = true; } else if (a === "--fixture-mb") { fixtureMb = parsePositiveInt(argv[++i], 16); + fixtureMbExplicit = true; } else if (a === "--fixture") { - // no-op: explicit marker; fixture is now the default + // no-op alias: synthetic smoke is the default when both paths are omitted } else if (a === "--keep-fixture") { keepFixture = true; } else if (a === "--bin") { @@ -275,15 +276,27 @@ function parseArgs(argv: string[]): CliArgs { } else if (a === "--json-only") { jsonOnly = true; } else if (a === "--help" || a === "-h") { - console.log("Usage: npm run eval:perf -- [--source ] [--root ] [--db ] [--runs ] [--read-runs ] [--status-runs ] [--skip-sync] [--fixture-mb ] [--keep-fixture] [--bin | --cli-argv-json ] [--artifact ] [--collect-rss] [--dogfood ] [--best-effort] [--json-only]"); + console.log("Usage: npm run eval:perf -- [--source ] [--fixture-mb ] [--keep-fixture] | --root --db [--skip-sync] [--runs ] [--read-runs ] [--status-runs ] [--bin | --cli-argv-json ] [--artifact ] [--collect-rss] [--dogfood ] [--best-effort] [--json-only]"); process.exit(0); } } - // Default to deterministic synthetic fixture when no explicit data source - // is provided. Real local data is opt-in via explicit --root or --db. + let workload; + try { + workload = resolvePerfWorkload({ explicitRoot, explicitDb, fixtureMbExplicit }); + } catch (error) { + if (error instanceof PerfDataSourceError) { + console.error(`error: ${error.message}`); + process.exit(1); + } + throw error; + } + + // Default: isolated synthetic smoke. Real local data is opt-in only when + // BOTH --root and --db are explicit — one flag must not revive the other + // developer-machine default. let fixture: FixturePaths | null = null; - if (!explicitRoot && !explicitDb) { + if (workload.kind === "synthetic_smoke") { fixture = generateFixture(fixtureMb, source); root = fixture.root; db = fixture.db; diff --git a/eval/perf-data-source.test.ts b/eval/perf-data-source.test.ts new file mode 100644 index 0000000..dc20842 --- /dev/null +++ b/eval/perf-data-source.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, test } from "vitest"; +import { PerfDataSourceError, resolvePerfWorkload } from "./perf-data-source"; + +describe("resolvePerfWorkload", () => { + test("omitting both paths selects synthetic smoke", () => { + expect(resolvePerfWorkload({ explicitRoot: false, explicitDb: false })).toEqual({ + kind: "synthetic_smoke", + }); + }); + + test("omitting both paths still allows --fixture-mb", () => { + expect(resolvePerfWorkload({ + explicitRoot: false, + explicitDb: false, + fixtureMbExplicit: true, + })).toEqual({ kind: "synthetic_smoke" }); + }); + + test("both paths select private calibration", () => { + expect(resolvePerfWorkload({ explicitRoot: true, explicitDb: true })).toEqual({ + kind: "private_calibration", + }); + }); + + test("only --root is rejected", () => { + expect(() => resolvePerfWorkload({ explicitRoot: true, explicitDb: false })) + .toThrow(PerfDataSourceError); + expect(() => resolvePerfWorkload({ explicitRoot: true, explicitDb: false })) + .toThrow(/both --root and --db/); + }); + + test("only --db is rejected", () => { + expect(() => resolvePerfWorkload({ explicitRoot: false, explicitDb: true })) + .toThrow(/both --root and --db/); + }); + + test("mixing --fixture-mb with both real paths is rejected", () => { + expect(() => resolvePerfWorkload({ + explicitRoot: true, + explicitDb: true, + fixtureMbExplicit: true, + })).toThrow(/do not mix --fixture-mb/); + }); +}); diff --git a/eval/perf-data-source.ts b/eval/perf-data-source.ts new file mode 100644 index 0000000..d7fb3d9 --- /dev/null +++ b/eval/perf-data-source.ts @@ -0,0 +1,44 @@ +/** + * Shared workload selection for eval:perf and eval:perf:concurrency. + * + * Two modes only: + * - omit both --root and --db → synthetic smoke fixture (isolated, not representative) + * - pass both --root and --db → private calibration against an existing corpus + * + * Passing only one path used to fall back to the other real default + * (~/.codex/sessions or the developer state DB). That is rejected. + */ + +export type PerfWorkloadKind = "synthetic_smoke" | "private_calibration"; + +export class PerfDataSourceError extends Error { + constructor(message: string) { + super(message); + this.name = "PerfDataSourceError"; + } +} + +export const PERF_DATA_SOURCE_USAGE = + "omit both --root and --db for the synthetic smoke fixture; pass both --root and --db for private read-only calibration"; + +export function resolvePerfWorkload(input: { + explicitRoot: boolean; + explicitDb: boolean; + fixtureMbExplicit?: boolean; +}): { kind: PerfWorkloadKind } { + const { explicitRoot, explicitDb, fixtureMbExplicit = false } = input; + if (explicitRoot && explicitDb) { + if (fixtureMbExplicit) { + throw new PerfDataSourceError( + `do not mix --fixture-mb with --root/--db; ${PERF_DATA_SOURCE_USAGE}`, + ); + } + return { kind: "private_calibration" }; + } + if (!explicitRoot && !explicitDb) { + return { kind: "synthetic_smoke" }; + } + throw new PerfDataSourceError( + `private calibration requires both --root and --db; ${PERF_DATA_SOURCE_USAGE}`, + ); +} diff --git a/eval/perf-fixture.ts b/eval/perf-fixture.ts index e2f98fd..5f6042b 100644 --- a/eval/perf-fixture.ts +++ b/eval/perf-fixture.ts @@ -1,9 +1,10 @@ /** - * Deterministic CJK-heavy fixture generator for Sherlog performance benchmarks. + * Deterministic Codex-format smoke fixture for Sherlog performance harnesses. * - * Generates Codex-format session transcripts into a temporary directory. - * Content is ~60% CJK (Chinese), ~25% Latin (English tech), ~15% paths/commands - * to exercise the full tokenizer path while staying realistic. + * Generates short session transcripts into a temporary directory. Message + * draws are ~60% CJK / ~25% Latin / ~15% paths (per message, not per byte, + * and not a real size histogram). This is isolated regression smoke — it is + * not a shape-faithful model of developer Codex/Pi/Claude corpora. * * Volume is controlled by `--fixture-mb` (default 16 MB of body text). * All output is deterministic given the same megabyte parameter — same seed