Architecture doc, mock-fm stub, and integration smoke test

.gitignore

@@ -5,3 +5,4 @@ coverage/
 reports/
 dist/
 tmp/
+.lore/data/

AGENTS.md

@@ -12,6 +12,13 @@
 
 The product benchmarks Apple's `fm` CLI, which only exists on macOS 27+ with Apple Intelligence. On the Linux cloud VM `fm` is absent, so `doctor`/`models`/the default benchmark fail with `spawn fm ENOENT`.
 
-The CLI reads the `fm` binary from the `FM_BIN` env var (or `--fm-bin <path>`). To exercise the full benchmark/report pipeline end-to-end here, point `FM_BIN` at a stub that emulates the subcommands `fm-bench` calls: `--help` (must print `Apple Foundation Models CLI` and/or a `MODELS` section), `available --model <m>`, `quota-usage --model <m>`, `token-count --quiet` (reads stdin, prints a token count), and `respond --model <m>` (reads the prompt on stdin, streams the answer to stdout). Example: `FM_BIN=/path/to/mock-fm node bin/fm-bench.js --models system --runs 2 --profile quick`.
+The CLI reads the `fm` binary from the `FM_BIN` env var (or `--fm-bin <path>`). Use the repo stub:
+
+```sh
+chmod +x scripts/mock-fm.sh
+FM_BIN="$(pwd)/scripts/mock-fm.sh" node bin/fm-bench.js --models system --runs 2 --profile quick
+```
+
+Architecture and module map: `docs/architecture.md`. Report schema: `docs/report-format.md`.
 
 These commands need **no** `fm` and work as-is: `legend`, `validate <report.json>`, `export <report.json>`, `compare <a.json> <b.json>`, `history <dir>`.

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## Unreleased
+
+- **`scripts/mock-fm.sh`**: documented stub for running the benchmark pipeline without Apple's `fm` (Linux/CI dev).
+- **`docs/architecture.md`**: explicit module map for agents and contributors.
+- **AGENTS.md** / **CONTRIBUTING.md**: mock-fm quick start and architecture cross-links.
+- Integration test: mock-fm quick benchmark JSON smoke.
+
 ## 0.6.1
 
 - **Release** workflow: GitHub release bodies are generated from `CHANGELOG.md` (not empty auto-notes). Re-running Release on an existing tag updates release notes.

CONTRIBUTING.md

@@ -12,6 +12,15 @@ npm run lint
 
 Use macOS 27 or newer when testing against the real Apple `fm` command. Unit tests do not require `fm`.
 
+On Linux or CI-like environments, run a quick benchmark with the mock `fm` stub:
+
+```sh
+chmod +x scripts/mock-fm.sh
+FM_BIN="$(pwd)/scripts/mock-fm.sh" node bin/fm-bench.js --models system --runs 2 --profile quick
+```
+
+See `docs/architecture.md` for how CLI, benchmark, and report layers fit together.
+
 ## Pull Requests
 
 - Keep benchmark behavior deterministic where possible.

README.md

@@ -279,7 +279,7 @@ A live single-line progress indicator runs on stderr during interactive sessions
 
 `pcc` (Private Cloud Compute) availability depends on Apple's current eligibility. `fm-bench` shows it as skipped if `fm available --model pcc` reports unavailable.
 
-See [docs/methodology.md](docs/methodology.md) for benchmark methodology and metric references.
+See [docs/methodology.md](docs/methodology.md) for benchmark methodology and metric references. Module layout: [docs/architecture.md](docs/architecture.md).
 
 ## Development
 

docs/architecture.md

@@ -0,0 +1,29 @@
+# fm-bench architecture
+
+Explicit overview for agents and contributors (not inferred from folder names alone).
+
+## Decision
+
+fm-bench is a **single-process Node.js CLI** (ESM, no runtime npm dependencies). Entry: `bin/fm-bench.js` → `src/cli.js` routes subcommands; benchmark work lives in `src/bench.js` with `src/fm.js` + `src/process.js` wrapping the external `fm` binary.
+
+## Layers
+
+| Layer | Paths | Responsibility |
+|-------|--------|----------------|
+| CLI | `src/cli.js`, `bin/fm-bench.js` | Args, doctor, compare, validate, export, history, legend, default benchmark |
+| Benchmark | `src/bench.js`, `src/prompts.js`, `src/fm.js` | Model discovery, prompt suite, timed runs, streaming metrics |
+| Reports | `src/report.js`, `src/schema.js`, `src/stats.js`, `src/export.js` | Schema v1 JSON, CSV flatten, HTML export, validation |
+| Compare / history | `src/compare.js`, `src/history.js` | Regression diffs, trend tables |
+| Presentation | `src/table.js`, `src/progress.js`, `src/ansi.js` | Terminal tables, progress, colors |
+
+## External dependency
+
+All inference goes through the **`fm` executable** (`FM_BIN` or `--fm-bin`). fm-bench does not embed models. Off macOS, use `scripts/mock-fm.sh` for pipeline tests.
+
+## Tests
+
+`node --test` under `test/` — parsers, schema, compare, CLI routing. No `fm` required for unit tests.
+
+## CI
+
+`.github/workflows/ci.yml` — `npm ci`, `npm test`, `npm run lint`, `publish:dry-run` on `macos-15`.

scripts/mock-fm.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# Minimal fm stub for fm-bench off macOS (see AGENTS.md). Usage:
+#   FM_BIN="$(pwd)/scripts/mock-fm.sh" node bin/fm-bench.js --models system --runs 2 --profile quick
+set -euo pipefail
+
+cmd="${1:-}"
+shift || true
+
+case "$cmd" in
+  --help|-h|'')
+    cat <<'EOF'
+Apple Foundation Models CLI (mock)
+
+MODELS
+  system        On-device Apple Foundation Model (default)
+  pcc           Apple Foundation Model on Private Cloud Compute
+
+USAGE
+  fm respond --model <model>
+  fm available --model <model>
+  fm quota-usage --model <model>
+  fm token-count --quiet
+EOF
+    exit 0
+    ;;
+  available)
+    model=""
+    while [[ $# -gt 0 ]]; do
+      case "$1" in
+        --model) model="$2"; shift 2 ;;
+        *) shift ;;
+      esac
+    done
+    echo "System model available: ${model:-system}"
+    exit 0
+    ;;
+  quota-usage)
+    echo "quota: 1000 remaining"
+    exit 0
+    ;;
+  token-count)
+    wc -c | awk '{print int($1/4)+1}'
+    exit 0
+    ;;
+  respond)
+    model="system"
+    while [[ $# -gt 0 ]]; do
+      case "$1" in
+        --model) model="$2"; shift 2 ;;
+        *) shift ;;
+      esac
+    done
+    prompt="$(cat)"
+    words="$(echo "$prompt" | wc -w | tr -d ' ')"
+    i=0
+    while [[ $i -lt $words ]]; do
+      echo -n "token "
+      i=$((i + 1))
+    done
+    echo "ok from mock-fm model=$model"
+    exit 0
+    ;;
+  *)
+    echo "mock-fm: unknown command $cmd" >&2
+    exit 1
+    ;;
+esac

test/mock-fm.integration.test.js

@@ -0,0 +1,46 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { spawn } from 'node:child_process';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const root = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
+const mockFm = path.join(root, 'scripts', 'mock-fm.sh');
+const cli = path.join(root, 'bin', 'fm-bench.js');
+
+function runBenchJson() {
+  return new Promise((resolve, reject) => {
+    const child = spawn(process.execPath, [
+      cli,
+      '--models', 'system',
+      '--runs', '1',
+      '--profile', 'quick',
+      '--format', 'json'
+    ], {
+      cwd: root,
+      env: { ...process.env, FM_BIN: mockFm },
+      stdio: ['ignore', 'pipe', 'pipe']
+    });
+    let out = '';
+    child.stdout.on('data', (c) => { out += c; });
+    child.on('error', reject);
+    child.on('close', (code) => {
+      if (code !== 0) {
+        reject(new Error(`exit ${code}`));
+        return;
+      }
+      try {
+        resolve(JSON.parse(out));
+      } catch (err) {
+        reject(err);
+      }
+    });
+  });
+}
+
+test('mock-fm runs quick benchmark and emits schema fields', async () => {
+  const report = await runBenchJson();
+  assert.equal(report.tool, 'fm-bench');
+  assert.ok(Array.isArray(report.results));
+  assert.ok(report.results.length >= 1);
+});