From df95de73780f233cf94f8ceae34eb4755de22cc0 Mon Sep 17 00:00:00 2001 From: Devin Oldenburg Date: Sun, 28 Jun 2026 06:01:02 +0200 Subject: [PATCH] Add architecture doc, mock-fm stub, and off-macOS benchmark smoke test. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lore-guided gap: no explicit architecture decision — document CLI/bench/report layers in docs/architecture.md. Ship scripts/mock-fm.sh for FM_BIN pipeline tests on Linux/CI; wire AGENTS/CONTRIBUTING/README and an integration test. Co-authored-by: Cursor --- .gitignore | 1 + AGENTS.md | 9 ++++- CHANGELOG.md | 7 ++++ CONTRIBUTING.md | 9 +++++ README.md | 2 +- docs/architecture.md | 29 ++++++++++++++ scripts/mock-fm.sh | 67 ++++++++++++++++++++++++++++++++ test/mock-fm.integration.test.js | 46 ++++++++++++++++++++++ 8 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 docs/architecture.md create mode 100755 scripts/mock-fm.sh create mode 100644 test/mock-fm.integration.test.js diff --git a/.gitignore b/.gitignore index bcd2e7a..200699d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ coverage/ reports/ dist/ tmp/ +.lore/data/ diff --git a/AGENTS.md b/AGENTS.md index d9b9422..cf35c65 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,13 @@ The product benchmarks Apple's `fm` CLI, which only exists on macOS 27+ with Apple Intelligence. On the Linux cloud VM `fm` is absent, so `doctor`/`models`/the default benchmark fail with `spawn fm ENOENT`. -The CLI reads the `fm` binary from the `FM_BIN` env var (or `--fm-bin `). To exercise the full benchmark/report pipeline end-to-end here, point `FM_BIN` at a stub that emulates the subcommands `fm-bench` calls: `--help` (must print `Apple Foundation Models CLI` and/or a `MODELS` section), `available --model `, `quota-usage --model `, `token-count --quiet` (reads stdin, prints a token count), and `respond --model ` (reads the prompt on stdin, streams the answer to stdout). Example: `FM_BIN=/path/to/mock-fm node bin/fm-bench.js --models system --runs 2 --profile quick`. +The CLI reads the `fm` binary from the `FM_BIN` env var (or `--fm-bin `). Use the repo stub: + +```sh +chmod +x scripts/mock-fm.sh +FM_BIN="$(pwd)/scripts/mock-fm.sh" node bin/fm-bench.js --models system --runs 2 --profile quick +``` + +Architecture and module map: `docs/architecture.md`. Report schema: `docs/report-format.md`. These commands need **no** `fm` and work as-is: `legend`, `validate `, `export `, `compare `, `history `. diff --git a/CHANGELOG.md b/CHANGELOG.md index b5bc4aa..5cee34b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Unreleased + +- **`scripts/mock-fm.sh`**: documented stub for running the benchmark pipeline without Apple's `fm` (Linux/CI dev). +- **`docs/architecture.md`**: explicit module map for agents and contributors. +- **AGENTS.md** / **CONTRIBUTING.md**: mock-fm quick start and architecture cross-links. +- Integration test: mock-fm quick benchmark JSON smoke. + ## 0.6.1 - **Release** workflow: GitHub release bodies are generated from `CHANGELOG.md` (not empty auto-notes). Re-running Release on an existing tag updates release notes. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 14187e7..b77dcd5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,6 +12,15 @@ npm run lint Use macOS 27 or newer when testing against the real Apple `fm` command. Unit tests do not require `fm`. +On Linux or CI-like environments, run a quick benchmark with the mock `fm` stub: + +```sh +chmod +x scripts/mock-fm.sh +FM_BIN="$(pwd)/scripts/mock-fm.sh" node bin/fm-bench.js --models system --runs 2 --profile quick +``` + +See `docs/architecture.md` for how CLI, benchmark, and report layers fit together. + ## Pull Requests - Keep benchmark behavior deterministic where possible. diff --git a/README.md b/README.md index f0dd24a..4484caa 100644 --- a/README.md +++ b/README.md @@ -279,7 +279,7 @@ A live single-line progress indicator runs on stderr during interactive sessions `pcc` (Private Cloud Compute) availability depends on Apple's current eligibility. `fm-bench` shows it as skipped if `fm available --model pcc` reports unavailable. -See [docs/methodology.md](docs/methodology.md) for benchmark methodology and metric references. +See [docs/methodology.md](docs/methodology.md) for benchmark methodology and metric references. Module layout: [docs/architecture.md](docs/architecture.md). ## Development diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..9a5ace4 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,29 @@ +# fm-bench architecture + +Explicit overview for agents and contributors (not inferred from folder names alone). + +## Decision + +fm-bench is a **single-process Node.js CLI** (ESM, no runtime npm dependencies). Entry: `bin/fm-bench.js` → `src/cli.js` routes subcommands; benchmark work lives in `src/bench.js` with `src/fm.js` + `src/process.js` wrapping the external `fm` binary. + +## Layers + +| Layer | Paths | Responsibility | +|-------|--------|----------------| +| CLI | `src/cli.js`, `bin/fm-bench.js` | Args, doctor, compare, validate, export, history, legend, default benchmark | +| Benchmark | `src/bench.js`, `src/prompts.js`, `src/fm.js` | Model discovery, prompt suite, timed runs, streaming metrics | +| Reports | `src/report.js`, `src/schema.js`, `src/stats.js`, `src/export.js` | Schema v1 JSON, CSV flatten, HTML export, validation | +| Compare / history | `src/compare.js`, `src/history.js` | Regression diffs, trend tables | +| Presentation | `src/table.js`, `src/progress.js`, `src/ansi.js` | Terminal tables, progress, colors | + +## External dependency + +All inference goes through the **`fm` executable** (`FM_BIN` or `--fm-bin`). fm-bench does not embed models. Off macOS, use `scripts/mock-fm.sh` for pipeline tests. + +## Tests + +`node --test` under `test/` — parsers, schema, compare, CLI routing. No `fm` required for unit tests. + +## CI + +`.github/workflows/ci.yml` — `npm ci`, `npm test`, `npm run lint`, `publish:dry-run` on `macos-15`. \ No newline at end of file diff --git a/scripts/mock-fm.sh b/scripts/mock-fm.sh new file mode 100755 index 0000000..3b7ae8f --- /dev/null +++ b/scripts/mock-fm.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Minimal fm stub for fm-bench off macOS (see AGENTS.md). Usage: +# FM_BIN="$(pwd)/scripts/mock-fm.sh" node bin/fm-bench.js --models system --runs 2 --profile quick +set -euo pipefail + +cmd="${1:-}" +shift || true + +case "$cmd" in + --help|-h|'') + cat <<'EOF' +Apple Foundation Models CLI (mock) + +MODELS + system On-device Apple Foundation Model (default) + pcc Apple Foundation Model on Private Cloud Compute + +USAGE + fm respond --model + fm available --model + fm quota-usage --model + fm token-count --quiet +EOF + exit 0 + ;; + available) + model="" + while [[ $# -gt 0 ]]; do + case "$1" in + --model) model="$2"; shift 2 ;; + *) shift ;; + esac + done + echo "System model available: ${model:-system}" + exit 0 + ;; + quota-usage) + echo "quota: 1000 remaining" + exit 0 + ;; + token-count) + wc -c | awk '{print int($1/4)+1}' + exit 0 + ;; + respond) + model="system" + while [[ $# -gt 0 ]]; do + case "$1" in + --model) model="$2"; shift 2 ;; + *) shift ;; + esac + done + prompt="$(cat)" + words="$(echo "$prompt" | wc -w | tr -d ' ')" + i=0 + while [[ $i -lt $words ]]; do + echo -n "token " + i=$((i + 1)) + done + echo "ok from mock-fm model=$model" + exit 0 + ;; + *) + echo "mock-fm: unknown command $cmd" >&2 + exit 1 + ;; +esac \ No newline at end of file diff --git a/test/mock-fm.integration.test.js b/test/mock-fm.integration.test.js new file mode 100644 index 0000000..e175020 --- /dev/null +++ b/test/mock-fm.integration.test.js @@ -0,0 +1,46 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.dirname(path.dirname(fileURLToPath(import.meta.url))); +const mockFm = path.join(root, 'scripts', 'mock-fm.sh'); +const cli = path.join(root, 'bin', 'fm-bench.js'); + +function runBenchJson() { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [ + cli, + '--models', 'system', + '--runs', '1', + '--profile', 'quick', + '--format', 'json' + ], { + cwd: root, + env: { ...process.env, FM_BIN: mockFm }, + stdio: ['ignore', 'pipe', 'pipe'] + }); + let out = ''; + child.stdout.on('data', (c) => { out += c; }); + child.on('error', reject); + child.on('close', (code) => { + if (code !== 0) { + reject(new Error(`exit ${code}`)); + return; + } + try { + resolve(JSON.parse(out)); + } catch (err) { + reject(err); + } + }); + }); +} + +test('mock-fm runs quick benchmark and emits schema fields', async () => { + const report = await runBenchJson(); + assert.equal(report.tool, 'fm-bench'); + assert.ok(Array.isArray(report.results)); + assert.ok(report.results.length >= 1); +}); \ No newline at end of file