From 8e5a4bf809e9440f4256d31a93d19d2d3d7ae9d5 Mon Sep 17 00:00:00 2001 From: Devin Oldenburg Date: Fri, 17 Jul 2026 16:47:55 +0200 Subject: [PATCH 1/2] Enforce macOS 27+ before launching fm benchmarks Refuse run/models/doctor on older macOS or non-darwin platforms with an exit code 2 message that names the detected version and the latest supported release. Offline report commands stay available everywhere. --- src/cli.js | 38 ++++++++++++++++++++----- src/macos.js | 71 ++++++++++++++++++++++++++++++++++++++++++++++ test/cli.test.js | 48 ++++++++++++++++++++++++++++++- test/macos.test.js | 67 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 216 insertions(+), 8 deletions(-) create mode 100644 src/macos.js create mode 100644 test/macos.test.js diff --git a/src/cli.js b/src/cli.js index cfd466e..e518ddd 100644 --- a/src/cli.js +++ b/src/cli.js @@ -5,6 +5,7 @@ import { diffReports, renderCompareReport } from './compare.js'; import { renderHtmlReport } from './export.js'; import { validateReport } from './schema.js'; import { loadHistory, renderHistoryReport } from './history.js'; +import { detectMacosVersion, evaluateMacosSupport, formatMacosRequirementError, MIN_SUPPORTED_MACOS, parseMacosVersion } from './macos.js'; import { runProcess } from './process.js'; import { createProgress } from './progress.js'; import { flattenResults, toCsv, writeReport } from './report.js'; @@ -14,7 +15,7 @@ import { legendEntries, renderBenchmarkReport, renderLatencyHistogram, renderLeg const require = createRequire(import.meta.url); const packageJson = require('../package.json'); -export async function runCli(argv = process.argv.slice(2)) { +export async function runCli(argv = process.argv.slice(2), env = {}) { const parsed = parseArgs(argv); if (parsed.help) { @@ -27,6 +28,8 @@ export async function runCli(argv = process.argv.slice(2)) { return; } + await assertSupportedMacos(parsed, env); + if (parsed.command === 'legend') { if (parsed.format === 'json') { console.log(JSON.stringify(legendEntries(), null, 2)); @@ -153,6 +156,24 @@ export async function runCli(argv = process.argv.slice(2)) { } } +// Offline commands only read local report files; they do not launch `fm`, so +// they stay usable on any platform. Benchmarks and `fm` probes are gated. +const OFFLINE_COMMANDS = new Set(['compare', 'history', 'validate', 'export', 'legend']); + +async function assertSupportedMacos(parsed, env = {}) { + if (OFFLINE_COMMANDS.has(parsed.command)) return; + + const evaluation = evaluateMacosSupport( + env.platform ?? process.platform, + parseMacosVersion(await detectMacosVersion(env)) + ); + if (evaluation.supported) return; + + const error = new Error(formatMacosRequirementError(evaluation)); + error.exitCode = 2; + throw error; +} + function evaluateCi(payload) { const reasons = []; const totalFailed = payload.summary.reduce((sum, item) => sum + item.failures, 0); @@ -554,11 +575,14 @@ async function runDoctor(options) { checks.push(['node', process.version, true]); checks.push(['platform', `${process.platform}/${process.arch}`, process.platform === 'darwin']); - const swVers = await runProcess('sw_vers', [], { timeoutMs: 5_000 }); - const macOS = swVers.stdout || swVers.stderr; - const versionMatch = macOS.match(/ProductVersion:\s*([0-9.]+)/); - const major = versionMatch ? Number.parseInt(versionMatch[1].split('.')[0], 10) : null; - checks.push(['macOS', versionMatch?.[1] || 'unknown', major == null || major >= 27]); + const macOS = await detectMacosVersion(); + const parsedVersion = parseMacosVersion(macOS); + const support = evaluateMacosSupport(process.platform, parsedVersion); + checks.push(['macOS', parsedVersion?.version || 'unknown', support.supported]); + if (!support.supported) { + checks.push(['macOS support', support.reason, false]); + checks.push(['latest supported', support.latestSupported, false]); + } const hwModel = await runProcess('sysctl', ['-n', 'hw.model'], { timeoutMs: 3_000 }); const hwModelStr = (hwModel.stdout || '').trim(); @@ -669,7 +693,7 @@ function resolveProgress(parsed) { function helpText() { return `fm-bench ${packageJson.version} -Dynamic benchmark CLI for Apple's fm command on macOS 27+. +Dynamic benchmark CLI for Apple's fm command on macOS ${MIN_SUPPORTED_MACOS}+. Usage: fm-bench [run] [options] diff --git a/src/macos.js b/src/macos.js new file mode 100644 index 0000000..c285f9f --- /dev/null +++ b/src/macos.js @@ -0,0 +1,71 @@ +import { runProcess } from './process.js'; + +// macOS releases that ship Apple's `fm` CLI. Older macOS versions cannot run +// fm-bench at all because the benchmarked binary does not exist there. +export const SUPPORTED_MACOS_MAJOR = 27; + +export const MIN_SUPPORTED_MACOS = `${SUPPORTED_MACOS_MAJOR}.0`; + +export const MACOS_REQUIREMENT_MESSAGE = + `fm-bench requires macOS ${SUPPORTED_MACOS_MAJOR} or newer (Apple's fm CLI is preinstalled there).`; + +export function parseMacosVersion(text = '') { + const match = String(text).match(/ProductVersion:\s*(\d+)(?:\.(\d+))?(?:\.(\d+))?/); + if (!match) return null; + return { + major: Number.parseInt(match[1], 10), + minor: match[2] != null ? Number.parseInt(match[2], 10) : 0, + patch: match[3] != null ? Number.parseInt(match[3], 10) : 0, + version: [match[1], match[2], match[3]].filter((part) => part != null).join('.') + }; +} + +export function isMacosSupported(parsed) { + return parsed != null && Number.isInteger(parsed.major) && parsed.major >= SUPPORTED_MACOS_MAJOR; +} + +export function evaluateMacosSupport(platform, parsed) { + if (platform !== 'darwin') { + return { + supported: false, + reason: `fm-bench only runs on macOS with Apple's fm CLI (detected platform: ${platform}).`, + latestSupported: `macOS ${MIN_SUPPORTED_MACOS} or newer` + }; + } + if (parsed == null) { + return { + supported: true, + warnOnly: true, + reason: 'could not determine the macOS version', + latestSupported: `macOS ${MIN_SUPPORTED_MACOS} or newer` + }; + } + if (isMacosSupported(parsed)) { + return { + supported: true, + reason: null, + latestSupported: `macOS ${MIN_SUPPORTED_MACOS} or newer` + }; + } + return { + supported: false, + reason: `detected macOS ${parsed.version}, but ${MACOS_REQUIREMENT_MESSAGE}`, + latestSupported: `macOS ${MIN_SUPPORTED_MACOS} or newer` + }; +} + +export function formatMacosRequirementError({ reason, latestSupported }) { + return [ + `unsupported macOS: ${reason}`, + `Latest supported: ${latestSupported} (fm is not available on older macOS releases).` + ].join('\n'); +} + +export async function detectMacosVersion(options = {}) { + const timeoutMs = options.timeoutMs ?? 5_000; + if (typeof options.swVers === 'function') { + return options.swVers(); + } + const result = await runProcess('sw_vers', [], { timeoutMs }); + return result.stdout || result.stderr; +} diff --git a/test/cli.test.js b/test/cli.test.js index a7eacd1..5b4a9d5 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -1,6 +1,6 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { parseArgs } from '../src/cli.js'; +import { parseArgs, runCli } from '../src/cli.js'; test('parseArgs supports repeated models and prompts', () => { const args = parseArgs(['--models', 'system,pcc', '--model', 'future', '--prompt', 'one', '--prompt', 'two']); @@ -58,3 +58,49 @@ test('parseArgs supports validate export compare strict and export-html', () => assert.equal(run.exportHtml, true); assert.equal(run.strictCompare, true); }); + +const SW_VERS_26 = 'ProductName:\tmacOS\nProductVersion:\t26.1\nBuildVersion:\t25B78\n'; +const swVers26 = async () => SW_VERS_26; + +test('runCli blocks benchmarks on macOS older than 27 and names the latest supported version', async () => { + await assert.rejects( + () => runCli(['--profile', 'quick'], { platform: 'darwin', swVers: swVers26 }), + (error) => { + assert.match(error.message, /unsupported macOS/); + assert.match(error.message, /detected macOS 26\.1/); + assert.match(error.message, /Latest supported: macOS 27\.0 or newer/); + assert.equal(error.exitCode, 2); + return true; + } + ); +}); + +test('runCli blocks models and doctor on unsupported macOS', async () => { + await assert.rejects( + () => runCli(['models'], { platform: 'darwin', swVers: swVers26 }), + (error) => { + assert.equal(error.exitCode, 2); + assert.match(error.message, /Latest supported/); + return true; + } + ); + await assert.rejects( + () => runCli(['doctor'], { platform: 'darwin', swVers: swVers26 }), + (error) => error.exitCode === 2 + ); +}); + +test('runCli blocks benchmarks on non-macOS platforms', async () => { + await assert.rejects( + () => runCli(['--profile', 'quick'], { platform: 'linux', swVers: swVers26 }), + (error) => { + assert.match(error.message, /only runs on macOS/); + assert.equal(error.exitCode, 2); + return true; + } + ); +}); + +test('runCli keeps offline commands usable on unsupported macOS', async () => { + await assert.doesNotReject(() => runCli(['legend', '--json'], { platform: 'darwin', swVers: swVers26 })); +}); diff --git a/test/macos.test.js b/test/macos.test.js new file mode 100644 index 0000000..64566bf --- /dev/null +++ b/test/macos.test.js @@ -0,0 +1,67 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { evaluateMacosSupport, formatMacosRequirementError, isMacosSupported, MIN_SUPPORTED_MACOS, parseMacosVersion, SUPPORTED_MACOS_MAJOR } from '../src/macos.js'; + +const SW_VERS_27 = 'ProductName:\t\tmacOS\nProductVersion:\t\t27.0\nBuildVersion:\t\t26A5378n\n'; +const SW_VERS_26 = 'ProductName:\t\tmacOS\nProductVersion:\t\t26.1\nBuildVersion:\t\t25B78\n'; + +test('parseMacosVersion reads sw_vers output', () => { + assert.deepEqual(parseMacosVersion(SW_VERS_27), { major: 27, minor: 0, patch: 0, version: '27.0' }); + assert.deepEqual(parseMacosVersion(SW_VERS_26), { major: 26, minor: 1, patch: 0, version: '26.1' }); +}); + +test('parseMacosVersion reads bare version strings and two-part versions', () => { + assert.deepEqual(parseMacosVersion('ProductVersion: 27'), { major: 27, minor: 0, patch: 0, version: '27' }); + assert.deepEqual(parseMacosVersion('ProductVersion: 27.1.2'), { major: 27, minor: 1, patch: 2, version: '27.1.2' }); +}); + +test('parseMacosVersion returns null for unknown output', () => { + assert.equal(parseMacosVersion(''), null); + assert.equal(parseMacosVersion('hello world'), null); + assert.equal(parseMacosVersion(null), null); +}); + +test('isMacosSupported accepts macOS 27 and newer only', () => { + assert.equal(isMacosSupported(parseMacosVersion(SW_VERS_27)), true); + assert.equal(isMacosSupported(parseMacosVersion('ProductVersion: 28.0')), true); + assert.equal(isMacosSupported(parseMacosVersion(SW_VERS_26)), false); + assert.equal(isMacosSupported(parseMacosVersion('ProductVersion: 15.5')), false); + assert.equal(isMacosSupported(null), false); +}); + +test('evaluateMacosSupport rejects older macOS with latest supported version', () => { + const result = evaluateMacosSupport('darwin', parseMacosVersion(SW_VERS_26)); + assert.equal(result.supported, false); + assert.match(result.reason, /macOS 26\.1/); + assert.match(result.reason, /macOS 27 or newer/); + assert.equal(result.latestSupported, `macOS ${MIN_SUPPORTED_MACOS} or newer`); +}); + +test('evaluateMacosSupport accepts macOS 27+', () => { + assert.equal(evaluateMacosSupport('darwin', parseMacosVersion(SW_VERS_27)).supported, true); + assert.equal(evaluateMacosSupport('darwin', parseMacosVersion('ProductVersion: 29.0')).supported, true); +}); + +test('evaluateMacosSupport rejects non-macOS platforms', () => { + const result = evaluateMacosSupport('linux', null); + assert.equal(result.supported, false); + assert.match(result.reason, /linux/); +}); + +test('evaluateMacosSupport warns but does not block unknown versions', () => { + const result = evaluateMacosSupport('darwin', null); + assert.equal(result.supported, true); + assert.equal(result.warnOnly, true); +}); + +test('formatMacosRequirementError names the latest supported macOS', () => { + const evaluation = evaluateMacosSupport('darwin', parseMacosVersion(SW_VERS_26)); + const message = formatMacosRequirementError(evaluation); + assert.match(message, /unsupported macOS/); + assert.match(message, /Latest supported: macOS 27\.0 or newer/); +}); + +test('supported major is 27', () => { + assert.equal(SUPPORTED_MACOS_MAJOR, 27); + assert.equal(MIN_SUPPORTED_MACOS, '27.0'); +}); From a408cc817f7cab75d3117c093c063eb6494f9c47 Mon Sep 17 00:00:00 2001 From: Devin Oldenburg Date: Fri, 17 Jul 2026 16:48:41 +0200 Subject: [PATCH 2/2] Let doctor diagnose unsupported macOS instead of hard-exiting Keep hard gates for run/models. Doctor still reports the detected version, latest supported macOS, and a failed fm probe when the host is too old. --- src/cli.js | 25 ++++++++++++++++--------- test/cli.test.js | 6 +----- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/cli.js b/src/cli.js index e518ddd..2bfbb2d 100644 --- a/src/cli.js +++ b/src/cli.js @@ -156,12 +156,13 @@ export async function runCli(argv = process.argv.slice(2), env = {}) { } } -// Offline commands only read local report files; they do not launch `fm`, so -// they stay usable on any platform. Benchmarks and `fm` probes are gated. -const OFFLINE_COMMANDS = new Set(['compare', 'history', 'validate', 'export', 'legend']); +// Offline report tools stay usable anywhere. `doctor` is also allowed through so +// it can print the detected version and the latest supported macOS when the host +// is too old. Only commands that launch `fm` benchmarks are hard-gated. +const SKIP_MACOS_GATE = new Set(['compare', 'history', 'validate', 'export', 'legend', 'doctor']); async function assertSupportedMacos(parsed, env = {}) { - if (OFFLINE_COMMANDS.has(parsed.command)) return; + if (SKIP_MACOS_GATE.has(parsed.command)) return; const evaluation = evaluateMacosSupport( env.platform ?? process.platform, @@ -619,17 +620,23 @@ async function runDoctor(options) { checks.push(['battery', `${pct} (${source})`, ok]); } - const inspection = await inspectModels(options); - checks.push(['fm', inspection.fmBin, inspection.models.length > 0]); - for (const model of inspection.models) { - checks.push([`model:${model.name}`, model.available ? 'available' : model.reason || 'unavailable', model.available]); + let models = []; + try { + const inspection = await inspectModels(options); + models = inspection.models; + checks.push(['fm', inspection.fmBin, inspection.models.length > 0]); + for (const model of inspection.models) { + checks.push([`model:${model.name}`, model.available ? 'available' : model.reason || 'unavailable', model.available]); + } + } catch (error) { + checks.push(['fm', error.message || String(error), false]); } const lines = checks.map(([name, detail, ok]) => `${ok ? 'ok ' : 'warn'} ${name.padEnd(16)} ${String(detail).replace(/\s+/g, ' ').trim()}`); console.log(lines.join('\n')); if (options.out) { - await fs.writeFile(options.out, `${JSON.stringify({ checks, models: inspection.models }, null, 2)}\n`, 'utf8'); + await fs.writeFile(options.out, `${JSON.stringify({ checks, models }, null, 2)}\n`, 'utf8'); } } diff --git a/test/cli.test.js b/test/cli.test.js index 5b4a9d5..08f4e35 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -75,7 +75,7 @@ test('runCli blocks benchmarks on macOS older than 27 and names the latest suppo ); }); -test('runCli blocks models and doctor on unsupported macOS', async () => { +test('runCli blocks models on unsupported macOS', async () => { await assert.rejects( () => runCli(['models'], { platform: 'darwin', swVers: swVers26 }), (error) => { @@ -84,10 +84,6 @@ test('runCli blocks models and doctor on unsupported macOS', async () => { return true; } ); - await assert.rejects( - () => runCli(['doctor'], { platform: 'darwin', swVers: swVers26 }), - (error) => error.exitCode === 2 - ); }); test('runCli blocks benchmarks on non-macOS platforms', async () => {