From 25e1ada24ee07e1d3c36af88c65f7b52aad2b8b3 Mon Sep 17 00:00:00 2001 From: MrSibe Date: Wed, 30 Sep 2026 16:40:03 +0800 Subject: [PATCH] fix(eval): run the harness on Windows and keep the baseline POSIX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gates the v1.5 eval path needs to be trustworthy on every platform. **`npm run eval` could not start on Windows.** The three scripts resolve `node_modules/.bin/electron.cmd` and spawn it. Node 24 refuses to spawn a `.cmd`/`.bat` without `shell: true` and fails with `EINVAL`, so the harness was unrunnable on the platform this project is developed on. The `electron` package exports the path to the real executable the wrapper runs, which spawns directly on every platform and needs no shell. **A Windows run rewrote the committed baseline.** `path.relative` returns backslashes on Windows, so `config.corpus` was written as `eval\corpus` instead of `eval/corpus`. The file's whole contract is that it is identical on every machine — the CI determinism check diffs it — and a Windows run silently broke that. The label is now normalised to POSIX separators. Verified on Windows with the pinned model: `npm run eval` runs and leaves `docs/eval/baseline-v1.5.json` byte-identical to the committed file. --- scripts/eval-chunking.mjs | 8 ++++---- scripts/eval-retrieval.mjs | 8 ++++---- scripts/eval.mjs | 10 ++++++---- src/main/eval/run.ts | 20 ++++++++++++++++---- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/scripts/eval-chunking.mjs b/scripts/eval-chunking.mjs index ecc0d4a..70f4e45 100644 --- a/scripts/eval-chunking.mjs +++ b/scripts/eval-chunking.mjs @@ -63,10 +63,10 @@ function readArg(prefix, fallback) { return arg ? arg.slice(prefix.length) : fallback } -const executable = resolve( - 'node_modules/.bin', - process.platform === 'win32' ? 'electron.cmd' : 'electron' -) +// Node 24 refuses to spawn a `.cmd`/`.bat` without `shell: true` (EINVAL), and the +// `.bin` entry is exactly that on Windows. Use the real binary the wrapper runs. +const { default: electronBinary } = await import('electron') +const executable = resolve(electronBinary) if (!existsSync(executable)) { console.error('[chunking] could not find the electron binary. Run `npm install` first.') diff --git a/scripts/eval-retrieval.mjs b/scripts/eval-retrieval.mjs index d573e3b..bccc605 100644 --- a/scripts/eval-retrieval.mjs +++ b/scripts/eval-retrieval.mjs @@ -39,10 +39,10 @@ function readArg(prefix, fallback) { return arg ? arg.slice(prefix.length) : fallback } -const executable = resolve( - 'node_modules/.bin', - process.platform === 'win32' ? 'electron.cmd' : 'electron' -) +// Node 24 refuses to spawn a `.cmd`/`.bat` without `shell: true` (EINVAL), and the +// `.bin` entry is exactly that on Windows. Use the real binary the wrapper runs. +const { default: electronBinary } = await import('electron') +const executable = resolve(electronBinary) if (!existsSync(executable)) { console.error('[retrieval] could not find the electron binary. Run `npm install` first.') diff --git a/scripts/eval.mjs b/scripts/eval.mjs index 0f4d304..8b61229 100644 --- a/scripts/eval.mjs +++ b/scripts/eval.mjs @@ -18,10 +18,12 @@ import { resolve } from 'node:path' const prepare = process.argv.includes('--prepare') const flag = prepare ? '--eval-prepare' : '--eval-harness' -const executable = resolve( - 'node_modules/.bin', - process.platform === 'win32' ? 'electron.cmd' : 'electron' -) +// The `.bin` entry is a shell wrapper (`electron.cmd` on Windows), and Node 24 +// refuses to spawn `.cmd`/`.bat` without `shell: true` — it fails with EINVAL. The +// `electron` package exports the path to the real executable the wrapper runs, so +// spawning that directly works on every platform without a shell. +const { default: electronBinary } = await import('electron') +const executable = resolve(electronBinary) if (!existsSync(executable)) { console.error('[eval] could not find the electron binary. Run `npm install` first.') diff --git a/src/main/eval/run.ts b/src/main/eval/run.ts index 10857a6..58becb8 100644 --- a/src/main/eval/run.ts +++ b/src/main/eval/run.ts @@ -13,7 +13,7 @@ import { app } from 'electron' import { mkdtempSync, rmSync, writeFileSync } from 'fs' import { mkdir } from 'fs/promises' -import { join, relative, resolve } from 'path' +import { join, relative, resolve, sep } from 'path' import { tmpdir } from 'os' import { closeDatabase, getDatabase, initDatabase, initVectorStore, runMigrations } from '../db' import { ConnectionManager } from '../models/ConnectionManager' @@ -62,6 +62,18 @@ function readBoolOption(argv: readonly string[], prefix: string, fallback: boole return value === 'true' } +/** + * Repo-relative path with POSIX separators. + * + * `path.relative` returns backslashes on Windows, and the corpus label is committed + * in the baseline. Without this, a Windows run writes `eval\corpus` and the file + * stops being identical on every machine — which is the one property the committed + * report promises. + */ +function repoRelative(absolutePath: string): string { + return relative(process.cwd(), absolutePath).split(sep).join('/') +} + /** 检索策略（#77）。默认 dense，所以不带 flag 的 `npm run eval` 仍量的是生产默认。 */ function readRetrievalStrategy(argv: readonly string[]): RetrievalStrategy { const raw = readOption(argv, '--eval-retrieval=', 'dense') @@ -162,9 +174,9 @@ export async function runEvalCli(argv: readonly string[] = process.argv): Promis const knowledgeService = new KnowledgeService(embeddingService) const options: EvalHarnessOptions = { corpusDir, - // Recorded in the report as a repo-relative path so the committed JSON is - // identical on every machine and checkout. - corpusLabel: relative(process.cwd(), corpusDir) || 'eval/corpus', + // Recorded in the report as a repo-relative, POSIX-separated path so the + // committed JSON is identical on every machine and checkout. + corpusLabel: repoRelative(corpusDir) || 'eval/corpus', questionsPath, baseline: readOption(argv, '--eval-baseline=', 'v1.5'), topK: 10,