Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions scripts/eval-chunking.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,10 @@ function readArg(prefix, fallback) {
return arg ? arg.slice(prefix.length) : fallback
}

const executable = resolve(
'node_modules/.bin',
process.platform === 'win32' ? 'electron.cmd' : 'electron'
)
// Node 24 refuses to spawn a `.cmd`/`.bat` without `shell: true` (EINVAL), and the
// `.bin` entry is exactly that on Windows. Use the real binary the wrapper runs.
const { default: electronBinary } = await import('electron')
const executable = resolve(electronBinary)

if (!existsSync(executable)) {
console.error('[chunking] could not find the electron binary. Run `npm install` first.')
Expand Down
8 changes: 4 additions & 4 deletions scripts/eval-retrieval.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@ function readArg(prefix, fallback) {
return arg ? arg.slice(prefix.length) : fallback
}

const executable = resolve(
'node_modules/.bin',
process.platform === 'win32' ? 'electron.cmd' : 'electron'
)
// Node 24 refuses to spawn a `.cmd`/`.bat` without `shell: true` (EINVAL), and the
// `.bin` entry is exactly that on Windows. Use the real binary the wrapper runs.
const { default: electronBinary } = await import('electron')
const executable = resolve(electronBinary)

if (!existsSync(executable)) {
console.error('[retrieval] could not find the electron binary. Run `npm install` first.')
Expand Down
10 changes: 6 additions & 4 deletions scripts/eval.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@ import { resolve } from 'node:path'
const prepare = process.argv.includes('--prepare')
const flag = prepare ? '--eval-prepare' : '--eval-harness'

const executable = resolve(
'node_modules/.bin',
process.platform === 'win32' ? 'electron.cmd' : 'electron'
)
// The `.bin` entry is a shell wrapper (`electron.cmd` on Windows), and Node 24
// refuses to spawn `.cmd`/`.bat` without `shell: true` — it fails with EINVAL. The
// `electron` package exports the path to the real executable the wrapper runs, so
// spawning that directly works on every platform without a shell.
const { default: electronBinary } = await import('electron')
const executable = resolve(electronBinary)

if (!existsSync(executable)) {
console.error('[eval] could not find the electron binary. Run `npm install` first.')
Expand Down
20 changes: 16 additions & 4 deletions src/main/eval/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import { app } from 'electron'
import { mkdtempSync, rmSync, writeFileSync } from 'fs'
import { mkdir } from 'fs/promises'
import { join, relative, resolve } from 'path'
import { join, relative, resolve, sep } from 'path'
import { tmpdir } from 'os'
import { closeDatabase, getDatabase, initDatabase, initVectorStore, runMigrations } from '../db'
import { ConnectionManager } from '../models/ConnectionManager'
Expand Down Expand Up @@ -62,6 +62,18 @@ function readBoolOption(argv: readonly string[], prefix: string, fallback: boole
return value === 'true'
}

/**
* Repo-relative path with POSIX separators.
*
* `path.relative` returns backslashes on Windows, and the corpus label is committed
* in the baseline. Without this, a Windows run writes `eval\corpus` and the file
* stops being identical on every machine — which is the one property the committed
* report promises.
*/
function repoRelative(absolutePath: string): string {
return relative(process.cwd(), absolutePath).split(sep).join('/')
}

/** 检索策略(#77)。默认 dense,所以不带 flag 的 `npm run eval` 仍量的是生产默认。 */
function readRetrievalStrategy(argv: readonly string[]): RetrievalStrategy {
const raw = readOption(argv, '--eval-retrieval=', 'dense')
Expand Down Expand Up @@ -162,9 +174,9 @@ export async function runEvalCli(argv: readonly string[] = process.argv): Promis
const knowledgeService = new KnowledgeService(embeddingService)
const options: EvalHarnessOptions = {
corpusDir,
// Recorded in the report as a repo-relative path so the committed JSON is
// identical on every machine and checkout.
corpusLabel: relative(process.cwd(), corpusDir) || 'eval/corpus',
// Recorded in the report as a repo-relative, POSIX-separated path so the
// committed JSON is identical on every machine and checkout.
corpusLabel: repoRelative(corpusDir) || 'eval/corpus',
questionsPath,
baseline: readOption(argv, '--eval-baseline=', 'v1.5'),
topK: 10,
Expand Down
Loading