diff --git a/src/commands.ts b/src/commands.ts index 57b522b..628eed5 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -13,6 +13,7 @@ import { addTraceFile, getWorkspacePath, openTerminal, openTraceDirectoryExterna import { addTraceDiagnostics, clearTaceDiagnostics } from './traceDiagnostics' import { setStatusBarState } from './statusBar' import { afterWatches, projectPath, saveName, state, traceFiles, traceRunning } from './appState' +import { TRACE_RUN_METRICS_FILE, buildTraceRunMetrics, discoverTraceJsonFiles, summarizeTraceParse, writeTraceRunMetrics } from './traceRunMetrics' const readdir = promisify(readdirC) @@ -148,21 +149,50 @@ async function runTrace(args?: unknown[]) { setStatusBarState('traceError', false) log(`shell: ${process.env.SHELL}`) + const startedAtMs = Date.now() + const startedAt = new Date(startedAtMs).toISOString() const cmdProcess = spawn(fullCmd, [], { cwd: newProjectPath, shell: process.env.SHELL }) let err = '' + let out = '' cmdProcess.stderr.on('data', data => err += data.toString()) - cmdProcess.stdout.on('data', data => log(data.toString())) + cmdProcess.stdout.on('data', (data) => { + const chunk = data.toString() + out += chunk + log(chunk) + }) + + let metricsWritten = false + async function writeMetricsOnce(exitCode: number | null) { + if (metricsWritten) + return - cmdProcess.on('error', (error) => { + metricsWritten = true + await writeRunMetrics({ + command: fullCmd, + cwd: newProjectPath, + traceDir, + startedAt, + startedAtMs, + exitCode, + stdout: out, + stderr: err, + }) + } + + cmdProcess.on('error', async (error) => { + traceRunning.value = false + setStatusBarState('traceError', true) vscode.window.showErrorMessage(error.message) + await writeMetricsOnce(null) }) cmdProcess.on('exit', async (code) => { log('---- trace stderr -----') log(err) traceRunning.value = false + await writeMetricsOnce(code) if (code) { setStatusBarState('traceError', true) vscode.window.showErrorMessage('error running trace') @@ -174,6 +204,41 @@ async function runTrace(args?: unknown[]) { }) } +async function writeRunMetrics(input: { + command: string + cwd: string + traceDir: string + startedAt: string + startedAtMs: number + exitCode: number | null + stdout: string + stderr: string +}) { + try { + const traceJsonFiles = existsSync(input.traceDir) ? await discoverTraceJsonFiles(input.traceDir) : [] + const parseSummary = existsSync(input.traceDir) + ? await summarizeTraceParse(input.traceDir, traceJsonFiles) + : { status: 'missing' as const, topLevelWarning: 'Trace directory was not created.' } + + await writeTraceRunMetrics(input.traceDir, buildTraceRunMetrics({ + command: input.command, + cwd: input.cwd, + traceDir: input.traceDir, + startedAt: input.startedAt, + endedAt: new Date().toISOString(), + wallTimeMs: Date.now() - input.startedAtMs, + exitCode: input.exitCode, + stdout: input.stdout, + stderr: input.stderr, + traceJsonFiles, + parseSummary, + })) + } + catch (error) { + log(`error writing trace metrics: ${error instanceof Error ? error.message : `${error}`}`) + } +} + export async function sendTraceDir(traceDir: string) { try { if (!existsSync(traceDir)) { @@ -182,6 +247,9 @@ export async function sendTraceDir(traceDir: string) { const fileNames = await readdir(traceDir) for (const fileName of fileNames) { + if (fileName === TRACE_RUN_METRICS_FILE) + continue + sendTrace(traceDir, fileName) } } diff --git a/src/traceRunMetrics.ts b/src/traceRunMetrics.ts new file mode 100644 index 0000000..64b74a0 --- /dev/null +++ b/src/traceRunMetrics.ts @@ -0,0 +1,130 @@ +import { Buffer } from 'node:buffer' +import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' + +export const TRACE_RUN_METRICS_FILE = 'metrics.json' + +const DEFAULT_OUTPUT_LIMIT = 16_000 + +export interface OutputSummary { + bytes: number + truncated: boolean + text: string +} + +export interface TraceJsonFile { + fileName: string + bytes: number +} + +export type TraceParseStatus = 'missing' | 'ok' | 'warning' | 'error' + +export interface TraceParseSummary { + status: TraceParseStatus + topLevelWarning?: string +} + +export interface TraceRunMetricsInput { + command: string + cwd: string + traceDir: string + startedAt: string + endedAt: string + wallTimeMs: number + exitCode: number | null + stdout: string + stderr: string + traceJsonFiles: TraceJsonFile[] + parseSummary: TraceParseSummary +} + +export interface TraceRunMetrics { + version: 1 + command: string + cwd: string + traceDir: string + startedAt: string + endedAt: string + wallTimeMs: number + exitCode: number | null + stdout: OutputSummary + stderr: OutputSummary + traceJsonFiles: TraceJsonFile[] + parse: TraceParseSummary +} + +export function summarizeOutput(text: string, limit = DEFAULT_OUTPUT_LIMIT): OutputSummary { + const bytes = Buffer.byteLength(text) + if (text.length <= limit) { + return { bytes, truncated: false, text } + } + + return { + bytes, + truncated: true, + text: text.slice(text.length - limit), + } +} + +export function buildTraceRunMetrics(input: TraceRunMetricsInput): TraceRunMetrics { + return { + version: 1, + command: input.command, + cwd: input.cwd, + traceDir: input.traceDir, + startedAt: input.startedAt, + endedAt: input.endedAt, + wallTimeMs: input.wallTimeMs, + exitCode: input.exitCode, + stdout: summarizeOutput(input.stdout), + stderr: summarizeOutput(input.stderr), + traceJsonFiles: input.traceJsonFiles, + parse: input.parseSummary, + } +} + +export async function discoverTraceJsonFiles(traceDir: string): Promise { + const entries = await readdir(traceDir, { withFileTypes: true }) + const files = await Promise.all( + entries + .filter(entry => entry.isFile() && entry.name.endsWith('.json') && entry.name !== TRACE_RUN_METRICS_FILE) + .map(async (entry) => { + const fileStat = await stat(join(traceDir, entry.name)) + return { + fileName: entry.name, + bytes: fileStat.size, + } + }), + ) + + return files.sort((a, b) => a.fileName.localeCompare(b.fileName)) +} + +export async function summarizeTraceParse(traceDir: string, traceJsonFiles: TraceJsonFile[]): Promise { + if (traceJsonFiles.length === 0) { + return { + status: 'missing', + topLevelWarning: 'No trace JSON files were found.', + } + } + + for (const file of traceJsonFiles) { + try { + JSON.parse(await readFile(join(traceDir, file.fileName), 'utf8')) + } + catch (error) { + return { + status: 'error', + topLevelWarning: `${file.fileName}: ${error instanceof Error ? error.message : `${error}`}`, + } + } + } + + return { status: 'ok' } +} + +export async function writeTraceRunMetrics(traceDir: string, metrics: TraceRunMetrics): Promise { + const fileName = join(traceDir, TRACE_RUN_METRICS_FILE) + await mkdir(dirname(fileName), { recursive: true }) + await writeFile(fileName, `${JSON.stringify(metrics, null, 2)}\n`, 'utf8') +} diff --git a/test/traceRunMetrics.test.ts b/test/traceRunMetrics.test.ts new file mode 100644 index 0000000..40c7330 --- /dev/null +++ b/test/traceRunMetrics.test.ts @@ -0,0 +1,106 @@ +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, describe, expect, it } from 'vitest' +import { + TRACE_RUN_METRICS_FILE, + buildTraceRunMetrics, + discoverTraceJsonFiles, + summarizeOutput, + summarizeTraceParse, + writeTraceRunMetrics, +} from '../src/traceRunMetrics' + +const tempDirs: string[] = [] + +async function makeTempDir() { + const dir = await mkdtemp(join(tmpdir(), 'tsperf-trace-metrics-')) + tempDirs.push(dir) + return dir +} + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map(dir => rm(dir, { recursive: true, force: true }))) +}) + +describe('trace run metrics', () => { + it('summarizes output and keeps the tail when truncated', () => { + expect(summarizeOutput('hello', 10)).toEqual({ + bytes: 5, + truncated: false, + text: 'hello', + }) + + expect(summarizeOutput('0123456789', 4)).toEqual({ + bytes: 10, + truncated: true, + text: '6789', + }) + }) + + it('discovers trace JSON files without including metrics.json', async () => { + const traceDir = await makeTempDir() + await writeFile(join(traceDir, 'types.json'), '[]') + await writeFile(join(traceDir, 'trace.json'), '[]') + await writeFile(join(traceDir, TRACE_RUN_METRICS_FILE), '{}') + await writeFile(join(traceDir, 'notes.txt'), 'ignored') + + await expect(discoverTraceJsonFiles(traceDir)).resolves.toEqual([ + { fileName: 'trace.json', bytes: 2 }, + { fileName: 'types.json', bytes: 2 }, + ]) + }) + + it('reports parse status for missing, valid, and invalid trace files', async () => { + const traceDir = await makeTempDir() + + await expect(summarizeTraceParse(traceDir, [])).resolves.toEqual({ + status: 'missing', + topLevelWarning: 'No trace JSON files were found.', + }) + + await writeFile(join(traceDir, 'trace.json'), '[]') + const files = await discoverTraceJsonFiles(traceDir) + await expect(summarizeTraceParse(traceDir, files)).resolves.toEqual({ status: 'ok' }) + + await writeFile(join(traceDir, 'types.json'), '{') + const invalidFiles = await discoverTraceJsonFiles(traceDir) + const invalidSummary = await summarizeTraceParse(traceDir, invalidFiles) + expect(invalidSummary.status).toBe('error') + expect(invalidSummary.topLevelWarning).toContain('types.json:') + }) + + it('builds and writes a stable metrics artifact', async () => { + const traceDir = await makeTempDir() + const metrics = buildTraceRunMetrics({ + command: 'npx tsc --noEmit --generateTrace trace', + cwd: '/repo', + traceDir, + startedAt: '2026-05-18T00:00:00.000Z', + endedAt: '2026-05-18T00:00:01.250Z', + wallTimeMs: 1250, + exitCode: 0, + stdout: 'ok', + stderr: '', + traceJsonFiles: [{ fileName: 'trace.json', bytes: 2 }], + parseSummary: { status: 'ok' }, + }) + + expect(metrics).toMatchObject({ + version: 1, + command: 'npx tsc --noEmit --generateTrace trace', + cwd: '/repo', + traceDir, + wallTimeMs: 1250, + exitCode: 0, + stdout: { bytes: 2, truncated: false, text: 'ok' }, + stderr: { bytes: 0, truncated: false, text: '' }, + traceJsonFiles: [{ fileName: 'trace.json', bytes: 2 }], + parse: { status: 'ok' }, + }) + + await writeTraceRunMetrics(traceDir, metrics) + await expect(readFile(join(traceDir, TRACE_RUN_METRICS_FILE), 'utf8')) + .resolves.toBe(`${JSON.stringify(metrics, null, 2)}\n`) + }) +})