feat: write trace run metrics artifact

src/commands.ts

@@ -13,6 +13,7 @@ import { addTraceFile, getWorkspacePath, openTerminal, openTraceDirectoryExterna
 import { addTraceDiagnostics, clearTaceDiagnostics } from './traceDiagnostics'
 import { setStatusBarState } from './statusBar'
 import { afterWatches, projectPath, saveName, state, traceFiles, traceRunning } from './appState'
+import { TRACE_RUN_METRICS_FILE, buildTraceRunMetrics, discoverTraceJsonFiles, summarizeTraceParse, writeTraceRunMetrics } from './traceRunMetrics'
 
 const readdir = promisify(readdirC)
 
@@ -148,21 +149,50 @@ async function runTrace(args?: unknown[]) {
     setStatusBarState('traceError', false)
 
     log(`shell: ${process.env.SHELL}`)
+    const startedAtMs = Date.now()
+    const startedAt = new Date(startedAtMs).toISOString()
     const cmdProcess = spawn(fullCmd, [], { cwd: newProjectPath, shell: process.env.SHELL })
 
     let err = ''
+    let out = ''
     cmdProcess.stderr.on('data', data => err += data.toString())
 
-    cmdProcess.stdout.on('data', data => log(data.toString()))
+    cmdProcess.stdout.on('data', (data) => {
+      const chunk = data.toString()
+      out += chunk
+      log(chunk)
+    })
+
+    let metricsWritten = false
+    async function writeMetricsOnce(exitCode: number | null) {
+      if (metricsWritten)
+        return
 
-    cmdProcess.on('error', (error) => {
+      metricsWritten = true
+      await writeRunMetrics({
+        command: fullCmd,
+        cwd: newProjectPath,
+        traceDir,
+        startedAt,
+        startedAtMs,
+        exitCode,
+        stdout: out,
+        stderr: err,
+      })
+    }
+
+    cmdProcess.on('error', async (error) => {
+      traceRunning.value = false
+      setStatusBarState('traceError', true)
       vscode.window.showErrorMessage(error.message)
+      await writeMetricsOnce(null)
     })
 
     cmdProcess.on('exit', async (code) => {
       log('---- trace stderr -----')
       log(err)
       traceRunning.value = false
+      await writeMetricsOnce(code)
       if (code) {
         setStatusBarState('traceError', true)
         vscode.window.showErrorMessage('error running trace')
@@ -174,6 +204,41 @@ async function runTrace(args?: unknown[]) {
   })
 }
 
+async function writeRunMetrics(input: {
+  command: string
+  cwd: string
+  traceDir: string
+  startedAt: string
+  startedAtMs: number
+  exitCode: number | null
+  stdout: string
+  stderr: string
+}) {
+  try {
+    const traceJsonFiles = existsSync(input.traceDir) ? await discoverTraceJsonFiles(input.traceDir) : []
+    const parseSummary = existsSync(input.traceDir)
+      ? await summarizeTraceParse(input.traceDir, traceJsonFiles)
+      : { status: 'missing' as const, topLevelWarning: 'Trace directory was not created.' }
+
+    await writeTraceRunMetrics(input.traceDir, buildTraceRunMetrics({
+      command: input.command,
+      cwd: input.cwd,
+      traceDir: input.traceDir,
+      startedAt: input.startedAt,
+      endedAt: new Date().toISOString(),
+      wallTimeMs: Date.now() - input.startedAtMs,
+      exitCode: input.exitCode,
+      stdout: input.stdout,
+      stderr: input.stderr,
+      traceJsonFiles,
+      parseSummary,
+    }))
+  }
+  catch (error) {
+    log(`error writing trace metrics: ${error instanceof Error ? error.message : `${error}`}`)
+  }
+}
+
 export async function sendTraceDir(traceDir: string) {
   try {
     if (!existsSync(traceDir)) {
@@ -182,6 +247,9 @@ export async function sendTraceDir(traceDir: string) {
 
     const fileNames = await readdir(traceDir)
     for (const fileName of fileNames) {
+      if (fileName === TRACE_RUN_METRICS_FILE)
+        continue
+
       sendTrace(traceDir, fileName)
     }
   }

src/traceRunMetrics.ts

@@ -0,0 +1,130 @@
+import { Buffer } from 'node:buffer'
+import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+
+export const TRACE_RUN_METRICS_FILE = 'metrics.json'
+
+const DEFAULT_OUTPUT_LIMIT = 16_000
+
+export interface OutputSummary {
+  bytes: number
+  truncated: boolean
+  text: string
+}
+
+export interface TraceJsonFile {
+  fileName: string
+  bytes: number
+}
+
+export type TraceParseStatus = 'missing' | 'ok' | 'warning' | 'error'
+
+export interface TraceParseSummary {
+  status: TraceParseStatus
+  topLevelWarning?: string
+}
+
+export interface TraceRunMetricsInput {
+  command: string
+  cwd: string
+  traceDir: string
+  startedAt: string
+  endedAt: string
+  wallTimeMs: number
+  exitCode: number | null
+  stdout: string
+  stderr: string
+  traceJsonFiles: TraceJsonFile[]
+  parseSummary: TraceParseSummary
+}
+
+export interface TraceRunMetrics {
+  version: 1
+  command: string
+  cwd: string
+  traceDir: string
+  startedAt: string
+  endedAt: string
+  wallTimeMs: number
+  exitCode: number | null
+  stdout: OutputSummary
+  stderr: OutputSummary
+  traceJsonFiles: TraceJsonFile[]
+  parse: TraceParseSummary
+}
+
+export function summarizeOutput(text: string, limit = DEFAULT_OUTPUT_LIMIT): OutputSummary {
+  const bytes = Buffer.byteLength(text)
+  if (text.length <= limit) {
+    return { bytes, truncated: false, text }
+  }
+
+  return {
+    bytes,
+    truncated: true,
+    text: text.slice(text.length - limit),
+  }
+}
+
+export function buildTraceRunMetrics(input: TraceRunMetricsInput): TraceRunMetrics {
+  return {
+    version: 1,
+    command: input.command,
+    cwd: input.cwd,
+    traceDir: input.traceDir,
+    startedAt: input.startedAt,
+    endedAt: input.endedAt,
+    wallTimeMs: input.wallTimeMs,
+    exitCode: input.exitCode,
+    stdout: summarizeOutput(input.stdout),
+    stderr: summarizeOutput(input.stderr),
+    traceJsonFiles: input.traceJsonFiles,
+    parse: input.parseSummary,
+  }
+}
+
+export async function discoverTraceJsonFiles(traceDir: string): Promise<TraceJsonFile[]> {
+  const entries = await readdir(traceDir, { withFileTypes: true })
+  const files = await Promise.all(
+    entries
+      .filter(entry => entry.isFile() && entry.name.endsWith('.json') && entry.name !== TRACE_RUN_METRICS_FILE)
+      .map(async (entry) => {
+        const fileStat = await stat(join(traceDir, entry.name))
+        return {
+          fileName: entry.name,
+          bytes: fileStat.size,
+        }
+      }),
+  )
+
+  return files.sort((a, b) => a.fileName.localeCompare(b.fileName))
+}
+
+export async function summarizeTraceParse(traceDir: string, traceJsonFiles: TraceJsonFile[]): Promise<TraceParseSummary> {
+  if (traceJsonFiles.length === 0) {
+    return {
+      status: 'missing',
+      topLevelWarning: 'No trace JSON files were found.',
+    }
+  }
+
+  for (const file of traceJsonFiles) {
+    try {
+      JSON.parse(await readFile(join(traceDir, file.fileName), 'utf8'))
+    }
+    catch (error) {
+      return {
+        status: 'error',
+        topLevelWarning: `${file.fileName}: ${error instanceof Error ? error.message : `${error}`}`,
+      }
+    }
+  }
+
+  return { status: 'ok' }
+}
+
+export async function writeTraceRunMetrics(traceDir: string, metrics: TraceRunMetrics): Promise<void> {
+  const fileName = join(traceDir, TRACE_RUN_METRICS_FILE)
+  await mkdir(dirname(fileName), { recursive: true })
+  await writeFile(fileName, `${JSON.stringify(metrics, null, 2)}\n`, 'utf8')
+}

test/traceRunMetrics.test.ts

@@ -0,0 +1,106 @@
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { tmpdir } from 'node:os'
+import { afterEach, describe, expect, it } from 'vitest'
+import {
+  TRACE_RUN_METRICS_FILE,
+  buildTraceRunMetrics,
+  discoverTraceJsonFiles,
+  summarizeOutput,
+  summarizeTraceParse,
+  writeTraceRunMetrics,
+} from '../src/traceRunMetrics'
+
+const tempDirs: string[] = []
+
+async function makeTempDir() {
+  const dir = await mkdtemp(join(tmpdir(), 'tsperf-trace-metrics-'))
+  tempDirs.push(dir)
+  return dir
+}
+
+afterEach(async () => {
+  await Promise.all(tempDirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })))
+})
+
+describe('trace run metrics', () => {
+  it('summarizes output and keeps the tail when truncated', () => {
+    expect(summarizeOutput('hello', 10)).toEqual({
+      bytes: 5,
+      truncated: false,
+      text: 'hello',
+    })
+
+    expect(summarizeOutput('0123456789', 4)).toEqual({
+      bytes: 10,
+      truncated: true,
+      text: '6789',
+    })
+  })
+
+  it('discovers trace JSON files without including metrics.json', async () => {
+    const traceDir = await makeTempDir()
+    await writeFile(join(traceDir, 'types.json'), '[]')
+    await writeFile(join(traceDir, 'trace.json'), '[]')
+    await writeFile(join(traceDir, TRACE_RUN_METRICS_FILE), '{}')
+    await writeFile(join(traceDir, 'notes.txt'), 'ignored')
+
+    await expect(discoverTraceJsonFiles(traceDir)).resolves.toEqual([
+      { fileName: 'trace.json', bytes: 2 },
+      { fileName: 'types.json', bytes: 2 },
+    ])
+  })
+
+  it('reports parse status for missing, valid, and invalid trace files', async () => {
+    const traceDir = await makeTempDir()
+
+    await expect(summarizeTraceParse(traceDir, [])).resolves.toEqual({
+      status: 'missing',
+      topLevelWarning: 'No trace JSON files were found.',
+    })
+
+    await writeFile(join(traceDir, 'trace.json'), '[]')
+    const files = await discoverTraceJsonFiles(traceDir)
+    await expect(summarizeTraceParse(traceDir, files)).resolves.toEqual({ status: 'ok' })
+
+    await writeFile(join(traceDir, 'types.json'), '{')
+    const invalidFiles = await discoverTraceJsonFiles(traceDir)
+    const invalidSummary = await summarizeTraceParse(traceDir, invalidFiles)
+    expect(invalidSummary.status).toBe('error')
+    expect(invalidSummary.topLevelWarning).toContain('types.json:')
+  })
+
+  it('builds and writes a stable metrics artifact', async () => {
+    const traceDir = await makeTempDir()
+    const metrics = buildTraceRunMetrics({
+      command: 'npx tsc --noEmit --generateTrace trace',
+      cwd: '/repo',
+      traceDir,
+      startedAt: '2026-05-18T00:00:00.000Z',
+      endedAt: '2026-05-18T00:00:01.250Z',
+      wallTimeMs: 1250,
+      exitCode: 0,
+      stdout: 'ok',
+      stderr: '',
+      traceJsonFiles: [{ fileName: 'trace.json', bytes: 2 }],
+      parseSummary: { status: 'ok' },
+    })
+
+    expect(metrics).toMatchObject({
+      version: 1,
+      command: 'npx tsc --noEmit --generateTrace trace',
+      cwd: '/repo',
+      traceDir,
+      wallTimeMs: 1250,
+      exitCode: 0,
+      stdout: { bytes: 2, truncated: false, text: 'ok' },
+      stderr: { bytes: 0, truncated: false, text: '' },
+      traceJsonFiles: [{ fileName: 'trace.json', bytes: 2 }],
+      parse: { status: 'ok' },
+    })
+
+    await writeTraceRunMetrics(traceDir, metrics)
+    await expect(readFile(join(traceDir, TRACE_RUN_METRICS_FILE), 'utf8'))
+      .resolves.toBe(`${JSON.stringify(metrics, null, 2)}\n`)
+  })
+})