From ba39cf2e07fa28dd5b22220a2ed58e18031921a5 Mon Sep 17 00:00:00 2001 From: ozymandiashh <234437643+ozymandiashh@users.noreply.github.com> Date: Tue, 23 Jun 2026 00:52:26 +0300 Subject: [PATCH] feat(providers): add open-design provider for per-model usage tracking --- src/models.ts | 1 + src/providers/index.ts | 3 +- src/providers/open-design.ts | 259 ++++++++++++++++++ .../data/runs/run-mixed/events.jsonl | 5 + .../data/runs/run-no-usage/events.jsonl | 3 + .../data/runs/run-start-seeded/events.jsonl | 3 + tests/provider-registry.test.ts | 2 +- tests/providers/open-design.test.ts | 155 +++++++++++ 8 files changed, 429 insertions(+), 2 deletions(-) create mode 100644 src/providers/open-design.ts create mode 100644 tests/fixtures/open-design/namespaces/release-stable/data/runs/run-mixed/events.jsonl create mode 100644 tests/fixtures/open-design/namespaces/release-stable/data/runs/run-no-usage/events.jsonl create mode 100644 tests/fixtures/open-design/namespaces/release-stable/data/runs/run-start-seeded/events.jsonl create mode 100644 tests/providers/open-design.test.ts diff --git a/src/models.ts b/src/models.ts index 4d04c1ad..35305a1a 100644 --- a/src/models.ts +++ b/src/models.ts @@ -236,6 +236,7 @@ const BUILTIN_ALIASES: Record = { 'copilot-auto': 'claude-sonnet-4-5', 'copilot-openai-auto': 'gpt-5.3-codex', 'copilot-anthropic-auto': 'claude-sonnet-4-5', + 'openai-codex:gpt-5.5': 'gpt-5.5', 'ibm-bob-auto': 'claude-sonnet-4-5', 'kiro-auto': 'claude-sonnet-4-5', 'cline-auto': 'claude-sonnet-4-5', diff --git a/src/providers/index.ts b/src/providers/index.ts index dca275af..8a0921cf 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -14,6 +14,7 @@ import { kimi } from './kimi.js' import { mistralVibe } from './mistral-vibe.js' import { mux } from './mux.js' import { openclaw } from './openclaw.js' +import { openDesign } from './open-design.js' import { pi, omp } from './pi.js' import { qwen } from './qwen.js' import { rooCode } from './roo-code.js' @@ -170,7 +171,7 @@ async function loadZcode(): Promise { } } -const coreProviders: Provider[] = [claude, cline, codebuff, codex, copilot, devin, droid, gemini, hermes, ibmBob, kiloCode, kiro, kimi, mistralVibe, mux, openclaw, pi, omp, qwen, rooCode, zerostack, grok] +const coreProviders: Provider[] = [claude, cline, codebuff, codex, copilot, devin, droid, gemini, hermes, ibmBob, kiloCode, kiro, kimi, mistralVibe, mux, openclaw, openDesign, pi, omp, qwen, rooCode, zerostack, grok] // Lazily loaded providers, listed by name so --provider validation works even // when an optional module fails to load. Must stay in sync with getAllProviders. diff --git a/src/providers/open-design.ts b/src/providers/open-design.ts new file mode 100644 index 00000000..5a2a0790 --- /dev/null +++ b/src/providers/open-design.ts @@ -0,0 +1,259 @@ +import { readdir, stat } from 'fs/promises' +import { basename, dirname, join } from 'path' +import { homedir, platform } from 'os' + +import { readSessionLines } from '../fs-utils.js' +import { calculateCost } from '../models.js' +import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' + +const PROVIDER_NAME = 'open-design' +const ENV_DIR = 'CODEBURN_OPEN_DESIGN_DIR' + +const modelDisplayNames = new Map([ + ['openai-codex:gpt-5.5', 'GPT-5.5'], + ['glm-5.2', 'GLM-5.2'], + ['GLM-5.2', 'GLM-5.2'], +]) + +type OpenDesignEntry = { + id?: unknown + event?: unknown + data?: unknown + timestamp?: unknown +} + +type TokenUsage = { + inputTokens: number + outputTokens: number + cacheReadTokens: number + reasoningTokens: number +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function stringValue(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined +} + +function tokenValue(value: unknown): number { + return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0 +} + +function timestampValue(value: unknown): string { + const text = stringValue(value) + if (text) return text + if (typeof value !== 'number' || !Number.isFinite(value)) return '' + + const date = new Date(value) + return Number.isNaN(date.getTime()) ? '' : date.toISOString() +} + +function parseEvent(line: string | Buffer): OpenDesignEntry | null { + const text = (typeof line === 'string' ? line : line.toString('utf-8')).trim() + if (!text) return null + + try { + const parsed = JSON.parse(text) as unknown + return isRecord(parsed) ? parsed : null + } catch { + return null + } +} + +function parseUsage(data: unknown): TokenUsage | null { + if (!isRecord(data) || data['type'] !== 'usage') return null + const usage = data['usage'] + if (!isRecord(usage)) return null + + return { + inputTokens: tokenValue(usage['input_tokens']), + outputTokens: tokenValue(usage['output_tokens']), + cacheReadTokens: tokenValue(usage['cached_read_tokens']), + reasoningTokens: tokenValue(usage['thought_tokens']), + } +} + +function getOpenDesignDir(): string { + const override = process.env[ENV_DIR] + if (override) return override + + const home = homedir() + const os = platform() + if (os === 'darwin') { + return join(home, 'Library', 'Application Support', 'Open Design') + } + if (os === 'win32') { + return join(process.env['APPDATA'] ?? join(home, 'AppData', 'Roaming'), 'Open Design') + } + return join(home, '.config', 'Open Design') +} + +function namespaceFromDataDir(dataDir: string): string { + const ns = basename(dirname(dataDir)) + return ns && ns !== 'namespaces' ? ns : PROVIDER_NAME +} + +function namespaceFromRunsDir(runsDir: string): string { + return namespaceFromDataDir(dirname(runsDir)) +} + +async function discoverRunsDir(runsDir: string, project: string): Promise { + const sources: SessionSource[] = [] + let runDirs: string[] + try { + runDirs = await readdir(runsDir) + } catch { + return sources + } + + for (const runDir of runDirs) { + const eventsPath = join(runsDir, runDir, 'events.jsonl') + const s = await stat(eventsPath).catch(() => null) + if (!s?.isFile()) continue + sources.push({ path: eventsPath, project, provider: PROVIDER_NAME }) + } + + return sources +} + +async function discoverNamespacesDir(namespacesDir: string): Promise { + const sources: SessionSource[] = [] + let namespaces: string[] + try { + namespaces = await readdir(namespacesDir) + } catch { + return sources + } + + for (const ns of namespaces) { + const runsDir = join(namespacesDir, ns, 'data', 'runs') + sources.push(...await discoverRunsDir(runsDir, ns)) + } + + return sources +} + +function dedupeSources(sources: SessionSource[]): SessionSource[] { + const seen = new Set() + const out: SessionSource[] = [] + for (const source of sources) { + if (seen.has(source.path)) continue + seen.add(source.path) + out.push(source) + } + return out +} + +async function discoverOpenDesignSessions(baseDir: string): Promise { + const baseName = basename(baseDir) + if (baseName === 'runs') { + return discoverRunsDir(baseDir, namespaceFromRunsDir(baseDir)) + } + if (baseName === 'data') { + return discoverRunsDir(join(baseDir, 'runs'), namespaceFromDataDir(baseDir)) + } + + const sources: SessionSource[] = [] + sources.push(...await discoverRunsDir(join(baseDir, 'data', 'runs'), basename(baseDir) || PROVIDER_NAME)) + sources.push(...await discoverRunsDir(join(baseDir, 'runs'), basename(baseDir) || PROVIDER_NAME)) + sources.push(...await discoverNamespacesDir(baseName === 'namespaces' ? baseDir : join(baseDir, 'namespaces'))) + return dedupeSources(sources) +} + +function createParser(source: SessionSource, seenKeys: Set): SessionParser { + return { + async *parse(): AsyncGenerator { + const sessionId = basename(dirname(source.path)) + let currentModel = '' + let fallbackEventCounter = 0 + + for await (const line of readSessionLines(source.path)) { + const entry = parseEvent(line) + if (!entry) continue + + const eventName = stringValue(entry.event) + const data = entry.data + + if (eventName === 'start' && isRecord(data)) { + const model = stringValue(data['model']) + if (model) currentModel = model + continue + } + + if (eventName !== 'agent' || !isRecord(data)) continue + + if (data['type'] === 'status') { + const model = stringValue(data['model']) + if (model) currentModel = model + continue + } + + const usage = parseUsage(data) + if (!usage || !currentModel) continue + + const eventId = stringValue(entry.id) ?? `line-${fallbackEventCounter++}` + const dedupKey = `${PROVIDER_NAME}:${sessionId}:${eventId}` + if (seenKeys.has(dedupKey)) continue + seenKeys.add(dedupKey) + + const uncachedInputTokens = Math.max(0, usage.inputTokens - usage.cacheReadTokens) + const costUSD = calculateCost( + currentModel, + uncachedInputTokens, + usage.outputTokens + usage.reasoningTokens, + 0, + usage.cacheReadTokens, + 0, + ) + + yield { + provider: PROVIDER_NAME, + sessionId, + project: source.project, + model: currentModel, + inputTokens: uncachedInputTokens, + outputTokens: usage.outputTokens, + cacheCreationInputTokens: 0, + cacheReadInputTokens: usage.cacheReadTokens, + cachedInputTokens: usage.cacheReadTokens, + reasoningTokens: usage.reasoningTokens, + webSearchRequests: 0, + costUSD, + tools: [], + bashCommands: [], + timestamp: timestampValue(entry.timestamp), + speed: 'standard', + deduplicationKey: dedupKey, + userMessage: '', + } + } + }, + } +} + +export function createOpenDesignProvider(overrideDir?: string): Provider { + return { + name: PROVIDER_NAME, + displayName: 'Open Design', + + modelDisplayName(model: string): string { + return modelDisplayNames.get(model) ?? model + }, + + toolDisplayName(rawTool: string): string { + return rawTool + }, + + async discoverSessions(): Promise { + return discoverOpenDesignSessions(overrideDir ?? getOpenDesignDir()) + }, + + createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { + return createParser(source, seenKeys) + }, + } +} + +export const openDesign = createOpenDesignProvider() diff --git a/tests/fixtures/open-design/namespaces/release-stable/data/runs/run-mixed/events.jsonl b/tests/fixtures/open-design/namespaces/release-stable/data/runs/run-mixed/events.jsonl new file mode 100644 index 00000000..d8e58962 --- /dev/null +++ b/tests/fixtures/open-design/namespaces/release-stable/data/runs/run-mixed/events.jsonl @@ -0,0 +1,5 @@ +{"id":"evt-start","event":"start","timestamp":"2026-06-22T10:00:00.000Z","data":{"model":"openai-codex:gpt-5.5"}} +{"id":"evt-codex-usage","event":"agent","timestamp":1782122405000,"data":{"type":"usage","usage":{"input_tokens":1000,"output_tokens":200,"cached_read_tokens":50,"thought_tokens":25,"total_tokens":1200},"costUsd":null}} +{"id":"evt-status-glm","event":"agent","timestamp":"2026-06-22T10:00:10.000Z","data":{"type":"status","model":"glm-5.2"}} +{"id":"evt-glm-usage","event":"agent","timestamp":"2026-06-22T10:00:15.000Z","data":{"type":"usage","usage":{"input_tokens":3000,"output_tokens":400,"cached_read_tokens":100,"thought_tokens":60,"total_tokens":3560},"costUsd":null}} +{"id":"evt-glm-usage","event":"agent","timestamp":"2026-06-22T10:00:16.000Z","data":{"type":"usage","usage":{"input_tokens":9999,"output_tokens":9999,"cached_read_tokens":9999,"thought_tokens":9999,"total_tokens":39996},"costUsd":null}} diff --git a/tests/fixtures/open-design/namespaces/release-stable/data/runs/run-no-usage/events.jsonl b/tests/fixtures/open-design/namespaces/release-stable/data/runs/run-no-usage/events.jsonl new file mode 100644 index 00000000..aa3e0ed0 --- /dev/null +++ b/tests/fixtures/open-design/namespaces/release-stable/data/runs/run-no-usage/events.jsonl @@ -0,0 +1,3 @@ +{"id":"evt-start","event":"start","timestamp":"2026-06-22T11:00:00.000Z","data":{"model":"glm-5.2"}} +{"id":"evt-status","event":"agent","timestamp":"2026-06-22T11:00:05.000Z","data":{"type":"status","model":"openai-codex:gpt-5.5"}} +{"id":"evt-message","event":"agent","timestamp":"2026-06-22T11:00:10.000Z","data":{"type":"message","text":"no usage was emitted"}} diff --git a/tests/fixtures/open-design/namespaces/release-stable/data/runs/run-start-seeded/events.jsonl b/tests/fixtures/open-design/namespaces/release-stable/data/runs/run-start-seeded/events.jsonl new file mode 100644 index 00000000..a4d5aaac --- /dev/null +++ b/tests/fixtures/open-design/namespaces/release-stable/data/runs/run-start-seeded/events.jsonl @@ -0,0 +1,3 @@ +{"id":"evt-start","event":"start","timestamp":"2026-06-22T12:00:00.000Z","data":{"model":"glm-5.2"}} +{"id":"evt-before-transition","event":"agent","timestamp":"2026-06-22T12:00:05.000Z","data":{"type":"usage","usage":{"input_tokens":777,"output_tokens":33,"cached_read_tokens":7,"thought_tokens":3,"total_tokens":820},"costUsd":null}} +{"id":"evt-status","event":"agent","timestamp":"2026-06-22T12:00:10.000Z","data":{"type":"status","model":"openai-codex:gpt-5.5"}} diff --git a/tests/provider-registry.test.ts b/tests/provider-registry.test.ts index 6c9b6c03..6374c590 100644 --- a/tests/provider-registry.test.ts +++ b/tests/provider-registry.test.ts @@ -3,7 +3,7 @@ import { providers, getAllProviders, getProvider } from '../src/providers/index. describe('provider registry', () => { it('has core providers registered synchronously', () => { - expect(providers.map(p => p.name)).toEqual(['claude', 'cline', 'codebuff', 'codex', 'copilot', 'devin', 'droid', 'gemini', 'hermes', 'ibm-bob', 'kilo-code', 'kiro', 'kimi', 'mistral-vibe', 'mux', 'openclaw', 'pi', 'omp', 'qwen', 'roo-code', 'zerostack', 'grok']) + expect(providers.map(p => p.name)).toEqual(['claude', 'cline', 'codebuff', 'codex', 'copilot', 'devin', 'droid', 'gemini', 'hermes', 'ibm-bob', 'kilo-code', 'kiro', 'kimi', 'mistral-vibe', 'mux', 'openclaw', 'open-design', 'pi', 'omp', 'qwen', 'roo-code', 'zerostack', 'grok']) }) it('codebuff tool display names normalize codebuff-native names to canonical set', () => { diff --git a/tests/providers/open-design.test.ts b/tests/providers/open-design.test.ts new file mode 100644 index 00000000..1595c5fd --- /dev/null +++ b/tests/providers/open-design.test.ts @@ -0,0 +1,155 @@ +import { mkdtemp, rm } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' + +import { calculateCost } from '../../src/models.js' +import { clearSessionCache, filterProjectsByDateRange, parseAllSessions } from '../../src/parser.js' +import { allProviderNames } from '../../src/providers/index.js' +import { createOpenDesignProvider } from '../../src/providers/open-design.js' +import type { ParsedProviderCall, SessionSource } from '../../src/providers/types.js' + +const fixtureRoot = join(import.meta.dirname, '../fixtures/open-design') +const dataDir = join(fixtureRoot, 'namespaces', 'release-stable', 'data') + +let previousOverride: string | undefined +let previousCacheDir: string | undefined +let cacheDir: string | undefined + +async function collect(source: SessionSource, seenKeys = new Set()): Promise { + const provider = createOpenDesignProvider() + const calls: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser(source, seenKeys).parse()) calls.push(call) + return calls +} + +async function fixtureSource(runId: string): Promise { + const provider = createOpenDesignProvider() + const sources = await provider.discoverSessions() + const source = sources.find(s => s.path.includes(`${runId}/events.jsonl`)) + expect(source).toBeDefined() + return source! +} + +describe('open-design provider', () => { + beforeEach(async () => { + previousOverride = process.env['CODEBURN_OPEN_DESIGN_DIR'] + previousCacheDir = process.env['CODEBURN_CACHE_DIR'] + cacheDir = await mkdtemp(join(tmpdir(), 'codeburn-open-design-cache-')) + process.env['CODEBURN_OPEN_DESIGN_DIR'] = dataDir + process.env['CODEBURN_CACHE_DIR'] = cacheDir + clearSessionCache() + }) + + afterEach(async () => { + clearSessionCache() + if (previousOverride === undefined) { + delete process.env['CODEBURN_OPEN_DESIGN_DIR'] + } else { + process.env['CODEBURN_OPEN_DESIGN_DIR'] = previousOverride + } + if (previousCacheDir === undefined) { + delete process.env['CODEBURN_CACHE_DIR'] + } else { + process.env['CODEBURN_CACHE_DIR'] = previousCacheDir + } + if (cacheDir) await rm(cacheDir, { recursive: true, force: true }) + cacheDir = undefined + }) + + it('discovers per-run events.jsonl files from the env override data dir', async () => { + const provider = createOpenDesignProvider() + const sources = await provider.discoverSessions() + + expect(sources.map(s => s.provider).every(p => p === 'open-design')).toBe(true) + expect(sources.map(s => s.project)).toEqual(['release-stable', 'release-stable', 'release-stable']) + expect(sources.map(s => s.path).sort()).toEqual([ + join(dataDir, 'runs', 'run-mixed', 'events.jsonl'), + join(dataDir, 'runs', 'run-no-usage', 'events.jsonl'), + join(dataDir, 'runs', 'run-start-seeded', 'events.jsonl'), + ].sort()) + }) + + it('parses a mixed-model run into separate per-model usage calls', async () => { + const calls = await collect(await fixtureSource('run-mixed')) + + expect(calls).toHaveLength(2) + expect(calls.map(c => c.model)).toEqual(['openai-codex:gpt-5.5', 'glm-5.2']) + + const codex = calls[0]! + expect(codex.provider).toBe('open-design') + expect(codex.sessionId).toBe('run-mixed') + expect(codex.inputTokens).toBe(950) + expect(codex.outputTokens).toBe(200) + expect(codex.cacheCreationInputTokens).toBe(0) + expect(codex.cacheReadInputTokens).toBe(50) + expect(codex.cachedInputTokens).toBe(50) + expect(codex.reasoningTokens).toBe(25) + expect(codex.timestamp).toBe('2026-06-22T10:00:05.000Z') + expect(new Date(codex.timestamp).toISOString()).toBe(codex.timestamp) + expect(codex.costUSD).toBeCloseTo( + calculateCost(codex.model, 950, 225, 0, 50, 0), + 12, + ) + + const glm = calls[1]! + expect(glm.inputTokens).toBe(2900) + expect(glm.outputTokens).toBe(400) + expect(glm.cacheCreationInputTokens).toBe(0) + expect(glm.cacheReadInputTokens).toBe(100) + expect(glm.cachedInputTokens).toBe(100) + expect(glm.reasoningTokens).toBe(60) + expect(glm.timestamp).toBe('2026-06-22T10:00:15.000Z') + expect(glm.costUSD).toBeGreaterThan(0) + }) + + it('does not emit calls for a run with no usage events', async () => { + const calls = await collect(await fixtureSource('run-no-usage')) + expect(calls).toHaveLength(0) + }) + + it('uses the start-seeded model before any status transition', async () => { + const calls = await collect(await fixtureSource('run-start-seeded')) + + expect(calls).toHaveLength(1) + expect(calls[0]!.model).toBe('glm-5.2') + expect(calls[0]!.inputTokens).toBe(770) + expect(calls[0]!.outputTokens).toBe(33) + expect(calls[0]!.cacheReadInputTokens).toBe(7) + expect(calls[0]!.reasoningTokens).toBe(3) + expect(calls[0]!.costUSD).toBeGreaterThan(0) + }) + + it('keeps numeric epoch timestamp usage in date-scoped aggregation', async () => { + const projects = await parseAllSessions(undefined, 'open-design') + const filtered = filterProjectsByDateRange(projects, { + start: new Date('2026-06-22T00:00:00.000Z'), + end: new Date('2026-06-22T23:59:59.999Z'), + }) + const calls = filtered.flatMap(project => + project.sessions.flatMap(session => + session.turns.flatMap(turn => turn.assistantCalls), + ), + ) + const numericTimestampCall = calls.find(call => + call.deduplicationKey === 'open-design:run-mixed:evt-codex-usage', + ) + + expect(numericTimestampCall?.timestamp).toBe('2026-06-22T10:00:05.000Z') + }) + + it('deduplicates usage events per run and event id across parser runs', async () => { + const source = await fixtureSource('run-mixed') + const seenKeys = new Set() + + const first = await collect(source, seenKeys) + const second = await collect(source, seenKeys) + + expect(first).toHaveLength(2) + expect(second).toHaveLength(0) + }) + + it('registers open-design as a core provider', () => { + expect(allProviderNames()).toContain('open-design') + }) +})