From 6fa78ce8b28debef1988a4fefadd167f23a4e12b Mon Sep 17 00:00:00 2001 From: tyl3r-ch Date: Tue, 28 Apr 2026 00:55:00 +0200 Subject: [PATCH 1/5] Normalize toktrack provider-suffixed models --- shared/dashboard-domain.js | 36 +++++++++++++++++-- tests/unit/dashboard-aggregation.test.ts | 45 ++++++++++++++++++++++++ tests/unit/model-colors.test.ts | 22 +++++++++++- tests/unit/model-normalization.test.ts | 8 +++++ 4 files changed, 108 insertions(+), 3 deletions(-) diff --git a/shared/dashboard-domain.js b/shared/dashboard-domain.js index e47f371..e6830db 100644 --- a/shared/dashboard-domain.js +++ b/shared/dashboard-domain.js @@ -10,6 +10,19 @@ const PROVIDER_MATCHERS = modelNormalizationSpec.providerMatchers.map((matcher) matcher: new RegExp(matcher.pattern, 'i'), })) +const TOKTRACK_PROVIDER_SUFFIXES = new Map([ + ['alibaba', 'Alibaba'], + ['anthropic', 'Anthropic'], + ['cohere', 'Cohere'], + ['deepseek', 'DeepSeek'], + ['google', 'Google'], + ['mistral', 'Mistral'], + ['opencode', 'OpenCode'], + ['openai', 'OpenAI'], + ['xai', 'xAI'], + ['meta', 'Meta'], +]) + function titleCaseSegment(segment) { if (!segment) return segment if (/^\d+([.-]\d+)*$/.test(segment)) return segment.replace(/-/g, '.') @@ -26,9 +39,25 @@ function formatVersion(version) { return version.replace(/-/g, '.') } +function splitToktrackProviderSuffix(raw) { + const value = String(raw || '').trim() + const match = value.match(/^(.*)::([a-z][a-z0-9_-]*)$/i) + if (!match) { + return { model: value, provider: null } + } + + const model = match[1].trim() + const provider = TOKTRACK_PROVIDER_SUFFIXES.get(match[2].toLowerCase()) ?? null + if (!model || !provider) { + return { model: value, provider: null } + } + + return { model, provider } +} + function canonicalizeModelName(raw) { - const normalized = String(raw || '') - .trim() + const { model } = splitToktrackProviderSuffix(raw) + const normalized = model .toLowerCase() .replace(/^model[:/ -]*/i, '') .replace(/^(anthropic|openai|google|vertex|models)[/-]/i, '') @@ -191,6 +220,9 @@ function normalizeModelName(raw) { * @returns The normalized provider name. */ function getModelProvider(raw) { + const suffixProvider = splitToktrackProviderSuffix(raw).provider + if (suffixProvider) return suffixProvider + const canonical = canonicalizeModelName(raw) for (const matcher of PROVIDER_MATCHERS) { if (matcher.matcher.test(canonical)) return matcher.provider diff --git a/tests/unit/dashboard-aggregation.test.ts b/tests/unit/dashboard-aggregation.test.ts index 31c1587..184d606 100644 --- a/tests/unit/dashboard-aggregation.test.ts +++ b/tests/unit/dashboard-aggregation.test.ts @@ -78,6 +78,51 @@ describe('summarizeUsageBreakdowns', () => { expect(summary.providerMetrics.get('OpenAI')).toMatchObject({ cost: 3, requests: 2, days: 1 }) }) + it('aggregates toktrack provider-suffixed GPT models into the shared model bucket', () => { + const data: DailyUsage[] = [ + { + date: '2026-04-01', + inputTokens: 30, + outputTokens: 15, + cacheCreationTokens: 0, + cacheReadTokens: 0, + thinkingTokens: 0, + totalTokens: 45, + totalCost: 3, + requestCount: 2, + modelsUsed: ['gpt-5.4', 'GPT-5 4::openai'], + modelBreakdowns: [ + { + modelName: 'gpt-5.4', + inputTokens: 10, + outputTokens: 5, + cacheCreationTokens: 0, + cacheReadTokens: 0, + thinkingTokens: 0, + cost: 1, + requestCount: 1, + }, + { + modelName: 'GPT-5 4::openai', + inputTokens: 20, + outputTokens: 10, + cacheCreationTokens: 0, + cacheReadTokens: 0, + thinkingTokens: 0, + cost: 2, + requestCount: 1, + }, + ], + }, + ] + + const summary = summarizeUsageBreakdowns(data) + + expect(summary.allModels).toEqual(['GPT-5.4']) + expect(summary.modelCosts.get('GPT-5.4')).toMatchObject({ cost: 3, requests: 2, days: 1 }) + expect(summary.providerMetrics.get('OpenAI')).toMatchObject({ cost: 3, requests: 2, days: 1 }) + }) + it('keeps model options aligned with breakdown-backed chart series without dropping modelsUsed-only data', () => { const data: DailyUsage[] = [ { diff --git a/tests/unit/model-colors.test.ts b/tests/unit/model-colors.test.ts index 4a6933b..94efc12 100644 --- a/tests/unit/model-colors.test.ts +++ b/tests/unit/model-colors.test.ts @@ -1,6 +1,11 @@ import { createRequire } from 'node:module' import { afterEach, describe, expect, it } from 'vitest' -import { getModelColor, getModelColorAlpha, getProviderBadgeStyle } from '@/lib/model-utils' +import { + getModelColor, + getModelColorAlpha, + getProviderBadgeStyle, + normalizeModelName, +} from '@/lib/model-utils' const require = createRequire(import.meta.url) const { @@ -116,6 +121,21 @@ describe('model colors', () => { expect(getModelColor('Gemini 2.5 Pro', 'light')).toBe('hsl(38, 86%, 34%)') }) + it('uses canonical GPT colors for toktrack provider-suffixed model names', () => { + const normalizedName = normalizeModelName('GPT-5 4::openai') + const palette = createModelColorPalette([ + 'GPT-5', + normalizedName, + normalizeModelName('gpt-5.4'), + ]) + + expect(normalizedName).toBe('GPT-5.4') + expect(getModelColor(normalizedName, 'dark')).toBe(getModelColor('GPT-5.4', 'dark')) + expect(palette.getColor(normalizedName, { theme: 'dark' })).toBe( + palette.getColor('GPT-5.4', { theme: 'dark' }), + ) + }) + it('routes GPT-4.1 through the omni family instead of the main GPT family', () => { const palette = createModelColorPalette(['GPT-4.1', 'GPT-5.4']) diff --git a/tests/unit/model-normalization.test.ts b/tests/unit/model-normalization.test.ts index d5e7a8d..a19d1ef 100644 --- a/tests/unit/model-normalization.test.ts +++ b/tests/unit/model-normalization.test.ts @@ -28,6 +28,14 @@ const MODEL_CASES = [ { raw: 'gpt-5.3-codex', name: 'GPT-5.3 Codex', provider: 'OpenAI' }, { raw: 'gpt-5-4-codex', name: 'GPT-5.4 Codex', provider: 'OpenAI' }, { raw: 'gpt-5.4', name: 'GPT-5.4', provider: 'OpenAI' }, + { raw: 'GPT-5 4::openai', name: 'GPT-5.4', provider: 'OpenAI' }, + { raw: 'gpt-5.4::openai', name: 'GPT-5.4', provider: 'OpenAI' }, + { raw: 'gpt-5-4::openai', name: 'GPT-5.4', provider: 'OpenAI' }, + { + raw: 'claude-sonnet-4-5::anthropic', + name: 'Claude Sonnet 4.5', + provider: 'Anthropic', + }, { raw: 'gemini-2.5-pro', name: 'Gemini 2.5 Pro', provider: 'Google' }, { raw: 'gemini-2.5-flash', name: 'Gemini 2.5 Flash', provider: 'Google' }, { From c2494ae01e2f491262e3084615b203be8989d8c5 Mon Sep 17 00:00:00 2001 From: tyl3r-ch Date: Tue, 28 Apr 2026 01:11:04 +0200 Subject: [PATCH 2/5] Harden integration test helpers for CodeQL --- tests/integration/server-auto-import.test.ts | 4 +- .../server-background-registry.test.ts | 6 +- tests/integration/server-local-auth.test.ts | 21 ++- tests/integration/server-test-helpers.ts | 145 ++++++++++++++++-- 4 files changed, 156 insertions(+), 20 deletions(-) diff --git a/tests/integration/server-auto-import.test.ts b/tests/integration/server-auto-import.test.ts index 9ed7b1a..c01d241 100644 --- a/tests/integration/server-auto-import.test.ts +++ b/tests/integration/server-auto-import.test.ts @@ -178,9 +178,7 @@ describe('local server auto-import integration', () => { expect(firstStreamBody).toContain('event: done') expect(readFileSync(invocationCountPath, 'utf-8')).toBe('1') } finally { - if (!existsSync(releaseRunnerPath)) { - writeFileSync(releaseRunnerPath, 'release') - } + writeFileSync(releaseRunnerPath, 'release') if (standaloneServer) await stopProcess(standaloneServer.child) rmSync(runtimeRoot, { recursive: true, force: true }) } diff --git a/tests/integration/server-background-registry.test.ts b/tests/integration/server-background-registry.test.ts index 3de6ed2..46365c6 100644 --- a/tests/integration/server-background-registry.test.ts +++ b/tests/integration/server-background-registry.test.ts @@ -5,7 +5,6 @@ import { describe, expect, it } from 'vitest' import { createCliEnv, createSharedServerContext, - fetchWithAuth, readBackgroundRegistry, registerSharedServerLifecycle, runCli, @@ -21,18 +20,17 @@ describe('local server background registry pruning', () => { const backgroundEnv = createCliEnv(backgroundRoot) try { - const runtimeResponse = await fetchWithAuth(`${sharedServer.baseUrl}/api/runtime`) - const runtime = await runtimeResponse.json() const sharedServerPid = sharedServer.child?.pid if (!sharedServerPid) { throw new Error('Shared server child process was not started.') } + const sharedServerPort = Number.parseInt(new URL(sharedServer.baseUrl).port, 10) writeBackgroundRegistry(backgroundRoot, [ { id: 'stale-entry', pid: sharedServerPid, - port: runtime.port, + port: sharedServerPort, url: sharedServer.baseUrl, host: '127.0.0.1', authHeader: sharedServer.authHeader, diff --git a/tests/integration/server-local-auth.test.ts b/tests/integration/server-local-auth.test.ts index ca173d7..59b2ed7 100644 --- a/tests/integration/server-local-auth.test.ts +++ b/tests/integration/server-local-auth.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from 'node:os' import path from 'node:path' import { describe, expect, it } from 'vitest' import { + fetchLocalBootstrap, fetchTrusted, getLocalAuthSessionPath, isPosix, @@ -46,10 +47,19 @@ describe('local server session authentication', () => { it('protects loopback read APIs and accepts bearer or bootstrap cookie credentials', async () => { const runtimeRoot = mkdtempSync(path.join(tmpdir(), 'ttdash-local-auth-test-')) + const localToken = 'ttdash-local-auth-bootstrap-token-123456' let standaloneServer: Awaited> | null = null try { - standaloneServer = await startStandaloneServer({ root: runtimeRoot }) + standaloneServer = await startStandaloneServer({ + root: runtimeRoot, + envOverrides: { + TTDASH_LOCAL_AUTH_TOKEN: localToken, + }, + readinessHeaders: { + Authorization: createBearerAuthHeader(localToken), + }, + }) for (const apiPath of [ '/api/usage', @@ -66,9 +76,12 @@ describe('local server session authentication', () => { expect(authenticatedResponse.status).toBe(200) } - const bootstrapResponse = await fetch(standaloneServer.bootstrapUrl!, { - redirect: 'manual', - }) + const bootstrapResponse = await fetchLocalBootstrap( + `${standaloneServer.url}/?ttdash_token=${localToken}`, + { + redirect: 'manual', + }, + ) expect(bootstrapResponse.status).toBe(303) expect(bootstrapResponse.headers.get('location')).toBe('/') const cookieHeader = bootstrapResponse.headers.get('set-cookie')?.split(';', 1)[0] diff --git a/tests/integration/server-test-helpers.ts b/tests/integration/server-test-helpers.ts index 4cf339c..ef31965 100644 --- a/tests/integration/server-test-helpers.ts +++ b/tests/integration/server-test-helpers.ts @@ -53,6 +53,7 @@ const fetchProbeTimeoutMs = 1000 const fetchRequestTimeoutMs = 15_000 const processStopTimeoutMs = 7000 const cliCommandTimeoutMs = 30_000 +const trustedLoopbackHosts = new Set(['127.0.0.1', 'localhost', '[::1]', '::1']) export const hasTypst = (() => { const result = spawnSync('typst', ['--version'], { stdio: 'ignore' }) @@ -65,6 +66,118 @@ export function permissionBits(targetPath: string) { return statSync(targetPath).mode & 0o777 } +function toTrustedLoopbackUrl(value: string | URL, context = 'test server URL') { + const url = value instanceof URL ? value : new URL(value) + if (url.protocol !== 'http:' || !trustedLoopbackHosts.has(url.hostname)) { + throw new Error(`Refusing non-loopback ${context}: ${url.href}`) + } + return url +} + +function toTrustedLoopbackHref(value: string | URL, context = 'test server URL') { + return toTrustedLoopbackUrl(value, context).href +} + +function toTrustedLoopbackOrigin(value: string | URL, context = 'test server URL') { + return toTrustedLoopbackUrl(value, context).origin +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function toPortNumber(value: unknown) { + const port = typeof value === 'number' ? value : Number.parseInt(String(value), 10) + return Number.isInteger(port) && port > 0 && port <= 65535 ? port : null +} + +function toPositiveInteger(value: unknown) { + const number = typeof value === 'number' ? value : Number.parseInt(String(value), 10) + return Number.isInteger(number) && number > 0 ? number : null +} + +function normalizeOptionalString(value: unknown) { + return typeof value === 'string' && value.trim() ? value : null +} + +function normalizeBackgroundRegistryEntry(value: unknown): BackgroundRegistryEntry | null { + if (!isRecord(value)) { + return null + } + + const id = normalizeOptionalString(value.id) + const startedAt = normalizeOptionalString(value.startedAt) + const pid = toPositiveInteger(value.pid) + if (!id || !startedAt || pid === null) { + return null + } + + let registryUrl: URL + try { + registryUrl = toTrustedLoopbackUrl(String(value.url), 'background registry URL') + } catch { + return null + } + + const port = toPortNumber(value.port) ?? toPortNumber(registryUrl.port || 80) + if (port === null) { + return null + } + + const host = trustedLoopbackHosts.has(String(value.host)) + ? String(value.host) + : registryUrl.hostname + const entry: BackgroundRegistryEntry = { + id, + url: registryUrl.origin, + port, + pid, + host, + startedAt, + } + + const apiPrefix = normalizeOptionalString(value.apiPrefix) + if (apiPrefix) { + entry.apiPrefix = apiPrefix + } + + const authHeader = normalizeOptionalString(value.authHeader) + if (authHeader) { + entry.authHeader = authHeader + } + + const bootstrapUrl = normalizeOptionalString(value.bootstrapUrl) + if (bootstrapUrl) { + try { + entry.bootstrapUrl = toTrustedLoopbackHref(bootstrapUrl, 'background bootstrap URL') + } catch { + entry.bootstrapUrl = null + } + } else if (value.bootstrapUrl === null) { + entry.bootstrapUrl = null + } + + const logFile = normalizeOptionalString(value.logFile) + if (logFile) { + entry.logFile = logFile + } else if (value.logFile === null) { + entry.logFile = null + } + + return entry +} + +function normalizeBackgroundRegistryEntries(entries: unknown) { + if (!Array.isArray(entries)) { + return [] as BackgroundRegistryEntry[] + } + + return entries.flatMap((entry) => { + const normalized = normalizeBackgroundRegistryEntry(entry) + return normalized ? [normalized] : [] + }) +} + async function fetchWithTimeout( url: string, init: RequestInit = {}, @@ -72,9 +185,10 @@ async function fetchWithTimeout( ) { const controller = new AbortController() const timeoutId = setTimeout(() => controller.abort(), timeoutMs) + const trustedUrl = toTrustedLoopbackHref(url, 'fetch URL') try { - return await fetch(url, { + return await fetch(trustedUrl, { ...init, signal: controller.signal, }) @@ -485,17 +599,21 @@ export function registerAuthHeader(url: string, authorizationHeader: string | nu return } - authHeadersByOrigin.set(new URL(url).origin, authorizationHeader) + authHeadersByOrigin.set(toTrustedLoopbackOrigin(url, 'auth header origin'), authorizationHeader) } export function getRegisteredAuthHeaders(url: string) { - const authorizationHeader = authHeadersByOrigin.get(new URL(url).origin) + const authorizationHeader = authHeadersByOrigin.get( + toTrustedLoopbackOrigin(url, 'auth header origin'), + ) return authorizationHeader ? { Authorization: authorizationHeader } : undefined } function applyRegisteredAuthHeader(url: string, init: RequestInit = {}) { const headers = new Headers(init.headers) - const registeredAuthHeader = authHeadersByOrigin.get(new URL(url).origin) + const registeredAuthHeader = authHeadersByOrigin.get( + toTrustedLoopbackOrigin(url, 'auth header origin'), + ) if (registeredAuthHeader && !headers.has('Authorization')) { headers.set('Authorization', registeredAuthHeader) @@ -533,7 +651,7 @@ export async function fetchTrusted(url: string, init: RequestInit = {}) { }) } - headers.set('Origin', new URL(url).origin) + headers.set('Origin', toTrustedLoopbackOrigin(url, 'mutation origin')) return await fetchWithTimeout(url, { ...init, @@ -556,9 +674,13 @@ export async function fetchWithAuth( ) } +export async function fetchLocalBootstrap(url: string, init: RequestInit = {}) { + return await fetchWithTimeout(toTrustedLoopbackHref(url, 'local auth bootstrap URL'), init) +} + export function readBackgroundRegistry(root: string) { const registryPath = path.join(getCliConfigDir(root), 'background-instances.json') - return JSON.parse(readFileSync(registryPath, 'utf-8')) as BackgroundRegistryEntry[] + return normalizeBackgroundRegistryEntries(JSON.parse(readFileSync(registryPath, 'utf-8'))) } export function tryReadBackgroundRegistry(root: string) { @@ -568,16 +690,21 @@ export function tryReadBackgroundRegistry(root: string) { } try { - return JSON.parse(readFileSync(registryPath, 'utf-8')) as BackgroundRegistryEntry[] + return normalizeBackgroundRegistryEntries(JSON.parse(readFileSync(registryPath, 'utf-8'))) } catch { return [] } } -export function writeBackgroundRegistry(root: string, entries: unknown) { +export function writeBackgroundRegistry(root: string, entries: BackgroundRegistryEntry[]) { + const normalizedEntries = normalizeBackgroundRegistryEntries(entries) + if (normalizedEntries.length !== entries.length) { + throw new Error('Invalid test background registry entries.') + } + const registryPath = path.join(getCliConfigDir(root), 'background-instances.json') mkdirSync(path.dirname(registryPath), { recursive: true }) - writeFileSync(registryPath, JSON.stringify(entries, null, 2)) + writeFileSync(registryPath, JSON.stringify(normalizedEntries, null, 2)) } export async function waitForBackgroundRegistry( From 365254b89a14c95d54dd89773a0fdb6acfeb5a43 Mon Sep 17 00:00:00 2001 From: tyl3r-ch Date: Tue, 28 Apr 2026 01:18:53 +0200 Subject: [PATCH 3/5] Harden reporting locale E2E test --- src/components/layout/Header.tsx | 1 + tests/e2e/dashboard-reporting.spec.ts | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/components/layout/Header.tsx b/src/components/layout/Header.tsx index 8681ebd..57b47d3 100644 --- a/src/components/layout/Header.tsx +++ b/src/components/layout/Header.tsx @@ -254,6 +254,7 @@ export function Header({