Skip to content

Add model context length and manual model deletion #520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
roiding wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Add model context length and manual model deletion #520

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1ebac44
feat: add context_length to /v1/models response
Apr 29, 2026
af9aac3
feat: add delete button for manually added models
Apr 29, 2026
0b2201e
fix: address PR review feedback for model metadata
May 12, 2026
3c689a4
fix: preserve model context metadata across credential scans
codex May 12, 2026
039d0d9
fix: canonicalize endpoint model context scope
codex May 12, 2026
eaa3279
fix: isolate concurrent model context scan scopes
codex May 13, 2026
edcbce7
fix: use conservative model context lengths in models surface
codex May 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 62 additions & 5 deletions src/server/proxy-core/surfaces/modelsSurface.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,27 @@
import { describe, expect, it, vi } from 'vitest';

import { listModelsSurface } from './modelsSurface.js';
import {
buildAccountModelContextLengthScope,
clearModelContextLengthCache,
setModelContextLength,
} from '../../services/modelContextLengthCache.js';

describe('listModelsSurface', () => {
it('returns OpenAI list shape and hides models without a resolvable channel', async () => {
clearModelContextLengthCache();
setModelContextLength('routable-model', 128000, buildAccountModelContextLengthScope(11));

const result = await listModelsSurface({
downstreamPolicy: { type: 'all' },
responseFormat: 'openai',
tokenRouter: {
getAvailableModels: vi.fn().mockResolvedValue(['routable-model', 'orphan-model']),
explainSelection: vi.fn()
.mockResolvedValueOnce({ selectedChannelId: null })
.mockResolvedValueOnce({ selectedChannelId: 11 }),
explainSelection: vi.fn(async (modelName: string) => (
modelName === 'routable-model'
? { selectedChannelId: 11, selectedAccountId: 11 }
: { selectedChannelId: null }
)),
},
refreshModelsAndRebuildRoutes: vi.fn(),
isModelAllowed: vi.fn().mockResolvedValue(true),
Expand All @@ -26,18 +36,22 @@ describe('listModelsSurface', () => {
object: 'model',
created: 1773878400,
owned_by: 'metapi',
context_length: 128000,
},
],
});
});

it('returns Claude list shape when requested', async () => {
clearModelContextLengthCache();
setModelContextLength('claude-opus-4-6', 200000, buildAccountModelContextLengthScope(22));

const result = await listModelsSurface({
downstreamPolicy: { type: 'all' },
responseFormat: 'claude',
tokenRouter: {
getAvailableModels: vi.fn().mockResolvedValue(['claude-opus-4-6']),
explainSelection: vi.fn().mockResolvedValue({ selectedChannelId: 22 }),
explainSelection: vi.fn().mockResolvedValue({ selectedChannelId: 22, selectedAccountId: 22 }),
},
refreshModelsAndRebuildRoutes: vi.fn(),
isModelAllowed: vi.fn().mockResolvedValue(true),
Expand All @@ -51,6 +65,7 @@ describe('listModelsSurface', () => {
type: 'model',
display_name: 'claude-opus-4-6',
created_at: '2026-03-19T00:00:00.000Z',
context_length: 200000,
},
],
first_id: 'claude-opus-4-6',
Expand All @@ -59,15 +74,56 @@ describe('listModelsSurface', () => {
});
});

it('uses the most conservative context length across eligible routing candidates', async () => {
clearModelContextLengthCache();
setModelContextLength('shared-model', 200000, buildAccountModelContextLengthScope(41));
setModelContextLength('shared-model', 128000, buildAccountModelContextLengthScope(42));

const result = await listModelsSurface({
downstreamPolicy: { type: 'all' },
responseFormat: 'openai',
tokenRouter: {
getAvailableModels: vi.fn().mockResolvedValue(['shared-model']),
explainSelection: vi.fn().mockResolvedValue({
selectedChannelId: 4101,
selectedAccountId: 41,
candidates: [
{ accountId: 41, eligible: true },
{ accountId: 42, eligible: true },
],
}),
},
refreshModelsAndRebuildRoutes: vi.fn(),
isModelAllowed: vi.fn().mockResolvedValue(true),
now: () => new Date('2026-03-19T00:00:00.000Z'),
});

expect(result).toEqual({
object: 'list',
data: [
{
id: 'shared-model',
object: 'model',
created: 1773878400,
owned_by: 'metapi',
context_length: 128000,
},
],
});
});

it('applies downstream policy filtering before selection checks and refreshes once when the first read is empty', async () => {
clearModelContextLengthCache();
setModelContextLength('allowed-model', 64000, buildAccountModelContextLengthScope(33));

const getAvailableModels = vi.fn()
.mockResolvedValueOnce(['blocked-model'])
.mockResolvedValueOnce(['allowed-model']);
const refreshModelsAndRebuildRoutes = vi.fn().mockResolvedValue(undefined);
const isModelAllowed = vi.fn()
.mockResolvedValueOnce(false)
.mockResolvedValueOnce(true);
const explainSelection = vi.fn().mockResolvedValue({ selectedChannelId: 33 });
const explainSelection = vi.fn().mockResolvedValue({ selectedChannelId: 33, selectedAccountId: 33 });

const result = await listModelsSurface({
downstreamPolicy: { type: 'whitelist' },
Expand All @@ -90,6 +146,7 @@ describe('listModelsSurface', () => {
object: 'model',
created: 1773878400,
owned_by: 'metapi',
context_length: 64000,
},
],
});
Expand Down
88 changes: 77 additions & 11 deletions src/server/proxy-core/surfaces/modelsSurface.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
import {
buildAccountModelContextLengthScope,
getModelContextLength,
} from '../../services/modelContextLengthCache.js';

function isSearchPseudoModel(modelName: string): boolean {
const normalized = (modelName || '').trim().toLowerCase();
if (!normalized) return false;
Expand All @@ -11,25 +16,75 @@ type ModelsSurfaceInput = {
getAvailableModels(): Promise<string[]>;
explainSelection(modelName: string, excludeChannelIds: number[], downstreamPolicy: unknown): Promise<{
selectedChannelId?: number | null;
selectedAccountId?: number | null;
candidates?: Array<{
accountId?: number | null;
eligible?: boolean;
}>;
}>;
};
refreshModelsAndRebuildRoutes(): Promise<unknown>;
isModelAllowed(modelName: string, downstreamPolicy: unknown): Promise<boolean>;
now?: () => Date;
};

async function readVisibleModels(input: ModelsSurfaceInput): Promise<string[]> {
function resolveModelContextLength(
modelName: string,
selectedAccountId?: number | null,
candidates?: Array<{ accountId?: number | null; eligible?: boolean }>,
): number {
const eligibleAccountIds = Array.from(new Set(
(Array.isArray(candidates) ? candidates : [])
.filter((candidate) => candidate?.eligible !== false)
.map((candidate) => candidate?.accountId)
.filter((accountId): accountId is number => typeof accountId === 'number' && accountId > 0),
));

if (eligibleAccountIds.length > 0) {
return eligibleAccountIds.reduce((minValue, accountId) => {
const currentValue = getModelContextLength(
modelName,
buildAccountModelContextLengthScope(accountId),
);
return Math.min(minValue, currentValue);
}, Number.POSITIVE_INFINITY);
}

if (typeof selectedAccountId === 'number' && selectedAccountId > 0) {
return getModelContextLength(
modelName,
buildAccountModelContextLengthScope(selectedAccountId),
);
}
return getModelContextLength(modelName);
}

async function readVisibleModels(
input: ModelsSurfaceInput,
): Promise<Array<{
id: string;
selectedAccountId?: number | null;
candidates?: Array<{ accountId?: number | null; eligible?: boolean }>;
}>> {
const deduped = Array.from(new Set(await input.tokenRouter.getAvailableModels()))
.filter((modelName) => !isSearchPseudoModel(modelName))
.sort();
const allowed: string[] = [];
const allowed: Array<{
id: string;
selectedAccountId?: number | null;
candidates?: Array<{ accountId?: number | null; eligible?: boolean }>;
}> = [];
for (const modelName of deduped) {
if (!await input.isModelAllowed(modelName, input.downstreamPolicy)) {
continue;
}
const decision = await input.tokenRouter.explainSelection(modelName, [], input.downstreamPolicy);
if (typeof decision.selectedChannelId === 'number') {
allowed.push(modelName);
allowed.push({
id: modelName,
selectedAccountId: decision.selectedAccountId,
Comment on lines 81 to +85
Copy link
Copy Markdown

@chatgpt-codex-connector chatgpt-codex-connector Bot May 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Derive context_length from all eligible channels

/v1/models now picks context_length from a single explainSelection result, but route selection is strategy-dependent (round-robin/weighted/stable-first) and can choose different channels/accounts for the same model across requests. In multi-channel routes where the same model has different limits (or one channel has no cached metadata and falls back to 1_000_000), this can overstate the safe window and cause intermittent upstream failures when a lower-limit channel is chosen at dispatch time. The value should be computed from all eligible routing candidates (e.g., conservative minimum) rather than one sampled account.

Useful? React with 👍 / 👎.

candidates: decision.candidates,
});
}
}
return allowed;
Expand All @@ -44,12 +99,22 @@ export async function listModelsSurface(input: ModelsSurfaceInput) {

const now = input.now?.() ?? new Date();
if (input.responseFormat === 'claude') {
const data = models.map((id) => ({
id,
type: 'model' as const,
display_name: id,
created_at: now.toISOString(),
}));
const data: Array<{
id: string;
type: 'model';
display_name: string;
created_at: string;
context_length: number;
}> = [];
for (const model of models) {
data.push({
id: model.id,
type: 'model' as const,
display_name: model.id,
created_at: now.toISOString(),
context_length: resolveModelContextLength(model.id, model.selectedAccountId, model.candidates),
});
}
return {
data,
first_id: data[0]?.id || null,
Expand All @@ -60,11 +125,12 @@ export async function listModelsSurface(input: ModelsSurfaceInput) {

return {
object: 'list' as const,
data: models.map((id) => ({
id,
data: models.map((model) => ({
id: model.id,
object: 'model' as const,
created: Math.floor(now.getTime() / 1000),
owned_by: 'metapi',
context_length: resolveModelContextLength(model.id, model.selectedAccountId, model.candidates),
})),
};
}
104 changes: 101 additions & 3 deletions src/server/routes/api/accounts.manual-models.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ describe('accounts manual models endpoint', () => {

app = Fastify();
await app.register(routesModule.accountsRoutes);
});
}, 30_000);

beforeEach(async () => {
await db.delete(schema.proxyLogs).run();
Expand All @@ -40,9 +40,9 @@ describe('accounts manual models endpoint', () => {
});

afterAll(async () => {
await app.close();
await app?.close();
delete process.env.DATA_DIR;
});
}, 30_000);

it('adds manual models and sets isManual to true', async () => {
const site = await db.insert(schema.sites).values({
Expand Down Expand Up @@ -179,4 +179,102 @@ describe('accounts manual models endpoint', () => {
message: 'Invalid models. Expected string[].',
});
});

it('deletes only manual models for the target account', async () => {
const site = await db.insert(schema.sites).values({
name: 'Test Site',
url: 'https://test.example.com',
platform: 'new-api',
}).returning().get();

const account = await db.insert(schema.accounts).values({
siteId: site.id,
accessToken: 'test-token',
}).returning().get();

await db.insert(schema.modelAvailability).values([
{
accountId: account.id,
modelName: 'manual-a',
available: true,
isManual: true,
},
{
accountId: account.id,
modelName: 'manual-b',
available: true,
isManual: true,
},
{
accountId: account.id,
modelName: 'synced-model',
available: true,
isManual: false,
},
]);

const response = await app.inject({
method: 'DELETE',
url: `/api/accounts/${account.id}/models/manual`,
payload: {
models: ['manual-a'],
},
});

expect(response.statusCode).toBe(200);
expect(response.json()).toEqual({ success: true });

const models = await db.select().from(schema.modelAvailability)
.where(eq(schema.modelAvailability.accountId, account.id))
.all();

expect(models.map((model) => `${model.modelName}:${model.isManual}`).sort()).toEqual([
'manual-b:true',
'synced-model:false',
]);
});

it('ignores duplicate and whitespace model names when deleting manual models', async () => {
const site = await db.insert(schema.sites).values({
name: 'Test Site',
url: 'https://test.example.com',
platform: 'new-api',
}).returning().get();

const account = await db.insert(schema.accounts).values({
siteId: site.id,
accessToken: 'test-token',
}).returning().get();

await db.insert(schema.modelAvailability).values([
{
accountId: account.id,
modelName: 'manual-a',
available: true,
isManual: true,
},
{
accountId: account.id,
modelName: 'manual-b',
available: true,
isManual: true,
},
]);

const response = await app.inject({
method: 'DELETE',
url: `/api/accounts/${account.id}/models/manual`,
payload: {
models: [' manual-a ', 'manual-a', ' '],
},
});

expect(response.statusCode).toBe(200);

const models = await db.select().from(schema.modelAvailability)
.where(eq(schema.modelAvailability.accountId, account.id))
.all();

expect(models.map((model) => model.modelName)).toEqual(['manual-b']);
});
});
Loading
Loading