diff --git a/apps/web/src/app/(authenticated)/home/Home.client.test.tsx b/apps/web/src/app/(authenticated)/home/Home.client.test.tsx index a2aff21c8..44eee8e2f 100644 --- a/apps/web/src/app/(authenticated)/home/Home.client.test.tsx +++ b/apps/web/src/app/(authenticated)/home/Home.client.test.tsx @@ -272,40 +272,6 @@ const routedEnvironmentSuggestion: RoutingDecision = { }, }; -const routedEnvironmentSuggestionWithModel: RoutingDecision = { - status: 'routed', - result: { - workspace: { - type: 'environment', - id: 'env-routed', - name: 'Routed Workspace', - }, - model: { - id: 'openrouter/z-ai/glm-5.2', - displayName: 'GLM 5.2', - source: 'preference', - }, - reasoning: 'Best match', - }, -}; - -const routedEnvironmentSuggestionWithDefaultModel: RoutingDecision = { - status: 'routed', - result: { - workspace: { - type: 'environment', - id: 'env-routed', - name: 'Routed Workspace', - }, - model: { - id: 'openrouter/openai/gpt-5.4', - displayName: 'GPT 5.4', - source: 'default', - }, - reasoning: 'Best match', - }, -}; - describe('Home', () => { beforeEach(() => { currentSearchParams = ''; @@ -555,32 +521,8 @@ describe('Home', () => { ); }); - it('uses the routed model for auto-routed launches', async () => { - mockRouteHomeTask.mockResolvedValue(routedEnvironmentSuggestionWithModel); - - render(); - - fireEvent.click(screen.getByRole('button', { name: 'Submit prompt' })); - - await waitFor(() => { - expect(mockCreateStandardTaskRun).toHaveBeenCalledWith( - expect.objectContaining({ - model: 'openrouter/z-ai/glm-5.2', - payload: expect.objectContaining({ - repo: ALL_REPOSITORIES, - environmentId: 'env-routed', - description: 'Test prompt', - blank: false, - }), - }), - ); - }); - }); - - it('preserves the picker model for auto-routed launches when routing only returns the default model', async () => { - mockRouteHomeTask.mockResolvedValue( - routedEnvironmentSuggestionWithDefaultModel, - ); + it('preserves the picker model for auto-routed launches', async () => { + mockRouteHomeTask.mockResolvedValue(routedEnvironmentSuggestion); render(); diff --git a/apps/web/src/app/(authenticated)/home/Home.tsx b/apps/web/src/app/(authenticated)/home/Home.tsx index 0080d888e..4756fc630 100644 --- a/apps/web/src/app/(authenticated)/home/Home.tsx +++ b/apps/web/src/app/(authenticated)/home/Home.tsx @@ -548,11 +548,6 @@ export function Home({ } setRoutingState('launching'); - const routedModelId = - routedResult.result.model?.source === 'preference' - ? routedResult.result.model.id - : undefined; - const didLaunch = await launchTask({ repo: ALL_REPOSITORIES, branch: submission.branch, @@ -562,7 +557,6 @@ export function Home({ : undefined, description: submission.description, images: submission.images, - modelId: routedModelId, blank: submission.blank, }); diff --git a/packages/cloud-agents/evals/router/assertions/routing-assertions.ts b/packages/cloud-agents/evals/router/assertions/routing-assertions.ts index e6bb7aa53..9ad04fe25 100644 --- a/packages/cloud-agents/evals/router/assertions/routing-assertions.ts +++ b/packages/cloud-agents/evals/router/assertions/routing-assertions.ts @@ -10,8 +10,6 @@ interface RoutingResponse { kickoffMessage?: string | null; needsExternalLookup?: boolean; externalReference?: string | null; - requestedModelId?: string | null; - modelConfidence?: number | null; } interface AssertionResult { @@ -226,53 +224,6 @@ function reasoningContainsExpected( }; } -/** - * Asserts that requestedModelId matches the test's expectedRequestedModelId var - * and carries a model confidence at or above the runtime preference threshold. - * Used for model-bearing routing cases where the user expresses a model - * preference and the router should echo back the matching model id. Picks - * below 0.9 confidence are demoted at runtime, so a low-confidence match - * fails this assertion too. - */ -function requestedModelIdMatchesExpected( - output: string, - context: { vars: Record }, -): AssertionResult { - const json = extractJson(output); - if (!json) { - return { pass: false, score: 0, reason: 'Invalid JSON response' }; - } - - const expected = context.vars.expectedRequestedModelId; - if (!expected) { - return { - pass: false, - score: 0, - reason: 'No expectedRequestedModelId in test vars', - }; - } - - const actual = json.requestedModelId ?? null; - if (actual !== expected) { - return { - pass: false, - score: 0, - reason: `requestedModelId mismatch: expected ${expected}, got ${actual}`, - }; - } - - const modelConfidence = json.modelConfidence; - const isConfident = - typeof modelConfidence === 'number' && modelConfidence >= 0.9; - return { - pass: isConfident, - score: isConfident ? 1 : 0, - reason: isConfident - ? `requestedModelId matches: ${expected} (model confidence ${modelConfidence})` - : `requestedModelId matches but model confidence is below the runtime threshold: got ${String(modelConfidence)}`, - }; -} - /** * Asserts that kickoffMessage is present as short display text ending with a * single period (the router prompt contract for chat started messages). @@ -316,30 +267,6 @@ function hasValidKickoffMessage(output: string): AssertionResult { }; } -/** - * Asserts that the router did not report a model preference when the user did - * not express one: requestedModelId must be the explicit `__no_model__` - * sentinel (or a legacy null/empty response). - */ -function requestedModelIdIsNull(output: string): AssertionResult { - const json = extractJson(output); - if (!json) { - return { pass: false, score: 0, reason: 'Invalid JSON response' }; - } - - const isNoModel = - json.requestedModelId == null || - json.requestedModelId === '' || - json.requestedModelId === '__no_model__'; - return { - pass: isNoModel, - score: isNoModel ? 1 : 0, - reason: isNoModel - ? 'requestedModelId reports no model preference' - : `expected no model preference, got ${json.requestedModelId}`, - }; -} - /** * Asserts that routing used the task's supplied context without requesting a * follow-up external lookup. @@ -368,9 +295,7 @@ export { reasoningContains, workspaceValueMatchesExpected, reasoningContainsExpected, - requestedModelIdMatchesExpected, hasValidKickoffMessage, - requestedModelIdIsNull, doesNotRequestExternalLookup, extractJson, }; diff --git a/packages/cloud-agents/evals/router/datasets/model-routing.yaml b/packages/cloud-agents/evals/router/datasets/model-routing.yaml deleted file mode 100644 index 5c58698e1..000000000 --- a/packages/cloud-agents/evals/router/datasets/model-routing.yaml +++ /dev/null @@ -1,364 +0,0 @@ -# Model-bearing routing cases -# These make sure prompts that include a task-model preference still route to -# the right workspace. - -- description: "Explicit model plus explicit environment" - vars: - context: | - **Task Description**: - Use GLM 5.2 for this and work in the Full Stack environment to implement the new checkout flow. - - **Source**: Slack - **Channel**: #shop - - **Available Agents**: - - Generalist [id: agent-generalist-1] - - **Available Repositories**: - - shop/frontend: Storefront UI - - shop/api: E-commerce API - - shop/search: Product search - - **Available Environments**: - - Full Stack: Complete development environment (repos: shop/frontend, shop/api, shop/search) - expectedWorkspaceValue: "Full Stack" - expectedRequestedModelId: "openrouter/z-ai/glm-5.2" - assert: - - type: javascript - value: file://assertions/routing-assertions.ts:isValidRoutingJson - - type: javascript - value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - - type: javascript - value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - - type: javascript - value: file://assertions/routing-assertions.ts:requestedModelIdMatchesExpected - -- description: "Family-only model preference plus explicit environment" - vars: - context: | - **Task Description**: - Let's go with Opus for this. Use the Full Stack environment to fix the modal closing bug. - - **Source**: Slack - **Channel**: #frontend - - **Available Agents**: - - Generalist [id: agent-generalist-1] - - **Available Repositories**: - - acme/frontend: React frontend application - - acme/backend: Node.js backend API - - **Available Environments**: - - Full Stack: Frontend and backend (repos: acme/frontend, acme/backend) - expectedWorkspaceValue: "Full Stack" - expectedRequestedModelId: "openrouter/anthropic/claude-opus-4.8" - assert: - - type: javascript - value: file://assertions/routing-assertions.ts:isValidRoutingJson - - type: javascript - value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - - type: javascript - value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - - type: javascript - value: file://assertions/routing-assertions.ts:requestedModelIdMatchesExpected - -- description: "Alternate family-only preference phrasing plus explicit environment" - vars: - context: | - **Task Description**: - Prefer Opus for this one. Use the Full Stack environment to fix the modal closing bug. - - **Source**: Slack - **Channel**: #frontend - - **Available Agents**: - - Generalist [id: agent-generalist-1] - - **Available Repositories**: - - acme/frontend: React frontend application - - acme/backend: Node.js backend API - - **Available Environments**: - - Full Stack: Frontend and backend (repos: acme/frontend, acme/backend) - expectedWorkspaceValue: "Full Stack" - expectedRequestedModelId: "openrouter/anthropic/claude-opus-4.8" - assert: - - type: javascript - value: file://assertions/routing-assertions.ts:isValidRoutingJson - - type: javascript - value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - - type: javascript - value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - - type: javascript - value: file://assertions/routing-assertions.ts:requestedModelIdMatchesExpected - -- description: "Explicit GPT version plus explicit environment" - vars: - context: | - **Task Description**: - Select GPT 5.6 Terra for this and use the Analytics Lab environment to investigate the retention dashboard regression. - - **Source**: Slack - **Channel**: #analytics - - **Available Agents**: - - Generalist [id: agent-generalist-1] - - **Available Repositories**: - - acme/analytics-web: Analytics frontend - - acme/metrics-api: Metrics aggregation API - - **Available Environments**: - - Analytics Lab: Dashboard and metrics services (repos: acme/analytics-web, acme/metrics-api) - expectedWorkspaceValue: "Analytics Lab" - expectedRequestedModelId: "openrouter/openai/gpt-5.6-terra" - assert: - - type: javascript - value: file://assertions/routing-assertions.ts:isValidRoutingJson - - type: javascript - value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - - type: javascript - value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - - type: javascript - value: file://assertions/routing-assertions.ts:requestedModelIdMatchesExpected - -- description: "Run-on phrasing plus explicit environment" - vars: - context: | - **Task Description**: - Run this on Minimax M3 and use the Payments Ops environment to trace the duplicate webhook retries. - - **Source**: Slack - **Channel**: #payments - - **Available Agents**: - - Generalist [id: agent-generalist-1] - - **Available Repositories**: - - acme/payments-api: Payments backend - - acme/reconciliation: Reconciliation workers - - **Available Environments**: - - Payments Ops: Payments API and reconciliation jobs (repos: acme/payments-api, acme/reconciliation) - expectedWorkspaceValue: "Payments Ops" - expectedRequestedModelId: "openrouter/minimax/minimax-m3" - assert: - - type: javascript - value: file://assertions/routing-assertions.ts:isValidRoutingJson - - type: javascript - value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - - type: javascript - value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - - type: javascript - value: file://assertions/routing-assertions.ts:requestedModelIdMatchesExpected - -- description: "Using phrasing plus explicit environment" - vars: - context: | - **Task Description**: - Using MiMo for this, please work in the Mobile Release environment and fix the offline sync spinner. - - **Source**: Slack - **Channel**: #mobile - - **Available Agents**: - - Generalist [id: agent-generalist-1] - - **Available Repositories**: - - acme/mobile-app: React Native mobile app - - acme/mobile-api: Mobile backend services - - **Available Environments**: - - Mobile Release: Mobile app and API (repos: acme/mobile-app, acme/mobile-api) - expectedWorkspaceValue: "Mobile Release" - expectedRequestedModelId: "openrouter/xiaomi/mimo-v2.5" - assert: - - type: javascript - value: file://assertions/routing-assertions.ts:isValidRoutingJson - - type: javascript - value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - - type: javascript - value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - - type: javascript - value: file://assertions/routing-assertions.ts:requestedModelIdMatchesExpected - -- description: "Courtesy family phrasing plus explicit environment" - vars: - context: | - **Task Description**: - DeepSeek please. Use the Search Relevance environment to tune the bad ranking on brand queries. - - **Source**: Slack - **Channel**: #search - - **Available Agents**: - - Generalist [id: agent-generalist-1] - - **Available Repositories**: - - acme/search-api: Search backend - - acme/ranking: Ranking and relevance services - - **Available Environments**: - - Search Relevance: Search and ranking services (repos: acme/search-api, acme/ranking) - expectedWorkspaceValue: "Search Relevance" - expectedRequestedModelId: "openrouter/deepseek/deepseek-v4-pro" - assert: - - type: javascript - value: file://assertions/routing-assertions.ts:isValidRoutingJson - - type: javascript - value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - - type: javascript - value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - - type: javascript - value: file://assertions/routing-assertions.ts:requestedModelIdMatchesExpected - -- description: "Standalone family mention plus explicit environment" - vars: - context: | - **Task Description**: - GLM. Use the Backend Services environment to investigate the failing rollout health checks. - - **Source**: Slack - **Channel**: #backend - - **Available Agents**: - - Generalist [id: agent-generalist-1] - - **Available Repositories**: - - acme/service-api: Core service API - - acme/deployments: Deployment automation - - **Available Environments**: - - Backend Services: Service API and deployment tooling (repos: acme/service-api, acme/deployments) - expectedWorkspaceValue: "Backend Services" - expectedRequestedModelId: "openrouter/z-ai/glm-5.2" - assert: - - type: javascript - value: file://assertions/routing-assertions.ts:isValidRoutingJson - - type: javascript - value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - - type: javascript - value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - - type: javascript - value: file://assertions/routing-assertions.ts:requestedModelIdMatchesExpected - -- description: "Incidental octopus text does not change explicit environment routing" - vars: - context: | - **Task Description**: - Investigate the octopus migration failure in the Backend Services environment. - - **Source**: Slack - **Channel**: #backend - - **Available Agents**: - - Generalist [id: agent-generalist-1] - - **Available Repositories**: - - acme/reporting: Reporting and export management - - acme/api: Main API service - - **Available Environments**: - - Backend Services: Reporting and API (repos: acme/reporting, acme/api) - expectedWorkspaceValue: "Backend Services" - assert: - - type: javascript - value: file://assertions/routing-assertions.ts:isValidRoutingJson - - type: javascript - value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - - type: javascript - value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - - type: javascript - value: file://assertions/routing-assertions.ts:requestedModelIdIsNull - -- description: "Feature-deletion request without a model mention reports no model preference" - vars: - context: | - **Task Description**: - Delete the Google Drive integration and the associated feature flag - - **Source**: Slack - **Channel**: #tasks - - **Available Agents**: - - Generalist [id: agent-generalist-1] - - **Available Repositories**: - - acme/app: Main product application - - **Available Environments**: - - App: Main product app and services (repos: acme/app) - expectedWorkspaceValue: "App" - assert: - - type: javascript - value: file://assertions/routing-assertions.ts:isValidRoutingJson - - type: javascript - value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - - type: javascript - value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - - type: javascript - value: file://assertions/routing-assertions.ts:requestedModelIdIsNull - -- description: "Parenthetical directive naming a custom deployment-added model" - vars: - context: | - **Task Description**: - (Use Fable) how much could https://emulate.dev/ help us with testing integrations locally? - - **Source**: Slack - **Channel**: #tasks - - **Available Agents**: - - Generalist [id: agent-generalist-1] - - **Available Repositories**: - - acme/app: Main product application - - **Available Environments**: - - App: Main product app, integrations, and services (repos: acme/app) - extraModels: - - displayName: "Fable 5" - id: "openrouter/anthropic/claude-fable-5" - expectedWorkspaceValue: "App" - expectedRequestedModelId: "openrouter/anthropic/claude-fable-5" - assert: - - type: javascript - value: file://assertions/routing-assertions.ts:isValidRoutingJson - - type: javascript - value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - - type: javascript - value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - - type: javascript - value: file://assertions/routing-assertions.ts:requestedModelIdMatchesExpected - -- description: "Custom deployment-added model is listed but not mentioned" - vars: - context: | - **Task Description**: - Investigate the failing rollout health checks in the App environment. - - **Source**: Slack - **Channel**: #tasks - - **Available Agents**: - - Generalist [id: agent-generalist-1] - - **Available Repositories**: - - acme/app: Main product application - - **Available Environments**: - - App: Main product app and services (repos: acme/app) - extraModels: - - displayName: "Fable 5" - id: "openrouter/anthropic/claude-fable-5" - expectedWorkspaceValue: "App" - assert: - - type: javascript - value: file://assertions/routing-assertions.ts:isValidRoutingJson - - type: javascript - value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - - type: javascript - value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - - type: javascript - value: file://assertions/routing-assertions.ts:requestedModelIdIsNull diff --git a/packages/cloud-agents/evals/router/promptfooconfig.ts b/packages/cloud-agents/evals/router/promptfooconfig.ts index d78cdce04..3f54b686a 100644 --- a/packages/cloud-agents/evals/router/promptfooconfig.ts +++ b/packages/cloud-agents/evals/router/promptfooconfig.ts @@ -42,7 +42,6 @@ const config = { 'file://datasets/agent-selection.yaml', 'file://datasets/workspace-selection.yaml', 'file://datasets/explicit-preferences.yaml', - 'file://datasets/model-routing.yaml', 'file://datasets/linear-guidance.yaml', 'file://datasets/github-agent-selection.yaml', 'file://datasets/edge-cases.yaml', diff --git a/packages/cloud-agents/evals/router/prompts/routing.ts b/packages/cloud-agents/evals/router/prompts/routing.ts index aa71fc590..c680f224f 100644 --- a/packages/cloud-agents/evals/router/prompts/routing.ts +++ b/packages/cloud-agents/evals/router/prompts/routing.ts @@ -6,11 +6,6 @@ */ import { buildWorkspaceRoutingPrompt } from '../../../src/server/router/prompts/routing-prompt'; -import { NO_MODEL_MENTIONED_VALUE } from '../../../src/server/router/routing-resolution'; -import { - DEFAULT_TASK_MODEL_SETTINGS, - getEnabledTaskModels, -} from '@roomote/types'; interface Agent { name: string; @@ -29,18 +24,12 @@ interface Environment { repositoryNames?: string[]; } -interface ExtraModel { - displayName: string; - id: string; -} - interface PromptVars { context?: string; taskDescription?: string; agents?: Agent[]; repositories?: (string | Repository)[]; environments?: Environment[]; - extraModels?: ExtraModel[]; routingRules?: Array<{ description: string; target: string }>; } @@ -123,29 +112,6 @@ function buildContext(vars: PromptVars): string { return parts.join('\n\n'); } -// Build the Available Models section from the shipped default catalog so evals -// exercise the same model-selection surface production exposes to the router. -// Tests can append custom deployment-added models (like the ones organizations -// add from the settings UI) through the `extraModels` var. -function buildAvailableModelsSection(extraModels?: ExtraModel[]): string { - const models: Array<{ displayName: string; id: string }> = [ - ...getEnabledTaskModels(DEFAULT_TASK_MODEL_SETTINGS), - ...(extraModels ?? []), - ]; - - if (models.length === 0) { - return ''; - } - - const lines = models.map((m) => `- ${m.displayName} [id: ${m.id}]`); - - lines.push( - `- No model mentioned [id: ${NO_MODEL_MENTIONED_VALUE}] (choose this when the user does not name a model)`, - ); - - return `\n**Available Models**:\n${lines.join('\n')}`; -} - // Production routes through `generateObject` with `workspaceResponseSchema`, // which carries the response shape and per-field descriptions. Promptfoo sends // the raw text prompt instead, so evals restate that response contract here to @@ -157,17 +123,14 @@ Respond with a single JSON object containing exactly these fields: - "workspaceValue" (string): Name of the chosen environment or an available workspace value. - "reasoning" (string): Brief explanation of your workspace decision. - "confidence" (number): Confidence in the workspace choice from 0 to 1. -- "kickoffMessage" (string): Required short user-facing kickoff sentence (about 8-18 words) that ends with a period. Naturally include the chosen environment name, and naturally include the model display name when requestedModelId is a real model. Vary wording; no "Getting started on your task in…" boilerplate every time. No emojis, markdown, quotes, or mentions. Always provide a non-empty value for real routed tasks. +- "kickoffMessage" (string): Required short user-facing kickoff sentence (about 8-18 words) that ends with a period. Naturally include the chosen environment name. Vary wording; no "Getting started on your task in…" boilerplate every time. No emojis, markdown, quotes, or mentions. Always provide a non-empty value for real routed tasks. - "needsExternalLookup" (boolean): Whether an external reference must be fetched before routing, per the external lookup rules. -- "externalReference" (string or null): The exact external reference to fetch when needsExternalLookup is true, otherwise null. -- "requestedModelId" (string or null): The model id the user explicitly requested from the Available Models list, or the literal "${NO_MODEL_MENTIONED_VALUE}" when the user does not name a model. -- "modelConfidence" (number or null): Confidence from 0 to 1 in your requestedModelId choice.`; +- "externalReference" (string or null): The exact external reference to fetch when needsExternalLookup is true, otherwise null.`; // Export a function that receives variables and returns the prompt // This is the format expected by promptfoo for .js/.ts prompt files export default function generatePrompt({ vars }: PromptInput): string { const context = buildContext(vars); - const availableModels = buildAvailableModelsSection(vars.extraModels); const routingPrompt = buildWorkspaceRoutingPrompt(); return `${routingPrompt} @@ -176,5 +139,5 @@ ${OUTPUT_FORMAT_SECTION} ## Current Request -${context}${availableModels ? `\n${availableModels}` : ''}`; +${context}`; } diff --git a/packages/cloud-agents/src/server/router/__tests__/router-helpers.test.ts b/packages/cloud-agents/src/server/router/__tests__/router-helpers.test.ts index 86756ab2e..4693afb56 100644 --- a/packages/cloud-agents/src/server/router/__tests__/router-helpers.test.ts +++ b/packages/cloud-agents/src/server/router/__tests__/router-helpers.test.ts @@ -122,68 +122,11 @@ describe('router helpers', () => { ); }); - it('includes model selection rules in the routing prompt', () => { - const prompt = buildWorkspaceRoutingPrompt(); - - expect(prompt).toContain('## Model Selection'); - expect(prompt).toContain('requestedModelId'); - expect(prompt).toContain('Available Models'); - }); - - it('renders the enabled model catalog in the context prompt', () => { - const prompt = buildContextPrompt( - createContext({ - taskModelSettings: { - models: [ - { - id: 'openrouter/z-ai/glm-5.2', - displayName: 'GLM 5.2', - family: 'GLM', - }, - { - id: 'openrouter/anthropic/claude-opus-4.8', - displayName: 'Opus 4.8', - family: 'Opus', - }, - ], - allowedModelIds: [ - 'openrouter/z-ai/glm-5.2', - 'openrouter/anthropic/claude-opus-4.8', - ], - defaultModelId: 'openrouter/z-ai/glm-5.2', - }, - }), - ); - - expect(prompt).toContain('**Available Models**:'); - expect(prompt).toContain('- GLM 5.2 [id: openrouter/z-ai/glm-5.2]'); - expect(prompt).toContain( - '- Opus 4.8 [id: openrouter/anthropic/claude-opus-4.8]', - ); - expect(prompt).toContain('- No model mentioned [id: __no_model__]'); - }); - - it('renders the default model catalog when settings are null', () => { - const prompt = buildContextPrompt( - createContext({ - taskModelSettings: null, - }), - ); - - expect(prompt).toContain('**Available Models**:'); - expect(prompt).toContain( - '- Claude Sonnet 5 [id: openrouter/anthropic/claude-sonnet-5]', - ); - expect(prompt).toContain( - '- GPT 5.6 Terra [id: openrouter/openai/gpt-5.6-terra]', + it('keeps model selection out of workspace routing', () => { + expect(buildWorkspaceRoutingPrompt()).not.toContain('Model Selection'); + expect(buildContextPrompt(createContext())).not.toContain( + '**Available Models**:', ); - expect(prompt).toContain('- No model mentioned [id: __no_model__]'); - }); - - it('omits the available models section when no settings are provided', () => { - const prompt = buildContextPrompt(createContext()); - - expect(prompt).not.toContain('**Available Models**:'); }); it('maps environments by exact or normalized name', () => { @@ -214,8 +157,6 @@ describe('router helpers', () => { kickoffMessage: null, needsExternalLookup: false, externalReference: null, - requestedModelId: null, - modelConfidence: null, }); }); diff --git a/packages/cloud-agents/src/server/router/__tests__/router-service.test.ts b/packages/cloud-agents/src/server/router/__tests__/router-service.test.ts index 45b60f01c..ee72d020e 100644 --- a/packages/cloud-agents/src/server/router/__tests__/router-service.test.ts +++ b/packages/cloud-agents/src/server/router/__tests__/router-service.test.ts @@ -1,5 +1,3 @@ -import { DEFAULT_TASK_MODEL_SETTINGS } from '@roomote/types'; - import type { RoutingContext, RoutableEnvironment } from '../types'; import { routeTask } from '../router-service'; @@ -42,7 +40,6 @@ describe('routeTask', () => { taskDescription: 'Fix the login flow', source: { type: 'slack', channelName: 'engineering' }, availableEnvironments: environments, - taskModelSettings: DEFAULT_TASK_MODEL_SETTINGS, ...overrides, }; } @@ -305,162 +302,7 @@ describe('routeTask', () => { }); }); - it('uses the LLM-requested model as a preference when it is enabled and confident', async () => { - mockGenerateTrackedNonTaskObject.mockResolvedValue({ - object: { - workspaceValue: 'Full Stack', - reasoning: 'Full Stack is the best fit.', - confidence: 0.92, - needsExternalLookup: false, - externalReference: null, - requestedModelId: 'openrouter/anthropic/claude-opus-5', - modelConfidence: 0.97, - }, - }); - - const result = await routeTask(createContext()); - - expect(result.status).toBe('routed'); - if (result.status !== 'routed') { - throw new Error('Expected routed result'); - } - - expect(result.result.model).toEqual({ - id: 'openrouter/anthropic/claude-opus-5', - displayName: 'Claude Opus 5', - source: 'preference', - confidence: 0.97, - }); - expect(result.result.debug?.selectedTaskModel).toEqual(result.result.model); - }); - - it('uses the LLM-requested model against the default catalog when settings are null', async () => { - mockGenerateTrackedNonTaskObject.mockResolvedValue({ - object: { - workspaceValue: 'Full Stack', - reasoning: 'Full Stack is the best fit.', - confidence: 0.92, - needsExternalLookup: false, - externalReference: null, - requestedModelId: 'openrouter/anthropic/claude-opus-5', - modelConfidence: 0.95, - }, - }); - - const result = await routeTask( - createContext({ - taskModelSettings: null, - }), - ); - - expect(result.status).toBe('routed'); - if (result.status !== 'routed') { - throw new Error('Expected routed result'); - } - - expect(result.result.model).toEqual({ - id: 'openrouter/anthropic/claude-opus-5', - displayName: 'Claude Opus 5', - source: 'preference', - confidence: 0.95, - }); - }); - - it('demotes an LLM-requested model with confidence below the threshold and records the rejected pick', async () => { - mockGenerateTrackedNonTaskObject.mockResolvedValue({ - object: { - workspaceValue: 'Full Stack', - reasoning: 'Full Stack is the best fit.', - confidence: 0.92, - needsExternalLookup: false, - externalReference: null, - requestedModelId: 'openrouter/anthropic/claude-opus-5', - modelConfidence: 0.6, - }, - }); - - const result = await routeTask(createContext()); - - expect(result.status).toBe('routed'); - if (result.status !== 'routed') { - throw new Error('Expected routed result'); - } - - expect(result.result.model).toEqual({ - id: DEFAULT_TASK_MODEL_SETTINGS.defaultModelId, - displayName: expect.any(String), - source: 'default', - rejectedPick: { - id: 'openrouter/anthropic/claude-opus-5', - displayName: 'Claude Opus 5', - confidence: 0.6, - reason: 'below_threshold', - }, - }); - expect(result.result.debug?.selectedTaskModel).toEqual(result.result.model); - }); - - it('demotes an LLM-requested model when the model confidence is missing', async () => { - mockGenerateTrackedNonTaskObject.mockResolvedValue({ - object: { - workspaceValue: 'Full Stack', - reasoning: 'Full Stack is the best fit.', - confidence: 0.92, - needsExternalLookup: false, - externalReference: null, - requestedModelId: 'openrouter/anthropic/claude-opus-5', - }, - }); - - const result = await routeTask(createContext()); - - expect(result.status).toBe('routed'); - if (result.status !== 'routed') { - throw new Error('Expected routed result'); - } - - expect(result.result.model).toEqual({ - id: DEFAULT_TASK_MODEL_SETTINGS.defaultModelId, - displayName: expect.any(String), - source: 'default', - rejectedPick: { - id: 'openrouter/anthropic/claude-opus-5', - displayName: 'Claude Opus 5', - confidence: null, - reason: 'below_threshold', - }, - }); - }); - - it('treats the __no_model__ sentinel as no model preference and records its confidence', async () => { - mockGenerateTrackedNonTaskObject.mockResolvedValue({ - object: { - workspaceValue: 'Full Stack', - reasoning: 'Full Stack is the best fit.', - confidence: 0.92, - needsExternalLookup: false, - externalReference: null, - requestedModelId: '__no_model__', - modelConfidence: 0.98, - }, - }); - - const result = await routeTask(createContext()); - - expect(result.status).toBe('routed'); - if (result.status !== 'routed') { - throw new Error('Expected routed result'); - } - - expect(result.result.model).toEqual({ - id: DEFAULT_TASK_MODEL_SETTINGS.defaultModelId, - displayName: expect.any(String), - source: 'default', - noModelChoice: { confidence: 0.98 }, - }); - }); - - it('falls back to the deployment default when the LLM does not request a model', async () => { + it('routes without selecting a task model', async () => { mockGenerateTrackedNonTaskObject.mockResolvedValue({ object: { workspaceValue: 'Full Stack', @@ -468,7 +310,6 @@ describe('routeTask', () => { confidence: 0.92, needsExternalLookup: false, externalReference: null, - requestedModelId: null, }, }); @@ -479,120 +320,7 @@ describe('routeTask', () => { throw new Error('Expected routed result'); } - expect(result.result.model).toEqual({ - id: DEFAULT_TASK_MODEL_SETTINGS.defaultModelId, - displayName: expect.any(String), - source: 'default', - }); - }); - - it('preserves the previous suggestion model when correcting without a new model preference', async () => { - mockGenerateTrackedNonTaskObject.mockResolvedValue({ - object: { - workspaceValue: 'Full Stack', - reasoning: 'Full Stack is the best fit.', - confidence: 0.92, - needsExternalLookup: false, - externalReference: null, - requestedModelId: null, - }, - }); - - const result = await routeTask( - createContext({ - previousSuggestion: { - workspaceValue: 'Full Stack', - workspaceDisplayName: 'Full Stack', - modelId: 'openrouter/anthropic/claude-opus-5', - modelDisplayName: 'Claude Opus 5', - }, - }), - ); - - expect(result.status).toBe('routed'); - if (result.status !== 'routed') { - throw new Error('Expected routed result'); - } - - expect(result.result.model).toEqual({ - id: 'openrouter/anthropic/claude-opus-5', - displayName: 'Claude Opus 5', - source: 'preserved', - }); - }); - - it('preserves the previous suggestion model over a low-confidence pick and records the rejected pick', async () => { - mockGenerateTrackedNonTaskObject.mockResolvedValue({ - object: { - workspaceValue: 'Full Stack', - reasoning: 'Full Stack is the best fit.', - confidence: 0.92, - needsExternalLookup: false, - externalReference: null, - requestedModelId: 'openrouter/openai/gpt-5.6-terra', - modelConfidence: 0.4, - }, - }); - - const result = await routeTask( - createContext({ - previousSuggestion: { - workspaceValue: 'Full Stack', - workspaceDisplayName: 'Full Stack', - modelId: 'openrouter/anthropic/claude-opus-5', - modelDisplayName: 'Claude Opus 5', - }, - }), - ); - - expect(result.status).toBe('routed'); - if (result.status !== 'routed') { - throw new Error('Expected routed result'); - } - - expect(result.result.model).toEqual({ - id: 'openrouter/anthropic/claude-opus-5', - displayName: 'Claude Opus 5', - source: 'preserved', - rejectedPick: { - id: 'openrouter/openai/gpt-5.6-terra', - displayName: 'GPT 5.6 Terra', - confidence: 0.4, - reason: 'below_threshold', - }, - }); - }); - - it('ignores an LLM-requested model that is not in the deployment allow-list', async () => { - mockGenerateTrackedNonTaskObject.mockResolvedValue({ - object: { - workspaceValue: 'Full Stack', - reasoning: 'Full Stack is the best fit.', - confidence: 0.92, - needsExternalLookup: false, - externalReference: null, - requestedModelId: 'openrouter/unknown/disabled-model', - modelConfidence: 0.95, - }, - }); - - const result = await routeTask(createContext()); - - expect(result.status).toBe('routed'); - if (result.status !== 'routed') { - throw new Error('Expected routed result'); - } - - expect(result.result.model).toEqual({ - id: DEFAULT_TASK_MODEL_SETTINGS.defaultModelId, - displayName: expect.any(String), - source: 'default', - rejectedPick: { - id: 'openrouter/unknown/disabled-model', - displayName: 'openrouter/unknown/disabled-model', - confidence: 0.95, - reason: 'not_allowed', - }, - }); + expect(result.result).not.toHaveProperty('model'); + expect(result.result.debug).not.toHaveProperty('selectedTaskModel'); }); }); diff --git a/packages/cloud-agents/src/server/router/context-builders.ts b/packages/cloud-agents/src/server/router/context-builders.ts index 1189bda2b..eecfecad7 100644 --- a/packages/cloud-agents/src/server/router/context-builders.ts +++ b/packages/cloud-agents/src/server/router/context-builders.ts @@ -24,13 +24,11 @@ async function fetchDeploymentRoutingSettings() { const deployment = await db.query.deploymentSettings.findFirst({ where: eq(deploymentSettings.id, DEFAULT_DEPLOYMENT_ID), columns: { - taskModelSettings: true, workspaceRoutingSettings: true, }, }); return { - taskModelSettings: deployment?.taskModelSettings ?? null, routingRules: deployment?.workspaceRoutingSettings?.rules ?? [], }; } diff --git a/packages/cloud-agents/src/server/router/prompts/routing-context-prompt.ts b/packages/cloud-agents/src/server/router/prompts/routing-context-prompt.ts index d18f321bc..c9ca3d89a 100644 --- a/packages/cloud-agents/src/server/router/prompts/routing-context-prompt.ts +++ b/packages/cloud-agents/src/server/router/prompts/routing-context-prompt.ts @@ -1,6 +1,6 @@ import type { ImagePart, ModelMessage, TextPart } from 'ai'; -import { ALL_REPOSITORIES, getEnabledTaskModels } from '@roomote/types'; +import { ALL_REPOSITORIES } from '@roomote/types'; import { MAX_TASK_DESCRIPTION_LENGTH, @@ -8,7 +8,6 @@ import { PLATFORM_WORKSPACE_VALUE, } from '../types'; import type { RoutingContext, RoutingSource } from '../types'; -import { NO_MODEL_MENTIONED_VALUE } from '../routing-resolution'; const MAX_ROUTING_IMAGE_ATTACHMENTS = 3; const MAX_GITHUB_ROUTING_CONTEXT_CHARS = 250_000; @@ -121,19 +120,6 @@ Prefer a specific environment when one is a plausible home for the work. prompt += `- ${PLATFORM_WORKSPACE_VALUE}: ${PLATFORM_WORKSPACE_DESCRIPTION}\n`; } - const enabledModels = - context.taskModelSettings !== undefined - ? getEnabledTaskModels(context.taskModelSettings) - : []; - - if (enabledModels.length > 0) { - prompt += `\n**Available Models**:\n`; - for (const model of enabledModels) { - prompt += `- ${model.displayName} [id: ${model.id}]\n`; - } - prompt += `- No model mentioned [id: ${NO_MODEL_MENTIONED_VALUE}] (choose this when the user does not name a model)\n`; - } - return prompt; } diff --git a/packages/cloud-agents/src/server/router/prompts/routing-prompt.ts b/packages/cloud-agents/src/server/router/prompts/routing-prompt.ts index 7aacd05ea..eab35984f 100644 --- a/packages/cloud-agents/src/server/router/prompts/routing-prompt.ts +++ b/packages/cloud-agents/src/server/router/prompts/routing-prompt.ts @@ -59,16 +59,15 @@ Also always set \`kickoffMessage\` to a short user-facing kickoff sentence for c - Naturally include the exact environment name from your \`workspaceValue\` choice. - When the choice is \`__all_repositories__\`, say "all repositories" instead of an environment name. -- When \`requestedModelId\` is a real model id (not \`__no_model__\`) with high confidence (at least 0.9), naturally include that model's **display name** from the Available Models list. When the choice is \`__no_model__\`, or you are not highly confident the user named a model, do not mention any model. - Be dynamic and varied: do not always say "Getting started on your task in…". - Prefer lively, progressive phrasing such as "Diving into…", "Looking into…", "Checking…", "Spinning up on…". - Good examples: - "Looking into daily environment snapshots for faster startup in App." - - "Checking mobile login redirects in Payments with Opus 4.8." + - "Checking mobile login redirects in Payments." - "Digging into the flaky checkout email race in Full Stack." -- The environment and model names in the sentence must match the Available lists exactly (same spelling/casing as shown). +- The environment name in the sentence must match the Available Environments list exactly (same spelling/casing as shown). - Do not include emojis, markdown, quotes, @-mentions, or Slack markup. -- Do not invent environment or model names that are not in the provided lists. +- Do not invent environment names that are not in the provided list. - Always produce a non-empty kickoffMessage for real routed tasks. Keep routing justification in \`reasoning\`; keep the spoken kickoff in \`kickoffMessage\`.`; export function buildWorkspaceRoutingPrompt(options?: { @@ -153,20 +152,6 @@ ${WORKSPACE_NARROWING_RULES_BODY} ${platformOverride} **CRITICAL**: You may ONLY select workspaceValue from the Available Environments listed in the request. NEVER invent or hallucinate environment names that are not in the provided lists. -## Model Selection - -When the request includes an **Available Models** list, also populate the \`requestedModelId\` and \`modelConfidence\` fields: - -- \`requestedModelId\` is ALWAYS an explicit choice between two options: a model **id** from the Available Models list, or the literal \`__no_model__\` ("no model mentioned"). -- If the user explicitly requests a model by name or family (for example "use GLM 5.2", "let's go with Opus", "prefer GPT", "run this on Minimax M3"), set \`requestedModelId\` to the matching model **id** from the Available Models list. - - Match by display name or the model id. When the user names only a family (for example "Opus" or "GPT") and multiple versions of that family are listed, choose the highest version of that family. A "Latest" variant of a family counts as its highest version. - - Model requests are often short directives attached to the task rather than full sentences, including parenthesized or bracketed prefixes (for example "(Use Fable) fix the login bug" or "[GLM] investigate the crash"). Treat these directives as explicit model requests. - - The Available Models list can include custom or organization-added models whose names are not well-known public model families (for example "Fable"). Every listed entry is a valid match target: when the user names something that matches a listed model's display name or id, pick that model even if the name is unfamiliar to you. -- Otherwise set \`requestedModelId\` to \`__no_model__\`. Do not pick a model when the user did not express a model preference. Most requests do not mention a model, so \`__no_model__\` is the common answer. -- You may ONLY choose a model id from the Available Models list or \`__no_model__\`. NEVER invent or hallucinate model ids that are not listed. -- When \`requestedModelId\` is a model id, set \`modelConfidence\` to your confidence from 0 to 1 that the user explicitly asked for that model. Picks with confidence below 0.9 are ignored, so only pick a model when the request clearly names it. When \`requestedModelId\` is \`__no_model__\`, set \`modelConfidence\` to your confidence from 0 to 1 that the user did not request a model. Always provide a \`modelConfidence\` number for your choice. -- Model selection is independent of workspace selection: a model preference does not change the workspace, and the absence of a model preference does not affect routing. - ${EXTERNAL_LOOKUP_RULES} ${kickoffSection}`; } diff --git a/packages/cloud-agents/src/server/router/router-service.ts b/packages/cloud-agents/src/server/router/router-service.ts index bab5f11b8..ee018dbe4 100644 --- a/packages/cloud-agents/src/server/router/router-service.ts +++ b/packages/cloud-agents/src/server/router/router-service.ts @@ -1,11 +1,6 @@ import { z } from 'zod'; -import { - formatSingleLineLog, - getDefaultTaskModel, - getTaskModelOptionById, - isTaskModelIdAllowed, -} from '@roomote/types'; +import { formatSingleLineLog } from '@roomote/types'; import { resolveConfiguredGitHubAppSlug } from '@roomote/github'; import type { FollowUpClassification, @@ -16,7 +11,6 @@ import type { RoutingDecision, RoutingPhase, RoutingResult, - RoutingTaskModelSelection, WorkspaceResponse, } from './types'; import { R_SMALL_MODEL_LABEL, PLATFORM_WORKSPACE_VALUE } from './types'; @@ -32,7 +26,6 @@ import { import { buildWorkspaceRoutingPrompt } from './prompts/routing-prompt'; import { mapWorkspace, - NO_MODEL_MENTIONED_VALUE, normalizeWorkspaceSelectionValue, wasWorkspaceRemapped, workspaceResponseSchema, @@ -113,112 +106,6 @@ function isPlatformWorkspaceSelection(value: string): boolean { ); } -/** - * Minimum self-reported model confidence required before an LLM-picked - * `requestedModelId` is honored as a user preference. Picks below this - * threshold are demoted to the preserved/default resolution and surfaced in - * router debug output as a rejected pick. - */ -const MODEL_PREFERENCE_MIN_CONFIDENCE = 0.9; - -/** - * Resolves the routed task model from the LLM's `requestedModelId` pick, the - * previous correction suggestion, and the deployment default. The LLM must - * answer with either a model id (when the user expressed a model preference) - * or the explicit `__no_model__` sentinel, plus a `modelConfidence` score for - * that choice. A model pick is only honored when its self-reported confidence - * is at or above `MODEL_PREFERENCE_MIN_CONFIDENCE`; otherwise the router - * preserves a prior correction or falls back to the deployment default. The - * LLM's raw choice is always recorded on the returned selection — as the - * preference confidence, an explicit `noModelChoice`, or a `rejectedPick` — - * so router debug output can report the model decision on every routing. - */ -function resolveRoutedTaskModel( - response: WorkspaceResponse, - context: RoutingContext, -): RoutingTaskModelSelection | undefined { - const settings = context.taskModelSettings; - - if (settings === undefined) { - return undefined; - } - - const requestedModelId = response.requestedModelId?.trim() || null; - const modelConfidence = - typeof response.modelConfidence === 'number' - ? response.modelConfidence - : null; - let noModelChoice: RoutingTaskModelSelection['noModelChoice']; - let rejectedPick: RoutingTaskModelSelection['rejectedPick']; - - if (requestedModelId === NO_MODEL_MENTIONED_VALUE) { - noModelChoice = { confidence: modelConfidence }; - } else if (requestedModelId) { - const requestedModel = isTaskModelIdAllowed(settings, requestedModelId) - ? getTaskModelOptionById(requestedModelId, settings) - : undefined; - - if (requestedModel) { - if ( - modelConfidence !== null && - modelConfidence >= MODEL_PREFERENCE_MIN_CONFIDENCE - ) { - return { - id: requestedModel.id, - displayName: requestedModel.displayName, - source: 'preference', - confidence: modelConfidence, - }; - } - - rejectedPick = { - id: requestedModel.id, - displayName: requestedModel.displayName, - confidence: modelConfidence, - reason: 'below_threshold', - }; - } else { - rejectedPick = { - id: requestedModelId, - displayName: requestedModelId, - confidence: modelConfidence, - reason: 'not_allowed', - }; - } - } - - const llmChoiceFields = { - ...(noModelChoice ? { noModelChoice } : {}), - ...(rejectedPick ? { rejectedPick } : {}), - }; - - const previousModelId = context.previousSuggestion?.modelId?.trim() || null; - - if (previousModelId && isTaskModelIdAllowed(settings, previousModelId)) { - const previousModel = getTaskModelOptionById(previousModelId, settings); - - if (previousModel) { - return { - id: previousModel.id, - displayName: - context.previousSuggestion?.modelDisplayName ?? - previousModel.displayName, - source: 'preserved', - ...llmChoiceFields, - }; - } - } - - const defaultModel = getDefaultTaskModel(settings); - - return { - id: defaultModel.id, - displayName: defaultModel.displayName, - source: 'default', - ...llmChoiceFields, - }; -} - function buildStandardTaskRoutingResult( response: WorkspaceResponse, context: RoutingContext, @@ -236,7 +123,6 @@ function buildStandardTaskRoutingResult( status: 'routed', result: { workspace, - model: resolveRoutedTaskModel(response, context), reasoning: response.reasoning, ...(kickoffMessage ? { kickoffMessage } : {}), workspaceOnly: true, @@ -538,12 +424,7 @@ export async function routeTask( switch (decision.status) { case 'routed': - decision.result.debug = { - ...debug, - ...(decision.result.model - ? { selectedTaskModel: decision.result.model } - : {}), - }; + decision.result.debug = debug; console.info( formatSingleLineLog('[LLM Router] Routed task', { sourceType: context.source.type, @@ -557,8 +438,6 @@ export async function routeTask( decision.result.workspace.type === 'environment' ? decision.result.workspace.name : null, - taskModelId: decision.result.model?.id, - taskModelSource: decision.result.model?.source, reasoning: truncateText(decision.result.reasoning, 280), }), ); diff --git a/packages/cloud-agents/src/server/router/routing-resolution.ts b/packages/cloud-agents/src/server/router/routing-resolution.ts index 480fd780e..aa8b7b71d 100644 --- a/packages/cloud-agents/src/server/router/routing-resolution.ts +++ b/packages/cloud-agents/src/server/router/routing-resolution.ts @@ -6,14 +6,6 @@ import type { RoutingContext, RoutingWorkspace } from './types'; const WORKSPACE_SELECTION_PREFIX = /^(workspace|environment)\s*:\s*/i; const WORKSPACE_SELECTION_WRAPPERS = /^[`"'([{]+|[`"')\]}]+$/g; -/** - * Sentinel model id the router LLM must return when the user did not name a - * model. Forcing an explicit "no model mentioned" choice instead of a nullable - * field makes small routing models less likely to autofill a hallucinated - * model preference. - */ -export const NO_MODEL_MENTIONED_VALUE = '__no_model__'; - const needsExternalLookupField = z .boolean() .describe( @@ -52,28 +44,12 @@ export const workspaceResponseSchema = z.object({ .string() .nullable() .describe( - 'Short user-facing kickoff sentence posted in chat (about 8-18 words) that ends with a period. Naturally include the exact chosen environment name, and when requestedModelId is a real model also naturally include that model display name from the Available Models list. Vary the wording; do not always use "Getting started on your task in…". No emojis, markdown, quotes, or mentions. Always provide a non-empty value for real routed tasks.', + 'Short user-facing kickoff sentence posted in chat (about 8-18 words) that ends with a period. Naturally include the exact chosen environment name. Vary the wording; do not always use "Getting started on your task in…". No emojis, markdown, quotes, or mentions. Always provide a non-empty value for real routed tasks.', ) .optional() .default(null), needsExternalLookup: needsExternalLookupField.optional().default(false), externalReference: externalReferenceField.optional().default(null), - requestedModelId: z - .string() - .nullable() - .describe( - `The model ID the user explicitly requested, chosen from the Available Models list, or the literal "${NO_MODEL_MENTIONED_VALUE}" when the user does not name a model. Always make this an explicit choice: pick a listed model id only when the user expresses a model preference, and pick "${NO_MODEL_MENTIONED_VALUE}" otherwise.`, - ) - .optional() - .default(null), - modelConfidence: z - .number() - .nullable() - .describe( - `Confidence from 0 to 1 in your requestedModelId choice. Always provide a number. When requestedModelId is a model id, this is your confidence that the user explicitly requested that model; picks below 0.9 are ignored. When requestedModelId is "${NO_MODEL_MENTIONED_VALUE}", this is your confidence that the user did not request a model.`, - ) - .optional() - .default(null), }); /** diff --git a/packages/cloud-agents/src/server/router/types.ts b/packages/cloud-agents/src/server/router/types.ts index bb614ed2c..91a004a44 100644 --- a/packages/cloud-agents/src/server/router/types.ts +++ b/packages/cloud-agents/src/server/router/types.ts @@ -1,7 +1,5 @@ import type { RequestedWorkKindDecision, - TaskModelOption, - TaskModelSettings, WorkspaceRoutingSettings, } from '@roomote/types'; @@ -40,16 +38,6 @@ export interface RoutingContext { routingModel?: string; source: RoutingSource; availableEnvironments: RoutableEnvironment[]; - /** - * Deployment task-model settings. When present, the enabled model catalog is - * exposed to the routing prompt so the LLM can pick a requested model, and - * the router resolves the final model from that pick (or the deployment - * default) instead of parsing the task text in code. - * `null` means the deployment settings row is missing and the built-in - * default catalog should be used; `undefined` means model selection is not - * available for this routing context. - */ - taskModelSettings?: TaskModelSettings | null; routingRules?: WorkspaceRoutingSettings['rules']; routingActor?: { userId: string; @@ -58,39 +46,6 @@ export interface RoutingContext { previousSuggestion?: { workspaceValue: string | null; workspaceDisplayName: string; - modelId?: string | null; - modelDisplayName?: string | null; - }; -} - -export interface RoutingTaskModelSelection extends Pick< - TaskModelOption, - 'id' | 'displayName' -> { - source: 'default' | 'preference' | 'preserved'; - /** - * The router LLM's self-reported confidence that the user explicitly - * requested this model. Only set when `source` is `'preference'`. - */ - confidence?: number | null; - /** - * Present when the router LLM explicitly chose "no model mentioned" - * (`__no_model__`). Carries its self-reported confidence in that choice so - * router debug output can report the model decision on every routing. - */ - noModelChoice?: { - confidence: number | null; - }; - /** - * Present when the router LLM picked a model but the pick was demoted: - * either its confidence was missing or below the preference threshold, or - * the picked id was not in the deployment allow-list. - */ - rejectedPick?: { - id: string; - displayName: string; - confidence: number | null; - reason: 'below_threshold' | 'not_allowed'; }; } @@ -175,7 +130,6 @@ export interface RoutingDebugInfo { needsExternalLookup: boolean | null; confidence?: number | null; workspaceRemapped?: boolean; - selectedTaskModel?: RoutingTaskModelSelection; } /** @@ -199,12 +153,11 @@ export function getRoutingAutoConfirmDelayMs( export interface RoutingResult { workspace: RoutingWorkspace; - model?: RoutingTaskModelSelection; reasoning: string; /** * Full short user-facing kickoff sentence generated with the routing decision. - * Should naturally include the chosen environment (and model override when - * the user requested one). Surfaces may post this directly when valid. + * Should naturally include the chosen environment. Surfaces may post this + * directly when valid. */ kickoffMessage?: string; requestedWorkKindDecision?: RequestedWorkKindDecision; @@ -267,8 +220,6 @@ export interface WorkspaceResponse { kickoffMessage?: string | null; needsExternalLookup: boolean; externalReference: string | null; - requestedModelId?: string | null; - modelConfidence?: number | null; } export type FollowUpIntent = 'confirm' | 'cancel' | 'correct'; diff --git a/packages/communication/src/__tests__/chat-messages.test.ts b/packages/communication/src/__tests__/chat-messages.test.ts index 94c5066af..8a4a23858 100644 --- a/packages/communication/src/__tests__/chat-messages.test.ts +++ b/packages/communication/src/__tests__/chat-messages.test.ts @@ -15,8 +15,6 @@ import { buildTaskStartingText, buildThreadReplyFooterText, formatMarkdownLink, - getUserRequestedModelDisplayName, - resolveUserFacingModelDisplayName, } from '../chat-messages'; describe('chat message copy builders', () => { @@ -139,57 +137,6 @@ describe('chat message copy builders', () => { ).toBe('Digging into the flaky checkout race in Full Stack'); }); - it('only returns model names the router treated as an explicit preference', () => { - expect( - getUserRequestedModelDisplayName({ - displayName: 'Grok 4.5', - source: 'default', - }), - ).toBeUndefined(); - - expect( - getUserRequestedModelDisplayName({ - displayName: 'Claude Opus 4.8', - source: 'preserved', - }), - ).toBeUndefined(); - - expect( - getUserRequestedModelDisplayName({ - displayName: 'Anthropic Claude Fable 5', - source: 'preference', - }), - ).toBe('Anthropic Claude Fable 5'); - }); - - it('clears previous preference names once a non-preference model is resolved', () => { - expect( - resolveUserFacingModelDisplayName({ - model: { - displayName: 'Grok 4.5', - source: 'default', - }, - previousDisplayName: 'Anthropic Claude Fable 5', - }), - ).toBeUndefined(); - - expect( - resolveUserFacingModelDisplayName({ - model: { - displayName: 'Anthropic Claude Fable 5', - source: 'preference', - }, - previousDisplayName: 'Claude Opus 4.8', - }), - ).toBe('Anthropic Claude Fable 5'); - - expect( - resolveUserFacingModelDisplayName({ - previousDisplayName: 'Anthropic Claude Fable 5', - }), - ).toBe('Anthropic Claude Fable 5'); - }); - it('builds queue count, launch, and snapshot resume acknowledgements', () => { expect(buildOtherRunningTasksText(1)).toBe( '1 other task currently running', diff --git a/packages/communication/src/chat-messages.ts b/packages/communication/src/chat-messages.ts index fc2ae90f2..3ac5add3a 100644 --- a/packages/communication/src/chat-messages.ts +++ b/packages/communication/src/chat-messages.ts @@ -95,49 +95,6 @@ export function buildRoutingConfirmationText({ return `I'll get started in ${workspace}${model}, OK?`; } -/** - * Returns a model display name for chat acknowledgements only when the router - * treated the pick as an explicit user preference. Default / preserved models - * stay out of the "getting started" copy. - */ -export function getUserRequestedModelDisplayName( - model?: { - displayName?: string | null; - source?: string | null; - } | null, -): string | undefined { - if (model?.source !== 'preference') { - return undefined; - } - - const displayName = model.displayName?.trim(); - return displayName || undefined; -} - -/** - * Resolves the user-facing model display name for routing state that can carry - * either a freshly routed model selection or a previous correction prefill. - * When a new model selection is present, only preference-sourced names are kept - * so display names cannot outlive a later default/preserved `modelId`. - */ -export function resolveUserFacingModelDisplayName({ - model, - previousDisplayName, -}: { - model?: { - displayName?: string | null; - source?: string | null; - } | null; - previousDisplayName?: string | null; -}): string | undefined { - if (model) { - return getUserRequestedModelDisplayName(model); - } - - const previous = previousDisplayName?.trim(); - return previous || undefined; -} - /** * Legacy started-message prefix used by the static kickoff template. * Dynamic LLM kickoffs are free-form and are detected via stored message ts diff --git a/packages/slack/src/__tests__/router-debug.test.ts b/packages/slack/src/__tests__/router-debug.test.ts index 5a5452f45..9c74c79b2 100644 --- a/packages/slack/src/__tests__/router-debug.test.ts +++ b/packages/slack/src/__tests__/router-debug.test.ts @@ -127,7 +127,7 @@ describe('postRouterDebugMessage', () => { vi.restoreAllMocks(); }); - it('includes the routed task model and selection source in debug channel messages', async () => { + it('includes routed environment details in debug channel messages', async () => { await postRouterDebugMessage({ source: 'Slack C123', sourceLink: @@ -141,12 +141,6 @@ describe('postRouterDebugMessage', () => { needsExternalLookup: false, confidence: 0.97, workspaceRemapped: false, - selectedTaskModel: { - id: 'openrouter/openai/gpt-5.4', - displayName: 'GPT 5.4', - source: 'preference', - confidence: 0.96, - }, }, }); @@ -179,14 +173,6 @@ describe('postRouterDebugMessage', () => { ), }), }), - expect.objectContaining({ - type: 'section', - text: expect.objectContaining({ - text: expect.stringContaining( - '• *Model:* GPT 5.4 `openrouter/openai/gpt-5.4` — user preference, confidence 0.96', - ), - }), - }), ]), }), ); @@ -215,18 +201,6 @@ describe('postRouterDebugMessage', () => { needsExternalLookup: true, confidence: 0.97, workspaceRemapped: true, - selectedTaskModel: { - id: 'openrouter/openai/gpt-5.4', - displayName: 'GPT 5.4', - source: 'preference', - confidence: 0.96, - rejectedPick: { - id: 'openrouter/anthropic/claude-opus-4.8', - displayName: 'Claude Opus 4.8', - confidence: 0.55, - reason: 'below_threshold', - }, - }, }, }); @@ -240,12 +214,6 @@ describe('postRouterDebugMessage', () => { ); const text = discordPostMessageMock.mock.calls[0]?.[0]?.text as string; expect(text).toContain('Environment: App — confidence 0.97'); - expect(text).toContain( - 'Model: GPT 5.4 `openrouter/openai/gpt-5.4` — user preference, confidence 0.96', - ); - expect(text).toContain( - 'Rejected model pick: Claude Opus 4.8 `openrouter/anthropic/claude-opus-4.8` — confidence 0.55 (below threshold)', - ); expect(text).toContain('Environment remapped:'); expect(text).toContain('Duration: 420ms'); expect(text).toContain('Tools: `search_workspaces`'); @@ -274,130 +242,4 @@ describe('postRouterDebugMessage', () => { }), ); }); - - it('includes a rejected low-confidence model pick in debug channel messages', async () => { - await postRouterDebugMessage({ - source: 'Slack C123', - taskDescription: 'Delete the Google Drive integration.', - selectedWorkspace: { name: 'App', type: 'environment' }, - reasoning: 'App is the best fit.', - routingDebug: { - phase: 'direct', - toolsUsed: [], - needsExternalLookup: false, - confidence: 0.97, - workspaceRemapped: false, - selectedTaskModel: { - id: 'openrouter/z-ai/glm-5.2', - displayName: 'GLM 5.2', - source: 'default', - rejectedPick: { - id: 'openrouter/anthropic/claude-opus-4.8', - displayName: 'Claude Opus 4.8', - confidence: 0.55, - reason: 'below_threshold', - }, - }, - }, - }); - - expect(chatPostMessageMock).toHaveBeenCalledWith( - expect.objectContaining({ - channel: 'CDEBUG', - blocks: expect.arrayContaining([ - expect.objectContaining({ - type: 'section', - text: expect.objectContaining({ - text: expect.stringContaining( - '• *Model:* GLM 5.2 `openrouter/z-ai/glm-5.2` — default', - ), - }), - }), - expect.objectContaining({ - type: 'section', - text: expect.objectContaining({ - text: expect.stringContaining( - '*Rejected model pick:* Claude Opus 4.8 `openrouter/anthropic/claude-opus-4.8` — confidence 0.55 (below threshold)', - ), - }), - }), - ]), - }), - ); - }); - - it('reports an explicit no-model choice with its confidence on the model line', async () => { - await postRouterDebugMessage({ - source: 'Slack C123', - taskDescription: 'Delete the Google Drive integration.', - selectedWorkspace: { name: 'App', type: 'environment' }, - reasoning: 'App is the best fit.', - routingDebug: { - phase: 'direct', - toolsUsed: [], - needsExternalLookup: false, - confidence: 0.97, - workspaceRemapped: false, - selectedTaskModel: { - id: 'openrouter/z-ai/glm-5.2', - displayName: 'GLM 5.2', - source: 'default', - noModelChoice: { confidence: 0.98 }, - }, - }, - }); - - expect(chatPostMessageMock).toHaveBeenCalledWith( - expect.objectContaining({ - channel: 'CDEBUG', - blocks: expect.arrayContaining([ - expect.objectContaining({ - type: 'section', - text: expect.objectContaining({ - text: expect.stringContaining( - '• *Model:* GLM 5.2 `openrouter/z-ai/glm-5.2` — default (router choice: no model mentioned, confidence: 0.98)', - ), - }), - }), - ]), - }), - ); - }); - - it('reports when the router did not report a model choice', async () => { - await postRouterDebugMessage({ - source: 'Slack C123', - taskDescription: 'Fix the login flow.', - selectedWorkspace: { name: 'App', type: 'environment' }, - reasoning: 'App is the best fit.', - routingDebug: { - phase: 'direct', - toolsUsed: [], - needsExternalLookup: false, - confidence: 0.97, - workspaceRemapped: false, - selectedTaskModel: { - id: 'openrouter/z-ai/glm-5.2', - displayName: 'GLM 5.2', - source: 'default', - }, - }, - }); - - expect(chatPostMessageMock).toHaveBeenCalledWith( - expect.objectContaining({ - channel: 'CDEBUG', - blocks: expect.arrayContaining([ - expect.objectContaining({ - type: 'section', - text: expect.objectContaining({ - text: expect.stringContaining( - '• *Model:* GLM 5.2 `openrouter/z-ai/glm-5.2` — default (router model choice: not reported)', - ), - }), - }), - ]), - }), - ); - }); }); diff --git a/packages/slack/src/block-kit.ts b/packages/slack/src/block-kit.ts index 2504974f1..65f99e512 100644 --- a/packages/slack/src/block-kit.ts +++ b/packages/slack/src/block-kit.ts @@ -21,8 +21,6 @@ import { buildAccountLinkConnectCopy, buildAccountLinkThreadReplyText as buildSharedAccountLinkThreadReplyText, buildRoutingConfirmationText, - getUserRequestedModelDisplayName, - resolveUserFacingModelDisplayName, } from '@roomote/communication/chat-messages'; import { type SlackInstallation, @@ -1150,10 +1148,6 @@ export async function showTaskConfiguration({ replaceMessageTs, threadMessages: routingThreadMessages, latestOwnBotReply, - modelId: routingResult.model?.id, - modelDisplayName: getUserRequestedModelDisplayName( - routingResult.model, - ), kickoffMessage: routingResult.kickoffMessage, reasoning: routingResult.reasoning, routingDebug: routingResult.debug, @@ -1188,7 +1182,7 @@ export async function showTaskConfiguration({ const confirmNonce = randomUUID(); const confirmBlocks: SlackBlock[] = buildRoutingConfirmBlocks( workspaceDisplayName, - getUserRequestedModelDisplayName(routingResult.model), + undefined, { threadId, confirmNonce }, warningText, ); @@ -1208,8 +1202,6 @@ export async function showTaskConfiguration({ workspaceOnly, workspaceValue, workspaceDisplayName, - modelId: routingResult.model?.id, - modelDisplayName: getUserRequestedModelDisplayName(routingResult.model), ...(routingResult.kickoffMessage ? { kickoffMessage: routingResult.kickoffMessage } : {}), @@ -2436,8 +2428,6 @@ export async function handleSlackRoutingCorrection({ suggestion: { workspaceValue: oldPrefill.workspaceValue, workspaceDisplayName: oldPrefill.workspaceDisplayName, - modelId: oldPrefill.modelId, - modelDisplayName: oldPrefill.modelDisplayName, }, userResponse: correctionText, userId: userMapping.userId, @@ -2647,18 +2637,13 @@ export async function handleSlackRoutingCorrection({ // Store new confirmation with fresh nonce. const correctionNonce = randomUUID(); - const modelDisplayName = resolveUserFacingModelDisplayName({ - model: result.model, - previousDisplayName: oldPrefill.modelDisplayName, - }); - const newPrefill: RoutingPrefillData = { agentName: newAgentName, workspaceOnly: result.workspaceOnly === true, workspaceValue: newWorkspaceValue, workspaceDisplayName: newWorkspaceDisplayName, - modelId: result.model?.id ?? oldPrefill.modelId, - modelDisplayName, + modelId: oldPrefill.modelId, + modelDisplayName: oldPrefill.modelDisplayName, ...(result.kickoffMessage ? { kickoffMessage: result.kickoffMessage } : {}), @@ -2688,7 +2673,7 @@ export async function handleSlackRoutingCorrection({ message: { blocks: buildRoutingConfirmBlocks( newWorkspaceDisplayName, - modelDisplayName, + oldPrefill.modelDisplayName, { threadId, confirmNonce: correctionNonce, @@ -2709,7 +2694,7 @@ export async function handleSlackRoutingCorrection({ thread_ts: threadId, blocks: buildRoutingConfirmBlocks( newWorkspaceDisplayName, - modelDisplayName, + oldPrefill.modelDisplayName, { threadId, confirmNonce: correctionNonce, diff --git a/packages/slack/src/router-debug.ts b/packages/slack/src/router-debug.ts index fd2bcd915..91b05f450 100644 --- a/packages/slack/src/router-debug.ts +++ b/packages/slack/src/router-debug.ts @@ -79,19 +79,6 @@ function formatToolsUsed(toolsUsed: string[]): string { return visibleTools.join(', '); } -function formatModelSource(source: string): string { - switch (source) { - case 'preference': - return 'user preference'; - case 'preserved': - return 'preserved'; - case 'default': - return 'default'; - default: - return source; - } -} - function formatSummaryFields( fields: Array<{ label: string; value: string | undefined }>, ): string { @@ -101,53 +88,7 @@ function formatSummaryFields( .join('\n'); } -function formatSelectedTaskModel( - selectedTaskModel: NonNullable, -): string { - let routerChoiceText: string | null = null; - - if (selectedTaskModel.source !== 'preference') { - if (selectedTaskModel.noModelChoice) { - const noModelConfidence = selectedTaskModel.noModelChoice.confidence; - routerChoiceText = - noModelConfidence != null - ? `(router choice: no model mentioned, confidence: ${String(noModelConfidence)})` - : '(router choice: no model mentioned)'; - } else if (!selectedTaskModel.rejectedPick) { - routerChoiceText = '(router model choice: not reported)'; - } - } - - const modelDetails = [formatModelSource(selectedTaskModel.source)]; - - if (selectedTaskModel.confidence != null) { - modelDetails.push(`confidence ${String(selectedTaskModel.confidence)}`); - } - - const modelValue = `${selectedTaskModel.displayName} \`${selectedTaskModel.id}\` — ${modelDetails.join(', ')}`; - return routerChoiceText ? `${modelValue} ${routerChoiceText}` : modelValue; -} - -function formatRejectedModelPick( - rejectedPick: NonNullable< - NonNullable['rejectedPick'] - >, -): string { - const confidenceText = - rejectedPick.confidence != null - ? String(rejectedPick.confidence) - : 'not provided'; - const reasonText = - rejectedPick.reason === 'not_allowed' - ? 'not in allow-list' - : 'below threshold'; - - return `${truncate(rejectedPick.displayName, 80)} \`${truncate(rejectedPick.id, 80)}\` — confidence ${confidenceText} (${reasonText})`; -} - function formatPlainRouterDebugMessage(params: RouterDebugParams): string { - const selectedTaskModel = params.routingDebug?.selectedTaskModel; - const rejectedPick = selectedTaskModel?.rejectedPick; const visibleToolsUsed = params.routingDebug ? getVisibleToolsUsed(params.routingDebug.toolsUsed) : []; @@ -162,14 +103,8 @@ function formatPlainRouterDebugMessage(params: RouterDebugParams): string { `Source: ${source}`, `Environment: ${environment}`, params.userRoute ? `User override: ${params.userRoute}` : null, - selectedTaskModel - ? `Model: ${formatSelectedTaskModel(selectedTaskModel)}` - : null, `Message:\n${truncate(params.taskDescription, 500) || '(empty)'}`, `Why this route:\n${truncate(params.reasoning, 2500) || '(none)'}`, - rejectedPick - ? `Rejected model pick: ${formatRejectedModelPick(rejectedPick)}` - : null, params.routingDebug?.workspaceRemapped ? 'Environment remapped: Suggested environment was unavailable, so the final route fell back to the resolved selection above.' : null, @@ -481,10 +416,6 @@ export async function postRouterDebugMessage( ? `${environmentName} — confidence ${String(params.routingDebug.confidence)}` : environmentName; - const modelValue = params.routingDebug?.selectedTaskModel - ? formatSelectedTaskModel(params.routingDebug.selectedTaskModel) - : undefined; - const blocks: RouterDebugBlocks = [ { type: 'section', @@ -503,7 +434,6 @@ export async function postRouterDebugMessage( label: 'User override', value: params.userRoute, }, - { label: 'Model', value: modelValue }, ]), }, }, @@ -525,18 +455,6 @@ export async function postRouterDebugMessage( }, }); - const rejectedPick = params.routingDebug?.selectedTaskModel?.rejectedPick; - - if (rejectedPick) { - blocks.push({ - type: 'section', - text: { - type: 'mrkdwn', - text: `*Rejected model pick:* ${formatRejectedModelPick(rejectedPick)}`, - }, - }); - } - if (params.routingDebug?.workspaceRemapped) { blocks.push({ type: 'section', diff --git a/packages/slack/src/start-auto-routed-slack-task.ts b/packages/slack/src/start-auto-routed-slack-task.ts index 2c8cd8f6c..929ca3af8 100644 --- a/packages/slack/src/start-auto-routed-slack-task.ts +++ b/packages/slack/src/start-auto-routed-slack-task.ts @@ -29,7 +29,6 @@ import { type RoutingResult, type SlackMcpSetupRequirement, } from '@roomote/cloud-agents/server'; -import { getUserRequestedModelDisplayName } from '@roomote/communication/chat-messages'; import { mapRoutingWorkspaceToSelectionValue, @@ -484,9 +483,6 @@ export async function startAutoRoutedSlackTask({ // initiator becomes the run's acting user immediately so their task gets // integration MCP access from the first turn. const initiatorLinkedUserId = getTaskInitiatorLinkedUserId(initiator); - const userRequestedModelDisplayName = getUserRequestedModelDisplayName( - decision.result.model, - ); let taskRun: Awaited>; try { taskRun = await startSlackAppMentionTask({ @@ -507,7 +503,7 @@ export async function startAutoRoutedSlackTask({ branch, sha, harness, - model: model ?? decision.result.model?.id, + model, environmentId: workspace.environmentId, reasoningEffort, images: allProcessedImages.length ? allProcessedImages : undefined, @@ -524,9 +520,6 @@ export async function startAutoRoutedSlackTask({ agentName: AGENT_DISPLAY_NAME, initiatingSlackUserId, workspaceDisplayName: workspace.workspaceDisplayName, - ...(userRequestedModelDisplayName - ? { modelDisplayName: userRequestedModelDisplayName } - : {}), ...(decision.result.kickoffMessage ? { kickoffMessage: decision.result.kickoffMessage } : {}), @@ -582,7 +575,6 @@ export async function startAutoRoutedSlackTask({ initiatingSlackUserId, agentName: AGENT_DISPLAY_NAME, workspaceDisplayName: workspace.workspaceDisplayName, - modelDisplayName: userRequestedModelDisplayName, kickoffMessage: decision.result.kickoffMessage, workspaceType: decision.result.workspace.type, workspaceValue,