From 7500638cf19bdb76595b2e345b53a7e0bf7ba0d0 Mon Sep 17 00:00:00 2001 From: Jack D Date: Wed, 15 Apr 2026 22:45:02 -0600 Subject: [PATCH 1/8] Use any model for AI Agent --- apps/web/src/app/settings/AIAgentSection.tsx | 24 ++- .../hooks/useSettingsSectionsConvex.ts | 46 +++++- packages/convex/convex/aiAgent.ts | 144 ++++++++++++++++-- packages/convex/convex/aiAgentActions.ts | 28 ++-- packages/convex/convex/lib/aiGateway.ts | 69 ++++++++- .../convex/schema/inboxConversationTables.ts | 1 + .../schema/operationsReportingTables.ts | 3 +- packages/convex/tests/aiAgent.test.ts | 14 +- .../convex/tests/aiAgentRuntimeSafety.test.ts | 22 ++- 9 files changed, 302 insertions(+), 49 deletions(-) diff --git a/apps/web/src/app/settings/AIAgentSection.tsx b/apps/web/src/app/settings/AIAgentSection.tsx index 738d87b..e001ea9 100644 --- a/apps/web/src/app/settings/AIAgentSection.tsx +++ b/apps/web/src/app/settings/AIAgentSection.tsx @@ -6,6 +6,10 @@ import { AlertTriangle, Bot } from "lucide-react"; import type { Id } from "@opencom/convex/dataModel"; import { useAIAgentSectionConvex } from "./hooks/useSettingsSectionsConvex"; +function normalizeModelValue(value: string): string { + return value.trim(); +} + export function AIAgentSection({ workspaceId, }: { @@ -22,6 +26,11 @@ export function AIAgentSection({ const [suggestionsEnabled, setSuggestionsEnabled] = useState(false); const [embeddingModel, setEmbeddingModel] = useState("text-embedding-3-small"); const [isSaving, setIsSaving] = useState(false); + const normalizedModel = normalizeModelValue(model); + const selectedDiscoveredModel = + availableModels?.some((availableModel) => availableModel.id === normalizedModel) ?? false + ? normalizedModel + : ""; useEffect(() => { if (aiSettings) { @@ -39,11 +48,12 @@ export function AIAgentSection({ const handleSave = async () => { if (!workspaceId) return; setIsSaving(true); + const nextModel = normalizeModelValue(model); try { await updateSettings({ workspaceId, enabled, - model, + model: nextModel, confidenceThreshold, knowledgeSources: knowledgeSources as ("articles" | "internalArticles" | "snippets")[], personality: personality || undefined, @@ -51,6 +61,7 @@ export function AIAgentSection({ suggestionsEnabled, embeddingModel, }); + setModel(nextModel); } catch (error) { console.error("Failed to save AI settings:", error); } finally { @@ -131,18 +142,25 @@ export function AIAgentSection({
+ setModel(e.target.value)} + placeholder="openai/gpt-5-nano" + />

- Choose the AI model for generating responses. + Choose a discovered model or enter one manually. Raw model IDs are interpreted + against the currently configured AI gateway runtime.

diff --git a/apps/web/src/app/settings/hooks/useSettingsSectionsConvex.ts b/apps/web/src/app/settings/hooks/useSettingsSectionsConvex.ts index c1711aa..a97ce62 100644 --- a/apps/web/src/app/settings/hooks/useSettingsSectionsConvex.ts +++ b/apps/web/src/app/settings/hooks/useSettingsSectionsConvex.ts @@ -1,5 +1,6 @@ "use client"; +import { useEffect, useState } from "react"; import type { Id } from "@opencom/convex/dataModel"; import type { HomeCard, HomeConfig, HomeDefaultSpace, HomeTab } from "@opencom/types"; import { @@ -83,8 +84,8 @@ const AI_SETTINGS_QUERY_REF = webQueryRef< } | null; } | null >("aiAgent:getSettings"); -const AVAILABLE_MODELS_QUERY_REF = webQueryRef< - Record, +const AVAILABLE_MODELS_ACTION_REF = webActionRef< + { workspaceId: Id<"workspaces">; selectedModel?: string }, Array<{ id: string; name: string; provider: string }> >("aiAgent:listAvailableModels"); const UPDATE_AI_SETTINGS_REF = webMutationRef< @@ -278,9 +279,46 @@ const TRANSFER_OWNERSHIP_REF = webMutationRef) { + const aiSettings = useWebQuery(AI_SETTINGS_QUERY_REF, workspaceId ? { workspaceId } : "skip"); + const listAvailableModels = useWebAction(AVAILABLE_MODELS_ACTION_REF); + const [availableModels, setAvailableModels] = useState< + Array<{ id: string; name: string; provider: string }> | undefined + >(undefined); + + useEffect(() => { + let cancelled = false; + + if (!workspaceId) { + setAvailableModels(undefined); + return () => { + cancelled = true; + }; + } + + void listAvailableModels({ + workspaceId, + selectedModel: aiSettings?.model, + }) + .then((models) => { + if (!cancelled) { + setAvailableModels(models); + } + }) + .catch((error) => { + console.error("Failed to load available AI models:", error); + if (!cancelled) { + setAvailableModels(undefined); + } + }); + + return () => { + cancelled = true; + }; + }, [workspaceId, aiSettings?.model, listAvailableModels]); + return { - aiSettings: useWebQuery(AI_SETTINGS_QUERY_REF, workspaceId ? { workspaceId } : "skip"), - availableModels: useWebQuery(AVAILABLE_MODELS_QUERY_REF, {}), + aiSettings, + availableModels, updateSettings: useWebMutation(UPDATE_AI_SETTINGS_REF), }; } diff --git a/packages/convex/convex/aiAgent.ts b/packages/convex/convex/aiAgent.ts index 3b95393..41d7198 100644 --- a/packages/convex/convex/aiAgent.ts +++ b/packages/convex/convex/aiAgent.ts @@ -12,6 +12,7 @@ import { Doc, Id } from "./_generated/dataModel"; import { getAuthenticatedUserFromSession } from "./auth"; import { getWorkspaceMembership, requirePermission } from "./permissions"; import { authAction, authMutation, authQuery } from "./lib/authWrappers"; +import { getAIGatewayApiKey, getAIBaseURL, getAIGatewayProviderLabel } from "./lib/aiGateway"; import { getShallowRunAfter, routeEventRef } from "./notifications/functionRefs"; import { resolveVisitorFromSession } from "./widgetSessions"; @@ -31,6 +32,19 @@ type RuntimeKnowledgeResult = { relevanceScore: number; }; +type AvailableAIModel = { + id: string; + name: string; + provider: string; +}; + +type OpenAIModelListResponse = { + data?: Array<{ + id?: string; + created?: number; + }>; +}; + type GetRelevantKnowledgeForRuntimeActionArgs = { workspaceId: Id<"workspaces">; query: string; @@ -81,6 +95,105 @@ const DEFAULT_AI_SETTINGS = { lastConfigError: null, }; +const AVAILABLE_MODEL_DISCOVERY_TIMEOUT_MS = 5000; +const NON_GENERATION_MODEL_PREFIXES = [ + "text-embedding-", + "omni-moderation-", + "whisper-", + "tts-", + "gpt-image-", + "dall-e-", + "babbage-", + "davinci-", +]; + +function normalizeAvailableModelId(value: string | undefined): string | null { + const normalized = value?.trim(); + if (!normalized) { + return null; + } + + return normalized; +} + +function createAvailableAIModel(value: string | undefined): AvailableAIModel | null { + const normalizedId = normalizeAvailableModelId(value); + if (!normalizedId) { + return null; + } + + const [provider, ...modelParts] = normalizedId.split("/"); + const model = modelParts.join("/") || normalizedId; + return { + id: normalizedId, + name: model, + provider, + }; +} + +function isLikelyGenerationModel(modelId: string): boolean { + const normalized = modelId.trim().toLowerCase(); + if (!normalized) { + return false; + } + + return !NON_GENERATION_MODEL_PREFIXES.some((prefix) => normalized.startsWith(prefix)); +} + +function dedupeAvailableAIModels(models: AvailableAIModel[]): AvailableAIModel[] { + const seen = new Set(); + return models.filter((model) => { + if (seen.has(model.id)) { + return false; + } + + seen.add(model.id); + return true; + }); +} + +async function discoverAvailableAIModels(): Promise { + const apiKey = getAIGatewayApiKey(); + if (!apiKey) { + return null; + } + + const providerLabel = getAIGatewayProviderLabel(getAIBaseURL(apiKey)); + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), AVAILABLE_MODEL_DISCOVERY_TIMEOUT_MS); + try { + const response = await fetch(`${getAIBaseURL(apiKey)}/models`, { + headers: { + Accept: "application/json", + Authorization: `Bearer ${apiKey}`, + }, + signal: controller.signal, + }); + if (!response.ok) { + return null; + } + + const payload = (await response.json()) as OpenAIModelListResponse; + const models = Array.isArray(payload.data) ? payload.data : []; + + return models + .filter( + (model): model is { id: string; created?: number } => + typeof model.id === "string" && isLikelyGenerationModel(model.id) + ) + .sort((left, right) => (right.created ?? 0) - (left.created ?? 0) || left.id.localeCompare(right.id)) + .map((model) => + createAvailableAIModel(model.id.includes("/") ? model.id : `${providerLabel}/${model.id}`) + ) + .filter((model): model is AvailableAIModel => model !== null); + } catch { + return null; + } finally { + clearTimeout(timeout); + } +} + function withAISettingDefaults(settings: Doc<"aiAgentSettings"> | null): { _id?: Id<"aiAgentSettings">; _creationTime?: number; @@ -489,7 +602,7 @@ export const getConversationResponses = query({ .withIndex("by_conversation", (q) => q.eq("conversationId", args.conversationId)) .collect(); - return responses.map((response) => ({ + return responses.filter((response) => response.messageId !== undefined).map((response) => ({ ...response, deliveredResponseContext: { response: response.response, @@ -711,20 +824,19 @@ export const getAnalytics = authQuery({ }); // List available AI models (from AI Gateway) -export const listAvailableModels = query({ - args: {}, - handler: async () => { - // Return a static list of supported models - // In production, this could query the AI Gateway API - return [ - { id: "openai/gpt-5-nano", name: "GPT-5.1 Mini", provider: "openai" }, - // { id: "openai/gpt-5.1", name: "GPT-5.1", provider: "openai" }, - // { id: "anthropic/claude-3-haiku-20240307", name: "Claude 3 Haiku", provider: "anthropic" }, - // { - // id: "anthropic/claude-3-5-sonnet-20241022", - // name: "Claude 3.5 Sonnet", - // provider: "anthropic", - // }, - ]; +export const listAvailableModels = authAction({ + args: { + workspaceId: v.id("workspaces"), + selectedModel: v.optional(v.string()), + }, + permission: "settings.workspace", + handler: async (_ctx, args): Promise => { + const discoveredModels = await discoverAvailableAIModels(); + const fallbackModels = [ + createAvailableAIModel(args.selectedModel), + createAvailableAIModel(DEFAULT_AI_SETTINGS.model), + ].filter((model): model is AvailableAIModel => model !== null); + + return dedupeAvailableAIModels([...(discoveredModels ?? []), ...fallbackModels]); }, }); diff --git a/packages/convex/convex/aiAgentActions.ts b/packages/convex/convex/aiAgentActions.ts index 2830050..e0ac402 100644 --- a/packages/convex/convex/aiAgentActions.ts +++ b/packages/convex/convex/aiAgentActions.ts @@ -5,7 +5,7 @@ import { v } from "convex/values"; import type { Id } from "./_generated/dataModel"; import { action } from "./_generated/server"; import { generateText } from "ai"; -import { createAIClient } from "./lib/aiGateway"; +import { createAIClient, getAIGatewayProviderLabel } from "./lib/aiGateway"; type AIConfigurationDiagnostic = { code: string; @@ -242,7 +242,6 @@ function getShallowRunAction(ctx: { runAction: unknown }) { ) => Promise; } -const SUPPORTED_AI_PROVIDERS = new Set(["openai"]); const GENERATION_FAILURE_FALLBACK_RESPONSE = "I'm having trouble processing your request right now. Let me connect you with a human agent."; const EMPTY_RESPONSE_RETRY_LIMIT = 1; @@ -263,17 +262,20 @@ const isGPT5ReasoningModel = (provider: string, model: string): boolean => // Parse model string to get provider and model name export const parseModel = (modelString: string): { provider: string; model: string } => { - const parts = modelString.split("/"); + const trimmedModel = modelString.trim(); + const parts = trimmedModel.split("/"); if (parts.length === 2) { return { provider: parts[0], model: parts[1] }; } - return { provider: "openai", model: modelString }; + + return { provider: getAIGatewayProviderLabel(), model: trimmedModel }; }; export const getAIConfigurationDiagnostic = ( modelString: string, - environment: { aiGatewayApiKey?: string } = { + environment: { aiGatewayApiKey?: string; aiGatewayProviderLabel?: string } = { aiGatewayApiKey: process.env.AI_GATEWAY_API_KEY, + aiGatewayProviderLabel: getAIGatewayProviderLabel(), } ): AIConfigurationDiagnostic | null => { const trimmedModel = modelString.trim(); @@ -285,20 +287,22 @@ export const getAIConfigurationDiagnostic = ( } const parts = trimmedModel.split("/"); - if (parts.length !== 2 || !parts[0] || !parts[1]) { + if (parts.length > 2 || (parts.length === 2 && (!parts[0] || !parts[1]))) { return { code: "INVALID_MODEL_FORMAT", - message: "AI model format is invalid. Use provider/model (for example openai/gpt-5-nano).", + message: + "AI model format is invalid. Use provider/model or a raw model ID exposed by the configured AI gateway.", model: trimmedModel, }; } - const provider = parts[0]; - if (!SUPPORTED_AI_PROVIDERS.has(provider)) { + const provider = parts.length === 2 ? parts[0] : getAIGatewayProviderLabel(); + const model = parts.length === 2 ? parts[1] : trimmedModel; + if (!provider || !model) { return { - code: "UNSUPPORTED_PROVIDER", - message: `Provider '${provider}' is not supported in this runtime.`, - provider, + code: "INVALID_MODEL_FORMAT", + message: + "AI model format is invalid. Use provider/model or a raw model ID exposed by the configured AI gateway.", model: trimmedModel, }; } diff --git a/packages/convex/convex/lib/aiGateway.ts b/packages/convex/convex/lib/aiGateway.ts index 829254e..747029b 100644 --- a/packages/convex/convex/lib/aiGateway.ts +++ b/packages/convex/convex/lib/aiGateway.ts @@ -1,17 +1,74 @@ import { createOpenAI } from "@ai-sdk/openai"; +const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1"; +const DEFAULT_VERCEL_AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh/v1"; +const GENERIC_PROVIDER_LABEL = "gateway"; + +function sanitizeProviderLabel(value: string): string { + const sanitized = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, ""); + return sanitized || GENERIC_PROVIDER_LABEL; +} + +export function getAIGatewayApiKey(): string | undefined { + const apiKey = process.env.AI_GATEWAY_API_KEY?.trim(); + return apiKey && apiKey.length > 0 ? apiKey : undefined; +} + +export function getAIBaseURL(apiKey = getAIGatewayApiKey()): string { + const configuredBaseURL = process.env.AI_GATEWAY_BASE_URL?.trim(); + if (configuredBaseURL && configuredBaseURL.length > 0) { + return configuredBaseURL.replace(/\/+$/, ""); + } + + return apiKey?.startsWith("vck_") + ? DEFAULT_VERCEL_AI_GATEWAY_BASE_URL + : DEFAULT_OPENAI_BASE_URL; +} + +export function getAIGatewayProviderLabel(baseURL = getAIBaseURL()): string { + try { + const hostname = new URL(baseURL).hostname.toLowerCase(); + if (hostname === "api.openai.com" || hostname.endsWith(".openai.com")) { + return "openai"; + } + if (hostname === "ai-gateway.vercel.sh") { + return GENERIC_PROVIDER_LABEL; + } + + const segments = hostname + .split(".") + .filter(Boolean) + .filter((segment) => !["api", "chat", "www", "gateway"].includes(segment)); + + if (segments.length >= 2 && segments[segments.length - 1] === "ai") { + const providerRoot = segments[segments.length - 2] ?? ""; + if (providerRoot.length === 1) { + return sanitizeProviderLabel(`${providerRoot}ai`); + } + } + + for (let index = segments.length - 1; index >= 0; index -= 1) { + const segment = segments[index]; + if (!segment || ["com", "ai", "net", "dev", "app", "io", "sh", "co"].includes(segment)) { + continue; + } + return sanitizeProviderLabel(segment); + } + } catch { + return GENERIC_PROVIDER_LABEL; + } + + return GENERIC_PROVIDER_LABEL; +} + export function createAIClient() { - const apiKey = process.env.AI_GATEWAY_API_KEY; + const apiKey = getAIGatewayApiKey(); if (!apiKey) { throw new Error("AI_GATEWAY_API_KEY environment variable is not set"); } - const baseURL = - process.env.AI_GATEWAY_BASE_URL || - (apiKey.startsWith("vck_") ? "https://ai-gateway.vercel.sh/v1" : "https://api.openai.com/v1"); - return createOpenAI({ apiKey, - baseURL, + baseURL: getAIBaseURL(apiKey), }); } diff --git a/packages/convex/convex/schema/inboxConversationTables.ts b/packages/convex/convex/schema/inboxConversationTables.ts index 26a6074..9e13b5d 100644 --- a/packages/convex/convex/schema/inboxConversationTables.ts +++ b/packages/convex/convex/schema/inboxConversationTables.ts @@ -87,6 +87,7 @@ export const inboxConversationTables = { aiWorkflowState: v.optional( v.union(v.literal("none"), v.literal("ai_handled"), v.literal("handoff")) ), + aiTurnSequence: v.optional(v.number()), aiHandoffReason: v.optional(v.string()), aiLastConfidence: v.optional(v.number()), aiLastResponseAt: v.optional(v.number()), diff --git a/packages/convex/convex/schema/operationsReportingTables.ts b/packages/convex/convex/schema/operationsReportingTables.ts index 5f028bc..6a31a71 100644 --- a/packages/convex/convex/schema/operationsReportingTables.ts +++ b/packages/convex/convex/schema/operationsReportingTables.ts @@ -5,7 +5,7 @@ import { jsonRecordValidator } from "../validators"; export const operationsReportingTables = { aiResponses: defineTable({ conversationId: v.id("conversations"), - messageId: v.id("messages"), + messageId: v.optional(v.id("messages")), query: v.string(), response: v.string(), generatedCandidateResponse: v.optional(v.string()), @@ -28,6 +28,7 @@ export const operationsReportingTables = { articleId: v.optional(v.string()), }) ), + attemptStatus: v.optional(v.string()), confidence: v.number(), feedback: v.optional(v.union(v.literal("helpful"), v.literal("not_helpful"))), handedOff: v.boolean(), diff --git a/packages/convex/tests/aiAgent.test.ts b/packages/convex/tests/aiAgent.test.ts index 1902f86..218bdc0 100644 --- a/packages/convex/tests/aiAgent.test.ts +++ b/packages/convex/tests/aiAgent.test.ts @@ -274,12 +274,24 @@ describe("aiAgent", () => { describe("listAvailableModels", () => { it("should return list of available models", async () => { - const models = await client.query(api.aiAgent.listAvailableModels, {}); + const models = await client.action(api.aiAgent.listAvailableModels, { + workspaceId: testWorkspaceId, + selectedModel: "openai/gpt-5-nano", + }); expect(models.length).toBeGreaterThan(0); expect(models[0]).toHaveProperty("id"); expect(models[0]).toHaveProperty("name"); expect(models[0]).toHaveProperty("provider"); }); + + it("should preserve the selected model when discovery falls back", async () => { + const models = await client.action(api.aiAgent.listAvailableModels, { + workspaceId: testWorkspaceId, + selectedModel: "openai/gpt-5.1-mini", + }); + + expect(models.some((model) => model.id === "openai/gpt-5.1-mini")).toBe(true); + }); }); }); diff --git a/packages/convex/tests/aiAgentRuntimeSafety.test.ts b/packages/convex/tests/aiAgentRuntimeSafety.test.ts index e16aad3..8ef3b3b 100644 --- a/packages/convex/tests/aiAgentRuntimeSafety.test.ts +++ b/packages/convex/tests/aiAgentRuntimeSafety.test.ts @@ -19,19 +19,29 @@ describe("aiAgentActions runtime safety", () => { beforeEach(() => { vi.clearAllMocks(); process.env.AI_GATEWAY_API_KEY = "test-ai-key"; + process.env.AI_GATEWAY_BASE_URL = "https://api.openai.com/v1"; }); it("returns explicit diagnostics for invalid runtime configuration", () => { expect(getAIConfigurationDiagnostic("")).toMatchObject({ code: "MISSING_MODEL", }); - expect(getAIConfigurationDiagnostic("invalid-model-format")).toMatchObject({ + expect(getAIConfigurationDiagnostic("invalid/model/format")).toMatchObject({ code: "INVALID_MODEL_FORMAT", }); - expect(getAIConfigurationDiagnostic("anthropic/claude-3-5-sonnet")).toMatchObject({ - code: "UNSUPPORTED_PROVIDER", - provider: "anthropic", - }); + expect( + getAIConfigurationDiagnostic("zai/glm-5-turbo", { + aiGatewayApiKey: "test-ai-key", + aiGatewayProviderLabel: "zai", + }) + ).toBeNull(); + expect( + getAIConfigurationDiagnostic("glm-5-turbo", { + aiGatewayApiKey: "test-ai-key", + aiGatewayProviderLabel: "zai", + }) + ).toBeNull(); + expect(getAIConfigurationDiagnostic("anthropic/claude-3-5-sonnet")).toBeNull(); }); it("falls back to handoff and records diagnostics when configuration is invalid", async () => { @@ -42,7 +52,7 @@ describe("aiAgentActions runtime safety", () => { if ("workspaceId" in args && "conversationId" in args === false) { return { enabled: true, - model: "invalid-model-format", + model: "invalid/model/format", confidenceThreshold: 0.6, knowledgeSources: ["articles"], personality: null, From f630437da55aac44e03c9d5a40caa5894d56de4b Mon Sep 17 00:00:00 2001 From: Jack D Date: Wed, 15 Apr 2026 22:54:26 -0600 Subject: [PATCH 2/8] restrict embedding model choice to vectors with 1536 dimensions, not 3072. --- apps/web/src/app/settings/AIAgentSection.tsx | 3 -- packages/convex/convex/_generated/api.d.ts | 2 + packages/convex/convex/aiAgent.ts | 14 +++++-- .../convex/convex/aiAgentActionsKnowledge.ts | 6 ++- packages/convex/convex/embeddings.ts | 6 ++- packages/convex/convex/lib/embeddingModels.ts | 40 +++++++++++++++++++ .../convex/schema/operationsAiTables.ts | 3 +- packages/convex/convex/suggestions.ts | 12 ++++-- packages/convex/tests/embeddingModels.test.ts | 33 +++++++++++++++ 9 files changed, 103 insertions(+), 16 deletions(-) create mode 100644 packages/convex/convex/lib/embeddingModels.ts create mode 100644 packages/convex/tests/embeddingModels.test.ts diff --git a/apps/web/src/app/settings/AIAgentSection.tsx b/apps/web/src/app/settings/AIAgentSection.tsx index e001ea9..944bd27 100644 --- a/apps/web/src/app/settings/AIAgentSection.tsx +++ b/apps/web/src/app/settings/AIAgentSection.tsx @@ -270,9 +270,6 @@ export function AIAgentSection({ -

diff --git a/packages/convex/convex/_generated/api.d.ts b/packages/convex/convex/_generated/api.d.ts index 7c39510..07bd412 100644 --- a/packages/convex/convex/_generated/api.d.ts +++ b/packages/convex/convex/_generated/api.d.ts @@ -52,6 +52,7 @@ import type * as internalArticles from "../internalArticles.js"; import type * as knowledge from "../knowledge.js"; import type * as lib_aiGateway from "../lib/aiGateway.js"; import type * as lib_authWrappers from "../lib/authWrappers.js"; +import type * as lib_embeddingModels from "../lib/embeddingModels.js"; import type * as lib_notificationPreferences from "../lib/notificationPreferences.js"; import type * as lib_seriesRuntimeAdapter from "../lib/seriesRuntimeAdapter.js"; import type * as lib_unifiedArticles from "../lib/unifiedArticles.js"; @@ -238,6 +239,7 @@ declare const fullApi: ApiFromModules<{ knowledge: typeof knowledge; "lib/aiGateway": typeof lib_aiGateway; "lib/authWrappers": typeof lib_authWrappers; + "lib/embeddingModels": typeof lib_embeddingModels; "lib/notificationPreferences": typeof lib_notificationPreferences; "lib/seriesRuntimeAdapter": typeof lib_seriesRuntimeAdapter; "lib/unifiedArticles": typeof lib_unifiedArticles; diff --git a/packages/convex/convex/aiAgent.ts b/packages/convex/convex/aiAgent.ts index 41d7198..ffd2342 100644 --- a/packages/convex/convex/aiAgent.ts +++ b/packages/convex/convex/aiAgent.ts @@ -12,6 +12,7 @@ import { Doc, Id } from "./_generated/dataModel"; import { getAuthenticatedUserFromSession } from "./auth"; import { getWorkspaceMembership, requirePermission } from "./permissions"; import { authAction, authMutation, authQuery } from "./lib/authWrappers"; +import { DEFAULT_CONTENT_EMBEDDING_MODEL, resolveContentEmbeddingModel } from "./lib/embeddingModels"; import { getAIGatewayApiKey, getAIBaseURL, getAIGatewayProviderLabel } from "./lib/aiGateway"; import { getShallowRunAfter, routeEventRef } from "./notifications/functionRefs"; import { resolveVisitorFromSession } from "./widgetSessions"; @@ -91,7 +92,7 @@ const DEFAULT_AI_SETTINGS = { workingHours: null, model: "openai/gpt-5-nano", suggestionsEnabled: false, - embeddingModel: "text-embedding-3-small", + embeddingModel: DEFAULT_CONTENT_EMBEDDING_MODEL, lastConfigError: null, }; @@ -228,7 +229,9 @@ function withAISettingDefaults(settings: Doc<"aiAgentSettings"> | null): { handoffMessage: settings.handoffMessage ?? DEFAULT_AI_SETTINGS.handoffMessage, workingHours: settings.workingHours ?? null, suggestionsEnabled: settings.suggestionsEnabled ?? false, - embeddingModel: settings.embeddingModel ?? DEFAULT_AI_SETTINGS.embeddingModel, + embeddingModel: resolveContentEmbeddingModel( + settings.embeddingModel ?? DEFAULT_AI_SETTINGS.embeddingModel + ), lastConfigError: settings.lastConfigError ?? null, }; } @@ -377,7 +380,8 @@ export const updateSettings = authMutation({ if (args.model !== undefined) updates.model = args.model; if (args.suggestionsEnabled !== undefined) updates.suggestionsEnabled = args.suggestionsEnabled; - if (args.embeddingModel !== undefined) updates.embeddingModel = args.embeddingModel; + if (args.embeddingModel !== undefined) + updates.embeddingModel = resolveContentEmbeddingModel(args.embeddingModel); await ctx.db.patch(existing._id, updates); return existing._id; @@ -394,7 +398,9 @@ export const updateSettings = authMutation({ workingHours: args.workingHours ?? undefined, model: args.model ?? "openai/gpt-5-nano", suggestionsEnabled: args.suggestionsEnabled ?? false, - embeddingModel: args.embeddingModel ?? "text-embedding-3-small", + embeddingModel: resolveContentEmbeddingModel( + args.embeddingModel ?? DEFAULT_CONTENT_EMBEDDING_MODEL + ), createdAt: now, updatedAt: now, }); diff --git a/packages/convex/convex/aiAgentActionsKnowledge.ts b/packages/convex/convex/aiAgentActionsKnowledge.ts index 67debcb..fdf6809 100644 --- a/packages/convex/convex/aiAgentActionsKnowledge.ts +++ b/packages/convex/convex/aiAgentActionsKnowledge.ts @@ -2,10 +2,10 @@ import { internalAction } from "./_generated/server"; import { v } from "convex/values"; import { embed } from "ai"; import { createAIClient } from "./lib/aiGateway"; +import { DEFAULT_CONTENT_EMBEDDING_MODEL, resolveContentEmbeddingModel } from "./lib/embeddingModels"; import { makeFunctionReference, type FunctionReference } from "convex/server"; import type { Id } from "./_generated/dataModel"; -const DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small"; const RUNTIME_KNOWLEDGE_DEFAULT_LIMIT = 5; const RUNTIME_KNOWLEDGE_MAX_LIMIT = 20; @@ -75,7 +75,9 @@ export const getRelevantKnowledgeForRuntimeAction = internalAction({ const aiClient = createAIClient(); const runQuery = getShallowRunQuery(ctx); - const embeddingModel = args.embeddingModel?.trim() || DEFAULT_EMBEDDING_MODEL; + const embeddingModel = resolveContentEmbeddingModel( + args.embeddingModel?.trim() || DEFAULT_CONTENT_EMBEDDING_MODEL + ); // 1. Embed the query const { embedding } = await embed({ diff --git a/packages/convex/convex/embeddings.ts b/packages/convex/convex/embeddings.ts index 82d6f7d..be1d123 100644 --- a/packages/convex/convex/embeddings.ts +++ b/packages/convex/convex/embeddings.ts @@ -5,13 +5,13 @@ import { Doc, Id } from "./_generated/dataModel"; import { embedMany } from "ai"; import { authAction } from "./lib/authWrappers"; import { createAIClient } from "./lib/aiGateway"; +import { DEFAULT_CONTENT_EMBEDDING_MODEL, resolveContentEmbeddingModel } from "./lib/embeddingModels"; import { isInternalArticle, isPublicArticle, listUnifiedArticlesWithLegacyFallback, } from "./lib/unifiedArticles"; -const DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small"; const EMBEDDING_GENERATE_CONCURRENCY = 4; const EMBEDDING_BACKFILL_BATCH_CONCURRENCY = 2; @@ -315,7 +315,9 @@ export const generateInternal = internalAction({ return { id: existing[0]._id, skipped: true }; } - const modelName = args.model || DEFAULT_EMBEDDING_MODEL; + const modelName = resolveContentEmbeddingModel( + args.model?.trim() || DEFAULT_CONTENT_EMBEDDING_MODEL + ); const aiClient = createAIClient(); const chunkTexts = chunks.map((chunk) => `${args.title}\n\n${chunk}`); const { embeddings } = await embedMany({ diff --git a/packages/convex/convex/lib/embeddingModels.ts b/packages/convex/convex/lib/embeddingModels.ts new file mode 100644 index 0000000..9903eb1 --- /dev/null +++ b/packages/convex/convex/lib/embeddingModels.ts @@ -0,0 +1,40 @@ +export const CONTENT_EMBEDDING_INDEX_DIMENSIONS = 1536; +export const DEFAULT_CONTENT_EMBEDDING_MODEL = "text-embedding-3-small"; +export const LARGE_CONTENT_EMBEDDING_MODEL = "text-embedding-3-large"; +export const LEGACY_CONTENT_EMBEDDING_MODEL = "text-embedding-ada-002"; + +function normalizeModelName(model: string | undefined): string { + return model?.trim().toLowerCase() ?? ""; +} + +export function getDefaultContentEmbeddingModel(): string { + return DEFAULT_CONTENT_EMBEDDING_MODEL; +} + +export function getContentEmbeddingIndexDimensions(): number { + return CONTENT_EMBEDDING_INDEX_DIMENSIONS; +} + +export function isContentEmbeddingModelSupportedByCurrentIndex(model: string | undefined): boolean { + const normalized = normalizeModelName(model); + return ( + normalized === "" || + normalized === DEFAULT_CONTENT_EMBEDDING_MODEL || + normalized === LEGACY_CONTENT_EMBEDDING_MODEL + ); +} + +export function resolveContentEmbeddingModel(model: string | undefined): string { + return isContentEmbeddingModelSupportedByCurrentIndex(model) + ? model?.trim() || DEFAULT_CONTENT_EMBEDDING_MODEL + : DEFAULT_CONTENT_EMBEDDING_MODEL; +} + +export function getContentEmbeddingModelCompatibilityMessage(model: string | undefined): string | null { + const normalized = normalizeModelName(model); + if (normalized === LARGE_CONTENT_EMBEDDING_MODEL) { + return `The current content embedding index expects ${CONTENT_EMBEDDING_INDEX_DIMENSIONS}-dimension vectors, so ${LARGE_CONTENT_EMBEDDING_MODEL} is not compatible yet. Falling back to ${DEFAULT_CONTENT_EMBEDDING_MODEL}.`; + } + + return null; +} diff --git a/packages/convex/convex/schema/operationsAiTables.ts b/packages/convex/convex/schema/operationsAiTables.ts index eb2c20a..b5e8a83 100644 --- a/packages/convex/convex/schema/operationsAiTables.ts +++ b/packages/convex/convex/schema/operationsAiTables.ts @@ -1,5 +1,6 @@ import { defineTable } from "convex/server"; import { v } from "convex/values"; +import { CONTENT_EMBEDDING_INDEX_DIMENSIONS } from "../lib/embeddingModels"; export const operationsAiTables = { aiAgentSettings: defineTable({ @@ -49,7 +50,7 @@ export const operationsAiTables = { .index("by_workspace", ["workspaceId"]) .vectorIndex("by_embedding", { vectorField: "embedding", - dimensions: 1536, + dimensions: CONTENT_EMBEDDING_INDEX_DIMENSIONS, filterFields: ["workspaceId", "contentType"], }), diff --git a/packages/convex/convex/suggestions.ts b/packages/convex/convex/suggestions.ts index 60a5bb7..f399112 100644 --- a/packages/convex/convex/suggestions.ts +++ b/packages/convex/convex/suggestions.ts @@ -5,10 +5,10 @@ import { Doc, Id } from "./_generated/dataModel"; import { embed } from "ai"; import { authAction, authMutation, authQuery } from "./lib/authWrappers"; import { createAIClient } from "./lib/aiGateway"; +import { DEFAULT_CONTENT_EMBEDDING_MODEL, resolveContentEmbeddingModel } from "./lib/embeddingModels"; import { getUnifiedArticleByIdOrLegacyInternalId, isInternalArticle } from "./lib/unifiedArticles"; import type { Permission } from "./permissions"; -const DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small"; const FEEDBACK_STATS_DEFAULT_LIMIT = 5000; const FEEDBACK_STATS_MAX_LIMIT = 20000; const SUGGESTIONS_DEFAULT_LIMIT = 10; @@ -258,7 +258,9 @@ export const searchSimilar = authAction({ 1, Math.min(args.limit ?? SUGGESTIONS_DEFAULT_LIMIT, SUGGESTIONS_MAX_LIMIT) ); - const modelName = args.model || DEFAULT_EMBEDDING_MODEL; + const modelName = resolveContentEmbeddingModel( + args.model?.trim() || DEFAULT_CONTENT_EMBEDDING_MODEL + ); const aiClient = createAIClient(); const runQuery = getShallowRunQuery(ctx); @@ -403,7 +405,9 @@ export const searchSimilarInternal = internalAction({ 1, Math.min(args.limit ?? SUGGESTIONS_DEFAULT_LIMIT, SUGGESTIONS_MAX_LIMIT) ); - const modelName = args.model || DEFAULT_EMBEDDING_MODEL; + const modelName = resolveContentEmbeddingModel( + args.model?.trim() || DEFAULT_CONTENT_EMBEDDING_MODEL + ); const aiClient = createAIClient(); const runQuery = getShallowRunQuery(ctx); @@ -715,7 +719,7 @@ export const searchForWidget = action({ const aiClient = createAIClient(); const { embedding } = await embed({ - model: aiClient.embedding(DEFAULT_EMBEDDING_MODEL), + model: aiClient.embedding(DEFAULT_CONTENT_EMBEDDING_MODEL), value: normalizedQuery, }); diff --git a/packages/convex/tests/embeddingModels.test.ts b/packages/convex/tests/embeddingModels.test.ts new file mode 100644 index 0000000..fb6cc9e --- /dev/null +++ b/packages/convex/tests/embeddingModels.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; + +import { + CONTENT_EMBEDDING_INDEX_DIMENSIONS, + DEFAULT_CONTENT_EMBEDDING_MODEL, + getContentEmbeddingModelCompatibilityMessage, + isContentEmbeddingModelSupportedByCurrentIndex, + LEGACY_CONTENT_EMBEDDING_MODEL, + resolveContentEmbeddingModel, +} from "../convex/lib/embeddingModels"; + +describe("content embedding model compatibility", () => { + it("keeps index-compatible models unchanged", () => { + expect(resolveContentEmbeddingModel(DEFAULT_CONTENT_EMBEDDING_MODEL)).toBe( + DEFAULT_CONTENT_EMBEDDING_MODEL + ); + expect(resolveContentEmbeddingModel(LEGACY_CONTENT_EMBEDDING_MODEL)).toBe( + LEGACY_CONTENT_EMBEDDING_MODEL + ); + expect(isContentEmbeddingModelSupportedByCurrentIndex(DEFAULT_CONTENT_EMBEDDING_MODEL)).toBe(true); + expect(isContentEmbeddingModelSupportedByCurrentIndex(LEGACY_CONTENT_EMBEDDING_MODEL)).toBe(true); + }); + + it("falls back from incompatible large embeddings to the default index-compatible model", () => { + expect(resolveContentEmbeddingModel("text-embedding-3-large")).toBe( + DEFAULT_CONTENT_EMBEDDING_MODEL + ); + expect(isContentEmbeddingModelSupportedByCurrentIndex("text-embedding-3-large")).toBe(false); + expect(getContentEmbeddingModelCompatibilityMessage("text-embedding-3-large")).toContain( + String(CONTENT_EMBEDDING_INDEX_DIMENSIONS) + ); + }); +}); From 01b005d84dfbb254ac1b7c3efd9f58148ea48d99 Mon Sep 17 00:00:00 2001 From: Jack D Date: Wed, 15 Apr 2026 23:11:35 -0600 Subject: [PATCH 3/8] fix tests --- apps/web/e2e/global-teardown.ts | 2 +- apps/web/e2e/helpers/test-data.ts | 6 ++- packages/convex/convex/aiAgentActions.ts | 37 ++++++++++++------- packages/convex/convex/testAdmin.ts | 25 +++++++++++-- .../aiAgentAuthorizationSemantics.test.ts | 2 + packages/convex/tests/helpers/testHelpers.ts | 2 +- .../convex/tests/setupTestAdminFallback.ts | 2 +- scripts/screenshots/seed.ts | 2 +- security/convex-v-any-arg-exceptions.json | 14 +------ 9 files changed, 57 insertions(+), 35 deletions(-) diff --git a/apps/web/e2e/global-teardown.ts b/apps/web/e2e/global-teardown.ts index 2cb6bf9..84bf2f7 100644 --- a/apps/web/e2e/global-teardown.ts +++ b/apps/web/e2e/global-teardown.ts @@ -46,7 +46,7 @@ async function globalTeardown() { args: { secret: adminSecret, name: "testing/helpers:cleanupE2ETestData", - mutationArgs: {}, + mutationArgsJson: JSON.stringify({}), }, format: "json", }), diff --git a/apps/web/e2e/helpers/test-data.ts b/apps/web/e2e/helpers/test-data.ts index 78835e9..cf9df43 100644 --- a/apps/web/e2e/helpers/test-data.ts +++ b/apps/web/e2e/helpers/test-data.ts @@ -37,7 +37,11 @@ async function callInternalMutation(path: string, args: Record Promise; } +function maybeGetShallowRunAction(ctx: { runAction?: unknown }) { + if (typeof ctx.runAction !== "function") { + return null; + } + return getShallowRunAction(ctx as { runAction: unknown }); +} + const GENERATION_FAILURE_FALLBACK_RESPONSE = "I'm having trouble processing your request right now. Let me connect you with a human agent."; const EMPTY_RESPONSE_RETRY_LIMIT = 1; @@ -477,7 +484,7 @@ export const generateResponse = action({ const runQuery = getShallowRunQuery(ctx); const runMutation = getShallowRunMutation(ctx); - const runAction = getShallowRunAction(ctx); + const runAction = maybeGetShallowRunAction(ctx); const access = await runQuery(AUTHORIZE_CONVERSATION_ACCESS_REF, { conversationId: args.conversationId, visitorId: args.visitorId, @@ -580,19 +587,21 @@ export const generateResponse = action({ // Get relevant knowledge let knowledgeResults: RelevantKnowledgeResult[] = []; - try { - knowledgeResults = await runAction(GET_RELEVANT_KNOWLEDGE_FOR_RUNTIME_ACTION_REF, { - workspaceId: args.workspaceId, - query: args.query, - knowledgeSources: settings.knowledgeSources, - limit: 5, - embeddingModel: settings.embeddingModel, - }); - } catch (retrievalError) { - console.error( - "Knowledge retrieval failed; continuing without knowledge context:", - retrievalError - ); + if (runAction) { + try { + knowledgeResults = await runAction(GET_RELEVANT_KNOWLEDGE_FOR_RUNTIME_ACTION_REF, { + workspaceId: args.workspaceId, + query: args.query, + knowledgeSources: settings.knowledgeSources, + limit: 5, + embeddingModel: settings.embeddingModel, + }); + } catch (retrievalError) { + console.error( + "Knowledge retrieval failed; continuing without knowledge context:", + retrievalError + ); + } } // Build knowledge context for prompt diff --git a/packages/convex/convex/testAdmin.ts b/packages/convex/convex/testAdmin.ts index abbeaef..e30a000 100644 --- a/packages/convex/convex/testAdmin.ts +++ b/packages/convex/convex/testAdmin.ts @@ -38,6 +38,25 @@ function getShallowRunMutation(ctx: { runMutation: unknown }) { ) => Promise; } +function parseMutationArgsJson(mutationArgsJson: string): Record { + if (!mutationArgsJson.trim()) { + return {}; + } + + let parsed: unknown; + try { + parsed = JSON.parse(mutationArgsJson); + } catch { + throw new Error("mutationArgsJson must be valid JSON."); + } + + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("mutationArgsJson must decode to a JSON object."); + } + + return parsed as Record; +} + export function isAuthorizedAdminSecret(providedSecret: string, expectedSecret: string): boolean { const providedBytes = toSecretBytes(providedSecret); const expectedBytes = toSecretBytes(expectedSecret); @@ -62,9 +81,9 @@ export const runTestMutation = action({ args: { secret: v.string(), name: v.string(), - mutationArgs: v.any(), + mutationArgsJson: v.string(), }, - handler: async (ctx, { secret, name, mutationArgs }) => { + handler: async (ctx, { secret, name, mutationArgsJson }) => { // Validate admin secret const expected = process.env.TEST_ADMIN_SECRET; if (!expected) { @@ -92,6 +111,6 @@ export const runTestMutation = action({ } const runMutation = getShallowRunMutation(ctx); - return await runMutation(getInternalRef(name), (mutationArgs ?? {}) as Record); + return await runMutation(getInternalRef(name), parseMutationArgsJson(mutationArgsJson)); }, }); diff --git a/packages/convex/tests/aiAgentAuthorizationSemantics.test.ts b/packages/convex/tests/aiAgentAuthorizationSemantics.test.ts index 3d64608..41a636f 100644 --- a/packages/convex/tests/aiAgentAuthorizationSemantics.test.ts +++ b/packages/convex/tests/aiAgentAuthorizationSemantics.test.ts @@ -67,12 +67,14 @@ describe("aiAgent authorization semantics", () => { { _id: "ai_response_1", conversationId, + messageId: "message_auth_1", response: "Response 1", confidence: 0.8, }, { _id: "ai_response_2", conversationId, + messageId: "message_auth_2", response: "Response 2", confidence: 0.7, }, diff --git a/packages/convex/tests/helpers/testHelpers.ts b/packages/convex/tests/helpers/testHelpers.ts index 37d6659..253eff1 100644 --- a/packages/convex/tests/helpers/testHelpers.ts +++ b/packages/convex/tests/helpers/testHelpers.ts @@ -24,7 +24,7 @@ async function callInternalTestMutation( headers: { "Content-Type": "application/json" }, body: JSON.stringify({ path: "testAdmin:runTestMutation", - args: { secret, name, mutationArgs }, + args: { secret, name, mutationArgsJson: JSON.stringify(mutationArgs) }, format: "json", }), }); diff --git a/packages/convex/tests/setupTestAdminFallback.ts b/packages/convex/tests/setupTestAdminFallback.ts index c5a4525..bf19b5e 100644 --- a/packages/convex/tests/setupTestAdminFallback.ts +++ b/packages/convex/tests/setupTestAdminFallback.ts @@ -26,7 +26,7 @@ async function callInternalTestMutation(name: string, mutationArgs: Record Date: Thu, 16 Apr 2026 11:19:55 -0600 Subject: [PATCH 4/8] Address type issues etc --- apps/web/src/app/settings/AIAgentSection.tsx | 33 ++--- .../hooks/useSettingsSectionsConvex.ts | 130 ++++++++++++------ packages/convex/convex/aiAgent.ts | 95 +++++++++---- packages/convex/convex/aiAgentActions.ts | 3 +- packages/convex/convex/lib/embeddingModels.ts | 11 +- .../convex/tests/aiAgentRuntimeSafety.test.ts | 108 ++++++++++++++- packages/convex/tests/embeddingModels.test.ts | 9 ++ 7 files changed, 293 insertions(+), 96 deletions(-) diff --git a/apps/web/src/app/settings/AIAgentSection.tsx b/apps/web/src/app/settings/AIAgentSection.tsx index 944bd27..f0b6e36 100644 --- a/apps/web/src/app/settings/AIAgentSection.tsx +++ b/apps/web/src/app/settings/AIAgentSection.tsx @@ -15,7 +15,7 @@ export function AIAgentSection({ }: { workspaceId?: Id<"workspaces">; }): React.JSX.Element | null { - const { aiSettings, availableModels, updateSettings } = useAIAgentSectionConvex(workspaceId); + const { aiSettings, availableModels, isSaving, saveSettings } = useAIAgentSectionConvex(workspaceId); const [enabled, setEnabled] = useState(false); const [model, setModel] = useState("openai/gpt-5-nano"); @@ -25,7 +25,6 @@ export function AIAgentSection({ const [handoffMessage, setHandoffMessage] = useState(""); const [suggestionsEnabled, setSuggestionsEnabled] = useState(false); const [embeddingModel, setEmbeddingModel] = useState("text-embedding-3-small"); - const [isSaving, setIsSaving] = useState(false); const normalizedModel = normalizeModelValue(model); const selectedDiscoveredModel = availableModels?.some((availableModel) => availableModel.id === normalizedModel) ?? false @@ -47,25 +46,19 @@ export function AIAgentSection({ const handleSave = async () => { if (!workspaceId) return; - setIsSaving(true); - const nextModel = normalizeModelValue(model); - try { - await updateSettings({ - workspaceId, - enabled, - model: nextModel, - confidenceThreshold, - knowledgeSources: knowledgeSources as ("articles" | "internalArticles" | "snippets")[], - personality: personality || undefined, - handoffMessage: handoffMessage || undefined, - suggestionsEnabled, - embeddingModel, - }); + const nextModel = await saveSettings({ + workspaceId, + enabled, + model, + confidenceThreshold, + knowledgeSources: knowledgeSources as ("articles" | "internalArticles" | "snippets")[], + personality, + handoffMessage, + suggestionsEnabled, + embeddingModel, + }); + if (nextModel) { setModel(nextModel); - } catch (error) { - console.error("Failed to save AI settings:", error); - } finally { - setIsSaving(false); } }; diff --git a/apps/web/src/app/settings/hooks/useSettingsSectionsConvex.ts b/apps/web/src/app/settings/hooks/useSettingsSectionsConvex.ts index a97ce62..34d4ecb 100644 --- a/apps/web/src/app/settings/hooks/useSettingsSectionsConvex.ts +++ b/apps/web/src/app/settings/hooks/useSettingsSectionsConvex.ts @@ -1,6 +1,6 @@ "use client"; -import { useEffect, useState } from "react"; +import { useCallback, useEffect, useState } from "react"; import type { Id } from "@opencom/convex/dataModel"; import type { HomeCard, HomeConfig, HomeDefaultSpace, HomeTab } from "@opencom/types"; import { @@ -16,6 +16,55 @@ type WorkspaceArgs = { workspaceId: Id<"workspaces">; }; +type AIAgentKnowledgeSource = "articles" | "internalArticles" | "snippets"; + +type AIAgentSettingsQueryResult = { + enabled: boolean; + model: string; + confidenceThreshold: number; + knowledgeSources: AIAgentKnowledgeSource[]; + personality?: string; + handoffMessage?: string; + suggestionsEnabled?: boolean; + embeddingModel?: string; + lastConfigError?: { + message: string; + code: string; + provider?: string; + model?: string; + } | null; +} | null; + +type AvailableAIAgentModel = { + id: string; + name: string; + provider: string; +}; + +type UpdateAIAgentSettingsArgs = { + workspaceId: Id<"workspaces">; + enabled?: boolean; + model?: string; + confidenceThreshold?: number; + knowledgeSources?: AIAgentKnowledgeSource[]; + personality?: string; + handoffMessage?: string; + suggestionsEnabled?: boolean; + embeddingModel?: string; +}; + +type SaveAIAgentSettingsArgs = { + workspaceId: Id<"workspaces">; + enabled: boolean; + model: string; + confidenceThreshold: number; + knowledgeSources: AIAgentKnowledgeSource[]; + personality: string; + handoffMessage: string; + suggestionsEnabled: boolean; + embeddingModel: string; +}; + type AuditAccessRecord = { status: "unauthenticated" | "forbidden" | "ok"; canManageSecurity?: boolean; @@ -65,43 +114,16 @@ type SuccessResponse = { success: boolean; }; -const AI_SETTINGS_QUERY_REF = webQueryRef< - WorkspaceArgs, - { - enabled: boolean; - model: string; - confidenceThreshold: number; - knowledgeSources: string[]; - personality?: string; - handoffMessage?: string; - suggestionsEnabled?: boolean; - embeddingModel?: string; - lastConfigError?: { - message: string; - code: string; - provider?: string; - model?: string; - } | null; - } | null ->("aiAgent:getSettings"); +const AI_SETTINGS_QUERY_REF = webQueryRef( + "aiAgent:getSettings" +); const AVAILABLE_MODELS_ACTION_REF = webActionRef< { workspaceId: Id<"workspaces">; selectedModel?: string }, - Array<{ id: string; name: string; provider: string }> + AvailableAIAgentModel[] >("aiAgent:listAvailableModels"); -const UPDATE_AI_SETTINGS_REF = webMutationRef< - { - workspaceId: Id<"workspaces">; - enabled?: boolean; - model?: string; - confidenceThreshold?: number; - knowledgeSources?: Array<"articles" | "internalArticles" | "snippets">; - personality?: string; - handoffMessage?: string; - suggestionsEnabled?: boolean; - embeddingModel?: string; - }, - null ->("aiAgent:updateSettings"); +const UPDATE_AI_SETTINGS_REF = webMutationRef>( + "aiAgent:updateSettings" +); const AUTOMATION_SETTINGS_QUERY_REF = webQueryRef< WorkspaceArgs, { @@ -281,9 +303,38 @@ const TRANSFER_OWNERSHIP_REF = webMutationRef) { const aiSettings = useWebQuery(AI_SETTINGS_QUERY_REF, workspaceId ? { workspaceId } : "skip"); const listAvailableModels = useWebAction(AVAILABLE_MODELS_ACTION_REF); - const [availableModels, setAvailableModels] = useState< - Array<{ id: string; name: string; provider: string }> | undefined - >(undefined); + const updateAIAgentSettings = useWebMutation(UPDATE_AI_SETTINGS_REF); + const [availableModels, setAvailableModels] = useState( + undefined + ); + const [isSaving, setIsSaving] = useState(false); + + const saveSettings = useCallback( + async (settings: SaveAIAgentSettingsArgs): Promise => { + const normalizedModel = settings.model.trim(); + setIsSaving(true); + try { + await updateAIAgentSettings({ + workspaceId: settings.workspaceId, + enabled: settings.enabled, + model: normalizedModel, + confidenceThreshold: settings.confidenceThreshold, + knowledgeSources: settings.knowledgeSources, + personality: settings.personality || undefined, + handoffMessage: settings.handoffMessage || undefined, + suggestionsEnabled: settings.suggestionsEnabled, + embeddingModel: settings.embeddingModel, + }); + return normalizedModel; + } catch (error) { + console.error("Failed to save AI settings:", error); + return null; + } finally { + setIsSaving(false); + } + }, + [updateAIAgentSettings] + ); useEffect(() => { let cancelled = false; @@ -319,7 +370,8 @@ export function useAIAgentSectionConvex(workspaceId?: Id<"workspaces">) { return { aiSettings, availableModels, - updateSettings: useWebMutation(UPDATE_AI_SETTINGS_REF), + isSaving, + saveSettings, }; } diff --git a/packages/convex/convex/aiAgent.ts b/packages/convex/convex/aiAgent.ts index ffd2342..bb114b5 100644 --- a/packages/convex/convex/aiAgent.ts +++ b/packages/convex/convex/aiAgent.ts @@ -46,6 +46,28 @@ type OpenAIModelListResponse = { }>; }; +type StoredAIResponse = Doc<"aiResponses">; +type StoredAIResponseSource = StoredAIResponse["sources"][number]; +type StoredAIResponseWithMessage = StoredAIResponse & { + messageId: Id<"messages">; +}; + +type ConversationResponse = Omit & { + messageId: Id<"messages">; + deliveredResponseContext: { + response: string; + sources: StoredAIResponseSource[]; + confidence: number | null; + }; + generatedResponseContext: + | { + response: string; + sources: StoredAIResponseSource[]; + confidence: number; + } + | null; +}; + type GetRelevantKnowledgeForRuntimeActionArgs = { workspaceId: Id<"workspaces">; query: string; @@ -123,12 +145,22 @@ function createAvailableAIModel(value: string | undefined): AvailableAIModel | n return null; } - const [provider, ...modelParts] = normalizedId.split("/"); - const model = modelParts.join("/") || normalizedId; + const defaultProvider = getAIGatewayProviderLabel(getAIBaseURL(getAIGatewayApiKey())); + const providerSeparatorIndex = normalizedId.indexOf("/"); + if (providerSeparatorIndex === -1) { + return { + id: normalizedId, + name: normalizedId, + provider: defaultProvider, + }; + } + + const provider = normalizedId.slice(0, providerSeparatorIndex).trim(); + const model = normalizedId.slice(providerSeparatorIndex + 1).trim(); return { id: normalizedId, - name: model, - provider, + name: model || normalizedId, + provider: provider || defaultProvider, }; } @@ -138,7 +170,13 @@ function isLikelyGenerationModel(modelId: string): boolean { return false; } - return !NON_GENERATION_MODEL_PREFIXES.some((prefix) => normalized.startsWith(prefix)); + const modelPathParts = normalized.split("/").map((part) => part.trim()).filter(Boolean); + const modelName = modelPathParts[modelPathParts.length - 1] ?? ""; + if (!modelName) { + return false; + } + + return !NON_GENERATION_MODEL_PREFIXES.some((prefix) => modelName.startsWith(prefix)); } function dedupeAvailableAIModels(models: AvailableAIModel[]): AvailableAIModel[] { @@ -245,6 +283,7 @@ async function getWorkspaceAISettings( .withIndex("by_workspace", (q) => q.eq("workspaceId", workspaceId)) .first(); } + async function requireConversationAccess( ctx: QueryCtx | MutationCtx, args: { @@ -357,7 +396,7 @@ export const updateSettings = authMutation({ embeddingModel: v.optional(v.string()), }, permission: "settings.workspace", - handler: async (ctx, args) => { + handler: async (ctx, args): Promise> => { const now = Date.now(); const existing = await ctx.db @@ -501,7 +540,7 @@ export const getRelevantKnowledgeForRuntime = internalQuery({ knowledgeSources: v.optional(v.array(knowledgeSourceValidator)), limit: v.optional(v.number()), }, - handler: async () => { + handler: async (): Promise => { return []; }, }); @@ -527,7 +566,7 @@ export const storeResponse = mutation({ model: v.string(), provider: v.string(), }, - handler: async (ctx, args) => { + handler: async (ctx, args): Promise> => { await requireConversationAccess(ctx, { conversationId: args.conversationId, visitorId: args.visitorId, @@ -573,7 +612,7 @@ export const submitFeedback = mutation({ visitorId: v.optional(v.id("visitors")), sessionToken: v.optional(v.string()), }, - handler: async (ctx, args) => { + handler: async (ctx, args): Promise => { const response = await ctx.db.get(args.responseId); if (!response) { throw new Error("AI response not found"); @@ -596,7 +635,7 @@ export const getConversationResponses = query({ visitorId: v.optional(v.id("visitors")), sessionToken: v.optional(v.string()), }, - handler: async (ctx, args) => { + handler: async (ctx, args): Promise => { await requireConversationAccess(ctx, { conversationId: args.conversationId, visitorId: args.visitorId, @@ -608,22 +647,26 @@ export const getConversationResponses = query({ .withIndex("by_conversation", (q) => q.eq("conversationId", args.conversationId)) .collect(); - return responses.filter((response) => response.messageId !== undefined).map((response) => ({ - ...response, - deliveredResponseContext: { - response: response.response, - sources: response.sources, - confidence: response.handedOff ? null : response.confidence, - }, - generatedResponseContext: - response.generatedCandidateResponse === undefined - ? null - : { - response: response.generatedCandidateResponse, - sources: response.generatedCandidateSources ?? [], - confidence: response.generatedCandidateConfidence ?? response.confidence, - }, - })); + return responses + .filter( + (response): response is StoredAIResponseWithMessage => response.messageId !== undefined + ) + .map((response) => ({ + ...response, + deliveredResponseContext: { + response: response.response, + sources: response.sources, + confidence: response.handedOff ? null : response.confidence, + }, + generatedResponseContext: + response.generatedCandidateResponse === undefined + ? null + : { + response: response.generatedCandidateResponse, + sources: response.generatedCandidateSources ?? [], + confidence: response.generatedCandidateConfidence ?? response.confidence, + }, + })); }, }); diff --git a/packages/convex/convex/aiAgentActions.ts b/packages/convex/convex/aiAgentActions.ts index d649f89..10e977d 100644 --- a/packages/convex/convex/aiAgentActions.ts +++ b/packages/convex/convex/aiAgentActions.ts @@ -293,6 +293,7 @@ export const getAIConfigurationDiagnostic = ( }; } + const providerLabel = environment.aiGatewayProviderLabel ?? getAIGatewayProviderLabel(); const parts = trimmedModel.split("/"); if (parts.length > 2 || (parts.length === 2 && (!parts[0] || !parts[1]))) { return { @@ -303,7 +304,7 @@ export const getAIConfigurationDiagnostic = ( }; } - const provider = parts.length === 2 ? parts[0] : getAIGatewayProviderLabel(); + const provider = parts.length === 2 ? parts[0] : providerLabel; const model = parts.length === 2 ? parts[1] : trimmedModel; if (!provider || !model) { return { diff --git a/packages/convex/convex/lib/embeddingModels.ts b/packages/convex/convex/lib/embeddingModels.ts index 9903eb1..83ee9a9 100644 --- a/packages/convex/convex/lib/embeddingModels.ts +++ b/packages/convex/convex/lib/embeddingModels.ts @@ -25,9 +25,14 @@ export function isContentEmbeddingModelSupportedByCurrentIndex(model: string | u } export function resolveContentEmbeddingModel(model: string | undefined): string { - return isContentEmbeddingModelSupportedByCurrentIndex(model) - ? model?.trim() || DEFAULT_CONTENT_EMBEDDING_MODEL - : DEFAULT_CONTENT_EMBEDDING_MODEL; + const normalized = normalizeModelName(model); + if (normalized === "" || normalized === DEFAULT_CONTENT_EMBEDDING_MODEL) { + return DEFAULT_CONTENT_EMBEDDING_MODEL; + } + if (normalized === LEGACY_CONTENT_EMBEDDING_MODEL) { + return LEGACY_CONTENT_EMBEDDING_MODEL; + } + return DEFAULT_CONTENT_EMBEDDING_MODEL; } export function getContentEmbeddingModelCompatibilityMessage(model: string | undefined): string | null { diff --git a/packages/convex/tests/aiAgentRuntimeSafety.test.ts b/packages/convex/tests/aiAgentRuntimeSafety.test.ts index 8ef3b3b..ef98aaf 100644 --- a/packages/convex/tests/aiAgentRuntimeSafety.test.ts +++ b/packages/convex/tests/aiAgentRuntimeSafety.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("ai", () => ({ generateText: vi.fn(), @@ -11,11 +11,38 @@ vi.mock("@ai-sdk/openai", () => ({ })); import { generateText } from "ai"; -import { generateResponse, getAIConfigurationDiagnostic } from "../convex/aiAgentActions"; +import { listAvailableModels as listAvailableModelsDefinition } from "../convex/aiAgent"; +import { + generateResponse as generateResponseDefinition, + getAIConfigurationDiagnostic, +} from "../convex/aiAgentActions"; const mockGenerateText = vi.mocked(generateText); +type GenerateResponseHandlerResult = { + response: string; + confidence: number; + sources: Array>; + handoff: boolean; + handoffReason: string | null; + messageId: string | null; +}; +const generateResponse = generateResponseDefinition as unknown as { + _handler: (ctx: unknown, args: Record) => Promise; +}; +const generateResponseHandler = generateResponse; +const listAvailableModels = listAvailableModelsDefinition as unknown as { + _handler: ( + ctx: unknown, + args: Record + ) => Promise>; +}; +const listAvailableModelsHandler = listAvailableModels; describe("aiAgentActions runtime safety", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + beforeEach(() => { vi.clearAllMocks(); process.env.AI_GATEWAY_API_KEY = "test-ai-key"; @@ -44,6 +71,73 @@ describe("aiAgentActions runtime safety", () => { expect(getAIConfigurationDiagnostic("anthropic/claude-3-5-sonnet")).toBeNull(); }); + it("uses the provided gateway provider label for raw-model diagnostics", () => { + expect( + getAIConfigurationDiagnostic("glm-5-turbo", { + aiGatewayApiKey: undefined, + aiGatewayProviderLabel: "zai", + }) + ).toMatchObject({ + code: "MISSING_PROVIDER_CREDENTIALS", + provider: "zai", + model: "glm-5-turbo", + }); + }); + + it("filters provider-prefixed non-generation models and labels raw fallback models correctly", async () => { + const fetchMock = vi.fn(async () => ({ + ok: true, + json: async () => ({ + data: [ + { id: "openai/text-embedding-3-small", created: 30 }, + { id: "openai/gpt-5-nano", created: 20 }, + { id: "gpt-5.1-mini", created: 10 }, + ], + }), + })); + vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); + + const runQuery = vi.fn(async (_reference: unknown, args: Record) => { + if (Object.keys(args).length === 0) { + return { + user: { + _id: "user_1", + }, + }; + } + if ("permission" in args) { + return null; + } + throw new Error(`Unexpected query args: ${JSON.stringify(args)}`); + }); + + const result = await listAvailableModelsHandler._handler( + { + runQuery, + } as any, + { + workspaceId: "workspace_1" as any, + selectedModel: "custom-raw-model", + } + ); + + expect(result.some((model) => model.id === "openai/text-embedding-3-small")).toBe(false); + expect(result).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "openai/gpt-5-nano", + name: "gpt-5-nano", + provider: "openai", + }), + expect.objectContaining({ + id: "custom-raw-model", + name: "custom-raw-model", + provider: "openai", + }), + ]) + ); + }); + it("falls back to handoff and records diagnostics when configuration is invalid", async () => { const runQuery = vi.fn(async (_reference: unknown, args: Record) => { if ("query" in args) { @@ -78,7 +172,7 @@ describe("aiAgentActions runtime safety", () => { throw new Error(`Unexpected mutation args: ${JSON.stringify(args)}`); }); - const result = await generateResponse._handler( + const result = await generateResponseHandler._handler( { runQuery, runMutation, @@ -131,7 +225,7 @@ describe("aiAgentActions runtime safety", () => { throw new Error(`Unexpected mutation args: ${JSON.stringify(args)}`); }); - const result = await generateResponse._handler( + const result = await generateResponseHandler._handler( { runQuery, runMutation, @@ -161,7 +255,7 @@ describe("aiAgentActions runtime safety", () => { const runMutation = vi.fn(); - const result = await generateResponse._handler( + const result = await generateResponseHandler._handler( { runQuery, runMutation, @@ -417,7 +511,7 @@ describe("aiAgentActions runtime safety", () => { throw new Error(`Unexpected mutation args: ${JSON.stringify(args)}`); }); - await generateResponse._handler( + await generateResponseHandler._handler( { runQuery, runMutation, @@ -862,7 +956,7 @@ describe("aiAgentActions runtime safety", () => { const runMutation = vi.fn(); await expect( - generateResponse._handler( + generateResponseHandler._handler( { runQuery, runMutation, diff --git a/packages/convex/tests/embeddingModels.test.ts b/packages/convex/tests/embeddingModels.test.ts index fb6cc9e..df6f7fa 100644 --- a/packages/convex/tests/embeddingModels.test.ts +++ b/packages/convex/tests/embeddingModels.test.ts @@ -30,4 +30,13 @@ describe("content embedding model compatibility", () => { String(CONTENT_EMBEDDING_INDEX_DIMENSIONS) ); }); + + it("normalizes supported embedding model ids to canonical constants", () => { + expect(resolveContentEmbeddingModel(" Text-Embedding-3-Small ")).toBe( + DEFAULT_CONTENT_EMBEDDING_MODEL + ); + expect(resolveContentEmbeddingModel("TEXT-EMBEDDING-ADA-002")).toBe( + LEGACY_CONTENT_EMBEDDING_MODEL + ); + }); }); From 971d865997e71d1e9f3a0d5253e7b80a1af579d6 Mon Sep 17 00:00:00 2001 From: Jack D Date: Thu, 16 Apr 2026 11:52:02 -0600 Subject: [PATCH 5/8] Address typing issues etc. add docs, tests and display the models in UI --- .../src/app/inbox/InboxAiReviewPanel.test.tsx | 46 +++++++ apps/web/src/app/inbox/InboxAiReviewPanel.tsx | 6 + apps/web/src/app/inbox/inboxRenderTypes.ts | 2 + .../src/components/SuggestionsPanel.test.tsx | 95 ++++++++++++++ apps/web/src/components/SuggestionsPanel.tsx | 14 +++ .../hooks/useSuggestionsPanelConvex.ts | 4 +- docs/api-reference.md | 55 +++++++- .../convex/schema/operationsAiTables.ts | 1 + packages/convex/convex/suggestions.ts | 14 ++- packages/convex/convex/testing/helpers.ts | 2 + packages/convex/convex/testing/helpers/ai.ts | 14 +++ packages/convex/tests/aiAgent.test.ts | 2 + .../aiAgentAuthorizationSemantics.test.ts | 19 ++- packages/convex/tests/suggestions.test.ts | 14 +++ .../suggestionsMetadataSemantics.test.ts | 117 ++++++++++++++++++ 15 files changed, 399 insertions(+), 6 deletions(-) create mode 100644 apps/web/src/app/inbox/InboxAiReviewPanel.test.tsx create mode 100644 apps/web/src/components/SuggestionsPanel.test.tsx create mode 100644 packages/convex/tests/suggestionsMetadataSemantics.test.ts diff --git a/apps/web/src/app/inbox/InboxAiReviewPanel.test.tsx b/apps/web/src/app/inbox/InboxAiReviewPanel.test.tsx new file mode 100644 index 0000000..b50d431 --- /dev/null +++ b/apps/web/src/app/inbox/InboxAiReviewPanel.test.tsx @@ -0,0 +1,46 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import type { Id } from "@opencom/convex/dataModel"; +import { InboxAiReviewPanel } from "./InboxAiReviewPanel"; +import type { InboxAiResponse } from "./inboxRenderTypes"; + +function messageId(value: string): Id<"messages"> { + return value as Id<"messages">; +} + +function responseId(value: string): Id<"aiResponses"> { + return value as Id<"aiResponses">; +} + +describe("InboxAiReviewPanel", () => { + it("renders persisted model and provider metadata for AI responses", () => { + const response: InboxAiResponse = { + _id: responseId("response_1"), + createdAt: Date.now(), + query: "How do I reset my password?", + response: "Go to Settings > Security > Reset Password.", + confidence: 0.82, + model: "openai/gpt-5-nano", + provider: "openai", + handedOff: false, + messageId: messageId("message_1"), + sources: [], + deliveredResponseContext: null, + generatedResponseContext: null, + }; + + render( + reason ?? "No reason"} + /> + ); + + expect(screen.getByText("Model openai/gpt-5-nano")).toBeInTheDocument(); + expect(screen.getByText("Provider openai")).toBeInTheDocument(); + }); +}); diff --git a/apps/web/src/app/inbox/InboxAiReviewPanel.tsx b/apps/web/src/app/inbox/InboxAiReviewPanel.tsx index 14cfcda..91b5b74 100644 --- a/apps/web/src/app/inbox/InboxAiReviewPanel.tsx +++ b/apps/web/src/app/inbox/InboxAiReviewPanel.tsx @@ -136,6 +136,12 @@ export function InboxAiReviewPanel({ {confidenceLabel} {Math.round(confidenceValue * 100)}% + + Model {response.model} + + + Provider {response.provider} + {response.feedback && ( Feedback {response.feedback === "helpful" ? "helpful" : "not helpful"} diff --git a/apps/web/src/app/inbox/inboxRenderTypes.ts b/apps/web/src/app/inbox/inboxRenderTypes.ts index f3809de..1bee376 100644 --- a/apps/web/src/app/inbox/inboxRenderTypes.ts +++ b/apps/web/src/app/inbox/inboxRenderTypes.ts @@ -87,6 +87,8 @@ export interface InboxAiResponse { query: string; response: string; confidence: number; + model: string; + provider: string; handedOff: boolean; handoffReason?: string | null; messageId: Id<"messages">; diff --git a/apps/web/src/components/SuggestionsPanel.test.tsx b/apps/web/src/components/SuggestionsPanel.test.tsx new file mode 100644 index 0000000..0b15d04 --- /dev/null +++ b/apps/web/src/components/SuggestionsPanel.test.tsx @@ -0,0 +1,95 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { Id } from "@opencom/convex/dataModel"; +import { SuggestionsPanel } from "./SuggestionsPanel"; + +const mocks = vi.hoisted(() => ({ + getSuggestions: vi.fn(), + trackUsage: vi.fn(), + trackDismissal: vi.fn(), +})); + +vi.mock("@/components/hooks/useSuggestionsPanelConvex", () => ({ + useSuggestionsPanelConvex: () => ({ + settings: { + suggestionsEnabled: true, + embeddingModel: "text-embedding-3-small", + }, + getSuggestions: mocks.getSuggestions, + trackUsage: mocks.trackUsage, + trackDismissal: mocks.trackDismissal, + }), +})); + +function workspaceId(value: string): Id<"workspaces"> { + return value as Id<"workspaces">; +} + +function conversationId(value: string): Id<"conversations"> { + return value as Id<"conversations">; +} + +describe("SuggestionsPanel", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.getSuggestions.mockResolvedValue([ + { + id: "article_1", + type: "article", + title: "Reset Password", + snippet: "Go to Settings > Security.", + content: "Full reset password content", + score: 0.91, + embeddingModel: "text-embedding-3-small", + }, + ]); + }); + + it("shows the resolved embedding model and passes it into usage tracking", async () => { + render( + + ); + + expect(await screen.findByText("Using embedding model: text-embedding-3-small")).toBeInTheDocument(); + expect(screen.getByText("Embedding model text-embedding-3-small")).toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: /insert/i })); + + await waitFor(() => { + expect(mocks.trackUsage).toHaveBeenCalledWith({ + workspaceId: workspaceId("workspace_1"), + conversationId: conversationId("conversation_1"), + contentType: "article", + contentId: "article_1", + embeddingModel: "text-embedding-3-small", + }); + }); + }); + + it("passes the embedding model into dismissal tracking", async () => { + render( + + ); + + await screen.findByText("Reset Password"); + fireEvent.click(screen.getByRole("button", { name: /dismiss/i })); + + await waitFor(() => { + expect(mocks.trackDismissal).toHaveBeenCalledWith({ + workspaceId: workspaceId("workspace_2"), + conversationId: conversationId("conversation_2"), + contentType: "article", + contentId: "article_1", + embeddingModel: "text-embedding-3-small", + }); + }); + }); +}); diff --git a/apps/web/src/components/SuggestionsPanel.tsx b/apps/web/src/components/SuggestionsPanel.tsx index b12c7f8..720e5e6 100644 --- a/apps/web/src/components/SuggestionsPanel.tsx +++ b/apps/web/src/components/SuggestionsPanel.tsx @@ -37,6 +37,8 @@ export function SuggestionsPanel({ const [dismissedIds, setDismissedIds] = useState>(new Set()); const { settings, getSuggestions, trackUsage, trackDismissal } = useSuggestionsPanelConvex(workspaceId); + const resolvedEmbeddingModel = + suggestions[0]?.embeddingModel ?? settings?.embeddingModel ?? "text-embedding-3-small"; const fetchSuggestions = useCallback(async () => { if (!settings?.suggestionsEnabled) { @@ -82,6 +84,7 @@ export function SuggestionsPanel({ conversationId, contentType: suggestion.type, contentId: suggestion.id, + embeddingModel: suggestion.embeddingModel, }); } catch (err) { console.error("Failed to track usage:", err); @@ -97,6 +100,7 @@ export function SuggestionsPanel({ conversationId, contentType: suggestion.type, contentId: suggestion.id, + embeddingModel: suggestion.embeddingModel, }); } catch (err) { console.error("Failed to track dismissal:", err); @@ -166,6 +170,10 @@ export function SuggestionsPanel({ +

+ Using embedding model: {resolvedEmbeddingModel} +

+ {isLoading && visibleSuggestions.length === 0 && (
@@ -198,6 +206,11 @@ export function SuggestionsPanel({

{suggestion.snippet}

+ {suggestion.embeddingModel && ( +

+ Embedding model {suggestion.embeddingModel} +

+ )}