diff --git a/.changeset/nvidia-nim-availability-fail-open.md b/.changeset/nvidia-nim-availability-fail-open.md new file mode 100644 index 000000000..4c1e610bb --- /dev/null +++ b/.changeset/nvidia-nim-availability-fail-open.md @@ -0,0 +1,5 @@ +--- +"openwiki": patch +--- + +fix: keep NVIDIA NIM availability validation fail-open on inconclusive catalogue lookups diff --git a/.changeset/nvidia-nim-availability.md b/.changeset/nvidia-nim-availability.md new file mode 100644 index 000000000..364aa959c --- /dev/null +++ b/.changeset/nvidia-nim-availability.md @@ -0,0 +1,5 @@ +--- +"openwiki": patch +--- + +feat: validate selected models against custom NVIDIA NIM endpoints before inference diff --git a/src/agent/index.ts b/src/agent/index.ts index fc4b5620f..86b603e14 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -111,6 +111,7 @@ import { OPENWIKI_PROVIDER_ENV_KEY, OPENWIKI_PROVIDER_RETRY_ATTEMPTS_ENV_KEY, OPENWIKI_STREAM_IDLE_TIMEOUT_ENV_KEY, + providerBaseUrlIsCustom, providerRequiresBaseUrl, providerRequiresRegion, providerRequiresSecretKey, @@ -295,6 +296,7 @@ async function resolveRunConfig( onProviderResolved(provider); const providerBaseUrl = resolveProviderBaseUrl(provider); + const baseUrlIsCustom = providerBaseUrlIsCustom(provider); emitDebug(options, `provider=${provider}`); if (providerBaseUrl) { emitDebug( @@ -331,10 +333,11 @@ async function resolveRunConfig( modelId, apiKey: getProviderApiKey(provider), baseUrl: providerBaseUrl, + baseUrlIsCustom, }); if (modelAvailability.status === "unavailable") { throw new Error( - `${getProviderLabel(provider)} does not make model "${modelId}" available to the configured credentials. Set ${OPENWIKI_MODEL_ID_ENV_KEY} to an available model.`, + `${getProviderLabel(provider)} does not make model "${modelId}" available. ${modelAvailability.reason} Set ${OPENWIKI_MODEL_ID_ENV_KEY} to an available model.`, ); } if (modelAvailability.status === "unknown") { diff --git a/src/config/constants.ts b/src/config/constants.ts index 5488cd8db..5e91f1af4 100644 --- a/src/config/constants.ts +++ b/src/config/constants.ts @@ -648,6 +648,38 @@ export function resolveProviderBaseUrl( return config.baseURL; } +/** + * Reports whether the provider points at a user-supplied endpoint rather than + * its built-in default. Compares the resolved base URL against + * {@link ProviderConfig.baseURL} instead of testing whether the environment + * variable is set, so a user who pins the default value explicitly is not + * mistaken for a self-hosted deployment. + */ +export function providerBaseUrlIsCustom( + provider: OpenWikiProvider, + env: NodeJS.ProcessEnv = process.env, +): boolean { + const resolved = resolveProviderBaseUrl(provider, env); + + if (!resolved) { + return false; + } + + const defaultBaseUrl = getProviderConfig(provider).baseURL; + if (!defaultBaseUrl) { + return true; + } + + return ( + normalizeBaseUrlForComparison(resolved) !== + normalizeBaseUrlForComparison(defaultBaseUrl) + ); +} + +function normalizeBaseUrlForComparison(value: string): string { + return value.trim().replace(/\/+$/u, ""); +} + export function getProviderBaseUrlEnvKey( provider: OpenWikiProvider, ): string | undefined { diff --git a/src/model-availability.ts b/src/model-availability.ts index a737cd5ae..d5bf8a7e7 100644 --- a/src/model-availability.ts +++ b/src/model-availability.ts @@ -8,16 +8,24 @@ export type ModelAvailability = interface ModelAvailabilityCheck { apiKey?: string; baseUrl?: string; + baseUrlIsCustom?: boolean; modelId: string; provider: OpenWikiProvider; } -type OpenAIModelListResponse = { +type ModelListResponse = { data?: Array<{ id?: unknown }>; }; const OPENAI_API_BASE_URL = "https://api.openai.com/v1"; +/** + * Wall-clock cap on the catalogue lookup. This runs before inference on an + * endpoint the user controls, so an unresponsive host must not stall the run: + * the abort surfaces as `unknown` and inference proceeds. + */ +const AVAILABILITY_LOOKUP_TIMEOUT_MS = 5_000; + /** * Checks whether a selected model is exposed to the configured provider * credential. `unknown` deliberately preserves the existing inference path: @@ -27,21 +35,88 @@ export async function getSelectedModelAvailability( check: ModelAvailabilityCheck, fetchImpl: typeof fetch = globalThis.fetch, ): Promise { - if (check.provider !== "openai") { - return { - status: "unknown", - reason: "No availability adapter is configured.", - }; + if (check.provider === "openai") { + if (check.baseUrl !== undefined) { + return { + status: "unknown", + reason: "Custom OpenAI-compatible endpoints are not validated.", + }; + } + + return checkOpenAIModelAvailability(check, fetchImpl); } - if (check.baseUrl !== undefined) { + if (check.provider === "nvidia") { + const baseUrl = check.baseUrl; + + if (!check.baseUrlIsCustom || !baseUrl) { + return { + status: "unknown", + reason: "The NVIDIA hosted endpoint is not validated.", + }; + } + + return checkNvidiaNimModelAvailability(check, baseUrl, fetchImpl); + } + + return { + status: "unknown", + reason: "No availability adapter is configured.", + }; +} + +async function checkOpenAIModelAvailability( + check: ModelAvailabilityCheck, + fetchImpl: typeof fetch, +): Promise { + return checkModelListAvailability({ + apiKey: check.apiKey, + endpoint: `${OPENAI_API_BASE_URL}/models`, + fetchImpl, + modelId: check.modelId, + providerLabel: "OpenAI", + }); +} + +async function checkNvidiaNimModelAvailability( + check: ModelAvailabilityCheck, + baseUrl: string, + fetchImpl: typeof fetch, +): Promise { + let endpoint: string; + + try { + endpoint = resolveModelListEndpoint(baseUrl); + } catch { return { status: "unknown", - reason: "Custom OpenAI-compatible endpoints are not validated.", + reason: "The configured base URL could not be resolved to a catalogue.", }; } - if (!check.apiKey) { + return checkModelListAvailability({ + apiKey: check.apiKey, + endpoint, + fetchImpl, + modelId: check.modelId, + providerLabel: "NVIDIA NIM", + }); +} + +async function checkModelListAvailability({ + apiKey, + endpoint, + fetchImpl, + modelId, + providerLabel, +}: { + apiKey?: string; + endpoint: string; + fetchImpl: typeof fetch; + modelId: string; + providerLabel: string; +}): Promise { + if (!apiKey) { return { status: "unknown", reason: "No API key is available for validation.", @@ -49,8 +124,9 @@ export async function getSelectedModelAvailability( } try { - const response = await fetchImpl(`${OPENAI_API_BASE_URL}/models`, { - headers: { Authorization: `Bearer ${check.apiKey}` }, + const response = await fetchImpl(endpoint, { + headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(AVAILABILITY_LOOKUP_TIMEOUT_MS), }); if (!response.ok) { @@ -60,7 +136,7 @@ export async function getSelectedModelAvailability( }; } - const body = (await response.json()) as OpenAIModelListResponse; + const body = (await response.json()) as ModelListResponse; if (!Array.isArray(body.data)) { return { status: "unknown", @@ -68,13 +144,22 @@ export async function getSelectedModelAvailability( }; } - if (body.data.some((model) => model.id === check.modelId)) { + if (body.data.length === 0) { + // A gateway that hides its catalogue from a key without list scope still + // serves inference, so an empty listing is no evidence of unavailability. + return { + status: "unknown", + reason: "Model availability lookup returned an empty catalogue.", + }; + } + + if (body.data.some((model) => model.id === modelId)) { return { status: "available" }; } return { status: "unavailable", - reason: "The selected model is not available to this OpenAI API key.", + reason: `The selected model is not available through this ${providerLabel} endpoint.`, }; } catch { return { @@ -83,3 +168,17 @@ export async function getSelectedModelAvailability( }; } } + +/** + * Appends `/models` to the API root's path. Building the URL from `pathname` + * rather than resolving a relative reference keeps a base URL that carries a + * query string or fragment from losing its last path segment. + */ +function resolveModelListEndpoint(baseUrl: string): string { + const url = new URL(baseUrl.trim()); + + url.pathname = `${url.pathname.replace(/\/+$/u, "")}/models`; + url.hash = ""; + + return url.toString(); +} diff --git a/test/config/constants.test.ts b/test/config/constants.test.ts index e1f8c88eb..93e0ff2a3 100644 --- a/test/config/constants.test.ts +++ b/test/config/constants.test.ts @@ -25,6 +25,8 @@ import { NVIDIA_BASE_URL_ENV_KEY, normalizeModelId, normalizeProvider, + OPENAI_BASE_URL_ENV_KEY, + providerBaseUrlIsCustom, providerRequiresApiKey, providerRequiresRegion, providerRequiresSecretKey, @@ -267,6 +269,50 @@ describe("resolveProviderBaseUrl", () => { }); }); +describe("providerBaseUrlIsCustom", () => { + test("is false when the provider falls back to its built-in default", () => { + expect(providerBaseUrlIsCustom("nvidia", {})).toBe(false); + expect(providerBaseUrlIsCustom("openai", {})).toBe(false); + }); + + test("is false when the override restates the built-in default", () => { + // Copying the documented endpoint into the env var is not a self-hosted + // deployment, so it must not make the hosted catalogue authoritative. + expect( + providerBaseUrlIsCustom("nvidia", { + [NVIDIA_BASE_URL_ENV_KEY]: "https://integrate.api.nvidia.com/v1", + }), + ).toBe(false); + expect( + providerBaseUrlIsCustom("nvidia", { + [NVIDIA_BASE_URL_ENV_KEY]: " https://integrate.api.nvidia.com/v1/ ", + }), + ).toBe(false); + }); + + test("is true for an endpoint that differs from the built-in default", () => { + expect( + providerBaseUrlIsCustom("nvidia", { + [NVIDIA_BASE_URL_ENV_KEY]: "https://nim.internal/v1", + }), + ).toBe(true); + }); + + test("is true for a provider with no default once an override is set", () => { + expect( + providerBaseUrlIsCustom("openai", { + [OPENAI_BASE_URL_ENV_KEY]: "https://gateway.example/openai/v1", + }), + ).toBe(true); + }); + + test("ignores a whitespace-only override", () => { + expect( + providerBaseUrlIsCustom("nvidia", { [NVIDIA_BASE_URL_ENV_KEY]: " " }), + ).toBe(false); + }); +}); + describe("resolveProviderRetryAttempts", () => { test("uses the OpenWiki default when no override is set", () => { expect(resolveProviderRetryAttempts({})).toBe( diff --git a/test/model-availability.test.ts b/test/model-availability.test.ts index 1872a33f5..b13775fcc 100644 --- a/test/model-availability.test.ts +++ b/test/model-availability.test.ts @@ -7,6 +7,14 @@ const OPENAI_CHECK = { provider: "openai" as const, }; +const NVIDIA_NIM_CHECK = { + apiKey: "test-api-key", + baseUrl: "https://nim.example/v1", + baseUrlIsCustom: true, + modelId: "nvidia/nemotron-test", + provider: "nvidia" as const, +}; + describe("getSelectedModelAvailability", () => { test("accepts a selected model returned by the OpenAI Models API", async () => { const result = await getSelectedModelAvailability(OPENAI_CHECK, () => @@ -49,4 +57,110 @@ describe("getSelectedModelAvailability", () => { expect(result).toMatchObject({ status: "unknown" }); }); + + test("accepts a model loaded by a custom NVIDIA NIM endpoint", async () => { + const result = await getSelectedModelAvailability( + NVIDIA_NIM_CHECK, + (input, init) => { + expect(input).toBe("https://nim.example/v1/models"); + expect(new Headers(init?.headers).get("Authorization")).toBe( + "Bearer test-api-key", + ); + return Promise.resolve( + Response.json({ data: [{ id: "nvidia/nemotron-test" }] }), + ); + }, + ); + + expect(result).toEqual({ status: "available" }); + }); + + test("rejects a model absent from a custom NVIDIA NIM endpoint", async () => { + const result = await getSelectedModelAvailability(NVIDIA_NIM_CHECK, () => + Promise.resolve(Response.json({ data: [{ id: "another-model" }] })), + ); + + expect(result).toMatchObject({ status: "unavailable" }); + }); + + test("does not block custom NVIDIA NIM when no API key is available", async () => { + const result = await getSelectedModelAvailability( + { ...NVIDIA_NIM_CHECK, apiKey: undefined }, + () => Promise.reject(new Error("fetch must not be called")), + ); + + expect(result).toMatchObject({ status: "unknown" }); + }); + + test("does not block custom NVIDIA NIM when lookup fails", async () => { + const result = await getSelectedModelAvailability(NVIDIA_NIM_CHECK, () => + Promise.reject(new Error("offline")), + ); + + expect(result).toMatchObject({ status: "unknown" }); + }); + + test("does not assume the NVIDIA hosted endpoint exposes entitlement data", async () => { + const result = await getSelectedModelAvailability( + { + ...NVIDIA_NIM_CHECK, + baseUrl: "https://integrate.api.nvidia.com/v1", + baseUrlIsCustom: false, + }, + () => Promise.reject(new Error("fetch must not be called")), + ); + + expect(result).toMatchObject({ status: "unknown" }); + }); + + test("does not block when a custom NVIDIA NIM endpoint hides its catalogue", async () => { + const result = await getSelectedModelAvailability(NVIDIA_NIM_CHECK, () => + Promise.resolve(Response.json({ object: "list", data: [] })), + ); + + expect(result).toMatchObject({ status: "unknown" }); + }); + + test("keeps the API root path when the base URL carries a query string", async () => { + const result = await getSelectedModelAvailability( + { ...NVIDIA_NIM_CHECK, baseUrl: "https://nim.example/v1?api-version=1" }, + (input) => { + expect(input).toBe("https://nim.example/v1/models?api-version=1"); + return Promise.resolve( + Response.json({ data: [{ id: "nvidia/nemotron-test" }] }), + ); + }, + ); + + expect(result).toEqual({ status: "available" }); + }); + + test("bounds a custom NVIDIA NIM lookup with an abort signal", async () => { + let signal: AbortSignal | undefined; + + const result = await getSelectedModelAvailability( + NVIDIA_NIM_CHECK, + (_input, init) => { + signal = init?.signal ?? undefined; + + return Promise.resolve( + Response.json({ data: [{ id: "nvidia/nemotron-test" }] }), + ); + }, + ); + + expect(result).toEqual({ status: "available" }); + expect(signal).toBeInstanceOf(AbortSignal); + expect(signal?.aborted).toBe(false); + }); + + test("treats an aborted lookup as non-blocking", async () => { + const result = await getSelectedModelAvailability(NVIDIA_NIM_CHECK, () => + Promise.reject( + new DOMException("The operation was aborted.", "TimeoutError"), + ), + ); + + expect(result).toMatchObject({ status: "unknown" }); + }); });