From a7ba3038c835470cb214a0066c8055a653169724 Mon Sep 17 00:00:00 2001 From: jyje Date: Mon, 27 Jul 2026 23:29:10 +0900 Subject: [PATCH 1/2] feat: validate custom NVIDIA NIM model availability --- .changeset/nvidia-nim-availability.md | 5 ++ src/agent/index.ts | 5 ++ src/model-availability.ts | 97 ++++++++++++++++++++++----- test/model-availability.test.ts | 63 +++++++++++++++++ 4 files changed, 153 insertions(+), 17 deletions(-) create mode 100644 .changeset/nvidia-nim-availability.md diff --git a/.changeset/nvidia-nim-availability.md b/.changeset/nvidia-nim-availability.md new file mode 100644 index 000000000..364aa959c --- /dev/null +++ b/.changeset/nvidia-nim-availability.md @@ -0,0 +1,5 @@ +--- +"openwiki": patch +--- + +feat: validate selected models against custom NVIDIA NIM endpoints before inference diff --git a/src/agent/index.ts b/src/agent/index.ts index fc4b5620f..e198047d6 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -295,6 +295,10 @@ async function resolveRunConfig( onProviderResolved(provider); const providerBaseUrl = resolveProviderBaseUrl(provider); + const providerBaseUrlEnvKey = getProviderBaseUrlEnvKey(provider); + const providerBaseUrlIsCustom = + providerBaseUrlEnvKey !== undefined && + Boolean(process.env[providerBaseUrlEnvKey]?.trim()); emitDebug(options, `provider=${provider}`); if (providerBaseUrl) { emitDebug( @@ -331,6 +335,7 @@ async function resolveRunConfig( modelId, apiKey: getProviderApiKey(provider), baseUrl: providerBaseUrl, + baseUrlIsCustom: providerBaseUrlIsCustom, }); if (modelAvailability.status === "unavailable") { throw new Error( diff --git a/src/model-availability.ts b/src/model-availability.ts index a737cd5ae..632b70e5b 100644 --- a/src/model-availability.ts +++ b/src/model-availability.ts @@ -8,11 +8,12 @@ export type ModelAvailability = interface ModelAvailabilityCheck { apiKey?: string; baseUrl?: string; + baseUrlIsCustom?: boolean; modelId: string; provider: OpenWikiProvider; } -type OpenAIModelListResponse = { +type ModelListResponse = { data?: Array<{ id?: unknown }>; }; @@ -27,21 +28,79 @@ export async function getSelectedModelAvailability( check: ModelAvailabilityCheck, fetchImpl: typeof fetch = globalThis.fetch, ): Promise { - if (check.provider !== "openai") { - return { - status: "unknown", - reason: "No availability adapter is configured.", - }; + if (check.provider === "openai") { + if (check.baseUrl !== undefined) { + return { + status: "unknown", + reason: "Custom OpenAI-compatible endpoints are not validated.", + }; + } + + return checkOpenAIModelAvailability(check, fetchImpl); } - if (check.baseUrl !== undefined) { - return { - status: "unknown", - reason: "Custom OpenAI-compatible endpoints are not validated.", - }; + if (check.provider === "nvidia") { + if (!check.baseUrlIsCustom || !check.baseUrl) { + return { + status: "unknown", + reason: "The NVIDIA hosted endpoint is not validated.", + }; + } + + return checkNvidiaNimModelAvailability(check, fetchImpl); } - if (!check.apiKey) { + return { + status: "unknown", + reason: "No availability adapter is configured.", + }; +} + +async function checkOpenAIModelAvailability( + check: ModelAvailabilityCheck, + fetchImpl: typeof fetch, +): Promise { + return checkModelListAvailability({ + apiKey: check.apiKey, + endpoint: `${OPENAI_API_BASE_URL}/models`, + fetchImpl, + modelId: check.modelId, + providerLabel: "OpenAI", + }); +} + +async function checkNvidiaNimModelAvailability( + check: ModelAvailabilityCheck, + fetchImpl: typeof fetch, +): Promise { + const endpoint = new URL( + "models", + ensureTrailingSlash(check.baseUrl!), + ).toString(); + + return checkModelListAvailability({ + apiKey: check.apiKey, + endpoint, + fetchImpl, + modelId: check.modelId, + providerLabel: "NVIDIA NIM", + }); +} + +async function checkModelListAvailability({ + apiKey, + endpoint, + fetchImpl, + modelId, + providerLabel, +}: { + apiKey?: string; + endpoint: string; + fetchImpl: typeof fetch; + modelId: string; + providerLabel: string; +}): Promise { + if (!apiKey) { return { status: "unknown", reason: "No API key is available for validation.", @@ -49,8 +108,8 @@ export async function getSelectedModelAvailability( } try { - const response = await fetchImpl(`${OPENAI_API_BASE_URL}/models`, { - headers: { Authorization: `Bearer ${check.apiKey}` }, + const response = await fetchImpl(endpoint, { + headers: { Authorization: `Bearer ${apiKey}` }, }); if (!response.ok) { @@ -60,7 +119,7 @@ export async function getSelectedModelAvailability( }; } - const body = (await response.json()) as OpenAIModelListResponse; + const body = (await response.json()) as ModelListResponse; if (!Array.isArray(body.data)) { return { status: "unknown", @@ -68,13 +127,13 @@ export async function getSelectedModelAvailability( }; } - if (body.data.some((model) => model.id === check.modelId)) { + if (body.data.some((model) => model.id === modelId)) { return { status: "available" }; } return { status: "unavailable", - reason: "The selected model is not available to this OpenAI API key.", + reason: `The selected model is not available through this ${providerLabel} endpoint.`, }; } catch { return { @@ -83,3 +142,7 @@ export async function getSelectedModelAvailability( }; } } + +function ensureTrailingSlash(value: string): string { + return value.endsWith("/") ? value : `${value}/`; +} diff --git a/test/model-availability.test.ts b/test/model-availability.test.ts index 1872a33f5..65f9580bd 100644 --- a/test/model-availability.test.ts +++ b/test/model-availability.test.ts @@ -7,6 +7,14 @@ const OPENAI_CHECK = { provider: "openai" as const, }; +const NVIDIA_NIM_CHECK = { + apiKey: "test-api-key", + baseUrl: "https://nim.example/v1", + baseUrlIsCustom: true, + modelId: "nvidia/nemotron-test", + provider: "nvidia" as const, +}; + describe("getSelectedModelAvailability", () => { test("accepts a selected model returned by the OpenAI Models API", async () => { const result = await getSelectedModelAvailability(OPENAI_CHECK, () => @@ -49,4 +57,59 @@ describe("getSelectedModelAvailability", () => { expect(result).toMatchObject({ status: "unknown" }); }); + + test("accepts a model loaded by a custom NVIDIA NIM endpoint", async () => { + const result = await getSelectedModelAvailability( + NVIDIA_NIM_CHECK, + (input, init) => { + expect(input).toBe("https://nim.example/v1/models"); + expect(new Headers(init?.headers).get("Authorization")).toBe( + "Bearer test-api-key", + ); + return Promise.resolve( + Response.json({ data: [{ id: "nvidia/nemotron-test" }] }), + ); + }, + ); + + expect(result).toEqual({ status: "available" }); + }); + + test("rejects a model absent from a custom NVIDIA NIM endpoint", async () => { + const result = await getSelectedModelAvailability(NVIDIA_NIM_CHECK, () => + Promise.resolve(Response.json({ data: [{ id: "another-model" }] })), + ); + + expect(result).toMatchObject({ status: "unavailable" }); + }); + + test("does not block custom NVIDIA NIM when no API key is available", async () => { + const result = await getSelectedModelAvailability( + { ...NVIDIA_NIM_CHECK, apiKey: undefined }, + () => Promise.reject(new Error("fetch must not be called")), + ); + + expect(result).toMatchObject({ status: "unknown" }); + }); + + test("does not block custom NVIDIA NIM when lookup fails", async () => { + const result = await getSelectedModelAvailability(NVIDIA_NIM_CHECK, () => + Promise.reject(new Error("offline")), + ); + + expect(result).toMatchObject({ status: "unknown" }); + }); + + test("does not assume the NVIDIA hosted endpoint exposes entitlement data", async () => { + const result = await getSelectedModelAvailability( + { + ...NVIDIA_NIM_CHECK, + baseUrl: "https://integrate.api.nvidia.com/v1", + baseUrlIsCustom: false, + }, + () => Promise.reject(new Error("fetch must not be called")), + ); + + expect(result).toMatchObject({ status: "unknown" }); + }); }); From ade29caed76acec5e6eb7a25754d7c47a53f6be8 Mon Sep 17 00:00:00 2001 From: jyje Date: Fri, 14 Aug 2026 13:21:54 +0900 Subject: [PATCH 2/2] fix: keep NVIDIA NIM availability validation fail-open (#13) `unavailable` aborts the run before inference with no override, so every inconclusive catalogue lookup that resolves to it is a total blocker. Four paths could reach it without evidence that the model cannot be invoked. - Derive "custom endpoint" by comparing the resolved base URL against the provider's built-in default rather than testing whether the env var is set. Pinning `NVIDIA_BASE_URL` to the documented hosted endpoint made the public hosted catalogue authoritative over entitlement it does not model. - Treat an empty `data` array as `unknown`. A proxied gateway that hides its catalogue from a key without list scope still serves inference. - Bound the lookup with a 5s `AbortSignal.timeout`. The request now targets user-supplied infrastructure, where a stalled host held every run for undici's 300s default. - Build the catalogue URL from `pathname` so a base URL carrying a query string keeps its last path segment, and fall back to `unknown` if the URL cannot be resolved at all. Also surface the availability reason in the thrown error, which previously blamed credentials for a model simply not loaded on the endpoint. Co-authored-by: Claude Opus 5 --- .../nvidia-nim-availability-fail-open.md | 5 ++ src/agent/index.ts | 10 ++-- src/config/constants.ts | 32 ++++++++++++ src/model-availability.ts | 52 ++++++++++++++++--- test/config/constants.test.ts | 46 ++++++++++++++++ test/model-availability.test.ts | 51 ++++++++++++++++++ 6 files changed, 182 insertions(+), 14 deletions(-) create mode 100644 .changeset/nvidia-nim-availability-fail-open.md diff --git a/.changeset/nvidia-nim-availability-fail-open.md b/.changeset/nvidia-nim-availability-fail-open.md new file mode 100644 index 000000000..4c1e610bb --- /dev/null +++ b/.changeset/nvidia-nim-availability-fail-open.md @@ -0,0 +1,5 @@ +--- +"openwiki": patch +--- + +fix: keep NVIDIA NIM availability validation fail-open on inconclusive catalogue lookups diff --git a/src/agent/index.ts b/src/agent/index.ts index e198047d6..86b603e14 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -111,6 +111,7 @@ import { OPENWIKI_PROVIDER_ENV_KEY, OPENWIKI_PROVIDER_RETRY_ATTEMPTS_ENV_KEY, OPENWIKI_STREAM_IDLE_TIMEOUT_ENV_KEY, + providerBaseUrlIsCustom, providerRequiresBaseUrl, providerRequiresRegion, providerRequiresSecretKey, @@ -295,10 +296,7 @@ async function resolveRunConfig( onProviderResolved(provider); const providerBaseUrl = resolveProviderBaseUrl(provider); - const providerBaseUrlEnvKey = getProviderBaseUrlEnvKey(provider); - const providerBaseUrlIsCustom = - providerBaseUrlEnvKey !== undefined && - Boolean(process.env[providerBaseUrlEnvKey]?.trim()); + const baseUrlIsCustom = providerBaseUrlIsCustom(provider); emitDebug(options, `provider=${provider}`); if (providerBaseUrl) { emitDebug( @@ -335,11 +333,11 @@ async function resolveRunConfig( modelId, apiKey: getProviderApiKey(provider), baseUrl: providerBaseUrl, - baseUrlIsCustom: providerBaseUrlIsCustom, + baseUrlIsCustom, }); if (modelAvailability.status === "unavailable") { throw new Error( - `${getProviderLabel(provider)} does not make model "${modelId}" available to the configured credentials. Set ${OPENWIKI_MODEL_ID_ENV_KEY} to an available model.`, + `${getProviderLabel(provider)} does not make model "${modelId}" available. ${modelAvailability.reason} Set ${OPENWIKI_MODEL_ID_ENV_KEY} to an available model.`, ); } if (modelAvailability.status === "unknown") { diff --git a/src/config/constants.ts b/src/config/constants.ts index 5488cd8db..5e91f1af4 100644 --- a/src/config/constants.ts +++ b/src/config/constants.ts @@ -648,6 +648,38 @@ export function resolveProviderBaseUrl( return config.baseURL; } +/** + * Reports whether the provider points at a user-supplied endpoint rather than + * its built-in default. Compares the resolved base URL against + * {@link ProviderConfig.baseURL} instead of testing whether the environment + * variable is set, so a user who pins the default value explicitly is not + * mistaken for a self-hosted deployment. + */ +export function providerBaseUrlIsCustom( + provider: OpenWikiProvider, + env: NodeJS.ProcessEnv = process.env, +): boolean { + const resolved = resolveProviderBaseUrl(provider, env); + + if (!resolved) { + return false; + } + + const defaultBaseUrl = getProviderConfig(provider).baseURL; + if (!defaultBaseUrl) { + return true; + } + + return ( + normalizeBaseUrlForComparison(resolved) !== + normalizeBaseUrlForComparison(defaultBaseUrl) + ); +} + +function normalizeBaseUrlForComparison(value: string): string { + return value.trim().replace(/\/+$/u, ""); +} + export function getProviderBaseUrlEnvKey( provider: OpenWikiProvider, ): string | undefined { diff --git a/src/model-availability.ts b/src/model-availability.ts index 632b70e5b..d5bf8a7e7 100644 --- a/src/model-availability.ts +++ b/src/model-availability.ts @@ -19,6 +19,13 @@ type ModelListResponse = { const OPENAI_API_BASE_URL = "https://api.openai.com/v1"; +/** + * Wall-clock cap on the catalogue lookup. This runs before inference on an + * endpoint the user controls, so an unresponsive host must not stall the run: + * the abort surfaces as `unknown` and inference proceeds. + */ +const AVAILABILITY_LOOKUP_TIMEOUT_MS = 5_000; + /** * Checks whether a selected model is exposed to the configured provider * credential. `unknown` deliberately preserves the existing inference path: @@ -40,14 +47,16 @@ export async function getSelectedModelAvailability( } if (check.provider === "nvidia") { - if (!check.baseUrlIsCustom || !check.baseUrl) { + const baseUrl = check.baseUrl; + + if (!check.baseUrlIsCustom || !baseUrl) { return { status: "unknown", reason: "The NVIDIA hosted endpoint is not validated.", }; } - return checkNvidiaNimModelAvailability(check, fetchImpl); + return checkNvidiaNimModelAvailability(check, baseUrl, fetchImpl); } return { @@ -71,12 +80,19 @@ async function checkOpenAIModelAvailability( async function checkNvidiaNimModelAvailability( check: ModelAvailabilityCheck, + baseUrl: string, fetchImpl: typeof fetch, ): Promise { - const endpoint = new URL( - "models", - ensureTrailingSlash(check.baseUrl!), - ).toString(); + let endpoint: string; + + try { + endpoint = resolveModelListEndpoint(baseUrl); + } catch { + return { + status: "unknown", + reason: "The configured base URL could not be resolved to a catalogue.", + }; + } return checkModelListAvailability({ apiKey: check.apiKey, @@ -110,6 +126,7 @@ async function checkModelListAvailability({ try { const response = await fetchImpl(endpoint, { headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(AVAILABILITY_LOOKUP_TIMEOUT_MS), }); if (!response.ok) { @@ -127,6 +144,15 @@ async function checkModelListAvailability({ }; } + if (body.data.length === 0) { + // A gateway that hides its catalogue from a key without list scope still + // serves inference, so an empty listing is no evidence of unavailability. + return { + status: "unknown", + reason: "Model availability lookup returned an empty catalogue.", + }; + } + if (body.data.some((model) => model.id === modelId)) { return { status: "available" }; } @@ -143,6 +169,16 @@ async function checkModelListAvailability({ } } -function ensureTrailingSlash(value: string): string { - return value.endsWith("/") ? value : `${value}/`; +/** + * Appends `/models` to the API root's path. Building the URL from `pathname` + * rather than resolving a relative reference keeps a base URL that carries a + * query string or fragment from losing its last path segment. + */ +function resolveModelListEndpoint(baseUrl: string): string { + const url = new URL(baseUrl.trim()); + + url.pathname = `${url.pathname.replace(/\/+$/u, "")}/models`; + url.hash = ""; + + return url.toString(); } diff --git a/test/config/constants.test.ts b/test/config/constants.test.ts index e1f8c88eb..93e0ff2a3 100644 --- a/test/config/constants.test.ts +++ b/test/config/constants.test.ts @@ -25,6 +25,8 @@ import { NVIDIA_BASE_URL_ENV_KEY, normalizeModelId, normalizeProvider, + OPENAI_BASE_URL_ENV_KEY, + providerBaseUrlIsCustom, providerRequiresApiKey, providerRequiresRegion, providerRequiresSecretKey, @@ -267,6 +269,50 @@ describe("resolveProviderBaseUrl", () => { }); }); +describe("providerBaseUrlIsCustom", () => { + test("is false when the provider falls back to its built-in default", () => { + expect(providerBaseUrlIsCustom("nvidia", {})).toBe(false); + expect(providerBaseUrlIsCustom("openai", {})).toBe(false); + }); + + test("is false when the override restates the built-in default", () => { + // Copying the documented endpoint into the env var is not a self-hosted + // deployment, so it must not make the hosted catalogue authoritative. + expect( + providerBaseUrlIsCustom("nvidia", { + [NVIDIA_BASE_URL_ENV_KEY]: "https://integrate.api.nvidia.com/v1", + }), + ).toBe(false); + expect( + providerBaseUrlIsCustom("nvidia", { + [NVIDIA_BASE_URL_ENV_KEY]: " https://integrate.api.nvidia.com/v1/ ", + }), + ).toBe(false); + }); + + test("is true for an endpoint that differs from the built-in default", () => { + expect( + providerBaseUrlIsCustom("nvidia", { + [NVIDIA_BASE_URL_ENV_KEY]: "https://nim.internal/v1", + }), + ).toBe(true); + }); + + test("is true for a provider with no default once an override is set", () => { + expect( + providerBaseUrlIsCustom("openai", { + [OPENAI_BASE_URL_ENV_KEY]: "https://gateway.example/openai/v1", + }), + ).toBe(true); + }); + + test("ignores a whitespace-only override", () => { + expect( + providerBaseUrlIsCustom("nvidia", { [NVIDIA_BASE_URL_ENV_KEY]: " " }), + ).toBe(false); + }); +}); + describe("resolveProviderRetryAttempts", () => { test("uses the OpenWiki default when no override is set", () => { expect(resolveProviderRetryAttempts({})).toBe( diff --git a/test/model-availability.test.ts b/test/model-availability.test.ts index 65f9580bd..b13775fcc 100644 --- a/test/model-availability.test.ts +++ b/test/model-availability.test.ts @@ -112,4 +112,55 @@ describe("getSelectedModelAvailability", () => { expect(result).toMatchObject({ status: "unknown" }); }); + + test("does not block when a custom NVIDIA NIM endpoint hides its catalogue", async () => { + const result = await getSelectedModelAvailability(NVIDIA_NIM_CHECK, () => + Promise.resolve(Response.json({ object: "list", data: [] })), + ); + + expect(result).toMatchObject({ status: "unknown" }); + }); + + test("keeps the API root path when the base URL carries a query string", async () => { + const result = await getSelectedModelAvailability( + { ...NVIDIA_NIM_CHECK, baseUrl: "https://nim.example/v1?api-version=1" }, + (input) => { + expect(input).toBe("https://nim.example/v1/models?api-version=1"); + return Promise.resolve( + Response.json({ data: [{ id: "nvidia/nemotron-test" }] }), + ); + }, + ); + + expect(result).toEqual({ status: "available" }); + }); + + test("bounds a custom NVIDIA NIM lookup with an abort signal", async () => { + let signal: AbortSignal | undefined; + + const result = await getSelectedModelAvailability( + NVIDIA_NIM_CHECK, + (_input, init) => { + signal = init?.signal ?? undefined; + + return Promise.resolve( + Response.json({ data: [{ id: "nvidia/nemotron-test" }] }), + ); + }, + ); + + expect(result).toEqual({ status: "available" }); + expect(signal).toBeInstanceOf(AbortSignal); + expect(signal?.aborted).toBe(false); + }); + + test("treats an aborted lookup as non-blocking", async () => { + const result = await getSelectedModelAvailability(NVIDIA_NIM_CHECK, () => + Promise.reject( + new DOMException("The operation was aborted.", "TimeoutError"), + ), + ); + + expect(result).toMatchObject({ status: "unknown" }); + }); });