feat: validate custom NVIDIA NIM model availability

.changeset/nvidia-nim-availability-fail-open.md

@@ -0,0 +1,5 @@
+---
+"openwiki": patch
+---
+
+fix: keep NVIDIA NIM availability validation fail-open on inconclusive catalogue lookups

.changeset/nvidia-nim-availability.md

@@ -0,0 +1,5 @@
+---
+"openwiki": patch
+---
+
+feat: validate selected models against custom NVIDIA NIM endpoints before inference

src/agent/index.ts

@@ -111,6 +111,7 @@ import {
   OPENWIKI_PROVIDER_ENV_KEY,
   OPENWIKI_PROVIDER_RETRY_ATTEMPTS_ENV_KEY,
   OPENWIKI_STREAM_IDLE_TIMEOUT_ENV_KEY,
+  providerBaseUrlIsCustom,
   providerRequiresBaseUrl,
   providerRequiresRegion,
   providerRequiresSecretKey,
@@ -295,6 +296,7 @@ async function resolveRunConfig(
     onProviderResolved(provider);
 
     const providerBaseUrl = resolveProviderBaseUrl(provider);
+    const baseUrlIsCustom = providerBaseUrlIsCustom(provider);
     emitDebug(options, `provider=${provider}`);
     if (providerBaseUrl) {
       emitDebug(
@@ -331,10 +333,11 @@ async function resolveRunConfig(
       modelId,
       apiKey: getProviderApiKey(provider),
       baseUrl: providerBaseUrl,
+      baseUrlIsCustom,
     });
     if (modelAvailability.status === "unavailable") {
       throw new Error(
-        `${getProviderLabel(provider)} does not make model "${modelId}" available to the configured credentials. Set ${OPENWIKI_MODEL_ID_ENV_KEY} to an available model.`,
+        `${getProviderLabel(provider)} does not make model "${modelId}" available. ${modelAvailability.reason} Set ${OPENWIKI_MODEL_ID_ENV_KEY} to an available model.`,
       );
     }
     if (modelAvailability.status === "unknown") {

src/config/constants.ts

@@ -648,6 +648,38 @@ export function resolveProviderBaseUrl(
   return config.baseURL;
 }
 
+/**
+ * Reports whether the provider points at a user-supplied endpoint rather than
+ * its built-in default. Compares the resolved base URL against
+ * {@link ProviderConfig.baseURL} instead of testing whether the environment
+ * variable is set, so a user who pins the default value explicitly is not
+ * mistaken for a self-hosted deployment.
+ */
+export function providerBaseUrlIsCustom(
+  provider: OpenWikiProvider,
+  env: NodeJS.ProcessEnv = process.env,
+): boolean {
+  const resolved = resolveProviderBaseUrl(provider, env);
+
+  if (!resolved) {
+    return false;
+  }
+
+  const defaultBaseUrl = getProviderConfig(provider).baseURL;
+  if (!defaultBaseUrl) {
+    return true;
+  }
+
+  return (
+    normalizeBaseUrlForComparison(resolved) !==
+    normalizeBaseUrlForComparison(defaultBaseUrl)
+  );
+}
+
+function normalizeBaseUrlForComparison(value: string): string {
+  return value.trim().replace(/\/+$/u, "");
+}
+
 export function getProviderBaseUrlEnvKey(
   provider: OpenWikiProvider,
 ): string | undefined {

src/model-availability.ts

@@ -8,16 +8,24 @@ export type ModelAvailability =
 interface ModelAvailabilityCheck {
   apiKey?: string;
   baseUrl?: string;
+  baseUrlIsCustom?: boolean;
   modelId: string;
   provider: OpenWikiProvider;
 }
 
-type OpenAIModelListResponse = {
+type ModelListResponse = {
   data?: Array<{ id?: unknown }>;
 };
 
 const OPENAI_API_BASE_URL = "https://api.openai.com/v1";
 
+/**
+ * Wall-clock cap on the catalogue lookup. This runs before inference on an
+ * endpoint the user controls, so an unresponsive host must not stall the run:
+ * the abort surfaces as `unknown` and inference proceeds.
+ */
+const AVAILABILITY_LOOKUP_TIMEOUT_MS = 5_000;
+
 /**
  * Checks whether a selected model is exposed to the configured provider
  * credential. `unknown` deliberately preserves the existing inference path:
@@ -27,30 +35,98 @@ export async function getSelectedModelAvailability(
   check: ModelAvailabilityCheck,
   fetchImpl: typeof fetch = globalThis.fetch,
 ): Promise<ModelAvailability> {
-  if (check.provider !== "openai") {
-    return {
-      status: "unknown",
-      reason: "No availability adapter is configured.",
-    };
+  if (check.provider === "openai") {
+    if (check.baseUrl !== undefined) {
+      return {
+        status: "unknown",
+        reason: "Custom OpenAI-compatible endpoints are not validated.",
+      };
+    }
+
+    return checkOpenAIModelAvailability(check, fetchImpl);
   }
 
-  if (check.baseUrl !== undefined) {
+  if (check.provider === "nvidia") {
+    const baseUrl = check.baseUrl;
+
+    if (!check.baseUrlIsCustom || !baseUrl) {
+      return {
+        status: "unknown",
+        reason: "The NVIDIA hosted endpoint is not validated.",
+      };
+    }
+
+    return checkNvidiaNimModelAvailability(check, baseUrl, fetchImpl);
+  }
+
+  return {
+    status: "unknown",
+    reason: "No availability adapter is configured.",
+  };
+}
+
+async function checkOpenAIModelAvailability(
+  check: ModelAvailabilityCheck,
+  fetchImpl: typeof fetch,
+): Promise<ModelAvailability> {
+  return checkModelListAvailability({
+    apiKey: check.apiKey,
+    endpoint: `${OPENAI_API_BASE_URL}/models`,
+    fetchImpl,
+    modelId: check.modelId,
+    providerLabel: "OpenAI",
+  });
+}
+
+async function checkNvidiaNimModelAvailability(
+  check: ModelAvailabilityCheck,
+  baseUrl: string,
+  fetchImpl: typeof fetch,
+): Promise<ModelAvailability> {
+  let endpoint: string;
+
+  try {
+    endpoint = resolveModelListEndpoint(baseUrl);
+  } catch {
     return {
       status: "unknown",
-      reason: "Custom OpenAI-compatible endpoints are not validated.",
+      reason: "The configured base URL could not be resolved to a catalogue.",
     };
   }
 
-  if (!check.apiKey) {
+  return checkModelListAvailability({
+    apiKey: check.apiKey,
+    endpoint,
+    fetchImpl,
+    modelId: check.modelId,
+    providerLabel: "NVIDIA NIM",
+  });
+}
+
+async function checkModelListAvailability({
+  apiKey,
+  endpoint,
+  fetchImpl,
+  modelId,
+  providerLabel,
+}: {
+  apiKey?: string;
+  endpoint: string;
+  fetchImpl: typeof fetch;
+  modelId: string;
+  providerLabel: string;
+}): Promise<ModelAvailability> {
+  if (!apiKey) {
     return {
       status: "unknown",
       reason: "No API key is available for validation.",
     };
   }
 
   try {
-    const response = await fetchImpl(`${OPENAI_API_BASE_URL}/models`, {
-      headers: { Authorization: `Bearer ${check.apiKey}` },
+    const response = await fetchImpl(endpoint, {
+      headers: { Authorization: `Bearer ${apiKey}` },
+      signal: AbortSignal.timeout(AVAILABILITY_LOOKUP_TIMEOUT_MS),
     });
 
     if (!response.ok) {
@@ -60,21 +136,30 @@ export async function getSelectedModelAvailability(
       };
     }
 
-    const body = (await response.json()) as OpenAIModelListResponse;
+    const body = (await response.json()) as ModelListResponse;
     if (!Array.isArray(body.data)) {
       return {
         status: "unknown",
         reason: "Model availability lookup returned an unexpected response.",
       };
     }
 
-    if (body.data.some((model) => model.id === check.modelId)) {
+    if (body.data.length === 0) {
+      // A gateway that hides its catalogue from a key without list scope still
+      // serves inference, so an empty listing is no evidence of unavailability.
+      return {
+        status: "unknown",
+        reason: "Model availability lookup returned an empty catalogue.",
+      };
+    }
+
+    if (body.data.some((model) => model.id === modelId)) {
       return { status: "available" };
     }
 
     return {
       status: "unavailable",
-      reason: "The selected model is not available to this OpenAI API key.",
+      reason: `The selected model is not available through this ${providerLabel} endpoint.`,
     };
   } catch {
     return {
@@ -83,3 +168,17 @@ export async function getSelectedModelAvailability(
     };
   }
 }
+
+/**
+ * Appends `/models` to the API root's path. Building the URL from `pathname`
+ * rather than resolving a relative reference keeps a base URL that carries a
+ * query string or fragment from losing its last path segment.
+ */
+function resolveModelListEndpoint(baseUrl: string): string {
+  const url = new URL(baseUrl.trim());
+
+  url.pathname = `${url.pathname.replace(/\/+$/u, "")}/models`;
+  url.hash = "";
+
+  return url.toString();
+}

test/config/constants.test.ts

@@ -25,6 +25,8 @@ import {
   NVIDIA_BASE_URL_ENV_KEY,
   normalizeModelId,
   normalizeProvider,
+  OPENAI_BASE_URL_ENV_KEY,
+  providerBaseUrlIsCustom,
   providerRequiresApiKey,
   providerRequiresRegion,
   providerRequiresSecretKey,
@@ -267,6 +269,50 @@ describe("resolveProviderBaseUrl", () => {
   });
 });
 
+describe("providerBaseUrlIsCustom", () => {
+  test("is false when the provider falls back to its built-in default", () => {
+    expect(providerBaseUrlIsCustom("nvidia", {})).toBe(false);
+    expect(providerBaseUrlIsCustom("openai", {})).toBe(false);
+  });
+
+  test("is false when the override restates the built-in default", () => {
+    // Copying the documented endpoint into the env var is not a self-hosted
+    // deployment, so it must not make the hosted catalogue authoritative.
+    expect(
+      providerBaseUrlIsCustom("nvidia", {
+        [NVIDIA_BASE_URL_ENV_KEY]: "https://integrate.api.nvidia.com/v1",
+      }),
+    ).toBe(false);
+    expect(
+      providerBaseUrlIsCustom("nvidia", {
+        [NVIDIA_BASE_URL_ENV_KEY]: " https://integrate.api.nvidia.com/v1/ ",
+      }),
+    ).toBe(false);
+  });
+
+  test("is true for an endpoint that differs from the built-in default", () => {
+    expect(
+      providerBaseUrlIsCustom("nvidia", {
+        [NVIDIA_BASE_URL_ENV_KEY]: "https://nim.internal/v1",
+      }),
+    ).toBe(true);
+  });
+
+  test("is true for a provider with no default once an override is set", () => {
+    expect(
+      providerBaseUrlIsCustom("openai", {
+        [OPENAI_BASE_URL_ENV_KEY]: "https://gateway.example/openai/v1",
+      }),
+    ).toBe(true);
+  });
+
+  test("ignores a whitespace-only override", () => {
+    expect(
+      providerBaseUrlIsCustom("nvidia", { [NVIDIA_BASE_URL_ENV_KEY]: "   " }),
+    ).toBe(false);
+  });
+});
+
 describe("resolveProviderRetryAttempts", () => {
   test("uses the OpenWiki default when no override is set", () => {
     expect(resolveProviderRetryAttempts({})).toBe(