feat: validate GitHub Copilot model availability

.changeset/copilot-model-availability.md

@@ -0,0 +1,5 @@
+---
+"openwiki": patch
+---
+
+feat: validate selected github copilot models against account availability before inference

src/model-availability.ts

@@ -16,7 +16,17 @@ type OpenAIModelListResponse = {
   data?: Array<{ id?: unknown }>;
 };
 
+type CopilotModelListResponse = {
+  data?: Array<{
+    id?: unknown;
+    capabilities?: { type?: unknown };
+    model_picker_enabled?: unknown;
+    policy?: { state?: unknown };
+  }>;
+};
+
 const OPENAI_API_BASE_URL = "https://api.openai.com/v1";
+const COPILOT_API_BASE_URL = "https://api.githubcopilot.com";
 
 /**
  * Checks whether a selected model is exposed to the configured provider
@@ -27,13 +37,24 @@ export async function getSelectedModelAvailability(
   check: ModelAvailabilityCheck,
   fetchImpl: typeof fetch = globalThis.fetch,
 ): Promise<ModelAvailability> {
-  if (check.provider !== "openai") {
-    return {
-      status: "unknown",
-      reason: "No availability adapter is configured.",
-    };
+  if (check.provider === "openai") {
+    return getOpenAIModelAvailability(check, fetchImpl);
+  }
+
+  if (check.provider === "copilot") {
+    return getCopilotModelAvailability(check, fetchImpl);
   }
 
+  return {
+    status: "unknown",
+    reason: "No availability adapter is configured.",
+  };
+}
+
+async function getOpenAIModelAvailability(
+  check: ModelAvailabilityCheck,
+  fetchImpl: typeof fetch,
+): Promise<ModelAvailability> {
   if (check.baseUrl !== undefined) {
     return {
       status: "unknown",
@@ -83,3 +104,72 @@ export async function getSelectedModelAvailability(
     };
   }
 }
+
+async function getCopilotModelAvailability(
+  check: ModelAvailabilityCheck,
+  fetchImpl: typeof fetch,
+): Promise<ModelAvailability> {
+  if (!check.apiKey) {
+    return {
+      status: "unknown",
+      reason: "No API key is available for validation.",
+    };
+  }
+
+  const baseUrl = check.baseUrl ?? COPILOT_API_BASE_URL;
+
+  try {
+    const response = await fetchImpl(`${baseUrl.replace(/\/+$/u, "")}/models`, {
+      headers: { Authorization: `Bearer ${check.apiKey}` },
+    });
+
+    if (!response.ok) {
+      return {
+        status: "unknown",
+        reason: `Model availability lookup returned HTTP ${response.status}.`,
+      };
+    }
+
+    const body = (await response.json()) as CopilotModelListResponse;
+    if (!Array.isArray(body.data)) {
+      return {
+        status: "unknown",
+        reason: "Model availability lookup returned an unexpected response.",
+      };
+    }
+
+    const model = body.data.find((candidate) => candidate.id === check.modelId);
+    if (!model || model.capabilities?.type !== "chat") {
+      return {
+        status: "unavailable",
+        reason:
+          "The selected model is not available to this GitHub Copilot account.",
+      };
+    }
+
+    if (
+      model.policy?.state === "enabled" ||
+      (model.policy?.state === undefined && model.model_picker_enabled === true)
+    ) {
+      return { status: "available" };
+    }
+
+    if (model.policy?.state === "disabled") {
+      return {
+        status: "unavailable",
+        reason:
+          "The selected model is not available to this GitHub Copilot account.",
+      };
+    }
+
+    return {
+      status: "unknown",
+      reason: "The Copilot Models API did not report an eligibility state.",
+    };
+  } catch {
+    return {
+      status: "unknown",
+      reason: "Model availability lookup could not be completed.",
+    };
+  }
+}

test/model-availability.test.ts

@@ -7,6 +7,24 @@ const OPENAI_CHECK = {
   provider: "openai" as const,
 };
 
+const COPILOT_CHECK = {
+  apiKey: "test-copilot-token",
+  modelId: "claude-fable-5",
+  provider: "copilot" as const,
+};
+
+function fetchInputUrl(input: Parameters<typeof fetch>[0]): string {
+  if (typeof input === "string") {
+    return input;
+  }
+
+  if (input instanceof URL) {
+    return input.toString();
+  }
+
+  return input.url;
+}
+
 describe("getSelectedModelAvailability", () => {
   test("accepts a selected model returned by the OpenAI Models API", async () => {
     const result = await getSelectedModelAvailability(OPENAI_CHECK, () =>
@@ -41,6 +59,175 @@ describe("getSelectedModelAvailability", () => {
     expect(result).toMatchObject({ status: "unknown" });
   });
 
+  test("accepts a selected chat model enabled by Copilot account policy", async () => {
+    const requests: string[] = [];
+    const result = await getSelectedModelAvailability(
+      COPILOT_CHECK,
+      (input, init) => {
+        requests.push(fetchInputUrl(input));
+        expect(new Headers(init?.headers).get("Authorization")).toBe(
+          "Bearer test-copilot-token",
+        );
+        return Promise.resolve(
+          Response.json({
+            data: [
+              {
+                id: "claude-fable-5",
+                capabilities: { type: "chat" },
+                model_picker_enabled: false,
+                policy: { state: "enabled" },
+              },
+            ],
+          }),
+        );
+      },
+    );
+
+    expect(result).toEqual({ status: "available" });
+    expect(requests).toEqual(["https://api.githubcopilot.com/models"]);
+  });
+
+  test("rejects a selected Copilot model missing from the account model list", async () => {
+    const result = await getSelectedModelAvailability(COPILOT_CHECK, () =>
+      Promise.resolve(
+        Response.json({
+          data: [
+            {
+              id: "gpt-5.6-terra",
+              capabilities: { type: "chat" },
+              model_picker_enabled: true,
+            },
+          ],
+        }),
+      ),
+    );
+
+    expect(result).toMatchObject({ status: "unavailable" });
+  });
+
+  test("rejects a selected Copilot model disabled by account policy", async () => {
+    const result = await getSelectedModelAvailability(COPILOT_CHECK, () =>
+      Promise.resolve(
+        Response.json({
+          data: [
+            {
+              id: "claude-fable-5",
+              capabilities: { type: "chat" },
+              model_picker_enabled: false,
+              policy: { state: "disabled" },
+            },
+          ],
+        }),
+      ),
+    );
+
+    expect(result).toMatchObject({ status: "unavailable" });
+  });
+
+  test("rejects a selected Copilot model that is not a chat model", async () => {
+    const result = await getSelectedModelAvailability(COPILOT_CHECK, () =>
+      Promise.resolve(
+        Response.json({
+          data: [
+            {
+              id: "claude-fable-5",
+              capabilities: { type: "embeddings" },
+              model_picker_enabled: true,
+            },
+          ],
+        }),
+      ),
+    );
+
+    expect(result).toMatchObject({ status: "unavailable" });
+  });
+
+  test("accepts a picker-enabled chat model when policy state is absent", async () => {
+    const result = await getSelectedModelAvailability(COPILOT_CHECK, () =>
+      Promise.resolve(
+        Response.json({
+          data: [
+            {
+              id: "claude-fable-5",
+              capabilities: { type: "chat" },
+              model_picker_enabled: true,
+            },
+          ],
+        }),
+      ),
+    );
+
+    expect(result).toEqual({ status: "available" });
+  });
+
+  test("does not block inference when Copilot eligibility is ambiguous", async () => {
+    const result = await getSelectedModelAvailability(COPILOT_CHECK, () =>
+      Promise.resolve(
+        Response.json({
+          data: [
+            {
+              id: "claude-fable-5",
+              capabilities: { type: "chat" },
+              model_picker_enabled: false,
+            },
+          ],
+        }),
+      ),
+    );
+
+    expect(result).toMatchObject({ status: "unknown" });
+  });
+
+  test("does not block inference when no Copilot token is available", async () => {
+    const result = await getSelectedModelAvailability(
+      { ...COPILOT_CHECK, apiKey: undefined },
+      () => Promise.reject(new Error("fetch must not be called")),
+    );
+
+    expect(result).toMatchObject({ status: "unknown" });
+  });
+
+  test("does not block inference when the Copilot Models API fails", async () => {
+    const result = await getSelectedModelAvailability(COPILOT_CHECK, () =>
+      Promise.resolve(new Response("forbidden", { status: 403 })),
+    );
+
+    expect(result).toMatchObject({ status: "unknown" });
+  });
+
+  test("does not block inference when the Copilot Models API shape is unexpected", async () => {
+    const result = await getSelectedModelAvailability(COPILOT_CHECK, () =>
+      Promise.resolve(Response.json({ models: [] })),
+    );
+
+    expect(result).toMatchObject({ status: "unknown" });
+  });
+
+  test("uses the configured Copilot base URL without duplicating slashes", async () => {
+    const requests: string[] = [];
+    const result = await getSelectedModelAvailability(
+      { ...COPILOT_CHECK, baseUrl: "https://tenant.ghe.com/api/copilot/" },
+      (input) => {
+        requests.push(fetchInputUrl(input));
+        return Promise.resolve(
+          Response.json({
+            data: [
+              {
+                id: "claude-fable-5",
+                capabilities: { type: "chat" },
+                model_picker_enabled: false,
+                policy: { state: "enabled" },
+              },
+            ],
+          }),
+        );
+      },
+    );
+
+    expect(result).toEqual({ status: "available" });
+    expect(requests).toEqual(["https://tenant.ghe.com/api/copilot/models"]);
+  });
+
   test("does not validate providers without an availability adapter", async () => {
     const result = await getSelectedModelAvailability({
       ...OPENAI_CHECK,