diff --git a/.changeset/copilot-model-availability.md b/.changeset/copilot-model-availability.md new file mode 100644 index 000000000..6533834cc --- /dev/null +++ b/.changeset/copilot-model-availability.md @@ -0,0 +1,5 @@ +--- +"openwiki": patch +--- + +feat: validate selected github copilot models against account availability before inference diff --git a/src/model-availability.ts b/src/model-availability.ts index a737cd5ae..6125de3d9 100644 --- a/src/model-availability.ts +++ b/src/model-availability.ts @@ -16,7 +16,17 @@ type OpenAIModelListResponse = { data?: Array<{ id?: unknown }>; }; +type CopilotModelListResponse = { + data?: Array<{ + id?: unknown; + capabilities?: { type?: unknown }; + model_picker_enabled?: unknown; + policy?: { state?: unknown }; + }>; +}; + const OPENAI_API_BASE_URL = "https://api.openai.com/v1"; +const COPILOT_API_BASE_URL = "https://api.githubcopilot.com"; /** * Checks whether a selected model is exposed to the configured provider @@ -27,13 +37,24 @@ export async function getSelectedModelAvailability( check: ModelAvailabilityCheck, fetchImpl: typeof fetch = globalThis.fetch, ): Promise { - if (check.provider !== "openai") { - return { - status: "unknown", - reason: "No availability adapter is configured.", - }; + if (check.provider === "openai") { + return getOpenAIModelAvailability(check, fetchImpl); + } + + if (check.provider === "copilot") { + return getCopilotModelAvailability(check, fetchImpl); } + return { + status: "unknown", + reason: "No availability adapter is configured.", + }; +} + +async function getOpenAIModelAvailability( + check: ModelAvailabilityCheck, + fetchImpl: typeof fetch, +): Promise { if (check.baseUrl !== undefined) { return { status: "unknown", @@ -83,3 +104,72 @@ export async function getSelectedModelAvailability( }; } } + +async function getCopilotModelAvailability( + check: ModelAvailabilityCheck, + fetchImpl: typeof fetch, +): Promise { + if (!check.apiKey) { + return { + status: "unknown", + reason: "No API key is available for validation.", + }; + } + + const baseUrl = check.baseUrl ?? COPILOT_API_BASE_URL; + + try { + const response = await fetchImpl(`${baseUrl.replace(/\/+$/u, "")}/models`, { + headers: { Authorization: `Bearer ${check.apiKey}` }, + }); + + if (!response.ok) { + return { + status: "unknown", + reason: `Model availability lookup returned HTTP ${response.status}.`, + }; + } + + const body = (await response.json()) as CopilotModelListResponse; + if (!Array.isArray(body.data)) { + return { + status: "unknown", + reason: "Model availability lookup returned an unexpected response.", + }; + } + + const model = body.data.find((candidate) => candidate.id === check.modelId); + if (!model || model.capabilities?.type !== "chat") { + return { + status: "unavailable", + reason: + "The selected model is not available to this GitHub Copilot account.", + }; + } + + if ( + model.policy?.state === "enabled" || + (model.policy?.state === undefined && model.model_picker_enabled === true) + ) { + return { status: "available" }; + } + + if (model.policy?.state === "disabled") { + return { + status: "unavailable", + reason: + "The selected model is not available to this GitHub Copilot account.", + }; + } + + return { + status: "unknown", + reason: "The Copilot Models API did not report an eligibility state.", + }; + } catch { + return { + status: "unknown", + reason: "Model availability lookup could not be completed.", + }; + } +} diff --git a/test/model-availability.test.ts b/test/model-availability.test.ts index 1872a33f5..4cdd2d741 100644 --- a/test/model-availability.test.ts +++ b/test/model-availability.test.ts @@ -7,6 +7,24 @@ const OPENAI_CHECK = { provider: "openai" as const, }; +const COPILOT_CHECK = { + apiKey: "test-copilot-token", + modelId: "claude-fable-5", + provider: "copilot" as const, +}; + +function fetchInputUrl(input: Parameters[0]): string { + if (typeof input === "string") { + return input; + } + + if (input instanceof URL) { + return input.toString(); + } + + return input.url; +} + describe("getSelectedModelAvailability", () => { test("accepts a selected model returned by the OpenAI Models API", async () => { const result = await getSelectedModelAvailability(OPENAI_CHECK, () => @@ -41,6 +59,175 @@ describe("getSelectedModelAvailability", () => { expect(result).toMatchObject({ status: "unknown" }); }); + test("accepts a selected chat model enabled by Copilot account policy", async () => { + const requests: string[] = []; + const result = await getSelectedModelAvailability( + COPILOT_CHECK, + (input, init) => { + requests.push(fetchInputUrl(input)); + expect(new Headers(init?.headers).get("Authorization")).toBe( + "Bearer test-copilot-token", + ); + return Promise.resolve( + Response.json({ + data: [ + { + id: "claude-fable-5", + capabilities: { type: "chat" }, + model_picker_enabled: false, + policy: { state: "enabled" }, + }, + ], + }), + ); + }, + ); + + expect(result).toEqual({ status: "available" }); + expect(requests).toEqual(["https://api.githubcopilot.com/models"]); + }); + + test("rejects a selected Copilot model missing from the account model list", async () => { + const result = await getSelectedModelAvailability(COPILOT_CHECK, () => + Promise.resolve( + Response.json({ + data: [ + { + id: "gpt-5.6-terra", + capabilities: { type: "chat" }, + model_picker_enabled: true, + }, + ], + }), + ), + ); + + expect(result).toMatchObject({ status: "unavailable" }); + }); + + test("rejects a selected Copilot model disabled by account policy", async () => { + const result = await getSelectedModelAvailability(COPILOT_CHECK, () => + Promise.resolve( + Response.json({ + data: [ + { + id: "claude-fable-5", + capabilities: { type: "chat" }, + model_picker_enabled: false, + policy: { state: "disabled" }, + }, + ], + }), + ), + ); + + expect(result).toMatchObject({ status: "unavailable" }); + }); + + test("rejects a selected Copilot model that is not a chat model", async () => { + const result = await getSelectedModelAvailability(COPILOT_CHECK, () => + Promise.resolve( + Response.json({ + data: [ + { + id: "claude-fable-5", + capabilities: { type: "embeddings" }, + model_picker_enabled: true, + }, + ], + }), + ), + ); + + expect(result).toMatchObject({ status: "unavailable" }); + }); + + test("accepts a picker-enabled chat model when policy state is absent", async () => { + const result = await getSelectedModelAvailability(COPILOT_CHECK, () => + Promise.resolve( + Response.json({ + data: [ + { + id: "claude-fable-5", + capabilities: { type: "chat" }, + model_picker_enabled: true, + }, + ], + }), + ), + ); + + expect(result).toEqual({ status: "available" }); + }); + + test("does not block inference when Copilot eligibility is ambiguous", async () => { + const result = await getSelectedModelAvailability(COPILOT_CHECK, () => + Promise.resolve( + Response.json({ + data: [ + { + id: "claude-fable-5", + capabilities: { type: "chat" }, + model_picker_enabled: false, + }, + ], + }), + ), + ); + + expect(result).toMatchObject({ status: "unknown" }); + }); + + test("does not block inference when no Copilot token is available", async () => { + const result = await getSelectedModelAvailability( + { ...COPILOT_CHECK, apiKey: undefined }, + () => Promise.reject(new Error("fetch must not be called")), + ); + + expect(result).toMatchObject({ status: "unknown" }); + }); + + test("does not block inference when the Copilot Models API fails", async () => { + const result = await getSelectedModelAvailability(COPILOT_CHECK, () => + Promise.resolve(new Response("forbidden", { status: 403 })), + ); + + expect(result).toMatchObject({ status: "unknown" }); + }); + + test("does not block inference when the Copilot Models API shape is unexpected", async () => { + const result = await getSelectedModelAvailability(COPILOT_CHECK, () => + Promise.resolve(Response.json({ models: [] })), + ); + + expect(result).toMatchObject({ status: "unknown" }); + }); + + test("uses the configured Copilot base URL without duplicating slashes", async () => { + const requests: string[] = []; + const result = await getSelectedModelAvailability( + { ...COPILOT_CHECK, baseUrl: "https://tenant.ghe.com/api/copilot/" }, + (input) => { + requests.push(fetchInputUrl(input)); + return Promise.resolve( + Response.json({ + data: [ + { + id: "claude-fable-5", + capabilities: { type: "chat" }, + model_picker_enabled: false, + policy: { state: "enabled" }, + }, + ], + }), + ); + }, + ); + + expect(result).toEqual({ status: "available" }); + expect(requests).toEqual(["https://tenant.ghe.com/api/copilot/models"]); + }); + test("does not validate providers without an availability adapter", async () => { const result = await getSelectedModelAvailability({ ...OPENAI_CHECK,