diff --git a/src/node/services/providerModelFactory.ts b/src/node/services/providerModelFactory.ts index f5d7fb31bcf..28e73b46fef 100644 --- a/src/node/services/providerModelFactory.ts +++ b/src/node/services/providerModelFactory.ts @@ -83,6 +83,7 @@ import { normalizeGatewayStreamUsage, normalizeGatewayGenerateResult, } from "@/node/utils/gatewayStreamNormalization"; +import { wrapFetchWithGatewayFilePartNormalization } from "@/node/utils/gatewayFilePartNormalization"; import { EnvHttpProxyAgent, type Dispatcher } from "undici"; import packageJson from "../../../package.json"; @@ -2096,7 +2097,12 @@ export class ProviderModelFactory { ); // For Anthropic models via gateway, normalize cache_control on the final payload. // Use getProviderFetch to preserve any user-configured custom fetch (e.g., proxies) - const baseFetch = getProviderFetch(providerConfig); + // The gateway server still expects spec-v3 string-encoded file parts; + // rewrite the v4 `{ type: "data" | "url" }` objects the 4.x SDK emits + // so image attachments are not rejected as "invalid request". + const baseFetch = wrapFetchWithGatewayFilePartNormalization( + getProviderFetch(providerConfig) + ); const isAnthropicModel = modelId.startsWith("anthropic/"); const disableBeta = muxProviderOptions?.anthropic?.disableBetaFeatures === true; // For Anthropic models via gateway, wrap for cache_control normalization; diff --git a/src/node/utils/gatewayFilePartNormalization.test.ts b/src/node/utils/gatewayFilePartNormalization.test.ts new file mode 100644 index 00000000000..ba776008f7b --- /dev/null +++ b/src/node/utils/gatewayFilePartNormalization.test.ts @@ -0,0 +1,239 @@ +import { describe, expect, it } from "bun:test"; +import { + normalizeFileDataToV3, + normalizeGatewayPromptFileParts, + wrapFetchWithGatewayFilePartNormalization, +} from "./gatewayFilePartNormalization"; + +describe("normalizeFileDataToV3", () => { + it("converts inline data objects to a data: URL string", () => { + expect(normalizeFileDataToV3({ type: "data", data: "aGVsbG8=" }, "image/png")).toBe( + "data:image/png;base64,aGVsbG8=" + ); + }); + + it("falls back to application/octet-stream when mediaType is missing", () => { + expect(normalizeFileDataToV3({ type: "data", data: "aGVsbG8=" }, undefined)).toBe( + "data:application/octet-stream;base64,aGVsbG8=" + ); + }); + + it("converts url objects to the url string", () => { + expect( + normalizeFileDataToV3({ type: "url", url: "https://example.com/a.png" }, "image/png") + ).toBe("https://example.com/a.png"); + }); + + it("passes strings and unknown shapes through untouched", () => { + expect(normalizeFileDataToV3("data:image/png;base64,abc", "image/png")).toBe( + "data:image/png;base64,abc" + ); + const weird = { type: "mystery" }; + expect(normalizeFileDataToV3(weird, "image/png")).toBe(weird); + }); +}); + +describe("normalizeGatewayPromptFileParts", () => { + it("rewrites user file parts in place and reports a change", () => { + const body: Record = { + prompt: [ + { role: "system", content: "sys" }, + { + role: "user", + content: [ + { type: "text", text: "look at this." }, + { + type: "file", + mediaType: "image/png", + filename: "image.png", + data: { type: "data", data: "iVBOR" }, + providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } }, + }, + ], + }, + ], + }; + + expect(normalizeGatewayPromptFileParts(body)).toBe(true); + const user = (body.prompt as Array<{ content: unknown }>)[1]; + expect(user.content).toEqual([ + { type: "text", text: "look at this." }, + { + type: "file", + mediaType: "image/png", + filename: "image.png", + data: "data:image/png;base64,iVBOR", + providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } }, + }, + ]); + }); + + it("rewrites reasoning-file parts and files nested in tool-result content", () => { + const body: Record = { + prompt: [ + { + role: "assistant", + content: [ + { + type: "reasoning-file", + mediaType: "image/jpeg", + data: { type: "data", data: "abc" }, + }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "t1", + toolName: "screenshot", + output: { + type: "content", + value: [ + { type: "text", text: "captured" }, + { type: "file", mediaType: "image/png", data: { type: "data", data: "xyz" } }, + ], + }, + }, + { + type: "tool-result", + toolCallId: "t2", + toolName: "echo", + output: { type: "text", value: "plain" }, + }, + ], + }, + ], + }; + + expect(normalizeGatewayPromptFileParts(body)).toBe(true); + expect(body).toEqual({ + prompt: [ + { + role: "assistant", + content: [ + { type: "reasoning-file", mediaType: "image/jpeg", data: "data:image/jpeg;base64,abc" }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "t1", + toolName: "screenshot", + output: { + type: "content", + value: [ + { type: "text", text: "captured" }, + { type: "file", mediaType: "image/png", data: "data:image/png;base64,xyz" }, + ], + }, + }, + { + type: "tool-result", + toolCallId: "t2", + toolName: "echo", + output: { type: "text", value: "plain" }, + }, + ], + }, + ], + }); + }); + + it("returns false and leaves text-only prompts alone", () => { + const body: Record = { + prompt: [{ role: "user", content: [{ type: "text", text: "hi" }] }], + }; + const snapshot = structuredClone(body); + expect(normalizeGatewayPromptFileParts(body)).toBe(false); + expect(body).toEqual(snapshot); + }); + + it("returns false when prompt is missing or malformed", () => { + expect(normalizeGatewayPromptFileParts({})).toBe(false); + expect(normalizeGatewayPromptFileParts({ prompt: "nope" })).toBe(false); + expect(normalizeGatewayPromptFileParts({ prompt: [null, 42, { content: "str" }] })).toBe(false); + }); +}); + +describe("wrapFetchWithGatewayFilePartNormalization", () => { + function captureFetch() { + const calls: Array<{ input: unknown; init: RequestInit | undefined }> = []; + const base = ((input: unknown, init?: RequestInit) => { + calls.push({ input, init }); + return Promise.resolve(new Response("ok")); + }) as unknown as typeof fetch; + return { base, calls }; + } + + it("rewrites file parts in POST JSON bodies and drops content-length", async () => { + const { base, calls } = captureFetch(); + const wrapped = wrapFetchWithGatewayFilePartNormalization(base); + const body = JSON.stringify({ + prompt: [ + { + role: "user", + content: [ + { type: "file", mediaType: "image/png", data: { type: "data", data: "iVBOR" } }, + ], + }, + ], + }); + + await wrapped("https://gateway.example/language-model", { + method: "POST", + headers: { "content-type": "application/json", "content-length": String(body.length) }, + body, + }); + + expect(calls).toHaveLength(1); + const init = calls[0].init!; + const sent = JSON.parse(init.body as string) as { + prompt: Array<{ content: Array<{ data: unknown }> }>; + }; + expect(sent.prompt[0].content[0].data).toBe("data:image/png;base64,iVBOR"); + const headers = new Headers(init.headers); + expect(headers.get("content-length")).toBeNull(); + expect(headers.get("content-type")).toBe("application/json"); + }); + + it("forwards text-only requests with the original init object", async () => { + const { base, calls } = captureFetch(); + const wrapped = wrapFetchWithGatewayFilePartNormalization(base); + const init: RequestInit = { + method: "POST", + body: JSON.stringify({ prompt: [{ role: "user", content: [{ type: "text", text: "hi" }] }] }), + }; + + await wrapped("https://gateway.example/language-model", init); + + expect(calls).toHaveLength(1); + expect(calls[0].init).toBe(init); + }); + + it("forwards non-POST and non-string bodies unchanged", async () => { + const { base, calls } = captureFetch(); + const wrapped = wrapFetchWithGatewayFilePartNormalization(base); + const getInit: RequestInit = { method: "GET" }; + const streamInit: RequestInit = { method: "POST", body: new Uint8Array([1, 2, 3]) }; + + await wrapped("https://gateway.example/models", getInit); + await wrapped("https://gateway.example/language-model", streamInit); + + expect(calls[0].init).toBe(getInit); + expect(calls[1].init).toBe(streamInit); + }); + + it("forwards unparseable bodies unchanged", async () => { + const { base, calls } = captureFetch(); + const wrapped = wrapFetchWithGatewayFilePartNormalization(base); + const init: RequestInit = { method: "POST", body: '{"type":"file" broken' }; + + await wrapped("https://gateway.example/language-model", init); + + expect(calls[0].init).toBe(init); + }); +}); diff --git a/src/node/utils/gatewayFilePartNormalization.ts b/src/node/utils/gatewayFilePartNormalization.ts new file mode 100644 index 00000000000..b710fb8c84a --- /dev/null +++ b/src/node/utils/gatewayFilePartNormalization.ts @@ -0,0 +1,117 @@ +/** + * Gateway file-part wire normalization. + * + * @ai-sdk/gateway 4.x (AI SDK v7, spec v4) serializes inline file data as a + * tagged object on the wire: + * + * { type: "file", mediaType, data: { type: "data", data: "" } } + * { type: "file", mediaType, data: { type: "url", url: "https://…" } } + * + * The 3.x SDK (spec v3) sent a plain string instead — a `data:` URL for inline + * bytes, or the URL string for remote files: + * + * { type: "file", mediaType, data: "data:image/png;base64," } + * + * The mux gateway server still validates the v3 shape and rejects the tagged + * object with "invalid request", so attaching an image to a chat message fails + * while text-only turns work. This module rewrites file parts in the outgoing + * `prompt` back to the v3 string encoding at the final wire-shaping step. + * + * Remove once the gateway server accepts spec v4 file parts. + */ + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value != null && !Array.isArray(value); +} + +/** + * Convert a v4 `LanguageModelV4DataContent` wire object to the v3 string form. + * Returns the input unchanged when it is already a string or not recognizable. + */ +export function normalizeFileDataToV3(data: unknown, mediaType: unknown): unknown { + if (!isRecord(data)) return data; + if (data.type === "data" && typeof data.data === "string") { + const mime = + typeof mediaType === "string" && mediaType.length > 0 + ? mediaType + : "application/octet-stream"; + return `data:${mime};base64,${data.data}`; + } + if (data.type === "url" && typeof data.url === "string") { + return data.url; + } + return data; +} + +function normalizeFilePart(part: Record): boolean { + const normalized = normalizeFileDataToV3(part.data, part.mediaType); + if (normalized === part.data) return false; + part.data = normalized; + return true; +} + +/** + * Rewrite v4 file-part data objects in a gateway `language-model` request body + * to the v3 string encoding, in place. Handles user/assistant `file` and + * `reasoning-file` parts as well as files nested inside tool-result content. + * + * @returns true when at least one part was rewritten. + */ +export function normalizeGatewayPromptFileParts(body: Record): boolean { + const prompt = body.prompt; + if (!Array.isArray(prompt)) return false; + + let changed = false; + for (const message of prompt) { + if (!isRecord(message) || !Array.isArray(message.content)) continue; + for (const part of message.content) { + if (!isRecord(part)) continue; + if (part.type === "file" || part.type === "reasoning-file") { + changed = normalizeFilePart(part) || changed; + } else if (part.type === "tool-result" && isRecord(part.output)) { + const output = part.output; + if (output.type === "content" && Array.isArray(output.value)) { + for (const contentPart of output.value) { + if (isRecord(contentPart) && contentPart.type === "file") { + changed = normalizeFilePart(contentPart) || changed; + } + } + } + } + } + } + return changed; +} + +/** + * Wrap fetch so gateway `language-model` POST bodies carry v3-encoded file + * parts. Requests without file parts are forwarded byte-for-byte unchanged. + */ +export function wrapFetchWithGatewayFilePartNormalization(baseFetch: typeof fetch): typeof fetch { + const wrappedFetch = async ( + input: Parameters[0], + init?: Parameters[1] + ): Promise => { + if (init?.method?.toUpperCase() !== "POST" || typeof init?.body !== "string") { + return baseFetch(input, init); + } + // Cheap pre-check so text-only turns skip a full JSON round-trip. + if (!init.body.includes('"type":"file"') && !init.body.includes('"type":"reasoning-file"')) { + return baseFetch(input, init); + } + try { + const json = JSON.parse(init.body) as unknown; + if (!isRecord(json) || !normalizeGatewayPromptFileParts(json)) { + return baseFetch(input, init); + } + const outHeaders = new Headers(init.headers); + outHeaders.delete("content-length"); // Body size changed + return baseFetch(input, { ...init, headers: outHeaders, body: JSON.stringify(json) }); + } catch { + // Not JSON we understand; forward unchanged. + return baseFetch(input, init); + } + }; + + return Object.assign(wrappedFetch, baseFetch) as typeof fetch; +}