🤖 fix: normalize mux-gateway file parts so image attachments are accepted

src/node/services/providerModelFactory.ts

@@ -82,6 +82,7 @@ import {
   normalizeGatewayStreamUsage,
   normalizeGatewayGenerateResult,
 } from "@/node/utils/gatewayStreamNormalization";
+import { wrapFetchWithGatewayFilePartNormalization } from "@/node/utils/gatewayFilePartNormalization";
 import { EnvHttpProxyAgent, type Dispatcher } from "undici";
 import packageJson from "../../../package.json";
 
@@ -2083,7 +2084,12 @@ export class ProviderModelFactory {
           );
           // For Anthropic models via gateway, normalize cache_control on the final payload.
           // Use getProviderFetch to preserve any user-configured custom fetch (e.g., proxies)
-          const baseFetch = getProviderFetch(providerConfig);
+          // The gateway server still expects spec-v3 string-encoded file parts;
+          // rewrite the v4 `{ type: "data" | "url" }` objects the 4.x SDK emits
+          // so image attachments are not rejected as "invalid request".
+          const baseFetch = wrapFetchWithGatewayFilePartNormalization(
+            getProviderFetch(providerConfig)
+          );
           const isAnthropicModel = modelId.startsWith("anthropic/");
           const disableBeta = muxProviderOptions?.anthropic?.disableBetaFeatures === true;
           // For Anthropic models via gateway, wrap for cache_control normalization;

src/node/utils/gatewayFilePartNormalization.test.ts

@@ -0,0 +1,239 @@
+import { describe, expect, it } from "bun:test";
+import {
+  normalizeFileDataToV3,
+  normalizeGatewayPromptFileParts,
+  wrapFetchWithGatewayFilePartNormalization,
+} from "./gatewayFilePartNormalization";
+
+describe("normalizeFileDataToV3", () => {
+  it("converts inline data objects to a data: URL string", () => {
+    expect(normalizeFileDataToV3({ type: "data", data: "aGVsbG8=" }, "image/png")).toBe(
+      "data:image/png;base64,aGVsbG8="
+    );
+  });
+
+  it("falls back to application/octet-stream when mediaType is missing", () => {
+    expect(normalizeFileDataToV3({ type: "data", data: "aGVsbG8=" }, undefined)).toBe(
+      "data:application/octet-stream;base64,aGVsbG8="
+    );
+  });
+
+  it("converts url objects to the url string", () => {
+    expect(
+      normalizeFileDataToV3({ type: "url", url: "https://example.com/a.png" }, "image/png")
+    ).toBe("https://example.com/a.png");
+  });
+
+  it("passes strings and unknown shapes through untouched", () => {
+    expect(normalizeFileDataToV3("data:image/png;base64,abc", "image/png")).toBe(
+      "data:image/png;base64,abc"
+    );
+    const weird = { type: "mystery" };
+    expect(normalizeFileDataToV3(weird, "image/png")).toBe(weird);
+  });
+});
+
+describe("normalizeGatewayPromptFileParts", () => {
+  it("rewrites user file parts in place and reports a change", () => {
+    const body: Record<string, unknown> = {
+      prompt: [
+        { role: "system", content: "sys" },
+        {
+          role: "user",
+          content: [
+            { type: "text", text: "look at this." },
+            {
+              type: "file",
+              mediaType: "image/png",
+              filename: "image.png",
+              data: { type: "data", data: "iVBOR" },
+              providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },
+            },
+          ],
+        },
+      ],
+    };
+
+    expect(normalizeGatewayPromptFileParts(body)).toBe(true);
+    const user = (body.prompt as Array<{ content: unknown }>)[1];
+    expect(user.content).toEqual([
+      { type: "text", text: "look at this." },
+      {
+        type: "file",
+        mediaType: "image/png",
+        filename: "image.png",
+        data: "data:image/png;base64,iVBOR",
+        providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },
+      },
+    ]);
+  });
+
+  it("rewrites reasoning-file parts and files nested in tool-result content", () => {
+    const body: Record<string, unknown> = {
+      prompt: [
+        {
+          role: "assistant",
+          content: [
+            {
+              type: "reasoning-file",
+              mediaType: "image/jpeg",
+              data: { type: "data", data: "abc" },
+            },
+          ],
+        },
+        {
+          role: "tool",
+          content: [
+            {
+              type: "tool-result",
+              toolCallId: "t1",
+              toolName: "screenshot",
+              output: {
+                type: "content",
+                value: [
+                  { type: "text", text: "captured" },
+                  { type: "file", mediaType: "image/png", data: { type: "data", data: "xyz" } },
+                ],
+              },
+            },
+            {
+              type: "tool-result",
+              toolCallId: "t2",
+              toolName: "echo",
+              output: { type: "text", value: "plain" },
+            },
+          ],
+        },
+      ],
+    };
+
+    expect(normalizeGatewayPromptFileParts(body)).toBe(true);
+    expect(body).toEqual({
+      prompt: [
+        {
+          role: "assistant",
+          content: [
+            { type: "reasoning-file", mediaType: "image/jpeg", data: "data:image/jpeg;base64,abc" },
+          ],
+        },
+        {
+          role: "tool",
+          content: [
+            {
+              type: "tool-result",
+              toolCallId: "t1",
+              toolName: "screenshot",
+              output: {
+                type: "content",
+                value: [
+                  { type: "text", text: "captured" },
+                  { type: "file", mediaType: "image/png", data: "data:image/png;base64,xyz" },
+                ],
+              },
+            },
+            {
+              type: "tool-result",
+              toolCallId: "t2",
+              toolName: "echo",
+              output: { type: "text", value: "plain" },
+            },
+          ],
+        },
+      ],
+    });
+  });
+
+  it("returns false and leaves text-only prompts alone", () => {
+    const body: Record<string, unknown> = {
+      prompt: [{ role: "user", content: [{ type: "text", text: "hi" }] }],
+    };
+    const snapshot = structuredClone(body);
+    expect(normalizeGatewayPromptFileParts(body)).toBe(false);
+    expect(body).toEqual(snapshot);
+  });
+
+  it("returns false when prompt is missing or malformed", () => {
+    expect(normalizeGatewayPromptFileParts({})).toBe(false);
+    expect(normalizeGatewayPromptFileParts({ prompt: "nope" })).toBe(false);
+    expect(normalizeGatewayPromptFileParts({ prompt: [null, 42, { content: "str" }] })).toBe(false);
+  });
+});
+
+describe("wrapFetchWithGatewayFilePartNormalization", () => {
+  function captureFetch() {
+    const calls: Array<{ input: unknown; init: RequestInit | undefined }> = [];
+    const base = ((input: unknown, init?: RequestInit) => {
+      calls.push({ input, init });
+      return Promise.resolve(new Response("ok"));
+    }) as unknown as typeof fetch;
+    return { base, calls };
+  }
+
+  it("rewrites file parts in POST JSON bodies and drops content-length", async () => {
+    const { base, calls } = captureFetch();
+    const wrapped = wrapFetchWithGatewayFilePartNormalization(base);
+    const body = JSON.stringify({
+      prompt: [
+        {
+          role: "user",
+          content: [
+            { type: "file", mediaType: "image/png", data: { type: "data", data: "iVBOR" } },
+          ],
+        },
+      ],
+    });
+
+    await wrapped("https://gateway.example/language-model", {
+      method: "POST",
+      headers: { "content-type": "application/json", "content-length": String(body.length) },
+      body,
+    });
+
+    expect(calls).toHaveLength(1);
+    const init = calls[0].init!;
+    const sent = JSON.parse(init.body as string) as {
+      prompt: Array<{ content: Array<{ data: unknown }> }>;
+    };
+    expect(sent.prompt[0].content[0].data).toBe("data:image/png;base64,iVBOR");
+    const headers = new Headers(init.headers);
+    expect(headers.get("content-length")).toBeNull();
+    expect(headers.get("content-type")).toBe("application/json");
+  });
+
+  it("forwards text-only requests with the original init object", async () => {
+    const { base, calls } = captureFetch();
+    const wrapped = wrapFetchWithGatewayFilePartNormalization(base);
+    const init: RequestInit = {
+      method: "POST",
+      body: JSON.stringify({ prompt: [{ role: "user", content: [{ type: "text", text: "hi" }] }] }),
+    };
+
+    await wrapped("https://gateway.example/language-model", init);
+
+    expect(calls).toHaveLength(1);
+    expect(calls[0].init).toBe(init);
+  });
+
+  it("forwards non-POST and non-string bodies unchanged", async () => {
+    const { base, calls } = captureFetch();
+    const wrapped = wrapFetchWithGatewayFilePartNormalization(base);
+    const getInit: RequestInit = { method: "GET" };
+    const streamInit: RequestInit = { method: "POST", body: new Uint8Array([1, 2, 3]) };
+
+    await wrapped("https://gateway.example/models", getInit);
+    await wrapped("https://gateway.example/language-model", streamInit);
+
+    expect(calls[0].init).toBe(getInit);
+    expect(calls[1].init).toBe(streamInit);
+  });
+
+  it("forwards unparseable bodies unchanged", async () => {
+    const { base, calls } = captureFetch();
+    const wrapped = wrapFetchWithGatewayFilePartNormalization(base);
+    const init: RequestInit = { method: "POST", body: '{"type":"file" broken' };
+
+    await wrapped("https://gateway.example/language-model", init);
+
+    expect(calls[0].init).toBe(init);
+  });
+});

src/node/utils/gatewayFilePartNormalization.ts

@@ -0,0 +1,117 @@
+/**
+ * Gateway file-part wire normalization.
+ *
+ * @ai-sdk/gateway 4.x (AI SDK v7, spec v4) serializes inline file data as a
+ * tagged object on the wire:
+ *
+ *   { type: "file", mediaType, data: { type: "data", data: "<base64>" } }
+ *   { type: "file", mediaType, data: { type: "url",  url:  "https://…" } }
+ *
+ * The 3.x SDK (spec v3) sent a plain string instead — a `data:` URL for inline
+ * bytes, or the URL string for remote files:
+ *
+ *   { type: "file", mediaType, data: "data:image/png;base64,<base64>" }
+ *
+ * The mux gateway server still validates the v3 shape and rejects the tagged
+ * object with "invalid request", so attaching an image to a chat message fails
+ * while text-only turns work. This module rewrites file parts in the outgoing
+ * `prompt` back to the v3 string encoding at the final wire-shaping step.
+ *
+ * Remove once the gateway server accepts spec v4 file parts.
+ */
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value != null && !Array.isArray(value);
+}
+
+/**
+ * Convert a v4 `LanguageModelV4DataContent` wire object to the v3 string form.
+ * Returns the input unchanged when it is already a string or not recognizable.
+ */
+export function normalizeFileDataToV3(data: unknown, mediaType: unknown): unknown {
+  if (!isRecord(data)) return data;
+  if (data.type === "data" && typeof data.data === "string") {
+    const mime =
+      typeof mediaType === "string" && mediaType.length > 0
+        ? mediaType
+        : "application/octet-stream";
+    return `data:${mime};base64,${data.data}`;
+  }
+  if (data.type === "url" && typeof data.url === "string") {
+    return data.url;
+  }
+  return data;
+}
+
+function normalizeFilePart(part: Record<string, unknown>): boolean {
+  const normalized = normalizeFileDataToV3(part.data, part.mediaType);
+  if (normalized === part.data) return false;
+  part.data = normalized;
+  return true;
+}
+
+/**
+ * Rewrite v4 file-part data objects in a gateway `language-model` request body
+ * to the v3 string encoding, in place. Handles user/assistant `file` and
+ * `reasoning-file` parts as well as files nested inside tool-result content.
+ *
+ * @returns true when at least one part was rewritten.
+ */
+export function normalizeGatewayPromptFileParts(body: Record<string, unknown>): boolean {
+  const prompt = body.prompt;
+  if (!Array.isArray(prompt)) return false;
+
+  let changed = false;
+  for (const message of prompt) {
+    if (!isRecord(message) || !Array.isArray(message.content)) continue;
+    for (const part of message.content) {
+      if (!isRecord(part)) continue;
+      if (part.type === "file" || part.type === "reasoning-file") {
+        changed = normalizeFilePart(part) || changed;
+      } else if (part.type === "tool-result" && isRecord(part.output)) {
+        const output = part.output;
+        if (output.type === "content" && Array.isArray(output.value)) {
+          for (const contentPart of output.value) {
+            if (isRecord(contentPart) && contentPart.type === "file") {
+              changed = normalizeFilePart(contentPart) || changed;
+            }
+          }
+        }
+      }
+    }
+  }
+  return changed;
+}
+
+/**
+ * Wrap fetch so gateway `language-model` POST bodies carry v3-encoded file
+ * parts. Requests without file parts are forwarded byte-for-byte unchanged.
+ */
+export function wrapFetchWithGatewayFilePartNormalization(baseFetch: typeof fetch): typeof fetch {
+  const wrappedFetch = async (
+    input: Parameters<typeof fetch>[0],
+    init?: Parameters<typeof fetch>[1]
+  ): Promise<Response> => {
+    if (init?.method?.toUpperCase() !== "POST" || typeof init?.body !== "string") {
+      return baseFetch(input, init);
+    }
+    // Cheap pre-check so text-only turns skip a full JSON round-trip.
+    if (!init.body.includes('"type":"file"') && !init.body.includes('"type":"reasoning-file"')) {
+      return baseFetch(input, init);
+    }
+    try {
+      const json = JSON.parse(init.body) as unknown;
+      if (!isRecord(json) || !normalizeGatewayPromptFileParts(json)) {
+        return baseFetch(input, init);
+      }
+      const outHeaders = new Headers(init.headers);
+      outHeaders.delete("content-length"); // Body size changed
+      return baseFetch(input, { ...init, headers: outHeaders, body: JSON.stringify(json) });
+    } catch {
+      // Not JSON we understand; forward unchanged.
+      return baseFetch(input, init);
+    }
+  };
+
+  return Object.assign(wrappedFetch, baseFetch) as typeof fetch;
+}