From 4ddc53c70a81e8e06709a0ead4f527acddc0a290 Mon Sep 17 00:00:00 2001 From: Michal Kubenka Date: Tue, 21 Jul 2026 15:24:04 +0200 Subject: [PATCH 1/2] fix(gemini): type mp4/mov/webm uploads so Gemini receives them Gemini silently discards uploads it cannot type. Any extension missing from GEMINI_UPLOAD_MIME_TYPES fell back to `application/octet-stream`, so the run reported the file as attached (`files=1`) while the model answered as though nothing had been sent. That made video look unsupported when the upload path itself was fine. Declaring `video/mp4` for the same MP4 works immediately: Gemini describes the footage and quotes on-screen text. Add only the three formats confirmed against the live endpoint, and pull the lookup into an exported `resolveGeminiUploadMimeType` so it can be tested. Probed each candidate with a sentinel prompt that distinguishes "received" from "actually viewable": .mp4 .mov .webm viewable - describe people, clothing and on-screen text .avi received but not viewable; only echoed container metadata .m4v dropped, despite identical bytes to a working .mp4 and the same declared video/mp4 - the endpoint gates on the extension too .mkv .3gp .heic Gemini returns an error .wav .mp3 .m4a never delivered, with audio/mpeg, audio/mp3, audio/wav or audio/x-wav .txt never delivered, with text/plain or text/md So audio and plain text are not supported by this endpoint at all, and the accepted set cannot be extrapolated from the Gemini API's documented formats. Mapping them would advertise support that does not exist. Note the 1 MB DEFAULT_MAX_FILE_SIZE_BYTES still applies; video generally needs `--max-file-size-bytes` / ORACLE_MAX_FILE_SIZE_BYTES. --- CHANGELOG.md | 4 +++ src/gemini-web/client.ts | 24 +++++++++++++++-- tests/gemini-upload-mime.test.ts | 45 ++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 tests/gemini-upload-mime.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cffd52dc..066e3b927 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ ### Fixed - Browser: ignore transient `/c/WEB:` routes until ChatGPT exposes the durable conversation URL, preventing completed GPT-5.6 and Pro answers from hanging until timeout under a mismatched response scope. Fixes #333. Thanks @dbachko and @kesslerio! +- Gemini: type `.mp4`, `.mov` and `.webm` uploads as video instead of sending them as + `application/octet-stream`, which Gemini silently discards. Previously a `--file clip.mp4` + run reported the attachment as sent (`files=1`) while the model replied as though it had + received nothing, making video look unsupported. Verified against gemini-3.1-pro. - Browser: recover completed answers after a recoverable DevTools disconnect by confirming target liveness and attempting bounded reattachment, while preserving fail-closed handling for unavailable targets. Fixes #326. Thanks @piyushbag! - CLI: avoid inheriting `browser.thinkingTime` from config when `--browser-model-strategy current` is explicit, while preserving an explicit `--browser-thinking-time` override. Thanks @jung0han! - Browser/Serve: keep the authenticated manual-login Chrome process alive while closing each successfully captured service-owned run tab, preventing renderer and memory accumulation across repeated remote consultations without changing explicit `--browser-keep-browser`, attached-tab, or incomplete-run recovery behavior. Thanks @rtl-ai! diff --git a/src/gemini-web/client.ts b/src/gemini-web/client.ts index 4c5a31e9d..c400131b9 100644 --- a/src/gemini-web/client.ts +++ b/src/gemini-web/client.ts @@ -49,12 +49,33 @@ const GEMINI_UPLOAD_MIME_TYPES: Record = { ".gif": "image/gif", ".jpeg": "image/jpeg", ".jpg": "image/jpeg", + ".mov": "video/quicktime", + ".mp4": "video/mp4", ".pdf": "application/pdf", ".png": "image/png", ".svg": "image/svg+xml", + ".webm": "video/webm", ".webp": "image/webp", }; +/** + * Resolve the MIME type Gemini should be told an upload carries. + * + * Gemini silently discards uploads it cannot type: the run still reports the file as + * attached, but the model never receives it and answers as though nothing was sent. + * Anything falling back to `application/octet-stream` is therefore invisible to the model. + * + * Only formats confirmed to work against the Gemini web upload endpoint are listed. The + * endpoint also gates on the file extension, not just the declared type — an `.m4v` byte + * for byte identical to a working `.mp4`, and declared `video/mp4`, is still dropped — so + * entries here cannot be extrapolated from what the Gemini API documents. + */ +export function resolveGeminiUploadMimeType(filePath: string): string { + return ( + GEMINI_UPLOAD_MIME_TYPES[path.extname(filePath).toLowerCase()] ?? "application/octet-stream" + ); +} + function getNestedValue(value: unknown, pathParts: Array, fallback: T): T { let current: unknown = value; for (const part of pathParts) { @@ -187,8 +208,7 @@ async function uploadGeminiFile( const absPath = path.resolve(process.cwd(), filePath); const data = await readFile(absPath); const fileName = path.basename(absPath); - const mimeType = - GEMINI_UPLOAD_MIME_TYPES[path.extname(absPath).toLowerCase()] ?? "application/octet-stream"; + const mimeType = resolveGeminiUploadMimeType(absPath); const form = new FormData(); form.append("file", new Blob([data], { type: mimeType }), fileName); diff --git a/tests/gemini-upload-mime.test.ts b/tests/gemini-upload-mime.test.ts new file mode 100644 index 000000000..69c1ff107 --- /dev/null +++ b/tests/gemini-upload-mime.test.ts @@ -0,0 +1,45 @@ +import { describe, it, expect } from "vitest"; +import { resolveGeminiUploadMimeType } from "../src/gemini-web/client.js"; + +describe("resolveGeminiUploadMimeType", () => { + it("types the video formats Gemini can actually read", () => { + // Untyped uploads are silently discarded: the run reports the file as attached + // while the model answers as though nothing was sent. + expect(resolveGeminiUploadMimeType("/tmp/clip.mp4")).toBe("video/mp4"); + expect(resolveGeminiUploadMimeType("/tmp/clip.mov")).toBe("video/quicktime"); + expect(resolveGeminiUploadMimeType("/tmp/clip.webm")).toBe("video/webm"); + }); + + it("keeps existing image and document types unchanged", () => { + expect(resolveGeminiUploadMimeType("/tmp/shot.png")).toBe("image/png"); + expect(resolveGeminiUploadMimeType("/tmp/shot.jpg")).toBe("image/jpeg"); + expect(resolveGeminiUploadMimeType("/tmp/shot.webp")).toBe("image/webp"); + expect(resolveGeminiUploadMimeType("/tmp/doc.pdf")).toBe("application/pdf"); + }); + + it("is case insensitive", () => { + expect(resolveGeminiUploadMimeType("/tmp/CLIP.MP4")).toBe("video/mp4"); + expect(resolveGeminiUploadMimeType("/tmp/SHOT.PNG")).toBe("image/png"); + }); + + it("leaves formats the endpoint rejects on the octet-stream fallback", () => { + // Audio and plain text are never delivered by this endpoint whatever type is + // declared, and .m4v/.mkv/.3gp are dropped or error out, so mapping them would + // only advertise support that does not exist. + for (const p of [ + "/tmp/a.wav", + "/tmp/a.mp3", + "/tmp/a.m4a", + "/tmp/n.txt", + "/tmp/c.m4v", + "/tmp/c.mkv", + ]) { + expect(resolveGeminiUploadMimeType(p)).toBe("application/octet-stream"); + } + }); + + it("falls back to octet-stream for unknown extensions", () => { + expect(resolveGeminiUploadMimeType("/tmp/archive.xyz")).toBe("application/octet-stream"); + expect(resolveGeminiUploadMimeType("/tmp/noext")).toBe("application/octet-stream"); + }); +}); From b7461a72b90dbcd1850c5fa73212f217c8c8e0b7 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 1 Aug 2026 22:24:49 -0700 Subject: [PATCH 2/2] docs: reconcile Gemini upload changelog --- CHANGELOG.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 066e3b927..658b619ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,11 @@ # Changelog -## 0.16.1 — Unreleased +## 0.16.2 — Unreleased ### Fixed - Browser: ignore transient `/c/WEB:` routes until ChatGPT exposes the durable conversation URL, preventing completed GPT-5.6 and Pro answers from hanging until timeout under a mismatched response scope. Fixes #333. Thanks @dbachko and @kesslerio! -- Gemini: type `.mp4`, `.mov` and `.webm` uploads as video instead of sending them as - `application/octet-stream`, which Gemini silently discards. Previously a `--file clip.mp4` - run reported the attachment as sent (`files=1`) while the model replied as though it had - received nothing, making video look unsupported. Verified against gemini-3.1-pro. +- Gemini: type `.mp4`, `.mov`, and `.webm` uploads as video so Gemini receives them instead of silently discarding generic binary uploads. Thanks @mkubenka! - Browser: recover completed answers after a recoverable DevTools disconnect by confirming target liveness and attempting bounded reattachment, while preserving fail-closed handling for unavailable targets. Fixes #326. Thanks @piyushbag! - CLI: avoid inheriting `browser.thinkingTime` from config when `--browser-model-strategy current` is explicit, while preserving an explicit `--browser-thinking-time` override. Thanks @jung0han! - Browser/Serve: keep the authenticated manual-login Chrome process alive while closing each successfully captured service-owned run tab, preventing renderer and memory accumulation across repeated remote consultations without changing explicit `--browser-keep-browser`, attached-tab, or incomplete-run recovery behavior. Thanks @rtl-ai!