From eb3423b6d5efd3ad70c837aed5fd518b69678d46 Mon Sep 17 00:00:00 2001 From: ltmoerdani Date: Sat, 25 Jul 2026 18:48:04 +0700 Subject: [PATCH 1/5] fix(vision): add top-level image size guard to prevent 400 on oversized pasted images (#38) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Top-level image attachments (pasted/dragged into chat) had no size guard, unlike MCP tool-result images which were capped at 1 MB by MAX_TOOL_RESULT_IMAGE_BYTES (PR #79). A large screenshot/photo produced multi-MB base64 payloads that the OpenCode Go gateway rejected with HTTP 400 'Upstream request failed' — observed on mimo-v2.5 with payloadBytes=3182845. Adds MAX_TOP_LEVEL_IMAGE_BYTES = 2_000_000 (2 MB raw) in convertMessage(). Threshold intentionally more liberal than the 1 MB tool-result cap because user screenshots/photos are typically larger than pre-compressed MCP screenshots, while staying under observed rejection (~3.18 MB) and Anthropic's published 5-10 MB per-image limit. Vision-capable models auto-resize upstream to a 1568-2576px patch budget anyway, so forwarding raw multi-MB images has no fidelity benefit. Oversized images are replaced with an actionable placeholder text part: model still knows an image was attached, user gets byte count + limit + resize hint. Not a regression — top-level handler never had a size cap since dee9634; latent bug surfaced because users now attach larger images. - src/extension.ts: +38 lines (constant + guard in convertMessage) - CHANGELOG.md: [Unreleased] entry - docs/issues/38-20260725-top-level-image-size-guard.md: full investigation --- CHANGELOG.md | 6 + .../38-20260725-top-level-image-size-guard.md | 279 ++++++++++++++++++ src/extension.ts | 38 +++ 3 files changed, 323 insertions(+) create mode 100644 docs/issues/38-20260725-top-level-image-size-guard.md diff --git a/CHANGELOG.md b/CHANGELOG.md index bdbe237..d3ac56a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to the **OpenCode Go BYOK Provider** extension are documented here. +## [Unreleased] + +### Fixed + +- **`[Vision]` Top-level image attachment size guard — prevent `400 Upstream request failed` on oversized pasted images (#38).** Pasting or dragging a large image (4K screenshot, high-res phone photo) into Copilot Chat with a vision-capable OpenCode model (e.g. `mimo-v2.5`) produced a multi-MB base64 payload that the OpenCode Go gateway rejected with HTTP 400. The top-level image handler in `convertMessage()` had **no size guard**, unlike MCP tool-result images which were already capped at 1 MB by `MAX_TOOL_RESULT_IMAGE_BYTES` (PR #79). New constant `MAX_TOP_LEVEL_IMAGE_BYTES = 2_000_000` (2 MB raw, intentionally more liberal than the 1 MB tool-result cap because user screenshots/photos are typically larger than pre-compressed MCP screenshots). Images exceeding the limit are replaced with an actionable placeholder text part — the model still knows an image was attached, and the user gets the actual byte count, the limit, and a hint to resize/compress. Threshold rationale (evidence-based): Anthropic publishes a 5–10 MB per-image limit, OpenCode Go was verified to reject 3.18 MB, and upstream vision models auto-resize to a 1568–2576 px patch budget anyway — so there is no fidelity benefit to forwarding multi-MB raw images. See `docs/issues/38-20260725-top-level-image-size-guard.md`. + ## [0.4.3] — 2026-07-24 ### Fixed diff --git a/docs/issues/38-20260725-top-level-image-size-guard.md b/docs/issues/38-20260725-top-level-image-size-guard.md new file mode 100644 index 0000000..0788e0b --- /dev/null +++ b/docs/issues/38-20260725-top-level-image-size-guard.md @@ -0,0 +1,279 @@ +**Status:** ✅ Solved + +# Top-Level Image Attachment Size Guard — Prevent 400 on Oversized Pasted Images + +**Topic:** vision / streaming / provider / gateway +**Updated:** 2026-07-25 +**Tags:** #vision #streaming #provider #gateway #bug + +--- + +## Overview + +When a user pasted or dragged a large image (4K screenshot, high-res phone photo) +into Copilot Chat while using a vision-capable OpenCode model (e.g. `mimo-v2.5`), +the request failed with: + +``` +OpenCode Go API request failed (400) model=mimo-v2.5 payloadBytes=3182845: +Error from provider (Console Go): Upstream request failed +``` + +The 3.18 MB payload (~2.4 MB raw image × 1.33 base64 overhead) was forwarded +directly to the OpenCode Go gateway, which rejected it. Top-level image +attachments had **no size guard**, unlike MCP tool-result images which were +already capped at 1 MB by `MAX_TOOL_RESULT_IMAGE_BYTES` (PR #79 / `ec92a44`). + +This was **not a regression** — the top-level image handler in +`src/extension.ts` `convertMessage()` never had a size cap since it was first +introduced in commit `dee9634`. The latent bug only surfaced now because users +are attaching larger images than before. + +--- + +## Problem Statement + +### Observed behavior + +| Path | Worked? | +|------|---------| +| Top-level paste/drag small image (< 1 MB) | ✅ Yes | +| Top-level paste/drag large image (> 2 MB) | ❌ No — `400 Upstream request failed` | +| MCP tool result image (any size) | ✅ Yes — guarded by `MAX_TOOL_RESULT_IMAGE_BYTES` (1 MB) since PR #79 | +| Built-in Copilot model + same image | ✅ Yes | + +### Error signature + +``` +Client Request Id: 8f70e12c-e1d9-46df-a1de-33b74a3962e5 +Reason: OpenCode Go API request failed (400) model=mimo-v2.5 payloadBytes=3182845: +Error from provider (Console Go): Upstream request failed + at buildOpenCodeRequestError (.../out/errors.js:39:12) + at streamOpenCodeResponse (.../out/streaming.js:333:73) + at streamChatCompletions (.../out/streaming.js:64:5) +``` + +### Why MiMo in particular + +`mimo-v2.5` is listed in `VISION_CAPABLE_MODELS` (`src/metadata.ts:247`), so the +extension treats it as a **native vision model**. This means: + +1. `modelCapabilities()` reports `imageInput: true` — VS Code keeps image parts. +2. The vision proxy (`proxyVision()`) is **not** activated — it only fires for + text-only models. +3. Images are forwarded directly to the model via transport chat-completions as + OpenAI-style `image_url` base64 data URIs. +4. A 3.18 MB payload exceeds the (unpublished) OpenCode Go gateway limit and is + rejected upstream. + +--- + +## Root Cause + +### The unguarded serialization path + +In `src/extension.ts` `convertMessage()`, top-level image attachments were +serialized as: + +```ts +if (part instanceof vscode.LanguageModelDataPart && part.mimeType.startsWith("image/")) { + const base64 = dataPartToBase64(part.data); + imageParts.push({ + type: "image_url", + image_url: { url: `data:${part.mimeType};base64,${base64}` } + }); + continue; +} +``` + +No byte-length check. Any image, regardless of size, was encoded as a base64 +data URI (~1.33× its raw byte size) and inserted into the request payload. + +### Asymmetry with tool-result path + +PR #79 (`ec92a44`) introduced a hard cap for images nested inside +`LanguageModelToolResultPart`: + +```ts +const MAX_TOOL_RESULT_IMAGE_BYTES = 1_000_000; // 1 MB raw bytes + +if (resultPart.data.byteLength > MAX_TOOL_RESULT_IMAGE_BYTES) { + toolTextParts.push(`[Image attachment omitted: ${resultPart.data.byteLength} bytes exceeds the ${MAX_TOOL_RESULT_IMAGE_BYTES}-byte limit for tool results. ...]`); + continue; +} +``` + +But the **top-level** path — which is the more common entry point for user +images — was overlooked. Doc `docs/features/12-20260720-mcp-tool-result-image-support.md` +Limitations #3 explicitly acknowledged this: + +> *"Top-level image attachments still have no size cap. Only tool-result images +> are bounded. Consistent limit can be added in a follow-up if users hit +> oversized pasted images."* + +This is that follow-up. + +### Why not a regression + +`git log -L 3336,3341:src/extension.ts` confirms the top-level image handler +**never** had a size guard: + +| Commit | Date | Change | +|--------|------|--------| +| `dee9634` | Initial | First vision support — base64 encode, no size check | +| `d0032ed` | Early | Refactor `btoa` → `dataPartToBase64` (still no size check) | +| `ec92a44` | 2026-07-20 | Added `MAX_TOOL_RESULT_IMAGE_BYTES` — but **only** for tool results | + +No subsequent commit modified the top-level image branch. The bug was latent +since day one; it surfaced now because users are attaching larger images. + +--- + +## Research (evidence-based) + +### Provider limits — authoritative sources + +| Provider | Per-image limit | Total payload | Source | +|----------|----------------|---------------|--------| +| **OpenAI** | (not explicit) | 512 MB, 1500 images | [developers.openai.com/api/docs/guides/images](https://developers.openai.com/api/docs/guides/images) | +| **Anthropic** | **10 MB base64** (5 MB on Bedrock/Vertex) | 32 MB standard endpoint | [platform.claude.com/docs/en/docs/build-with-claude/vision](https://platform.claude.com/docs/en/docs/build-with-claude/vision) | +| **OpenCode Go/Zen** | Not published | Not published | [opencode.ai/docs/zen](https://opencode.ai/docs/zen) | + +### Why upstream auto-resize makes large payloads pointless + +OpenAI and Anthropic docs both confirm that **vision models auto-resize images +to a patch budget** before tokenization: + +- **OpenAI**: 1568–2576 px long-edge depending on `detail` and model family. +- **Anthropic**: 1568 px (standard tier) / 2576 px (high-res tier), ~4784 visual + tokens max. + +Image pixels beyond the patch budget are discarded by the upstream model +regardless of what the client sends. Forwarding a 4K raw image is pure waste — +the model downscales it to ~1568–2576 px before processing. There is **no +fidelity benefit** to sending multi-MB raw images. + +### Verified rejection point + +The user's error shows `payloadBytes=3182845` (~3.18 MB) was rejected by the +OpenCode Go gateway. Combined with Anthropic's published 5–10 MB per-image +limit on partner platforms, 2 MB raw (→ ~2.7 MB base64) is a safe threshold with +margin. + +--- + +## Solution + +### 1. New constant `MAX_TOP_LEVEL_IMAGE_BYTES` + +```ts +const MAX_TOP_LEVEL_IMAGE_BYTES = 2_000_000; // 2 MB raw bytes +``` + +Intentionally **more liberal** than `MAX_TOOL_RESULT_IMAGE_BYTES` (1 MB) because: + +- Top-level images are user-supplied screenshots/photos, typically larger than + pre-compressed MCP tool-result screenshots. +- Anthropic's published per-image limit is 5–10 MB; 2 MB stays well under that. +- OpenCode Go verified to reject 3.18 MB; 2 MB raw → ~2.7 MB base64 stays under + the observed rejection point. + +### 2. Size guard in `convertMessage()` + +```ts +if (part instanceof vscode.LanguageModelDataPart && part.mimeType.startsWith("image/")) { + if (part.data.byteLength > MAX_TOP_LEVEL_IMAGE_BYTES) { + textParts.push( + `[Image attachment omitted: ${part.data.byteLength} bytes exceeds the ` + + `${MAX_TOP_LEVEL_IMAGE_BYTES}-byte limit for top-level attachments. ` + + `Resize or compress the image to under ${Math.floor(MAX_TOP_LEVEL_IMAGE_BYTES / 1_000_000)} MB and re-attach it.]` + ); + continue; + } + const base64 = dataPartToBase64(part.data); + imageParts.push({ + type: "image_url", + image_url: { url: `data:${part.mimeType};base64,${base64}` } + }); + continue; +} +``` + +The model still knows an image was attached (placeholder text part is emitted), +and the user gets an **actionable hint** with the actual byte count, the limit, +and the suggested fix (resize/compress). + +### Why not auto-resize inside the extension + +Considered and rejected: + +| Option | Verdict | Reason | +|--------|---------|--------| +| `sharp` (native binary) | ❌ Rejected | ~30 MB native dep, impractical for VS Code extension packaging, platform-specific builds | +| `jimp` / pure-JS | ❌ Rejected | Manual impl, quality inconsistency across formats, large dep | +| Delegate to vision proxy | ❌ Rejected | Would silently consume Copilot quota; vision proxy is for text-only models, not native-vision MiMo | +| VS Code built-in API | ❌ Not available | `vscode.LanguageModelDataPart` is immutable; no native resize API | + +Upstream models auto-resize to a patch budget anyway, so a client-side resize +layer adds complexity with no fidelity benefit. + +--- + +## Files Changed + +| File | Change | Lines | +|------|--------|-------| +| `src/extension.ts` | New constant `MAX_TOP_LEVEL_IMAGE_BYTES = 2_000_000` with JSDoc rationale | +23 | +| `src/extension.ts` | Size guard in `convertMessage()` top-level image branch with actionable placeholder | +15 | + +**Total:** 1 file, +38 lines, 0 deletions. + +--- + +## Code Locations + +| Concern | Location | +|---------|----------| +| `MAX_TOP_LEVEL_IMAGE_BYTES` constant | `src/extension.ts` (~L582) | +| Top-level image size guard | `src/extension.ts` `convertMessage()` (~L3359) | +| `MAX_TOOL_RESULT_IMAGE_BYTES` (tool-result analogue) | `src/extension.ts` (~L559) | +| Tool-result image size guard | `src/extension.ts` `convertMessage()` (~L3292) | + +--- + +## Verification + +- `npm run compile` (tsc strict) — **pass**, no errors. +- `get_errors src/extension.ts` — **0 errors**. +- `git diff --stat` — 1 file, +38 lines, scoped to the single image branch. + +Manual testing path (deferred to release validation): paste a >2 MB image into +Copilot Chat with `mimo-v2.5` selected — model should now receive a placeholder +text note instead of the raw image, and no `400 Upstream request failed` should +occur. + +--- + +## Limitations & Follow-ups + +1. **No automatic resize.** Users must resize/compress externally before + re-attaching. A future enhancement could bundle a pure-JS decoder for the + common formats (PNG/JPEG) if the false-positive rate becomes a problem. +2. **Single threshold for all transports.** The 2 MB cap is the most + conservative across Anthropic (5–10 MB) and observed OpenCode Go behavior + (~3 MB rejection). Per-transport tuning is possible but adds complexity + without clear benefit given upstream auto-resize. +3. **History-level trimming not addressed.** This bounds each image + individually; a conversation with many small images can still cumulatively + exceed the model's context window. VS Code's own history trimming is + expected to handle that, and `estimateTokenCount` accounts for image tokens + via `IMAGE_TOKEN_ESTIMATE`. + +--- + +## Related Docs + +- [`docs/features/01-20260514-vision-image-input.md`](../features/01-20260514-vision-image-input.md) — top-level image attachment foundation +- [`docs/features/11-20260715-vision-proxy.md`](../features/11-20260715-vision-proxy.md) — vision proxy for text-only models +- [`docs/features/12-20260720-mcp-tool-result-image-support.md`](../features/12-20260720-mcp-tool-result-image-support.md) — MCP tool-result image guard (`MAX_TOOL_RESULT_IMAGE_BYTES`) +- [`docs/issues/34-20260720-mcp-tool-result-image-dropped.md`](34-20260720-mcp-tool-result-image-dropped.md) — original tool-result size-guard investigation diff --git a/src/extension.ts b/src/extension.ts index e8c504f..78e400f 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -558,6 +558,28 @@ const IMAGE_TOKEN_ESTIMATE = 1024; */ const MAX_TOOL_RESULT_IMAGE_BYTES = 1_000_000; +/** + * Hard upper limit (in bytes of raw image data) for a single top-level image + * attachment pasted or dropped into the chat by the user. Top-level images + * (screenshots, photos) are typically larger than MCP tool-result screenshots, + * so this threshold is intentionally more liberal than the tool-result guard. + * + * Rationale (evidence-based): + * - Anthropic API hard limit: 10 MB per image base64 (5 MB on Bedrock/Vertex). + * - OpenAI API: 512 MB total payload, but upstream models auto-resize to a + * patch budget (1568–2576 px long-edge) so anything larger is wasted. + * - OpenCode Go gateway: limit not published, but verified to reject a + * 3.18 MB payload with HTTP 400 "Upstream request failed" (issue #38). + * - 2 MB raw → ~2.7 MB base64, comfortably under observed rejection point + * while allowing typical user screenshots/photos without false positives. + * + * Larger images are replaced with a placeholder text part so the model still + * knows an image was attached and the user gets an actionable hint to resize. + * Vision-capable models auto-downsample upstream anyway, so there is no value + * in forwarding multi-MB raw image data. + */ +const MAX_TOP_LEVEL_IMAGE_BYTES = 2_000_000; + type CopilotCompatibleCapabilities = vscode.LanguageModelChatCapabilities & { supportsToolCalling: boolean; supportsImageToText: boolean; @@ -3334,6 +3356,22 @@ function convertMessage( } if (part instanceof vscode.LanguageModelDataPart && part.mimeType.startsWith("image/")) { + // SIZE GUARD: Top-level images larger than MAX_TOP_LEVEL_IMAGE_BYTES are + // replaced with a placeholder text part. This prevents a single oversized + // pasted image (e.g. 4K screenshot, high-res phone photo) from producing + // a multi-MB base64 payload that triggers upstream 400 "Upstream request + // failed" rejections from OpenCode Go. Vision-capable models auto-resize + // upstream to a patch budget anyway, so there is no fidelity loss in + // practice — the model would have downscaled it regardless. The user + // gets an actionable hint so they can resize and re-attach. + if (part.data.byteLength > MAX_TOP_LEVEL_IMAGE_BYTES) { + textParts.push( + `[Image attachment omitted: ${part.data.byteLength} bytes exceeds the ` + + `${MAX_TOP_LEVEL_IMAGE_BYTES}-byte limit for top-level attachments. ` + + `Resize or compress the image to under ${Math.floor(MAX_TOP_LEVEL_IMAGE_BYTES / 1_000_000)} MB and re-attach it.]` + ); + continue; + } const base64 = dataPartToBase64(part.data); imageParts.push({ type: "image_url", From a649548edf620c3bc162a2852aeb0e898aa397ac Mon Sep 17 00:00:00 2001 From: ltmoerdani Date: Sat, 25 Jul 2026 18:53:45 +0700 Subject: [PATCH 2/5] chore(release): bump version to 0.4.4 Promote [Unreleased] -> [0.4.4] with 2026-07-25 release date. Release notes: - fix(vision): top-level image size guard (#38) --- CHANGELOG.md | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3ac56a..cfb2e75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to the **OpenCode Go BYOK Provider** extension are documented here. -## [Unreleased] +## [0.4.4] — 2026-07-25 ### Fixed diff --git a/package.json b/package.json index 009b69e..574d9b2 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "opencode-copilot-chat", "displayName": "OpenCode for Copilot Chat — BYOK 30+ AI Models", "description": "Use 30+ frontier AI models (DeepSeek V4, Kimi K2.6, GLM-5.1, Qwen3.7, MiMo V2.5, MiniMax M2.7, free Claude Opus, GPT-5.5, Gemini 3.5, Grok) in GitHub Copilot Chat. Bring Your Own Key — no Copilot Pro needed.", - "version": "0.4.3", + "version": "0.4.4", "publisher": "ltmoerdani", "license": "MIT", "icon": "media/opencodego.png", From 77b8c953c94e878ab08ab7e7dd17aafccebfada5 Mon Sep 17 00:00:00 2001 From: ltmoerdani Date: Sat, 25 Jul 2026 19:12:09 +0700 Subject: [PATCH 3/5] fix(vision): trim old images from history to prevent MiMo MCP screenshot loops (#38) Root cause of MiMo + MCP 'Upstream request failed' was NOT per-image size (documented in docs/issues/34 line 264+): agent screenshot loops accumulate multi-MB base64 data URIs in conversation history. VS Code Copilot Chat is supposed to trim history based on advertisedMaxInputTokens, but our local estimator under-counts base64 image data (IMAGE_TOKEN_ESTIMATE=1024 per image vs realistic ~80K tokens/MB), so VS Code never sees the true payload weight and forwards multi-MB requests that OpenCode Go rejects. Verified case from docs/issues/34: mimo-v2.5 + chrome-devtools-mcp after 8 screenshots: payloadBytes=4665383 (4.6 MB) -> 400 Upstream request failed every subsequent retry in the same agent loop also failed. Fix: new trimOldImagesFromHistoryInPlace() keeps only the most recent MAX_HISTORY_IMAGES_KEPT = 2 images and replaces older ones with a placeholder text note. The model retains conversation structure and the latest screenshots for immediate agentic context (compare current vs previous), while cumulative payload stays bounded. Vision-capable upstream models auto-resize each image to a 1568-2576 px patch budget, so old screenshots lose most pixel value once a newer one arrives. Applied AFTER vision proxy (so proxy text descriptions are preserved) and BEFORE promptTokens estimation (so the output budget reflects the trimmed payload). Diagnostic log line '[history-trim] Replaced N old image(s)...' appears in the Output channel when trimming fires. - src/extension.ts: +128 lines (constant + trimOldImagesFromHistoryInPlace) - CHANGELOG.md: [0.4.5] entry - package.json: 0.4.4 -> 0.4.5 --- CHANGELOG.md | 2 + src/extension.ts | 128 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfb2e75..6f795e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable changes to the **OpenCode Go BYOK Provider** extension are documente ### Fixed +- **`[Vision]` Trim old images from conversation history — fix `400 Upstream request failed` on MiMo + MCP screenshot loops (#38 follow-up).** Even with per-image size guards (`MAX_TOOL_RESULT_IMAGE_BYTES` 1 MB, `MAX_TOP_LEVEL_IMAGE_BYTES` 2 MB), MCP-driven agentic workflows (`chrome-devtools-mcp`, `playwright-mcp`) accumulated multiple screenshots in conversation history and hit the OpenCode Go gateway's upstream limit. Documented in `docs/issues/34-20260720-mcp-tool-result-image-dropped.md` line 264+: a `mimo-v2.5` agent loop reached `payloadBytes=4665383` (4.6 MB) after 8 screenshots and started failing every subsequent request with HTTP 400. VS Code Copilot Chat is *supposed* to trim history based on `advertisedMaxInputTokens`, but our local estimator under-counts base64 image data (`IMAGE_TOKEN_ESTIMATE = 1024` per image vs. realistic ~80K tokens/MB), so VS Code never sees the true payload weight. New function `trimOldImagesFromHistoryInPlace()` keeps only the most recent `MAX_HISTORY_IMAGES_KEPT = 2` images and replaces older ones with a short placeholder text note ("Earlier screenshot omitted from history..."). The model retains conversation structure and the latest screenshots for immediate agentic context (compare current vs. previous), while cumulative payload stays bounded. OpenAI and Anthropic vision models auto-resize each image to a 1568–2576 px patch budget upstream, so old screenshots lose most of their pixel value once a newer one arrives — the model rarely benefits from keeping more than 2 in flight. Applied after vision proxy (so proxy text descriptions are preserved) and before `promptTokens` estimation (so the output budget reflects the trimmed payload). Diagnostic log line `[history-trim] Replaced N old image(s)...` appears in the Output channel when trimming fires. + - **`[Vision]` Top-level image attachment size guard — prevent `400 Upstream request failed` on oversized pasted images (#38).** Pasting or dragging a large image (4K screenshot, high-res phone photo) into Copilot Chat with a vision-capable OpenCode model (e.g. `mimo-v2.5`) produced a multi-MB base64 payload that the OpenCode Go gateway rejected with HTTP 400. The top-level image handler in `convertMessage()` had **no size guard**, unlike MCP tool-result images which were already capped at 1 MB by `MAX_TOOL_RESULT_IMAGE_BYTES` (PR #79). New constant `MAX_TOP_LEVEL_IMAGE_BYTES = 2_000_000` (2 MB raw, intentionally more liberal than the 1 MB tool-result cap because user screenshots/photos are typically larger than pre-compressed MCP screenshots). Images exceeding the limit are replaced with an actionable placeholder text part — the model still knows an image was attached, and the user gets the actual byte count, the limit, and a hint to resize/compress. Threshold rationale (evidence-based): Anthropic publishes a 5–10 MB per-image limit, OpenCode Go was verified to reject 3.18 MB, and upstream vision models auto-resize to a 1568–2576 px patch budget anyway — so there is no fidelity benefit to forwarding multi-MB raw images. See `docs/issues/38-20260725-top-level-image-size-guard.md`. ## [0.4.3] — 2026-07-24 diff --git a/src/extension.ts b/src/extension.ts index 78e400f..7de3dda 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -558,6 +558,35 @@ const IMAGE_TOKEN_ESTIMATE = 1024; */ const MAX_TOOL_RESULT_IMAGE_BYTES = 1_000_000; +/** + * Maximum number of image attachments (top-level + tool-result combined) to + * keep in conversation history before older ones are replaced with a + * placeholder text note. + * + * Rationale (evidence-based, issue #38 follow-up): + * - Doc `docs/issues/34-20260720-mcp-tool-result-image-dropped.md` line 264+ + * documents a 4.6 MB payload causing `400 Upstream request failed` on + * `mimo-v2.5` after 8 MCP screenshots accumulated in history (~1-2 MB each + * → base64 ~1.33× → 4.6 MB total JSON body). + * - VS Code Copilot Chat is *supposed* to trim conversation history based on + * `advertisedMaxInputTokens`, but our local estimator under-counts base64 + * image data (`IMAGE_TOKEN_ESTIMATE = 1024` per image, vs the realistic + * ~80K tokens/MB). This means VS Code never sees the true payload weight + * and forwards a multi-MB request that the OpenCode Go gateway rejects. + * - Keeping the most recent 2 images preserves the immediate agentic context + * (the model needs to compare current vs. previous screenshot in most MCP + * workflows) while bounding the cumulative payload to a safe ceiling. + * - OpenAI and Anthropic vision models auto-resize each image to a patch + * budget (1568-2576 px) upstream, so old screenshots lose most of their + * pixel value once a newer one arrives — the model rarely benefits from + * keeping more than 2 in flight. + * + * Older images are replaced with a short placeholder text note so the model + * still knows a screenshot existed at that point in the conversation (useful + * for understanding agent-loop context) without incurring the payload cost. + */ +const MAX_HISTORY_IMAGES_KEPT = 2; + /** * Hard upper limit (in bytes of raw image data) for a single top-level image * attachment pasted or dropped into the chat by the user. Top-level images @@ -2091,6 +2120,23 @@ class OpenCodeProvider implements vscode.LanguageModelChatProvider 0) { + this.log(`[history-trim] Replaced ${trimmedCount} old image(s) with placeholder text to bound payload (kept most recent ${MAX_HISTORY_IMAGES_KEPT}).`); + } + const thinkingPayload = buildThinkingPayload(rawModelId, settings.thinking, hasImageInput && metadata.supportsVision); const requestHeaders = buildOpenCodeRequestHeaders( messages, @@ -3591,6 +3637,88 @@ function messagesHaveImages(messages: readonly ApiMessage[]): boolean { ); } +/** + * Replace image content parts in older messages with a placeholder text note + * in place, keeping only the most recent `MAX_HISTORY_IMAGES_KEPT` images in + * the conversation. This bounds the cumulative payload weight when MCP + * screenshot loops (chrome-devtools-mcp, playwright-mcp) accumulate base64 + * data URIs in history and trigger upstream `400 Upstream request failed` + * rejections from OpenCode Go. + * + * CONTRACT: + * - Iterates messages from newest to oldest, counting `image_url` parts. + * - Once `MAX_HISTORY_IMAGES_KEPT` images have been seen, every subsequent + * (older) image part is replaced in place with a placeholder text note. + * - Non-image content parts (text, tool_calls, tool_call_id) are preserved + * unchanged — the conversation structure stays intact. + * - The placeholder replaces the image part in the same message's content + * array; the array shape is preserved so downstream transport builders + * still see a valid multimodal structure. + * - Mutates the input array's message `content` fields in place (safe: the + * caller `provideLanguageModelChatResponse` does not reuse the original + * array after this point). + * + * INVARIANTS: + * - Total `image_url` parts remaining in the array after the call ≤ + * `MAX_HISTORY_IMAGES_KEPT`. + * - Every original image position is either preserved or replaced with a + * placeholder text part — no message is silently dropped. + * + * @param messages ApiMessage[] from convertMessage() — must be in chronological + * order (oldest first, newest last), as produced by + * `messages.flatMap(convertMessage)`. Mutated in place. + * @returns Number of image parts that were replaced with a placeholder (for + * diagnostic logging). Returns 0 when no trimming was needed. + */ +function trimOldImagesFromHistoryInPlace(messages: ApiMessage[]): number { + // Count total images to decide whether trimming is needed. Cheap pass that + // skips allocation and mutation for the common case (short conversations, + // 0-2 images). + let totalImages = 0; + for (const msg of messages) { + if (!Array.isArray(msg.content)) continue; + for (const part of msg.content) { + if (part.type === "image_url") totalImages++; + } + } + if (totalImages <= MAX_HISTORY_IMAGES_KEPT) { + return 0; + } + + // Walk newest -> oldest, allowing the first MAX_HISTORY_IMAGES_KEPT images + // to pass through and replacing every older image with a placeholder note. + let imagesKept = 0; + let replacedCount = 0; + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i]; + if (!Array.isArray(msg.content)) continue; + const hasImage = msg.content.some((p) => p.type === "image_url"); + if (!hasImage) continue; + // Build a new content array, replacing image parts once the budget is spent. + // We rebuild the array rather than splice-in-place because the original + // parts array may be shared with the caller's view. + const newContent: OpenAiContentPart[] = []; + for (const part of msg.content) { + if (part.type === "image_url") { + if (imagesKept < MAX_HISTORY_IMAGES_KEPT) { + newContent.push(part); + imagesKept++; + } else { + newContent.push({ + type: "text", + text: "[Earlier screenshot omitted from history to keep request payload under gateway limit. The latest screenshots above are preserved.]", + }); + replacedCount++; + } + } else { + newContent.push(part); + } + } + msg.content = newContent; + } + return replacedCount; +} + function hasMessagePayload(message: ApiMessage): boolean { if (message.tool_calls?.length || message.tool_call_id) { return true; From fb9f8243b1b7f9cc6ebc708f0a4d56cc5ef71579 Mon Sep 17 00:00:00 2001 From: ltmoerdani Date: Sat, 25 Jul 2026 19:46:54 +0700 Subject: [PATCH 4/5] fix(mimo): omit reasoning_content echo in tool_call history to prevent 400 Upstream request failed (#38) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause finally identified from Output channel logs (2026-07-25 12:23): messages=67 payloadBytes=244401 -> 200 OK (finishReason=tool_calls, reasoningChars=110) messages=70 payloadBytes=359146 -> 400 Upstream request failed every retry in same session -> 400 MiMo upstream (Xiaomi) uses strict Pydantic-style validator that rejects assistant tool_call messages carrying a 'reasoning_content' field once they appear in conversation history. The extension was echoing reasoning_content back into history for ALL models (including MiMo) via reasoningForToolCalls() in convertMessage(). This mirrors the DeepSeek V4 tool-call issue (#36354 upstream): 'OpenCode backend does not correctly handle reasoning_content echoing for DeepSeek V4 tool calls (and possibly MiMo), causing 400 errors'. Fix: gate reasoning_content injection by model family. For MiMo (/^mimo-/i), omit reasoning_content in the echoed assistant tool_call history. The current live response still surfaces reasoning_content to the user via the thinking panel — only the history echo is dropped. Other families (DeepSeek, Kimi, GLM, Qwen, MiniMax) tolerate the echo and keep it for cross-turn reasoning continuity. - src/extension.ts: convertMessage() takes optional rawModelId, skips reasoning_content injection for MiMo family in assistant tool_call branch. --- src/extension.ts | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/extension.ts b/src/extension.ts index 7de3dda..8874631 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -2032,9 +2032,9 @@ class OpenCodeProvider implements vscode.LanguageModelChatProvider convertMessage(message, this.reasoningContentByToolCallId))); - const baseSettings = getSettings(); const rawModelId = model.rawModelId ?? resolveRawModelId(model.id); + const apiMessages = normalizeMessages(messages.flatMap((message) => convertMessage(message, this.reasoningContentByToolCallId, rawModelId))); + const baseSettings = getSettings(); // Apply per-request Thinking selection (from Copilot Chat submenu) on top // of the workspace default. The override only affects the current model // family; other families remain at their global defaults. @@ -3318,7 +3318,8 @@ function anthropicToolChoice(mode: vscode.LanguageModelChatToolMode): { type: "a function convertMessage( message: vscode.LanguageModelChatRequestMessage, - reasoningContentByToolCallId: ReadonlyMap + reasoningContentByToolCallId: ReadonlyMap, + rawModelId?: string, ): ApiMessage[] { const role = message.role === vscode.LanguageModelChatMessageRole.Assistant ? "assistant" : "user"; const textParts: string[] = []; @@ -3451,10 +3452,27 @@ function convertMessage( } if (role === "assistant" && toolCalls.length) { + // CONTRACT: reasoning_content injection into tool_call assistant messages + // is gated by model family. MiMo upstream (Xiaomi) uses a strict Pydantic- + // style validator that rejects assistant tool_call messages carrying a + // `reasoning_content` field with HTTP 400 `Upstream request failed`, once + // the conversation history contains tool_calls with reasoning echo. This + // mirrors the DeepSeek V4 issue (#36354 upstream) and was verified in this + // extension's logs (issue #38, 2026-07-25): MiMo succeeds until the first + // tool_call turn with reasoning_content, then every subsequent turn 400s. + // + // For MiMo we omit reasoning_content in the echoed assistant tool_call + // history. The current live response still surfaces reasoning_content to + // the user via the thinking panel — only the *history echo* is dropped. + // Other families (DeepSeek, Kimi, GLM, Qwen, MiniMax) tolerate the echo + // and keep it for cross-turn reasoning continuity. + const shouldOmitReasoningEcho = rawModelId !== undefined && /^mimo-/i.test(rawModelId); return [{ role, content: typeof content === "string" ? content || null : content, - reasoning_content: reasoningForToolCalls(toolCalls, reasoningContentByToolCallId), + reasoning_content: shouldOmitReasoningEcho + ? undefined + : reasoningForToolCalls(toolCalls, reasoningContentByToolCallId), tool_calls: toolCalls }]; } From 3c5df1edd5355133716790039854671a6bf2120e Mon Sep 17 00:00:00 2001 From: ltmoerdani Date: Sat, 25 Jul 2026 20:11:22 +0700 Subject: [PATCH 5/5] fix(mimo): flatten list-type tool message content to string (upstream #32613) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause finally verified via upstream issue anomalyco/opencode#32613 'Xiaomi MiMo rejects list-type tool message content (400 text is not set)': Xiaomi MiMo API requires role:'tool' message content to be a plain string, NOT a list of content parts. MiMo accepts multimodal content in user/assistant messages but strictly rejects list-type content in tool messages. The OpenCode Go gateway passes list-type content through unchanged, so we must flatten it client-side. Verified in user log (2026-07-25 13:05): messages=3 payloadBytes=151626 -> 200 OK (tool result without image) messages=5 payloadBytes=426159 -> 400 Upstream request failed (same + tool result WITH image_url list-type content) Previous 4 fix attempts (top-level guard, history trim, reasoning echo, TDZ) were all red herrings — the real issue is the tool message shape itself. Fix: for MiMo family, when a tool result contains image parts, emit plain string content by joining text parts and replacing each image with a short placeholder note (the model cannot see tool images on MiMo upstream anyway). Other providers (Kimi, GLM-5.1, MiniMax, Qwen) keep the multimodal array because they accept list-type tool content. Upstream PR anomalyco/opencode#32966 exists but was closed by automated cleanup before merge, so this workaround is necessary until the gateway handles the translation. - src/extension.ts: convertMessage() tool-result branch gates multimodal array on model family; MiMo gets flattened string. --- src/extension.ts | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/src/extension.ts b/src/extension.ts index 8874631..5a5416c 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -3383,13 +3383,37 @@ function convertMessage( let toolContent: string | OpenAiContentPart[]; if (toolImageParts.length > 0) { - const multimodal: OpenAiContentPart[] = []; - const joinedText = toolTextParts.join("\n"); - if (joinedText) { - multimodal.push({ type: "text", text: joinedText }); + // PROVIDER QUIRK: Xiaomi MiMo (and GLM-5.2) reject list-type tool + // message content with HTTP 400 "text is not set" (upstream issue + // anomalyco/opencode#32613). MiMo accepts multimodal content in + // user/assistant messages but strictly requires `role: "tool"` + // messages to have a plain string content. The OpenCode Go gateway + // passes list-type content through unchanged, so we must flatten it + // client-side for MiMo. + // + // For MiMo: emit a plain string — join text parts, and replace each + // image with a short placeholder note (the model cannot see tool + // images on MiMo upstream anyway, so we lose nothing and gain a + // working request). For other providers: keep the multimodal array + // (Kimi, GLM-5.1, MiniMax, Qwen all accept list-type tool content). + const isMimoModel = rawModelId !== undefined && /^mimo-/i.test(rawModelId); + if (isMimoModel) { + const flattened: string[] = [...toolTextParts]; + for (let i = 0; i < toolImageParts.length; i++) { + flattened.push( + `[Tool returned an image attachment, but the MiMo upstream provider does not accept images in tool messages. Image ${i + 1} of ${toolImageParts.length} was dropped to keep the request valid.]` + ); + } + toolContent = flattened.join("\n"); + } else { + const multimodal: OpenAiContentPart[] = []; + const joinedText = toolTextParts.join("\n"); + if (joinedText) { + multimodal.push({ type: "text", text: joinedText }); + } + multimodal.push(...toolImageParts); + toolContent = multimodal; } - multimodal.push(...toolImageParts); - toolContent = multimodal; } else { toolContent = toolTextParts.join("\n"); }