From 956a9cc6544e94e074fc656d28ee7d0fade76f00 Mon Sep 17 00:00:00 2001 From: badcuban <108198679+badcuban@users.noreply.github.com> Date: Sun, 6 Sep 2026 04:40:34 -0400 Subject: [PATCH] fix(chat): images an agent reads no longer show as broken The server capped every string in a tool row at 4,000 characters before saving it, keeping only the tail behind a "..." prefix. A Claude Read of an image carries the bytes as one base64 string in the tool result, so nearly every image was cut, and the chat built a data URL from the cut string that the browser could not draw. The activity trimmer now leaves a base64 image block's data whole, the same way it already preserved Codex image results. On the client, an image block whose data is not valid base64 (the trimmed rows already saved) is ignored so the row falls back to loading the named path over the workspace RPC. --- .../Layers/ProviderActivityProjection.test.ts | 36 ++++++++++++++++ .../Layers/ProviderActivityProjection.ts | 27 +++++++++--- apps/web/src/session-logic.test.ts | 42 +++++++++++++++++++ apps/web/src/session-logic.ts | 9 +++- 4 files changed, 108 insertions(+), 6 deletions(-) diff --git a/apps/server/src/orchestration/Layers/ProviderActivityProjection.test.ts b/apps/server/src/orchestration/Layers/ProviderActivityProjection.test.ts index 0711ca2d..7e093604 100644 --- a/apps/server/src/orchestration/Layers/ProviderActivityProjection.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderActivityProjection.test.ts @@ -433,6 +433,42 @@ describe("ProviderActivityProjection", () => { expect(payload?.data?.item?.result?.startsWith("...")).toBe(false); }); + it("preserves base64 image blocks on Claude tool results", () => { + const imageData = `iVBORw0KGgo${"A".repeat(MAX_THREAD_ACTIVITY_PAYLOAD_TEXT_LENGTH + 128)}`; + const activities = projectRuntimeEventToActivities({ + type: "item.completed", + eventId: EventId.make("evt-read-image-completed"), + provider: ProviderDriverKind.make("claude"), + threadId: ThreadId.make("thread-1"), + turnId: TurnId.make("turn-1"), + createdAt: "2026-06-01T12:00:00.000Z", + payload: { + itemType: "image_view", + title: "Read screenshot.png", + data: { + toolName: "Read", + input: { file_path: "C:\\shots\\screenshot.png" }, + result: { + type: "tool_result", + tool_use_id: "tool-read-1", + content: [ + { + type: "image", + source: { type: "base64", media_type: "image/png", data: imageData }, + }, + ], + }, + }, + }, + } satisfies ProviderRuntimeEvent); + + const payload = activities[0]?.payload as + | { data?: { result?: { content?: Array<{ source?: { data?: string } }> } } } + | undefined; + + expect(payload?.data?.result?.content?.[0]?.source?.data).toBe(imageData); + }); + it("still compacts non-image result payload strings", () => { const largeResult = Array.from({ length: 1_000 }, (_, index) => `line ${index}`).join("\n"); const activities = projectRuntimeEventToActivities({ diff --git a/apps/server/src/orchestration/Layers/ProviderActivityProjection.ts b/apps/server/src/orchestration/Layers/ProviderActivityProjection.ts index 5f73868f..9efbbb88 100644 --- a/apps/server/src/orchestration/Layers/ProviderActivityProjection.ts +++ b/apps/server/src/orchestration/Layers/ProviderActivityProjection.ts @@ -112,6 +112,18 @@ function isImageGenerationPayloadRecord(value: Record): boolean ); } +/** A Claude `tool_result` image block's `source`: `{type: "base64", media_type: + * "image/...", data}`. The data only works whole -- a trimmed copy draws as a + * broken image -- so it keeps its full length. */ +function isBase64ImageSourceRecord(value: Record): boolean { + return ( + value.type === "base64" && + typeof value.media_type === "string" && + value.media_type.toLowerCase().startsWith("image/") && + typeof value.data === "string" + ); +} + function compactActivityPayloadData( value: unknown, key?: string | undefined, @@ -147,12 +159,17 @@ function compactActivityPayloadData( const entries = Object.entries(value as Record); const compacted: Record = {}; - const preserveImageResult = isImageGenerationPayloadRecord(value as Record); + const record = value as Record; + const preserveImageResult = isImageGenerationPayloadRecord(record); + const preserveImageData = isBase64ImageSourceRecord(record); for (const [entryKey, entryValue] of entries.slice(0, MAX_THREAD_ACTIVITY_PAYLOAD_OBJECT_KEYS)) { - compacted[entryKey] = - preserveImageResult && entryKey === "result" && typeof entryValue === "string" - ? entryValue - : compactActivityPayloadData(entryValue, entryKey, depth + 1, limits); + const preserved = + typeof entryValue === "string" && + ((preserveImageResult && entryKey === "result") || + (preserveImageData && entryKey === "data")); + compacted[entryKey] = preserved + ? entryValue + : compactActivityPayloadData(entryValue, entryKey, depth + 1, limits); } if (entries.length > MAX_THREAD_ACTIVITY_PAYLOAD_OBJECT_KEYS) { compacted.__truncatedKeys = entries.length - MAX_THREAD_ACTIVITY_PAYLOAD_OBJECT_KEYS; diff --git a/apps/web/src/session-logic.test.ts b/apps/web/src/session-logic.test.ts index 1079af1b..a88ee137 100644 --- a/apps/web/src/session-logic.test.ts +++ b/apps/web/src/session-logic.test.ts @@ -2843,6 +2843,48 @@ describe("deriveWorkLogEntries", () => { expect(entry?.images?.[0]?.previewUrl).toBeUndefined(); }); + it("falls back to the file path when a stored image block was trimmed", () => { + const activities: OrchestrationThreadActivity[] = [ + makeActivity({ + id: "claude-read-trimmed", + kind: "tool.completed", + summary: "Read icon.png", + payload: { + itemType: "image_view", + title: "Read icon.png", + data: { + toolName: "Read", + input: { file_path: "C:\\Users\\wilfr\\AppData\\Local\\Temp\\icon.png" }, + result: { + type: "tool_result", + tool_use_id: "toolu_read", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "...uLW518pMPMrvYRRZoMTJG", + }, + }, + ], + }, + item: { id: "toolu_read" }, + }, + }, + }), + ]; + + const [entry] = deriveWorkLogEntries(activities); + expect(entry?.images).toEqual([ + { + id: "toolu_read", + name: "icon.png", + path: "C:\\Users\\wilfr\\AppData\\Local\\Temp\\icon.png", + }, + ]); + }); + it("previews the image blocks a screenshot tool returned inline", () => { const screenshotBase64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII="; diff --git a/apps/web/src/session-logic.ts b/apps/web/src/session-logic.ts index d9b9b2bb..cb23918f 100644 --- a/apps/web/src/session-logic.ts +++ b/apps/web/src/session-logic.ts @@ -3904,7 +3904,14 @@ function imagePathFromPayload(payload: Record | null): unknown * screenshot tool's `{type: "image", source: {type: "base64", ...}}` content * blocks are already here; reading them is what makes an MCP screenshot row a * picture without the event log holding a second copy. + * + * Rows saved before the server learned to keep image bytes whole hold a + * trimmed copy (`...` plus the tail). That is not base64, and a data URL built + * from it draws as a broken image, so such a block is skipped and the row falls + * back to loading the named path. */ +const BASE64_PATTERN = /^[A-Za-z0-9+/]+={0,2}$/u; + function imageBlocksFromPayload( payload: Record | null, ): Array<{ mimeType: string; base64: string }> { @@ -3922,7 +3929,7 @@ function imageBlocksFromPayload( } const mimeType = asTrimmedString(source.media_type); const base64 = asTrimmedString(source.data); - return mimeType && base64 ? [{ mimeType, base64 }] : []; + return mimeType && base64 && BASE64_PATTERN.test(base64) ? [{ mimeType, base64 }] : []; }); }