From e069e0c5df553e5f9c35637b0bacf30ab4c627d2 Mon Sep 17 00:00:00 2001 From: huchenxi Date: Wed, 22 Apr 2026 10:45:02 +0800 Subject: [PATCH 1/2] feat(web): inline image rendering in tool results with lightbox preview MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tool results (Read, Bash, Markdown, Generic views) currently render only text content — any base64 image blocks returned by tools are dropped. This means screenshots read by the Read tool, image output from bash commands, etc. are invisible in the chat. Changes: - Add ImageLightbox component with Portal overlay, ESC/backdrop close, and open-in-new-tab action. - Extract and render base64 image blocks in tool result content. - Integrate ResultImages into Read/Bash/Markdown/Generic result views. - Add click-to-preview for user-uploaded image attachments. --- .../messages/MessageAttachments.tsx | 30 +++-- web/src/components/ImageLightbox.tsx | 100 ++++++++++++++++ .../components/ToolCard/views/_results.tsx | 110 +++++++++++++++++- 3 files changed, 228 insertions(+), 12 deletions(-) create mode 100644 web/src/components/ImageLightbox.tsx diff --git a/web/src/components/AssistantChat/messages/MessageAttachments.tsx b/web/src/components/AssistantChat/messages/MessageAttachments.tsx index 11eb6b3637..f6b9e961a3 100644 --- a/web/src/components/AssistantChat/messages/MessageAttachments.tsx +++ b/web/src/components/AssistantChat/messages/MessageAttachments.tsx @@ -1,6 +1,8 @@ +import { useState } from 'react' import type { AttachmentMetadata } from '@/types/api' import { FileIcon } from '@/components/FileIcon' import { isImageMimeType } from '@/lib/fileAttachments' +import { ImageLightbox } from '@/components/ImageLightbox' function formatFileSize(bytes: number): string { if (bytes < 1024) return `${bytes} B` @@ -10,19 +12,25 @@ function formatFileSize(bytes: number): string { function ImageAttachment(props: { attachment: AttachmentMetadata }) { const { attachment } = props + const [open, setOpen] = useState(false) return ( -
- {attachment.filename} -
- - {attachment.filename} - + <> +
setOpen(true)}> + {attachment.filename} +
+ + {attachment.filename} + +
-
+ {attachment.previewUrl && ( + setOpen(false)} /> + )} + ) } diff --git a/web/src/components/ImageLightbox.tsx b/web/src/components/ImageLightbox.tsx new file mode 100644 index 0000000000..4fa1e8a8b8 --- /dev/null +++ b/web/src/components/ImageLightbox.tsx @@ -0,0 +1,100 @@ +import { useEffect, useCallback, useState } from 'react' +import { createPortal } from 'react-dom' + +interface ImageLightboxProps { + src: string + alt?: string + open: boolean + onClose: () => void +} + +export function ImageLightbox({ src, alt, open, onClose }: ImageLightboxProps) { + const handleKeyDown = useCallback( + (e: KeyboardEvent) => { + if (e.key === 'Escape') onClose() + }, + [onClose] + ) + + useEffect(() => { + if (!open) return + document.addEventListener('keydown', handleKeyDown) + document.body.style.overflow = 'hidden' + return () => { + document.removeEventListener('keydown', handleKeyDown) + document.body.style.overflow = '' + } + }, [open, handleKeyDown]) + + if (!open) return null + + return createPortal( +
+ {/* Top-right buttons */} +
+ + +
+ + {/* Image */} + {alt e.stopPropagation()} + /> +
, + document.body + ) +} + +export function useImageLightbox() { + const [lightbox, setLightbox] = useState<{ src: string; alt?: string } | null>(null) + + const openLightbox = useCallback((src: string, alt?: string) => { + setLightbox({ src, alt }) + }, []) + + const closeLightbox = useCallback(() => { + setLightbox(null) + }, []) + + const LightboxPortal = lightbox ? ( + + ) : null + + return { openLightbox, LightboxPortal } +} diff --git a/web/src/components/ToolCard/views/_results.tsx b/web/src/components/ToolCard/views/_results.tsx index 26d7b15a88..57b53cd9cd 100644 --- a/web/src/components/ToolCard/views/_results.tsx +++ b/web/src/components/ToolCard/views/_results.tsx @@ -1,3 +1,4 @@ +import { useState } from 'react' import type { ToolViewComponent, ToolViewProps } from '@/components/ToolCard/views/_all' import { isObject, safeStringify } from '@hapi/protocol' import { CodeBlock } from '@/components/CodeBlock' @@ -5,6 +6,7 @@ import { MarkdownRenderer } from '@/components/MarkdownRenderer' import { ChecklistList, extractTodoChecklist } from '@/components/ToolCard/checklist' import { basename, resolveDisplayPath } from '@/utils/path' import { getInputStringAny } from '@/lib/toolInputUtils' +import { ImageLightbox } from '@/components/ImageLightbox' function parseToolUseError(message: string): { isToolUseError: boolean; errorMessage: string | null } { const regex = /(.*?)<\/tool_use_error>/s @@ -28,6 +30,66 @@ function extractTextFromContentBlock(block: unknown): string | null { return null } +interface ImageBlock { + mediaType: string + dataUrl: string +} + +function extractImageFromContentBlock(block: unknown): ImageBlock | null { + if (!isObject(block)) return null + if (block.type !== 'image') return null + const source = isObject(block.source) ? block.source : null + if (!source) return null + if (source.type === 'base64' && typeof source.media_type === 'string' && typeof source.data === 'string') { + return { mediaType: source.media_type, dataUrl: `data:${source.media_type};base64,${source.data}` } + } + return null +} + +function extractImagesFromResult(result: unknown): ImageBlock[] { + if (!result) return [] + + if (Array.isArray(result)) { + return result.map(extractImageFromContentBlock).filter((img): img is ImageBlock => img !== null) + } + + if (isObject(result)) { + const contentArray = Array.isArray(result.content) ? result.content : null + if (contentArray) { + return contentArray.map(extractImageFromContentBlock).filter((img): img is ImageBlock => img !== null) + } + } + + return [] +} + +function InlineImage({ image }: { image: ImageBlock }) { + const [open, setOpen] = useState(false) + return ( + <> + Tool result image setOpen(true)} + /> + setOpen(false)} /> + + ) +} + +function ResultImages({ result }: { result: unknown }) { + const images = extractImagesFromResult(result) + if (images.length === 0) return null + return ( +
+ {images.map((img, i) => ( + + ))} +
+ ) +} + export function extractTextFromResult(result: unknown, depth: number = 0): string | null { if (depth > 2) return null if (result === null || result === undefined) return null @@ -249,6 +311,7 @@ const BashResultView: ToolViewComponent = (props: ToolViewProps) => { {stdio.stdout ? : null} {stdio.stderr ? : null}
+ ) @@ -259,6 +322,17 @@ const BashResultView: ToolViewComponent = (props: ToolViewProps) => { return ( <> {renderText(text, { mode: 'code', language: 'text' })} + + + + ) + } + + const images = extractImagesFromResult(result) + if (images.length > 0) { + return ( + <> + ) @@ -284,6 +358,17 @@ const MarkdownResultView: ToolViewComponent = (props: ToolViewProps) => { return ( <> {renderText(text, { mode: 'auto' })} + + + + ) + } + + const images = extractImagesFromResult(result) + if (images.length > 0) { + return ( + <> + ) @@ -354,6 +439,8 @@ const ReadResultView: ToolViewComponent = (props: ToolViewProps) => { return
{placeholderForState(props.block.tool.state)}
} + const images = extractImagesFromResult(result) + const file = extractReadFileContent(result) if (file) { const path = file.filePath ? resolveDisplayPath(file.filePath, props.metadata) : null @@ -364,7 +451,16 @@ const ReadResultView: ToolViewComponent = (props: ToolViewProps) => { {basename(path)} ) : null} - + {images.length > 0 ? : } + + + ) + } + + if (images.length > 0) { + return ( + <> + ) @@ -595,6 +691,7 @@ const GenericResultView: ToolViewComponent = (props: ToolViewProps) => { {parsed.wallTime && `Wall time: ${parsed.wallTime}`} {renderText(parsed.output.trim(), { mode: 'code' })} + ) @@ -606,11 +703,22 @@ const GenericResultView: ToolViewComponent = (props: ToolViewProps) => { return ( <> {renderText(text, { mode: 'auto' })} + {typeof result === 'object' ? : null} ) } + const images = extractImagesFromResult(result) + if (images.length > 0) { + return ( + <> + + + + ) + } + if (typeof result === 'string') { return renderText(result, { mode: 'auto' }) } From ed86b3ac3fc98cec04b2cc96ec8c9492707be660 Mon Sep 17 00:00:00 2001 From: huchenxi Date: Wed, 22 Apr 2026 14:59:16 +0800 Subject: [PATCH 2/2] fix(web): render both text and images for Read tool results The Read tool view used a ternary that swapped file content for images when a result contained image blocks, so mixed text+image payloads (e.g. Read on a notebook with embedded images) lost the textual output. Render the CodeBlock unconditionally and append ResultImages when images are present, mirroring how Bash / Markdown / Generic views already handle the same extraction. --- web/src/components/ToolCard/views/_results.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/src/components/ToolCard/views/_results.tsx b/web/src/components/ToolCard/views/_results.tsx index 57b53cd9cd..efea2b2c1f 100644 --- a/web/src/components/ToolCard/views/_results.tsx +++ b/web/src/components/ToolCard/views/_results.tsx @@ -451,7 +451,8 @@ const ReadResultView: ToolViewComponent = (props: ToolViewProps) => { {basename(path)} ) : null} - {images.length > 0 ? : } + + {images.length > 0 ? : null} )