From 50331ef9c8d4dbd62f0ae6c60b50e8d87c9e8e09 Mon Sep 17 00:00:00 2001 From: Ayman Hamed Date: Tue, 4 Aug 2026 13:12:12 +0300 Subject: [PATCH] feat(tools): carry images through tool results + MCP bridge (PoC) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lets a tool return images (e.g. a browser screenshot) that reach a vision model on the next turn, instead of being dropped. - AgentToolResult gains `images: [LLMImage]` (default []), Codable/Equatable; `.success(…, images:)` convenience. - MCPToolBridge.execute now captures `.image` content items (base64 → LLMImage) instead of dropping everything but text — the ingestion point for a chrome-devtools screenshot. - AgentMessage.toLLMMessages() forwards tool-result images onto the `.tool` LLMMessage (uses the full LLMMessage initializer, so it builds against the released LLMProviderKit). Tests: images survive Codable; toLLMMessages forwards them; text-only carries none. Encoding to a provider's wire format is proven in LLMProviderKit. PoC: pairs with LLMProviderKit feat/tool-result-images. Runtime image encoding needs that published; provider generalization + Naseem integration follow. Co-Authored-By: Claude Opus 4.8 --- Sources/SwiftAgentKit/Core/AgentMessage.swift | 20 ++++++++- Sources/SwiftAgentKitMCP/MCPToolBridge.swift | 24 ++++++++--- .../ToolResultImageTests.swift | 43 +++++++++++++++++++ 3 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 Tests/SwiftAgentKitTests/ToolResultImageTests.swift diff --git a/Sources/SwiftAgentKit/Core/AgentMessage.swift b/Sources/SwiftAgentKit/Core/AgentMessage.swift index 92fc2d7..e8d0a61 100644 --- a/Sources/SwiftAgentKit/Core/AgentMessage.swift +++ b/Sources/SwiftAgentKit/Core/AgentMessage.swift @@ -98,7 +98,13 @@ public struct AgentMessage: Identifiable, @unchecked Sendable, Codable { let status = result.isError ? "ERROR" : "OK" let toolName = result.toolName ?? "unknown" let content = "[Tool: \(toolName)] \(status)\n\(result.result)" - return .tool(content, toolCallId: result.toolCallId) + if result.images.isEmpty { + return .tool(content, toolCallId: result.toolCallId) + } + // Full initializer (available in the released LLMProviderKit) so + // this compiles without the newer `.tool(_:images:)` convenience. + return LLMMessage(role: .tool, content: content, + images: result.images, toolCallId: result.toolCallId) } default: @@ -224,19 +230,24 @@ public struct AgentToolResult: Sendable, Identifiable, Equatable, Codable { public let toolName: String? public let result: String public let isError: Bool + /// Images the tool produced (e.g. a browser screenshot). Fed back to a + /// vision-capable model alongside the text result so it can "see" them. + public let images: [LLMImage] public init( id: String = UUID().uuidString, toolCallId: String, toolName: String? = nil, result: String, - isError: Bool = false + isError: Bool = false, + images: [LLMImage] = [] ) { self.id = id self.toolCallId = toolCallId self.toolName = toolName self.result = result self.isError = isError + self.images = images } /// Create a successful result. @@ -244,6 +255,11 @@ public struct AgentToolResult: Sendable, Identifiable, Equatable, Codable { AgentToolResult(toolCallId: toolCallId, toolName: toolName, result: result, isError: false) } + /// Create a successful result carrying images (e.g. a screenshot). + public static func success(toolCallId: String, toolName: String?, result: String, images: [LLMImage]) -> AgentToolResult { + AgentToolResult(toolCallId: toolCallId, toolName: toolName, result: result, isError: false, images: images) + } + /// Create an error result. public static func error(toolCallId: String, toolName: String?, message: String) -> AgentToolResult { AgentToolResult(toolCallId: toolCallId, toolName: toolName, result: message, isError: true) diff --git a/Sources/SwiftAgentKitMCP/MCPToolBridge.swift b/Sources/SwiftAgentKitMCP/MCPToolBridge.swift index e0e5aa8..b8928cd 100644 --- a/Sources/SwiftAgentKitMCP/MCPToolBridge.swift +++ b/Sources/SwiftAgentKitMCP/MCPToolBridge.swift @@ -1,5 +1,6 @@ import Foundation import SwiftAgentKit +import LLMProviderKit import MCP /// Bridges an MCP server tool into SwiftAgentKit's `AgentTool` protocol. @@ -40,19 +41,30 @@ public struct MCPToolBridge: AgentTool { try await mcpClient.callTool(name: toolName, arguments: arguments) } - // Extract text from content items - let textParts = content.compactMap { item -> String? in - if case .text(let text, _, _) = item { - return text + // Extract text and images from content items. Images (e.g. a browser + // tool's screenshot) are carried through so a vision model can see them + // instead of being dropped. + var textParts: [String] = [] + var images: [LLMImage] = [] + for item in content { + switch item { + case .text(let text, _, _): + textParts.append(text) + case .image(let data, let mimeType, _, _): + if let bytes = Data(base64Encoded: data) { + images.append(LLMImage(data: bytes, mimeType: mimeType)) + } + default: + break } - return nil } let result = textParts.joined(separator: "\n") if isError ?? false { return .error(toolCallId: "", toolName: name, message: result.isEmpty ? "MCP tool error" : result) } - return .success(toolCallId: "", toolName: name, result: result) + let text = result.isEmpty && !images.isEmpty ? "[image returned]" : result + return .success(toolCallId: "", toolName: name, result: text, images: images) } public func execute(context: ToolContext) async throws -> AgentToolResult { diff --git a/Tests/SwiftAgentKitTests/ToolResultImageTests.swift b/Tests/SwiftAgentKitTests/ToolResultImageTests.swift new file mode 100644 index 0000000..a6145ea --- /dev/null +++ b/Tests/SwiftAgentKitTests/ToolResultImageTests.swift @@ -0,0 +1,43 @@ +import Testing +import Foundation +import LLMProviderKit +@testable import SwiftAgentKit + +/// A tool result can carry images, they survive Codable, and they flow through +/// `toLLMMessages()` onto the `.tool` LLM message so a vision model sees them. +struct ToolResultImageTests { + + private var sampleImage: LLMImage { + LLMImage(data: Data([0x89, 0x50, 0x4E, 0x47]), mimeType: "image/png") // "‰PNG" + } + + @Test func imagesSurviveCodableRoundTrip() throws { + let result = AgentToolResult.success( + toolCallId: "tu_1", toolName: "take_snapshot", + result: "captured", images: [sampleImage]) + let data = try JSONEncoder().encode(result) + let decoded = try JSONDecoder().decode(AgentToolResult.self, from: data) + #expect(decoded.images == [sampleImage]) + #expect(decoded.result == "captured") + } + + @Test func toLLMMessagesForwardsImagesOntoToolMessage() { + let msg = AgentMessage.tool(results: [ + .success(toolCallId: "tu_1", toolName: "take_snapshot", + result: "captured", images: [sampleImage]) + ]) + let llm = msg.toLLMMessages() + #expect(llm.count == 1) + #expect(llm[0].role == .tool) + #expect(llm[0].toolCallId == "tu_1") + #expect(llm[0].images == [sampleImage]) + } + + @Test func textOnlyResultCarriesNoImages() { + let msg = AgentMessage.tool(results: [ + .success(toolCallId: "tu_2", toolName: "noop", result: "ok") + ]) + let llm = msg.toLLMMessages() + #expect(llm[0].images.isEmpty) + } +}