diff --git a/docs/image-model-context.md b/docs/image-model-context.md new file mode 100644 index 00000000..702ef2e5 --- /dev/null +++ b/docs/image-model-context.md @@ -0,0 +1,17 @@ +# ChatGPT 图片上下文 + +启用 MCP Apps 后,直连 AgentDock 的 `view_image` 提供共享图片组件。组件预览标准 MCP 图片结果;用户点击“让 ChatGPT 查看图片”后,使用宿主上传接口和 `widgetState.imageIds` 将真实图片加入后续模型轮次。不支持该接口的宿主只提供预览。 + +这不是同轮无人值守 Computer Use。标准图片字节、输出 schema 和结构化结果保持不变;不使用 OCR,不新增外部网络域名。上传由用户点击触发,不默认存入文件库。关闭 MCP Apps 或工具返回错误时,不添加图片组件结果绑定。 + +## 共享实现与合并依赖 + +HTML、宿主桥接和组件行为测试由 `agentdock-protocol/mcpapps` 统一维护,NexusDock 使用同一实现。依赖的上游改动为 [agentdock-protocol #6](https://github.com/uvwt/agentdock-protocol/pull/6)。 + +当前是配套草稿:`go.mod` 中的 v0.8.1 尚未包含新接口,因此独立构建暂不成立。合并前必须等待协议改动合并并发布,再更新 `go.mod` / `go.sum` 到真实上游版本、移除本段待办并重新运行完整检查。不提交本地路径 replace 或私人 fork 依赖。 + +审阅时可将本仓库和协议 PR 分支放入临时 Go workspace,使用 `go work init <协议仓库路径> <本仓库路径>`,再设置 `GOWORK` 为该文件绝对路径运行 `make check` 和 `go test -race ./...`。协议仓库另运行 `node --test mcpapps/image.test.mjs`。 + +## 验收边界 + +直接 MCP 集成测试覆盖真实 PNG 的 `view_image` 调用、开关、错误结果和共享 HTML 内容一致性。真实 ChatGPT 的识图验收仍须使用 Chrome:提示词不得提供答案,点击组件后核对只能从图片取得的细节。MCP 集成测试通过不等于已完成直连 ChatGPT 的端到端验收。 diff --git a/internal/app/mcp_apps.go b/internal/app/mcp_apps.go index 8f1c86e9..a3168dc1 100644 --- a/internal/app/mcp_apps.go +++ b/internal/app/mcp_apps.go @@ -10,6 +10,7 @@ type UIBinding struct { } var toolUIBindings = map[string]UIBinding{ + "view_image": {ResourceURI: protocol.ImageUIResourceURI}, "agentdock_context": {ResourceURI: protocol.ContextUIResourceURI}, "file_edit": {ResourceURI: protocol.FileChangeUIResourceURI}, "task_manage": {ResourceURI: protocol.TaskProgressUIResourceURI}, diff --git a/internal/mcp/apps.go b/internal/mcp/apps.go index 056f6f4b..f998212e 100644 --- a/internal/mcp/apps.go +++ b/internal/mcp/apps.go @@ -24,6 +24,13 @@ func (s *Server) appResourceDefinitions() []appResourceDefinition { return nil } definitions := []appResourceDefinition{ + { + URI: protocol.ImageUIResourceURI, + Name: "agentdock-image", + Title: "AgentDock image", + Description: "Display the requested image and attach it to a ChatGPT follow-up.", + HTML: mcpapps.HTML("view_image", "Image"), + }, { URI: protocol.ContextUIResourceURI, Name: "agentdock-context", diff --git a/internal/mcp/apps_test.go b/internal/mcp/apps_test.go index 725e4b8e..ca1ace66 100644 --- a/internal/mcp/apps_test.go +++ b/internal/mcp/apps_test.go @@ -112,8 +112,8 @@ func TestUIResourcesMatchServedResourceRegistry(t *testing.T) { server := &Server{cfg: config.Config{NexusEndpoint: "https://nexus.example.test", ACPEnabled: true, MCPAppsEnabled: true}} definitions := server.appResourceDefinitions() resources := server.UIResources() - if len(definitions) != 8 || len(resources) != len(definitions) { - t.Fatalf("resource registry=%d bridge capabilities=%d, want 8", len(definitions), len(resources)) + if len(definitions) != 9 || len(resources) != len(definitions) { + t.Fatalf("resource registry=%d bridge capabilities=%d, want 9", len(definitions), len(resources)) } byURI := make(map[string]protocol.UIResourceCapability, len(resources)) for _, resource := range resources { @@ -251,8 +251,8 @@ func TestMCPAppsBindResourcesDirectlyToBusinessTools(t *testing.T) { } resources[resource.URI] = resource } - if len(resources) != 5 { - t.Fatalf("resources/list count = %d, want 5", len(resources)) + if len(resources) != 6 { + t.Fatalf("resources/list count = %d, want 6", len(resources)) } for _, uri := range []string{ protocol.ContextUIResourceURI, @@ -534,8 +534,8 @@ func TestMCPAppsExposeNexusViewsWhenNexusEnabled(t *testing.T) { } resources[resource.URI] = resource } - if len(resources) != 7 { - t.Fatalf("resources/list count = %d, want 7", len(resources)) + if len(resources) != 8 { + t.Fatalf("resources/list count = %d, want 8", len(resources)) } for _, tc := range []struct { uri string diff --git a/internal/mcp/image_app_test.go b/internal/mcp/image_app_test.go new file mode 100644 index 00000000..9d296b6d --- /dev/null +++ b/internal/mcp/image_app_test.go @@ -0,0 +1,79 @@ +package mcp + +import ( + "bytes" + "image" + "image/color" + "image/png" + "os" + "path/filepath" + "testing" + + mcpsdk "github.com/modelcontextprotocol/go-sdk/mcp" + protocol "github.com/uvwt/agentdock-protocol" + "github.com/uvwt/agentdock-protocol/mcpapps" + "github.com/uvwt/agentdock/internal/config" +) + +func TestDirectImageAppUsesSharedRendererAndPreservesPixels(t *testing.T) { + var original []byte + for _, enabled := range []bool{false, true} { + root := t.TempDir() + img := image.NewRGBA(image.Rect(0, 0, 2, 2)) + img.Set(0, 0, color.RGBA{R: 255, A: 255}) + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatal(err) + } + path := filepath.Join(root, "image.png") + if err := os.WriteFile(path, buf.Bytes(), 0600); err != nil { + t.Fatal(err) + } + h := newMCPAppTestHarnessWithApps(t, config.Config{AgentDockDefaultDir: root, AgentDockHome: filepath.Join(root, ".agentdock")}, enabled) + var found *mcpsdk.Tool + for tool, err := range h.session.Tools(t.Context(), nil) { + if err != nil { + t.Fatal(err) + } + if tool.Name == "view_image" { + found = tool + } + } + if found == nil || found.OutputSchema == nil { + t.Fatal("missing image contract") + } + if enabled { + assertToolUIResource(t, found, protocol.ImageUIResourceURI, "openai/outputTemplate") + } else if found.Meta["ui"] != nil { + t.Fatal("Apps disabled") + } + result, err := h.session.CallTool(t.Context(), &mcpsdk.CallToolParams{Name: "view_image", Arguments: map[string]any{"path": path, "format": "png"}}) + if err != nil || result.IsError { + t.Fatalf("image failed: %v %#v", err, result) + } + pixels, ok := result.Content[0].(*mcpsdk.ImageContent) + if !ok || len(pixels.Data) == 0 || result.StructuredContent == nil { + t.Fatal("lost image or metadata") + } + if !enabled { + original = pixels.Data + } else if !bytes.Equal(original, pixels.Data) { + t.Fatal("Apps changed image bytes") + } + if enabled { + if result.Meta["ui"] == nil { + t.Fatal("missing result binding") + } + read, err := h.session.ReadResource(t.Context(), &mcpsdk.ReadResourceParams{URI: protocol.ImageUIResourceURI}) + if err != nil || len(read.Contents) != 1 || read.Contents[0].Text != mcpapps.HTML("view_image", "Image") { + t.Fatalf("not using shared renderer: %v", err) + } + } else if result.Meta["ui"] != nil || len(result.Content) != 1 { + t.Fatal("disabled result decorated") + } + failed, err := h.session.CallTool(t.Context(), &mcpsdk.CallToolParams{Name: "view_image", Arguments: map[string]any{"path": filepath.Join(root, "missing.png")}}) + if err != nil || !failed.IsError || failed.Meta["ui"] != nil { + t.Fatalf("failure acquired image widget: %v %#v", err, failed) + } + } +} diff --git a/internal/mcp/server.go b/internal/mcp/server.go index b52a62f4..7be61242 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -13,6 +13,7 @@ import ( sdkjsonrpc "github.com/modelcontextprotocol/go-sdk/jsonrpc" mcpsdk "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/uvwt/agentdock-protocol/mcpapps" "github.com/uvwt/agentdock/internal/app" "github.com/uvwt/agentdock/internal/buildinfo" "github.com/uvwt/agentdock/internal/config" @@ -171,6 +172,16 @@ func (s *Server) callTool(ctx context.Context, name string, request *mcpsdk.Call if meta := toolResultMetadata(def, arguments, s.cfg.MCPAppsEnabled); len(meta) > 0 { response.Meta = meta } + if name == "view_image" && s.cfg.MCPAppsEnabled && !response.IsError { + for _, content := range response.Content { + if _, ok := content.(*mcpsdk.ImageContent); ok { + // 图像宿主需在结果上获得展示绑定;原图字节和结构化协议保持不变。 + response.Meta = mcpsdk.Meta(toolMetadata(def, true)) + response.Content = append(response.Content, &mcpsdk.TextContent{Text: mcpapps.ImageResultText}) + break + } + } + } } return &response, nil } @@ -179,6 +190,9 @@ func toolMetadata(def ToolDefinition, mcpAppsEnabled bool) map[string]any { meta := map[string]any{} if mcpAppsEnabled && def.UIBinding != nil && def.UIBinding.Action == "" { meta["ui"] = map[string]any{"resourceUri": def.UIBinding.ResourceURI} + if def.Name == "view_image" { + meta["openai/outputTemplate"] = def.UIBinding.ResourceURI + } } if len(def.FileArgRewritePaths) > 0 { paths := append([]string(nil), def.FileArgRewritePaths...)