Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions docs/image-model-context.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# ChatGPT 图片上下文

启用 MCP Apps 后,直连 AgentDock 的 `view_image` 提供共享图片组件。组件预览标准 MCP 图片结果;用户点击“让 ChatGPT 查看图片”后,使用宿主上传接口和 `widgetState.imageIds` 将真实图片加入后续模型轮次。不支持该接口的宿主只提供预览。

这不是同轮无人值守 Computer Use。标准图片字节、输出 schema 和结构化结果保持不变;不使用 OCR,不新增外部网络域名。上传由用户点击触发,不默认存入文件库。关闭 MCP Apps 或工具返回错误时,不添加图片组件结果绑定。

## 共享实现与合并依赖

HTML、宿主桥接和组件行为测试由 `agentdock-protocol/mcpapps` 统一维护,NexusDock 使用同一实现。依赖的上游改动为 [agentdock-protocol #6](https://github.com/uvwt/agentdock-protocol/pull/6)。

当前是配套草稿:`go.mod` 中的 v0.8.1 尚未包含新接口,因此独立构建暂不成立。合并前必须等待协议改动合并并发布,再更新 `go.mod` / `go.sum` 到真实上游版本、移除本段待办并重新运行完整检查。不提交本地路径 replace 或私人 fork 依赖。

审阅时可将本仓库和协议 PR 分支放入临时 Go workspace,使用 `go work init <协议仓库路径> <本仓库路径>`,再设置 `GOWORK` 为该文件绝对路径运行 `make check` 和 `go test -race ./...`。协议仓库另运行 `node --test mcpapps/image.test.mjs`。

## 验收边界

直接 MCP 集成测试覆盖真实 PNG 的 `view_image` 调用、开关、错误结果和共享 HTML 内容一致性。真实 ChatGPT 的识图验收仍须使用 Chrome:提示词不得提供答案,点击组件后核对只能从图片取得的细节。MCP 集成测试通过不等于已完成直连 ChatGPT 的端到端验收。
1 change: 1 addition & 0 deletions internal/app/mcp_apps.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ type UIBinding struct {
}

var toolUIBindings = map[string]UIBinding{
"view_image": {ResourceURI: protocol.ImageUIResourceURI},
"agentdock_context": {ResourceURI: protocol.ContextUIResourceURI},
"file_edit": {ResourceURI: protocol.FileChangeUIResourceURI},
"task_manage": {ResourceURI: protocol.TaskProgressUIResourceURI},
Expand Down
7 changes: 7 additions & 0 deletions internal/mcp/apps.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,13 @@ func (s *Server) appResourceDefinitions() []appResourceDefinition {
return nil
}
definitions := []appResourceDefinition{
{
URI: protocol.ImageUIResourceURI,
Name: "agentdock-image",
Title: "AgentDock image",
Description: "Display the requested image and attach it to a ChatGPT follow-up.",
HTML: mcpapps.HTML("view_image", "Image"),
},
{
URI: protocol.ContextUIResourceURI,
Name: "agentdock-context",
Expand Down
12 changes: 6 additions & 6 deletions internal/mcp/apps_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,8 @@ func TestUIResourcesMatchServedResourceRegistry(t *testing.T) {
server := &Server{cfg: config.Config{NexusEndpoint: "https://nexus.example.test", ACPEnabled: true, MCPAppsEnabled: true}}
definitions := server.appResourceDefinitions()
resources := server.UIResources()
if len(definitions) != 8 || len(resources) != len(definitions) {
t.Fatalf("resource registry=%d bridge capabilities=%d, want 8", len(definitions), len(resources))
if len(definitions) != 9 || len(resources) != len(definitions) {
t.Fatalf("resource registry=%d bridge capabilities=%d, want 9", len(definitions), len(resources))
}
byURI := make(map[string]protocol.UIResourceCapability, len(resources))
for _, resource := range resources {
Expand Down Expand Up @@ -251,8 +251,8 @@ func TestMCPAppsBindResourcesDirectlyToBusinessTools(t *testing.T) {
}
resources[resource.URI] = resource
}
if len(resources) != 5 {
t.Fatalf("resources/list count = %d, want 5", len(resources))
if len(resources) != 6 {
t.Fatalf("resources/list count = %d, want 6", len(resources))
}
for _, uri := range []string{
protocol.ContextUIResourceURI,
Expand Down Expand Up @@ -534,8 +534,8 @@ func TestMCPAppsExposeNexusViewsWhenNexusEnabled(t *testing.T) {
}
resources[resource.URI] = resource
}
if len(resources) != 7 {
t.Fatalf("resources/list count = %d, want 7", len(resources))
if len(resources) != 8 {
t.Fatalf("resources/list count = %d, want 8", len(resources))
}
for _, tc := range []struct {
uri string
Expand Down
79 changes: 79 additions & 0 deletions internal/mcp/image_app_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package mcp

import (
"bytes"
"image"
"image/color"
"image/png"
"os"
"path/filepath"
"testing"

mcpsdk "github.com/modelcontextprotocol/go-sdk/mcp"
protocol "github.com/uvwt/agentdock-protocol"
"github.com/uvwt/agentdock-protocol/mcpapps"
"github.com/uvwt/agentdock/internal/config"
)

func TestDirectImageAppUsesSharedRendererAndPreservesPixels(t *testing.T) {
var original []byte
for _, enabled := range []bool{false, true} {
root := t.TempDir()
img := image.NewRGBA(image.Rect(0, 0, 2, 2))
img.Set(0, 0, color.RGBA{R: 255, A: 255})
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
t.Fatal(err)
}
path := filepath.Join(root, "image.png")
if err := os.WriteFile(path, buf.Bytes(), 0600); err != nil {
t.Fatal(err)
}
h := newMCPAppTestHarnessWithApps(t, config.Config{AgentDockDefaultDir: root, AgentDockHome: filepath.Join(root, ".agentdock")}, enabled)
var found *mcpsdk.Tool
for tool, err := range h.session.Tools(t.Context(), nil) {
if err != nil {
t.Fatal(err)
}
if tool.Name == "view_image" {
found = tool
}
}
if found == nil || found.OutputSchema == nil {
t.Fatal("missing image contract")
}
if enabled {
assertToolUIResource(t, found, protocol.ImageUIResourceURI, "openai/outputTemplate")
} else if found.Meta["ui"] != nil {
t.Fatal("Apps disabled")
}
result, err := h.session.CallTool(t.Context(), &mcpsdk.CallToolParams{Name: "view_image", Arguments: map[string]any{"path": path, "format": "png"}})
if err != nil || result.IsError {
t.Fatalf("image failed: %v %#v", err, result)
}
pixels, ok := result.Content[0].(*mcpsdk.ImageContent)
if !ok || len(pixels.Data) == 0 || result.StructuredContent == nil {
t.Fatal("lost image or metadata")
}
if !enabled {
original = pixels.Data
} else if !bytes.Equal(original, pixels.Data) {
t.Fatal("Apps changed image bytes")
}
if enabled {
if result.Meta["ui"] == nil {
t.Fatal("missing result binding")
}
read, err := h.session.ReadResource(t.Context(), &mcpsdk.ReadResourceParams{URI: protocol.ImageUIResourceURI})
if err != nil || len(read.Contents) != 1 || read.Contents[0].Text != mcpapps.HTML("view_image", "Image") {
t.Fatalf("not using shared renderer: %v", err)
}
} else if result.Meta["ui"] != nil || len(result.Content) != 1 {
t.Fatal("disabled result decorated")
}
failed, err := h.session.CallTool(t.Context(), &mcpsdk.CallToolParams{Name: "view_image", Arguments: map[string]any{"path": filepath.Join(root, "missing.png")}})
if err != nil || !failed.IsError || failed.Meta["ui"] != nil {
t.Fatalf("failure acquired image widget: %v %#v", err, failed)
}
}
}
14 changes: 14 additions & 0 deletions internal/mcp/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (

sdkjsonrpc "github.com/modelcontextprotocol/go-sdk/jsonrpc"
mcpsdk "github.com/modelcontextprotocol/go-sdk/mcp"
"github.com/uvwt/agentdock-protocol/mcpapps"
"github.com/uvwt/agentdock/internal/app"
"github.com/uvwt/agentdock/internal/buildinfo"
"github.com/uvwt/agentdock/internal/config"
Expand Down Expand Up @@ -171,6 +172,16 @@ func (s *Server) callTool(ctx context.Context, name string, request *mcpsdk.Call
if meta := toolResultMetadata(def, arguments, s.cfg.MCPAppsEnabled); len(meta) > 0 {
response.Meta = meta
}
if name == "view_image" && s.cfg.MCPAppsEnabled && !response.IsError {
for _, content := range response.Content {
if _, ok := content.(*mcpsdk.ImageContent); ok {
// 图像宿主需在结果上获得展示绑定;原图字节和结构化协议保持不变。
response.Meta = mcpsdk.Meta(toolMetadata(def, true))
response.Content = append(response.Content, &mcpsdk.TextContent{Text: mcpapps.ImageResultText})
break
}
}
}
}
return &response, nil
}
Expand All @@ -179,6 +190,9 @@ func toolMetadata(def ToolDefinition, mcpAppsEnabled bool) map[string]any {
meta := map[string]any{}
if mcpAppsEnabled && def.UIBinding != nil && def.UIBinding.Action == "" {
meta["ui"] = map[string]any{"resourceUri": def.UIBinding.ResourceURI}
if def.Name == "view_image" {
meta["openai/outputTemplate"] = def.UIBinding.ResourceURI
}
}
if len(def.FileArgRewritePaths) > 0 {
paths := append([]string(nil), def.FileArgRewritePaths...)
Expand Down