Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,11 @@ The repository intentionally has two logical layers:
- `mcpcontract` owns only the canonical model-facing MCP contracts shared by the two entrypoints: input/output schemas, annotations, and bounded behavior vectors.

Neither package owns AgentDock runtime behavior, NexusDock stores, renderer HTML, Recall persistence, Workflow persistence, or HTTP handlers. Those remain in their application repositories.

## 共享 MCP Apps

独立的 `mcpapps` 包维护两个入口复用的 UI 文档。`HTML("view_image", ...)` 返回图片组件,资源身份与兼容契约由根包的 `ImageUIResourceURI` / `ImageUIContract` 定义。图片内容仍使用标准 MCP image 块,各入口自行注册资源和绑定工具,不将执行或 HTTP 逻辑放进共享包。

图片组件只预览本次工具结果,用户点击后通过 ChatGPT 宿主的 `uploadFile` 和 `widgetState.imageIds` 附加到后续对话。未提供宿主接口的客户端只显示预览;不保证同轮自动视觉,不读取额外文件、不使用 OCR、不默认保存到文件库,也不扩大 CSP 网络域名。

验证命令:`go test ./...`、`go test -race ./...`、`go vet ./...`、`node --test mcpapps/image.test.mjs`(Node 20+)。组件上传、宿主状态更新、错误、重复点击、换图竞争及非父 frame 消息均有行为测试。真实模型识图仍须在宿主中独立验收,不能仅凭单元测试推定。
9 changes: 9 additions & 0 deletions mcpapps/apps.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,17 @@ import (
//go:embed app.html
var appHTMLTemplate string

//go:embed image.html
var imageHTML string

// ImageResultText explains the optional, user-initiated follow-up without claiming the model saw pixels.
const ImageResultText = "图片已返回。若客户端未将图片直接提供给模型,可在图片组件中将它附加到下一轮对话。"

// HTML renders one shared MCP App document for a known AgentDock view.
// View and title are internal constants owned by AgentDock/NexusDock, not user input.
func HTML(view, title string) string {
if view == "view_image" {
return imageHTML
}
return strings.NewReplacer("{{VIEW}}", view, "{{TITLE}}", title).Replace(appHTMLTemplate)
}
76 changes: 76 additions & 0 deletions mcpapps/image.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
<!doctype html>
<html lang="zh-CN"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<style>
:root{color-scheme:light dark;font:14px system-ui,sans-serif}body{margin:0;padding:16px}img{display:block;max-width:100%;max-height:480px;object-fit:contain;margin:auto}img[hidden]{display:none}p{line-height:1.5;margin:12px 0}button{font:inherit;padding:9px 14px;border:1px solid #8886;border-radius:10px;cursor:pointer}button:disabled{opacity:.55;cursor:default}
</style></head>
<body><img id="preview" alt="本次工具返回的图片" hidden><p id="status" role="status">等待图片</p><button id="share" disabled>让 ChatGPT 查看图片</button>
<script>
(() => {
const preview=document.getElementById("preview"), status=document.getElementById("status"), share=document.getElementById("share");
let selected=null, busy=false, generation=0, lastImage=null;
const notify=(method,params)=>window.parent.postMessage({jsonrpc:"2.0",method,params},"*");
const canUpload=()=>typeof window.openai?.uploadFile==="function" && typeof window.openai?.setWidgetState==="function";
function ready(){
share.disabled=busy || !selected || !canUpload();
if(selected && !busy) status.textContent=canUpload()?"将这张图片附给 ChatGPT,并在下一轮识图。":"当前客户端可预览图片,但未提供 ChatGPT 图片附件接口。";
}
async function receive(result){
if(!result || !Array.isArray(result.content)) return;
const img=result.content.find(item=>item?.type==="image" && ["image/png","image/jpeg","image/webp","image/gif"].includes(item.mimeType));
// 宿主更新 widgetState 时也会发布 globals;同一图像不能中断正在进行的上传。
if(!result.isError && img && lastImage?.data===img.data && lastImage?.mimeType===img.mimeType) return;
lastImage=result.isError?null:img;
const version=++generation;
selected=null; preview.hidden=true; share.disabled=true;
if(result.isError){ status.textContent="图片工具返回错误,未上传图片。"; return; }
if(!img || typeof img.data!=="string" || img.data.length>28000000){status.textContent="没有可显示的图片,或图片超过 20 MB。";return;}
try{
const bytes=Uint8Array.from(atob(img.data),char=>char.charCodeAt(0));
if(!bytes.length || bytes.length>20*1024*1024) throw Error("size");
const digest=await crypto.subtle.digest("SHA-256",bytes);
if(version!==generation) return;
const key=img.mimeType+":"+Array.from(new Uint8Array(digest),n=>n.toString(16).padStart(2,"0")).join("");
const previous=window.openai?.widgetState;
const saved=previous?.privateContent?.imageKey===key && previous?.imageIds?.length===1 ? previous.imageIds[0] : null;
selected={bytes,key,mimeType:img.mimeType,fileId:typeof saved==="string"?saved:null};
preview.src="data:"+img.mimeType+";base64,"+img.data; preview.hidden=false; ready();
notify("ui/notifications/size-changed",{height:580});
}catch{ if(version===generation) status.textContent="图片解码失败,未上传图片。"; }
}
share.addEventListener("click",async()=>{
if(busy || !selected || !canUpload()) return;
busy=true;share.disabled=true;const image=selected,version=generation;
status.textContent="正在附加图片…";
try{
if(!image.fileId){
const ext={"image/png":"png","image/jpeg":"jpg","image/webp":"webp","image/gif":"gif"}[image.mimeType];
const uploaded=await window.openai.uploadFile(new File([image.bytes],"agentdock-image."+ext,{type:image.mimeType}));
if(typeof uploaded?.fileId!=="string" || !uploaded.fileId) throw Error("missing file ID");
image.fileId=uploaded.fileId;
}
// 上传途中换图时,不把旧图片绑定到新的工具结果。
if(version!==generation) return;
window.openai.setWidgetState({modelContent:"用户选择了本次 AgentDock 工具返回的图片。请直接观察所附图片,不要依据元数据猜测。",privateContent:{imageKey:image.key},imageIds:[image.fileId]});
status.textContent="图片已附加,可在下一轮直接识图。";
if(typeof window.openai.sendFollowUpMessage==="function"){
await window.openai.sendFollowUpMessage({prompt:"请直接观察刚刚附加的 AgentDock 图片,回答我上一条消息中的看图问题;若上一条没有具体问题,简要描述可见内容。不要重新调用工具或用 OCR 代替看图。",scrollToBottom:false});
}
}catch{status.textContent="附加图片或继续对话失败,请重试。";}
finally{busy=false;share.disabled=!selected || !canUpload();if(version!==generation)ready();}
});
function fromGlobals(){
const meta=window.openai?.toolResponseMetadata;
const result=meta?.mcp_tool_result || meta?.call_tool_result;
if(result) receive(result); else ready();
}
window.addEventListener("openai:set_globals",fromGlobals);
window.addEventListener("message",event=>{
if(event.source!==window.parent || event.data?.jsonrpc!=="2.0") return;
const message=event.data;
if(message.method==="ui/notifications/tool-result") receive(message.params);
if(message.id==="image-init" && message.result) notify("ui/notifications/initialized",{});
});
window.parent.postMessage({jsonrpc:"2.0",id:"image-init",method:"ui/initialize",params:{appInfo:{name:"agentdock-image",version:"1.0.0"},appCapabilities:{},protocolVersion:"2026-01-26"}},"*");
fromGlobals();
})();
</script></body></html>
82 changes: 82 additions & 0 deletions mcpapps/image.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import vm from 'node:vm';
import { webcrypto } from 'node:crypto';

const html = readFileSync(new URL('./image.html', import.meta.url), 'utf8');
const source = html.match(/<script>([\s\S]*?)<\/script>/)[1];
const result = { content: [{ type:'image', mimeType:'image/png', data:'AQID' }] };
const settle = () => new Promise(resolve => setTimeout(resolve, 20));
function mount(api = {}) {
const events = {}, elements = {}, messages = [];
for (const id of ['preview','status','share']) elements[id] = { hidden:false, disabled:false, textContent:'', addEventListener(name, cb) { this[name]=cb; } };
const parent = { postMessage(message) { messages.push(message); } };
const win = { parent, openai:api, addEventListener(name,cb) { events[name]=cb; } };
vm.runInNewContext(source, { window:win, document:{ getElementById:id=>elements[id] }, atob, Uint8Array, File, crypto:webcrypto, console, setTimeout, clearTimeout });
return { elements, api, messages, globals(){events['openai:set_globals']();}, receive(value, sender=parent) { events.message({source:sender,data:{jsonrpc:'2.0',method:'ui/notifications/tool-result',params:value}}); } };
}

test('preserves pixels, uploads only on selection and includes real file ID before follow-up', async () => {
const calls=[];
const ui=mount({uploadFile:async file=>{ calls.push(['upload',Array.from(new Uint8Array(await file.arrayBuffer())),file.type]); return {fileId:'file-real'}; }, setWidgetState:state=>calls.push(['state',state]),sendFollowUpMessage:async()=>calls.push(['follow'])});
ui.receive(result); await settle();
assert.equal(calls.length,0);
assert.equal(ui.elements.preview.src,'data:image/png;base64,AQID');
await ui.elements.share.click();
assert.deepEqual(calls[0],['upload',[1,2,3],'image/png']);
assert.deepEqual(Array.from(calls[1][1].imageIds),['file-real']);
assert.equal(calls[2][0],'follow');
await ui.elements.share.click();
assert.equal(calls.filter(c=>c[0]==='upload').length,1);
});

test('host envelope can hydrate initial image, without automatically uploading it', async () => {
const ui=mount({toolResponseMetadata:{mcp_tool_result:result}}); await settle();
assert.equal(ui.elements.preview.src,'data:image/png;base64,AQID');
assert.equal(ui.elements.share.disabled,true);
});

test('untrusted frames, errors and active content are not images', async () => {
const ui=mount(); ui.receive(result,{}); await settle();
assert.equal(ui.elements.preview.src,undefined);
ui.receive({isError:true,...result}); await settle();
assert.equal(ui.elements.preview.src,undefined);
ui.receive({content:[{type:'image',mimeType:'image/svg+xml',data:'AQID'}]}); await settle();
assert.equal(ui.elements.preview.src,undefined);
});

test('upload errors do not claim success or send a follow-up', async () => {
let sent=false;
const ui=mount({uploadFile:async()=>{throw Error('private upstream detail');},setWidgetState:()=>{},sendFollowUpMessage:async()=>{sent=true;}});
ui.receive(result); await settle(); await ui.elements.share.click();
assert.equal(sent,false); assert.match(ui.elements.status.textContent,/失败/);
assert.ok(!ui.elements.status.textContent.includes('private upstream'));
assert.equal(ui.elements.share.disabled,false);
});

test('concurrent clicks cannot duplicate uploads and no fake file IDs are accepted', async () => {
let count=0,finish;
const ui=mount({uploadFile:()=>{count++;return new Promise(resolve=>finish=resolve);},setWidgetState:()=>{throw Error('must not store absent file');}});
ui.receive(result); await settle(); const first=ui.elements.share.click(); await ui.elements.share.click();
assert.equal(count,1); finish({}); await first;
assert.match(ui.elements.status.textContent,/失败/);
});

test('host globals updates during upload do not discard the selected image', async () => {
let finish,state;
const ui=mount({toolResponseMetadata:{mcp_tool_result:result},uploadFile:()=>new Promise(resolve=>finish=resolve),setWidgetState:value=>{state=value;}});
await settle(); const action=ui.elements.share.click(); ui.globals(); await settle();
finish({fileId:'file-real'}); await action;
assert.deepEqual(Array.from(state.imageIds),['file-real']);
});

test('switching images during upload never attaches stale pixels', async () => {
let finish,stored=false;
const ui=mount({uploadFile:()=>new Promise(resolve=>finish=resolve),setWidgetState:()=>{stored=true;}});
ui.receive(result); await settle(); const action=ui.elements.share.click();
ui.receive({content:[{type:'image',mimeType:'image/png',data:'BAUG'}]}); await settle();
finish({fileId:'file-old'}); await action;
assert.equal(stored,false);
assert.match(ui.elements.status.textContent,/将这张图片/);
});
26 changes: 26 additions & 0 deletions mcpapps/image_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package mcpapps

import (
protocol "github.com/uvwt/agentdock-protocol"
"strings"
"testing"
)

func TestImageViewIsSharedAndHasStableContract(t *testing.T) {
contract, ok := protocol.UIResourceContract("ui://agentdock/view-image/v1.html")
if !ok || contract != "agentdock.view-image.v1" {
t.Fatal("missing shared image contract")
}
html := HTML("view_image", "Image")
for _, marker := range []string{"imageIds", "uploadFile", "ui/notifications/tool-result", "image-init"} {
if !strings.Contains(html, marker) {
t.Fatalf("missing image behavior: %s", marker)
}
}
if strings.Contains(html, "nexusdock-image") {
t.Fatal("shared renderer is tied to NexusDock")
}
if strings.Contains(HTML("agentdock_context", "Context"), "image-init") {
t.Fatal("changed another view")
}
}
4 changes: 4 additions & 0 deletions protocol.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ const (
DynamicMCPUIResourceURI = "ui://agentdock/dynamic-mcp"
ArtifactUIResourceURI = "ui://agentdock/artifact"
ACPStatusUIResourceURI = "ui://agentdock/acp-status"
ImageUIResourceURI = "ui://agentdock/view-image/v1.html"
)

const (
Expand All @@ -49,6 +50,7 @@ const (
DynamicMCPUIContract = "agentdock.dynamic-mcp.v1"
ArtifactUIContract = "agentdock.artifact.v1"
ACPStatusUIContract = "agentdock.acp-status.v1"
ImageUIContract = "agentdock.view-image.v1"
)

const MCPAppMIMEType = "text/html;profile=mcp-app"
Expand All @@ -57,6 +59,8 @@ const MCPAppMIMEType = "text/html;profile=mcp-app"
// URIs identify resources and remain stable; renderer compatibility evolves through the contract string.
func UIResourceContract(uri string) (string, bool) {
switch uri {
case ImageUIResourceURI:
return ImageUIContract, true
case ContextUIResourceURI:
return ContextUIContract, true
case TaskProgressUIResourceURI:
Expand Down