支持从输出文本 Markdown 里解析图片链接

src/lib/agentApi.test.ts

@@ -212,6 +212,64 @@ describe('callAgentResponsesApi', () => {
     }])
   })
 
+  it('extracts Markdown base64 images with arbitrary alt text and removes them from Agent text', async () => {
+    vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify({
+      output: [{
+        type: 'message',
+        content: [{
+          type: 'output_text',
+          text: [
+            '## 生成完成',
+            '',
+            '这是 **说明文字** 和 [帮助链接](https://docs.example.com)。',
+            '',
+            '![不是固定名称](data:image/jpeg;base64,aW1hZ2U=)',
+            '',
+            '- 列表内容与 `inline code` 应保留。',
+            '',
+            '```text',
+            '![代码示例](data:image/png;base64,example)',
+            '```',
+            '',
+            '结尾文字。',
+          ].join('\n'),
+        }],
+      }],
+    }), {
+      status: 200,
+      headers: { 'Content-Type': 'application/json' },
+    }))
+    const profile = createDefaultOpenAIProfile({
+      apiKey: 'test-key',
+      apiMode: 'responses',
+    })
+
+    const result = await callAgentResponsesApi({
+      settings: DEFAULT_SETTINGS,
+      profile,
+      params: DEFAULT_PARAMS,
+      input: [{ role: 'user', content: [{ type: 'input_text', text: 'prompt' }] }],
+    })
+
+    expect(result.text).toBe([
+      '## 生成完成',
+      '',
+      '这是 **说明文字** 和 [帮助链接](https://docs.example.com)。',
+      '',
+      '- 列表内容与 `inline code` 应保留。',
+      '',
+      '```text',
+      '![代码示例](data:image/png;base64,example)',
+      '```',
+      '',
+      '结尾文字。',
+    ].join('\n'))
+    expect(result.images).toEqual([{
+      dataUrl: 'data:image/jpeg;base64,aW1hZ2U=',
+      actualParams: {},
+    }])
+  })
+
   it('stops reading a stream when the caller aborts after output starts', async () => {
     const streamBody = [
       'data: {"type":"response.output_text.delta","delta":"Hel"}',

src/lib/agentApi.ts

@@ -3,13 +3,15 @@ import { buildApiUrl, readClientDevProxyConfig, shouldUseApiProxy } from './devP
 import { appendStreamingFormatHint, getApiErrorMessage, getResponsesImageResultBase64, maybeAppendStreamingHint, MIME_MAP, normalizeBase64Image, pickActualParams, PROMPT_REWRITE_GUARD_PREFIX } from './imageApiShared'
 import { normalizeResponsesOutputItems } from './responsesOutputState'
 import { isEventStreamResponse, readJsonServerSentEvents, throwIfAborted } from './serverSentEvents'
+import { removeMarkdownImages, resolveMarkdownImages } from './markdownImages'
 
 export interface AgentApiResultImage {
   toolCallId?: string
   action?: string
   dataUrl: string
   actualParams?: Partial<TaskParams>
   revisedPrompt?: string
+  rawImageUrl?: string
 }
 
 export interface AgentApiImageToolFailure {
@@ -23,6 +25,7 @@ export interface AgentApiResult {
   images: AgentApiResultImage[]
   outputItems: ResponsesApiResponse['output']
   rawResponsePayload?: string
+  unresolvedImageUrls?: string[]
 }
 
 const AGENT_IMAGE_INSTRUCTIONS = [
@@ -343,14 +346,14 @@ function extractText(payload: ResponsesApiResponse) {
     if (item.type !== 'message') continue
     for (const part of item.content ?? []) {
       if ((part.type === 'output_text' || part.type === 'text') && typeof part.text === 'string') {
-        chunks.push(applyUrlCitations(part.text, part.annotations))
+        chunks.push(removeMarkdownImages(applyUrlCitations(part.text, part.annotations)))
       } else if (part.type === 'refusal' && typeof part.refusal === 'string') {
         chunks.push(part.refusal)
       }
     }
   }
 
-  return chunks.join('\n').trim()
+  return chunks.filter(Boolean).join('\n').trim()
 }
 
 function decodeXmlText(text: string) {
@@ -376,24 +379,42 @@ function parseAgentConversationTitleXml(text: string) {
   return `${chars.slice(0, AGENT_TITLE_MAX_LENGTH - 3).join('')}...`
 }
 
-function extractImages(payload: ResponsesApiResponse, fallbackMime: string): AgentApiResultImage[] {
+async function extractImages(payload: ResponsesApiResponse, fallbackMime: string, signal?: AbortSignal): Promise<{
+  images: AgentApiResultImage[]
+  unresolvedImageUrls: string[]
+}> {
   const images: AgentApiResultImage[] = []
+  const unresolvedImageUrls: string[] = []
 
   for (const item of payload.output ?? []) {
-    if (item.type !== 'image_generation_call') continue
-
-    const b64 = getResponsesImageResultBase64(item.result)
-    if (!b64) continue
-    images.push({
-      toolCallId: typeof item.id === 'string' ? item.id : undefined,
-      action: typeof item.action === 'string' ? item.action : undefined,
-      dataUrl: normalizeBase64Image(b64, fallbackMime),
-      actualParams: pickActualParams(item),
-      revisedPrompt: typeof item.revised_prompt === 'string' ? item.revised_prompt : undefined,
-    })
+    if (item.type === 'image_generation_call') {
+      const b64 = getResponsesImageResultBase64(item.result)
+      if (b64) {
+        images.push({
+          toolCallId: typeof item.id === 'string' ? item.id : undefined,
+          action: typeof item.action === 'string' ? item.action : undefined,
+          dataUrl: normalizeBase64Image(b64, fallbackMime),
+          actualParams: pickActualParams(item),
+          revisedPrompt: typeof item.revised_prompt === 'string' ? item.revised_prompt : undefined,
+        })
+      }
+      continue
+    }
+
+    if (item.type !== 'message') continue
+    for (const part of item.content ?? []) {
+      if ((part.type !== 'output_text' && part.type !== 'text') || typeof part.text !== 'string') continue
+      const markdownImages = await resolveMarkdownImages(part.text, fallbackMime, signal)
+      images.push(...markdownImages.images.map((image) => ({
+        dataUrl: image.dataUrl,
+        actualParams: pickActualParams(item),
+        rawImageUrl: image.rawImageUrl,
+      })))
+      unresolvedImageUrls.push(...markdownImages.unresolvedImageUrls)
+    }
   }
 
-  return images
+  return { images, unresolvedImageUrls: [...new Set(unresolvedImageUrls)] }
 }
 
 function extractImageFromOutputItem(item: ResponsesOutputItem, fallbackMime: string): AgentApiResultImage | null {
@@ -577,13 +598,15 @@ async function parseAgentStreamResponse(
   const payload: ResponsesApiResponse | null = completedPayload ?? (outputItems.length ? { output: outputItems } : null)
   if (!payload) throw new Error('Agent 流式接口未返回最终响应数据')
 
-  const text = extractText(payload) || streamedText.trim()
+  const text = extractText(payload) || removeMarkdownImages(streamedText)
+  const extractedImages = await extractImages(payload, mime, signal)
   return {
     responseId: payload.id,
     text,
-    images: extractImages(payload, mime),
+    images: extractedImages.images,
     outputItems: payload.output ?? [],
     rawResponsePayload: JSON.stringify(payload, null, 2),
+    ...(extractedImages.unresolvedImageUrls.length ? { unresolvedImageUrls: extractedImages.unresolvedImageUrls } : {}),
   }
 }
 
@@ -645,12 +668,14 @@ export async function callAgentResponsesApi(opts: {
     const payload = normalizeResponsePayload(rawPayload)
     if (!payload) throw new Error('Agent 接口返回格式无效')
     throwIfAborted(controller.signal, signal)
+    const extractedImages = await extractImages(payload, mime, controller.signal)
     return {
       responseId: payload.id,
       text: extractText(payload),
-      images: extractImages(payload, mime),
+      images: extractedImages.images,
       outputItems: payload.output,
       rawResponsePayload: JSON.stringify(payload, null, 2),
+      ...(extractedImages.unresolvedImageUrls.length ? { unresolvedImageUrls: extractedImages.unresolvedImageUrls } : {}),
     }
   } finally {
     clearTimeout(timeoutId)
@@ -721,6 +746,7 @@ export interface BatchImageCallResult {
   image: AgentApiResultImage | null
   error: string | null
   rawResponsePayload?: string
+  unresolvedImageUrls?: string[]
 }
 
 /**
@@ -819,6 +845,7 @@ export async function callBatchImageSingle(opts: {
       await onImageToolStarted?.()
       let completedImage: AgentApiResultImage | null = null
       let rawPayload: string | undefined
+      let unresolvedImageUrls: string[] = []
 
       await readJsonServerSentEvents(response, async (event) => {
         const type = getStringValue(event, 'type')
@@ -851,10 +878,12 @@ export async function callBatchImageSingle(opts: {
           const payload = getStreamResponsePayload(event)
           if (payload) rawPayload = JSON.stringify(payload, null, 2)
           if (!completedImage && payload) {
-            const images = extractImages(payload, mime)
-            if (images.length > 0) {
-              completedImage = images[0]
-              await onImageToolCompleted?.(completedImage)
+            const extractedImages = await extractImages(payload, mime, controller.signal)
+            unresolvedImageUrls = extractedImages.unresolvedImageUrls
+            const image = extractedImages.images[0]
+            if (image) {
+              completedImage = image
+              await onImageToolCompleted?.(image)
             }
           }
         }
@@ -869,20 +898,22 @@ export async function callBatchImageSingle(opts: {
         image: completedImage,
         error: completedImage ? null : '流式响应未返回图片',
         rawResponsePayload: rawPayload,
+        ...(unresolvedImageUrls.length ? { unresolvedImageUrls } : {}),
       }
     }
 
     // Non-streaming
     const payload = normalizeResponsePayload(await response.json())
     if (!payload) throw new Error('图像接口返回格式无效')
-    const images = extractImages(payload, mime)
-    const image = images[0] ?? null
+    const extractedImages = await extractImages(payload, mime, controller.signal)
+    const image = extractedImages.images[0] ?? null
     if (image) await onImageToolCompleted?.(image)
     return {
       batchItemId,
       image,
       error: image ? null : '接口未返回图片数据',
       rawResponsePayload: JSON.stringify(payload, null, 2),
+      ...(extractedImages.unresolvedImageUrls.length ? { unresolvedImageUrls: extractedImages.unresolvedImageUrls } : {}),
     }
   } catch (err) {
     if (controller.signal.aborted || signal?.aborted) {

src/lib/api.test.ts

@@ -608,6 +608,43 @@ describe('callImageApi', () => {
     })
   })
 
+  it('parses Markdown base64 images from Responses API gallery messages', async () => {
+    vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify({
+      output: [{
+        type: 'message',
+        content: [{
+          type: 'output_text',
+          text: [
+            '结果如下，详见 [生成说明](https://docs.example.com)。',
+            '',
+            '![任意替代文本](data:image/jpeg;base64,aW1hZ2U=)',
+            '',
+            '> 这是一段 **引用文本**，不是图片。',
+            '',
+            '![第二张图](data:image/png;base64,c2Vjb25k)',
+            '',
+            '`![行内示例](data:image/webp;base64,example)` 不应被视为图片。',
+          ].join('\n'),
+        }],
+      }],
+    }), {
+      status: 200,
+      headers: { 'Content-Type': 'application/json' },
+    }))
+
+    const result = await callImageApi({
+      settings: { ...DEFAULT_SETTINGS, apiKey: 'test-key', apiMode: 'responses' },
+      prompt: 'prompt',
+      params: { ...DEFAULT_PARAMS },
+      inputImageDataUrls: [],
+    })
+
+    expect(result.images).toEqual([
+      'data:image/jpeg;base64,aW1hZ2U=',
+      'data:image/png;base64,c2Vjb25k',
+    ])
+  })
+
   it('keeps Responses API stream output item images when completed response omits result', async () => {
     const streamBody = [
       'data: {"type":"response.output_item.done","item":{"id":"img-call-1","type":"image_generation_call","status":"generating","action":"generate","result":"ZmluYWw=","size":"1024x1024"},"output_index":0}',

src/lib/markdownImages.test.ts

@@ -0,0 +1,79 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { extractMarkdownImageSources, removeMarkdownImages, resolveMarkdownImages } from './markdownImages'
+
+describe('markdownImages', () => {
+  afterEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  it('extracts only supported images among surrounding rich Markdown content', () => {
+    const text = [
+      '# 生成结果',
+      '',
+      '这是 **加粗**、*斜体* 和 [说明链接](https://docs.example.com) 前的文字。',
+      '',
+      '![任意名称](data:image/jpeg;base64,aW1hZ2U=)',
+      '',
+      '- 列表中的 `inline code` 与 [普通链接](https://example.com/page) 必须保留。',
+      '- ![另一个](<https://example.com/image.png> "标题")',
+      '',
+      '```text',
+      '![代码块内容](data:image/png;base64,should-not-be-special)',
+      '```',
+      '',
+      '尾部 `![行内示例](data:image/webp;base64,example)` 和 ![忽略](javascript:alert(1)) 也必须保留。',
+    ].join('\n')
+
+    expect(extractMarkdownImageSources(text)).toEqual([
+      {
+        url: 'data:image/jpeg;base64,aW1hZ2U=',
+        markdown: '![任意名称](data:image/jpeg;base64,aW1hZ2U=)',
+      },
+      {
+        url: 'https://example.com/image.png',
+        markdown: '![另一个](<https://example.com/image.png> "标题")',
+      },
+    ])
+    expect(removeMarkdownImages(text)).toBe([
+      '# 生成结果',
+      '',
+      '这是 **加粗**、*斜体* 和 [说明链接](https://docs.example.com) 前的文字。',
+      '',
+      '- 列表中的 `inline code` 与 [普通链接](https://example.com/page) 必须保留。',
+      '-',
+      '',
+      '```text',
+      '![代码块内容](data:image/png;base64,should-not-be-special)',
+      '```',
+      '',
+      '尾部 `![行内示例](data:image/webp;base64,example)` 和 ![忽略](javascript:alert(1)) 也必须保留。',
+    ].join('\n'))
+  })
+
+  it('keeps an unreachable remote image URL while returning other resolved images', async () => {
+    vi.spyOn(globalThis, 'fetch').mockRejectedValue(new TypeError('Failed to fetch'))
+
+    await expect(resolveMarkdownImages([
+      '![Base64](data:image/png;base64,aW1hZ2U=)',
+      '![远程图](https://example.com/image.png)',
+    ].join('\n'), 'image/jpeg')).resolves.toEqual({
+      images: [{ dataUrl: 'data:image/png;base64,aW1hZ2U=' }],
+      unresolvedImageUrls: ['https://example.com/image.png'],
+    })
+  })
+
+  it('downloads remote Markdown images as local data URLs', async () => {
+    vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(new Uint8Array([1, 2, 3]), {
+      status: 200,
+      headers: { 'Content-Type': 'image/webp' },
+    }))
+
+    await expect(resolveMarkdownImages('![远程图](https://example.com/image.webp)', 'image/png')).resolves.toEqual({
+      images: [{
+        dataUrl: 'data:image/webp;base64,AQID',
+        rawImageUrl: 'https://example.com/image.webp',
+      }],
+      unresolvedImageUrls: [],
+    })
+  })
+})