diff --git a/src/lib/agentApi.test.ts b/src/lib/agentApi.test.ts index a9c05b310..ff0a7fbbf 100644 --- a/src/lib/agentApi.test.ts +++ b/src/lib/agentApi.test.ts @@ -212,6 +212,64 @@ describe('callAgentResponsesApi', () => { }]) }) + it('extracts Markdown base64 images with arbitrary alt text and removes them from Agent text', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify({ + output: [{ + type: 'message', + content: [{ + type: 'output_text', + text: [ + '## 生成完成', + '', + '这是 **说明文字** 和 [帮助链接](https://docs.example.com)。', + '', + '![不是固定名称](data:image/jpeg;base64,aW1hZ2U=)', + '', + '- 列表内容与 `inline code` 应保留。', + '', + '```text', + '![代码示例](data:image/png;base64,example)', + '```', + '', + '结尾文字。', + ].join('\n'), + }], + }], + }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + })) + const profile = createDefaultOpenAIProfile({ + apiKey: 'test-key', + apiMode: 'responses', + }) + + const result = await callAgentResponsesApi({ + settings: DEFAULT_SETTINGS, + profile, + params: DEFAULT_PARAMS, + input: [{ role: 'user', content: [{ type: 'input_text', text: 'prompt' }] }], + }) + + expect(result.text).toBe([ + '## 生成完成', + '', + '这是 **说明文字** 和 [帮助链接](https://docs.example.com)。', + '', + '- 列表内容与 `inline code` 应保留。', + '', + '```text', + '![代码示例](data:image/png;base64,example)', + '```', + '', + '结尾文字。', + ].join('\n')) + expect(result.images).toEqual([{ + dataUrl: 'data:image/jpeg;base64,aW1hZ2U=', + actualParams: {}, + }]) + }) + it('stops reading a stream when the caller aborts after output starts', async () => { const streamBody = [ 'data: {"type":"response.output_text.delta","delta":"Hel"}', diff --git a/src/lib/agentApi.ts b/src/lib/agentApi.ts index 5fdf13a7f..e1c9071e9 100644 --- a/src/lib/agentApi.ts +++ b/src/lib/agentApi.ts @@ -3,6 +3,7 @@ import { buildApiUrl, readClientDevProxyConfig, shouldUseApiProxy } from './devP import { appendStreamingFormatHint, getApiErrorMessage, getResponsesImageResultBase64, maybeAppendStreamingHint, MIME_MAP, normalizeBase64Image, pickActualParams, PROMPT_REWRITE_GUARD_PREFIX } from './imageApiShared' import { normalizeResponsesOutputItems } from './responsesOutputState' import { isEventStreamResponse, readJsonServerSentEvents, throwIfAborted } from './serverSentEvents' +import { removeMarkdownImages, resolveMarkdownImages } from './markdownImages' export interface AgentApiResultImage { toolCallId?: string @@ -10,6 +11,7 @@ export interface AgentApiResultImage { dataUrl: string actualParams?: Partial revisedPrompt?: string + rawImageUrl?: string } export interface AgentApiImageToolFailure { @@ -23,6 +25,7 @@ export interface AgentApiResult { images: AgentApiResultImage[] outputItems: ResponsesApiResponse['output'] rawResponsePayload?: string + unresolvedImageUrls?: string[] } const AGENT_IMAGE_INSTRUCTIONS = [ @@ -343,14 +346,14 @@ function extractText(payload: ResponsesApiResponse) { if (item.type !== 'message') continue for (const part of item.content ?? []) { if ((part.type === 'output_text' || part.type === 'text') && typeof part.text === 'string') { - chunks.push(applyUrlCitations(part.text, part.annotations)) + chunks.push(removeMarkdownImages(applyUrlCitations(part.text, part.annotations))) } else if (part.type === 'refusal' && typeof part.refusal === 'string') { chunks.push(part.refusal) } } } - return chunks.join('\n').trim() + return chunks.filter(Boolean).join('\n').trim() } function decodeXmlText(text: string) { @@ -376,24 +379,42 @@ function parseAgentConversationTitleXml(text: string) { return `${chars.slice(0, AGENT_TITLE_MAX_LENGTH - 3).join('')}...` } -function extractImages(payload: ResponsesApiResponse, fallbackMime: string): AgentApiResultImage[] { +async function extractImages(payload: ResponsesApiResponse, fallbackMime: string, signal?: AbortSignal): Promise<{ + images: AgentApiResultImage[] + unresolvedImageUrls: string[] +}> { const images: AgentApiResultImage[] = [] + const unresolvedImageUrls: string[] = [] for (const item of payload.output ?? []) { - if (item.type !== 'image_generation_call') continue - - const b64 = getResponsesImageResultBase64(item.result) - if (!b64) continue - images.push({ - toolCallId: typeof item.id === 'string' ? item.id : undefined, - action: typeof item.action === 'string' ? item.action : undefined, - dataUrl: normalizeBase64Image(b64, fallbackMime), - actualParams: pickActualParams(item), - revisedPrompt: typeof item.revised_prompt === 'string' ? item.revised_prompt : undefined, - }) + if (item.type === 'image_generation_call') { + const b64 = getResponsesImageResultBase64(item.result) + if (b64) { + images.push({ + toolCallId: typeof item.id === 'string' ? item.id : undefined, + action: typeof item.action === 'string' ? item.action : undefined, + dataUrl: normalizeBase64Image(b64, fallbackMime), + actualParams: pickActualParams(item), + revisedPrompt: typeof item.revised_prompt === 'string' ? item.revised_prompt : undefined, + }) + } + continue + } + + if (item.type !== 'message') continue + for (const part of item.content ?? []) { + if ((part.type !== 'output_text' && part.type !== 'text') || typeof part.text !== 'string') continue + const markdownImages = await resolveMarkdownImages(part.text, fallbackMime, signal) + images.push(...markdownImages.images.map((image) => ({ + dataUrl: image.dataUrl, + actualParams: pickActualParams(item), + rawImageUrl: image.rawImageUrl, + }))) + unresolvedImageUrls.push(...markdownImages.unresolvedImageUrls) + } } - return images + return { images, unresolvedImageUrls: [...new Set(unresolvedImageUrls)] } } function extractImageFromOutputItem(item: ResponsesOutputItem, fallbackMime: string): AgentApiResultImage | null { @@ -577,13 +598,15 @@ async function parseAgentStreamResponse( const payload: ResponsesApiResponse | null = completedPayload ?? (outputItems.length ? { output: outputItems } : null) if (!payload) throw new Error('Agent 流式接口未返回最终响应数据') - const text = extractText(payload) || streamedText.trim() + const text = extractText(payload) || removeMarkdownImages(streamedText) + const extractedImages = await extractImages(payload, mime, signal) return { responseId: payload.id, text, - images: extractImages(payload, mime), + images: extractedImages.images, outputItems: payload.output ?? [], rawResponsePayload: JSON.stringify(payload, null, 2), + ...(extractedImages.unresolvedImageUrls.length ? { unresolvedImageUrls: extractedImages.unresolvedImageUrls } : {}), } } @@ -645,12 +668,14 @@ export async function callAgentResponsesApi(opts: { const payload = normalizeResponsePayload(rawPayload) if (!payload) throw new Error('Agent 接口返回格式无效') throwIfAborted(controller.signal, signal) + const extractedImages = await extractImages(payload, mime, controller.signal) return { responseId: payload.id, text: extractText(payload), - images: extractImages(payload, mime), + images: extractedImages.images, outputItems: payload.output, rawResponsePayload: JSON.stringify(payload, null, 2), + ...(extractedImages.unresolvedImageUrls.length ? { unresolvedImageUrls: extractedImages.unresolvedImageUrls } : {}), } } finally { clearTimeout(timeoutId) @@ -721,6 +746,7 @@ export interface BatchImageCallResult { image: AgentApiResultImage | null error: string | null rawResponsePayload?: string + unresolvedImageUrls?: string[] } /** @@ -819,6 +845,7 @@ export async function callBatchImageSingle(opts: { await onImageToolStarted?.() let completedImage: AgentApiResultImage | null = null let rawPayload: string | undefined + let unresolvedImageUrls: string[] = [] await readJsonServerSentEvents(response, async (event) => { const type = getStringValue(event, 'type') @@ -851,10 +878,12 @@ export async function callBatchImageSingle(opts: { const payload = getStreamResponsePayload(event) if (payload) rawPayload = JSON.stringify(payload, null, 2) if (!completedImage && payload) { - const images = extractImages(payload, mime) - if (images.length > 0) { - completedImage = images[0] - await onImageToolCompleted?.(completedImage) + const extractedImages = await extractImages(payload, mime, controller.signal) + unresolvedImageUrls = extractedImages.unresolvedImageUrls + const image = extractedImages.images[0] + if (image) { + completedImage = image + await onImageToolCompleted?.(image) } } } @@ -869,20 +898,22 @@ export async function callBatchImageSingle(opts: { image: completedImage, error: completedImage ? null : '流式响应未返回图片', rawResponsePayload: rawPayload, + ...(unresolvedImageUrls.length ? { unresolvedImageUrls } : {}), } } // Non-streaming const payload = normalizeResponsePayload(await response.json()) if (!payload) throw new Error('图像接口返回格式无效') - const images = extractImages(payload, mime) - const image = images[0] ?? null + const extractedImages = await extractImages(payload, mime, controller.signal) + const image = extractedImages.images[0] ?? null if (image) await onImageToolCompleted?.(image) return { batchItemId, image, error: image ? null : '接口未返回图片数据', rawResponsePayload: JSON.stringify(payload, null, 2), + ...(extractedImages.unresolvedImageUrls.length ? { unresolvedImageUrls: extractedImages.unresolvedImageUrls } : {}), } } catch (err) { if (controller.signal.aborted || signal?.aborted) { diff --git a/src/lib/api.test.ts b/src/lib/api.test.ts index 5d693d09d..1d75fd079 100644 --- a/src/lib/api.test.ts +++ b/src/lib/api.test.ts @@ -608,6 +608,43 @@ describe('callImageApi', () => { }) }) + it('parses Markdown base64 images from Responses API gallery messages', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify({ + output: [{ + type: 'message', + content: [{ + type: 'output_text', + text: [ + '结果如下，详见 [生成说明](https://docs.example.com)。', + '', + '![任意替代文本](data:image/jpeg;base64,aW1hZ2U=)', + '', + '> 这是一段 **引用文本**，不是图片。', + '', + '![第二张图](data:image/png;base64,c2Vjb25k)', + '', + '`![行内示例](data:image/webp;base64,example)` 不应被视为图片。', + ].join('\n'), + }], + }], + }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + })) + + const result = await callImageApi({ + settings: { ...DEFAULT_SETTINGS, apiKey: 'test-key', apiMode: 'responses' }, + prompt: 'prompt', + params: { ...DEFAULT_PARAMS }, + inputImageDataUrls: [], + }) + + expect(result.images).toEqual([ + 'data:image/jpeg;base64,aW1hZ2U=', + 'data:image/png;base64,c2Vjb25k', + ]) + }) + it('keeps Responses API stream output item images when completed response omits result', async () => { const streamBody = [ 'data: {"type":"response.output_item.done","item":{"id":"img-call-1","type":"image_generation_call","status":"generating","action":"generate","result":"ZmluYWw=","size":"1024x1024"},"output_index":0}', diff --git a/src/lib/markdownImages.test.ts b/src/lib/markdownImages.test.ts new file mode 100644 index 000000000..3438afc26 --- /dev/null +++ b/src/lib/markdownImages.test.ts @@ -0,0 +1,79 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { extractMarkdownImageSources, removeMarkdownImages, resolveMarkdownImages } from './markdownImages' + +describe('markdownImages', () => { + afterEach(() => { + vi.restoreAllMocks() + }) + + it('extracts only supported images among surrounding rich Markdown content', () => { + const text = [ + '# 生成结果', + '', + '这是 **加粗**、*斜体* 和 [说明链接](https://docs.example.com) 前的文字。', + '', + '![任意名称](data:image/jpeg;base64,aW1hZ2U=)', + '', + '- 列表中的 `inline code` 与 [普通链接](https://example.com/page) 必须保留。', + '- ![另一个]( "标题")', + '', + '```text', + '![代码块内容](data:image/png;base64,should-not-be-special)', + '```', + '', + '尾部 `![行内示例](data:image/webp;base64,example)` 和 ![忽略](javascript:alert(1)) 也必须保留。', + ].join('\n') + + expect(extractMarkdownImageSources(text)).toEqual([ + { + url: 'data:image/jpeg;base64,aW1hZ2U=', + markdown: '![任意名称](data:image/jpeg;base64,aW1hZ2U=)', + }, + { + url: 'https://example.com/image.png', + markdown: '![另一个]( "标题")', + }, + ]) + expect(removeMarkdownImages(text)).toBe([ + '# 生成结果', + '', + '这是 **加粗**、*斜体* 和 [说明链接](https://docs.example.com) 前的文字。', + '', + '- 列表中的 `inline code` 与 [普通链接](https://example.com/page) 必须保留。', + '-', + '', + '```text', + '![代码块内容](data:image/png;base64,should-not-be-special)', + '```', + '', + '尾部 `![行内示例](data:image/webp;base64,example)` 和 ![忽略](javascript:alert(1)) 也必须保留。', + ].join('\n')) + }) + + it('keeps an unreachable remote image URL while returning other resolved images', async () => { + vi.spyOn(globalThis, 'fetch').mockRejectedValue(new TypeError('Failed to fetch')) + + await expect(resolveMarkdownImages([ + '![Base64](data:image/png;base64,aW1hZ2U=)', + '![远程图](https://example.com/image.png)', + ].join('\n'), 'image/jpeg')).resolves.toEqual({ + images: [{ dataUrl: 'data:image/png;base64,aW1hZ2U=' }], + unresolvedImageUrls: ['https://example.com/image.png'], + }) + }) + + it('downloads remote Markdown images as local data URLs', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(new Uint8Array([1, 2, 3]), { + status: 200, + headers: { 'Content-Type': 'image/webp' }, + })) + + await expect(resolveMarkdownImages('![远程图](https://example.com/image.webp)', 'image/png')).resolves.toEqual({ + images: [{ + dataUrl: 'data:image/webp;base64,AQID', + rawImageUrl: 'https://example.com/image.webp', + }], + unresolvedImageUrls: [], + }) + }) +}) diff --git a/src/lib/markdownImages.ts b/src/lib/markdownImages.ts new file mode 100644 index 000000000..3647d7dd3 --- /dev/null +++ b/src/lib/markdownImages.ts @@ -0,0 +1,59 @@ +import { fetchImageUrlAsDataUrl, isHttpUrl, normalizeBase64Image } from './imageApiShared' + +export interface MarkdownImageSource { + url: string + markdown: string +} + +export interface ResolvedMarkdownImages { + images: Array<{ dataUrl: string; rawImageUrl?: string }> + unresolvedImageUrls: string[] +} + +const MARKDOWN_IMAGE_PATTERN = /!\[[^\]]*\]\(\s*(?:<([^>\r\n]+)>|([^\s)\r\n]+))(?:\s+(?:"[^"]*"|'[^']*'|\([^)]*\)))?\s*\)/ +const MARKDOWN_IMAGE_OR_CODE_PATTERN = new RegExp(`\\x60{3}[\\s\\S]*?\\x60{3}|\\x60[^\\x60\\r\\n]*\\x60|${MARKDOWN_IMAGE_PATTERN.source}`, 'g') + +export function extractMarkdownImageSources(text: string): MarkdownImageSource[] { + const images: MarkdownImageSource[] = [] + + for (const match of text.matchAll(MARKDOWN_IMAGE_OR_CODE_PATTERN)) { + if (match[0].startsWith('\x60\x60\x60') || match[0].startsWith('\x60')) continue + const url = (match[1] ?? match[2] ?? '').trim() + if (!/^data:image\/[\w.+-]+;base64,/i.test(url) && !isHttpUrl(url)) continue + images.push({ url, markdown: match[0] }) + } + + return images +} + +export function removeMarkdownImages(text: string): string { + return text.replace(MARKDOWN_IMAGE_OR_CODE_PATTERN, (markdown) => { + if (markdown.startsWith('\x60\x60\x60') || markdown.startsWith('\x60')) return markdown + const [source] = extractMarkdownImageSources(markdown) + return source ? '' : markdown + }).replace(/\n{3,}/g, '\n\n').trim() +} + +export async function resolveMarkdownImages(text: string, fallbackMime: string, signal?: AbortSignal): Promise { + const images: Array<{ dataUrl: string; rawImageUrl?: string }> = [] + const unresolvedImageUrls: string[] = [] + + for (const source of extractMarkdownImageSources(text)) { + if (/^data:image\/[\w.+-]+;base64,/i.test(source.url)) { + images.push({ dataUrl: normalizeBase64Image(source.url, fallbackMime) }) + continue + } + + try { + images.push({ + dataUrl: await fetchImageUrlAsDataUrl(source.url, fallbackMime, signal), + rawImageUrl: source.url, + }) + } catch (err) { + console.warn('Markdown 图片链接下载失败，已保留原始链接', err) + unresolvedImageUrls.push(source.url) + } + } + + return { images, unresolvedImageUrls } +} diff --git a/src/lib/openaiCompatibleImageApi.ts b/src/lib/openaiCompatibleImageApi.ts index 6b32b1441..a9662e4f3 100644 --- a/src/lib/openaiCompatibleImageApi.ts +++ b/src/lib/openaiCompatibleImageApi.ts @@ -22,6 +22,7 @@ import { PROMPT_REWRITE_GUARD_PREFIX, } from './imageApiShared' import { isEventStreamResponse, readJsonServerSentEvents } from './serverSentEvents' +import { resolveMarkdownImages } from './markdownImages' import { prependCodexCliSizePrompt } from './size' function getStreamPartialImages(profile: ApiProfile): number { @@ -179,11 +180,19 @@ function createResponsesInput(prompt: string, inputImageDataUrls: string[], allo ] } -function parseResponsesImageResults(payload: ResponsesApiResponse, fallbackMime: string): Array<{ +type ResponsesImageResult = { image: string actualParams?: Partial revisedPrompt?: string -}> { + rawImageUrl?: string +} + +type ParsedResponsesImageResults = { + results: ResponsesImageResult[] + rawImageUrls: string[] +} + +async function parseResponsesImageResults(payload: ResponsesApiResponse, fallbackMime: string, signal?: AbortSignal): Promise { const output = payload.output if (!Array.isArray(output) || !output.length) { const err = new Error('接口未返回图片数据') @@ -191,28 +200,45 @@ function parseResponsesImageResults(payload: ResponsesApiResponse, fallbackMime: throw err } - const results: Array<{ image: string; actualParams?: Partial; revisedPrompt?: string }> = [] + const results: ResponsesImageResult[] = [] + const rawImageUrls: string[] = [] for (const item of output) { - if (item?.type !== 'image_generation_call') continue - - const b64 = getResponsesImageResultBase64(item.result) - if (b64) { - results.push({ - image: normalizeBase64Image(b64, fallbackMime), - actualParams: mergeActualParams(pickActualParams(item)), - revisedPrompt: typeof item.revised_prompt === 'string' ? item.revised_prompt : undefined, - }) + if (item?.type === 'image_generation_call') { + const b64 = getResponsesImageResultBase64(item.result) + if (b64) { + results.push({ + image: normalizeBase64Image(b64, fallbackMime), + actualParams: mergeActualParams(pickActualParams(item)), + revisedPrompt: typeof item.revised_prompt === 'string' ? item.revised_prompt : undefined, + }) + } + continue + } + + if (item?.type !== 'message') continue + for (const part of item.content ?? []) { + if ((part.type !== 'output_text' && part.type !== 'text') || typeof part.text !== 'string') continue + const markdownImages = await resolveMarkdownImages(part.text, fallbackMime, signal) + for (const image of markdownImages.images) { + results.push({ + image: image.dataUrl, + actualParams: mergeActualParams(pickActualParams(item)), + rawImageUrl: image.rawImageUrl, + }) + } + rawImageUrls.push(...markdownImages.unresolvedImageUrls) } } if (!results.length) { const err = new Error('接口没有返回可识别的图片数据，请查看原始响应内容确认服务商实际返回的数据结构。如果使用的是中转或兼容接口，建议创建并使用「自定义服务商」配置。') ;(err as any).rawResponsePayload = JSON.stringify(payload, null, 2) + if (rawImageUrls.length) (err as any).rawImageUrls = [...new Set(rawImageUrls)] throw err } - return results + return { results, rawImageUrls: [...new Set(rawImageUrls)] } } async function parseImagesApiResponse(payload: ImageApiResponse, mime: string, signal?: AbortSignal): Promise { @@ -389,20 +415,25 @@ async function parseResponsesApiStreamResponse( const payload = completedPayload ?? (outputItems.length ? { output: outputItems } : null) if (!payload) throw new Error('流式接口未返回最终图片数据') - let imageResults: ReturnType + let parsedImages: ParsedResponsesImageResults try { - imageResults = parseResponsesImageResults(payload, mime) + parsedImages = await parseResponsesImageResults(payload, mime) } catch (err) { const collectedImageItems = outputItems.filter((item) => getResponsesImageResultBase64(item.result)) if (collectedImageItems.length === 0) throw err - imageResults = parseResponsesImageResults({ output: collectedImageItems }, mime) + parsedImages = await parseResponsesImageResults({ output: collectedImageItems }, mime) } - const actualParams = mergeActualParams(imageResults[0]?.actualParams ?? {}) + const actualParams = mergeActualParams(parsedImages.results[0]?.actualParams ?? {}) + const rawImageUrls = [ + ...parsedImages.results.map((result) => result.rawImageUrl).filter((url): url is string => Boolean(url)), + ...parsedImages.rawImageUrls, + ] return { - images: imageResults.map((result) => result.image), + images: parsedImages.results.map((result) => result.image), actualParams, - actualParamsList: imageResults.map((result) => mergeActualParams(result.actualParams ?? {})), - revisedPrompts: imageResults.map((result) => result.revisedPrompt), + actualParamsList: parsedImages.results.map((result) => mergeActualParams(result.actualParams ?? {})), + revisedPrompts: parsedImages.results.map((result) => result.revisedPrompt), + ...(rawImageUrls.length ? { rawImageUrls: [...new Set(rawImageUrls)] } : {}), } } @@ -1026,17 +1057,22 @@ async function callResponsesImageApiSingle(opts: CallApiOptions, profile: ApiPro } const payload = await response.json() as ResponsesApiResponse - const imageResults = parseResponsesImageResults(payload, mime) + const parsedImages = await parseResponsesImageResults(payload, mime, controller.signal) const actualParams = mergeActualParams( - imageResults[0]?.actualParams ?? {}, + parsedImages.results[0]?.actualParams ?? {}, ) + const rawImageUrls = [ + ...parsedImages.results.map((result) => result.rawImageUrl).filter((url): url is string => Boolean(url)), + ...parsedImages.rawImageUrls, + ] return { - images: imageResults.map((result) => result.image), + images: parsedImages.results.map((result) => result.image), actualParams, - actualParamsList: imageResults.map((result) => + actualParamsList: parsedImages.results.map((result) => mergeActualParams(result.actualParams ?? {}), ), - revisedPrompts: imageResults.map((result) => result.revisedPrompt), + revisedPrompts: parsedImages.results.map((result) => result.revisedPrompt), + ...(rawImageUrls.length ? { rawImageUrls: [...new Set(rawImageUrls)] } : {}), } } finally { clearTimeout(timeoutId) diff --git a/src/store.ts b/src/store.ts index 872a4dbf5..4ff727787 100644 --- a/src/store.ts +++ b/src/store.ts @@ -49,6 +49,7 @@ import { buildAgentApiInput, buildAgentContinuationInput } from './lib/agentInpu import { collectAgentRoundOutputImageSlots, extractAgentReferenceIds, getAgentCurrentReferenceId, getAgentGeneratedImageReferenceId } from './lib/agentImageReferences' import { showBrowserNotification } from './lib/browserNotification' import { IMAGE_FETCH_CORS_HINT } from './lib/imageApiShared' +import { removeMarkdownImages } from './lib/markdownImages' import { getFalErrorMessage, getFalQueuedImageResult } from './lib/falAiImageApi' import { getCustomQueuedImageResult } from './lib/openaiCompatibleImageApi' import { validateMaskMatchesImage } from './lib/canvasImage' @@ -2623,6 +2624,7 @@ async function executeAgentRound( actualParams, actualParamsByImage: { [stored.id]: actualParams }, revisedPromptByImage: image.revisedPrompt ? { [stored.id]: image.revisedPrompt } : undefined, + rawImageUrls: image.rawImageUrl ? [image.rawImageUrl] : undefined, rawResponsePayload, ...createTaskDonePatch(latestBeforeUpdate, Date.now()), agentToolAction: image.action, @@ -2631,7 +2633,7 @@ async function executeAgentRound( return { taskId, committed: true } } - const failAgentImageTask = (toolCallId: string, error: string, rawResponsePayload?: string) => { + const failAgentImageTask = (toolCallId: string, error: string, rawResponsePayload?: string, rawImageUrls?: string[]) => { const taskId = taskIdByToolCallId.get(toolCallId) if (!taskId) return const latestTask = useStore.getState().tasks.find((task) => task.id === taskId) @@ -2641,6 +2643,7 @@ async function executeAgentRound( updateTaskInStore(taskId, { ...createTaskErrorPatch(latestTask, error, Date.now()), rawResponsePayload, + rawImageUrls: rawImageUrls?.length ? rawImageUrls : undefined, falRecoverable: false, customRecoverable: false, }) @@ -2921,6 +2924,7 @@ async function executeAgentRound( const batchResult = requestSettings.agentApiConfigMode === 'hybrid' ? { batchItemId: item.id, + unresolvedImageUrls: undefined as string[] | undefined, ...(await callHybridImageApiSingle({ taskId: taskIdByToolCallId.get(batchToolCallId)!, prompt: item.prompt, @@ -2996,7 +3000,7 @@ async function executeAgentRound( const r = settled.value if (r.image && !r.committed) continue if (!r.image) { - failAgentImageTask(batchExecutionItems[i].batchToolCallId, r.error!, r.rawResponsePayload) + failAgentImageTask(batchExecutionItems[i].batchToolCallId, r.error!, r.rawResponsePayload, r.unresolvedImageUrls) } outputImages.push({ id: r.batchItemId, @@ -3115,13 +3119,21 @@ async function executeAgentRound( rounds: current.rounds.map((item) => item.id === roundId ? { ...item, responseId: lastResponseId, responseOutput: accumulatedOutputItems } : item), })) + const cleanedTextBeforeResponse = removeMarkdownImages(textBeforeResponse) + if (shouldStreamAssistantMessage && accumulatedText !== textBeforeResponse) { + accumulatedText = removeMarkdownImages(accumulatedText) + updateAgentConversation(conversationId, (current) => ({ + ...current, + messages: current.messages.map((message) => message.id === assistantMessageId ? { ...message, content: accumulatedText } : message), + })) + } const responseText = result.text.trim() - if (responseText && accumulatedText === textBeforeResponse) { + if (responseText && accumulatedText === cleanedTextBeforeResponse) { const textToAppend = accumulatedText ? `\n\n${responseText}` : responseText accumulatedText += textToAppend if (shouldStreamAssistantMessage) appendAgentAssistantMessageContent(conversationId, assistantMessageId, textToAppend) } - const newTextInThisResponse = accumulatedText.slice(textBeforeResponse.length).trim() + const newTextInThisResponse = accumulatedText.slice(cleanedTextBeforeResponse.length).trim() if (newTextInThisResponse) textSegments.push(newTextInThisResponse) // Process built-in image_generation_call results (single images) @@ -3166,6 +3178,7 @@ async function executeAgentRound( actualParams, actualParamsByImage: { [stored.id]: actualParams }, revisedPromptByImage: image.revisedPrompt ? { [stored.id]: image.revisedPrompt } : undefined, + rawImageUrls: image.rawImageUrl ? [image.rawImageUrl] : undefined, rawResponsePayload: result.rawResponsePayload, status: 'done', error: null, @@ -3184,6 +3197,17 @@ async function executeAgentRound( await putTask(task) } + for (const rawImageUrl of result.unresolvedImageUrls ?? []) { + const taskId = await ensureStreamingAgentTask(genId(), round?.prompt ?? userMessage.content) + const task = useStore.getState().tasks.find((item) => item.id === taskId) + if (!task) continue + updateTaskInStore(taskId, { + ...createTaskErrorPatch(task, `图片已生成，但因跨域限制无法下载。${IMAGE_FETCH_CORS_HINT}`, Date.now()), + rawImageUrls: [rawImageUrl], + rawResponsePayload: result.rawResponsePayload, + }) + } + if (result.rawResponsePayload && streamingTaskIds.length > 0) { for (const taskId of streamingTaskIds) { const latestTask = useStore.getState().tasks.find((task) => task.id === taskId)