Skip to content

Commit 4032213

Browse files
committed
feat(web): render extracted images inline in the document reader
Follow-up to #197: long/short source docs embed `![image](sources/images/...)` references whose files were extracted at ingest but never displayed (they showed as a literal "!image"). Now they render inline in the reader. - Backend: GET /api/v1/document/image serves wiki/sources/images/** — narrowed to the images dir + raster suffixes with a traversal guard (read-only; cannot reach wiki pages/skills/arbitrary files). - MarkdownView: gain an optional image token, ENABLED ONLY when a resolveImageSrc prop is passed (the doc reader). Without it the `![](…)` token is omitted from the regex, so chat/wiki rendering is byte-for-byte unchanged. A new AuthedImage fetches the API path as a blob (the bearer token can't ride on <img src>; mirrors artifacts.ts) and revokes the object URL on unmount. - Doc reader: passes a resolver that normalizes both source path conventions (wiki-root `sources/images/...` and note-relative `images/...`) to the image endpoint; external/data URLs stay literal text. Images are now visible in the UI. (LLM answers are still image-blind — ingestion does no caption/OCR — which is a separate pipeline change.) Backend adds serve/traversal/suffix/404/auth tests (1251 passed). Verified in a browser: the Soochow Securities report's figures render inline.
1 parent fc8197b commit 4032213

4 files changed

Lines changed: 209 additions & 19 deletions

File tree

‎frontend/src/components/MarkdownView.tsx‎

Lines changed: 87 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import React, { useEffect, useId, useRef, useState } from 'react'
22
import katex from 'katex'
33
import 'katex/dist/katex.min.css'
4+
import { fetchAsBlobUrl } from '@/api/client'
45
import { useTheme } from '@/lib/theme'
56

67
/** Guard for [text](url) links: only render a real anchor for http(s) or
@@ -31,11 +32,19 @@ const isSafeUrl = (u: string) => {
3132
* wikilink target is NOT recursed). Single `*`/`_` obey a minimal left/right
3233
* flanking rule so intraword runs (`a_b_c`, `2*3*4`) stay literal.
3334
*/
34-
function inline(text: string, onWikiLink?: (target: string) => void): React.ReactNode[] {
35+
function inline(
36+
text: string,
37+
onWikiLink?: (target: string) => void,
38+
resolveImageSrc?: (rawSrc: string) => string | null,
39+
): React.ReactNode[] {
3540
const parts: React.ReactNode[] = []
3641
// Bold is non-greedy (`.+?`) so it tolerates an inner opposite-emphasis
37-
// (`**a *b* c**`); italic keeps a bounded `[^*]+`/`[^_]+` class.
38-
const re = /(\[\[[^\]]+\]\]|\\\([\s\S]+?\\\)|\*\*.+?\*\*|~~[^~]+~~|`[^`]+`|\[[^\]]+\]\([^)]+\)|\*[^*]+\*|_[^_]+_)/g
42+
// (`**a *b* c**`); italic keeps a bounded `[^*]+`/`[^_]+` class. The image
43+
// alternative `![alt](url)` is included ONLY when a resolveImageSrc is given
44+
// (the document reader), so chat/wiki rendering stays byte-for-byte unchanged.
45+
const re = resolveImageSrc
46+
? /(\[\[[^\]]+\]\]|!\[[^\]]*\]\([^)]+\)|\\\([\s\S]+?\\\)|\*\*.+?\*\*|~~[^~]+~~|`[^`]+`|\[[^\]]+\]\([^)]+\)|\*[^*]+\*|_[^_]+_)/g
47+
: /(\[\[[^\]]+\]\]|\\\([\s\S]+?\\\)|\*\*.+?\*\*|~~[^~]+~~|`[^`]+`|\[[^\]]+\]\([^)]+\)|\*[^*]+\*|_[^_]+_)/g
3948
let last = 0, m: RegExpExecArray | null, k = 0
4049
while ((m = re.exec(text))) {
4150
if (m.index > last) parts.push(text.slice(last, m.index))
@@ -85,11 +94,28 @@ function inline(text: string, onWikiLink?: (target: string) => void): React.Reac
8594
)
8695
} else if (tok.startsWith('**')) {
8796
// Recurse so nested markup (e.g. `**[docs](url)**`) resolves. slice is strictly shorter.
88-
parts.push(<strong key={k++} className="font-semibold text-foreground">{inline(tok.slice(2, -2), onWikiLink)}</strong>)
97+
parts.push(<strong key={k++} className="font-semibold text-foreground">{inline(tok.slice(2, -2), onWikiLink, resolveImageSrc)}</strong>)
8998
} else if (tok.startsWith('~~')) {
90-
parts.push(<del key={k++} className="line-through">{inline(tok.slice(2, -2), onWikiLink)}</del>)
99+
parts.push(<del key={k++} className="line-through">{inline(tok.slice(2, -2), onWikiLink, resolveImageSrc)}</del>)
91100
} else if (tok.startsWith('`')) {
92101
parts.push(<code key={k++} className="font-mono2 text-[12px] bg-muted rounded px-1 py-px">{tok.slice(1, -1)}</code>)
102+
} else if (tok.startsWith('![')) {
103+
// Markdown image — only reached when resolveImageSrc is provided (the
104+
// regex omits this token otherwise). resolveImageSrc maps a KB-relative
105+
// source path to an authed API path, or null for external/data URLs
106+
// (left as literal text). Cannot collide with the link/wikilink branches:
107+
// those match tokens starting with '[', this one starts with '!['.
108+
const im = /^!\[([^\]]*)\]\(([^)]+)\)$/.exec(tok)
109+
const alt = im ? im[1] : ''
110+
const raw = im ? im[2].trim() : ''
111+
const apiPath = raw ? (resolveImageSrc?.(raw) ?? null) : null
112+
parts.push(
113+
apiPath ? (
114+
<AuthedImage key={k++} apiPath={apiPath} alt={alt} />
115+
) : (
116+
<span key={k++}>{tok}</span>
117+
),
118+
)
93119
} else if (tok.startsWith('[')) {
94120
// Inline link [text](url). `[[…]]` was already consumed above, so any
95121
// `[` reaching here is a genuine link. Only emit an anchor when the URL
@@ -109,7 +135,7 @@ function inline(text: string, onWikiLink?: (target: string) => void): React.Reac
109135
rel="noopener noreferrer"
110136
className="text-accent-brand hover:underline"
111137
>
112-
{inline(label, onWikiLink)}
138+
{inline(label, onWikiLink, resolveImageSrc)}
113139
</a>
114140
) : (
115141
<span key={k++}>{tok}</span>
@@ -135,7 +161,7 @@ function inline(text: string, onWikiLink?: (target: string) => void): React.Reac
135161
re.lastIndex = m.index + 1
136162
continue
137163
}
138-
parts.push(<em key={k++} className="italic">{inline(innerEm, onWikiLink)}</em>)
164+
parts.push(<em key={k++} className="italic">{inline(innerEm, onWikiLink, resolveImageSrc)}</em>)
139165
}
140166
last = m.index + tok.length
141167
}
@@ -216,14 +242,59 @@ function MermaidBlock({ code }: { code: string }) {
216242
)
217243
}
218244

245+
/** Render a bearer-authed KB image. The API token can't ride on `<img src>`,
246+
* so we fetch the API path as a blob (mirrors `artifacts.ts`), show it, and
247+
* revoke the object URL on unmount / src change. On failure it renders nothing
248+
* rather than a broken-image glyph; while loading it shows a muted placeholder. */
249+
function AuthedImage({ apiPath, alt }: { apiPath: string; alt: string }) {
250+
const [url, setUrl] = useState<string | null>(null)
251+
const [failed, setFailed] = useState(false)
252+
useEffect(() => {
253+
let cancelled = false
254+
let objectUrl: string | null = null
255+
setUrl(null)
256+
setFailed(false)
257+
fetchAsBlobUrl(apiPath)
258+
.then((u) => {
259+
if (cancelled) {
260+
URL.revokeObjectURL(u)
261+
return
262+
}
263+
objectUrl = u
264+
setUrl(u)
265+
})
266+
.catch(() => {
267+
if (!cancelled) setFailed(true)
268+
})
269+
return () => {
270+
cancelled = true
271+
if (objectUrl) URL.revokeObjectURL(objectUrl)
272+
}
273+
}, [apiPath])
274+
if (failed) return null
275+
if (!url) return <span className="my-3 block h-32 animate-pulse rounded-lg bg-muted" aria-hidden />
276+
return (
277+
<img
278+
src={url}
279+
alt={alt}
280+
className="my-3 max-w-full rounded-lg border border-[hsl(var(--glass-border))]"
281+
/>
282+
)
283+
}
284+
219285
export default function MarkdownView({
220286
source,
221287
onWikiLink,
288+
resolveImageSrc,
222289
}: {
223290
source: string
224291
/** Navigate to a `[[target]]` wikilink's page. Omit to render plain,
225292
* non-interactive tokens (no `cursor-pointer` implying a dead click). */
226293
onWikiLink?: (target: string) => void
294+
/** Map a Markdown image's raw src to an authed API path (or null to leave it
295+
* as text). Only the document reader passes this; without it, `![](…)` is
296+
* not tokenized as an image, so chat/wiki rendering is unchanged. */
297+
resolveImageSrc?: (rawSrc: string) => string | null
227298
}) {
228299
const lines = source.split('\n')
229300
const out: React.ReactNode[] = []
@@ -240,7 +311,7 @@ export default function MarkdownView({
240311
{list.map((li, i) => (
241312
<li key={i} className="flex gap-2 text-[14px] leading-relaxed text-muted-foreground">
242313
<span className="mt-[9px] w-1 h-1 rounded-full bg-muted-foreground shrink-0" />
243-
<span>{inline(li, onWikiLink)}</span>
314+
<span>{inline(li, onWikiLink, resolveImageSrc)}</span>
244315
</li>
245316
))}
246317
</ul>,
@@ -254,7 +325,7 @@ export default function MarkdownView({
254325
<ol key={key++} start={olistStart} className="my-2.5 space-y-1.5 pl-6 list-decimal">
255326
{olist.map((li, i) => (
256327
<li key={i} className="pl-1 text-[14px] leading-relaxed text-muted-foreground marker:text-muted-foreground">
257-
<span>{inline(li, onWikiLink)}</span>
328+
<span>{inline(li, onWikiLink, resolveImageSrc)}</span>
258329
</li>
259330
))}
260331
</ol>,
@@ -382,7 +453,7 @@ export default function MarkdownView({
382453
key={c}
383454
className={`border border-[hsl(var(--glass-border))] bg-muted/50 px-3 py-1.5 font-semibold text-foreground ${alignOf(c)}`}
384455
>
385-
{inline(h, onWikiLink)}
456+
{inline(h, onWikiLink, resolveImageSrc)}
386457
</th>
387458
))}
388459
</tr>
@@ -395,7 +466,7 @@ export default function MarkdownView({
395466
key={c}
396467
className={`border border-[hsl(var(--glass-border))] px-3 py-1.5 text-muted-foreground ${alignOf(c)}`}
397468
>
398-
{inline(r[c] ?? '', onWikiLink)}
469+
{inline(r[c] ?? '', onWikiLink, resolveImageSrc)}
399470
</td>
400471
))}
401472
</tr>
@@ -422,11 +493,11 @@ export default function MarkdownView({
422493
}
423494
flushBlocks()
424495
if (!line.trim()) { out.push(<div key={key++} className="h-2" />); continue }
425-
if (line.startsWith('### ')) out.push(<h3 key={key++} className="mt-4 mb-1.5 text-[14px] font-semibold text-foreground">{inline(line.slice(4), onWikiLink)}</h3>)
426-
else if (line.startsWith('## ')) out.push(<h2 key={key++} className="mt-5 mb-2 text-[16px] font-bold text-foreground">{inline(line.slice(3), onWikiLink)}</h2>)
427-
else if (line.startsWith('# ')) out.push(<h1 key={key++} className="mb-3 text-[22px] font-extrabold tracking-tight text-foreground">{inline(line.slice(2), onWikiLink)}</h1>)
428-
else if (line.startsWith('> ')) out.push(<div key={key++} className="my-2.5 border-l-2 border-amber-400/70 bg-amber-400/10 rounded-r-lg px-3 py-2 text-[13px] text-muted-foreground">{inline(line.slice(2), onWikiLink)}</div>)
429-
else out.push(<p key={key++} className="my-1.5 text-[14px] leading-relaxed text-muted-foreground">{inline(line, onWikiLink)}</p>)
496+
if (line.startsWith('### ')) out.push(<h3 key={key++} className="mt-4 mb-1.5 text-[14px] font-semibold text-foreground">{inline(line.slice(4), onWikiLink, resolveImageSrc)}</h3>)
497+
else if (line.startsWith('## ')) out.push(<h2 key={key++} className="mt-5 mb-2 text-[16px] font-bold text-foreground">{inline(line.slice(3), onWikiLink, resolveImageSrc)}</h2>)
498+
else if (line.startsWith('# ')) out.push(<h1 key={key++} className="mb-3 text-[22px] font-extrabold tracking-tight text-foreground">{inline(line.slice(2), onWikiLink, resolveImageSrc)}</h1>)
499+
else if (line.startsWith('> ')) out.push(<div key={key++} className="my-2.5 border-l-2 border-amber-400/70 bg-amber-400/10 rounded-r-lg px-3 py-2 text-[13px] text-muted-foreground">{inline(line.slice(2), onWikiLink, resolveImageSrc)}</div>)
500+
else out.push(<p key={key++} className="my-1.5 text-[14px] leading-relaxed text-muted-foreground">{inline(line, onWikiLink, resolveImageSrc)}</p>)
430501
}
431502
flushBlocks()
432503
return <div>{out}</div>

‎frontend/src/pages/KbDetail.tsx‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1081,11 +1081,27 @@ function DocumentsPane({
10811081
}
10821082
}, [kb, openHash, docReloadSeq])
10831083

1084+
// Map a source image ref to the authed image endpoint. Long-doc JSON stores
1085+
// wiki-root-relative `sources/images/...`; short-doc MD uses note-relative
1086+
// `images/...` — normalize both to a wiki-relative path. Non-matching refs
1087+
// (external / data URLs) return null and render as plain text.
1088+
const resolveDocImageSrc = useCallback(
1089+
(rawSrc: string): string | null => {
1090+
let rel: string | null = null
1091+
if (rawSrc.startsWith('sources/images/')) rel = rawSrc
1092+
else if (rawSrc.startsWith('images/')) rel = `sources/${rawSrc}`
1093+
if (!rel) return null
1094+
return `/api/v1/document/image?kb=${encodeURIComponent(kb)}&path=${encodeURIComponent(rel)}`
1095+
},
1096+
[kb],
1097+
)
10841098
// Parse Markdown once per fetched source (stable cache ref → no re-parse).
10851099
const readerBody = useMemo(
10861100
() =>
1087-
docSource && docSource.content.trim() ? <MarkdownView source={docSource.content} /> : null,
1088-
[docSource],
1101+
docSource && docSource.content.trim() ? (
1102+
<MarkdownView source={docSource.content} resolveImageSrc={resolveDocImageSrc} />
1103+
) : null,
1104+
[docSource, resolveDocImageSrc],
10891105
)
10901106
const readerEmpty = docSource != null && docSource.content.trim().length === 0
10911107

‎openkb/api_documents_router.py‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88

99
from __future__ import annotations
1010

11-
from fastapi import APIRouter, Depends, HTTPException
11+
from fastapi import APIRouter, Depends, HTTPException, Query
12+
from fastapi.responses import FileResponse
1213
from starlette.concurrency import run_in_threadpool
1314

1415
from openkb.api_helpers import _resolve_kb, require_bearer_token
@@ -17,6 +18,9 @@
1718

1819
documents_router = APIRouter()
1920

21+
# Raster types the image extractor produces; SVG is excluded (inline-script risk).
22+
_IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg", ".gif", ".webp")
23+
2024

2125
@documents_router.post("/api/v1/document/source", response_model=DocumentSourceResponse)
2226
async def document_source_endpoint(
@@ -33,3 +37,28 @@ async def document_source_endpoint(
3337
if result is None:
3438
raise HTTPException(status_code=404, detail="Document source not found.")
3539
return DocumentSourceResponse(**result)
40+
41+
42+
@documents_router.get("/api/v1/document/image")
43+
async def document_image_endpoint(
44+
kb: str = Query(...),
45+
path: str = Query(..., min_length=1),
46+
_: None = Depends(require_bearer_token),
47+
) -> FileResponse:
48+
"""Serve an extracted document image (``wiki/sources/images/**``).
49+
50+
``path`` is the image reference from the source text, resolved relative to
51+
the KB's ``wiki/`` dir (source text stores ``sources/images/<doc>/...``).
52+
Narrowed to the images dir + raster suffixes with a traversal guard, so this
53+
read-only sink can never serve wiki pages, skills, or arbitrary files.
54+
"""
55+
kb_dir = _resolve_kb(kb)
56+
images_root = (kb_dir / "wiki" / "sources" / "images").resolve()
57+
full = (kb_dir / "wiki" / path).resolve()
58+
if not full.is_relative_to(images_root):
59+
raise HTTPException(status_code=400, detail="Invalid image path.")
60+
if full.suffix.lower() not in _IMAGE_SUFFIXES:
61+
raise HTTPException(status_code=400, detail="Only extracted images are served.")
62+
if not full.is_file():
63+
raise HTTPException(status_code=404, detail="Image not found.")
64+
return FileResponse(full)

‎tests/test_api_documents.py‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,3 +186,77 @@ def test_document_source_requires_auth(monkeypatch, kb_dir):
186186
resp = client.post("/api/v1/document/source", json={"kb": "test-kb", "hash": "h1"})
187187

188188
assert resp.status_code == 401
189+
190+
191+
def test_document_image_serves_extracted_image(monkeypatch, kb_dir):
192+
client = _client(monkeypatch)
193+
kb = _use_named_kb(monkeypatch, kb_dir)
194+
img_dir = kb_dir / "wiki" / "sources" / "images" / "doc"
195+
img_dir.mkdir(parents=True)
196+
(img_dir / "p1_img1.png").write_bytes(b"\x89PNG\r\n\x1a\nfake-bytes")
197+
198+
resp = client.get(
199+
"/api/v1/document/image",
200+
params={"kb": kb, "path": "sources/images/doc/p1_img1.png"},
201+
headers=_auth(),
202+
)
203+
204+
assert resp.status_code == 200
205+
assert resp.headers["content-type"].startswith("image/")
206+
assert resp.content == b"\x89PNG\r\n\x1a\nfake-bytes"
207+
208+
209+
def test_document_image_rejects_traversal(monkeypatch, kb_dir):
210+
"""A path escaping wiki/sources/images (even to another .png) is rejected by
211+
the containment guard before the suffix check."""
212+
client = _client(monkeypatch)
213+
kb = _use_named_kb(monkeypatch, kb_dir)
214+
(kb_dir / "wiki" / "sources" / "evil.png").write_bytes(b"x") # inside sources/, outside images/
215+
216+
resp = client.get(
217+
"/api/v1/document/image",
218+
params={"kb": kb, "path": "sources/images/../evil.png"},
219+
headers=_auth(),
220+
)
221+
222+
assert resp.status_code == 400
223+
224+
225+
def test_document_image_rejects_non_image_suffix(monkeypatch, kb_dir):
226+
client = _client(monkeypatch)
227+
kb = _use_named_kb(monkeypatch, kb_dir)
228+
d = kb_dir / "wiki" / "sources" / "images" / "doc"
229+
d.mkdir(parents=True)
230+
(d / "notes.md").write_text("secret", encoding="utf-8")
231+
232+
resp = client.get(
233+
"/api/v1/document/image",
234+
params={"kb": kb, "path": "sources/images/doc/notes.md"},
235+
headers=_auth(),
236+
)
237+
238+
assert resp.status_code == 400
239+
240+
241+
def test_document_image_missing_404(monkeypatch, kb_dir):
242+
client = _client(monkeypatch)
243+
kb = _use_named_kb(monkeypatch, kb_dir)
244+
245+
resp = client.get(
246+
"/api/v1/document/image",
247+
params={"kb": kb, "path": "sources/images/doc/nope.png"},
248+
headers=_auth(),
249+
)
250+
251+
assert resp.status_code == 404
252+
253+
254+
def test_document_image_requires_auth(monkeypatch, kb_dir):
255+
client = _client(monkeypatch)
256+
_use_named_kb(monkeypatch, kb_dir)
257+
258+
resp = client.get(
259+
"/api/v1/document/image", params={"kb": "test-kb", "path": "sources/images/doc/p.png"}
260+
)
261+
262+
assert resp.status_code == 401

0 commit comments

Comments
 (0)