[codex] add PDF extraction preview

alembic/versions/0024_add_source_extraction_metadata.py

@@ -0,0 +1,54 @@
+"""Add source extraction metadata columns.
+
+Revision ID: 0024
+Revises: 0023
+Create Date: 2026-06-08
+
+Records the PDF extraction engine and page count used during ingest so the UI
+can explain the Extract step without needing to re-run document conversion.
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from sqlalchemy import inspect as sa_inspect
+
+from alembic import op
+
+revision: str = "0024"
+down_revision: str = "0023"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _column_exists(conn: sa.engine.Connection, table: str, column: str) -> bool:
+    inspector = sa_inspect(conn)
+    return any(c["name"] == column for c in inspector.get_columns(table))
+
+
+def upgrade() -> None:
+    """Add nullable extraction metadata columns to source."""
+    conn = op.get_bind()
+    if not _column_exists(conn, "source", "extraction_engine"):
+        op.add_column("source", sa.Column("extraction_engine", sa.String(), nullable=True))
+    if not _column_exists(conn, "source", "extraction_page_count"):
+        op.add_column("source", sa.Column("extraction_page_count", sa.Integer(), nullable=True))
+
+
+def downgrade() -> None:
+    """Remove extraction metadata columns from source."""
+    conn = op.get_bind()
+    is_sqlite = conn.dialect.name == "sqlite"
+
+    if is_sqlite:
+        with op.batch_alter_table("source") as batch_op:
+            if _column_exists(conn, "source", "extraction_page_count"):
+                batch_op.drop_column("extraction_page_count")
+            if _column_exists(conn, "source", "extraction_engine"):
+                batch_op.drop_column("extraction_engine")
+        return
+
+    if _column_exists(conn, "source", "extraction_page_count"):
+        op.drop_column("source", "extraction_page_count")
+    if _column_exists(conn, "source", "extraction_engine"):
+        op.drop_column("source", "extraction_engine")

apps/web/src/api/sources.ts

@@ -61,7 +61,7 @@ export function retryCompile(sourceId: string): Promise<TriggerCompileResponse>
 
 export function getSourceContent(sourceId: string): Promise<SourceContentResponse> {
   return apiFetch<SourceContentResponse>(
-    `/ingest/sources/${encodeURIComponent(sourceId)}/content`,
+    `/api/ingest/sources/${encodeURIComponent(sourceId)}/content`,
   );
 }
 

apps/web/src/components/inbox/SourceDetailView.tsx

@@ -5,6 +5,7 @@ import { getBaseUrl } from "../../api/client";
 import { Badge, type BadgeTone } from "../shared/Badge";
 import { Card } from "../shared/Card";
 import { Spinner } from "../shared/Spinner";
+import { ExtractionPreviewModal } from "../viewers/ExtractionPreviewModal";
 import { SourceSpansPanel } from "../viewers/SourceSpansPanel";
 import type { IngestStatus, PipelineStep, SourceType } from "../../types/api";
 
@@ -86,6 +87,7 @@ export function SourceDetailView() {
   const { id } = useParams<{ id: string }>();
   const { data: source, isLoading, isError } = useSourceDetail(id);
   const [selectedImg, setSelectedImg] = useState<string | null>(null);
+  const [extractPreviewOpen, setExtractPreviewOpen] = useState(false);
 
   if (isLoading) {
     return (
@@ -194,20 +196,46 @@ export function SourceDetailView() {
               Processing Pipeline
             </h2>
             <div className="relative">
-              {source.pipeline_steps.map((step, idx) => (
-                <div key={step.name} className="flex gap-3 pb-4 last:pb-0">
-                  <div className="flex flex-col items-center">
-                    <StepIcon status={step.status} />
-                    {idx < source.pipeline_steps.length - 1 ? (
-                      <div className="mt-1 h-full w-px bg-slate-200" />
-                    ) : null}
-                  </div>
-                  <div className="pt-0.5">
-                    <p className="text-sm font-medium text-slate-900">{step.name}</p>
-                    <p className="text-xs text-slate-500">{step.description}</p>
-                  </div>
-                </div>
-              ))}
+              {source.pipeline_steps.map((step, idx) => {
+                const canPreviewExtraction =
+                  step.name === "Extract" &&
+                  step.status === "complete" &&
+                  source.source_type === "pdf" &&
+                  source.has_original;
+                const row = (
+                  <>
+                    <div className="flex flex-col items-center">
+                      <StepIcon status={step.status} />
+                      {idx < source.pipeline_steps.length - 1 ? (
+                        <div className="mt-1 h-full w-px bg-slate-200" />
+                      ) : null}
+                    </div>
+                    <div className="pt-0.5">
+                      <p className="text-sm font-medium text-slate-900">{step.name}</p>
+                      <p className="text-xs text-slate-500">{step.description}</p>
+                    </div>
+                  </>
+                );
+
+                if (!canPreviewExtraction) {
+                  return (
+                    <div key={step.name} className="flex gap-3 pb-4 last:pb-0">
+                      {row}
+                    </div>
+                  );
+                }
+
+                return (
+                  <button
+                    key={step.name}
+                    type="button"
+                    onClick={() => setExtractPreviewOpen(true)}
+                    className="flex w-full gap-3 rounded-md pb-4 text-left transition hover:bg-slate-50 focus:outline-none focus:ring-2 focus:ring-brand-500 focus:ring-offset-2 last:pb-0"
+                  >
+                    {row}
+                  </button>
+                );
+              })}
             </div>
           </Card>
 
@@ -320,6 +348,13 @@ export function SourceDetailView() {
           </div>
         </div>
       ) : null}
+
+      {extractPreviewOpen ? (
+        <ExtractionPreviewModal
+          source={source}
+          onClose={() => setExtractPreviewOpen(false)}
+        />
+      ) : null}
     </div>
   );
 }

apps/web/src/components/viewers/ExtractionPreviewModal.tsx

@@ -0,0 +1,92 @@
+import { useQuery } from "@tanstack/react-query";
+import { getOriginalUrl, getSourceContent } from "../../api/sources";
+import type { SourceDetailResponse } from "../../types/api";
+import { Badge } from "../shared/Badge";
+import { Button } from "../shared/Button";
+import { Spinner } from "../shared/Spinner";
+import { PdfViewer } from "./PdfViewer";
+
+interface ExtractionPreviewModalProps {
+  source: SourceDetailResponse;
+  onClose: () => void;
+}
+
+function engineLabel(engine: string | null): string {
+  switch (engine) {
+    case "docling-serve":
+      return "Docling";
+    case "pymupdf":
+      return "PyMuPDF";
+    default:
+      return "Unknown";
+  }
+}
+
+function formatNumber(value: number | null): string {
+  return value == null ? "Unknown" : value.toLocaleString();
+}
+
+export function ExtractionPreviewModal({ source, onClose }: ExtractionPreviewModalProps) {
+  const contentQuery = useQuery({
+    queryKey: ["source-extraction-content", source.id],
+    queryFn: () => getSourceContent(source.id),
+  });
+
+  return (
+    <div className="fixed inset-0 z-50 flex items-center justify-center bg-black/50">
+      <div className="flex h-[92vh] w-[94vw] flex-col rounded-lg border border-slate-200 bg-white shadow-xl">
+        <div className="flex flex-wrap items-center justify-between gap-3 border-b border-slate-200 px-5 py-3">
+          <div className="min-w-0">
+            <h2 className="truncate text-lg font-semibold text-slate-900">
+              {source.title || "Extraction preview"}
+            </h2>
+            <div className="mt-1 flex flex-wrap items-center gap-2">
+              <Badge tone="brand">Extract</Badge>
+              <Badge tone={source.extraction_engine ? "info" : "neutral"}>
+                Engine: {engineLabel(source.extraction_engine)}
+              </Badge>
+              <Badge tone="neutral">
+                Pages: {formatNumber(source.extraction_page_count)}
+              </Badge>
+              <Badge tone="neutral">
+                Tokens: {formatNumber(source.token_count)}
+              </Badge>
+            </div>
+          </div>
+          <Button variant="ghost" size="sm" onClick={onClose}>
+            Close
+          </Button>
+        </div>
+
+        <div className="grid min-h-0 flex-1 grid-cols-1 lg:grid-cols-2">
+          <div className="min-h-0 overflow-hidden border-b border-slate-200 lg:border-b-0 lg:border-r">
+            <PdfViewer url={getOriginalUrl(source.id)} />
+          </div>
+
+          <div className="flex min-h-0 flex-col bg-white">
+            <div className="flex items-center justify-between border-b border-slate-200 px-4 py-3">
+              <h3 className="text-sm font-semibold text-slate-900">Extracted text</h3>
+              {contentQuery.data?.truncated ? (
+                <Badge tone="warning">Truncated</Badge>
+              ) : null}
+            </div>
+
+            {contentQuery.isLoading ? (
+              <div className="flex flex-1 items-center justify-center gap-2 p-8 text-sm text-slate-500">
+                <Spinner size={16} /> Loading extracted text...
+              </div>
+            ) : contentQuery.isError ? (
+              <div className="m-4 rounded-md border border-rose-200 bg-rose-50 p-3 text-sm text-rose-800">
+                Failed to load extracted text.
+              </div>
+            ) : (
+              <pre className="min-h-0 flex-1 overflow-auto whitespace-pre-wrap p-4 font-mono text-sm leading-relaxed text-slate-800">
+                {contentQuery.data?.content ?? ""}
+              </pre>
+            )}
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+}

apps/web/src/types/api.ts

@@ -49,13 +49,16 @@ export interface Source {
   token_count: number | null;
   error_message: string | null;
   file_path: string | null;
+  extraction_engine: string | null;
+  extraction_page_count: number | null;
   has_original: boolean;
 }
 
 export interface SourceContentResponse {
   content: string;
   source_type: SourceType;
   title: string | null;
+  truncated?: boolean;
 }
 
 export interface PipelineStep {
@@ -90,6 +93,8 @@ export interface SourceDetailResponse {
   token_count: number | null;
   error_message: string | null;
   has_original: boolean;
+  extraction_engine: string | null;
+  extraction_page_count: number | null;
   pipeline_steps: PipelineStep[];
   images: SourceImageEntry[];
   linked_articles: LinkedArticleSummary[];

src/wikimind/api/routes/ingest.py

@@ -221,6 +221,8 @@ async def get_source_detail(
         token_count=source.token_count,
         error_message=source.error_message,
         has_original=source.has_original,
+        extraction_engine=source.extraction_engine,
+        extraction_page_count=source.extraction_page_count,
         pipeline_steps=pipeline_steps,
         images=images,
         linked_articles=linked_articles,

src/wikimind/ingest/adapters/pdf.py

@@ -219,9 +219,13 @@ async def ingest(  # noqa: PLR0915
         # when docling-serve is unavailable.
         try:
             clean_text, page_count = await self._extract_via_docling(raw_pdf_path, source.id, user_id)
+            extraction_engine = "docling-serve"
         except (httpx.HTTPError, httpx.ConnectError) as exc:
             log.warning("docling-serve unavailable, falling back to fitz", error=str(exc))
             clean_text, page_count = self._extract_via_fitz(file_bytes)
+            extraction_engine = "pymupdf"
+        source.extraction_engine = extraction_engine
+        source.extraction_page_count = page_count
 
         # If the title is still the filename fallback (no PDF metadata title),
         # try extracting the first markdown heading from the converted text.

src/wikimind/models/dto/ingest.py

@@ -119,6 +119,8 @@ class SourceDetailResponse(BaseModel):
     token_count: int | None
     error_message: str | None
     has_original: bool
+    extraction_engine: str | None = None
+    extraction_page_count: int | None = None
     pipeline_steps: list[PipelineStep]
     images: list[SourceImageEntry]
     linked_articles: list[LinkedArticleSummary]

src/wikimind/models/tables/ingest.py

@@ -33,6 +33,8 @@ class Source(SQLModel, table=True):
         sa_type=Text,
         exclude=True,
     )  # DB-backed source content; excluded from API responses
+    extraction_engine: str | None = None  # e.g. "docling-serve" or "pymupdf"
+    extraction_page_count: int | None = None
     # SHA-256 hex digest of the raw payload (issue #67). Used by the ingest
     # layer to detect duplicates: re-ingesting the same content returns the
     # existing source instead of creating a second row.

tests/unit/test_pdf_adapter.py

@@ -116,6 +116,8 @@ async def test_ingest_falls_back_to_fitz_when_docling_serve_down(
         assert source.source_type == SourceType.PDF
         assert source.title == "fallback"
         assert source.status == IngestStatus.PROCESSING
+        assert source.extraction_engine == "pymupdf"
+        assert source.extraction_page_count == 1
         assert "Fallback page text" in doc.clean_text
         assert doc.estimated_tokens > 0
         assert doc.chunks  # at least one chunk
@@ -215,6 +217,8 @@ async def test_ingest_uses_docling_serve_when_available(
 
         assert doc.clean_text == markdown
         assert "# Slide deck" in doc.clean_text
+        assert source.extraction_engine == "docling-serve"
+        assert source.extraction_page_count == 1
         assert source.file_path == f"{source.id}.txt"
         assert (isolated_data_dir / "raw" / TEST_USER_ID / f"{source.id}.txt").read_text(encoding="utf-8") == markdown
 

tests/unit/test_source_content.py

@@ -12,7 +12,7 @@
 from wikimind.database import get_session
 from wikimind.errors import NotFoundError
 from wikimind.main import app
-from wikimind.models import Source, SourceContentResponse, SourceType
+from wikimind.models import IngestStatus, Source, SourceContentResponse, SourceType
 from wikimind.services.factories import get_ingest_service
 from wikimind.services.ingest import IngestService
 from wikimind.storage import LocalFileStorage
@@ -153,6 +153,29 @@ async def test_content_endpoint_passes_user_id() -> None:
     assert call_kwargs[1]["user_id"] == "test-user-456"
 
 
+async def test_source_detail_includes_extraction_metadata(client: AsyncClient, db_session) -> None:
+    """Source detail exposes lightweight PDF extraction metadata."""
+    source = Source(
+        id="src-meta",
+        source_type=SourceType.PDF,
+        title="Metadata PDF",
+        user_id=TEST_USER_ID,
+        status=IngestStatus.PROCESSING,
+        clean_text="Extracted text",
+        extraction_engine="docling-serve",
+        extraction_page_count=7,
+    )
+    db_session.add(source)
+    await db_session.commit()
+
+    resp = await client.get("/api/ingest/sources/src-meta/detail")
+
+    assert resp.status_code == 200
+    data = resp.json()
+    assert data["extraction_engine"] == "docling-serve"
+    assert data["extraction_page_count"] == 7
+
+
 async def test_service_get_source_content_reads_file(tmp_path: Path) -> None:
     """IngestService.get_source_content reads the text file from storage."""
     raw_text = "Original source text here."