diff --git a/alembic/versions/0024_add_source_extraction_metadata.py b/alembic/versions/0024_add_source_extraction_metadata.py new file mode 100644 index 00000000..752f2e22 --- /dev/null +++ b/alembic/versions/0024_add_source_extraction_metadata.py @@ -0,0 +1,54 @@ +"""Add source extraction metadata columns. + +Revision ID: 0024 +Revises: 0023 +Create Date: 2026-06-08 + +Records the PDF extraction engine and page count used during ingest so the UI +can explain the Extract step without needing to re-run document conversion. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy import inspect as sa_inspect + +from alembic import op + +revision: str = "0024" +down_revision: str = "0023" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _column_exists(conn: sa.engine.Connection, table: str, column: str) -> bool: + inspector = sa_inspect(conn) + return any(c["name"] == column for c in inspector.get_columns(table)) + + +def upgrade() -> None: + """Add nullable extraction metadata columns to source.""" + conn = op.get_bind() + if not _column_exists(conn, "source", "extraction_engine"): + op.add_column("source", sa.Column("extraction_engine", sa.String(), nullable=True)) + if not _column_exists(conn, "source", "extraction_page_count"): + op.add_column("source", sa.Column("extraction_page_count", sa.Integer(), nullable=True)) + + +def downgrade() -> None: + """Remove extraction metadata columns from source.""" + conn = op.get_bind() + is_sqlite = conn.dialect.name == "sqlite" + + if is_sqlite: + with op.batch_alter_table("source") as batch_op: + if _column_exists(conn, "source", "extraction_page_count"): + batch_op.drop_column("extraction_page_count") + if _column_exists(conn, "source", "extraction_engine"): + batch_op.drop_column("extraction_engine") + return + + if _column_exists(conn, "source", "extraction_page_count"): + op.drop_column("source", "extraction_page_count") + if _column_exists(conn, "source", "extraction_engine"): + op.drop_column("source", "extraction_engine") diff --git a/apps/web/src/api/sources.ts b/apps/web/src/api/sources.ts index 205ab387..3fd0e802 100644 --- a/apps/web/src/api/sources.ts +++ b/apps/web/src/api/sources.ts @@ -61,7 +61,7 @@ export function retryCompile(sourceId: string): Promise export function getSourceContent(sourceId: string): Promise { return apiFetch( - `/ingest/sources/${encodeURIComponent(sourceId)}/content`, + `/api/ingest/sources/${encodeURIComponent(sourceId)}/content`, ); } diff --git a/apps/web/src/components/inbox/SourceDetailView.tsx b/apps/web/src/components/inbox/SourceDetailView.tsx index 6fa62631..926cbce7 100644 --- a/apps/web/src/components/inbox/SourceDetailView.tsx +++ b/apps/web/src/components/inbox/SourceDetailView.tsx @@ -5,6 +5,7 @@ import { getBaseUrl } from "../../api/client"; import { Badge, type BadgeTone } from "../shared/Badge"; import { Card } from "../shared/Card"; import { Spinner } from "../shared/Spinner"; +import { ExtractionPreviewModal } from "../viewers/ExtractionPreviewModal"; import { SourceSpansPanel } from "../viewers/SourceSpansPanel"; import type { IngestStatus, PipelineStep, SourceType } from "../../types/api"; @@ -86,6 +87,7 @@ export function SourceDetailView() { const { id } = useParams<{ id: string }>(); const { data: source, isLoading, isError } = useSourceDetail(id); const [selectedImg, setSelectedImg] = useState(null); + const [extractPreviewOpen, setExtractPreviewOpen] = useState(false); if (isLoading) { return ( @@ -194,20 +196,46 @@ export function SourceDetailView() { Processing Pipeline
- {source.pipeline_steps.map((step, idx) => ( -
-
- - {idx < source.pipeline_steps.length - 1 ? ( -
- ) : null} -
-
-

{step.name}

-

{step.description}

-
-
- ))} + {source.pipeline_steps.map((step, idx) => { + const canPreviewExtraction = + step.name === "Extract" && + step.status === "complete" && + source.source_type === "pdf" && + source.has_original; + const row = ( + <> +
+ + {idx < source.pipeline_steps.length - 1 ? ( +
+ ) : null} +
+
+

{step.name}

+

{step.description}

+
+ + ); + + if (!canPreviewExtraction) { + return ( +
+ {row} +
+ ); + } + + return ( + + ); + })}
@@ -320,6 +348,13 @@ export function SourceDetailView() {
) : null} + + {extractPreviewOpen ? ( + setExtractPreviewOpen(false)} + /> + ) : null} ); } diff --git a/apps/web/src/components/viewers/ExtractionPreviewModal.tsx b/apps/web/src/components/viewers/ExtractionPreviewModal.tsx new file mode 100644 index 00000000..bb5bbe3a --- /dev/null +++ b/apps/web/src/components/viewers/ExtractionPreviewModal.tsx @@ -0,0 +1,92 @@ +import { useQuery } from "@tanstack/react-query"; +import { getOriginalUrl, getSourceContent } from "../../api/sources"; +import type { SourceDetailResponse } from "../../types/api"; +import { Badge } from "../shared/Badge"; +import { Button } from "../shared/Button"; +import { Spinner } from "../shared/Spinner"; +import { PdfViewer } from "./PdfViewer"; + +interface ExtractionPreviewModalProps { + source: SourceDetailResponse; + onClose: () => void; +} + +function engineLabel(engine: string | null): string { + switch (engine) { + case "docling-serve": + return "Docling"; + case "pymupdf": + return "PyMuPDF"; + default: + return "Unknown"; + } +} + +function formatNumber(value: number | null): string { + return value == null ? "Unknown" : value.toLocaleString(); +} + +export function ExtractionPreviewModal({ source, onClose }: ExtractionPreviewModalProps) { + const contentQuery = useQuery({ + queryKey: ["source-extraction-content", source.id], + queryFn: () => getSourceContent(source.id), + }); + + return ( +
+
+
+
+

+ {source.title || "Extraction preview"} +

+
+ Extract + + Engine: {engineLabel(source.extraction_engine)} + + + Pages: {formatNumber(source.extraction_page_count)} + + + Tokens: {formatNumber(source.token_count)} + +
+
+ +
+ +
+
+ +
+ +
+
+

Extracted text

+ {contentQuery.data?.truncated ? ( + Truncated + ) : null} +
+ + {contentQuery.isLoading ? ( +
+ Loading extracted text... +
+ ) : contentQuery.isError ? ( +
+ Failed to load extracted text. +
+ ) : ( + 
+                {contentQuery.data?.content ?? ""}
+
+ )} +
+
+
+
+ ); +} diff --git a/apps/web/src/types/api.ts b/apps/web/src/types/api.ts index 6e8b486c..76b9b72e 100644 --- a/apps/web/src/types/api.ts +++ b/apps/web/src/types/api.ts @@ -49,6 +49,8 @@ export interface Source { token_count: number | null; error_message: string | null; file_path: string | null; + extraction_engine: string | null; + extraction_page_count: number | null; has_original: boolean; } @@ -56,6 +58,7 @@ export interface SourceContentResponse { content: string; source_type: SourceType; title: string | null; + truncated?: boolean; } export interface PipelineStep { @@ -90,6 +93,8 @@ export interface SourceDetailResponse { token_count: number | null; error_message: string | null; has_original: boolean; + extraction_engine: string | null; + extraction_page_count: number | null; pipeline_steps: PipelineStep[]; images: SourceImageEntry[]; linked_articles: LinkedArticleSummary[]; diff --git a/src/wikimind/api/routes/ingest.py b/src/wikimind/api/routes/ingest.py index 48295e8f..d9b347f4 100644 --- a/src/wikimind/api/routes/ingest.py +++ b/src/wikimind/api/routes/ingest.py @@ -221,6 +221,8 @@ async def get_source_detail( token_count=source.token_count, error_message=source.error_message, has_original=source.has_original, + extraction_engine=source.extraction_engine, + extraction_page_count=source.extraction_page_count, pipeline_steps=pipeline_steps, images=images, linked_articles=linked_articles, diff --git a/src/wikimind/ingest/adapters/pdf.py b/src/wikimind/ingest/adapters/pdf.py index 39eac6b0..7c211354 100644 --- a/src/wikimind/ingest/adapters/pdf.py +++ b/src/wikimind/ingest/adapters/pdf.py @@ -219,9 +219,13 @@ async def ingest( # noqa: PLR0915 # when docling-serve is unavailable. try: clean_text, page_count = await self._extract_via_docling(raw_pdf_path, source.id, user_id) + extraction_engine = "docling-serve" except (httpx.HTTPError, httpx.ConnectError) as exc: log.warning("docling-serve unavailable, falling back to fitz", error=str(exc)) clean_text, page_count = self._extract_via_fitz(file_bytes) + extraction_engine = "pymupdf" + source.extraction_engine = extraction_engine + source.extraction_page_count = page_count # If the title is still the filename fallback (no PDF metadata title), # try extracting the first markdown heading from the converted text. diff --git a/src/wikimind/models/dto/ingest.py b/src/wikimind/models/dto/ingest.py index 9bb6e41b..72b1b414 100644 --- a/src/wikimind/models/dto/ingest.py +++ b/src/wikimind/models/dto/ingest.py @@ -119,6 +119,8 @@ class SourceDetailResponse(BaseModel): token_count: int | None error_message: str | None has_original: bool + extraction_engine: str | None = None + extraction_page_count: int | None = None pipeline_steps: list[PipelineStep] images: list[SourceImageEntry] linked_articles: list[LinkedArticleSummary] diff --git a/src/wikimind/models/tables/ingest.py b/src/wikimind/models/tables/ingest.py index 5d702d46..8318c188 100644 --- a/src/wikimind/models/tables/ingest.py +++ b/src/wikimind/models/tables/ingest.py @@ -33,6 +33,8 @@ class Source(SQLModel, table=True): sa_type=Text, exclude=True, ) # DB-backed source content; excluded from API responses + extraction_engine: str | None = None # e.g. "docling-serve" or "pymupdf" + extraction_page_count: int | None = None # SHA-256 hex digest of the raw payload (issue #67). Used by the ingest # layer to detect duplicates: re-ingesting the same content returns the # existing source instead of creating a second row. diff --git a/tests/unit/test_pdf_adapter.py b/tests/unit/test_pdf_adapter.py index 5de8eab5..613421ea 100644 --- a/tests/unit/test_pdf_adapter.py +++ b/tests/unit/test_pdf_adapter.py @@ -116,6 +116,8 @@ async def test_ingest_falls_back_to_fitz_when_docling_serve_down( assert source.source_type == SourceType.PDF assert source.title == "fallback" assert source.status == IngestStatus.PROCESSING + assert source.extraction_engine == "pymupdf" + assert source.extraction_page_count == 1 assert "Fallback page text" in doc.clean_text assert doc.estimated_tokens > 0 assert doc.chunks # at least one chunk @@ -215,6 +217,8 @@ async def test_ingest_uses_docling_serve_when_available( assert doc.clean_text == markdown assert "# Slide deck" in doc.clean_text + assert source.extraction_engine == "docling-serve" + assert source.extraction_page_count == 1 assert source.file_path == f"{source.id}.txt" assert (isolated_data_dir / "raw" / TEST_USER_ID / f"{source.id}.txt").read_text(encoding="utf-8") == markdown diff --git a/tests/unit/test_source_content.py b/tests/unit/test_source_content.py index a8cfc5e3..4bf007b9 100644 --- a/tests/unit/test_source_content.py +++ b/tests/unit/test_source_content.py @@ -12,7 +12,7 @@ from wikimind.database import get_session from wikimind.errors import NotFoundError from wikimind.main import app -from wikimind.models import Source, SourceContentResponse, SourceType +from wikimind.models import IngestStatus, Source, SourceContentResponse, SourceType from wikimind.services.factories import get_ingest_service from wikimind.services.ingest import IngestService from wikimind.storage import LocalFileStorage @@ -153,6 +153,29 @@ async def test_content_endpoint_passes_user_id() -> None: assert call_kwargs[1]["user_id"] == "test-user-456" +async def test_source_detail_includes_extraction_metadata(client: AsyncClient, db_session) -> None: + """Source detail exposes lightweight PDF extraction metadata.""" + source = Source( + id="src-meta", + source_type=SourceType.PDF, + title="Metadata PDF", + user_id=TEST_USER_ID, + status=IngestStatus.PROCESSING, + clean_text="Extracted text", + extraction_engine="docling-serve", + extraction_page_count=7, + ) + db_session.add(source) + await db_session.commit() + + resp = await client.get("/api/ingest/sources/src-meta/detail") + + assert resp.status_code == 200 + data = resp.json() + assert data["extraction_engine"] == "docling-serve" + assert data["extraction_page_count"] == 7 + + async def test_service_get_source_content_reads_file(tmp_path: Path) -> None: """IngestService.get_source_content reads the text file from storage.""" raw_text = "Original source text here."