From bc6a01c0ee4137dd65fbce390910ebd1fda5af45 Mon Sep 17 00:00:00 2001 From: manavgup Date: Sat, 23 May 2026 20:07:53 -0400 Subject: [PATCH 1/3] feat(ingest): add span re-anchoring and stale-span linter (Phase 5) (#450) When a source is re-ingested, existing spans are matched to new spans by fingerprint so claim references (source_span_ids) stay valid. Unmatched old spans are marked stale. A new linter rule surfaces articles with claims pointing to stale spans as structural warnings. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../0023_add_sourcespan_stale_column.py | 48 ++++ src/wikimind/engine/linter/__init__.py | 3 +- src/wikimind/engine/linter/runner.py | 5 + src/wikimind/engine/linter/stale_spans.py | 120 +++++++++ src/wikimind/ingest/spans.py | 96 ++++++- src/wikimind/models/dto/wiki.py | 1 + src/wikimind/models/tables/wiki.py | 1 + tests/unit/test_source_spans.py | 244 +++++++++++++++++- 8 files changed, 515 insertions(+), 3 deletions(-) create mode 100644 alembic/versions/0023_add_sourcespan_stale_column.py create mode 100644 src/wikimind/engine/linter/stale_spans.py diff --git a/alembic/versions/0023_add_sourcespan_stale_column.py b/alembic/versions/0023_add_sourcespan_stale_column.py new file mode 100644 index 00000000..a75e9322 --- /dev/null +++ b/alembic/versions/0023_add_sourcespan_stale_column.py @@ -0,0 +1,48 @@ +"""Add stale column to sourcespan table. + +Revision ID: 0023 +Revises: 0022 +Create Date: 2026-05-23 + +Adds a boolean ``stale`` column to the ``sourcespan`` table so that spans +whose content no longer matches after a source re-ingestion can be flagged. +Claims referencing stale spans surface as linter warnings. See issue #450, +Phase 5. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy import inspect as sa_inspect + +from alembic import op + +revision: str = "0023" +down_revision: str = "0022" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + conn = op.get_bind() + inspector = sa_inspect(conn) + existing = inspector.get_table_names() + + if "sourcespan" in existing: + columns = [c["name"] for c in inspector.get_columns("sourcespan")] + if "stale" not in columns: + op.add_column( + "sourcespan", + sa.Column("stale", sa.Boolean(), nullable=False, server_default=sa.text("0")), + ) + + +def downgrade() -> None: + conn = op.get_bind() + inspector = sa_inspect(conn) + existing = inspector.get_table_names() + + if "sourcespan" in existing: + columns = [c["name"] for c in inspector.get_columns("sourcespan")] + if "stale" in columns: + op.drop_column("sourcespan", "stale") diff --git a/src/wikimind/engine/linter/__init__.py b/src/wikimind/engine/linter/__init__.py index aea18916..ea0af9dc 100644 --- a/src/wikimind/engine/linter/__init__.py +++ b/src/wikimind/engine/linter/__init__.py @@ -6,5 +6,6 @@ from wikimind.engine.linter.contradictions import detect_contradictions from wikimind.engine.linter.orphans import detect_orphans from wikimind.engine.linter.runner import run_lint +from wikimind.engine.linter.stale_spans import detect_stale_spans -__all__ = ["detect_contradictions", "detect_orphans", "run_lint"] +__all__ = ["detect_contradictions", "detect_orphans", "detect_stale_spans", "run_lint"] diff --git a/src/wikimind/engine/linter/runner.py b/src/wikimind/engine/linter/runner.py index 9f53d39f..412b0e25 100644 --- a/src/wikimind/engine/linter/runner.py +++ b/src/wikimind/engine/linter/runner.py @@ -26,6 +26,7 @@ from wikimind.engine.backlink_enforcer import enforce_backlinks from wikimind.engine.linter.contradictions import detect_contradictions from wikimind.engine.linter.orphans import detect_orphans +from wikimind.engine.linter.stale_spans import detect_stale_spans from wikimind.engine.linter.staleness import detect_stale_articles from wikimind.engine.llm_router import get_llm_router from wikimind.models import ( @@ -379,6 +380,10 @@ async def run_lint( stale_findings = await detect_stale_articles(session, settings, report.id, user_id=user_id) structurals.extend(stale_findings) + # Phase 5: Stale source-span detection (issue #450) + stale_span_findings = await detect_stale_spans(session, report.id, user_id=user_id) + structurals.extend(stale_span_findings) + # Apply dismiss suppression await _apply_dismiss_suppression(session, contradictions, orphans, structurals) diff --git a/src/wikimind/engine/linter/stale_spans.py b/src/wikimind/engine/linter/stale_spans.py new file mode 100644 index 00000000..ded68e7a --- /dev/null +++ b/src/wikimind/engine/linter/stale_spans.py @@ -0,0 +1,120 @@ +"""Stale-span detection — surface articles with claims pointing to stale source spans. + +When a source is re-ingested and some paragraphs no longer match, the +corresponding ``SourceSpan`` rows are marked ``stale=True``. Claims that +still reference those stale spans lose their citation anchor. This check +generates :class:`StructuralFinding` lint warnings so users know which +articles need attention. +""" + +from __future__ import annotations + +import hashlib +import json +from typing import TYPE_CHECKING + +import structlog +from sqlmodel import select + +from wikimind.models import ( + Article, + CompiledClaim, + LintFindingKind, + LintSeverity, + SourceSpan, + StructuralFinding, +) + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + +log = structlog.get_logger() + +VIOLATION_TYPE = "stale_source_spans" + + +def _content_hash(article_id: str) -> str: + """Compute a stable sha256 for cross-run dedup of stale-span findings.""" + raw = f"{LintFindingKind.STRUCTURAL}|{article_id}|{VIOLATION_TYPE}" + return hashlib.sha256(raw.encode()).hexdigest() + + +async def detect_stale_spans( + session: AsyncSession, + report_id: str, + user_id: str, +) -> list[StructuralFinding]: + """Find articles with claims pointing to stale source spans. + + For each article, loads its compiled claims and checks whether any + ``source_span_ids`` reference a span whose ``stale`` flag is True. + + Args: + session: Async database session. + report_id: The parent LintReport ID. + user_id: User ID for data isolation. + + Returns: + List of StructuralFinding instances for articles with stale span refs. + """ + # Load all stale span IDs for this user in one query + stale_stmt = ( + select(SourceSpan.id).where(SourceSpan.user_id == user_id).where(SourceSpan.stale.is_(True)) # type: ignore[attr-defined] + ) + stale_result = await session.execute(stale_stmt) + stale_span_ids: set[str] = {row[0] for row in stale_result.all()} + + if not stale_span_ids: + log.info("Stale-span detection: no stale spans found") + return [] + + # Load all claims for this user that have span references + claim_stmt = ( + select(CompiledClaim.article_id, CompiledClaim.source_span_ids) + .where(CompiledClaim.user_id == user_id) + .where(CompiledClaim.source_span_ids != "[]") + ) + claim_result = await session.execute(claim_stmt) + claim_rows = claim_result.all() + + # Group stale-span-referencing claims by article + articles_with_stale: dict[str, int] = {} + for article_id, span_ids_json in claim_rows: + try: + span_ids = json.loads(span_ids_json) + except (json.JSONDecodeError, TypeError): + continue + stale_count = sum(1 for sid in span_ids if sid in stale_span_ids) + if stale_count > 0: + articles_with_stale[article_id] = articles_with_stale.get(article_id, 0) + stale_count + + if not articles_with_stale: + log.info("Stale-span detection: no claims reference stale spans") + return [] + + # Look up article titles for readable descriptions + article_stmt = select(Article.id, Article.title).where( + Article.id.in_(list(articles_with_stale.keys())), # type: ignore[attr-defined] + ) + article_result = await session.execute(article_stmt) + article_titles: dict[str, str] = {row[0]: row[1] for row in article_result.all()} + + findings: list[StructuralFinding] = [] + for article_id, stale_count in articles_with_stale.items(): + title = article_titles.get(article_id, article_id) + findings.append( + StructuralFinding( + report_id=report_id, + severity=LintSeverity.WARN, + description=(f"Article '{title}' has {stale_count} claim(s) referencing stale source spans"), + content_hash=_content_hash(article_id), + article_id=article_id, + violation_type=VIOLATION_TYPE, + auto_repaired=False, + detail=f"stale_span_references={stale_count}", + user_id=user_id, + ) + ) + + log.info("Stale-span detection complete", articles_affected=len(findings)) + return findings diff --git a/src/wikimind/ingest/spans.py b/src/wikimind/ingest/spans.py index 986d831b..ed1ec54a 100644 --- a/src/wikimind/ingest/spans.py +++ b/src/wikimind/ingest/spans.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING import structlog +from sqlmodel import select from wikimind.models.enums import LocatorKind from wikimind.models.tables.wiki import SourceSpan @@ -245,13 +246,106 @@ async def persist_spans( ) -> None: """Persist a batch of SourceSpan instances to the database. + If the source already has spans in the database, delegates to + :func:`reanchor_spans` for fingerprint-based matching so that + existing span IDs (and thus claim references) are preserved. + Args: spans: List of SourceSpan instances to save. session: Async database session. """ if not spans: return + + # Check if this source already has spans — if so, re-anchor + source_id = spans[0].source_id + existing_stmt = select(SourceSpan).where(SourceSpan.source_id == source_id) + existing_result = await session.execute(existing_stmt) + existing_spans = list(existing_result.scalars().all()) + + if existing_spans: + await reanchor_spans(source_id, spans, session) + return + for span in spans: session.add(span) await session.flush() - log.info("Persisted source spans", count=len(spans), source_id=spans[0].source_id) + log.info("Persisted source spans", count=len(spans), source_id=source_id) + + +# --------------------------------------------------------------------------- +# Re-anchoring on source update +# --------------------------------------------------------------------------- + + +async def reanchor_spans( + source_id: str, + new_spans: list[SourceSpan], + session: AsyncSession, +) -> list[SourceSpan]: + """Re-anchor existing spans after a source is re-ingested. + + Matches old spans to new spans by fingerprint so that claim references + (via ``CompiledClaim.source_span_ids``) remain valid after content + updates. Unmatched old spans are marked stale; genuinely new spans + are created normally. + + Args: + source_id: The source whose spans are being refreshed. + new_spans: Freshly extracted spans from the updated content. + session: Async database session. + + Returns: + The final list of spans (updated + new) that were persisted. + """ + # Load existing spans for this source + stmt = select(SourceSpan).where(SourceSpan.source_id == source_id) + result = await session.execute(stmt) + old_spans = list(result.scalars().all()) + + if not old_spans: + # No existing spans — just persist the new ones directly + await persist_spans(new_spans, session) + return new_spans + + # Build lookup from fingerprint -> old span (first match wins) + old_by_fingerprint: dict[str, SourceSpan] = {} + for span in old_spans: + if span.fingerprint not in old_by_fingerprint: + old_by_fingerprint[span.fingerprint] = span + + matched_old_ids: set[str] = set() + final_spans: list[SourceSpan] = [] + + for new_span in new_spans: + old_span = old_by_fingerprint.get(new_span.fingerprint) + if old_span and old_span.id not in matched_old_ids: + # Matched: update locator to new position, keep same ID + old_span.locator = new_span.locator + old_span.text = new_span.text + old_span.stale = False + session.add(old_span) + matched_old_ids.add(old_span.id) + final_spans.append(old_span) + else: + # Genuinely new span — persist it + session.add(new_span) + final_spans.append(new_span) + + # Mark unmatched old spans as stale + stale_count = 0 + for old_span in old_spans: + if old_span.id not in matched_old_ids: + old_span.stale = True + session.add(old_span) + stale_count += 1 + + await session.flush() + log.info( + "Re-anchored source spans", + source_id=source_id, + matched=len(matched_old_ids), + new=len(final_spans) - len(matched_old_ids), + stale=stale_count, + ) + return final_spans diff --git a/src/wikimind/models/dto/wiki.py b/src/wikimind/models/dto/wiki.py index 97de9643..8f1da8b8 100644 --- a/src/wikimind/models/dto/wiki.py +++ b/src/wikimind/models/dto/wiki.py @@ -405,6 +405,7 @@ class SourceSpanResponse(BaseModel): locator: dict text: str fingerprint: str + stale: bool = False created_at: datetime diff --git a/src/wikimind/models/tables/wiki.py b/src/wikimind/models/tables/wiki.py index 96ae39d0..3727360c 100644 --- a/src/wikimind/models/tables/wiki.py +++ b/src/wikimind/models/tables/wiki.py @@ -28,6 +28,7 @@ class SourceSpan(SQLModel, table=True): locator: dict = Field(sa_column=Column(JSON, nullable=False)) # adapter-specific anchor text: str = Field(sa_type=Text) # verbatim quoted text fingerprint: str = Field(index=True) # SHA-256 of normalized text for re-anchoring + stale: bool = Field(default=False) # True when source re-ingested and span no longer matches created_at: datetime = Field(default_factory=utcnow_naive) diff --git a/tests/unit/test_source_spans.py b/tests/unit/test_source_spans.py index 60a3aa00..e1597d04 100644 --- a/tests/unit/test_source_spans.py +++ b/tests/unit/test_source_spans.py @@ -1,7 +1,8 @@ """Tests for span-level citation extraction and fingerprinting (issue #450). Covers the fingerprint utility, span extraction functions for each adapter, -the SourceSpan persistence, and the GET /api/sources/{id}/spans endpoint. +the SourceSpan persistence, re-anchoring on source updates, and the +GET /api/sources/{id}/spans endpoint. """ from __future__ import annotations @@ -9,6 +10,7 @@ import uuid import pytest +from sqlmodel import select from sqlmodel.ext.asyncio.session import AsyncSession from tests.conftest import TEST_USER_ID @@ -19,6 +21,7 @@ extract_url_spans, normalize_text, persist_spans, + reanchor_spans, ) from wikimind.models import LocatorKind, Source, SourceSpan @@ -229,6 +232,245 @@ async def test_persist_empty_list(self, db_session: AsyncSession) -> None: # --------------------------------------------------------------------------- +class TestReanchorSpans: + """Verify re-anchoring preserves span IDs and marks stale spans.""" + + @pytest.mark.asyncio + async def test_matching_fingerprint_preserves_id(self, db_session: AsyncSession) -> None: + """Spans with matching fingerprints keep their original IDs.""" + source_id = _uid() + source = Source( + id=source_id, + user_id=TEST_USER_ID, + source_type="text", + title="Test Source", + ) + db_session.add(source) + await db_session.flush() + + # Create original spans + old_spans = extract_text_spans("Hello world.\n\nGoodbye world.", source_id, TEST_USER_ID) + for span in old_spans: + db_session.add(span) + await db_session.flush() + + original_ids = {s.id for s in old_spans} + original_fingerprints = {s.fingerprint for s in old_spans} + + # Re-ingest with same paragraphs in different positions + new_text = "Extra paragraph.\n\nHello world.\n\nGoodbye world." + new_spans = extract_text_spans(new_text, source_id, TEST_USER_ID) + + result = await reanchor_spans(source_id, new_spans, db_session) + await db_session.flush() + + # The two matching paragraphs should keep their original IDs + result_ids = {s.id for s in result if s.fingerprint in original_fingerprints} + assert result_ids == original_ids + + @pytest.mark.asyncio + async def test_unmatched_old_spans_become_stale(self, db_session: AsyncSession) -> None: + """Old spans with no matching fingerprint are marked stale.""" + source_id = _uid() + source = Source( + id=source_id, + user_id=TEST_USER_ID, + source_type="text", + title="Test Source", + ) + db_session.add(source) + await db_session.flush() + + # Create original spans + old_spans = extract_text_spans("Will be removed.\n\nStays the same.", source_id, TEST_USER_ID) + removed_id = old_spans[0].id + for span in old_spans: + db_session.add(span) + await db_session.flush() + + # Re-ingest without the first paragraph + new_spans = extract_text_spans("Stays the same.\n\nBrand new.", source_id, TEST_USER_ID) + await reanchor_spans(source_id, new_spans, db_session) + await db_session.flush() + + # The removed paragraph's span should be stale + stmt = select(SourceSpan).where(SourceSpan.id == removed_id) + result = await db_session.execute(stmt) + stale_span = result.scalar_one() + assert stale_span.stale is True + + @pytest.mark.asyncio + async def test_new_spans_are_created(self, db_session: AsyncSession) -> None: + """Spans with no matching fingerprint in old set are created normally.""" + source_id = _uid() + source = Source( + id=source_id, + user_id=TEST_USER_ID, + source_type="text", + title="Test Source", + ) + db_session.add(source) + await db_session.flush() + + # Create original spans + old_spans = extract_text_spans("Old paragraph.", source_id, TEST_USER_ID) + for span in old_spans: + db_session.add(span) + await db_session.flush() + + old_id = old_spans[0].id + + # Re-ingest with a new paragraph added + new_spans = extract_text_spans("Old paragraph.\n\nBrand new paragraph.", source_id, TEST_USER_ID) + result = await reanchor_spans(source_id, new_spans, db_session) + await db_session.flush() + + assert len(result) == 2 + # Old span preserved + assert any(s.id == old_id for s in result) + # New span is genuinely new (different ID) + new_ids = {s.id for s in result} - {old_id} + assert len(new_ids) == 1 + + @pytest.mark.asyncio + async def test_persist_spans_auto_reanchors(self, db_session: AsyncSession) -> None: + """persist_spans delegates to reanchor when existing spans are found.""" + source_id = _uid() + source = Source( + id=source_id, + user_id=TEST_USER_ID, + source_type="text", + title="Test Source", + ) + db_session.add(source) + await db_session.flush() + + # First persist — normal path + spans_v1 = extract_text_spans("Paragraph A.\n\nParagraph B.", source_id, TEST_USER_ID) + await persist_spans(spans_v1, db_session) + await db_session.flush() + + original_ids = {s.id for s in spans_v1} + + # Second persist — should auto-reanchor + spans_v2 = extract_text_spans("Paragraph A.\n\nParagraph C.", source_id, TEST_USER_ID) + await persist_spans(spans_v2, db_session) + await db_session.flush() + + # Check Paragraph A kept its ID + stmt = select(SourceSpan).where(SourceSpan.source_id == source_id) + result = await db_session.execute(stmt) + all_spans = list(result.scalars().all()) + + a_spans = [s for s in all_spans if "paragraph a" in s.text.lower()] + assert len(a_spans) == 1 + assert a_spans[0].id in original_ids + assert a_spans[0].stale is False + + # Check Paragraph B is stale + b_spans = [s for s in all_spans if "paragraph b" in s.text.lower()] + assert len(b_spans) == 1 + assert b_spans[0].stale is True + + @pytest.mark.asyncio + async def test_reanchor_no_existing_spans(self, db_session: AsyncSession) -> None: + """When no existing spans exist, reanchor just persists normally.""" + source_id = _uid() + source = Source( + id=source_id, + user_id=TEST_USER_ID, + source_type="text", + title="Test Source", + ) + db_session.add(source) + await db_session.flush() + + new_spans = extract_text_spans("Brand new text.", source_id, TEST_USER_ID) + result = await reanchor_spans(source_id, new_spans, db_session) + + assert len(result) == 1 + assert result[0].text == "Brand new text." + + +# --------------------------------------------------------------------------- +# Stale-span linter tests +# --------------------------------------------------------------------------- + + +class TestDetectStaleSpans: + """Verify stale-span linter detection.""" + + @pytest.mark.asyncio + async def test_no_stale_spans_returns_empty(self, db_session: AsyncSession) -> None: + from wikimind.engine.linter.stale_spans import detect_stale_spans + + findings = await detect_stale_spans(db_session, "report-1", TEST_USER_ID) + assert findings == [] + + @pytest.mark.asyncio + async def test_stale_spans_with_claim_refs(self, db_session: AsyncSession) -> None: + """Claims referencing stale spans produce linter findings.""" + import json + + from wikimind.engine.linter.stale_spans import detect_stale_spans + from wikimind.models import Article, CompiledClaim + + source_id = _uid() + source = Source( + id=source_id, + user_id=TEST_USER_ID, + source_type="text", + title="Test Source", + ) + db_session.add(source) + + # Create a stale span + span_id = _uid() + span = SourceSpan( + id=span_id, + source_id=source_id, + user_id=TEST_USER_ID, + locator_kind=LocatorKind.TEXT_BYTE_RANGE, + locator={"start": 0, "end": 10}, + text="Old text.", + fingerprint=compute_fingerprint("Old text."), + stale=True, + ) + db_session.add(span) + + # Create article and claim referencing the stale span + article_id = _uid() + article = Article( + id=article_id, + user_id=TEST_USER_ID, + slug="test-article", + title="Test Article", + file_path="test.md", + ) + db_session.add(article) + + claim = CompiledClaim( + article_id=article_id, + user_id=TEST_USER_ID, + text="Some claim text", + confidence_level="sourced", + source_span_ids=json.dumps([span_id]), + ) + db_session.add(claim) + await db_session.flush() + + findings = await detect_stale_spans(db_session, "report-1", TEST_USER_ID) + assert len(findings) == 1 + assert findings[0].article_id == article_id + assert findings[0].violation_type == "stale_source_spans" + assert "1 claim(s) referencing stale source spans" in findings[0].description + + +# --------------------------------------------------------------------------- +# API endpoint tests +# --------------------------------------------------------------------------- + + class TestSourceSpansEndpoint: """Verify GET /api/sources/{id}/spans endpoint.""" From c0250cf046daa95e113b70e629f98f62d63fbb0c Mon Sep 17 00:00:00 2001 From: manavgup Date: Sat, 23 May 2026 20:23:28 -0400 Subject: [PATCH 2/3] fix: use boolean default in stale column migration Co-Authored-By: Claude Opus 4.6 (1M context) --- alembic/versions/0023_add_sourcespan_stale_column.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alembic/versions/0023_add_sourcespan_stale_column.py b/alembic/versions/0023_add_sourcespan_stale_column.py index a75e9322..aa1e0b70 100644 --- a/alembic/versions/0023_add_sourcespan_stale_column.py +++ b/alembic/versions/0023_add_sourcespan_stale_column.py @@ -33,7 +33,7 @@ def upgrade() -> None: if "stale" not in columns: op.add_column( "sourcespan", - sa.Column("stale", sa.Boolean(), nullable=False, server_default=sa.text("0")), + sa.Column("stale", sa.Boolean(), nullable=False, server_default=sa.text("false")), ) From c2e910126330cf3a32dbef13d59ea275db01e82b Mon Sep 17 00:00:00 2001 From: manavgup Date: Mon, 25 May 2026 10:27:00 -0400 Subject: [PATCH 3/3] fix: line length, inline imports, and redundant DB query (#450) Co-Authored-By: Claude Opus 4.6 (1M context) --- src/wikimind/engine/linter/stale_spans.py | 8 +++++--- src/wikimind/ingest/spans.py | 17 +++++++++++----- tests/unit/test_source_spans.py | 24 ++++++++--------------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/wikimind/engine/linter/stale_spans.py b/src/wikimind/engine/linter/stale_spans.py index ded68e7a..490c4ac6 100644 --- a/src/wikimind/engine/linter/stale_spans.py +++ b/src/wikimind/engine/linter/stale_spans.py @@ -58,8 +58,9 @@ async def detect_stale_spans( List of StructuralFinding instances for articles with stale span refs. """ # Load all stale span IDs for this user in one query - stale_stmt = ( - select(SourceSpan.id).where(SourceSpan.user_id == user_id).where(SourceSpan.stale.is_(True)) # type: ignore[attr-defined] + stale_stmt = select(SourceSpan.id).where( + SourceSpan.user_id == user_id, + SourceSpan.stale.is_(True), # type: ignore[attr-defined] ) stale_result = await session.execute(stale_stmt) stale_span_ids: set[str] = {row[0] for row in stale_result.all()} @@ -102,11 +103,12 @@ async def detect_stale_spans( findings: list[StructuralFinding] = [] for article_id, stale_count in articles_with_stale.items(): title = article_titles.get(article_id, article_id) + desc = f"Article '{title}' has {stale_count} claim(s) referencing stale source spans" findings.append( StructuralFinding( report_id=report_id, severity=LintSeverity.WARN, - description=(f"Article '{title}' has {stale_count} claim(s) referencing stale source spans"), + description=desc, content_hash=_content_hash(article_id), article_id=article_id, violation_type=VIOLATION_TYPE, diff --git a/src/wikimind/ingest/spans.py b/src/wikimind/ingest/spans.py index ed1ec54a..a588c8e9 100644 --- a/src/wikimind/ingest/spans.py +++ b/src/wikimind/ingest/spans.py @@ -264,7 +264,7 @@ async def persist_spans( existing_spans = list(existing_result.scalars().all()) if existing_spans: - await reanchor_spans(source_id, spans, session) + await reanchor_spans(source_id, spans, session, existing_spans=existing_spans) return for span in spans: @@ -282,6 +282,8 @@ async def reanchor_spans( source_id: str, new_spans: list[SourceSpan], session: AsyncSession, + *, + existing_spans: list[SourceSpan] | None = None, ) -> list[SourceSpan]: """Re-anchor existing spans after a source is re-ingested. @@ -294,14 +296,19 @@ async def reanchor_spans( source_id: The source whose spans are being refreshed. new_spans: Freshly extracted spans from the updated content. session: Async database session. + existing_spans: Pre-loaded existing spans to avoid a redundant + database query when the caller already has them. Returns: The final list of spans (updated + new) that were persisted. """ - # Load existing spans for this source - stmt = select(SourceSpan).where(SourceSpan.source_id == source_id) - result = await session.execute(stmt) - old_spans = list(result.scalars().all()) + if existing_spans is not None: + old_spans = existing_spans + else: + # Load existing spans for this source + stmt = select(SourceSpan).where(SourceSpan.source_id == source_id) + result = await session.execute(stmt) + old_spans = list(result.scalars().all()) if not old_spans: # No existing spans — just persist the new ones directly diff --git a/tests/unit/test_source_spans.py b/tests/unit/test_source_spans.py index e1597d04..67447df3 100644 --- a/tests/unit/test_source_spans.py +++ b/tests/unit/test_source_spans.py @@ -7,13 +7,16 @@ from __future__ import annotations +import json import uuid import pytest +from sqlalchemy.ext.asyncio import async_sessionmaker from sqlmodel import select from sqlmodel.ext.asyncio.session import AsyncSession from tests.conftest import TEST_USER_ID +from wikimind.engine.linter.stale_spans import detect_stale_spans from wikimind.ingest.spans import ( compute_fingerprint, extract_pdf_spans, @@ -23,7 +26,7 @@ persist_spans, reanchor_spans, ) -from wikimind.models import LocatorKind, Source, SourceSpan +from wikimind.models import Article, CompiledClaim, LocatorKind, Source, SourceSpan def _uid() -> str: @@ -214,8 +217,6 @@ async def test_persist_spans(self, db_session: AsyncSession) -> None: await db_session.flush() # Read back - from sqlmodel import select - stmt = select(SourceSpan).where(SourceSpan.source_id == source_id) result = (await db_session.exec(stmt)).all() assert len(result) == 2 @@ -282,7 +283,8 @@ async def test_unmatched_old_spans_become_stale(self, db_session: AsyncSession) await db_session.flush() # Create original spans - old_spans = extract_text_spans("Will be removed.\n\nStays the same.", source_id, TEST_USER_ID) + text = "Will be removed.\n\nStays the same." + old_spans = extract_text_spans(text, source_id, TEST_USER_ID) removed_id = old_spans[0].id for span in old_spans: db_session.add(span) @@ -321,7 +323,8 @@ async def test_new_spans_are_created(self, db_session: AsyncSession) -> None: old_id = old_spans[0].id # Re-ingest with a new paragraph added - new_spans = extract_text_spans("Old paragraph.\n\nBrand new paragraph.", source_id, TEST_USER_ID) + new_text = "Old paragraph.\n\nBrand new paragraph." + new_spans = extract_text_spans(new_text, source_id, TEST_USER_ID) result = await reanchor_spans(source_id, new_spans, db_session) await db_session.flush() @@ -402,19 +405,12 @@ class TestDetectStaleSpans: @pytest.mark.asyncio async def test_no_stale_spans_returns_empty(self, db_session: AsyncSession) -> None: - from wikimind.engine.linter.stale_spans import detect_stale_spans - findings = await detect_stale_spans(db_session, "report-1", TEST_USER_ID) assert findings == [] @pytest.mark.asyncio async def test_stale_spans_with_claim_refs(self, db_session: AsyncSession) -> None: """Claims referencing stale spans produce linter findings.""" - import json - - from wikimind.engine.linter.stale_spans import detect_stale_spans - from wikimind.models import Article, CompiledClaim - source_id = _uid() source = Source( id=source_id, @@ -481,8 +477,6 @@ async def test_spans_endpoint_not_found(self, client) -> None: @pytest.mark.asyncio async def test_spans_endpoint_empty(self, client, async_engine) -> None: - from sqlalchemy.ext.asyncio import async_sessionmaker - factory = async_sessionmaker(async_engine, class_=AsyncSession, expire_on_commit=False) source_id = _uid() async with factory() as session: @@ -501,8 +495,6 @@ async def test_spans_endpoint_empty(self, client, async_engine) -> None: @pytest.mark.asyncio async def test_spans_endpoint_with_spans(self, client, async_engine) -> None: - from sqlalchemy.ext.asyncio import async_sessionmaker - factory = async_sessionmaker(async_engine, class_=AsyncSession, expire_on_commit=False) source_id = _uid()