Audit/mesa data autonomy final

.mesa_data_agent_state.json

@@ -4,7 +4,7 @@
   "workflow": "mesa_data_autonomy",
   "branch": "audit/mesa-data-autonomy-final",
   "baseline_commit": "84c61db716c10cb4110b673f9a976215afe71cd3",
-  "current_phase": "complete",
+  "current_phase": "correctness_hardening",
   "overall_status": "partially_ready",
   "phases": {
     "prompt_1": {
@@ -191,6 +191,44 @@
       ],
       "blockers": []
     },
+    "correctness_hardening": {
+      "status": "completed",
+      "started_at": "2026-08-29T00:00:00Z",
+      "completed_at": "2026-08-29T00:00:00Z",
+      "findings": [
+        {"id": "QG_ZERO_ARTICLE_PASS", "severity": "P0", "status": "fixed"},
+        {"id": "PUBLICATION_DATE_BRIDGE_LOSS", "severity": "P0", "status": "fixed"},
+        {"id": "STALE_APPROVED_VERSION_PUBLISH", "severity": "P0", "status": "fixed"},
+        {"id": "CURRENT_VERSION_MESA_STATUS", "severity": "P0", "status": "fixed"},
+        {"id": "DERIVED_REPROCESS_EARLY_RETURN", "severity": "P0", "status": "fixed"},
+        {"id": "MESA_TARGET_KEY_EXFILTRATION", "severity": "P0", "status": "fixed"},
+        {"id": "MUTATION_TIMEOUT_MARKED_FAILED", "severity": "P1", "status": "fixed"},
+        {"id": "ARTIFACT_SHA_CROSS_DOCUMENT_COLLISION", "severity": "P1", "status": "fixed"}
+      ],
+      "fixes": [
+        "Legislation zero-article and abnormal large uncovered-area outcomes cannot PASS; article ordinal is retained in canonical records.",
+        "Discovery publication_date is forwarded into artifact metadata and canonical provenance; retrieval time remains fallback-only.",
+        "Release and MESA selection now joins documents.current_version_id, excluding stale approved versions and unrelated blockers.",
+        "Reprocess refreshes only derived output, resets approval, and preserves version identity and revision number.",
+        "MESA requests require local development host or HTTPS host exactly allowed by MESA_DATA_MESA_ALLOWED_HOST; 401/403 are reachable but unauthenticated failures.",
+        "Pending mutation timeouts use AWAITING_MUTATION and schema support; retries poll known mutation IDs before resubmitting.",
+        "Same payload for another document produces an explicit ARTIFACT_DOCUMENT_COLLISION rather than reassociation."
+      ],
+      "commits": [
+        "a6656bb fix: harden parsing and version correctness",
+        "c20aa17 fix: secure current-version MESA publishing",
+        "71f4306 fix: report MESA state from current version"
+      ],
+      "migrations": ["0009_correctness_hardening.sql"],
+      "tests": [
+        {"name": "focused correctness regression", "command": "MESA_DATA_MESA_ALLOWED_HOST=mock-mesa.internal UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run pytest -q tests/unit/test_correctness_hardening.py tests/unit/test_quality_gate.py tests/unit/test_catalog_release_iterator.py tests/unit/test_publisher_client_and_ledger.py tests/integration/test_mesa_v4_publisher_e2e.py", "result": "19 passed"},
+        {"name": "compile", "command": "UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run python -m compileall -q src/mesa_legal_data", "result": "passed"},
+        {"name": "ruff", "command": "UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run ruff check src tests", "result": "passed"}
+        ,{"name": "current-version MESA status regression", "command": "MESA_DATA_MESA_ALLOWED_HOST=mock-mesa.internal UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run pytest -q tests/unit/test_correctness_hardening.py tests/unit/test_publisher_client_and_ledger.py", "result": "10 passed"}
+      ],
+      "blockers": ["The all-test pytest invocation did not complete within the desktop command window and is not recorded as passed."],
+      "unverified": ["MESA live HTTP payload and server-side mutation contract remain UNVERIFIED because no live endpoint or MESA source code is available."]
+    },
     "prompt_5": {
       "status": "completed",
       "completed_at": "2026-08-29T15:13:48Z",
@@ -266,5 +304,5 @@
       "Configure documented MESA routes and verify health, publish, polling, COMMITTED truth, and server-side idempotency against a live endpoint."
     ]
   },
-  "last_updated_at": "2026-08-29T15:13:48Z"
+  "last_updated_at": "2026-08-29T00:00:00Z"
 }

migrations/0009_correctness_hardening.sql

@@ -0,0 +1,31 @@
+-- Preserve state truth: a local polling timeout is pending, not failure.
+PRAGMA foreign_keys = OFF;
+
+CREATE TABLE mesa_delivery_items_v2 (
+    item_id TEXT PRIMARY KEY,
+    delivery_id TEXT NOT NULL,
+    document_id TEXT NOT NULL,
+    version_id TEXT NOT NULL,
+    chunk_id TEXT NOT NULL,
+    content_hash TEXT NOT NULL,
+    idempotency_key TEXT NOT NULL,
+    remote_mutation_id TEXT,
+    remote_state TEXT NOT NULL CHECK (remote_state IN ('PLANNED', 'SENDING', 'QUEUED', 'PROCESSING', 'AWAITING_MUTATION', 'COMMITTED', 'FAILED', 'REJECTED', 'SKIPPED')),
+    payload_json TEXT NOT NULL,
+    last_error TEXT,
+    updated_at TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    FOREIGN KEY (delivery_id) REFERENCES mesa_deliveries(delivery_id)
+);
+INSERT INTO mesa_delivery_items_v2 SELECT * FROM mesa_delivery_items;
+DROP TABLE mesa_delivery_items;
+ALTER TABLE mesa_delivery_items_v2 RENAME TO mesa_delivery_items;
+CREATE INDEX idx_mesa_items_delivery_state ON mesa_delivery_items(delivery_id, remote_state);
+CREATE INDEX idx_mesa_items_doc_ver_chunk ON mesa_delivery_items(document_id, version_id, chunk_id, content_hash);
+CREATE INDEX idx_mesa_items_idempotency ON mesa_delivery_items(idempotency_key);
+CREATE INDEX idx_mesa_items_remote_state ON mesa_delivery_items(remote_state);
+
+-- Do not silently merge historical collisions: applying this migration fails
+-- visibly if existing data violates the invariant.
+CREATE UNIQUE INDEX idx_versions_document_revision_unique ON versions(document_id, revision_number);
+PRAGMA foreign_keys = ON;

schemas/article.schema.json

@@ -22,6 +22,7 @@
     "article_number": { "type": "string" },
     "article_kind": { "type": "string" },
     "heading": { "type": ["string", "null"] },
+    "ordinal": { "type": ["integer", "null"], "minimum": 1 },
     "text": { "type": "string" },
     "structure": {
       "type": ["object", "null"],
@@ -55,6 +56,7 @@
         "source_id": { "type": "string" },
         "source_url": { "type": "string" },
         "retrieved_at": { "type": "string" },
+        "publication_date": { "type": ["string", "null"] },
         "artifact_sha256": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" },
         "artifact_path": { "type": ["string", "null"] }
       },

schemas/citation.schema.json

@@ -42,6 +42,7 @@
         "source_id": { "type": "string" },
         "source_url": { "type": "string" },
         "retrieved_at": { "type": "string" },
+        "publication_date": { "type": ["string", "null"] },
         "artifact_sha256": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" },
         "artifact_path": { "type": ["string", "null"] }
       },

schemas/common.schema.json

@@ -25,6 +25,7 @@
         "source_id": { "type": "string" },
         "source_url": { "type": "string" },
         "retrieved_at": { "type": "string" },
+        "publication_date": { "type": ["string", "null"] },
         "artifact_sha256": { "type": "string" },
         "artifact_path": { "type": ["string", "null"] }
       },

schemas/decision.schema.json

@@ -35,6 +35,7 @@
         "source_id": { "type": "string" },
         "source_url": { "type": "string" },
         "retrieved_at": { "type": "string" },
+        "publication_date": { "type": ["string", "null"] },
         "artifact_sha256": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" },
         "artifact_path": { "type": ["string", "null"] }
       },

schemas/legislation.schema.json

@@ -57,6 +57,7 @@
         "source_id": { "type": "string" },
         "source_url": { "type": "string" },
         "retrieved_at": { "type": "string" },
+        "publication_date": { "type": ["string", "null"] },
         "artifact_sha256": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" },
         "artifact_path": { "type": ["string", "null"] }
       },

src/mesa_legal_data/catalog.py

@@ -462,6 +462,54 @@ def insert_version(
         )
 
 
+def replace_derived_version_output(
+    conn: sqlite3.Connection,
+    *,
+    version_id: str,
+    document_id: str,
+    artifact_id: str,
+    canonical_path: str,
+    canonical_line: int,
+    canonical_sha256: str,
+    parser_name: str,
+    parser_version: str,
+    validation_status: str,
+    privacy_status: str,
+    quality_status: str,
+    quality_json: str,
+) -> None:
+    """Replace only reproducible output for one immutable legal version."""
+    current = get_version(conn, version_id)
+    if not current:
+        raise CatalogError(f"Cannot reprocess unknown version {version_id}")
+    if current["document_id"] != document_id or current["artifact_id"] != artifact_id:
+        raise CatalogError(f"Immutable version identity mismatch for {version_id}")
+
+    with transaction(conn):
+        conn.execute(
+            """UPDATE versions
+               SET canonical_path = ?, canonical_line = ?, canonical_sha256 = ?,
+                   parser_name = ?, parser_version = ?, validation_status = ?,
+                   privacy_status = ?, approval_status = 'pending',
+                   quality_status = ?, quality_json = ?, auto_approved = 0,
+                   is_audit_sample = 0, audit_sample_reason = NULL
+               WHERE version_id = ?""",
+            (
+                canonical_path,
+                canonical_line,
+                canonical_sha256,
+                parser_name,
+                parser_version,
+                validation_status,
+                privacy_status,
+                quality_status,
+                quality_json,
+                version_id,
+            ),
+        )
+        conn.execute("DELETE FROM records WHERE version_id = ?", (version_id,))
+
+
 def get_version(conn: sqlite3.Connection, version_id: str) -> dict[str, Any] | None:
     cursor = conn.cursor()
     cursor.execute(
@@ -649,20 +697,17 @@ def iter_records_for_release(
     cursor = conn.cursor()
     cursor.execute("""
         WITH eligible_versions AS (
-            SELECT v.version_id,
-                   ROW_NUMBER() OVER (
-                       PARTITION BY v.document_id
-                       ORDER BY COALESCE(v.revision_number, 1) DESC, v.created_at DESC
-                   ) AS version_rank
-            FROM versions v
+            SELECT v.version_id
+            FROM documents d
+            JOIN versions v ON v.version_id = d.current_version_id
             WHERE v.approval_status = 'approved'
               AND v.validation_status = 'valid'
-              AND (v.quality_status IS NULL OR v.quality_status != 'BLOCK')
+              AND v.quality_status = 'PASS'
               AND v.privacy_status IN ('clean', 'approved')
         )
         SELECT r.record_id, r.record_type, r.record_sha256, r.canonical_path, r.canonical_line, r.version_id
         FROM records r
-        JOIN eligible_versions ev ON r.version_id = ev.version_id AND ev.version_rank = 1
+        JOIN eligible_versions ev ON r.version_id = ev.version_id
         WHERE r.approval_status = 'approved'
           AND r.validation_status = 'valid'
         ORDER BY r.canonical_path ASC, r.canonical_line ASC

src/mesa_legal_data/harvest/runner.py

@@ -353,6 +353,11 @@ def _run_harvest_batch_impl(
                 current_attempt, sbe.code, max_attempts=harvest_cfg.runner.max_attempts
             )
 
+            # A content-address collision across logical documents is a data
+            # correctness conflict, not a transient transport condition.
+            if sbe.code == "ARTIFACT_DOCUMENT_COLLISION":
+                should_retry = False
+
             if should_retry:
                 update_item_status(
                     item_id,
@@ -366,7 +371,12 @@ def _run_harvest_batch_impl(
             else:
                 target_terminal = (
                     ItemStatus.BLOCKED
-                    if sbe.code in ("SOURCE_HOST_NOT_ALLOWED", "PRIVATE_IP_NOT_ALLOWED")
+                    if sbe.code
+                    in (
+                        "SOURCE_HOST_NOT_ALLOWED",
+                        "PRIVATE_IP_NOT_ALLOWED",
+                        "ARTIFACT_DOCUMENT_COLLISION",
+                    )
                     else ItemStatus.FAILED
                 )
                 update_item_status(

src/mesa_legal_data/harvest/service_bridge.py

@@ -4,7 +4,7 @@
 from mesa_legal_data.harvest.models import CollectResult, HarvestItem, PipelineResult
 from mesa_legal_data.pipeline import InvalidStateTransition, process_artifact_pipeline
 from mesa_legal_data.schema_validation import SchemaValidationError
-from mesa_legal_data.sources.manual import import_manual_url
+from mesa_legal_data.sources.manual import ArtifactDocumentCollisionError, import_manual_url
 from mesa_legal_data.sources.url_fetcher import (
     SourcePolicyError,
     SSRFError,
@@ -34,6 +34,7 @@ def collect_url_item(item: HarvestItem, sources_yaml_path: Path | None = None) -
             jurisdiction="TR",
             title=item.title,
             stable_key=item.document_id,
+            publication_date=item.publication_date,
             sources_yaml_path=sources_yaml_path,
         )
         return CollectResult(
@@ -70,6 +71,8 @@ def collect_url_item(item: HarvestItem, sources_yaml_path: Path | None = None) -
                 err_code = "HTTP_429"
             elif any(s in err_msg for s in ("500", "502", "503", "504")):
                 err_code = "HTTP_SERVER_ERROR"
+        elif isinstance(e, ArtifactDocumentCollisionError):
+            err_code = "ARTIFACT_DOCUMENT_COLLISION"
         else:
             if "Host" in err_msg and "not allowed" in err_msg:
                 err_code = "SOURCE_HOST_NOT_ALLOWED"

src/mesa_legal_data/parsers/legislation.py

@@ -4,6 +4,9 @@
 
 from mesa_legal_data.parsers.text_normalizer import normalize_text
 
+PARSER_NAME = "legislation_parser"
+PARSER_VERSION = "1.0.0"
+
 
 class ParsedArticle(BaseModel):
     model_config = ConfigDict(frozen=True)