Audit/mesa data autonomy final

docs/BUILD_STATE.json

@@ -1,7 +1,9 @@
 {
   "status": "mvp_closed",
   "scope": "feature_freeze",
-  "pilot_ready": true,
+  "pilot_ready": false,
+  "external_mesa_ready": false,
+  "final_audit_status": "partially_ready",
   "current_task": "CLOSED",
   "completed": [
     "FREEZE-001",
@@ -35,12 +37,24 @@
     "MVP-CLOSURE-004",
     "BLOCKER-1-REAL-RESMI-GAZETE-PILOT-GENUINE-E2E",
     "BLOCKER-2-OPERATOR-RETRY-RESTRICTED",
-    "BLOCKER-3-RUN-SCOPED-REQUEST-BUDGET"
+    "BLOCKER-3-RUN-SCOPED-REQUEST-BUDGET",
+    "PROMPT-5-FINAL-ADVERSARIAL-AUDIT"
+  ],
+  "last_test_command": "uv run pytest -ra",
+  "last_test_result": "passed (304 passed in 79.53s)",
+  "verification_checks": {
+    "black": "passed (198 Python files, line length 120)",
+    "ruff": "passed",
+    "mypy": "passed (77 source files)",
+    "compileall": "passed",
+    "javascript_syntax": "passed",
+    "live_local_browser_journey": "passed",
+    "live_mesa_contract": "LIVE_MESA_NOT_VERIFIED"
+  },
+  "blockers": [
+    "Live MESA HTTP routes, schemas, mutation semantics, and server-side idempotency remain unverified because no endpoint or source access is available."
   ],
-  "last_test_command": "uv run pytest -m \"not scale\" -ra && uv run pytest -m \"scale\" -ra",
-  "last_test_result": "passed (210 passed: 206 fast, 4 scale)",
   "polish_status": "completed",
   "polish_current_task": "CLOSED",
-  "updated_at": "2026-08-12T23:36:00Z"
+  "updated_at": "2026-08-29T15:13:48Z"
 }
-

migrations/0005_version_aware_records_and_quality.sql

@@ -0,0 +1,130 @@
+-- Migration 0005: Version-aware records and Quality Gate support
+
+PRAGMA foreign_keys = OFF;
+
+-- 1. Upgrade versions table with revision tracking and quality gate fields
+ALTER TABLE versions ADD COLUMN revision_number INTEGER DEFAULT 1;
+ALTER TABLE versions ADD COLUMN supersedes_version_id TEXT;
+ALTER TABLE versions ADD COLUMN quality_status TEXT;
+ALTER TABLE versions ADD COLUMN quality_json TEXT;
+
+CREATE INDEX IF NOT EXISTS idx_versions_doc_revision ON versions(document_id, revision_number);
+CREATE INDEX IF NOT EXISTS idx_versions_quality_status ON versions(quality_status);
+
+-- 2. Upgrade release_items without obsolete single-column record_id FK
+CREATE TABLE IF NOT EXISTS release_items_v2 (
+    release_id TEXT NOT NULL,
+    record_id TEXT NOT NULL,
+    record_sha256 TEXT NOT NULL,
+    PRIMARY KEY (release_id, record_id),
+    FOREIGN KEY (release_id) REFERENCES releases(release_id)
+);
+
+INSERT INTO release_items_v2 (release_id, record_id, record_sha256)
+SELECT release_id, record_id, record_sha256 FROM release_items;
+
+DROP TABLE release_items;
+ALTER TABLE release_items_v2 RENAME TO release_items;
+
+-- 3. Upgrade record_reviews without obsolete single-column record_id FK
+CREATE TABLE IF NOT EXISTS record_reviews_v2 (
+    review_id TEXT PRIMARY KEY,
+    record_id TEXT NOT NULL,
+    record_sha256 TEXT NOT NULL,
+    decision TEXT NOT NULL CHECK (decision IN ('approved', 'rejected')),
+    reviewer TEXT NOT NULL,
+    note TEXT,
+    reviewed_at TEXT NOT NULL
+);
+
+INSERT INTO record_reviews_v2 (review_id, record_id, record_sha256, decision, reviewer, note, reviewed_at)
+SELECT review_id, record_id, record_sha256, decision, reviewer, note, reviewed_at FROM record_reviews;
+
+DROP TABLE record_reviews;
+ALTER TABLE record_reviews_v2 RENAME TO record_reviews;
+
+CREATE INDEX IF NOT EXISTS idx_record_reviews_record ON record_reviews(record_id, reviewed_at);
+
+-- 4. Upgrade record_annotations without obsolete single-column record_id FK
+CREATE TABLE IF NOT EXISTS record_annotations_v2 (
+    annotation_id TEXT PRIMARY KEY,
+    record_id TEXT NOT NULL,
+    annotation_type TEXT NOT NULL CHECK (annotation_type IN ('tag', 'note', 'custom_field')),
+    namespace TEXT NOT NULL,
+    key TEXT NOT NULL,
+    value_json TEXT NOT NULL,
+    created_by TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL
+);
+
+INSERT INTO record_annotations_v2 (annotation_id, record_id, annotation_type, namespace, key, value_json, created_by, created_at, updated_at)
+SELECT annotation_id, record_id, annotation_type, namespace, key, value_json, created_by, created_at, updated_at FROM record_annotations;
+
+DROP TABLE record_annotations;
+ALTER TABLE record_annotations_v2 RENAME TO record_annotations;
+
+CREATE INDEX IF NOT EXISTS idx_record_annotations_record ON record_annotations(record_id);
+
+-- 5. Upgrade record_revisions without obsolete single-column record_id FK
+CREATE TABLE IF NOT EXISTS record_revisions_v2 (
+    revision_id TEXT PRIMARY KEY,
+    original_record_id TEXT NOT NULL,
+    original_record_sha256 TEXT NOT NULL,
+    revised_record_id TEXT NOT NULL,
+    revised_record_sha256 TEXT NOT NULL,
+    version_id TEXT NOT NULL,
+    change_type TEXT NOT NULL,
+    patch_json TEXT NOT NULL,
+    reason TEXT NOT NULL,
+    created_by TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    status TEXT NOT NULL CHECK (status IN ('draft', 'validated', 'needs_review', 'approved', 'rejected', 'superseded'))
+);
+
+INSERT INTO record_revisions_v2 (revision_id, original_record_id, original_record_sha256, revised_record_id, revised_record_sha256, version_id, change_type, patch_json, reason, created_by, created_at, status)
+SELECT revision_id, original_record_id, original_record_sha256, revised_record_id, revised_record_sha256, version_id, change_type, patch_json, reason, created_by, created_at, status FROM record_revisions;
+
+DROP TABLE record_revisions;
+ALTER TABLE record_revisions_v2 RENAME TO record_revisions;
+
+CREATE INDEX IF NOT EXISTS idx_record_revisions_original ON record_revisions(original_record_id);
+
+-- 6. Upgrade records table to support multi-version record instances and composite uniqueness
+CREATE TABLE IF NOT EXISTS records_v2 (
+    record_instance_id TEXT PRIMARY KEY,
+    record_id TEXT NOT NULL,
+    version_id TEXT NOT NULL,
+    record_type TEXT NOT NULL,
+    canonical_path TEXT NOT NULL,
+    canonical_line INTEGER NOT NULL,
+    record_sha256 TEXT NOT NULL,
+    validation_status TEXT NOT NULL,
+    approval_status TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    FOREIGN KEY (version_id) REFERENCES versions(version_id)
+);
+
+INSERT INTO records_v2 (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at)
+SELECT
+    version_id || ':' || record_id,
+    record_id,
+    version_id,
+    record_type,
+    canonical_path,
+    canonical_line,
+    record_sha256,
+    validation_status,
+    approval_status,
+    created_at
+FROM records;
+
+DROP TABLE records;
+ALTER TABLE records_v2 RENAME TO records;
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_records_version_record ON records(version_id, record_id);
+CREATE INDEX IF NOT EXISTS idx_records_record_id ON records(record_id);
+CREATE INDEX IF NOT EXISTS idx_records_version_type ON records(version_id, record_type);
+CREATE INDEX IF NOT EXISTS idx_records_approval_status ON records(approval_status);
+
+PRAGMA foreign_keys = ON;

migrations/0006_citations_certifications_and_panel.sql

@@ -0,0 +1,39 @@
+-- Migration 0006: Citations semantics, parser certifications, operational source settings, and audit sampling
+
+-- 1. Parser certifications table
+CREATE TABLE IF NOT EXISTS parser_certifications (
+    source_id TEXT NOT NULL,
+    parser_name TEXT NOT NULL,
+    parser_version TEXT NOT NULL,
+    certified INTEGER NOT NULL DEFAULT 0,
+    certified_at TEXT,
+    certified_by TEXT,
+    PRIMARY KEY (source_id, parser_name, parser_version)
+);
+
+-- 2. Operational source settings table
+CREATE TABLE IF NOT EXISTS source_operational_settings (
+    source_id TEXT PRIMARY KEY,
+    enabled INTEGER NOT NULL DEFAULT 1,
+    auto_approval_enabled INTEGER NOT NULL DEFAULT 0,
+    weekly_sample_count INTEGER NOT NULL DEFAULT 10,
+    updated_at TEXT NOT NULL,
+    updated_by TEXT
+);
+
+-- 3. Version audit sampling and auto approval tracking
+ALTER TABLE versions ADD COLUMN is_audit_sample INTEGER DEFAULT 0;
+ALTER TABLE versions ADD COLUMN audit_sample_reason TEXT;
+ALTER TABLE versions ADD COLUMN auto_approved INTEGER DEFAULT 0;
+
+CREATE INDEX IF NOT EXISTS idx_versions_audit_sample ON versions(is_audit_sample);
+CREATE INDEX IF NOT EXISTS idx_versions_auto_approved ON versions(auto_approved);
+
+-- 4. Initial seed for certified parsers for built-in sources
+INSERT OR IGNORE INTO parser_certifications (source_id, parser_name, parser_version, certified, certified_at, certified_by)
+VALUES
+    ('mevzuat', 'legislation_parser', '1.0.0', 1, '2026-01-01T00:00:00Z', 'system'),
+    ('resmi_gazete', 'legislation_parser', '1.0.0', 1, '2026-01-01T00:00:00Z', 'system'),
+    ('yargitay', 'decision_parser', '1.0.0', 1, '2026-01-01T00:00:00Z', 'system'),
+    ('danistay', 'decision_parser', '1.0.0', 1, '2026-01-01T00:00:00Z', 'system'),
+    ('anayasa_mahkemesi', 'decision_parser', '1.0.0', 1, '2026-01-01T00:00:00Z', 'system');

migrations/0007_mesa_v4_publisher_and_deliveries.sql

@@ -0,0 +1,62 @@
+-- Migration 0007: MESA v4 Publisher target settings, delivery ledger, and mutation tracking
+
+-- 1. MESA Target Settings (Single active target in MVP, non-secret parameters)
+CREATE TABLE IF NOT EXISTS mesa_target_settings (
+    target_key TEXT PRIMARY KEY DEFAULT 'default',
+    base_url TEXT NOT NULL DEFAULT 'http://localhost:8000',
+    tenant_id TEXT NOT NULL DEFAULT 'default',
+    workspace_id TEXT NOT NULL DEFAULT 'legal',
+    dataset_id TEXT NOT NULL DEFAULT 'tr_legislation',
+    agent_id TEXT NOT NULL DEFAULT 'mesa_data_publisher',
+    content_limit_chars INTEGER NOT NULL DEFAULT 32768,
+    updated_at TEXT NOT NULL
+);
+
+-- Seed default target settings
+INSERT OR IGNORE INTO mesa_target_settings (target_key, base_url, tenant_id, workspace_id, dataset_id, agent_id, content_limit_chars, updated_at)
+VALUES ('default', 'http://localhost:8000', 'default', 'legal', 'tr_legislation', 'mesa_data_publisher', 32768, '2026-01-01T00:00:00Z');
+
+-- 2. MESA Deliveries Ledger
+CREATE TABLE IF NOT EXISTS mesa_deliveries (
+    delivery_id TEXT PRIMARY KEY,
+    release_id TEXT,
+    target_key TEXT NOT NULL DEFAULT 'default',
+    status TEXT NOT NULL
+        CHECK (status IN ('PLANNED', 'SENDING', 'AWAITING_MUTATION', 'COMMITTED', 'PARTIAL', 'FAILED', 'CANCELLED')),
+    started_at TEXT NOT NULL,
+    finished_at TEXT,
+    total_items INTEGER NOT NULL DEFAULT 0,
+    committed_items INTEGER NOT NULL DEFAULT 0,
+    failed_items INTEGER NOT NULL DEFAULT 0,
+    skipped_items INTEGER NOT NULL DEFAULT 0,
+    last_error TEXT,
+    created_at TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_mesa_deliveries_status ON mesa_deliveries(status);
+CREATE INDEX IF NOT EXISTS idx_mesa_deliveries_release ON mesa_deliveries(release_id);
+CREATE INDEX IF NOT EXISTS idx_mesa_deliveries_created ON mesa_deliveries(created_at);
+
+-- 3. MESA Delivery Items (Granular chunk mutation tracking)
+CREATE TABLE IF NOT EXISTS mesa_delivery_items (
+    item_id TEXT PRIMARY KEY,
+    delivery_id TEXT NOT NULL,
+    document_id TEXT NOT NULL,
+    version_id TEXT NOT NULL,
+    chunk_id TEXT NOT NULL,
+    content_hash TEXT NOT NULL,
+    idempotency_key TEXT NOT NULL,
+    remote_mutation_id TEXT,
+    remote_state TEXT NOT NULL
+        CHECK (remote_state IN ('PLANNED', 'SENDING', 'QUEUED', 'PROCESSING', 'COMMITTED', 'FAILED', 'REJECTED', 'SKIPPED')),
+    payload_json TEXT NOT NULL,
+    last_error TEXT,
+    updated_at TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    FOREIGN KEY (delivery_id) REFERENCES mesa_deliveries(delivery_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_mesa_items_delivery_state ON mesa_delivery_items(delivery_id, remote_state);
+CREATE INDEX IF NOT EXISTS idx_mesa_items_doc_ver_chunk ON mesa_delivery_items(document_id, version_id, chunk_id, content_hash);
+CREATE INDEX IF NOT EXISTS idx_mesa_items_idempotency ON mesa_delivery_items(idempotency_key);
+CREATE INDEX IF NOT EXISTS idx_mesa_items_remote_state ON mesa_delivery_items(remote_state);

migrations/0008_final_audit_hardening.sql

@@ -0,0 +1,31 @@
+-- Migration 0008: final audit hardening for explicit MESA HTTP contracts
+-- and version-aware release evidence.
+
+ALTER TABLE mesa_target_settings ADD COLUMN contract_source TEXT NOT NULL DEFAULT 'unknown'
+    CHECK (contract_source IN ('unknown', 'configured', 'live_verified'));
+ALTER TABLE mesa_target_settings ADD COLUMN health_path TEXT NOT NULL DEFAULT '';
+ALTER TABLE mesa_target_settings ADD COLUMN publish_path TEXT NOT NULL DEFAULT '';
+ALTER TABLE mesa_target_settings ADD COLUMN mutation_status_path_template TEXT NOT NULL DEFAULT '';
+
+-- Routes shipped before a local or live HTTP contract was available were assumptions.
+-- Do not silently preserve them as if they were verified.
+UPDATE mesa_target_settings
+SET contract_source = 'unknown',
+    health_path = '',
+    publish_path = '',
+    mutation_status_path_template = '';
+
+ALTER TABLE release_items ADD COLUMN version_id TEXT;
+
+UPDATE release_items
+SET version_id = (
+    SELECT r.version_id
+    FROM records r
+    WHERE r.record_id = release_items.record_id
+      AND r.record_sha256 = release_items.record_sha256
+    ORDER BY r.created_at DESC
+    LIMIT 1
+)
+WHERE version_id IS NULL;
+
+CREATE INDEX IF NOT EXISTS idx_release_items_version ON release_items(release_id, version_id);

pyproject.toml

@@ -40,6 +40,7 @@ build-backend = "setuptools.build_meta"
 
 [dependency-groups]
 dev = [
+    "httpx2>=0.1.0",
     "mypy>=2.3.0",
     "pip-audit>=2.10.1",
     "pytest>=9.1.1",
@@ -59,4 +60,3 @@ ignore = ["BLE001", "B008", "DTZ007", "E501", "W293", "W291"]
 markers = [
     "scale: Large-scale or heavy benchmark tests processing thousands of records",
 ]
-

schemas/citation.schema.json

@@ -18,9 +18,11 @@
     "id": { "type": "string" },
     "record_type": { "type": "string", "const": "citation" },
     "source_record_id": { "type": "string" },
-    "target_legislation_id": { "type": "string" },
+    "target_legislation_id": { "type": ["string", "null"] },
     "target_article_id": { "type": ["string", "null"] },
     "raw_text": { "type": "string" },
+    "citation_status": { "type": "string", "enum": ["EXTRACTED", "RESOLVED", "UNRESOLVED", "HUMAN_VERIFIED"] },
+    "relation_hint": { "type": ["string", "null"], "enum": ["amends", "repeals", "adds", null] },
     "source_span": {
       "type": ["object", "null"],
       "properties": {

scripts/run_resmi_gazete_pilot.py

@@ -242,9 +242,8 @@ def run_pilot():
         active_after_rollback = cur.fetchone()[0]
         stg_final.close()
         print(f"✓ Active release after rollback: {active_after_rollback}")
-        assert active_after_rollback == baseline_release_id, (
-            f"Expected active release {baseline_release_id}, got {active_after_rollback}"
-        )
+        rollback_mismatch = f"Expected active release {baseline_release_id}, got {active_after_rollback}"
+        assert active_after_rollback == baseline_release_id, rollback_mismatch
 
         # Verify provenance after rollback: tracked_record_id is no longer in active release
         prov_post_rollback = get_record_provenance(tracked_record_id)