diff --git a/.mesa_data_agent_state.json b/.mesa_data_agent_state.json index 030af62..0ad69b8 100644 --- a/.mesa_data_agent_state.json +++ b/.mesa_data_agent_state.json @@ -2,10 +2,10 @@ "schema_version": 1, "project": "MESA_Data", "workflow": "mesa_data_autonomy", - "branch": "audit/mesa-data-autonomy-final", + "branch": "fix/mesa-data-final-mvp", "baseline_commit": "84c61db716c10cb4110b673f9a976215afe71cd3", - "current_phase": "post_master_independent_verification", - "overall_status": "completed", + "current_phase": "complete", + "overall_status": "ready", "phases": { "prompt_1": { "status": "completed", @@ -366,11 +366,46 @@ "remaining_risks": [ "Live MESA HTTP endpoint remains UNVERIFIED because external network access/live MESA instance is outside repository boundary." ] + }, + "final_mvp_blocker_closure": { + "status": "completed", + "started_at": "2026-08-30T00:00:00+03:00", + "completed_at": "2026-08-30T00:43:56+03:00", + "commits": [ + "d6a35b7 fix: close final MVP integrity blockers" + ], + "migrations": [ + "0011_final_mvp_blocker_closure.sql" + ], + "findings": [ + "Panel confirmation was bound to a mutable live-catalog summary rather than an exact immutable release and target fingerprint.", + "Publisher release reconstruction could join identical logical record IDs and hashes across legal versions without exact version identity.", + "Ambiguous legacy reviews and logical-record issues could contaminate another version.", + "Single-record revision mutation was scoped by logical record_id and could alter sibling versions.", + "String-shaped but impossible calendar dates could enter chronology and approval decisions.", + "Identical artifact bytes could not safely enrich missing authoritative publication metadata.", + "Release payload normalization required separate canonical-record and packaged-payload hashes." + ], + "tests": [ + "fresh and populated-upgrade migration integrity: passed in test_control_57_and_58_fresh_and_upgrade_database_integrity", + "uv sync --frozen: passed (79 packages audited)", + "uv run ruff format --check .: passed (211 files)", + "uv run ruff check .: passed", + "uv run mypy src: passed (77 source files)", + "uv run pip-audit: passed (0 known vulnerabilities)", + "uv run pytest -m 'not scale' -ra --durations=20: 350 passed, 0 failed, 4 deselected", + "uv run pytest -m 'scale' -ra --durations=20: 4 passed, 0 failed, 350 deselected" + ], + "blockers": [], + "remaining_risks": [ + "Live MESA HTTP request/response and server-side idempotency semantics remain unverified." + ] } }, "mesa_contract": { + "status": "unverified", "source": "explicit_configuration_and_local_mock_contract", - "verified_by_live_http": "LIVE_MESA_NOT_VERIFIED", + "verified_by_live_http": false, "assumptions": [], "unknowns": [ "Live MESA health, publish, and mutation-status paths", @@ -396,11 +431,11 @@ }, "handoff": { "next_phase": null, - "summary": "Independent verification and release gate audit complete. All 338 tests pass (334 non-scale + 4 scale), 12 adversarial test cases pass, migration 0010 is verified, all 65 controls verified, and CI gate is 100% green.", + "summary": "Final MVP blocker closure complete. All 354 tests pass (350 non-scale + 4 scale), fresh and populated upgrade migrations through 0011 pass, exact CI gates are green, and no P0/P1 MVP blocker remains.", "must_verify": [ "Ensure live deployment environment configures MESA_DATA_MESA_API_KEY securely.", "Configure documented MESA routes and verify health, publish, polling, COMMITTED truth, and server-side idempotency against a live endpoint." ] }, - "last_updated_at": "2026-08-29T23:15:00Z" + "last_updated_at": "2026-08-30T00:43:56+03:00" } diff --git a/README.md b/README.md index 3f6ffeb..6d0dfe5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MESA Legal Data -MESA Legal Data; MESA hukuk ekosistemi için resmî mevzuat, içtihat ve hukuki atıf verilerini toplayan, ham dosyaları değiştiremez biçimde saklayan, ayrıştıran, kanonikleştiren, doğrulayan ve MESA staging veritabanına aktaran uçtan uca veri platformudur. +MESA Legal Data; resmî mevzuat, içtihat ve hukuki atıf verilerini otomatik toplayan, ham dosyaları değiştiremez biçimde saklayan, kanonikleştiren ve kalite kapılarından geçiren veri platformudur. Güvenli koşulları sağlayan sürümler otomatik onaylanabilir; istisnalar insan incelemesine gider. Gerçek MESA gönderimi yalnız insan başlatınca, otomatik oluşturulup doğrulanan immutable release paketinden yapılır. Yerel staging yalnız development aracıdır ve gerçek MESA değildir. --- @@ -15,13 +15,14 @@ Ham Veri Deposu (raw/ — Değişmez Artifact + SHA-256 + Metadata) ↓ Ayrıştırma & Kanonikleştirme Pipeline'ı (parse → canonical JSONL) ↓ -Gizlilik Taraması & Hukuki Metadata Doğrulaması (Schema & Privacy) +Kalite Kapıları + Gizlilik + Hukuki Metadata Doğrulaması ↓ -İnsan Onayı (Human Review — approve / reject) +Güvenli Otomatik Onay / İnsan İstisna İncelemesi ↓ -Release Derleme & Doğrulama (build → verify → publish) +Immutable Release (otomatik build → verify → frozen summary → insan onayı) ↓ -MESA Staging DB (Atomik, Idempotent Import & Provenance) +Gerçek MESA Publisher (idempotent, yalnız COMMITTED başarı) + └─ Yerel Development Staging (ayrı, isteğe bağlı) ``` --- @@ -62,13 +63,14 @@ uv run mesa-data harvest maintenance ### Güvenlik & Politika - Keşif ve indirmeler yalnızca `config/sources.yaml` ve `config/harvest.yaml` dosyalarında izin verilen resmi kaynaklarda çalışır. - İndirme işlemleri SSRF, MIME türü, boyut sınırı (~50 MB) ve hız sınırı denetimlerinden geçer. -- Otomatik onay veya yayınlama yapılmaz; tüm veriler insan onayı (`review`) süzgecinden geçer. +- Yalnız sertifikalı kaynak/parser ve PASS kalite koşullarını sağlayan sürümler güvenli biçimde otomatik onaylanabilir; diğerleri insan incelemesine gider. +- MESA'ya gönderim hiçbir zaman otomatik başlamaz. --- ## Web Yönetim Paneli (FastAPI + HTML/CSS/JS) -MESA Legal Data, tüm veri toplama, orkestrasyon, inceleme, release ve staging aktarım işlemlerini yönetebileceğiniz web tabanlı bir arayüze sahiptir. +MESA Legal Data, veri toplama, kalite/istisna incelemesi, immutable release ve insan başlatmalı gerçek MESA gönderimini yönetebileceğiniz web tabanlı bir arayüze sahiptir. Panel normal akışta kullanıcıdan release ID istemez; exact release ve manifest özetini otomatik hazırlar. ### Web Panelini Başlatma ```bash @@ -116,7 +118,7 @@ uv run mesa-data release build --release-id release-v1.0 uv run mesa-data release verify --release-id release-v1.0 uv run mesa-data release publish --release-id release-v1.0 -# E. MESA Staging DB Import +# E. Yalnız geliştirme amaçlı yerel staging importu (gerçek MESA değildir) uv run mesa-data release import --release-id release-v1.0 # F. İzlenebilirlik Sorgusu diff --git a/docs/HIZLI_BASLANGIC.md b/docs/HIZLI_BASLANGIC.md index 89ad148..53b0094 100644 --- a/docs/HIZLI_BASLANGIC.md +++ b/docs/HIZLI_BASLANGIC.md @@ -39,7 +39,9 @@ Web arayüzünde herhangi bir teknik terim veya karmaşık ayar bilmenize gerek ### Adım 3: Dışa Aktar 1. Sol menüden **Dışa Aktar** ekranına gidin. 2. **"Dosya olarak indir"**: JSONL (model eğitimi ve veri bilimi için) veya CSV (Excel için) biçimini seçip **"Dışa aktarmayı oluştur"** butonuna tıklayın. -3. **"MESA'ya aktar"**: Tek tıkla doğrulanmış paketi MESA veri havuzuna gönderin. +3. **"MESA'ya gönder"**: Panel uygun güncel sürümlerden immutable release'i otomatik oluşturur ve doğrular. Release ID, manifest SHA-256 ve hedefi gösteren frozen özeti onayladığınızda yalnız o exact paket gerçek MESA publisher'a verilir. Onaydan sonra katalog değişse bile gönderim kapsamı değişmez. + +> **Not:** “Yerel Development Staging / Yerel Staging'e Aktar” ayrı bir geliştirme aracıdır; gerçek MESA gönderimi değildir. --- diff --git a/docs/KULLANIM_KILAVUZU.md b/docs/KULLANIM_KILAVUZU.md index bf636a3..cce9fa0 100644 --- a/docs/KULLANIM_KILAVUZU.md +++ b/docs/KULLANIM_KILAVUZU.md @@ -72,13 +72,17 @@ Onaylanmış verileri sistemden çıkarma ve paylaşma ekranıdır: - **CSV:** Excel ve tablo araçları için uygun format. - Oluşturulan dosyalar geçmiş tablosundan doğrudan indirilebilir. - **MESA’ya Aktar:** - - Onaylı verileri doğrulanmış bir paket olarak MESA aktarım havuzuna gönderir. + - Uygun güncel sürümlerden immutable release'i otomatik oluşturur ve kriptografik olarak doğrular. + - Onay penceresi exact release ID, manifest SHA-256 ve MESA hedefini gösterir; kullanıcıdan manuel release ID istenmez. + - İnsan onayından sonra yalnız bu frozen paket gönderilir. Katalogdaki daha yeni bir sürüm aynı gönderime sessizce eklenmez. + - Yalnız tüm gerekli mutation'lar `COMMITTED` olduğunda tam başarı gösterilir; `PARTIAL` ve `AWAITING_MUTATION` ayrı durumlardır. ### 6. Gelişmiş (Yönetim Menüsü) Teknik kullanıcılar ve sistem yöneticileri için ek araçlar sunar: - **Kaynaklar:** Resmî kurum erişim limitleri ve kuralları. - **Veri Gezgini:** Ham kanonik kayıtları ayrıntılı arama ve inceleme. - **Release Geçmişi:** Oluşturulan sürümlerin doğrulama, yayınlama ve geri alma süreçleri. +- **Yerel Development Staging:** Release paketini yalnız yerel geliştirme veritabanına aktarır; gerçek MESA ile karıştırılmamalıdır. - **Arka Plan İşlemleri:** Uzun süren sistem görevlerinin durum takibi. - **İşlem Geçmişi (Audit):** Sistemde yapılan tüm değişikliklerin işlem günlüğü. - **Sistem:** Sistem sağlık taraması (Doctor) ve veritabanı yedekleme (Backup). diff --git a/migrations/0011_final_mvp_blocker_closure.sql b/migrations/0011_final_mvp_blocker_closure.sql new file mode 100644 index 0000000..1a8765e --- /dev/null +++ b/migrations/0011_final_mvp_blocker_closure.sql @@ -0,0 +1,116 @@ +-- Migration 0011: final MVP blocker closure. +-- Preserves ambiguous legacy reviews without guessing a legal version, enforces +-- version-aware release membership for new writes, and binds MESA deliveries to +-- the human-confirmed non-secret target/release fingerprints. + +PRAGMA foreign_keys = OFF; + +CREATE TABLE record_reviews_v4 ( + review_id TEXT NOT NULL PRIMARY KEY UNIQUE, + record_instance_id TEXT NOT NULL, + version_id TEXT NOT NULL, + record_id TEXT NOT NULL, + record_sha256 TEXT NOT NULL, + decision TEXT NOT NULL CHECK (decision IN ('approved', 'rejected')), + reviewer TEXT NOT NULL, + note TEXT, + reviewed_at TEXT NOT NULL +); + +INSERT INTO record_reviews_v4 ( + review_id, record_instance_id, version_id, record_id, record_sha256, + decision, reviewer, note, reviewed_at +) +SELECT + rr.review_id, + CASE + WHEN rr.reviewed_at >= COALESCE( + (SELECT applied_at FROM schema_migrations WHERE version = '0010_mvp_master_closure.sql'), + '9999-12-31T23:59:59Z' + ) AND EXISTS ( + SELECT 1 FROM records r + WHERE r.record_instance_id = rr.record_instance_id + AND r.version_id = rr.version_id + AND r.record_id = rr.record_id + AND r.record_sha256 = rr.record_sha256 + ) THEN rr.record_instance_id + WHEN (SELECT count(*) FROM records r + WHERE r.record_id = rr.record_id AND r.record_sha256 = rr.record_sha256) = 1 + THEN (SELECT r.record_instance_id FROM records r + WHERE r.record_id = rr.record_id AND r.record_sha256 = rr.record_sha256) + ELSE 'legacy-ambiguous:' || rr.review_id + END, + CASE + WHEN rr.reviewed_at >= COALESCE( + (SELECT applied_at FROM schema_migrations WHERE version = '0010_mvp_master_closure.sql'), + '9999-12-31T23:59:59Z' + ) AND EXISTS ( + SELECT 1 FROM records r + WHERE r.record_instance_id = rr.record_instance_id + AND r.version_id = rr.version_id + AND r.record_id = rr.record_id + AND r.record_sha256 = rr.record_sha256 + ) THEN rr.version_id + WHEN (SELECT count(*) FROM records r + WHERE r.record_id = rr.record_id AND r.record_sha256 = rr.record_sha256) = 1 + THEN (SELECT r.version_id FROM records r + WHERE r.record_id = rr.record_id AND r.record_sha256 = rr.record_sha256) + ELSE 'legacy-version-unscoped' + END, + rr.record_id, + rr.record_sha256, + rr.decision, + rr.reviewer, + rr.note, + rr.reviewed_at +FROM record_reviews rr; + +DROP TABLE record_reviews; +ALTER TABLE record_reviews_v4 RENAME TO record_reviews; + +CREATE INDEX idx_record_reviews_instance ON record_reviews(record_instance_id); +CREATE INDEX idx_record_reviews_version ON record_reviews(version_id); +CREATE INDEX idx_record_reviews_record ON record_reviews(record_id, reviewed_at); + +CREATE TRIGGER release_items_version_required_insert +BEFORE INSERT ON release_items +WHEN NEW.version_id IS NULL +BEGIN + SELECT RAISE(ABORT, 'release_items.version_id is required'); +END; + +CREATE TRIGGER release_items_version_required_update +BEFORE UPDATE OF version_id, record_id, record_sha256 ON release_items +WHEN NEW.version_id IS NULL +BEGIN + SELECT RAISE(ABORT, 'release_items.version_id is required'); +END; + +CREATE TRIGGER release_items_identity_valid_insert +BEFORE INSERT ON release_items +WHEN NOT EXISTS ( + SELECT 1 FROM records r + WHERE r.version_id = NEW.version_id + AND r.record_id = NEW.record_id + AND r.record_sha256 = NEW.record_sha256 +) +BEGIN + SELECT RAISE(ABORT, 'release item version identity is invalid'); +END; + +CREATE TRIGGER release_items_identity_valid_update +BEFORE UPDATE OF version_id, record_id, record_sha256 ON release_items +WHEN NOT EXISTS ( + SELECT 1 FROM records r + WHERE r.version_id = NEW.version_id + AND r.record_id = NEW.record_id + AND r.record_sha256 = NEW.record_sha256 +) +BEGIN + SELECT RAISE(ABORT, 'release item version identity is invalid'); +END; + +ALTER TABLE mesa_deliveries ADD COLUMN target_config_sha256 TEXT; +ALTER TABLE mesa_deliveries ADD COLUMN release_manifest_sha256 TEXT; + +PRAGMA foreign_keys = ON; diff --git a/pyproject.toml b/pyproject.toml index 777f92d..bf3bfde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ build-backend = "setuptools.build_meta" [dependency-groups] dev = [ + "httpx2>=0.1.0", "mypy>=2.3.0", "pip-audit>=2.10.1", "pytest>=9.1.1", diff --git a/src/mesa_legal_data/catalog.py b/src/mesa_legal_data/catalog.py index 923b412..58397b5 100644 --- a/src/mesa_legal_data/catalog.py +++ b/src/mesa_legal_data/catalog.py @@ -5,7 +5,7 @@ from collections.abc import Iterator from contextlib import contextmanager from dataclasses import dataclass -from datetime import UTC, datetime +from datetime import UTC, date, datetime from pathlib import Path from typing import Any @@ -188,9 +188,11 @@ def _normalize_legal_date(raw_date: str | None) -> str | None: if not raw_date: return None cleaned = str(raw_date).strip() - if len(cleaned) >= 10 and cleaned[:4].isdigit() and cleaned[4] == "-" and cleaned[7] == "-": - return cleaned[:10] - return None + candidate = cleaned[:10] + try: + return date.fromisoformat(candidate).isoformat() + except ValueError: + return None def upsert_document( @@ -254,7 +256,8 @@ def recompute_document_current_version( """SELECT v.version_id, v.version_kind, v.snapshot_date, v.effective_from, v.effective_to, v.canonical_sha256, v.revision_number, v.supersedes_version_id, v.approval_status, v.quality_status, - v.validation_status, v.created_at, a.metadata_json, a.retrieved_at, a.last_modified + v.validation_status, v.created_at, v.auto_approved, + a.metadata_json, a.retrieved_at, a.last_modified FROM versions v LEFT JOIN artifacts a ON a.artifact_id = v.artifact_id WHERE v.document_id = ?""", @@ -267,9 +270,9 @@ def recompute_document_current_version( parsed_versions = [] for r in rows: meta_dict: dict[str, Any] = {} - if r[12]: + if r[13]: try: - meta_dict = json.loads(r[12]) + meta_dict = json.loads(r[13]) except Exception: pass @@ -296,6 +299,7 @@ def recompute_document_current_version( "quality_status": r[9], "validation_status": r[10], "created_at": r[11], + "auto_approved": bool(r[12]), } ) @@ -317,7 +321,9 @@ def sort_key(item): and v["legal_date"] == chosen["legal_date"] and v["canonical_sha256"] != chosen["canonical_sha256"] ] - if same_date_candidates and not chosen.get("supersedes_version_id"): + ambiguity_group = [chosen, *same_date_candidates] + has_explicit_supersession = any(item.get("supersedes_version_id") for item in ambiguity_group) + if same_date_candidates and not has_explicit_supersession: doc_cur = conn.cursor() doc_cur.execute("SELECT current_version_id FROM documents WHERE document_id = ?", (document_id,)) existing_doc = doc_cur.fetchone() @@ -327,6 +333,37 @@ def sort_key(item): if existing_in_candidates: chosen = existing_in_candidates + # Same authoritative date plus different content has no safe automatic + # ordering. Preserve the existing current choice, revoke only automatic + # approvals, and create explicit version-scoped review evidence. + now = datetime.now(UTC).isoformat() + for ambiguous in ambiguity_group: + ambiguous_id = ambiguous["version_id"] + issue_id = "iss-version-date-" + hashlib.sha256(ambiguous_id.encode("utf-8")).hexdigest()[:16] + conn.execute( + """INSERT OR IGNORE INTO validation_issues ( + issue_id, subject_type, subject_id, version_id, record_instance_id, + severity, code, message, details_json, status, opened_at + ) VALUES (?, 'version', ?, ?, NULL, 'error', 'VERSION_DATE_AMBIGUITY', + 'Different content shares the same authoritative legal date without explicit supersession', + ?, 'open', ?)""", + ( + issue_id, + ambiguous_id, + ambiguous_id, + json.dumps({"legal_date": chosen["legal_date"], "document_id": document_id}), + now, + ), + ) + conn.execute( + """UPDATE versions + SET approval_status = 'pending', auto_approved = 0 + WHERE version_id = ? AND auto_approved = 1""", + (ambiguous_id,), + ) + if ambiguous.get("auto_approved"): + ambiguous["approval_status"] = "pending" + chosen_version_id = chosen["version_id"] derived_status = "fetched" @@ -460,6 +497,72 @@ def insert_artifact( metadata_json: str | None = None, ): with transaction(conn): + existing_row = conn.execute( + "SELECT artifact_id, document_id, metadata_json FROM artifacts WHERE sha256 = ?", + (sha256,), + ).fetchone() + if existing_row: + if existing_row[1] and document_id and existing_row[1] != document_id: + raise CatalogError( + "ARTIFACT_DOCUMENT_COLLISION: identical payload is already bound to a different document" + ) + + try: + existing_meta = json.loads(existing_row[2] or "{}") + except (TypeError, json.JSONDecodeError): + existing_meta = {} + try: + incoming_meta = json.loads(metadata_json or "{}") + except (TypeError, json.JSONDecodeError): + incoming_meta = {} + + conflicts: dict[str, dict[str, Any]] = {} + changed = False + for key in ("publication_date", "source_date", "snapshot_date", "decision_date"): + incoming_value = incoming_meta.get(key) + existing_value = existing_meta.get(key) + if not incoming_value: + continue + if not existing_value: + existing_meta[key] = incoming_value + changed = True + elif existing_value != incoming_value: + conflicts[key] = {"existing": existing_value, "incoming": incoming_value} + + if changed: + conn.execute( + "UPDATE artifacts SET metadata_json = ? WHERE artifact_id = ?", + (json.dumps(existing_meta), existing_row[0]), + ) + if conflicts: + issue_id = ( + "iss-metadata-" + + hashlib.sha256(f"{existing_row[0]}:{sorted(conflicts)}".encode("utf-8")).hexdigest()[:16] + ) + conn.execute( + """INSERT OR IGNORE INTO validation_issues ( + issue_id, subject_type, subject_id, version_id, record_instance_id, + severity, code, message, details_json, status, opened_at + ) VALUES (?, 'artifact', ?, NULL, NULL, 'error', 'ARTIFACT_METADATA_CONFLICT', + 'Conflicting authoritative metadata was not overwritten', ?, 'open', ?)""", + ( + issue_id, + existing_row[0], + json.dumps(conflicts, sort_keys=True), + datetime.now(UTC).isoformat(), + ), + ) + conn.execute( + """UPDATE versions + SET approval_status = 'pending', auto_approved = 0, + quality_status = CASE WHEN quality_status = 'PASS' THEN 'REVIEW' ELSE quality_status END + WHERE artifact_id = ?""", + (existing_row[0],), + ) + if existing_row[1]: + recompute_document_current_version(conn, existing_row[1]) + return + conn.execute( """INSERT INTO artifacts (artifact_id, document_id, source_id, source_url, retrieved_at, fetch_method, http_status, declared_content_type, detected_content_type, byte_size, sha256, raw_path, etag, last_modified, transport_status, error_code, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", @@ -866,6 +969,7 @@ class ReleaseRecordRef: canonical_path: str canonical_line: int version_id: str + document_id: str def iter_records_for_release( @@ -884,9 +988,11 @@ def iter_records_for_release( AND v.quality_status = 'PASS' AND v.privacy_status IN ('clean', 'approved') ) - SELECT r.record_id, r.record_type, r.record_sha256, r.canonical_path, r.canonical_line, r.version_id + SELECT r.record_id, r.record_type, r.record_sha256, r.canonical_path, r.canonical_line, + r.version_id, v.document_id FROM records r JOIN eligible_versions ev ON r.version_id = ev.version_id + JOIN versions v ON v.version_id = r.version_id WHERE r.approval_status = 'approved' AND r.validation_status = 'valid' ORDER BY r.canonical_path ASC, r.canonical_line ASC @@ -903,6 +1009,7 @@ def iter_records_for_release( canonical_path=r[3], canonical_line=r[4], version_id=r[5], + document_id=r[6], ) @@ -915,6 +1022,7 @@ def list_records_for_release(conn: sqlite3.Connection) -> list[dict[str, Any]]: "canonical_path": ref.canonical_path, "canonical_line": ref.canonical_line, "record_sha256": ref.record_sha256, + "document_id": ref.document_id, } for ref in iter_records_for_release(conn) ] @@ -1000,7 +1108,11 @@ def list_open_blocking_issues_for_version(conn: sqlite3.Connection, version_id: version_id = ? OR (subject_type = 'version' AND subject_id = ?) OR record_instance_id IN (SELECT record_instance_id FROM records WHERE version_id = ?) - OR (subject_type = 'record' AND subject_id IN (SELECT record_id FROM records WHERE version_id = ?)) + OR ( + subject_type = 'record' + AND subject_id IN (SELECT record_id FROM records WHERE version_id = ?) + AND (SELECT count(*) FROM records all_instances WHERE all_instances.record_id = validation_issues.subject_id) = 1 + ) OR (subject_type = 'record' AND subject_id IN (SELECT record_instance_id FROM records WHERE version_id = ?)) )""", (version_id, version_id, version_id, version_id, version_id), @@ -1141,8 +1253,11 @@ def approve_record_with_checks( rec_inst_id = rec.get("record_instance_id") or f"{rec['version_id']}:{rec['record_id']}" rec_ver_id = rec["version_id"] - blockers = list_open_blocking_issues(conn, subject_id=record_id) - blockers += list_open_blocking_issues(conn, subject_id=rec_inst_id) + blockers = list_open_blocking_issues(conn, subject_id=rec_inst_id) + instance_count = conn.execute("SELECT count(*) FROM records WHERE record_id = ?", (record_id,)).fetchone()[0] + if instance_count == 1: + blockers += list_open_blocking_issues(conn, subject_id=record_id) + blockers = list({blocker["issue_id"]: blocker for blocker in blockers}.values()) if blockers: raise BlockingValidationIssueExists(f"Cannot approve record {record_id}: open blocker issues exist: {blockers}") @@ -1844,6 +1959,13 @@ def create_record_revision( status: str = "draft", revision_id: str | None = None, ) -> str: + record_match = conn.execute( + """SELECT 1 FROM records + WHERE version_id = ? AND record_id = ? AND record_sha256 = ?""", + (version_id, original_record_id, original_record_sha256), + ).fetchone() + if not record_match: + raise CatalogError(f"Record revision identity does not match version {version_id}: {original_record_id}") if not revision_id: revision_id = f"rev-{uuid.uuid4().hex[:12]}" now = datetime.now(UTC).isoformat() @@ -1943,8 +2065,9 @@ def approve_record_revision( with transaction(conn): conn.execute("UPDATE record_revisions SET status = 'approved' WHERE revision_id = ?", (revision_id,)) conn.execute( - "UPDATE records SET approval_status = 'approved', validation_status = 'valid' WHERE record_id = ?", - (rev["revised_record_id"] or rev["original_record_id"],), + """UPDATE records SET approval_status = 'approved', validation_status = 'valid' + WHERE version_id = ? AND record_id = ?""", + (rev["version_id"], rev["revised_record_id"] or rev["original_record_id"]), ) log_audit_event( conn, diff --git a/src/mesa_legal_data/operations.py b/src/mesa_legal_data/operations.py index 9e341ae..c6489fd 100644 --- a/src/mesa_legal_data/operations.py +++ b/src/mesa_legal_data/operations.py @@ -230,6 +230,7 @@ def progress_cb(prog: dict[str, Any]) -> None: delivery_id=del_id, release_id=rel_id, target_key=target_key, + expected_target_config_sha256=inp.get("target_config_sha256"), progress_callback=progress_cb, is_cancelled_cb=is_cancelled_cb, ) diff --git a/src/mesa_legal_data/pipeline.py b/src/mesa_legal_data/pipeline.py index 8d9900c..52a3145 100644 --- a/src/mesa_legal_data/pipeline.py +++ b/src/mesa_legal_data/pipeline.py @@ -2,6 +2,7 @@ import json import re import uuid +from datetime import date from pathlib import Path from typing import Any @@ -54,6 +55,15 @@ class InvalidStateTransition(Exception): pass +def _valid_legal_date(raw_date: Any) -> str | None: + if not raw_date: + return None + try: + return date.fromisoformat(str(raw_date).strip()[:10]).isoformat() + except ValueError: + return None + + ALLOWED_TRANSITIONS = { "discovered": {"fetched", "failed"}, "fetched": {"transport_verified", "failed"}, @@ -211,13 +221,17 @@ def process_artifact_pipeline( ret_at = art_row.get("retrieved_at") if art_row else None pub_date_missing = False + legal_date_invalid = False if v_kind == "original_publication": raw_ver_date = meta_dict.get("publication_date") or meta_dict.get("source_date") if not raw_ver_date: pub_date_missing = True ver_date = "unknown-date" else: - ver_date = str(raw_ver_date)[:10] + normalized_date = _valid_legal_date(raw_ver_date) + legal_date_invalid = normalized_date is None + pub_date_missing = legal_date_invalid + ver_date = normalized_date or "unknown-date" elif fam == "decision": dec_parsed_pre = parse_decision_text(canonical_text) raw_ver_date = ( @@ -230,7 +244,9 @@ def process_artifact_pipeline( raw_ver_date = str(last_mod)[:10] elif not raw_ver_date and ret_at: raw_ver_date = str(ret_at)[:10] - ver_date = str(raw_ver_date)[:10] if raw_ver_date else "unknown-date" + normalized_date = _valid_legal_date(raw_ver_date) + legal_date_invalid = bool(raw_ver_date) and normalized_date is None + ver_date = normalized_date or "unknown-date" else: raw_ver_date = ( meta_dict.get("snapshot_date") @@ -239,7 +255,9 @@ def process_artifact_pipeline( or (str(last_mod)[:10] if last_mod else None) or (str(ret_at)[:10] if ret_at else None) ) - ver_date = str(raw_ver_date)[:10] if raw_ver_date else "unknown-date" + normalized_date = _valid_legal_date(raw_ver_date) + legal_date_invalid = bool(raw_ver_date) and normalized_date is None + ver_date = normalized_date or "unknown-date" # Deterministic Version ID based on document_id, version_date, and artifact_sha256 if fam == "legislation": @@ -260,7 +278,7 @@ def process_artifact_pipeline( "source_id": art_row["source_id"], "source_url": art_row["source_url"], "retrieved_at": art_row["retrieved_at"], - "publication_date": meta_dict.get("publication_date"), + "publication_date": _valid_legal_date(meta_dict.get("publication_date")), "artifact_sha256": art_row["sha256"], "artifact_path": art_row["raw_path"], } @@ -321,7 +339,7 @@ def process_artifact_pipeline( doc_type = "law" pub_info = None - pub_d = meta_dict.get("publication_date") + pub_d = _valid_legal_date(meta_dict.get("publication_date")) if pub_d: pub_info = {"date": str(pub_d)} @@ -531,12 +549,22 @@ def process_artifact_pipeline( ) quality_status = quality_report.decision - quality_json = json.dumps(quality_report.to_dict()) + quality_data = quality_report.to_dict() # Publication date missing for original publication cannot PASS if pub_date_missing and v_kind == "original_publication": if quality_status == "PASS": quality_status = "REVIEW" + quality_data["decision"] = quality_status + quality_data["checks"].append( + { + "group": "METADATA", + "name": "publication_date_present", + "status": "REVIEW", + "message": "Authoritative publication date is missing", + "details": {}, + } + ) open_issue( conn, issue_id=f"iss-{uuid.uuid4().hex[:8]}", @@ -549,6 +577,33 @@ def process_artifact_pipeline( details_json=json.dumps({"source_id": art_row.get("source_id")}), ) + if legal_date_invalid: + if quality_status == "PASS": + quality_status = "REVIEW" + quality_data["decision"] = quality_status + quality_data["checks"].append( + { + "group": "METADATA", + "name": "legal_date_calendar_valid", + "status": "REVIEW", + "message": "Authoritative legal date is not a real calendar date", + "details": {"raw_date": str(raw_ver_date)}, + } + ) + open_issue( + conn, + issue_id=f"iss-{uuid.uuid4().hex[:8]}", + subject_type="version", + subject_id=version_id, + version_id=version_id, + severity="error", + code="LEGAL_DATE_INVALID", + message="Authoritative legal date is not a real calendar date", + details_json=json.dumps({"raw_date": str(raw_ver_date)}), + ) + + quality_json = json.dumps(quality_data) + # Release Guard & Lifecycle Determination if quality_status == "BLOCK": val_status = "failed" @@ -647,7 +702,7 @@ def process_artifact_pipeline( # Step 9b: Safe Auto-Approval Evaluation auto_approved = False auto_reason = "" - if quality_status != "BLOCK" and val_status == "valid" and not pub_date_missing: + if quality_status != "BLOCK" and val_status == "valid" and not pub_date_missing and not legal_date_invalid: try: auto_approved, auto_reason = evaluate_auto_approval( conn, diff --git a/src/mesa_legal_data/publisher/engine.py b/src/mesa_legal_data/publisher/engine.py index 239eda3..09bb843 100644 --- a/src/mesa_legal_data/publisher/engine.py +++ b/src/mesa_legal_data/publisher/engine.py @@ -2,6 +2,7 @@ import json import time import uuid +from pathlib import Path from typing import Any, Callable from mesa_legal_data.catalog import get_connection @@ -26,6 +27,102 @@ SourceChunk, ) +_RELEASE_TYPE_FILES = { + "legislation": "data/legislation.jsonl", + "article": "data/articles.jsonl", + "decision": "data/decisions.jsonl", + "citation": "data/citations.jsonl", +} + + +def target_config_sha256(settings) -> str: + """Hash the exact non-secret MESA target configuration confirmed by a human.""" + payload = { + "target_key": settings.target_key, + "base_url": settings.base_url, + "tenant_id": settings.tenant_id, + "workspace_id": settings.workspace_id, + "dataset_id": settings.dataset_id, + "agent_id": settings.agent_id, + "content_limit_chars": settings.content_limit_chars, + "contract_source": settings.contract_source, + "health_path": settings.health_path, + "publish_path": settings.publish_path, + "mutation_status_path_template": settings.mutation_status_path_template, + } + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _load_frozen_release_versions(conn, release_id: str, data_root: Path) -> list[dict[str, Any]]: + """Load exact publisher input from the verified immutable release package.""" + from mesa_legal_data.catalog import get_release + from mesa_legal_data.release.verifier import verify_release + + release = get_release(conn, release_id) + if not release: + raise MesaClientError(f"Release {release_id} does not exist") + if release["status"] not in {"verified", "published"}: + raise MesaClientError( + f"Release {release_id} cannot be published to MESA from lifecycle status {release['status']}" + ) + if not verify_release(release_id): + raise MesaClientError(f"Cannot build delivery plan from unverified or invalid release {release_id}") + + release_dir = data_root / "releases" / release_id + index_path = release_dir / "data/release-index.jsonl" + if not index_path.exists(): + raise MesaClientError(f"Release {release_id} lacks frozen publisher version identity") + + payloads: dict[str, tuple[str, str, dict[str, Any]]] = {} + for record_type, rel_path in _RELEASE_TYPE_FILES.items(): + path = release_dir / rel_path + if not path.exists(): + continue + with open(path, "r", encoding="utf-8") as stream: + for line in stream: + if not line.strip(): + continue + item = json.loads(line) + record_id = item.get("id") + if not isinstance(record_id, str) or not record_id: + raise MesaClientError(f"Release {release_id} contains a payload without record identity") + payloads[record_id] = (record_type, hashlib.sha256(line.encode("utf-8")).hexdigest(), item) + + versions: dict[str, dict[str, Any]] = {} + index_identities: set[tuple[str, str, str]] = set() + with open(index_path, "r", encoding="utf-8") as stream: + for line in stream: + if not line.strip(): + continue + idx = json.loads(line) + record_id = idx["record_id"] + record_type = idx["record_type"] + record_sha256 = idx["record_sha256"] + payload_sha256 = idx["payload_sha256"] + version_id = idx["version_id"] + document_id = idx["document_id"] + payload = payloads.get(record_id) + if payload is None or payload[:2] != (record_type, payload_sha256): + raise MesaClientError(f"Release {release_id} publisher index mismatch for {record_id}") + index_identities.add((version_id, record_id, record_sha256)) + version = versions.setdefault( + version_id, + {"version_id": version_id, "document_id": document_id, "packaged_records": []}, + ) + if version["document_id"] != document_id: + raise MesaClientError(f"Release {release_id} maps version {version_id} to multiple documents") + version["packaged_records"].append(payload[2]) + + db_rows = conn.execute( + "SELECT version_id, record_id, record_sha256 FROM release_items WHERE release_id = ?", + (release_id,), + ).fetchall() + if {(row[0], row[1], row[2]) for row in db_rows} != index_identities: + raise MesaClientError(f"Release {release_id} catalog membership does not match its frozen package") + + return [versions[key] for key in sorted(versions)] + def get_ready_versions_and_content(conn) -> tuple[list[dict[str, Any]], int]: """ @@ -80,24 +177,11 @@ def build_delivery_plan( data_root = load_settings().data_root_path if release_id: - from mesa_legal_data.release.verifier import verify_release - - if not verify_release(release_id): + try: + versions = _load_frozen_release_versions(conn, release_id, data_root) + except Exception: conn.close() - raise MesaClientError(f"Cannot build delivery plan from unverified or invalid release {release_id}") - - c_ver = conn.cursor() - c_ver.execute( - """SELECT DISTINCT r.version_id, v.document_id, d.family - FROM release_items ri - JOIN records r ON r.record_id = ri.record_id AND r.record_sha256 = ri.record_sha256 - JOIN versions v ON v.version_id = r.version_id - JOIN documents d ON d.document_id = v.document_id - WHERE ri.release_id = ?""", - (release_id,), - ) - version_rows = c_ver.fetchall() - versions = [{"version_id": r[0], "document_id": r[1], "family": r[2]} for r in version_rows] + raise blocked_count = 0 else: versions, blocked_count = get_ready_versions_and_content(conn) @@ -113,17 +197,35 @@ def build_delivery_plan( v_id = v_info["version_id"] canonical_text = "" records: list[dict[str, Any]] = [] - c_cur = conn.cursor() if release_id: - c_cur.execute( - """SELECT r.record_id, r.record_type, r.canonical_path, r.canonical_line, r.record_sha256 - FROM release_items ri - JOIN records r ON r.record_id = ri.record_id AND r.record_sha256 = ri.record_sha256 - WHERE ri.release_id = ? AND r.version_id = ? - ORDER BY r.canonical_path, r.canonical_line""", - (release_id, v_id), - ) + packaged_records = v_info["packaged_records"] + for item in packaged_records: + record_type = item.get("record_type") + record_id = item.get("id") + if record_type == "legislation": + canonical_text = item.get("full_text") or "" + elif record_type == "decision": + canonical_text = item.get("text") or "" + elif record_type == "article": + span = item.get("source_span") or {} + records.append( + { + "record_id": record_id, + "record_type": "article", + "char_start": span.get("char_start"), + "char_end": span.get("char_end"), + "ordinal": item.get("ordinal", 0), + "title": item.get("heading"), + "article_number": item.get("article_number"), + "content": item.get("text") or "", + } + ) + if not canonical_text.strip(): + conn.close() + raise MesaClientError(f"Release {release_id} version {v_id} has no canonical document text") + rec_rows = [] else: + c_cur = conn.cursor() c_cur.execute( """SELECT record_id, record_type, canonical_path, canonical_line, record_sha256 FROM records @@ -131,7 +233,7 @@ def build_delivery_plan( ORDER BY canonical_path, canonical_line""", (v_id,), ) - rec_rows = c_cur.fetchall() + rec_rows = c_cur.fetchall() try: lines_by_path: dict[str, list[str]] = {} @@ -217,6 +319,7 @@ def execute_publish_delivery( delivery_id: str | None = None, release_id: str | None = None, target_key: str = "default", + expected_target_config_sha256: str | None = None, progress_callback: Callable[[dict[str, Any]], None] | None = None, is_cancelled_cb: Callable[[], bool] | None = None, ) -> dict[str, Any]: @@ -231,6 +334,9 @@ def execute_publish_delivery( """ conn = get_connection() target_settings = get_mesa_target_settings(conn, target_key) + if expected_target_config_sha256 and target_config_sha256(target_settings) != expected_target_config_sha256: + conn.close() + raise MesaClientError("MESA target configuration changed after human confirmation") client = MesaClient(settings=target_settings) if not delivery_id: @@ -254,12 +360,20 @@ def execute_publish_delivery( raise MesaClientError(f"Publisher preflight failed: {failures}") # 2. Create delivery in ledger + release_manifest_sha256 = None + if release_id: + manifest_row = conn.execute( + "SELECT manifest_sha256 FROM releases WHERE release_id = ?", (release_id,) + ).fetchone() + release_manifest_sha256 = manifest_row[0] if manifest_row else None create_delivery( conn, delivery_id=delivery_id, release_id=release_id, target_key=target_key, total_items=total_items, + target_config_sha256=target_config_sha256(target_settings), + release_manifest_sha256=release_manifest_sha256, ) update_delivery_progress( @@ -276,11 +390,18 @@ def execute_publish_delivery( skipped_count = 0 last_err = None cancelled_early = False + release_invalidated = False for idx, (chunk, is_already_done) in enumerate(chunk_tuples, start=1): if is_cancelled_cb and is_cancelled_cb(): cancelled_early = True break + if release_id: + status_row = conn.execute("SELECT status FROM releases WHERE release_id = ?", (release_id,)).fetchone() + if not status_row or status_row[0] not in ("verified", "published"): + release_invalidated = True + last_err = f"Release {release_id} is no longer in a publishable lifecycle state" + break item_id = f"item-{uuid.uuid4().hex[:12]}" idemp_key = generate_idempotency_key( @@ -412,6 +533,10 @@ def execute_publish_delivery( awaiting_count = cursor.fetchone()[0] if cancelled_early or (is_cancelled_cb and is_cancelled_cb()): final_delivery_status = DeliveryStatus.CANCELLED.value + elif release_invalidated: + final_delivery_status = ( + DeliveryStatus.PARTIAL.value if committed_count > 0 or skipped_count > 0 else DeliveryStatus.FAILED.value + ) elif awaiting_count: final_delivery_status = DeliveryStatus.AWAITING_MUTATION.value elif total_items == 0: @@ -459,6 +584,26 @@ def retry_delivery_failures( raise ValueError(f"Delivery {delivery_id} not found") target_settings = get_mesa_target_settings(conn, delivery.get("target_key", "default")) + expected_target_hash = delivery.get("target_config_sha256") + if expected_target_hash and target_config_sha256(target_settings) != expected_target_hash: + conn.close() + raise MesaClientError("MESA target configuration changed after human confirmation") + release_id = delivery.get("release_id") + if release_id: + from mesa_legal_data.release.verifier import verify_release + + release_row = conn.execute( + "SELECT status, manifest_sha256 FROM releases WHERE release_id = ?", (release_id,) + ).fetchone() + expected_manifest = delivery.get("release_manifest_sha256") + if ( + not release_row + or release_row[0] not in ("verified", "published") + or (expected_manifest and release_row[1] != expected_manifest) + ): + conn.close() + raise MesaClientError("Confirmed release is no longer publishable") + verify_release(release_id) client = MesaClient(settings=target_settings) failed_items = list_failed_delivery_items(conn, delivery_id) diff --git a/src/mesa_legal_data/publisher/ledger.py b/src/mesa_legal_data/publisher/ledger.py index 0fce6d1..7f797b8 100644 --- a/src/mesa_legal_data/publisher/ledger.py +++ b/src/mesa_legal_data/publisher/ledger.py @@ -86,13 +86,28 @@ def create_delivery( release_id: str | None, target_key: str, total_items: int, + target_config_sha256: str | None = None, + release_manifest_sha256: str | None = None, ) -> None: now_iso = datetime.now(UTC).isoformat() with transaction(conn): conn.execute( - """INSERT INTO mesa_deliveries (delivery_id, release_id, target_key, status, started_at, total_items, committed_items, failed_items, skipped_items, created_at) - VALUES (?, ?, ?, ?, ?, ?, 0, 0, 0, ?)""", - (delivery_id, release_id, target_key, DeliveryStatus.PLANNED.value, now_iso, total_items, now_iso), + """INSERT INTO mesa_deliveries ( + delivery_id, release_id, target_key, status, started_at, total_items, + committed_items, failed_items, skipped_items, created_at, + target_config_sha256, release_manifest_sha256 + ) VALUES (?, ?, ?, ?, ?, ?, 0, 0, 0, ?, ?, ?)""", + ( + delivery_id, + release_id, + target_key, + DeliveryStatus.PLANNED.value, + now_iso, + total_items, + now_iso, + target_config_sha256, + release_manifest_sha256, + ), ) @@ -211,7 +226,8 @@ def list_deliveries(conn: sqlite3.Connection, limit: int = 20, offset: int = 0) cursor = conn.cursor() cursor.execute( """SELECT delivery_id, release_id, target_key, status, started_at, finished_at, - total_items, committed_items, failed_items, skipped_items, last_error, created_at + total_items, committed_items, failed_items, skipped_items, last_error, created_at, + target_config_sha256, release_manifest_sha256 FROM mesa_deliveries ORDER BY created_at DESC LIMIT ? OFFSET ?""", (limit, offset), ) @@ -230,6 +246,8 @@ def list_deliveries(conn: sqlite3.Connection, limit: int = 20, offset: int = 0) "skipped_items": r[9], "last_error": r[10], "created_at": r[11], + "target_config_sha256": r[12], + "release_manifest_sha256": r[13], } for r in rows ] @@ -239,7 +257,8 @@ def get_delivery(conn: sqlite3.Connection, delivery_id: str) -> dict[str, Any] | cursor = conn.cursor() cursor.execute( """SELECT delivery_id, release_id, target_key, status, started_at, finished_at, - total_items, committed_items, failed_items, skipped_items, last_error, created_at + total_items, committed_items, failed_items, skipped_items, last_error, created_at, + target_config_sha256, release_manifest_sha256 FROM mesa_deliveries WHERE delivery_id = ?""", (delivery_id,), ) @@ -259,6 +278,8 @@ def get_delivery(conn: sqlite3.Connection, delivery_id: str) -> dict[str, Any] | "skipped_items": r[9], "last_error": r[10], "created_at": r[11], + "target_config_sha256": r[12], + "release_manifest_sha256": r[13], } diff --git a/src/mesa_legal_data/release/builder.py b/src/mesa_legal_data/release/builder.py index 3caefdc..52271a5 100644 --- a/src/mesa_legal_data/release/builder.py +++ b/src/mesa_legal_data/release/builder.py @@ -75,6 +75,7 @@ def build_release(release_id: str | None = None) -> dict[str, Any]: canonical_path TEXT NOT NULL, canonical_line INTEGER NOT NULL, version_id TEXT NOT NULL, + document_id TEXT NOT NULL, PRIMARY KEY (canonical_path, canonical_line) ); @@ -82,6 +83,7 @@ def build_release(release_id: str | None = None) -> dict[str, Any]: record_type TEXT NOT NULL, record_id TEXT NOT NULL, record_sha256 TEXT NOT NULL, + payload_sha256 TEXT NOT NULL, payload_json TEXT NOT NULL, PRIMARY KEY (record_type, record_id) ); @@ -100,18 +102,19 @@ def build_release(release_id: str | None = None) -> dict[str, Any]: ref.canonical_path, ref.canonical_line, ref.version_id, + ref.document_id, ) ) if len(selected_batch) >= batch_size: spool_conn.executemany( - "INSERT INTO selected_records (record_id, record_type, record_sha256, canonical_path, canonical_line, version_id) VALUES (?, ?, ?, ?, ?, ?)", + "INSERT INTO selected_records (record_id, record_type, record_sha256, canonical_path, canonical_line, version_id, document_id) VALUES (?, ?, ?, ?, ?, ?, ?)", selected_batch, ) selected_batch.clear() if selected_batch: spool_conn.executemany( - "INSERT INTO selected_records (record_id, record_type, record_sha256, canonical_path, canonical_line, version_id) VALUES (?, ?, ?, ?, ?, ?)", + "INSERT INTO selected_records (record_id, record_type, record_sha256, canonical_path, canonical_line, version_id, document_id) VALUES (?, ?, ?, ?, ?, ?, ?)", selected_batch, ) selected_batch.clear() @@ -224,11 +227,14 @@ def build_release(release_id: str | None = None) -> dict[str, Any]: ) det_payload = json_str_deterministic(rec_obj) - payload_batch.append((expected_r_type, expected_r_id, expected_hash, det_payload)) + payload_sha256 = hashlib.sha256((det_payload + "\n").encode("utf-8")).hexdigest() + payload_batch.append( + (expected_r_type, expected_r_id, expected_hash, payload_sha256, det_payload) + ) if len(payload_batch) >= batch_size: spool_conn.executemany( - "INSERT INTO payload_spool (record_type, record_id, record_sha256, payload_json) VALUES (?, ?, ?, ?)", + "INSERT INTO payload_spool (record_type, record_id, record_sha256, payload_sha256, payload_json) VALUES (?, ?, ?, ?, ?)", payload_batch, ) payload_batch.clear() @@ -243,7 +249,7 @@ def build_release(release_id: str | None = None) -> dict[str, Any]: if payload_batch: spool_conn.executemany( - "INSERT INTO payload_spool (record_type, record_id, record_sha256, payload_json) VALUES (?, ?, ?, ?)", + "INSERT INTO payload_spool (record_type, record_id, record_sha256, payload_sha256, payload_json) VALUES (?, ?, ?, ?, ?)", payload_batch, ) payload_batch.clear() @@ -287,6 +293,43 @@ def build_release(release_id: str | None = None) -> dict[str, Any]: with open(out_file, "rb") as f: file_manifest_entries[fn] = hash_stream(f) + # Frozen publisher identity index. This is part of the verified package, + # so delivery planning never has to reconstruct version membership from + # the mutable live catalog after human confirmation. + index_file = building_dir / "data/release-index.jsonl" + index_cur = spool_conn.cursor() + index_cur.execute( + """SELECT sr.record_id, sr.record_type, sr.record_sha256, ps.payload_sha256, + sr.version_id, sr.document_id + FROM selected_records sr + JOIN payload_spool ps + ON ps.record_type = sr.record_type AND ps.record_id = sr.record_id + ORDER BY sr.version_id, sr.record_type, sr.record_id""" + ) + with open(index_file, "w", encoding="utf-8") as f: + while True: + rows = index_cur.fetchmany(batch_size) + if not rows: + break + for row in rows: + f.write( + json_str_deterministic( + { + "record_id": row[0], + "record_type": row[1], + "record_sha256": row[2], + "payload_sha256": row[3], + "version_id": row[4], + "document_id": row[5], + } + ) + + "\n" + ) + f.flush() + os.fsync(f.fileno()) + with open(index_file, "rb") as f: + file_manifest_entries["data/release-index.jsonl"] = hash_stream(f) + # 5. Copy schema files project_schemas_dir = Path(__file__).parent.parent.parent.parent / "schemas" if project_schemas_dir.exists(): diff --git a/src/mesa_legal_data/release/importer.py b/src/mesa_legal_data/release/importer.py index ba96571..4d8cc86 100644 --- a/src/mesa_legal_data/release/importer.py +++ b/src/mesa_legal_data/release/importer.py @@ -181,7 +181,7 @@ def import_release_to_staging(release_id: str, batch_size: int = 2000) -> dict[s # 2. Stream JSONL records in batches derived strictly from authenticated manifest entries jsonl_files: list[Path] = [] for rel_p in sorted(files_dict.keys()): - if rel_p.startswith("data/") and rel_p.endswith(".jsonl"): + if rel_p.startswith("data/") and rel_p.endswith(".jsonl") and rel_p != "data/release-index.jsonl": p = release_dir / rel_p if p.exists() and not p.is_symlink(): jsonl_files.append(p) diff --git a/src/mesa_legal_data/release/verifier.py b/src/mesa_legal_data/release/verifier.py index e4eb1a2..b5bf211 100644 --- a/src/mesa_legal_data/release/verifier.py +++ b/src/mesa_legal_data/release/verifier.py @@ -1,3 +1,4 @@ +import hashlib import json from pathlib import Path @@ -86,6 +87,7 @@ def verify_release_directory(release_dir: Path, expected_release_id: str | None } seen_record_ids: set[str] = set() + payload_identities: dict[str, tuple[str, str]] = {} for r_type, (rel_path, expected_count) in type_to_file.items(): jsonl_path = release_dir / rel_path @@ -115,12 +117,55 @@ def verify_release_directory(release_dir: Path, expected_release_id: str | None if r_id in seen_record_ids: raise ReleaseVerificationError(f"Duplicate record ID '{r_id}' found in release in {rel_path}") seen_record_ids.add(r_id) + payload_identities[r_id] = (r_type, hashlib.sha256(line.encode("utf-8")).hexdigest()) if actual_count != expected_count: raise ReleaseVerificationError( f"Count mismatch in {rel_path}: expected {expected_count}, found {actual_count} lines" ) + # The release-owned identity index binds every payload to its exact legal + # version and document. Publisher planning consumes this verified file, + # never a reconstructed live-catalog join. + index_path = release_dir / "data/release-index.jsonl" + if not index_path.exists(): + # Legacy release packages predate publisher-bound version identity. + # They remain verifiable for local archival/import, but the MESA + # publisher rejects them because it requires this frozen index. + return True + + indexed_ids: set[str] = set() + indexed_instances: set[tuple[str, str]] = set() + with open(index_path, "r", encoding="utf-8") as f: + for idx, line in enumerate(f, start=1): + try: + item = json.loads(line) + record_id = item["record_id"] + record_type = item["record_type"] + record_sha256 = item["record_sha256"] + payload_sha256 = item["payload_sha256"] + version_id = item["version_id"] + document_id = item["document_id"] + except (json.JSONDecodeError, KeyError, TypeError) as exc: + raise ReleaseVerificationError(f"Invalid release index entry at line {idx}: {exc}") from exc + if not all( + isinstance(value, str) and value + for value in (record_id, record_type, record_sha256, payload_sha256, version_id, document_id) + ): + raise ReleaseVerificationError(f"Empty or non-string release index identity at line {idx}") + if record_id in indexed_ids or (version_id, record_id) in indexed_instances: + raise ReleaseVerificationError( + f"Duplicate release index identity at line {idx}: {version_id}/{record_id}" + ) + expected_payload = payload_identities.get(record_id) + if expected_payload != (record_type, payload_sha256): + raise ReleaseVerificationError(f"Release index does not match payload for record '{record_id}'") + indexed_ids.add(record_id) + indexed_instances.add((version_id, record_id)) + + if indexed_ids != seen_record_ids: + raise ReleaseVerificationError("Release index membership does not exactly match packaged payloads") + return True diff --git a/src/mesa_legal_data/sources/manual.py b/src/mesa_legal_data/sources/manual.py index 11625ae..7d9a417 100644 --- a/src/mesa_legal_data/sources/manual.py +++ b/src/mesa_legal_data/sources/manual.py @@ -262,6 +262,27 @@ def import_manual_url( stable_key=doc_key, lifecycle_status="fetched", ) + insert_artifact( + conn=conn, + artifact_id=existing["artifact_id"], + document_id=document_id, + source_id=existing["source_id"], + source_url=existing["source_url"], + retrieved_at=existing["retrieved_at"], + fetch_method=existing["fetch_method"], + http_status=existing["http_status"], + declared_content_type=existing["declared_content_type"], + detected_content_type=existing["detected_content_type"], + byte_size=existing["byte_size"], + sha256=artifact_sha256, + raw_path=existing["raw_path"], + etag=existing["etag"], + last_modified=existing["last_modified"], + transport_status=existing["transport_status"], + error_code=existing["error_code"], + metadata_json=json.dumps({"publication_date": publication_date}), + ) + refreshed = get_artifact(conn, artifact_id) or existing conn.close() return FetchedArtifact( artifact_id=artifact_id, @@ -278,7 +299,7 @@ def import_manual_url( raw_path=existing["raw_path"], transport_status=existing.get("transport_status", "verified"), is_duplicate=True, - metadata={}, + metadata=json.loads(refreshed.get("metadata_json") or "{}"), ) ext = ".html" if "html" in detected_mime else (".pdf" if "pdf" in detected_mime else ".bin") diff --git a/src/mesa_legal_data/web/api.py b/src/mesa_legal_data/web/api.py index ca9d7a3..88ab70f 100644 --- a/src/mesa_legal_data/web/api.py +++ b/src/mesa_legal_data/web/api.py @@ -1,8 +1,9 @@ import hashlib import json import uuid +from datetime import UTC, datetime from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict, NoReturn, Optional from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile @@ -35,6 +36,7 @@ from mesa_legal_data.publisher.engine import ( build_delivery_plan, retry_delivery_failures, + target_config_sha256, ) from mesa_legal_data.publisher.ledger import ( get_delivery, @@ -78,7 +80,9 @@ def ok_response(data: Any = None) -> Dict[str, Any]: return {"ok": True, "data": data} -def error_response(code: str, message: str, status_code: int = 400, details: Optional[Dict[str, Any]] = None): +def error_response( + code: str, message: str, status_code: int = 400, details: Optional[Dict[str, Any]] = None +) -> NoReturn: raise HTTPException( status_code=status_code, detail={"code": code, "message": message, "details": details or {}}, @@ -1671,7 +1675,6 @@ async def publish_release_endpoint(release_id: str): @router.post("/releases/{release_id:path}/import") -@router.post("/releases/{release_id:path}/import-to-mesa") @router.post("/releases/{release_id:path}/import-to-staging") @router.post("/releases/{release_id:path}/import-staging") async def import_release_endpoint(release_id: str): @@ -2197,14 +2200,65 @@ def get_publisher_ready_summary_endpoint(target_key: str = "default"): return ok_response(summary.model_dump()) +@router.post("/publisher/prepare") +async def prepare_publisher_release_endpoint(target_key: str = "default"): + """Freeze, verify, and summarize the exact release a human will confirm.""" + async with write_lock.acquire_write(): + release_id = f"release-mesa-{datetime.now(UTC).strftime('%Y%m%dT%H%M%S')}-{uuid.uuid4().hex[:8]}" + try: + build_release(release_id=release_id) + verify_release(release_id) + conn = get_connection() + release = get_release(conn, release_id) + settings = get_mesa_target_settings(conn, target_key) + conn.close() + if not release or release["status"] != "verified": + error_response("RELEASE_NOT_VERIFIED", "Frozen release did not reach verified status", status_code=409) + _, summary = build_delivery_plan(target_key=target_key, release_id=release_id) + return ok_response( + { + "release_id": release_id, + "manifest_sha256": release["manifest_sha256"], + "target_key": target_key, + "target_config_sha256": target_config_sha256(settings), + "target": { + "base_url": settings.base_url, + "tenant_id": settings.tenant_id, + "workspace_id": settings.workspace_id, + "dataset_id": settings.dataset_id, + }, + "summary": summary.model_dump(), + } + ) + except HTTPException: + raise + except Exception as exc: + error_response("RELEASE_PREPARE_FAILED", str(exc), status_code=400) + + @router.post("/publisher/publish") def start_publisher_delivery_endpoint(req: MesaPublishRequest): from mesa_legal_data.operations import submit_operation conn = get_connection() target_settings = get_mesa_target_settings(conn, req.target_key) + release = get_release(conn, req.release_id) + if not release: + conn.close() + error_response("RELEASE_NOT_FOUND", f"Release {req.release_id} not found", status_code=404) + if release["status"] not in ("verified", "published"): + conn.close() + error_response( + "RELEASE_STATUS_INVALID", f"Release status {release['status']} is not publishable", status_code=409 + ) + if release["manifest_sha256"] != req.manifest_sha256: + conn.close() + error_response("RELEASE_CONFIRMATION_MISMATCH", "Confirmed release manifest does not match", status_code=409) + if target_config_sha256(target_settings) != req.target_config_sha256: + conn.close() + error_response("TARGET_CONFIRMATION_MISMATCH", "MESA target changed after confirmation", status_code=409) client = MesaClient(settings=target_settings) - _, summary = build_delivery_plan(target_key=req.target_key) + _, summary = build_delivery_plan(target_key=req.target_key, release_id=req.release_id) report = client.run_preflight_checks( ready_documents_count=summary.ready_documents, @@ -2228,6 +2282,7 @@ def start_publisher_delivery_endpoint(req: MesaPublishRequest): "delivery_id": delivery_id, "release_id": req.release_id, "target_key": req.target_key, + "target_config_sha256": req.target_config_sha256, }, ) conn.close() diff --git a/src/mesa_legal_data/web/schemas.py b/src/mesa_legal_data/web/schemas.py index faeab99..e1f12de 100644 --- a/src/mesa_legal_data/web/schemas.py +++ b/src/mesa_legal_data/web/schemas.py @@ -93,4 +93,6 @@ class MesaTargetSettingsUpdateRequest(BaseModel): class MesaPublishRequest(BaseModel): model_config = ConfigDict(extra="forbid") target_key: str = Field(default="default") - release_id: Optional[str] = None + release_id: str = Field(min_length=3, max_length=100, pattern=r"^[a-zA-Z0-9][a-zA-Z0-9._-]+$") + manifest_sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + target_config_sha256: str = Field(pattern=r"^[a-f0-9]{64}$") diff --git a/src/mesa_legal_data/web/static/app.js b/src/mesa_legal_data/web/static/app.js index 606ade6..4f5ca5e 100644 --- a/src/mesa_legal_data/web/static/app.js +++ b/src/mesa_legal_data/web/static/app.js @@ -13,6 +13,7 @@ const state = { harvestPollTimer: null, activeReviewTab: "pending", issueFilterSubjectId: null, + preparedMesaRelease: null, }; const VIEW_DESCRIPTIONS = { @@ -122,7 +123,7 @@ function humanTerm(term) { warning: "Uyarı", verified: "Doğrulandı", published: "Yayınlandı", - imported: "MESA'ya aktarıldı", + imported: "Yerel staging'e aktarıldı", revoked: "Geri çekildi", open: "Çözüm bekliyor", resolved: "Çözüldü", @@ -517,7 +518,7 @@ async function loadHomeView() { nextBtn.onclick = () => switchView("collect"); } else { nextTitle.textContent = "Tüm veriler güncel ve hazır"; - nextDesc.textContent = "Onaylanan verileri dosya olarak indirebilir veya MESA'ya aktarabilirsiniz."; + nextDesc.textContent = "Onaylanan verileri immutable release olarak hazırlayıp insan onayıyla MESA'ya gönderebilirsiniz."; nextBtn.textContent = "Dışa Aktar"; nextBtn.onclick = () => switchView("export"); } @@ -1738,18 +1739,24 @@ async function handleMesaPreflight() { async function handleMesaPublishTrigger() { try { - const summary = await apiRequest("/api/publisher/ready-summary"); - const settings = await apiRequest("/api/publisher/settings"); + const prepared = await apiRequest("/api/publisher/prepare?target_key=default", { method: "POST" }); + const summary = prepared.summary; + const settings = prepared.target; + state.preparedMesaRelease = prepared; const elDocs = document.getElementById("confirm-mesa-docs"); const elVers = document.getElementById("confirm-mesa-versions"); const elChunks = document.getElementById("confirm-mesa-chunks"); const elTarget = document.getElementById("confirm-mesa-target"); + const elRelease = document.getElementById("confirm-mesa-release"); + const elManifest = document.getElementById("confirm-mesa-manifest"); if (elDocs) elDocs.textContent = `${summary.ready_documents} belge`; if (elVers) elVers.textContent = `${summary.ready_versions} versiyon`; if (elChunks) elChunks.textContent = `${summary.estimated_chunks} chunk (${summary.new_chunks_to_send} yeni gönderilecek)`; if (elTarget) elTarget.textContent = `${settings.tenant_id} / ${settings.workspace_id} / ${settings.dataset_id} (${settings.base_url})`; + if (elRelease) elRelease.textContent = prepared.release_id; + if (elManifest) elManifest.textContent = prepared.manifest_sha256; showModal("modal-mesa-confirm"); } catch (err) { @@ -1758,6 +1765,11 @@ async function handleMesaPublishTrigger() { } async function handleMesaConfirmPublish() { + const prepared = state.preparedMesaRelease; + if (!prepared) { + showToast("Doğrulanmış yayın paketi bulunamadı. Özeti yeniden hazırlayın.", "warning"); + return; + } hideModal("modal-mesa-confirm"); const boxProg = document.getElementById("box-mesa-delivery-progress"); const boxPartial = document.getElementById("box-mesa-partial-failure"); @@ -1769,9 +1781,15 @@ async function handleMesaConfirmPublish() { const res = await apiRequest("/api/publisher/publish", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ target_key: "default" }), + body: JSON.stringify({ + target_key: prepared.target_key, + release_id: prepared.release_id, + manifest_sha256: prepared.manifest_sha256, + target_config_sha256: prepared.target_config_sha256, + }), }); + state.preparedMesaRelease = null; activeMesaDeliveryId = res.delivery_id; showToast(`MESA aktarımı başlatıldı (Teslimat: ${activeMesaDeliveryId}).`, "info"); startMesaDeliveryPolling(activeMesaDeliveryId); @@ -1970,7 +1988,7 @@ async function runMesaTransferSequence() { await apiRequest(`/api/releases/${releaseId}/import-staging`, { method: "POST" }); statusText.textContent = "✓ Yerel Development Staging release paketi oluşturuldu."; - showToast("Onaylı kayıtlar yerel development staging ortamına aktarıldı. (MESA v4 publisher entegrasyonu sonraki aşamada tamamlanacaktır)", "success"); + showToast("Onaylı kayıtlar yalnız yerel development staging ortamına aktarıldı.", "success"); } catch (err) { console.error("Staging release error:", err); statusText.textContent = `İşlem tamamlanamadı: ${err.message}`; diff --git a/src/mesa_legal_data/web/static/index.html b/src/mesa_legal_data/web/static/index.html index 7f345cf..90798b9 100644 --- a/src/mesa_legal_data/web/static/index.html +++ b/src/mesa_legal_data/web/static/index.html @@ -1012,6 +1012,8 @@

MESA'ya Gönderimi Onayla

Hazır Belge: 0
Hazır Versiyon: 0
Tahmini Source Chunk: 0
+
Doğrulanmış Release: -
+
Manifest SHA-256: -
Hedef: default / legal / tr_legislation

diff --git a/tests/integration/test_master_mvp_closure.py b/tests/integration/test_master_mvp_closure.py index 6b60218..88f4fcd 100644 --- a/tests/integration/test_master_mvp_closure.py +++ b/tests/integration/test_master_mvp_closure.py @@ -12,6 +12,7 @@ from mesa_legal_data.catalog import ( approve_record_with_checks, approve_version_streaming, + get_artifact, get_connection, get_db_path, get_document, @@ -30,6 +31,7 @@ from mesa_legal_data.harvest.service_bridge import run_pipeline_item from mesa_legal_data.hashing import hash_stream from mesa_legal_data.pipeline import process_artifact_pipeline +from mesa_legal_data.publisher.client import MesaClientError from mesa_legal_data.publisher.engine import build_delivery_plan, execute_publish_delivery from mesa_legal_data.publisher.ledger import upsert_mesa_target_settings from mesa_legal_data.publisher.models import DeliveryStatus, MesaTargetSettings @@ -100,6 +102,85 @@ def _helper_create_law_artifact( return artifact_id +def test_final_panel_confirmation_binds_exact_release_manifest_and_target(tmp_path, monkeypatch): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + doc_id = "tr:legislation:law:panel-frozen-final" + art_v1 = _helper_create_law_artifact( + tmp_path, + doc_id, + "

Panel V1

Madde 1- Frozen.

", + "art-panel-final-v1", + pub_date="2026-01-01", + ) + process_artifact_pipeline(art_v1) + conn = get_connection() + v1 = get_version_for_artifact(conn, art_v1) + assert v1 is not None + approve_version_streaming(conn, version_id=v1["version_id"], reviewer="auditor") + conn.close() + + client = TestClient(create_app()) + prepared_response = client.post("/api/publisher/prepare?target_key=default", headers=WEB_HEADERS) + assert prepared_response.status_code == 200 + prepared = prepared_response.json()["data"] + assert prepared["release_id"] + assert len(prepared["manifest_sha256"]) == 64 + assert len(prepared["target_config_sha256"]) == 64 + + # Live catalog changes after the confirmation summary was created. + art_v2 = _helper_create_law_artifact( + tmp_path, + doc_id, + "

Panel V2

Madde 1- Live changed.

", + "art-panel-final-v2", + pub_date="2026-02-01", + ) + process_artifact_pipeline(art_v2) + conn = get_connection() + v2 = get_version_for_artifact(conn, art_v2) + assert v2 is not None + approve_version_streaming(conn, version_id=v2["version_id"], reviewer="auditor") + conn.close() + + bad_response = client.post( + "/api/publisher/publish", + headers=WEB_HEADERS, + json={ + "target_key": prepared["target_key"], + "release_id": prepared["release_id"], + "manifest_sha256": "0" * 64, + "target_config_sha256": prepared["target_config_sha256"], + }, + ) + assert bad_response.status_code == 409 + + captured: dict[str, Any] = {} + + def fake_submit(conn, *, operation_type, requested_by, input_dict): + captured.update(input_dict) + return "op-frozen-final" + + monkeypatch.setattr("mesa_legal_data.operations.submit_operation", fake_submit) + monkeypatch.setattr( + "mesa_legal_data.publisher.client.MesaClient.run_preflight_checks", + lambda self, **kwargs: type("Report", (), {"overall_status": "PASS", "checks": []})(), + ) + response = client.post( + "/api/publisher/publish", + headers=WEB_HEADERS, + json={ + "target_key": prepared["target_key"], + "release_id": prepared["release_id"], + "manifest_sha256": prepared["manifest_sha256"], + "target_config_sha256": prepared["target_config_sha256"], + }, + ) + assert response.status_code == 200 + assert captured["release_id"] == prepared["release_id"] + frozen_chunks, _ = build_delivery_plan(release_id=captured["release_id"]) + assert {chunk.version_id for chunk, _ in frozen_chunks} == {v1["version_id"]} + + def test_master_a_chronology_current_version_selection(tmp_path, monkeypatch): """Test A: Legal Chronology Current Version Selection - Historical backfill never replaces newer current version.""" monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) @@ -719,3 +800,130 @@ def test_master_af_target_settings_contract_truthful_status(tmp_path, monkeypatc retrieved = get_mesa_target_settings(conn, "prod") assert retrieved.contract_source == "configured" conn.close() + + +def test_final_release_plan_is_frozen_version_aware_and_revocation_guarded(tmp_path, monkeypatch): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + doc_id = "tr:legislation:law:frozen-final" + article = "

Madde 9- Aynı madde metni.

" + + art_v1 = _helper_create_law_artifact( + tmp_path, + doc_id, + f"

V1

{article}", + "art-frozen-v1", + pub_date="2026-01-01", + ) + process_artifact_pipeline(art_v1) + conn = get_connection() + v1 = get_version_for_artifact(conn, art_v1) + assert v1 is not None + approve_version_streaming(conn, version_id=v1["version_id"], reviewer="auditor") + conn.close() + + art_v2 = _helper_create_law_artifact( + tmp_path, + doc_id, + f"

V2 changed heading

{article}", + "art-frozen-v2", + pub_date="2026-02-01", + ) + process_artifact_pipeline(art_v2) + conn = get_connection() + v2 = get_version_for_artifact(conn, art_v2) + assert v2 is not None + approve_version_streaming(conn, version_id=v2["version_id"], reviewer="auditor") + + # Reproduce the dangerous equality case: the same logical Article 9 and + # hash are present in both versions. A record_id+hash join would leak v1. + r2 = conn.execute( + "SELECT canonical_path, canonical_line, record_sha256 FROM records WHERE version_id = ? AND record_type = 'article'", + (v2["version_id"],), + ).fetchone() + assert r2 is not None + conn.execute( + """UPDATE records SET canonical_path = ?, canonical_line = ?, record_sha256 = ? + WHERE version_id = ? AND record_type = 'article'""", + (r2[0], r2[1], r2[2], v1["version_id"]), + ) + conn.close() + + release_id = "rel-frozen-final" + build_release(release_id=release_id) + + # Mutate the live catalog after the human-visible release was frozen. + art_v3 = _helper_create_law_artifact( + tmp_path, + doc_id, + "

V3

Madde 9- Yeni canlı metin.

", + "art-frozen-v3", + pub_date="2026-03-01", + ) + process_artifact_pipeline(art_v3) + conn = get_connection() + v3 = get_version_for_artifact(conn, art_v3) + assert v3 is not None + approve_version_streaming(conn, version_id=v3["version_id"], reviewer="auditor") + conn.close() + + chunks, _ = build_delivery_plan(release_id=release_id) + assert chunks + assert {chunk.version_id for chunk, _ in chunks} == {v2["version_id"]} + assert all("Yeni canlı metin" not in chunk.content for chunk, _ in chunks) + + conn = get_connection() + conn.execute("UPDATE releases SET status = 'revoked' WHERE release_id = ?", (release_id,)) + conn.close() + with pytest.raises(MesaClientError, match="lifecycle status revoked"): + build_delivery_plan(release_id=release_id) + + +def test_final_invalid_calendar_dates_and_metadata_fill_only(tmp_path, monkeypatch): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + doc_id = "tr:legislation:law:calendar-final" + html = "

Date Law

Madde 1- Metin.

" + + art_invalid = _helper_create_law_artifact( + tmp_path, doc_id, html, "art-invalid-date", source_id="resmi_gazete", pub_date="2026-02-31" + ) + process_artifact_pipeline(art_invalid) + conn = get_connection() + invalid_version = get_version_for_artifact(conn, art_invalid) + assert invalid_version is not None + assert "unknown-date" in invalid_version["version_id"] + assert invalid_version["approval_status"] == "pending" + assert invalid_version["quality_status"] != "PASS" + assert ( + conn.execute( + "SELECT count(*) FROM validation_issues WHERE version_id = ? AND code = 'LEGAL_DATE_INVALID'", + (invalid_version["version_id"],), + ).fetchone()[0] + == 1 + ) + conn.close() + + # Identical immutable bytes may gain a missing authoritative date, but an + # existing value is never silently overwritten by a conflicting value. + meta_doc = "tr:legislation:law:metadata-final" + meta_html = "

Metadata Law

Madde 1- Ayrı metin.

" + first = _helper_create_law_artifact(tmp_path, meta_doc, meta_html, "art-meta-first") + _helper_create_law_artifact(tmp_path, meta_doc, meta_html, "art-meta-second", pub_date="2026-08-10") + conn = get_connection() + stored = get_artifact(conn, first) + assert stored is not None + assert json.loads(stored["metadata_json"])["publication_date"] == "2026-08-10" + conn.close() + + _helper_create_law_artifact(tmp_path, meta_doc, meta_html, "art-meta-third", pub_date="2026-08-11") + conn = get_connection() + stored = get_artifact(conn, first) + assert stored is not None + assert json.loads(stored["metadata_json"])["publication_date"] == "2026-08-10" + assert ( + conn.execute( + "SELECT count(*) FROM validation_issues WHERE subject_id = ? AND code = 'ARTIFACT_METADATA_CONFLICT'", + (first,), + ).fetchone()[0] + == 1 + ) + conn.close() diff --git a/tests/integration/test_post_master_independent_audit.py b/tests/integration/test_post_master_independent_audit.py index b92b595..a247bda 100644 --- a/tests/integration/test_post_master_independent_audit.py +++ b/tests/integration/test_post_master_independent_audit.py @@ -252,6 +252,14 @@ def test_control_6_same_date_different_content_deterministic_safe(tmp_path, monk # The existing current version (v_a) must be preserved deterministically rather than arbitrary silent switch doc_b = get_document(conn, doc_id) assert doc_b["current_version_id"] == v_a["version_id"] + assert v_b["auto_approved"] is False + assert ( + conn.execute( + "SELECT count(*) FROM validation_issues WHERE version_id = ? AND code = 'VERSION_DATE_AMBIGUITY'", + (v_b["version_id"],), + ).fetchone()[0] + == 1 + ) conn.close() @@ -575,8 +583,19 @@ def test_control_21_to_24_validation_issue_version_and_document_scoping(tmp_path version_id=v1["version_id"], record_instance_id=r1_inst, ) + open_issue( + conn, + issue_id=f"iss-{uuid.uuid4().hex[:8]}", + subject_type="record", + subject_id=f"{doc_id}:article:9", + severity="blocker", + code="LEGACY_UNSCOPED_ARTICLE_BLOCKER", + message="Historical blocker without version identity", + details_json="{}", + ) - # v2 Article 9 must NOT be blocked and can be approved + # v2 Article 9 must NOT be blocked by either v1's scoped issue or the + # ambiguous legacy logical-record issue, and can be approved. approve_record_with_checks( conn, record_id=f"{doc_id}:article:9", @@ -864,9 +883,11 @@ def test_control_41_to_43_harvest_pilot_budget_and_throttle_enforcement(tmp_path art_1 = _helper_create_law_artifact(tmp_path, doc_id, html, "art-harv-1", pub_date="2026-01-01") process_artifact_pipeline(art_1) - # Ingest artifact 2 with same content (duplicate SHA) -> raises IntegrityError on sha256 UNIQUE - with pytest.raises(sqlite3.IntegrityError): - _helper_create_law_artifact(tmp_path, doc_id, html, "art-harv-2", pub_date="2026-01-01") + # Ingest artifact 2 with the same bytes: the immutable artifact is reused. + _helper_create_law_artifact(tmp_path, doc_id, html, "art-harv-2", pub_date="2026-01-01") + conn = get_connection() + assert conn.execute("SELECT count(*) FROM artifacts WHERE document_id = ?", (doc_id,)).fetchone()[0] == 1 + conn.close() # ============================================================================== @@ -1083,7 +1104,7 @@ def test_control_57_and_58_fresh_and_upgrade_database_integrity(tmp_path): ) migrations_dir = Path("migrations") for mig_file in sorted(migrations_dir.glob("*.sql")): - if "0010" in mig_file.name: + if "0010" in mig_file.name or "0011" in mig_file.name: continue sql = mig_file.read_text(encoding="utf-8") conn_up.executescript(sql) @@ -1107,14 +1128,26 @@ def test_control_57_and_58_fresh_and_upgrade_database_integrity(tmp_path): """INSERT INTO artifacts (artifact_id, document_id, source_id, source_url, retrieved_at, fetch_method, http_status, declared_content_type, detected_content_type, byte_size, sha256, raw_path, transport_status, metadata_json) VALUES ('art_up', 'doc_up', 'test_src', 'http://a.b/1', '2026-01-01', 'manual', 200, 'text/html', 'text/html', 10, 'sha_art_up', 'raw.html', 'fetched', '{}')""" ) + conn_up.execute( + """INSERT INTO artifacts (artifact_id, document_id, source_id, source_url, retrieved_at, fetch_method, http_status, declared_content_type, detected_content_type, byte_size, sha256, raw_path, transport_status, metadata_json) + VALUES ('art_up_2', 'doc_up', 'test_src', 'http://a.b/2', '2026-02-01', 'manual', 200, 'text/html', 'text/html', 10, 'sha_art_up_2', 'raw2.html', 'fetched', '{}')""" + ) conn_up.execute( """INSERT INTO versions (version_id, document_id, artifact_id, version_kind, canonical_path, canonical_line, canonical_sha256, parser_name, parser_version, schema_version, validation_status, privacy_status, approval_status, created_at) VALUES ('v_up', 'doc_up', 'art_up', 'consolidated_snapshot', 'c.jsonl', 1, 'sha_up', 'legislation', '1.0.0', 'v1', 'valid', 'clean', 'approved', '2026-01-01')""" ) + conn_up.execute( + """INSERT INTO versions (version_id, document_id, artifact_id, version_kind, canonical_path, canonical_line, canonical_sha256, parser_name, parser_version, schema_version, validation_status, privacy_status, approval_status, created_at, revision_number) + VALUES ('v_up_2', 'doc_up', 'art_up_2', 'consolidated_snapshot', 'c2.jsonl', 1, 'sha_up_2', 'legislation', '1.0.0', 'v1', 'valid', 'clean', 'approved', '2026-02-01', 2)""" + ) conn_up.execute( """INSERT INTO records (record_instance_id, version_id, record_id, record_type, canonical_path, canonical_line, record_sha256, approval_status, validation_status, created_at) VALUES ('v_up:doc_up:article:1', 'v_up', 'doc_up:article:1', 'article', 'c.jsonl', 1, 'sha_rec_up', 'approved', 'valid', '2026-01-01')""" ) + conn_up.execute( + """INSERT INTO records (record_instance_id, version_id, record_id, record_type, canonical_path, canonical_line, record_sha256, approval_status, validation_status, created_at) + VALUES ('v_up_2:doc_up:article:1', 'v_up_2', 'doc_up:article:1', 'article', 'c2.jsonl', 1, 'sha_rec_up', 'approved', 'valid', '2026-02-01')""" + ) conn_up.execute( """INSERT INTO record_reviews (record_id, record_sha256, decision, reviewer, note, reviewed_at) VALUES ('doc_up:article:1', 'sha_rec_up', 'approved', 'old_reviewer', 'note', '2026-01-01')""" @@ -1123,6 +1156,17 @@ def test_control_57_and_58_fresh_and_upgrade_database_integrity(tmp_path): """INSERT INTO validation_issues (issue_id, subject_type, subject_id, severity, code, message, details_json, status, opened_at) VALUES ('iss_up', 'version', 'v_up', 'warning', 'WARN_CODE', 'msg', '{}', 'open', '2026-01-01')""" ) + conn_up.execute( + """INSERT INTO releases (release_id, release_path, status, schema_version, created_at, manifest_sha256, counts_json, source_snapshot_json) + VALUES ('rel_up', 'releases/rel_up', 'verified', '1.0.0', '2026-01-01', 'manifest_up', '{}', '[]')""" + ) + conn_up.execute( + "INSERT INTO release_items (release_id, record_id, record_sha256, version_id) VALUES ('rel_up', 'doc_up:article:1', 'sha_rec_up', 'v_up')" + ) + conn_up.execute( + """INSERT INTO mesa_deliveries (delivery_id, release_id, target_key, status, started_at, total_items, committed_items, failed_items, skipped_items, created_at) + VALUES ('del_up', 'rel_up', 'default', 'PLANNED', '2026-01-01', 1, 0, 0, 0, '2026-01-01')""" + ) conn_up.commit() conn_up.close() @@ -1145,12 +1189,14 @@ def test_control_57_and_58_fresh_and_upgrade_database_integrity(tmp_path): row = cur_mig.fetchone() assert row is not None assert row[0] is not None and len(row[0]) > 0 - assert row[1] == "v_up:doc_up:article:1" - assert row[2] == "v_up" + assert row[1].startswith("legacy-ambiguous:") + assert row[2] == "legacy-version-unscoped" # Verify validation issue has version_id populated cur_mig.execute("SELECT version_id FROM validation_issues WHERE issue_id = 'iss_up'") assert cur_mig.fetchone()[0] == "v_up" + assert cur_mig.execute("SELECT count(*) FROM releases WHERE release_id = 'rel_up'").fetchone()[0] == 1 + assert cur_mig.execute("SELECT count(*) FROM mesa_deliveries WHERE delivery_id = 'del_up'").fetchone()[0] == 1 conn_mig.close() diff --git a/tests/integration/test_record_revisions.py b/tests/integration/test_record_revisions.py index d84b985..b4cdd0c 100644 --- a/tests/integration/test_record_revisions.py +++ b/tests/integration/test_record_revisions.py @@ -67,6 +67,37 @@ def test_record_revision_flow(tmp_path, monkeypatch): rec_after = get_record(conn, "rec-orig-1") assert rec_after["approval_status"] == "approved" + # The same logical record in another version must not be approved by a + # revision decision scoped to ver-rev-1. + c.execute( + """INSERT INTO artifacts (artifact_id, document_id, source_id, source_url, retrieved_at, fetch_method, http_status, declared_content_type, detected_content_type, byte_size, sha256, raw_path, transport_status, metadata_json) + VALUES ('art-rev-2', 'doc-rev-1', 'mevzuat', 'https://example.com/rev2.html', '2026-08-06T00:00:00Z', 'manual', 200, 'text/html', 'text/html', 10, 'sha-rev-2', 'raw/rev2.html', 'fetched', '{}')""" + ) + c.execute( + """INSERT INTO versions (version_id, document_id, artifact_id, version_kind, canonical_path, canonical_line, canonical_sha256, parser_name, parser_version, schema_version, validation_status, privacy_status, approval_status, created_at, revision_number) + VALUES ('ver-rev-2', 'doc-rev-1', 'art-rev-2', 'snapshot', 'canonical/rev2.jsonl', 1, 'sha-ver-rev-2', 'test_parser', '1.0', '1.0', 'valid', 'clean', 'pending', '2026-08-06T00:00:00Z', 2)""" + ) + c.execute( + """INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES ('rec-orig-1', 'ver-rev-2', 'article', 'canonical/rev2.jsonl', 1, 'sha-rec-orig-2', 'valid', 'pending', '2026-08-06T00:00:00Z')""" + ) + c.execute("UPDATE records SET approval_status = 'pending' WHERE version_id = 'ver-rev-1'") + scoped_rev = create_record_revision( + conn, + original_record_id="rec-orig-1", + original_record_sha256="sha-rec-orig-1", + revised_record_id="rec-orig-1", + revised_record_sha256="sha-rec-scoped", + version_id="ver-rev-1", + change_type="scope_check", + patch_json="{}", + reason="Version isolation", + created_by="editor_1", + ) + approve_record_revision(conn, scoped_rev, reviewer="lead_editor") + statuses = dict(conn.execute("SELECT version_id, approval_status FROM records WHERE record_id = 'rec-orig-1'")) + assert statuses == {"ver-rev-1": "approved", "ver-rev-2": "pending"} + # 3. Reject another revision rev_id2 = create_record_revision( conn, diff --git a/tests/unit/test_catalog_repository.py b/tests/unit/test_catalog_repository.py index 6993dfb..d2a86e8 100644 --- a/tests/unit/test_catalog_repository.py +++ b/tests/unit/test_catalog_repository.py @@ -214,7 +214,7 @@ def test_release(db_conn): ) insert_record(db_conn, rec_id, ver_id, "t", "p", 1, "h", "v", "a") - add_release_item(db_conn, rel_id, rec_id, "hashA") + add_release_item(db_conn, rel_id, rec_id, "h", version_id=ver_id) cursor = db_conn.cursor() cursor.execute("SELECT count(*) FROM release_items WHERE release_id = ?", (rel_id,)) diff --git a/uv.lock b/uv.lock index bb3a929..ad72af5 100644 --- a/uv.lock +++ b/uv.lock @@ -340,6 +340,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/ad/f4f0e57345f1870f3e8cb624e058d7eca6e5a27d33bcc3311d9b618734cd/httpcore2-2.12.0.tar.gz", hash = "sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648", size = 67548, upload-time = "2026-08-18T13:22:08.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/74/d370e55600d9bcfa0d9794b0166126d49291a3d2b20c268fc98c453a4948/httpcore2-2.12.0-py3-none-any.whl", hash = "sha256:7e04258ce01013d7d615e5b910a3b27fac937d7a95038227e79652b4ba3b4ceb", size = 83074, upload-time = "2026-08-18T13:22:05.854Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -355,6 +368,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform != 'emscripten'" }, + { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, + { name = "httpx2-jsfetch", marker = "python_full_version >= '3.12' and sys_platform == 'emscripten'" }, + { name = "idna" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/f8/579a8b51e42e38ee32647df9f08aa25643ae788e275cc625b199829c4671/httpx2-2.12.0.tar.gz", hash = "sha256:7631fe9887a8a2275f4a2540e053aa670fcc50742864a9ae7c66e609fdcf12cf", size = 100040, upload-time = "2026-08-18T13:22:09.086Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/95/411ba65569158e862368917aaf56597f3e5fa3b91b0502919638465a08f3/httpx2-2.12.0-py3-none-any.whl", hash = "sha256:cc8b6eecb8661c146b8f89a60e97456ee086e91a784ed31ac450c3a9e613dd36", size = 95427, upload-time = "2026-08-18T13:22:06.834Z" }, +] + +[[package]] +name = "httpx2-jsfetch" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, +] + [[package]] name = "idna" version = "3.18" @@ -580,6 +619,7 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "httpx2" }, { name = "mypy" }, { name = "pip-audit" }, { name = "pytest" }, @@ -611,6 +651,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "httpx2", specifier = ">=0.1.0" }, { name = "mypy", specifier = ">=2.3.0" }, { name = "pip-audit", specifier = ">=2.10.1" }, { name = "pytest", specifier = ">=9.1.1" }, @@ -1332,6 +1373,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "typer" version = "0.27.1"