diff --git a/.mesa_data_agent_state.json b/.mesa_data_agent_state.json index 0ad69b8..a801a20 100644 --- a/.mesa_data_agent_state.json +++ b/.mesa_data_agent_state.json @@ -4,7 +4,7 @@ "workflow": "mesa_data_autonomy", "branch": "fix/mesa-data-final-mvp", "baseline_commit": "84c61db716c10cb4110b673f9a976215afe71cd3", - "current_phase": "complete", + "current_phase": "final_p1_record_instance_not_null", "overall_status": "ready", "phases": { "prompt_1": { @@ -92,7 +92,7 @@ "Web panel needed 6 key metrics, truthful local staging publish labels, multi-version document history, reprocess actions, and source automation settings." ], "decisions": [ - "Implemented 4-state citation lifecycle with Turkish legal aliases (TCK, CMK, HMK, TMK, TBK, TTK, İİK, İYUK, VUK, KVKK, AY, AvK, PVSK, İşK) and temporal relation verbs (amends, repeals, adds).", + "Implemented 4-state citation lifecycle with Turkish legal aliases (TCK, CMK, HMK, TMK, TBK, TTK, \u0130\u0130K, \u0130YUK, VUK, KVKK, AY, AvK, PVSK, \u0130\u015fK) and temporal relation verbs (amends, repeals, adds).", "Enforced exact canonical coordinate slicing (canonical_text[char_start:char_end] == raw_text) and context-aware law number matching.", "Implemented 7-condition Safe Auto-Approval Engine in catalog.py with parser certifications and weekly audit sampling.", "Completed web management panel with health metrics, honest MESA banner, multi-version history, safe reprocessing, and operational settings." @@ -149,7 +149,7 @@ "Deterministic hashing: Canonical revision hash is pure SHA256 of sorted chunk content hashes. Stable idempotency key mesa-data: contains zero runtime volatile data.", "Delivery ledger & O(1) deduplication: Persists mesa_deliveries and mesa_delivery_items; skips already COMMITTED chunks across releases.", "Truthful mutation lifecycle: COMMITTED, PROCESSING, QUEUED, FAILED, REJECTED tracked accurately; partial failure banner displays retry CTA.", - "Web Panel Yayınla tab: Target settings manager, preflight checklist, ready summary, confirmation modal, live progress polling, delivery detail modal, retry CTA.", + "Web Panel Yay\u0131nla tab: Target settings manager, preflight checklist, ready summary, confirmation modal, live progress polling, delivery detail modal, retry CTA.", "CLI sync: mesa-data sync command discovers, fetches, processes, validates quality, auto-approves, and summarizes ready items, stopping strictly before MESA push." ], "tests": [ @@ -196,14 +196,46 @@ "started_at": "2026-08-29T00:00:00Z", "completed_at": "2026-08-29T00:00:00Z", "findings": [ - {"id": "QG_ZERO_ARTICLE_PASS", "severity": "P0", "status": "fixed"}, - {"id": "PUBLICATION_DATE_BRIDGE_LOSS", "severity": "P0", "status": "fixed"}, - {"id": "STALE_APPROVED_VERSION_PUBLISH", "severity": "P0", "status": "fixed"}, - {"id": "CURRENT_VERSION_MESA_STATUS", "severity": "P0", "status": "fixed"}, - {"id": "DERIVED_REPROCESS_EARLY_RETURN", "severity": "P0", "status": "fixed"}, - {"id": "MESA_TARGET_KEY_EXFILTRATION", "severity": "P0", "status": "fixed"}, - {"id": "MUTATION_TIMEOUT_MARKED_FAILED", "severity": "P1", "status": "fixed"}, - {"id": "ARTIFACT_SHA_CROSS_DOCUMENT_COLLISION", "severity": "P1", "status": "fixed"} + { + "id": "QG_ZERO_ARTICLE_PASS", + "severity": "P0", + "status": "fixed" + }, + { + "id": "PUBLICATION_DATE_BRIDGE_LOSS", + "severity": "P0", + "status": "fixed" + }, + { + "id": "STALE_APPROVED_VERSION_PUBLISH", + "severity": "P0", + "status": "fixed" + }, + { + "id": "CURRENT_VERSION_MESA_STATUS", + "severity": "P0", + "status": "fixed" + }, + { + "id": "DERIVED_REPROCESS_EARLY_RETURN", + "severity": "P0", + "status": "fixed" + }, + { + "id": "MESA_TARGET_KEY_EXFILTRATION", + "severity": "P0", + "status": "fixed" + }, + { + "id": "MUTATION_TIMEOUT_MARKED_FAILED", + "severity": "P1", + "status": "fixed" + }, + { + "id": "ARTIFACT_SHA_CROSS_DOCUMENT_COLLISION", + "severity": "P1", + "status": "fixed" + } ], "fixes": [ "Legislation zero-article and abnormal large uncovered-area outcomes cannot PASS; article ordinal is retained in canonical records.", @@ -219,15 +251,37 @@ "c20aa17 fix: secure current-version MESA publishing", "71f4306 fix: report MESA state from current version" ], - "migrations": ["0009_correctness_hardening.sql"], + "migrations": [ + "0009_correctness_hardening.sql" + ], "tests": [ - {"name": "focused correctness regression", "command": "MESA_DATA_MESA_ALLOWED_HOST=mock-mesa.internal UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run pytest -q tests/unit/test_correctness_hardening.py tests/unit/test_quality_gate.py tests/unit/test_catalog_release_iterator.py tests/unit/test_publisher_client_and_ledger.py tests/integration/test_mesa_v4_publisher_e2e.py", "result": "19 passed"}, - {"name": "compile", "command": "UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run python -m compileall -q src/mesa_legal_data", "result": "passed"}, - {"name": "ruff", "command": "UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run ruff check src tests", "result": "passed"} - ,{"name": "current-version MESA status regression", "command": "MESA_DATA_MESA_ALLOWED_HOST=mock-mesa.internal UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run pytest -q tests/unit/test_correctness_hardening.py tests/unit/test_publisher_client_and_ledger.py", "result": "10 passed"} + { + "name": "focused correctness regression", + "command": "MESA_DATA_MESA_ALLOWED_HOST=mock-mesa.internal UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run pytest -q tests/unit/test_correctness_hardening.py tests/unit/test_quality_gate.py tests/unit/test_catalog_release_iterator.py tests/unit/test_publisher_client_and_ledger.py tests/integration/test_mesa_v4_publisher_e2e.py", + "result": "19 passed" + }, + { + "name": "compile", + "command": "UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run python -m compileall -q src/mesa_legal_data", + "result": "passed" + }, + { + "name": "ruff", + "command": "UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run ruff check src tests", + "result": "passed" + }, + { + "name": "current-version MESA status regression", + "command": "MESA_DATA_MESA_ALLOWED_HOST=mock-mesa.internal UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run pytest -q tests/unit/test_correctness_hardening.py tests/unit/test_publisher_client_and_ledger.py", + "result": "10 passed" + } + ], + "blockers": [ + "The all-test pytest invocation did not complete within the desktop command window and is not recorded as passed." ], - "blockers": ["The all-test pytest invocation did not complete within the desktop command window and is not recorded as passed."], - "unverified": ["MESA live HTTP payload and server-side mutation contract remain UNVERIFIED because no live endpoint or MESA source code is available."] + "unverified": [ + "MESA live HTTP payload and server-side mutation contract remain UNVERIFIED because no live endpoint or MESA source code is available." + ] }, "prompt_5": { "status": "completed", @@ -262,7 +316,7 @@ "uv run mypy src/mesa_legal_data --ignore-missing-imports --explicit-package-bases --follow-imports=skip: passed (77 source files)", "python -m compileall -q src/mesa_legal_data: passed", "node --check src/mesa_legal_data/web/static/app.js: passed", - "Live browser journey: Ana Sayfa, Veri Topla, Kütüphane, İnceleme, Yayınla, advanced navigation and explicit MESA contract fields verified", + "Live browser journey: Ana Sayfa, Veri Topla, K\u00fct\u00fcphane, \u0130nceleme, Yay\u0131nla, advanced navigation and explicit MESA contract fields verified", "Two bounded real Resmi Gazete sync runs in isolated storage: one item per run, zero MESA deliveries" ], "blockers": [ @@ -400,6 +454,57 @@ "remaining_risks": [ "Live MESA HTTP request/response and server-side idempotency semantics remain unverified." ] + }, + "final_p1_record_instance_not_null": { + "status": "completed", + "started_at": "2026-08-30T23:20:00+03:00", + "completed_at": "2026-08-30T23:35:00+03:00", + "commits": [ + "fix: enforce non-null record instance identities" + ], + "migrations": [ + "0012_record_instance_not_null.sql" + ], + "files_changed": [ + "migrations/0012_record_instance_not_null.sql", + "tests/unit/test_migration_0012.py", + "tests/integration/test_post_master_independent_audit.py", + "tests/acceptance/test_workspace_e2e.py", + "tests/acceptance/test_operations_scale_gate.py", + "tests/integration/test_annotations_and_issues.py", + "tests/integration/test_data_explorer_api.py", + "tests/integration/test_large_export_streaming.py", + "tests/integration/test_record_revisions.py", + "tests/unit/test_final_closure_gate.py", + "tests/unit/test_release.py", + "docs/BUILD_STATE.json" + ], + "findings": [ + "records.record_instance_id was TEXT PRIMARY KEY without explicit NOT NULL, allowing SQLite to accept NULL values on insert.", + "Deterministic authoritative generator version_id:record_id backfills any legacy NULL record_instance_id values." + ], + "decisions": [ + "Added migration 0012_record_instance_not_null.sql performing safe table rebuild to declare record_instance_id TEXT NOT NULL PRIMARY KEY.", + "Preserved (version_id, record_id) UNIQUE constraint, foreign keys, and indexes (idx_records_version_record, idx_records_record_id, idx_records_version_type, idx_records_approval_status).", + "Preserved and restored triggers on release_items referencing records." + ], + "tests": [ + "Test A (Fresh DB): passed in test_migration_0012.py::test_migration_0012_test_a_fresh_db_table_info", + "Test B (NULL insert rejected): passed in test_migration_0012.py::test_migration_0012_test_b_null_insert_rejected", + "Test C (Normal insert works): passed in test_migration_0012.py::test_migration_0012_test_c_normal_insert_works", + "Test D (Upgrade populated DB): passed in test_migration_0012.py::test_migration_0012_test_d_upgrade_populated_db", + "Test E (Legacy NULL backfill): passed in test_migration_0012.py::test_migration_0012_test_e_legacy_null_backfill", + "Test F (Version-aware invariant): passed in test_migration_0012.py::test_migration_0012_test_f_version_aware_invariant", + "uv sync --frozen: passed", + "uv run ruff format --check .: passed (212 files)", + "uv run ruff check .: passed", + "uv run mypy src: passed (77 source files)", + "uv run pip-audit: passed (0 known vulnerabilities)", + "uv run pytest -m \"not scale\" -ra --durations=20: 356 passed, 0 failed, 4 deselected", + "uv run pytest -m \"scale\" -ra --durations=20: 4 passed, 0 failed, 356 deselected", + "uv run pytest -ra: 360 passed in 72.83s" + ], + "blockers": [] } }, "mesa_contract": { @@ -431,11 +536,11 @@ }, "handoff": { "next_phase": null, - "summary": "Final MVP blocker closure complete. All 354 tests pass (350 non-scale + 4 scale), fresh and populated upgrade migrations through 0011 pass, exact CI gates are green, and no P0/P1 MVP blocker remains.", + "summary": "Final P1 blocker fix complete. records.record_instance_id is strictly NOT NULL PRIMARY KEY at database level via migration 0012. All 360 tests pass (356 non-scale + 4 scale), fresh and upgrade migrations pass, CI gates green, 0 P0/P1 blockers remain.", "must_verify": [ "Ensure live deployment environment configures MESA_DATA_MESA_API_KEY securely.", "Configure documented MESA routes and verify health, publish, polling, COMMITTED truth, and server-side idempotency against a live endpoint." ] }, - "last_updated_at": "2026-08-30T00:43:56+03:00" + "last_updated_at": "2026-08-30T23:35:00+03:00" } diff --git a/docs/BUILD_STATE.json b/docs/BUILD_STATE.json index 91c1fa6..108472a 100644 --- a/docs/BUILD_STATE.json +++ b/docs/BUILD_STATE.json @@ -40,16 +40,17 @@ "BLOCKER-3-RUN-SCOPED-REQUEST-BUDGET", "PROMPT-5-FINAL-ADVERSARIAL-AUDIT", "MASTER-MVP-CLOSURE", - "POST-MASTER-INDEPENDENT-VERIFICATION" + "POST-MASTER-INDEPENDENT-VERIFICATION", + "P1-RECORD-INSTANCE-NOT-NULL-ENFORCED" ], "last_test_command": "uv run pytest -ra", - "last_test_result": "passed (351 passed: 347 non-scale + 4 scale in 73.34s)", + "last_test_result": "passed (360 passed: 356 non-scale + 4 scale in 72.83s)", "verification_checks": { - "ruff_format": "passed (211 Python files)", + "ruff_format": "passed (212 Python files)", "ruff_check": "passed", "mypy": "passed (77 source files)", "pip_audit": "passed (no known vulnerabilities)", - "pytest_not_scale": "passed (347 passed)", + "pytest_not_scale": "passed (356 passed)", "pytest_scale": "passed (4 passed)", "live_mesa_contract": "LIVE_MESA_NOT_VERIFIED" }, @@ -58,5 +59,5 @@ ], "polish_status": "completed", "polish_current_task": "CLOSED", - "updated_at": "2026-08-29T23:15:00Z" + "updated_at": "2026-08-30T23:30:00Z" } diff --git a/migrations/0012_record_instance_not_null.sql b/migrations/0012_record_instance_not_null.sql new file mode 100644 index 0000000..a197a6f --- /dev/null +++ b/migrations/0012_record_instance_not_null.sql @@ -0,0 +1,83 @@ +-- Migration 0012: Enforce NOT NULL invariant on records.record_instance_id. +-- Rebuilds the records table to ensure record_instance_id is declared +-- as TEXT NOT NULL PRIMARY KEY, backfills any legacy NULL record_instance_id +-- values with deterministic version_id:record_id identities, and preserves +-- all composite uniqueness, foreign keys, and indexes. + +PRAGMA foreign_keys = OFF; + +DROP TRIGGER IF EXISTS release_items_identity_valid_insert; +DROP TRIGGER IF EXISTS release_items_identity_valid_update; + +CREATE TABLE records_v3 ( + record_instance_id TEXT NOT NULL PRIMARY KEY, + record_id TEXT NOT NULL, + version_id TEXT NOT NULL, + record_type TEXT NOT NULL, + canonical_path TEXT NOT NULL, + canonical_line INTEGER NOT NULL, + record_sha256 TEXT NOT NULL, + validation_status TEXT NOT NULL, + approval_status TEXT NOT NULL, + created_at TEXT NOT NULL, + FOREIGN KEY (version_id) REFERENCES versions(version_id) +); + +INSERT INTO records_v3 ( + record_instance_id, + record_id, + version_id, + record_type, + canonical_path, + canonical_line, + record_sha256, + validation_status, + approval_status, + created_at +) +SELECT + COALESCE(record_instance_id, version_id || ':' || record_id), + record_id, + version_id, + record_type, + canonical_path, + canonical_line, + record_sha256, + validation_status, + approval_status, + created_at +FROM records; + +DROP TABLE records; +ALTER TABLE records_v3 RENAME TO records; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_records_version_record ON records(version_id, record_id); +CREATE INDEX IF NOT EXISTS idx_records_record_id ON records(record_id); +CREATE INDEX IF NOT EXISTS idx_records_version_type ON records(version_id, record_type); +CREATE INDEX IF NOT EXISTS idx_records_approval_status ON records(approval_status); + +CREATE TRIGGER release_items_identity_valid_insert +BEFORE INSERT ON release_items +WHEN NOT EXISTS ( + SELECT 1 FROM records r + WHERE r.version_id = NEW.version_id + AND r.record_id = NEW.record_id + AND r.record_sha256 = NEW.record_sha256 +) +BEGIN + SELECT RAISE(ABORT, 'release item version identity is invalid'); +END; + +CREATE TRIGGER release_items_identity_valid_update +BEFORE UPDATE OF version_id, record_id, record_sha256 ON release_items +WHEN NOT EXISTS ( + SELECT 1 FROM records r + WHERE r.version_id = NEW.version_id + AND r.record_id = NEW.record_id + AND r.record_sha256 = NEW.record_sha256 +) +BEGIN + SELECT RAISE(ABORT, 'release item version identity is invalid'); +END; + +PRAGMA foreign_keys = ON; diff --git a/tests/acceptance/test_operations_scale_gate.py b/tests/acceptance/test_operations_scale_gate.py index ce5e2f7..1316242 100644 --- a/tests/acceptance/test_operations_scale_gate.py +++ b/tests/acceptance/test_operations_scale_gate.py @@ -88,9 +88,9 @@ def test_operations_scale_gate(tmp_path, monkeypatch): (v_id, i, i), ) c.execute( - """INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) - VALUES (?, ?, 'article', 'canonical/scale.jsonl', ?, ?, 'valid', 'pending', '2026-08-05T00:00:00Z')""", - (r_id, v_id, i, rec_sha), + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES (?, ?, ?, 'article', 'canonical/scale.jsonl', ?, ?, 'valid', 'pending', '2026-08-05T00:00:00Z')""", + (f"{v_id}:{r_id}", r_id, v_id, i, rec_sha), ) f_canon.write(line_str) diff --git a/tests/acceptance/test_workspace_e2e.py b/tests/acceptance/test_workspace_e2e.py index 0b337e1..1e7d17d 100644 --- a/tests/acceptance/test_workspace_e2e.py +++ b/tests/acceptance/test_workspace_e2e.py @@ -104,13 +104,13 @@ def _seed_data(tmp_path): canon_file.write_text(line1 + line2, encoding="utf-8") conn.execute( - """INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) - VALUES ('rec-e2e-art-1', 'ver-e2e-1', 'article', 'canonical/e2e/article.jsonl', 1, ?, 'valid', 'pending', '2026-08-05T00:00:00Z')""", + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES ('ver-e2e-1:rec-e2e-art-1', 'rec-e2e-art-1', 'ver-e2e-1', 'article', 'canonical/e2e/article.jsonl', 1, ?, 'valid', 'pending', '2026-08-05T00:00:00Z')""", (sha1,), ) conn.execute( - """INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) - VALUES ('rec-e2e-art-2', 'ver-e2e-1', 'article', 'canonical/e2e/article.jsonl', 2, ?, 'valid', 'pending', '2026-08-05T00:00:00Z')""", + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES ('ver-e2e-1:rec-e2e-art-2', 'rec-e2e-art-2', 'ver-e2e-1', 'article', 'canonical/e2e/article.jsonl', 2, ?, 'valid', 'pending', '2026-08-05T00:00:00Z')""", (sha2,), ) conn.commit() diff --git a/tests/integration/test_annotations_and_issues.py b/tests/integration/test_annotations_and_issues.py index f222a98..4482cd8 100644 --- a/tests/integration/test_annotations_and_issues.py +++ b/tests/integration/test_annotations_and_issues.py @@ -39,8 +39,8 @@ def test_annotations_issues_and_quarantine(tmp_path, monkeypatch): VALUES ('ver-q1', 'doc-q1', 'art-q1', 'snapshot', 'canonical/q1.jsonl', 1, 'sha-ver-q1', 'test_parser', '1.0', '1.0', 'valid', 'clean', 'pending', '2026-08-05T00:00:00Z')""" ) c.execute( - """INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) - VALUES ('rec-q1', 'ver-q1', 'article', 'canonical/q1.jsonl', 1, 'sha-rec-q1', 'invalid', 'pending', '2026-08-05T00:00:00Z')""" + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES ('ver-q1:rec-q1', 'rec-q1', 'ver-q1', 'article', 'canonical/q1.jsonl', 1, 'sha-rec-q1', 'invalid', 'pending', '2026-08-05T00:00:00Z')""" ) # 1. Add Annotation diff --git a/tests/integration/test_data_explorer_api.py b/tests/integration/test_data_explorer_api.py index 2c9b223..d1d2ad8 100644 --- a/tests/integration/test_data_explorer_api.py +++ b/tests/integration/test_data_explorer_api.py @@ -29,8 +29,8 @@ def test_data_explorer_search_and_facets(tmp_path, monkeypatch): VALUES ('ver-ex-1', 'doc-ex-1', 'art-ex-1', 'snapshot', 'canonical/ex1.jsonl', 1, 'sha-ver-ex-1', 'test_parser', '1.0', '1.0', 'valid', 'clean', 'approved', '2026-08-05T00:00:00Z')""" ) c.execute( - """INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) - VALUES ('rec-ex-1', 'ver-ex-1', 'article', 'canonical/ex1.jsonl', 1, 'sha-rec-ex-1', 'valid', 'approved', '2026-08-05T00:00:00Z')""" + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES ('ver-ex-1:rec-ex-1', 'rec-ex-1', 'ver-ex-1', 'article', 'canonical/ex1.jsonl', 1, 'sha-rec-ex-1', 'valid', 'approved', '2026-08-05T00:00:00Z')""" ) conn.commit() conn.close() diff --git a/tests/integration/test_large_export_streaming.py b/tests/integration/test_large_export_streaming.py index 7fdbed3..f1b9471 100644 --- a/tests/integration/test_large_export_streaming.py +++ b/tests/integration/test_large_export_streaming.py @@ -41,9 +41,9 @@ def test_large_export_streaming(tmp_path, monkeypatch): lines.append(json.dumps(rec_obj) + "\n") line_num = i + 1 c.execute( - """INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) - VALUES (?, 'ver-bulk', 'article', 'canonical/bulk.jsonl', ?, ?, 'valid', 'approved', '2026-08-05T00:00:00Z')""", - (rec_id, line_num, f"sha-rec-{i}"), + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES (?, ?, 'ver-bulk', 'article', 'canonical/bulk.jsonl', ?, ?, 'valid', 'approved', '2026-08-05T00:00:00Z')""", + (f"ver-bulk:{rec_id}", rec_id, line_num, f"sha-rec-{i}"), ) canon_file.write_text("".join(lines), encoding="utf-8") conn.commit() diff --git a/tests/integration/test_post_master_independent_audit.py b/tests/integration/test_post_master_independent_audit.py index a247bda..173632d 100644 --- a/tests/integration/test_post_master_independent_audit.py +++ b/tests/integration/test_post_master_independent_audit.py @@ -1104,7 +1104,7 @@ def test_control_57_and_58_fresh_and_upgrade_database_integrity(tmp_path): ) migrations_dir = Path("migrations") for mig_file in sorted(migrations_dir.glob("*.sql")): - if "0010" in mig_file.name or "0011" in mig_file.name: + if any(v in mig_file.name for v in ("0010", "0011", "0012")): continue sql = mig_file.read_text(encoding="utf-8") conn_up.executescript(sql) diff --git a/tests/integration/test_record_revisions.py b/tests/integration/test_record_revisions.py index b4cdd0c..f51e806 100644 --- a/tests/integration/test_record_revisions.py +++ b/tests/integration/test_record_revisions.py @@ -33,8 +33,8 @@ def test_record_revision_flow(tmp_path, monkeypatch): VALUES ('ver-rev-1', 'doc-rev-1', 'art-rev-1', 'snapshot', 'canonical/rev.jsonl', 1, 'sha-ver-rev-1', 'test_parser', '1.0', '1.0', 'valid', 'clean', 'pending', '2026-08-05T00:00:00Z')""" ) c.execute( - """INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) - VALUES ('rec-orig-1', 'ver-rev-1', 'article', 'canonical/rev.jsonl', 1, 'sha-rec-orig-1', 'valid', 'pending', '2026-08-05T00:00:00Z')""" + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES ('ver-rev-1:rec-orig-1', 'rec-orig-1', 'ver-rev-1', 'article', 'canonical/rev.jsonl', 1, 'sha-rec-orig-1', 'valid', 'pending', '2026-08-05T00:00:00Z')""" ) conn.commit() @@ -78,8 +78,8 @@ def test_record_revision_flow(tmp_path, monkeypatch): VALUES ('ver-rev-2', 'doc-rev-1', 'art-rev-2', 'snapshot', 'canonical/rev2.jsonl', 1, 'sha-ver-rev-2', 'test_parser', '1.0', '1.0', 'valid', 'clean', 'pending', '2026-08-06T00:00:00Z', 2)""" ) c.execute( - """INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) - VALUES ('rec-orig-1', 'ver-rev-2', 'article', 'canonical/rev2.jsonl', 1, 'sha-rec-orig-2', 'valid', 'pending', '2026-08-06T00:00:00Z')""" + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES ('ver-rev-2:rec-orig-1', 'rec-orig-1', 'ver-rev-2', 'article', 'canonical/rev2.jsonl', 1, 'sha-rec-orig-2', 'valid', 'pending', '2026-08-06T00:00:00Z')""" ) c.execute("UPDATE records SET approval_status = 'pending' WHERE version_id = 'ver-rev-1'") scoped_rev = create_record_revision( diff --git a/tests/unit/test_final_closure_gate.py b/tests/unit/test_final_closure_gate.py index 4db5bef..8ccfa89 100644 --- a/tests/unit/test_final_closure_gate.py +++ b/tests/unit/test_final_closure_gate.py @@ -618,9 +618,9 @@ def test_flagged_privacy_human_approval_resolves_eligibility(tmp_path, monkeypat conn.execute( """ INSERT INTO records ( - record_id, version_id, record_type, canonical_path, canonical_line, + record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at - ) VALUES ('rec-flagged', 'ver-flagged', 'legislation', 'canonical/test.jsonl', 1, + ) VALUES ('ver-flagged:rec-flagged', 'rec-flagged', 'ver-flagged', 'legislation', 'canonical/test.jsonl', 1, 'hash1', 'valid', 'pending', ?) """, (now,), @@ -763,8 +763,8 @@ def test_provenance_actual_membership(tmp_path, monkeypatch): ) conn.execute( """ - INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) - VALUES ('rec1', 'ver1', 'legislation', 'c/1', 1, 'h1', 'valid', 'approved', ?) + INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES ('ver1:rec1', 'rec1', 'ver1', 'legislation', 'c/1', 1, 'h1', 'valid', 'approved', ?) """, (now,), ) diff --git a/tests/unit/test_migration_0012.py b/tests/unit/test_migration_0012.py new file mode 100644 index 0000000..0ff709e --- /dev/null +++ b/tests/unit/test_migration_0012.py @@ -0,0 +1,339 @@ +import sqlite3 +from pathlib import Path + +import pytest + +from mesa_legal_data.catalog import hash_file, insert_record, migrate + + +def _seed_base_hierarchy(conn: sqlite3.Connection) -> None: + """Helper to seed minimal valid sources, documents, artifacts, and versions.""" + conn.execute( + """INSERT INTO sources (source_id, name, authority, base_url, access_mode, enabled, policy_version, config_json, created_at, updated_at) + VALUES ('src_test', 'Source Test', 'Authority', 'https://example.com', 'manual', 1, '1.0', '{}', '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')""" + ) + conn.execute( + """INSERT INTO documents (document_id, family, document_type, stable_key, lifecycle_status, created_at, updated_at) + VALUES ('doc_test', 'legislation', 'law', 'law_test', 'active', '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')""" + ) + conn.execute( + """INSERT INTO artifacts (artifact_id, document_id, source_id, source_url, retrieved_at, fetch_method, detected_content_type, byte_size, sha256, raw_path, transport_status, metadata_json) + VALUES ('art_1', 'doc_test', 'src_test', 'https://example.com/1', '2026-01-01T00:00:00Z', 'manual', 'text/html', 100, 'sha256_art_1', 'raw/1.html', 'ok', '{}')""" + ) + conn.execute( + """INSERT INTO artifacts (artifact_id, document_id, source_id, source_url, retrieved_at, fetch_method, detected_content_type, byte_size, sha256, raw_path, transport_status, metadata_json) + VALUES ('art_2', 'doc_test', 'src_test', 'https://example.com/2', '2026-02-01T00:00:00Z', 'manual', 'text/html', 100, 'sha256_art_2', 'raw/2.html', 'ok', '{}')""" + ) + conn.execute( + """INSERT INTO versions (version_id, document_id, artifact_id, version_kind, canonical_path, canonical_line, canonical_sha256, parser_name, parser_version, schema_version, validation_status, privacy_status, approval_status, created_at, revision_number) + VALUES ('v1', 'doc_test', 'art_1', 'consolidated_snapshot', 'canonical/v1.jsonl', 1, 'sha256_can_1', 'law_parser', '1.0', '1.0', 'passed', 'clean', 'approved', '2026-01-01T00:00:00Z', 1)""" + ) + conn.execute( + """INSERT INTO versions (version_id, document_id, artifact_id, version_kind, canonical_path, canonical_line, canonical_sha256, parser_name, parser_version, schema_version, validation_status, privacy_status, approval_status, created_at, revision_number) + VALUES ('v2', 'doc_test', 'art_2', 'consolidated_snapshot', 'canonical/v2.jsonl', 1, 'sha256_can_2', 'law_parser', '1.0', '1.0', 'passed', 'clean', 'approved', '2026-02-01T00:00:00Z', 2)""" + ) + + +def test_migration_0012_test_a_fresh_db_table_info(tmp_path): + """Test A — Fresh DB: Empty DB migrated 0001 -> 0012 has record_instance_id NOT NULL.""" + db_file = tmp_path / "fresh_0012.sqlite" + migrate(None, db_file) + + conn = sqlite3.connect(db_file) + cur = conn.cursor() + + # Inspect records schema via PRAGMA table_info + cur.execute("PRAGMA table_info(records)") + columns = {r[1]: {"type": r[2], "notnull": r[3], "pk": r[5]} for r in cur.fetchall()} + + assert "record_instance_id" in columns + assert columns["record_instance_id"]["type"] == "TEXT" + assert columns["record_instance_id"]["notnull"] == 1, "record_instance_id must be NOT NULL (notnull=1)" + assert columns["record_instance_id"]["pk"] == 1, "record_instance_id must be primary key (pk=1)" + + # Check PRAGMA pragmas + cur.execute("PRAGMA foreign_key_check") + assert cur.fetchall() == [] + + cur.execute("PRAGMA integrity_check") + assert cur.fetchone()[0] == "ok" + conn.close() + + +def test_migration_0012_test_b_null_insert_rejected(tmp_path): + """Test B — NULL insert rejected: INSERT INTO records with record_instance_id=NULL raises IntegrityError.""" + db_file = tmp_path / "null_rejected.sqlite" + migrate(None, db_file) + + conn = sqlite3.connect(db_file) + conn.execute("PRAGMA foreign_keys = ON") + _seed_base_hierarchy(conn) + + with pytest.raises(sqlite3.IntegrityError, match="NOT NULL constraint failed: records.record_instance_id"): + conn.execute( + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES (NULL, 'doc_test:art:1', 'v1', 'article', 'canonical/v1.jsonl', 1, 'sha_rec_1', 'passed', 'approved', '2026-01-01T00:00:00Z')""" + ) + conn.close() + + +def test_migration_0012_test_c_normal_insert_works(tmp_path): + """Test C — Normal insert works: Normal application-generated record_instance_id succeeds.""" + db_file = tmp_path / "normal_insert.sqlite" + migrate(None, db_file) + + conn = sqlite3.connect(db_file) + conn.execute("PRAGMA foreign_keys = ON") + _seed_base_hierarchy(conn) + + # 1. Direct SQL insert + conn.execute( + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES ('v1:doc_test:art:1', 'doc_test:art:1', 'v1', 'article', 'canonical/v1.jsonl', 1, 'sha_rec_1', 'passed', 'approved', '2026-01-01T00:00:00Z')""" + ) + conn.commit() + + # 2. insert_record catalog API + insert_record( + conn=conn, + record_id="doc_test:art:2", + version_id="v1", + record_type="article", + canonical_path="canonical/v1.jsonl", + canonical_line=2, + record_sha256="sha_rec_2", + validation_status="passed", + approval_status="approved", + ) + + cur = conn.cursor() + cur.execute("SELECT count(*) FROM records") + assert cur.fetchone()[0] == 2 + + cur.execute("SELECT record_instance_id, record_id FROM records ORDER BY canonical_line ASC") + rows = cur.fetchall() + assert rows == [ + ("v1:doc_test:art:1", "doc_test:art:1"), + ("v1:doc_test:art:2", "doc_test:art:2"), + ] + conn.close() + + +def test_migration_0012_test_d_upgrade_populated_db(tmp_path): + """Test D — Upgrade populated DB: Seeded 0011-level DB upgraded to 0012 preserves all rows, IDs, hashes, and invariants.""" + db_file = tmp_path / "upgrade_populated.sqlite" + + # 1. Set up pre-0012 DB (0001 -> 0011) + conn_pre = sqlite3.connect(db_file) + conn_pre.execute( + """CREATE TABLE schema_migrations (version TEXT PRIMARY KEY, applied_at TEXT NOT NULL, file_hash TEXT NOT NULL)""" + ) + migrations_dir = Path("migrations") + for mig_file in sorted(migrations_dir.glob("*.sql")): + if "0012" in mig_file.name: + continue + sql = mig_file.read_text(encoding="utf-8") + conn_pre.executescript(sql) + f_hash = hash_file(mig_file) + conn_pre.execute( + "INSERT INTO schema_migrations (version, applied_at, file_hash) VALUES (?, '2026-01-01', ?)", + (mig_file.name, f_hash), + ) + conn_pre.commit() + + # Populate realistic records, releases, release_items, reviews, and validation issues + _seed_base_hierarchy(conn_pre) + + records_data = [ + ( + "v1:doc_test:art:1", + "doc_test:art:1", + "v1", + "article", + "c1.jsonl", + 1, + "sha_art_1", + "passed", + "approved", + "2026-01-01T00:00:00Z", + ), + ( + "v1:doc_test:art:2", + "doc_test:art:2", + "v1", + "article", + "c1.jsonl", + 2, + "sha_art_2", + "passed", + "approved", + "2026-01-01T00:00:00Z", + ), + ( + "v2:doc_test:art:1", + "doc_test:art:1", + "v2", + "article", + "c2.jsonl", + 1, + "sha_art_1_v2", + "passed", + "approved", + "2026-02-01T00:00:00Z", + ), + ] + for r in records_data: + conn_pre.execute( + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + r, + ) + + conn_pre.execute( + """INSERT INTO releases (release_id, release_path, status, schema_version, created_at, manifest_sha256, counts_json, source_snapshot_json) + VALUES ('rel_1', 'releases/rel_1', 'verified', '1.0.0', '2026-01-01T00:00:00Z', 'man_sha', '{}', '[]')""" + ) + conn_pre.execute( + """INSERT INTO release_items (release_id, record_id, record_sha256, version_id) + VALUES ('rel_1', 'doc_test:art:1', 'sha_art_1', 'v1')""" + ) + conn_pre.execute( + """INSERT INTO record_reviews (review_id, record_instance_id, version_id, record_id, record_sha256, decision, reviewer, note, reviewed_at) + VALUES ('rev_1', 'v1:doc_test:art:1', 'v1', 'doc_test:art:1', 'sha_art_1', 'approved', 'auditor', 'ok', '2026-01-01T00:00:00Z')""" + ) + conn_pre.commit() + conn_pre.close() + + # 2. Run migration to apply 0012 + migrate(None, db_file) + + # 3. Verify upgraded state + conn_post = sqlite3.connect(db_file) + cur = conn_post.cursor() + + cur.execute("PRAGMA foreign_key_check") + assert cur.fetchall() == [] + + cur.execute("PRAGMA integrity_check") + assert cur.fetchone()[0] == "ok" + + # Verify table schema has NOT NULL + cur.execute("PRAGMA table_info(records)") + columns = {r[1]: {"notnull": r[3], "pk": r[5]} for r in cur.fetchall()} + assert columns["record_instance_id"]["notnull"] == 1 + assert columns["record_instance_id"]["pk"] == 1 + + # Verify row counts and exact data preservation + cur.execute("SELECT count(*) FROM records") + assert cur.fetchone()[0] == 3 + + cur.execute( + "SELECT record_instance_id, record_id, version_id, record_sha256, approval_status FROM records ORDER BY record_instance_id ASC" + ) + rows = cur.fetchall() + assert rows == [ + ("v1:doc_test:art:1", "doc_test:art:1", "v1", "sha_art_1", "approved"), + ("v1:doc_test:art:2", "doc_test:art:2", "v1", "sha_art_2", "approved"), + ("v2:doc_test:art:1", "doc_test:art:1", "v2", "sha_art_1_v2", "approved"), + ] + + # Verify dependent trigger release_items_identity_valid_insert still functions + with pytest.raises(sqlite3.IntegrityError, match="release item version identity is invalid"): + conn_post.execute( + """INSERT INTO release_items (release_id, record_id, record_sha256, version_id) + VALUES ('rel_1', 'nonexistent_rec', 'invalid_sha', 'v1')""" + ) + + conn_post.close() + + +def test_migration_0012_test_e_legacy_null_backfill(tmp_path): + """Test E — Legacy NULL backfill: Pre-0012 record with NULL record_instance_id is deterministically backfilled.""" + db_file = tmp_path / "legacy_null_backfill.sqlite" + + conn_pre = sqlite3.connect(db_file) + conn_pre.execute( + """CREATE TABLE schema_migrations (version TEXT PRIMARY KEY, applied_at TEXT NOT NULL, file_hash TEXT NOT NULL)""" + ) + migrations_dir = Path("migrations") + for mig_file in sorted(migrations_dir.glob("*.sql")): + if "0012" in mig_file.name: + continue + sql = mig_file.read_text(encoding="utf-8") + conn_pre.executescript(sql) + f_hash = hash_file(mig_file) + conn_pre.execute( + "INSERT INTO schema_migrations (version, applied_at, file_hash) VALUES (?, '2026-01-01', ?)", + (mig_file.name, f_hash), + ) + conn_pre.commit() + + _seed_base_hierarchy(conn_pre) + + # Insert pre-0012 record where record_instance_id IS NULL + conn_pre.execute( + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES (NULL, 'doc_test:art:9', 'v1', 'article', 'c1.jsonl', 9, 'sha_art_9', 'passed', 'approved', '2026-01-01T00:00:00Z')""" + ) + conn_pre.commit() + conn_pre.close() + + # Apply 0012 + migrate(None, db_file) + + conn_post = sqlite3.connect(db_file) + cur = conn_post.cursor() + + cur.execute("SELECT record_instance_id, record_id, version_id FROM records WHERE record_id = 'doc_test:art:9'") + row = cur.fetchone() + assert row is not None + assert row[0] == "v1:doc_test:art:9", ( + "Backfilled record_instance_id must match authoritative generator (version_id:record_id)" + ) + assert row[0] is not None + + cur.execute("SELECT count(*) FROM records WHERE record_instance_id IS NULL") + assert cur.fetchone()[0] == 0 + + conn_post.close() + + +def test_migration_0012_test_f_version_aware_invariant(tmp_path): + """Test F — Version-aware invariant: v1 Article 9 and v2 Article 9 maintain separate record_instance_ids and composite uniqueness.""" + db_file = tmp_path / "version_aware.sqlite" + migrate(None, db_file) + + conn = sqlite3.connect(db_file) + conn.execute("PRAGMA foreign_keys = ON") + _seed_base_hierarchy(conn) + + # Insert v1 Article 9 + conn.execute( + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES ('v1:doc_test:art:9', 'doc_test:art:9', 'v1', 'article', 'c1.jsonl', 9, 'sha_v1_9', 'passed', 'approved', '2026-01-01T00:00:00Z')""" + ) + + # Insert v2 Article 9 (same logical record_id, different version_id and record_instance_id) + conn.execute( + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES ('v2:doc_test:art:9', 'doc_test:art:9', 'v2', 'article', 'c2.jsonl', 9, 'sha_v2_9', 'passed', 'approved', '2026-02-01T00:00:00Z')""" + ) + conn.commit() + + cur = conn.cursor() + cur.execute( + "SELECT record_instance_id, version_id, record_id FROM records WHERE record_id = 'doc_test:art:9' ORDER BY version_id ASC" + ) + rows = cur.fetchall() + assert len(rows) == 2 + assert rows[0] == ("v1:doc_test:art:9", "v1", "doc_test:art:9") + assert rows[1] == ("v2:doc_test:art:9", "v2", "doc_test:art:9") + + # Verify duplicate (version_id, record_id) is rejected by unique constraint + with pytest.raises(sqlite3.IntegrityError, match="UNIQUE constraint failed"): + conn.execute( + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES ('v1:doc_test:art:9:dup', 'doc_test:art:9', 'v1', 'article', 'c1.jsonl', 9, 'sha_v1_9_dup', 'passed', 'approved', '2026-01-01T00:00:00Z')""" + ) + + conn.close() diff --git a/tests/unit/test_release.py b/tests/unit/test_release.py index 575505c..13c43c5 100644 --- a/tests/unit/test_release.py +++ b/tests/unit/test_release.py @@ -93,8 +93,8 @@ def test_release_lifecycle_cli(tmp_path, monkeypatch): ) conn.execute("UPDATE documents SET current_version_id = 'ver1' WHERE document_id = 'doc1'") conn.execute( - """INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) - VALUES ('rec1', 'ver1', 'legislation', 'canonical/test.jsonl', 1, ?, 'valid', 'approved', ?)""", + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES ('ver1:rec1', 'rec1', 'ver1', 'legislation', 'canonical/test.jsonl', 1, ?, 'valid', 'approved', ?)""", (rec_sha, now), ) conn.commit()