Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 126 additions & 21 deletions .mesa_data_agent_state.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"workflow": "mesa_data_autonomy",
"branch": "fix/mesa-data-final-mvp",
"baseline_commit": "84c61db716c10cb4110b673f9a976215afe71cd3",
"current_phase": "complete",
"current_phase": "final_p1_record_instance_not_null",
"overall_status": "ready",
"phases": {
"prompt_1": {
Expand Down Expand Up @@ -92,7 +92,7 @@
"Web panel needed 6 key metrics, truthful local staging publish labels, multi-version document history, reprocess actions, and source automation settings."
],
"decisions": [
"Implemented 4-state citation lifecycle with Turkish legal aliases (TCK, CMK, HMK, TMK, TBK, TTK, İİK, İYUK, VUK, KVKK, AY, AvK, PVSK, İşK) and temporal relation verbs (amends, repeals, adds).",
"Implemented 4-state citation lifecycle with Turkish legal aliases (TCK, CMK, HMK, TMK, TBK, TTK, \u0130\u0130K, \u0130YUK, VUK, KVKK, AY, AvK, PVSK, \u0130\u015fK) and temporal relation verbs (amends, repeals, adds).",
"Enforced exact canonical coordinate slicing (canonical_text[char_start:char_end] == raw_text) and context-aware law number matching.",
"Implemented 7-condition Safe Auto-Approval Engine in catalog.py with parser certifications and weekly audit sampling.",
"Completed web management panel with health metrics, honest MESA banner, multi-version history, safe reprocessing, and operational settings."
Expand Down Expand Up @@ -149,7 +149,7 @@
"Deterministic hashing: Canonical revision hash is pure SHA256 of sorted chunk content hashes. Stable idempotency key mesa-data:<sha256(canonical_json)> contains zero runtime volatile data.",
"Delivery ledger & O(1) deduplication: Persists mesa_deliveries and mesa_delivery_items; skips already COMMITTED chunks across releases.",
"Truthful mutation lifecycle: COMMITTED, PROCESSING, QUEUED, FAILED, REJECTED tracked accurately; partial failure banner displays retry CTA.",
"Web Panel Yayınla tab: Target settings manager, preflight checklist, ready summary, confirmation modal, live progress polling, delivery detail modal, retry CTA.",
"Web Panel Yay\u0131nla tab: Target settings manager, preflight checklist, ready summary, confirmation modal, live progress polling, delivery detail modal, retry CTA.",
"CLI sync: mesa-data sync command discovers, fetches, processes, validates quality, auto-approves, and summarizes ready items, stopping strictly before MESA push."
],
"tests": [
Expand Down Expand Up @@ -196,14 +196,46 @@
"started_at": "2026-08-29T00:00:00Z",
"completed_at": "2026-08-29T00:00:00Z",
"findings": [
{"id": "QG_ZERO_ARTICLE_PASS", "severity": "P0", "status": "fixed"},
{"id": "PUBLICATION_DATE_BRIDGE_LOSS", "severity": "P0", "status": "fixed"},
{"id": "STALE_APPROVED_VERSION_PUBLISH", "severity": "P0", "status": "fixed"},
{"id": "CURRENT_VERSION_MESA_STATUS", "severity": "P0", "status": "fixed"},
{"id": "DERIVED_REPROCESS_EARLY_RETURN", "severity": "P0", "status": "fixed"},
{"id": "MESA_TARGET_KEY_EXFILTRATION", "severity": "P0", "status": "fixed"},
{"id": "MUTATION_TIMEOUT_MARKED_FAILED", "severity": "P1", "status": "fixed"},
{"id": "ARTIFACT_SHA_CROSS_DOCUMENT_COLLISION", "severity": "P1", "status": "fixed"}
{
"id": "QG_ZERO_ARTICLE_PASS",
"severity": "P0",
"status": "fixed"
},
{
"id": "PUBLICATION_DATE_BRIDGE_LOSS",
"severity": "P0",
"status": "fixed"
},
{
"id": "STALE_APPROVED_VERSION_PUBLISH",
"severity": "P0",
"status": "fixed"
},
{
"id": "CURRENT_VERSION_MESA_STATUS",
"severity": "P0",
"status": "fixed"
},
{
"id": "DERIVED_REPROCESS_EARLY_RETURN",
"severity": "P0",
"status": "fixed"
},
{
"id": "MESA_TARGET_KEY_EXFILTRATION",
"severity": "P0",
"status": "fixed"
},
{
"id": "MUTATION_TIMEOUT_MARKED_FAILED",
"severity": "P1",
"status": "fixed"
},
{
"id": "ARTIFACT_SHA_CROSS_DOCUMENT_COLLISION",
"severity": "P1",
"status": "fixed"
}
],
"fixes": [
"Legislation zero-article and abnormal large uncovered-area outcomes cannot PASS; article ordinal is retained in canonical records.",
Expand All @@ -219,15 +251,37 @@
"c20aa17 fix: secure current-version MESA publishing",
"71f4306 fix: report MESA state from current version"
],
"migrations": ["0009_correctness_hardening.sql"],
"migrations": [
"0009_correctness_hardening.sql"
],
"tests": [
{"name": "focused correctness regression", "command": "MESA_DATA_MESA_ALLOWED_HOST=mock-mesa.internal UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run pytest -q tests/unit/test_correctness_hardening.py tests/unit/test_quality_gate.py tests/unit/test_catalog_release_iterator.py tests/unit/test_publisher_client_and_ledger.py tests/integration/test_mesa_v4_publisher_e2e.py", "result": "19 passed"},
{"name": "compile", "command": "UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run python -m compileall -q src/mesa_legal_data", "result": "passed"},
{"name": "ruff", "command": "UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run ruff check src tests", "result": "passed"}
,{"name": "current-version MESA status regression", "command": "MESA_DATA_MESA_ALLOWED_HOST=mock-mesa.internal UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run pytest -q tests/unit/test_correctness_hardening.py tests/unit/test_publisher_client_and_ledger.py", "result": "10 passed"}
{
"name": "focused correctness regression",
"command": "MESA_DATA_MESA_ALLOWED_HOST=mock-mesa.internal UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run pytest -q tests/unit/test_correctness_hardening.py tests/unit/test_quality_gate.py tests/unit/test_catalog_release_iterator.py tests/unit/test_publisher_client_and_ledger.py tests/integration/test_mesa_v4_publisher_e2e.py",
"result": "19 passed"
},
{
"name": "compile",
"command": "UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run python -m compileall -q src/mesa_legal_data",
"result": "passed"
},
{
"name": "ruff",
"command": "UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run ruff check src tests",
"result": "passed"
},
{
"name": "current-version MESA status regression",
"command": "MESA_DATA_MESA_ALLOWED_HOST=mock-mesa.internal UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run pytest -q tests/unit/test_correctness_hardening.py tests/unit/test_publisher_client_and_ledger.py",
"result": "10 passed"
}
],
"blockers": [
"The all-test pytest invocation did not complete within the desktop command window and is not recorded as passed."
],
"blockers": ["The all-test pytest invocation did not complete within the desktop command window and is not recorded as passed."],
"unverified": ["MESA live HTTP payload and server-side mutation contract remain UNVERIFIED because no live endpoint or MESA source code is available."]
"unverified": [
"MESA live HTTP payload and server-side mutation contract remain UNVERIFIED because no live endpoint or MESA source code is available."
]
},
"prompt_5": {
"status": "completed",
Expand Down Expand Up @@ -262,7 +316,7 @@
"uv run mypy src/mesa_legal_data --ignore-missing-imports --explicit-package-bases --follow-imports=skip: passed (77 source files)",
"python -m compileall -q src/mesa_legal_data: passed",
"node --check src/mesa_legal_data/web/static/app.js: passed",
"Live browser journey: Ana Sayfa, Veri Topla, Kütüphane, İnceleme, Yayınla, advanced navigation and explicit MESA contract fields verified",
"Live browser journey: Ana Sayfa, Veri Topla, K\u00fct\u00fcphane, \u0130nceleme, Yay\u0131nla, advanced navigation and explicit MESA contract fields verified",
"Two bounded real Resmi Gazete sync runs in isolated storage: one item per run, zero MESA deliveries"
],
"blockers": [
Expand Down Expand Up @@ -400,6 +454,57 @@
"remaining_risks": [
"Live MESA HTTP request/response and server-side idempotency semantics remain unverified."
]
},
"final_p1_record_instance_not_null": {
"status": "completed",
"started_at": "2026-08-30T23:20:00+03:00",
"completed_at": "2026-08-30T23:35:00+03:00",
"commits": [
"fix: enforce non-null record instance identities"
],
"migrations": [
"0012_record_instance_not_null.sql"
],
"files_changed": [
"migrations/0012_record_instance_not_null.sql",
"tests/unit/test_migration_0012.py",
"tests/integration/test_post_master_independent_audit.py",
"tests/acceptance/test_workspace_e2e.py",
"tests/acceptance/test_operations_scale_gate.py",
"tests/integration/test_annotations_and_issues.py",
"tests/integration/test_data_explorer_api.py",
"tests/integration/test_large_export_streaming.py",
"tests/integration/test_record_revisions.py",
"tests/unit/test_final_closure_gate.py",
"tests/unit/test_release.py",
"docs/BUILD_STATE.json"
],
"findings": [
"records.record_instance_id was TEXT PRIMARY KEY without explicit NOT NULL, allowing SQLite to accept NULL values on insert.",
"Deterministic authoritative generator version_id:record_id backfills any legacy NULL record_instance_id values."
],
"decisions": [
"Added migration 0012_record_instance_not_null.sql performing safe table rebuild to declare record_instance_id TEXT NOT NULL PRIMARY KEY.",
"Preserved (version_id, record_id) UNIQUE constraint, foreign keys, and indexes (idx_records_version_record, idx_records_record_id, idx_records_version_type, idx_records_approval_status).",
"Preserved and restored triggers on release_items referencing records."
],
"tests": [
"Test A (Fresh DB): passed in test_migration_0012.py::test_migration_0012_test_a_fresh_db_table_info",
"Test B (NULL insert rejected): passed in test_migration_0012.py::test_migration_0012_test_b_null_insert_rejected",
"Test C (Normal insert works): passed in test_migration_0012.py::test_migration_0012_test_c_normal_insert_works",
"Test D (Upgrade populated DB): passed in test_migration_0012.py::test_migration_0012_test_d_upgrade_populated_db",
"Test E (Legacy NULL backfill): passed in test_migration_0012.py::test_migration_0012_test_e_legacy_null_backfill",
"Test F (Version-aware invariant): passed in test_migration_0012.py::test_migration_0012_test_f_version_aware_invariant",
"uv sync --frozen: passed",
"uv run ruff format --check .: passed (212 files)",
"uv run ruff check .: passed",
"uv run mypy src: passed (77 source files)",
"uv run pip-audit: passed (0 known vulnerabilities)",
"uv run pytest -m \"not scale\" -ra --durations=20: 356 passed, 0 failed, 4 deselected",
"uv run pytest -m \"scale\" -ra --durations=20: 4 passed, 0 failed, 356 deselected",
"uv run pytest -ra: 360 passed in 72.83s"
],
"blockers": []
}
},
"mesa_contract": {
Expand Down Expand Up @@ -431,11 +536,11 @@
},
"handoff": {
"next_phase": null,
"summary": "Final MVP blocker closure complete. All 354 tests pass (350 non-scale + 4 scale), fresh and populated upgrade migrations through 0011 pass, exact CI gates are green, and no P0/P1 MVP blocker remains.",
"summary": "Final P1 blocker fix complete. records.record_instance_id is strictly NOT NULL PRIMARY KEY at database level via migration 0012. All 360 tests pass (356 non-scale + 4 scale), fresh and upgrade migrations pass, CI gates green, 0 P0/P1 blockers remain.",
"must_verify": [
"Ensure live deployment environment configures MESA_DATA_MESA_API_KEY securely.",
"Configure documented MESA routes and verify health, publish, polling, COMMITTED truth, and server-side idempotency against a live endpoint."
]
},
"last_updated_at": "2026-08-30T00:43:56+03:00"
"last_updated_at": "2026-08-30T23:35:00+03:00"
}
11 changes: 6 additions & 5 deletions docs/BUILD_STATE.json
Original file line number Diff line number Diff line change
Expand Up @@ -40,16 +40,17 @@
"BLOCKER-3-RUN-SCOPED-REQUEST-BUDGET",
"PROMPT-5-FINAL-ADVERSARIAL-AUDIT",
"MASTER-MVP-CLOSURE",
"POST-MASTER-INDEPENDENT-VERIFICATION"
"POST-MASTER-INDEPENDENT-VERIFICATION",
"P1-RECORD-INSTANCE-NOT-NULL-ENFORCED"
],
"last_test_command": "uv run pytest -ra",
"last_test_result": "passed (351 passed: 347 non-scale + 4 scale in 73.34s)",
"last_test_result": "passed (360 passed: 356 non-scale + 4 scale in 72.83s)",
"verification_checks": {
"ruff_format": "passed (211 Python files)",
"ruff_format": "passed (212 Python files)",
"ruff_check": "passed",
"mypy": "passed (77 source files)",
"pip_audit": "passed (no known vulnerabilities)",
"pytest_not_scale": "passed (347 passed)",
"pytest_not_scale": "passed (356 passed)",
"pytest_scale": "passed (4 passed)",
"live_mesa_contract": "LIVE_MESA_NOT_VERIFIED"
},
Expand All @@ -58,5 +59,5 @@
],
"polish_status": "completed",
"polish_current_task": "CLOSED",
"updated_at": "2026-08-29T23:15:00Z"
"updated_at": "2026-08-30T23:30:00Z"
}
83 changes: 83 additions & 0 deletions migrations/0012_record_instance_not_null.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
-- Migration 0012: Enforce NOT NULL invariant on records.record_instance_id.
-- Rebuilds the records table to ensure record_instance_id is declared
-- as TEXT NOT NULL PRIMARY KEY, backfills any legacy NULL record_instance_id
-- values with deterministic version_id:record_id identities, and preserves
-- all composite uniqueness, foreign keys, and indexes.

PRAGMA foreign_keys = OFF;

DROP TRIGGER IF EXISTS release_items_identity_valid_insert;
DROP TRIGGER IF EXISTS release_items_identity_valid_update;

CREATE TABLE records_v3 (
record_instance_id TEXT NOT NULL PRIMARY KEY,
record_id TEXT NOT NULL,
version_id TEXT NOT NULL,
record_type TEXT NOT NULL,
canonical_path TEXT NOT NULL,
canonical_line INTEGER NOT NULL,
record_sha256 TEXT NOT NULL,
validation_status TEXT NOT NULL,
approval_status TEXT NOT NULL,
created_at TEXT NOT NULL,
FOREIGN KEY (version_id) REFERENCES versions(version_id)
);

INSERT INTO records_v3 (
record_instance_id,
record_id,
version_id,
record_type,
canonical_path,
canonical_line,
record_sha256,
validation_status,
approval_status,
created_at
)
SELECT
COALESCE(record_instance_id, version_id || ':' || record_id),
record_id,
version_id,
record_type,
canonical_path,
canonical_line,
record_sha256,
validation_status,
approval_status,
created_at
FROM records;

DROP TABLE records;
ALTER TABLE records_v3 RENAME TO records;

CREATE UNIQUE INDEX IF NOT EXISTS idx_records_version_record ON records(version_id, record_id);
CREATE INDEX IF NOT EXISTS idx_records_record_id ON records(record_id);
CREATE INDEX IF NOT EXISTS idx_records_version_type ON records(version_id, record_type);
CREATE INDEX IF NOT EXISTS idx_records_approval_status ON records(approval_status);

CREATE TRIGGER release_items_identity_valid_insert
BEFORE INSERT ON release_items
WHEN NOT EXISTS (
SELECT 1 FROM records r
WHERE r.version_id = NEW.version_id
AND r.record_id = NEW.record_id
AND r.record_sha256 = NEW.record_sha256
)
BEGIN
SELECT RAISE(ABORT, 'release item version identity is invalid');
END;

CREATE TRIGGER release_items_identity_valid_update
BEFORE UPDATE OF version_id, record_id, record_sha256 ON release_items
WHEN NOT EXISTS (
SELECT 1 FROM records r
WHERE r.version_id = NEW.version_id
AND r.record_id = NEW.record_id
AND r.record_sha256 = NEW.record_sha256
)
BEGIN
SELECT RAISE(ABORT, 'release item version identity is invalid');
END;

PRAGMA foreign_keys = ON;
6 changes: 3 additions & 3 deletions tests/acceptance/test_operations_scale_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,9 @@ def test_operations_scale_gate(tmp_path, monkeypatch):
(v_id, i, i),
)
c.execute(
"""INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at)
VALUES (?, ?, 'article', 'canonical/scale.jsonl', ?, ?, 'valid', 'pending', '2026-08-05T00:00:00Z')""",
(r_id, v_id, i, rec_sha),
"""INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at)
VALUES (?, ?, ?, 'article', 'canonical/scale.jsonl', ?, ?, 'valid', 'pending', '2026-08-05T00:00:00Z')""",
(f"{v_id}:{r_id}", r_id, v_id, i, rec_sha),
)
f_canon.write(line_str)

Expand Down
8 changes: 4 additions & 4 deletions tests/acceptance/test_workspace_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,13 +104,13 @@ def _seed_data(tmp_path):
canon_file.write_text(line1 + line2, encoding="utf-8")

conn.execute(
"""INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at)
VALUES ('rec-e2e-art-1', 'ver-e2e-1', 'article', 'canonical/e2e/article.jsonl', 1, ?, 'valid', 'pending', '2026-08-05T00:00:00Z')""",
"""INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at)
VALUES ('ver-e2e-1:rec-e2e-art-1', 'rec-e2e-art-1', 'ver-e2e-1', 'article', 'canonical/e2e/article.jsonl', 1, ?, 'valid', 'pending', '2026-08-05T00:00:00Z')""",
(sha1,),
)
conn.execute(
"""INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at)
VALUES ('rec-e2e-art-2', 'ver-e2e-1', 'article', 'canonical/e2e/article.jsonl', 2, ?, 'valid', 'pending', '2026-08-05T00:00:00Z')""",
"""INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at)
VALUES ('ver-e2e-1:rec-e2e-art-2', 'rec-e2e-art-2', 'ver-e2e-1', 'article', 'canonical/e2e/article.jsonl', 2, ?, 'valid', 'pending', '2026-08-05T00:00:00Z')""",
(sha2,),
)
conn.commit()
Expand Down
4 changes: 2 additions & 2 deletions tests/integration/test_annotations_and_issues.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ def test_annotations_issues_and_quarantine(tmp_path, monkeypatch):
VALUES ('ver-q1', 'doc-q1', 'art-q1', 'snapshot', 'canonical/q1.jsonl', 1, 'sha-ver-q1', 'test_parser', '1.0', '1.0', 'valid', 'clean', 'pending', '2026-08-05T00:00:00Z')"""
)
c.execute(
"""INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at)
VALUES ('rec-q1', 'ver-q1', 'article', 'canonical/q1.jsonl', 1, 'sha-rec-q1', 'invalid', 'pending', '2026-08-05T00:00:00Z')"""
"""INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at)
VALUES ('ver-q1:rec-q1', 'rec-q1', 'ver-q1', 'article', 'canonical/q1.jsonl', 1, 'sha-rec-q1', 'invalid', 'pending', '2026-08-05T00:00:00Z')"""
)

# 1. Add Annotation
Expand Down
Loading
Loading