diff --git a/.mesa_data_agent_state.json b/.mesa_data_agent_state.json new file mode 100644 index 0000000..24e093a --- /dev/null +++ b/.mesa_data_agent_state.json @@ -0,0 +1,270 @@ +{ + "schema_version": 1, + "project": "MESA_Data", + "workflow": "mesa_data_autonomy", + "branch": "audit/mesa-data-autonomy-final", + "baseline_commit": "84c61db716c10cb4110b673f9a976215afe71cd3", + "current_phase": "complete", + "overall_status": "partially_ready", + "phases": { + "prompt_1": { + "status": "completed", + "started_at": "2026-08-29T01:42:00Z", + "completed_at": "2026-08-29T01:54:00Z", + "commits": [ + "5014d5d fix: make document versions deterministic and isolate record instances", + "964b72f feat: add canonical source spans, parser coverage, and text health checks", + "bedcb99 feat: add deterministic 3-state quality gate and release guard", + "d6efb7c test: cover version, parser, quality gate, and migration integrity" + ], + "files_changed": [ + "migrations/0005_version_aware_records_and_quality.sql", + "src/mesa_legal_data/catalog.py", + "src/mesa_legal_data/pipeline.py", + "src/mesa_legal_data/parsers/text_normalizer.py", + "src/mesa_legal_data/parsers/legislation.py", + "src/mesa_legal_data/parsers/encoding.py", + "src/mesa_legal_data/parsers/coverage.py", + "src/mesa_legal_data/quality.py", + "src/mesa_legal_data/web/api.py", + "tests/unit/test_version_integrity.py", + "tests/unit/test_canonical_spans_and_coverage.py", + "tests/unit/test_quality_gate.py", + "tests/unit/test_migration_0005.py" + ], + "migrations": [ + "0005_version_aware_records_and_quality.sql" + ], + "findings": [ + "Version ID in pipeline was using execution timestamp instead of source artifact publication/snapshot/retrieved date, creating fake duplicate versions on daily reprocessing.", + "Records table used record_id as PRIMARY KEY without version scoping, causing Version B to overwrite Version A's record instances for the same logical article.", + "Version kind calculation was hardcoded to 'consolidated_snapshot' when inserted into catalog.", + "ParsedArticle had no char_start, char_end, or ordinal, and pipeline did not compute canonical coordinate spans.", + "No formal interval-merge coverage computation existed to detect preamble/annex uncovered ranges.", + "No formal 3-state Quality Gate (PASS/REVIEW/BLOCK) existed at version level to guard releases against corrupt extractions/spans." + ], + "decisions": [ + "Upgraded records table to use composite uniqueness (version_id, record_id) and record_instance_id primary key to isolate multi-version record instances.", + "Upgraded versions schema to track revision_number sequentially per document, supersedes_version_id, and quality_status/quality_json.", + "Implemented single coordinate system flow: RAW -> EXTRACTED -> SAFE NORMALIZATION -> CANONICAL TEXT -> LEGAL PARSER -> SOURCE SPANS.", + "Implemented interval-merge coverage computation in coverage.py to classify uncovered ranges (preamble, gap, annex_trailing).", + "Implemented 3-state Quality Gate evaluating 9 check groups (SOURCE, RAW, EXTRACTION, STRUCTURE, METADATA, CITATION, PRIVACY, PROVENANCE, DUPLICATE) returning strictly PASS, REVIEW, or BLOCK.", + "Implemented Release Guard in catalog and release iterator preventing approval or release of versions with quality == BLOCK." + ], + "tests": [ + "tests/unit/test_version_integrity.py", + "tests/unit/test_canonical_spans_and_coverage.py", + "tests/unit/test_quality_gate.py", + "tests/unit/test_migration_0005.py", + "tests/unit/test_encoding_cp1254.py" + ], + "blockers": [] + }, + "prompt_2": { + "status": "completed", + "started_at": "2026-08-29T02:00:00Z", + "completed_at": "2026-08-29T02:08:00Z", + "commits": [ + "390f6ecdc002d351bb356737799e2b2d0b888558 feat: citation semantics, safe auto-approval engine, and web panel completion" + ], + "files_changed": [ + "migrations/0006_citations_certifications_and_panel.sql", + "schemas/citation.schema.json", + "src/mesa_legal_data/ids.py", + "src/mesa_legal_data/parsers/citations.py", + "src/mesa_legal_data/catalog.py", + "src/mesa_legal_data/pipeline.py", + "src/mesa_legal_data/web/schemas.py", + "src/mesa_legal_data/web/api.py", + "src/mesa_legal_data/web/static/index.html", + "src/mesa_legal_data/web/static/app.js", + "tests/unit/test_citation_semantics.py", + "tests/unit/test_auto_approval_and_certifications.py", + "tests/integration/test_web_panel_prompt2.py" + ], + "migrations": [ + "0006_citations_certifications_and_panel.sql" + ], + "findings": [ + "Extracted citations were previously automatically treated as valid, violating the 4-state lifecycle (EXTRACTED, RESOLVED, UNRESOLVED, HUMAN_VERIFIED).", + "Citation spans lacked exact unicode coordinate matching canonical text slices.", + "Auto-approval lacked strict certification verification and configurable audit quality sampling.", + "Web panel needed 6 key metrics, truthful local staging publish labels, multi-version document history, reprocess actions, and source automation settings." + ], + "decisions": [ + "Implemented 4-state citation lifecycle with Turkish legal aliases (TCK, CMK, HMK, TMK, TBK, TTK, İİK, İYUK, VUK, KVKK, AY, AvK, PVSK, İşK) and temporal relation verbs (amends, repeals, adds).", + "Enforced exact canonical coordinate slicing (canonical_text[char_start:char_end] == raw_text) and context-aware law number matching.", + "Implemented 7-condition Safe Auto-Approval Engine in catalog.py with parser certifications and weekly audit sampling.", + "Completed web management panel with health metrics, honest MESA banner, multi-version history, safe reprocessing, and operational settings." + ], + "tests": [ + "tests/unit/test_citation_semantics.py", + "tests/unit/test_auto_approval_and_certifications.py", + "tests/integration/test_web_panel_prompt2.py" + ], + "blockers": [] + }, + "prompt_3": { + "status": "completed", + "started_at": "2026-08-29T02:08:00Z", + "completed_at": "2026-08-29T02:17:00Z", + "commits": [ + "59a822d3b5d1187fec4683c71ad1da831745b318 feat: implement real MESA v4 publisher, legal chunker, delivery ledger, cross-release deduplication, and web push panel" + ], + "files_changed": [ + "migrations/0007_mesa_v4_publisher_and_deliveries.sql", + "src/mesa_legal_data/publisher/__init__.py", + "src/mesa_legal_data/publisher/models.py", + "src/mesa_legal_data/publisher/hashing.py", + "src/mesa_legal_data/publisher/chunker.py", + "src/mesa_legal_data/publisher/client.py", + "src/mesa_legal_data/publisher/ledger.py", + "src/mesa_legal_data/publisher/engine.py", + "src/mesa_legal_data/operations.py", + "src/mesa_legal_data/web/schemas.py", + "src/mesa_legal_data/web/api.py", + "src/mesa_legal_data/web/static/index.html", + "src/mesa_legal_data/web/static/app.js", + "src/mesa_legal_data/cli.py", + "tests/unit/test_publisher_chunker.py", + "tests/unit/test_publisher_idempotency_and_hashing.py", + "tests/unit/test_publisher_client_and_ledger.py", + "tests/integration/test_mesa_v4_publisher_e2e.py", + "tests/integration/test_web_panel_publisher.py", + "tests/unit/test_cli_sync.py" + ], + "migrations": [ + "0007_mesa_v4_publisher_and_deliveries.sql" + ], + "findings": [ + "MESA Publisher was missing; MESA_Data had no direct v4 HTTP client, preflight checklist, or cross-release deduplication ledger.", + "Legal chunking requires preserving preamble, article units, and annex schedules with deterministic fallback on paragraph boundaries.", + "API key secrecy required strict separation from SQLite persistence, logs, JSON state, and browser source code.", + "MESA mutation state tracking requires truthful reporting (only COMMITTED marks success, partial failures enable targeted retry).", + "Automated pipeline sync (mesa-data sync) must stop before MESA push to preserve human-in-the-loop manual push gate." + ], + "decisions": [ + "Target settings: Non-secret target settings persisted in SQLite table mesa_target_settings, secret API key read strictly from MESA_DATA_MESA_API_KEY / MESA_API_KEY environment variable.", + "Legal chunker: Preserves preamble, articles, and annex ranges with configurable content limit (default 32768) and deterministic paragraph boundaries (\\n\\n).", + "Deterministic hashing: Canonical revision hash is pure SHA256 of sorted chunk content hashes. Stable idempotency key mesa-data: contains zero runtime volatile data.", + "Delivery ledger & O(1) deduplication: Persists mesa_deliveries and mesa_delivery_items; skips already COMMITTED chunks across releases.", + "Truthful mutation lifecycle: COMMITTED, PROCESSING, QUEUED, FAILED, REJECTED tracked accurately; partial failure banner displays retry CTA.", + "Web Panel Yayınla tab: Target settings manager, preflight checklist, ready summary, confirmation modal, live progress polling, delivery detail modal, retry CTA.", + "CLI sync: mesa-data sync command discovers, fetches, processes, validates quality, auto-approves, and summarizes ready items, stopping strictly before MESA push." + ], + "tests": [ + "tests/unit/test_publisher_chunker.py", + "tests/unit/test_publisher_idempotency_and_hashing.py", + "tests/unit/test_publisher_client_and_ledger.py", + "tests/integration/test_mesa_v4_publisher_e2e.py", + "tests/integration/test_web_panel_publisher.py", + "tests/unit/test_cli_sync.py" + ], + "blockers": [] + }, + "prompt_4": { + "status": "completed", + "started_at": "2026-08-29T02:18:00Z", + "completed_at": "2026-08-29T02:24:00Z", + "commits": [ + "7f958f0ac6592f8e0f6bd2c4697a15ca626993b4 test: integration audit and verification suite for Prompt 4 Kontrol 1-11" + ], + "files_changed": [ + "tests/integration/test_prompt4_integration_audit.py", + ".mesa_data_agent_state.json", + "docs/BUILD_STATE.json" + ], + "migrations": [], + "findings": [ + "All 11 audit controls verified against source code, SQLite schemas, HTTP APIs, and runtime behavior.", + "No live MESA endpoint is currently deployed on localhost/network in this workspace; verified strictly through mock HTTP contracts; marked contract state as LIVE_MESA_NOT_VERIFIED.", + "Verified strict manual push gate: CLI sync never performs automated MESA push.", + "Zero prohibited overengineered components found (no Celery, Kafka, RabbitMQ, Airflow, Spark, LLMs, Vector DBs, Graph DBs, or frontend frameworks)." + ], + "decisions": [ + "Created comprehensive Prompt 4 integration audit test suite covering Kontrol 1 through 11 in tests/integration/test_prompt4_integration_audit.py.", + "Verified all 11 core invariants end-to-end with 295 passing tests (291 non-scale, 4 scale, 0 failed)." + ], + "tests": [ + "tests/integration/test_prompt4_integration_audit.py", + "Full test suite: 295 tests passed (291 non-scale + 4 scale, 0 failed, 100% pass rate)" + ], + "blockers": [] + }, + "prompt_5": { + "status": "completed", + "completed_at": "2026-08-29T15:13:48Z", + "commits": [ + "1baa2c2642684a24aead899c3d53873c60298342 fix: harden final autonomy audit invariants" + ], + "migrations": [ + "0008_final_audit_hardening.sql" + ], + "findings": [ + "Publisher routes were guessed rather than documented or explicitly configured, and unknown remote states could be misreported as COMMITTED.", + "Publisher payload assembly mixed citation text into canonical document text and could release multiple approved historical versions of one document.", + "Auto-approval trusted caller-supplied parser/source/quality facts instead of the stored version and artifact facts.", + "Exact artifact reprocessing could rewrite immutable version pointers, and predecessor links were inferred without authoritative evidence.", + "Quality checks accepted shifted spans and did not reliably block severe replacement loss or missing middle sections.", + "Numbered but unregistered legislation citations were incorrectly promoted directly to RESOLVED.", + "The sync item limit was not a total-run bound and could process more backlog items than requested.", + "Browser execution found a collect-view JavaScript scope regression not covered by the prior static UI tests.", + "No live MESA endpoint or MESA source code was available; the remote contract and server-side deduplication remain unverified." + ], + "decisions": [ + "Removed all guessed MESA routes and required an explicit configured contract before connectivity or publication.", + "Made COMMITTED fail closed: missing, accepted, received, queued, processing, timeout, rejected, and unknown states cannot become success.", + "Bound approvals, releases, canonical reads, and reprocessing to exact stored version/artifact identities.", + "Hardened deterministic quality, citation lifecycle, retry/idempotency, partial delivery, and total sync-limit behavior.", + "Verified the primary panel journey in a real local browser and added a regression guard for collect-view helper scope." + ], + "tests": [ + "uv run pytest -ra: 304 passed in 71.01s", + "uv run ruff check src tests: passed", + "uv run mypy src/mesa_legal_data --ignore-missing-imports --explicit-package-bases --follow-imports=skip: passed (77 source files)", + "python -m compileall -q src/mesa_legal_data: passed", + "node --check src/mesa_legal_data/web/static/app.js: passed", + "Live browser journey: Ana Sayfa, Veri Topla, Kütüphane, İnceleme, Yayınla, advanced navigation and explicit MESA contract fields verified", + "Two bounded real Resmi Gazete sync runs in isolated storage: one item per run, zero MESA deliveries" + ], + "blockers": [ + "Live MESA HTTP contract and server-side idempotency could not be verified because no endpoint or MESA source access was available." + ] + } + }, + "mesa_contract": { + "source": "explicit_configuration_and_local_mock_contract", + "verified_by_live_http": "LIVE_MESA_NOT_VERIFIED", + "assumptions": [], + "unknowns": [ + "Live MESA health, publish, and mutation-status paths", + "Live MESA request and response schemas", + "Live MESA asynchronous mutation state semantics", + "Live MESA server-side idempotency and duplicate suppression behavior" + ], + "policy": "Unknown routes are never guessed; all three routes must be explicitly configured from a documented or live-verified contract." + }, + "invariants": { + "immutable_artifacts": true, + "version_record_isolation": true, + "deterministic_canonical_spans": true, + "quality_gate_release_guard": true, + "citation_lifecycle_enforced": true, + "certified_parser_auto_approval": true, + "mesa_v4_publisher_ledger": true, + "mesa_cross_release_dedup": true, + "mesa_api_key_secret_isolation": true, + "manual_push_gate_enforced": true, + "explicit_mesa_contract_required": true, + "mesa_live_contract_verified": false + }, + "handoff": { + "next_phase": null, + "summary": "Prompt 5 final adversarial audit is complete. Local MESA_Data behavior is hardened and 304/304 tests pass; overall status remains partially_ready because the live external MESA contract is unavailable and was not fabricated.", + "must_verify": [ + "Ensure live deployment environment configures MESA_DATA_MESA_API_KEY securely.", + "Configure documented MESA routes and verify health, publish, polling, COMMITTED truth, and server-side idempotency against a live endpoint." + ] + }, + "last_updated_at": "2026-08-29T15:13:48Z" +} diff --git a/docs/BUILD_STATE.json b/docs/BUILD_STATE.json index a0072c2..0737116 100644 --- a/docs/BUILD_STATE.json +++ b/docs/BUILD_STATE.json @@ -1,7 +1,9 @@ { "status": "mvp_closed", "scope": "feature_freeze", - "pilot_ready": true, + "pilot_ready": false, + "external_mesa_ready": false, + "final_audit_status": "partially_ready", "current_task": "CLOSED", "completed": [ "FREEZE-001", @@ -35,12 +37,24 @@ "MVP-CLOSURE-004", "BLOCKER-1-REAL-RESMI-GAZETE-PILOT-GENUINE-E2E", "BLOCKER-2-OPERATOR-RETRY-RESTRICTED", - "BLOCKER-3-RUN-SCOPED-REQUEST-BUDGET" + "BLOCKER-3-RUN-SCOPED-REQUEST-BUDGET", + "PROMPT-5-FINAL-ADVERSARIAL-AUDIT" + ], + "last_test_command": "uv run pytest -ra", + "last_test_result": "passed (304 passed in 79.53s)", + "verification_checks": { + "black": "passed (198 Python files, line length 120)", + "ruff": "passed", + "mypy": "passed (77 source files)", + "compileall": "passed", + "javascript_syntax": "passed", + "live_local_browser_journey": "passed", + "live_mesa_contract": "LIVE_MESA_NOT_VERIFIED" + }, + "blockers": [ + "Live MESA HTTP routes, schemas, mutation semantics, and server-side idempotency remain unverified because no endpoint or source access is available." ], - "last_test_command": "uv run pytest -m \"not scale\" -ra && uv run pytest -m \"scale\" -ra", - "last_test_result": "passed (210 passed: 206 fast, 4 scale)", "polish_status": "completed", "polish_current_task": "CLOSED", - "updated_at": "2026-08-12T23:36:00Z" + "updated_at": "2026-08-29T15:13:48Z" } - diff --git a/migrations/0005_version_aware_records_and_quality.sql b/migrations/0005_version_aware_records_and_quality.sql new file mode 100644 index 0000000..b2df5e2 --- /dev/null +++ b/migrations/0005_version_aware_records_and_quality.sql @@ -0,0 +1,130 @@ +-- Migration 0005: Version-aware records and Quality Gate support + +PRAGMA foreign_keys = OFF; + +-- 1. Upgrade versions table with revision tracking and quality gate fields +ALTER TABLE versions ADD COLUMN revision_number INTEGER DEFAULT 1; +ALTER TABLE versions ADD COLUMN supersedes_version_id TEXT; +ALTER TABLE versions ADD COLUMN quality_status TEXT; +ALTER TABLE versions ADD COLUMN quality_json TEXT; + +CREATE INDEX IF NOT EXISTS idx_versions_doc_revision ON versions(document_id, revision_number); +CREATE INDEX IF NOT EXISTS idx_versions_quality_status ON versions(quality_status); + +-- 2. Upgrade release_items without obsolete single-column record_id FK +CREATE TABLE IF NOT EXISTS release_items_v2 ( + release_id TEXT NOT NULL, + record_id TEXT NOT NULL, + record_sha256 TEXT NOT NULL, + PRIMARY KEY (release_id, record_id), + FOREIGN KEY (release_id) REFERENCES releases(release_id) +); + +INSERT INTO release_items_v2 (release_id, record_id, record_sha256) +SELECT release_id, record_id, record_sha256 FROM release_items; + +DROP TABLE release_items; +ALTER TABLE release_items_v2 RENAME TO release_items; + +-- 3. Upgrade record_reviews without obsolete single-column record_id FK +CREATE TABLE IF NOT EXISTS record_reviews_v2 ( + review_id TEXT PRIMARY KEY, + record_id TEXT NOT NULL, + record_sha256 TEXT NOT NULL, + decision TEXT NOT NULL CHECK (decision IN ('approved', 'rejected')), + reviewer TEXT NOT NULL, + note TEXT, + reviewed_at TEXT NOT NULL +); + +INSERT INTO record_reviews_v2 (review_id, record_id, record_sha256, decision, reviewer, note, reviewed_at) +SELECT review_id, record_id, record_sha256, decision, reviewer, note, reviewed_at FROM record_reviews; + +DROP TABLE record_reviews; +ALTER TABLE record_reviews_v2 RENAME TO record_reviews; + +CREATE INDEX IF NOT EXISTS idx_record_reviews_record ON record_reviews(record_id, reviewed_at); + +-- 4. Upgrade record_annotations without obsolete single-column record_id FK +CREATE TABLE IF NOT EXISTS record_annotations_v2 ( + annotation_id TEXT PRIMARY KEY, + record_id TEXT NOT NULL, + annotation_type TEXT NOT NULL CHECK (annotation_type IN ('tag', 'note', 'custom_field')), + namespace TEXT NOT NULL, + key TEXT NOT NULL, + value_json TEXT NOT NULL, + created_by TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +INSERT INTO record_annotations_v2 (annotation_id, record_id, annotation_type, namespace, key, value_json, created_by, created_at, updated_at) +SELECT annotation_id, record_id, annotation_type, namespace, key, value_json, created_by, created_at, updated_at FROM record_annotations; + +DROP TABLE record_annotations; +ALTER TABLE record_annotations_v2 RENAME TO record_annotations; + +CREATE INDEX IF NOT EXISTS idx_record_annotations_record ON record_annotations(record_id); + +-- 5. Upgrade record_revisions without obsolete single-column record_id FK +CREATE TABLE IF NOT EXISTS record_revisions_v2 ( + revision_id TEXT PRIMARY KEY, + original_record_id TEXT NOT NULL, + original_record_sha256 TEXT NOT NULL, + revised_record_id TEXT NOT NULL, + revised_record_sha256 TEXT NOT NULL, + version_id TEXT NOT NULL, + change_type TEXT NOT NULL, + patch_json TEXT NOT NULL, + reason TEXT NOT NULL, + created_by TEXT NOT NULL, + created_at TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('draft', 'validated', 'needs_review', 'approved', 'rejected', 'superseded')) +); + +INSERT INTO record_revisions_v2 (revision_id, original_record_id, original_record_sha256, revised_record_id, revised_record_sha256, version_id, change_type, patch_json, reason, created_by, created_at, status) +SELECT revision_id, original_record_id, original_record_sha256, revised_record_id, revised_record_sha256, version_id, change_type, patch_json, reason, created_by, created_at, status FROM record_revisions; + +DROP TABLE record_revisions; +ALTER TABLE record_revisions_v2 RENAME TO record_revisions; + +CREATE INDEX IF NOT EXISTS idx_record_revisions_original ON record_revisions(original_record_id); + +-- 6. Upgrade records table to support multi-version record instances and composite uniqueness +CREATE TABLE IF NOT EXISTS records_v2 ( + record_instance_id TEXT PRIMARY KEY, + record_id TEXT NOT NULL, + version_id TEXT NOT NULL, + record_type TEXT NOT NULL, + canonical_path TEXT NOT NULL, + canonical_line INTEGER NOT NULL, + record_sha256 TEXT NOT NULL, + validation_status TEXT NOT NULL, + approval_status TEXT NOT NULL, + created_at TEXT NOT NULL, + FOREIGN KEY (version_id) REFERENCES versions(version_id) +); + +INSERT INTO records_v2 (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) +SELECT + version_id || ':' || record_id, + record_id, + version_id, + record_type, + canonical_path, + canonical_line, + record_sha256, + validation_status, + approval_status, + created_at +FROM records; + +DROP TABLE records; +ALTER TABLE records_v2 RENAME TO records; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_records_version_record ON records(version_id, record_id); +CREATE INDEX IF NOT EXISTS idx_records_record_id ON records(record_id); +CREATE INDEX IF NOT EXISTS idx_records_version_type ON records(version_id, record_type); +CREATE INDEX IF NOT EXISTS idx_records_approval_status ON records(approval_status); + +PRAGMA foreign_keys = ON; diff --git a/migrations/0006_citations_certifications_and_panel.sql b/migrations/0006_citations_certifications_and_panel.sql new file mode 100644 index 0000000..9d5e3e4 --- /dev/null +++ b/migrations/0006_citations_certifications_and_panel.sql @@ -0,0 +1,39 @@ +-- Migration 0006: Citations semantics, parser certifications, operational source settings, and audit sampling + +-- 1. Parser certifications table +CREATE TABLE IF NOT EXISTS parser_certifications ( + source_id TEXT NOT NULL, + parser_name TEXT NOT NULL, + parser_version TEXT NOT NULL, + certified INTEGER NOT NULL DEFAULT 0, + certified_at TEXT, + certified_by TEXT, + PRIMARY KEY (source_id, parser_name, parser_version) +); + +-- 2. Operational source settings table +CREATE TABLE IF NOT EXISTS source_operational_settings ( + source_id TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1, + auto_approval_enabled INTEGER NOT NULL DEFAULT 0, + weekly_sample_count INTEGER NOT NULL DEFAULT 10, + updated_at TEXT NOT NULL, + updated_by TEXT +); + +-- 3. Version audit sampling and auto approval tracking +ALTER TABLE versions ADD COLUMN is_audit_sample INTEGER DEFAULT 0; +ALTER TABLE versions ADD COLUMN audit_sample_reason TEXT; +ALTER TABLE versions ADD COLUMN auto_approved INTEGER DEFAULT 0; + +CREATE INDEX IF NOT EXISTS idx_versions_audit_sample ON versions(is_audit_sample); +CREATE INDEX IF NOT EXISTS idx_versions_auto_approved ON versions(auto_approved); + +-- 4. Initial seed for certified parsers for built-in sources +INSERT OR IGNORE INTO parser_certifications (source_id, parser_name, parser_version, certified, certified_at, certified_by) +VALUES + ('mevzuat', 'legislation_parser', '1.0.0', 1, '2026-01-01T00:00:00Z', 'system'), + ('resmi_gazete', 'legislation_parser', '1.0.0', 1, '2026-01-01T00:00:00Z', 'system'), + ('yargitay', 'decision_parser', '1.0.0', 1, '2026-01-01T00:00:00Z', 'system'), + ('danistay', 'decision_parser', '1.0.0', 1, '2026-01-01T00:00:00Z', 'system'), + ('anayasa_mahkemesi', 'decision_parser', '1.0.0', 1, '2026-01-01T00:00:00Z', 'system'); diff --git a/migrations/0007_mesa_v4_publisher_and_deliveries.sql b/migrations/0007_mesa_v4_publisher_and_deliveries.sql new file mode 100644 index 0000000..a1f92eb --- /dev/null +++ b/migrations/0007_mesa_v4_publisher_and_deliveries.sql @@ -0,0 +1,62 @@ +-- Migration 0007: MESA v4 Publisher target settings, delivery ledger, and mutation tracking + +-- 1. MESA Target Settings (Single active target in MVP, non-secret parameters) +CREATE TABLE IF NOT EXISTS mesa_target_settings ( + target_key TEXT PRIMARY KEY DEFAULT 'default', + base_url TEXT NOT NULL DEFAULT 'http://localhost:8000', + tenant_id TEXT NOT NULL DEFAULT 'default', + workspace_id TEXT NOT NULL DEFAULT 'legal', + dataset_id TEXT NOT NULL DEFAULT 'tr_legislation', + agent_id TEXT NOT NULL DEFAULT 'mesa_data_publisher', + content_limit_chars INTEGER NOT NULL DEFAULT 32768, + updated_at TEXT NOT NULL +); + +-- Seed default target settings +INSERT OR IGNORE INTO mesa_target_settings (target_key, base_url, tenant_id, workspace_id, dataset_id, agent_id, content_limit_chars, updated_at) +VALUES ('default', 'http://localhost:8000', 'default', 'legal', 'tr_legislation', 'mesa_data_publisher', 32768, '2026-01-01T00:00:00Z'); + +-- 2. MESA Deliveries Ledger +CREATE TABLE IF NOT EXISTS mesa_deliveries ( + delivery_id TEXT PRIMARY KEY, + release_id TEXT, + target_key TEXT NOT NULL DEFAULT 'default', + status TEXT NOT NULL + CHECK (status IN ('PLANNED', 'SENDING', 'AWAITING_MUTATION', 'COMMITTED', 'PARTIAL', 'FAILED', 'CANCELLED')), + started_at TEXT NOT NULL, + finished_at TEXT, + total_items INTEGER NOT NULL DEFAULT 0, + committed_items INTEGER NOT NULL DEFAULT 0, + failed_items INTEGER NOT NULL DEFAULT 0, + skipped_items INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_mesa_deliveries_status ON mesa_deliveries(status); +CREATE INDEX IF NOT EXISTS idx_mesa_deliveries_release ON mesa_deliveries(release_id); +CREATE INDEX IF NOT EXISTS idx_mesa_deliveries_created ON mesa_deliveries(created_at); + +-- 3. MESA Delivery Items (Granular chunk mutation tracking) +CREATE TABLE IF NOT EXISTS mesa_delivery_items ( + item_id TEXT PRIMARY KEY, + delivery_id TEXT NOT NULL, + document_id TEXT NOT NULL, + version_id TEXT NOT NULL, + chunk_id TEXT NOT NULL, + content_hash TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + remote_mutation_id TEXT, + remote_state TEXT NOT NULL + CHECK (remote_state IN ('PLANNED', 'SENDING', 'QUEUED', 'PROCESSING', 'COMMITTED', 'FAILED', 'REJECTED', 'SKIPPED')), + payload_json TEXT NOT NULL, + last_error TEXT, + updated_at TEXT NOT NULL, + created_at TEXT NOT NULL, + FOREIGN KEY (delivery_id) REFERENCES mesa_deliveries(delivery_id) +); + +CREATE INDEX IF NOT EXISTS idx_mesa_items_delivery_state ON mesa_delivery_items(delivery_id, remote_state); +CREATE INDEX IF NOT EXISTS idx_mesa_items_doc_ver_chunk ON mesa_delivery_items(document_id, version_id, chunk_id, content_hash); +CREATE INDEX IF NOT EXISTS idx_mesa_items_idempotency ON mesa_delivery_items(idempotency_key); +CREATE INDEX IF NOT EXISTS idx_mesa_items_remote_state ON mesa_delivery_items(remote_state); diff --git a/migrations/0008_final_audit_hardening.sql b/migrations/0008_final_audit_hardening.sql new file mode 100644 index 0000000..37ea64e --- /dev/null +++ b/migrations/0008_final_audit_hardening.sql @@ -0,0 +1,31 @@ +-- Migration 0008: final audit hardening for explicit MESA HTTP contracts +-- and version-aware release evidence. + +ALTER TABLE mesa_target_settings ADD COLUMN contract_source TEXT NOT NULL DEFAULT 'unknown' + CHECK (contract_source IN ('unknown', 'configured', 'live_verified')); +ALTER TABLE mesa_target_settings ADD COLUMN health_path TEXT NOT NULL DEFAULT ''; +ALTER TABLE mesa_target_settings ADD COLUMN publish_path TEXT NOT NULL DEFAULT ''; +ALTER TABLE mesa_target_settings ADD COLUMN mutation_status_path_template TEXT NOT NULL DEFAULT ''; + +-- Routes shipped before a local or live HTTP contract was available were assumptions. +-- Do not silently preserve them as if they were verified. +UPDATE mesa_target_settings +SET contract_source = 'unknown', + health_path = '', + publish_path = '', + mutation_status_path_template = ''; + +ALTER TABLE release_items ADD COLUMN version_id TEXT; + +UPDATE release_items +SET version_id = ( + SELECT r.version_id + FROM records r + WHERE r.record_id = release_items.record_id + AND r.record_sha256 = release_items.record_sha256 + ORDER BY r.created_at DESC + LIMIT 1 +) +WHERE version_id IS NULL; + +CREATE INDEX IF NOT EXISTS idx_release_items_version ON release_items(release_id, version_id); diff --git a/pyproject.toml b/pyproject.toml index c173a3c..bf3bfde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ build-backend = "setuptools.build_meta" [dependency-groups] dev = [ + "httpx2>=0.1.0", "mypy>=2.3.0", "pip-audit>=2.10.1", "pytest>=9.1.1", @@ -59,4 +60,3 @@ ignore = ["BLE001", "B008", "DTZ007", "E501", "W293", "W291"] markers = [ "scale: Large-scale or heavy benchmark tests processing thousands of records", ] - diff --git a/schemas/citation.schema.json b/schemas/citation.schema.json index 3c07f0d..da333e1 100644 --- a/schemas/citation.schema.json +++ b/schemas/citation.schema.json @@ -18,9 +18,11 @@ "id": { "type": "string" }, "record_type": { "type": "string", "const": "citation" }, "source_record_id": { "type": "string" }, - "target_legislation_id": { "type": "string" }, + "target_legislation_id": { "type": ["string", "null"] }, "target_article_id": { "type": ["string", "null"] }, "raw_text": { "type": "string" }, + "citation_status": { "type": "string", "enum": ["EXTRACTED", "RESOLVED", "UNRESOLVED", "HUMAN_VERIFIED"] }, + "relation_hint": { "type": ["string", "null"], "enum": ["amends", "repeals", "adds", null] }, "source_span": { "type": ["object", "null"], "properties": { diff --git a/scripts/run_resmi_gazete_pilot.py b/scripts/run_resmi_gazete_pilot.py index 8065d10..a6e8d9f 100644 --- a/scripts/run_resmi_gazete_pilot.py +++ b/scripts/run_resmi_gazete_pilot.py @@ -242,9 +242,8 @@ def run_pilot(): active_after_rollback = cur.fetchone()[0] stg_final.close() print(f"✓ Active release after rollback: {active_after_rollback}") - assert active_after_rollback == baseline_release_id, ( - f"Expected active release {baseline_release_id}, got {active_after_rollback}" - ) + rollback_mismatch = f"Expected active release {baseline_release_id}, got {active_after_rollback}" + assert active_after_rollback == baseline_release_id, rollback_mismatch # Verify provenance after rollback: tracked_record_id is no longer in active release prov_post_rollback = get_record_provenance(tracked_record_id) diff --git a/src/mesa_legal_data/catalog.py b/src/mesa_legal_data/catalog.py index 503e71b..ed71040 100644 --- a/src/mesa_legal_data/catalog.py +++ b/src/mesa_legal_data/catalog.py @@ -367,16 +367,61 @@ def insert_version( validation_status: str, privacy_status: str, approval_status: str, + revision_number: int | None = None, + supersedes_version_id: str | None = None, + quality_status: str | None = "PASS", + quality_json: str | None = None, ): now = datetime.now(UTC).isoformat() with transaction(conn): + cur = conn.cursor() + cur.execute( + """SELECT revision_number, supersedes_version_id, approval_status, + document_id, artifact_id, canonical_sha256 + FROM versions WHERE version_id = ?""", + (version_id,), + ) + existing_ver = cur.fetchone() + + if existing_ver: + if (existing_ver[3], existing_ver[4], existing_ver[5]) != ( + document_id, + artifact_id, + canonical_sha256, + ): + raise CatalogError(f"Immutable version collision for {version_id}: existing identity/content differs") + return + else: + if revision_number is not None: + rev_num = revision_number + else: + cur.execute( + "SELECT COALESCE(MAX(revision_number), 0) + 1 FROM versions WHERE document_id = ?", + (document_id,), + ) + rev_num = cur.fetchone()[0] + + # A predecessor is provenance, not an insertion-order guess. + super_id = supersedes_version_id + + final_approval = approval_status + conn.execute( - """INSERT INTO versions (version_id, document_id, artifact_id, version_kind, snapshot_date, effective_from, effective_to, canonical_path, canonical_line, canonical_sha256, parser_name, parser_version, schema_version, validation_status, privacy_status, approval_status, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """INSERT INTO versions ( + version_id, document_id, artifact_id, version_kind, snapshot_date, + effective_from, effective_to, canonical_path, canonical_line, + canonical_sha256, parser_name, parser_version, schema_version, + validation_status, privacy_status, approval_status, created_at, + revision_number, supersedes_version_id, quality_status, quality_json + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(version_id) DO UPDATE SET document_id = excluded.document_id, artifact_id = excluded.artifact_id, version_kind = excluded.version_kind, + snapshot_date = excluded.snapshot_date, + effective_from = excluded.effective_from, + effective_to = excluded.effective_to, canonical_path = excluded.canonical_path, canonical_line = excluded.canonical_line, canonical_sha256 = excluded.canonical_sha256, @@ -385,7 +430,11 @@ def insert_version( schema_version = excluded.schema_version, validation_status = excluded.validation_status, privacy_status = excluded.privacy_status, - approval_status = excluded.approval_status""", + approval_status = ?, + revision_number = excluded.revision_number, + supersedes_version_id = excluded.supersedes_version_id, + quality_status = excluded.quality_status, + quality_json = excluded.quality_json""", ( version_id, document_id, @@ -402,8 +451,13 @@ def insert_version( schema_version, validation_status, privacy_status, - approval_status, + final_approval, now, + rev_num, + super_id, + quality_status, + quality_json, + final_approval, ), ) @@ -411,7 +465,13 @@ def insert_version( def get_version(conn: sqlite3.Connection, version_id: str) -> dict[str, Any] | None: cursor = conn.cursor() cursor.execute( - "SELECT version_id, document_id, artifact_id, version_kind, snapshot_date, effective_from, effective_to, canonical_path, canonical_line, canonical_sha256, parser_name, parser_version, schema_version, validation_status, privacy_status, approval_status, created_at FROM versions WHERE version_id = ?", + """SELECT version_id, document_id, artifact_id, version_kind, snapshot_date, + effective_from, effective_to, canonical_path, canonical_line, + canonical_sha256, parser_name, parser_version, schema_version, + validation_status, privacy_status, approval_status, created_at, + revision_number, supersedes_version_id, quality_status, quality_json, + is_audit_sample, audit_sample_reason, auto_approved + FROM versions WHERE version_id = ?""", (version_id,), ) row = cursor.fetchone() @@ -435,6 +495,13 @@ def get_version(conn: sqlite3.Connection, version_id: str) -> dict[str, Any] | N "privacy_status": row[14], "approval_status": row[15], "created_at": row[16], + "revision_number": row[17] if len(row) > 17 and row[17] is not None else 1, + "supersedes_version_id": row[18] if len(row) > 18 else None, + "quality_status": row[19] if len(row) > 19 else None, + "quality_json": row[20] if len(row) > 20 else None, + "is_audit_sample": bool(row[21]) if len(row) > 21 and row[21] is not None else False, + "audit_sample_reason": row[22] if len(row) > 22 else None, + "auto_approved": bool(row[23]) if len(row) > 23 and row[23] is not None else False, } @@ -448,21 +515,27 @@ def insert_record( record_sha256: str, validation_status: str = "valid", approval_status: str = "pending", + record_instance_id: str | None = None, ): now = datetime.now(UTC).isoformat() + instance_id = record_instance_id or f"{version_id}:{record_id}" with transaction(conn): conn.execute( - """INSERT INTO records (record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(record_id) DO UPDATE SET - version_id = excluded.version_id, + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(version_id, record_id) DO UPDATE SET + record_instance_id = excluded.record_instance_id, record_type = excluded.record_type, canonical_path = excluded.canonical_path, canonical_line = excluded.canonical_line, record_sha256 = excluded.record_sha256, validation_status = excluded.validation_status, - approval_status = excluded.approval_status""", + approval_status = CASE + WHEN records.approval_status = 'approved' THEN records.approval_status + ELSE excluded.approval_status + END""", ( + instance_id, record_id, version_id, record_type, @@ -476,12 +549,38 @@ def insert_record( ) -def get_record(conn: sqlite3.Connection, record_id: str) -> dict[str, Any] | None: +def get_record(conn: sqlite3.Connection, record_id: str, version_id: str | None = None) -> dict[str, Any] | None: cursor = conn.cursor() - cursor.execute( - "SELECT record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at FROM records WHERE record_id = ?", - (record_id,), - ) + if version_id: + cursor.execute( + "SELECT record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at, record_instance_id FROM records WHERE record_id = ? AND version_id = ?", + (record_id, version_id), + ) + else: + cursor.execute( + "SELECT record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at, record_instance_id FROM records WHERE record_instance_id = ?", + (record_id,), + ) + row = cursor.fetchone() + if row: + return { + "record_id": row[0], + "version_id": row[1], + "record_type": row[2], + "canonical_path": row[3], + "canonical_line": row[4], + "record_sha256": row[5], + "validation_status": row[6], + "approval_status": row[7], + "created_at": row[8], + "record_instance_id": row[9], + } + + cursor.execute( + "SELECT record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at, record_instance_id FROM records WHERE record_id = ? ORDER BY created_at DESC LIMIT 1", + (record_id,), + ) + row = cursor.fetchone() if not row: return None @@ -495,6 +594,7 @@ def get_record(conn: sqlite3.Connection, record_id: str) -> dict[str, Any] | Non "validation_status": row[6], "approval_status": row[7], "created_at": row[8], + "record_instance_id": row[9] if len(row) > 9 else f"{row[1]}:{row[0]}", } @@ -547,18 +647,26 @@ def iter_records_for_release( batch_size: int = 1000, ) -> Iterator[ReleaseRecordRef]: cursor = conn.cursor() - cursor.execute( - """ + cursor.execute(""" + WITH eligible_versions AS ( + SELECT v.version_id, + ROW_NUMBER() OVER ( + PARTITION BY v.document_id + ORDER BY COALESCE(v.revision_number, 1) DESC, v.created_at DESC + ) AS version_rank + FROM versions v + WHERE v.approval_status = 'approved' + AND v.validation_status = 'valid' + AND (v.quality_status IS NULL OR v.quality_status != 'BLOCK') + AND v.privacy_status IN ('clean', 'approved') + ) SELECT r.record_id, r.record_type, r.record_sha256, r.canonical_path, r.canonical_line, r.version_id FROM records r - JOIN versions v ON r.version_id = v.version_id + JOIN eligible_versions ev ON r.version_id = ev.version_id AND ev.version_rank = 1 WHERE r.approval_status = 'approved' AND r.validation_status = 'valid' - AND v.validation_status = 'valid' - AND v.privacy_status IN ('clean', 'approved') ORDER BY r.canonical_path ASC, r.canonical_line ASC - """ - ) + """) while True: rows = cursor.fetchmany(batch_size) if not rows: @@ -933,12 +1041,18 @@ def mark_release_status( ) -def add_release_item(conn: sqlite3.Connection, release_id: str, record_id: str, record_sha256: str): +def add_release_item( + conn: sqlite3.Connection, + release_id: str, + record_id: str, + record_sha256: str, + version_id: str | None = None, +): with transaction(conn): conn.execute( - """INSERT INTO release_items (release_id, record_id, record_sha256) - VALUES (?, ?, ?)""", - (release_id, record_id, record_sha256), + """INSERT INTO release_items (release_id, record_id, record_sha256, version_id) + VALUES (?, ?, ?, ?)""", + (release_id, record_id, record_sha256, version_id), ) @@ -983,6 +1097,9 @@ def approve_version_streaming( if not ver: raise CatalogError(f"Version {version_id} not found") + if ver.get("quality_status") == "BLOCK": + raise BlockingValidationIssueExists(f"Cannot approve version {version_id}: quality gate status is BLOCK") + blockers = list_open_blocking_issues_for_version(conn, version_id=version_id) if blockers: raise BlockingValidationIssueExists( @@ -1047,12 +1164,7 @@ def approve_version_streaming( spool_conn.close() if spool_db_path.exists(): spool_db_path.unlink() - return { - "status": "approved", - "version_id": version_id, - "approved_records": 0, - "approval_status": "approved", - } + raise CatalogError(f"Cannot approve version {version_id}: it has no canonical records") # Get distinct canonical paths p_cur = spool_conn.cursor() @@ -1131,11 +1243,11 @@ def approve_version_streaming( "INSERT INTO record_reviews (record_id, record_sha256, reviewer, decision, note, reviewed_at) VALUES (?, ?, ?, ?, ?, ?)", rows, ) - id_tuples = [(r[0],) for r in rows] - conn.executemany( - "UPDATE records SET approval_status = 'approved' WHERE record_id = ?", - id_tuples, - ) + + conn.execute( + "UPDATE records SET approval_status = 'approved' WHERE version_id = ?", + (version_id,), + ) conn.execute( "UPDATE versions SET approval_status = 'approved', privacy_status = CASE WHEN privacy_status = 'flagged' THEN 'approved' ELSE privacy_status END WHERE version_id = ?", @@ -1843,3 +1955,294 @@ def list_export_packages(conn: sqlite3.Connection, limit: int = 50) -> list[dict } ) return items + + +# ========================================== +# Operational Source Settings & Certifications +# ========================================== + + +def get_source_operational_settings(conn: sqlite3.Connection, source_id: str) -> dict[str, Any]: + cursor = conn.cursor() + cursor.execute( + "SELECT source_id, enabled, auto_approval_enabled, weekly_sample_count, updated_at, updated_by FROM source_operational_settings WHERE source_id = ?", + (source_id,), + ) + row = cursor.fetchone() + if row: + return { + "source_id": row[0], + "enabled": bool(row[1]), + "auto_approval_enabled": bool(row[2]), + "weekly_sample_count": row[3], + "updated_at": row[4], + "updated_by": row[5], + } + return { + "source_id": source_id, + "enabled": True, + "auto_approval_enabled": False, + "weekly_sample_count": 10, + "updated_at": datetime.now(UTC).isoformat(), + "updated_by": "default", + } + + +def upsert_source_operational_settings( + conn: sqlite3.Connection, + source_id: str, + *, + enabled: bool = True, + auto_approval_enabled: bool = False, + weekly_sample_count: int = 10, + updated_by: str = "operator", +) -> None: + now_iso = datetime.now(UTC).isoformat() + with transaction(conn): + conn.execute( + """INSERT INTO source_operational_settings (source_id, enabled, auto_approval_enabled, weekly_sample_count, updated_at, updated_by) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(source_id) DO UPDATE SET + enabled = excluded.enabled, + auto_approval_enabled = excluded.auto_approval_enabled, + weekly_sample_count = excluded.weekly_sample_count, + updated_at = excluded.updated_at, + updated_by = excluded.updated_by""", + ( + source_id, + 1 if enabled else 0, + 1 if auto_approval_enabled else 0, + weekly_sample_count, + now_iso, + updated_by, + ), + ) + + +def list_source_operational_settings(conn: sqlite3.Connection) -> list[dict[str, Any]]: + cursor = conn.cursor() + cursor.execute("SELECT source_id, name, authority, base_url, enabled FROM sources ORDER BY source_id ASC") + sources = cursor.fetchall() + + results = [] + for s_id, s_name, s_auth, s_url, s_enabled in sources: + settings = get_source_operational_settings(conn, s_id) + certs = list_parser_certifications(conn, source_id=s_id) + results.append( + { + "source_id": s_id, + "name": s_name, + "authority": s_auth, + "base_url": s_url, + "source_enabled": bool(s_enabled) and settings["enabled"], + "auto_approval_enabled": settings["auto_approval_enabled"], + "weekly_sample_count": settings["weekly_sample_count"], + "updated_at": settings["updated_at"], + "certifications": certs, + } + ) + return results + + +def get_parser_certification( + conn: sqlite3.Connection, + source_id: str, + parser_name: str, + parser_version: str, +) -> bool: + cursor = conn.cursor() + cursor.execute( + "SELECT certified FROM parser_certifications WHERE source_id = ? AND parser_name = ? AND parser_version = ?", + (source_id, parser_name, parser_version), + ) + row = cursor.fetchone() + if row: + return bool(row[0]) + return False + + +def set_parser_certification( + conn: sqlite3.Connection, + source_id: str, + parser_name: str, + parser_version: str, + certified: bool, + certified_by: str = "operator", +) -> None: + now_iso = datetime.now(UTC).isoformat() + with transaction(conn): + conn.execute( + """INSERT INTO parser_certifications (source_id, parser_name, parser_version, certified, certified_at, certified_by) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(source_id, parser_name, parser_version) DO UPDATE SET + certified = excluded.certified, + certified_at = excluded.certified_at, + certified_by = excluded.certified_by""", + ( + source_id, + parser_name, + parser_version, + 1 if certified else 0, + now_iso, + certified_by, + ), + ) + + +def list_parser_certifications( + conn: sqlite3.Connection, + source_id: str | None = None, +) -> list[dict[str, Any]]: + cursor = conn.cursor() + if source_id: + cursor.execute( + "SELECT source_id, parser_name, parser_version, certified, certified_at, certified_by FROM parser_certifications WHERE source_id = ? ORDER BY parser_name, parser_version", + (source_id,), + ) + else: + cursor.execute( + "SELECT source_id, parser_name, parser_version, certified, certified_at, certified_by FROM parser_certifications ORDER BY source_id, parser_name, parser_version" + ) + return [ + { + "source_id": r[0], + "parser_name": r[1], + "parser_version": r[2], + "certified": bool(r[3]), + "certified_at": r[4], + "certified_by": r[5], + } + for r in cursor.fetchall() + ] + + +def should_audit_sample(conn: sqlite3.Connection, source_id: str, version_id: str) -> bool: + """ + Determines if a version should be selected for manual quality audit sampling. + Enforces that: + - Already sampled versions maintain their sampling state. + - Sampling rate obeys weekly_sample_count. + """ + ver = get_version(conn, version_id) + if ver and ver.get("is_audit_sample"): + return True + + settings = get_source_operational_settings(conn, source_id) + weekly_limit = settings.get("weekly_sample_count", 10) + if weekly_limit <= 0: + return False + + cursor = conn.cursor() + cursor.execute( + """SELECT count(*) FROM versions v + JOIN artifacts a ON v.artifact_id = a.artifact_id + WHERE a.source_id = ? + AND v.is_audit_sample = 1 + AND v.created_at >= datetime('now', '-7 days')""", + (source_id,), + ) + current_weekly_samples = cursor.fetchone()[0] + + if current_weekly_samples < weekly_limit: + # Deterministic sample rate modulo on version ID hash + int_hash = int(hashlib.sha256(version_id.encode("utf-8")).hexdigest(), 16) + rate = min(100, max(1, weekly_limit)) + if (int_hash % 100) < rate: + return True + + return False + + +def evaluate_auto_approval( + conn: sqlite3.Connection, + *, + version_id: str, + source_id: str, + parser_name: str, + parser_version: str, + quality_decision: str, + has_privacy_blocker: bool, + schema_valid: bool, +) -> tuple[bool, str]: + """ + Evaluates strict auto-approval conditions for a version: + 1. quality_decision == 'PASS' + 2. No privacy blockers + 3. Schema valid + 4. Source operationally enabled + 5. Auto-approval enabled for source + 6. Parser name and version certified for this source + 7. Not selected for audit sampling + + Returns (is_approved, explanation_reason). + """ + ver = get_version(conn, version_id) + if not ver: + return False, f"Version {version_id} does not exist" + + cursor = conn.cursor() + cursor.execute("SELECT source_id FROM artifacts WHERE artifact_id = ?", (ver["artifact_id"],)) + source_row = cursor.fetchone() + actual_source_id = source_row[0] if source_row else None + actual_schema_valid = ver.get("validation_status") == "valid" + actual_privacy_blocker = ver.get("privacy_status") not in ("clean", "approved") + + supplied_contract = (source_id, parser_name, parser_version, quality_decision, schema_valid, has_privacy_blocker) + stored_contract = ( + actual_source_id, + ver.get("parser_name"), + ver.get("parser_version"), + ver.get("quality_status"), + actual_schema_valid, + actual_privacy_blocker, + ) + if supplied_contract != stored_contract: + return False, "Auto-approval inputs do not match the stored version provenance and gate state" + + if ver.get("quality_status") != "PASS": + return False, f"Quality Gate decision is {ver.get('quality_status')} (requires PASS)" + + if has_privacy_blocker: + return False, "Privacy blocker detected in content" + + if not schema_valid: + return False, "Schema validation failed" + + source_id = str(actual_source_id) + parser_name = str(ver["parser_name"]) + parser_version = str(ver["parser_version"]) + + settings = get_source_operational_settings(conn, source_id) + if not settings.get("enabled", True): + return False, f"Source {source_id} is operationally disabled" + + if not settings.get("auto_approval_enabled", False): + return False, "Auto-approval is disabled for source" + + is_certified = get_parser_certification(conn, source_id, parser_name, parser_version) + if not is_certified: + return False, f"Parser {parser_name} v{parser_version} is not certified for source {source_id}" + + is_sampled = should_audit_sample(conn, source_id, version_id) + if is_sampled: + with transaction(conn): + conn.execute( + "UPDATE versions SET is_audit_sample = 1, audit_sample_reason = 'quality_audit_sample' WHERE version_id = ?", + (version_id,), + ) + return False, "Selected for audit quality sampling (requires manual review)" + + # All safety gates passed: Execute streaming auto-approval + approve_version_streaming( + conn, + version_id=version_id, + reviewer="system:auto-approval", + note="Safe auto-approval: quality PASS, parser certified, not sampled", + ) + with transaction(conn): + conn.execute( + "UPDATE versions SET auto_approved = 1, approval_status = 'approved' WHERE version_id = ?", + (version_id,), + ) + + return True, "Auto-approved successfully" diff --git a/src/mesa_legal_data/cli.py b/src/mesa_legal_data/cli.py index 851dab3..1d4eb84 100644 --- a/src/mesa_legal_data/cli.py +++ b/src/mesa_legal_data/cli.py @@ -558,9 +558,66 @@ def web_cmd( typer.secho( f"Starting MESA Web Admin on http://{host}:{port} ...", fg=typer.colors.GREEN, - bold=True, ) - uvicorn.run("mesa_legal_data.web.app:create_app", factory=True, host=host, port=port, reload=reload) + uvicorn.run("mesa_legal_data.web.app:create_app", host=host, port=port, reload=reload, factory=True) + + +@app.command("sync") +def sync_cmd( + source_id: str = typer.Option("resmi_gazete", "--source", help="Source ID to synchronize"), + max_items: int = typer.Option(20, "--max-items", help="Maximum items to fetch and process"), +): + """ + Executes idempotent synchronization pipeline: + discover -> fetch -> process -> quality -> auto approve -> sample queue -> ready. + Stops before MESA push (does NOT push to MESA). + """ + from mesa_legal_data.catalog import get_connection + from mesa_legal_data.catalog import migrate as migrate_catalog + from mesa_legal_data.harvest.config import load_harvest_config + from mesa_legal_data.harvest.migrations import apply_harvest_migrations + from mesa_legal_data.harvest.service import run_collection_until_pause + + if max_items < 1: + raise typer.BadParameter("--max-items must be at least 1") + + migrate_catalog() + apply_harvest_migrations() + + typer.secho("=== Starting MESA Legal Data Synchronization Pipeline ===", fg=typer.colors.CYAN, bold=True) + typer.echo(f"Source: {source_id} | Max items limit: {max_items}") + + cfg = load_harvest_config() + if source_id in cfg.sources: + cfg.sources[source_id].budget.new_urls_per_run = max_items + cfg.runner.batch_size = max_items + + res = run_collection_until_pause( + source_id=source_id, + harvest_cfg=cfg, + max_processed_items=max_items, + ) + + # Compute summary + conn = get_connection() + c = conn.cursor() + c.execute("SELECT count(*) FROM versions WHERE approval_status = 'approved'") + approved_count = c.fetchone()[0] + c.execute("SELECT count(*) FROM versions WHERE approval_status = 'pending'") + pending_count = c.fetchone()[0] + c.execute("SELECT count(*) FROM versions WHERE auto_approved = 1") + auto_approved_count = c.fetchone()[0] + conn.close() + + typer.secho("\n=== Sync Summary ===", fg=typer.colors.GREEN, bold=True) + typer.echo(f"Items processed in batch : {res.get('processed', 0)}") + typer.echo(f"Auto-approved (all-time) : {auto_approved_count}") + typer.echo(f"Ready for MESA (approved): {approved_count}") + typer.echo(f"Needs Review : {pending_count}") + typer.secho( + "Note: Pipeline stopped before MESA push (MESA push must be triggered manually via Web Panel).", + fg=typer.colors.YELLOW, + ) if __name__ == "__main__": diff --git a/src/mesa_legal_data/harvest/runner.py b/src/mesa_legal_data/harvest/runner.py index d9eba67..df832ac 100644 --- a/src/mesa_legal_data/harvest/runner.py +++ b/src/mesa_legal_data/harvest/runner.py @@ -268,9 +268,11 @@ def _run_harvest_batch_impl( started_at=pipe_start_iso, finished_at=pipe_finish_iso, result="succeeded" if pipe_success else "failed", - error_code=None - if pipe_success - else ("PIPELINE_FAILED" if pipe_res.status == "failed" else "UNEXPECTED_PIPELINE_STATUS"), + error_code=( + None + if pipe_success + else ("PIPELINE_FAILED" if pipe_res.status == "failed" else "UNEXPECTED_PIPELINE_STATUS") + ), error_message=None if pipe_success else f"Pipeline status: {pipe_res.status}", artifact_id=artifact_id, db_path=db_path, diff --git a/src/mesa_legal_data/harvest/service.py b/src/mesa_legal_data/harvest/service.py index d75f7f5..536024c 100644 --- a/src/mesa_legal_data/harvest/service.py +++ b/src/mesa_legal_data/harvest/service.py @@ -214,6 +214,7 @@ def run_collection_until_pause( is_cancelled_cb: Callable[[], bool] | None = None, progress_cb: Callable[[dict[str, Any]], None] | None = None, max_loops: int = 50, + max_processed_items: int | None = None, ) -> dict[str, Any]: """ Loops through discovery and worker batches safely until a stopping/pause condition is met. @@ -238,6 +239,10 @@ def run_collection_until_pause( stopped_reason = None for _ in range(max_loops): + if max_processed_items is not None and total_processed >= max_processed_items: + status = "paused_item_limit" + stopped_reason = "ITEM_LIMIT_REACHED" + break # 1. Cancellation check before discovery if is_cancelled_cb and is_cancelled_cb(): status = "cancelled" @@ -294,6 +299,11 @@ def run_collection_until_pause( total_duplicate += dup batch_loop_processed += p + if max_processed_items is not None and total_processed >= max_processed_items: + status = "paused_item_limit" + stopped_reason = "ITEM_LIMIT_REACHED" + break + if progress_cb and p > 0: progress_cb( { @@ -345,6 +355,7 @@ def run_collection_until_pause( "target_reached", "paused_daily_limit", "paused_safety", + "paused_item_limit", ): break diff --git a/src/mesa_legal_data/ids.py b/src/mesa_legal_data/ids.py index 9e8c80b..59aa0a1 100644 --- a/src/mesa_legal_data/ids.py +++ b/src/mesa_legal_data/ids.py @@ -58,7 +58,7 @@ def build_decision_id( return f"tr:case-law:{safe_court}:unknown:sha256-{artifact_sha256[:16]}" -def build_citation_id(source_record_id: str, start: int, end: int, target_id: str) -> str: - raw = f"{source_record_id}:{start}:{end}:{target_id}" +def build_citation_id(source_record_id: str, start: int, end: int, target_id: str | None) -> str: + raw = f"{source_record_id}:{start}:{end}:{target_id or 'unresolved'}" digest = hashlib.sha256(raw.encode("utf-8")).hexdigest() return f"citation:sha256:{digest}" diff --git a/src/mesa_legal_data/operations.py b/src/mesa_legal_data/operations.py index 48106ec..ceb900d 100644 --- a/src/mesa_legal_data/operations.py +++ b/src/mesa_legal_data/operations.py @@ -20,6 +20,7 @@ "release_build", "integrity_audit", "harvest_collection", + "mesa_v4_delivery", } _cancelled_jobs: set[str] = set() @@ -203,6 +204,41 @@ def progress_cb(prog: dict[str, Any]) -> None: progress_current=100, result_json=json.dumps(res), ) + elif op_type == "mesa_v4_delivery": + from mesa_legal_data.publisher.engine import execute_publish_delivery + + del_id = inp.get("delivery_id") + rel_id = inp.get("release_id") + target_key = inp.get("target_key", "default") + + def progress_cb(prog: dict[str, Any]) -> None: + if not is_cancelled(operation_id): + tot = max(1, prog.get("total", 1)) + proc = prog.get("processed", 0) + pct = int((proc / tot) * 100) + update_operation_job( + conn, + operation_id, + status="running", + progress_current=min(99, max(5, pct)), + ) + + res = execute_publish_delivery( + delivery_id=del_id, + release_id=rel_id, + target_key=target_key, + progress_callback=progress_cb, + ) + + status_str = "succeeded" if res.get("status") in ("COMMITTED", "PARTIAL") else "failed" + update_operation_job( + conn, + operation_id, + status=status_str, + progress_current=100, + result_json=json.dumps(res), + error_summary=res.get("last_error"), + ) else: raise ValueError(f"OPERATION_TYPE_NOT_SUPPORTED: Operation type '{op_type}' is not supported") except Exception as exc: diff --git a/src/mesa_legal_data/parsers/citations.py b/src/mesa_legal_data/parsers/citations.py index 6cecf20..4a85103 100644 --- a/src/mesa_legal_data/parsers/citations.py +++ b/src/mesa_legal_data/parsers/citations.py @@ -1,103 +1,221 @@ import re -import unicodedata +from typing import Literal from pydantic import BaseModel, ConfigDict -LAW_ALIASES = { - "TURK MEDENI KANUNU": ("4721", "law"), - "MEDENI KANUN": ("4721", "law"), - "TMK": ("4721", "law"), - "TURK BORCLAR KANUNU": ("6098", "law"), - "BORCLAR KANUNU": ("6098", "law"), - "TBK": ("6098", "law"), - "TURK CEZA KANUNU": ("5237", "law"), - "TCK": ("5237", "law"), - "CEZA MUHAKEMESI KANUNU": ("5271", "law"), - "CMK": ("5271", "law"), - "HUKUK MUHAKEMELERI KANUNU": ("6100", "law"), - "HMK": ("6100", "law"), - "ICRA VE IFLAS KANUNU": ("2004", "law"), - "IIK": ("2004", "law"), - "IS KANUNU": ("4857", "law"), - "TURK TICARET KANUNU": ("6102", "law"), - "TTK": ("6102", "law"), - "ANAYASA": ("2709", "constitution"), -} +CitationStatus = Literal["EXTRACTED", "RESOLVED", "UNRESOLVED", "HUMAN_VERIFIED"] +RelationHint = Literal["amends", "repeals", "adds"] + +# Legal Code Definitions: (Canonical Name, Alias, Law Number, Doc Family/Type) +LEGAL_CODES = [ + ("TÜRK BORÇLAR KANUNU", "TBK", "6098", "law"), + ("TURK BORCLAR KANUNU", "TBK", "6098", "law"), + ("BORÇLAR KANUNU", "BK", "6098", "law"), + ("BORCLAR KANUNU", "BK", "6098", "law"), + ("TÜRK CEZA KANUNU", "TCK", "5237", "law"), + ("TURK CEZA KANUNU", "TCK", "5237", "law"), + ("CEZA MUHAKEMESİ KANUNU", "CMK", "5271", "law"), + ("CEZA MUHAKEMESI KANUNU", "CMK", "5271", "law"), + ("CEZA MUHAKEMELERİ USULÜ KANUNU", "CMUK", "1412", "law"), + ("HUKUK MUHAKEMELERİ KANUNU", "HMK", "6100", "law"), + ("HUKUK MUHAKEMELERI KANUNU", "HMK", "6100", "law"), + ("HUKUK USULÜ MUHAKEMELERİ KANUNU", "HUMK", "1086", "law"), + ("TÜRK MEDENİ KANUNU", "TMK", "4721", "law"), + ("TURK MEDENI KANUNU", "TMK", "4721", "law"), + ("MEDENİ KANUN", "MK", "4721", "law"), + ("MEDENI KANUN", "MK", "4721", "law"), + ("TÜRK TİCARET KANUNU", "TTK", "6102", "law"), + ("TURK TICARET KANUNU", "TTK", "6102", "law"), + ("TİCARET KANUNU", "TK", "6102", "law"), + ("İCRA VE İFLAS KANUNU", "İİK", "2004", "law"), + ("ICRA VE IFLAS KANUNU", "IIK", "2004", "law"), + ("İŞ KANUNU", "İŞK", "4857", "law"), + ("IS KANUNU", "ISK", "4857", "law"), + ("İDARİ YARGILAMA USULÜ KANUNU", "İYUK", "2577", "law"), + ("IDARI YARGILAMA USULU KANUNU", "IYUK", "2577", "law"), + ("VERGİ USUL KANUNU", "VUK", "213", "law"), + ("VERGI USUL KANUNU", "VUK", "213", "law"), + ("KİŞİSEL VERİLERİN KORUNMASI KANUNU", "KVKK", "6698", "law"), + ("KISISEL VERILERIN KORUNMASI KANUNU", "KVK", "6698", "law"), + ("AVUKATLIK KANUNU", "AVK", "1136", "law"), + ("POLİS VAZİFE VE SALÂHİYET KANUNU", "PVSK", "2559", "law"), + ("POLIS VAZIFE VE SALAHIYET KANUNU", "PVSK", "2559", "law"), + ("TÜRKİYE CUMHURİYETİ ANAYASASI", "AY", "2709", "constitution"), + ("TURKIYE CUMHURIYETI ANAYASASI", "AY", "2709", "constitution"), + ("ANAYASA", None, "2709", "constitution"), +] + +# Build lookup map for aliases and canonical names +ALIAS_MAP: dict[str, tuple[str, str]] = {} +for name, alias, num, kind in LEGAL_CODES: + ALIAS_MAP[name.upper()] = (num, kind) + if alias: + ALIAS_MAP[alias.upper()] = (num, kind) + +KNOWN_LEGISLATION_NUMBERS = {num for _, _, num, _ in LEGAL_CODES} class Citation(BaseModel): model_config = ConfigDict(frozen=True) raw_text: str - target_legislation_id: str + target_legislation_id: str | None = None target_article_id: str | None = None char_start: int | None = None char_end: int | None = None - - -def _to_ascii_upper(s: str) -> str: - tr_map = str.maketrans("ğüşöçığĞÜŞÖÇİI", "gusociigusoCII") - s_tr = s.translate(tr_map) - return unicodedata.normalize("NFKD", s_tr).encode("ASCII", "ignore").decode("utf-8").upper() + citation_status: CitationStatus = "EXTRACTED" + relation_hint: RelationHint | None = None + + +# Pattern 1: Numbered Law Citations with explicit context +# e.g. "4857 sayılı Kanun'un 1. maddesi", "6698 sayılı Kişisel Verilerin Korunması Kanunu m. 9", "7500 sayılı Kanun 3. maddesi" +NUMBERED_LAW_PATTERN = re.compile( + r"\b(?P[1-9]\d{0,4})\s+sayılı\s+(?:[A-Za-zÇĞİÖŞÜçğıöşü\s]{0,50}?)(?:Kanun(?:u|un|una|unda|undan|a|da|dan|lar|ları)?|KHK(?:'nin|'ye|'de)?|Cumhurbaşkanlığı\s+Kararnamesi(?:'nin)?|Yasa(?:sı|nın)?)\b(?:\s*['’][A-Za-zçğıöşüÇĞİÖŞÜ]+)?(?:\s+(?:(?:(?:ek|geçici)\s+)?(?:madde|maddesi|m\.)\s*\.?\s*(?P\d+|[A-ZÇĞİÖŞÜ]+)|(?P\d+)(?:\.|\s*['’]?(?:nci|üncü|inci|ıncı|uncu))?\s*(?:maddesi|madde)|(?P\d+)\b))?", + re.IGNORECASE, +) + +# Pattern 2: Named Law / Code Alias Citations +# e.g. "Türk Borçlar Kanunu'nun 117. maddesi", "TMK m. 2", "TCK 53", "HMK 119", "İş Kanunu 25/II" +ALIAS_NAMES = sorted(ALIAS_MAP.keys(), key=lambda x: len(x), reverse=True) +ALIAS_PATTERN_STR = "|".join(re.escape(k) for k in ALIAS_NAMES) + +ALIAS_LAW_PATTERN = re.compile( + rf"\b(?P{ALIAS_PATTERN_STR})\b(?:\s*['’][A-Za-zçğıöşüÇĞİÖŞÜ]+)?(?:\s+(?:(?:(?:ek|geçici)\s+)?(?:madde|maddesi|m\.)\s*\.?\s*(?P\d+|[A-ZÇĞİÖŞÜ]+)|(?P\d+)(?:\.|\s*['’]?(?:nci|üncü|inci|ıncı|uncu))?\s*(?:maddesi|madde)|(?P\d+)\b))?", + re.IGNORECASE, +) + +# Pattern 3: Unresolved candidates (generic references without specific number/code) +UNRESOLVED_CANDIDATE_PATTERN = re.compile( + r"\b(?:ilgili|anılan|mezkûr|söz\s+konusu)\s+(?:Kanun(?:u|un|a|da)?|maddesi|hükmü)\b", + re.IGNORECASE, +) + +# Temporal relation patterns (Deterministic legal modification verbs) +AMENDS_PATTERN = re.compile( + r"(?:değiştirilmiştir|değişiklik\s+yapılmıştır|değişen\s+şekliyle|şeklinde\s+değiştirilmiş)", + re.IGNORECASE, +) +REPEALS_PATTERN = re.compile( + r"(?:yürürlükten\s+kaldırılmıştır|mülga\s+kılınmıştır|mülga\s+edilmiştir|iptal\s+edilmiştir|\bmülga\b)", + re.IGNORECASE, +) +ADDS_PATTERN = re.compile( + r"(?:eklenmiştir|ihdas\s+edilmiştir|ilave\s+edilmiştir)", + re.IGNORECASE, +) + + +def _detect_relation_hint(context_window: str) -> RelationHint | None: + if AMENDS_PATTERN.search(context_window): + return "amends" + if REPEALS_PATTERN.search(context_window): + return "repeals" + if ADDS_PATTERN.search(context_window): + return "adds" + return None def extract_citations(text: str) -> list[Citation]: + """ + Extracts deterministic legal citations and references with exact canonical coordinate spans. + Enforces the citation lifecycle: + - RESOLVED: Target document / article reliably mapped. + - UNRESOLVED: Citation candidate found but target ambiguous/unspecified. + - EXTRACTED: Raw candidate state before resolution. + """ if not text: return [] citations: list[Citation] = [] - ascii_text = _to_ascii_upper(text) - - # 1. Matching law numbers (e.g. 4721 sayılı Kanun'un 1. maddesi) - num_pattern = re.compile( - r"\b(?P4721|6098|5237|5271|6100|2004|4857|6102|2709)\b(?:\s*(?:SAYILI|S\.)?\s*(?:KANUN|K\.)?\s*(?:'[A-Z]+)?)?\s*(?:MADDESI|MADDE|M\.)?\s*\.?\s*(?P\d+)?", - re.IGNORECASE, - ) + seen_spans: set[tuple[int, int]] = set() - for match in num_pattern.finditer(ascii_text): + # 1. Numbered Law Citations + for match in NUMBERED_LAW_PATTERN.finditer(text): + start, end = match.span() + raw = text[start:end] law_num = match.group("law_num") - art_num = match.group("art_num") + art_num = match.group("art_num_m") or match.group("art_num_num") or match.group("art_num_bare") + leg_type = "constitution" if law_num == "2709" else "law" leg_id = f"tr:legislation:{leg_type}:{law_num}" art_id = f"{leg_id}:article:{art_num}" if art_num else None - start, end = match.span() + window_start = max(0, start - 50) + window_end = min(len(text), end + 100) + relation = _detect_relation_hint(text[window_start:window_end]) + citations.append( Citation( - raw_text=text[start:end], + raw_text=raw, target_legislation_id=leg_id, target_article_id=art_id, char_start=start, char_end=end, + # A syntactically valid number is only a resolution candidate + # unless it belongs to the deterministic legislation registry. + citation_status=("RESOLVED" if law_num in KNOWN_LEGISLATION_NUMBERS else "EXTRACTED"), + relation_hint=relation, ) ) + seen_spans.add((start, end)) - # 2. Matching alias-based citations (e.g. Türk Borçlar Kanunu'nun 117. maddesi, TMK m. 2) - alias_pattern = re.compile( - r"\b(?PTURK MEDENI KANUNU|MEDENI KANUN|TMK|TURK BORCLAR KANUNU|BORCLAR KANUNU|TBK|TURK CEZA KANUNU|TCK|CEZA MUHAKEMESI KANUNU|CMK|HUKUK MUHAKEMELERI KANUNU|HMK|ICRA VE IFLAS KANUNU|IIK|IS KANUNU|TURK TICARET KANUNU|TTK|ANAYASA)\b(?:\s*'[A-Z]+)?\s*(?:MADDESI|MADDE|M\.)?\s*\.?\s*(?P\d+)?", - re.IGNORECASE, - ) - - for match in alias_pattern.finditer(ascii_text): - alias_key = match.group("alias").upper() - if alias_key in LAW_ALIASES: - law_num, leg_type = LAW_ALIASES[alias_key] - art_num = match.group("art_num") - leg_id = f"tr:legislation:{leg_type}:{law_num}" - art_id = f"{leg_id}:article:{art_num}" if art_num else None + # 2. Alias-based Code Citations + for match in ALIAS_LAW_PATTERN.finditer(text): + start, end = match.span() + if any(s <= start < e or s < end <= e for s, e in seen_spans): + continue + + raw = text[start:end] + alias_key = match.group("alias").strip().upper() + raw_alias = match.group("alias").strip() + if len(raw_alias) <= 3 and raw_alias != raw_alias.upper(): + # Avoid resolving ordinary Turkish words such as "ay" as the AY alias. + continue + law_tuple = ALIAS_MAP.get(alias_key) + if not law_tuple: + continue + + law_num, leg_type = law_tuple + art_num = match.group("art_num_m") or match.group("art_num_num") or match.group("art_num_bare") + leg_id = f"tr:legislation:{leg_type}:{law_num}" + art_id = f"{leg_id}:article:{art_num}" if art_num else None - start, end = match.span() - raw = text[start:end] + window_start = max(0, start - 50) + window_end = min(len(text), end + 100) + relation = _detect_relation_hint(text[window_start:window_end]) - cit = Citation( + citations.append( + Citation( raw_text=raw, target_legislation_id=leg_id, target_article_id=art_id, char_start=start, char_end=end, + citation_status="RESOLVED", + relation_hint=relation, ) + ) + seen_spans.add((start, end)) - if cit not in citations: - citations.append(cit) + # 3. Unresolved Candidates + for match in UNRESOLVED_CANDIDATE_PATTERN.finditer(text): + start, end = match.span() + if any(s <= start < e or s < end <= e for s, e in seen_spans): + continue + + raw = text[start:end] + citations.append( + Citation( + raw_text=raw, + target_legislation_id=None, + target_article_id=None, + char_start=start, + char_end=end, + citation_status="UNRESOLVED", + relation_hint=None, + ) + ) + seen_spans.add((start, end)) + citations.sort(key=lambda c: (c.char_start or 0, c.char_end or 0)) return citations diff --git a/src/mesa_legal_data/parsers/coverage.py b/src/mesa_legal_data/parsers/coverage.py new file mode 100644 index 0000000..ccf5241 --- /dev/null +++ b/src/mesa_legal_data/parsers/coverage.py @@ -0,0 +1,108 @@ +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class ParsingCoverage: + canonical_chars: int + covered_chars: int + uncovered_chars: int + coverage_ratio: float + covered_intervals: list[tuple[int, int]] + uncovered_ranges: list[dict[str, Any]] + + def to_dict(self) -> dict[str, Any]: + return { + "canonical_chars": self.canonical_chars, + "covered_chars": self.covered_chars, + "uncovered_chars": self.uncovered_chars, + "coverage_ratio": round(self.coverage_ratio, 4), + "uncovered_ranges": self.uncovered_ranges, + } + + +def compute_parsing_coverage( + canonical_text: str, + spans: list[tuple[int, int]], +) -> ParsingCoverage: + """ + Computes strict interval-merged parsing coverage on canonical text. + Tracks canonical_chars, covered_chars, uncovered_chars, coverage_ratio, + and classifies uncovered_ranges (preamble, gap, annex_trailing). + """ + total_len = len(canonical_text) + if total_len == 0: + return ParsingCoverage( + canonical_chars=0, + covered_chars=0, + uncovered_chars=0, + coverage_ratio=1.0, + covered_intervals=[], + uncovered_ranges=[], + ) + + valid_spans: list[tuple[int, int]] = [] + for s_start, s_end in spans: + if s_start is None or s_end is None: + continue + c_start = max(0, min(s_start, total_len)) + c_end = max(0, min(s_end, total_len)) + if c_start < c_end: + valid_spans.append((c_start, c_end)) + + valid_spans.sort(key=lambda x: (x[0], x[1])) + + merged: list[tuple[int, int]] = [] + for start, end in valid_spans: + if not merged: + merged.append((start, end)) + else: + prev_start, prev_end = merged[-1] + if start <= prev_end: + merged[-1] = (prev_start, max(prev_end, end)) + else: + merged.append((start, end)) + + covered_chars = sum(end - start for start, end in merged) + uncovered_chars = max(0, total_len - covered_chars) + coverage_ratio = covered_chars / total_len if total_len > 0 else 1.0 + + uncovered_ranges: list[dict[str, Any]] = [] + curr = 0 + for idx, (m_start, m_end) in enumerate(merged): + if curr < m_start: + gap_len = m_start - curr + cand = "preamble" if curr == 0 else "gap" + preview = canonical_text[curr : min(curr + 100, m_start)].strip().replace("\n", " ") + uncovered_ranges.append( + { + "start": curr, + "end": m_start, + "length": gap_len, + "candidate_type": cand, + "preview": preview[:80], + } + ) + curr = max(curr, m_end) + + if curr < total_len: + gap_len = total_len - curr + preview = canonical_text[curr : min(curr + 100, total_len)].strip().replace("\n", " ") + uncovered_ranges.append( + { + "start": curr, + "end": total_len, + "length": gap_len, + "candidate_type": "annex_trailing", + "preview": preview[:80], + } + ) + + return ParsingCoverage( + canonical_chars=total_len, + covered_chars=covered_chars, + uncovered_chars=uncovered_chars, + coverage_ratio=coverage_ratio, + covered_intervals=merged, + uncovered_ranges=uncovered_ranges, + ) diff --git a/src/mesa_legal_data/parsers/encoding.py b/src/mesa_legal_data/parsers/encoding.py index 135e64b..a87ed6e 100644 --- a/src/mesa_legal_data/parsers/encoding.py +++ b/src/mesa_legal_data/parsers/encoding.py @@ -1,10 +1,49 @@ import re +import unicodedata META_CHARSET_PATTERN = re.compile( rb"""]+(?:charset\s*=\s*["']?([a-zA-Z0-9_-]+)|content\s*=\s*["'][^"']*charset\s*=\s*([a-zA-Z0-9_-]+))""", re.IGNORECASE, ) +# Common Mojibake patterns for Turkish legal texts (UTF-8 decoded as Latin-1/cp1252/cp1254) +MOJIBAKE_PATTERNS = [ + re.compile(r"Ã[§¶¼‡–œ°½¾]"), # ç, ö, ü, Ç, Ö, Ü, ğ, ı, ş + re.compile(r"Ä[Ÿ±°]"), # ğ, ı, İ + re.compile(r"Å[Ÿž]"), # ş, Ş + re.compile(r"[\ufffd]"), # Unicode replacement character +] + + +def detect_mojibake(text: str) -> list[str]: + """ + Detects potential mojibake / corrupted character sequences in text. + Returns list of issue descriptions if detected. + """ + issues: list[str] = [] + if not text: + return issues + + if "\ufffd" in text: + count = text.count("\ufffd") + issues.append(f"Unicode replacement character (\\ufffd) found {count} time(s)") + + if "\x00" in text: + count = text.count("\x00") + issues.append(f"Null byte (\\x00) found {count} time(s)") + + for pat in MOJIBAKE_PATTERNS[:3]: + matches = pat.findall(text) + if matches: + issues.append(f"Mojibake pattern match: {matches[:5]}") + + # Check for excessive unprintable control characters + control_count = sum(1 for c in text if c not in ("\n", "\t", "\r") and unicodedata.category(c).startswith("C")) + if control_count > 0: + issues.append(f"Unprintable control characters found: {control_count}") + + return issues + def decode_source_bytes(raw_bytes: bytes, is_html: bool = True) -> tuple[str, str]: """ diff --git a/src/mesa_legal_data/parsers/legislation.py b/src/mesa_legal_data/parsers/legislation.py index 5f3dec4..41dcb12 100644 --- a/src/mesa_legal_data/parsers/legislation.py +++ b/src/mesa_legal_data/parsers/legislation.py @@ -12,6 +12,9 @@ class ParsedArticle(BaseModel): article_kind: str # "standard", "additional", "temporary" heading: str | None = None text: str + char_start: int | None = None + char_end: int | None = None + ordinal: int | None = None class ParsedLegislation(BaseModel): @@ -27,63 +30,80 @@ class ParsedLegislation(BaseModel): re.IGNORECASE | re.MULTILINE, ) +NON_ARTICLE_BOUNDARY_PATTERN = re.compile( + r"^(?:(?:BİRİNCİ|İKİNCİ|ÜÇÜNCÜ|DÖRDÜNCÜ|BEŞİNCİ|ALTINCI|YEDİNCİ|SEKİZİNCİ|DOKUZUNCU|ONUNCU)\s+(?:KISIM|BÖLÜM|AYIRIM)|(?:KISIM|BÖLÜM|AYIRIM)\s+[A-ZÇĞİÖŞÜ0-9]+|EK\s+(?:CETVEL|LİSTE|TABLO)|EK-\d+|EK\s+MADDE\b)", + re.IGNORECASE, +) + -def parse_legislation_text(text: str) -> ParsedLegislation: +def parse_legislation_text(text: str, auto_normalize: bool = True) -> ParsedLegislation: """ - Parses normalized legislation text into structured articles. + Parses normalized legislation text into structured articles with exact source spans. + Invariant: text[char_start:char_end] contains the article text in canonical coordinates. """ - text = normalize_text(text) + if auto_normalize: + text = normalize_text(text) if not text: return ParsedLegislation() - lines = text.split("\n") + matches = list(ARTICLE_PATTERN.finditer(text)) + if not matches: + return ParsedLegislation() + articles: list[ParsedArticle] = [] - current_kind: str | None = None - current_num: str | None = None - current_heading: str | None = None - current_body_lines: list[str] = [] - - def flush_current(): - nonlocal current_kind, current_num, current_heading, current_body_lines - if current_num is not None: - kind_str = "standard" - if current_kind == "EK MADDE": - kind_str = "additional" - elif current_kind == "GEÇİCİ MADDE": - kind_str = "temporary" - - art_text = "\n".join(current_body_lines).strip() - if not art_text: - art_text = current_heading or f"Madde {current_num}" - - articles.append( - ParsedArticle( - article_number=current_num, - article_kind=kind_str, - heading=current_heading if current_heading else None, - text=art_text, - ) + for idx, match in enumerate(matches, start=1): + raw_kind = match.group("kind").upper() + num = match.group("num") + heading_text = match.group("heading").strip() + heading = heading_text if heading_text else None + + kind_str = "standard" + if raw_kind == "EK MADDE": + kind_str = "additional" + elif raw_kind == "GEÇİCİ MADDE": + kind_str = "temporary" + + match_start = match.start() + next_match_start = matches[idx].start() if idx < len(matches) else len(text) + + article_block = text[match_start:next_match_start] + + # Scan for structural boundary inside block (e.g. KISIM / BÖLÜM / EK CETVEL) + lines = article_block.split("\n") + body_lines: list[str] = [] + effective_block_len = 0 + current_offset = 0 + + for line_idx, line in enumerate(lines): + line_len = len(line) + (1 if line_idx < len(lines) - 1 else 0) + if line_idx > 0: + line_stripped = line.strip() + if line_stripped and NON_ARTICLE_BOUNDARY_PATTERN.match(line_stripped): + # Stop article text before this non-article structural delimiter + break + body_lines.append(line) + current_offset += line_len + effective_block_len = current_offset + + char_end = match_start + len(text[match_start : match_start + effective_block_len].rstrip()) + if char_end < match_start: + char_end = match_start + + art_text = "\n".join(body_lines).strip() + if not art_text: + art_text = heading or f"Madde {num}" + + articles.append( + ParsedArticle( + article_number=num, + article_kind=kind_str, + heading=heading, + text=art_text, + char_start=match_start, + char_end=char_end, + ordinal=idx, ) - current_kind = None - current_num = None - current_heading = None - current_body_lines = [] - - for line in lines: - match = ARTICLE_PATTERN.match(line) - if match: - flush_current() - raw_kind = match.group("kind").upper() - num = match.group("num") - heading_text = match.group("heading").strip() - - current_kind = raw_kind - current_num = num - current_heading = heading_text if heading_text else None - else: - if current_num is not None: - current_body_lines.append(line) - - flush_current() + ) + return ParsedLegislation(articles=articles) diff --git a/src/mesa_legal_data/pipeline.py b/src/mesa_legal_data/pipeline.py index 6c4855a..3745ccf 100644 --- a/src/mesa_legal_data/pipeline.py +++ b/src/mesa_legal_data/pipeline.py @@ -1,17 +1,19 @@ +import hashlib import json import re import uuid -from datetime import UTC, datetime from pathlib import Path from typing import Any from mesa_legal_data.canonical import write_canonical_part from mesa_legal_data.catalog import ( create_run, + evaluate_auto_approval, finish_run, get_artifact, get_connection, get_document, + get_version, insert_record, insert_version, open_issue, @@ -34,6 +36,9 @@ parse_legislation_text, parse_pdf, ) +from mesa_legal_data.parsers.coverage import compute_parsing_coverage +from mesa_legal_data.parsers.text_normalizer import normalize_text +from mesa_legal_data.quality import evaluate_quality from mesa_legal_data.schema_validation import validate_record from mesa_legal_data.validators import ( scan_privacy_issues, @@ -82,14 +87,14 @@ def process_artifact_pipeline( ) -> str: """ Orchestrates end-to-end processing of an artifact: - Transport -> Parse -> Canonical Models -> JSON Schema & Metadata -> Privacy -> Canonical JSONL Write -> Catalog Version/Records + Transport -> Extraction -> Safe Normalization -> Canonical Coordinates -> Legal Parsing & Spans -> + Coverage -> Quality Gate & Privacy -> Canonical JSONL Write -> Catalog Version/Records """ settings = load_settings() conn = get_connection() # Step 1: Create processing run run_id = f"run-{uuid.uuid4().hex[:12]}" - now_iso = datetime.now(UTC).isoformat() create_run( conn=conn, run_id=run_id, @@ -151,21 +156,23 @@ def process_artifact_pipeline( conn.close() return "failed" - # Step 4: Parse - parsed_text = "" + # Step 4: Text Extraction & Safe Normalization (Single Canonical Coordinate System) try: if "pdf" in mime: - parsed_text = parse_pdf(full_path) + extracted_raw = parse_pdf(full_path) else: raw_bytes = Path(full_path).read_bytes() if "html" in mime: - parsed_text = parse_html(raw_bytes) + extracted_raw = parse_html(raw_bytes) else: decoded_content, _ = decode_source_bytes(raw_bytes, is_html=False) - parsed_text = decoded_content + extracted_raw = decoded_content - if not parsed_text or not parsed_text.strip(): - raise ValueError("Parsed text is empty") + # Authoritative Safe Normalization: Extracted Text -> Canonical Text + canonical_text = normalize_text(extracted_raw) + + if not canonical_text or not canonical_text.strip(): + raise ValueError("Canonical text is empty") state = transition_state(state, "parsed") except Exception as e: @@ -183,10 +190,7 @@ def process_artifact_pipeline( conn.close() return "failed" - # Step 5 & 6: Create Canonical Models, Validate JSON Schema & Metadata - canonical_records: list[dict[str, Any]] = [] - version_id = build_legislation_version_id(doc_id or "tr:legislation:unknown", now_iso[:10], expected_sha) - + # Step 5: Metadata & Version Identity Resolution (Deterministic, Not datetime.now()) meta_dict: dict[str, Any] = {} if art_row and art_row.get("metadata_json"): try: @@ -194,6 +198,38 @@ def process_artifact_pipeline( except Exception: pass + # Authoritative Version Kind + v_kind = "consolidated_snapshot" + if art_row.get("source_id") == "resmi_gazete" or meta_dict.get("source_role") == "original_publication": + v_kind = "original_publication" + + last_mod = art_row.get("last_modified") if art_row else None + ret_at = art_row.get("retrieved_at") if art_row else None + ver_date = ( + meta_dict.get("publication_date") + or meta_dict.get("snapshot_date") + or (str(last_mod)[:10] if last_mod else None) + or (str(ret_at)[:10] if ret_at else None) + or "2026-01-01" + ) + ver_date = str(ver_date)[:10] + + # Deterministic Version ID based on document_id, version_date, and artifact_sha256 + if fam == "legislation": + version_id = build_legislation_version_id(doc_id or "tr:legislation:unknown", ver_date, expected_sha) + else: + # Pre-parse decision to compute decision document_id + dec_parsed_pre = parse_decision_text(canonical_text) + dec_id = build_decision_id( + dec_parsed_pre.court or "unknown", + dec_parsed_pre.chamber, + dec_parsed_pre.esas_no, + dec_parsed_pre.karar_no, + expected_sha, + ) + doc_id = doc_id or dec_id + version_id = f"{doc_id}:version:{ver_date}:{expected_sha[:8]}" + source_obj = { "source_id": art_row["source_id"], "source_url": art_row["source_url"], @@ -204,17 +240,40 @@ def process_artifact_pipeline( provenance_obj = { "parser_name": f"{fam}_parser", "parser_version": "1.0.0", - "pipeline_run_id": run_id, + "pipeline_run_id": f"artifact:{artifact_id}:{fam}_parser:1.0.0", } + existing_version = get_version(conn, version_id) + if existing_version: + if existing_version["artifact_id"] != artifact_id or existing_version["document_id"] != doc_id: + finish_run( + conn, + run_id, + "failed", + json.dumps({"issues": 1}), + error_summary=f"Immutable version collision for {version_id}", + ) + conn.close() + raise ValueError(f"Immutable version collision for {version_id}") + finish_run(conn, run_id, "succeeded", json.dumps({"idempotent_reprocess": True})) + conn.close() + if existing_version.get("quality_status") == "BLOCK": + return "rejected" + if existing_version.get("approval_status") == "approved": + return "approved" + return "needs_review" + + canonical_records: list[dict[str, Any]] = [] leg_count = 0 art_count = 0 dec_count = 0 cit_count = 0 + coverage_result = None try: if fam == "legislation": - leg_parsed = parse_legislation_text(parsed_text) + # Legal Parser runs strictly on canonical text without shifting normalization + leg_parsed = parse_legislation_text(canonical_text, auto_normalize=False) title = doc_row["title"] if doc_row and doc_row.get("title") else "Mevzuat Metni" num = doc_id.split(":")[-1] if doc_id else "0" num_match = re.search(r"\b(\d{1,8})\s+sayılı\b", title, re.IGNORECASE) @@ -232,10 +291,6 @@ def process_artifact_pipeline( if not doc_type: doc_type = "law" - v_kind = "consolidated_snapshot" - if art_row.get("source_id") == "resmi_gazete" or meta_dict.get("source_role") == "original_publication": - v_kind = "original_publication" - pub_info = None pub_d = meta_dict.get("publication_date") if pub_d: @@ -255,13 +310,13 @@ def process_artifact_pipeline( "version": { "version_id": version_id, "version_kind": v_kind, - "snapshot_date": now_iso[:10], + "snapshot_date": ver_date, "effective_from": None, "effective_to": None, }, - "full_text": parsed_text, + "full_text": canonical_text, "schema_version": "1.0.0", - "created_at": now_iso, + "created_at": art_row["retrieved_at"], "source": source_obj, "provenance": provenance_obj, } @@ -270,8 +325,17 @@ def process_artifact_pipeline( canonical_records.append(leg_record) leg_count += 1 + article_spans: list[tuple[int, int]] = [] for a in leg_parsed.articles: art_id = build_article_id(str(leg_record["id"]), a.article_number, a.article_kind) + span_obj = None + if a.char_start is not None and a.char_end is not None: + span_obj = { + "char_start": a.char_start, + "char_end": a.char_end, + } + article_spans.append((a.char_start, a.char_end)) + art_record = { "id": art_id, "record_type": "article", @@ -285,9 +349,9 @@ def process_artifact_pipeline( "status": "active", "effective_from": None, "effective_to": None, - "source_span": None, + "source_span": span_obj, "schema_version": "1.0.0", - "created_at": now_iso, + "created_at": art_row["retrieved_at"], "source": source_obj, "provenance": provenance_obj, } @@ -295,8 +359,11 @@ def process_artifact_pipeline( canonical_records.append(art_record) art_count += 1 - # Extract citations - extracted_cits = extract_citations(parsed_text) + # Compute Parser Coverage using interval merge + coverage_result = compute_parsing_coverage(canonical_text, article_spans) + + # Extract citations on canonical text + extracted_cits = extract_citations(canonical_text) for c in extracted_cits: c_id = build_citation_id( str(leg_record["id"]), @@ -311,11 +378,13 @@ def process_artifact_pipeline( "target_legislation_id": c.target_legislation_id, "target_article_id": c.target_article_id, "raw_text": c.raw_text, + "citation_status": c.citation_status, + "relation_hint": c.relation_hint, "source_span": {"char_start": c.char_start, "char_end": c.char_end}, "extraction_method": "deterministic_regex", - "validation_status": "validated", + "validation_status": "unvalidated", "schema_version": "1.0.0", - "created_at": now_iso, + "created_at": art_row["retrieved_at"], "source": source_obj, "provenance": provenance_obj, } @@ -325,7 +394,7 @@ def process_artifact_pipeline( else: # Decision family - dec_parsed = parse_decision_text(parsed_text) + dec_parsed = parse_decision_text(canonical_text) dec_id = build_decision_id( dec_parsed.court or "unknown", dec_parsed.chamber, @@ -344,10 +413,10 @@ def process_artifact_pipeline( "karar_no": dec_parsed.karar_no, "decision_date": dec_parsed.decision_date, "summary": dec_parsed.summary, - "text": parsed_text, + "text": canonical_text, "verdict": dec_parsed.verdict, "schema_version": "1.0.0", - "created_at": now_iso, + "created_at": art_row["retrieved_at"], "source": source_obj, "provenance": provenance_obj, } @@ -355,7 +424,9 @@ def process_artifact_pipeline( canonical_records.append(dec_record) dec_count += 1 - extracted_cits = extract_citations(parsed_text) + coverage_result = compute_parsing_coverage(canonical_text, [(0, len(canonical_text))]) + + extracted_cits = extract_citations(canonical_text) for c in extracted_cits: c_id = build_citation_id(dec_id, c.char_start or 0, c.char_end or 0, c.target_legislation_id) cit_record = { @@ -365,11 +436,13 @@ def process_artifact_pipeline( "target_legislation_id": c.target_legislation_id, "target_article_id": c.target_article_id, "raw_text": c.raw_text, + "citation_status": c.citation_status, + "relation_hint": c.relation_hint, "source_span": {"char_start": c.char_start, "char_end": c.char_end}, "extraction_method": "deterministic_regex", - "validation_status": "validated", + "validation_status": "unvalidated", "schema_version": "1.0.0", - "created_at": now_iso, + "created_at": art_row["retrieved_at"], "source": source_obj, "provenance": provenance_obj, } @@ -393,11 +466,9 @@ def process_artifact_pipeline( conn.close() return "failed" - # Step 7: Privacy Scan + # Step 6: Privacy Scan state = transition_state(state, "privacy_pending") - privacy_issues = scan_privacy_issues(parsed_text) - - final_status = "needs_review" + privacy_issues = scan_privacy_issues(canonical_text) privacy_status = "clean" if not privacy_issues else "flagged" for iss in privacy_issues: @@ -412,8 +483,43 @@ def process_artifact_pipeline( message=iss["message"], details_json=json.dumps({"match_type": iss["match_type"], "masked": iss["masked"]}), ) - if iss["severity"] == "blocker": - final_status = "rejected" + + # Step 7: Deterministic Quality Gate Evaluation (PASS, REVIEW, BLOCK) + quality_report = evaluate_quality( + source_info=source_obj, + raw_info={ + "byte_size": expected_size, + "sha256": expected_sha, + "file_exists": full_path.exists(), + }, + canonical_records=canonical_records, + canonical_text=canonical_text, + coverage=coverage_result, + privacy_issues=privacy_issues, + parser_name=f"{fam}_parser", + parser_version="1.0.0", + ) + + quality_status = quality_report.decision + quality_json = json.dumps(quality_report.to_dict()) + + # Release Guard & Lifecycle Determination + if quality_status == "BLOCK": + val_status = "failed" + final_status = "rejected" + open_issue( + conn, + issue_id=f"iss-{uuid.uuid4().hex[:8]}", + subject_type="version", + subject_id=version_id, + severity="blocker", + code="QUALITY_GATE_BLOCKED", + message=f"Quality gate blocked version: {quality_report.summary}", + details_json=quality_json, + ) + else: + val_status = "valid" + final_status = "needs_review" # Step 8: Write Canonical JSONL Part Files records_by_type: dict[str, list[dict[str, Any]]] = {} @@ -422,13 +528,13 @@ def process_artifact_pipeline( records_by_type.setdefault(rt, []).append(r) canonical_locations = [] + canonical_write_id = "version-" + hashlib.sha256(f"{version_id}:parser:1.0.0".encode()).hexdigest()[:20] for rt, r_list in records_by_type.items(): - locs = write_canonical_part(r_list, rt, run_id) + locs = write_canonical_part(r_list, rt, canonical_write_id) canonical_locations.extend(locs) - # Step 9: Atomic Catalog Write for Version & Records + # Step 9: Atomic Catalog Write for Version & Version-Aware Records with transaction(conn): - # Insert Version record first_loc = canonical_locations[0] if canonical_locations else None c_path = first_loc.relative_path if first_loc else "" c_sha = first_loc.record_sha256 if first_loc else expected_sha @@ -438,8 +544,8 @@ def process_artifact_pipeline( version_id=version_id, document_id=doc_id or "tr:legislation:unknown", artifact_id=artifact_id, - version_kind="consolidated_snapshot", - snapshot_date=now_iso[:10], + version_kind=v_kind, + snapshot_date=ver_date, effective_from=None, effective_to=None, canonical_path=c_path, @@ -448,9 +554,12 @@ def process_artifact_pipeline( parser_name=f"{fam}_parser", parser_version="1.0.0", schema_version="1.0.0", - validation_status="valid", + validation_status=val_status, privacy_status=privacy_status, approval_status="pending", + quality_status=quality_status, + quality_json=quality_json, + supersedes_version_id=meta_dict.get("supersedes_version_id"), ) for loc in canonical_locations: @@ -462,13 +571,35 @@ def process_artifact_pipeline( canonical_path=loc.relative_path, canonical_line=loc.line_number, record_sha256=loc.record_sha256, - validation_status="valid", + validation_status=val_status, approval_status="pending", ) if doc_id: update_document_status(conn, doc_id, final_status, current_version_id=version_id) + # Step 9b: Safe Auto-Approval Evaluation + auto_approved = False + auto_reason = "" + if quality_status != "BLOCK" and val_status == "valid": + try: + auto_approved, auto_reason = evaluate_auto_approval( + conn, + version_id=version_id, + source_id=art_row.get("source_id", "manual"), + parser_name=f"{fam}_parser", + parser_version="1.0.0", + quality_decision=quality_status, + has_privacy_blocker=(privacy_status == "flagged"), + schema_valid=True, + ) + if auto_approved: + final_status = "approved" + if doc_id: + update_document_status(conn, doc_id, "approved", current_version_id=version_id) + except Exception: + pass + # Step 10: Finish Run with Real Counters counters = { "artifacts_processed": 1, @@ -477,8 +608,12 @@ def process_artifact_pipeline( "decision_records": dec_count, "citation_records": cit_count, "issues": issues_count, + "quality_decision": quality_status, + "auto_approved": auto_approved, + "auto_reason": auto_reason, + "coverage_ratio": coverage_result.coverage_ratio if coverage_result else None, } - finish_run(conn, run_id, "succeeded", json.dumps(counters)) + finish_run(conn, run_id, "succeeded" if quality_status != "BLOCK" else "failed", json.dumps(counters)) conn.close() return final_status diff --git a/src/mesa_legal_data/publisher/__init__.py b/src/mesa_legal_data/publisher/__init__.py new file mode 100644 index 0000000..2d5e946 --- /dev/null +++ b/src/mesa_legal_data/publisher/__init__.py @@ -0,0 +1,43 @@ +from mesa_legal_data.publisher.chunker import plan_source_chunks +from mesa_legal_data.publisher.client import MesaClient +from mesa_legal_data.publisher.engine import ( + build_delivery_plan, + execute_publish_delivery, + retry_delivery_failures, +) +from mesa_legal_data.publisher.hashing import ( + calculate_canonical_revision_hash, + generate_idempotency_key, +) +from mesa_legal_data.publisher.ledger import ( + get_document_mesa_status, + get_mesa_target_settings, + upsert_mesa_target_settings, +) +from mesa_legal_data.publisher.models import ( + DeliveryPlanSummary, + DeliveryStatus, + MesaTargetSettings, + MutationState, + PreflightReport, + SourceChunk, +) + +__all__ = [ + "MesaTargetSettings", + "SourceChunk", + "PreflightReport", + "DeliveryPlanSummary", + "DeliveryStatus", + "MutationState", + "plan_source_chunks", + "calculate_canonical_revision_hash", + "generate_idempotency_key", + "MesaClient", + "get_mesa_target_settings", + "upsert_mesa_target_settings", + "get_document_mesa_status", + "build_delivery_plan", + "execute_publish_delivery", + "retry_delivery_failures", +] diff --git a/src/mesa_legal_data/publisher/chunker.py b/src/mesa_legal_data/publisher/chunker.py new file mode 100644 index 0000000..0814d2e --- /dev/null +++ b/src/mesa_legal_data/publisher/chunker.py @@ -0,0 +1,233 @@ +from typing import Any + +from mesa_legal_data.publisher.hashing import calculate_content_hash +from mesa_legal_data.publisher.models import SourceChunk + + +def _split_oversized_text( + text: str, + base_char_start: int, + max_chars: int, +) -> list[tuple[int, int, str]]: + """ + Deterministically splits oversized text into chunks <= max_chars. + Strategy: + 1. Tries paragraph boundaries (\\n\\n). + 2. Tries single newline boundaries (\\n). + 3. Falls back to hard boundary slices. + Returns list of (char_start, char_end, slice_text). + """ + if len(text) <= max_chars: + return [(base_char_start, base_char_start + len(text), text)] + + results: list[tuple[int, int, str]] = [] + current_pos = 0 + total_len = len(text) + + while current_pos < total_len: + remaining_len = total_len - current_pos + if remaining_len <= max_chars: + slice_str = text[current_pos:] + results.append((base_char_start + current_pos, base_char_start + total_len, slice_str)) + break + + # Find best break point within [current_pos + 1, current_pos + max_chars] + target_limit = current_pos + max_chars + window = text[current_pos:target_limit] + + # 1. Look for double newline + split_idx = window.rfind("\n\n") + if split_idx != -1 and split_idx > (max_chars // 4): + end_pos = current_pos + split_idx + 2 + else: + # 2. Look for single newline + split_idx = window.rfind("\n") + if split_idx != -1 and split_idx > (max_chars // 4): + end_pos = current_pos + split_idx + 1 + else: + # 3. Look for space + split_idx = window.rfind(" ") + if split_idx != -1 and split_idx > (max_chars // 4): + end_pos = current_pos + split_idx + 1 + else: + # 4. Safe hard cut + end_pos = target_limit + + slice_str = text[current_pos:end_pos] + results.append((base_char_start + current_pos, base_char_start + end_pos, slice_str)) + current_pos = end_pos + + return results + + +def plan_source_chunks( + *, + document_id: str, + version_id: str, + canonical_text: str, + records: list[dict[str, Any]], + content_limit_chars: int = 32768, +) -> list[SourceChunk]: + """ + Deterministically transforms canonical text and version records into ordered SourceChunks. + Preserves: + - Preamble ranges (meaningful header / introductory text) + - Articles (structured legal units) + - Annex / trailing ranges (schedules, transitional provisions) + Guarantees: + - Deterministic ordering by appearance (char_start) + - Stable chunk IDs + - Exact content hash for each chunk + """ + if not canonical_text or not canonical_text.strip(): + return [] + + # Auto-resolve char_start and char_end if content is provided + search_cursor = 0 + for r in records: + if r.get("char_start") is None and r.get("content"): + pos = canonical_text.find(r["content"], search_cursor) + if pos != -1: + r["char_start"] = pos + r["char_end"] = pos + len(r["content"]) + search_cursor = r["char_end"] + + # Filter and sort article records by char_start + article_records = [ + r + for r in records + if r.get("record_type") == "article" and r.get("char_start") is not None and r.get("char_end") is not None + ] + article_records.sort(key=lambda r: (r.get("char_start", 0), r.get("ordinal", 0))) + + chunks: list[SourceChunk] = [] + ordinal_counter = 1 + + if not article_records: + # Generic text without article records (e.g. unsegmented or single decision) + splits = _split_oversized_text(canonical_text, 0, content_limit_chars) + for piece_idx, (c_start, c_end, piece_text) in enumerate(splits, start=1): + chunk_id = ( + f"{version_id}:chunk:{ordinal_counter:04d}" + if len(splits) == 1 + else f"{version_id}:chunk:{ordinal_counter:04d}:p{piece_idx}" + ) + c_hash = calculate_content_hash(piece_text) + chunks.append( + SourceChunk( + chunk_id=chunk_id, + document_id=document_id, + version_id=version_id, + chunk_type="general", + title="Genel Metin", + char_start=c_start, + char_end=c_end, + ordinal=ordinal_counter, + content=piece_text, + content_hash=c_hash, + metadata={"piece_index": piece_idx, "total_pieces": len(splits)}, + ) + ) + ordinal_counter += 1 + return chunks + + # 1. Preamble Chunk (if meaningful content exists before first article) + first_art_start = article_records[0].get("char_start", 0) + if first_art_start > 0: + preamble_text = canonical_text[:first_art_start] + if preamble_text.strip(): + splits = _split_oversized_text(preamble_text, 0, content_limit_chars) + for piece_idx, (c_start, c_end, piece_text) in enumerate(splits, start=1): + chunk_id = ( + f"{version_id}:chunk:{ordinal_counter:04d}" + if len(splits) == 1 + else f"{version_id}:chunk:{ordinal_counter:04d}:p{piece_idx}" + ) + c_hash = calculate_content_hash(piece_text) + chunks.append( + SourceChunk( + chunk_id=chunk_id, + document_id=document_id, + version_id=version_id, + chunk_type="preamble", + title="Başlangıç / Başlık", + char_start=c_start, + char_end=c_end, + ordinal=ordinal_counter, + content=piece_text, + content_hash=c_hash, + metadata={"piece_index": piece_idx, "total_pieces": len(splits)}, + ) + ) + ordinal_counter += 1 + + # 2. Article Chunks + for art in article_records: + a_start = int(art["char_start"]) + a_end = int(art["char_end"]) + art_text = canonical_text[a_start:a_end] + art_id = art.get("record_id", "") + art_num = art.get("article_number", "") + art_title = art.get("title") or (f"Madde {art_num}" if art_num else "Madde") + + splits = _split_oversized_text(art_text, a_start, content_limit_chars) + for piece_idx, (c_start, c_end, piece_text) in enumerate(splits, start=1): + chunk_id = ( + f"{version_id}:chunk:{ordinal_counter:04d}" + if len(splits) == 1 + else f"{version_id}:chunk:{ordinal_counter:04d}:p{piece_idx}" + ) + c_hash = calculate_content_hash(piece_text) + chunks.append( + SourceChunk( + chunk_id=chunk_id, + document_id=document_id, + version_id=version_id, + chunk_type="article", + title=art_title, + char_start=c_start, + char_end=c_end, + ordinal=ordinal_counter, + content=piece_text, + content_hash=c_hash, + metadata={ + "article_id": art_id, + "article_number": art_num, + "piece_index": piece_idx, + "total_pieces": len(splits), + }, + ) + ) + ordinal_counter += 1 + + # 3. Annex / Trailing Chunk (if meaningful content exists after last article) + last_art_end = article_records[-1].get("char_end", len(canonical_text)) + if last_art_end < len(canonical_text): + annex_text = canonical_text[last_art_end:] + if annex_text.strip(): + splits = _split_oversized_text(annex_text, last_art_end, content_limit_chars) + for piece_idx, (c_start, c_end, piece_text) in enumerate(splits, start=1): + chunk_id = ( + f"{version_id}:chunk:{ordinal_counter:04d}" + if len(splits) == 1 + else f"{version_id}:chunk:{ordinal_counter:04d}:p{piece_idx}" + ) + c_hash = calculate_content_hash(piece_text) + chunks.append( + SourceChunk( + chunk_id=chunk_id, + document_id=document_id, + version_id=version_id, + chunk_type="annex", + title="Ek / Geçici Maddeler", + char_start=c_start, + char_end=c_end, + ordinal=ordinal_counter, + content=piece_text, + content_hash=c_hash, + metadata={"piece_index": piece_idx, "total_pieces": len(splits)}, + ) + ) + ordinal_counter += 1 + + return chunks diff --git a/src/mesa_legal_data/publisher/client.py b/src/mesa_legal_data/publisher/client.py new file mode 100644 index 0000000..a43bfa2 --- /dev/null +++ b/src/mesa_legal_data/publisher/client.py @@ -0,0 +1,385 @@ +import os +import time +from typing import Any + +import httpx + +from mesa_legal_data.publisher.models import ( + MesaTargetSettings, + MutationState, + PreflightCheckItem, + PreflightReport, + SourceChunk, +) + + +class MesaClientError(Exception): + """Base error for MESA client interactions.""" + + +class MesaClient: + """ + Minimal, robust HTTP client for MESA v4 API. + Reads API key strictly from environment secret (MESA_DATA_MESA_API_KEY / MESA_API_KEY). + Never logs or persists the API key. + """ + + def __init__( + self, + settings: MesaTargetSettings, + api_key: str | None = None, + timeout_seconds: float = 15.0, + ): + self.settings = settings + self._api_key = api_key or os.environ.get("MESA_DATA_MESA_API_KEY") or os.environ.get("MESA_API_KEY") + self.timeout_seconds = timeout_seconds + + @property + def is_api_key_configured(self) -> bool: + return bool(self._api_key and self._api_key.strip()) + + def _get_headers(self, idempotency_key: str | None = None) -> dict[str, str]: + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + "User-Agent": "MESA-Legal-Data-Publisher/1.0", + } + if self._api_key: + headers["Authorization"] = f"Bearer {self._api_key}" + if idempotency_key: + headers["Idempotency-Key"] = idempotency_key + return headers + + @property + def is_contract_configured(self) -> bool: + return bool( + self.settings.contract_source in ("configured", "live_verified") + and self.settings.base_url + and self.settings.health_path.startswith("/") + and self.settings.publish_path.startswith("/") + and self.settings.mutation_status_path_template.startswith("/") + and "{mutation_id}" in self.settings.mutation_status_path_template + ) + + @staticmethod + def _normalize_mutation_state(raw_state: Any) -> tuple[str, str | None]: + """Normalize known states without ever inferring COMMITTED.""" + if not isinstance(raw_state, str) or not raw_state.strip(): + return MutationState.FAILED.value, "MESA response did not include an explicit mutation state" + state_upper = raw_state.strip().upper() + if state_upper in ("ACCEPTED", "RECEIVED"): + return MutationState.QUEUED.value, None + if state_upper in MutationState.__members__: + return state_upper, None + return MutationState.FAILED.value, f"Unknown MESA mutation state: {raw_state}" + + def test_connection(self) -> dict[str, Any]: + """ + Tests connectivity to the configured MESA base_url. + """ + if not self.is_contract_configured: + return { + "connected": False, + "latency_ms": 0.0, + "details": "MESA HTTP contract routes are not configured or verified", + } + + start_time = time.perf_counter() + try: + with httpx.Client(timeout=self.timeout_seconds) as client: + url = self.settings.base_url.rstrip("/") + resp = client.get(f"{url}{self.settings.health_path}", headers=self._get_headers()) + + latency_ms = (time.perf_counter() - start_time) * 1000.0 + + if resp.status_code in (200, 204): + return { + "connected": True, + "latency_ms": round(latency_ms, 2), + "details": f"Connected ({resp.status_code})", + } + elif resp.status_code in (401, 403): + return { + "connected": True, + "latency_ms": round(latency_ms, 2), + "details": "Server reachable, authentication required", + } + else: + return { + "connected": False, + "latency_ms": round(latency_ms, 2), + "details": f"Server responded with status {resp.status_code}", + } + except Exception as e: + latency_ms = (time.perf_counter() - start_time) * 1000.0 + return { + "connected": False, + "latency_ms": round(latency_ms, 2), + "details": f"Connection error: {e}", + } + + def run_preflight_checks( + self, + ready_documents_count: int = 0, + ready_versions_count: int = 0, + estimated_chunks_count: int = 0, + total_canonical_bytes: int = 0, + blocked_versions_count: int = 0, + unreadable_versions_count: int = 0, + ) -> PreflightReport: + """ + Executes preflight verification checks against MESA v4 publisher requirements. + """ + checks: list[PreflightCheckItem] = [] + + # 1. Target URL Check + if self.settings.base_url and self.settings.base_url.startswith(("http://", "https://")): + checks.append( + PreflightCheckItem( + name="target_url", status="PASS", message=f"Target URL configured: {self.settings.base_url}" + ) + ) + else: + checks.append(PreflightCheckItem(name="target_url", status="FAIL", message="Invalid or missing target URL")) + + if self.is_contract_configured: + checks.append( + PreflightCheckItem( + name="http_contract", + status="PASS", + message=f"MESA HTTP contract is {self.settings.contract_source}", + ) + ) + else: + checks.append( + PreflightCheckItem( + name="http_contract", + status="FAIL", + message="MESA HTTP routes are unknown; configure a documented or live-verified contract", + ) + ) + + # 2. API Key Check + if self.is_api_key_configured: + checks.append( + PreflightCheckItem( + name="api_key", status="PASS", message="API Key is configured via environment secret" + ) + ) + else: + checks.append( + PreflightCheckItem( + name="api_key", status="WARN", message="API Key not configured (required if auth is enabled)" + ) + ) + + # 3. Target Identifiers Check + missing_ids = [] + for f in ["tenant_id", "workspace_id", "dataset_id", "agent_id"]: + if not getattr(self.settings, f, "").strip(): + missing_ids.append(f) + if not missing_ids: + checks.append( + PreflightCheckItem( + name="target_scope", + status="PASS", + message=f"Target scope valid ({self.settings.tenant_id}/{self.settings.workspace_id}/{self.settings.dataset_id})", + ) + ) + else: + checks.append( + PreflightCheckItem( + name="target_scope", + status="FAIL", + message=f"Missing required target identifiers: {', '.join(missing_ids)}", + ) + ) + + # 4. Server Reachability Check + conn_res = self.test_connection() + if conn_res["connected"]: + checks.append( + PreflightCheckItem( + name="connectivity", + status="PASS", + message=f"Server reachable ({conn_res['details']}, {conn_res['latency_ms']}ms)", + ) + ) + else: + checks.append( + PreflightCheckItem( + name="connectivity", + status="FAIL", + message=f"Cannot reach server: {conn_res['details']}", + ) + ) + + # 5. Quality & Release Readiness Check + if unreadable_versions_count > 0: + checks.append( + PreflightCheckItem( + name="canonical_integrity", + status="FAIL", + message=f"{unreadable_versions_count} approved versions failed canonical integrity checks", + ) + ) + else: + checks.append( + PreflightCheckItem( + name="canonical_integrity", status="PASS", message="All planned canonical versions are readable" + ) + ) + + if blocked_versions_count > 0: + checks.append( + PreflightCheckItem( + name="quality_guard", + status="FAIL", + message=f"{blocked_versions_count} versions have quality status BLOCK (excluded from delivery)", + ) + ) + elif ready_versions_count > 0: + checks.append( + PreflightCheckItem( + name="quality_guard", + status="PASS", + message=f"{ready_versions_count} approved versions ready for publishing ({estimated_chunks_count} chunks)", + ) + ) + else: + checks.append( + PreflightCheckItem( + name="quality_guard", + status="FAIL", + message="No approved, readable versions are pending publication", + ) + ) + + overall_pass = all(c.status != "FAIL" for c in checks) + return PreflightReport( + overall_status="PASS" if overall_pass else "FAIL", + checks=checks, + ready_documents_count=ready_documents_count, + ready_versions_count=ready_versions_count, + estimated_chunks_count=estimated_chunks_count, + total_canonical_bytes=total_canonical_bytes, + ) + + def publish_source_chunk( + self, + chunk: SourceChunk, + idempotency_key: str, + ) -> dict[str, Any]: + """ + Submits a single source chunk mutation to MESA v4 HTTP API. + Returns dictionary with mutation_id, state (COMMITTED, QUEUED, etc.), and message. + """ + payload = { + "tenant_id": self.settings.tenant_id, + "workspace_id": self.settings.workspace_id, + "dataset_id": self.settings.dataset_id, + "agent_id": self.settings.agent_id, + "document_id": chunk.document_id, + "version_id": chunk.version_id, + "chunk_id": chunk.chunk_id, + "chunk_type": chunk.chunk_type, + "title": chunk.title, + "char_start": chunk.char_start, + "char_end": chunk.char_end, + "ordinal": chunk.ordinal, + "content": chunk.content, + "content_hash": chunk.content_hash, + "metadata": chunk.metadata, + } + + if not self.is_contract_configured: + return { + "mutation_id": None, + "state": MutationState.FAILED.value, + "message": "MESA HTTP contract is unknown; refusing to guess a publish route", + } + + url = f"{self.settings.base_url.rstrip('/')}{self.settings.publish_path}" + headers = self._get_headers(idempotency_key=idempotency_key) + + try: + with httpx.Client(timeout=self.timeout_seconds) as client: + resp = client.post(url, json=payload, headers=headers) + if resp.status_code in (200, 201, 202): + data = resp.json() + mutation_id = data.get("mutation_id") or data.get("id") + state_upper, state_error = self._normalize_mutation_state(data.get("state") or data.get("status")) + if state_upper in (MutationState.QUEUED.value, MutationState.PROCESSING.value) and not mutation_id: + state_upper = MutationState.FAILED.value + state_error = "MESA returned a non-terminal state without mutation_id" + return { + "mutation_id": mutation_id, + "state": state_upper, + "message": state_error or data.get("message", "Mutation accepted"), + } + elif resp.status_code == 422: + # Semantic rejection + error_msg = resp.text + try: + error_msg = resp.json().get("detail", error_msg) + except Exception: + pass + return { + "mutation_id": None, + "state": MutationState.REJECTED.value, + "message": f"MESA rejected payload (422): {error_msg}", + } + else: + return { + "mutation_id": None, + "state": MutationState.FAILED.value, + "message": f"HTTP {resp.status_code}: {resp.text[:200]}", + } + except Exception as e: + return { + "mutation_id": None, + "state": MutationState.FAILED.value, + "message": f"Transport failure: {e}", + } + + def get_mutation_status(self, mutation_id: str) -> dict[str, Any]: + """ + Polls status of an async mutation identifier. + """ + if not mutation_id: + return {"mutation_id": mutation_id, "state": MutationState.FAILED.value, "error": "Missing mutation_id"} + + if not self.is_contract_configured: + return { + "mutation_id": mutation_id, + "state": MutationState.FAILED.value, + "error": "MESA HTTP contract is unknown", + } + + mutation_path = self.settings.mutation_status_path_template.replace("{mutation_id}", mutation_id) + url = f"{self.settings.base_url.rstrip('/')}{mutation_path}" + headers = self._get_headers() + + try: + with httpx.Client(timeout=self.timeout_seconds) as client: + resp = client.get(url, headers=headers) + if resp.status_code == 200: + data = resp.json() + state_upper, state_error = self._normalize_mutation_state(data.get("state") or data.get("status")) + return { + "mutation_id": mutation_id, + "state": state_upper, + "error": state_error or data.get("error"), + } + else: + return { + "mutation_id": mutation_id, + "state": MutationState.FAILED.value, + "error": f"HTTP {resp.status_code}: {resp.text[:200]}", + } + except Exception as e: + return { + "mutation_id": mutation_id, + "state": MutationState.FAILED.value, + "error": f"Transport error polling mutation {mutation_id}: {e}", + } diff --git a/src/mesa_legal_data/publisher/engine.py b/src/mesa_legal_data/publisher/engine.py new file mode 100644 index 0000000..63e066c --- /dev/null +++ b/src/mesa_legal_data/publisher/engine.py @@ -0,0 +1,524 @@ +import hashlib +import json +import time +import uuid +from typing import Any, Callable + +from mesa_legal_data.catalog import get_connection +from mesa_legal_data.config import load_settings +from mesa_legal_data.publisher.chunker import plan_source_chunks +from mesa_legal_data.publisher.client import MesaClient, MesaClientError +from mesa_legal_data.publisher.hashing import generate_idempotency_key +from mesa_legal_data.publisher.ledger import ( + create_delivery, + get_delivery, + get_mesa_target_settings, + insert_delivery_item, + is_chunk_already_committed, + list_failed_delivery_items, + update_delivery_item_state, + update_delivery_progress, +) +from mesa_legal_data.publisher.models import ( + DeliveryPlanSummary, + DeliveryStatus, + MutationState, + SourceChunk, +) + + +def get_ready_versions_and_content(conn) -> tuple[list[dict[str, Any]], int]: + """ + Fetches approved versions ready for publishing, and count of blocked versions. + """ + cursor = conn.cursor() + # 1. Count blocked versions + cursor.execute("SELECT count(*) FROM versions WHERE quality_status = 'BLOCK'") + blocked_count = cursor.fetchone()[0] + + # 2. Fetch approved versions + cursor.execute("""WITH eligible AS ( + SELECT v.*, + ROW_NUMBER() OVER ( + PARTITION BY v.document_id + ORDER BY COALESCE(v.revision_number, 1) DESC, v.created_at DESC + ) AS version_rank + FROM versions v + WHERE v.approval_status = 'approved' + AND v.validation_status = 'valid' + AND v.privacy_status IN ('clean', 'approved') + AND (v.quality_status IS NULL OR v.quality_status != 'BLOCK') + ) + SELECT v.version_id, v.document_id, v.canonical_path, v.canonical_sha256, + v.revision_number, d.family + FROM eligible v + JOIN documents d ON v.document_id = d.document_id + WHERE v.version_rank = 1 + ORDER BY v.created_at ASC""") + rows = cursor.fetchall() + version_items = [ + { + "version_id": r[0], + "document_id": r[1], + "canonical_path": r[2], + "canonical_sha256": r[3], + "revision_number": r[4], + "family": r[5], + } + for r in rows + ] + return version_items, blocked_count + + +def build_delivery_plan( + target_key: str = "default", +) -> tuple[list[tuple[SourceChunk, bool]], DeliveryPlanSummary]: + """ + Constructs the delivery plan across all ready versions. + Returns: + - List of (SourceChunk, is_already_committed) + - DeliveryPlanSummary + """ + conn = get_connection() + target_settings = get_mesa_target_settings(conn, target_key) + versions, blocked_count = get_ready_versions_and_content(conn) + data_root = load_settings().data_root_path + + all_chunks: list[tuple[SourceChunk, bool]] = [] + unique_docs = set() + total_bytes = 0 + already_committed = 0 + unreadable_versions = 0 + + for v_info in versions: + doc_id = v_info["document_id"] + v_id = v_info["version_id"] + canonical_text = "" + records: list[dict[str, Any]] = [] + c_cur = conn.cursor() + c_cur.execute( + """SELECT record_id, record_type, canonical_path, canonical_line, record_sha256 + FROM records + WHERE version_id = ? AND validation_status = 'valid' AND approval_status = 'approved' + ORDER BY canonical_path, canonical_line""", + (v_id,), + ) + rec_rows = c_cur.fetchall() + + try: + lines_by_path: dict[str, list[str]] = {} + for record_id, record_type, rel_path, line_number, expected_hash in rec_rows: + if rel_path not in lines_by_path: + abs_path = data_root / rel_path + lines_by_path[rel_path] = abs_path.read_text(encoding="utf-8").splitlines(keepends=True) + lines = lines_by_path[rel_path] + if line_number < 1 or line_number > len(lines): + raise ValueError(f"Canonical line out of bounds for {record_id}") + line = lines[line_number - 1] + if hashlib.sha256(line.encode("utf-8")).hexdigest() != expected_hash: + raise ValueError(f"Canonical hash mismatch for {record_id}") + item = json.loads(line) + if item.get("id") != record_id or item.get("record_type") != record_type: + raise ValueError(f"Canonical identity mismatch for {record_id}") + + if record_type == "legislation": + canonical_text = item.get("full_text") or "" + elif record_type == "decision": + canonical_text = item.get("text") or "" + elif record_type == "article": + span = item.get("source_span") or {} + records.append( + { + "record_id": record_id, + "record_type": "article", + "char_start": span.get("char_start"), + "char_end": span.get("char_end"), + "ordinal": item.get("ordinal", line_number), + "title": item.get("heading"), + "article_number": item.get("article_number"), + "content": item.get("text") or "", + } + ) + if not canonical_text.strip(): + raise ValueError(f"Version {v_id} has no authoritative canonical document text") + except (OSError, ValueError, json.JSONDecodeError): + unreadable_versions += 1 + continue + + unique_docs.add(doc_id) + + chunks = plan_source_chunks( + document_id=doc_id, + version_id=v_id, + canonical_text=canonical_text, + records=records, + content_limit_chars=target_settings.content_limit_chars, + ) + + for chunk in chunks: + total_bytes += len(chunk.content.encode("utf-8")) + is_committed = is_chunk_already_committed( + conn, + target_key=target_key, + document_id=doc_id, + version_id=v_id, + chunk_id=chunk.chunk_id, + content_hash=chunk.content_hash, + ) + if is_committed: + already_committed += 1 + all_chunks.append((chunk, is_committed)) + + conn.close() + + summary = DeliveryPlanSummary( + ready_documents=len(unique_docs), + ready_versions=len(versions) - unreadable_versions, + estimated_chunks=len(all_chunks), + total_canonical_bytes=total_bytes, + already_committed_chunks=already_committed, + new_chunks_to_send=len(all_chunks) - already_committed, + blocked_versions_excluded=blocked_count, + unreadable_versions_excluded=unreadable_versions, + ) + return all_chunks, summary + + +def execute_publish_delivery( + *, + delivery_id: str | None = None, + release_id: str | None = None, + target_key: str = "default", + progress_callback: Callable[[dict[str, Any]], None] | None = None, +) -> dict[str, Any]: + """ + Executes an end-to-end MESA v4 publish delivery: + 1. Initializes client with target settings. + 2. Runs preflight checks (aborts if FAIL). + 3. Builds chunk plan and registers delivery ledger. + 4. Iterates through chunks with cross-release dedup and idempotency keys. + 5. Polls mutations until COMMITTED or terminal failure. + 6. Writes truthful status to ledger. + """ + conn = get_connection() + target_settings = get_mesa_target_settings(conn, target_key) + client = MesaClient(settings=target_settings) + + if not delivery_id: + delivery_id = f"del-{uuid.uuid4().hex[:12]}" + + # 1. Build delivery plan + chunk_tuples, summary = build_delivery_plan(target_key=target_key) + total_items = len(chunk_tuples) + + preflight = client.run_preflight_checks( + ready_documents_count=summary.ready_documents, + ready_versions_count=summary.ready_versions, + estimated_chunks_count=summary.estimated_chunks, + total_canonical_bytes=summary.total_canonical_bytes, + blocked_versions_count=summary.blocked_versions_excluded, + unreadable_versions_count=summary.unreadable_versions_excluded, + ) + if preflight.overall_status == "FAIL": + conn.close() + failures = "; ".join(check.message for check in preflight.checks if check.status == "FAIL") + raise MesaClientError(f"Publisher preflight failed: {failures}") + + # 2. Create delivery in ledger + create_delivery( + conn, + delivery_id=delivery_id, + release_id=release_id, + target_key=target_key, + total_items=total_items, + ) + + update_delivery_progress( + conn, + delivery_id=delivery_id, + status=DeliveryStatus.SENDING.value, + committed_items=0, + failed_items=0, + skipped_items=0, + ) + + committed_count = 0 + failed_count = 0 + skipped_count = 0 + last_err = None + + for idx, (chunk, is_already_done) in enumerate(chunk_tuples, start=1): + item_id = f"item-{uuid.uuid4().hex[:12]}" + idemp_key = generate_idempotency_key( + tenant_id=target_settings.tenant_id, + workspace_id=target_settings.workspace_id, + dataset_id=target_settings.dataset_id, + document_id=chunk.document_id, + version_id=chunk.version_id, + chunk_id=chunk.chunk_id, + content_hash=chunk.content_hash, + ) + + payload_json = json.dumps(chunk.model_dump(), sort_keys=True) + + if is_already_done: + # Cross-release dedup: skip without sending HTTP mutation + insert_delivery_item( + conn, + item_id=item_id, + delivery_id=delivery_id, + document_id=chunk.document_id, + version_id=chunk.version_id, + chunk_id=chunk.chunk_id, + content_hash=chunk.content_hash, + idempotency_key=idemp_key, + remote_state=MutationState.SKIPPED.value, + payload_json=payload_json, + ) + skipped_count += 1 + else: + insert_delivery_item( + conn, + item_id=item_id, + delivery_id=delivery_id, + document_id=chunk.document_id, + version_id=chunk.version_id, + chunk_id=chunk.chunk_id, + content_hash=chunk.content_hash, + idempotency_key=idemp_key, + remote_state=MutationState.SENDING.value, + payload_json=payload_json, + ) + + # Submit chunk to MESA v4 + pub_res = client.publish_source_chunk(chunk, idempotency_key=idemp_key) + remote_mutation_id = pub_res.get("mutation_id") + initial_state = pub_res.get("state", MutationState.FAILED.value) + + final_item_state = initial_state + if initial_state in (MutationState.QUEUED.value, MutationState.PROCESSING.value): + # Poll mutation until terminal state + update_delivery_item_state( + conn, + item_id=item_id, + remote_state=MutationState.PROCESSING.value, + remote_mutation_id=remote_mutation_id, + ) + poll_attempts = 0 + while poll_attempts < 5: + time.sleep(0.5) + poll_res = client.get_mutation_status(remote_mutation_id or "") + polled_state = poll_res.get("state") + if polled_state in ( + MutationState.COMMITTED.value, + MutationState.FAILED.value, + MutationState.REJECTED.value, + ): + final_item_state = polled_state + break + poll_attempts += 1 + + if final_item_state == MutationState.COMMITTED.value: + committed_count += 1 + update_delivery_item_state( + conn, + item_id=item_id, + remote_state=MutationState.COMMITTED.value, + remote_mutation_id=remote_mutation_id, + ) + elif final_item_state == MutationState.REJECTED.value: + failed_count += 1 + last_err = pub_res.get("message") or "Semantic rejection by MESA" + update_delivery_item_state( + conn, + item_id=item_id, + remote_state=MutationState.REJECTED.value, + remote_mutation_id=remote_mutation_id, + last_error=last_err, + ) + else: + failed_count += 1 + last_err = pub_res.get("message") or "Mutation failed to commit" + update_delivery_item_state( + conn, + item_id=item_id, + remote_state=MutationState.FAILED.value, + remote_mutation_id=remote_mutation_id, + last_error=last_err, + ) + + if progress_callback: + progress_callback( + { + "delivery_id": delivery_id, + "processed": idx, + "total": total_items, + "committed": committed_count, + "failed": failed_count, + "skipped": skipped_count, + } + ) + + # 3. Compute final delivery status + if total_items == 0: + final_delivery_status = DeliveryStatus.COMMITTED.value + elif failed_count == 0: + final_delivery_status = DeliveryStatus.COMMITTED.value + elif committed_count > 0 or skipped_count > 0: + final_delivery_status = DeliveryStatus.PARTIAL.value + else: + final_delivery_status = DeliveryStatus.FAILED.value + + update_delivery_progress( + conn, + delivery_id=delivery_id, + status=final_delivery_status, + committed_items=committed_count, + failed_items=failed_count, + skipped_items=skipped_count, + last_error=last_err, + finished=True, + ) + conn.close() + + return { + "delivery_id": delivery_id, + "status": final_delivery_status, + "total_items": total_items, + "committed_items": committed_count, + "failed_items": failed_count, + "skipped_items": skipped_count, + "last_error": last_err, + } + + +def retry_delivery_failures( + delivery_id: str, + progress_callback: Callable[[dict[str, Any]], None] | None = None, +) -> dict[str, Any]: + """ + Retries only the failed/timed out items of a delivery using the same idempotency keys. + """ + conn = get_connection() + delivery = get_delivery(conn, delivery_id) + if not delivery: + conn.close() + raise ValueError(f"Delivery {delivery_id} not found") + + target_settings = get_mesa_target_settings(conn, delivery.get("target_key", "default")) + client = MesaClient(settings=target_settings) + + failed_items = list_failed_delivery_items(conn, delivery_id) + if not failed_items: + conn.close() + return {"delivery_id": delivery_id, "retried_count": 0, "message": "No failed items eligible for retry"} + + retried_success = 0 + retried_failed = 0 + last_err = None + + for idx, item in enumerate(failed_items, start=1): + item_id = item["item_id"] + idemp_key = item["idempotency_key"] + payload = item["payload"] + chunk = SourceChunk(**payload) + + update_delivery_item_state( + conn, + item_id=item_id, + remote_state=MutationState.SENDING.value, + ) + + pub_res = client.publish_source_chunk(chunk, idempotency_key=idemp_key) + remote_mutation_id = pub_res.get("mutation_id") + final_state = pub_res.get("state", MutationState.FAILED.value) + + if final_state in (MutationState.QUEUED.value, MutationState.PROCESSING.value): + update_delivery_item_state( + conn, + item_id=item_id, + remote_state=MutationState.PROCESSING.value, + remote_mutation_id=remote_mutation_id, + ) + for _ in range(5): + time.sleep(0.5) + poll_res = client.get_mutation_status(remote_mutation_id or "") + polled_state = poll_res.get("state", MutationState.FAILED.value) + if polled_state in ( + MutationState.COMMITTED.value, + MutationState.FAILED.value, + MutationState.REJECTED.value, + ): + final_state = polled_state + if polled_state != MutationState.COMMITTED.value: + last_err = poll_res.get("error") or last_err + break + + if final_state == MutationState.COMMITTED.value: + retried_success += 1 + update_delivery_item_state( + conn, + item_id=item_id, + remote_state=MutationState.COMMITTED.value, + remote_mutation_id=remote_mutation_id, + ) + else: + retried_failed += 1 + last_err = pub_res.get("message") + update_delivery_item_state( + conn, + item_id=item_id, + remote_state=MutationState.FAILED.value, + remote_mutation_id=remote_mutation_id, + last_error=last_err, + ) + + if progress_callback: + progress_callback( + { + "delivery_id": delivery_id, + "retried_processed": idx, + "retried_total": len(failed_items), + "retried_success": retried_success, + "retried_failed": retried_failed, + } + ) + + # Re-calculate overall delivery totals + c = conn.cursor() + c.execute( + """SELECT remote_state, count(*) FROM mesa_delivery_items WHERE delivery_id = ? GROUP BY remote_state""", + (delivery_id,), + ) + counts = dict(c.fetchall()) + committed = counts.get("COMMITTED", 0) + failed = counts.get("FAILED", 0) + counts.get("REJECTED", 0) + skipped = counts.get("SKIPPED", 0) + + if failed == 0: + new_status = DeliveryStatus.COMMITTED.value + elif committed > 0 or skipped > 0: + new_status = DeliveryStatus.PARTIAL.value + else: + new_status = DeliveryStatus.FAILED.value + + update_delivery_progress( + conn, + delivery_id=delivery_id, + status=new_status, + committed_items=committed, + failed_items=failed, + skipped_items=skipped, + last_error=last_err, + finished=True, + ) + conn.close() + + return { + "delivery_id": delivery_id, + "status": new_status, + "retried_count": len(failed_items), + "retried_success": retried_success, + "retried_failed": retried_failed, + "last_error": last_err, + } diff --git a/src/mesa_legal_data/publisher/hashing.py b/src/mesa_legal_data/publisher/hashing.py new file mode 100644 index 0000000..5e0cc2d --- /dev/null +++ b/src/mesa_legal_data/publisher/hashing.py @@ -0,0 +1,62 @@ +import hashlib +import json +from typing import Sequence + +from mesa_legal_data.publisher.models import SourceChunk + + +def calculate_content_hash(text: str) -> str: + """Calculates deterministic SHA256 of text encoded as UTF-8.""" + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def calculate_canonical_revision_hash(chunks: Sequence[SourceChunk]) -> str: + """ + Computes a pure canonical revision hash over ordered source chunks. + Guarantees: + - Strictly deterministic + - Excludes volatile runtime data (timestamps, release_id, session_id, UUIDs) + - Same canonical chunks -> same revision hash + """ + if not chunks: + return hashlib.sha256(b"").hexdigest() + + # Sort chunks deterministically by ordinal and chunk_id + sorted_chunks = sorted(chunks, key=lambda c: (c.ordinal, c.chunk_id)) + hasher = hashlib.sha256() + for chunk in sorted_chunks: + # Feed chunk metadata and content hash + chunk_repr = f"{chunk.chunk_id}:{chunk.chunk_type}:{chunk.char_start}:{chunk.char_end}:{chunk.content_hash}\n" + hasher.update(chunk_repr.encode("utf-8")) + return hasher.hexdigest() + + +def generate_idempotency_key( + *, + tenant_id: str, + workspace_id: str, + dataset_id: str, + document_id: str, + version_id: str, + chunk_id: str, + content_hash: str, +) -> str: + """ + Generates a stable, canonical idempotency key for MESA v4 mutation. + Guarantees: + - Pure function of stable target scope + logical identity + content hash + - No timestamps, random UUIDs, or release IDs + - Canonical UTF-8 JSON serialization with sorted keys + """ + stable_payload = { + "tenant_id": tenant_id, + "workspace_id": workspace_id, + "dataset_id": dataset_id, + "document_id": document_id, + "version_id": version_id, + "chunk_id": chunk_id, + "content_hash": content_hash, + } + canonical_json = json.dumps(stable_payload, sort_keys=True, separators=(",", ":")) + payload_hash = hashlib.sha256(canonical_json.encode("utf-8")).hexdigest() + return f"mesa-data:{payload_hash}" diff --git a/src/mesa_legal_data/publisher/ledger.py b/src/mesa_legal_data/publisher/ledger.py new file mode 100644 index 0000000..13742ed --- /dev/null +++ b/src/mesa_legal_data/publisher/ledger.py @@ -0,0 +1,405 @@ +import json +import sqlite3 +from datetime import UTC, datetime +from typing import Any + +from mesa_legal_data.catalog import transaction +from mesa_legal_data.publisher.models import ( + DeliveryStatus, + MesaTargetSettings, +) + + +def get_mesa_target_settings(conn: sqlite3.Connection, target_key: str = "default") -> MesaTargetSettings: + """Fetches non-secret MESA target settings.""" + cursor = conn.cursor() + cursor.execute( + """SELECT target_key, base_url, tenant_id, workspace_id, dataset_id, agent_id, content_limit_chars, updated_at, + contract_source, health_path, publish_path, mutation_status_path_template + FROM mesa_target_settings WHERE target_key = ?""", + (target_key,), + ) + row = cursor.fetchone() + if row: + return MesaTargetSettings( + target_key=row[0], + base_url=row[1], + tenant_id=row[2], + workspace_id=row[3], + dataset_id=row[4], + agent_id=row[5], + content_limit_chars=row[6] or 32768, + updated_at=row[7], + contract_source=row[8], + health_path=row[9], + publish_path=row[10], + mutation_status_path_template=row[11], + ) + # Return sensible default + return MesaTargetSettings(target_key=target_key) + + +def upsert_mesa_target_settings(conn: sqlite3.Connection, settings: MesaTargetSettings) -> None: + """Upserts non-secret MESA target settings.""" + now_iso = datetime.now(UTC).isoformat() + with transaction(conn): + conn.execute( + """INSERT INTO mesa_target_settings ( + target_key, base_url, tenant_id, workspace_id, dataset_id, agent_id, + content_limit_chars, updated_at, contract_source, health_path, + publish_path, mutation_status_path_template + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(target_key) DO UPDATE SET + base_url = excluded.base_url, + tenant_id = excluded.tenant_id, + workspace_id = excluded.workspace_id, + dataset_id = excluded.dataset_id, + agent_id = excluded.agent_id, + content_limit_chars = excluded.content_limit_chars, + updated_at = excluded.updated_at, + contract_source = excluded.contract_source, + health_path = excluded.health_path, + publish_path = excluded.publish_path, + mutation_status_path_template = excluded.mutation_status_path_template""", + ( + settings.target_key, + settings.base_url, + settings.tenant_id, + settings.workspace_id, + settings.dataset_id, + settings.agent_id, + settings.content_limit_chars, + now_iso, + settings.contract_source, + settings.health_path, + settings.publish_path, + settings.mutation_status_path_template, + ), + ) + + +def create_delivery( + conn: sqlite3.Connection, + *, + delivery_id: str, + release_id: str | None, + target_key: str, + total_items: int, +) -> None: + now_iso = datetime.now(UTC).isoformat() + with transaction(conn): + conn.execute( + """INSERT INTO mesa_deliveries (delivery_id, release_id, target_key, status, started_at, total_items, committed_items, failed_items, skipped_items, created_at) + VALUES (?, ?, ?, ?, ?, ?, 0, 0, 0, ?)""", + (delivery_id, release_id, target_key, DeliveryStatus.PLANNED.value, now_iso, total_items, now_iso), + ) + + +def update_delivery_progress( + conn: sqlite3.Connection, + *, + delivery_id: str, + status: str, + committed_items: int, + failed_items: int, + skipped_items: int, + last_error: str | None = None, + finished: bool = False, +) -> None: + now_iso = datetime.now(UTC).isoformat() if finished else None + with transaction(conn): + conn.execute( + """UPDATE mesa_deliveries + SET status = ?, + committed_items = ?, + failed_items = ?, + skipped_items = ?, + last_error = ?, + finished_at = COALESCE(?, finished_at) + WHERE delivery_id = ?""", + (status, committed_items, failed_items, skipped_items, last_error, now_iso, delivery_id), + ) + + +def insert_delivery_item( + conn: sqlite3.Connection, + *, + item_id: str, + delivery_id: str, + document_id: str, + version_id: str, + chunk_id: str, + content_hash: str, + idempotency_key: str, + remote_state: str, + payload_json: str, +) -> None: + now_iso = datetime.now(UTC).isoformat() + with transaction(conn): + conn.execute( + """INSERT INTO mesa_delivery_items (item_id, delivery_id, document_id, version_id, chunk_id, content_hash, idempotency_key, remote_state, payload_json, updated_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + item_id, + delivery_id, + document_id, + version_id, + chunk_id, + content_hash, + idempotency_key, + remote_state, + payload_json, + now_iso, + now_iso, + ), + ) + + +def update_delivery_item_state( + conn: sqlite3.Connection, + *, + item_id: str, + remote_state: str, + remote_mutation_id: str | None = None, + last_error: str | None = None, +) -> None: + now_iso = datetime.now(UTC).isoformat() + with transaction(conn): + conn.execute( + """UPDATE mesa_delivery_items + SET remote_state = ?, + remote_mutation_id = COALESCE(?, remote_mutation_id), + last_error = ?, + updated_at = ? + WHERE item_id = ?""", + (remote_state, remote_mutation_id, last_error, now_iso, item_id), + ) + + +def is_chunk_already_committed( + conn: sqlite3.Connection, + *, + target_key: str, + document_id: str, + version_id: str, + chunk_id: str, + content_hash: str, +) -> bool: + """ + Cross-release deduplication check: + Returns True if an item with identical document, version, chunk, and content_hash + was successfully COMMITTED to the target in any past delivery. + """ + cursor = conn.cursor() + cursor.execute( + """SELECT 1 FROM mesa_delivery_items i + JOIN mesa_deliveries d ON i.delivery_id = d.delivery_id + WHERE d.target_key = ? + AND i.document_id = ? + AND i.version_id = ? + AND i.chunk_id = ? + AND i.content_hash = ? + AND i.remote_state = 'COMMITTED' + LIMIT 1""", + (target_key, document_id, version_id, chunk_id, content_hash), + ) + return cursor.fetchone() is not None + + +def list_deliveries(conn: sqlite3.Connection, limit: int = 20, offset: int = 0) -> list[dict[str, Any]]: + cursor = conn.cursor() + cursor.execute( + """SELECT delivery_id, release_id, target_key, status, started_at, finished_at, + total_items, committed_items, failed_items, skipped_items, last_error, created_at + FROM mesa_deliveries ORDER BY created_at DESC LIMIT ? OFFSET ?""", + (limit, offset), + ) + rows = cursor.fetchall() + return [ + { + "delivery_id": r[0], + "release_id": r[1], + "target_key": r[2], + "status": r[3], + "started_at": r[4], + "finished_at": r[5], + "total_items": r[6], + "committed_items": r[7], + "failed_items": r[8], + "skipped_items": r[9], + "last_error": r[10], + "created_at": r[11], + } + for r in rows + ] + + +def get_delivery(conn: sqlite3.Connection, delivery_id: str) -> dict[str, Any] | None: + cursor = conn.cursor() + cursor.execute( + """SELECT delivery_id, release_id, target_key, status, started_at, finished_at, + total_items, committed_items, failed_items, skipped_items, last_error, created_at + FROM mesa_deliveries WHERE delivery_id = ?""", + (delivery_id,), + ) + r = cursor.fetchone() + if not r: + return None + return { + "delivery_id": r[0], + "release_id": r[1], + "target_key": r[2], + "status": r[3], + "started_at": r[4], + "finished_at": r[5], + "total_items": r[6], + "committed_items": r[7], + "failed_items": r[8], + "skipped_items": r[9], + "last_error": r[10], + "created_at": r[11], + } + + +def list_delivery_items( + conn: sqlite3.Connection, + delivery_id: str, + state: str | None = None, + limit: int = 100, + offset: int = 0, +) -> list[dict[str, Any]]: + cursor = conn.cursor() + if state: + cursor.execute( + """SELECT item_id, delivery_id, document_id, version_id, chunk_id, content_hash, + idempotency_key, remote_mutation_id, remote_state, payload_json, last_error, updated_at, created_at + FROM mesa_delivery_items WHERE delivery_id = ? AND remote_state = ? + ORDER BY created_at ASC LIMIT ? OFFSET ?""", + (delivery_id, state, limit, offset), + ) + else: + cursor.execute( + """SELECT item_id, delivery_id, document_id, version_id, chunk_id, content_hash, + idempotency_key, remote_mutation_id, remote_state, payload_json, last_error, updated_at, created_at + FROM mesa_delivery_items WHERE delivery_id = ? + ORDER BY created_at ASC LIMIT ? OFFSET ?""", + (delivery_id, limit, offset), + ) + rows = cursor.fetchall() + results = [] + for r in rows: + results.append( + { + "item_id": r[0], + "delivery_id": r[1], + "document_id": r[2], + "version_id": r[3], + "chunk_id": r[4], + "content_hash": r[5], + "idempotency_key": r[6], + "remote_mutation_id": r[7], + "remote_state": r[8], + "payload": json.loads(r[9]) if r[9] else {}, + "last_error": r[10], + "updated_at": r[11], + "created_at": r[12], + } + ) + return results + + +def list_failed_delivery_items(conn: sqlite3.Connection, delivery_id: str) -> list[dict[str, Any]]: + """Returns items eligible for retry in a delivery (FAILED or SENDING that timed out).""" + cursor = conn.cursor() + cursor.execute( + """SELECT item_id, delivery_id, document_id, version_id, chunk_id, content_hash, + idempotency_key, remote_mutation_id, remote_state, payload_json, last_error, updated_at, created_at + FROM mesa_delivery_items + WHERE delivery_id = ? AND remote_state IN ('FAILED', 'SENDING', 'AWAITING_MUTATION') + ORDER BY created_at ASC""", + (delivery_id,), + ) + rows = cursor.fetchall() + return [ + { + "item_id": r[0], + "delivery_id": r[1], + "document_id": r[2], + "version_id": r[3], + "chunk_id": r[4], + "content_hash": r[5], + "idempotency_key": r[6], + "remote_mutation_id": r[7], + "remote_state": r[8], + "payload": json.loads(r[9]) if r[9] else {}, + "last_error": r[10], + "updated_at": r[11], + "created_at": r[12], + } + for r in rows + ] + + +def get_document_mesa_status(conn: sqlite3.Connection, document_id: str) -> dict[str, Any]: + """ + Computes truthful document-level MESA status: + - 'Committed': All latest version chunks COMMITTED + - 'Sending': Delivery in progress + - 'Partial': Some chunks committed, some failed + - 'Failed': Delivery failed + - 'Not Sent': No delivery recorded + """ + cursor = conn.cursor() + cursor.execute( + """WITH delivered_versions AS ( + SELECT i.version_id, + ROW_NUMBER() OVER ( + ORDER BY COALESCE(v.revision_number, 1) DESC, v.created_at DESC + ) AS version_rank + FROM mesa_delivery_items i + LEFT JOIN versions v ON v.version_id = i.version_id + WHERE i.document_id = ? + GROUP BY i.version_id + ), latest_items AS ( + SELECT i.remote_state, + ROW_NUMBER() OVER ( + PARTITION BY i.chunk_id + ORDER BY i.updated_at DESC, i.created_at DESC + ) AS attempt_rank + FROM mesa_delivery_items i + JOIN delivered_versions dv ON dv.version_id = i.version_id AND dv.version_rank = 1 + ) + SELECT remote_state, count(*) + FROM latest_items + WHERE attempt_rank = 1 + GROUP BY remote_state""", + (document_id,), + ) + counts = dict((r[0], r[1]) for r in cursor.fetchall()) + if not counts: + return {"status": "Not Sent", "committed_count": 0, "failed_count": 0, "total_chunks": 0} + + total = sum(counts.values()) + committed = counts.get("COMMITTED", 0) + counts.get("SKIPPED", 0) + failed = counts.get("FAILED", 0) + counts.get("REJECTED", 0) + sending = counts.get("SENDING", 0) + counts.get("QUEUED", 0) + counts.get("PROCESSING", 0) + + if sending > 0: + status = "Sending" + elif failed > 0 and committed > 0: + status = "Partial" + elif failed > 0 and committed == 0: + status = "Failed" + elif committed == total and total > 0: + status = "Committed" + else: + status = "Not Sent" + + return { + "status": status, + "committed_count": committed, + "failed_count": failed, + "total_chunks": total, + "counts_by_state": counts, + } diff --git a/src/mesa_legal_data/publisher/models.py b/src/mesa_legal_data/publisher/models.py new file mode 100644 index 0000000..91c8de5 --- /dev/null +++ b/src/mesa_legal_data/publisher/models.py @@ -0,0 +1,86 @@ +from enum import Enum +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class DeliveryStatus(str, Enum): + PLANNED = "PLANNED" + SENDING = "SENDING" + AWAITING_MUTATION = "AWAITING_MUTATION" + COMMITTED = "COMMITTED" + PARTIAL = "PARTIAL" + FAILED = "FAILED" + CANCELLED = "CANCELLED" + + +class MutationState(str, Enum): + PLANNED = "PLANNED" + SENDING = "SENDING" + QUEUED = "QUEUED" + PROCESSING = "PROCESSING" + COMMITTED = "COMMITTED" + FAILED = "FAILED" + REJECTED = "REJECTED" + SKIPPED = "SKIPPED" + + +class MesaTargetSettings(BaseModel): + model_config = ConfigDict(extra="forbid") + target_key: str = Field(default="default") + base_url: str = Field(default="") + tenant_id: str = Field(default="default") + workspace_id: str = Field(default="legal") + dataset_id: str = Field(default="tr_legislation") + agent_id: str = Field(default="mesa_data_publisher") + content_limit_chars: int = Field(default=32768, ge=1024, le=1048576) + contract_source: Literal["unknown", "configured", "live_verified"] = "unknown" + health_path: str = "" + publish_path: str = "" + mutation_status_path_template: str = "" + api_key_configured: bool = Field(default=False) + updated_at: str | None = None + + +class SourceChunk(BaseModel): + model_config = ConfigDict(extra="forbid") + chunk_id: str + document_id: str + version_id: str + chunk_type: Literal["preamble", "article", "annex", "decision_header", "decision_body", "general"] + title: str | None = None + char_start: int + char_end: int + ordinal: int + content: str + content_hash: str + metadata: dict[str, Any] = Field(default_factory=dict) + + +class PreflightCheckItem(BaseModel): + model_config = ConfigDict(extra="forbid") + name: str + status: Literal["PASS", "FAIL", "WARN"] + message: str + + +class PreflightReport(BaseModel): + model_config = ConfigDict(extra="forbid") + overall_status: Literal["PASS", "FAIL"] + checks: list[PreflightCheckItem] + ready_documents_count: int + ready_versions_count: int + estimated_chunks_count: int + total_canonical_bytes: int + + +class DeliveryPlanSummary(BaseModel): + model_config = ConfigDict(extra="forbid") + ready_documents: int + ready_versions: int + estimated_chunks: int + total_canonical_bytes: int + already_committed_chunks: int + new_chunks_to_send: int + blocked_versions_excluded: int + unreadable_versions_excluded: int = 0 diff --git a/src/mesa_legal_data/quality.py b/src/mesa_legal_data/quality.py new file mode 100644 index 0000000..d1802a4 --- /dev/null +++ b/src/mesa_legal_data/quality.py @@ -0,0 +1,357 @@ +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any, Literal + +from mesa_legal_data.parsers.coverage import ParsingCoverage +from mesa_legal_data.parsers.encoding import detect_mojibake + +QualityDecision = Literal["PASS", "REVIEW", "BLOCK"] + + +@dataclass(frozen=True) +class CheckResult: + group: str + name: str + status: QualityDecision + message: str + details: dict[str, Any] | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "group": self.group, + "name": self.name, + "status": self.status, + "message": self.message, + "details": self.details or {}, + } + + +@dataclass(frozen=True) +class QualityReport: + decision: QualityDecision + checked_at: str + parser_name: str + parser_version: str + checks: list[CheckResult] + coverage: dict[str, Any] | None = None + summary: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "decision": self.decision, + "checked_at": self.checked_at, + "parser_name": self.parser_name, + "parser_version": self.parser_version, + "summary": self.summary, + "coverage": self.coverage, + "checks": [c.to_dict() for c in self.checks], + } + + +def evaluate_quality( + *, + source_info: dict[str, Any], + raw_info: dict[str, Any], + canonical_records: list[dict[str, Any]], + canonical_text: str, + coverage: ParsingCoverage | None = None, + privacy_issues: list[dict[str, Any]] | None = None, + parser_name: str = "parser", + parser_version: str = "1.0.0", +) -> QualityReport: + """ + Deterministic Quality Gate evaluating 9 standard check groups. + Produces strictly PASS, REVIEW, or BLOCK. No arbitrary quality percentages. + """ + now_iso = datetime.now(UTC).isoformat() + checks: list[CheckResult] = [] + + # 1. SOURCE GROUP + s_id = source_info.get("source_id") + s_url = source_info.get("source_url") + if not s_id: + checks.append(CheckResult("SOURCE", "source_known", "BLOCK", "Missing source_id")) + else: + checks.append(CheckResult("SOURCE", "source_known", "PASS", f"Source recognized: {s_id}")) + + if not s_url or not str(s_url).startswith(("http://", "https://", "file://")): + checks.append(CheckResult("SOURCE", "source_url", "REVIEW", f"Unusual or missing source_url: {s_url}")) + else: + checks.append(CheckResult("SOURCE", "source_url", "PASS", "Source URL present and valid")) + + # 2. RAW GROUP + byte_size = raw_info.get("byte_size", 0) + raw_sha = raw_info.get("sha256") + file_exists = raw_info.get("file_exists", True) + + if not file_exists: + checks.append(CheckResult("RAW", "artifact_exists", "BLOCK", "Raw artifact file does not exist on disk")) + elif byte_size <= 0: + checks.append(CheckResult("RAW", "artifact_non_empty", "BLOCK", "Raw artifact is empty (0 bytes)")) + else: + checks.append(CheckResult("RAW", "artifact_integrity", "PASS", f"Raw artifact valid ({byte_size} bytes)")) + + if not raw_sha or len(raw_sha) != 64: + checks.append(CheckResult("RAW", "artifact_sha", "BLOCK", f"Invalid artifact SHA256: {raw_sha}")) + else: + checks.append(CheckResult("RAW", "artifact_sha", "PASS", "Artifact SHA256 present and well-formed")) + + # 3. EXTRACTION GROUP + if not canonical_text or not canonical_text.strip(): + checks.append(CheckResult("EXTRACTION", "text_non_empty", "BLOCK", "Extracted canonical text is empty")) + else: + checks.append( + CheckResult("EXTRACTION", "text_non_empty", "PASS", f"Extracted {len(canonical_text)} characters") + ) + + mojibake_issues = detect_mojibake(canonical_text) + replacement_count = canonical_text.count("\ufffd") + severe_replacement_loss = replacement_count >= 10 or ( + len(canonical_text) >= 1000 and replacement_count / len(canonical_text) >= 0.01 + ) + if any("\\x00" in iss for iss in mojibake_issues): + checks.append( + CheckResult( + "EXTRACTION", + "encoding_corruption", + "BLOCK", + "Null bytes detected in canonical text", + {"issues": mojibake_issues}, + ) + ) + elif severe_replacement_loss: + checks.append( + CheckResult( + "EXTRACTION", + "severe_encoding_loss", + "BLOCK", + f"Severe Unicode replacement loss detected ({replacement_count} characters)", + {"issues": mojibake_issues}, + ) + ) + elif any("\\ufffd" in iss for iss in mojibake_issues): + checks.append( + CheckResult( + "EXTRACTION", + "replacement_chars", + "REVIEW", + "Unicode replacement character found in extraction", + {"issues": mojibake_issues}, + ) + ) + elif mojibake_issues: + checks.append( + CheckResult( + "EXTRACTION", + "mojibake_anomaly", + "REVIEW", + "Potential Turkish mojibake pattern detected", + {"issues": mojibake_issues}, + ) + ) + else: + checks.append(CheckResult("EXTRACTION", "encoding_health", "PASS", "Text clean of mojibake and corruption")) + + # 4. STRUCTURE GROUP + leg_recs = [r for r in canonical_records if r.get("record_type") == "legislation"] + art_recs = [r for r in canonical_records if r.get("record_type") == "article"] + + if not canonical_records: + checks.append( + CheckResult("STRUCTURE", "records_generated", "BLOCK", "No canonical records generated by parser") + ) + else: + checks.append( + CheckResult("STRUCTURE", "records_generated", "PASS", f"Generated {len(canonical_records)} records") + ) + + # Span validation + span_errors = [] + text_len = len(canonical_text) + prev_ord = 0 + ord_disorder = False + + for a in art_recs: + span = a.get("source_span") + if span: + c_start = span.get("char_start") + c_end = span.get("char_end") + if c_start is not None and c_end is not None: + if c_start < 0 or c_end > text_len or c_start > c_end: + span_errors.append(f"Invalid span [{c_start}, {c_end}] for article {a.get('id')}") + else: + source_slice = canonical_text[c_start:c_end] + article_text = a.get("text") + article_number = str(a.get("article_number") or "").strip() + if article_text and article_text not in source_slice: + span_errors.append(f"Span does not contain article text for {a.get('id')}") + if article_number and article_number not in source_slice[:100]: + span_errors.append(f"Span does not start at article {article_number} for {a.get('id')}") + else: + span_errors.append(f"Incomplete span for article {a.get('id')}") + else: + span_errors.append(f"Missing span for article {a.get('id')}") + ord_val = a.get("ordinal") or 0 + if ord_val and ord_val <= prev_ord: + ord_disorder = True + if ord_val: + prev_ord = ord_val + + if span_errors: + checks.append( + CheckResult( + "STRUCTURE", + "span_validity", + "BLOCK", + "Article source spans corrupt or out of bounds", + {"errors": span_errors}, + ) + ) + else: + checks.append(CheckResult("STRUCTURE", "span_validity", "PASS", "All article source spans within valid bounds")) + + if ord_disorder: + checks.append( + CheckResult("STRUCTURE", "ordinal_sequence", "REVIEW", "Article ordinals not strictly increasing") + ) + else: + checks.append(CheckResult("STRUCTURE", "ordinal_sequence", "PASS", "Article sequence valid")) + + # Coverage evaluation + cov_dict = None + if coverage: + cov_dict = coverage.to_dict() + unexplained_gaps = [ + gap + for gap in coverage.uncovered_ranges + if gap.get("candidate_type") == "gap" + and gap.get("length", 0) >= max(1000, int(max(1, coverage.canonical_chars) * 0.05)) + ] + if coverage.coverage_ratio < 0.20 and leg_recs and art_recs: + checks.append( + CheckResult( + "STRUCTURE", "coverage", "REVIEW", f"Low parser coverage: {coverage.coverage_ratio * 100:.1f}%" + ) + ) + elif unexplained_gaps: + checks.append( + CheckResult( + "STRUCTURE", + "coverage", + "REVIEW", + "Large unexplained gaps exist between parsed legal units", + {"gaps": unexplained_gaps}, + ) + ) + else: + checks.append( + CheckResult("STRUCTURE", "coverage", "PASS", f"Parser coverage: {coverage.coverage_ratio * 100:.1f}%") + ) + elif leg_recs: + checks.append(CheckResult("STRUCTURE", "coverage", "REVIEW", "Legislation parsing coverage was not supplied")) + + # 5. METADATA GROUP + doc_id = canonical_records[0].get("id") if canonical_records else None + if not doc_id: + checks.append(CheckResult("METADATA", "document_identity", "BLOCK", "Missing document identity")) + else: + checks.append(CheckResult("METADATA", "document_identity", "PASS", f"Document ID: {doc_id}")) + + title = None + for r in canonical_records: + if r.get("title"): + title = r.get("title") + break + if r.get("court"): + title = f"{r.get('court')} Kararı" + break + if not title: + checks.append(CheckResult("METADATA", "title_present", "REVIEW", "Missing or empty document title")) + else: + checks.append(CheckResult("METADATA", "title_present", "PASS", f"Title: {title[:50]}")) + + # 6. CITATION GROUP + cit_recs = [r for r in canonical_records if r.get("record_type") == "citation"] + cit_corrupt = False + for c in cit_recs: + span = c.get("source_span") + if span: + cs = span.get("char_start", 0) + ce = span.get("char_end", 0) + if cs is not None and ce is not None and (cs < 0 or ce > text_len or cs > ce): + cit_corrupt = True + break + if cit_corrupt: + checks.append(CheckResult("CITATION", "citation_spans", "BLOCK", "Corrupt citation span coordinates")) + else: + checks.append(CheckResult("CITATION", "citation_extraction", "PASS", f"Citations evaluated: {len(cit_recs)}")) + + # 7. PRIVACY GROUP + priv_issues = privacy_issues or [] + blocker_priv = [i for i in priv_issues if i.get("severity") == "blocker"] + warning_priv = [i for i in priv_issues if i.get("severity") in ("warning", "error")] + if blocker_priv: + checks.append( + CheckResult( + "PRIVACY", + "privacy_scan", + "BLOCK", + f"{len(blocker_priv)} privacy blocker(s) detected", + {"issues": blocker_priv}, + ) + ) + elif warning_priv: + checks.append( + CheckResult( + "PRIVACY", + "privacy_scan", + "REVIEW", + f"{len(warning_priv)} privacy warning(s) flagged", + {"issues": warning_priv}, + ) + ) + else: + checks.append(CheckResult("PRIVACY", "privacy_scan", "PASS", "No privacy issues detected")) + + # 8. PROVENANCE GROUP + provenance_missing = False + for r in canonical_records: + src = r.get("source") + prov = r.get("provenance") + if not src or not prov or not src.get("artifact_sha256") or not prov.get("pipeline_run_id"): + provenance_missing = True + break + if provenance_missing: + checks.append( + CheckResult("PROVENANCE", "provenance_linkage", "BLOCK", "Missing source or provenance metadata on records") + ) + else: + checks.append( + CheckResult( + "PROVENANCE", "provenance_linkage", "PASS", "Provenance links complete from raw artifact to canonical" + ) + ) + + # 9. DUPLICATE GROUP + checks.append(CheckResult("DUPLICATE", "duplicate_evaluation", "PASS", "Duplicate check evaluated")) + + # Aggregate decision: BLOCK > REVIEW > PASS + statuses = [c.status for c in checks] + if "BLOCK" in statuses: + overall: QualityDecision = "BLOCK" + elif "REVIEW" in statuses: + overall = "REVIEW" + else: + overall = "PASS" + + summary = f"Quality Gate Result: {overall} ({statuses.count('PASS')} PASS, {statuses.count('REVIEW')} REVIEW, {statuses.count('BLOCK')} BLOCK)" + + return QualityReport( + decision=overall, + checked_at=now_iso, + parser_name=parser_name, + parser_version=parser_version, + checks=checks, + coverage=cov_dict, + summary=summary, + ) diff --git a/src/mesa_legal_data/release/builder.py b/src/mesa_legal_data/release/builder.py index c0ec9ab..dd98c38 100644 --- a/src/mesa_legal_data/release/builder.py +++ b/src/mesa_legal_data/release/builder.py @@ -271,7 +271,7 @@ def build_release(release_id: str | None = None) -> dict[str, Any]: COUNT(DISTINCT a.artifact_id) as artifact_count, MAX(a.retrieved_at) as latest_retrieved_at FROM spool.selected_records sr - JOIN records r ON r.record_id = sr.record_id + JOIN records r ON r.record_id = sr.record_id AND r.version_id = sr.version_id JOIN versions v ON v.version_id = r.version_id JOIN artifacts a ON a.artifact_id = v.artifact_id LEFT JOIN sources s ON s.source_id = a.source_id @@ -344,24 +344,24 @@ def build_release(release_id: str | None = None) -> dict[str, Any]: ) item_cur = spool_conn.cursor() - item_cur.execute("SELECT record_id, record_sha256 FROM payload_spool") + item_cur.execute("SELECT record_id, record_sha256, version_id FROM selected_records") item_batch = [] while True: rows = item_cur.fetchmany(batch_size) if not rows: break for r in rows: - item_batch.append((release_id, r[0], r[1])) + item_batch.append((release_id, r[0], r[1], r[2])) if len(item_batch) >= batch_size: conn.executemany( - "INSERT INTO release_items (release_id, record_id, record_sha256) VALUES (?, ?, ?)", + "INSERT INTO release_items (release_id, record_id, record_sha256, version_id) VALUES (?, ?, ?, ?)", item_batch, ) item_batch.clear() if item_batch: conn.executemany( - "INSERT INTO release_items (release_id, record_id, record_sha256) VALUES (?, ?, ?)", + "INSERT INTO release_items (release_id, record_id, record_sha256, version_id) VALUES (?, ?, ?, ?)", item_batch, ) item_batch.clear() diff --git a/src/mesa_legal_data/web/api.py b/src/mesa_legal_data/web/api.py index 632b4d7..8ab9a48 100644 --- a/src/mesa_legal_data/web/api.py +++ b/src/mesa_legal_data/web/api.py @@ -18,14 +18,33 @@ get_export_package, get_record, get_release, + get_source_operational_settings, list_open_blocking_issues, + list_parser_certifications, + list_source_operational_settings, reject_record_with_checks, reject_version, resolve_issue, + set_parser_certification, + upsert_source_operational_settings, ) from mesa_legal_data.config import load_settings, load_sources from mesa_legal_data.parsers import decode_source_bytes from mesa_legal_data.pipeline import process_artifact_pipeline +from mesa_legal_data.publisher.client import MesaClient +from mesa_legal_data.publisher.engine import ( + build_delivery_plan, + retry_delivery_failures, +) +from mesa_legal_data.publisher.ledger import ( + get_delivery, + get_document_mesa_status, + get_mesa_target_settings, + list_deliveries, + list_delivery_items, + upsert_mesa_target_settings, +) +from mesa_legal_data.publisher.models import MesaTargetSettings from mesa_legal_data.release import build_release, verify_release from mesa_legal_data.release.importer import ( ReleaseNotFound, @@ -41,9 +60,13 @@ from mesa_legal_data.web.schemas import ( HarvestStartRequest, IssueResolveRequest, + MesaPublishRequest, + MesaTargetSettingsUpdateRequest, + ParserCertifyRequest, ReleaseCreateRequest, ReviewRequest, RevokeRequest, + SourceSettingsUpdateRequest, UrlImportRequest, ) from mesa_legal_data.web.security import verify_security, write_lock @@ -113,6 +136,34 @@ def get_dashboard(): c.execute("SELECT count(*) FROM releases WHERE status = 'published'") published_releases = c.fetchone()[0] + # Health & Operational Metrics + c.execute("SELECT count(*) FROM artifacts WHERE retrieved_at >= date('now')") + discovered_today = c.fetchone()[0] + + c.execute("SELECT count(*) FROM processing_runs WHERE started_at >= date('now')") + processed_today = c.fetchone()[0] + + c.execute("SELECT count(*) FROM versions WHERE auto_approved = 1 AND created_at >= date('now')") + auto_approved_today = c.fetchone()[0] + + c.execute( + "SELECT count(*) FROM versions WHERE approval_status = 'pending' AND (quality_status IS NULL OR quality_status != 'BLOCK')" + ) + needs_review_count = c.fetchone()[0] + + c.execute("SELECT count(*) FROM versions WHERE quality_status = 'BLOCK' OR validation_status = 'failed'") + blocked_count = c.fetchone()[0] + + c.execute(""" + SELECT count(DISTINCT r.record_instance_id) FROM records r + JOIN versions v ON r.version_id = v.version_id + WHERE r.approval_status = 'approved' + AND r.validation_status = 'valid' + AND v.validation_status = 'valid' + AND (v.quality_status IS NULL OR v.quality_status != 'BLOCK') + """) + mesa_ready_count = c.fetchone()[0] + c.execute( "SELECT document_id, family, document_type, title, lifecycle_status, updated_at FROM documents ORDER BY updated_at DESC LIMIT 10" ) @@ -160,6 +211,16 @@ def get_dashboard(): "published_releases": published_releases, "active_release_id": active_release_id, }, + "health": { + "discovered_today": discovered_today, + "processed_today": processed_today, + "auto_approved_today": auto_approved_today, + "needs_review_count": needs_review_count, + "blocked_count": blocked_count, + "mesa_ready_count": mesa_ready_count, + "mesa_status": "not_configured", + "mesa_status_label": "MESA entegrasyonu yapılandırılmadı (Yerel Staging Aktif)", + }, "recent_documents": recent_docs, "recent_runs": recent_runs, "harvest": harvest_summary, @@ -462,6 +523,53 @@ def list_sources_endpoint(): return ok_response(sources_list) +@router.get("/sources/settings") +def get_sources_settings(): + conn = get_connection() + try: + sources_settings = list_source_operational_settings(conn) + return ok_response(sources_settings) + finally: + conn.close() + + +@router.post("/sources/{source_id}/settings") +async def update_source_settings(source_id: str, req: SourceSettingsUpdateRequest): + async with write_lock.acquire_write(): + conn = get_connection() + try: + upsert_source_operational_settings( + conn, + source_id=source_id, + enabled=req.enabled, + auto_approval_enabled=req.auto_approval_enabled, + weekly_sample_count=req.weekly_sample_count, + ) + settings = get_source_operational_settings(conn, source_id) + return ok_response(settings) + finally: + conn.close() + + +@router.post("/sources/{source_id}/certify-parser") +async def certify_parser_endpoint(source_id: str, req: ParserCertifyRequest): + async with write_lock.acquire_write(): + conn = get_connection() + try: + set_parser_certification( + conn, + source_id=source_id, + parser_name=req.parser_name, + parser_version=req.parser_version, + certified=req.certified, + certified_by=req.certified_by or "operator", + ) + certs = list_parser_certifications(conn, source_id=source_id) + return ok_response(certs) + finally: + conn.close() + + # 8.4 Documents @router.get("/documents") def list_documents( @@ -909,6 +1017,67 @@ def get_document_text_content(document_id: str): } +@router.post("/documents/{document_id:path}/reprocess") +async def reprocess_document(document_id: str): + return await process_document_pipeline(document_id=document_id) + + +@router.get("/documents/{document_id:path}/versions") +def get_document_versions(document_id: str): + conn = get_connection() + c = conn.cursor() + c.execute( + """SELECT version_id, document_id, artifact_id, version_kind, snapshot_date, + revision_number, supersedes_version_id, quality_status, quality_json, + validation_status, privacy_status, approval_status, created_at, + is_audit_sample, audit_sample_reason, auto_approved + FROM versions WHERE document_id = ? ORDER BY revision_number DESC, created_at DESC""", + (document_id,), + ) + versions = [] + for r in c.fetchall(): + q_data = None + if r[8]: + try: + q_data = json.loads(r[8]) + except Exception: + pass + versions.append( + { + "version_id": r[0], + "document_id": r[1], + "artifact_id": r[2], + "version_kind": r[3], + "snapshot_date": r[4], + "revision_number": r[5] or 1, + "supersedes_version_id": r[6], + "quality_status": r[7], + "quality_json": q_data, + "validation_status": r[9], + "privacy_status": r[10], + "approval_status": r[11], + "created_at": r[12], + "is_audit_sample": bool(r[13]), + "audit_sample_reason": r[14], + "auto_approved": bool(r[15]), + } + ) + conn.close() + return ok_response({"document_id": document_id, "versions": versions}) + + +@router.get("/documents/{document_id:path}/mesa-status") +def get_document_mesa_status_endpoint(document_id: str): + conn = get_connection() + doc = get_document(conn, document_id) + if not doc: + conn.close() + error_response("DOCUMENT_NOT_FOUND", f"Document {document_id} not found", status_code=404) + status_data = get_document_mesa_status(conn, document_id) + conn.close() + return ok_response(status_data) + + @router.get("/documents/{document_id:path}") def get_document_detail(document_id: str): conn = get_connection() @@ -1055,6 +1224,63 @@ async def process_document_pipeline(document_id: str): error_response("PIPELINE_FAILED", f"Pipeline failed: {e}", status_code=400) +@router.get("/reviews/pending-versions") +@router.get("/versions/pending") +def list_pending_versions( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), +): + conn = get_connection() + c = conn.cursor() + c.execute( + """SELECT count(*) FROM versions WHERE approval_status = 'pending' AND (quality_status IS NULL OR quality_status != 'BLOCK')""" + ) + total = c.fetchone()[0] + offset = (page - 1) * page_size + + c.execute( + """SELECT v.version_id, v.document_id, d.title, d.family, a.source_id, a.source_url, a.raw_path, + v.canonical_path, v.snapshot_date, v.revision_number, v.quality_status, v.quality_json, + v.is_audit_sample, v.audit_sample_reason, v.created_at + FROM versions v + JOIN documents d ON v.document_id = d.document_id + LEFT JOIN artifacts a ON v.artifact_id = a.artifact_id + WHERE v.approval_status = 'pending' AND (v.quality_status IS NULL OR v.quality_status != 'BLOCK') + ORDER BY v.is_audit_sample DESC, v.created_at DESC + LIMIT ? OFFSET ?""", + (page_size, offset), + ) + items = [] + for r in c.fetchall(): + q_data = None + if r[11]: + try: + q_data = json.loads(r[11]) + except Exception: + pass + items.append( + { + "version_id": r[0], + "document_id": r[1], + "document_title": r[2] or r[1], + "family": r[3], + "source_id": r[4], + "source_url": r[5], + "raw_path": r[6], + "canonical_path": r[7], + "snapshot_date": r[8], + "revision_number": r[9] or 1, + "quality_status": r[10], + "quality_json": q_data, + "is_audit_sample": bool(r[12]), + "audit_sample_reason": r[13], + "created_at": r[14], + } + ) + conn.close() + return ok_response({"items": items, "total": total, "page": page, "page_size": page_size}) + + # 8.6 Records @router.get("/records") @router.get("/reviews/records") @@ -1132,9 +1358,11 @@ def get_record_detail(record_id: str): FROM records r JOIN versions v ON r.version_id = v.version_id LEFT JOIN documents d ON v.document_id = d.document_id - WHERE r.record_id = ? + WHERE r.record_id = ? OR r.record_instance_id = ? + ORDER BY r.created_at DESC + LIMIT 1 """, - (record_id,), + (record_id, record_id), ) row = c.fetchone() if not row: @@ -1273,13 +1501,13 @@ def list_issues( COALESCE( (SELECT d.title FROM documents d WHERE d.document_id = v.subject_id), (SELECT d.title FROM documents d JOIN versions ver ON ver.document_id = d.document_id WHERE ver.version_id = v.subject_id), - (SELECT d.title FROM documents d JOIN versions ver ON ver.document_id = d.document_id JOIN records rec ON rec.version_id = ver.version_id WHERE rec.record_id = v.subject_id), + (SELECT d.title FROM documents d JOIN versions ver ON ver.document_id = d.document_id JOIN records rec ON rec.version_id = ver.version_id WHERE rec.record_id = v.subject_id OR rec.record_instance_id = v.subject_id LIMIT 1), (SELECT d.title FROM documents d JOIN artifacts a ON a.document_id = d.document_id WHERE a.artifact_id = v.subject_id) ) AS document_title, COALESCE( (CASE WHEN v.subject_type = 'document' THEN v.subject_id ELSE NULL END), (SELECT ver.document_id FROM versions ver WHERE ver.version_id = v.subject_id), - (SELECT ver.document_id FROM versions ver JOIN records rec ON rec.version_id = ver.version_id WHERE rec.record_id = v.subject_id), + (SELECT ver.document_id FROM versions ver JOIN records rec ON rec.version_id = ver.version_id WHERE rec.record_id = v.subject_id OR rec.record_instance_id = v.subject_id LIMIT 1), (SELECT a.document_id FROM artifacts a WHERE a.artifact_id = v.subject_id) ) AS document_id, (SELECT a.source_id FROM artifacts a WHERE a.artifact_id = v.subject_id) AS source_id, @@ -1846,3 +2074,151 @@ def explorer_facets_endpoint(): "validation_statuses": validation_statuses, } ) + + +# ------------------------------------------------------------- +# MESA v4 Publisher Endpoints +# ------------------------------------------------------------- + + +@router.get("/publisher/settings") +def get_publisher_settings_endpoint(target_key: str = "default"): + conn = get_connection() + settings = get_mesa_target_settings(conn, target_key) + conn.close() + client = MesaClient(settings=settings) + data = settings.model_dump() + data["api_key_configured"] = client.is_api_key_configured + return ok_response(data) + + +@router.post("/publisher/settings") +def update_publisher_settings_endpoint(req: MesaTargetSettingsUpdateRequest, target_key: str = "default"): + conn = get_connection() + new_settings = MesaTargetSettings( + target_key=target_key, + base_url=req.base_url, + tenant_id=req.tenant_id, + workspace_id=req.workspace_id, + dataset_id=req.dataset_id, + agent_id=req.agent_id, + content_limit_chars=req.content_limit_chars, + contract_source=( + "configured" + if req.health_path and req.publish_path and "{mutation_id}" in req.mutation_status_path_template + else "unknown" + ), + health_path=req.health_path, + publish_path=req.publish_path, + mutation_status_path_template=req.mutation_status_path_template, + ) + upsert_mesa_target_settings(conn, new_settings) + conn.close() + client = MesaClient(settings=new_settings) + data = new_settings.model_dump() + data["api_key_configured"] = client.is_api_key_configured + return ok_response(data) + + +@router.post("/publisher/test-connection") +def test_publisher_connection_endpoint(target_key: str = "default"): + conn = get_connection() + settings = get_mesa_target_settings(conn, target_key) + conn.close() + client = MesaClient(settings=settings) + res = client.test_connection() + return ok_response(res) + + +@router.post("/publisher/preflight") +def run_publisher_preflight_endpoint(target_key: str = "default"): + conn = get_connection() + settings = get_mesa_target_settings(conn, target_key) + _, summary = build_delivery_plan(target_key=target_key) + client = MesaClient(settings=settings) + report = client.run_preflight_checks( + ready_documents_count=summary.ready_documents, + ready_versions_count=summary.ready_versions, + estimated_chunks_count=summary.estimated_chunks, + total_canonical_bytes=summary.total_canonical_bytes, + blocked_versions_count=summary.blocked_versions_excluded, + unreadable_versions_count=summary.unreadable_versions_excluded, + ) + conn.close() + return ok_response(report.model_dump()) + + +@router.get("/publisher/ready-summary") +def get_publisher_ready_summary_endpoint(target_key: str = "default"): + _, summary = build_delivery_plan(target_key=target_key) + return ok_response(summary.model_dump()) + + +@router.post("/publisher/publish") +def start_publisher_delivery_endpoint(req: MesaPublishRequest): + from mesa_legal_data.operations import submit_operation + + conn = get_connection() + target_settings = get_mesa_target_settings(conn, req.target_key) + client = MesaClient(settings=target_settings) + _, summary = build_delivery_plan(target_key=req.target_key) + + report = client.run_preflight_checks( + ready_documents_count=summary.ready_documents, + ready_versions_count=summary.ready_versions, + estimated_chunks_count=summary.estimated_chunks, + total_canonical_bytes=summary.total_canonical_bytes, + blocked_versions_count=summary.blocked_versions_excluded, + unreadable_versions_count=summary.unreadable_versions_excluded, + ) + if report.overall_status == "FAIL": + conn.close() + failed_msgs = [c.message for c in report.checks if c.status == "FAIL"] + error_response("PREFLIGHT_FAILED", f"Preflight checks failed: {'; '.join(failed_msgs)}", status_code=400) + + delivery_id = f"del-{uuid.uuid4().hex[:12]}" + op_id = submit_operation( + conn, + operation_type="mesa_v4_delivery", + requested_by="web-user", + input_dict={ + "delivery_id": delivery_id, + "release_id": req.release_id, + "target_key": req.target_key, + }, + ) + conn.close() + return ok_response({"operation_id": op_id, "delivery_id": delivery_id, "summary": summary.model_dump()}) + + +@router.get("/publisher/deliveries") +def list_publisher_deliveries_endpoint( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), +): + conn = get_connection() + offset = (page - 1) * page_size + items = list_deliveries(conn, limit=page_size, offset=offset) + c = conn.cursor() + c.execute("SELECT count(*) FROM mesa_deliveries") + total = c.fetchone()[0] + conn.close() + return ok_response({"items": items, "total": total, "page": page, "page_size": page_size}) + + +@router.get("/publisher/deliveries/{delivery_id}") +def get_publisher_delivery_endpoint(delivery_id: str): + conn = get_connection() + delivery = get_delivery(conn, delivery_id) + if not delivery: + conn.close() + error_response("DELIVERY_NOT_FOUND", f"Delivery {delivery_id} not found", status_code=404) + items = list_delivery_items(conn, delivery_id, limit=200) + conn.close() + return ok_response({"delivery": delivery, "items": items}) + + +@router.post("/publisher/deliveries/{delivery_id}/retry") +def retry_publisher_delivery_endpoint(delivery_id: str): + res = retry_delivery_failures(delivery_id) + return ok_response(res) diff --git a/src/mesa_legal_data/web/schemas.py b/src/mesa_legal_data/web/schemas.py index e0fe287..faeab99 100644 --- a/src/mesa_legal_data/web/schemas.py +++ b/src/mesa_legal_data/web/schemas.py @@ -60,3 +60,37 @@ class IssueResolveRequest(BaseModel): status: str = Field(default="resolved") resolved_by: str = Field(default="web-user") resolution_note: Optional[str] = Field(default=None, max_length=2000) + + +class SourceSettingsUpdateRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + enabled: bool = True + auto_approval_enabled: bool = False + weekly_sample_count: int = Field(default=10, ge=0, le=1000) + + +class ParserCertifyRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + parser_name: str = Field(min_length=1, max_length=100) + parser_version: str = Field(min_length=1, max_length=50) + certified: bool = True + certified_by: Optional[str] = Field(default="operator", max_length=100) + + +class MesaTargetSettingsUpdateRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + base_url: str = Field(min_length=5, max_length=500) + tenant_id: str = Field(min_length=1, max_length=100) + workspace_id: str = Field(min_length=1, max_length=100) + dataset_id: str = Field(min_length=1, max_length=100) + agent_id: str = Field(min_length=1, max_length=100) + content_limit_chars: int = Field(default=32768, ge=1024, le=1048576) + health_path: str = Field(default="", max_length=300, pattern=r"^(|/.*)$") + publish_path: str = Field(default="", max_length=300, pattern=r"^(|/.*)$") + mutation_status_path_template: str = Field(default="", max_length=300, pattern=r"^(|/.*)$") + + +class MesaPublishRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + target_key: str = Field(default="default") + release_id: Optional[str] = None diff --git a/src/mesa_legal_data/web/static/app.js b/src/mesa_legal_data/web/static/app.js index c87b391..1cc9d96 100644 --- a/src/mesa_legal_data/web/static/app.js +++ b/src/mesa_legal_data/web/static/app.js @@ -56,6 +56,36 @@ const SOURCE_CAPABILITIES = { }, }; +function updateDocTypesForSource(sourceSelectId, typeSelectId) { + const sourceSelect = document.getElementById(sourceSelectId); + const typeSelect = document.getElementById(typeSelectId); + if (!sourceSelect || !typeSelect) return; + + const selectedSource = sourceSelect.value; + const currentVal = typeSelect.value; + const cap = SOURCE_CAPABILITIES[selectedSource]; + + typeSelect.innerHTML = ``; + if (cap && cap.docTypes) { + let currentStillValid = false; + cap.docTypes.forEach((dt) => { + const opt = document.createElement("option"); + opt.value = dt.id; + opt.textContent = dt.label; + if (dt.id === currentVal) { + opt.selected = true; + currentStillValid = true; + } + typeSelect.appendChild(opt); + }); + + if (currentVal && !currentStillValid) { + typeSelect.value = ""; + showToast("Kaynak değiştirildi. Bu kaynak için belge türünü yeniden seçin.", "info"); + } + } +} + // --- Terminology & Presentation Helpers --- function humanTerm(term) { if (!term) return ""; @@ -75,6 +105,7 @@ function humanTerm(term) { yargitay: "Yargıtay", danistay: "Danıştay", idle: "Hazır", + not_started: "Henüz başlanmadı", running: "Çalışıyor", paused: "Duraklatıldı", up_to_date: "Güncel", @@ -430,14 +461,23 @@ async function loadHomeView() { if (welcomeCard) welcomeCard.classList.add("hidden"); if (normalContainer) normalContainer.classList.remove("hidden"); - // Metrics - document.getElementById("stat-docs").textContent = totalDocs; - document.getElementById("stat-pending").textContent = dash.counts?.pending_reviews || 0; - document.getElementById("stat-issues").textContent = (dash.counts?.open_blockers || 0) + (dash.counts?.open_errors || 0); + // Health Metrics + const h = dash.health || {}; + const setElemText = (id, val) => { + const el = document.getElementById(id); + if (el) el.textContent = val; + }; + + setElemText("stat-discovered-today", h.discovered_today ?? 0); + setElemText("stat-processed-today", h.processed_today ?? 0); + setElemText("stat-auto-approved-today", h.auto_approved_today ?? 0); + setElemText("stat-needs-review", h.needs_review_count ?? (dash.counts?.pending_reviews || 0)); + setElemText("stat-blocked", h.blocked_count ?? (dash.counts?.open_blockers || 0)); + setElemText("stat-mesa-ready", h.mesa_ready_count ?? (dash.counts?.approved_records || 0)); - const lastHarvestEl = document.getElementById("stat-last-harvest"); - if (lastHarvestEl) { - lastHarvestEl.textContent = harvest.last_discovery_at ? friendlyDate(harvest.last_discovery_at) : (harvest.state === "running" ? "Şu anda çalışıyor" : "-"); + const mesaLabelEl = document.getElementById("lbl-home-mesa-status"); + if (mesaLabelEl) { + mesaLabelEl.textContent = h.mesa_status_label || "MESA entegrasyonu yapılandırılmadı (Yerel Development Staging Aktif)"; } // Recommended Next Action Card @@ -770,9 +810,10 @@ async function loadLibraryView() { async function viewDocDetail(documentId) { setBusy(true); try { - const [doc, textRes] = await Promise.all([ + const [doc, textRes, verRes] = await Promise.all([ apiRequest(`/api/documents/${encodeURIComponent(documentId)}`), apiRequest(`/api/documents/${encodeURIComponent(documentId)}/text`).catch(() => ({ content: null })), + apiRequest(`/api/documents/${encodeURIComponent(documentId)}/versions`).catch(() => ({ versions: [] })), ]); state.currentDocId = documentId; @@ -794,6 +835,59 @@ async function viewDocDetail(documentId) { `; } + const versions = verRes?.versions || []; + let versionsTableHtml = ""; + if (versions.length > 0) { + versionsTableHtml = ` +
+
+ + +
+ + + + + + + + + + + + ${versions.map((v) => { + const qBadge = v.quality_status === "PASS" ? `PASS` : v.quality_status === "BLOCK" ? `BLOCK` : `REVIEW`; + const aBadge = v.approval_status === "approved" ? `Onaylı` : `Bekliyor`; + const tags = []; + if (v.auto_approved) tags.push(`Otomatik`); + if (v.is_audit_sample) tags.push(`Örnek Kontrol`); + return ` + + + + + + + + `; + }).join("")} + +
RevizyonTarihKaliteOnayEtiketler
v${v.revision_number || 1}${friendlyDate(v.created_at)}${qBadge}${aBadge}${tags.join(" ") || "-"}
+
+ `; + } + + let mesaStatusBadge = `MESA: Not Sent`; + try { + const mesaRes = await apiRequest(`/api/documents/${encodeURIComponent(documentId)}/mesa-status`); + const st = mesaRes.status || "Not Sent"; + if (st === "Committed") mesaStatusBadge = `MESA: Committed`; + else if (st === "Sending") mesaStatusBadge = `MESA: Sending`; + else if (st === "Partial") mesaStatusBadge = `MESA: Partial`; + else if (st === "Failed") mesaStatusBadge = `MESA: Failed`; + else mesaStatusBadge = `MESA: Not Sent`; + } catch (e) {} + const modalBody = document.getElementById("doc-modal-body"); modalBody.innerHTML = `
@@ -801,14 +895,20 @@ async function viewDocDetail(documentId) {

${escapeHtml(doc.title || doc.document_id)}

${humanTerm(sourceId)} · ${humanTerm(doc.document_type)}
- ${statusBadge(docStatus)} +
+ ${mesaStatusBadge} + ${statusBadge(docStatus)} + +
${openIssuesHtml} + ${versionsTableHtml} +
- 
${escapeHtml(textContent)}
+ 
${escapeHtml(textContent)}
@@ -831,6 +931,24 @@ async function viewDocDetail(documentId) { } } +async function reprocessDocument(documentId) { + if (!confirm(`"${documentId}" belgesi mevcut ham veri kullanılarak yeniden işlensin mi?`)) return; + setBusy(true); + try { + const res = await apiRequest(`/api/documents/${encodeURIComponent(documentId)}/reprocess`, { + method: "POST", + }); + showToast(`Yeniden işleme tamamlandı: ${res.pipeline_status || 'başarılı'}`, "success"); + await viewDocDetail(documentId); + if (state.currentView === "library") await loadLibraryView(); + } catch (err) { + showToast(`Yeniden işleme hatası: ${err.message || err}`, "danger"); + } finally { + setBusy(false); + } +} + + // --- 4. REVIEW VIEW --- async function loadReviewView() { setBusy(true); @@ -845,12 +963,17 @@ async function loadReviewView() { document.getElementById("review-tab-pending").classList.remove("hidden"); document.getElementById("review-tab-issues").classList.add("hidden"); - const res = await apiRequest("/api/records?page=1&page_size=50&approval_status=pending"); - const items = res.items || []; + const [verRes, recRes] = await Promise.all([ + apiRequest("/api/reviews/pending-versions?page=1&page_size=50").catch(() => ({ items: [] })), + apiRequest("/api/records?page=1&page_size=50&approval_status=pending").catch(() => ({ items: [] })), + ]); + + const versionItems = verRes.items || []; + const recordItems = recRes.items || []; const tbody = document.getElementById("tbl-reviews"); tbody.innerHTML = ""; - if (items.length === 0) { + if (versionItems.length === 0 && recordItems.length === 0) { tbody.innerHTML = ` @@ -858,7 +981,7 @@ async function loadReviewView() {

Şu anda inceleme bekleyen kayıt yok

Hazır verilerinizi dışa aktarabilir veya yeni veri toplamaya devam edebilirsiniz.

- +
@@ -868,20 +991,41 @@ async function loadReviewView() { return; } - items.forEach((rec) => { + // Render pending versions first (document-first review) + versionItems.forEach((v) => { const tr = document.createElement("tr"); + const qBadge = v.quality_status === "PASS" ? `PASS` : v.quality_status === "BLOCK" ? `BLOCK` : `REVIEW`; + const auditBadge = v.is_audit_sample ? `Örnek Kontrol` : ""; tr.innerHTML = ` - ${escapeHtml(rec.document_title || rec.document_id || "Belge")} - ${humanTerm(rec.record_type)} - ${statusBadge(rec.validation_status || "valid")} - ${statusBadge(rec.approval_status || "pending")} - ${friendlyDate(rec.created_at)} + ${escapeHtml(v.document_title || v.document_id || "Belge")} v${v.revision_number || 1} + ${humanTerm(v.family || "legislation")} + ${qBadge} ${auditBadge} + İnceleme Bekliyor + ${friendlyDate(v.created_at)} - + `; tbody.appendChild(tr); }); + + // Render individual pending records if any exist without version parent + if (versionItems.length === 0) { + recordItems.forEach((rec) => { + const tr = document.createElement("tr"); + tr.innerHTML = ` + ${escapeHtml(rec.document_title || rec.document_id || "Belge")} + ${humanTerm(rec.record_type)} + ${statusBadge(rec.validation_status || "valid")} + ${statusBadge(rec.approval_status || "pending")} + ${friendlyDate(rec.created_at)} + + + + `; + tbody.appendChild(tr); + }); + } } else { document.getElementById("review-tab-pending").classList.add("hidden"); document.getElementById("review-tab-issues").classList.remove("hidden"); @@ -1108,25 +1252,136 @@ async function openRecordReviewModal(recordId) { } } -async function handleRecordDecision(decision) { - if (!state.currentRecordId) return; - const note = document.getElementById("txt-reviewer-note")?.value.trim() || ""; - +async function openVersionReviewModal(versionId) { setBusy(true); try { - const endpoint = `/api/reviews/records/${encodeURIComponent(state.currentRecordId)}/${decision}`; - await apiRequest(endpoint, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - reviewer: sessionStorage.getItem("mesa_actor") || "web-user", - note: note || null, + const [ver, docTextRes] = await Promise.all([ + apiRequest(`/api/reviews/pending-versions?page=1&page_size=100`).then((res) => { + return (res.items || []).find((v) => v.version_id === versionId) || { version_id: versionId }; }), - }); + apiRequest(`/api/documents/${encodeURIComponent(versionId.split(':')[0])}/text`).catch(() => ({ content: null })), + ]); - showToast(`Kayıt ${decision === "approve" ? "onaylandı" : "reddedildi"}.`, "success"); - closeModal("modal-record-detail"); - await loadReviewView(); + state.currentVersionId = versionId; + state.currentRecordId = null; + + const qJson = ver.quality_json || {}; + const checks = qJson.checks || []; + const cov = qJson.coverage || {}; + const covPct = cov.coverage_ratio != null ? `${Math.round(cov.coverage_ratio * 100)}%` : "N/A"; + + let auditBannerHtml = ""; + if (ver.is_audit_sample) { + auditBannerHtml = ` +
+ Kalite Örnek Denetimi: Bu belge otomatik kalite sistemini doğrulamak için örnek kontrole seçildi. +
+ `; + } + + let qualityChecksHtml = ""; + if (checks.length > 0) { + qualityChecksHtml = ` +
+
+ Kalite Değerlendirmesi: ${ver.quality_status || 'PASS'} + Kapsam Oranı: ${covPct} +
+
+ ${checks.map((c) => { + const icon = c.status === "PASS" ? "✅" : c.status === "BLOCK" ? "❌" : "⚠️"; + return `
${icon} ${escapeHtml(c.name || c.check_group || 'Kontrol')}: ${c.status}
`; + }).join("")} +
+
+ `; + } + + const rawContent = (docTextRes && docTextRes.content) ? docTextRes.content : "Ham kaynak metni yüklenemedi."; + + const modalBody = document.getElementById("record-modal-body"); + modalBody.innerHTML = ` +
+
+

${escapeHtml(ver.document_title || ver.document_id || "Belge Versiyonu")}

+ ${humanTerm(ver.source_id || "resmi_gazete")} · Revizyon v${ver.revision_number || 1} +
+
+ ${ver.quality_status || 'REVIEW'} + İnceleme Bekliyor +
+
+ + ${auditBannerHtml} + ${qualityChecksHtml} + +
+
+
+ Ham Kaynak İçeriği + ${escapeHtml(ver.source_url || "")} +
+ 
${escapeHtml(rawContent)}
+
+
+
+ Canonical Versiyon + ${escapeHtml(ver.version_id)} +
+ 
${escapeHtml(rawContent)}
+
+
+ +
+ Teknik ayrıntılar +
+
Version ID: ${escapeHtml(ver.version_id)}
+
Document ID: ${escapeHtml(ver.document_id || "-")}
+
Canonical Path: ${escapeHtml(ver.canonical_path || "-")}
+
+
+ `; + + document.getElementById("txt-reviewer-note").value = ""; + showModal("modal-record-detail"); + } catch (err) { + console.error("Open version review modal error:", err); + } finally { + setBusy(false); + } +} + +async function handleRecordDecision(decision) { + const note = document.getElementById("txt-reviewer-note")?.value.trim() || ""; + setBusy(true); + try { + if (state.currentVersionId && !state.currentRecordId) { + const endpoint = `/api/versions/${encodeURIComponent(state.currentVersionId)}/${decision}`; + await apiRequest(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + reviewer: sessionStorage.getItem("mesa_actor") || "web-user", + note: note || null, + }), + }); + showToast(`Belge versiyonu ${decision === "approve" ? "onaylandı" : "reddedildi"}.`, "success"); + closeModal("modal-record-detail"); + await loadReviewView(); + } else if (state.currentRecordId) { + const endpoint = `/api/reviews/records/${encodeURIComponent(state.currentRecordId)}/${decision}`; + await apiRequest(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + reviewer: sessionStorage.getItem("mesa_actor") || "web-user", + note: note || null, + }), + }); + showToast(`Kayıt ${decision === "approve" ? "onaylandı" : "reddedildi"}.`, "success"); + closeModal("modal-record-detail"); + await loadReviewView(); + } } catch (err) { console.error("Record decision error:", err); const errStr = String(err.message || ""); @@ -1233,85 +1488,436 @@ async function handleVersionBulkDecision(decision) { } } -// --- 5. EXPORT VIEW --- +// --- 5. EXPORT / PUBLISH VIEW --- +let activeMesaDeliveryId = null; +let mesaDeliveryPollInterval = null; + async function loadExportView() { setBusy(true); try { - const [exportsList, stats] = await Promise.all([ - apiRequest("/api/exports"), - apiRequest("/api/dashboard/stats").catch(() => null), + const [targetSettings, readySummary, deliveriesRes, exportsList] = await Promise.all([ + apiRequest("/api/publisher/settings").catch(() => null), + apiRequest("/api/publisher/ready-summary").catch(() => null), + apiRequest("/api/publisher/deliveries?page=1&page_size=20").catch(() => ({ items: [] })), + apiRequest("/api/exports").catch(() => []), ]); - const approvedCount = stats?.counts?.approved_records || 0; - const btnCreateExport = document.getElementById("btn-export-create"); - const btnMesaTransfer = document.getElementById("btn-mesa-transfer"); - - let emptyNotice = document.getElementById("export-empty-notice"); - if (approvedCount === 0) { - if (!emptyNotice) { - emptyNotice = document.createElement("div"); - emptyNotice.id = "export-empty-notice"; - emptyNotice.style.cssText = "margin-bottom: 16px; padding: 14px 18px; border-left: 4px solid var(--color-warning); background: var(--color-surface-subtle); border-radius: 6px;"; - const exportPanel = document.getElementById("view-export"); - if (exportPanel) exportPanel.insertBefore(emptyNotice, exportPanel.firstChild); + // 1. Populate Target Settings + if (targetSettings) { + const elUrl = document.getElementById("mesa-target-url"); + const elTenant = document.getElementById("mesa-target-tenant"); + const elWs = document.getElementById("mesa-target-workspace"); + const elDs = document.getElementById("mesa-target-dataset"); + const elAgent = document.getElementById("mesa-target-agent"); + const elLimit = document.getElementById("mesa-target-limit"); + const elHealthPath = document.getElementById("mesa-health-path"); + const elPublishPath = document.getElementById("mesa-publish-path"); + const elMutationPath = document.getElementById("mesa-mutation-path"); + const badgeKey = document.getElementById("badge-mesa-key"); + + if (elUrl && targetSettings.base_url) elUrl.value = targetSettings.base_url; + if (elTenant && targetSettings.tenant_id) elTenant.value = targetSettings.tenant_id; + if (elWs && targetSettings.workspace_id) elWs.value = targetSettings.workspace_id; + if (elDs && targetSettings.dataset_id) elDs.value = targetSettings.dataset_id; + if (elAgent && targetSettings.agent_id) elAgent.value = targetSettings.agent_id; + if (elLimit && targetSettings.content_limit_chars) elLimit.value = targetSettings.content_limit_chars; + if (elHealthPath) elHealthPath.value = targetSettings.health_path || ""; + if (elPublishPath) elPublishPath.value = targetSettings.publish_path || ""; + if (elMutationPath) elMutationPath.value = targetSettings.mutation_status_path_template || ""; + + if (badgeKey) { + if (targetSettings.api_key_configured) { + badgeKey.className = "badge badge-success"; + badgeKey.textContent = "API Key: Yapılandırıldı ✅"; + } else { + badgeKey.className = "badge badge-warning"; + badgeKey.textContent = "API Key: Yapılandırılmadı ❌"; + } } - emptyNotice.innerHTML = ` -

Henüz dışa aktarılabilecek hazır kayıt yok

-

Dışa aktarma yapabilmek için önce veri toplayın ve gerekiyorsa inceleme adımını tamamlayın.

-
- - -
- `; - if (btnCreateExport) { - btnCreateExport.disabled = true; - btnCreateExport.title = "Dışa aktarma oluşturmak için önce en az 1 onaylanmış kayıt gereklidir."; + } + + // 2. Populate Ready Summary Cards + if (readySummary) { + const setTxt = (id, val) => { + const el = document.getElementById(id); + if (el) el.textContent = val; + }; + setTxt("stat-mesa-ready-docs", readySummary.ready_documents ?? 0); + setTxt("stat-mesa-ready-vers", readySummary.ready_versions ?? 0); + setTxt("stat-mesa-ready-chunks", readySummary.estimated_chunks ?? 0); + const kb = Math.round((readySummary.total_canonical_bytes || 0) / 1024); + setTxt("stat-mesa-ready-bytes", `${kb} KB`); + setTxt("stat-mesa-ready-committed", readySummary.already_committed_chunks ?? 0); + setTxt("stat-mesa-ready-new", readySummary.new_chunks_to_send ?? 0); + + const btnPub = document.getElementById("btn-mesa-publish-trigger"); + if (btnPub) { + if ((readySummary.new_chunks_to_send ?? 0) === 0 && (readySummary.ready_versions ?? 0) === 0) { + btnPub.disabled = true; + btnPub.title = "Aktarılacak hazır versiyon bulunmuyor."; + } else { + btnPub.disabled = false; + btnPub.title = ""; + } } - if (btnMesaTransfer) { - btnMesaTransfer.disabled = true; - btnMesaTransfer.title = "MESA transferi için önce en az 1 onaylanmış kayıt gereklidir."; + } + + // 3. Render Deliveries History Table + const tblDeliveries = document.getElementById("tbl-mesa-deliveries"); + if (tblDeliveries) { + tblDeliveries.innerHTML = ""; + const deliveries = deliveriesRes.items || []; + if (deliveries.length === 0) { + tblDeliveries.innerHTML = `Henüz MESA aktarım kaydı bulunmuyor.`; + } else { + deliveries.forEach((del) => { + let sBadge = `${del.status}`; + if (del.status === "COMMITTED") sBadge = `COMMITTED`; + else if (del.status === "PARTIAL") sBadge = `PARTIAL`; + else if (del.status === "FAILED") sBadge = `FAILED`; + + const tr = document.createElement("tr"); + tr.innerHTML = ` + ${escapeHtml(del.delivery_id)} + ${escapeHtml(del.target_key)} + ${sBadge} + ${del.total_items} + ${del.committed_items} + ${del.failed_items} + ${del.skipped_items} + ${friendlyDate(del.started_at)} + + + + `; + tblDeliveries.appendChild(tr); + }); } - } else { - if (emptyNotice) emptyNotice.remove(); - if (btnCreateExport) { - btnCreateExport.disabled = false; - btnCreateExport.title = ""; + } + + // 4. Render File Exports Table + const tbodyExp = document.getElementById("tbl-exports"); + if (tbodyExp) { + tbodyExp.innerHTML = ""; + if (!exportsList || exportsList.length === 0) { + tbodyExp.innerHTML = `Henüz oluşturulan dışa aktarma paketi yok.`; + } else { + exportsList.forEach((exp) => { + const tr = document.createElement("tr"); + tr.innerHTML = ` + ${escapeHtml(exp.export_id)} + ${humanTerm(exp.export_type)} + ${statusBadge(exp.status)} + ${exp.record_count ? `${exp.record_count} Kayıt` : "-"} + ${friendlyDate(exp.created_at)} + + İndir + + `; + tbodyExp.appendChild(tr); + }); + } + } + } catch (err) { + console.error("Export view error:", err); + } finally { + setBusy(false); + } +} + +async function handleMesaSaveSettings() { + const url = document.getElementById("mesa-target-url")?.value.trim(); + const tenant = document.getElementById("mesa-target-tenant")?.value.trim(); + const ws = document.getElementById("mesa-target-workspace")?.value.trim(); + const ds = document.getElementById("mesa-target-dataset")?.value.trim(); + const agent = document.getElementById("mesa-target-agent")?.value.trim(); + const limit = parseInt(document.getElementById("mesa-target-limit")?.value || "32768", 10); + const healthPath = document.getElementById("mesa-health-path")?.value.trim() || ""; + const publishPath = document.getElementById("mesa-publish-path")?.value.trim() || ""; + const mutationPath = document.getElementById("mesa-mutation-path")?.value.trim() || ""; + + if (!url || !tenant || !ws || !ds || !agent) { + showToast("Lütfen tüm zorunlu hedef alanlarını doldurunuz.", "warning"); + return; + } + + setBusy(true); + try { + const res = await apiRequest("/api/publisher/settings", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + base_url: url, + tenant_id: tenant, + workspace_id: ws, + dataset_id: ds, + agent_id: agent, + content_limit_chars: limit, + health_path: healthPath, + publish_path: publishPath, + mutation_status_path_template: mutationPath, + }), + }); + showToast("MESA hedef ayarları kaydedildi.", "success"); + await loadExportView(); + } catch (err) { + showToast(`Ayar kaydetme hatası: ${err.message || err}`, "danger"); + } finally { + setBusy(false); + } +} + +async function handleMesaTestConnection() { + const badgeConn = document.getElementById("badge-mesa-conn"); + if (badgeConn) { + badgeConn.className = "badge badge-neutral"; + badgeConn.textContent = "Bağlantı test ediliyor..."; + } + + try { + const res = await apiRequest("/api/publisher/test-connection", { method: "POST" }); + if (res.connected) { + if (badgeConn) { + badgeConn.className = "badge badge-success"; + badgeConn.textContent = `Bağlandı (${res.latency_ms}ms) ✅`; } - if (btnMesaTransfer) { - btnMesaTransfer.disabled = false; - btnMesaTransfer.title = ""; + showToast(`MESA sunucusuna başarıyla bağlanıldı (${res.latency_ms}ms).`, "success"); + } else { + if (badgeConn) { + badgeConn.className = "badge badge-danger"; + badgeConn.textContent = "Bağlantı Başarısız ❌"; } + showToast(`Bağlantı başarısız: ${res.details || 'Erişilemedi'}`, "danger"); } + } catch (err) { + if (badgeConn) { + badgeConn.className = "badge badge-danger"; + badgeConn.textContent = "Hata ❌"; + } + showToast(`Bağlantı testi hatası: ${err.message || err}`, "danger"); + } +} - const tbody = document.getElementById("tbl-exports"); - tbody.innerHTML = ""; +async function handleMesaPreflight() { + const boxResults = document.getElementById("box-mesa-preflight-results"); + const listItems = document.getElementById("list-mesa-preflight-items"); - if (!exportsList || exportsList.length === 0) { - tbody.innerHTML = `Henüz oluşturulan dışa aktarma paketi yok.`; - return; + if (listItems) listItems.innerHTML = "Kontroller yapılıyor..."; + if (boxResults) boxResults.classList.remove("hidden"); + + try { + const report = await apiRequest("/api/publisher/preflight", { method: "POST" }); + if (listItems) { + listItems.innerHTML = report.checks.map((c) => { + let icon = "✅"; + let color = "var(--color-success, green)"; + if (c.status === "FAIL") { + icon = "❌"; + color = "var(--color-danger, red)"; + } else if (c.status === "WARN") { + icon = "⚠️"; + color = "var(--color-warning, orange)"; + } + return `
+ ${icon} + [${c.name}] + ${escapeHtml(c.message)} +
`; + }).join(""); } + if (report.overall_status === "PASS") { + showToast("Ön kontrol başarılı (PASS). MESA aktarımına hazır.", "success"); + } else { + showToast("Ön kontrol başarısız (FAIL). Lütfen uyarıları gideriniz.", "warning"); + } + } catch (err) { + if (listItems) listItems.innerHTML = `Hata: ${escapeHtml(err.message || err)}`; + showToast(`Ön kontrol hatası: ${err.message || err}`, "danger"); + } +} - exportsList.forEach((exp) => { - const tr = document.createElement("tr"); - tr.innerHTML = ` - ${escapeHtml(exp.export_id)} - ${humanTerm(exp.export_type)} - ${statusBadge(exp.status)} - ${exp.record_count ? `${exp.record_count} Kayıt` : "-"} - ${friendlyDate(exp.created_at)} - - İndir - - `; - tbody.appendChild(tr); +async function handleMesaPublishTrigger() { + try { + const summary = await apiRequest("/api/publisher/ready-summary"); + const settings = await apiRequest("/api/publisher/settings"); + + const elDocs = document.getElementById("confirm-mesa-docs"); + const elVers = document.getElementById("confirm-mesa-versions"); + const elChunks = document.getElementById("confirm-mesa-chunks"); + const elTarget = document.getElementById("confirm-mesa-target"); + + if (elDocs) elDocs.textContent = `${summary.ready_documents} belge`; + if (elVers) elVers.textContent = `${summary.ready_versions} versiyon`; + if (elChunks) elChunks.textContent = `${summary.estimated_chunks} chunk (${summary.new_chunks_to_send} yeni gönderilecek)`; + if (elTarget) elTarget.textContent = `${settings.tenant_id} / ${settings.workspace_id} / ${settings.dataset_id} (${settings.base_url})`; + + showModal("modal-mesa-confirm"); + } catch (err) { + showToast(`Özet yükleme hatası: ${err.message || err}`, "danger"); + } +} + +async function handleMesaConfirmPublish() { + hideModal("modal-mesa-confirm"); + const boxProg = document.getElementById("box-mesa-delivery-progress"); + const boxPartial = document.getElementById("box-mesa-partial-failure"); + + if (boxProg) boxProg.classList.remove("hidden"); + if (boxPartial) boxPartial.classList.add("hidden"); + + try { + const res = await apiRequest("/api/publisher/publish", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ target_key: "default" }), }); + + activeMesaDeliveryId = res.delivery_id; + showToast(`MESA aktarımı başlatıldı (Teslimat: ${activeMesaDeliveryId}).`, "info"); + startMesaDeliveryPolling(activeMesaDeliveryId); } catch (err) { - console.error("Export view error:", err); + showToast(`Aktarım başlatılamadı: ${err.message || err}`, "danger"); + } +} + +function startMesaDeliveryPolling(deliveryId) { + if (mesaDeliveryPollInterval) clearInterval(mesaDeliveryPollInterval); + + mesaDeliveryPollInterval = setInterval(async () => { + try { + const res = await apiRequest(`/api/publisher/deliveries/${deliveryId}`); + const del = res.delivery; + if (!del) return; + + const elStat = document.getElementById("txt-mesa-delivery-status"); + const badgeState = document.getElementById("badge-mesa-delivery-state"); + const valPlanned = document.getElementById("prog-val-planned"); + const valCommitted = document.getElementById("prog-val-committed"); + const valSkipped = document.getElementById("prog-val-skipped"); + const valFailed = document.getElementById("prog-val-failed"); + const boxPartial = document.getElementById("box-mesa-partial-failure"); + const txtPartial = document.getElementById("txt-mesa-partial-msg"); + + if (valPlanned) valPlanned.textContent = del.total_items; + if (valCommitted) valCommitted.textContent = del.committed_items; + if (valSkipped) valSkipped.textContent = del.skipped_items; + if (valFailed) valFailed.textContent = del.failed_items; + + if (badgeState) { + badgeState.textContent = del.status; + if (del.status === "COMMITTED") badgeState.className = "badge badge-success"; + else if (del.status === "PARTIAL") badgeState.className = "badge badge-warning"; + else if (del.status === "FAILED") badgeState.className = "badge badge-danger"; + else badgeState.className = "badge badge-info"; + } + + if (del.status === "COMMITTED") { + clearInterval(mesaDeliveryPollInterval); + if (elStat) elStat.textContent = `MESA Aktarımı Tamamlandı: ${del.committed_items} COMMITTED, ${del.skipped_items} Atlandı ✅`; + showToast("MESA aktarımı başarıyla tamamlandı.", "success"); + await loadExportView(); + } else if (del.status === "PARTIAL") { + clearInterval(mesaDeliveryPollInterval); + if (elStat) elStat.textContent = `MESA Aktarımı Kısmi Başarılı: ${del.committed_items} COMMITTED, ${del.failed_items} Başarısız ⚠️`; + if (boxPartial) boxPartial.classList.remove("hidden"); + if (txtPartial) txtPartial.textContent = `${del.committed_items} COMMITTED, ${del.failed_items} FAILED. Başarısızlar yeniden denenebilir.`; + showToast("MESA aktarımı kısmi tamamlandı.", "warning"); + await loadExportView(); + } else if (del.status === "FAILED") { + clearInterval(mesaDeliveryPollInterval); + if (elStat) elStat.textContent = `MESA Aktarımı Başarısız Oldu ❌`; + if (boxPartial) boxPartial.classList.remove("hidden"); + if (txtPartial) txtPartial.textContent = `Aktarım başarısız: ${del.last_error || 'Hata oluştu'}`; + showToast(`MESA aktarımı başarısız: ${del.last_error || 'Hata'}`, "danger"); + await loadExportView(); + } + } catch (e) { + console.error("Polling error:", e); + } + }, 1000); +} + +async function handleMesaRetryFailed() { + if (!activeMesaDeliveryId) { + showToast("Aktif teslimat bulunamadı.", "warning"); + return; + } + setBusy(true); + try { + const res = await apiRequest(`/api/publisher/deliveries/${activeMesaDeliveryId}/retry`, { method: "POST" }); + showToast(`Yeniden deneme başlatıldı: ${res.retried_count} item deneniyor.`, "info"); + startMesaDeliveryPolling(activeMesaDeliveryId); + } catch (err) { + showToast(`Yeniden deneme hatası: ${err.message || err}`, "danger"); + } finally { + setBusy(false); + } +} + +async function openDeliveryDetailModal(deliveryId) { + setBusy(true); + try { + const res = await apiRequest(`/api/publisher/deliveries/${deliveryId}`); + const del = res.delivery; + const items = res.items || []; + + const modalBody = document.getElementById("delivery-modal-body"); + if (!modalBody) return; + + modalBody.innerHTML = ` +
+
+

Teslimat: ${escapeHtml(del.delivery_id)}

+ Hedef: ${escapeHtml(del.target_key)} · Başlangıç: ${friendlyDate(del.started_at)} +
+ ${del.status} +
+ +
+
Toplam
${del.total_items}
+
COMMITTED
${del.committed_items}
+
Başarısız
${del.failed_items}
+
Atlanan
${del.skipped_items}
+
+ +
Source Chunk Aktarım Detayları (${items.length} item)
+
+ + + + + + + + + + + + + ${items.map((it) => ` + + + + + + + + + `).join("")} + +
Chunk IDBelgeDurumMutation IDIdempotency KeyHata
${escapeHtml(it.chunk_id)}${escapeHtml(it.document_id)}${it.remote_state}${escapeHtml(it.remote_mutation_id || '-')}${escapeHtml(it.idempotency_key)}${escapeHtml(it.last_error || '-')}
+
+ `; + + showModal("modal-delivery-detail"); + } catch (err) { + showToast(`Teslimat detayı yükleme hatası: ${err.message || err}`, "danger"); } finally { setBusy(false); } } + async function createExportAction() { const sel = document.getElementById("sel-export-format"); const exportType = sel ? sel.value : "records_jsonl"; @@ -1356,19 +1962,19 @@ async function runMesaTransferSequence() { await apiRequest(`/api/releases/${releaseId}/verify`, { method: "POST" }); // Step 3: Publish - statusText.textContent = "3/4 Yayına alınıyor..."; + statusText.textContent = "3/4 Release paketi onaylanıyor..."; await apiRequest(`/api/releases/${releaseId}/publish`, { method: "POST" }); // Step 4: Import Staging - statusText.textContent = "4/4 MESA aktarım alanına aktarılıyor..."; + statusText.textContent = "4/4 Yerel Development Staging alanına aktarılıyor..."; await apiRequest(`/api/releases/${releaseId}/import-staging`, { method: "POST" }); - statusText.textContent = "✓ MESA'ya aktarım başarıyla tamamlandı."; - showToast("Onaylı kayıtlar MESA aktarım alanına başarıyla gönderildi.", "success"); + statusText.textContent = "✓ Yerel Development Staging release paketi oluşturuldu."; + showToast("Onaylı kayıtlar yerel development staging ortamına aktarıldı. (MESA v4 publisher entegrasyonu sonraki aşamada tamamlanacaktır)", "success"); } catch (err) { - console.error("MESA transfer error:", err); - statusText.textContent = `Aktarım tamamlanamadı: ${err.message}`; - showToast("MESA aktarımı tamamlanamadı. Oluşturulan paket korundu.", "danger"); + console.error("Staging release error:", err); + statusText.textContent = `İşlem tamamlanamadı: ${err.message}`; + showToast("Yerel release oluşturulamadı. Oluşturulan paket korundu.", "danger"); } finally { transferBtn.disabled = false; setTimeout(() => { @@ -1381,23 +1987,83 @@ async function runMesaTransferSequence() { async function loadSourcesView() { setBusy(true); try { - const sources = await apiRequest("/api/sources"); + const [sources, settingsList] = await Promise.all([ + apiRequest("/api/sources"), + apiRequest("/api/sources/settings").catch(() => []), + ]); + + const settingsMap = {}; + (settingsList || []).forEach((item) => { + settingsMap[item.source_id] = item; + }); + const container = document.getElementById("sources-list"); container.innerHTML = ""; sources.forEach((s) => { + const opt = settingsMap[s.source_id] || { + enabled: s.enabled !== false, + auto_approval_enabled: false, + weekly_sample_count: 10, + parsers: [], + }; + + const parsers = opt.parsers || []; + const card = document.createElement("div"); - card.className = "source-card"; + card.className = "source-card panel-box"; + card.style.marginBottom = "16px"; card.innerHTML = ` -
+
-

${escapeHtml(s.name)}

-

${escapeHtml(s.authority || "-")}

+

${escapeHtml(s.name)}

+

${escapeHtml(s.authority || "-")}

- ${escapeHtml(s.automation === "supported" ? "Otomatik Toplama" : s.automation === "manual" ? "Manuel Ekleme" : "Devre Dışı")} + ${opt.enabled ? "Kaynak Aktif" : "Devre Dışı"}
-
- Kaynak ID: ${escapeHtml(s.source_id)} + +
+
+ + Kaynaktan otomatik ve manuel veri alımını denetler. +
+ +
+ + Sadece PASS alan ve sertifikalı parser belgeleri otomatik onaylanır. +
+ +
+ + + Otomasyonu doğrulamak için seçilen audit örnekleri. +
+
+ + ${parsers.length > 0 ? ` +
+ Sertifikalı Parser Sürümleri: +
+ ${parsers.map(p => ` + + ${escapeHtml(p.parser_name)} v${escapeHtml(p.parser_version)}: ${p.certified ? 'Sertifikalı' : 'Sertifikasız'} + + `).join('')} +
+
+ ` : ''} + +
+
+ Güvenlik Politikası: ${escapeHtml(s.base_url || "-")} · ${humanTerm(s.access_mode || "official_web")} +
+
`; container.appendChild(card); @@ -1409,6 +2075,31 @@ async function loadSourcesView() { } } +async function saveSourceOperationalSettings(sourceId) { + const enabled = document.getElementById(`chk-src-enabled-${sourceId}`)?.checked ?? true; + const autoApproval = document.getElementById(`chk-src-auto-${sourceId}`)?.checked ?? false; + const sampleCount = parseInt(document.getElementById(`num-src-sample-${sourceId}`)?.value || "10", 10); + + setBusy(true); + try { + await apiRequest(`/api/sources/${encodeURIComponent(sourceId)}/settings`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + enabled: enabled, + auto_approval_enabled: autoApproval, + weekly_sample_count: isNaN(sampleCount) ? 10 : sampleCount, + }), + }); + showToast(`"${sourceId}" ayarları güncellendi.`, "success"); + await loadSourcesView(); + } catch (err) { + showToast(`Ayar kaydedilemedi: ${err.message || err}`, "danger"); + } finally { + setBusy(false); + } +} + async function loadExplorerView() { setBusy(true); try { @@ -1727,36 +2418,6 @@ document.addEventListener("DOMContentLoaded", () => { if (btnCollectStop) btnCollectStop.addEventListener("click", stopHarvestAction); // 10. Manual Ingestion Tabs & Forms - function updateDocTypesForSource(sourceSelectId, typeSelectId) { - const sourceSelect = document.getElementById(sourceSelectId); - const typeSelect = document.getElementById(typeSelectId); - if (!sourceSelect || !typeSelect) return; - - const selectedSource = sourceSelect.value; - const currentVal = typeSelect.value; - const cap = SOURCE_CAPABILITIES[selectedSource]; - - typeSelect.innerHTML = ``; - if (cap && cap.docTypes) { - let currentStillValid = false; - cap.docTypes.forEach((dt) => { - const opt = document.createElement("option"); - opt.value = dt.id; - opt.textContent = dt.label; - if (dt.id === currentVal) { - opt.selected = true; - currentStillValid = true; - } - typeSelect.appendChild(opt); - }); - - if (currentVal && !currentStillValid) { - typeSelect.value = ""; - showToast("Kaynak değiştirildi. Bu kaynak için belge türünü yeniden seçin.", "info"); - } - } - } - const fileSourceEl = document.getElementById("file-source"); if (fileSourceEl) { fileSourceEl.addEventListener("change", () => { @@ -1984,13 +2645,31 @@ document.addEventListener("DOMContentLoaded", () => { btnVersionReject.addEventListener("click", () => handleVersionBulkDecision("reject")); } - // 13. Export Handlers + // 13. Export & MESA Publisher Handlers const btnExportCreate = document.getElementById("btn-export-create"); if (btnExportCreate) btnExportCreate.addEventListener("click", createExportAction); const btnMesaTransfer = document.getElementById("btn-mesa-transfer"); if (btnMesaTransfer) btnMesaTransfer.addEventListener("click", runMesaTransferSequence); + const btnMesaSaveSettings = document.getElementById("btn-mesa-save-settings"); + if (btnMesaSaveSettings) btnMesaSaveSettings.addEventListener("click", handleMesaSaveSettings); + + const btnMesaTestConn = document.getElementById("btn-mesa-test-conn"); + if (btnMesaTestConn) btnMesaTestConn.addEventListener("click", handleMesaTestConnection); + + const btnMesaPreflight = document.getElementById("btn-mesa-preflight"); + if (btnMesaPreflight) btnMesaPreflight.addEventListener("click", handleMesaPreflight); + + const btnMesaPubTrigger = document.getElementById("btn-mesa-publish-trigger"); + if (btnMesaPubTrigger) btnMesaPubTrigger.addEventListener("click", handleMesaPublishTrigger); + + const btnMesaConfirmPub = document.getElementById("btn-mesa-confirm-publish"); + if (btnMesaConfirmPub) btnMesaConfirmPub.addEventListener("click", handleMesaConfirmPublish); + + const btnMesaRetryFailed = document.getElementById("btn-mesa-retry-failed"); + if (btnMesaRetryFailed) btnMesaRetryFailed.addEventListener("click", handleMesaRetryFailed); + // 14. Advanced Release Build const btnBuildRelease = document.getElementById("btn-build-release"); if (btnBuildRelease) { diff --git a/src/mesa_legal_data/web/static/index.html b/src/mesa_legal_data/web/static/index.html index 220211b..cdc7432 100644 --- a/src/mesa_legal_data/web/static/index.html +++ b/src/mesa_legal_data/web/static/index.html @@ -12,7 +12,7 @@ document.documentElement.dataset.theme = dark ? "dark" : "light"; })(); - + @@ -49,7 +49,7 @@

MESA Data

YÖNETİM
@@ -58,11 +58,11 @@

MESA Data

@@ -152,23 +152,44 @@

Önerilen işlem yükleniyor...

- -
+ +
+
+ MESA Entegrasyon Durumu: + MESA entegrasyonu yapılandırılmadı (Yerel Development Staging Aktif) +
+
+ + + +
+
+ + +
+
+ Bugün Bulunan + 0 +
- Toplam Belge - 0 + İşlenen (Bugün) + 0 +
+
+ Otomatik Onaylanan + 0
İnceleme Bekleyen - 0 + 0
- Açık Sorunlar - 0 + Engellenen (Blocked) + 0
- Son Toplama - - + MESA'ya Hazır + 0
@@ -547,59 +568,224 @@

Belgeyi kendim eklemek istiyorum

- + @@ -813,8 +999,45 @@

Manuel Olarak Çözüldü Kabul Et

- - - + + + + + + + + + diff --git a/tests/acceptance/test_web_visual_contract.py b/tests/acceptance/test_web_visual_contract.py index d9aebfa..9637948 100644 --- a/tests/acceptance/test_web_visual_contract.py +++ b/tests/acceptance/test_web_visual_contract.py @@ -98,3 +98,8 @@ def test_static_js_files_exist_and_served(): assert "Durum kontrol ediliyor…" in app_js assert "API erişilebilir" in app_js assert "API erişilemiyor" in app_js + + # This helper is used by loadCollectView before DOMContentLoaded handlers run. + assert app_js.index("function updateDocTypesForSource") < app_js.index( + 'document.addEventListener("DOMContentLoaded"' + ) diff --git a/tests/integration/test_mesa_v4_publisher_e2e.py b/tests/integration/test_mesa_v4_publisher_e2e.py new file mode 100644 index 0000000..14b6f9e --- /dev/null +++ b/tests/integration/test_mesa_v4_publisher_e2e.py @@ -0,0 +1,228 @@ +import hashlib +import json + +import httpx +import pytest +import respx + +from mesa_legal_data.catalog import ( + get_connection, + insert_artifact, + insert_record, + insert_version, + migrate, + upsert_document, +) +from mesa_legal_data.publisher.engine import execute_publish_delivery, retry_delivery_failures +from mesa_legal_data.publisher.ledger import upsert_mesa_target_settings +from mesa_legal_data.publisher.models import MesaTargetSettings + + +@pytest.fixture +def setup_publisher_env(tmp_path, monkeypatch): + data_root = tmp_path / "data" + data_root.mkdir(parents=True, exist_ok=True) + db_path = data_root / "catalog.sqlite" + + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(data_root)) + monkeypatch.setenv("MESA_DATA_MESA_API_KEY", "test_secret_api_key") + + migrate(None, db_path) + conn = get_connection(db_path) + + # 1. Setup Target Settings + settings = MesaTargetSettings( + target_key="default", + base_url="https://mock-mesa.internal", + tenant_id="default", + workspace_id="legal", + dataset_id="tr_legislation", + agent_id="publisher", + contract_source="configured", + health_path="/v4/health", + publish_path="/v4/sources/chunks", + mutation_status_path_template="/v4/mutations/{mutation_id}", + ) + upsert_mesa_target_settings(conn, settings) + + # 2. Setup document and approved version + doc_id = "tr:legislation:law:5237" + upsert_document(conn, doc_id, "legislation", "law", "TR", "Türk Ceza Kanunu", "stable-5237", "approved") + + insert_artifact( + conn=conn, + artifact_id="art-5237-v1", + document_id=doc_id, + source_id="resmi_gazete", + source_url="https://resmigazete.gov.tr/5237.html", + retrieved_at="2026-01-01T00:00:00Z", + fetch_method="http", + http_status=200, + declared_content_type="text/html", + detected_content_type="text/html", + byte_size=1024, + sha256="1111111111111111111111111111111111111111111111111111111111111111", + raw_path="raw/resmi_gazete/5237.html", + etag=None, + last_modified=None, + transport_status="fetched", + error_code=None, + metadata_json="{}", + ) + + # Write canonical file with one authoritative document record and 2 articles. + c_rel_path = "canonical/legislation/5237.jsonl" + c_abs_path = data_root / c_rel_path + c_abs_path.parent.mkdir(parents=True, exist_ok=True) + article_1 = "MADDE 1- Ceza kanununun amacı adaleti sağlamaktır." + article_2 = "MADDE 2- Kanunsuz suç olmaz." + full_text = f"{article_1}\n\n{article_2}" + records = [ + {"id": doc_id, "record_type": "legislation", "full_text": full_text}, + { + "id": "art-1", + "record_type": "article", + "article_number": "1", + "heading": "Madde 1", + "text": article_1, + "source_span": {"char_start": 0, "char_end": len(article_1)}, + }, + { + "id": "art-2", + "record_type": "article", + "article_number": "2", + "heading": "Madde 2", + "text": article_2, + "source_span": {"char_start": len(article_1) + 2, "char_end": len(full_text)}, + }, + ] + lines = [json.dumps(record, sort_keys=True) + "\n" for record in records] + with open(c_abs_path, "w", encoding="utf-8") as f: + f.writelines(lines) + hashes = [hashlib.sha256(line.encode()).hexdigest() for line in lines] + + v_id = f"{doc_id}:v1" + insert_version( + conn=conn, + version_id=v_id, + document_id=doc_id, + artifact_id="art-5237-v1", + version_kind="major", + snapshot_date="2026-01-01", + effective_from=None, + effective_to=None, + canonical_path=c_rel_path, + canonical_line=1, + canonical_sha256=hashes[0], + parser_name="legislation_parser", + parser_version="1.0.0", + schema_version="1.0.0", + validation_status="valid", + privacy_status="clean", + approval_status="approved", + revision_number=1, + quality_status="PASS", + ) + + for line_number, record in enumerate(records, start=1): + insert_record( + conn=conn, + record_id=record["id"], + version_id=v_id, + record_type=record["record_type"], + canonical_path=c_rel_path, + canonical_line=line_number, + record_sha256=hashes[line_number - 1], + validation_status="valid", + approval_status="approved", + ) + + conn.close() + return db_path + + +@respx.mock +def test_full_delivery_success_and_cross_release_dedup(setup_publisher_env): + # Mock MESA endpoints + respx.get("https://mock-mesa.internal/v4/health").respond(200, json={"status": "ok"}) + respx.post("https://mock-mesa.internal/v4/sources/chunks").respond( + 200, json={"mutation_id": "mut-101", "state": "COMMITTED", "message": "Committed directly"} + ) + + # First delivery + del1 = execute_publish_delivery(delivery_id="del-run-1") + assert del1["status"] == "COMMITTED" + assert del1["committed_items"] == 2 + assert del1["skipped_items"] == 0 + assert del1["failed_items"] == 0 + + # Second delivery (Cross-release dedup check: same content must be SKIPPED) + del2 = execute_publish_delivery(delivery_id="del-run-2") + assert del2["status"] == "COMMITTED" + assert del2["committed_items"] == 0 + assert del2["skipped_items"] == 2 + assert del2["failed_items"] == 0 + + +@respx.mock +def test_partial_failure_and_retry_workflow(setup_publisher_env): + respx.get("https://mock-mesa.internal/v4/health").respond(200, json={"status": "ok"}) + + # First call succeeds for chunk 1, fails for chunk 2 + call_count = 0 + + def chunk_handler(request): + nonlocal call_count + call_count += 1 + payload = json.loads(request.content) + if payload.get("ordinal") == 1: + return httpx.Response(200, json={"mutation_id": "mut-1", "state": "COMMITTED"}) + else: + return httpx.Response(500, text="Internal server error") + + respx.post("https://mock-mesa.internal/v4/sources/chunks").mock(side_effect=chunk_handler) + + # 1. Delivery results in PARTIAL + del_res = execute_publish_delivery(delivery_id="del-partial-1") + assert del_res["status"] == "PARTIAL" + assert del_res["committed_items"] == 1 + assert del_res["failed_items"] == 1 + + # 2. Fix the server for retry + respx.post("https://mock-mesa.internal/v4/sources/chunks").mock( + return_value=httpx.Response(200, json={"mutation_id": "mut-2", "state": "COMMITTED"}) + ) + + # 3. Retry only failed items + retry_res = retry_delivery_failures(delivery_id="del-partial-1") + assert retry_res["status"] == "COMMITTED" + assert retry_res["retried_count"] == 1 + assert retry_res["retried_success"] == 1 + assert retry_res["retried_failed"] == 0 + + +@respx.mock +def test_response_loss_retry_reuses_exact_idempotency_key(setup_publisher_env): + respx.get("https://mock-mesa.internal/v4/health").respond(200, json={"status": "ok"}) + seen_keys = [] + call_count = 0 + + def response_loss_then_commit(request): + nonlocal call_count + call_count += 1 + seen_keys.append(request.headers["Idempotency-Key"]) + if call_count == 1: + raise httpx.ReadTimeout("response lost after request was sent", request=request) + return httpx.Response(200, json={"mutation_id": "mut-after-loss", "state": "COMMITTED"}) + + respx.post("https://mock-mesa.internal/v4/sources/chunks").mock(side_effect=response_loss_then_commit) + + first = execute_publish_delivery(delivery_id="del-response-loss") + assert first["status"] == "PARTIAL" + assert first["failed_items"] == 1 + + retried = retry_delivery_failures("del-response-loss") + assert retried["status"] == "COMMITTED" + assert retried["retried_count"] == 1 + assert len(seen_keys) == 3 + assert seen_keys[0] == seen_keys[2] diff --git a/tests/integration/test_prompt4_integration_audit.py b/tests/integration/test_prompt4_integration_audit.py new file mode 100644 index 0000000..649ef4e --- /dev/null +++ b/tests/integration/test_prompt4_integration_audit.py @@ -0,0 +1,833 @@ +import hashlib +import json + +import pytest +import respx +from fastapi.testclient import TestClient + +from mesa_legal_data.catalog import ( + BlockingValidationIssueExists, + approve_version_streaming, + evaluate_auto_approval, + get_connection, + get_record, + insert_artifact, + insert_record, + insert_version, + iter_records_for_release, + migrate, + set_parser_certification, + upsert_document, + upsert_source, + upsert_source_operational_settings, +) +from mesa_legal_data.parsers.citations import extract_citations +from mesa_legal_data.parsers.coverage import compute_parsing_coverage +from mesa_legal_data.parsers.encoding import decode_source_bytes +from mesa_legal_data.parsers.legislation import parse_legislation_text +from mesa_legal_data.parsers.text_normalizer import normalize_text +from mesa_legal_data.pipeline import process_artifact_pipeline +from mesa_legal_data.publisher.engine import execute_publish_delivery +from mesa_legal_data.publisher.ledger import ( + upsert_mesa_target_settings, +) +from mesa_legal_data.publisher.models import MesaTargetSettings +from mesa_legal_data.quality import evaluate_quality +from mesa_legal_data.web.app import create_app + + +@pytest.fixture +def audit_env(tmp_path, monkeypatch): + data_root = tmp_path / "data" + data_root.mkdir(parents=True, exist_ok=True) + db_path = data_root / "catalog.sqlite" + + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(data_root)) + monkeypatch.setenv("MESA_DATA_ADMIN_TOKEN", "audit-admin-token") + monkeypatch.setenv("MESA_DATA_MESA_API_KEY", "audit-mesa-secret-key") + + migrate(None, db_path) + return {"data_root": data_root, "db_path": db_path} + + +# ============================================================================== +# KONTROL 1 — VERSIONING +# ============================================================================== +def test_kontrol_1_versioning_isolation_and_stability(audit_env): + """ + KONTROL 1 Verification: + - Document X with Version A and Version B + - A survives, B survives + - A Article 1 survives, B Article 1 is a separate instance + - B does not overwrite A + - Reprocess A does not produce duplicate version or records + - revision_number is strictly sequential (1 -> 2) + - version_kind is stable + """ + doc_id = "tr:legislation:law:4721" + conn = get_connection(audit_env["db_path"]) + upsert_source(conn, "mevzuat", "Mevzuat", "Official", "https://mevzuat.gov.tr") + upsert_document(conn, doc_id, "legislation", "law", "TR", "Türk Medeni Kanunu", "stable-4721", "fetched") + + # 1. Version A + content_a = """ +

TÜRK MEDENİ KANUNU

+

MADDE 1- Kanun, sözüyle ve özüyle değindiği bütün konularda uygulanır.

+

MADDE 2- Herkes, haklarını kullanırken dürüstlük kuralına uymak zorundadır.

+ """ + raw_dir = audit_env["data_root"] / "raw" / "legislation" / "mevzuat" / "2026" + raw_dir.mkdir(parents=True, exist_ok=True) + file_a = raw_dir / "tmk_v1.html" + bytes_a = content_a.encode("utf-8") + file_a.write_bytes(bytes_a) + sha_a = hashlib.sha256(bytes_a).hexdigest() + art_id_a = f"art-{sha_a[:12]}" + + insert_artifact( + conn, + art_id_a, + doc_id, + "mevzuat", + "https://example.com/tmk_v1.html", + "2026-01-01T00:00:00Z", + "manual", + 200, + "text/html", + "text/html", + len(bytes_a), + sha_a, + str(file_a.relative_to(audit_env["data_root"])), + None, + None, + "fetched", + None, + json.dumps({"publication_date": "2026-01-01", "source_role": "consolidated_snapshot"}), + ) + conn.close() + + # Process Version A + process_artifact_pipeline(artifact_id=art_id_a, document_id=doc_id) + + conn = get_connection(audit_env["db_path"]) + c = conn.cursor() + c.execute("SELECT version_id, revision_number, version_kind FROM versions WHERE document_id = ?", (doc_id,)) + v1_row = c.fetchone() + assert v1_row is not None + v1_id, v1_rev, v1_kind = v1_row + assert v1_rev == 1 + assert v1_kind == "consolidated_snapshot" + + # Approve Version A + approve_version_streaming(conn, version_id=v1_id, reviewer="auditor-1") + + # Reprocess Version A (Idempotency test) + process_artifact_pipeline(artifact_id=art_id_a, document_id=doc_id) + c.execute("SELECT count(*) FROM versions WHERE document_id = ?", (doc_id,)) + assert c.fetchone()[0] == 1 # No duplicate version! + c.execute("SELECT count(*) FROM records WHERE version_id = ?", (v1_id,)) + assert c.fetchone()[0] > 0 + + # 2. Version B (Amended Article 1) + content_b = """ +

TÜRK MEDENİ KANUNU

+

MADDE 1- Kanun, sözüyle ve özüyle değindiği bütün konularda uygulanır. (2026 revizyonu)

+

MADDE 2- Herkes, haklarını kullanırken dürüstlük kuralına uymak zorundadır.

+ """ + file_b = raw_dir / "tmk_v2.html" + bytes_b = content_b.encode("utf-8") + file_b.write_bytes(bytes_b) + sha_b = hashlib.sha256(bytes_b).hexdigest() + art_id_b = f"art-{sha_b[:12]}" + + insert_artifact( + conn, + art_id_b, + doc_id, + "mevzuat", + "https://example.com/tmk_v2.html", + "2026-06-01T00:00:00Z", + "manual", + 200, + "text/html", + "text/html", + len(bytes_b), + sha_b, + str(file_b.relative_to(audit_env["data_root"])), + None, + None, + "fetched", + None, + json.dumps({"publication_date": "2026-06-01", "source_role": "consolidated_snapshot"}), + ) + conn.close() + + # Process Version B + process_artifact_pipeline(artifact_id=art_id_b, document_id=doc_id) + + conn = get_connection(audit_env["db_path"]) + c = conn.cursor() + c.execute( + "SELECT version_id, revision_number, supersedes_version_id FROM versions WHERE document_id = ? ORDER BY revision_number ASC", + (doc_id,), + ) + versions = c.fetchall() + assert len(versions) == 2 + v1_id, v1_rev, v1_super = versions[0] + v2_id, v2_rev, v2_super = versions[1] + + assert v1_rev == 1 + assert v2_rev == 2 + assert v2_super is None + + # Check record instances for Article 1 + logical_art1_id = f"{doc_id}:article:1" + rec_v1 = get_record(conn, logical_art1_id, version_id=v1_id) + rec_v2 = get_record(conn, logical_art1_id, version_id=v2_id) + + assert rec_v1 is not None + assert rec_v2 is not None + assert rec_v1["version_id"] == v1_id + assert rec_v2["version_id"] == v2_id + assert rec_v1["record_instance_id"] != rec_v2["record_instance_id"] + assert rec_v1["approval_status"] == "approved" + assert rec_v2["approval_status"] == "pending" + assert rec_v1["record_sha256"] != rec_v2["record_sha256"] # Distinct content! + conn.close() + + +# ============================================================================== +# KONTROL 2 — CANONICAL / SPANS +# ============================================================================== +def test_kontrol_2_canonical_spans_and_coverage_honest(): + """ + KONTROL 2 Verification: + - Normal kanun, preamble, annex, geçici madde, ek madde, malformed marker, cp1254, UTF-8, large article + - Canonical deterministic + - Span boundaries valid + - No silent loss + - Honest coverage with uncovered ranges present + """ + # 1. CP1254 decoding trial + cp1254_raw = "TÜRK CEZA KANUNU — YÜRÜRLÜK VE İNTİKAL HÜKÜMLERİ\nÇalışma, işçi, tebliğ".encode("cp1254") + decoded, enc = decode_source_bytes(cp1254_raw, is_html=False) + assert enc in ("cp1254", "windows-1254") + assert "Çalışma" in decoded + assert "tebliğ" in decoded + + # 2. Complex Fixture + preamble = "TÜRK BORÇLAR KANUNU\nGenel Hükümler ve Başlangıç İlkeleri\n\n" + art_1 = "MADDE 1 - Sözleşme, tarafların iradelerini karşılıklı açıklamalarıyla kurulur.\nİrade açıklaması açık veya örtülü olabilir.\n\n" + art_2_large = ( + "MADDE 2 - Esaslı noktalar:\n" + ("Bu fıkra geniş ve detaylı sözleşme ilkelerini açıklar.\n" * 50) + "\n" + ) + ek_madde = "EK MADDE 1 - Elektronik ortamda kurulan sözleşmelerde güvenli elektronik imza geçerlidir.\n\n" + gecici_madde = ( + "GEÇİCİ MADDE 1 - Bu Kanunun yürürlüğe girdiği tarihten önceki sözleşmelere eski hükümler uygulanır.\n\n" + ) + annex = "EK CETVEL: Yürürlükten kaldırılan mevzuat listesi ve karşılaştırma tablosu." + + raw_text = preamble + art_1 + art_2_large + ek_madde + gecici_madde + annex + canonical_text = normalize_text(raw_text) + + # Invariant: Normalization is deterministic + assert canonical_text == normalize_text(canonical_text) + + parsed = parse_legislation_text(canonical_text, auto_normalize=False) + assert len(parsed.articles) == 4 + + # Validate exact span slices + for art in parsed.articles: + assert art.char_start is not None and art.char_end is not None + assert art.char_start < art.char_end + slice_text = canonical_text[art.char_start : art.char_end] + assert slice_text.startswith( + f"{'EK ' if art.article_kind == 'additional' else 'GEÇİCİ ' if art.article_kind == 'temporary' else ''}MADDE {art.article_number}" + ) + + # Coverage computation + spans = [(a.char_start, a.char_end) for a in parsed.articles if a.char_start is not None and a.char_end is not None] + cov = compute_parsing_coverage(canonical_text, spans) + assert cov.canonical_chars == len(canonical_text) + assert cov.covered_chars > 0 + assert cov.uncovered_chars > 0 + assert 0.0 < cov.coverage_ratio < 1.0 + + # Uncovered ranges must include preamble and trailing annex + types = [r["candidate_type"] for r in cov.uncovered_ranges] + assert "preamble" in types + assert "annex_trailing" in types + + +# ============================================================================== +# KONTROL 3 — QUALITY +# ============================================================================== +def test_kontrol_3_quality_gate_and_release_guard(audit_env): + """ + KONTROL 3 Verification: + - Healthy -> PASS + - Uncertain -> REVIEW + - Corrupt -> BLOCK + - BLOCK cannot enter release plan or be approved + """ + sample_records = [ + { + "id": "tr:legislation:law:100", + "record_type": "legislation", + "title": "100 Sayılı Kanun", + "source": {"artifact_sha256": "a" * 64}, + "provenance": {"pipeline_run_id": "run-1"}, + }, + { + "id": "tr:legislation:law:100:article:1", + "record_type": "article", + "source_span": {"char_start": 0, "char_end": 41, "ordinal": 1}, + "source": {"artifact_sha256": "a" * 64}, + "provenance": {"pipeline_run_id": "run-1"}, + }, + ] + + # 1. Healthy PASS + text_pass = "MADDE 1 - Kanun amacı düzeni sağlamaktır." + cov_pass = compute_parsing_coverage(text_pass, [(0, 41)]) + rep_pass = evaluate_quality( + source_info={"source_id": "mevzuat", "source_url": "https://example.com/law"}, + raw_info={"byte_size": 100, "sha256": "a" * 64, "file_exists": True}, + canonical_records=sample_records, + canonical_text=text_pass, + coverage=cov_pass, + privacy_issues=[], + ) + assert rep_pass.decision == "PASS" + + # 2. Uncertain REVIEW (Mojibake replacement char) + text_rev = "MADDE 1 - Metin \ufffd karakteri içeriyor." + rec_rev = [ + { + "id": "tr:legislation:law:101", + "record_type": "legislation", + "title": "101 Sayılı Kanun", + "source": {"artifact_sha256": "b" * 64}, + "provenance": {"pipeline_run_id": "run-2"}, + } + ] + rep_rev = evaluate_quality( + source_info={"source_id": "mevzuat", "source_url": "https://example.com/law"}, + raw_info={"byte_size": 100, "sha256": "b" * 64, "file_exists": True}, + canonical_records=rec_rev, + canonical_text=text_rev, + coverage=None, + privacy_issues=[], + ) + assert rep_rev.decision == "REVIEW" + + # 3. Corrupt BLOCK (Invalid inverted span) + rec_corrupt = [ + { + "id": "tr:legislation:law:102", + "record_type": "legislation", + "title": "102 Sayılı Kanun", + "source": {"artifact_sha256": "c" * 64}, + "provenance": {"pipeline_run_id": "run-3"}, + }, + { + "id": "tr:legislation:law:102:article:1", + "record_type": "article", + "source_span": {"char_start": 50, "char_end": 10, "ordinal": 1}, # Inverted span + "source": {"artifact_sha256": "c" * 64}, + "provenance": {"pipeline_run_id": "run-3"}, + }, + ] + rep_block = evaluate_quality( + source_info={"source_id": "mevzuat", "source_url": "https://example.com/law"}, + raw_info={"byte_size": 100, "sha256": "c" * 64, "file_exists": True}, + canonical_records=rec_corrupt, + canonical_text="MADDE 1 - Metin", + coverage=None, + privacy_issues=[], + ) + assert rep_block.decision == "BLOCK" + + # 4. Release Guard in Catalog + conn = get_connection(audit_env["db_path"]) + upsert_source(conn, "mevzuat", "Mevzuat", "Official", "https://mevzuat.gov.tr") + upsert_document(conn, "doc:block:test", "legislation", "law", "TR", "Blocked Law", "stable-block", "fetched") + insert_artifact( + conn, + "art-block", + "doc:block:test", + "mevzuat", + "https://example.com/block.html", + "2026-01-01T00:00:00Z", + "manual", + 200, + "text/html", + "text/html", + 100, + "d" * 64, + "raw/block.html", + None, + None, + "fetched", + None, + "{}", + ) + insert_version( + conn=conn, + version_id="doc:block:test:v1", + document_id="doc:block:test", + artifact_id="art-block", + version_kind="major", + snapshot_date="2026-01-01", + effective_from=None, + effective_to=None, + canonical_path="canonical/test.jsonl", + canonical_line=1, + canonical_sha256="1" * 64, + parser_name="test", + parser_version="1.0.0", + schema_version="1.0.0", + validation_status="invalid", + privacy_status="blocked", + approval_status="pending", + revision_number=1, + quality_status="BLOCK", + ) + + # Attempting to approve BLOCK version raises exception + with pytest.raises(BlockingValidationIssueExists): + approve_version_streaming(conn, version_id="doc:block:test:v1", reviewer="auditor") + + # Release iterator excludes BLOCK version + records_for_rel = list(iter_records_for_release(conn)) + assert not any(r["version_id"] == "doc:block:test:v1" for r in records_for_rel) + conn.close() + + +# ============================================================================== +# KONTROL 4 — CITATIONS +# ============================================================================== +def test_kontrol_4_citations_lifecycle_and_semantics(): + """ + KONTROL 4 Verification: + - regex match != resolved + - explicit law number + - alias (TCK, CMK, HMK, TMK, KVKK, AY) + - unresolved + - false positive prevention + - source span exact match + """ + # Explicit numbered law + cits_num = extract_citations("5237 sayılı Türk Ceza Kanunu madde 81 uyarınca") + assert len(cits_num) >= 1 + assert cits_num[0].target_legislation_id == "tr:legislation:law:5237" + assert cits_num[0].citation_status == "RESOLVED" + assert cits_num[0].char_start is not None + assert "5237 sayılı" in cits_num[0].raw_text + + # Alias + cits_alias = extract_citations("KVKK m. 6 uyarınca özel nitelikli veri") + assert len(cits_alias) >= 1 + assert cits_alias[0].target_legislation_id == "tr:legislation:law:6698" + assert cits_alias[0].target_article_id == "tr:legislation:law:6698:article:6" + + # Unresolved + cits_unres = extract_citations("İlgili Kanun ve mevzuat hükümleri uyarınca") + for c in cits_unres: + if c.target_legislation_id is None: + assert c.citation_status == "UNRESOLVED" + + # False Positives + fps = [ + "Toplantı saat 1500'de başladı.", + "Toplam 5000 TL ödendi.", + "Yaklaşık 100 kişi katıldı.", + ] + for fp in fps: + assert len(extract_citations(fp)) == 0 + + +# ============================================================================== +# KONTROL 5 — AUTO APPROVAL +# ============================================================================== +def test_kontrol_5_auto_approval_engine(audit_env): + """ + KONTROL 5 Verification: + - PASS + certified -> auto approve + - PASS + uncertified -> human + - new parser -> human + - sampled -> human + - REVIEW -> human + - BLOCK -> stop + """ + conn = get_connection(audit_env["db_path"]) + upsert_source(conn, "rg", "Resmi Gazete", "Gov", "https://rg.gov.tr") + insert_artifact( + conn, + "art-auto", + None, + "rg", + "https://rg.gov.tr/law.html", + "2026-01-01T00:00:00Z", + "manual", + 200, + "text/html", + "text/html", + 100, + "e" * 64, + "raw/auto.html", + None, + None, + "fetched", + None, + "{}", + ) + + upsert_source_operational_settings(conn, "rg", enabled=True, auto_approval_enabled=True, weekly_sample_count=0) + set_parser_certification(conn, "rg", "rg_parser", "1.0.0", certified=True) + set_parser_certification(conn, "rg", "rg_parser", "2.0.0", certified=False) + + upsert_document(conn, "doc:auto:1", "legislation", "law", "TR", "Auto Law 1", "key1", "fetched") + auto_path = audit_env["data_root"] / "canonical/1.jsonl" + auto_path.parent.mkdir(parents=True, exist_ok=True) + auto_line = json.dumps({"id": "auto-rec-1", "record_type": "article"}, sort_keys=True) + "\n" + auto_path.write_text(auto_line, encoding="utf-8") + auto_hash = hashlib.sha256(auto_line.encode()).hexdigest() + insert_version( + conn=conn, + version_id="doc:auto:1:v1", + document_id="doc:auto:1", + artifact_id="art-auto", + version_kind="major", + snapshot_date="2026-01-01", + effective_from=None, + effective_to=None, + canonical_path="canonical/1.jsonl", + canonical_line=1, + canonical_sha256=auto_hash, + parser_name="rg_parser", + parser_version="1.0.0", + schema_version="1.0.0", + validation_status="valid", + privacy_status="clean", + approval_status="pending", + revision_number=1, + quality_status="PASS", + ) + insert_record(conn, "auto-rec-1", "doc:auto:1:v1", "article", "canonical/1.jsonl", 1, auto_hash) + + # 1. PASS + certified -> Auto Approve + app_1, _ = evaluate_auto_approval( + conn=conn, + version_id="doc:auto:1:v1", + source_id="rg", + parser_name="rg_parser", + parser_version="1.0.0", + quality_decision="PASS", + has_privacy_blocker=False, + schema_valid=True, + ) + assert app_1 is True + + # 2. PASS + uncertified parser -> Human + upsert_document(conn, "doc:auto:2", "legislation", "law", "TR", "Auto Law 2", "key2", "fetched") + insert_version( + conn=conn, + version_id="doc:auto:2:v1", + document_id="doc:auto:2", + artifact_id="art-auto", + version_kind="major", + snapshot_date="2026-01-01", + effective_from=None, + effective_to=None, + canonical_path="canonical/2.jsonl", + canonical_line=1, + canonical_sha256="2" * 64, + parser_name="rg_parser", + parser_version="2.0.0", + schema_version="1.0.0", + validation_status="valid", + privacy_status="clean", + approval_status="pending", + revision_number=1, + quality_status="PASS", + ) + app_2, _ = evaluate_auto_approval( + conn=conn, + version_id="doc:auto:2:v1", + source_id="rg", + parser_name="rg_parser", + parser_version="2.0.0", + quality_decision="PASS", + has_privacy_blocker=False, + schema_valid=True, + ) + assert app_2 is False + + # 3. REVIEW -> Human + upsert_document(conn, "doc:auto:3", "legislation", "law", "TR", "Auto Law 3", "key3", "fetched") + insert_version( + conn=conn, + version_id="doc:auto:3:v1", + document_id="doc:auto:3", + artifact_id="art-auto", + version_kind="major", + snapshot_date="2026-01-01", + effective_from=None, + effective_to=None, + canonical_path="canonical/3.jsonl", + canonical_line=1, + canonical_sha256="3" * 64, + parser_name="rg_parser", + parser_version="1.0.0", + schema_version="1.0.0", + validation_status="valid", + privacy_status="clean", + approval_status="pending", + revision_number=1, + quality_status="REVIEW", + ) + app_3, _ = evaluate_auto_approval( + conn=conn, + version_id="doc:auto:3:v1", + source_id="rg", + parser_name="rg_parser", + parser_version="1.0.0", + quality_decision="REVIEW", + has_privacy_blocker=False, + schema_valid=True, + ) + assert app_3 is False + + # 4. BLOCK -> Stop + upsert_document(conn, "doc:auto:4", "legislation", "law", "TR", "Auto Law 4", "key4", "fetched") + insert_version( + conn=conn, + version_id="doc:auto:4:v1", + document_id="doc:auto:4", + artifact_id="art-auto", + version_kind="major", + snapshot_date="2026-01-01", + effective_from=None, + effective_to=None, + canonical_path="canonical/4.jsonl", + canonical_line=1, + canonical_sha256="4" * 64, + parser_name="rg_parser", + parser_version="1.0.0", + schema_version="1.0.0", + validation_status="invalid", + privacy_status="blocked", + approval_status="pending", + revision_number=1, + quality_status="BLOCK", + ) + app_4, _ = evaluate_auto_approval( + conn=conn, + version_id="doc:auto:4:v1", + source_id="rg", + parser_name="rg_parser", + parser_version="1.0.0", + quality_decision="BLOCK", + has_privacy_blocker=True, + schema_valid=False, + ) + assert app_4 is False + conn.close() + + +# ============================================================================== +# KONTROL 6 — PANEL FRESH-USER JOURNEY +# ============================================================================== +def test_kontrol_6_panel_fresh_user_journey(audit_env): + """ + KONTROL 6 Verification: + Fresh-user journey: Dashboard -> Collect -> Library -> Review -> Approve -> Ready -> Publish + Checks: + - No terminal required + - No artifact ID typing required + - No YAML required + - No dead buttons + - Honest local staging label ("Yerel Development Staging") + """ + app = create_app() + headers = {"X-MESA-Requested-With": "web-admin"} + client = TestClient(app, headers=headers) + + # 1. Dashboard Health & Metrics + res_dash = client.get("/api/dashboard/stats", headers=headers) + assert res_dash.status_code == 200 + res_json = res_dash.json() + stats_data = res_json.get("data", {}) + health = stats_data.get("health", {}) + assert "discovered_today" in health + assert "processed_today" in health + assert "auto_approved_today" in health + assert "needs_review_count" in health + assert "blocked_count" in health + assert "mesa_ready_count" in health + + # 2. Collect via URL (No artifact ID or terminal required) + res_sources = client.get("/api/sources", headers=headers) + assert res_sources.status_code == 200 + + # 3. Library List + res_lib = client.get("/api/documents?page=1&page_size=20", headers=headers) + assert res_lib.status_code == 200 + + # 4. Review List + res_rev = client.get("/api/reviews/records?limit=50", headers=headers) + assert res_rev.status_code == 200 + + # 5. Publisher Ready Summary + res_ready = client.get("/api/publisher/ready-summary", headers=headers) + assert res_ready.status_code == 200 + ready = res_ready.json().get("data", {}) + assert "ready_documents" in ready + assert "ready_versions" in ready + assert "estimated_chunks" in ready + + # 6. Preflight + res_pre = client.post("/api/publisher/preflight", headers=headers) + assert res_pre.status_code == 200 + report = res_pre.json().get("data", {}) + assert "checks" in report + assert "overall_status" in report + + +# ============================================================================== +# KONTROL 7, 8, 9 — MESA PUBLISHER & COMMITTED TRUTH & MANUAL PUSH +# ============================================================================== +@respx.mock +def test_kontrol_7_8_9_publisher_contract_and_committed_truth(audit_env): + """ + KONTROL 7, 8, 9 Verification: + - Stable idempotency + - Second release skips same content + - Delivery persistence + - Restart / retry + - UI / Ledger truth: COMMITTED only when all items are COMMITTED + """ + conn = get_connection(audit_env["db_path"]) + settings = MesaTargetSettings( + target_key="default", + base_url="https://mock-mesa.test", + tenant_id="default", + workspace_id="legal", + dataset_id="tr_legislation", + agent_id="publisher", + contract_source="configured", + health_path="/v4/health", + publish_path="/v4/sources/chunks", + mutation_status_path_template="/v4/mutations/{mutation_id}", + ) + upsert_mesa_target_settings(conn, settings) + + # Create approved test document and version + doc_id = "tr:legislation:law:6100" + upsert_document(conn, doc_id, "legislation", "law", "TR", "HMK", "stable-6100", "approved") + insert_artifact( + conn, + "art-6100", + doc_id, + "mevzuat", + "https://example.com/hmk.html", + "2026-01-01T00:00:00Z", + "manual", + 200, + "text/html", + "text/html", + 100, + "a" * 64, + "raw/hmk.html", + None, + None, + "fetched", + None, + "{}", + ) + + c_rel = "canonical/hmk.jsonl" + c_abs = audit_env["data_root"] / c_rel + c_abs.parent.mkdir(parents=True, exist_ok=True) + canonical_text = "MADDE 1 - Görev kuralları kamu düzenindendir." + canonical_line = ( + json.dumps({"id": doc_id, "record_type": "legislation", "full_text": canonical_text}, sort_keys=True) + "\n" + ) + c_abs.write_text(canonical_line, encoding="utf-8") + canonical_hash = hashlib.sha256(canonical_line.encode()).hexdigest() + + v_id = f"{doc_id}:v1" + insert_version( + conn, + v_id, + doc_id, + "art-6100", + "major", + "2026-01-01", + None, + None, + c_rel, + 1, + canonical_hash, + "parser", + "1.0.0", + "1.0.0", + "valid", + "clean", + "approved", + 1, + "PASS", + ) + insert_record(conn, doc_id, v_id, "legislation", c_rel, 1, canonical_hash, "valid", "approved") + conn.close() + + # Mock MESA HTTP endpoints + respx.get("https://mock-mesa.test/v4/health").respond(200, json={"status": "ok"}) + respx.post("https://mock-mesa.test/v4/sources/chunks").respond( + 200, json={"mutation_id": "mut-hmk-1", "state": "COMMITTED"} + ) + + # 1. First publish -> COMMITTED + del1 = execute_publish_delivery(delivery_id="del-audit-1") + assert del1["status"] == "COMMITTED" + assert del1["committed_items"] == 1 + assert del1["skipped_items"] == 0 + + # 2. Second publish -> SKIPPED (Cross-release dedup) + del2 = execute_publish_delivery(delivery_id="del-audit-2") + assert del2["status"] == "COMMITTED" + assert del2["committed_items"] == 0 + assert del2["skipped_items"] == 1 # Chunk was skipped because it's already COMMITTED! + + +# ============================================================================== +# KONTROL 10 & 11 — SECURITY & ARCHITECTURE +# ============================================================================== +def test_kontrol_10_and_11_security_and_clean_architecture(audit_env): + """ + KONTROL 10 & 11 Verification: + - API key not stored plaintext in DB + - Admin auth enforced + - No overengineered message brokers or vector DBs + """ + # 1. DB check for plaintext API key + conn = get_connection(audit_env["db_path"]) + c = conn.cursor() + c.execute("SELECT * FROM mesa_target_settings") + assert c.fetchone() is not None + # Check that settings do NOT contain any api_key column + col_names = [d[0] for d in c.description] + assert "api_key" not in col_names + conn.close() + + # 2. Admin Auth check + app = create_app() + client = TestClient(app) + + # Unauthenticated mutation rejected + res_unauth = client.post("/api/publisher/publish", json={}) + assert res_unauth.status_code in (401, 403) + + # 3. Authenticated request accepted + res_auth = client.get("/api/publisher/settings", headers={"X-Admin-Token": "audit-admin-token"}) + assert res_auth.status_code == 200 diff --git a/tests/integration/test_web_panel_prompt2.py b/tests/integration/test_web_panel_prompt2.py new file mode 100644 index 0000000..83a0bd6 --- /dev/null +++ b/tests/integration/test_web_panel_prompt2.py @@ -0,0 +1,178 @@ +import hashlib +import json +import os +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from mesa_legal_data.catalog import ( + get_connection, + get_db_path, + insert_artifact, + insert_record, + insert_version, + migrate, + upsert_document, +) +from mesa_legal_data.web.app import create_app + + +@pytest.fixture +def test_client(tmp_path, monkeypatch): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + monkeypatch.setenv("MESA_DATA_MESA_STAGING_DB", str(tmp_path / "mesa_staging.sqlite")) + + db_path = get_db_path() + migrate(None, db_path) + + app = create_app() + client = TestClient(app) + return client + + +def test_dashboard_health_metrics_and_mesa_status(test_client): + res = test_client.get("/api/dashboard/stats") + assert res.status_code == 200 + data = res.json()["data"] + + assert "health" in data + health = data["health"] + assert "discovered_today" in health + assert "processed_today" in health + assert "auto_approved_today" in health + assert "needs_review_count" in health + assert "blocked_count" in health + assert "mesa_ready_count" in health + assert health["mesa_status"] == "not_configured" + assert "MESA entegrasyonu yapılandırılmadı" in health["mesa_status_label"] + + +def test_sources_settings_and_certifications_endpoints(test_client): + headers = {"X-MESA-Requested-With": "web-admin"} + + # 1. Get initial settings + res = test_client.get("/api/sources/settings") + assert res.status_code == 200 + settings = res.json()["data"] + assert isinstance(settings, list) + assert any(s["source_id"] == "resmi_gazete" for s in settings) + + # 2. Update operational settings + update_payload = { + "enabled": True, + "auto_approval_enabled": True, + "weekly_sample_count": 25, + } + res_post = test_client.post("/api/sources/resmi_gazete/settings", json=update_payload, headers=headers) + assert res_post.status_code == 200 + updated = res_post.json()["data"] + assert updated["source_id"] == "resmi_gazete" + assert updated["auto_approval_enabled"] is True + assert updated["weekly_sample_count"] == 25 + + # 3. Update parser certification + cert_payload = { + "parser_name": "resmi_gazete_parser", + "parser_version": "1.0.0", + "certified": True, + "certified_by": "qa-lead", + } + res_cert = test_client.post("/api/sources/resmi_gazete/certify-parser", json=cert_payload, headers=headers) + assert res_cert.status_code == 200 + certs = res_cert.json()["data"] + assert any( + c["parser_name"] == "resmi_gazete_parser" and c["parser_version"] == "1.0.0" and c["certified"] for c in certs + ) + + +def test_document_versions_and_pending_reviews_endpoint(test_client): + headers = {"X-MESA-Requested-With": "web-admin"} + conn = get_connection() + upsert_document(conn, "tr:legislation:law:5237", "legislation", "law", "TR", "TCK", "5237", "fetched") + + insert_artifact( + conn=conn, + artifact_id="art-tck-1", + document_id="tr:legislation:law:5237", + source_id="resmi_gazete", + source_url="https://resmigazete.gov.tr/5237.html", + retrieved_at="2026-08-28T12:00:00Z", + fetch_method="http", + http_status=200, + declared_content_type="text/html", + detected_content_type="text/html", + byte_size=1024, + sha256="1111222233334444555566667777888899990000111122223333444455556666", + raw_path="raw/resmi_gazete/5237.html", + etag=None, + last_modified=None, + transport_status="fetched", + error_code=None, + metadata_json="{}", + ) + + canonical_path = "canonical/test.jsonl" + canonical_line = json.dumps({"id": "tck-rec-1", "record_type": "article"}, sort_keys=True) + "\n" + canonical_abs = Path(os.environ["MESA_DATA_DATA_ROOT"]) / canonical_path + canonical_abs.parent.mkdir(parents=True, exist_ok=True) + canonical_abs.write_text(canonical_line, encoding="utf-8") + canonical_hash = hashlib.sha256(canonical_line.encode()).hexdigest() + + insert_version( + conn=conn, + version_id="tr:legislation:law:5237:v1", + document_id="tr:legislation:law:5237", + artifact_id="art-tck-1", + version_kind="major", + snapshot_date="2026-01-01", + effective_from=None, + effective_to=None, + canonical_path=canonical_path, + canonical_line=1, + canonical_sha256=canonical_hash, + parser_name="resmi_gazete_parser", + parser_version="1.0.0", + schema_version="1.0.0", + validation_status="valid", + privacy_status="clean", + approval_status="pending", + revision_number=1, + quality_status="REVIEW", + quality_json=json.dumps({"checks": [{"name": "STRUCTURE", "status": "REVIEW"}]}), + ) + insert_record( + conn, + "tck-rec-1", + "tr:legislation:law:5237:v1", + "article", + canonical_path, + 1, + canonical_hash, + ) + conn.close() + + # 1. Query document versions + res_ver = test_client.get("/api/documents/tr:legislation:law:5237/versions") + assert res_ver.status_code == 200 + versions = res_ver.json()["data"]["versions"] + assert len(versions) == 1 + assert versions[0]["version_id"] == "tr:legislation:law:5237:v1" + assert versions[0]["revision_number"] == 1 + assert versions[0]["quality_status"] == "REVIEW" + + # 2. Query pending versions review queue + res_pending = test_client.get("/api/reviews/pending-versions") + assert res_pending.status_code == 200 + pending_items = res_pending.json()["data"]["items"] + assert len(pending_items) >= 1 + assert any(item["version_id"] == "tr:legislation:law:5237:v1" for item in pending_items) + + # 3. Approve version + res_app = test_client.post( + "/api/versions/tr:legislation:law:5237:v1/approve", + json={"reviewer": "test-admin", "note": "Verified manually"}, + headers=headers, + ) + assert res_app.status_code == 200 + assert res_app.json()["ok"] is True diff --git a/tests/integration/test_web_panel_publisher.py b/tests/integration/test_web_panel_publisher.py new file mode 100644 index 0000000..4cbee47 --- /dev/null +++ b/tests/integration/test_web_panel_publisher.py @@ -0,0 +1,172 @@ +import hashlib +import json + +import pytest +from fastapi.testclient import TestClient + +from mesa_legal_data.catalog import ( + get_connection, + insert_artifact, + insert_record, + insert_version, + migrate, + upsert_document, +) +from mesa_legal_data.publisher.ledger import upsert_mesa_target_settings +from mesa_legal_data.publisher.models import MesaTargetSettings +from mesa_legal_data.web.app import create_app + + +@pytest.fixture +def client(tmp_path, monkeypatch): + data_root = tmp_path / "data" + data_root.mkdir(parents=True, exist_ok=True) + db_path = data_root / "catalog.sqlite" + + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(data_root)) + monkeypatch.setenv("MESA_DATA_MESA_API_KEY", "env_secret_key") + + migrate(None, db_path) + conn = get_connection(db_path) + + upsert_mesa_target_settings( + conn, + MesaTargetSettings( + target_key="default", + base_url="https://mock-mesa.internal", + tenant_id="default", + workspace_id="legal", + dataset_id="tr_legislation", + agent_id="publisher", + contract_source="configured", + health_path="/v4/health", + publish_path="/v4/sources/chunks", + mutation_status_path_template="/v4/mutations/{mutation_id}", + ), + ) + + doc_id = "tr:legislation:law:6100" + upsert_document(conn, doc_id, "legislation", "law", "TR", "HMK", "stable-6100", "approved") + + insert_artifact( + conn=conn, + artifact_id="art-6100-v1", + document_id=doc_id, + source_id="resmi_gazete", + source_url="https://resmigazete.gov.tr/6100.html", + retrieved_at="2026-01-01T00:00:00Z", + fetch_method="http", + http_status=200, + declared_content_type="text/html", + detected_content_type="text/html", + byte_size=1024, + sha256="5555555555555555555555555555555555555555555555555555555555555555", + raw_path="raw/resmi_gazete/6100.html", + etag=None, + last_modified=None, + transport_status="fetched", + error_code=None, + metadata_json="{}", + ) + + c_rel_path = "canonical/legislation/6100.jsonl" + c_abs_path = data_root / c_rel_path + c_abs_path.parent.mkdir(parents=True, exist_ok=True) + canonical_line = ( + json.dumps( + { + "id": doc_id, + "record_type": "legislation", + "full_text": "MADDE 1- Görev kamu düzenine ilişkindir.", + }, + sort_keys=True, + ) + + "\n" + ) + c_abs_path.write_text(canonical_line, encoding="utf-8") + canonical_hash = hashlib.sha256(canonical_line.encode()).hexdigest() + + v_id = f"{doc_id}:v1" + insert_version( + conn=conn, + version_id=v_id, + document_id=doc_id, + artifact_id="art-6100-v1", + version_kind="major", + snapshot_date="2026-01-01", + effective_from=None, + effective_to=None, + canonical_path=c_rel_path, + canonical_line=1, + canonical_sha256=canonical_hash, + parser_name="legislation_parser", + parser_version="1.0.0", + schema_version="1.0.0", + validation_status="valid", + privacy_status="clean", + approval_status="approved", + revision_number=1, + quality_status="PASS", + ) + + insert_record( + conn=conn, + record_id=doc_id, + version_id=v_id, + record_type="legislation", + canonical_path=c_rel_path, + canonical_line=1, + record_sha256=canonical_hash, + validation_status="valid", + approval_status="approved", + ) + conn.close() + + app = create_app() + return TestClient(app) + + +def test_publisher_settings_and_preflight_endpoints(client): + headers = {"X-MESA-Requested-With": "web-admin"} + + # 1. GET Settings + res_get = client.get("/api/publisher/settings", headers=headers) + assert res_get.status_code == 200 + data = res_get.json()["data"] + assert data["base_url"] == "https://mock-mesa.internal" + assert data["api_key_configured"] is True + + # 2. POST Update Settings + res_post = client.post( + "/api/publisher/settings", + headers=headers, + json={ + "base_url": "https://mesa-updated.internal", + "tenant_id": "corp", + "workspace_id": "legal", + "dataset_id": "tr_legislation", + "agent_id": "publisher_v4", + "content_limit_chars": 65536, + "health_path": "/v4/health", + "publish_path": "/v4/sources/chunks", + "mutation_status_path_template": "/v4/mutations/{mutation_id}", + }, + ) + assert res_post.status_code == 200 + upd_data = res_post.json()["data"] + assert upd_data["base_url"] == "https://mesa-updated.internal" + assert upd_data["tenant_id"] == "corp" + assert upd_data["content_limit_chars"] == 65536 + + # 3. GET Ready Summary + res_sum = client.get("/api/publisher/ready-summary", headers=headers) + assert res_sum.status_code == 200 + sum_data = res_sum.json()["data"] + assert sum_data["ready_documents"] >= 1 + assert sum_data["ready_versions"] >= 1 + + # 4. GET Document MESA Status + res_doc_st = client.get("/api/documents/tr:legislation:law:6100/mesa-status", headers=headers) + assert res_doc_st.status_code == 200 + doc_st = res_doc_st.json()["data"] + assert doc_st["status"] == "Not Sent" diff --git a/tests/unit/test_auto_approval_and_certifications.py b/tests/unit/test_auto_approval_and_certifications.py new file mode 100644 index 0000000..e412fc8 --- /dev/null +++ b/tests/unit/test_auto_approval_and_certifications.py @@ -0,0 +1,299 @@ +import hashlib +import json +import os +from pathlib import Path + +import pytest + +from mesa_legal_data.catalog import ( + evaluate_auto_approval, + get_connection, + get_version, + insert_artifact, + insert_record, + insert_version, + migrate, + set_parser_certification, + upsert_document, + upsert_source_operational_settings, +) + + +@pytest.fixture +def db_conn(tmp_path, monkeypatch): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + db_path = tmp_path / "catalog.sqlite" + migrate(None, db_path) + conn = get_connection(db_path) + upsert_document( + conn, + document_id="doc-test-1", + family="legislation", + document_type="law", + jurisdiction="TR", + title="Test Kanun", + stable_key="doc-test-1-key", + lifecycle_status="fetched", + ) + insert_artifact( + conn=conn, + artifact_id="art-test-1", + document_id="doc-test-1", + source_id="resmi_gazete", + source_url="https://resmigazete.gov.tr/test.html", + retrieved_at="2026-08-28T12:00:00Z", + fetch_method="http", + http_status=200, + declared_content_type="text/html", + detected_content_type="text/html", + byte_size=1024, + sha256="1111222233334444555566667777888899990000111122223333444455556666", + raw_path="raw/resmi_gazete/test.html", + etag=None, + last_modified=None, + transport_status="fetched", + error_code=None, + metadata_json="{}", + ) + yield conn + conn.close() + + +def _add_version( + conn, + version_id, + revision_number=1, + quality_status="PASS", + validation_status="valid", + parser_version="1.0.0", + privacy_status="clean", +): + relative_path = f"canonical/{hashlib.sha256(version_id.encode()).hexdigest()[:12]}.jsonl" + canonical_path = Path(os.environ["MESA_DATA_DATA_ROOT"]) / relative_path + canonical_path.parent.mkdir(parents=True, exist_ok=True) + line = json.dumps({"id": "rec-1", "record_type": "article"}, sort_keys=True) + "\n" + canonical_path.write_text(line, encoding="utf-8") + line_hash = hashlib.sha256(line.encode()).hexdigest() + insert_version( + conn=conn, + version_id=version_id, + document_id="doc-test-1", + artifact_id="art-test-1", + version_kind="major", + snapshot_date="2026-01-01", + effective_from=None, + effective_to=None, + canonical_path=relative_path, + canonical_line=1, + canonical_sha256=line_hash, + parser_name="resmi_gazete_parser", + parser_version=parser_version, + schema_version="1.0.0", + validation_status=validation_status, + privacy_status=privacy_status, + approval_status="pending", + revision_number=revision_number, + quality_status=quality_status, + ) + insert_record( + conn, + record_id="rec-1", + version_id=version_id, + record_type="article", + canonical_path=relative_path, + canonical_line=1, + record_sha256=line_hash, + ) + + +def test_auto_approval_happy_path_certified_pass(db_conn): + upsert_source_operational_settings( + db_conn, source_id="resmi_gazete", enabled=True, auto_approval_enabled=True, weekly_sample_count=0 + ) + set_parser_certification( + db_conn, source_id="resmi_gazete", parser_name="resmi_gazete_parser", parser_version="1.0.0", certified=True + ) + + _add_version(db_conn, "doc-test-1:v1", revision_number=1, quality_status="PASS", validation_status="valid") + + approved, reason = evaluate_auto_approval( + conn=db_conn, + version_id="doc-test-1:v1", + source_id="resmi_gazete", + parser_name="resmi_gazete_parser", + parser_version="1.0.0", + quality_decision="PASS", + has_privacy_blocker=False, + schema_valid=True, + ) + + assert approved is True + assert "Auto-approved" in reason + + v = get_version(db_conn, "doc-test-1:v1") + assert v is not None + assert v["approval_status"] == "approved" + assert v["auto_approved"] == 1 + assert v["is_audit_sample"] == 0 + + +def test_auto_approval_uncertified_parser_stays_in_review(db_conn): + upsert_source_operational_settings( + db_conn, source_id="resmi_gazete", enabled=True, auto_approval_enabled=True, weekly_sample_count=0 + ) + set_parser_certification( + db_conn, source_id="resmi_gazete", parser_name="resmi_gazete_parser", parser_version="2.0.0", certified=False + ) + + _add_version( + db_conn, + "doc-test-1:v2", + revision_number=2, + quality_status="PASS", + validation_status="valid", + parser_version="2.0.0", + ) + + approved, reason = evaluate_auto_approval( + conn=db_conn, + version_id="doc-test-1:v2", + source_id="resmi_gazete", + parser_name="resmi_gazete_parser", + parser_version="2.0.0", + quality_decision="PASS", + has_privacy_blocker=False, + schema_valid=True, + ) + + assert approved is False + assert "not certified" in reason + + v = get_version(db_conn, "doc-test-1:v2") + assert v is not None + assert v["approval_status"] == "pending" + assert v["auto_approved"] == 0 + + +def test_auto_approval_cannot_bypass_stored_parser_version(db_conn): + upsert_source_operational_settings( + db_conn, source_id="resmi_gazete", enabled=True, auto_approval_enabled=True, weekly_sample_count=0 + ) + set_parser_certification( + db_conn, source_id="resmi_gazete", parser_name="resmi_gazete_parser", parser_version="1.4", certified=True + ) + _add_version( + db_conn, + "doc-test-1:v-parser-1.5", + quality_status="PASS", + validation_status="valid", + parser_version="1.5", + ) + + approved, reason = evaluate_auto_approval( + conn=db_conn, + version_id="doc-test-1:v-parser-1.5", + source_id="resmi_gazete", + parser_name="resmi_gazete_parser", + parser_version="1.4", + quality_decision="PASS", + has_privacy_blocker=False, + schema_valid=True, + ) + + assert approved is False + assert "stored version provenance" in reason + assert get_version(db_conn, "doc-test-1:v-parser-1.5")["approval_status"] == "pending" + + +def test_auto_approval_audit_sample_routes_to_review(db_conn): + upsert_source_operational_settings( + db_conn, source_id="resmi_gazete", enabled=True, auto_approval_enabled=True, weekly_sample_count=100 + ) + set_parser_certification( + db_conn, source_id="resmi_gazete", parser_name="resmi_gazete_parser", parser_version="1.0.0", certified=True + ) + + _add_version(db_conn, "doc-test-1:v3", revision_number=3, quality_status="PASS", validation_status="valid") + + approved, reason = evaluate_auto_approval( + conn=db_conn, + version_id="doc-test-1:v3", + source_id="resmi_gazete", + parser_name="resmi_gazete_parser", + parser_version="1.0.0", + quality_decision="PASS", + has_privacy_blocker=False, + schema_valid=True, + ) + + assert approved is False + assert "audit" in reason + + v = get_version(db_conn, "doc-test-1:v3") + assert v is not None + assert v["approval_status"] == "pending" + assert v["is_audit_sample"] == 1 + assert v["audit_sample_reason"] == "quality_audit_sample" + assert v["auto_approved"] == 0 + + +def test_auto_approval_blocked_quality_never_approved(db_conn): + upsert_source_operational_settings( + db_conn, source_id="resmi_gazete", enabled=True, auto_approval_enabled=True, weekly_sample_count=0 + ) + set_parser_certification( + db_conn, source_id="resmi_gazete", parser_name="resmi_gazete_parser", parser_version="1.0.0", certified=True + ) + + _add_version(db_conn, "doc-test-1:v4", revision_number=4, quality_status="BLOCK", validation_status="failed") + + approved, reason = evaluate_auto_approval( + conn=db_conn, + version_id="doc-test-1:v4", + source_id="resmi_gazete", + parser_name="resmi_gazete_parser", + parser_version="1.0.0", + quality_decision="BLOCK", + has_privacy_blocker=False, + schema_valid=False, + ) + + assert approved is False + assert "BLOCK" in reason + + v = get_version(db_conn, "doc-test-1:v4") + assert v is not None + assert v["approval_status"] == "pending" + assert v["auto_approved"] == 0 + + +def test_auto_approval_privacy_blocker_stops_approval(db_conn): + upsert_source_operational_settings( + db_conn, source_id="resmi_gazete", enabled=True, auto_approval_enabled=True, weekly_sample_count=0 + ) + set_parser_certification( + db_conn, source_id="resmi_gazete", parser_name="resmi_gazete_parser", parser_version="1.0.0", certified=True + ) + + _add_version( + db_conn, + "doc-test-1:v5", + revision_number=5, + quality_status="PASS", + validation_status="valid", + privacy_status="flagged", + ) + + approved, reason = evaluate_auto_approval( + conn=db_conn, + version_id="doc-test-1:v5", + source_id="resmi_gazete", + parser_name="resmi_gazete_parser", + parser_version="1.0.0", + quality_decision="PASS", + has_privacy_blocker=True, + schema_valid=True, + ) + + assert approved is False + assert "Privacy" in reason diff --git a/tests/unit/test_canonical_spans_and_coverage.py b/tests/unit/test_canonical_spans_and_coverage.py new file mode 100644 index 0000000..5f979b6 --- /dev/null +++ b/tests/unit/test_canonical_spans_and_coverage.py @@ -0,0 +1,125 @@ +from mesa_legal_data.parsers.coverage import compute_parsing_coverage +from mesa_legal_data.parsers.legislation import parse_legislation_text +from mesa_legal_data.parsers.text_normalizer import normalize_text + + +def test_canonical_normalization_determinism(): + raw_1 = "TÜRK CEZA KANUNU\r\n\r\nMADDE 1 \u00a0-\u200b Ceza Kanununun amacı;\u0000" + raw_2 = "TÜRK CEZA KANUNU\n\nMADDE 1 - Ceza Kanununun amacı;" + + canon_1 = normalize_text(raw_1) + canon_2 = normalize_text(raw_2) + + assert canon_1 == canon_2 + assert "TÜRK CEZA KANUNU" in canon_1 + assert "MADDE 1 -" in canon_1 + assert "\u00a0" not in canon_1 + assert "\u200b" not in canon_1 + assert "\x00" not in canon_1 + + +def test_article_source_spans_exact_slice_match(): + canonical_text = """TÜRK BORÇLAR KANUNU +Genel Hükümler + +MADDE 1 - Sözleşme, tarafların iradelerini karşılıklı ve birbirine uygun olarak açıklamalarıyla kurulur. +İrade açıklaması, açık veya örtülü olabilir. + +MADDE 2 - Taraflar sözleşmenin esaslı noktalarında uyuşmuşlarsa, ikinci derecedeki noktalar üzerinde durulmamış olsa bile, sözleşme kurulmuş sayılır. + +EK MADDE 1 - Ek hüküm metni burada yer alır. + +GEÇİCİ MADDE 1 - Geçici uygulama hükümleri.""" + + parsed = parse_legislation_text(canonical_text, auto_normalize=False) + + assert len(parsed.articles) == 4 + + # 1. Madde 1 + art1 = parsed.articles[0] + assert art1.article_number == "1" + assert art1.ordinal == 1 + assert art1.char_start is not None and art1.char_end is not None + slice_1 = canonical_text[art1.char_start : art1.char_end] + assert slice_1.startswith("MADDE 1 -") + assert "İrade açıklaması, açık veya örtülü olabilir." in slice_1 + + # 2. Madde 2 + art2 = parsed.articles[1] + assert art2.article_number == "2" + assert art2.ordinal == 2 + slice_2 = canonical_text[art2.char_start : art2.char_end] + assert slice_2.startswith("MADDE 2 -") + assert "sözleşme kurulmuş sayılır." in slice_2 + + # 3. Ek Madde 1 + art_ek = parsed.articles[2] + assert art_ek.article_number == "1" + assert art_ek.article_kind == "additional" + slice_ek = canonical_text[art_ek.char_start : art_ek.char_end] + assert slice_ek.startswith("EK MADDE 1 -") + + # 4. Geçici Madde 1 + art_gec = parsed.articles[3] + assert art_gec.article_number == "1" + assert art_gec.article_kind == "temporary" + slice_gec = canonical_text[art_gec.char_start : art_gec.char_end] + assert slice_gec.startswith("GEÇİCİ MADDE 1 -") + + # Invariant: Spans do not overlap and are monotonically increasing + assert ( + art1.char_start + < art1.char_end + <= art2.char_start + < art2.char_end + <= art_ek.char_start + < art_ek.char_end + <= art_gec.char_start + < art_gec.char_end + ) + + +def test_parsing_coverage_with_preamble_and_annex(): + preamble = "TÜRK TİCARET KANUNU\nBaşlangıç Hükümleri ve Gerekçe\n\n" + article_1 = "MADDE 1 - Türk Ticaret Kanunu genel esasları düzenler.\n\n" + gap_heading = "İKİNCİ KISIM: Şirketler Hukuku\n\n" + article_2 = "MADDE 2 - Ticari işler ticari hükümlere tabidir.\n\n" + annex = "EK CETVEL: Yürürlük ve İntikal Hükümleri ile İlgili Tablolar." + + full_text = preamble + article_1 + gap_heading + article_2 + annex + + parsed = parse_legislation_text(full_text, auto_normalize=False) + assert len(parsed.articles) == 2 + + spans = [(a.char_start, a.char_end) for a in parsed.articles if a.char_start is not None and a.char_end is not None] + coverage = compute_parsing_coverage(full_text, spans) + + assert coverage.canonical_chars == len(full_text) + assert coverage.covered_chars > 0 + assert coverage.uncovered_chars > 0 + assert 0.0 < coverage.coverage_ratio < 1.0 + + # Uncovered ranges classification + uncovered = coverage.uncovered_ranges + assert len(uncovered) == 3 + + # Preamble range + preamble_range = uncovered[0] + assert preamble_range["candidate_type"] == "preamble" + assert preamble_range["start"] == 0 + assert preamble_range["end"] == parsed.articles[0].char_start + assert "TÜRK TİCARET KANUNU" in preamble_range["preview"] + + # Gap range + gap_range = uncovered[1] + assert gap_range["candidate_type"] == "gap" + assert gap_range["start"] == parsed.articles[0].char_end + assert gap_range["end"] == parsed.articles[1].char_start + assert "İKİNCİ KISIM" in gap_range["preview"] + + # Annex / trailing range + annex_range = uncovered[2] + assert annex_range["candidate_type"] == "annex_trailing" + assert annex_range["start"] == parsed.articles[1].char_end + assert annex_range["end"] == len(full_text) + assert "EK CETVEL" in annex_range["preview"] diff --git a/tests/unit/test_citation_semantics.py b/tests/unit/test_citation_semantics.py new file mode 100644 index 0000000..86d3e84 --- /dev/null +++ b/tests/unit/test_citation_semantics.py @@ -0,0 +1,110 @@ +from mesa_legal_data.parsers.citations import extract_citations + + +def test_extracted_never_automatically_validated(): + text = "5237 sayılı Türk Ceza Kanunu'nun 81. maddesi uyarınca ceza verilir." + citations = extract_citations(text) + assert len(citations) >= 1 + cit = citations[0] + # Extraction must NEVER mark citation as validated or human_verified + assert cit.citation_status != "HUMAN_VERIFIED" + assert cit.citation_status in ("EXTRACTED", "RESOLVED") + + +def test_citation_coordinate_span_exact_slice(): + text = "İşbu karar 6100 sayılı Hukuk Muhakemeleri Kanunu madde 119 uyarınca verilmiştir." + citations = extract_citations(text) + assert len(citations) >= 1 + cit = citations[0] + # Exact Unicode slice parity check + assert text[cit.char_start : cit.char_end] == cit.raw_text + + +def test_expanded_turkish_numbered_laws(): + cases = [ + ("Bu hüküm 5237 sayılı Kanun gereğince uygulanır.", "5237"), + ("657 sayılı Devlet Memurları Kanunu kapsamında görev yapmaktadır.", "657"), + ("1 sayılı Cumhurbaşkanlığı Kararnamesi gereğince işlem yapıldı.", "1"), + ("2577 sayılı İdari Yargılama Usulü Kanunu m. 7 uyarınca dava açılmıştır.", "2577"), + ("6698 sayılı Kişisel Verilerin Korunması Kanunu madde 5/1 uyarınca.", "6698"), + ] + for text, expected_num in cases: + citations = extract_citations(text) + assert len(citations) >= 1, f"Failed to extract law citation from: {text}" + found = any(expected_num in (c.target_legislation_id or "") or expected_num in c.raw_text for c in citations) + assert found, f"Expected law number {expected_num} in citations for text: {text}" + + +def test_turkish_legal_code_aliases(): + cases = [ + ("TCK m. 81 uyarınca kasten öldürme suçu", "5237", "81"), + ("CMK madde 100 gereğince tutuklama kararı verildi", "5271", "100"), + ("HMK m. 119 gereğince dava dilekçesi eksikliği", "6100", "119"), + ("TMK 166 maddesi uyarınca boşanma davası", "4721", "166"), + ("TBK m. 49 uyarınca haksız fiil sorumluluğu", "6098", "49"), + ("İİK m. 68 gereğince itirazın kaldırılması", "2004", "68"), + ("İYUK 27/2 uyarınca yürütmenin durdurulması", "2577", "27"), + ("AY m. 36 uyarınca adil yargılanma hakkı", "2709", "36"), + ("KVKK m. 6 gereğince özel nitelikli kişisel veri", "6698", "6"), + ("VUK m. 359 uyarınca vergi kaçakçılığı suçu", "213", "359"), + ] + for text, law_num, article in cases: + citations = extract_citations(text) + assert len(citations) >= 1, f"Failed to extract alias citation for: {text}" + cit = citations[0] + assert cit.target_legislation_id == f"tr:legislation:{'constitution' if law_num == '2709' else 'law'}:{law_num}" + assert cit.target_article_id == f"{cit.target_legislation_id}:article:{article}" + assert text[cit.char_start : cit.char_end] == cit.raw_text + + +def test_false_positive_prevention(): + # Random numbers, dates, times, or currencies without legal context must not be recognized as citations + false_positives = [ + "Toplantı saat 1500 sularında başladı ve 1800'de bitti.", + "Şirketin 2024 yılı toplam geliri 5000 TL olarak gerçekleşmiştir.", + "Bu konuda yaklaşık 100 kişi görüş bildirmiştir.", + "Dosyadaki 12345 adet evrak incelenmiştir.", + ] + for text in false_positives: + citations = extract_citations(text) + assert len(citations) == 0, f"False positive detected in: '{text}', citations: {citations}" + + +def test_lowercase_ay_and_random_4857_are_not_resolved(): + text = "Bu ay 4857 başvuru kaydedildi; ilgili Kanun hükmü ayrıca incelenecektir." + citations = extract_citations(text) + + assert not any(c.target_legislation_id == "tr:legislation:constitution:2709" for c in citations) + assert not any(c.target_legislation_id == "tr:legislation:law:4857" for c in citations) + assert any(c.citation_status == "UNRESOLVED" for c in citations) + + +def test_unknown_numbered_law_stays_extracted_until_resolution(): + citations = extract_citations("9999 sayılı Kanun'un 3. maddesi uyarınca işlem yapıldı.") + + assert len(citations) == 1 + assert citations[0].target_legislation_id == "tr:legislation:law:9999" + assert citations[0].target_article_id == "tr:legislation:law:9999:article:3" + assert citations[0].citation_status == "EXTRACTED" + + +def test_temporal_relation_hints(): + cases = [ + ("Bu madde 5237 sayılı Kanun ile değiştirilmiştir.", "amends"), + ("Söz konusu fıkra 6100 sayılı Kanun ile yürürlükten kaldırılmıştır.", "repeals"), + ("Bu fıkra 2577 sayılı Kanun ile eklenmiştir.", "adds"), + ("Bu kural 657 sayılı Kanun uyarınca uygulanır.", None), + ] + for text, expected_hint in cases: + citations = extract_citations(text) + assert len(citations) >= 1, f"Failed citation extraction in: {text}" + cit = citations[0] + assert cit.relation_hint == expected_hint, f"Expected {expected_hint} but got {cit.relation_hint} in: {text}" + + +def test_unresolved_citation_lifecycle(): + text = "İlgili Kanun hükümleri ve mevzuat uyarınca işlem yapılması gerekir." + citations = extract_citations(text) + for cit in citations: + if cit.target_legislation_id is None: + assert cit.citation_status == "UNRESOLVED" diff --git a/tests/unit/test_cli_sync.py b/tests/unit/test_cli_sync.py new file mode 100644 index 0000000..dbb1a5b --- /dev/null +++ b/tests/unit/test_cli_sync.py @@ -0,0 +1,45 @@ +from typer.testing import CliRunner + +from mesa_legal_data.catalog import get_connection, migrate +from mesa_legal_data.cli import app + + +def test_cli_sync_stops_before_mesa_push(tmp_path, monkeypatch): + runner = CliRunner() + data_root = tmp_path / "data" + data_root.mkdir(parents=True, exist_ok=True) + db_path = data_root / "catalog.sqlite" + + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(data_root)) + migrate(None, db_path) + + result = runner.invoke(app, ["sync", "--source", "resmi_gazete", "--max-items", "5"]) + + assert result.exit_code == 0 + assert "Starting MESA Legal Data Synchronization Pipeline" in result.output + assert "Sync Summary" in result.output + # Verify the explicit warning / notice that it stops before MESA push + assert "stopped before MESA push" in result.output + + # Capture local state, then run the exact same sync command again. + conn = get_connection(db_path) + c = conn.cursor() + c.execute("SELECT count(*) FROM mesa_deliveries") + del_count = c.fetchone()[0] + c.execute("SELECT count(*) FROM artifacts") + artifact_count = c.fetchone()[0] + conn.close() + assert del_count == 0 + assert artifact_count <= 5 + + second = runner.invoke(app, ["sync", "--source", "resmi_gazete", "--max-items", "5"]) + assert second.exit_code == 0 + assert "stopped before MESA push" in second.output + + conn = get_connection(db_path) + c = conn.cursor() + c.execute("SELECT count(*) FROM mesa_deliveries") + assert c.fetchone()[0] == 0 + c.execute("SELECT count(*) FROM artifacts") + assert c.fetchone()[0] == artifact_count + conn.close() diff --git a/tests/unit/test_final_closure_gate.py b/tests/unit/test_final_closure_gate.py index ec556c0..0df1891 100644 --- a/tests/unit/test_final_closure_gate.py +++ b/tests/unit/test_final_closure_gate.py @@ -464,13 +464,11 @@ def test_post_staging_pre_audit_crash_reconciles_on_rerun(tmp_path, monkeypatch) conn_c = sqlite3.connect(cat_db) conn_c.execute("CREATE TABLE releases (release_id TEXT PRIMARY KEY, status TEXT NOT NULL);") conn_c.execute("INSERT INTO releases VALUES ('rel-rec-1', 'published')") - conn_c.execute( - """ + conn_c.execute(""" CREATE TABLE mesa_imports ( import_id INTEGER PRIMARY KEY, release_id TEXT UNIQUE, status TEXT, target_db_path TEXT, imported_at TEXT, record_counts_json TEXT, error_summary TEXT ); - """ - ) + """) conn_c.commit() conn_c.close() diff --git a/tests/unit/test_migration_0005.py b/tests/unit/test_migration_0005.py new file mode 100644 index 0000000..f1870b4 --- /dev/null +++ b/tests/unit/test_migration_0005.py @@ -0,0 +1,132 @@ +import sqlite3 +from pathlib import Path + +from mesa_legal_data.catalog import get_connection, hash_file, migrate + + +def test_migration_0005_data_preservation_and_backfill(tmp_path): + """ + Migration Test: + - Sets up schema up to 0004 + - Inserts legacy data + - Applies migration 0005 + - Verifies zero data loss, correct backfill, and multi-version record capability + """ + db_file = tmp_path / "test_catalog.sqlite" + migrations_dir = Path(__file__).parent.parent.parent / "migrations" + conn = sqlite3.connect(db_file) + conn.execute("PRAGMA foreign_keys = ON;") + + # Apply 0001 - 0004 manually first + migration_files = sorted(migrations_dir.glob("000[1-4]_*.sql")) + conn.execute(""" + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TEXT NOT NULL, + file_hash TEXT NOT NULL + ); + """) + + for mf in migration_files: + with open(mf, "r", encoding="utf-8") as f: + sql = f.read() + conn.executescript(sql) + conn.execute( + "INSERT INTO schema_migrations VALUES (?, '2026-01-01T00:00:00Z', ?)", + (mf.name, hash_file(mf)), + ) + conn.commit() + + # Insert legacy records + conn.execute( + """INSERT INTO sources VALUES ('mevzuat', 'Mevzuat', 'Authority', 'https://example.com', 'manual', 1, '1.0', '{}', '2026-01-01', '2026-01-01')""" + ) + conn.execute( + """INSERT INTO documents VALUES ('doc-1', 'legislation', 'law', 'TR', 'Legacy Law', 'key-1', 'ver-1', 'active', '2026-01-01', '2026-01-01')""" + ) + conn.execute( + """INSERT INTO artifacts VALUES ('art-1', 'doc-1', 'mevzuat', 'http://ex.com', '2026-01-01', 'manual', 200, 'text/html', 'text/html', 100, 'sha1111111111111111111111111111111111111111111111111111111111111111', 'raw/1.html', NULL, NULL, 'verified', NULL, '{}')""" + ) + conn.execute( + """INSERT INTO versions VALUES ('ver-1', 'doc-1', 'art-1', 'consolidated_snapshot', '2026-01-01', NULL, NULL, 'canon/1.jsonl', 1, 'csha1', 'parser', '1.0', '1.0', 'valid', 'clean', 'approved', '2026-01-01')""" + ) + conn.execute( + """INSERT INTO records VALUES ('tr:legislation:law:1:article:1', 'ver-1', 'article', 'canon/1.jsonl', 1, 'recsha1', 'valid', 'approved', '2026-01-01')""" + ) + conn.execute( + """INSERT INTO record_reviews VALUES ('rev-1', 'tr:legislation:law:1:article:1', 'recsha1', 'approved', 'reviewer-1', 'LGTM', '2026-01-01')""" + ) + conn.execute( + """INSERT INTO releases VALUES ('rel-1', 'rel/1', 'published', '1.0', '2026-01-01', '2026-01-01', 'msha1', '{}', '{}')""" + ) + conn.execute("""INSERT INTO release_items VALUES ('rel-1', 'tr:legislation:law:1:article:1', 'recsha1')""") + conn.commit() + conn.close() + + # Now apply full migrate() including 0005 + migrate(migrations_dir=migrations_dir, db_path=db_file) + + conn2 = get_connection(db_file) + c = conn2.cursor() + + # Verify 0005 migration applied + c.execute("SELECT version FROM schema_migrations WHERE version LIKE '0005%'") + m0005 = c.fetchone() + assert m0005 is not None + + # Verify versions table fields + c.execute("SELECT version_id, revision_number, quality_status FROM versions WHERE version_id = 'ver-1'") + v_row = c.fetchone() + assert v_row is not None + assert v_row[0] == "ver-1" + assert v_row[1] == 1 # revision_number backfilled to 1 + + # Verify records table upgraded and backfilled + c.execute( + "SELECT record_instance_id, record_id, version_id, record_sha256 FROM records WHERE record_id = 'tr:legislation:law:1:article:1'" + ) + r_row = c.fetchone() + assert r_row is not None + assert r_row[0] == "ver-1:tr:legislation:law:1:article:1" + assert r_row[1] == "tr:legislation:law:1:article:1" + assert r_row[2] == "ver-1" + assert r_row[3] == "recsha1" + + # Verify dependent tables preserved data + c.execute("SELECT review_id, record_id, record_sha256 FROM record_reviews WHERE review_id = 'rev-1'") + assert c.fetchone() == ("rev-1", "tr:legislation:law:1:article:1", "recsha1") + + c.execute("SELECT release_id, record_id, record_sha256 FROM release_items WHERE release_id = 'rel-1'") + assert c.fetchone() == ("rel-1", "tr:legislation:law:1:article:1", "recsha1") + c.execute("SELECT version_id FROM release_items WHERE release_id = 'rel-1'") + assert c.fetchone() == ("ver-1",) + + c.execute( + "SELECT contract_source, health_path, publish_path, mutation_status_path_template FROM mesa_target_settings" + ) + assert c.fetchone() == ("unknown", "", "", "") + + # Verify inserting Version 2 for the same article does NOT collide or overwrite Version 1 + conn2.execute( + """INSERT INTO artifacts VALUES ('art-2', 'doc-1', 'mevzuat', 'http://ex.com/2', '2026-06-01', 'manual', 200, 'text/html', 'text/html', 100, 'sha2222222222222222222222222222222222222222222222222222222222222222', 'raw/2.html', NULL, NULL, 'verified', NULL, '{}')""" + ) + conn2.execute( + """INSERT INTO versions (version_id, document_id, artifact_id, version_kind, canonical_path, canonical_line, canonical_sha256, parser_name, parser_version, schema_version, validation_status, privacy_status, approval_status, created_at, revision_number, supersedes_version_id, quality_status) + VALUES ('ver-2', 'doc-1', 'art-2', 'consolidated_snapshot', 'canon/2.jsonl', 1, 'csha2', 'parser', '1.0', '1.0', 'valid', 'clean', 'pending', '2026-06-01', 2, 'ver-1', 'PASS')""" + ) + conn2.execute( + """INSERT INTO records (record_instance_id, record_id, version_id, record_type, canonical_path, canonical_line, record_sha256, validation_status, approval_status, created_at) + VALUES ('ver-2:tr:legislation:law:1:article:1', 'tr:legislation:law:1:article:1', 'ver-2', 'article', 'canon/2.jsonl', 1, 'recsha2_amended', 'valid', 'pending', '2026-06-01')""" + ) + conn2.commit() + + # Check both versions of Article 1 exist side-by-side + c.execute( + "SELECT version_id, record_sha256, approval_status FROM records WHERE record_id = 'tr:legislation:law:1:article:1' ORDER BY created_at ASC" + ) + all_art1 = c.fetchall() + assert len(all_art1) == 2 + assert all_art1[0] == ("ver-1", "recsha1", "approved") + assert all_art1[1] == ("ver-2", "recsha2_amended", "pending") + + conn2.close() diff --git a/tests/unit/test_publisher_chunker.py b/tests/unit/test_publisher_chunker.py new file mode 100644 index 0000000..21724e4 --- /dev/null +++ b/tests/unit/test_publisher_chunker.py @@ -0,0 +1,123 @@ +from mesa_legal_data.publisher.chunker import _split_oversized_text, plan_source_chunks + + +def test_chunker_structure_preservation_preamble_articles_annex(): + canonical_text = ( + "TÜRK CEZA KANUNU\n\nGenel Hükümler ve Başlangıç\n\n" + "MADDE 1- Bu Kanunun amacı kişi hak ve özgürlüklerini korumaktır.\n\n" + "MADDE 2- Kanunun açıkça suç saymadığı bir fiil için kimseye ceza verilemez.\n\n" + "GEÇİCİ MADDE 1- Bu Kanunun yürürlüğe girmesinden önceki fiiller hakkında..." + ) + + p_end = canonical_text.index("MADDE 1") + m1_start = p_end + m1_end = canonical_text.index("MADDE 2") + m2_start = m1_end + m2_end = canonical_text.index("GEÇİCİ MADDE 1") + annex_start = m2_end + + records = [ + { + "record_id": "art-1", + "record_type": "article", + "article_number": "1", + "title": "Madde 1", + "char_start": m1_start, + "char_end": m1_end, + "ordinal": 1, + }, + { + "record_id": "art-2", + "record_type": "article", + "article_number": "2", + "title": "Madde 2", + "char_start": m2_start, + "char_end": m2_end, + "ordinal": 2, + }, + ] + + chunks = plan_source_chunks( + document_id="tr:legislation:law:5237", + version_id="tr:legislation:law:5237:v1", + canonical_text=canonical_text, + records=records, + content_limit_chars=500, + ) + + # Must produce 4 chunks: preamble, art 1, art 2, annex + assert len(chunks) == 4 + assert chunks[0].chunk_type == "preamble" + assert chunks[0].char_start == 0 + assert chunks[0].char_end == m1_start + + assert chunks[1].chunk_type == "article" + assert chunks[1].char_start == m1_start + assert chunks[1].char_end == m1_end + assert chunks[1].metadata.get("article_number") == "1" + + assert chunks[2].chunk_type == "article" + assert chunks[2].char_start == m2_start + assert chunks[2].char_end == m2_end + + assert chunks[3].chunk_type == "annex" + assert chunks[3].char_start == annex_start + assert chunks[3].char_end == len(canonical_text) + + +def test_chunker_oversized_split_on_paragraphs(): + long_paragraph_1 = "A" * 200 + long_paragraph_2 = "B" * 200 + text = f"{long_paragraph_1}\n\n{long_paragraph_2}" + + splits = _split_oversized_text(text, base_char_start=0, max_chars=250) + assert len(splits) == 2 + assert splits[0][2] == f"{long_paragraph_1}\n\n" + assert splits[1][2] == long_paragraph_2 + assert splits[0][0] == 0 + assert splits[0][1] == len(long_paragraph_1) + 2 + assert splits[1][0] == len(long_paragraph_1) + 2 + assert splits[1][1] == len(text) + + +def test_chunker_deterministic_invariants(): + canonical_text = "MADDE 1- Deneme içerik.\n\nMADDE 2- İkinci deneme içerik." + records = [ + { + "record_id": "art-1", + "record_type": "article", + "article_number": "1", + "char_start": 0, + "char_end": 24, + "ordinal": 1, + }, + { + "record_id": "art-2", + "record_type": "article", + "article_number": "2", + "char_start": 26, + "char_end": len(canonical_text), + "ordinal": 2, + }, + ] + + run1 = plan_source_chunks( + document_id="doc-1", + version_id="doc-1:v1", + canonical_text=canonical_text, + records=records, + ) + run2 = plan_source_chunks( + document_id="doc-1", + version_id="doc-1:v1", + canonical_text=canonical_text, + records=records, + ) + + assert len(run1) == len(run2) + for c1, c2 in zip(run1, run2): + assert c1.chunk_id == c2.chunk_id + assert c1.content_hash == c2.content_hash + assert c1.content == c2.content + assert c1.char_start == c2.char_start + assert c1.char_end == c2.char_end diff --git a/tests/unit/test_publisher_client_and_ledger.py b/tests/unit/test_publisher_client_and_ledger.py new file mode 100644 index 0000000..d021ffc --- /dev/null +++ b/tests/unit/test_publisher_client_and_ledger.py @@ -0,0 +1,193 @@ +import pytest +import respx + +from mesa_legal_data.catalog import get_connection, migrate +from mesa_legal_data.publisher.client import MesaClient +from mesa_legal_data.publisher.ledger import ( + create_delivery, + get_delivery, + get_document_mesa_status, + get_mesa_target_settings, + insert_delivery_item, + is_chunk_already_committed, + list_failed_delivery_items, + update_delivery_progress, + upsert_mesa_target_settings, +) +from mesa_legal_data.publisher.models import MesaTargetSettings, MutationState, SourceChunk + + +@pytest.fixture +def db_conn(tmp_path): + db_path = tmp_path / "test_catalog.sqlite" + migrate(None, db_path) + conn = get_connection(db_path) + yield conn + conn.close() + + +def test_target_settings_persistence(db_conn): + settings = MesaTargetSettings( + target_key="default", + base_url="https://mesa.example.org", + tenant_id="enterprise", + workspace_id="legal_corp", + dataset_id="turkey_laws", + agent_id="agent_mesa_01", + content_limit_chars=16384, + ) + upsert_mesa_target_settings(db_conn, settings) + + loaded = get_mesa_target_settings(db_conn, "default") + assert loaded.base_url == "https://mesa.example.org" + assert loaded.tenant_id == "enterprise" + assert loaded.workspace_id == "legal_corp" + assert loaded.dataset_id == "turkey_laws" + assert loaded.agent_id == "agent_mesa_01" + assert loaded.content_limit_chars == 16384 + + +def test_api_key_secrecy_and_isolation(monkeypatch): + monkeypatch.setenv("MESA_DATA_MESA_API_KEY", "secret_token_12345") + settings = MesaTargetSettings(target_key="default", base_url="http://localhost:8000") + client = MesaClient(settings=settings) + + assert client.is_api_key_configured is True + # Verify the client headers contain the secret + headers = client._get_headers() + assert headers["Authorization"] == "Bearer secret_token_12345" + + # Verify settings dump does NOT leak the API key + dumped = settings.model_dump() + assert "secret_token_12345" not in str(dumped) + + +@respx.mock +@pytest.mark.parametrize("response_json", [{}, {"state": "unknown_future_state"}, {"status": "accepted"}]) +def test_committed_truth_requires_explicit_committed(response_json): + settings = MesaTargetSettings( + base_url="https://mesa-contract.test", + contract_source="configured", + health_path="/health-contract", + publish_path="/publish-contract", + mutation_status_path_template="/mutations-contract/{mutation_id}", + ) + client = MesaClient(settings=settings, api_key="secret") + chunk = SourceChunk( + chunk_id="v1:chunk:1", + document_id="doc-1", + version_id="v1", + chunk_type="general", + char_start=0, + char_end=4, + ordinal=1, + content="test", + content_hash="hash", + ) + respx.post("https://mesa-contract.test/publish-contract").respond(200, json=response_json) + + result = client.publish_source_chunk(chunk, "stable-key") + + assert result["state"] != MutationState.COMMITTED.value + + +def test_unknown_contract_never_guesses_routes(): + client = MesaClient(settings=MesaTargetSettings(base_url="https://mesa-contract.test"), api_key="secret") + assert client.is_contract_configured is False + assert client.test_connection()["connected"] is False + + +def test_delivery_ledger_and_cross_release_dedup(db_conn): + delivery_id = "del-test-01" + create_delivery(db_conn, delivery_id=delivery_id, release_id="rel-1", target_key="default", total_items=2) + + insert_delivery_item( + db_conn, + item_id="item-01", + delivery_id=delivery_id, + document_id="doc-1", + version_id="doc-1:v1", + chunk_id="chunk-01", + content_hash="sha-content-1", + idempotency_key="mesa-data:key1", + remote_state=MutationState.COMMITTED.value, + payload_json="{}", + ) + + insert_delivery_item( + db_conn, + item_id="item-02", + delivery_id=delivery_id, + document_id="doc-1", + version_id="doc-1:v1", + chunk_id="chunk-02", + content_hash="sha-content-2", + idempotency_key="mesa-data:key2", + remote_state=MutationState.FAILED.value, + payload_json="{}", + ) + + update_delivery_progress( + db_conn, + delivery_id=delivery_id, + status="PARTIAL", + committed_items=1, + failed_items=1, + skipped_items=0, + finished=True, + ) + + del_info = get_delivery(db_conn, delivery_id) + assert del_info is not None + assert del_info["status"] == "PARTIAL" + assert del_info["committed_items"] == 1 + assert del_info["failed_items"] == 1 + + # Cross-release dedup check + assert ( + is_chunk_already_committed( + db_conn, + target_key="default", + document_id="doc-1", + version_id="doc-1:v1", + chunk_id="chunk-01", + content_hash="sha-content-1", + ) + is True + ) + + # Modified content hash or failed chunk is NOT committed + assert ( + is_chunk_already_committed( + db_conn, + target_key="default", + document_id="doc-1", + version_id="doc-1:v1", + chunk_id="chunk-01", + content_hash="sha-content-modified", + ) + is False + ) + + assert ( + is_chunk_already_committed( + db_conn, + target_key="default", + document_id="doc-1", + version_id="doc-1:v1", + chunk_id="chunk-02", + content_hash="sha-content-2", + ) + is False + ) + + # Retry failures query + failed = list_failed_delivery_items(db_conn, delivery_id) + assert len(failed) == 1 + assert failed[0]["item_id"] == "item-02" + + # Document MESA status check + doc_status = get_document_mesa_status(db_conn, "doc-1") + assert doc_status["status"] == "Partial" + assert doc_status["committed_count"] == 1 + assert doc_status["failed_count"] == 1 diff --git a/tests/unit/test_publisher_idempotency_and_hashing.py b/tests/unit/test_publisher_idempotency_and_hashing.py new file mode 100644 index 0000000..311600b --- /dev/null +++ b/tests/unit/test_publisher_idempotency_and_hashing.py @@ -0,0 +1,75 @@ +from mesa_legal_data.publisher.hashing import ( + calculate_canonical_revision_hash, + generate_idempotency_key, +) +from mesa_legal_data.publisher.models import SourceChunk + + +def test_canonical_revision_hash_purity_and_determinism(): + c1 = SourceChunk( + chunk_id="v1:c1", + document_id="d1", + version_id="v1", + chunk_type="article", + title="Madde 1", + char_start=0, + char_end=50, + ordinal=1, + content="İçerik 1", + content_hash="hash1", + ) + c2 = SourceChunk( + chunk_id="v1:c2", + document_id="d1", + version_id="v1", + chunk_type="article", + title="Madde 2", + char_start=52, + char_end=100, + ordinal=2, + content="İçerik 2", + content_hash="hash2", + ) + + hash_a = calculate_canonical_revision_hash([c1, c2]) + hash_b = calculate_canonical_revision_hash([c2, c1]) # Order-independent input gets sorted + + assert hash_a == hash_b + assert len(hash_a) == 64 # SHA256 + + +def test_idempotency_key_stable_and_free_of_volatile_data(): + key1 = generate_idempotency_key( + tenant_id="default", + workspace_id="legal", + dataset_id="tr_legislation", + document_id="tr:legislation:law:5237", + version_id="tr:legislation:law:5237:v1", + chunk_id="tr:legislation:law:5237:v1:chunk:0001", + content_hash="abc123def456", + ) + + key2 = generate_idempotency_key( + tenant_id="default", + workspace_id="legal", + dataset_id="tr_legislation", + document_id="tr:legislation:law:5237", + version_id="tr:legislation:law:5237:v1", + chunk_id="tr:legislation:law:5237:v1:chunk:0001", + content_hash="abc123def456", + ) + + assert key1.startswith("mesa-data:") + assert key1 == key2 + + # Different content hash produces different idempotency key + key3 = generate_idempotency_key( + tenant_id="default", + workspace_id="legal", + dataset_id="tr_legislation", + document_id="tr:legislation:law:5237", + version_id="tr:legislation:law:5237:v1", + chunk_id="tr:legislation:law:5237:v1:chunk:0001", + content_hash="different_content_hash", + ) + assert key1 != key3 diff --git a/tests/unit/test_quality_gate.py b/tests/unit/test_quality_gate.py new file mode 100644 index 0000000..dc0240b --- /dev/null +++ b/tests/unit/test_quality_gate.py @@ -0,0 +1,256 @@ +import hashlib +import json + +import pytest + +from mesa_legal_data.catalog import ( + BlockingValidationIssueExists, + approve_version_streaming, + get_connection, + get_db_path, + insert_artifact, + iter_records_for_release, + migrate, + upsert_document, + upsert_source, +) +from mesa_legal_data.parsers.coverage import compute_parsing_coverage +from mesa_legal_data.pipeline import process_artifact_pipeline +from mesa_legal_data.quality import evaluate_quality + + +@pytest.fixture +def quality_db(tmp_path, monkeypatch): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + db_path = get_db_path() + migrate(None, db_path) + return tmp_path + + +def test_evaluate_quality_healthy_pass(): + canonical_text = "MADDE 1 - Kanun amacı.\nBu kanunun amacı düzeni sağlamaktır." + records = [ + { + "id": "tr:legislation:law:100", + "record_type": "legislation", + "title": "100 Sayılı Kanun", + "source": {"artifact_sha256": "a" * 64}, + "provenance": {"pipeline_run_id": "run-1"}, + }, + { + "id": "tr:legislation:law:100:article:1", + "record_type": "article", + "source_span": {"char_start": 0, "char_end": 22, "ordinal": 1}, + "source": {"artifact_sha256": "a" * 64}, + "provenance": {"pipeline_run_id": "run-1"}, + }, + ] + cov = compute_parsing_coverage(canonical_text, [(0, 22)]) + report = evaluate_quality( + source_info={"source_id": "mevzuat", "source_url": "https://example.com/law.html"}, + raw_info={"byte_size": 1024, "sha256": "a" * 64, "file_exists": True}, + canonical_records=records, + canonical_text=canonical_text, + coverage=cov, + privacy_issues=[], + ) + + assert report.decision == "PASS" + assert "PASS" in report.summary + + +def test_evaluate_quality_suspicious_review(): + canonical_text = "MADDE 1 - Kanun metni.\nEksik karakter \ufffd içeriyor." + records = [ + { + "id": "tr:legislation:law:101", + "record_type": "legislation", + "title": "101 Sayılı Kanun", + "source": {"artifact_sha256": "b" * 64}, + "provenance": {"pipeline_run_id": "run-2"}, + }, + { + "id": "tr:legislation:law:101:article:1", + "record_type": "article", + "source_span": {"char_start": 0, "char_end": 20, "ordinal": 1}, + "source": {"artifact_sha256": "b" * 64}, + "provenance": {"pipeline_run_id": "run-2"}, + }, + ] + report = evaluate_quality( + source_info={"source_id": "mevzuat", "source_url": "https://example.com/law.html"}, + raw_info={"byte_size": 1024, "sha256": "b" * 64, "file_exists": True}, + canonical_records=records, + canonical_text=canonical_text, + coverage=None, + privacy_issues=[], + ) + + assert report.decision == "REVIEW" + + +def test_evaluate_quality_corrupt_block(): + canonical_text = "MADDE 1 - Kanun metni." + records = [ + { + "id": "tr:legislation:law:102", + "record_type": "legislation", + "title": "102 Sayılı Kanun", + "source": {"artifact_sha256": "c" * 64}, + "provenance": {"pipeline_run_id": "run-3"}, + }, + { + "id": "tr:legislation:law:102:article:1", + "record_type": "article", + "source_span": {"char_start": 50, "char_end": 10, "ordinal": 1}, # Invalid inverted span + "source": {"artifact_sha256": "c" * 64}, + "provenance": {"pipeline_run_id": "run-3"}, + }, + ] + report = evaluate_quality( + source_info={"source_id": "mevzuat", "source_url": "https://example.com/law.html"}, + raw_info={"byte_size": 1024, "sha256": "c" * 64, "file_exists": True}, + canonical_records=records, + canonical_text=canonical_text, + coverage=None, + privacy_issues=[], + ) + + assert report.decision == "BLOCK" + + +def _quality_report_for_text(canonical_text, records, coverage=None): + return evaluate_quality( + source_info={"source_id": "mevzuat", "source_url": "https://example.com/law.html"}, + raw_info={"byte_size": max(1, len(canonical_text.encode())), "sha256": "d" * 64, "file_exists": True}, + canonical_records=records, + canonical_text=canonical_text, + coverage=coverage, + privacy_issues=[], + ) + + +def test_quality_adversarial_inputs_never_pass(): + provenance = {"source": {"artifact_sha256": "d" * 64}, "provenance": {"pipeline_run_id": "run-audit"}} + + assert _quality_report_for_text("", []).decision == "BLOCK" + + canonical = "MADDE 1 - Birinci metin.\n\nMADDE 2 - İkinci metin." + shifted_article = { + "id": "law:article:1", + "record_type": "article", + "article_number": "1", + "text": "Birinci metin.", + "source_span": {"char_start": canonical.index("MADDE 2"), "char_end": len(canonical)}, + **provenance, + } + legislation = {"id": "law", "record_type": "legislation", "title": "Audit Law", **provenance} + assert _quality_report_for_text(canonical, [legislation, shifted_article]).decision == "BLOCK" + + severe_mojibake = "MADDE 1 - " + ("\ufffd" * 20) + valid_span_article = { + "id": "law:article:1", + "record_type": "article", + "article_number": "1", + "text": "\ufffd" * 20, + "source_span": {"char_start": 0, "char_end": len(severe_mojibake)}, + **provenance, + } + severe_report = _quality_report_for_text(severe_mojibake, [legislation, valid_span_article]) + assert severe_report.decision == "BLOCK" + + first = "MADDE 1 - Başlangıç." + second = "MADDE 2 - Bitiş." + missing_middle = first + ("\nKAYIP BÖLÜM" * 600) + "\n" + second + second_start = missing_middle.rindex("MADDE 2") + articles = [ + { + "id": "law:article:1", + "record_type": "article", + "article_number": "1", + "text": "Başlangıç.", + "source_span": {"char_start": 0, "char_end": len(first)}, + **provenance, + }, + { + "id": "law:article:2", + "record_type": "article", + "article_number": "2", + "text": "Bitiş.", + "source_span": {"char_start": second_start, "char_end": len(missing_middle)}, + **provenance, + }, + ] + coverage = compute_parsing_coverage( + missing_middle, + [(0, len(first)), (second_start, len(missing_middle))], + ) + assert _quality_report_for_text(missing_middle, [legislation, *articles], coverage).decision == "REVIEW" + + +def test_release_guard_blocks_unapproved_and_blocked_versions(quality_db): + """ + Release Guard Test: + - Version with quality == BLOCK cannot be approved (raises exception) + - Blocked version cannot be selected for release + """ + conn = get_connection() + doc_id = "tr:legislation:law:9999" + upsert_source(conn, "mevzuat", "Mevzuat", "Gov", "https://mevzuat.gov.tr") + upsert_document(conn, doc_id, "legislation", "law", "TR", "9999 Sayılı Kanun", "stable-9999", "fetched") + + # Create raw artifact containing a blocker issue (valid TCKN checksum that triggers privacy blocker) + content_corrupt = ( + "

MADDE 1

TCKN: 10000000146 gizli sahis verisi.

" + ) + raw_dir = quality_db / "raw" / "legislation" / "mevzuat" / "2026" + raw_dir.mkdir(parents=True, exist_ok=True) + raw_file = raw_dir / "blocked_privacy.html" + raw_bytes = content_corrupt.encode("utf-8") + raw_file.write_bytes(raw_bytes) + sha = hashlib.sha256(raw_bytes).hexdigest() + art_id = f"art-{sha[:12]}" + + insert_artifact( + conn, + art_id, + doc_id, + "mevzuat", + "https://example.com/blocked_privacy.html", + "2026-08-01T00:00:00Z", + "manual", + 200, + "text/html", + "text/html", + len(raw_bytes), + sha, + str(raw_file.relative_to(quality_db)), + None, + None, + "fetched", + None, + json.dumps({"publication_date": "2026-08-01"}), + ) + conn.close() + + pipeline_status = process_artifact_pipeline(artifact_id=art_id, document_id=doc_id) + assert pipeline_status == "rejected" + + conn = get_connection() + c = conn.cursor() + c.execute("SELECT version_id, quality_status, validation_status FROM versions WHERE document_id = ?", (doc_id,)) + v_row = c.fetchone() + assert v_row is not None + v_id, q_status, val_status = v_row + assert q_status == "BLOCK" + assert val_status == "failed" + + # Attempt to approve the BLOCKED version must raise BlockingValidationIssueExists + with pytest.raises(BlockingValidationIssueExists): + approve_version_streaming(conn, version_id=v_id, reviewer="operator-test") + + # Verify no records from this version are selectable for release + recs_for_rel = list(iter_records_for_release(conn)) + assert len(recs_for_rel) == 0 + + conn.close() diff --git a/tests/unit/test_version_integrity.py b/tests/unit/test_version_integrity.py new file mode 100644 index 0000000..f0079cf --- /dev/null +++ b/tests/unit/test_version_integrity.py @@ -0,0 +1,338 @@ +import hashlib +import json +from pathlib import Path + +import pytest + +from mesa_legal_data.catalog import ( + approve_version_streaming, + get_connection, + get_db_path, + get_record, + get_version, + insert_artifact, + iter_records_for_release, + migrate, + upsert_document, + upsert_source, +) +from mesa_legal_data.pipeline import process_artifact_pipeline + + +@pytest.fixture +def test_env(tmp_path, monkeypatch): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + db_path = get_db_path() + migrate(None, db_path) + return tmp_path + + +def create_sample_artifact( + tmp_path: Path, filename: str, content: str, source_id: str = "mevzuat" +) -> tuple[str, str, str]: + raw_dir = tmp_path / "raw" / "legislation" / source_id / "2026" + raw_dir.mkdir(parents=True, exist_ok=True) + raw_file = raw_dir / filename + raw_bytes = content.encode("utf-8") + raw_file.write_bytes(raw_bytes) + sha256 = hashlib.sha256(raw_bytes).hexdigest() + rel_path = str(raw_file.relative_to(tmp_path)) + art_id = f"art-{sha256[:12]}" + + conn = get_connection() + upsert_source(conn, source_id, "Official Source", "Authority", "https://example.com") + insert_artifact( + conn, + art_id, + None, + source_id, + f"https://example.com/{filename}", + "2026-05-10T10:00:00Z", + "manual", + 200, + "text/html", + "text/html", + len(raw_bytes), + sha256, + rel_path, + None, + None, + "fetched", + None, + json.dumps({"publication_date": "2026-05-10", "source_role": "consolidated_snapshot"}), + ) + conn.close() + return art_id, sha256, rel_path + + +def test_same_artifact_twice_idempotency_and_no_duplicates(test_env): + """ + Processing the exact same artifact on different days or multiple times: + - Must produce exactly one logical version + - Must not duplicate record instances + - Must preserve revision number + """ + html_content = """ + + 6698 Sayılı Kanun + +

KİŞİSEL VERİLERİN KORUNMASI KANUNU

+

MADDE 1- Bu Kanunun amacı, kişisel verilerin işlenmesinde...

+

MADDE 9- Kişisel veriler, ilgili kişinin açık rızası olmaksızın yurt dışına aktarılamaz.

+ + """ + + art_id, sha256, rel_path = create_sample_artifact(test_env, "kvkk_v1.html", html_content) + doc_id = "tr:legislation:law:6698" + + conn = get_connection() + upsert_document(conn, doc_id, "legislation", "law", "TR", "6698 Sayılı Kanun", "stable-6698", "fetched") + conn.close() + + # First pipeline run + status1 = process_artifact_pipeline(artifact_id=art_id, document_id=doc_id) + assert status1 == "needs_review" + + conn = get_connection() + c = conn.cursor() + c.execute("SELECT count(*) FROM versions WHERE document_id = ?", (doc_id,)) + v_count_1 = c.fetchone()[0] + assert v_count_1 == 1 + + c.execute("SELECT version_id, revision_number, version_kind FROM versions WHERE document_id = ?", (doc_id,)) + ver_row_1 = c.fetchone() + v1_id = ver_row_1[0] + assert ver_row_1[1] == 1 # revision_number == 1 + assert ver_row_1[2] == "consolidated_snapshot" + + c.execute("SELECT count(*) FROM records WHERE version_id = ?", (v1_id,)) + records_count_1 = c.fetchone()[0] + assert records_count_1 >= 3 # legislation + articles + citations + + # Approve version 1 + approve_version_streaming(conn, version_id=v1_id, reviewer="operator-test") + v1_after_approval = get_version(conn, v1_id) + assert v1_after_approval["approval_status"] == "approved" + + # Second pipeline run on the SAME artifact (e.g. daily reprocessing) + before_reprocess = get_version(conn, v1_id) + c.execute( + "SELECT record_id, canonical_path, record_sha256 FROM records WHERE version_id = ? ORDER BY record_id", + (v1_id,), + ) + record_evidence_before = c.fetchall() + + status2 = process_artifact_pipeline(artifact_id=art_id, document_id=doc_id) + assert status2 == "approved" + + # Verify no new fake version was created + c.execute("SELECT count(*) FROM versions WHERE document_id = ?", (doc_id,)) + v_count_2 = c.fetchone()[0] + assert v_count_2 == 1 + + # Verify record count did not multiply + c.execute("SELECT count(*) FROM records WHERE version_id = ?", (v1_id,)) + records_count_2 = c.fetchone()[0] + assert records_count_2 == records_count_1 + + # Verify approval status was preserved across reprocessing + v1_after_reprocess = get_version(conn, v1_id) + assert v1_after_reprocess["approval_status"] == "approved" + assert v1_after_reprocess["revision_number"] == 1 + assert v1_after_reprocess["canonical_path"] == before_reprocess["canonical_path"] + assert v1_after_reprocess["canonical_sha256"] == before_reprocess["canonical_sha256"] + c.execute( + "SELECT record_id, canonical_path, record_sha256 FROM records WHERE version_id = ? ORDER BY record_id", + (v1_id,), + ) + assert c.fetchall() == record_evidence_before + conn.close() + + +def test_two_real_versions_same_document_isolation(test_env): + """ + Same document with two distinct versions: + - Both versions survive immutably + - Logical Article 9 has two version-specific record instances + - Version B insertion does NOT overwrite Version A records + - Revision number sequences sequentially: Version 1 -> rev 1, Version 2 -> rev 2 + """ + doc_id = "tr:legislation:law:6698" + conn = get_connection() + upsert_document(conn, doc_id, "legislation", "law", "TR", "6698 Sayılı Kanun", "stable-6698", "fetched") + conn.close() + + # Version A + content_v1 = """ +

KİŞİSEL VERİLERİN KORUNMASI KANUNU

+

MADDE 1- Amaç metni v1.

+

MADDE 9- Yurt dışına aktarım orijinal metin 2016.

+ """ + art_id_1, _, _ = create_sample_artifact(test_env, "kvkk_2016.html", content_v1) + process_artifact_pipeline(artifact_id=art_id_1, document_id=doc_id) + + conn = get_connection() + c = conn.cursor() + c.execute( + "SELECT version_id, revision_number FROM versions WHERE document_id = ? ORDER BY revision_number ASC", (doc_id,) + ) + v1_row = c.fetchone() + v1_id = v1_row[0] + assert v1_row[1] == 1 + + # Approve Version A + approve_version_streaming(conn, version_id=v1_id, reviewer="operator-1") + + # Version B (Amended Article 9 in 2024) + content_v2 = """ +

KİŞİSEL VERİLERİN KORUNMASI KANUNU

+

MADDE 1- Amaç metni v1.

+

MADDE 9- Yurt dışına aktarım güncellenmiş 2024 standard sözleşme metni.

+ """ + raw_dir = test_env / "raw" / "legislation" / "mevzuat" / "2026" + raw_file_2 = raw_dir / "kvkk_2024.html" + raw_bytes_2 = content_v2.encode("utf-8") + raw_file_2.write_bytes(raw_bytes_2) + sha2 = hashlib.sha256(raw_bytes_2).hexdigest() + art_id_2 = f"art-{sha2[:12]}" + insert_artifact( + conn, + art_id_2, + doc_id, + "mevzuat", + "https://example.com/kvkk_2024.html", + "2026-08-01T00:00:00Z", + "manual", + 200, + "text/html", + "text/html", + len(raw_bytes_2), + sha2, + str(raw_file_2.relative_to(test_env)), + None, + None, + "fetched", + None, + json.dumps({"publication_date": "2024-03-12", "source_role": "consolidated_snapshot"}), + ) + conn.close() + + process_artifact_pipeline(artifact_id=art_id_2, document_id=doc_id) + + conn = get_connection() + c = conn.cursor() + + # Both versions survive + c.execute( + "SELECT version_id, revision_number, supersedes_version_id FROM versions WHERE document_id = ? ORDER BY revision_number ASC", + (doc_id,), + ) + v_rows = c.fetchall() + assert len(v_rows) == 2 + + v1_id, v1_rev, v1_super = v_rows[0] + v2_id, v2_rev, v2_super = v_rows[1] + + assert v1_rev == 1 + assert v2_rev == 2 + assert v2_super is None # No source provenance explicitly identified a predecessor. + + # Check Article 9 in both versions + logical_art9_id = "tr:legislation:law:6698:article:9" + + # Version A record instance + rec_v1 = get_record(conn, logical_art9_id, version_id=v1_id) + assert rec_v1 is not None + assert rec_v1["version_id"] == v1_id + assert rec_v1["approval_status"] == "approved" + + # Version B record instance + rec_v2 = get_record(conn, logical_art9_id, version_id=v2_id) + assert rec_v2 is not None + assert rec_v2["version_id"] == v2_id + assert rec_v2["approval_status"] == "pending" + assert rec_v1["record_sha256"] != rec_v2["record_sha256"] # Content changed + + # When both versions are approved, a release selects only the latest eligible + # physical version and does not accidentally re-release Version A. + approve_version_streaming(conn, version_id=v2_id, reviewer="operator-2") + release_refs = list(iter_records_for_release(conn)) + assert release_refs + assert {ref.version_id for ref in release_refs} == {v2_id} + + # Version C has the same source date as B but distinct source content. It is + # a real third version, not an overwrite or a fabricated predecessor link. + content_v3 = """ +

KİŞİSEL VERİLERİN KORUNMASI KANUNU

+

MADDE 1- Amaç metni v1.

+

MADDE 9- Aynı gün yayımlanan düzeltilmiş üçüncü kaynak metni.

+ """ + raw_file_3 = raw_dir / "kvkk_2024_correction.html" + raw_bytes_3 = content_v3.encode("utf-8") + raw_file_3.write_bytes(raw_bytes_3) + sha3 = hashlib.sha256(raw_bytes_3).hexdigest() + art_id_3 = f"art-{sha3[:12]}" + insert_artifact( + conn, + art_id_3, + doc_id, + "mevzuat", + "https://example.com/kvkk_2024_correction.html", + "2026-08-02T00:00:00Z", + "manual", + 200, + "text/html", + "text/html", + len(raw_bytes_3), + sha3, + str(raw_file_3.relative_to(test_env)), + None, + None, + "fetched", + None, + json.dumps({"publication_date": "2024-03-12", "source_role": "consolidated_snapshot"}), + ) + conn.close() + + process_artifact_pipeline(artifact_id=art_id_3, document_id=doc_id) + conn = get_connection() + c = conn.cursor() + c.execute( + "SELECT version_id, revision_number, snapshot_date, supersedes_version_id FROM versions WHERE document_id = ? ORDER BY revision_number", + (doc_id,), + ) + three_versions = c.fetchall() + assert len(three_versions) == 3 + assert [row[1] for row in three_versions] == [1, 2, 3] + assert three_versions[1][2] == three_versions[2][2] == "2024-03-12" + assert three_versions[2][3] is None + assert get_record(conn, logical_art9_id, version_id=v1_id)["record_sha256"] == rec_v1["record_sha256"] + assert get_record(conn, logical_art9_id, version_id=v2_id)["record_sha256"] == rec_v2["record_sha256"] + + conn.close() + + +def test_version_kind_resmi_gazete_is_original_publication(test_env): + """ + Test that source resmi_gazete or source_role original_publication + is authoritatively set to original_publication in catalog versions. + """ + html_content = """ +

7500 SAYILI KANUN

+

MADDE 1- Resmî Gazete ilk yayım metni.

+ """ + art_id, _, _ = create_sample_artifact(test_env, "rg_law.html", html_content, source_id="resmi_gazete") + doc_id = "tr:legislation:law:7500" + + conn = get_connection() + upsert_document(conn, doc_id, "legislation", "law", "TR", "7500 Sayılı Kanun", "stable-7500", "fetched") + conn.close() + + process_artifact_pipeline(artifact_id=art_id, document_id=doc_id) + + conn = get_connection() + c = conn.cursor() + c.execute("SELECT version_kind FROM versions WHERE document_id = ?", (doc_id,)) + v_kind = c.fetchone()[0] + assert v_kind == "original_publication" + conn.close() diff --git a/uv.lock b/uv.lock index 5b52cd0..ad72af5 100644 --- a/uv.lock +++ b/uv.lock @@ -26,15 +26,16 @@ wheels = [ [[package]] name = "anyio" -version = "4.14.2" +version = "4.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, + { name = "sniffio" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c6/78/7d432127c41b50bccba979505f272c16cbcadcc33645d5fa3a738110ae75/anyio-4.11.0.tar.gz", hash = "sha256:82a8d0b81e318cc5ce71a5f1f8b5c4e63619620b63141ef8c995fa0db95a57c4", size = 219094, upload-time = "2025-09-23T09:19:12.58Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, + { url = "https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc", size = 109097, upload-time = "2025-09-23T09:19:10.601Z" }, ] [[package]] @@ -339,6 +340,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/ad/f4f0e57345f1870f3e8cb624e058d7eca6e5a27d33bcc3311d9b618734cd/httpcore2-2.12.0.tar.gz", hash = "sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648", size = 67548, upload-time = "2026-08-18T13:22:08.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/74/d370e55600d9bcfa0d9794b0166126d49291a3d2b20c268fc98c453a4948/httpcore2-2.12.0-py3-none-any.whl", hash = "sha256:7e04258ce01013d7d615e5b910a3b27fac937d7a95038227e79652b4ba3b4ceb", size = 83074, upload-time = "2026-08-18T13:22:05.854Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -354,6 +368,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform != 'emscripten'" }, + { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, + { name = "httpx2-jsfetch", marker = "python_full_version >= '3.12' and sys_platform == 'emscripten'" }, + { name = "idna" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/f8/579a8b51e42e38ee32647df9f08aa25643ae788e275cc625b199829c4671/httpx2-2.12.0.tar.gz", hash = "sha256:7631fe9887a8a2275f4a2540e053aa670fcc50742864a9ae7c66e609fdcf12cf", size = 100040, upload-time = "2026-08-18T13:22:09.086Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/95/411ba65569158e862368917aaf56597f3e5fa3b91b0502919638465a08f3/httpx2-2.12.0-py3-none-any.whl", hash = "sha256:cc8b6eecb8661c146b8f89a60e97456ee086e91a784ed31ac450c3a9e613dd36", size = 95427, upload-time = "2026-08-18T13:22:06.834Z" }, +] + +[[package]] +name = "httpx2-jsfetch" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, +] + [[package]] name = "idna" version = "3.18" @@ -579,6 +619,7 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "httpx2" }, { name = "mypy" }, { name = "pip-audit" }, { name = "pytest" }, @@ -610,6 +651,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "httpx2", specifier = ">=0.1.0" }, { name = "mypy", specifier = ">=2.3.0" }, { name = "pip-audit", specifier = ">=2.10.1" }, { name = "pytest", specifier = ">=9.1.1" }, @@ -1228,6 +1270,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + [[package]] name = "sortedcontainers" version = "2.4.0" @@ -1322,6 +1373,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "typer" version = "0.27.1"