From f356dde42ee0fcf487cfc88a8d6b30dcc311923c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yasin=20B=C3=BCy=C3=BCktepe?= Date: Sat, 29 Aug 2026 00:55:36 +0300 Subject: [PATCH 1/4] fix: preserve Turkish HTML encoding during parsing --- src/mesa_legal_data/parsers/__init__.py | 2 + src/mesa_legal_data/parsers/encoding.py | 78 ++++++++++++ src/mesa_legal_data/parsers/html.py | 10 +- src/mesa_legal_data/pipeline.py | 10 +- src/mesa_legal_data/sources/url_fetcher.py | 9 +- src/mesa_legal_data/web/api.py | 10 +- tests/integration/test_pipeline_encoding.py | 115 ++++++++++++++++++ .../test_web_document_text_encoding.py | 86 +++++++++++++ tests/unit/test_encoding_cp1254.py | 67 ++++++++++ 9 files changed, 376 insertions(+), 11 deletions(-) create mode 100644 src/mesa_legal_data/parsers/encoding.py create mode 100644 tests/integration/test_pipeline_encoding.py create mode 100644 tests/integration/test_web_document_text_encoding.py create mode 100644 tests/unit/test_encoding_cp1254.py diff --git a/src/mesa_legal_data/parsers/__init__.py b/src/mesa_legal_data/parsers/__init__.py index 8cb63da..20363c8 100644 --- a/src/mesa_legal_data/parsers/__init__.py +++ b/src/mesa_legal_data/parsers/__init__.py @@ -1,5 +1,6 @@ from .citations import Citation, extract_citations from .decision import ParsedDecision, parse_decision_text +from .encoding import decode_source_bytes from .html import HTMLParseError, parse_html from .legislation import ParsedArticle, ParsedLegislation, parse_legislation_text from .pdf import OCRRequiredError, PDFParseError, parse_pdf @@ -13,6 +14,7 @@ "ParsedArticle", "ParsedDecision", "ParsedLegislation", + "decode_source_bytes", "extract_citations", "normalize_text", "parse_decision_text", diff --git a/src/mesa_legal_data/parsers/encoding.py b/src/mesa_legal_data/parsers/encoding.py new file mode 100644 index 0000000..135e64b --- /dev/null +++ b/src/mesa_legal_data/parsers/encoding.py @@ -0,0 +1,78 @@ +import re + +META_CHARSET_PATTERN = re.compile( + rb"""]+(?:charset\s*=\s*["']?([a-zA-Z0-9_-]+)|content\s*=\s*["'][^"']*charset\s*=\s*([a-zA-Z0-9_-]+))""", + re.IGNORECASE, +) + + +def decode_source_bytes(raw_bytes: bytes, is_html: bool = True) -> tuple[str, str]: + """ + Decodes raw bytes into a Python string preserving Turkish characters + without silent character deletion (`errors='ignore'` is forbidden). + + Supported encodings: + - UTF-8 with or without BOM + - Windows-1254 / cp1254 + - ISO-8859-9 / Latin-5 + - Declared HTML meta charset + + Returns: + tuple[str, str]: (decoded_text, detected_charset) + Raises: + UnicodeDecodeError: If bytes cannot be decoded safely. + """ + if not raw_bytes: + return "", "utf-8" + + # 1. UTF-8 BOM detection + if raw_bytes.startswith(b"\xef\xbb\xbf"): + return raw_bytes.decode("utf-8-sig"), "utf-8-sig" + + # 2. HTML meta charset inspection + if is_html: + sample = raw_bytes[:4096] + match = META_CHARSET_PATTERN.search(sample) + if match: + charset_bytes = match.group(1) or match.group(2) + if charset_bytes: + charset_name = charset_bytes.decode("ascii", errors="ignore").lower().strip() + if charset_name in ("windows-1254", "cp1254", "1254"): + try: + return raw_bytes.decode("cp1254"), "windows-1254" + except UnicodeDecodeError: + pass + elif charset_name in ("iso-8859-9", "latin5", "8859-9"): + try: + return raw_bytes.decode("iso-8859-9"), "iso-8859-9" + except UnicodeDecodeError: + pass + elif charset_name in ("utf-8", "utf8"): + try: + return raw_bytes.decode("utf-8"), "utf-8" + except UnicodeDecodeError: + pass + else: + try: + return raw_bytes.decode(charset_name), charset_name + except Exception: + pass + + # 3. Strict UTF-8 trial + try: + return raw_bytes.decode("utf-8"), "utf-8" + except UnicodeDecodeError: + pass + + # 4. Turkish fallback trial: cp1254 and iso-8859-9 + try: + return raw_bytes.decode("cp1254"), "windows-1254" + except UnicodeDecodeError: + pass + + try: + return raw_bytes.decode("iso-8859-9"), "iso-8859-9" + except UnicodeDecodeError: + pass + + raise UnicodeDecodeError("utf-8", raw_bytes, 0, len(raw_bytes), "Unable to safely decode source bytes") diff --git a/src/mesa_legal_data/parsers/html.py b/src/mesa_legal_data/parsers/html.py index 1469eaf..ca2e831 100644 --- a/src/mesa_legal_data/parsers/html.py +++ b/src/mesa_legal_data/parsers/html.py @@ -1,5 +1,6 @@ from bs4 import BeautifulSoup +from mesa_legal_data.parsers.encoding import decode_source_bytes from mesa_legal_data.parsers.text_normalizer import normalize_text @@ -18,11 +19,16 @@ def parse_html(html_content: str | bytes) -> str: if not html_content: return "" + if isinstance(html_content, bytes): + text, _ = decode_source_bytes(html_content, is_html=True) + else: + text = html_content + try: - soup = BeautifulSoup(html_content, "lxml") + soup = BeautifulSoup(text, "lxml") except Exception: # Fallback parser if lxml fails - soup = BeautifulSoup(html_content, "html.parser") + soup = BeautifulSoup(text, "html.parser") # Remove non-content tags for tag in soup(["script", "style", "noscript", "iframe", "svg", "head", "meta"]): diff --git a/src/mesa_legal_data/pipeline.py b/src/mesa_legal_data/pipeline.py index e91c519..6c4855a 100644 --- a/src/mesa_legal_data/pipeline.py +++ b/src/mesa_legal_data/pipeline.py @@ -2,6 +2,7 @@ import re import uuid from datetime import UTC, datetime +from pathlib import Path from typing import Any from mesa_legal_data.canonical import write_canonical_part @@ -26,6 +27,7 @@ build_legislation_version_id, ) from mesa_legal_data.parsers import ( + decode_source_bytes, extract_citations, parse_decision_text, parse_html, @@ -155,12 +157,12 @@ def process_artifact_pipeline( if "pdf" in mime: parsed_text = parse_pdf(full_path) else: - with open(full_path, "r", encoding="utf-8", errors="ignore") as f: - content = f.read() + raw_bytes = Path(full_path).read_bytes() if "html" in mime: - parsed_text = parse_html(content) + parsed_text = parse_html(raw_bytes) else: - parsed_text = content + decoded_content, _ = decode_source_bytes(raw_bytes, is_html=False) + parsed_text = decoded_content if not parsed_text or not parsed_text.strip(): raise ValueError("Parsed text is empty") diff --git a/src/mesa_legal_data/sources/url_fetcher.py b/src/mesa_legal_data/sources/url_fetcher.py index 1e48738..7028bec 100644 --- a/src/mesa_legal_data/sources/url_fetcher.py +++ b/src/mesa_legal_data/sources/url_fetcher.py @@ -639,7 +639,10 @@ def fetch_discovery_html( if ct and not ("html" in ct or "text" in ct or "xml" in ct): raise SourcePolicyError(f"Discovery page returned invalid Content-Type: {ct}") + from mesa_legal_data.parsers.encoding import decode_source_bytes + try: - return raw_bytes.decode("utf-8") - except UnicodeDecodeError: - return raw_bytes.decode("iso-8859-9", errors="ignore") + decoded_text, _ = decode_source_bytes(raw_bytes, is_html=True) + return decoded_text + except Exception: + return raw_bytes.decode("utf-8", errors="replace") diff --git a/src/mesa_legal_data/web/api.py b/src/mesa_legal_data/web/api.py index c217fb3..5bc16fb 100644 --- a/src/mesa_legal_data/web/api.py +++ b/src/mesa_legal_data/web/api.py @@ -23,6 +23,7 @@ resolve_issue, ) from mesa_legal_data.config import load_settings, load_sources +from mesa_legal_data.parsers import decode_source_bytes from mesa_legal_data.pipeline import process_artifact_pipeline from mesa_legal_data.release import build_release, verify_release from mesa_legal_data.release.importer import ( @@ -854,11 +855,14 @@ def get_document_text_content(document_id: str): data_root = load_settings().data_root_path content_text = "" source_type = "raw" + detected_charset = "utf-8" if raw_path_rel: try: safe_p = validate_file_download(data_root, raw_path_rel) - content_text = safe_p.read_text(encoding="utf-8", errors="ignore") + raw_bytes = safe_p.read_bytes() + is_html = raw_path_rel.lower().endswith((".html", ".htm")) + content_text, detected_charset = decode_source_bytes(raw_bytes, is_html=is_html) except Exception: pass @@ -871,7 +875,8 @@ def get_document_text_content(document_id: str): if v_row and v_row[0]: try: safe_can = validate_file_download(data_root, v_row[0]) - content_text = safe_can.read_text(encoding="utf-8", errors="ignore") + raw_bytes = safe_can.read_bytes() + content_text, detected_charset = decode_source_bytes(raw_bytes, is_html=False) source_type = "canonical" except Exception: pass @@ -888,6 +893,7 @@ def get_document_text_content(document_id: str): "document_id": document_id, "title": dict(doc).get("title"), "source_type": source_type, + "charset": detected_charset, "truncated": truncated, "content": content_text or "Metin içeriği bulunamadı.", } diff --git a/tests/integration/test_pipeline_encoding.py b/tests/integration/test_pipeline_encoding.py new file mode 100644 index 0000000..7926175 --- /dev/null +++ b/tests/integration/test_pipeline_encoding.py @@ -0,0 +1,115 @@ +import json + +from mesa_legal_data.catalog import ( + get_connection, + get_db_path, + insert_artifact, + migrate, + upsert_document, + upsert_source, +) +from mesa_legal_data.hashing import hash_stream +from mesa_legal_data.pipeline import process_artifact_pipeline + + +def test_pipeline_cp1254_turkish_encoding_preservation(tmp_path, monkeypatch): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + + db_path = get_db_path() + migrate(None, db_path) + + raw_dir = tmp_path / "raw" / "legislation" / "resmi_gazete" / "2026" / "rg-20260826-4" / "hashrg" + raw_dir.mkdir(parents=True, exist_ok=True) + raw_file = raw_dir / "payload.html" + + html_content = """ + + + + +

KONKORDATO GİDER AVANSI TARİFESİ

+

Madde 9- (1) Bu Tarife yayım tarihinde yürürlüğe girer.

+

Çalışma, işçi, işveren, değişiklik, tebliğ ve yönetmelik.

+ +""" + raw_bytes = html_content.encode("cp1254") + raw_file.write_bytes(raw_bytes) + + with open(raw_file, "rb") as f: + sha256 = hash_stream(f) + byte_size = raw_file.stat().st_size + + conn = get_connection() + upsert_source(conn, "resmi_gazete", "Resmî Gazete", "T.C. Cumhurbaşkanlığı", "https://www.resmigazete.gov.tr") + upsert_document( + conn, + "tr:legislation:communique:rg-20260826-4", + "legislation", + "communique", + "TR", + "Konkordato Gider Avansı Tarifesi", + "20260826-4", + "fetched", + ) + insert_artifact( + conn, + artifact_id="art-rg-encoding-1", + document_id="tr:legislation:communique:rg-20260826-4", + source_id="resmi_gazete", + source_url="https://www.resmigazete.gov.tr/eskiler/2026/08/20260826-4.htm", + retrieved_at="2026-08-26T00:00:00Z", + fetch_method="manual", + http_status=200, + declared_content_type="text/html; charset=Windows-1254", + detected_content_type="text/html", + byte_size=byte_size, + sha256=sha256, + raw_path=str(raw_file.relative_to(tmp_path)), + etag=None, + last_modified=None, + transport_status="fetched", + error_code=None, + metadata_json=json.dumps({"source_role": "original_publication", "publication_date": "2026-08-26"}), + ) + conn.close() + + status = process_artifact_pipeline(artifact_id="art-rg-encoding-1") + assert status == "needs_review" + + # Verify canonical jsonl content + conn = get_connection() + cur = conn.cursor() + cur.execute("SELECT record_id, canonical_path FROM records WHERE record_id = 'tr:legislation:communique:rg-20260826-4:article:9'") + row = cur.fetchone() + assert row is not None, "Article record was not created in records table" + art_record_id, canonical_rel_path = row + conn.close() + + canonical_file = tmp_path / canonical_rel_path + assert canonical_file.exists() + + with open(canonical_file, "r", encoding="utf-8") as f: + lines = [json.loads(line) for line in f] + + article_records = [r for r in lines if r.get("id") == art_record_id] + assert len(article_records) >= 1 + art_rec = article_records[0] + art_heading = art_rec.get("heading", "") + art_text = art_rec.get("text", "") + full_content = f"{art_heading} {art_text}" + + # Exact Turkish characters must be present + assert "yayım" in full_content, f"Expected 'yayım' in canonical content, got: {full_content}" + assert "yürürlüğe" in full_content, f"Expected 'yürürlüğe' in canonical content, got: {full_content}" + assert "Çalışma" in full_content + assert "işçi" in full_content + assert "işveren" in full_content + assert "değişiklik" in full_content + assert "tebliğ" in full_content + assert "yönetmelik" in full_content + assert "yaym" not in full_content + assert "yrrle" not in full_content + + # Raw artifact SHA256 must be identical + with open(raw_file, "rb") as f: + assert hash_stream(f) == sha256 diff --git a/tests/integration/test_web_document_text_encoding.py b/tests/integration/test_web_document_text_encoding.py new file mode 100644 index 0000000..0bf7ac2 --- /dev/null +++ b/tests/integration/test_web_document_text_encoding.py @@ -0,0 +1,86 @@ +from fastapi.testclient import TestClient + +from mesa_legal_data.catalog import ( + get_connection, + get_db_path, + insert_artifact, + migrate, + upsert_document, + upsert_source, +) +from mesa_legal_data.hashing import hash_stream +from mesa_legal_data.web.app import create_app + + +def test_web_document_text_cp1254_encoding(tmp_path, monkeypatch): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + + db_path = get_db_path() + migrate(None, db_path) + + raw_dir = tmp_path / "raw" / "legislation" / "resmi_gazete" / "2026" / "rg-20260826-4" / "hashwebenc" + raw_dir.mkdir(parents=True, exist_ok=True) + raw_file = raw_dir / "payload.html" + + html_content = """ + + + + +

KONKORDATO GİDER AVANSI TARİFESİ

+

Madde 9- (1) Bu Tarife yayım tarihinde yürürlüğe girer.

+ +""" + raw_bytes = html_content.encode("cp1254") + raw_file.write_bytes(raw_bytes) + + with open(raw_file, "rb") as f: + sha256 = hash_stream(f) + + conn = get_connection() + upsert_source(conn, "resmi_gazete", "Resmî Gazete", "T.C. Cumhurbaşkanlığı", "https://www.resmigazete.gov.tr") + upsert_document( + conn, + "tr:legislation:communique:rg-20260826-4", + "legislation", + "communique", + "TR", + "Konkordato Gider Avansı Tarifesi", + "20260826-4", + "fetched", + ) + insert_artifact( + conn, + artifact_id="art-web-enc-1", + document_id="tr:legislation:communique:rg-20260826-4", + source_id="resmi_gazete", + source_url="https://www.resmigazete.gov.tr/eskiler/2026/08/20260826-4.htm", + retrieved_at="2026-08-26T00:00:00Z", + fetch_method="manual", + http_status=200, + declared_content_type="text/html; charset=Windows-1254", + detected_content_type="text/html", + byte_size=len(raw_bytes), + sha256=sha256, + raw_path=str(raw_file.relative_to(tmp_path)), + etag=None, + last_modified=None, + transport_status="fetched", + error_code=None, + metadata_json="{}", + ) + conn.close() + + app = create_app() + client = TestClient(app) + + res = client.get("/api/documents/tr:legislation:communique:rg-20260826-4/text") + assert res.status_code == 200 + data = res.json()["data"] + + content = data["content"] + assert "yayım" in content, f"Expected 'yayım' in web raw text, got: {content}" + assert "yürürlüğe" in content, f"Expected 'yürürlüğe' in web raw text, got: {content}" + assert "yaym" not in content + assert "yrrle" not in content + assert data.get("charset") in ("windows-1254", "cp1254") diff --git a/tests/unit/test_encoding_cp1254.py b/tests/unit/test_encoding_cp1254.py new file mode 100644 index 0000000..582f872 --- /dev/null +++ b/tests/unit/test_encoding_cp1254.py @@ -0,0 +1,67 @@ +from mesa_legal_data.parsers.encoding import decode_source_bytes +from mesa_legal_data.parsers.html import parse_html + + +def test_parse_html_cp1254_turkish_characters(): + html = """ + + + + +

Madde 9- Bu Tarife yayım tarihinde yürürlüğe girer.

+

Çalışma, işçi, işveren, değişiklik, tebliğ ve yönetmelik.

+ +""" + raw_bytes = html.encode("cp1254") + + parsed = parse_html(raw_bytes) + + assert "yayım" in parsed, f"Expected 'yayım' in parsed text, got: {parsed}" + assert "yürürlüğe" in parsed, f"Expected 'yürürlüğe' in parsed text, got: {parsed}" + assert "Çalışma" in parsed + assert "işçi" in parsed + assert "işveren" in parsed + assert "değişiklik" in parsed + assert "tebliğ" in parsed + assert "yönetmelik" in parsed + assert "yaym" not in parsed + assert "yrrle" not in parsed + + +def test_decode_source_bytes_all_encodings(): + # 1. UTF-8 + utf8_str = "İş Sağlığı ve Güvenliği Yönetmeliği — ÇALIŞMA" + raw_utf8 = utf8_str.encode("utf-8") + text, enc = decode_source_bytes(raw_utf8, is_html=False) + assert text == utf8_str + assert enc == "utf-8" + + # 2. UTF-8 BOM + raw_bom = b"\xef\xbb\xbf" + utf8_str.encode("utf-8") + text, enc = decode_source_bytes(raw_bom, is_html=False) + assert text == utf8_str + assert enc == "utf-8-sig" + + # 3. Windows-1254 with HTML meta + html_cp1254 = """(1) Bu Tarife yayım tarihinde yürürlüğe girer.""" + raw_cp1254 = html_cp1254.encode("cp1254") + text, enc = decode_source_bytes(raw_cp1254, is_html=True) + assert "yayım" in text + assert "yürürlüğe" in text + assert enc == "windows-1254" + + # 4. ISO-8859-9 with HTML meta + html_iso = """(1) Bu Tarife yayım tarihinde yürürlüğe girer.""" + raw_iso = html_iso.encode("iso-8859-9") + text, enc = decode_source_bytes(raw_iso, is_html=True) + assert "yayım" in text + assert "yürürlüğe" in text + assert enc == "iso-8859-9" + + # 5. Raw cp1254 text without HTML meta + plain_cp1254 = "Çalışma ve Sosyal Güvenlik Bakanlığı tebliği yayım tarihi".encode("cp1254") + text, enc = decode_source_bytes(plain_cp1254, is_html=False) + assert "Çalışma" in text + assert "Güvenlik" in text + assert "tebliği" in text + assert "yayım" in text From 30fad2adea8f655a32f234f3541def02922e129e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yasin=20B=C3=BCy=C3=BCktepe?= Date: Sat, 29 Aug 2026 00:57:05 +0300 Subject: [PATCH 2/4] feat: add version-level bulk review actions --- src/mesa_legal_data/catalog.py | 106 +++++++- src/mesa_legal_data/cli.py | 23 ++ src/mesa_legal_data/web/api.py | 14 + tests/integration/test_bulk_version_review.py | 247 ++++++++++++++++++ 4 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_bulk_version_review.py diff --git a/src/mesa_legal_data/catalog.py b/src/mesa_legal_data/catalog.py index ecb1e0d..249bfc4 100644 --- a/src/mesa_legal_data/catalog.py +++ b/src/mesa_legal_data/catalog.py @@ -646,6 +646,38 @@ def list_open_blocking_issues(conn: sqlite3.Connection, subject_id: str | None = ] +def list_open_blocking_issues_for_version(conn: sqlite3.Connection, version_id: str) -> list[dict[str, Any]]: + """Lists open blocking issues on the version itself or any child records under that version.""" + cursor = conn.cursor() + cursor.execute( + """SELECT issue_id, subject_type, subject_id, severity, code, message + FROM validation_issues + WHERE status = 'open' AND severity IN ('blocker', 'error') + AND ( + subject_id = ? + OR (subject_type = 'record' AND subject_id IN (SELECT record_id FROM records WHERE version_id = ?)) + )""", + (version_id, version_id), + ) + rows = [] + while True: + batch = cursor.fetchmany(1000) + if not batch: + break + rows.extend(batch) + return [ + { + "issue_id": r[0], + "subject_type": r[1], + "subject_id": r[2], + "severity": r[3], + "code": r[4], + "message": r[5], + } + for r in rows + ] + + def resolve_issue( conn: sqlite3.Connection, issue_id: str, @@ -762,6 +794,78 @@ def reject_record_with_checks( return {"status": "rejected", "record_id": record_id, "review_id": review_id} +def reject_version( + conn: sqlite3.Connection, + *, + version_id: str, + reviewer: str, + note: str | None = None, +) -> dict[str, Any]: + ver = get_version(conn, version_id) + if not ver: + raise CatalogError(f"Version {version_id} not found") + + now_iso = datetime.now(UTC).isoformat() + + with transaction(conn): + cur = conn.cursor() + cur.execute("SELECT record_id, record_sha256 FROM records WHERE version_id = ?", (version_id,)) + records = cur.fetchall() + + if records: + review_rows = [ + (r[0], r[1], reviewer, "rejected", note, now_iso) + for r in records + ] + conn.executemany( + "INSERT INTO record_reviews (record_id, record_sha256, reviewer, decision, note, reviewed_at) VALUES (?, ?, ?, ?, ?, ?)", + review_rows, + ) + conn.execute( + "UPDATE records SET approval_status = 'rejected' WHERE version_id = ?", + (version_id,), + ) + + conn.execute( + "UPDATE versions SET approval_status = 'rejected' WHERE version_id = ?", + (version_id,), + ) + conn.execute( + "UPDATE documents SET lifecycle_status = 'rejected', updated_at = ? WHERE document_id = ?", + (now_iso, ver["document_id"]), + ) + + log_audit_event( + conn, + actor=reviewer, + action="version_reject", + subject_type="version", + subject_id=version_id, + reason=note, + details_json=json.dumps({"rejected_records": len(records)}), + ) + + try: + from mesa_legal_data.harvest.queue import reconcile_harvest_review_status + + reconcile_harvest_review_status(version_id) + except Exception: + pass + + return { + "status": "rejected", + "version_id": version_id, + "rejected_records": len(records), + "approval_status": "rejected", + } + + +def reject_version_with_checks( + conn: sqlite3.Connection, version_id: str, reviewer: str, note: str | None = None +) -> dict[str, Any]: + return reject_version(conn, version_id=version_id, reviewer=reviewer, note=note) + + def get_latest_valid_review(conn: sqlite3.Connection, record_id: str, record_sha256: str) -> dict[str, Any] | None: cursor = conn.cursor() cursor.execute( @@ -882,7 +986,7 @@ def approve_version_streaming( if not ver: raise CatalogError(f"Version {version_id} not found") - blockers = list_open_blocking_issues(conn, subject_id=version_id) + blockers = list_open_blocking_issues_for_version(conn, version_id=version_id) if blockers: raise BlockingValidationIssueExists( f"Cannot approve version {version_id}: open blocking issues exist: {blockers}" diff --git a/src/mesa_legal_data/cli.py b/src/mesa_legal_data/cli.py index bf878fe..851dab3 100644 --- a/src/mesa_legal_data/cli.py +++ b/src/mesa_legal_data/cli.py @@ -358,6 +358,29 @@ def review_reject( conn.close() +@review_app.command("reject-version") +def review_reject_version( + version_id: str = typer.Argument(..., help="Version ID to reject completely"), + reviewer: str = typer.Option("reviewer", "--reviewer", help="Reviewer name"), + note: str | None = typer.Option(None, "--note", help="Rejection note"), +): + """Rejects all records under a version.""" + from mesa_legal_data.catalog import get_connection, reject_version + + conn = get_connection() + try: + res = reject_version(conn, version_id=version_id, reviewer=reviewer, note=note) + typer.secho( + f"Successfully REJECTED version {version_id} ({res['rejected_records']} records)", + fg=typer.colors.YELLOW, + ) + except Exception as e: + typer.secho(f"Error rejecting version: {e}", fg=typer.colors.RED) + raise typer.Exit(code=1) + finally: + conn.close() + + release_app = typer.Typer(help="Manage release packages for MESA consumption.") app.add_typer(release_app, name="release") diff --git a/src/mesa_legal_data/web/api.py b/src/mesa_legal_data/web/api.py index 5bc16fb..632b4d7 100644 --- a/src/mesa_legal_data/web/api.py +++ b/src/mesa_legal_data/web/api.py @@ -20,6 +20,7 @@ get_release, list_open_blocking_issues, reject_record_with_checks, + reject_version, resolve_issue, ) from mesa_legal_data.config import load_settings, load_sources @@ -1227,6 +1228,19 @@ async def approve_version(version_id: str, req: ReviewRequest): conn.close() +@router.post("/versions/{version_id:path}/reject") +async def reject_version_endpoint(version_id: str, req: ReviewRequest): + async with write_lock.acquire_write(): + conn = get_connection() + try: + res = reject_version(conn, version_id=version_id, reviewer=req.reviewer, note=req.note) + return ok_response(res) + except Exception as e: + error_response("VERSION_REJECT_FAILED", str(e), status_code=400) + finally: + conn.close() + + # 8.8 Issues @router.get("/issues") def list_issues( diff --git a/tests/integration/test_bulk_version_review.py b/tests/integration/test_bulk_version_review.py new file mode 100644 index 0000000..9010fe0 --- /dev/null +++ b/tests/integration/test_bulk_version_review.py @@ -0,0 +1,247 @@ +import pytest +from fastapi.testclient import TestClient + +from mesa_legal_data.catalog import ( + BlockingValidationIssueExists, + approve_version_streaming, + get_connection, + get_db_path, + get_document, + get_record, + get_version, + insert_artifact, + migrate, + open_issue, + reject_version, + upsert_document, + upsert_source, +) +from mesa_legal_data.hashing import hash_stream +from mesa_legal_data.pipeline import process_artifact_pipeline +from mesa_legal_data.web.app import create_app + + +def _setup_multi_record_version(tmp_path, doc_num="100"): + db_path = get_db_path() + migrate(None, db_path) + + doc_id = f"tr:legislation:law:{doc_num}" + raw_dir = tmp_path / "raw" / "legislation" / "mevzuat" / "2026" / f"law{doc_num}" / "hash1" + raw_dir.mkdir(parents=True, exist_ok=True) + raw_file = raw_dir / "payload.html" + + html = f""" + + +

DENEME KANUNU {doc_num}

+

Madde 1- İlk madde metni 4721 sayılı Kanun uyarınca düzenlenmiştir.

+

Madde 2- İkinci madde metni 5237 sayılı Kanun gereğince uygulanır.

+

Madde 3- Üçüncü madde yürürlük maddesidir.

+ +""" + raw_bytes = html.encode("utf-8") + raw_file.write_bytes(raw_bytes) + + with open(raw_file, "rb") as f: + sha256 = hash_stream(f) + + conn = get_connection() + upsert_source(conn, "mevzuat", "Mevzuat", "T.C. Cumhurbaşkanlığı", "https://www.mevzuat.gov.tr") + upsert_document(conn, doc_id, "legislation", "law", "TR", f"Deneme Kanunu {doc_num}", doc_num, "fetched") + insert_artifact( + conn, + artifact_id=f"art-bulk-{doc_num}", + document_id=doc_id, + source_id="mevzuat", + source_url=f"https://www.mevzuat.gov.tr/{doc_num}", + retrieved_at="2026-08-01T00:00:00Z", + fetch_method="manual", + http_status=200, + declared_content_type="text/html", + detected_content_type="text/html", + byte_size=len(raw_bytes), + sha256=sha256, + raw_path=str(raw_file.relative_to(tmp_path)), + etag=None, + last_modified=None, + transport_status="fetched", + error_code=None, + metadata_json="{}", + ) + conn.close() + + status = process_artifact_pipeline(artifact_id=f"art-bulk-{doc_num}") + assert status == "needs_review" + + conn = get_connection() + c = conn.cursor() + c.execute("SELECT version_id FROM versions WHERE document_id = ?", (doc_id,)) + ver_id = c.fetchone()[0] + + c.execute("SELECT record_id, record_type FROM records WHERE version_id = ?", (ver_id,)) + records = c.fetchall() + conn.close() + + return doc_id, ver_id, records + + +def test_bulk_approve_success_all_records_and_audit(tmp_path, monkeypatch): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + doc_id, ver_id, records = _setup_multi_record_version(tmp_path, "101") + + # Ensure we have 1 legislation, 3 articles, and citations + rec_types = [r[1] for r in records] + assert "legislation" in rec_types + assert rec_types.count("article") == 3 + assert "citation" in rec_types + + conn = get_connection() + res = approve_version_streaming(conn, version_id=ver_id, reviewer="lead_reviewer", note="Toplu onay") + assert res["status"] == "approved" + assert res["approved_records"] == len(records) + + # Check version status + ver = get_version(conn, ver_id) + assert ver["approval_status"] == "approved" + + # Check document status + doc = get_document(conn, doc_id) + assert doc["lifecycle_status"] == "approved" + + # Check all records status & review trail + for r_id, _ in records: + rec = get_record(conn, r_id) + assert rec["approval_status"] == "approved" + + c = conn.cursor() + c.execute("SELECT count(*) FROM record_reviews WHERE reviewer = 'lead_reviewer' AND decision = 'approved'") + assert c.fetchone()[0] == len(records) + + c.execute("SELECT action, subject_type, subject_id, actor FROM audit_events WHERE action = 'version_approve'") + audit_row = c.fetchone() + assert audit_row is not None + assert audit_row[1] == "version" + assert audit_row[2] == ver_id + assert audit_row[3] == "lead_reviewer" + conn.close() + + +def test_bulk_approve_fail_closed_on_child_record_blocker(tmp_path, monkeypatch): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + doc_id, ver_id, records = _setup_multi_record_version(tmp_path, "102") + + article_record_id = [r[0] for r in records if r[1] == "article"][0] + + # Open a blocker issue on one child record + conn = get_connection() + open_issue( + conn, + issue_id="iss-blk-child-1", + subject_type="record", + subject_id=article_record_id, + severity="blocker", + code="CONTENT_VALIDATION_ERROR", + message="Maddede kritik hukuki format hatasi", + details_json="{}", + ) + + # Attempting to bulk approve must FAIL-CLOSED + with pytest.raises(BlockingValidationIssueExists): + approve_version_streaming(conn, version_id=ver_id, reviewer="lead_reviewer", note="Toplu onay denemesi") + + # Verify atomicity: no records approved + for r_id, _ in records: + rec = get_record(conn, r_id) + assert rec["approval_status"] == "pending" + + ver = get_version(conn, ver_id) + assert ver["approval_status"] == "pending" + conn.close() + + +def test_bulk_reject_success_and_audit(tmp_path, monkeypatch): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + doc_id, ver_id, records = _setup_multi_record_version(tmp_path, "103") + + conn = get_connection() + res = reject_version(conn, version_id=ver_id, reviewer="auditor", note="Yetersiz kaynak kalitesi") + assert res["status"] == "rejected" + assert res["rejected_records"] == len(records) + + # Check version & document status + ver = get_version(conn, ver_id) + assert ver["approval_status"] == "rejected" + + doc = get_document(conn, doc_id) + assert doc["lifecycle_status"] == "rejected" + + # Check all records + for r_id, _ in records: + rec = get_record(conn, r_id) + assert rec["approval_status"] == "rejected" + + c = conn.cursor() + c.execute("SELECT count(*) FROM record_reviews WHERE reviewer = 'auditor' AND decision = 'rejected'") + assert c.fetchone()[0] == len(records) + + c.execute("SELECT action, subject_type, subject_id, actor FROM audit_events WHERE action = 'version_reject'") + audit_row = c.fetchone() + assert audit_row is not None + assert audit_row[1] == "version" + assert audit_row[2] == ver_id + assert audit_row[3] == "auditor" + conn.close() + + +def test_bulk_review_web_api_endpoints(tmp_path, monkeypatch): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + doc_id, ver_id, records = _setup_multi_record_version(tmp_path, "104") + + app = create_app() + client = TestClient(app) + + # 1. Test POST /api/versions/{version_id}/reject + res_rej = client.post( + f"/api/versions/{ver_id}/reject", + json={"reviewer": "web_admin", "note": "Gecersiz surum"}, + headers={"X-MESA-Requested-With": "web-admin"}, + ) + assert res_rej.status_code == 200 + assert res_rej.json()["data"]["status"] == "rejected" + assert res_rej.json()["data"]["rejected_records"] == len(records) + + # 2. Test POST /api/versions/{version_id}/approve on another version + doc_id_2, ver_id_2, records_2 = _setup_multi_record_version(tmp_path, "105") + res_app = client.post( + f"/api/versions/{ver_id_2}/approve", + json={"reviewer": "web_admin", "note": "Onaylandi"}, + headers={"X-MESA-Requested-With": "web-admin"}, + ) + assert res_app.status_code == 200 + assert res_app.json()["data"]["status"] == "approved" + assert res_app.json()["data"]["approved_records"] == len(records_2) + + # 3. Test POST /api/versions/{version_id}/approve with child blocker -> 400 + doc_id_3, ver_id_3, records_3 = _setup_multi_record_version(tmp_path, "106") + child_art_id = [r[0] for r in records_3 if r[1] == "article"][0] + + conn = get_connection() + open_issue( + conn, + issue_id="iss-blk-api-1", + subject_type="record", + subject_id=child_art_id, + severity="blocker", + code="CONTENT_ERROR", + message="Kritik engel", + details_json="{}", + ) + conn.close() + + res_blk = client.post( + f"/api/versions/{ver_id_3}/approve", + json={"reviewer": "web_admin", "note": "Onay denemesi"}, + headers={"X-MESA-Requested-With": "web-admin"}, + ) + assert res_blk.status_code == 400 + assert "VERSION_APPROVE_FAILED" in res_blk.text From d3a27a08d0ce44e04a6d0299f2ae808b6677beae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yasin=20B=C3=BCy=C3=BCktepe?= Date: Sat, 29 Aug 2026 00:58:27 +0300 Subject: [PATCH 3/4] feat: show raw HTML beside canonical review text --- src/mesa_legal_data/web/static/app.js | 121 +++++++++++++++- src/mesa_legal_data/web/static/index.html | 16 ++- src/mesa_legal_data/web/static/styles.css | 56 ++++++++ .../test_web_side_by_side_review_contract.py | 130 ++++++++++++++++++ 4 files changed, 315 insertions(+), 8 deletions(-) create mode 100644 tests/acceptance/test_web_side_by_side_review_contract.py diff --git a/src/mesa_legal_data/web/static/app.js b/src/mesa_legal_data/web/static/app.js index 34195c4..c87b391 100644 --- a/src/mesa_legal_data/web/static/app.js +++ b/src/mesa_legal_data/web/static/app.js @@ -1038,8 +1038,26 @@ async function openRecordReviewModal(recordId) { setBusy(true); try { const rec = await apiRequest(`/api/records/${encodeURIComponent(recordId)}`); + state.currentRecord = rec; state.currentRecordId = recordId; state.currentVersionId = rec.version_id; + state.currentDocTitle = rec.document_title || rec.document_id || "Belge"; + + let rawContent = "Ham kaynak içeriği yüklenemedi."; + let rawCharset = ""; + try { + const docTextRes = await apiRequest(`/api/documents/${encodeURIComponent(rec.document_id)}/text`); + if (docTextRes && docTextRes.content) { + rawContent = docTextRes.content; + } + if (docTextRes && docTextRes.charset) { + rawCharset = ` · Charset: ${escapeHtml(docTextRes.charset)}`; + } + } catch (e) { + console.warn("Failed to load raw document text for comparison:", e); + } + + const canonicalPreview = rec.text_preview || (typeof rec.data_json === "string" ? rec.data_json : JSON.stringify(rec.data_json, null, 2)); const modalBody = document.getElementById("record-modal-body"); modalBody.innerHTML = ` @@ -1054,9 +1072,21 @@ async function openRecordReviewModal(recordId) { -
- - 
${escapeHtml(rec.text_preview || (typeof rec.data_json === "string" ? rec.data_json : JSON.stringify(rec.data_json, null, 2)))}
+
+
+
+ Ham Kaynak (HTML) + ${escapeHtml(rec.source_url || "")}${rawCharset} +
+ 
${escapeHtml(rawContent)}
+
+
+
+ Canonical Kayıt + ${escapeHtml(rec.record_id)} +
+ 
${escapeHtml(canonicalPreview)}
+
@@ -1128,6 +1158,81 @@ async function handleRecordDecision(decision) { } } +async function handleVersionBulkDecision(decision) { + if (!state.currentVersionId) return; + const versionId = state.currentVersionId; + const docTitle = state.currentDocTitle || "bu belgenin"; + const note = document.getElementById("txt-reviewer-note")?.value.trim() || ""; + const actionVerb = decision === "approve" ? "onaylayacaktır" : "reddedecektir"; + const actionPast = decision === "approve" ? "onaylandı" : "reddedildi"; + + let recordCountText = "tüm"; + try { + const docId = state.currentRecord?.document_id; + if (docId) { + const recsRes = await apiRequest(`/api/records?document_id=${encodeURIComponent(docId)}`); + const items = recsRes?.items || []; + const count = items.filter(r => r.version_id === versionId).length || recsRes?.total || ""; + if (count) recordCountText = `${count}`; + } + } catch (e) { + // Non-critical count lookup fallback + } + + const confirmMsg = `Bu işlem '${docTitle}' belgesinin bu sürümündeki ${recordCountText} kaydın tamamını ${actionVerb}.\n\nDevam etmek istiyor musunuz?`; + if (!window.confirm(confirmMsg)) { + return; + } + + setBusy(true); + try { + const endpoint = `/api/versions/${encodeURIComponent(versionId)}/${decision}`; + const res = await apiRequest(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + reviewer: sessionStorage.getItem("mesa_actor") || "web-user", + note: note || null, + }), + }); + + const affectedCount = res?.approved_records ?? res?.rejected_records ?? ""; + showToast(`Belge sürümü (${affectedCount} kayıt) başarıyla ${actionPast}.`, "success"); + closeModal("modal-record-detail"); + await loadReviewView(); + } catch (err) { + console.error("Bulk version decision error:", err); + const errStr = String(err.message || ""); + const isBlocker = errStr.includes("blocking") || errStr.includes("blocker") || errStr.includes("BLOCKING_ISSUES_EXIST") || errStr.includes("çözülmesi gereken") || errStr.includes("VERSION_APPROVE_FAILED"); + if (isBlocker && decision === "approve") { + showToast("Bu belge toplu olarak onaylanamaz. Belge içindeki çözülmemiş kritik kayıt sorunları bulunmaktadır.", "danger"); + const modalBody = document.getElementById("record-modal-body"); + if (modalBody) { + let blockerNotice = document.getElementById("record-blocker-notice"); + if (!blockerNotice) { + blockerNotice = document.createElement("div"); + blockerNotice.id = "record-blocker-notice"; + blockerNotice.style.cssText = "margin-top: 12px; padding: 12px; background: rgba(239, 68, 68, 0.1); border: 1px solid var(--color-danger); border-radius: 6px;"; + modalBody.appendChild(blockerNotice); + } + blockerNotice.innerHTML = ` +
+
+ Bu belge toplu olarak onaylanamaz. +
Belge içindeki çözülmemiş kritik kayıt sorunları bulunmaktadır.
+
+ +
+ `; + } + } else { + showToast(`İşlem gerçekleştirilemedi: ${err.message || "Bilinmeyen hata"}`, "danger"); + } + } finally { + setBusy(false); + } +} + // --- 5. EXPORT VIEW --- async function loadExportView() { setBusy(true); @@ -1869,6 +1974,16 @@ document.addEventListener("DOMContentLoaded", () => { btnRecordReject.addEventListener("click", () => handleRecordDecision("reject")); } + const btnVersionApprove = document.getElementById("btn-version-approve"); + if (btnVersionApprove) { + btnVersionApprove.addEventListener("click", () => handleVersionBulkDecision("approve")); + } + + const btnVersionReject = document.getElementById("btn-version-reject"); + if (btnVersionReject) { + btnVersionReject.addEventListener("click", () => handleVersionBulkDecision("reject")); + } + // 13. Export Handlers const btnExportCreate = document.getElementById("btn-export-create"); if (btnExportCreate) btnExportCreate.addEventListener("click", createExportAction); diff --git a/src/mesa_legal_data/web/static/index.html b/src/mesa_legal_data/web/static/index.html index 86b7f8b..220211b 100644 --- a/src/mesa_legal_data/web/static/index.html +++ b/src/mesa_legal_data/web/static/index.html @@ -769,16 +769,22 @@

Belge Detayı