diff --git a/src/mesa_legal_data/catalog.py b/src/mesa_legal_data/catalog.py index ecb1e0d..503e71b 100644 --- a/src/mesa_legal_data/catalog.py +++ b/src/mesa_legal_data/catalog.py @@ -646,6 +646,38 @@ def list_open_blocking_issues(conn: sqlite3.Connection, subject_id: str | None = ] +def list_open_blocking_issues_for_version(conn: sqlite3.Connection, version_id: str) -> list[dict[str, Any]]: + """Lists open blocking issues on the version itself or any child records under that version.""" + cursor = conn.cursor() + cursor.execute( + """SELECT issue_id, subject_type, subject_id, severity, code, message + FROM validation_issues + WHERE status = 'open' AND severity IN ('blocker', 'error') + AND ( + subject_id = ? + OR (subject_type = 'record' AND subject_id IN (SELECT record_id FROM records WHERE version_id = ?)) + )""", + (version_id, version_id), + ) + rows = [] + while True: + batch = cursor.fetchmany(1000) + if not batch: + break + rows.extend(batch) + return [ + { + "issue_id": r[0], + "subject_type": r[1], + "subject_id": r[2], + "severity": r[3], + "code": r[4], + "message": r[5], + } + for r in rows + ] + + def resolve_issue( conn: sqlite3.Connection, issue_id: str, @@ -762,6 +794,75 @@ def reject_record_with_checks( return {"status": "rejected", "record_id": record_id, "review_id": review_id} +def reject_version( + conn: sqlite3.Connection, + *, + version_id: str, + reviewer: str, + note: str | None = None, +) -> dict[str, Any]: + ver = get_version(conn, version_id) + if not ver: + raise CatalogError(f"Version {version_id} not found") + + now_iso = datetime.now(UTC).isoformat() + + with transaction(conn): + cur = conn.cursor() + cur.execute("SELECT record_id, record_sha256 FROM records WHERE version_id = ?", (version_id,)) + records = cur.fetchall() + + if records: + review_rows = [(r[0], r[1], reviewer, "rejected", note, now_iso) for r in records] + conn.executemany( + "INSERT INTO record_reviews (record_id, record_sha256, reviewer, decision, note, reviewed_at) VALUES (?, ?, ?, ?, ?, ?)", + review_rows, + ) + conn.execute( + "UPDATE records SET approval_status = 'rejected' WHERE version_id = ?", + (version_id,), + ) + + conn.execute( + "UPDATE versions SET approval_status = 'rejected' WHERE version_id = ?", + (version_id,), + ) + conn.execute( + "UPDATE documents SET lifecycle_status = 'rejected', updated_at = ? WHERE document_id = ?", + (now_iso, ver["document_id"]), + ) + + log_audit_event( + conn, + actor=reviewer, + action="version_reject", + subject_type="version", + subject_id=version_id, + reason=note, + details_json=json.dumps({"rejected_records": len(records)}), + ) + + try: + from mesa_legal_data.harvest.queue import reconcile_harvest_review_status + + reconcile_harvest_review_status(version_id) + except Exception: + pass + + return { + "status": "rejected", + "version_id": version_id, + "rejected_records": len(records), + "approval_status": "rejected", + } + + +def reject_version_with_checks( + conn: sqlite3.Connection, version_id: str, reviewer: str, note: str | None = None +) -> dict[str, Any]: + return reject_version(conn, version_id=version_id, reviewer=reviewer, note=note) + + def get_latest_valid_review(conn: sqlite3.Connection, record_id: str, record_sha256: str) -> dict[str, Any] | None: cursor = conn.cursor() cursor.execute( @@ -882,7 +983,7 @@ def approve_version_streaming( if not ver: raise CatalogError(f"Version {version_id} not found") - blockers = list_open_blocking_issues(conn, subject_id=version_id) + blockers = list_open_blocking_issues_for_version(conn, version_id=version_id) if blockers: raise BlockingValidationIssueExists( f"Cannot approve version {version_id}: open blocking issues exist: {blockers}" diff --git a/src/mesa_legal_data/cli.py b/src/mesa_legal_data/cli.py index bf878fe..851dab3 100644 --- a/src/mesa_legal_data/cli.py +++ b/src/mesa_legal_data/cli.py @@ -358,6 +358,29 @@ def review_reject( conn.close() +@review_app.command("reject-version") +def review_reject_version( + version_id: str = typer.Argument(..., help="Version ID to reject completely"), + reviewer: str = typer.Option("reviewer", "--reviewer", help="Reviewer name"), + note: str | None = typer.Option(None, "--note", help="Rejection note"), +): + """Rejects all records under a version.""" + from mesa_legal_data.catalog import get_connection, reject_version + + conn = get_connection() + try: + res = reject_version(conn, version_id=version_id, reviewer=reviewer, note=note) + typer.secho( + f"Successfully REJECTED version {version_id} ({res['rejected_records']} records)", + fg=typer.colors.YELLOW, + ) + except Exception as e: + typer.secho(f"Error rejecting version: {e}", fg=typer.colors.RED) + raise typer.Exit(code=1) + finally: + conn.close() + + release_app = typer.Typer(help="Manage release packages for MESA consumption.") app.add_typer(release_app, name="release") diff --git a/src/mesa_legal_data/parsers/__init__.py b/src/mesa_legal_data/parsers/__init__.py index 8cb63da..20363c8 100644 --- a/src/mesa_legal_data/parsers/__init__.py +++ b/src/mesa_legal_data/parsers/__init__.py @@ -1,5 +1,6 @@ from .citations import Citation, extract_citations from .decision import ParsedDecision, parse_decision_text +from .encoding import decode_source_bytes from .html import HTMLParseError, parse_html from .legislation import ParsedArticle, ParsedLegislation, parse_legislation_text from .pdf import OCRRequiredError, PDFParseError, parse_pdf @@ -13,6 +14,7 @@ "ParsedArticle", "ParsedDecision", "ParsedLegislation", + "decode_source_bytes", "extract_citations", "normalize_text", "parse_decision_text", diff --git a/src/mesa_legal_data/parsers/encoding.py b/src/mesa_legal_data/parsers/encoding.py new file mode 100644 index 0000000..135e64b --- /dev/null +++ b/src/mesa_legal_data/parsers/encoding.py @@ -0,0 +1,78 @@ +import re + +META_CHARSET_PATTERN = re.compile( + rb"""]+(?:charset\s*=\s*["']?([a-zA-Z0-9_-]+)|content\s*=\s*["'][^"']*charset\s*=\s*([a-zA-Z0-9_-]+))""", + re.IGNORECASE, +) + + +def decode_source_bytes(raw_bytes: bytes, is_html: bool = True) -> tuple[str, str]: + """ + Decodes raw bytes into a Python string preserving Turkish characters + without silent character deletion (`errors='ignore'` is forbidden). + + Supported encodings: + - UTF-8 with or without BOM + - Windows-1254 / cp1254 + - ISO-8859-9 / Latin-5 + - Declared HTML meta charset + + Returns: + tuple[str, str]: (decoded_text, detected_charset) + Raises: + UnicodeDecodeError: If bytes cannot be decoded safely. + """ + if not raw_bytes: + return "", "utf-8" + + # 1. UTF-8 BOM detection + if raw_bytes.startswith(b"\xef\xbb\xbf"): + return raw_bytes.decode("utf-8-sig"), "utf-8-sig" + + # 2. HTML meta charset inspection + if is_html: + sample = raw_bytes[:4096] + match = META_CHARSET_PATTERN.search(sample) + if match: + charset_bytes = match.group(1) or match.group(2) + if charset_bytes: + charset_name = charset_bytes.decode("ascii", errors="ignore").lower().strip() + if charset_name in ("windows-1254", "cp1254", "1254"): + try: + return raw_bytes.decode("cp1254"), "windows-1254" + except UnicodeDecodeError: + pass + elif charset_name in ("iso-8859-9", "latin5", "8859-9"): + try: + return raw_bytes.decode("iso-8859-9"), "iso-8859-9" + except UnicodeDecodeError: + pass + elif charset_name in ("utf-8", "utf8"): + try: + return raw_bytes.decode("utf-8"), "utf-8" + except UnicodeDecodeError: + pass + else: + try: + return raw_bytes.decode(charset_name), charset_name + except Exception: + pass + + # 3. Strict UTF-8 trial + try: + return raw_bytes.decode("utf-8"), "utf-8" + except UnicodeDecodeError: + pass + + # 4. Turkish fallback trial: cp1254 and iso-8859-9 + try: + return raw_bytes.decode("cp1254"), "windows-1254" + except UnicodeDecodeError: + pass + + try: + return raw_bytes.decode("iso-8859-9"), "iso-8859-9" + except UnicodeDecodeError: + pass + + raise UnicodeDecodeError("utf-8", raw_bytes, 0, len(raw_bytes), "Unable to safely decode source bytes") diff --git a/src/mesa_legal_data/parsers/html.py b/src/mesa_legal_data/parsers/html.py index 1469eaf..ca2e831 100644 --- a/src/mesa_legal_data/parsers/html.py +++ b/src/mesa_legal_data/parsers/html.py @@ -1,5 +1,6 @@ from bs4 import BeautifulSoup +from mesa_legal_data.parsers.encoding import decode_source_bytes from mesa_legal_data.parsers.text_normalizer import normalize_text @@ -18,11 +19,16 @@ def parse_html(html_content: str | bytes) -> str: if not html_content: return "" + if isinstance(html_content, bytes): + text, _ = decode_source_bytes(html_content, is_html=True) + else: + text = html_content + try: - soup = BeautifulSoup(html_content, "lxml") + soup = BeautifulSoup(text, "lxml") except Exception: # Fallback parser if lxml fails - soup = BeautifulSoup(html_content, "html.parser") + soup = BeautifulSoup(text, "html.parser") # Remove non-content tags for tag in soup(["script", "style", "noscript", "iframe", "svg", "head", "meta"]): diff --git a/src/mesa_legal_data/pipeline.py b/src/mesa_legal_data/pipeline.py index e91c519..6c4855a 100644 --- a/src/mesa_legal_data/pipeline.py +++ b/src/mesa_legal_data/pipeline.py @@ -2,6 +2,7 @@ import re import uuid from datetime import UTC, datetime +from pathlib import Path from typing import Any from mesa_legal_data.canonical import write_canonical_part @@ -26,6 +27,7 @@ build_legislation_version_id, ) from mesa_legal_data.parsers import ( + decode_source_bytes, extract_citations, parse_decision_text, parse_html, @@ -155,12 +157,12 @@ def process_artifact_pipeline( if "pdf" in mime: parsed_text = parse_pdf(full_path) else: - with open(full_path, "r", encoding="utf-8", errors="ignore") as f: - content = f.read() + raw_bytes = Path(full_path).read_bytes() if "html" in mime: - parsed_text = parse_html(content) + parsed_text = parse_html(raw_bytes) else: - parsed_text = content + decoded_content, _ = decode_source_bytes(raw_bytes, is_html=False) + parsed_text = decoded_content if not parsed_text or not parsed_text.strip(): raise ValueError("Parsed text is empty") diff --git a/src/mesa_legal_data/sources/url_fetcher.py b/src/mesa_legal_data/sources/url_fetcher.py index 1e48738..7028bec 100644 --- a/src/mesa_legal_data/sources/url_fetcher.py +++ b/src/mesa_legal_data/sources/url_fetcher.py @@ -639,7 +639,10 @@ def fetch_discovery_html( if ct and not ("html" in ct or "text" in ct or "xml" in ct): raise SourcePolicyError(f"Discovery page returned invalid Content-Type: {ct}") + from mesa_legal_data.parsers.encoding import decode_source_bytes + try: - return raw_bytes.decode("utf-8") - except UnicodeDecodeError: - return raw_bytes.decode("iso-8859-9", errors="ignore") + decoded_text, _ = decode_source_bytes(raw_bytes, is_html=True) + return decoded_text + except Exception: + return raw_bytes.decode("utf-8", errors="replace") diff --git a/src/mesa_legal_data/web/api.py b/src/mesa_legal_data/web/api.py index c217fb3..632b4d7 100644 --- a/src/mesa_legal_data/web/api.py +++ b/src/mesa_legal_data/web/api.py @@ -20,9 +20,11 @@ get_release, list_open_blocking_issues, reject_record_with_checks, + reject_version, resolve_issue, ) from mesa_legal_data.config import load_settings, load_sources +from mesa_legal_data.parsers import decode_source_bytes from mesa_legal_data.pipeline import process_artifact_pipeline from mesa_legal_data.release import build_release, verify_release from mesa_legal_data.release.importer import ( @@ -854,11 +856,14 @@ def get_document_text_content(document_id: str): data_root = load_settings().data_root_path content_text = "" source_type = "raw" + detected_charset = "utf-8" if raw_path_rel: try: safe_p = validate_file_download(data_root, raw_path_rel) - content_text = safe_p.read_text(encoding="utf-8", errors="ignore") + raw_bytes = safe_p.read_bytes() + is_html = raw_path_rel.lower().endswith((".html", ".htm")) + content_text, detected_charset = decode_source_bytes(raw_bytes, is_html=is_html) except Exception: pass @@ -871,7 +876,8 @@ def get_document_text_content(document_id: str): if v_row and v_row[0]: try: safe_can = validate_file_download(data_root, v_row[0]) - content_text = safe_can.read_text(encoding="utf-8", errors="ignore") + raw_bytes = safe_can.read_bytes() + content_text, detected_charset = decode_source_bytes(raw_bytes, is_html=False) source_type = "canonical" except Exception: pass @@ -888,6 +894,7 @@ def get_document_text_content(document_id: str): "document_id": document_id, "title": dict(doc).get("title"), "source_type": source_type, + "charset": detected_charset, "truncated": truncated, "content": content_text or "Metin içeriği bulunamadı.", } @@ -1221,6 +1228,19 @@ async def approve_version(version_id: str, req: ReviewRequest): conn.close() +@router.post("/versions/{version_id:path}/reject") +async def reject_version_endpoint(version_id: str, req: ReviewRequest): + async with write_lock.acquire_write(): + conn = get_connection() + try: + res = reject_version(conn, version_id=version_id, reviewer=req.reviewer, note=req.note) + return ok_response(res) + except Exception as e: + error_response("VERSION_REJECT_FAILED", str(e), status_code=400) + finally: + conn.close() + + # 8.8 Issues @router.get("/issues") def list_issues( diff --git a/src/mesa_legal_data/web/static/app.js b/src/mesa_legal_data/web/static/app.js index 34195c4..c87b391 100644 --- a/src/mesa_legal_data/web/static/app.js +++ b/src/mesa_legal_data/web/static/app.js @@ -1038,8 +1038,26 @@ async function openRecordReviewModal(recordId) { setBusy(true); try { const rec = await apiRequest(`/api/records/${encodeURIComponent(recordId)}`); + state.currentRecord = rec; state.currentRecordId = recordId; state.currentVersionId = rec.version_id; + state.currentDocTitle = rec.document_title || rec.document_id || "Belge"; + + let rawContent = "Ham kaynak içeriği yüklenemedi."; + let rawCharset = ""; + try { + const docTextRes = await apiRequest(`/api/documents/${encodeURIComponent(rec.document_id)}/text`); + if (docTextRes && docTextRes.content) { + rawContent = docTextRes.content; + } + if (docTextRes && docTextRes.charset) { + rawCharset = ` · Charset: ${escapeHtml(docTextRes.charset)}`; + } + } catch (e) { + console.warn("Failed to load raw document text for comparison:", e); + } + + const canonicalPreview = rec.text_preview || (typeof rec.data_json === "string" ? rec.data_json : JSON.stringify(rec.data_json, null, 2)); const modalBody = document.getElementById("record-modal-body"); modalBody.innerHTML = ` @@ -1054,9 +1072,21 @@ async function openRecordReviewModal(recordId) { -
- - 
${escapeHtml(rec.text_preview || (typeof rec.data_json === "string" ? rec.data_json : JSON.stringify(rec.data_json, null, 2)))}
+
+
+
+ Ham Kaynak (HTML) + ${escapeHtml(rec.source_url || "")}${rawCharset} +
+ 
${escapeHtml(rawContent)}
+
+
+
+ Canonical Kayıt + ${escapeHtml(rec.record_id)} +
+ 
${escapeHtml(canonicalPreview)}
+
@@ -1128,6 +1158,81 @@ async function handleRecordDecision(decision) { } } +async function handleVersionBulkDecision(decision) { + if (!state.currentVersionId) return; + const versionId = state.currentVersionId; + const docTitle = state.currentDocTitle || "bu belgenin"; + const note = document.getElementById("txt-reviewer-note")?.value.trim() || ""; + const actionVerb = decision === "approve" ? "onaylayacaktır" : "reddedecektir"; + const actionPast = decision === "approve" ? "onaylandı" : "reddedildi"; + + let recordCountText = "tüm"; + try { + const docId = state.currentRecord?.document_id; + if (docId) { + const recsRes = await apiRequest(`/api/records?document_id=${encodeURIComponent(docId)}`); + const items = recsRes?.items || []; + const count = items.filter(r => r.version_id === versionId).length || recsRes?.total || ""; + if (count) recordCountText = `${count}`; + } + } catch (e) { + // Non-critical count lookup fallback + } + + const confirmMsg = `Bu işlem '${docTitle}' belgesinin bu sürümündeki ${recordCountText} kaydın tamamını ${actionVerb}.\n\nDevam etmek istiyor musunuz?`; + if (!window.confirm(confirmMsg)) { + return; + } + + setBusy(true); + try { + const endpoint = `/api/versions/${encodeURIComponent(versionId)}/${decision}`; + const res = await apiRequest(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + reviewer: sessionStorage.getItem("mesa_actor") || "web-user", + note: note || null, + }), + }); + + const affectedCount = res?.approved_records ?? res?.rejected_records ?? ""; + showToast(`Belge sürümü (${affectedCount} kayıt) başarıyla ${actionPast}.`, "success"); + closeModal("modal-record-detail"); + await loadReviewView(); + } catch (err) { + console.error("Bulk version decision error:", err); + const errStr = String(err.message || ""); + const isBlocker = errStr.includes("blocking") || errStr.includes("blocker") || errStr.includes("BLOCKING_ISSUES_EXIST") || errStr.includes("çözülmesi gereken") || errStr.includes("VERSION_APPROVE_FAILED"); + if (isBlocker && decision === "approve") { + showToast("Bu belge toplu olarak onaylanamaz. Belge içindeki çözülmemiş kritik kayıt sorunları bulunmaktadır.", "danger"); + const modalBody = document.getElementById("record-modal-body"); + if (modalBody) { + let blockerNotice = document.getElementById("record-blocker-notice"); + if (!blockerNotice) { + blockerNotice = document.createElement("div"); + blockerNotice.id = "record-blocker-notice"; + blockerNotice.style.cssText = "margin-top: 12px; padding: 12px; background: rgba(239, 68, 68, 0.1); border: 1px solid var(--color-danger); border-radius: 6px;"; + modalBody.appendChild(blockerNotice); + } + blockerNotice.innerHTML = ` +
+
+ Bu belge toplu olarak onaylanamaz. +
Belge içindeki çözülmemiş kritik kayıt sorunları bulunmaktadır.
+
+ +
+ `; + } + } else { + showToast(`İşlem gerçekleştirilemedi: ${err.message || "Bilinmeyen hata"}`, "danger"); + } + } finally { + setBusy(false); + } +} + // --- 5. EXPORT VIEW --- async function loadExportView() { setBusy(true); @@ -1869,6 +1974,16 @@ document.addEventListener("DOMContentLoaded", () => { btnRecordReject.addEventListener("click", () => handleRecordDecision("reject")); } + const btnVersionApprove = document.getElementById("btn-version-approve"); + if (btnVersionApprove) { + btnVersionApprove.addEventListener("click", () => handleVersionBulkDecision("approve")); + } + + const btnVersionReject = document.getElementById("btn-version-reject"); + if (btnVersionReject) { + btnVersionReject.addEventListener("click", () => handleVersionBulkDecision("reject")); + } + // 13. Export Handlers const btnExportCreate = document.getElementById("btn-export-create"); if (btnExportCreate) btnExportCreate.addEventListener("click", createExportAction); diff --git a/src/mesa_legal_data/web/static/index.html b/src/mesa_legal_data/web/static/index.html index 86b7f8b..220211b 100644 --- a/src/mesa_legal_data/web/static/index.html +++ b/src/mesa_legal_data/web/static/index.html @@ -769,16 +769,22 @@

Belge Detayı