Fix/encoding bulk review

src/mesa_legal_data/catalog.py

@@ -646,6 +646,38 @@ def list_open_blocking_issues(conn: sqlite3.Connection, subject_id: str | None =
     ]
 
 
+def list_open_blocking_issues_for_version(conn: sqlite3.Connection, version_id: str) -> list[dict[str, Any]]:
+    """Lists open blocking issues on the version itself or any child records under that version."""
+    cursor = conn.cursor()
+    cursor.execute(
+        """SELECT issue_id, subject_type, subject_id, severity, code, message 
+           FROM validation_issues 
+           WHERE status = 'open' AND severity IN ('blocker', 'error') 
+             AND (
+                 subject_id = ? 
+                 OR (subject_type = 'record' AND subject_id IN (SELECT record_id FROM records WHERE version_id = ?))
+             )""",
+        (version_id, version_id),
+    )
+    rows = []
+    while True:
+        batch = cursor.fetchmany(1000)
+        if not batch:
+            break
+        rows.extend(batch)
+    return [
+        {
+            "issue_id": r[0],
+            "subject_type": r[1],
+            "subject_id": r[2],
+            "severity": r[3],
+            "code": r[4],
+            "message": r[5],
+        }
+        for r in rows
+    ]
+
+
 def resolve_issue(
     conn: sqlite3.Connection,
     issue_id: str,
@@ -762,6 +794,75 @@ def reject_record_with_checks(
     return {"status": "rejected", "record_id": record_id, "review_id": review_id}
 
 
+def reject_version(
+    conn: sqlite3.Connection,
+    *,
+    version_id: str,
+    reviewer: str,
+    note: str | None = None,
+) -> dict[str, Any]:
+    ver = get_version(conn, version_id)
+    if not ver:
+        raise CatalogError(f"Version {version_id} not found")
+
+    now_iso = datetime.now(UTC).isoformat()
+
+    with transaction(conn):
+        cur = conn.cursor()
+        cur.execute("SELECT record_id, record_sha256 FROM records WHERE version_id = ?", (version_id,))
+        records = cur.fetchall()
+
+        if records:
+            review_rows = [(r[0], r[1], reviewer, "rejected", note, now_iso) for r in records]
+            conn.executemany(
+                "INSERT INTO record_reviews (record_id, record_sha256, reviewer, decision, note, reviewed_at) VALUES (?, ?, ?, ?, ?, ?)",
+                review_rows,
+            )
+            conn.execute(
+                "UPDATE records SET approval_status = 'rejected' WHERE version_id = ?",
+                (version_id,),
+            )
+
+        conn.execute(
+            "UPDATE versions SET approval_status = 'rejected' WHERE version_id = ?",
+            (version_id,),
+        )
+        conn.execute(
+            "UPDATE documents SET lifecycle_status = 'rejected', updated_at = ? WHERE document_id = ?",
+            (now_iso, ver["document_id"]),
+        )
+
+        log_audit_event(
+            conn,
+            actor=reviewer,
+            action="version_reject",
+            subject_type="version",
+            subject_id=version_id,
+            reason=note,
+            details_json=json.dumps({"rejected_records": len(records)}),
+        )
+
+        try:
+            from mesa_legal_data.harvest.queue import reconcile_harvest_review_status
+
+            reconcile_harvest_review_status(version_id)
+        except Exception:
+            pass
+
+    return {
+        "status": "rejected",
+        "version_id": version_id,
+        "rejected_records": len(records),
+        "approval_status": "rejected",
+    }
+
+
+def reject_version_with_checks(
+    conn: sqlite3.Connection, version_id: str, reviewer: str, note: str | None = None
+) -> dict[str, Any]:
+    return reject_version(conn, version_id=version_id, reviewer=reviewer, note=note)
+
+
 def get_latest_valid_review(conn: sqlite3.Connection, record_id: str, record_sha256: str) -> dict[str, Any] | None:
     cursor = conn.cursor()
     cursor.execute(
@@ -882,7 +983,7 @@ def approve_version_streaming(
     if not ver:
         raise CatalogError(f"Version {version_id} not found")
 
-    blockers = list_open_blocking_issues(conn, subject_id=version_id)
+    blockers = list_open_blocking_issues_for_version(conn, version_id=version_id)
     if blockers:
         raise BlockingValidationIssueExists(
             f"Cannot approve version {version_id}: open blocking issues exist: {blockers}"

src/mesa_legal_data/cli.py

@@ -358,6 +358,29 @@ def review_reject(
         conn.close()
 
 
+@review_app.command("reject-version")
+def review_reject_version(
+    version_id: str = typer.Argument(..., help="Version ID to reject completely"),
+    reviewer: str = typer.Option("reviewer", "--reviewer", help="Reviewer name"),
+    note: str | None = typer.Option(None, "--note", help="Rejection note"),
+):
+    """Rejects all records under a version."""
+    from mesa_legal_data.catalog import get_connection, reject_version
+
+    conn = get_connection()
+    try:
+        res = reject_version(conn, version_id=version_id, reviewer=reviewer, note=note)
+        typer.secho(
+            f"Successfully REJECTED version {version_id} ({res['rejected_records']} records)",
+            fg=typer.colors.YELLOW,
+        )
+    except Exception as e:
+        typer.secho(f"Error rejecting version: {e}", fg=typer.colors.RED)
+        raise typer.Exit(code=1)
+    finally:
+        conn.close()
+
+
 release_app = typer.Typer(help="Manage release packages for MESA consumption.")
 app.add_typer(release_app, name="release")
 

src/mesa_legal_data/parsers/__init__.py

@@ -1,5 +1,6 @@
 from .citations import Citation, extract_citations
 from .decision import ParsedDecision, parse_decision_text
+from .encoding import decode_source_bytes
 from .html import HTMLParseError, parse_html
 from .legislation import ParsedArticle, ParsedLegislation, parse_legislation_text
 from .pdf import OCRRequiredError, PDFParseError, parse_pdf
@@ -13,6 +14,7 @@
     "ParsedArticle",
     "ParsedDecision",
     "ParsedLegislation",
+    "decode_source_bytes",
     "extract_citations",
     "normalize_text",
     "parse_decision_text",

src/mesa_legal_data/parsers/encoding.py

@@ -0,0 +1,78 @@
+import re
+
+META_CHARSET_PATTERN = re.compile(
+    rb"""<meta[^>]+(?:charset\s*=\s*["']?([a-zA-Z0-9_-]+)|content\s*=\s*["'][^"']*charset\s*=\s*([a-zA-Z0-9_-]+))""",
+    re.IGNORECASE,
+)
+
+
+def decode_source_bytes(raw_bytes: bytes, is_html: bool = True) -> tuple[str, str]:
+    """
+    Decodes raw bytes into a Python string preserving Turkish characters
+    without silent character deletion (`errors='ignore'` is forbidden).
+
+    Supported encodings:
+    - UTF-8 with or without BOM
+    - Windows-1254 / cp1254
+    - ISO-8859-9 / Latin-5
+    - Declared HTML meta charset
+
+    Returns:
+        tuple[str, str]: (decoded_text, detected_charset)
+    Raises:
+        UnicodeDecodeError: If bytes cannot be decoded safely.
+    """
+    if not raw_bytes:
+        return "", "utf-8"
+
+    # 1. UTF-8 BOM detection
+    if raw_bytes.startswith(b"\xef\xbb\xbf"):
+        return raw_bytes.decode("utf-8-sig"), "utf-8-sig"
+
+    # 2. HTML meta charset inspection
+    if is_html:
+        sample = raw_bytes[:4096]
+        match = META_CHARSET_PATTERN.search(sample)
+        if match:
+            charset_bytes = match.group(1) or match.group(2)
+            if charset_bytes:
+                charset_name = charset_bytes.decode("ascii", errors="ignore").lower().strip()
+                if charset_name in ("windows-1254", "cp1254", "1254"):
+                    try:
+                        return raw_bytes.decode("cp1254"), "windows-1254"
+                    except UnicodeDecodeError:
+                        pass
+                elif charset_name in ("iso-8859-9", "latin5", "8859-9"):
+                    try:
+                        return raw_bytes.decode("iso-8859-9"), "iso-8859-9"
+                    except UnicodeDecodeError:
+                        pass
+                elif charset_name in ("utf-8", "utf8"):
+                    try:
+                        return raw_bytes.decode("utf-8"), "utf-8"
+                    except UnicodeDecodeError:
+                        pass
+                else:
+                    try:
+                        return raw_bytes.decode(charset_name), charset_name
+                    except Exception:
+                        pass
+
+    # 3. Strict UTF-8 trial
+    try:
+        return raw_bytes.decode("utf-8"), "utf-8"
+    except UnicodeDecodeError:
+        pass
+
+    # 4. Turkish fallback trial: cp1254 and iso-8859-9
+    try:
+        return raw_bytes.decode("cp1254"), "windows-1254"
+    except UnicodeDecodeError:
+        pass
+
+    try:
+        return raw_bytes.decode("iso-8859-9"), "iso-8859-9"
+    except UnicodeDecodeError:
+        pass
+
+    raise UnicodeDecodeError("utf-8", raw_bytes, 0, len(raw_bytes), "Unable to safely decode source bytes")

src/mesa_legal_data/parsers/html.py

@@ -1,5 +1,6 @@
 from bs4 import BeautifulSoup
 
+from mesa_legal_data.parsers.encoding import decode_source_bytes
 from mesa_legal_data.parsers.text_normalizer import normalize_text
 
 
@@ -18,11 +19,16 @@ def parse_html(html_content: str | bytes) -> str:
     if not html_content:
         return ""
 
+    if isinstance(html_content, bytes):
+        text, _ = decode_source_bytes(html_content, is_html=True)
+    else:
+        text = html_content
+
     try:
-        soup = BeautifulSoup(html_content, "lxml")
+        soup = BeautifulSoup(text, "lxml")
     except Exception:
         # Fallback parser if lxml fails
-        soup = BeautifulSoup(html_content, "html.parser")
+        soup = BeautifulSoup(text, "html.parser")
 
     # Remove non-content tags
     for tag in soup(["script", "style", "noscript", "iframe", "svg", "head", "meta"]):

src/mesa_legal_data/pipeline.py

@@ -2,6 +2,7 @@
 import re
 import uuid
 from datetime import UTC, datetime
+from pathlib import Path
 from typing import Any
 
 from mesa_legal_data.canonical import write_canonical_part
@@ -26,6 +27,7 @@
     build_legislation_version_id,
 )
 from mesa_legal_data.parsers import (
+    decode_source_bytes,
     extract_citations,
     parse_decision_text,
     parse_html,
@@ -155,12 +157,12 @@ def process_artifact_pipeline(
         if "pdf" in mime:
             parsed_text = parse_pdf(full_path)
         else:
-            with open(full_path, "r", encoding="utf-8", errors="ignore") as f:
-                content = f.read()
+            raw_bytes = Path(full_path).read_bytes()
             if "html" in mime:
-                parsed_text = parse_html(content)
+                parsed_text = parse_html(raw_bytes)
             else:
-                parsed_text = content
+                decoded_content, _ = decode_source_bytes(raw_bytes, is_html=False)
+                parsed_text = decoded_content
 
         if not parsed_text or not parsed_text.strip():
             raise ValueError("Parsed text is empty")

src/mesa_legal_data/sources/url_fetcher.py

@@ -639,7 +639,10 @@ def fetch_discovery_html(
     if ct and not ("html" in ct or "text" in ct or "xml" in ct):
         raise SourcePolicyError(f"Discovery page returned invalid Content-Type: {ct}")
 
+    from mesa_legal_data.parsers.encoding import decode_source_bytes
+
     try:
-        return raw_bytes.decode("utf-8")
-    except UnicodeDecodeError:
-        return raw_bytes.decode("iso-8859-9", errors="ignore")
+        decoded_text, _ = decode_source_bytes(raw_bytes, is_html=True)
+        return decoded_text
+    except Exception:
+        return raw_bytes.decode("utf-8", errors="replace")

src/mesa_legal_data/web/api.py

@@ -20,9 +20,11 @@
     get_release,
     list_open_blocking_issues,
     reject_record_with_checks,
+    reject_version,
     resolve_issue,
 )
 from mesa_legal_data.config import load_settings, load_sources
+from mesa_legal_data.parsers import decode_source_bytes
 from mesa_legal_data.pipeline import process_artifact_pipeline
 from mesa_legal_data.release import build_release, verify_release
 from mesa_legal_data.release.importer import (
@@ -854,11 +856,14 @@ def get_document_text_content(document_id: str):
     data_root = load_settings().data_root_path
     content_text = ""
     source_type = "raw"
+    detected_charset = "utf-8"
 
     if raw_path_rel:
         try:
             safe_p = validate_file_download(data_root, raw_path_rel)
-            content_text = safe_p.read_text(encoding="utf-8", errors="ignore")
+            raw_bytes = safe_p.read_bytes()
+            is_html = raw_path_rel.lower().endswith((".html", ".htm"))
+            content_text, detected_charset = decode_source_bytes(raw_bytes, is_html=is_html)
         except Exception:
             pass
 
@@ -871,7 +876,8 @@ def get_document_text_content(document_id: str):
         if v_row and v_row[0]:
             try:
                 safe_can = validate_file_download(data_root, v_row[0])
-                content_text = safe_can.read_text(encoding="utf-8", errors="ignore")
+                raw_bytes = safe_can.read_bytes()
+                content_text, detected_charset = decode_source_bytes(raw_bytes, is_html=False)
                 source_type = "canonical"
             except Exception:
                 pass
@@ -888,6 +894,7 @@ def get_document_text_content(document_id: str):
             "document_id": document_id,
             "title": dict(doc).get("title"),
             "source_type": source_type,
+            "charset": detected_charset,
             "truncated": truncated,
             "content": content_text or "Metin içeriği bulunamadı.",
         }
@@ -1221,6 +1228,19 @@ async def approve_version(version_id: str, req: ReviewRequest):
             conn.close()
 
 
+@router.post("/versions/{version_id:path}/reject")
+async def reject_version_endpoint(version_id: str, req: ReviewRequest):
+    async with write_lock.acquire_write():
+        conn = get_connection()
+        try:
+            res = reject_version(conn, version_id=version_id, reviewer=req.reviewer, note=req.note)
+            return ok_response(res)
+        except Exception as e:
+            error_response("VERSION_REJECT_FAILED", str(e), status_code=400)
+        finally:
+            conn.close()
+
+
 # 8.8 Issues
 @router.get("/issues")
 def list_issues(