[codex] Audit Python dependencies and contain PDF parsing

.github/workflows/ci.yml

@@ -5,6 +5,8 @@ on:
     branches: [main]
     tags: ["v*"]
   pull_request:
+  schedule:
+    - cron: "17 2 * * 1"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -21,6 +23,7 @@ env:
 
 jobs:
   quality:
+    if: github.event_name != 'schedule'
     strategy:
       fail-fast: false
       matrix:
@@ -119,6 +122,7 @@ jobs:
         run: npm run pack:check
 
   clean-install-e2e:
+    if: github.event_name != 'schedule'
     strategy:
       fail-fast: false
       matrix:
@@ -198,3 +202,23 @@ jobs:
           Add-Content -Path $env:GITHUB_ENV -Value "TYPST_PACKAGE_CACHE_PATH=$packageCache"
       - name: Run clean npm and Python delivery flow
         run: python scripts/check_clean_install.py --json
+
+  python-dependency-audit:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        python: ["3.10", "3.11"]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+          cache: pip
+      - run: python -m pip install --upgrade pip
+      - run: python -m pip install -r requirements-audit.txt
+      - name: Audit exact Python runtime dependency closure
+        run: python scripts/audit_python_dependencies.py

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ## [Unreleased]
 
+- Python 运行时升级并锁定 Flask 3.1.3 与 pypdf 6.14.2；Windows/Ubuntu CI 使用固定 pip-audit 审计完整依赖闭包，临时例外必须绑定包名、原因、到期日和追踪链接。PDF 页数读取、文本边界扫描与切页迁入受 timeout、内存、输出和进程数限制的独立 worker，损坏或异常 PDF 不再无界占用构建进程或 WebUI 请求线程。
 - 非交互 stdout/stderr 及 Checker feedback 改为原子共享单一输出预算，完成、超时、取消和资源超限均在进程树终止后执行确定性公平前缀截断；无法测量或截断时 fail closed，沙箱缓存 Schema 升至 6。stress schema 2 反例以 `E + min(E, 8 MiB)` 限制单次持久化，消除 `generator.out` 输入副本，记录逐文件预算与截断证据，并拒绝重放不完整的 Generator OLE 输入。
 - WebUI 完整沙箱与上传评测增加最多 8 个请求线程的有界 HTTP 接入、前置 admission gate、共享固定 worker pool、有界队列、同题进程内互斥与跨进程 Judge 锁，新增结构化 `429 queue_full`、单调排队/执行/整任务 deadline、完整沙箱取消，以及日志、协议输出、结构化结果和完成记录上限；取消与协议错误状态保持单调，上传准备与清理纳入任务生命周期，超时、清理失败与服务关闭不再虚报成功并继续回收完整进程树。
 - WebUI 沙箱改为检查已安装 Core 内的 `local_judge.py`，不再要求赛事仓库复制 Judge 运行时；点击运行会先通过当前写入队列完成题面保存，修复沙箱按钮被错误禁用以及旧 `_doSave` 调用导致任务无法入队的问题。

package.json

@@ -50,6 +50,7 @@
     "!scripts/webui/tailwind.input.css",
     "!scripts/check_clean_install.py",
     "!scripts/check_release.py",
+    "!scripts/audit_python_dependencies.py",
     "probhub/**/*.py"
   ],
   "devDependencies": {

probhub/pdf_processing.py

@@ -0,0 +1,182 @@
+"""Bounded PDF inspection and extraction through an isolated worker."""
+
+import json
+import os
+import stat
+import sys
+import tempfile
+from pathlib import Path
+
+from .errors import ProbHubError
+from .process_control import OutputBudgetError, run_managed_to_files
+
+
+PACKAGE_ROOT = Path(__file__).resolve().parents[1]
+PDF_TIMEOUT_SECONDS = 30
+PDF_UI_TIMEOUT_SECONDS = 10
+PDF_MEMORY_LIMIT_MB = 512
+PDF_OUTPUT_LIMIT_BYTES = 1024 * 1024
+PDF_PROCESS_LIMIT = 4
+PDF_REQUEST_LIMIT_BYTES = 2 * 1024 * 1024
+
+
+def _regular_file(path, *, label, require_nonempty=True):
+    try:
+        path = Path(path)
+    except (TypeError, ValueError) as exc:
+        raise ProbHubError(
+            f"{label} must be a path",
+            code="pdf_processing_failed",
+        ) from exc
+    try:
+        info = os.lstat(path)
+    except OSError as exc:
+        raise ProbHubError(f"{label} is unavailable: {path}: {exc}", code="pdf_processing_failed") from exc
+    reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)
+    if stat.S_ISLNK(info.st_mode) or (
+        reparse_flag and getattr(info, "st_file_attributes", 0) & reparse_flag
+    ):
+        raise ProbHubError(f"{label} must not be a link: {path}", code="pdf_processing_failed")
+    if not stat.S_ISREG(info.st_mode):
+        raise ProbHubError(f"{label} is not a regular file: {path}", code="pdf_processing_failed")
+    if require_nonempty and info.st_size <= 0:
+        raise ProbHubError(f"{label} is empty: {path}", code="pdf_processing_failed")
+    return path.resolve()
+
+
+def _worker_command(operation, request_path):
+    return [sys.executable, "-m", "probhub.pdf_worker", operation, str(request_path)]
+
+
+def _worker_error(result, stdout_path, stderr_path):
+    reason = result.get("reason")
+    messages = {
+        "time_limit": "PDF processing timed out",
+        "memory_limit": "PDF processing exceeded the memory limit",
+        "output_limit": "PDF processing exceeded the diagnostic output limit",
+        "process_limit": "PDF processing exceeded the process limit",
+    }
+    message = messages.get(reason)
+    stdout = stdout_path.read_text(encoding="utf-8", errors="replace") if stdout_path.is_file() else ""
+    stderr = stderr_path.read_text(encoding="utf-8", errors="replace") if stderr_path.is_file() else ""
+    try:
+        payload = json.loads(stdout)
+    except json.JSONDecodeError:
+        payload = None
+    if message is None and isinstance(payload, dict) and isinstance(payload.get("error"), str):
+        message = "PDF processing failed: " + payload["error"][:2000]
+    if message is None:
+        detail = stderr.strip()[-2000:] or stdout.strip()[-2000:]
+        message = result.get("message") or "PDF processing worker failed"
+        if detail:
+            message += ": " + detail
+    return ProbHubError(message, code="pdf_processing_failed")
+
+
+def _run_worker(operation, payload, *, timeout=None):
+    timeout = PDF_TIMEOUT_SECONDS if timeout is None else float(timeout)
+    with tempfile.TemporaryDirectory(prefix="probhub-pdf-") as temp:
+        temp_dir = Path(temp)
+        request_path = temp_dir / "request.json"
+        encoded = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
+        if len(encoded) > PDF_REQUEST_LIMIT_BYTES:
+            raise ProbHubError("PDF processing request is too large", code="pdf_processing_failed")
+        request_path.write_bytes(encoded)
+        stdout_path = temp_dir / "stdout"
+        stderr_path = temp_dir / "stderr"
+        try:
+            result = run_managed_to_files(
+                _worker_command(operation, request_path),
+                stdout_path=stdout_path,
+                stderr_path=stderr_path,
+                timeout=timeout,
+                memory_limit_mb=PDF_MEMORY_LIMIT_MB,
+                output_limit_bytes=PDF_OUTPUT_LIMIT_BYTES,
+                process_limit=PDF_PROCESS_LIMIT,
+                cwd=PACKAGE_ROOT,
+            )
+        except (OSError, OutputBudgetError) as exc:
+            raise ProbHubError(
+                f"PDF processing worker could not start: {exc}",
+                code="pdf_processing_failed",
+            ) from exc
+        if result.get("reason") != "completed" or result.get("returncode") != 0:
+            raise _worker_error(result, stdout_path, stderr_path)
+        try:
+            response = json.loads(stdout_path.read_text(encoding="utf-8"))
+        except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+            raise ProbHubError(
+                "PDF processing worker returned invalid JSON",
+                code="pdf_processing_failed",
+            ) from exc
+        if not isinstance(response, dict) or response.get("ok") is not True:
+            raise ProbHubError(
+                "PDF processing worker returned an invalid result",
+                code="pdf_processing_failed",
+            )
+        return response
+
+
+def inspect_pdf(pdf_path, *, scan_text=False, timeout=None):
+    pdf_path = _regular_file(pdf_path, label="PDF input")
+    response = _run_worker(
+        "inspect",
+        {"input": str(pdf_path), "scan_text": bool(scan_text)},
+        timeout=timeout,
+    )
+    pages = response.get("pages")
+    if isinstance(pages, bool) or not isinstance(pages, int) or pages < 0:
+        raise ProbHubError("PDF worker returned an invalid page count", code="pdf_processing_failed")
+    if scan_text:
+        if not isinstance(response.get("markers"), list) or not isinstance(response.get("legacy"), list):
+            raise ProbHubError("PDF worker returned invalid boundary data", code="pdf_processing_failed")
+    return response
+
+
+def pdf_page_count(pdf_path, *, timeout=PDF_UI_TIMEOUT_SECONDS):
+    return inspect_pdf(pdf_path, timeout=timeout)["pages"]
+
+
+def split_pdf(pdf_path, outputs, *, timeout=None):
+    pdf_path = _regular_file(pdf_path, label="PDF input")
+    if not isinstance(outputs, (list, tuple)):
+        raise ProbHubError("PDF split outputs must be a list", code="pdf_processing_failed")
+    normalized = []
+    seen = set()
+    for item in outputs:
+        if not isinstance(item, dict):
+            raise ProbHubError("PDF split entry must be an object", code="pdf_processing_failed")
+        try:
+            raw_output = os.fspath(item.get("path"))
+        except TypeError as exc:
+            raise ProbHubError(
+                "PDF split output path must be a non-empty path",
+                code="pdf_processing_failed",
+            ) from exc
+        if not isinstance(raw_output, str) or not raw_output:
+            raise ProbHubError(
+                "PDF split output path must be a non-empty path",
+                code="pdf_processing_failed",
+            )
+        output = Path(raw_output).resolve()
+        start = item.get("start")
+        end = item.get("end")
+        if output == pdf_path or output in seen:
+            raise ProbHubError("PDF split outputs must be unique", code="pdf_processing_failed")
+        if any(isinstance(value, bool) or not isinstance(value, int) for value in (start, end)):
+            raise ProbHubError("PDF split ranges must be integers", code="pdf_processing_failed")
+        seen.add(output)
+        normalized.append({"path": str(output), "start": start, "end": end})
+    if not normalized:
+        return {"ok": True, "pages": inspect_pdf(pdf_path, timeout=timeout)["pages"], "outputs": []}
+    response = _run_worker(
+        "split",
+        {"input": str(pdf_path), "outputs": normalized},
+        timeout=timeout,
+    )
+    returned = response.get("outputs")
+    if not isinstance(returned, list) or len(returned) != len(normalized):
+        raise ProbHubError("PDF worker returned invalid split results", code="pdf_processing_failed")
+    for item in normalized:
+        _regular_file(item["path"], label="PDF split output")
+    return response

probhub/pdf_worker.py

@@ -0,0 +1,139 @@
+"""Isolated pypdf worker. Invoke through probhub.pdf_processing only."""
+
+import json
+import os
+import re
+import stat
+import sys
+import uuid
+from pathlib import Path
+
+from pypdf import PdfReader, PdfWriter
+
+from .metadata import BOUNDARY_MARKER_PREFIX
+
+
+REQUEST_LIMIT_BYTES = 2 * 1024 * 1024
+BOUNDARY_MARKER_PATTERN = re.compile(
+    rf"{re.escape(BOUNDARY_MARKER_PREFIX)}[0-9a-f]{{64}}"
+)
+LEGACY_HEADING_PATTERN = re.compile(r"题目\s+[A-Z]+\.\s*(.+)")
+
+
+def _regular_file(path, *, label):
+    path = Path(path)
+    info = os.lstat(path)
+    reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)
+    if stat.S_ISLNK(info.st_mode) or (
+        reparse_flag and getattr(info, "st_file_attributes", 0) & reparse_flag
+    ):
+        raise ValueError(f"{label} must not be a link")
+    if not stat.S_ISREG(info.st_mode) or info.st_size <= 0:
+        raise ValueError(f"{label} must be a non-empty regular file")
+    return path.resolve()
+
+
+def _load_request(path):
+    path = _regular_file(path, label="request")
+    if path.stat().st_size > REQUEST_LIMIT_BYTES:
+        raise ValueError("request is too large")
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(payload, dict):
+        raise ValueError("request must be an object")
+    return payload
+
+
+def _inspect(payload):
+    input_path = _regular_file(payload.get("input", ""), label="PDF input")
+    scan_text = payload.get("scan_text", False)
+    if not isinstance(scan_text, bool):
+        raise ValueError("scan_text must be boolean")
+    markers = []
+    legacy = []
+    with input_path.open("rb") as stream:
+        reader = PdfReader(stream)
+        pages = len(reader.pages)
+        if scan_text:
+            for index, page in enumerate(reader.pages):
+                text = page.extract_text() or ""
+                markers.extend(
+                    {"marker": marker, "page": index + 1}
+                    for marker in BOUNDARY_MARKER_PATTERN.findall(text)
+                )
+                for line in text.splitlines():
+                    match = LEGACY_HEADING_PATTERN.search(line.strip())
+                    if match:
+                        legacy.append({"display_name": match.group(1).strip(), "page": index + 1})
+                        break
+    return {"ok": True, "pages": pages, "markers": markers, "legacy": legacy}
+
+
+def _split(payload):
+    input_path = _regular_file(payload.get("input", ""), label="PDF input")
+    entries = payload.get("outputs")
+    if not isinstance(entries, list) or not entries:
+        raise ValueError("outputs must be a non-empty list")
+    normalized = []
+    seen = set()
+    with input_path.open("rb") as stream:
+        reader = PdfReader(stream)
+        total_pages = len(reader.pages)
+        for item in entries:
+            if not isinstance(item, dict):
+                raise ValueError("split entry must be an object")
+            output = Path(item.get("path", "")).resolve()
+            start = item.get("start")
+            end = item.get("end")
+            if output == input_path or output in seen:
+                raise ValueError("split outputs must be unique")
+            if any(isinstance(value, bool) or not isinstance(value, int) for value in (start, end)):
+                raise ValueError("split ranges must be integers")
+            if start < 0 or end <= start or end > total_pages:
+                raise ValueError("split range is outside the PDF")
+            seen.add(output)
+            normalized.append((output, start, end))
+
+        prepared = []
+        try:
+            for output, start, end in normalized:
+                output.parent.mkdir(parents=True, exist_ok=True)
+                temporary = output.with_name(f".{output.name}.probhub-{uuid.uuid4().hex}.tmp")
+                prepared.append((temporary, output, end - start))
+                writer = PdfWriter()
+                for index in range(start, end):
+                    writer.add_page(reader.pages[index])
+                with temporary.open("wb") as target:
+                    writer.write(target)
+            for temporary, output, _ in prepared:
+                os.replace(temporary, output)
+        finally:
+            for temporary, _, _ in prepared:
+                temporary.unlink(missing_ok=True)
+    return {
+        "ok": True,
+        "pages": total_pages,
+        "outputs": [
+            {"path": str(output), "pages": pages}
+            for _, output, pages in prepared
+        ],
+    }
+
+
+def main(argv=None):
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if len(argv) != 2 or argv[0] not in {"inspect", "split"}:
+        print(json.dumps({"ok": False, "error": "usage: pdf_worker inspect|split request.json"}))
+        return 2
+    try:
+        payload = _load_request(argv[1])
+        response = _inspect(payload) if argv[0] == "inspect" else _split(payload)
+    except Exception as exc:
+        detail = f"{type(exc).__name__}: {exc}"[:2000]
+        print(json.dumps({"ok": False, "error": detail}, ensure_ascii=True))
+        return 2
+    print(json.dumps(response, ensure_ascii=True, separators=(",", ":")))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())