diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 308dbf7..3c8fdee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,8 @@ on: branches: [main] tags: ["v*"] pull_request: + schedule: + - cron: "17 2 * * 1" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -21,6 +23,7 @@ env: jobs: quality: + if: github.event_name != 'schedule' strategy: fail-fast: false matrix: @@ -119,6 +122,7 @@ jobs: run: npm run pack:check clean-install-e2e: + if: github.event_name != 'schedule' strategy: fail-fast: false matrix: @@ -198,3 +202,23 @@ jobs: Add-Content -Path $env:GITHUB_ENV -Value "TYPST_PACKAGE_CACHE_PATH=$packageCache" - name: Run clean npm and Python delivery flow run: python scripts/check_clean_install.py --json + + python-dependency-audit: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python: ["3.10", "3.11"] + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + cache: pip + - run: python -m pip install --upgrade pip + - run: python -m pip install -r requirements-audit.txt + - name: Audit exact Python runtime dependency closure + run: python scripts/audit_python_dependencies.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fbaf2a8..3949fa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ## [Unreleased] +- Python 运行时升级并锁定 Flask 3.1.3 与 pypdf 6.14.2；Windows/Ubuntu CI 使用固定 pip-audit 审计完整依赖闭包，临时例外必须绑定包名、原因、到期日和追踪链接。PDF 页数读取、文本边界扫描与切页迁入受 timeout、内存、输出和进程数限制的独立 worker，损坏或异常 PDF 不再无界占用构建进程或 WebUI 请求线程。 - 非交互 stdout/stderr 及 Checker feedback 改为原子共享单一输出预算，完成、超时、取消和资源超限均在进程树终止后执行确定性公平前缀截断；无法测量或截断时 fail closed，沙箱缓存 Schema 升至 6。stress schema 2 反例以 `E + min(E, 8 MiB)` 限制单次持久化，消除 `generator.out` 输入副本，记录逐文件预算与截断证据，并拒绝重放不完整的 Generator OLE 输入。 - WebUI 完整沙箱与上传评测增加最多 8 个请求线程的有界 HTTP 接入、前置 admission gate、共享固定 worker pool、有界队列、同题进程内互斥与跨进程 Judge 锁，新增结构化 `429 queue_full`、单调排队/执行/整任务 deadline、完整沙箱取消，以及日志、协议输出、结构化结果和完成记录上限；取消与协议错误状态保持单调，上传准备与清理纳入任务生命周期，超时、清理失败与服务关闭不再虚报成功并继续回收完整进程树。 - WebUI 沙箱改为检查已安装 Core 内的 `local_judge.py`，不再要求赛事仓库复制 Judge 运行时；点击运行会先通过当前写入队列完成题面保存，修复沙箱按钮被错误禁用以及旧 `_doSave` 调用导致任务无法入队的问题。 diff --git a/package.json b/package.json index 4ce4e1f..7dd11a5 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "!scripts/webui/tailwind.input.css", "!scripts/check_clean_install.py", "!scripts/check_release.py", + "!scripts/audit_python_dependencies.py", "probhub/**/*.py" ], "devDependencies": { diff --git a/probhub/pdf_processing.py b/probhub/pdf_processing.py new file mode 100644 index 0000000..fd88cf4 --- /dev/null +++ b/probhub/pdf_processing.py @@ -0,0 +1,182 @@ +"""Bounded PDF inspection and extraction through an isolated worker.""" + +import json +import os +import stat +import sys +import tempfile +from pathlib import Path + +from .errors import ProbHubError +from .process_control import OutputBudgetError, run_managed_to_files + + +PACKAGE_ROOT = Path(__file__).resolve().parents[1] +PDF_TIMEOUT_SECONDS = 30 +PDF_UI_TIMEOUT_SECONDS = 10 +PDF_MEMORY_LIMIT_MB = 512 +PDF_OUTPUT_LIMIT_BYTES = 1024 * 1024 +PDF_PROCESS_LIMIT = 4 +PDF_REQUEST_LIMIT_BYTES = 2 * 1024 * 1024 + + +def _regular_file(path, *, label, require_nonempty=True): + try: + path = Path(path) + except (TypeError, ValueError) as exc: + raise ProbHubError( + f"{label} must be a path", + code="pdf_processing_failed", + ) from exc + try: + info = os.lstat(path) + except OSError as exc: + raise ProbHubError(f"{label} is unavailable: {path}: {exc}", code="pdf_processing_failed") from exc + reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + if stat.S_ISLNK(info.st_mode) or ( + reparse_flag and getattr(info, "st_file_attributes", 0) & reparse_flag + ): + raise ProbHubError(f"{label} must not be a link: {path}", code="pdf_processing_failed") + if not stat.S_ISREG(info.st_mode): + raise ProbHubError(f"{label} is not a regular file: {path}", code="pdf_processing_failed") + if require_nonempty and info.st_size <= 0: + raise ProbHubError(f"{label} is empty: {path}", code="pdf_processing_failed") + return path.resolve() + + +def _worker_command(operation, request_path): + return [sys.executable, "-m", "probhub.pdf_worker", operation, str(request_path)] + + +def _worker_error(result, stdout_path, stderr_path): + reason = result.get("reason") + messages = { + "time_limit": "PDF processing timed out", + "memory_limit": "PDF processing exceeded the memory limit", + "output_limit": "PDF processing exceeded the diagnostic output limit", + "process_limit": "PDF processing exceeded the process limit", + } + message = messages.get(reason) + stdout = stdout_path.read_text(encoding="utf-8", errors="replace") if stdout_path.is_file() else "" + stderr = stderr_path.read_text(encoding="utf-8", errors="replace") if stderr_path.is_file() else "" + try: + payload = json.loads(stdout) + except json.JSONDecodeError: + payload = None + if message is None and isinstance(payload, dict) and isinstance(payload.get("error"), str): + message = "PDF processing failed: " + payload["error"][:2000] + if message is None: + detail = stderr.strip()[-2000:] or stdout.strip()[-2000:] + message = result.get("message") or "PDF processing worker failed" + if detail: + message += ": " + detail + return ProbHubError(message, code="pdf_processing_failed") + + +def _run_worker(operation, payload, *, timeout=None): + timeout = PDF_TIMEOUT_SECONDS if timeout is None else float(timeout) + with tempfile.TemporaryDirectory(prefix="probhub-pdf-") as temp: + temp_dir = Path(temp) + request_path = temp_dir / "request.json" + encoded = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + if len(encoded) > PDF_REQUEST_LIMIT_BYTES: + raise ProbHubError("PDF processing request is too large", code="pdf_processing_failed") + request_path.write_bytes(encoded) + stdout_path = temp_dir / "stdout" + stderr_path = temp_dir / "stderr" + try: + result = run_managed_to_files( + _worker_command(operation, request_path), + stdout_path=stdout_path, + stderr_path=stderr_path, + timeout=timeout, + memory_limit_mb=PDF_MEMORY_LIMIT_MB, + output_limit_bytes=PDF_OUTPUT_LIMIT_BYTES, + process_limit=PDF_PROCESS_LIMIT, + cwd=PACKAGE_ROOT, + ) + except (OSError, OutputBudgetError) as exc: + raise ProbHubError( + f"PDF processing worker could not start: {exc}", + code="pdf_processing_failed", + ) from exc + if result.get("reason") != "completed" or result.get("returncode") != 0: + raise _worker_error(result, stdout_path, stderr_path) + try: + response = json.loads(stdout_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ProbHubError( + "PDF processing worker returned invalid JSON", + code="pdf_processing_failed", + ) from exc + if not isinstance(response, dict) or response.get("ok") is not True: + raise ProbHubError( + "PDF processing worker returned an invalid result", + code="pdf_processing_failed", + ) + return response + + +def inspect_pdf(pdf_path, *, scan_text=False, timeout=None): + pdf_path = _regular_file(pdf_path, label="PDF input") + response = _run_worker( + "inspect", + {"input": str(pdf_path), "scan_text": bool(scan_text)}, + timeout=timeout, + ) + pages = response.get("pages") + if isinstance(pages, bool) or not isinstance(pages, int) or pages < 0: + raise ProbHubError("PDF worker returned an invalid page count", code="pdf_processing_failed") + if scan_text: + if not isinstance(response.get("markers"), list) or not isinstance(response.get("legacy"), list): + raise ProbHubError("PDF worker returned invalid boundary data", code="pdf_processing_failed") + return response + + +def pdf_page_count(pdf_path, *, timeout=PDF_UI_TIMEOUT_SECONDS): + return inspect_pdf(pdf_path, timeout=timeout)["pages"] + + +def split_pdf(pdf_path, outputs, *, timeout=None): + pdf_path = _regular_file(pdf_path, label="PDF input") + if not isinstance(outputs, (list, tuple)): + raise ProbHubError("PDF split outputs must be a list", code="pdf_processing_failed") + normalized = [] + seen = set() + for item in outputs: + if not isinstance(item, dict): + raise ProbHubError("PDF split entry must be an object", code="pdf_processing_failed") + try: + raw_output = os.fspath(item.get("path")) + except TypeError as exc: + raise ProbHubError( + "PDF split output path must be a non-empty path", + code="pdf_processing_failed", + ) from exc + if not isinstance(raw_output, str) or not raw_output: + raise ProbHubError( + "PDF split output path must be a non-empty path", + code="pdf_processing_failed", + ) + output = Path(raw_output).resolve() + start = item.get("start") + end = item.get("end") + if output == pdf_path or output in seen: + raise ProbHubError("PDF split outputs must be unique", code="pdf_processing_failed") + if any(isinstance(value, bool) or not isinstance(value, int) for value in (start, end)): + raise ProbHubError("PDF split ranges must be integers", code="pdf_processing_failed") + seen.add(output) + normalized.append({"path": str(output), "start": start, "end": end}) + if not normalized: + return {"ok": True, "pages": inspect_pdf(pdf_path, timeout=timeout)["pages"], "outputs": []} + response = _run_worker( + "split", + {"input": str(pdf_path), "outputs": normalized}, + timeout=timeout, + ) + returned = response.get("outputs") + if not isinstance(returned, list) or len(returned) != len(normalized): + raise ProbHubError("PDF worker returned invalid split results", code="pdf_processing_failed") + for item in normalized: + _regular_file(item["path"], label="PDF split output") + return response diff --git a/probhub/pdf_worker.py b/probhub/pdf_worker.py new file mode 100644 index 0000000..fe5e30e --- /dev/null +++ b/probhub/pdf_worker.py @@ -0,0 +1,139 @@ +"""Isolated pypdf worker. Invoke through probhub.pdf_processing only.""" + +import json +import os +import re +import stat +import sys +import uuid +from pathlib import Path + +from pypdf import PdfReader, PdfWriter + +from .metadata import BOUNDARY_MARKER_PREFIX + + +REQUEST_LIMIT_BYTES = 2 * 1024 * 1024 +BOUNDARY_MARKER_PATTERN = re.compile( + rf"{re.escape(BOUNDARY_MARKER_PREFIX)}[0-9a-f]{{64}}" +) +LEGACY_HEADING_PATTERN = re.compile(r"题目\s+[A-Z]+\.\s*(.+)") + + +def _regular_file(path, *, label): + path = Path(path) + info = os.lstat(path) + reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + if stat.S_ISLNK(info.st_mode) or ( + reparse_flag and getattr(info, "st_file_attributes", 0) & reparse_flag + ): + raise ValueError(f"{label} must not be a link") + if not stat.S_ISREG(info.st_mode) or info.st_size <= 0: + raise ValueError(f"{label} must be a non-empty regular file") + return path.resolve() + + +def _load_request(path): + path = _regular_file(path, label="request") + if path.stat().st_size > REQUEST_LIMIT_BYTES: + raise ValueError("request is too large") + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError("request must be an object") + return payload + + +def _inspect(payload): + input_path = _regular_file(payload.get("input", ""), label="PDF input") + scan_text = payload.get("scan_text", False) + if not isinstance(scan_text, bool): + raise ValueError("scan_text must be boolean") + markers = [] + legacy = [] + with input_path.open("rb") as stream: + reader = PdfReader(stream) + pages = len(reader.pages) + if scan_text: + for index, page in enumerate(reader.pages): + text = page.extract_text() or "" + markers.extend( + {"marker": marker, "page": index + 1} + for marker in BOUNDARY_MARKER_PATTERN.findall(text) + ) + for line in text.splitlines(): + match = LEGACY_HEADING_PATTERN.search(line.strip()) + if match: + legacy.append({"display_name": match.group(1).strip(), "page": index + 1}) + break + return {"ok": True, "pages": pages, "markers": markers, "legacy": legacy} + + +def _split(payload): + input_path = _regular_file(payload.get("input", ""), label="PDF input") + entries = payload.get("outputs") + if not isinstance(entries, list) or not entries: + raise ValueError("outputs must be a non-empty list") + normalized = [] + seen = set() + with input_path.open("rb") as stream: + reader = PdfReader(stream) + total_pages = len(reader.pages) + for item in entries: + if not isinstance(item, dict): + raise ValueError("split entry must be an object") + output = Path(item.get("path", "")).resolve() + start = item.get("start") + end = item.get("end") + if output == input_path or output in seen: + raise ValueError("split outputs must be unique") + if any(isinstance(value, bool) or not isinstance(value, int) for value in (start, end)): + raise ValueError("split ranges must be integers") + if start < 0 or end <= start or end > total_pages: + raise ValueError("split range is outside the PDF") + seen.add(output) + normalized.append((output, start, end)) + + prepared = [] + try: + for output, start, end in normalized: + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_name(f".{output.name}.probhub-{uuid.uuid4().hex}.tmp") + prepared.append((temporary, output, end - start)) + writer = PdfWriter() + for index in range(start, end): + writer.add_page(reader.pages[index]) + with temporary.open("wb") as target: + writer.write(target) + for temporary, output, _ in prepared: + os.replace(temporary, output) + finally: + for temporary, _, _ in prepared: + temporary.unlink(missing_ok=True) + return { + "ok": True, + "pages": total_pages, + "outputs": [ + {"path": str(output), "pages": pages} + for _, output, pages in prepared + ], + } + + +def main(argv=None): + argv = list(sys.argv[1:] if argv is None else argv) + if len(argv) != 2 or argv[0] not in {"inspect", "split"}: + print(json.dumps({"ok": False, "error": "usage: pdf_worker inspect|split request.json"})) + return 2 + try: + payload = _load_request(argv[1]) + response = _inspect(payload) if argv[0] == "inspect" else _split(payload) + except Exception as exc: + detail = f"{type(exc).__name__}: {exc}"[:2000] + print(json.dumps({"ok": False, "error": detail}, ensure_ascii=True)) + return 2 + print(json.dumps(response, ensure_ascii=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/probhub/typesetting.py b/probhub/typesetting.py index 72a41b9..8fd11cb 100644 --- a/probhub/typesetting.py +++ b/probhub/typesetting.py @@ -3,25 +3,19 @@ import uuid from pathlib import Path -from pypdf import PdfReader, PdfWriter - from .errors import ProbHubError from .metadata import ( - BOUNDARY_MARKER_PREFIX, normalize_display_name, problem_boundary_marker, write_typst_collection, ) +from .pdf_processing import inspect_pdf, split_pdf from .process_control import run_managed_to_files TYPST_TIMEOUT_SECONDS = 120 TYPST_OUTPUT_LIMIT_BYTES = 4 * 1024 * 1024 TYPST_TEMP_SOURCE_PREFIX = ".probhub-" -BOUNDARY_MARKER_PATTERN = re.compile( - rf"{re.escape(BOUNDARY_MARKER_PREFIX)}[0-9a-f]{{64}}" -) -LEGACY_HEADING_PATTERN = re.compile(r"题目\s+[A-Z]+\.\s*(.+)") TYPST_BOUNDARY_FIELD_PATTERN = re.compile( r"(?:\.\s*boundary_marker\b|at\(\s*[\"']boundary_marker[\"']\s*\))" ) @@ -238,21 +232,9 @@ def _validate_legacy_boundaries(boundaries, expected): return validated -def problem_boundaries(pdf_path, loaded_problems=None): - reader = PdfReader(pdf_path) - markers = [] - legacy = [] - for index, page in enumerate(reader.pages): - text = page.extract_text() or "" - markers.extend( - {"marker": marker, "page": index + 1} - for marker in BOUNDARY_MARKER_PATTERN.findall(text) - ) - for line in text.splitlines(): - match = LEGACY_HEADING_PATTERN.search(line.strip()) - if match: - legacy.append({"display_name": match.group(1).strip(), "page": index + 1}) - break +def _boundaries_from_inspection(inspection, loaded_problems=None): + markers = inspection["markers"] + legacy = inspection["legacy"] if markers: if loaded_problems is None: return markers @@ -262,6 +244,13 @@ def problem_boundaries(pdf_path, loaded_problems=None): return _validate_legacy_boundaries(legacy, _expected_boundaries(loaded_problems)) +def problem_boundaries(pdf_path, loaded_problems=None): + return _boundaries_from_inspection( + inspect_pdf(pdf_path, scan_text=True), + loaded_problems, + ) + + def compile_collection(root, workspace, loaded_problems): typst = workspace.get("typst") or {} typst_dir, problems = write_typst_collection(root, workspace, loaded_problems) @@ -304,11 +293,12 @@ def compile_collection(root, workspace, loaded_problems): def extract_problem_pdfs(main_pdf, loaded_problems, only_ids=None): - boundaries = problem_boundaries(main_pdf, loaded_problems) + inspection = inspect_pdf(main_pdf, scan_text=True) + boundaries = _boundaries_from_inspection(inspection, loaded_problems) if not boundaries: raise ProbHubError("no problem headings found in compiled PDF") - reader = PdfReader(main_pdf) outputs = {} + split_plan = [] boundary_indexes = {item["id"]: index for index, item in enumerate(boundaries)} for problem_dir, config in loaded_problems: problem_id = config["id"] @@ -321,13 +311,14 @@ def extract_problem_pdfs(main_pdf, loaded_problems, only_ids=None): boundaries[index]["page"], boundaries[index + 1]["page"] if index + 1 < len(boundaries) - else len(reader.pages) + 1, + else inspection["pages"] + 1, ) - writer = PdfWriter() - for page_number in range(found[0] - 1, found[1] - 1): - writer.add_page(reader.pages[page_number]) output = problem_dir / "problem.pdf" - with output.open("wb") as stream: - writer.write(stream) + split_plan.append({ + "path": output, + "start": found[0] - 1, + "end": found[1] - 1, + }) outputs[problem_id] = {"path": str(output), "pages": found[1] - found[0]} + split_pdf(main_pdf, split_plan) return outputs diff --git a/references/legacy-workflow.md b/references/legacy-workflow.md index 4f77748..3692b3a 100644 --- a/references/legacy-workflow.md +++ b/references/legacy-workflow.md @@ -68,7 +68,7 @@ `python scripts/extract_new_problem.py “typst-statement/” “<英文目录名>”` 2. 脚本自动选择提取模式： - **新增题目**（该题在 `problems.json` 中为最后一题）→ **页数差值模式**：记录编译前 `main.pdf` 的页数，编译后计算差值 `x`，提取最后 `x` 页。首次编译时自动扣除封面和空白页（2 页）。 - - **修改旧题**（该题不是最后一题）→ **PDF 文本扫描模式**：编译后用 `pypdf` 扫描每页文本，搜索”题目 X. {题名}”标题建立页码映射，精确裁剪目标页码范围。 + - **修改旧题**（该题不是最后一题）→ **PDF 文本扫描模式**：编译后由受控 `pypdf` worker 扫描每页文本，搜索”题目 X. {题名}”标题建立页码映射，精确裁剪目标页码范围。 3. 观察脚本输出：会显示使用的模式、页码范围或差值信息。 4. 如果脚本执行成功，提示用户检查 `<英文目录名>/problem.pdf`，确认题目页数和内容是否无误。如果脚本报错”未找到题目标题”（文本扫描模式）或”页数 <= 0”（差值模式），你需要检查 Typst 语法或 `display_name` 匹配情况并自行 Debug。 diff --git a/references/process-control.md b/references/process-control.md index 8a4a588..1c15bf8 100644 --- a/references/process-control.md +++ b/references/process-control.md @@ -31,6 +31,7 @@ limits: - Validator； - C++ 编译器及其后代； - stress 的 Generator、Validator、accepted、brute、Checker 和编译器。 +- Core 与 WebUI 的 pypdf 页数读取、边界扫描和切页 worker。 所有超时、超限、异常和正常退出路径都会回收直接进程并清理后代。若父进程创建后台子进程后先正常退出，ProbHub 仍会终止遗留后代，避免污染下一测试点、占用文件或持续消耗 CPU。 @@ -105,6 +106,10 @@ Flask 多线程请求和 CLI 都不会在父进程中使用 Python `preexec_fn` 这一区分对出题自检很重要：官方工具失败必须修复题目基础设施，不能作为错解“被击杀”的证据。 +PDF 解析不会在 CLI 主构建进程或 Flask 请求线程内直接运行。Core 通过独立 Python worker 调用固定版本 pypdf，默认限制为 30 秒、512 MiB、1 MiB stdout/stderr 共享预算和 4 个进程；WebUI 页数检查使用 10 秒 deadline，页面渲染继续由受控 Poppler 进程完成。worker 超时、内存/输出/进程超限、损坏 JSON、链接输入、畸形 PDF 或缺失切页输出统一返回 `pdf_processing_failed`；正式 build/typeset 仍在 staging 中处理，失败不会覆盖最后正确产物。 + +这层隔离用于避免损坏 PDF 无界占用本地出题流程，不是面向任意敌意文件的强安全容器。正式运行时依赖闭包由 `requirements.txt` 逐项锁定，仓库 CI 在 Windows 与 Ubuntu 上按精确版本审计，并每周自动复查一次。 + ## 8. Stress `probhub stress` 的每个阶段都使用相同底层控制： diff --git a/requirements-audit.txt b/requirements-audit.txt new file mode 100644 index 0000000..684087b --- /dev/null +++ b/requirements-audit.txt @@ -0,0 +1 @@ +pip-audit==2.10.1 diff --git a/requirements.txt b/requirements.txt index 4a0307f..2d3b114 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,11 @@ -Flask==3.1.1 +Flask==3.1.3 +blinker==1.9.0 +click==8.4.2 +colorama==0.4.6; platform_system == "Windows" +itsdangerous==2.2.0 +Jinja2==3.1.6 +MarkupSafe==3.0.3 PyYAML==6.0.3 -pypdf==6.12.1 +pypdf==6.14.2 +typing_extensions==4.16.0; python_version < "3.11" +Werkzeug==3.1.8 diff --git a/scripts/audit_python_dependencies.py b/scripts/audit_python_dependencies.py new file mode 100644 index 0000000..56765a1 --- /dev/null +++ b/scripts/audit_python_dependencies.py @@ -0,0 +1,188 @@ +"""Audit the pinned Python dependency closure with explicit exceptions only.""" + +import argparse +import json +import sys +from datetime import date +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +ROOT = SCRIPT_DIR.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.check_release import run_bounded + + +DEFAULT_REQUIREMENTS = ROOT / "requirements.txt" +DEFAULT_EXCEPTIONS = ROOT / "security/python-audit-exceptions.json" +AUDIT_TIMEOUT_SECONDS = 180 +AUDIT_OUTPUT_LIMIT_BYTES = 4 * 1024 * 1024 + + +class DependencyAuditError(RuntimeError): + pass + + +def load_exceptions(path, *, today=None): + path = Path(path) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise DependencyAuditError(f"cannot read dependency audit exceptions: {exc}") from exc + if not isinstance(payload, dict) or payload.get("schema_version") != 1: + raise DependencyAuditError("dependency audit exceptions must use schema_version 1") + entries = payload.get("exceptions") + if not isinstance(entries, list): + raise DependencyAuditError("dependency audit exceptions must be a list") + + today = today or date.today() + identifiers = set() + validated = [] + for index, entry in enumerate(entries): + label = f"dependency audit exception {index + 1}" + if not isinstance(entry, dict): + raise DependencyAuditError(f"{label} must be an object") + required = ("id", "package", "reason", "expires", "tracking_url") + missing = [name for name in required if not isinstance(entry.get(name), str) or not entry[name].strip()] + if missing: + raise DependencyAuditError(f"{label} is missing: {', '.join(missing)}") + identifier = entry["id"].strip() + if identifier in identifiers: + raise DependencyAuditError(f"duplicate dependency audit exception: {identifier}") + identifiers.add(identifier) + try: + expires = date.fromisoformat(entry["expires"].strip()) + except ValueError as exc: + raise DependencyAuditError(f"{label} has an invalid expires date") from exc + if expires < today: + raise DependencyAuditError(f"dependency audit exception expired: {identifier}") + tracking_url = entry["tracking_url"].strip() + if not tracking_url.startswith(("https://", "http://")): + raise DependencyAuditError(f"{label} tracking_url must be HTTP(S)") + validated.append({**entry, "id": identifier}) + return validated + + +def audit_command(requirements): + return [ + sys.executable, + "-m", + "pip_audit", + "--requirement", + str(Path(requirements)), + "--no-deps", + "--disable-pip", + "--strict", + "--format", + "json", + "--desc", + "off", + "--aliases", + "on", + "--progress-spinner", + "off", + "--timeout", + "30", + ] + + +def evaluate_audit(payload, exceptions): + dependencies = payload.get("dependencies") if isinstance(payload, dict) else None + if not isinstance(dependencies, list): + raise DependencyAuditError("Python dependency audit returned an unexpected schema") + matched = set() + unhandled = [] + vulnerability_count = 0 + for dependency in dependencies: + if not isinstance(dependency, dict) or not isinstance(dependency.get("name"), str): + raise DependencyAuditError("Python dependency audit returned an invalid dependency") + package = dependency["name"].casefold() + vulnerabilities = dependency.get("vulns") + if not isinstance(vulnerabilities, list): + raise DependencyAuditError("Python dependency audit returned invalid vulnerabilities") + for vulnerability in vulnerabilities: + if not isinstance(vulnerability, dict) or not isinstance(vulnerability.get("id"), str): + raise DependencyAuditError("Python dependency audit returned an invalid vulnerability") + vulnerability_count += 1 + identifiers = {vulnerability["id"], *(vulnerability.get("aliases") or [])} + accepted = None + for entry in exceptions: + if entry["id"] in identifiers and entry["package"].casefold() == package: + accepted = entry + break + if accepted is None: + fixes = ", ".join(vulnerability.get("fix_versions") or []) or "none published" + unhandled.append(f"{dependency['name']} {vulnerability['id']} (fix: {fixes})") + else: + matched.add(accepted["id"]) + stale = sorted(entry["id"] for entry in exceptions if entry["id"] not in matched) + if unhandled: + raise DependencyAuditError( + "unhandled Python dependency vulnerabilities: " + "; ".join(unhandled) + ) + if stale: + raise DependencyAuditError( + "dependency audit exception no longer matches an applicable vulnerability: " + + ", ".join(stale) + ) + return len(dependencies), vulnerability_count + + +def run_audit(requirements=DEFAULT_REQUIREMENTS, exceptions_path=DEFAULT_EXCEPTIONS): + requirements = Path(requirements) + if not requirements.is_file(): + raise DependencyAuditError(f"requirements file is missing: {requirements}") + exceptions = load_exceptions(exceptions_path) + result = run_bounded( + audit_command(requirements), + cwd=ROOT, + timeout=AUDIT_TIMEOUT_SECONDS, + output_limit=AUDIT_OUTPUT_LIMIT_BYTES, + ) + if result["reason"] != "completed" or result["returncode"] not in {0, 1}: + detail = (result["stdout"] + "\n" + result["stderr"]).strip()[-12000:] + raise DependencyAuditError( + "Python dependency audit failed " + f"({result['returncode']} / {result['reason']}): {detail or result.get('message') or 'no diagnostics'}" + ) + try: + payload = json.loads(result["stdout"]) + except json.JSONDecodeError as exc: + if result["returncode"]: + detail = (result["stdout"] + "\n" + result["stderr"]).strip()[-12000:] + raise DependencyAuditError( + "Python dependency audit failed " + f"({result['returncode']} / completed): {detail or 'invalid or empty JSON output'}" + ) from exc + raise DependencyAuditError("Python dependency audit returned invalid JSON") from exc + dependencies, vulnerability_count = evaluate_audit(payload, exceptions) + if result["returncode"] == 1 and vulnerability_count == 0: + raise DependencyAuditError( + "Python dependency audit exited with status 1 but reported no vulnerabilities" + ) + return { + "ok": True, + "requirements": str(requirements), + "exceptions": [entry["id"] for entry in exceptions], + "dependencies": dependencies, + "vulnerabilities": vulnerability_count, + } + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--requirements", type=Path, default=DEFAULT_REQUIREMENTS) + parser.add_argument("--exceptions", type=Path, default=DEFAULT_EXCEPTIONS) + args = parser.parse_args(argv) + try: + payload = run_audit(args.requirements, args.exceptions) + except DependencyAuditError as exc: + print(json.dumps({"ok": False, "error": str(exc)}, ensure_ascii=False)) + return 1 + print(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_release.py b/scripts/check_release.py index c0e5ed5..9a13318 100644 --- a/scripts/check_release.py +++ b/scripts/check_release.py @@ -212,6 +212,7 @@ def validate_pack_inventories(*, dry_run=True, destination=None): if ( "__pycache__" in parts or ".probhub" in parts + or lowered == "scripts/audit_python_dependencies.py" or lowered.endswith((".pyc", ".pyo", ".exe", ".o", ".obj", ".zip")) or lowered.endswith("problem.pdf") ): diff --git a/scripts/extract_new_problem.py b/scripts/extract_new_problem.py index 9ea724f..9d25a89 100644 --- a/scripts/extract_new_problem.py +++ b/scripts/extract_new_problem.py @@ -3,7 +3,14 @@ import os import json import subprocess -from pypdf import PdfReader, PdfWriter +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +PACKAGE_ROOT = SCRIPT_DIR.parent +if str(PACKAGE_ROOT) not in sys.path: + sys.path.insert(0, str(PACKAGE_ROOT)) + +from probhub.pdf_processing import inspect_pdf, split_pdf BOILERPLATE_PAGES = 2 # cover + blank page @@ -16,15 +23,7 @@ def get_page_count(pdf_path): """Return page count of a PDF, or 0 if file doesn't exist.""" if not os.path.exists(pdf_path): return 0 - return len(PdfReader(pdf_path).pages) - -def extract_text_from_page(reader, page_idx): - """Extract text from a single PDF page.""" - try: - text = reader.pages[page_idx].extract_text() - return text if text else "" - except Exception: - return "" + return inspect_pdf(pdf_path)["pages"] def find_problem_boundaries(pdf_path): """ @@ -32,23 +31,7 @@ def find_problem_boundaries(pdf_path): for each problem found, ordered by page number. Problems are identified by the heading pattern "题目 X. NAME". """ - reader = PdfReader(pdf_path) - found = [] - - for i in range(len(reader.pages)): - text = extract_text_from_page(reader, i) - if not text: - continue - for line in text.split('\n'): - line = line.strip() - if line.startswith("题目 ") and ". " in line: - dot_idx = line.index(". ") - display_name = line[dot_idx + 2:].strip() - if display_name: - found.append({"display_name": display_name, "page": i + 1}) - break - - return found + return inspect_pdf(pdf_path, scan_text=True)["legacy"] def is_last_problem(typst_dir, target_name): """Check if target problem is the last one in problems.json.""" @@ -84,14 +67,12 @@ def extract_by_page_count(main_pdf_path, output_pdf_path, prev_pages): print("[*] Page count unchanged, falling back to text-scan mode...") return False - reader = PdfReader(main_pdf_path) - writer = PdfWriter() - for i in range(new_pages - x, new_pages): - writer.add_page(reader.pages[i]) - os.makedirs(os.path.dirname(output_pdf_path) or ".", exist_ok=True) - with open(output_pdf_path, "wb") as f: - writer.write(f) + split_pdf(main_pdf_path, [{ + "path": output_pdf_path, + "start": new_pages - x, + "end": new_pages, + }]) print(f"[+] Extracted last {x} pages -> {output_pdf_path}") return True @@ -123,18 +104,16 @@ def extract_by_text_scan(main_pdf_path, output_pdf_path, target_name): print(f"[-] Target problem '{target_name}' not found in PDF headings.") sys.exit(1) - reader = PdfReader(main_pdf_path) - total_pages = len(reader.pages) + total_pages = get_page_count(main_pdf_path) if end_page == -1: end_page = total_pages + 1 - writer = PdfWriter() - for page_num in range(start_page - 1, end_page - 1): - writer.add_page(reader.pages[page_num]) - os.makedirs(os.path.dirname(output_pdf_path) or ".", exist_ok=True) - with open(output_pdf_path, "wb") as f: - writer.write(f) + split_pdf(main_pdf_path, [{ + "path": output_pdf_path, + "start": start_page - 1, + "end": end_page - 1, + }]) page_count = end_page - start_page print(f"[+] Extracted! ({page_count} pages, P{start_page} - P{end_page - 1}) -> {output_pdf_path}") diff --git a/scripts/ui.py b/scripts/ui.py index 1d9dbfa..3b041a7 100644 --- a/scripts/ui.py +++ b/scripts/ui.py @@ -36,6 +36,7 @@ from probhub.build_lock import workspace_build_lock from probhub.building import build_workspace, create_build_plan, create_build_snapshot from probhub.errors import ProbHubError +from probhub.pdf_processing import pdf_page_count as bounded_pdf_page_count from probhub.process_control import ( run_managed_to_files, spawn_managed, @@ -2174,14 +2175,12 @@ def save_contest_config(subtitle): @app.route('/api/pdf-pages/') def pdf_page_count(subtitle): """Return the number of pages in main.pdf.""" - import pypdf preview_path = _preview_pdf_path(subtitle) pdf_path = str(preview_path if preview_path.is_file() else Path(secure_path(subtitle, "main.pdf"))) if not os.path.exists(pdf_path): return jsonify({"pages": 0}) try: - reader = pypdf.PdfReader(pdf_path) - return jsonify({"pages": len(reader.pages)}) + return jsonify({"pages": bounded_pdf_page_count(pdf_path)}) except Exception: return jsonify({"pages": 0}) @@ -2190,7 +2189,6 @@ def pdf_page_count(subtitle): def serve_pdf_page(subtitle, page): """Render a single PDF page through Poppler into the process temp cache.""" from flask import send_file - import pypdf preview_path = _preview_pdf_path(subtitle) pdf_path = str(preview_path if preview_path.is_file() else Path(secure_path(subtitle, "main.pdf"))) @@ -2199,8 +2197,7 @@ def serve_pdf_page(subtitle, page): # Validate page number try: - reader = pypdf.PdfReader(pdf_path) - total = len(reader.pages) + total = bounded_pdf_page_count(pdf_path) if page < 0 or page >= total: return "Page out of range", 404 except Exception: diff --git a/security/README.md b/security/README.md new file mode 100644 index 0000000..2557c4b --- /dev/null +++ b/security/README.md @@ -0,0 +1,17 @@ +# Python dependency audit + +`requirements.txt` pins the exact runtime dependency closure shipped with the npm package, including Flask's transitive runtime dependencies. CI installs the separately pinned `pip-audit` version from `requirements-audit.txt` and audits exactly those listed versions with dependency resolution disabled. The audit tool's own CI-only dependencies do not ship in the npm package. + +The audit runs on every pull request, push, release tag, and on a weekly schedule on both Windows and Ubuntu. Scheduled runs skip the full quality and clean-install suites. + +The audit fails when the advisory service is unavailable, dependency collection is incomplete, output is invalid, or any applicable vulnerability is not reviewed. It does not silently downgrade network or tool failures to warnings. + +Temporary exceptions belong in `python-audit-exceptions.json`. Every entry must contain: + +- `id`: the primary or alias vulnerability identifier; +- `package`: the affected package name; +- `reason`: why an immediate upgrade is not currently possible; +- `expires`: an ISO `YYYY-MM-DD` review deadline; +- `tracking_url`: an HTTP(S) issue or advisory URL. + +Expired, duplicate, package-mismatched, and no-longer-applicable exceptions fail closed. Keep the list empty whenever possible. diff --git a/security/python-audit-exceptions.json b/security/python-audit-exceptions.json new file mode 100644 index 0000000..18a7268 --- /dev/null +++ b/security/python-audit-exceptions.json @@ -0,0 +1,4 @@ +{ + "schema_version": 1, + "exceptions": [] +} diff --git a/tests/test_dependency_audit.py b/tests/test_dependency_audit.py new file mode 100644 index 0000000..f24863c --- /dev/null +++ b/tests/test_dependency_audit.py @@ -0,0 +1,231 @@ +import importlib.util +import json +import tempfile +import unittest +from datetime import date +from pathlib import Path +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "audit_python_dependencies", + ROOT / "scripts/audit_python_dependencies.py", +) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class DependencyAuditTests(unittest.TestCase): + def audit_result(self, *, returncode=0, reason="completed", stdout=None, stderr="", message=None): + if stdout is None: + stdout = json.dumps({"dependencies": []}) + return { + "returncode": returncode, + "reason": reason, + "stdout": stdout, + "stderr": stderr, + "message": message, + } + + def write_exceptions(self, root, entries): + path = Path(root) / "exceptions.json" + path.write_text( + json.dumps({"schema_version": 1, "exceptions": entries}), + encoding="utf-8", + ) + return path + + def valid_entry(self, **changes): + entry = { + "id": "PYSEC-2099-1", + "package": "example", + "reason": "Upgrade is blocked by an upstream compatibility issue.", + "expires": "2099-12-31", + "tracking_url": "https://example.invalid/issues/1", + } + entry.update(changes) + return entry + + def test_empty_checked_in_exception_file_is_valid(self): + self.assertEqual(MODULE.load_exceptions(MODULE.DEFAULT_EXCEPTIONS), []) + + def test_exception_requires_reason_expiry_and_tracking_url(self): + with tempfile.TemporaryDirectory() as temp: + for changes in ( + {"reason": ""}, + {"expires": "not-a-date"}, + {"tracking_url": "issue-1"}, + ): + with self.subTest(changes=changes): + path = self.write_exceptions(temp, [self.valid_entry(**changes)]) + with self.assertRaises(MODULE.DependencyAuditError): + MODULE.load_exceptions(path, today=date(2099, 1, 1)) + + def test_expired_and_duplicate_exceptions_fail_closed(self): + with tempfile.TemporaryDirectory() as temp: + expired = self.write_exceptions( + temp, + [self.valid_entry(expires="2020-01-01")], + ) + with self.assertRaisesRegex(MODULE.DependencyAuditError, "expired"): + MODULE.load_exceptions(expired, today=date(2020, 1, 2)) + + duplicate = self.write_exceptions( + temp, + [self.valid_entry(), self.valid_entry(package="other")], + ) + with self.assertRaisesRegex(MODULE.DependencyAuditError, "duplicate"): + MODULE.load_exceptions(duplicate, today=date(2099, 1, 1)) + + def test_audit_command_does_not_hide_vulnerabilities_from_evaluation(self): + command = MODULE.audit_command(ROOT / "requirements.txt") + self.assertNotIn("--ignore-vuln", command) + self.assertIn("--strict", command) + self.assertIn("--no-deps", command) + self.assertIn("--disable-pip", command) + + def test_requirements_pin_the_complete_runtime_closure(self): + requirements = (ROOT / "requirements.txt").read_text(encoding="utf-8").splitlines() + packages = { + line.split("==", 1)[0].casefold() + for line in requirements + if line.strip() and not line.lstrip().startswith("#") + } + self.assertEqual( + packages, + { + "blinker", + "click", + "colorama", + "flask", + "itsdangerous", + "jinja2", + "markupsafe", + "pypdf", + "pyyaml", + "typing_extensions", + "werkzeug", + }, + ) + self.assertTrue(all("==" in line for line in requirements if line.strip())) + self.assertIn( + 'typing_extensions==4.16.0; python_version < "3.11"', + requirements, + ) + + def test_exception_must_match_vulnerability_and_package(self): + payload = { + "dependencies": [{ + "name": "Example", + "version": "1.0", + "vulns": [{ + "id": "PYSEC-2099-1", + "aliases": ["GHSA-test-0000-0000"], + "fix_versions": ["2.0"], + }], + }], + } + self.assertEqual( + MODULE.evaluate_audit(payload, [self.valid_entry()]), + (1, 1), + ) + with self.assertRaisesRegex(MODULE.DependencyAuditError, "unhandled"): + MODULE.evaluate_audit(payload, [self.valid_entry(package="other")]) + + def test_stale_exception_fails_closed(self): + payload = {"dependencies": [{"name": "example", "version": "2.0", "vulns": []}]} + with self.assertRaisesRegex(MODULE.DependencyAuditError, "no longer matches"): + MODULE.evaluate_audit(payload, [self.valid_entry()]) + + def test_run_audit_accepts_clean_result(self): + with patch.object(MODULE, "run_bounded", return_value=self.audit_result()): + result = MODULE.run_audit() + self.assertTrue(result["ok"]) + self.assertEqual(result["dependencies"], 0) + self.assertEqual(result["vulnerabilities"], 0) + + def test_run_audit_fails_closed_on_process_failures(self): + results = ( + self.audit_result(returncode=None, reason="time_limit", stdout="", message="timed out"), + self.audit_result(returncode=None, reason="output_limit", stdout="", message="too much output"), + self.audit_result(returncode=2, stdout="", stderr="tool failure"), + ) + for result in results: + with self.subTest(result=result): + with ( + patch.object(MODULE, "run_bounded", return_value=result), + self.assertRaisesRegex(MODULE.DependencyAuditError, "audit failed"), + ): + MODULE.run_audit() + + def test_run_audit_reports_network_failure_with_invalid_json(self): + result = self.audit_result( + returncode=1, + stdout="", + stderr="advisory service unavailable", + ) + with ( + patch.object(MODULE, "run_bounded", return_value=result), + self.assertRaisesRegex(MODULE.DependencyAuditError, "advisory service unavailable"), + ): + MODULE.run_audit() + + def test_run_audit_rejects_malformed_success_json(self): + result = self.audit_result(stdout="not-json") + with ( + patch.object(MODULE, "run_bounded", return_value=result), + self.assertRaisesRegex(MODULE.DependencyAuditError, "invalid JSON"), + ): + MODULE.run_audit() + + def test_run_audit_rejects_unhandled_vulnerability(self): + payload = { + "dependencies": [{ + "name": "example", + "version": "1.0", + "vulns": [{ + "id": "PYSEC-2099-1", + "aliases": [], + "fix_versions": ["2.0"], + }], + }], + } + result = self.audit_result(returncode=1, stdout=json.dumps(payload)) + with ( + patch.object(MODULE, "run_bounded", return_value=result), + self.assertRaisesRegex(MODULE.DependencyAuditError, "unhandled"), + ): + MODULE.run_audit() + + def test_run_audit_accepts_exit_one_for_reviewed_vulnerability(self): + payload = { + "dependencies": [{ + "name": "example", + "version": "1.0", + "vulns": [{ + "id": "PYSEC-2099-1", + "aliases": [], + "fix_versions": [], + }], + }], + } + result = self.audit_result(returncode=1, stdout=json.dumps(payload)) + with tempfile.TemporaryDirectory() as temp: + exceptions = self.write_exceptions(temp, [self.valid_entry()]) + with patch.object(MODULE, "run_bounded", return_value=result): + audited = MODULE.run_audit(exceptions_path=exceptions) + self.assertTrue(audited["ok"]) + self.assertEqual(audited["vulnerabilities"], 1) + + def test_run_audit_rejects_exit_one_without_reported_vulnerability(self): + result = self.audit_result(returncode=1) + with ( + patch.object(MODULE, "run_bounded", return_value=result), + self.assertRaisesRegex(MODULE.DependencyAuditError, "reported no vulnerabilities"), + ): + MODULE.run_audit() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_extract_new_problem.py b/tests/test_extract_new_problem.py index 439bdaa..2de0e8f 100644 --- a/tests/test_extract_new_problem.py +++ b/tests/test_extract_new_problem.py @@ -1,7 +1,10 @@ import importlib.util +import tempfile import unittest from pathlib import Path +from pypdf import PdfReader, PdfWriter + ROOT = Path(__file__).resolve().parents[1] SPEC = importlib.util.spec_from_file_location("extract_new_problem", ROOT / "scripts" / "extract_new_problem.py") MODULE = importlib.util.module_from_spec(SPEC) @@ -12,6 +15,20 @@ class DisplayNameTests(unittest.TestCase): def test_pdf_inserted_spaces_do_not_break_matching(self): self.assertEqual(MODULE.normalize_display_name("同步棋路 1"), MODULE.normalize_display_name("同步棋路1")) + def test_page_count_extraction_uses_pdf_worker(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / "main.pdf" + output = root / "problem.pdf" + writer = PdfWriter() + for _ in range(3): + writer.add_blank_page(width=200, height=200) + with source.open("wb") as stream: + writer.write(stream) + + self.assertTrue(MODULE.extract_by_page_count(source, output, 1)) + self.assertEqual(len(PdfReader(output).pages), 2) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_npm_packages.py b/tests/test_npm_packages.py index d7c35f3..fbde652 100644 --- a/tests/test_npm_packages.py +++ b/tests/test_npm_packages.py @@ -41,6 +41,7 @@ def test_main_and_compatibility_packages_share_exact_version(self): self.assertIn("--require-head-tag", compat["scripts"]["prepublishOnly"]) self.assertIn("CHANGELOG.md", main["files"]) self.assertIn("scripts/webui/**", main["files"]) + self.assertIn("!scripts/audit_python_dependencies.py", main["files"]) def test_both_packages_expose_cli_and_skill_installer(self): expected_bins = { diff --git a/tests/test_pdf_processing.py b/tests/test_pdf_processing.py new file mode 100644 index 0000000..8d5a622 --- /dev/null +++ b/tests/test_pdf_processing.py @@ -0,0 +1,151 @@ +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest import mock + +from pypdf import PdfReader, PdfWriter + +from probhub.errors import ProbHubError +from probhub.pdf_processing import inspect_pdf, split_pdf +from probhub.process_control import process_alive +from probhub import pdf_worker + + +class PdfProcessingTests(unittest.TestCase): + def make_pdf(self, path, pages=3): + writer = PdfWriter() + for _ in range(pages): + writer.add_blank_page(width=200, height=200) + with Path(path).open("wb") as stream: + writer.write(stream) + + def test_inspect_and_split_run_in_bounded_worker(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / "source.pdf" + first = root / "first.pdf" + second = root / "second.pdf" + self.make_pdf(source) + + inspected = inspect_pdf(source) + self.assertEqual(inspected["pages"], 3) + split = split_pdf(source, [ + {"path": first, "start": 0, "end": 1}, + {"path": second, "start": 1, "end": 3}, + ]) + + self.assertTrue(split["ok"]) + self.assertEqual(len(PdfReader(first).pages), 1) + self.assertEqual(len(PdfReader(second).pages), 2) + + def test_malformed_pdf_fails_with_stable_error(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "malformed.pdf" + path.write_bytes(b"%PDF-1.7\nmalformed\n%%EOF\n") + with self.assertRaises(ProbHubError) as raised: + inspect_pdf(path, scan_text=True) + self.assertEqual(raised.exception.code, "pdf_processing_failed") + + def test_worker_timeout_terminates_the_process_tree(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + path = root / "input.pdf" + child_pid_path = root / "child.pid" + path.write_bytes(b"not parsed by injected worker") + child_code = ( + "import os,pathlib,time;" + f"pathlib.Path({str(child_pid_path)!r}).write_text(str(os.getpid()),encoding='utf-8');" + "time.sleep(30)" + ) + parent_code = ( + "import subprocess,sys,time;" + f"subprocess.Popen([sys.executable,'-c',{child_code!r}]);" + "time.sleep(30)" + ) + command = [sys.executable, "-c", parent_code] + with mock.patch( + "probhub.pdf_processing._worker_command", + return_value=command, + ): + with self.assertRaises(ProbHubError) as raised: + inspect_pdf(path, timeout=1.0) + self.assertEqual(raised.exception.code, "pdf_processing_failed") + self.assertIn("timed out", str(raised.exception)) + self.assertTrue(child_pid_path.is_file(), "worker child did not start") + child_pid = int(child_pid_path.read_text(encoding="utf-8")) + deadline = time.time() + 5 + while process_alive(child_pid) and time.time() < deadline: + time.sleep(0.05) + self.assertFalse(process_alive(child_pid), f"PDF worker child {child_pid} survived") + + def test_split_rejects_invalid_ranges_without_publishing_output(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / "source.pdf" + output = root / "output.pdf" + self.make_pdf(source, pages=1) + with self.assertRaises(ProbHubError): + split_pdf(source, [{"path": output, "start": 0, "end": 2}]) + self.assertFalse(output.exists()) + + def test_split_rejects_invalid_container_and_output_path(self): + with tempfile.TemporaryDirectory() as temp: + source = Path(temp) / "source.pdf" + self.make_pdf(source, pages=1) + for outputs in (None, [{"path": None, "start": 0, "end": 1}]): + with self.subTest(outputs=outputs): + with self.assertRaises(ProbHubError) as raised: + split_pdf(source, outputs) + self.assertEqual(raised.exception.code, "pdf_processing_failed") + + def test_invalid_input_path_uses_stable_error(self): + for call in ( + lambda: inspect_pdf(None), + lambda: split_pdf(None, []), + ): + with self.subTest(call=call): + with self.assertRaises(ProbHubError) as raised: + call() + self.assertEqual(raised.exception.code, "pdf_processing_failed") + + def test_worker_write_failure_cleans_partial_temporary_file(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / "source.pdf" + output = root / "output.pdf" + self.make_pdf(source, pages=1) + payload = { + "input": str(source), + "outputs": [{"path": str(output), "start": 0, "end": 1}], + } + with mock.patch.object(pdf_worker.PdfWriter, "write", side_effect=RuntimeError("write failed")): + with self.assertRaisesRegex(RuntimeError, "write failed"): + pdf_worker._split(payload) + self.assertFalse(output.exists()) + self.assertEqual(list(root.glob(".output.pdf.probhub-*.tmp")), []) + + def test_worker_resource_failures_map_to_stable_error(self): + with tempfile.TemporaryDirectory() as temp: + source = Path(temp) / "source.pdf" + source.write_bytes(b"handled by mocked worker") + for reason, text in ( + ("memory_limit", "memory limit"), + ("output_limit", "diagnostic output limit"), + ("process_limit", "process limit"), + ): + with self.subTest(reason=reason): + result = {"reason": reason, "returncode": None, "message": reason} + with mock.patch( + "probhub.pdf_processing.run_managed_to_files", + return_value=result, + ): + with self.assertRaises(ProbHubError) as raised: + inspect_pdf(source) + self.assertEqual(raised.exception.code, "pdf_processing_failed") + self.assertIn(text, str(raised.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ui_theme.py b/tests/test_ui_theme.py index ffceca2..116cb7a 100644 --- a/tests/test_ui_theme.py +++ b/tests/test_ui_theme.py @@ -92,7 +92,7 @@ def fake_render(command, **kwargs): return {"reason": "completed", "returncode": 0} with ( - mock.patch("pypdf.PdfReader", return_value=mock.Mock(pages=[object()])), + mock.patch.object(self.ui, "bounded_pdf_page_count", return_value=1), mock.patch.object(self.ui, "run_managed_to_files", side_effect=fake_render), ): response = self.client.get("/api/pdf-page/QA/0") @@ -106,6 +106,37 @@ def fake_render(command, **kwargs): os.chdir(original_cwd) temp.cleanup() + def test_invalid_pdf_is_bounded_and_preserves_api_shape(self): + original_cwd = Path.cwd() + original_base_dir = self.ui.BASE_DIR + temp = tempfile.TemporaryDirectory() + try: + root = Path(temp.name) + statement = root / "typst-statement" / "QA" + statement.mkdir(parents=True) + (statement / "main.pdf").write_bytes(b"malformed pdf") + os.chdir(root) + self.ui.BASE_DIR = "typst-statement" + error = self.ui.ProbHubError( + "PDF processing timed out", + code="pdf_processing_failed", + ) + with mock.patch.object( + self.ui, + "bounded_pdf_page_count", + side_effect=error, + ): + count = self.client.get("/api/pdf-pages/QA") + page = self.client.get("/api/pdf-page/QA/0") + self.assertEqual(count.status_code, 200) + self.assertEqual(count.get_json(), {"pages": 0}) + self.assertEqual(page.status_code, 500) + self.assertEqual(page.get_data(as_text=True), "Invalid PDF") + finally: + self.ui.BASE_DIR = original_base_dir + os.chdir(original_cwd) + temp.cleanup() + if __name__ == "__main__": unittest.main()