From 029f21eed81a06717d1f29e094723225f423b5d4 Mon Sep 17 00:00:00 2001 From: greenthree <1395214327@qq.com> Date: Wed, 5 Aug 2026 23:00:39 +0800 Subject: [PATCH] add judge QA delivery gates --- README.md | 19 ++ SKILL.md | 14 +- probhub/building.py | 39 +++ probhub/cli.py | 77 ++++- probhub/judge_qa.py | 4 + probhub/judge_qa_evidence.py | 517 ++++++++++++++++++++++++++++++ probhub/judge_qa_runtime.py | 339 ++++++++++++++++++-- probhub/linting.py | 47 ++- probhub/reporting.py | 122 ++++++- references/checker-interactor.md | 33 ++ references/cli.md | 43 ++- references/generations.md | 2 +- references/workspace-schema-v1.md | 56 ++++ tests/test_batch_build.py | 204 ++++++++++++ tests/test_generations.py | 120 ++++++- tests/test_judge_qa_cli.py | 77 +++++ tests/test_judge_qa_evidence.py | 302 +++++++++++++++++ tests/test_judge_qa_execution.py | 119 ++++++- 18 files changed, 2079 insertions(+), 55 deletions(-) create mode 100644 probhub/judge_qa_evidence.py create mode 100644 tests/test_judge_qa_cli.py create mode 100644 tests/test_judge_qa_evidence.py diff --git a/README.md b/README.md index 95db042..3575950 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ ProbHub 会在这条流程中提供： - 面向 Agent 的 Skill，让 Agent 了解规范文件、验证顺序和交付标准； - standard、custom checker、浮点比较和 interactive 四类常见评测场景； - AC、WA、TLE、MLE、OLE、RE、FAIL 等结果和完整进程树清理； +- Checker/Interactor 的题目级主动 Judge QA：fixture、鲁棒性探针、隔离执行和有界 evidence； - 可复现的数据生成、差分测试、反例重放和错解击杀矩阵； - 题面与 Validator 的范围对账、多组数据累计总量的静态复核提示，以及按复杂度、测试需求和资源余量推导 `T` 与累计上限的 Agent 指引； - Typst 全卷排版、单题 PDF、DOMjudge ZIP 和交付前验包； @@ -172,6 +173,7 @@ Agent 和 WebUI 都会调用同一套 Core。只有需要手动排查或编排 | `probhub lint L01` | 检查目录、配置、题面结构和约束对账 | | `probhub report L01` | 查看数据画像、错解击杀和累计约束状态 | | `probhub judge L01` | 编译并运行 Validator、标程、暴力和错解 | +| `probhub judge-qa L01 --no-cache` | 主动测试 Checker/Interactor 的 fixture 和鲁棒性 | | `probhub stress L01 --rounds 1000 --seed 12345` | 用随机小数据对拍 | | `probhub seal L01 --no-cache` | 验证并冻结当前题目版本 | | `probhub build L01 --no-cache` | 正式生成 PDF、ZIP 和 Manifest | @@ -223,6 +225,7 @@ Agent 完成题目后，应明确报告下列结果： - 命令退出码为 0； - Judge 最终结果为 `all_expectations_met`； +- 对 custom/interactive 题，Judge QA 已配置且最终状态为 `passed`，evidence 为 `current`； - `status` 为 `current`； - ZIP 深度验证没有错误； - 人工检查过单题 PDF 和整场 PDF。 @@ -231,6 +234,8 @@ Agent 完成题目后，应明确报告下列结果： 本机通过不等于目标 DOMjudge 机器一定具有相同速度。时间限制和内存限制仍应在目标 Linux/DOMjudge 环境校准。 +对 `judge.type: custom` 或 `judge.type: interactive` 的新题，或修改 Checker/Interactor 后，先在 `judge.qa` 中登记题目级 fixture，再运行 `probhub judge-qa --no-cache`。fixture 每次都会执行，只有编译结果可以缓存；`judge-qa-evidence-v1.json` 是本地有界证据，不会进入 ZIP、PDF 或 Manifest。lint/status 中的 evidence 缺失或过期是体检 warning，但 `seal` 和正式 `build` 不允许已配置题目绕过通过的 Judge QA。 + ## 并行出题时怎么做 多名出题人或多个 Agent 可以各自只修改自己的题目目录： @@ -310,6 +315,20 @@ probhub seal L01 --no-cache --seed 12345 所有题目完成后再运行多题 `build`。 +### `seal` 提示 `seal_judge_qa_failed` + +题目已配置 Checker/Interactor 主动测试，但 fixture 没有全部得到期望状态，或 Judge/清理基础设施失败。先运行： + +```bash +probhub judge-qa L01 --no-cache +``` + +按结构化结果修复 Checker、Interactor、模拟选手、fixture 或期望状态，再重新执行 `seal`。`FAIL` 表示题目基础设施错误，不是成功击杀错解。 + +### `lint` 或 `status` 显示 Judge QA evidence warning + +`judge_qa_evidence_missing`、`judge_qa_evidence_stale` 和 `judge_qa_evidence_invalid` 不会单独让 lint 失败，但正式交付前必须恢复为 `current`。旧的 standard 题或未配置 QA 的旧工作区仍可继续使用；新的 custom/interactive 题不能以 `not-configured` 交付。 + ### `status` 显示 `stale` 这表示题目、数据、题序、模板、构建工具链或正式产物在上次构建后发生了变化。读取 `stale_fields` 定位原因；重新 `seal` 并 `build`，不要手工修改 Manifest。若显示 `builder_fingerprint.unavailable`，先按诊断修复 Typst 或 Python 依赖。 diff --git a/SKILL.md b/SKILL.md index eb8a7f3..d0555f1 100644 --- a/SKILL.md +++ b/SKILL.md @@ -95,6 +95,7 @@ probhub build | `report [ID...]` | 只读汇总难度、数据画像、recipe、TL 余量和错解击杀矩阵；`--format markdown` 输出 Markdown | | `sample-check [ID...]` | 只运行样例与首个 accepted，严格核对 `.ans`；不发布 Judge 校准 evidence | | `judge [ID...]` | 编译并运行 Validator、accepted、brute、wrong | +| `judge-qa [ID...]` | 对已配置的 Checker/Interactor fixture 和鲁棒性探针做主动 Judge QA；只缓存编译结果 | | `stress ID...` | 反复生成小数据，对拍 accepted 与 brute，保存首个可重放反例；`--against <解法>` 反向找刀，`--fixate ` 把命中一步固化为 secret 数据 + 配方 + 定向数据组 | | `checkpoint ID` | 发布当前题目的不可变 draft checkpoint，供并行组卷使用 | | `seal ID` | lint、judge、stress 后冻结 revision，并自动生成一版完整试卷 | @@ -139,6 +140,14 @@ probhub build L01 --no-cache probhub build L01 --skip-judge ``` +对 `judge.type: custom` 或 `judge.type: interactive` 的新题、以及修改过 Checker/Interactor 的题，必须在 `judge.qa` 中登记题目级 fixture 后执行： + +```powershell +probhub judge-qa L01 --no-cache +``` + +Judge QA 每次都会重新执行 fixture 和内建探针；`--no-cache` 只额外强制重编官方 Judge 与模拟选手。成功会原子发布题目本地的 `judge-qa-evidence-v1.json`，该文件不进入 ZIP、PDF、Manifest 或正式数据。没有配置 QA 的旧题仍保持 Core 兼容，但 Agent 不得把 `not-configured` 当作新 custom/interactive 题的交付完成。 + 完整语法、产物、退出码和故障处理见 `references/cli.md`。配置或执行差分测试前读取 `references/stress.md`；修改资源限制、解释 OLE 或排查残留进程时读取 `references/process-control.md`。 # 4. Agent 验证模式 @@ -184,6 +193,8 @@ probhub build L01 --skip-judge 发现反例后先用输出的 `replay_command` 固定复现，再修复并重跑；完整协议见 `references/stress.md`。 + 对已配置 `judge.qa` 的 custom/interactive 题，再执行 `probhub judge-qa --no-cache`，确认最终状态为 `passed`；自动探针若返回 `AC`，还必须人工确认这不是 Checker/Interactor 误放行。 + 6. 完成后执行： ```powershell @@ -197,7 +208,7 @@ probhub build L01 --skip-judge probhub build --no-cache ``` -8. 只有命令退出码为 `0`、沙箱最终事件为 `all_expectations_met`、ZIP 深度验证成功且 `status` 为 `current` 时才可交付。独立复核正式包时使用 `probhub --workspace <工作区> verify-package .zip --require-pdf --problem `；不带 `--problem` 的 `verification_scope: structural` 不能替代题名、限制、数据和输入 Validator 对账。Manifest 的 `collection_hash` 会跟踪整场排版输入；其他题题面、题面媒体、样例、题序或模板变化后，受影响题目也必须重新构建。 +8. 只有命令退出码为 `0`、沙箱最终事件为 `all_expectations_met`、已配置 Judge QA 的最终状态为 `passed` 且 evidence 为 `current`、ZIP 深度验证成功且 `status` 为 `current` 时才可交付。独立复核正式包时使用 `probhub --workspace <工作区> verify-package .zip --require-pdf --problem `；不带 `--problem` 的 `verification_scope: structural` 不能替代题名、限制、数据和输入 Validator 对账。Manifest 的 `collection_hash` 会跟踪整场排版输入；其他题题面、题面媒体、样例、题序或模板变化后，受影响题目也必须重新构建。 9. 查看 judge summary 与 lint/status 的 `calibration`、`diagnostics`：默认 accepted 应满足 `max_time × 3 <= TL`，期望 TLE 的目标用例应有至少 `1.5 × TL` 的延长探针证据。缺失或低余量 warning 必须在交付前人工处理或在题目 `calibration` 中有意识地调整阈值。 本地 `max_time`、内存和输出余量不是正式评测承诺。Windows 与 Linux/DOMjudge 的启动、链接、调度、计时和内存口径不同；正式 TL/ML/OL 必须在目标 Linux 评测环境重新校准，结构化结果中的 `target_guarantee` 固定为 `false`。 @@ -220,6 +231,7 @@ probhub build L01 --skip-judge - 普通唯一答案题使用 `judge.type: standard`：忽略整个输出首尾空白和每行末尾空格/Tab，但行内空格与内部换行仍需一致。需要 Token 级宽松比较时改用 Checker。 - 非唯一答案和浮点题使用 `judge.type: custom` 与 `code/checker.cpp`；交互题使用 `judge.type: interactive` 与 `code/interactor.cpp`。实现前读取 `references/checker-interactor.md`。 - Checker/Interactor 必须使用附带的 DOMjudge/testlib 协议；交互题按需设置 `judge.interactive.idle_limit` 和 `transcript_limit`。Core 负责本地编译以及生成 `output_validators/validate/`，不得手工维护该生成目录。 +- Checker/Interactor 题应在 `judge.qa` 中登记真实 fixture；Checker 可声明 `AC/WA`，Interactor 可声明 `AC/WA/RE/TLE/MLE/OLE`，并可使用 `early-eof`、`idle`、`output-flood` 内建模拟行为。fixture 文件放在 `judge-fixtures/`，交互模拟选手源码放在 `code/judge-qa/`，都会按原始字节进入 `fixture_hash`，但永远不会进入正式题目包。 - 数据严格放在 `data/sample` 和 `data/secret`，每个 `.in` 必须有同名 `.ans`。 - 样例 `.ans` 必须由配置顺序中的首个 accepted 精确复现；只归一 CRLF/CR 为 LF，尾空格、缺少尾换行和其他字节差异仍失败。Custom Checker 的非唯一输出语义不能替代这条样例不变量；交互题明确不适用。 - 题面只能有一个 H1，必需 H2 依次为题目描述、输入格式、输出格式且内容非空；提示位于输出之后，样例输入/输出只来自 `data/sample`。lint 的约束对账会保守识别直接 LaTeX/中文累计上限与 Validator 直接累加器，并在多测但未发现累计上限时提示复核；结果始终是 `analysis_state: partial`，启发式 mismatch 只能 warning，不能替代复杂度分析或正确性证明。 diff --git a/probhub/building.py b/probhub/building.py index 33d8f6f..9ef9024 100644 --- a/probhub/building.py +++ b/probhub/building.py @@ -26,6 +26,12 @@ from .generations import checkpoint_revision, latest_checkpoint from .hashing import hash_file from .io import write_json +from .judge_qa import inspect_judge_qa +from .judge_qa_evidence import ( + JUDGE_QA_EVIDENCE_FILENAME, + JUDGE_QA_EVIDENCE_LOCK_FILENAME, + validate_judge_qa_evidence_document, +) from .judging import judge_problem from .linting import ( BUILD_MANIFEST_SCHEMA_VERSION, @@ -211,6 +217,9 @@ def require_collection_sealed(plan): f"{problem_id}: sealed revision does not match live {', '.join(mismatches)}" ) continue + if (qa_error := _sealed_judge_qa_error(item, checkpoint)) is not None: + rejected.append(f"{problem_id}: {qa_error}") + continue checkpoints[problem_id] = checkpoint if rejected: @@ -222,6 +231,27 @@ def require_collection_sealed(plan): return checkpoints +def _sealed_judge_qa_error(item, checkpoint): + inspection = inspect_judge_qa(item.problem_dir, item.config) + if not inspection.get("configured"): + return None + evidence = checkpoint.get("evidence") + judge_qa = evidence.get("judge_qa") if isinstance(evidence, dict) else None + if not isinstance(judge_qa, dict) or judge_qa.get("status") != "passed": + return "sealed revision has no passed Judge QA evidence" + evaluated = validate_judge_qa_evidence_document( + judge_qa, + item.config, + inspection, + item.source_hash, + item.data_hash, + ) + if evaluated.get("state") != "current": + reason = evaluated.get("reason") or evaluated.get("state") or "invalid" + return f"sealed Judge QA evidence is {reason}" + return None + + def assert_collection_seals_unchanged(plan, checkpoints): """Revalidate the exact sealed revisions immediately before publication.""" rejected = [] @@ -252,6 +282,8 @@ def assert_collection_seals_unchanged(plan, checkpoints): or checkpoint.get("data_hash") != item.data_hash ): rejected.append(f"{problem_id}: sealed revision changed during build") + elif (qa_error := _sealed_judge_qa_error(item, checkpoint)) is not None: + rejected.append(f"{problem_id}: {qa_error}") if rejected: raise ProbHubError( "sealed build evidence changed during the run: " + "; ".join(rejected), @@ -298,6 +330,9 @@ def _snapshot_ignore(plan): "generation.lock", "generations", EVIDENCE_LOCK_FILENAME, + JUDGE_QA_EVIDENCE_FILENAME, + JUDGE_QA_EVIDENCE_LOCK_FILENAME, + "compile", "sandbox-cache-v1.json.tmp", "stress", "submissions", @@ -321,6 +356,10 @@ def ignore(directory, names): result.append(name) if directory.name == ".probhub" and name.startswith(EVIDENCE_FILENAME + "."): result.append(name) + if directory.name == ".probhub" and name.startswith( + JUDGE_QA_EVIDENCE_FILENAME + "." + ): + result.append(name) return result return ignore diff --git a/probhub/cli.py b/probhub/cli.py index 71157e4..2d5cbf6 100644 --- a/probhub/cli.py +++ b/probhub/cli.py @@ -17,6 +17,7 @@ generation_status, ) from .io import atomic_write_bytes, atomic_write_text, write_yaml +from .judge_qa import judge_qa_problem from .judging import check_sample_answers, judge_problem from .linting import ( compute_collection_hash, @@ -492,6 +493,24 @@ def command_judge(args): return {"ok": all(item["ok"] for item in results.values()), "problems": results} +def command_judge_qa(args): + root, workspace = workspace_context(args) + ensure_no_pending_transactions(root, workspace) + _ensure_local_gitignore(root) + results = {} + for entry in select_entries(workspace, args.problem): + problem_dir, _ = load_problem(root, entry) + results[entry["id"]] = judge_qa_problem( + root, + problem_dir, + use_cache=not args.no_cache, + ) + return { + "ok": all(item["ok"] for item in results.values()), + "problems": results, + } + + def command_sample_check(args): root, workspace = workspace_context(args) ensure_no_pending_transactions(root, workspace) @@ -576,6 +595,9 @@ def command_seal(args): f"cannot seal {entry['id']}: " + "; ".join(messages), code="seal_lint_failed", ) + qa_configured = bool( + (lint["problems"][0].get("judge_qa") or {}).get("configured") + ) problem_dir, config = load_problem(root, entry) source_hash = compute_source_hash(problem_dir, config) @@ -586,10 +608,41 @@ def command_seal(args): f"cannot seal {entry['id']}: sandbox failed: {judge.get('final')}", code="seal_judge_failed", ) - # Judge publishes the successful local calibration evidence. Refresh the - # read-only lint view so the sealed checkpoint does not preserve the - # pre-judge "evidence missing" warning. + judge_qa = judge_qa_problem( + root, + problem_dir, + use_cache=not args.no_cache, + ) + qa_status = judge_qa.get("status") + qa_passed = ( + judge_qa.get("ok") is True + and ( + qa_status == "passed" + if qa_configured + else qa_status == "not-configured" + ) + ) + if not qa_passed: + raise ProbHubError( + f"cannot seal {entry['id']}: Judge QA failed: " + f"{qa_status or 'unknown'} ({judge_qa.get('code') or 'no-code'})", + code="seal_judge_qa_failed", + ) + # Judge and Judge QA publish local evidence. Refresh the read-only lint + # view so the checkpoint records the post-verification states. lint = lint_workspace(root, workspace, [entry]) + evidence_state = ( + ((lint["problems"][0].get("judge_qa") or {}).get("evidence") or {}).get( + "state" + ) + ) + expected_evidence_state = "current" if qa_configured else "not-configured" + if evidence_state != expected_evidence_state: + raise ProbHubError( + f"cannot seal {entry['id']}: Judge QA evidence is " + f"{evidence_state or 'missing'}", + code="seal_judge_qa_failed", + ) stress = None if config.get("stress"): @@ -629,6 +682,20 @@ def command_seal(args): "summaries": judge.get("summaries", []), "calibration": judge.get("calibration"), }, + "judge_qa": ( + { + "status": "not-configured", + "applicable": False, + "judge_type": judge_qa.get("judge_type"), + } + if qa_status == "not-configured" + else { + "status": "passed", + "applicable": True, + "code": judge_qa.get("code"), + **dict(judge_qa.get("evidence") or {}), + } + ), "stress": stress, } checkpoint = create_problem_checkpoint( @@ -756,12 +823,12 @@ def build_parser(): report.add_argument("--format", choices=("text", "markdown"), default="text") report.set_defaults(handler=command_report, renderer=render_report_result) - for name, handler in (("lint", command_lint), ("status", command_status), ("judge", command_judge), ("sample-check", command_sample_check), ("typeset", command_typeset), ("package", command_package), ("build", command_build)): + for name, handler in (("lint", command_lint), ("status", command_status), ("judge", command_judge), ("judge-qa", command_judge_qa), ("sample-check", command_sample_check), ("typeset", command_typeset), ("package", command_package), ("build", command_build)): item = sub.add_parser(name) item.add_argument("problem", nargs="*") if name == "package": item.add_argument("--allow-missing-pdf", action="store_true") - if name in {"judge", "sample-check", "build"}: + if name in {"judge", "judge-qa", "sample-check", "build"}: item.add_argument("--no-cache", action="store_true", help="ignore existing sandbox caches and refresh them") if name == "build": item.add_argument("--skip-judge", action="store_true") diff --git a/probhub/judge_qa.py b/probhub/judge_qa.py index 4eb8145..95ecf3c 100644 --- a/probhub/judge_qa.py +++ b/probhub/judge_qa.py @@ -3,6 +3,10 @@ import unicodedata from pathlib import Path, PurePosixPath +from .judge_qa_evidence import ( + evaluate_judge_qa_evidence, + validate_judge_qa_evidence_document, +) from .problem_paths import ProblemPathError, resolve_problem_regular_file diff --git a/probhub/judge_qa_evidence.py b/probhub/judge_qa_evidence.py new file mode 100644 index 0000000..a3ad538 --- /dev/null +++ b/probhub/judge_qa_evidence.py @@ -0,0 +1,517 @@ +"""Bounded Judge QA evidence reading and validation.""" + +import json +import platform +import stat +import unicodedata +from pathlib import Path, PurePosixPath + +from . import __version__ +from .calibration import SANDBOX_CACHE_SCHEMA_VERSION + + +JUDGE_QA_EVIDENCE_SCHEMA_VERSION = 1 +JUDGE_QA_POLICY_VERSION = 1 +JUDGE_QA_EVIDENCE_FILENAME = "judge-qa-evidence-v1.json" +JUDGE_QA_EVIDENCE_LOCK_FILENAME = "judge-qa-evidence.lock" +MAX_JUDGE_QA_EVIDENCE_BYTES = 2 * 1024 * 1024 +MAX_JUDGE_QA_EVIDENCE_MESSAGE_BYTES = 4096 +MAX_JUDGE_QA_EVIDENCE_NODES = 20000 + +_FORBIDDEN_CONTENT_KEYS = { + "entries", + "feedback_text", + "stderr", + "stderr_text", + "stdout", + "stdout_text", + "transcript_entries", + "message", + "message_text", +} +_ACTUAL_KEYS = ( + "status", + "timeout_kind", + "termination_reason", + "actor", + "execution_status", + "failure_kind", +) + + +def evidence_path(problem_dir): + return Path(problem_dir) / ".probhub" / JUDGE_QA_EVIDENCE_FILENAME + + +def _fold(value): + return unicodedata.normalize("NFC", str(value)).casefold() + + +def _normalise_path(value): + if not isinstance(value, str) or not value.strip(): + return None + return PurePosixPath(value.strip().replace("\\", "/")).as_posix() + + +def _bounded_detail(value): + payload = str(value or "").strip().encode("utf-8", errors="replace") + if len(payload) <= MAX_JUDGE_QA_EVIDENCE_MESSAGE_BYTES: + return payload.decode("utf-8", errors="replace") + return ( + payload[:MAX_JUDGE_QA_EVIDENCE_MESSAGE_BYTES].decode( + "utf-8", errors="replace" + ) + + "..." + ) + + +def _diagnostic(code, message, **payload): + return { + "code": code, + "severity": "warning", + "message": message, + **payload, + } + + +def _base_status(inspection, state, diagnostics=None, **payload): + diagnostics = list(diagnostics or []) + robustness = inspection.get("robustness") or {} + declared_probes = list(robustness.get("probes") or []) + return { + "state": state, + "configured": bool(inspection.get("configured")), + "applicable": bool(inspection.get("applicable")), + "judge_type": inspection.get("judge_type"), + "target_guarantee": False, + "declared_cases": len(inspection.get("cases") or []), + "evidence_cases": 0, + "matched_cases": 0, + "declared_probes": len(declared_probes), + "evidence_probes": 0, + "manual_review_probes": 0, + "cases": [], + "probes": [], + "diagnostics": diagnostics, + "warnings": [ + f"[{item['code']}] {item['message']}" for item in diagnostics + ], + **payload, + } + + +def _state_diagnostic(state, reason, detail=None): + if state == "missing": + return _diagnostic( + "judge_qa_evidence_missing", + "Judge QA evidence is missing; run probhub judge-qa", + reason=reason, + ) + if state == "stale": + return _diagnostic( + "judge_qa_evidence_stale", + "Judge QA evidence is stale; rerun probhub judge-qa", + reason=reason, + ) + return _diagnostic( + "judge_qa_evidence_invalid", + "Judge QA evidence is invalid; rerun probhub judge-qa", + reason=reason, + **({"detail": _bounded_detail(detail)} if detail else {}), + ) + + +def _read_evidence(problem_dir): + path = evidence_path(problem_dir) + try: + before = path.lstat() + except FileNotFoundError: + return None, "missing", None + except OSError as exc: + return None, "unreadable", exc + reparse = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + if stat.S_ISLNK(before.st_mode) or ( + reparse and getattr(before, "st_file_attributes", 0) & reparse + ): + return None, "unsafe_path", "evidence path is a link or reparse point" + if not stat.S_ISREG(before.st_mode): + return None, "unsafe_path", "evidence path is not a regular file" + if before.st_size > MAX_JUDGE_QA_EVIDENCE_BYTES: + return None, "too_large", ( + f"evidence is {before.st_size} bytes; limit is " + f"{MAX_JUDGE_QA_EVIDENCE_BYTES}" + ) + try: + with path.open("rb") as stream: + payload = stream.read(MAX_JUDGE_QA_EVIDENCE_BYTES + 1) + after = path.lstat() + except OSError as exc: + return None, "unreadable", exc + if len(payload) > MAX_JUDGE_QA_EVIDENCE_BYTES: + return None, "too_large", "evidence grew beyond the read limit" + if ( + before.st_dev, + before.st_ino, + before.st_mtime_ns, + before.st_size, + ) != ( + after.st_dev, + after.st_ino, + after.st_mtime_ns, + after.st_size, + ): + return None, "unreadable", "evidence changed while being read" + try: + decoded = payload.decode("utf-8") + evidence = json.loads(decoded) + except (UnicodeError, json.JSONDecodeError, RecursionError) as exc: + return None, "invalid_json", exc + if not isinstance(evidence, dict): + return None, "invalid_json", "evidence root must be a JSON object" + return evidence, None, None + + +def _payload_structure_error(evidence): + stack = [evidence] + visited = 0 + while stack: + value = stack.pop() + visited += 1 + if visited > MAX_JUDGE_QA_EVIDENCE_NODES: + return "evidence structure exceeds the fixed node limit" + if isinstance(value, dict): + forbidden = sorted(set(value).intersection(_FORBIDDEN_CONTENT_KEYS)) + if forbidden: + return "evidence contains forbidden stream content: " + ", ".join( + forbidden + ) + if "feedback" in value and ( + isinstance(value["feedback"], bool) + or not isinstance(value["feedback"], (int, float)) + ): + return "evidence contains feedback content instead of a byte count" + transcript = value.get("transcript") + if isinstance(transcript, dict) and not set(transcript).issubset( + {"bytes", "limit", "truncated"} + ): + return "evidence transcript contains unsupported fields" + stack.extend(value.values()) + elif isinstance(value, list): + stack.extend(value) + return None + + +def _index_records(records, label): + if not isinstance(records, list): + return None, f"{label} must be a list" + indexed = {} + for item in records: + if not isinstance(item, dict) or not isinstance(item.get("id"), str): + return None, f"{label} contains an invalid record" + folded = _fold(item["id"]) + if folded in indexed: + return None, f"{label} contains duplicate ID {item['id']}" + indexed[folded] = item + return indexed, None + + +def _diagnostic_error(item, label): + diagnostic = item.get("diagnostic") + if not isinstance(diagnostic, dict): + return f"{label} diagnostic must be a mapping" + if set(diagnostic) != {"present", "bytes", "truncated"}: + return f"{label} diagnostic contains unsupported fields" + if not isinstance(diagnostic.get("present"), bool): + return f"{label} diagnostic.present must be boolean" + count = diagnostic.get("bytes") + if isinstance(count, bool) or not isinstance(count, int) or count < 0: + return f"{label} diagnostic.bytes must be a non-negative integer" + if count > MAX_JUDGE_QA_EVIDENCE_MESSAGE_BYTES: + return f"{label} diagnostic.bytes exceeds the fixed byte limit" + if diagnostic["present"] != (count > 0): + return f"{label} diagnostic presence does not match byte count" + if not isinstance(diagnostic.get("truncated"), bool): + return f"{label} diagnostic.truncated must be boolean" + if diagnostic["truncated"] and count != MAX_JUDGE_QA_EVIDENCE_MESSAGE_BYTES: + return f"{label} truncated diagnostic must use the fixed byte limit" + return None + + +def _expected_compilers(config, inspection): + judge = config.get("judge") if isinstance(config.get("judge"), dict) else {} + expected = [] + for role in ("validator", inspection.get("judge_type") == "custom" and "checker" or "interactor"): + source = _normalise_path(judge.get(role)) + if source: + expected.append((role, source)) + for case in inspection.get("cases") or []: + contestant = case.get("contestant") or {} + source = _normalise_path(contestant.get("source")) + if source: + expected.append(("contestant", source)) + return set(expected) + + +def _structure_error(evidence, config, inspection): + if (error := _payload_structure_error(evidence)) is not None: + return error + measurement = evidence.get("measurement") + if not isinstance(measurement, dict) or measurement.get("target_guarantee") is not False: + return "measurement.target_guarantee must be false" + cleanup = evidence.get("cleanup") + if not isinstance(cleanup, dict) or cleanup.get("ok") is not True or cleanup.get("snapshot_removed") is not True: + return "cleanup must confirm successful snapshot removal" + + declared_cases, error = _index_records(inspection.get("cases") or [], "declared cases") + if error: + return error + observed_cases, error = _index_records(evidence.get("cases"), "evidence cases") + if error: + return error + if set(observed_cases) != set(declared_cases): + return "evidence case IDs do not match current Judge QA configuration" + for folded, declared in declared_cases.items(): + observed = observed_cases[folded] + if observed.get("purpose") != declared.get("purpose"): + return f"case purpose changed for {declared.get('id')}" + if observed.get("expected") != declared.get("expected"): + return f"case expectation changed for {declared.get('id')}" + if observed.get("matched") is not True or observed.get("infrastructure_failed") is not False: + return f"case did not pass cleanly: {declared.get('id')}" + actual = observed.get("actual") + if not isinstance(actual, dict): + return f"case actual result is missing: {declared.get('id')}" + if any(actual.get(key) != value for key, value in declared.get("expected", {}).items()): + return f"case actual result does not satisfy expectation: {declared.get('id')}" + if (error := _diagnostic_error(observed, f"case {declared.get('id')}")) is not None: + return error + + declared_probe_ids = list((inspection.get("robustness") or {}).get("probes") or []) + declared_probes = {_fold(item): item for item in declared_probe_ids} + observed_probes, error = _index_records(evidence.get("probes"), "evidence probes") + if error: + return error + if set(observed_probes) != set(declared_probes): + return "evidence probe IDs do not match current Judge QA configuration" + for folded, declared_id in declared_probes.items(): + observed = observed_probes[folded] + if observed.get("matched") is not True or observed.get("infrastructure_failed") is not False: + return f"robustness probe did not complete cleanly: {declared_id}" + actual = observed.get("actual") + if not isinstance(actual, dict) or actual.get("status") not in {"AC", "WA"}: + return f"robustness probe actual result is invalid: {declared_id}" + expected_manual = actual.get("status") == "AC" + if observed.get("manual_review_required") is not expected_manual: + return f"robustness probe review flag is invalid: {declared_id}" + if (error := _diagnostic_error(observed, f"probe {declared_id}")) is not None: + return error + + expected_inputs = { + _fold(case.get("input")): case.get("input") + for case in inspection.get("cases") or [] + if case.get("input") + } + validators = evidence.get("validators") + if not isinstance(validators, list): + return "validator summaries must be a list" + observed_inputs = {} + for item in validators: + if not isinstance(item, dict) or not isinstance(item.get("input"), str): + return "validator summary is invalid" + folded = _fold(item["input"]) + if folded in observed_inputs: + return f"validator input is duplicated: {item['input']}" + if item.get("ok") is not True: + return f"validator did not accept Judge QA input: {item['input']}" + if (error := _diagnostic_error(item, f"validator {item['input']}")) is not None: + return error + observed_inputs[folded] = item["input"] + if set(observed_inputs) != set(expected_inputs): + return "validator summaries do not cover current Judge QA inputs" + + compilers = evidence.get("compilers") + if not isinstance(compilers, list): + return "compiler summaries must be a list" + expected_compilers = _expected_compilers(config, inspection) + observed_compilers = set() + for item in compilers: + if not isinstance(item, dict): + return "compiler summary is invalid" + role = item.get("role") + source = _normalise_path(item.get("source")) + identity = (role, source) + if not isinstance(role, str) or source is None or identity in observed_compilers: + return "compiler identity is invalid or duplicated" + expected_kind = "python" if PurePosixPath(source).suffix.casefold() == ".py" else "cpp17" + if item.get("kind") != expected_kind: + return f"compiler kind is invalid for {source}" + if expected_kind == "cpp17" and not isinstance(item.get("compiler_identity"), str): + return f"compiler identity is missing for {source}" + observed_compilers.add(identity) + if observed_compilers != expected_compilers: + return "compiler summaries do not match current Judge QA programs" + return None + + +def _project_current(evidence, inspection, diagnostics): + cases = [ + { + "id": item.get("id"), + "purpose": item.get("purpose"), + "expected": dict(item.get("expected") or {}), + "actual": { + key: (item.get("actual") or {}).get(key) + for key in _ACTUAL_KEYS + if key in (item.get("actual") or {}) + }, + "matched": True, + "diagnostic": dict(item.get("diagnostic") or {}), + } + for item in evidence.get("cases") or [] + ] + probes = [ + { + "id": item.get("id"), + "purpose": item.get("purpose"), + "expected": dict(item.get("expected") or {}), + "actual": { + key: (item.get("actual") or {}).get(key) + for key in _ACTUAL_KEYS + if key in (item.get("actual") or {}) + }, + "matched": True, + "manual_review_required": bool(item.get("manual_review_required")), + "diagnostic": dict(item.get("diagnostic") or {}), + } + for item in evidence.get("probes") or [] + ] + manual = sorted( + str(item.get("id")) for item in probes if item["manual_review_required"] + ) + if manual: + diagnostics.append(_diagnostic( + "judge_qa_probe_manual_review_required", + "one or more automatic Judge QA probes returned AC and require author review", + probe_ids=manual, + )) + measurement = evidence.get("measurement") or {} + return _base_status( + inspection, + "current", + diagnostics, + published_at=evidence.get("published_at"), + probhub_version=evidence.get("probhub_version"), + platform={ + "system": measurement.get("platform"), + "machine": measurement.get("machine"), + }, + evidence_cases=len(cases), + matched_cases=len(cases), + evidence_probes=len(probes), + manual_review_probes=len(manual), + cases=cases, + probes=probes, + ) + + +def validate_judge_qa_evidence_document( + evidence, + config, + inspection, + source_hash, + data_hash, +): + """Validate an already-decoded evidence document and return its bounded view.""" + + if not inspection.get("configured"): + return _base_status(inspection, "not-configured") + if not inspection.get("ok"): + return _base_status(inspection, "invalid") + if not isinstance(evidence, dict): + diagnostic = _state_diagnostic("invalid", "invalid_json") + return _base_status(inspection, "invalid", [diagnostic], reason="invalid_json") + + stale_checks = ( + (evidence.get("schema_version") != JUDGE_QA_EVIDENCE_SCHEMA_VERSION, "schema"), + (evidence.get("policy_version") != JUDGE_QA_POLICY_VERSION, "policy"), + (evidence.get("sandbox_schema_version") != SANDBOX_CACHE_SCHEMA_VERSION, "sandbox"), + (evidence.get("probhub_version") != __version__, "core"), + ( + evidence.get("source_hash") != source_hash + or evidence.get("data_hash") != data_hash + or evidence.get("fixture_hash") != inspection.get("fixture_hash"), + "inputs", + ), + ) + for changed, reason in stale_checks: + if changed: + diagnostic = _state_diagnostic("stale", reason) + return _base_status( + inspection, + "stale", + [diagnostic], + reason=reason, + published_at=evidence.get("published_at"), + ) + measurement = evidence.get("measurement") + if not isinstance(measurement, dict): + structure_error = "measurement summary is missing" + elif ( + measurement.get("platform") != platform.system() + or ( + measurement.get("machine") + and measurement.get("machine") != platform.machine() + ) + ): + diagnostic = _state_diagnostic("stale", "platform") + return _base_status( + inspection, + "stale", + [diagnostic], + reason="platform", + published_at=evidence.get("published_at"), + ) + else: + structure_error = _structure_error(evidence, config, inspection) + if structure_error is not None: + diagnostic = _state_diagnostic("invalid", "structure", structure_error) + return _base_status( + inspection, + "invalid", + [diagnostic], + reason="structure", + published_at=evidence.get("published_at"), + ) + return _project_current(evidence, inspection, []) + + +def evaluate_judge_qa_evidence( + problem_dir, + config, + inspection, + source_hash, + data_hash, +): + """Read and evaluate Judge QA evidence without running external tools.""" + + if not inspection.get("configured"): + return _base_status(inspection, "not-configured") + if not inspection.get("ok"): + return _base_status(inspection, "invalid") + evidence, reason, detail = _read_evidence(problem_dir) + if reason is not None: + state = "missing" if reason == "missing" else "invalid" + diagnostic = _state_diagnostic(state, reason, detail) + return _base_status( + inspection, + state, + [diagnostic], + reason=reason, + ) + return validate_judge_qa_evidence_document( + evidence, + config, + inspection, + source_hash, + data_hash, + ) diff --git a/probhub/judge_qa_runtime.py b/probhub/judge_qa_runtime.py index a864c40..59f232e 100644 --- a/probhub/judge_qa_runtime.py +++ b/probhub/judge_qa_runtime.py @@ -1,11 +1,14 @@ import hashlib +import json import math import os import platform import shutil +import stat import sys import tempfile import time +import uuid from datetime import datetime, timezone from pathlib import Path @@ -13,9 +16,16 @@ from .build_lock import workspace_file_lock from .calibration import SANDBOX_CACHE_SCHEMA_VERSION from .errors import ProbHubError -from .hashing import files_under +from .hashing import files_under, hash_file from .io import atomic_write_json, normalize_newlines, read_bounded_text, read_yaml from .judge_qa import inspect_judge_qa +from .judge_qa_evidence import ( + JUDGE_QA_EVIDENCE_FILENAME, + JUDGE_QA_EVIDENCE_LOCK_FILENAME, + JUDGE_QA_EVIDENCE_SCHEMA_VERSION, + JUDGE_QA_POLICY_VERSION, + evidence_path, +) from .linting import compute_data_hash, compute_source_hash, problem_source_paths from .problem_paths import ProblemPathError, resolve_problem_regular_file from .process_control import ( @@ -32,10 +42,6 @@ from .special_judges import execute_interactive_session, run_checker_to_files -JUDGE_QA_EVIDENCE_SCHEMA_VERSION = 1 -JUDGE_QA_POLICY_VERSION = 1 -JUDGE_QA_EVIDENCE_FILENAME = "judge-qa-evidence-v1.json" -JUDGE_QA_EVIDENCE_LOCK_FILENAME = "judge-qa-evidence.lock" DEFAULT_JUDGE_QA_TIMEOUT_SECONDS = 600.0 MAX_JUDGE_QA_TIMEOUT_SECONDS = 3600.0 DEFAULT_TOOL_TIMEOUT_SECONDS = 10.0 @@ -48,6 +54,10 @@ MAX_JUDGE_QA_OUTPUT_BYTES = 8 * 1024 * 1024 MAX_JUDGE_QA_TRANSCRIPT_BYTES = 1024 * 1024 MIN_JUDGE_QA_IDLE_SECONDS = 0.1 +JUDGE_QA_COMPILE_CACHE_SCHEMA_VERSION = 1 +MAX_JUDGE_QA_COMPILE_CACHE_ENTRIES = 64 +MAX_JUDGE_QA_COMPILE_CACHE_BINARY_BYTES = 128 * 1024 * 1024 +MAX_JUDGE_QA_COMPILE_CACHE_METADATA_BYTES = 64 * 1024 class _OverallTimeout(Exception): @@ -110,6 +120,19 @@ def _bounded_message(value): return retained.decode("utf-8", errors="replace") + "..." +def _diagnostic_summary(value): + """Keep only whether untrusted diagnostic text existed and its byte count.""" + + if value is None: + return {"present": False, "bytes": 0, "truncated": False} + raw = str(value).encode("utf-8", errors="replace") + return { + "present": bool(raw), + "bytes": min(len(raw), DIAGNOSTIC_LIMIT_BYTES), + "truncated": len(raw) > DIAGNOSTIC_LIMIT_BYTES, + } + + def _cleanup_summary(cleanup): cleanup = cleanup if isinstance(cleanup, dict) else {} states = { @@ -127,7 +150,7 @@ def _cleanup_summary(cleanup): { "stage": item.get("stage"), "actor": item.get("actor"), - "message": _bounded_message(item.get("message")), + "diagnostic": _diagnostic_summary(item.get("message")), } for item in errors[:8] if isinstance(item, dict) @@ -279,6 +302,239 @@ def _compiler_identity(cancellation): return identity +def _compiler_cache_identity(): + executable = shutil.which("g++") + if not executable: + raise ProbHubError( + "failed to identify the Judge QA compiler: g++ was not found", + code="compiler_identity_failed", + ) + try: + path = Path(executable).resolve(strict=True) + info = path.stat() + if not path.is_file(): + raise OSError("compiler path is not a regular file") + return ( + f"sha256:{hash_file(path)};size:{info.st_size};" + f"path:{path.as_posix()}" + ) + except OSError as exc: + raise ProbHubError( + f"failed to identify the Judge QA compiler: {_bounded_message(exc)}", + code="compiler_identity_failed", + ) from exc + + +def _link_like(info): + reparse = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + return stat.S_ISLNK(info.st_mode) or ( + reparse and getattr(info, "st_file_attributes", 0) & reparse + ) + + +def _compile_cache_root(problem_dir): + problem_dir = Path(problem_dir).resolve() + current = problem_dir + for part in (".probhub", "compile", "judge-qa-v1"): + current /= part + try: + info = current.lstat() + except FileNotFoundError: + try: + current.mkdir() + except FileExistsError: + pass + info = current.lstat() + if _link_like(info) or not stat.S_ISDIR(info.st_mode): + raise ProbHubError( + f"Judge QA compile cache path is unsafe: {current}", + code="judge_qa_cache_unsafe", + ) + return current + + +def _new_compile_cache(problem_dir, source_hash, use_cache): + return { + "root": _compile_cache_root(problem_dir), + "source_hash": source_hash, + "use_cache": bool(use_cache), + "mode": "normal" if use_cache else "refresh", + "compiler_identity": None, + "compile_hits": 0, + "compile_misses": 0, + "write_errors": [], + } + + +def _compile_cache_summary(cache): + if cache is None: + return { + "mode": "disabled", + "compile_hits": 0, + "compile_misses": 0, + "write_errors": [], + } + return { + key: cache[key] + for key in ("mode", "compile_hits", "compile_misses", "write_errors") + } + + +def _program_output_path(source, build_dir, role): + digest = hashlib.sha256(f"{role}\0{source}".encode("utf-8")).hexdigest()[:16] + return Path(build_dir) / ( + f"{role}-{digest}.exe" if os.name == "nt" else f"{role}-{digest}" + ) + + +def _compile_cache_key(source, build_dir, role, cache, cancellation): + if cache["compiler_identity"] is None: + cancellation.remaining() + cache["compiler_identity"] = _compiler_cache_identity() + relative = Path(source).relative_to(Path(build_dir).parent).as_posix() + testlib = Path(__file__).resolve().parents[1] / "references" / "testlib.h" + identity = { + "schema_version": JUDGE_QA_COMPILE_CACHE_SCHEMA_VERSION, + "policy_version": JUDGE_QA_POLICY_VERSION, + "source_hash": cache["source_hash"], + "source": relative, + "role": role, + "compiler_identity": cache["compiler_identity"], + "platform": platform.system(), + "machine": platform.machine(), + "static": platform.system() == "Windows", + "for_linux": platform.system() == "Windows" and role == "validator", + "testlib_hash": hash_file(testlib), + } + encoded = json.dumps( + identity, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest(), identity + + +def _regular_cache_file(path, maximum): + try: + info = Path(path).lstat() + except OSError: + return False + return ( + stat.S_ISREG(info.st_mode) + and not _link_like(info) + and 0 <= info.st_size <= maximum + ) + + +def _restore_cached_binary(cache, key, identity, destination): + metadata_path = cache["root"] / f"{key}.json" + binary_path = cache["root"] / f"{key}.bin" + if not _regular_cache_file( + metadata_path, MAX_JUDGE_QA_COMPILE_CACHE_METADATA_BYTES + ) or not _regular_cache_file( + binary_path, MAX_JUDGE_QA_COMPILE_CACHE_BINARY_BYTES + ): + return False + try: + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + if ( + not isinstance(metadata, dict) + or metadata.get("schema_version") != JUDGE_QA_COMPILE_CACHE_SCHEMA_VERSION + or metadata.get("key") != key + or metadata.get("identity") != identity + or metadata.get("binary_bytes") != binary_path.stat().st_size + or metadata.get("binary_sha256") != hash_file(binary_path) + ): + return False + shutil.copyfile(binary_path, destination) + if os.name != "nt": + os.chmod(destination, 0o755) + return hash_file(destination) == metadata["binary_sha256"] + except (OSError, UnicodeError, ValueError, TypeError): + Path(destination).unlink(missing_ok=True) + return False + + +def _prune_compile_cache(cache): + try: + records = sorted( + ( + path + for path in cache["root"].glob("*.json") + if _regular_cache_file( + path, MAX_JUDGE_QA_COMPILE_CACHE_METADATA_BYTES + ) + ), + key=lambda path: path.stat().st_mtime_ns, + reverse=True, + ) + for metadata_path in records[MAX_JUDGE_QA_COMPILE_CACHE_ENTRIES:]: + key = metadata_path.stem + metadata_path.unlink(missing_ok=True) + (cache["root"] / f"{key}.bin").unlink(missing_ok=True) + except OSError as exc: + cache["write_errors"].append(_bounded_message(exc)) + + +def _publish_cached_binary(cache, key, identity, binary): + binary = Path(binary) + try: + size = binary.stat().st_size + if size > MAX_JUDGE_QA_COMPILE_CACHE_BINARY_BYTES: + raise OSError( + f"compiled binary is {size} bytes; cache limit is " + f"{MAX_JUDGE_QA_COMPILE_CACHE_BINARY_BYTES}" + ) + destination = cache["root"] / f"{key}.bin" + temporary = cache["root"] / f"{key}.{uuid.uuid4().hex}.tmp" + try: + shutil.copyfile(binary, temporary) + digest = hash_file(temporary) + os.replace(temporary, destination) + finally: + temporary.unlink(missing_ok=True) + atomic_write_json(cache["root"] / f"{key}.json", { + "schema_version": JUDGE_QA_COMPILE_CACHE_SCHEMA_VERSION, + "key": key, + "identity": identity, + "binary_bytes": size, + "binary_sha256": digest, + }) + _prune_compile_cache(cache) + except OSError as exc: + cache["write_errors"].append(_bounded_message(exc)) + + +def _compile_program_cached(source, build_dir, role, cancellation, cache=None): + source = Path(source) + if cache is None or source.suffix.casefold() != ".cpp": + return _compile_program(source, build_dir, role, cancellation) + key, identity = _compile_cache_key( + source, build_dir, role, cache, cancellation + ) + output = _program_output_path(source, build_dir, role) + if cache["use_cache"] and _restore_cached_binary( + cache, key, identity, output + ): + cache["compile_hits"] += 1 + return [str(output)], { + "role": role, + "source": identity["source"], + "kind": "cpp17", + "compiler": "g++", + "compiler_path": shutil.which("g++"), + "compiler_identity": cache["compiler_identity"], + "cache_hit": True, + } + cache["compile_misses"] += 1 + command, info = _compile_program(source, build_dir, role, cancellation) + info["compiler_identity"] = cache["compiler_identity"] + info["cache_hit"] = False + _publish_cached_binary(cache, key, identity, command[0]) + return command, info + + def _compile_program(source, build_dir, role, cancellation): source = Path(source) try: @@ -306,7 +562,7 @@ def _compile_program(source, build_dir, role, cancellation): ) digest = hashlib.sha256(f"{role}\0{source}".encode("utf-8")).hexdigest()[:16] - output = Path(build_dir) / (f"{role}-{digest}.exe" if os.name == "nt" else f"{role}-{digest}") + output = _program_output_path(source, build_dir, role) include_dir = Path(build_dir) / "include" include_dir.mkdir(parents=True, exist_ok=True) shutil.copyfile( @@ -479,12 +735,13 @@ def _validate_inputs(validator_command, problem_dir, cases, cancellation): "execution_status": execution.get("reason"), "returncode": execution.get("returncode"), "time": execution.get("time"), - "message": _bounded_message(message or execution.get("message")), + "diagnostic": _diagnostic_summary(message or execution.get("message")), } results.append(summary) if not summary["ok"]: raise ProbHubError( - f"Validator rejected Judge QA input {relative}: {summary['message']}", + f"Validator rejected Judge QA input {relative}: " + f"{summary['execution_status'] or 'unknown'}", code="judge_qa_validator_failed", ) finally: @@ -510,7 +767,7 @@ def _checker_summary(case, result): }, "matched": matched, "infrastructure_failed": infrastructure, - "message": _bounded_message(result.get("message")), + "diagnostic": _diagnostic_summary(result.get("message")), "resources": { key: result.get(key) for key in ( @@ -546,7 +803,7 @@ def _interactor_summary(case, result): "actual": actual, "matched": matched, "infrastructure_failed": infrastructure, - "message": _bounded_message(result.get("message")), + "diagnostic": _diagnostic_summary(result.get("message")), "resources": result.get("resources") or {}, "traffic": result.get("traffic") or {}, "transcript": { @@ -727,7 +984,7 @@ def _live_identity(problem_dir, cancellation=None): def _evidence_path(problem_dir): - return Path(problem_dir) / ".probhub" / JUDGE_QA_EVIDENCE_FILENAME + return evidence_path(problem_dir) def _publish_evidence(problem_dir, evidence, expected_identity, cancellation): @@ -759,6 +1016,7 @@ def _build_evidence( probes, elapsed, overall_timeout, + cache=None, ): now = datetime.now(timezone.utc).isoformat() return { @@ -809,6 +1067,7 @@ def _build_evidence( ), }, "elapsed": elapsed, + "cache": _compile_cache_summary(cache), "compilers": compilers, "validators": validators, "cases": cases, @@ -817,13 +1076,24 @@ def _build_evidence( } -def _execute_snapshot(snapshot_problem, config, report, cancellation): +def _execute_snapshot( + snapshot_problem, + config, + report, + cancellation, + *, + compile_cache=None, +): limits = _runtime_limits(config) build_dir = Path(snapshot_problem) / ".judge-qa-build" build_dir.mkdir(parents=True, exist_ok=True) compiled = [] - validator_command, compile_info = _compile_program( - _program_path(snapshot_problem, config, "validator"), build_dir, "validator", cancellation + validator_command, compile_info = _compile_program_cached( + _program_path(snapshot_problem, config, "validator"), + build_dir, + "validator", + cancellation, + compile_cache, ) compiled.append(compile_info) validators = _validate_inputs( @@ -831,8 +1101,12 @@ def _execute_snapshot(snapshot_problem, config, report, cancellation): ) if report["judge_type"] == "custom": - checker_command, compile_info = _compile_program( - _program_path(snapshot_problem, config, "checker"), build_dir, "checker", cancellation + checker_command, compile_info = _compile_program_cached( + _program_path(snapshot_problem, config, "checker"), + build_dir, + "checker", + cancellation, + compile_cache, ) compiled.append(compile_info) cases, probes, status = _run_checker_qa( @@ -840,8 +1114,12 @@ def _execute_snapshot(snapshot_problem, config, report, cancellation): ) else: limits.update(_interactive_limits(config, limits)) - interactor_command, compile_info = _compile_program( - _program_path(snapshot_problem, config, "interactor"), build_dir, "interactor", cancellation + interactor_command, compile_info = _compile_program_cached( + _program_path(snapshot_problem, config, "interactor"), + build_dir, + "interactor", + cancellation, + compile_cache, ) compiled.append(compile_info) contestant_commands = {} @@ -849,11 +1127,12 @@ def _execute_snapshot(snapshot_problem, config, report, cancellation): source = (case.get("contestant") or {}).get("source") if not source or source.casefold() in contestant_commands: continue - command, compile_info = _compile_program( + command, compile_info = _compile_program_cached( resolve_problem_regular_file(snapshot_problem, source), build_dir, "contestant", cancellation, + compile_cache, ) contestant_commands[source.casefold()] = command compiled.append(compile_info) @@ -867,11 +1146,6 @@ def _execute_snapshot(snapshot_problem, config, report, cancellation): cancellation, ) probes = [] - if any(item.get("kind") == "cpp17" for item in compiled): - compiler_identity = _compiler_identity(cancellation) - for item in compiled: - if item.get("kind") == "cpp17": - item["compiler_identity"] = compiler_identity return limits, validators, compiled, cases, probes, status @@ -881,6 +1155,7 @@ def judge_qa_problem( *, timeout=DEFAULT_JUDGE_QA_TIMEOUT_SECONDS, cancel_check=None, + use_cache=True, ): """Execute configured Checker/Interactor fixtures without mutating formal artifacts.""" @@ -947,6 +1222,11 @@ def judge_qa_problem( ), report.get("fixture_hash"), ) + compile_cache = _new_compile_cache( + problem_dir, + identity[0], + use_cache, + ) temporary_root = Path(tempfile.mkdtemp(prefix="probhub-judge-qa-")) snapshot_problem = temporary_root / "problem" execution = None @@ -962,7 +1242,11 @@ def judge_qa_problem( cancellation, ) execution = _execute_snapshot( - snapshot_problem, copied_config, copied_report, cancellation + snapshot_problem, + copied_config, + copied_report, + cancellation, + compile_cache=compile_cache, ) _, _, snapshot_identity = _live_identity( snapshot_problem, cancellation @@ -1019,6 +1303,7 @@ def judge_qa_problem( cases=cases, probes=probes, validators=validators, + cache=_compile_cache_summary(compile_cache), ) elapsed = time.monotonic() - started evidence = _build_evidence( @@ -1030,6 +1315,7 @@ def judge_qa_problem( probes, elapsed, timeout, + compile_cache, ) _publish_evidence(problem_dir, evidence, identity, cancellation) return _result( @@ -1042,6 +1328,7 @@ def judge_qa_problem( evidence=evidence, evidence_path=str(_evidence_path(problem_dir)), evidence_published=True, + cache=_compile_cache_summary(compile_cache), ) except ProcessCancelled as exc: if cancellation.cause == "overall_timeout": diff --git a/probhub/linting.py b/probhub/linting.py index a306c35..bbc20a3 100644 --- a/probhub/linting.py +++ b/probhub/linting.py @@ -15,7 +15,11 @@ from .datagen import recipe_coverage, resolve_data_dir from .errors import ProbHubError from .hashing import files_under, hash_file, hash_paths -from .judge_qa import inspect_judge_qa, judge_fixture_tree_paths +from .judge_qa import ( + evaluate_judge_qa_evidence, + inspect_judge_qa, + judge_fixture_tree_paths, +) from .metadata import build_meta, normalize_display_name from .problem_paths import ProblemPathError, resolve_problem_regular_file from .solutions import analyze_solution_verification @@ -611,6 +615,15 @@ def lint_problem(root, workspace, entry): warnings.extend(solution_verification.get("warnings") or []) source_hash = compute_source_hash(problem_dir, config) data_hash = compute_data_hash(problem_dir, config) + judge_qa_evidence = evaluate_judge_qa_evidence( + problem_dir, + config, + judge_qa, + source_hash, + data_hash, + ) + judge_qa["evidence"] = judge_qa_evidence + warnings.extend(judge_qa_evidence["warnings"]) calibration = evaluate_calibration( problem_dir, config, @@ -623,6 +636,7 @@ def lint_problem(root, workspace, entry): *constraint_reconciliation["diagnostics"], *(solution_verification.get("diagnostics") or []), *judge_qa["diagnostics"], + *judge_qa_evidence["diagnostics"], ] return { "id": entry["id"], @@ -741,6 +755,15 @@ def problem_status( "pdf_hash": hash_file(problem_dir / "problem.pdf"), "package_hash": hash_file(problem_dir.parent / f"{config['id']}.zip"), } + judge_qa = inspect_judge_qa(problem_dir, config) + judge_qa_evidence = evaluate_judge_qa_evidence( + problem_dir, + config, + judge_qa, + current["source_hash"], + current["data_hash"], + ) + judge_qa["evidence"] = judge_qa_evidence if root is not None and workspace is not None: current["workspace_hash"] = ( compute_workspace_hash(root, workspace) @@ -781,9 +804,16 @@ def problem_status( if builder_fingerprint_error is not None else {} ), - "warnings": calibration["warnings"], - "diagnostics": calibration["diagnostics"], + "warnings": [ + *calibration["warnings"], + *judge_qa_evidence["warnings"], + ], + "diagnostics": [ + *calibration["diagnostics"], + *judge_qa_evidence["diagnostics"], + ], "calibration": calibration, + "judge_qa": judge_qa, } stale = [] if manifest.get("schema_version") != BUILD_MANIFEST_SCHEMA_VERSION: @@ -828,7 +858,14 @@ def problem_status( else {} ), "manifest": manifest, - "warnings": calibration["warnings"], - "diagnostics": calibration["diagnostics"], + "warnings": [ + *calibration["warnings"], + *judge_qa_evidence["warnings"], + ], + "diagnostics": [ + *calibration["diagnostics"], + *judge_qa_evidence["diagnostics"], + ], "calibration": calibration, + "judge_qa": judge_qa, } diff --git a/probhub/reporting.py b/probhub/reporting.py index beb7ffd..ed65d93 100644 --- a/probhub/reporting.py +++ b/probhub/reporting.py @@ -360,6 +360,50 @@ def _calibration_profile(calibration): } +def _judge_qa_profile(lint_result): + inspection = (lint_result or {}).get("judge_qa") or {} + evidence = inspection.get("evidence") or {} + keys = ( + "state", + "configured", + "applicable", + "judge_type", + "target_guarantee", + "published_at", + "platform", + "declared_cases", + "evidence_cases", + "matched_cases", + "declared_probes", + "evidence_probes", + "manual_review_probes", + "cases", + "probes", + ) + defaults = { + "state": "not-configured", + "configured": False, + "applicable": bool(inspection.get("applicable")), + "judge_type": inspection.get("judge_type"), + "target_guarantee": False, + "declared_cases": len(inspection.get("cases") or []), + "evidence_cases": 0, + "matched_cases": 0, + "declared_probes": len( + (inspection.get("robustness") or {}).get("probes") or [] + ), + "evidence_probes": 0, + "manual_review_probes": 0, + "cases": [], + "probes": [], + } + return { + key: evidence.get(key, defaults.get(key)) + for key in keys + if key in evidence or key in defaults + } + + def _kill_matrix(config, groups, cases, calibration): wrong_entries = normalize_solution_entries(config)["wrong"] evidence_state = (calibration or {}).get("state", "missing") @@ -510,6 +554,7 @@ def _problem_report(root, workspace, entry, position, lint_result): }, }, "calibration": _calibration_profile(calibration), + "judge_qa": _judge_qa_profile(lint_result), "solution_verification": (lint_result or {}).get("solution_verification") or {}, "kill_matrix": _kill_matrix(config, groups, cases, calibration), "diagnostics": report_diagnostics, @@ -561,6 +606,9 @@ def build_workspace_report(root, workspace, selected=None): "calibration_states": dict(Counter( problem["calibration"]["state"] for problem in problems )), + "judge_qa_states": dict(Counter( + problem["judge_qa"]["state"] for problem in problems + )), "warnings": sum(item.get("severity") == "warning" for item in diagnostics), "errors": sum(item.get("severity") == "error" for item in diagnostics) + sum(len(problem["lint"]["errors"]) for problem in problems), @@ -594,6 +642,20 @@ def _format_headroom(value): return "—" if value is None else f"{float(value):.3g}x" +def _format_qa_actual(value): + value = value if isinstance(value, dict) else {} + parts = [value.get("status") or "—"] + if value.get("timeout_kind"): + parts.append(str(value["timeout_kind"])) + if value.get("termination_reason"): + parts.append(str(value["termination_reason"])) + return "/".join(parts) + + +def _format_qa_expected(value): + return _format_qa_actual(value) + + def render_markdown_report(report): workspace = report["workspace"] summary = report["summary"] @@ -607,12 +669,12 @@ def render_markdown_report(report): "", "## 题目概览", "", - "| 题号 | ID | 题名 | 难度 | 标签 | Sample | Secret | Recipe 覆盖 | TL 余量 |", - "|---|---|---|---:|---|---:|---:|---:|---:|", + "| 题号 | ID | 题名 | 难度 | 标签 | Sample | Secret | Recipe 覆盖 | TL 余量 | Judge QA |", + "|---|---|---|---:|---|---:|---:|---:|---:|---|", ] for problem in report["problems"]: lines.append( - "| {label} | {id} | {name} | {difficulty} | {tags} | {sample} | {secret} | {recipes} | {headroom} |".format( + "| {label} | {id} | {name} | {difficulty} | {tags} | {sample} | {secret} | {recipes} | {headroom} | {judge_qa} |".format( label=_markdown_escape(problem["label"]), id=_markdown_escape(problem["id"]), name=_markdown_escape(problem["name"]), @@ -622,6 +684,7 @@ def render_markdown_report(report): secret=problem["tests"]["secret"]["cases"], recipes=_format_ratio(problem["recipes"]["coverage_ratio"]), headroom=_format_headroom(problem["calibration"]["primary_headroom"]), + judge_qa=_markdown_escape(problem["judge_qa"]["state"]), ) ) @@ -636,6 +699,10 @@ def render_markdown_report(report): f"targeted {recipes['targeted']}/{recipes['total']}；near-boundary {recipes['near_boundary']}/{recipes['total']}", f"- 校准：{problem['calibration']['state']}；primary accepted TL 余量 " f"{_format_headroom(problem['calibration']['primary_headroom'])}；`target_guarantee: false`", + f"- Judge QA：{problem['judge_qa']['state']}；case " + f"{problem['judge_qa']['matched_cases']}/{problem['judge_qa']['declared_cases']}；" + f"probe {problem['judge_qa']['evidence_probes']}/{problem['judge_qa']['declared_probes']}；" + f"人工复核 {problem['judge_qa']['manual_review_probes']}", f"- 累计约束：{problem['aggregate_constraints']['state']}；" f"matched {problem['aggregate_constraints']['summary']['matched']}；" f"statement-only {problem['aggregate_constraints']['summary']['statement_only']}；" @@ -657,6 +724,31 @@ def render_markdown_report(report): f"{_format_ratio(group['secret_ratio'])} | " f"{_markdown_escape(', '.join(group['targets']) or '—')} |" ) + judge_qa = problem["judge_qa"] + if judge_qa["state"] == "current" and ( + judge_qa["cases"] or judge_qa["probes"] + ): + lines.extend([ + "", + "### Judge QA", + "", + "| ID | Purpose | Expected | Actual | Result |", + "|---|---|---|---|---|", + ]) + for item in judge_qa["cases"]: + lines.append( + f"| {_markdown_escape(item.get('id'))} | " + f"{_markdown_escape(item.get('purpose'))} | " + f"{_markdown_escape(_format_qa_expected(item.get('expected')))} | " + f"{_markdown_escape(_format_qa_actual(item.get('actual')))} | passed |" + ) + for item in judge_qa["probes"]: + result = "review" if item.get("manual_review_required") else "passed" + lines.append( + f"| {_markdown_escape(item.get('id'))} | " + f"{_markdown_escape(item.get('purpose'))} | probe | " + f"{_markdown_escape(_format_qa_actual(item.get('actual')))} | {result} |" + ) matrix = problem["kill_matrix"] if matrix["rows"]: columns = matrix["columns"] @@ -701,7 +793,8 @@ def render_text_report(report): lines.append( f"[{problem['label']}] {problem['id']} {problem['name']} | 难度 {problem['difficulty'] if problem['difficulty'] is not None else '—'} " f"| 标签 {tags} | sample {problem['tests']['sample']['cases']} / secret {problem['tests']['secret']['cases']} " - f"| recipe {_format_ratio(problem['recipes']['coverage_ratio'])} | TL {_format_headroom(problem['calibration']['primary_headroom'])}" + f"| recipe {_format_ratio(problem['recipes']['coverage_ratio'])} | TL {_format_headroom(problem['calibration']['primary_headroom'])} " + f"| QA {problem['judge_qa']['state']}" ) recipes = problem["recipes"] lines.append( @@ -722,6 +815,27 @@ def render_text_report(report): f"validator-only={aggregate['summary']['validator_only']} " f"dynamic={aggregate['summary']['dynamic']}" ) + judge_qa = problem["judge_qa"] + lines.append( + f" Judge QA: cases={judge_qa['matched_cases']}/{judge_qa['declared_cases']} " + f"probes={judge_qa['evidence_probes']}/{judge_qa['declared_probes']} " + f"manual-review={judge_qa['manual_review_probes']}" + ) + if judge_qa["state"] == "current": + for item in judge_qa["cases"]: + lines.append( + f" case {item.get('id')}: " + f"{_format_qa_expected(item.get('expected'))} -> " + f"{_format_qa_actual(item.get('actual'))} (passed)" + ) + for item in judge_qa["probes"]: + disposition = ( + "review" if item.get("manual_review_required") else "passed" + ) + lines.append( + f" probe {item.get('id')}: " + f"{_format_qa_actual(item.get('actual'))} ({disposition})" + ) if problem["groups"]: lines.append(" 数据组:") for group in problem["groups"]: diff --git a/references/checker-interactor.md b/references/checker-interactor.md index 437978b..9e3d828 100644 --- a/references/checker-interactor.md +++ b/references/checker-interactor.md @@ -167,6 +167,39 @@ validation: custom interactive `output_validators/` 是生成物，不要手工修改。修改 Checker/Interactor 后重新执行 `probhub judge ` 和 `probhub build `。 +## 3.1 Judge QA 主动测试 + +新题或修改过 Checker/Interactor 后，在 `probhub.yaml` 中增加 `judge.qa`，并把测试素材放在题目目录内： + +```text +/ +├── judge-fixtures/ # 显式 input、jury_answer、contestant_output +└── code/judge-qa/ # Interactor 的 C++/Python 模拟选手 +``` + +配置必须使用 `schema_version: 1`。Checker fixture 可引用 `case: sample/` 或 `secret/`，也可互斥地提供 `input`、`jury_answer`；`contestant_output` 始终是题目目录内 `judge-fixtures/` 下的普通文件，期望状态为 `AC` 或 `WA`。Interactor fixture 的 `contestant` 必须二选一：`source: code/judge-qa/`（`.cpp` 或 `.py`）或内建 `behavior: early-eof|idle|output-flood`，期望状态可为 `AC`、`WA`、`RE`、`TLE`、`MLE`、`OLE`，TLE 可附 `timeout_kind: idle|total`。 + +Checker 可额外声明： + +```yaml +judge: + qa: + schema_version: 1 + robustness: + baseline: accepts-alternative + probes: [empty, truncated, extra-token, oversized] + cases: + - id: accepts-alternative + purpose: valid alternative + case: sample/basic + contestant_output: judge-fixtures/checker/alternative.out + expected: {status: AC} +``` + +`probhub judge-qa --no-cache` 会以正式 Validator、Checker/Interactor 和进程控制策略执行全部 fixture；每次都执行 fixture verdict，`--no-cache` 只强制重编译。Checker 自身 `_fail`、Interactor 崩溃、资源/进程树/清理故障是基础设施失败，不能用 `expected` 声明为成功。自动 Checker 探针返回 AC 时需要人工确认没有误放行。fixture 以原始字节计算 `fixture_hash`，按 Windows 大小写不敏感 ID/路径去重，并受数量、单文件和总字节上限约束；它们不进入 DOMjudge ZIP。 + +成功 evidence 只保存有界状态、期望/实际 verdict 和脱敏原因，不保存 stdout、stderr、feedback 正文或 transcript 条目。完整成功原子发布 `judge-qa-evidence-v1.json`；失败、取消、超时、输入变化或发布错误保留上一份成功 evidence。`lint`/`status` 将 evidence 报告为 `not-configured`、`missing`、`current`、`stale` 或 `invalid`，后三者是 warning；已配置题目的 `seal` 仍要求 Judge QA `passed` 且 evidence `current`。 + ## 4. 工艺守则 ### Validator diff --git a/references/cli.md b/references/cli.md index dc9a57f..93f43c9 100644 --- a/references/cli.md +++ b/references/cli.md @@ -383,6 +383,20 @@ probhub judge L01 --no-cache 缓存事件包含 `mode`、`compile_hits/misses`、`validator_hits/misses`、`case_hits/misses` 和 `probe_hits/misses`。进程树、OLE、校准探针或资源限制语义变化会提升缓存 Schema，防止旧结果绕过新策略。 +## 9.1 `judge-qa` + +```powershell +probhub judge-qa [ID...] [--no-cache] +``` + +`judge-qa` 只适用于 `judge.type: custom` 和 `judge.type: interactive` 且已配置 `judge.qa.schema_version: 1` 的题目；standard 题未配置时返回 `ok: true, applicable: false, status: not-configured`。Checker fixture 直接把声明的 contestant output 交给正式 Checker，Interactor fixture 编译并运行题目级模拟选手；二者都先用正式 Validator 验证输入。每个 fixture 都必须得到声明的期望状态，Checker/Interactor 自身 `_fail`、崩溃、资源故障、无法建立完整进程树或清理失败始终是基础设施失败。 + +自动 Checker 探针可从 AC 基线派生 `empty`、`truncated`、`extra-token`、`oversized`；探针返回 AC 只表示执行成功，状态仍会带 `judge_qa_probe_manual_review_required`，必须由作者确认没有误放行。Interactor 可用 `early-eof`、`idle`、`output-flood` 作为受控模拟行为；题目特定协议必须写入 `code/judge-qa/`。 + +只缓存内容寻址的编译结果，fixture verdict、Validator 和探针每次都会真实执行；`--no-cache` 强制重新编译但不能跳过 QA。完整成功才原子发布 `/.probhub/judge-qa-evidence-v1.json`，失败、取消、超时、输入变化、锁竞争或发布失败保留上一份成功 evidence。evidence 只保存有界状态和脱敏摘要，不保存 stdout、stderr、feedback 正文或 transcript 条目。 + +命令结果状态为 `not-configured`、`passed`、`expectation-failed`、`infrastructure-failed` 或 `cancelled`。lint/status 会把 evidence 映射为 `not-configured`、`missing`、`current`、`stale` 或 `invalid`；missing/stale/invalid 是 warning，不改变正式产物 `current/stale` 或 lint 退出码。`seal` 对已配置 QA 要求命令状态 `passed` 且 evidence `current`，否则返回 `seal_judge_qa_failed`，不创建新的 sealed checkpoint。正式 build 还会拒绝没有当前 QA evidence 的旧 sealed checkpoint。 + ## 10. `stress` ```powershell @@ -473,10 +487,12 @@ probhub seal L10 --rounds 3000 --seed 12345 1. 单题 lint； 2. judge，`--no-cache` 时完整重跑； -3. 若配置了 `stress`，按配置或 `--rounds` 执行固定 seed 差分测试； -4. 复核 live source/data hash 未在验证期间变化； -5. 写入 sealed checkpoint 和验证证据； -6. 使用所有题目的最新 checkpoint 组装完整试卷 generation。 +3. 对已配置 `judge.qa` 的题目运行 Judge QA； +4. 刷新 lint 视图，确认 QA evidence 为 `current`； +5. 若配置了 `stress`，按配置或 `--rounds` 执行固定 seed 差分测试； +6. 复核 live source/data hash 未在验证期间变化； +7. 写入 sealed checkpoint 和验证证据； +8. 使用所有题目的最新 checkpoint 组装完整试卷 generation。 单独组装和检查当前 generation： @@ -542,11 +558,12 @@ probhub build [ID...] [--skip-judge] [--no-cache] 3. 要求 collection 中每道题的最新 checkpoint 均为与 live source/data 匹配的 sealed revision；否则以 `sealed_revision_required` 失败且不创建构建快照。 4. 复制受控输入快照；后续 judge、排版和打包只读取该快照。 5. judge 所选题目。 -6. 在快照中编译一次完整 Typst 集合并提取所选单题 PDF。 -7. 在快照中生成 DOMjudge 配置、构建并验证全部所选 ZIP。 -8. 为所有所选题目生成带同一 `batch_id` 与各自 `sealed_revision_id` 的 Manifest。 -9. 发布前重新计算 live 输入哈希，并复核所有 sealed revision；变化时以 `inputs_changed` 或 `sealed_revision_changed` 失败。 -10. 全部准备成功后发布共享产物与所选产物，Manifest 最后替换。 +6. 复核已配置 Judge QA 的 sealed checkpoint 仍包含当前通过的 QA evidence；否则以 `sealed_revision_required` 失败。 +7. 在快照中编译一次完整 Typst 集合并提取所选单题 PDF。 +8. 在快照中生成 DOMjudge 配置、构建并验证全部所选 ZIP。 +9. 为所有所选题目生成带同一 `batch_id` 与各自 `sealed_revision_id` 的 Manifest。 +10. 发布前重新计算 live 输入哈希，并复核所有 sealed revision；变化时以 `inputs_changed` 或 `sealed_revision_changed` 失败。 +11. 全部准备成功后发布共享产物与所选产物，Manifest 最后替换。 即使执行 `build L01`，也会为了题序与页码编译全卷，并要求整场所有题目都已 seal，但只评测、提取、打包和更新 L01。正式发布推荐在全部题目 seal 后一次传入全部 ID。 @@ -656,6 +673,14 @@ npm install -g . 正式 build 要求整场每道题都已 `seal`，并且 seal 后未再修改 live source/data。对提示中的题目重新运行 `seal `，待全部题目 sealed 后执行一次多 ID build。 +### `seal_judge_qa_failed` + +题目配置了 Checker/Interactor 主动测试，但 Judge QA 没有得到 `passed`，或上一次成功 evidence 已不再与当前输入匹配。先检查 `probhub judge-qa --no-cache` 的结构化结果和 fixture，再修复 `_fail`、协议、期望状态或清理问题后重新 seal。不要把 `FAIL` 当成错解被击杀，也不要手工删除旧 evidence。 + +### `judge_qa_evidence_missing` / `judge_qa_evidence_stale` / `judge_qa_evidence_invalid` + +这些是 lint/status 的非阻断体检 warning。对已配置的题目运行 `probhub judge-qa --no-cache` 会重新生成 evidence；如果要正式交付或重新 seal，必须让状态回到 `current`。旧未配置工作区不会因为升级 Core 而被强制补 QA。 + ### `problem.pdf` 缺失 先执行： diff --git a/references/generations.md b/references/generations.md index 0a0f365..e66431a 100644 --- a/references/generations.md +++ b/references/generations.md @@ -40,7 +40,7 @@ probhub checkpoint L10 probhub seal L10 --no-cache --seed 12345 ``` -`seal` 执行所选题目的 lint、judge，以及配置存在时的 stress。成功后写入带结构化证据的 sealed checkpoint，并自动组装一份完整试卷 generation。 +`seal` 执行所选题目的 lint、judge、已配置的 Judge QA，以及配置存在时的 stress。成功后写入带结构化证据的 sealed checkpoint，并自动组装一份完整试卷 generation。对已配置 `judge.qa` 的题目，QA 必须返回 `passed` 且 evidence 为 `current`；否则不会创建 sealed checkpoint。旧未配置工作区仍保持兼容。 单独组装或查看当前 generation： diff --git a/references/workspace-schema-v1.md b/references/workspace-schema-v1.md index 1b7b813..09b4dba 100644 --- a/references/workspace-schema-v1.md +++ b/references/workspace-schema-v1.md @@ -183,6 +183,62 @@ judge: `validator` 始终校验输入。`checker` 和 `interactor` 使用 ProbHub 附带的 DOMjudge/testlib 协议；完整参数、退出状态与模板见 `references/checker-interactor.md`。Core 会从规范源码生成 `output_validators/validate/validate.cpp` 和 `testlib.h`，不得手工维护生成目录。 +### Judge QA Schema v1 + +`judge.qa` 是 custom/interactive 题的可选题目级主动测试配置。standard 题不能配置它；未配置的旧题保持兼容，但新 custom/interactive 题在 Agent 交付前必须配置并通过 Judge QA。所有 QA 路径都必须是题目目录内的普通非符号链接文件，fixture 按原始字节参与 `fixture_hash`。 + +Checker 示例： + +```yaml +judge: + type: custom + validator: code/validator.cpp + checker: code/checker.cpp + qa: + schema_version: 1 + robustness: + baseline: accepts-alternative + probes: [empty, truncated, extra-token, oversized] + cases: + - id: accepts-alternative + purpose: valid alternative output + case: sample/basic + contestant_output: judge-fixtures/checker/alternative.out + expected: {status: AC} + - id: rejects-extra-token + purpose: extra token + input: judge-fixtures/checker/extra.in + jury_answer: judge-fixtures/checker/extra.ans + contestant_output: judge-fixtures/checker/extra.out + expected: {status: WA} +``` + +Interactor 示例： + +```yaml +judge: + type: interactive + validator: code/validator.cpp + interactor: code/interactor.cpp + qa: + schema_version: 1 + cases: + - id: normal-protocol + purpose: normal protocol + case: secret/basic + contestant: {source: code/judge-qa/normal.cpp} + expected: {status: AC} + - id: idle-player + purpose: idle contestant + case: secret/basic + contestant: {behavior: idle} + expected: {status: TLE, timeout_kind: idle} +``` + +Checker fixture 的期望状态只能是 `AC`/`WA`；Interactor 可使用 `AC`、`WA`、`RE`、`TLE`、`MLE`、`OLE`，其中 TLE 可声明 `timeout_kind: idle|total`。Interactor 的题目特定模拟选手源码放在 `code/judge-qa/`，内建行为只有 `early-eof`、`idle`、`output-flood`。`judge-fixtures/` 与 `code/judge-qa/` 都属于规范源，会被 source/checkpoint 跟踪，但不进入正式 PDF、ZIP、Manifest 或 DOMjudge 数据。 + +Schema 固定限制 fixture 数量、单文件大小、总字节、诊断、transcript 和运行时间；ID 与路径按 Windows 大小写不敏感去重。`probhub judge-qa ` 每次真实执行 fixture、Validator 和探针，只缓存内容寻址的编译结果。成功才原子发布 `/.probhub/judge-qa-evidence-v1.json`；失败、取消、超时、输入变化、锁竞争或发布失败保留上一份成功 evidence。evidence 状态由 lint/status 报告为 `not-configured`、`missing`、`current`、`stale` 或 `invalid`，missing/stale/invalid 是 warning；`seal` 和正式 `build` 对已配置 QA 要求当前通过的 evidence。 + ### Stress differential testing `stress` 是可选的单题差分测试配置： diff --git a/tests/test_batch_build.py b/tests/test_batch_build.py index c38ead7..8cc612a 100644 --- a/tests/test_batch_build.py +++ b/tests/test_batch_build.py @@ -1,11 +1,13 @@ import json import os +import platform import tempfile import unittest from pathlib import Path from types import SimpleNamespace from unittest.mock import patch +from probhub import __version__ from probhub.build_lock import workspace_build_lock, workspace_file_lock from probhub.builder_fingerprint import ( GENERATION_SCHEMA_VERSION, @@ -14,15 +16,24 @@ from probhub.building import ( _assert_publish_targets_available, _recover_build_publish_transactions, + assert_collection_seals_unchanged, build_workspace, + create_build_plan, package_workspace, publish_build, + require_collection_sealed, ) +from probhub.calibration import SANDBOX_CACHE_SCHEMA_VERSION from probhub.cli import command_status from probhub.errors import ProbHubError from probhub.generations import checkpoint_revision, create_problem_checkpoint from probhub.hashing import hash_file from probhub.io import write_yaml +from probhub.judge_qa import inspect_judge_qa +from probhub.judge_qa_evidence import ( + JUDGE_QA_EVIDENCE_SCHEMA_VERSION, + JUDGE_QA_POLICY_VERSION, +) from probhub.linting import ( BUILD_MANIFEST_SCHEMA_VERSION, compute_data_hash, @@ -130,6 +141,94 @@ def seal_problem(self, root, workspace, entry): expected_data_hash=compute_data_hash(problem_dir, config), ) + def configure_checker_qa(self, root, entry): + problem_dir, config = load_problem(root, entry) + (problem_dir / "code/checker.cpp").write_text( + "int main(){return 0;}\n", encoding="utf-8" + ) + output = problem_dir / "judge-fixtures/checker/accepted.out" + output.parent.mkdir(parents=True) + output.write_text("1\n", encoding="utf-8") + config["judge"].update({ + "type": "custom", + "checker": "code/checker.cpp", + "qa": { + "schema_version": 1, + "cases": [{ + "id": "accepted-output", + "purpose": "valid", + "case": "sample/1", + "contestant_output": "judge-fixtures/checker/accepted.out", + "expected": {"status": "AC"}, + }], + }, + }) + write_yaml(problem_dir / "probhub.yaml", config) + return load_problem(root, entry) + + def checker_qa_evidence(self, problem_dir, config): + inspection = inspect_judge_qa(problem_dir, config) + return { + "status": "passed", + "applicable": True, + "code": "judge_qa_passed", + "schema_version": JUDGE_QA_EVIDENCE_SCHEMA_VERSION, + "policy_version": JUDGE_QA_POLICY_VERSION, + "sandbox_schema_version": SANDBOX_CACHE_SCHEMA_VERSION, + "probhub_version": __version__, + "source_hash": compute_source_hash(problem_dir, config), + "data_hash": compute_data_hash(problem_dir, config), + "fixture_hash": inspection["fixture_hash"], + "published_at": "2026-08-05T00:00:00+00:00", + "measurement": { + "platform": platform.system(), + "machine": platform.machine(), + "target_guarantee": False, + }, + "limits": {}, + "cache": {}, + "compilers": [ + { + "role": "validator", + "source": "code/validator.cpp", + "kind": "cpp17", + "compiler_identity": "fixture", + }, + { + "role": "checker", + "source": "code/checker.cpp", + "kind": "cpp17", + "compiler_identity": "fixture", + }, + ], + "validators": [ + { + "input": "data/sample/1.in", + "ok": True, + "diagnostic": { + "present": False, + "bytes": 0, + "truncated": False, + }, + } + ], + "cases": [{ + "id": "accepted-output", + "purpose": "valid", + "expected": {"status": "AC"}, + "actual": {"status": "AC"}, + "matched": True, + "infrastructure_failed": False, + "diagnostic": { + "present": False, + "bytes": 0, + "truncated": False, + }, + }], + "probes": [], + "cleanup": {"ok": True, "snapshot_removed": True}, + } + def create_workspace(self, root, *, sealed=True): (root / ".probhub").mkdir(parents=True) (root / "typst/contest").mkdir(parents=True) @@ -564,6 +663,111 @@ def test_multi_problem_build_rejects_unsealed_batch_before_staging(self): for path, content in artifacts.items(): self.assertEqual(path.read_bytes(), content) + def test_build_requires_current_judge_qa_evidence_in_sealed_revision(self): + with tempfile.TemporaryDirectory() as temp: + root, workspace = self.create_workspace(Path(temp), sealed=False) + entries = problem_entries(workspace) + self.configure_checker_qa(root, entries[0]) + self.seal_problem(root, workspace, entries[0]) + self.seal_problem(root, workspace, entries[1]) + + with patch("probhub.building.create_build_snapshot") as snapshot: + with self.assertRaises(ProbHubError) as raised: + build_workspace(root, workspace, entries, run_judge=False) + + self.assertEqual(raised.exception.code, "sealed_revision_required") + self.assertIn("no passed Judge QA evidence", str(raised.exception)) + snapshot.assert_not_called() + + with tempfile.TemporaryDirectory() as temp: + root, workspace = self.create_workspace(Path(temp), sealed=False) + entries = problem_entries(workspace) + problem_dir, config = self.configure_checker_qa(root, entries[0]) + qa_evidence = self.checker_qa_evidence(problem_dir, config) + create_problem_checkpoint( + root, + workspace, + entries[0], + state="sealed", + evidence={"judge_qa": qa_evidence}, + expected_source_hash=compute_source_hash(problem_dir, config), + expected_data_hash=compute_data_hash(problem_dir, config), + ) + self.seal_problem(root, workspace, entries[1]) + + plan = create_build_plan(root, workspace, entries) + checkpoints = require_collection_sealed(plan) + + self.assertEqual(set(checkpoints), {"A", "B"}) + + def test_build_rejects_stale_or_invalid_judge_qa_checkpoint_evidence(self): + mutations = ( + ("inputs", lambda evidence: evidence.update(source_hash="stale")), + ( + "structure", + lambda evidence: evidence["cases"][0].update( + message="forbidden-feedback" + ), + ), + ) + for expected, mutate in mutations: + with self.subTest(expected=expected), tempfile.TemporaryDirectory() as temp: + root, workspace = self.create_workspace(Path(temp), sealed=False) + entries = problem_entries(workspace) + problem_dir, config = self.configure_checker_qa(root, entries[0]) + qa_evidence = self.checker_qa_evidence(problem_dir, config) + mutate(qa_evidence) + create_problem_checkpoint( + root, + workspace, + entries[0], + state="sealed", + evidence={"judge_qa": qa_evidence}, + expected_source_hash=compute_source_hash(problem_dir, config), + expected_data_hash=compute_data_hash(problem_dir, config), + ) + self.seal_problem(root, workspace, entries[1]) + + with patch("probhub.building.create_build_snapshot") as snapshot: + with self.assertRaises(ProbHubError) as raised: + build_workspace(root, workspace, entries, run_judge=False) + + self.assertEqual(raised.exception.code, "sealed_revision_required") + self.assertIn(f"sealed Judge QA evidence is {expected}", str(raised.exception)) + snapshot.assert_not_called() + + def test_build_rechecks_judge_qa_checkpoint_evidence_before_publish(self): + with tempfile.TemporaryDirectory() as temp: + root, workspace = self.create_workspace(Path(temp), sealed=False) + entries = problem_entries(workspace) + problem_dir, config = self.configure_checker_qa(root, entries[0]) + qa_evidence = self.checker_qa_evidence(problem_dir, config) + create_problem_checkpoint( + root, + workspace, + entries[0], + state="sealed", + evidence={"judge_qa": qa_evidence}, + expected_source_hash=compute_source_hash(problem_dir, config), + expected_data_hash=compute_data_hash(problem_dir, config), + ) + self.seal_problem(root, workspace, entries[1]) + plan = create_build_plan(root, workspace, entries) + with patch( + "probhub.building.validate_judge_qa_evidence_document", + side_effect=( + {"state": "current"}, + {"state": "stale", "reason": "core"}, + ), + ) as validate: + checkpoints = require_collection_sealed(plan) + with self.assertRaises(ProbHubError) as raised: + assert_collection_seals_unchanged(plan, checkpoints) + + self.assertEqual(raised.exception.code, "sealed_revision_changed") + self.assertIn("sealed Judge QA evidence is core", str(raised.exception)) + self.assertEqual(validate.call_count, 2) + def test_single_problem_build_rejects_unsealed_collection_peer(self): with tempfile.TemporaryDirectory() as temp: root, workspace = self.create_workspace(Path(temp), sealed=False) diff --git a/tests/test_generations.py b/tests/test_generations.py index a498c10..052bcef 100644 --- a/tests/test_generations.py +++ b/tests/test_generations.py @@ -28,6 +28,7 @@ from probhub.linting import compute_data_hash, compute_source_hash from probhub.workspace import load_problem from probhub.workspace import load_workspace, problem_entries +from tests.fixture_support import copy_workspace_fixture class GenerationTests(unittest.TestCase): @@ -502,9 +503,29 @@ def test_seal_records_evidence_and_requests_generation(self): "rounds_completed": 7, "master_seed": 99, } + order = [] + + def run_judge(*_args, **_kwargs): + order.append("judge") + return judge_result + + def run_qa(*_args, **_kwargs): + order.append("judge-qa") + return { + "ok": True, + "applicable": False, + "status": "not-configured", + "judge_type": "standard", + } + + def run_stress(*_args, **_kwargs): + order.append("stress") + return stress_result + with ( - patch("probhub.cli.judge_problem", return_value=judge_result), - patch("probhub.cli.stress_problem", return_value=stress_result) as stress, + patch("probhub.cli.judge_problem", side_effect=run_judge), + patch("probhub.cli.judge_qa_problem", side_effect=run_qa), + patch("probhub.cli.stress_problem", side_effect=run_stress) as stress, patch( "probhub.cli.assemble_exam_generation", return_value={"ok": True, "generation_id": "fixture-generation"}, @@ -532,7 +553,12 @@ def test_seal_records_evidence_and_requests_generation(self): checkpoint = payload["checkpoint"] self.assertEqual(checkpoint["state"], "sealed") self.assertTrue(checkpoint["evidence"]["judge"]["ok"]) + self.assertEqual( + checkpoint["evidence"]["judge_qa"]["status"], + "not-configured", + ) self.assertEqual(checkpoint["evidence"]["stress"]["rounds_completed"], 7) + self.assertEqual(order, ["judge", "judge-qa", "stress"]) self.assertEqual(payload["generation"]["generation_id"], "fixture-generation") self.assertEqual(stress.call_args.kwargs["rounds"], 7) self.assertEqual(stress.call_args.kwargs["master_seed"], 99) @@ -541,6 +567,96 @@ def test_seal_records_evidence_and_requests_generation(self): self.builder_fingerprint, ) + def test_configured_judge_qa_failure_does_not_create_sealed_checkpoint(self): + judge_result = { + "ok": True, + "returncode": 0, + "final": { + "type": "final", + "status": "passed", + "code": "all_expectations_met", + }, + "cache": {}, + } + for status in ( + "expectation-failed", + "infrastructure-failed", + "cancelled", + "not-configured", + ): + with self.subTest(status=status), tempfile.TemporaryDirectory() as temp: + fixture = copy_workspace_fixture("checker-qa", temp) + output = io.StringIO() + with ( + patch("probhub.cli.judge_problem", return_value=judge_result), + patch("probhub.cli.judge_qa_problem", return_value={ + "ok": status == "not-configured", + "applicable": status != "not-configured", + "status": status, + "code": "fixture-failure", + }), + patch( + "probhub.cli.compute_builder_fingerprint", + return_value=self.builder_fingerprint, + ), + patch("probhub.cli.assemble_exam_generation") as assemble, + redirect_stdout(output), + ): + code = cli_main([ + "--workspace", + str(fixture.root), + "--json", + "seal", + "F06", + "--no-cache", + ]) + + self.assertEqual(code, 1, output.getvalue()) + payload = json.loads(output.getvalue()) + self.assertEqual(payload["code"], "seal_judge_qa_failed") + self.assertIsNone(latest_checkpoint(fixture.root, "F06")) + assemble.assert_not_called() + + def test_configured_judge_qa_success_records_current_evidence(self): + judge_result = { + "ok": True, + "returncode": 0, + "final": { + "type": "final", + "status": "passed", + "code": "all_expectations_met", + }, + "cache": {}, + } + with tempfile.TemporaryDirectory() as temp: + fixture = copy_workspace_fixture("checker-qa", temp) + output = io.StringIO() + with ( + patch("probhub.cli.judge_problem", return_value=judge_result), + patch("probhub.cli.compute_builder_fingerprint", return_value=self.builder_fingerprint), + patch( + "probhub.cli.assemble_exam_generation", + return_value={"ok": True, "generation_id": "qa-generation"}, + ), + redirect_stdout(output), + ): + code = cli_main([ + "--workspace", + str(fixture.root), + "--json", + "seal", + "F06", + "--no-cache", + ]) + + self.assertEqual(code, 0, output.getvalue()) + payload = json.loads(output.getvalue()) + checkpoint = payload["checkpoint"] + self.assertEqual(checkpoint["state"], "sealed") + self.assertEqual(checkpoint["evidence"]["judge_qa"]["status"], "passed") + self.assertEqual(payload["generation"]["generation_id"], "qa-generation") + self.assertIsNotNone(latest_checkpoint(fixture.root, "F06")) + def test_generation_status_reports_schema_and_builder_staleness(self): with tempfile.TemporaryDirectory() as temp: root, workspace = self.create_workspace(Path(temp)) diff --git a/tests/test_judge_qa_cli.py b/tests/test_judge_qa_cli.py new file mode 100644 index 0000000..da3ab42 --- /dev/null +++ b/tests/test_judge_qa_cli.py @@ -0,0 +1,77 @@ +import io +import json +import tempfile +import unittest +from contextlib import redirect_stdout +from unittest.mock import patch + +from probhub.cli import main as cli_main +from tests.fixture_support import copy_workspace_fixture + + +class JudgeQACliTests(unittest.TestCase): + def copy_fixture(self, name="checker-qa"): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + return copy_workspace_fixture(name, temporary.name) + + def invoke(self, root, *arguments): + output = io.StringIO() + with redirect_stdout(output): + code = cli_main(["--workspace", str(root), "--json", *arguments]) + return code, json.loads(output.getvalue()) + + def test_no_cache_is_forwarded_without_changing_result_shape(self): + fixture = self.copy_fixture() + result = { + "ok": True, + "applicable": True, + "status": "passed", + "code": "judge_qa_passed", + "cache": { + "mode": "refresh", + "compile_hits": 0, + "compile_misses": 2, + }, + } + with patch("probhub.cli.judge_qa_problem", return_value=result) as execute: + code, payload = self.invoke( + fixture.root, + "judge-qa", + "F06", + "--no-cache", + ) + + self.assertEqual(code, 0, payload) + self.assertEqual(payload["problems"]["F06"]["status"], "passed") + self.assertFalse(execute.call_args.kwargs["use_cache"]) + + def test_not_configured_is_successful_but_not_reported_as_passed(self): + fixture = self.copy_fixture("custom") + + code, payload = self.invoke(fixture.root, "judge-qa") + + self.assertEqual(code, 0, payload) + result = next(iter(payload["problems"].values())) + self.assertTrue(result["ok"]) + self.assertFalse(result["applicable"]) + self.assertEqual(result["status"], "not-configured") + + def test_failure_states_return_nonzero(self): + fixture = self.copy_fixture() + for status in ( + "expectation-failed", + "infrastructure-failed", + "cancelled", + ): + with self.subTest(status=status), patch( + "probhub.cli.judge_qa_problem", + return_value={"ok": False, "applicable": True, "status": status}, + ): + code, payload = self.invoke(fixture.root, "judge-qa", "F06") + self.assertEqual(code, 1, payload) + self.assertEqual(payload["problems"]["F06"]["status"], status) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_judge_qa_evidence.py b/tests/test_judge_qa_evidence.py new file mode 100644 index 0000000..e823a28 --- /dev/null +++ b/tests/test_judge_qa_evidence.py @@ -0,0 +1,302 @@ +import copy +import json +import platform +import tempfile +import unittest +from pathlib import Path + +from probhub import __version__ +from probhub.calibration import SANDBOX_CACHE_SCHEMA_VERSION +from probhub.io import atomic_write_json +from probhub.judge_qa import evaluate_judge_qa_evidence, inspect_judge_qa +from probhub.judge_qa_evidence import ( + JUDGE_QA_EVIDENCE_SCHEMA_VERSION, + JUDGE_QA_POLICY_VERSION, + MAX_JUDGE_QA_EVIDENCE_BYTES, + evidence_path, +) +from probhub.linting import ( + compute_data_hash, + compute_source_hash, + lint_workspace, + problem_status, +) +from probhub.reporting import ( + build_workspace_report, + render_markdown_report, + render_text_report, +) +from probhub.workspace import load_problem, load_workspace, problem_entries +from tests.fixture_support import copy_workspace_fixture + + +class JudgeQAEvidenceTests(unittest.TestCase): + def copy_fixture(self, name="checker-qa"): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + fixture = copy_workspace_fixture(name, temporary.name) + root, workspace = load_workspace(fixture.root) + entry = problem_entries(workspace)[0] + problem, config = load_problem(root, entry) + inspection = inspect_judge_qa(problem, config) + return fixture, root, workspace, entry, problem, config, inspection + + @staticmethod + def valid_evidence(problem, config, inspection): + judge = config["judge"] + compilers = [ + { + "role": "validator", + "source": judge["validator"], + "kind": "cpp17", + "compiler_identity": "fixture-compiler", + }, + { + "role": "checker" if inspection["judge_type"] == "custom" else "interactor", + "source": judge[ + "checker" if inspection["judge_type"] == "custom" else "interactor" + ], + "kind": "cpp17", + "compiler_identity": "fixture-compiler", + }, + ] + contestant_sources = sorted({ + (case.get("contestant") or {}).get("source") + for case in inspection["cases"] + if (case.get("contestant") or {}).get("source") + }) + for source in contestant_sources: + compilers.append({ + "role": "contestant", + "source": source, + "kind": "python" if source.endswith(".py") else "cpp17", + **( + {"compiler_identity": "fixture-compiler"} + if source.endswith(".cpp") + else {} + ), + }) + validators = [ + { + "input": value, + "ok": True, + "diagnostic": {"present": False, "bytes": 0, "truncated": False}, + } + for value in sorted({case["input"] for case in inspection["cases"]}) + ] + cases = [] + for case in inspection["cases"]: + expected = dict(case["expected"]) + cases.append({ + "id": case["id"], + "purpose": case["purpose"], + "expected": expected, + "actual": { + **expected, + "actor": "session", + "execution_status": "completed", + }, + "matched": True, + "infrastructure_failed": False, + "diagnostic": {"present": True, "bytes": 7, "truncated": False}, + }) + probes = [ + { + "id": probe, + "purpose": "robustness-probe", + "expected": {}, + "actual": {"status": "WA"}, + "matched": True, + "infrastructure_failed": False, + "manual_review_required": False, + "diagnostic": {"present": True, "bytes": 7, "truncated": False}, + } + for probe in (inspection.get("robustness") or {}).get("probes") or [] + ] + return { + "schema_version": JUDGE_QA_EVIDENCE_SCHEMA_VERSION, + "policy_version": JUDGE_QA_POLICY_VERSION, + "sandbox_schema_version": SANDBOX_CACHE_SCHEMA_VERSION, + "probhub_version": __version__, + "source_hash": compute_source_hash(problem, config), + "data_hash": compute_data_hash(problem, config), + "fixture_hash": inspection["fixture_hash"], + "published_at": "2026-08-05T00:00:00+00:00", + "measurement": { + "platform": platform.system(), + "machine": platform.machine(), + "target_guarantee": False, + }, + "limits": {}, + "cache": { + "mode": "normal", + "compile_hits": 0, + "compile_misses": len(compilers), + "write_errors": [], + }, + "compilers": compilers, + "validators": validators, + "cases": cases, + "probes": probes, + "cleanup": {"ok": True, "snapshot_removed": True}, + } + + def evaluate(self, problem, config, inspection): + return evaluate_judge_qa_evidence( + problem, + config, + inspection, + compute_source_hash(problem, config), + compute_data_hash(problem, config), + ) + + def write_valid(self, problem, config, inspection): + evidence = self.valid_evidence(problem, config, inspection) + atomic_write_json(evidence_path(problem), evidence) + return evidence + + def test_not_configured_ignores_orphan_evidence(self): + _, _, _, _, problem, config, inspection = self.copy_fixture("custom") + evidence_path(problem).parent.mkdir(parents=True, exist_ok=True) + evidence_path(problem).write_text("not-json", encoding="utf-8") + + result = self.evaluate(problem, config, inspection) + + self.assertEqual(result["state"], "not-configured", result) + self.assertEqual(result["diagnostics"], []) + + def test_missing_and_current_states_are_non_blocking_lint_results(self): + _, root, workspace, _, problem, config, inspection = self.copy_fixture() + missing = lint_workspace(root, workspace) + self.assertTrue(missing["ok"], missing) + qa = missing["problems"][0]["judge_qa"]["evidence"] + self.assertEqual(qa["state"], "missing", qa) + self.assertIn("judge_qa_evidence_missing", { + item["code"] for item in qa["diagnostics"] + }) + + self.write_valid(problem, config, inspection) + current = lint_workspace(root, workspace) + qa = current["problems"][0]["judge_qa"]["evidence"] + self.assertTrue(current["ok"], current) + self.assertEqual(qa["state"], "current", qa) + self.assertEqual(qa["matched_cases"], qa["declared_cases"]) + + def test_stale_reasons_have_stable_precedence(self): + _, _, _, _, problem, config, inspection = self.copy_fixture() + base = self.valid_evidence(problem, config, inspection) + mutations = { + "schema": lambda item: item.update(schema_version=999), + "policy": lambda item: item.update(policy_version=999), + "sandbox": lambda item: item.update(sandbox_schema_version=999), + "core": lambda item: item.update(probhub_version="0.0.0"), + "inputs": lambda item: item.update(source_hash="changed"), + "platform": lambda item: item["measurement"].update(platform="Other"), + } + for reason, mutate in mutations.items(): + with self.subTest(reason=reason): + evidence = copy.deepcopy(base) + mutate(evidence) + evidence["cases"] = [] + atomic_write_json(evidence_path(problem), evidence) + result = self.evaluate(problem, config, inspection) + self.assertEqual(result["state"], "stale", result) + self.assertEqual(result["reason"], reason, result) + + def test_invalid_file_states_are_distinct_from_missing(self): + _, _, _, _, problem, config, inspection = self.copy_fixture() + path = evidence_path(problem) + path.parent.mkdir(parents=True, exist_ok=True) + cases = ( + (b"{", "invalid_json"), + (b"[]", "invalid_json"), + (b"x" * (MAX_JUDGE_QA_EVIDENCE_BYTES + 1), "too_large"), + ) + for payload, reason in cases: + with self.subTest(reason=reason): + path.write_bytes(payload) + result = self.evaluate(problem, config, inspection) + self.assertEqual(result["state"], "invalid", result) + self.assertEqual(result["reason"], reason, result) + + def test_structure_failures_cannot_be_reported_as_current(self): + _, _, _, _, problem, config, inspection = self.copy_fixture() + base = self.valid_evidence(problem, config, inspection) + + def add_stream(item): + item["cases"][0]["stdout"] = "forbidden" + + mutations = ( + lambda item: item.update(cleanup={"ok": False, "snapshot_removed": True}), + lambda item: item["measurement"].update(target_guarantee=True), + lambda item: item["cases"].pop(), + lambda item: item["cases"][0].update(matched=False), + lambda item: item["validators"].pop(), + lambda item: item["compilers"].pop(), + lambda item: item["probes"].pop(), + add_stream, + lambda item: item["cases"][0].update(message="feedback-secret"), + ) + for index, mutate in enumerate(mutations): + with self.subTest(index=index): + evidence = copy.deepcopy(base) + mutate(evidence) + atomic_write_json(evidence_path(problem), evidence) + result = self.evaluate(problem, config, inspection) + self.assertEqual(result["state"], "invalid", result) + self.assertEqual(result["reason"], "structure", result) + + def test_probe_ac_is_current_but_requires_manual_review(self): + _, _, _, _, problem, config, inspection = self.copy_fixture() + evidence = self.valid_evidence(problem, config, inspection) + evidence["probes"][0]["actual"]["status"] = "AC" + evidence["probes"][0]["manual_review_required"] = True + atomic_write_json(evidence_path(problem), evidence) + + result = self.evaluate(problem, config, inspection) + + self.assertEqual(result["state"], "current", result) + self.assertEqual(result["manual_review_probes"], 1) + self.assertIn("judge_qa_probe_manual_review_required", { + item["code"] for item in result["diagnostics"] + }) + + def test_status_exposes_qa_without_changing_formal_state(self): + _, root, workspace, entry, problem, config, inspection = self.copy_fixture() + self.write_valid(problem, config, inspection) + + status = problem_status(problem, config, root, workspace) + + self.assertEqual(status["state"], "never-built", status) + self.assertEqual(status["judge_qa"]["evidence"]["state"], "current") + + def test_report_exposes_only_bounded_current_qa_summaries(self): + _, root, workspace, _, problem, config, inspection = self.copy_fixture() + evidence = self.write_valid(problem, config, inspection) + evidence["cases"][0]["resources"] = {"sentinel_secret": "hidden"} + atomic_write_json(evidence_path(problem), evidence) + + report = build_workspace_report(root, workspace) + item = report["problems"][0]["judge_qa"] + text = render_text_report(report) + markdown = render_markdown_report(report) + + self.assertEqual(report["summary"]["judge_qa_states"], {"current": 1}) + self.assertEqual(item["state"], "current", item) + self.assertEqual(item["matched_cases"], item["declared_cases"]) + self.assertNotIn("resources", item["cases"][0]) + self.assertNotIn("sentinel_secret", json.dumps(report)) + self.assertIn("QA current", text) + self.assertIn("### Judge QA", markdown) + + evidence["source_hash"] = "stale" + atomic_write_json(evidence_path(problem), evidence) + stale = build_workspace_report(root, workspace) + stale_item = stale["problems"][0]["judge_qa"] + self.assertEqual(stale_item["state"], "stale") + self.assertEqual(stale_item["cases"], []) + self.assertNotIn("stale-result-must-stay-hidden", json.dumps(stale)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_judge_qa_execution.py b/tests/test_judge_qa_execution.py index 1311da7..c4d521c 100644 --- a/tests/test_judge_qa_execution.py +++ b/tests/test_judge_qa_execution.py @@ -13,7 +13,12 @@ from probhub.errors import ProbHubError from probhub.build_lock import workspace_file_lock from probhub.io import write_yaml -from probhub.judge_qa import judge_qa_problem +from probhub.judge_qa import ( + evaluate_judge_qa_evidence, + inspect_judge_qa, + judge_qa_problem, +) +from probhub.linting import compute_data_hash, compute_source_hash from probhub.process_control import process_alive import probhub.judge_qa_runtime as runtime from tests.fixture_support import FIXTURE_ROOT, copy_workspace_fixture @@ -62,6 +67,8 @@ def formal_workspace_bytes(fixture): path.relative_to(fixture.root).as_posix(): path.read_bytes() for path in fixture.root.rglob("*") if path.is_file() + and "/.probhub/compile/" + not in "/" + path.relative_to(fixture.root).as_posix() and not ( path.parent.name == ".probhub" and ( @@ -129,6 +136,19 @@ def assert_successful_evidence(self, fixture, result): ) self.assertTrue(result["evidence"]["cleanup"]["snapshot_removed"]) self.assert_evidence_has_no_stream_content(result["evidence"]) + serialized = json.dumps(result["evidence"], ensure_ascii=False) + self.assertNotIn("accepted absolute value", serialized) + self.assertNotIn("wrong absolute value", serialized) + config = fixture.config() + inspection = inspect_judge_qa(fixture.problem, config) + evaluated = evaluate_judge_qa_evidence( + fixture.problem, + config, + inspection, + compute_source_hash(fixture.problem, config), + compute_data_hash(fixture.problem, config), + ) + self.assertEqual(evaluated["state"], "current", evaluated) self.assertEqual( list(evidence_path.parent.glob(f"{evidence_path.name}.*.tmp")), [], @@ -174,6 +194,58 @@ def test_checker_fixture_executes_and_publishes_bounded_evidence(self): )) self.assertEqual(self.formal_workspace_bytes(fixture), before) + def test_compile_cache_hits_but_fixture_verdicts_execute_every_time(self): + fixture = self.copy_fixture("checker-qa") + original_checker = runtime.run_checker_to_files + calls = [] + + def counted_checker(*args, **kwargs): + calls.append(Path(args[3]).name) + return original_checker(*args, **kwargs) + + with mock.patch.object( + runtime, + "run_checker_to_files", + side_effect=counted_checker, + ): + first = judge_qa_problem(fixture.root, fixture.problem) + first_calls = len(calls) + second = judge_qa_problem(fixture.root, fixture.problem) + second_calls = len(calls) - first_calls + refreshed = judge_qa_problem( + fixture.root, + fixture.problem, + use_cache=False, + ) + + self.assertTrue(first["ok"], first) + self.assertEqual(first["cache"]["compile_hits"], 0, first) + self.assertEqual(first["cache"]["compile_misses"], 2, first) + self.assertTrue(second["ok"], second) + self.assertEqual(second["cache"]["compile_hits"], 2, second) + self.assertEqual(second["cache"]["compile_misses"], 0, second) + self.assertEqual(first_calls, 6) + self.assertEqual(second_calls, first_calls) + self.assertTrue(refreshed["ok"], refreshed) + self.assertEqual(refreshed["cache"]["mode"], "refresh") + self.assertEqual(refreshed["cache"]["compile_hits"], 0, refreshed) + self.assertEqual(refreshed["cache"]["compile_misses"], 2, refreshed) + + def test_corrupt_compile_cache_binary_is_not_executed(self): + fixture = self.copy_fixture("checker-qa") + first = judge_qa_problem(fixture.root, fixture.problem) + self.assertTrue(first["ok"], first) + binary = next( + (fixture.problem / ".probhub/compile/judge-qa-v1").glob("*.bin") + ) + binary.write_bytes(b"corrupt") + + second = judge_qa_problem(fixture.root, fixture.problem) + + self.assertTrue(second["ok"], second) + self.assertEqual(second["cache"]["compile_hits"], 1, second) + self.assertEqual(second["cache"]["compile_misses"], 1, second) + def test_cpp_checker_qa_runs_from_unicode_and_space_path(self): with tempfile.TemporaryDirectory() as temp: workspace = Path(temp) / "中文 workspace" @@ -258,6 +330,45 @@ def test_resource_and_control_failures_cannot_satisfy_checker_expectations(self) self.assertTrue(summary["infrastructure_failed"], summary) self.assertFalse(summary["matched"], summary) + def test_untrusted_checker_and_interactor_diagnostics_are_reduced_to_byte_counts(self): + case = {"id": "diagnostic", "purpose": "diagnostic", "expected": {"status": "WA"}} + sentinel = "judge-feedback-secret-7f2d" + checker = runtime._checker_summary(case, { + "verdict": "WA", + "execution_status": "completed", + "failure_kind": None, + "actor": "checker", + "message": sentinel, + "cleanup": {"ok": True, "errors": []}, + }) + interactor = runtime._interactor_summary(case, { + "status": "WA", + "actor": "interactor", + "execution_status": "completed", + "failure_kind": None, + "message": sentinel, + "cleanup": {"ok": True, "errors": []}, + }) + for summary in (checker, interactor): + serialized = json.dumps(summary, ensure_ascii=False) + self.assertNotIn(sentinel, serialized) + self.assertEqual(summary["diagnostic"]["bytes"], len(sentinel.encode("utf-8"))) + self.assertTrue(summary["diagnostic"]["present"]) + self.assertFalse(summary["diagnostic"]["truncated"]) + + oversized = runtime._checker_summary(case, { + "verdict": "WA", + "execution_status": "completed", + "failure_kind": None, + "actor": "checker", + "message": "x" * (runtime.DIAGNOSTIC_LIMIT_BYTES + 1), + "cleanup": {"ok": True, "errors": []}, + }) + self.assertEqual( + oversized["diagnostic"]["bytes"], runtime.DIAGNOSTIC_LIMIT_BYTES + ) + self.assertTrue(oversized["diagnostic"]["truncated"]) + def test_checker_timeout_matches_formal_judge_policy(self): fixture = self.copy_fixture("checker-qa") report = runtime.inspect_judge_qa(fixture.problem, fixture.config()) @@ -839,10 +950,14 @@ def fake_compile(_source, _build_dir, role, _cancellation): mock.patch.object(runtime, "_compile_program", side_effect=fake_compile), mock.patch.object(runtime.tempfile, "mkdtemp", side_effect=make_temporary), ): + # Leave enough startup budget for the controlled checker to publish + # both PID markers on busy Windows runners; the checker still sleeps + # beyond the overall deadline, so the timeout/cleanup assertion is + # unchanged. result = judge_qa_problem( fixture.root, fixture.problem, - timeout=1.5, + timeout=5.0, ) self.assertFalse(result["ok"], result)