From 4b1c0e74f308b3bd7f353460fb8621c35e2b72e9 Mon Sep 17 00:00:00 2001 From: zzy <17092805+mz2007@user.noreply.gitee.com> Date: Sat, 5 Sep 2026 16:16:22 +0800 Subject: [PATCH 1/6] feat: add ContinuousCasting domain (CuttingOptimization + CuttingOptimizationOnline) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two original benchmarks inspired by CUMCM 2021 Problem D, formalized into deterministic, stdlib-only, verifiable optimization tasks. - CuttingOptimization (offline): cut a continuously cast billet (with fixed 0.8 m scrap segments) into pieces to minimize total scrapped length, then match a customer target. Reference DP = 88.7, baseline = 72.5; the optimum is reachable (openevolve hits 88.7). - CuttingOptimizationOnline (closed-loop): scrap segments are revealed only within a reveal_lead horizon; the agent decides each cut with only the visible past and is scored against the full hidden defect set. Clairvoyant reference = 76.4, baseline = 52.4, agents (3 frameworks x 3 runs) = 70.43 +- 0.83 — an online agent cannot reach the clairvoyant optimum (info asymmetry), which is the key difference vs offline. Integrity/anti-cheat mirrors the accepted CVRP/TelecomBackup benchmarks: reference solver and generator excluded from the sandbox + forbidden by the validator, FRONTIER_* env stripping (host side-channel), runtime instance generation, EVOLVE-BLOCK fixed-region check, determinism probe, sandbox-evaluator test coverage, multiseed_stat tool, and honest README/Task docs. Unit tests: 35 (offline) + 23 (online) pass. --- TASK_DETAILS.md | 9 + TASK_DETAILS_zh-CN.md | 9 + .../CuttingOptimization/.gitignore | 2 + .../CuttingOptimization/README.md | 136 +++++++++ .../CuttingOptimization/README_zh-CN.md | 94 ++++++ .../CuttingOptimization/Task.md | 105 +++++++ .../baseline/result_log.txt | 31 ++ .../CuttingOptimization/baseline/solver.py | 76 +++++ .../frontier_eval/agent_files.txt | 4 + .../frontier_eval/artifact_files.txt | 2 + .../frontier_eval/constraints.txt | 19 ++ .../frontier_eval/copy_files.txt | 6 + .../frontier_eval/eval_command.txt | 1 + .../frontier_eval/eval_cwd.txt | 1 + .../frontier_eval/evaluator.py | 36 +++ .../frontier_eval/initial_program.txt | 1 + .../frontier_eval/readonly_files.txt | 4 + .../frontier_eval/run_eval.py | 126 ++++++++ .../data/instances/instance_1.json | 30 ++ .../data/instances/instance_2.json | 30 ++ .../data/instances/instance_3.json | 38 +++ .../data/instances/instance_4.json | 38 +++ .../data/instances/instance_5.json | 30 ++ .../data/instances/instance_6.json | 50 ++++ .../data/instances/instance_7.json | 42 +++ .../data/instances/instance_8.json | 42 +++ .../verification/docker/Dockerfile | 13 + .../verification/evaluate.py | 276 ++++++++++++++++++ .../verification/generator.py | 202 +++++++++++++ .../verification/multiseed_stat.py | 87 ++++++ .../verification/ref_solver.py | 111 +++++++ .../verification/requirements.txt | 2 + .../verification/simulator.py | 169 +++++++++++ .../verification/test_evaluator.py | 61 ++++ .../test_frontier_eval_evaluator.py | 103 +++++++ .../verification/test_generator.py | 57 ++++ .../verification/test_ref_solver.py | 50 ++++ .../verification/test_simulator.py | 124 ++++++++ .../verification/test_validator.py | 80 +++++ .../verification/validator.py | 154 ++++++++++ .../CuttingOptimizationOnline/.gitignore | 2 + .../CuttingOptimizationOnline/README.md | 151 ++++++++++ .../CuttingOptimizationOnline/README_zh-CN.md | 116 ++++++++ .../CuttingOptimizationOnline/Task.md | 80 +++++ .../baseline/result_log.txt | 28 ++ .../baseline/solver.py | 56 ++++ .../frontier_eval/agent_files.txt | 4 + .../frontier_eval/artifact_files.txt | 2 + .../frontier_eval/constraints.txt | 22 ++ .../frontier_eval/copy_files.txt | 6 + .../frontier_eval/eval_command.txt | 1 + .../frontier_eval/eval_cwd.txt | 1 + .../frontier_eval/evaluator.py | 31 ++ .../frontier_eval/initial_program.txt | 1 + .../frontier_eval/readonly_files.txt | 4 + .../frontier_eval/run_eval.py | 126 ++++++++ .../data/instances/instance_1.json | 33 +++ .../data/instances/instance_2.json | 33 +++ .../data/instances/instance_3.json | 41 +++ .../data/instances/instance_4.json | 53 ++++ .../data/instances/instance_5.json | 45 +++ .../data/instances/instance_6.json | 77 +++++ .../data/instances/instance_7.json | 69 +++++ .../data/instances/instance_8.json | 77 +++++ .../verification/docker/Dockerfile | 13 + .../verification/evaluate.py | 223 ++++++++++++++ .../verification/generator.py | 100 +++++++ .../verification/multiseed_stat.py | 87 ++++++ .../verification/ref_solver.py | 109 +++++++ .../verification/requirements.txt | 2 + .../verification/simulator.py | 154 ++++++++++ .../verification/test_evaluator.py | 63 ++++ .../test_frontier_eval_evaluator.py | 65 +++++ .../verification/test_generator.py | 41 +++ .../verification/test_ref_solver.py | 42 +++ .../verification/test_simulator.py | 67 +++++ .../verification/test_validator.py | 73 +++++ .../verification/validator.py | 93 ++++++ benchmarks/ContinuousCasting/README.md | 23 ++ benchmarks/ContinuousCasting/README_zh-CN.md | 19 ++ 80 files changed, 4684 insertions(+) create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/.gitignore create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/README.md create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/README_zh-CN.md create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/Task.md create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/baseline/result_log.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/baseline/solver.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/agent_files.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/artifact_files.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/constraints.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/copy_files.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/eval_command.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/eval_cwd.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/evaluator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/initial_program.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/readonly_files.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/run_eval.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_1.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_2.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_3.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_4.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_5.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_6.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_7.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_8.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/docker/Dockerfile create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/evaluate.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/generator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/multiseed_stat.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/ref_solver.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/requirements.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/simulator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/test_evaluator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/test_frontier_eval_evaluator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/test_generator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/test_ref_solver.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/test_simulator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/test_validator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimization/verification/validator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/.gitignore create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/README_zh-CN.md create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/Task.md create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/baseline/result_log.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/baseline/solver.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/agent_files.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/artifact_files.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/constraints.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/copy_files.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/eval_command.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/eval_cwd.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/evaluator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/initial_program.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/readonly_files.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/run_eval.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_1.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_2.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_3.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_4.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_5.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_6.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_7.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_8.json create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/docker/Dockerfile create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/evaluate.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/generator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/multiseed_stat.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/ref_solver.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/requirements.txt create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/simulator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_evaluator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_frontier_eval_evaluator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_generator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_ref_solver.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_simulator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_validator.py create mode 100644 benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/validator.py create mode 100644 benchmarks/ContinuousCasting/README.md create mode 100644 benchmarks/ContinuousCasting/README_zh-CN.md diff --git a/TASK_DETAILS.md b/TASK_DETAILS.md index 7475c4cc..c2d2c98b 100644 --- a/TASK_DETAILS.md +++ b/TASK_DETAILS.md @@ -346,6 +346,15 @@ We welcome new engineering problem ideas — even without complete verification EV2GymSmartCharging Upstream-aligned EV smart charging scheduling + + ContinuousCasting + CuttingOptimization + Cut a continuously cast billet into pieces to minimize scrapped length and match a customer target (offline; the optimum is reachable) + + + CuttingOptimizationOnline + Online (closed-loop) cutting where 0.8 m scrap segments are revealed only within a reveal horizon — info asymmetry keeps agents below the clairvoyant optimum + AdditiveManufacturing DiffSimThermalControl diff --git a/TASK_DETAILS_zh-CN.md b/TASK_DETAILS_zh-CN.md index e2a070a2..5e9f0559 100644 --- a/TASK_DETAILS_zh-CN.md +++ b/TASK_DETAILS_zh-CN.md @@ -346,6 +346,15 @@ Frontier-Eng 目前已覆盖以下领域的任务。每个任务均配有可运 EV2GymSmartCharging 上游对齐的电动车智能充电调度 + + ContinuousCasting + CuttingOptimization + 把连续浇铸的钢坯切成成品，最小化报废并贴近客户目标值（离线；最优可达） + + + CuttingOptimizationOnline + 在线（闭环）切割：0.8m 报废段只在揭示提前量内才告知 agent——信息不对称让 agent 达不到全知最优 + AdditiveManufacturing DiffSimThermalControl diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/.gitignore b/benchmarks/ContinuousCasting/CuttingOptimization/.gitignore new file mode 100644 index 00000000..7a60b85e --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/README.md b/benchmarks/ContinuousCasting/CuttingOptimization/README.md new file mode 100644 index 00000000..84d631f1 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/README.md @@ -0,0 +1,136 @@ +# CuttingOptimization: Online Optimization of Continuous-Casting Cutting (Frontier-Eng Benchmark) + +An **original** Frontier-Engineering benchmark inspired by the CUMCM 2021 Problem D +(«连铸切割的在线优化»), formalized into a self-contained, deterministic optimization task. + +A continuously cast steel billet is drawn at a fixed speed. Defects (crystalline-moulder +anomalies) create 0.8 m scrap segments inside the billet that must be cut out and scrapped. +A solver receives the billet length, the defect positions, and a customer target length with +an acceptance window, and must produce a **cutting plan** (a list of cut lengths that exactly +partition the whole billet) that minimizes the total scrapped length and then makes every +shipped piece as close to the target length as possible. + +The full game rules and evaluation semantics are in [Task.md](./Task.md) (Chinese). + +## Layout + +``` +benchmarks/ContinuousCasting/CuttingOptimization/ +├── baseline/solver.py # Candidate solver (EVOLVE-BLOCK region is the only editable part) +├── verification/ +│ ├── generator.py # Fixed-seed instance generator (defects + target window) +│ ├── simulator.py # Scoring simulator (validate + scrap / penalty metric) +│ ├── evaluate.py # Evaluation entry (subprocess + time budget + scoring) +│ ├── validator.py # Integrity checks (static + env stripping + determinism) +│ ├── ref_solver.py # Reference DP (1-D partition; documented "best" score) +│ ├── test_simulator.py # Unit tests: simulator correctness +│ ├── test_generator.py # Unit tests: deterministic / defect feasibility / headroom +│ ├── test_ref_solver.py # Unit tests: reference-solver validity + optimality +│ ├── test_validator.py # Unit tests: integrity checks / env stripping / determinism +│ ├── test_evaluator.py # Unit tests: end-to-end evaluation behavior +│ ├── data/instances/ # 8 fixed instances (seed-fixed, reproducible) +│ ├── docker/Dockerfile # Minimal stdlib-only python image +│ └── requirements.txt +├── frontier_eval/ # UnifiedTask metadata +├── Task.md # Task rules, interface, scoring, reference scores +└── README.md +``` + +## Requirements + +- Python >= 3.10, standard library only (no third-party dependencies). +- Runtime is pure-Python; the reference DP and the baseline solver evaluate in well under a + second per instance. + +## Run + +```powershell +# Score a solver on the fixed 8-instance set (default 60s time budget per instance) +python verification/evaluate.py baseline/solver.py + +# Add runtime-generated instances (anti-hardcoding) +python verification/evaluate.py baseline/solver.py --generate-seed + +# Tighter budget (challenge tier: 10s) +python verification/evaluate.py baseline/solver.py --time-budget 10 +``` + +### Docker + +The evaluator is pure stdlib, so a minimal `python` image suffices. Build it and use the +unified runtime's `isolation_mode=docker`: + +```bash +# Build (inside the CuttingOptimization directory) +docker build -t cutting-opt-benchmark -f verification/docker/Dockerfile . +``` + +## Tests + +```powershell +# From the task directory (stdlib unittest, no dependencies) +python -m unittest discover -s verification -p "test_*.py" +``` + +35 tests across six modules (simulator / generator / ref_solver / validator / evaluator / +sandbox evaluator): +piece-level scrap & penalty rules, feasibility checks (sum, length window, defect isolation), +determinism of generation and reference solver, reference-solver optimality vs the baseline, +validator integrity (EVOLVE-BLOCK / forbidden references / absolute paths / per-instance +hardcoding / env stripping / determinism probe), evaluator behavior (scoring, runtime +generation, cheating-candidate rejection), and the `frontier_eval/evaluator.py` sandbox entry +(consistency with `verification/evaluate.py` + cheat rejection). `verification/multiseed_stat.py` +computes multi-run mean ± std. + +## Integrity / threat model + +- **Runtime-generated instances**: with `CUTTING_EVAL_GENERATE_SEED` set, the evaluator + generates fresh instances at evaluation time (temp dir, never in the repo/sandbox), so a + candidate cannot pre-position solutions for them. +- **Candidate env stripping**: candidate subprocesses get `FRONTIER_*` / `CUTTING_EVAL_*` + variables stripped (see `verification/validator.py`), closing the host-env side channel. +- **Static checks**: EVOLVE-BLOCK markers + fixed-region byte diff vs the initial baseline, + forbidden imports of evaluation / generation / reference modules, absolute paths, + per-instance hardcoding, plus a determinism probe (two runs must match). Any violation + scores 0. +- **Sandbox scope**: the 8 fixed instances and the evaluator / validator sources are visible + to the candidate during evolution (they are needed for scoring and `verification/simulator.py` + is intentionally usable as a white-box scorer). Anti-hardcoding therefore relies on + `CUTTING_EVAL_GENERATE_SEED` (fresh instances at evaluation time — set a seed, do not use a + fixed one); the name-keyed hardcoding check is best-effort. `verification/ref_solver.py` and + `verification/generator.py` are **not** copied into the sandbox and are additionally forbidden + by the validator. +- Honest note: in process mode the candidate has host filesystem access (framework-wide + limitation); this benchmark relies on the layered defenses above. + +## Scoring + +- Instances = 8 fixed (difficulties easy/medium/hard, S = 24..150 m, 0..6 defects, target + window ±0.5 m around the customer target) + runtime-generated when `CUTTING_EVAL_GENERATE_SEED` + is set. +- **Metric**: material utilization = `100 * (billet_length - (scrap + 1e-4*penalty)) / billet_length`, + averaged over instances (0..100, higher is better). `scrap` = total scrapped length (defect + pieces + sub-8.0 m pieces + over-window excess); `penalty = Σ|delivered − target|` over shipped + pieces; the `1e-4` weight is so small that scrap strictly dominates, respecting the lexicographic + objective of the original problem (the penalty only breaks ties among equal-scrap plans). +- Malformed output / out-of-range cuts / cuts that fail to isolate a defect / crash / timeout + ⇒ 0 points for that instance. +- **Headroom guarantee**: the generator accepts only instances where the reference DP strictly + beats a naive equal-split baseline by ≥ 0.1 m of scrap, so every instance has real + optimization signal. +- Reference scores (verified on the fixed 8 instances, `verification/evaluate.py`): + - baseline (equal-split, no target-awareness): **72.5** utilization (mean scrap 25.2 m) + - reference DP (`verification/ref_solver.py`, 1-D partition): **88.7** utilization (mean scrap 10.6 m) + - per-instance reference utilization: 95.8 / 88.8 / 91.8 / 75.2 / 92.4 / 87.0 / 89.4 / 89.3 + - agent (openevolve, 10 generations, best saved program): **88.7** utilization + (run `20260903_130517`) — **equals the reference DP exactly**. + - agent (ShinkaEvolve, 15 generations, via reasoning proxy): **88.4** utilization + (near the optimum, run `20260904_182017`). + - agent (AB-MCTS, 15 iterations): **72.5** utilization (= baseline; this run did not improve — + AB-MCTS is weaker here, and low-reasoning mutations mostly regressed/invalidated). + - honest design note: the offline optimum is **reachable** (a strong agent can derive the + 1-D partition DP and hit ~88.7 = the ceiling). So the offline task's difficulty is *deriving* + the DP, not long-horizon search. The harder **online** companion + ([CuttingOptimizationOnline](../CuttingOptimizationOnline/README.md)) is where info + asymmetry keeps agents below the clairvoyant ceiling — see that README for the 3×2 matrix. + - `verification/multiseed_stat.py` gathers multi-run mean ± std across framework run dirs. diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/README_zh-CN.md b/benchmarks/ContinuousCasting/CuttingOptimization/README_zh-CN.md new file mode 100644 index 00000000..1ae3af2b --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/README_zh-CN.md @@ -0,0 +1,94 @@ +# CuttingOptimization：连铸切割的在线优化（Frontier-Eng 基准） + +一个**原创**的 Frontier-Engineering 基准，灵感来自 2021 全国大学生数学建模竞赛 D 题 +《连铸切割的在线优化》，并被形式化成**确定性、自包含**的切割优化任务。 + +一根连续浇铸的钢坯以固定速度被拉出；结晶器异常会在坯内产生 **0.8 m 的零废段**，必须被 +切出并报废。求解器收到钢坯长度、零废段位置、以及一个"目标值 + 可接受窗口"的用户需求后， +要给出一个**切割方案**（一组恰好铺满整根钢坯的切段长度），使**报废总长度最小**，并让 +**每块成品尽量贴近目标值**。 + +完整规则与评测语义见 [Task.md](./Task.md)。 + +## 目录结构 + +``` +benchmarks/ContinuousCasting/CuttingOptimization/ +├── baseline/solver.py # 候选求解器（仅 EVOLVE-BLOCK 区域可改） +├── verification/ +│ ├── generator.py # 固定种子实例生成器（零废段 + 目标窗口） +│ ├── simulator.py # 计分模拟器（校验 + 报废 / 贴合度指标） +│ ├── evaluate.py # 评测入口（subprocess + 时间预算 + 打分） +│ ├── validator.py # 完整性校验（静态 + 环境剥离 + 确定性） +│ ├── ref_solver.py # 参考解（一维划分 DP；文档化的"最优"分） +│ ├── test_simulator.py # 单测：计分器正确性 +│ ├── test_generator.py # 单测：确定性 / 零废段可行性 / headroom +│ ├── test_ref_solver.py # 单测：参考解合法性 + 最优性 +│ ├── test_validator.py # 单测：完整性校验 / 环境剥离 / 确定性 +│ ├── test_evaluator.py # 单测：端到端评测行为 +│ ├── data/instances/ # 8 个固定实例（种子固定、可复现） +│ ├── docker/Dockerfile # 极简纯标准库 python 镜像 +│ └── requirements.txt +├── frontier_eval/ # UnifiedTask 元数据 +├── Task.md # 任务规则、接口、评分、参考分 +└── README_zh-CN.md +``` + +## 运行 + +```powershell +# 在固定 8 实例上给求解器打分（默认每实例 60s） +python verification/evaluate.py baseline/solver.py + +# 加运行时生成实例（防硬编码） +python verification/evaluate.py baseline/solver.py --generate-seed + +# 更紧的时间预算（挑战档 10s） +python verification/evaluate.py baseline/solver.py --time-budget 10 +``` + +## 测试 + +```powershell +python -m unittest discover -s verification -p "test_*.py" +``` + +共 35 个单测（simulator / generator / ref_solver / validator / evaluator / 沙箱 evaluator 六个模块）： +单块报废与贴合度规则、合法校验（求和、长度窗口、零废段对齐）、生成与参考解的确定性、 +参考解优于 baseline 的最优性、validator 完整性（EVOLVE-BLOCK / 禁引用 / 绝对路径 / +按实例名硬编码 / 环境剥离 / 确定性探针）、以及 evaluator 行为（打分、运行时生成、 +作弊候选被拒）、`frontier_eval/evaluator.py` 沙箱入口一致性。`verification/multiseed_stat.py` +用于多轮均值±std。 + +## 完整性 / 威胁模型 + +- **运行时生成实例**：设置 `CUTTING_EVAL_GENERATE_SEED` 后，评测现场生成新实例（临时目录， + 不进仓库/沙箱），候选无法预先记忆。 +- **候选环境剥离**：候选子进程剥离 `FRONTIER_*` / `CUTTING_EVAL_*` 变量（见 validator.py）。 +- **静态检查**：EVOLVE-BLOCK 标记 + 固定区字节比对、禁引用评测/生成/参考解模块、绝对路径、 + 按实例名硬编码、确定性探针（两次运行输出一致）。任何违规记 0 分。 +- **沙箱范围**：8 个固定实例、evaluator/validator 源码对候选可见（打分需要，且 + `verification/simulator.py` 有意作为白盒计分器）；防硬编码依赖 `CUTTING_EVAL_GENERATE_SEED`。 + `verification/ref_solver.py` 与 `verification/generator.py` **不**复制进沙箱，并被 validator 额外禁用。 +- 说明：在进程模式下候选有主机文件系统访问（框架级限制），本基准依赖上述分层防御。 + +## 评分 + +- 实例 = 8 固定（easy/medium/hard，S=24..150 m，0..6 个零废段，目标窗口 ±0.5 m）+ 设置 + `CUTTING_EVAL_GENERATE_SEED` 时的运行时生成实例。 +- **指标**：材料利用率 `util = 100 * (S - (scrap + 1e-4*penalty)) / S`，多实例取平均（0~100，越高越好）。 + `scrap` = 总报废长度（零废段小块 + <8.0 m 整损 + 超出窗口的余量）；`penalty = Σ|交付 - 目标|`； + `1e-4` 权重极小，保证报废严格占主导（惩罚仅在报废相同时破平），符合赛题的字典序目标。 +- 非法输出 / 越界切段 / 未对齐零废段 / 崩溃 / 超时 ⇒ 该实例 0 分。 +- **headroom 保证**：生成器只接受"参考解严格优于朴素等分 ≥ 0.1 m 报废"的实例， + 保证每个实例都有真实优化信号。 +- 参考分（固定 8 实例实测）： + - baseline（均匀等分）：**72.5** 利用率（平均报废 25.2 m） + - ref_solver（一维划分 DP）：**88.7** 利用率（平均报废 10.6 m） + - agent（openevolve，10 代，best 保存程序）：**88.7** 利用率（run `20260903_130517`）——**恰好等于参考解 DP**。 + - agent（ShinkaEvolve，15 代，经推理代理）：**88.4** 利用率（run `20260904_182017`，接近最优）。 + - agent（AB-MCTS，15 迭代）：**72.5** 利用率（= baseline；本轮未提升——AB-MCTS 在此较弱，低推理下多数改进突变回归/无效）。 + - 诚实设计说明：离线最优是**可达的**（强 agent 能推导出精确的一维划分 DP 并打到 88.7 = 天花板）。 + 所以离线任务的难度在"推导 DP"，不在长程搜索。更难的**在线版** + （`CuttingOptimizationOnline`）才是信息不对称让 agent 无法达到全知天花板——见该 README 的 3×2 矩阵。 + - `verification/multiseed_stat.py` 用于跨框架运行目录做多轮均值±std。 diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/Task.md b/benchmarks/ContinuousCasting/CuttingOptimization/Task.md new file mode 100644 index 00000000..7192e868 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/Task.md @@ -0,0 +1,105 @@ +# 连铸切割的在线优化（CuttingOptimization） + +## 1. 背景 + +连铸是把钢水变成钢坯的生产过程：钢水从中间包连续浇入结晶器，按固定拉坯速度往下拉， +经二冷段凝固成钢坯，再按尺寸要求切割。切割机有一个固定的工作起点，切割必须从该起点开始。 + +当结晶器出现异常时，钢坯内会形成一小段"报废段"（本任务中统一为 **0.8 m**），必须被切出 +并报废。切割方案要同时满足两条要求（字典序）： + +1. **优先最小化切割损失**：切割损失 = 报废钢坯的总长度。 +2. **其次满足用户需求**：在损失相同的方案中，切出的成品尽量贴近用户目标值。 + +本任务把赛题抽象成一个**确定性的一维切割优化**问题：钢坯是一根一维线段，上面带若干 +零废段；求解器要给出一个把整根钢坯（含必须切出的零废段）完全切成段的切割方案，使报废 +长度最小、成品贴合目标值。 + +## 2. 输入（实例） + +实例是一个 JSON 文件，由 `verification/generator.py` 按种子生成（评测时实时生成，不可预记忆）： + +```json +{ + "seed": 3, + "process": {"speed": 1.0, "cut_time": 3, "return_time": 1, "buffer_len": 60, "scrap_len": 0.8}, + "billet": {"total_length": 106.2}, + "customer": {"target": 8.5, "target_min": 8.0, "target_max": 9.0}, + "defects": [[21.3, 22.1], [42.5, 43.3], [60.1, 60.9]], + "limits": {"min_basic": 4.8, "max_basic": 12.6, "min_process": 8.0, "max_process": 11.6} +} +``` + +含义： +- `billet.total_length`：钢坯总长 S，线段 `[0, S]`。 +- `customer.target` / `target_min` / `target_max`：用户目标值及可接受的切段长度窗口。 +- `defects`：零废段区间 `[a, b)`（每段长 0.8 m）。它们把 `[0, S]` 分成若干"干净坯段"。 +- `limits`：三段长度窗口 + - `[min_basic, max_basic] = [4.8, 12.6]`：能运走的最小/最大长度（硬约束）。 + - `[min_process, max_process] = [8.0, 11.6]`：下道工序可直接接受的长度。 + - 用户窗口 `[target_min, target_max]`：零报废的理想段长范围。 + +时间参数（拉坯速度 1.0 m/min、切一块 3 min、回程 1 min、结晶器到切割机 60 m）在 +`process` 里给出，仅作为物理背景。由于两次切割最小间隔对应材料长度 +`1.0×(3+1)=4 m < 4.8 m`，**切割机总能跟上，时间不构成约束**，故不参与评分。 + +## 3. 输出（求解格式） + +求解器以 `python baseline/solver.py ` 运行，向 stdout 打印一个 JSON： + +```json +{"cuts": [9.5, 9.4, 8.8, 0.8, 10.0, ...]} +``` + +`cuts` 是切段长度列表，依次排开**恰好铺满整个 `[0, S]`**（含每个零废段这个 0.8 m 的小块）。 +即 `sum(cuts) == total_length`（容差 1e-3）。 + +## 4. 合法性校验（硬约束） + +1. `sum(cuts) ≈ S`（容差 1e-3）。 +2. 每块长度必须在 `[4.8, 12.6]`，**除非**它恰好是某个零废段（0.8 m 小块，允许出现）。 +3. 切口必须对齐每个零废段的两个端点：零废段必须被单独切出（其两端必须是切点）； + 任何跨过零废段的成品块都会污染，判非法。 + +违反任一硬约束 ⇒ 该实例记 0 分。 + +## 5. 评分函数 + +每块长度 `c` 的报废与贴合度： + +- `c < 8.0`（且非零废段小块）：送不到下道工序 → 整块报废，`scrap += c`。 +- `c >= 8.0`：可送下道。超出 `target_max` 的部分报废，`scrap += c - min(c, target_max)`。 +- `target_min <= c <= target_max`：零报废、零惩罚（完美块）。 +- `c < target_min`（但 `c >= 8.0`）：能送但偏短 → 零报废、贴合度惩罚。 + +贴合度惩罚：`penalty = |min(c, target_max) - target|`（实际交付长度与目标值的距离）。 + +指标：**材料利用率** `util = 100 * (S - (scrap + 1e-4*penalty)) / S`，多实例取平均（0~100，越高越好）。 +其中 `score = scrap + 1e-4*penalty`，`1e-4` 极小，保证**先最小化报废、再最小化贴合度**（字典序）， +`util` 略随贴合度变化以破平。 + +> 说明：`penalty` 项只对非零废段的成品块计算；零废段小块只计入报废，不参与贴合度。 + +## 6. 参考分数（固定 8 实例，`verification/evaluate.py` 实测） + +| 求解器 | 平均利用率 | 平均报废(m) | +|---|---|---| +| baseline（均匀等分） | 72.5 | 25.2 | +| ref_solver（一维划分 DP，最优） | 88.7 | 10.6 | + +各实例参考利用率：95.8 / 88.8 / 91.8 / 75.2 / 92.4 / 87.0 / 89.4 / 89.3。 + +> agent/框架分数（统一 low 推理）：openevolve **88.7**（= 参考解）、ShinkaEvolve **88.4**、 +> AB-MCTS **72.5**（= baseline，本轮未提升）。诚实说明：**该任务的最优是可达的**——强 agent +> 能推导出精确划分 DP 并打到 ~88.7 = 天花板；难度在"推导 DP"而非长程搜索。更难的**在线版** +> （`CuttingOptimizationOnline`）才是信息不对称让 agent 打不满全知最优（见其 README）。 + +## 7. 接口契约与约束 + +- 只允许修改 `baseline/solver.py` 的 `EVOLVE-BLOCK-START` / `EVOLVE-BLOCK-END` 区域； + 标记外的代码必须与初始 baseline 逐字节一致。 +- 输出必须始终是合法切割方案；越界、未对齐零废段、崩溃或超时 ⇒ 该实例 0 分。 +- 允许 `import verification/simulator.py`（只读，用于搜索时评估候选方案）； + **禁止** import / 读取 `verification/generator.py`、`verification/ref_solver.py`、 + `verification/evaluate.py`。 +- 单实例时间预算默认 60 s，`--time-budget` 可调（挑战档 10 s）。 diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/baseline/result_log.txt b/benchmarks/ContinuousCasting/CuttingOptimization/baseline/result_log.txt new file mode 100644 index 00000000..715c2b25 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/baseline/result_log.txt @@ -0,0 +1,31 @@ +ContinuousCasting / CuttingOptimization — baseline & reference scores (8 fixed instances) + +Evaluation entry: verification/evaluate.py baseline/solver.py +Metric: material utilization = 100*(S - scrap)/S, averaged over instances (higher is better). +Reference DP: verification/ref_solver.py (1-D partition, stdlib only). + +Instance set: data/instances/instance_1.json .. instance_8.json (seed-fixed). +Mean billet length S = 94.4 m; difficulty easy/medium/hard; 0..6 defects; target window +/- 0.5 m. + +Per-instance (S, ref_util, base_util, ref_scrap, base_scrap): + i1 S= 68.9 ref= 95.79 base= 87.08 ref_scrap= 2.90 base_scrap= 8.90 + i2 S= 50.2 ref= 88.84 base= 69.32 ref_scrap= 5.60 base_scrap= 15.40 + i3 S=106.2 ref= 91.81 base= 75.71 ref_scrap= 8.70 base_scrap= 25.80 + i4 S= 71.8 ref= 75.21 base= 50.14 ref_scrap= 17.80 base_scrap= 35.80 + i5 S= 86.0 ref= 92.44 base= 69.77 ref_scrap= 6.50 base_scrap= 26.00 + i6 S=139.7 ref= 86.97 base= 76.16 ref_scrap= 18.20 base_scrap= 33.30 + i7 S=121.1 ref= 89.43 base= 72.83 ref_scrap= 12.80 base_scrap= 32.90 + i8 S=111.3 ref= 89.31 base= 78.89 ref_scrap= 11.90 base_scrap= 23.50 + +Summary: + baseline (equal-split, not target-aware): mean utilization = 72.49, mean scrap = 25.20 m + reference DP (optimal within model): mean utilization = 88.72, mean scrap = 10.55 m + mean headroom (ref_util - base_util) = 16.24 utilization points (reference beats baseline by this much) + +Every instance has headroom >= 0.1 m of scrap (generator enforces this via _interesting_ok), +so naive equal-splitting is always strictly beatable by target-aware optimization. + +Agent scores (fixed 8 instances, unified low-reasoning via reasoning proxy): + openevolve (10 generations) = 88.72 (run 20260903_130517) == reference DP + ShinkaEvolve (15 generations) = 88.43 (run 20260904_182017) near-optimum + AB-MCTS (15 iterations) = 72.49 (run 20260904_183826) == baseline (did not improve) diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/baseline/solver.py b/benchmarks/ContinuousCasting/CuttingOptimization/baseline/solver.py new file mode 100644 index 00000000..c7b2328b --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/baseline/solver.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""连铸切割 baseline 求解器（朴素基线：每个干净坯段均匀等分）。 + +用法：python solver.py +输出：stdout 打印 {"cuts": [c1, c2, ...]}，cuts 为切段长度列表，之和 = total_length。 + +只允许修改 EVOLVE-BLOCK 区域内的代码；接口契约（main/stdin-json/stdout-json）必须保留。 + +基线取"均匀等分"（朴素、不针对目标窗口）：每段长度取 [min_basic, max_basic] 内的均匀值， +但完全不考虑用户目标值 T，因此产生大量超出 target_max 的报废（或落在 target_min 下方 +的偏短块）。参考解（按目标窗口做一维划分 DP）能显著超越它，故基线是一个"正常且被明显超越"的起点。 +""" + +from __future__ import annotations + +import json +import math +import sys +from pathlib import Path + + +def clean_segments(inst: dict) -> list[list[float]]: + """干净坯段列表 [[start, length], ...]（零废段之外的可切材料）。""" + total = float(inst["billet"]["total_length"]) + defects = [list((a, b)) for a, b in (inst.get("defects") or [])] + segs: list[list[float]] = [] + prev = 0.0 + for a, b in defects: + segs.append([prev, a - prev]) + prev = b + segs.append([prev, total - prev]) + return [s for s in segs if s[1] > 1e-9] + + +def solve(inst: dict) -> list[float]: + """返回完整钢坯的切段长度列表（含零废段小块）。""" + # EVOLVE-BLOCK-START + def partition(length: float) -> list[float]: + """把一段干净坯 length 等分成若干块，每块在 [min_basic, max_basic]。""" + a = float(inst["limits"]["min_basic"]) + b = float(inst["limits"]["max_basic"]) + n = max(1, math.ceil(length / b)) + while n > 1 and length / n < a: + n -= 1 + while length / n > b: + n += 1 + base = length / n + pieces = [base] * n + pieces[-1] = length - base * (n - 1) + return [round(p, 4) for p in pieces] + + segs = clean_segments(inst) + defects = sorted((list((a, b)) for a, b in (inst.get("defects") or [])), key=lambda x: x[0]) + cuts: list[float] = [] + for i, (_start, length) in enumerate(segs): + cuts.extend(partition(length)) + if i < len(defects): + a, b = defects[i] + cuts.append(round(b - a, 4)) # 零废段强制切出 + return cuts + # EVOLVE-BLOCK-END + + +def main() -> int: + if len(sys.argv) < 2: + print("usage: python solver.py ", file=sys.stderr) + return 2 + inst_path = Path(sys.argv[1]) + inst = json.loads(inst_path.read_text(encoding="utf-8")) + cuts = solve(inst) + print(json.dumps({"cuts": cuts})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/agent_files.txt b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/agent_files.txt new file mode 100644 index 00000000..640607a1 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/agent_files.txt @@ -0,0 +1,4 @@ +README.md +Task.md +baseline/solver.py +frontier_eval/constraints.txt diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/artifact_files.txt b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/artifact_files.txt new file mode 100644 index 00000000..a52b8b16 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/artifact_files.txt @@ -0,0 +1,2 @@ +# No extra artifact files are auto-collected by default for this benchmark. +# metrics.json and artifacts.json are handled separately by UnifiedTask. diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/constraints.txt b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/constraints.txt new file mode 100644 index 00000000..8f605659 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/constraints.txt @@ -0,0 +1,19 @@ +UnifiedTask constraints: +1) Only modify `baseline/solver.py`, and only inside the EVOLVE-BLOCK-START / EVOLVE-BLOCK-END region. +2) Preserve the public contract: run as `python baseline/solver.py `, print a single JSON object + `{"cuts": [c1, c2, ...]}` to stdout -- one cut length per piece, with the concatenation summing to + `total_length` (the whole steel billet, including any 0.8 m scrap segments that must be cut out). +3) Do not modify benchmark assets, documentation, verification code, instance data, or `frontier_eval/` metadata. +4) Your output must always be a valid cutting plan. Every cut must be in [4.8, 12.6] metres (or be exactly + a 0.8 m scrap-segment piece); the cuts must align to both ends of every scrap segment (scrap segments are + fixed waste, they must be isolated); and the sum must equal the billet length. Malformed output, + out-of-range cuts, unaligned cuts, crashes, or timeouts score 0 for that instance (the simulator + validates every plan). +5) You may import `verification/simulator.py` (read-only) to evaluate candidate plans while searching, + but the plan you finally print must come from your own algorithm. You may NOT import or read + `verification/generator.py`, `verification/ref_solver.py`, or `verification/evaluate.py`. +6) Prioritize validity before optimization. The solver is given a fixed time budget per instance + (default 60s); keep it fast enough, or your plan is discarded. +7) Objective: minimize total scrap length first; among equal-scrap plans, make each shipped piece as close + to the customer target length as possible. Score = mean material utilization over instances (0-100, + higher is better), which equals 100 * (billet_length - scrap) / billet_length averaged. diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/copy_files.txt b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/copy_files.txt new file mode 100644 index 00000000..79f8b1a5 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/copy_files.txt @@ -0,0 +1,6 @@ +baseline +verification/evaluate.py +verification/simulator.py +verification/validator.py +verification/data/instances +frontier_eval diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/eval_command.txt b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/eval_command.txt new file mode 100644 index 00000000..3a31d525 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/eval_command.txt @@ -0,0 +1 @@ +FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR={benchmark_source} {python} frontier_eval/run_eval.py --candidate {candidate} --metrics-out metrics.json --artifacts-out artifacts.json diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/eval_cwd.txt b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/eval_cwd.txt new file mode 100644 index 00000000..9c558e35 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/eval_cwd.txt @@ -0,0 +1 @@ +. diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/evaluator.py b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/evaluator.py new file mode 100644 index 00000000..b4558786 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/evaluator.py @@ -0,0 +1,36 @@ +"""Unified evaluator entry point for the CuttingOptimization benchmark. + +This module is loaded by `frontier_eval/run_eval.py` and must expose a +top-level `evaluate(program_path, **kwargs)` callable. The real +implementation lives in `verification/evaluate.py`. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any + +TIME_BUDGET_S = 60.0 + + +def _load_verification_evaluator() -> Any: + evaluator_path = ( + Path(__file__).resolve().parent.parent / "verification" / "evaluate.py" + ) + spec = importlib.util.spec_from_file_location( + "_cutting_verification_evaluator", evaluator_path + ) + if spec is None or spec.loader is None: + raise RuntimeError(f"Failed to load verification evaluator from {evaluator_path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def evaluate(program_path: str, **kwargs: Any) -> Any: + module = _load_verification_evaluator() + result = module.evaluate(program_path, time_budget=TIME_BUDGET_S, **kwargs) + if isinstance(result, dict) and "metrics" in result: + return result + return {"metrics": result, "artifacts": {}} diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/initial_program.txt b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/initial_program.txt new file mode 100644 index 00000000..6645b02f --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/initial_program.txt @@ -0,0 +1 @@ +baseline/solver.py diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/readonly_files.txt b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/readonly_files.txt new file mode 100644 index 00000000..1205de75 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/readonly_files.txt @@ -0,0 +1,4 @@ +README.md +Task.md +verification +frontier_eval diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/run_eval.py b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/run_eval.py new file mode 100644 index 00000000..720d0e09 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/frontier_eval/run_eval.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import argparse +import json +import sys +import traceback +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +from typing import Any + +INVALID_COMBINED_SCORE = -1e18 + + +def _write_json(path: Path, obj: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(obj, ensure_ascii=False, indent=2, default=str) + "\n", + encoding="utf-8", + ) + + +def _normalize_result(result: Any) -> tuple[dict[str, Any], dict[str, Any]]: + if hasattr(result, "metrics") and hasattr(result, "artifacts"): + return dict(getattr(result, "metrics")), dict(getattr(result, "artifacts")) + + if isinstance(result, dict): + raw_metrics = result.get("metrics") + raw_artifacts = result.get("artifacts") + if isinstance(raw_metrics, dict): + return dict(raw_metrics), dict(raw_artifacts or {}) + return dict(result), {} + + raise TypeError( + "Evaluator must return an EvaluationResult-like object or a dict of metrics." + ) + + +def _load_local_evaluator() -> Any: + evaluator_path = Path(__file__).with_name("evaluator.py").resolve() + spec = spec_from_file_location("_frontier_eval_local_evaluator", evaluator_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Failed to load local evaluator from {evaluator_path}") + module = module_from_spec(spec) + spec.loader.exec_module(module) + try: + return getattr(module, "evaluate") + except AttributeError as exc: + raise RuntimeError( + f"Local evaluator does not define evaluate(): {evaluator_path}" + ) from exc + + +def _find_repo_root() -> Path: + import os + + env_root = os.environ.get("FRONTIER_ENGINEERING_ROOT") + if env_root: + return Path(env_root).expanduser().resolve() + + here = Path(__file__).resolve() + for parent in [here.parent, *here.parents]: + if (parent / "frontier_eval").is_dir() and (parent / "benchmarks").is_dir(): + return parent + return Path.cwd().resolve() + + +def _build_kwargs(evaluate_fn: Any) -> dict[str, Any]: + kwargs: dict[str, Any] = {} + try: + parameters = inspect_signature(evaluate_fn) + except Exception: + return kwargs + + if "repo_root" in parameters: + kwargs["repo_root"] = _find_repo_root() + return kwargs + + +def inspect_signature(fn: Any) -> set[str]: + import inspect + + return set(inspect.signature(fn).parameters) + + +def _parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run a benchmark-local unified evaluator and export metrics/artifacts JSON." + ) + parser.add_argument("--candidate", required=True) + parser.add_argument("--metrics-out", default="metrics.json") + parser.add_argument("--artifacts-out", default="artifacts.json") + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = _parse_args(argv) + + candidate_path = Path(args.candidate).expanduser().resolve() + metrics_out = Path(args.metrics_out).expanduser().resolve() + artifacts_out = Path(args.artifacts_out).expanduser().resolve() + + metrics: dict[str, Any] = { + "combined_score": INVALID_COMBINED_SCORE, + "valid": 0.0, + } + artifacts: dict[str, Any] = { + "local_evaluator_path": str(Path(__file__).with_name("evaluator.py").resolve()), + "candidate_path": str(candidate_path), + } + + try: + evaluate_fn = _load_local_evaluator() + result = evaluate_fn(str(candidate_path), **_build_kwargs(evaluate_fn)) + metrics, evaluator_artifacts = _normalize_result(result) + artifacts.update(evaluator_artifacts) + except Exception as exc: + artifacts["error_message"] = str(exc) + artifacts["traceback"] = traceback.format_exc() + + _write_json(metrics_out, metrics) + _write_json(artifacts_out, artifacts) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_1.json b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_1.json new file mode 100644 index 00000000..caf2b23a --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_1.json @@ -0,0 +1,30 @@ +{ + "seed": 4986, + "process": { + "speed": 1.0, + "cut_time": 3, + "return_time": 1, + "buffer_len": 60.0, + "scrap_len": 0.8 + }, + "billet": { + "total_length": 68.9 + }, + "customer": { + "target": 9.5, + "target_min": 9.0, + "target_max": 10.0 + }, + "defects": [ + [ + 46.0, + 46.8 + ] + ], + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + } +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_2.json b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_2.json new file mode 100644 index 00000000..0e199e45 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_2.json @@ -0,0 +1,30 @@ +{ + "seed": 1996, + "process": { + "speed": 1.0, + "cut_time": 3, + "return_time": 1, + "buffer_len": 60.0, + "scrap_len": 0.8 + }, + "billet": { + "total_length": 50.2 + }, + "customer": { + "target": 11.1, + "target_min": 10.6, + "target_max": 11.6 + }, + "defects": [ + [ + 34.9, + 35.7 + ] + ], + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + } +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_3.json b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_3.json new file mode 100644 index 00000000..4263c4e8 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_3.json @@ -0,0 +1,38 @@ +{ + "seed": 3991, + "process": { + "speed": 1.0, + "cut_time": 3, + "return_time": 1, + "buffer_len": 60.0, + "scrap_len": 0.8 + }, + "billet": { + "total_length": 106.2 + }, + "customer": { + "target": 8.5, + "target_min": 8.0, + "target_max": 9.0 + }, + "defects": [ + [ + 32.4, + 33.2 + ], + [ + 89.9, + 90.7 + ], + [ + 99.1, + 99.9 + ] + ], + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + } +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_4.json b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_4.json new file mode 100644 index 00000000..f314861f --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_4.json @@ -0,0 +1,38 @@ +{ + "seed": 4, + "process": { + "speed": 1.0, + "cut_time": 3, + "return_time": 1, + "buffer_len": 60.0, + "scrap_len": 0.8 + }, + "billet": { + "total_length": 71.8 + }, + "customer": { + "target": 8.5, + "target_min": 8.0, + "target_max": 9.0 + }, + "defects": [ + [ + 14.3, + 15.1 + ], + [ + 29.1, + 29.9 + ], + [ + 61.2, + 62.0 + ] + ], + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + } +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_5.json b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_5.json new file mode 100644 index 00000000..1c6f9601 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_5.json @@ -0,0 +1,30 @@ +{ + "seed": 1002, + "process": { + "speed": 1.0, + "cut_time": 3, + "return_time": 1, + "buffer_len": 60.0, + "scrap_len": 0.8 + }, + "billet": { + "total_length": 86.0 + }, + "customer": { + "target": 9.5, + "target_min": 9.0, + "target_max": 10.0 + }, + "defects": [ + [ + 15.7, + 16.5 + ] + ], + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + } +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_6.json b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_6.json new file mode 100644 index 00000000..8c9a4357 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_6.json @@ -0,0 +1,50 @@ +{ + "seed": 6, + "process": { + "speed": 1.0, + "cut_time": 3, + "return_time": 1, + "buffer_len": 60.0, + "scrap_len": 0.8 + }, + "billet": { + "total_length": 139.7 + }, + "customer": { + "target": 8.5, + "target_min": 8.0, + "target_max": 9.0 + }, + "defects": [ + [ + 9.6, + 10.4 + ], + [ + 23.6, + 24.4 + ], + [ + 46.1, + 46.9 + ], + [ + 80.6, + 81.4 + ], + [ + 103.4, + 104.2 + ], + [ + 129.6, + 130.4 + ] + ], + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + } +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_7.json b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_7.json new file mode 100644 index 00000000..a1420ab0 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_7.json @@ -0,0 +1,42 @@ +{ + "seed": 1004, + "process": { + "speed": 1.0, + "cut_time": 3, + "return_time": 1, + "buffer_len": 60.0, + "scrap_len": 0.8 + }, + "billet": { + "total_length": 121.1 + }, + "customer": { + "target": 11.1, + "target_min": 10.6, + "target_max": 11.6 + }, + "defects": [ + [ + 11.5, + 12.3 + ], + [ + 42.6, + 43.4 + ], + [ + 90.3, + 91.1 + ], + [ + 105.6, + 106.4 + ] + ], + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + } +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_8.json b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_8.json new file mode 100644 index 00000000..a23c91c9 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/data/instances/instance_8.json @@ -0,0 +1,42 @@ +{ + "seed": 8, + "process": { + "speed": 1.0, + "cut_time": 3, + "return_time": 1, + "buffer_len": 60.0, + "scrap_len": 0.8 + }, + "billet": { + "total_length": 111.3 + }, + "customer": { + "target": 9.5, + "target_min": 9.0, + "target_max": 10.0 + }, + "defects": [ + [ + 9.2, + 10.0 + ], + [ + 18.6, + 19.4 + ], + [ + 24.3, + 25.1 + ], + [ + 86.7, + 87.5 + ] + ], + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + } +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/docker/Dockerfile b/benchmarks/ContinuousCasting/CuttingOptimization/verification/docker/Dockerfile new file mode 100644 index 00000000..7c94d870 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/docker/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.11-slim + +WORKDIR /workspace + +# The evaluator, simulator, reference solver and baseline use only the Python +# standard library. The unified runtime mounts the benchmark sandbox into the +# container, so no benchmark files are baked into the image. The candidate may +# read `verification/simulator.py` (it is part of the scoring objective and is +# allowed by the constraints), but no instance data is baked in. + +ENV PYTHONUNBUFFERED=1 + +CMD ["python"] diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/evaluate.py b/benchmarks/ContinuousCasting/CuttingOptimization/verification/evaluate.py new file mode 100644 index 00000000..7d4a8405 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/evaluate.py @@ -0,0 +1,276 @@ +"""连铸切割评测入口。 + +CLI：python verification/evaluate.py [--time-budget 60] [--data-dir ...] +对每个实例：subprocess 运行候选求解器（超时 = 时间预算），解析输出 cuts，调 simulator 校验打分； +超时/格式错/越界 -> 该实例 0 分。 +分数 = 各实例"材料利用率"的平均值（0~100，越高越好）。 + +利用率 = 100 * (S - (scrap + lambda*penalty)) / S。废弃物越少、成品越贴合目标值，利用率越高； +它等价于"先最小化报废、再最小化贴合度惩罚"，符合赛题的字典序目标，且越高越好、跨实例可比。 + +完整性 / 防作弊（对齐 CVRP / TelecomBackup 基准的经验）： + * 评分前静态检查候选（EVOLVE-BLOCK 标记与标记外代码、禁引用评测/生成/参考解模块、 + 禁绝对路径、禁按实例名硬编码）——见 verification/validator.py； + * 候选子进程环境剥离 FRONTIER_*/CUTTING_EVAL_*（candidate_env），封侧信道； + * 运行时生成实例（CUTTING_EVAL_GENERATE_SEED 设置时，评测现场按种子生成新实例， + 候选无法预先记忆；生成的实例只存在于临时目录，不落仓库/沙箱）。 + * 确定性探针：跨规模选若干实例（小/中/大 + 一个生成实例）各跑两次，输出必须一致。 + +环境变量： + CUTTING_EVAL_GENERATE_SEED 设置后开启运行时生成（防硬编码） + CUTTING_EVAL_GENERATE_COUNT 生成实例数（默认 8） + +对外接口（供 frontier_eval/evaluator.py 包装）： + evaluate(program_path, *, time_budget=60.0) -> {"combined_score": float, + "valid": float, "per_instance": {...}} +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from simulator import load_instance, score # noqa: E402 +from validator import candidate_env, check_candidate, check_determinism # noqa: E402 + +INVALID_SCORE = 0.0 +DATA_DIR = Path(__file__).resolve().parent / "data" / "instances" +DEFAULT_GENERATE_COUNT = 8 +# 运行时生成实例的难度循环（与 generator 默认一致） +GEN_DIFFS = ("easy", "medium", "hard", "medium", "hard", "medium", "hard", "hard") + + +def _source_benchmark_dir() -> Path | None: + raw = os.environ.get("FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR", "").strip() + if raw: + path = Path(raw) + if path.is_dir(): + return path + return None + + +def _parse_cuts(raw: str) -> list[float] | None: + """解析候选 stdout 为 cuts 列表；非法返回 None。 + + 格式：{"cuts": [c1, c2, ...]} + """ + try: + text = raw.strip() + if not text: + return None + obj = json.loads(text) + if isinstance(obj, dict): + obj = obj.get("cuts") + if not isinstance(obj, list) or not obj: + return None + out: list[float] = [] + for c in obj: + if isinstance(c, bool) or not isinstance(c, (int, float)): + return None + if math.isnan(c) or math.isinf(c): + return None + out.append(float(c)) + return out + except Exception: + return None + + +def _utilization(inst: dict[str, Any], cuts: list[float]) -> float: + """给定切段方案 -> 材料利用率（0~100，越高越好）。无效则 0。""" + ok, m = score(inst, cuts) + if not ok: + return 0.0 + S = float(inst["billet"]["total_length"]) + return max(0.0, 100.0 * (S - m["score"]) / S) + + +def _run_one(program_path: Path, inst_path: Path, time_budget: float, + python: str) -> tuple[float, dict[str, Any]]: + inst = load_instance(inst_path) + started = time.time() + try: + proc = subprocess.run( + [python, str(program_path), str(inst_path)], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=time_budget, + cwd=str(program_path.parent), + env=candidate_env(), + ) + elapsed = time.time() - started + if proc.returncode != 0: + return 0.0, {"status": "crash", "stderr_tail": proc.stderr[-500:]} + cuts = _parse_cuts(proc.stdout) + if cuts is None: + return 0.0, {"status": "bad_output", "stdout_tail": proc.stdout[-500:]} + util = _utilization(inst, cuts) + return util, {"status": "ok", "cuts": cuts, "utilization": round(util, 2), + "elapsed_s": round(elapsed, 2)} + except subprocess.TimeoutExpired: + return 0.0, {"status": "timeout", "budget_s": time_budget} + except Exception as exc: + return 0.0, {"status": "error", "message": str(exc)} + + +def _load_host_module(mod_name: str): + """从宿主 benchmark 目录加载 `verification/.py`（沙箱内不含该模块时）。""" + import importlib.util + + src = os.environ.get("FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR", "").strip() + if src and Path(src).is_dir(): + path = Path(src) / "verification" / f"{mod_name}.py" + if path.is_file(): + spec = importlib.util.spec_from_file_location(f"_cc_{mod_name}", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + return None + + +def _generate_instances(base_seed: int, count: int, out_dir: Path) -> list[Path]: + """按种子现场生成新实例（防硬编码）。派生种子 base_seed*1000+i，可复现。""" + gen_mod = _load_host_module("generator") + if gen_mod is None: + import generator as gen_mod # 直跑（非沙箱）时的本地回退 + + paths: list[Path] = [] + i = 0 + attempts = 0 + while len(paths) < count and attempts < count * 64: + inst = gen_mod.generate(base_seed * 1000 + i, GEN_DIFFS[i % len(GEN_DIFFS)]) + i += 1 + attempts += 1 + if not gen_mod._interesting_ok(inst): + continue + path = out_dir / f"gen_{base_seed}_{len(paths) + 1}.json" + path.write_text(json.dumps(inst, ensure_ascii=False) + "\n", encoding="utf-8") + paths.append(path) + return paths + + +def _select_probes(instances: list[Path], n: int = 3) -> list[Path]: + """跨规模选确定性探针：按排序取 小/中/大 各一（外加一个生成实例，若有）。""" + if len(instances) <= n: + return list(instances) + idxs = sorted({0, len(instances) // 2, len(instances) - 1}) + probes = [instances[i] for i in idxs] + gen = [p for p in instances if p.name.startswith("gen_")] + if gen: + probes.append(gen[0]) + return probes + + +def evaluate(program_path: str, *, time_budget: float = 60.0, + data_dir: str | Path | None = None, + python: str | None = None) -> dict[str, Any]: + prog = Path(program_path).resolve() + if not prog.exists(): + return {"combined_score": 0.0, "valid": 0.0, "per_instance": {}, + "error": f"program not found: {prog}"} + + # 静态完整性检查（EVOLVE-BLOCK 标记/只读区比对、禁引用、禁绝对路径、禁硬编码） + baseline_path = None + src_dir = _source_benchmark_dir() + if src_dir is not None: + candidate_baseline = src_dir / "baseline" / "solver.py" + if candidate_baseline.is_file(): + baseline_path = candidate_baseline + violations = check_candidate(prog, baseline_path=baseline_path) + + # 实例池 = 固定实例（本地 data-dir 存在时）+ 运行时生成（设了生成种子时） + inst_dir = Path(data_dir).resolve() if data_dir else DATA_DIR + instances: list[Path] = [] + if inst_dir.is_dir(): + instances = sorted(inst_dir.glob("instance_*.json")) + + gen_seed_raw = os.environ.get("CUTTING_EVAL_GENERATE_SEED", "").strip() + tmp_dir: Path | None = None + if gen_seed_raw: + base_seed = 0 + try: + base_seed = int(gen_seed_raw) + except ValueError: + pass + gen_count = DEFAULT_GENERATE_COUNT + try: + gen_count = max(0, int(os.environ.get("CUTTING_EVAL_GENERATE_COUNT", "").strip())) + except ValueError: + pass + if gen_count > 0: + tmp_dir = Path(tempfile.mkdtemp(prefix="cutting_eval_")) + instances.extend(_generate_instances(base_seed, gen_count, tmp_dir)) + + if not instances: + return {"combined_score": 0.0, "valid": 0.0, "per_instance": {}, + "error": f"no instances (data_dir={inst_dir}, generate_seed={gen_seed_raw!r})"} + + py = python or sys.executable + + # 确定性探针：跨规模选若干实例各跑两次，输出必须一致。 + if not violations: + for probe in _select_probes(instances): + det_ok, det_note = check_determinism(py, prog, probe, time_budget) + if not det_ok: + violations = [f"determinism check failed on {probe.name}: {det_note}"] + break + + per_instance: dict[str, Any] = {} + total = 0.0 + all_valid = True + for inst_path in instances: + if violations: + per_instance[inst_path.name] = {"status": "preflight_failed", + "reasons": violations} + continue + util, info = _run_one(prog, inst_path, time_budget, py) + per_instance[inst_path.name] = info + if util <= 0 and info.get("status") != "ok": + all_valid = False + total += util + + combined = total / len(instances) if instances else 0.0 + if tmp_dir is not None: + shutil.rmtree(tmp_dir, ignore_errors=True) + return { + "combined_score": round(combined, 2), + "valid": 1.0 if all_valid and not violations else 0.0, + "per_instance": per_instance, + "num_instances": len(instances), + "time_budget_s": time_budget, + "generate_seed": gen_seed_raw or None, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="连铸切割评测") + parser.add_argument("solver", help="候选求解器脚本路径") + parser.add_argument("--time-budget", type=float, default=60.0, + help="求解时间预算（秒），默认 60") + parser.add_argument("--data-dir", type=str, default=None, + help="实例目录（默认 verification/data/instances）") + parser.add_argument("--generate-seed", type=int, default=None, + help="运行时生成实例的种子（防硬编码；等价于设 CUTTING_EVAL_GENERATE_SEED）") + args = parser.parse_args(argv) + + if args.generate_seed is not None: + os.environ["CUTTING_EVAL_GENERATE_SEED"] = str(args.generate_seed) + result = evaluate(args.solver, time_budget=args.time_budget, data_dir=args.data_dir) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/generator.py b/benchmarks/ContinuousCasting/CuttingOptimization/verification/generator.py new file mode 100644 index 00000000..ec2c7522 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/generator.py @@ -0,0 +1,202 @@ +"""连铸切割实例生成器：seed 固定，生成可复现的一维切割实例（JSON）。 + +参数语义： +- 钢坯是一维线段 [0, S]（S = total_length）。 +- 结晶器异常在坯内产生若干"零废段"（每段长 scrap_len=0.8m），必须被切出（报废）。 +- 用户目标值 T、范围 [target_min, target_max]；工艺参数与长度窗口见 LIMITS。 +- 难度由 total_length、零废段数量、目标窗口宽度共同决定。 + +零废段位置限制：相邻零废段、零废段与线段两端之间必须留出 >= min_basic 的干净坯段， +否则该段 < min_basic 无法成一块（不可行）。生成时强制该约束。 +""" + +from __future__ import annotations + +import json +import random +import sys +from pathlib import Path +from typing import Any + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parent)) # 保证同目录 import + +# 长度窗口（与 simulator.py 保持一致） +MIN_BASIC = 4.8 # 能运走的最小长度 +MAX_BASIC = 12.6 # 能运走的最大长度 +MIN_PROCESS = 8.0 # 下道工序可接受的最小长度 +MAX_PROCESS = 11.6 # 下道工序可接受的最大长度 + +PROCESS = { + "speed": 1.0, # 拉坯速度 (m/min) + "cut_time": 3, # 切一块耗时 (min) + "return_time": 1, # 切完回工作起点 (min) + "buffer_len": 60.0, # 结晶器中心到切割机工作起点 (m) + "scrap_len": 0.8, # 结晶器异常产生的零废段长度 (m) +} + + +def _clamp_times(T: float) -> tuple[float, float]: + return T - 0.5, T + 0.5 + + +def _target_options(rng: random.Random) -> tuple[float, float]: + """从赛题提及的目标值集合里挑一个（8.5 / 9.5 / 11.1）。""" + return float(rng.choice([8.5, 9.5, 11.1])) + + +def generate(seed: int, difficulty: str = "medium") -> dict[str, Any]: + """按种子生成一个切割实例。 + + difficulty: "easy" | "medium" | "hard"。easy 给短坯+零废段少， + hard 给长坯+零废段多（且窗口更紧）。 + """ + rng = random.Random(seed) + + if difficulty == "easy": + s_lo, s_hi = 24.0, 70.0 + n_def_lo, n_def_hi = 0, 1 + elif difficulty == "hard": + s_lo, s_hi = 100.0, 150.0 + n_def_lo, n_def_hi = 3, 6 + else: # medium + s_lo, s_hi = 60.0, 110.0 + n_def_lo, n_def_hi = 1, 3 + + S = round(rng.uniform(s_lo, s_hi), 1) + T = _target_options(rng) + t_min, t_max = _clamp_times(T) + + # 生成零废段位置：彼此、与两端之间都留出 >= MIN_BASIC 干净坯。 + n_def = rng.randint(n_def_lo, n_def_hi) + scrap_len = PROCESS["scrap_len"] + defects: list[list[float]] = [] + # 可用的"槽位"起点范围 [MIN_BASIC, S - MIN_BASIC - scrap_len] + low = MIN_BASIC + high = S - MIN_BASIC - scrap_len + attempts = 0 + while len(defects) < n_def and attempts < 500: + attempts += 1 + cand = round(rng.uniform(low, high), 1) + # 与已有零废段保持 >= MIN_BASIC 间距（即干净坯段 >= MIN_BASIC） + ok = True + for a, b in defects: + if cand < b + MIN_BASIC and a < cand + scrap_len + MIN_BASIC: + ok = False + break + if ok: + defects.append([cand, round(cand + scrap_len, 1)]) + defects.sort() + + inst: dict[str, Any] = { + "seed": seed, + "process": PROCESS, + "billet": {"total_length": S}, + "customer": {"target": T, "target_min": t_min, "target_max": t_max}, + "defects": defects, + "limits": { + "min_basic": MIN_BASIC, + "max_basic": MAX_BASIC, + "min_process": MIN_PROCESS, + "max_process": MAX_PROCESS, + }, + } + return inst + + +def clean_segments(inst: dict[str, Any]) -> list[list[float]]: + """返回干净坯段列表 [[start, length], ...]。 + + 零废段把 [0, S] 分成若干"干净坯段"，每段内部不含零废段、 + 可被切成若干成品。零废段本身是强制报废。 + """ + S = float(inst["billet"]["total_length"]) + defects = [list((a, b)) for a, b in inst["defects"]] or [] + segs: list[list[float]] = [] + prev = 0.0 + for a, b in defects: + segs.append([prev, a - prev]) + prev = b + segs.append([prev, S - prev]) + return [s for s in segs if s[1] > 1e-9] + + +def _equal_split_scrap(inst: dict[str, Any]) -> float: + """朴素"均匀等分"baseline 的总报废长度（用于验收 headroom）。""" + import math + + from simulator import piece_scrap + + a = MIN_BASIC + b = MAX_BASIC + total = 0.0 + for _start, length in clean_segments(inst): + n = max(1, math.ceil(length / b)) + while n > 1 and length / n < a: + n -= 1 + while length / n > b: + n += 1 + base = length / n + pieces = [base] * n + pieces[-1] = length - base * (n - 1) + # 与 baseline/solver.py 的 partition() 完全同口径（每块 round 到 4 位） + for c in pieces: + total += piece_scrap(round(c, 4), inst) + # 零废段强制报废（与 baseline/solver.py 的真实报废口径一致） + for a0, b0 in inst["defects"] or []: + total += (b0 - a0) + return total + + +def _interesting_ok(inst: dict[str, Any], min_extra: float = 0.2, min_headroom: float = 0.1) -> bool: + """验收：参考解必须严格优于朴素等分（存在真正的优化空间）。 + + 两层含义： + 1. 最优损失须显著超过"零废段强制报废"（即存在真正的余料损失）—— + 若最优损失只等于零废段长度，说明实例能被"一块不多不少"地切成目标长度， + 没有优化空间。 + 2. 朴素"均匀等分"baseline 须比参考解差 >= min_headroom（米）—— + 否则该实例上 agent 无区分度（优化也拿不到收益），对 benchmark 无意义。 + 另外保证至少存在一块干净坯段 >= min_basic（否则不可行）。 + """ + import ref_solver + + from simulator import validate + + ok_segs = clean_segments(inst) + if not ok_segs or max(s[1] for s in ok_segs) < MIN_BASIC: + return False + ref_cuts = ref_solver.solve(inst)["cuts"] + if not validate(inst, ref_cuts)[0]: + return False + ref_scrap = ref_solver.total_scrap(inst, ref_cuts) + defect_total = sum(b - a for a, b in inst["defects"] or []) + if ref_scrap < defect_total + min_extra: + return False + return _equal_split_scrap(inst) >= ref_scrap + min_headroom + + +def main() -> None: + out_dir = Path(__file__).resolve().parent / "data" / "instances" + out_dir.mkdir(parents=True, exist_ok=True) + specs = [ + (1, "easy"), (2, "easy"), (3, "medium"), (4, "medium"), + (5, "medium"), (6, "hard"), (7, "hard"), (8, "hard"), + ] + for seed, difficulty in specs: + inst = None + for trial in range(600): + s = seed + trial * 997 + cand = generate(s, difficulty) + if _interesting_ok(cand): + inst = cand + break + assert inst is not None, f"no acceptable instance for slot {seed}/{difficulty}" + path = out_dir / f"instance_{seed}.json" + path.write_text(json.dumps(inst, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") + print(f"instance_{seed}.json diff={difficulty} S={inst['billet']['total_length']} " + f"T={inst['customer']['target']} #defects={len(inst['defects'])} seed={inst['seed']}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/multiseed_stat.py b/benchmarks/ContinuousCasting/CuttingOptimization/verification/multiseed_stat.py new file mode 100644 index 00000000..e6df657a --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/multiseed_stat.py @@ -0,0 +1,87 @@ +"""多运行（多种子/多轮）agent 分数统计：对一组框架运行目录取 combined_score 的 mean±std。 + +用法： + python verification/multiseed_stat.py --runs-dir runs/unified__ContinuousCasting__CuttingOptimization/openevolve + python verification/multiseed_stat.py --runs-dir "runs/**/openevolve/deepseek-v4-flash" # glob + python verification/multiseed_stat.py --runs-dir --pattern "*openevolve*" + +从每个运行目录下读 `/best/best_program_info.json` 的 `metrics.combined_score` +（框架统一保存的 best 程序分数），对多次运行做 mean / std / min / max， +并给出"相对 reference 的余量"（gap = reference_util - mean）。 + +对齐 CVRP 评审点：agent 分数需多轮均值±std（而非单次），衡量稳定性与真实水平。 +纯标准库。 +""" + +from __future__ import annotations + +import argparse +import glob +import json +import math +import statistics +from pathlib import Path + +DEFAULT_REFERENCE = 88.72 # 全知参考解利用率（verification/ref_solver.py），可 --reference 覆盖 + + +def _best_score(run_dir: Path) -> float | None: + for rel in ("openevolve/best/best_program_info.json", + "shinkaevolve/best/best_program_info.json", + "abmcts/best/best_program_info.json", + "best/best_program_info.json"): + p = run_dir / rel + if p.is_file(): + try: + return float(json.loads(p.read_text(encoding="utf-8"))["metrics"]["combined_score"]) + except Exception: + return None + return None + + +def collect(runs_dir: str | Path, pattern: str | None = None) -> list[tuple[Path, float]]: + base = Path(runs_dir) + if pattern: + bests = sorted(Path(p) for p in glob.glob(str(base / pattern), recursive=True)) + else: + bests = sorted(base.rglob("best/best_program_info.json")) + out: list[tuple[Path, float]] = [] + seen: set[Path] = set() + for best in bests: + # best_program_info.json 位于 //best/ 下 + run_dir = best.parent.parent.parent + if run_dir in seen: + continue + score = _best_score(run_dir) + if score is not None: + out.append((run_dir, score)) + seen.add(run_dir) + return out + + +def main() -> int: + parser = argparse.ArgumentParser(description="多运行 agent 分数统计") + parser.add_argument("--runs-dir", required=True, help="运行目录或 glob 模式") + parser.add_argument("--pattern", default=None, help="追加的子 glob 模式") + parser.add_argument("--reference", type=float, default=DEFAULT_REFERENCE, + help="参考解利用率（默认 88.72）") + args = parser.parse_args() + + pairs = collect(args.runs_dir, args.pattern) + if not pairs: + print("no runs found under", args.runs_dir) + return 1 + + scores = [s for _, s in pairs] + mean = statistics.mean(scores) + std = statistics.stdev(scores) if len(scores) > 1 else 0.0 + print(f"runs: {len(scores)}") + for run_dir, s in pairs: + print(f" {run_dir.name}: {s}") + print(f"mean={mean:.2f} std={std:.2f} min={min(scores):.2f} max={max(scores):.2f}") + print(f"gap to reference ({args.reference}): {args.reference - mean:.2f}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/ref_solver.py b/benchmarks/ContinuousCasting/CuttingOptimization/verification/ref_solver.py new file mode 100644 index 00000000..ecd8846f --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/ref_solver.py @@ -0,0 +1,111 @@ +"""连铸切割参考解（一维划分 DP），纯 Python 标准库。 + +思路：零废段把钢坯 [0, S] 分成若干"干净坯段"；每段独立做切割优化。 +每段用一个网格化 DP：状态 = 已覆盖长度（以 GRID 为步长）， +转移 = 选一块长度 c 属于 [min_basic, max_basic]，代价 = 该块的 + scrap + lambda*penalty。段内最小总代价即最优。 + +注意：实例中的所有长度（S、零废段位置、scrap_len、min/max_basic）都取 +0.1 米的多倍，而 GRID=0.02，故 all 长度都是 GRID 的多倍 → DP 无舍入误差， +参考解切段长度之和恰好等于 S（可直接通过 sum≈S 校验）。 +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) # 保证同目录 import + +from simulator import ( # noqa: E402 + MAX_BASIC, + MIN_BASIC, + TARGET_PENALTY_WEIGHT, + clean_segments, + piece_penalty, + piece_scrap, +) + +GRID = 0.02 + + +def _build_cost_table(inst: dict[str, Any]) -> tuple[int, int, list[float]]: + """返回 (min_steps, max_steps, costs)；costs[s-min_steps] = 切一块 s*GRID 的代价。""" + min_steps = int(round(MIN_BASIC / GRID)) + max_steps = int(round(MAX_BASIC / GRID)) + costs = [] + for s in range(min_steps, max_steps + 1): + c = s * GRID + costs.append(piece_scrap(c, inst) + TARGET_PENALTY_WEIGHT * piece_penalty(c, inst)) + return min_steps, max_steps, costs + + +def _optimize_segment(inst: dict[str, Any], length: float) -> list[float]: + """对单个干净坯段做最优切割，返回该段的切段长度列表（之和 = length）。""" + n = int(round(length / GRID)) + if n <= 0: + return [] + min_steps, max_steps, costs = _build_cost_table(inst) + INF = float("inf") + f = [INF] * (n + 1) + bp = [-1] * (n + 1) + f[0] = 0.0 + for pos in range(min_steps, n + 1): + best = INF + best_s = -1 + hi = min(max_steps, pos) + for s in range(min_steps, hi + 1): + prev = f[pos - s] + if prev >= INF: + continue + val = prev + costs[s - min_steps] + if val < best: + best = val + best_s = s + f[pos] = best + bp[pos] = best_s + + if f[n] >= INF: + return _fallback(inst, length) + + pieces: list[float] = [] + pos = n + while pos > 0: + s = bp[pos] + pieces.append(round(s * GRID, 4)) + pos -= s + pieces.reverse() + return pieces + + +def _fallback(inst: dict[str, Any], length: float) -> list[float]: + """极少数非网格情况：尽量切成 [min_basic, max_basic] 的均匀段。""" + k = int(length // MAX_BASIC) + if k == 0: + return [length] + base = length / (k + 1) + if base < MIN_BASIC: + return [length] + return [round(base, 4)] * (k + 1) + + +def solve(inst: dict[str, Any]) -> dict[str, Any]: + """返回 {'cuts': [...]}：完整钢坯的最优切段（含零废段小块）。""" + segs = clean_segments(inst) + defects = sorted((list((a, b)) for a, b in inst["defects"] or []), key=lambda x: x[0]) + cuts: list[float] = [] + for i, (_start, length) in enumerate(segs): + cuts.extend(_optimize_segment(inst, length)) + if i < len(defects): + a, b = defects[i] + cuts.append(round(b - a, 4)) # 零废段强制切出 + return {"cuts": cuts} + + +def total_scrap(inst: dict[str, Any], cuts: list[float]) -> float: + """给定切段方案，返回总报废长度（用于实例验收/对比）。""" + from simulator import score + + ok, m = score(inst, cuts) + return m["scrap"] if ok else float("inf") diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/requirements.txt b/benchmarks/ContinuousCasting/CuttingOptimization/verification/requirements.txt new file mode 100644 index 00000000..4e66700e --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/requirements.txt @@ -0,0 +1,2 @@ +# The evaluator uses only the Python standard library. +# Runtime requirement: Python >= 3.10 (no third-party dependencies). diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/simulator.py b/benchmarks/ContinuousCasting/CuttingOptimization/verification/simulator.py new file mode 100644 index 00000000..09046df3 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/simulator.py @@ -0,0 +1,169 @@ +"""连铸切割计分模拟器：校验候选切割方案并给出评分。 + +规则（自编简化模型，物理语义见 Task.md / README）： +- 钢坯 = 一维线段 [0, S]；零废段（缺陷）必须被切出（强制报废）。 +- 切口必须对齐每个零废段端点：任何一块"成品"都不能跨过零废段边界， + 否则该方案非法（零废段无法被单独切出）。 +- 每块长度必须在 [min_basic, max_basic]；否则非法。 +- 评分 = 总报废长度 + lambda * 总贴合度惩罚（lambda 极小，保证字典序： + 先最小化报废，再在同报废下让成品长度尽量贴近目标值 T）。 + +报废判定（对齐赛题原文）： +- c < min_process：送不到下道工序 -> 整块报废，scrap += c。 +- c >= min_process：可送下道。超出 target_max 的部分报废，scrap += c - min(c, target_max)。 +- target_min <= c <= target_max：零报废、零惩罚（完美块）。 +- c < target_min（但 c >= min_process）：能送但偏短 -> 零报废、贴合度惩罚。 + +贴合度惩罚（惩罚项，用于同报废下的区分）： +- penalty = |min(c, target_max) - T|（即"实际交付长度"与目标值的距离）。 +""" + +from __future__ import annotations + +import json +import math +from pathlib import Path +from typing import Any + +# 默认长度窗口（与 generator.py 保持一致） +MIN_BASIC = 4.8 +MAX_BASIC = 12.6 +MIN_PROCESS = 8.0 +MAX_PROCESS = 11.6 +# 贴合度惩罚权重（远小于 1m 报废，保证字典序：报废优先，贴合度仅破平） +TARGET_PENALTY_WEIGHT = 1e-4 +SUM_TOL = 1e-3 +BOUND_TOL = 1e-3 + + +def load_instance(path: str | Path) -> dict[str, Any]: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _limits(inst: dict[str, Any]) -> dict[str, float]: + lim = inst.get("limits", {}) + return { + "min_basic": float(lim.get("min_basic", MIN_BASIC)), + "max_basic": float(lim.get("max_basic", MAX_BASIC)), + "min_process": float(lim.get("min_process", MIN_PROCESS)), + "max_process": float(lim.get("max_process", MAX_PROCESS)), + } + + +def clean_segments(inst: dict[str, Any]) -> list[list[float]]: + """干净坯段列表 [[start, length], ...]（与 generator 一致，供参考解使用）。""" + S = float(inst["billet"]["total_length"]) + defects = [list((a, b)) for a, b in inst["defects"]] or [] + segs: list[list[float]] = [] + prev = 0.0 + for a, b in defects: + segs.append([prev, a - prev]) + prev = b + segs.append([prev, S - prev]) + return [s for s in segs if s[1] > 1e-9] + + +def boundaries(cuts: list[float]) -> list[float]: + """从 cut 长度列表求切点坐标（含 0 和 S）。""" + b = [0.0] + for c in cuts: + b.append(b[-1] + c) + return b + + +def piece_scrap(c: float, inst: dict[str, Any]) -> float: + """单块长度 c 的报废长度（不含贴合度惩罚）。""" + lim = _limits(inst) + t_max = float(inst["customer"]["target_max"]) + if c < lim["min_process"]: + return c + return max(0.0, c - t_max) + + +def piece_penalty(c: float, inst: dict[str, Any]) -> float: + """单块长度 c 的实际交付长度与目标值的距离（贴合度惩罚）。""" + t = float(inst["customer"]["target"]) + t_max = float(inst["customer"]["target_max"]) + delivered = min(c, t_max) + return abs(delivered - t) + + +def _is_defect_piece(l: float, r: float, defects: list[list[float]]) -> bool: + """长度区间 [l, r) 是否恰好是某个零废段（允许其小于 min_basic）。""" + for a, b in defects: + if abs(l - a) <= BOUND_TOL and abs(r - b) <= BOUND_TOL: + return True + return False + + +def validate(inst: dict[str, Any], cuts: list[float]) -> tuple[bool, str]: + """校验候选方案：sum 近似 S、每块在 [min_basic, max_basic]（零废段除外）、切口对齐零废段。""" + lim = _limits(inst) + S = float(inst["billet"]["total_length"]) + if not cuts or any((not isinstance(c, (int, float))) or math.isnan(c) or math.isinf(c) for c in cuts): + return False, "empty or non-numeric cuts" + if any(c <= 0 for c in cuts): + return False, "non-positive cut" + if abs(sum(cuts) - S) > SUM_TOL: + return False, f"sum {sum(cuts)!r} != S {S!r} (tol {SUM_TOL})" + + defects = [list((a, b)) for a, b in inst["defects"]] or [] + b = boundaries(cuts) + for i, c in enumerate(cuts): + l, r = b[i], b[i + 1] + in_range = (lim["min_basic"] - BOUND_TOL <= c <= lim["max_basic"] + BOUND_TOL) + is_def = _is_defect_piece(l, r, defects) + if not in_range and not is_def: + return False, f"cut {c!r} out of [{lim['min_basic']}, {lim['max_basic']}] and not a defect" + # 零废段端点必须是切点（否则成品跨零废段 / 零废段被切成不可运的碎块） + for a, bb in defects: + for e in (a, bb): + if abs(e) > 1e-9 and abs(e - S) > 1e-9: + if not any(abs(p - e) <= BOUND_TOL for p in b): + return False, f"cut does not isolate defect endpoint {e!r}" + return True, "ok" + + +def score(inst: dict[str, Any], cuts: list[float], *, lam: float | None = None + ) -> tuple[bool, dict[str, Any]]: + """评分：返回 (valid, metrics)。metrics = {scrap, penalty, score, cuts}。""" + ok, reason = validate(inst, cuts) + if not ok: + return False, {"valid": False, "reason": reason} + + # 零废段以独立小段表示（0.8m）：scrap(0.8)=0.8，不参与贴合度惩罚。 + defects = [list((a, b)) for a, b in inst["defects"]] or [] + b = boundaries(cuts) + total_scrap = 0.0 + total_penalty = 0.0 + for i, c in enumerate(cuts): + l, r = b[i], b[i + 1] + total_scrap += piece_scrap(c, inst) + if not _is_defect_piece(l, r, defects): + total_penalty += piece_penalty(c, inst) + + lam = TARGET_PENALTY_WEIGHT if lam is None else float(lam) + return True, { + "valid": True, + "scrap": round(total_scrap, 4), + "penalty": round(total_penalty, 4), + "score": round(total_scrap + lam * total_penalty, 4), + "cuts": cuts, + } + + +def solve_value(inst: dict[str, Any], cuts: list[float]) -> float: + """便捷路由：无效 -> 返回极大值；有效 -> 返回 score。""" + ok, m = score(inst, cuts) + return m["score"] if ok else math.inf + + +if __name__ == "__main__": + import sys + if len(sys.argv) < 2: + print("usage: python simulator.py ", file=sys.stderr) + raise SystemExit(2) + inst = load_instance(sys.argv[1]) + print(json.dumps({"clean_segments": clean_segments(inst), + "defects": inst["defects"]}, ensure_ascii=False, indent=1)) diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_evaluator.py b/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_evaluator.py new file mode 100644 index 00000000..0dc41b7f --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_evaluator.py @@ -0,0 +1,61 @@ +"""evaluate.py 单测：完整评测链路（baseline 合法 + 作弊候选被拒绝 + 运行时生成）。""" + +from __future__ import annotations + +import json +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import evaluate # noqa: E402 + +BASELINE = Path(__file__).resolve().parent.parent / "baseline" / "solver.py" + +CHEAT = '''#!/usr/bin/env python3 +from __future__ import annotations +import json, sys +from pathlib import Path + +def solve(inst): + # EVOLVE-BLOCK-START + from ref_solver import solve as ref + return ref(inst)["cuts"] + # EVOLVE-BLOCK-END +''' + + +class TestEvaluator(unittest.TestCase): + def test_baseline_scores_positive_and_valid(self): + res = evaluate.evaluate(str(BASELINE), time_budget=20) + self.assertEqual(res["valid"], 1.0) + self.assertGreater(res["combined_score"], 0.0) + self.assertGreater(res["num_instances"], 0) + + def test_runtime_generation_adds_instances(self): + import os + os.environ["CUTTING_EVAL_GENERATE_SEED"] = "7" + os.environ["CUTTING_EVAL_GENERATE_COUNT"] = "3" + try: + res = evaluate.evaluate(str(BASELINE), time_budget=20) + finally: + del os.environ["CUTTING_EVAL_GENERATE_SEED"] + del os.environ["CUTTING_EVAL_GENERATE_COUNT"] + gen = [k for k in res["per_instance"] if k.startswith("gen_")] + self.assertEqual(len(gen), 3) + + def test_cheating_candidate_rejected(self): + with tempfile.TemporaryDirectory() as td: + cheat = Path(td) / "cheat.py" + cheat.write_text(CHEAT, encoding="utf-8") + res = evaluate.evaluate(str(cheat), time_budget=20, + data_dir=Path(__file__).parent / "data" / "instances") + self.assertEqual(res["valid"], 0.0) + # 所有实例都应 preflight_failed（或至少 combined_score 为 0） + self.assertLessEqual(res["combined_score"], 0.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_frontier_eval_evaluator.py b/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_frontier_eval_evaluator.py new file mode 100644 index 00000000..3522d1f4 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_frontier_eval_evaluator.py @@ -0,0 +1,103 @@ +"""frontier_eval/evaluator.py（沙箱入口）测试。 + +对齐 CVRP 评审点："沙箱版 evaluator 必须有测试覆盖"——测试 benchmark 的 uniform 入口 +（run_eval 加载的 evaluator.py）与 verification/evaluate.py 行为一致，且能拒绝作弊候选。 +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +import tempfile +import unittest +from pathlib import Path + +TASK = Path(__file__).resolve().parent.parent + + +def _load_evaluator(): + p = TASK / "frontier_eval" / "evaluator.py" + spec = importlib.util.spec_from_file_location("_oc_sandbox_evaluator", p) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + return m + + +def _write_cheat(d: Path) -> Path: + """作弊候选：在 EVOLVE-BLOCK 里 import ref_solver（应被 preflight 拒绝）。""" + p = d / "cheat.py" + p.write_text( + '#!/usr/bin/env python3\n' + 'from __future__ import annotations\n' + 'import json, sys\n' + 'def solve(inst):\n' + ' # EVOLVE-BLOCK-START\n' + ' from ref_solver import solve as ref\n' + ' return ref(inst)["cuts"]\n' + ' # EVOLVE-BLOCK-END\n' + 'def main():\n' + ' inst=json.load(open(sys.argv[1]))\n' + ' print(json.dumps({"cuts": solve(inst)}))\n' + 'if __name__ == "__main__": main()\n', + encoding="utf-8", + ) + return p + + +class TestSandboxEvaluator(unittest.TestCase): + @classmethod + def setUpClass(cls): + sys.path.insert(0, str(TASK / "verification")) + cls.ev = _load_evaluator() + + def test_baseline_scores_and_valid(self): + import os + os.environ.pop("CUTTING_EVAL_GENERATE_SEED", None) + r = self.ev.evaluate(str(TASK / "baseline" / "solver.py")) + if isinstance(r, dict) and "metrics" in r: + r = r["metrics"] + self.assertEqual(r.get("valid"), 1.0) + self.assertGreater(r.get("combined_score", 0.0), 0.0) + + def test_runtime_generation_adds_instances(self): + import os + os.environ["CUTTING_EVAL_GENERATE_SEED"] = "7" + os.environ["CUTTING_EVAL_GENERATE_COUNT"] = "3" + try: + r = self.ev.evaluate(str(TASK / "baseline" / "solver.py"), + data_dir=str(TASK / "verification" / "data" / "instances")) + if isinstance(r, dict) and "metrics" in r: + r = r["metrics"] + finally: + os.environ.pop("CUTTING_EVAL_GENERATE_SEED", None) + os.environ.pop("CUTTING_EVAL_GENERATE_COUNT", None) + gen = [k for k in r["per_instance"] if k.startswith("gen_")] + self.assertEqual(len(gen), 3) + + def test_cheating_candidate_rejected(self): + with tempfile.TemporaryDirectory() as td: + cheat = _write_cheat(Path(td)) + r = self.ev.evaluate(str(cheat), + data_dir=str(TASK / "verification" / "data" / "instances")) + if isinstance(r, dict) and "metrics" in r: + r = r["metrics"] + self.assertEqual(r.get("valid"), 0.0) + + def test_consistent_with_verification_evaluator(self): + """沙箱入口与 verification/evaluate.py 应给出同样的 combined_score（数值一致）。""" + import os + os.environ.pop("CUTTING_EVAL_GENERATE_SEED", None) + sandbox = self.ev.evaluate(str(TASK / "baseline" / "solver.py"), + data_dir=str(TASK / "verification" / "data" / "instances")) + if isinstance(sandbox, dict) and "metrics" in sandbox: + sandbox = sandbox["metrics"] + + from evaluate import evaluate as ver_evaluate + ver = ver_evaluate(str(TASK / "baseline" / "solver.py"), + data_dir=str(TASK / "verification" / "data" / "instances")) + self.assertAlmostEqual(sandbox["combined_score"], ver["combined_score"], places=2) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_generator.py b/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_generator.py new file mode 100644 index 00000000..7c8f20d2 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_generator.py @@ -0,0 +1,57 @@ +"""generator.py 单测：确定性、零废段约束、干净坯段可行性、interesting 验收。""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import generator # noqa: E402 +from simulator import MIN_BASIC # noqa: E402 + + +class TestGenerate(unittest.TestCase): + def test_deterministic_same_seed(self): + a = generator.generate(123, "medium") + b = generator.generate(123, "medium") + self.assertEqual(a, b) + + def test_lengths_are_grid_multiples(self): + """所有长度均应接近 0.1 的整数倍（0.02 网格可整除，保证 DP 无舍入误差）。""" + inst = generator.generate(7, "hard") + S = float(inst["billet"]["total_length"]) + self.assertAlmostEqual(round(S / 0.02) * 0.02, S, places=6) + for a, b in inst["defects"]: + self.assertEqual(abs((b - a) - 0.8) < 1e-6, True) + + def test_defects_within_bounds_and_sorted(self): + inst = generator.generate(6, "hard") + S = float(inst["billet"]["total_length"]) + prev_end = 0.0 + for a, b in inst["defects"]: + self.assertGreaterEqual(a, 0.0) + self.assertLessEqual(b, S) + self.assertGreater(a, prev_end) + self.assertGreaterEqual(a - prev_end, MIN_BASIC - 1e-6) + prev_end = b + self.assertGreaterEqual(S - prev_end, MIN_BASIC - 1e-6) + + def test_clean_segments_all_feasible(self): + inst = generator.generate(5, "medium") + segs = generator.clean_segments(inst) + self.assertTrue(segs) + for _s, length in segs: + self.assertGreaterEqual(length, MIN_BASIC - 1e-6) + + def test_interesting_ok_on_accepted_instances(self): + """main() 产出的每个实例都应通过 _interesting_ok（有 headroom、非退化）。""" + for p in sorted(Path(__file__).parent.glob("data/instances/instance_*.json")): + import json + inst = json.loads(p.read_text(encoding="utf-8")) + self.assertTrue(generator._interesting_ok(inst), p.name) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_ref_solver.py b/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_ref_solver.py new file mode 100644 index 00000000..883978c7 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_ref_solver.py @@ -0,0 +1,50 @@ +"""ref_solver.py 单测：求解合法性、确定性、显著优于 baseline、等于全局最优。""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import ref_solver # noqa: E402 +from simulator import load_instance, score, validate # noqa: E402 + +DATA = Path(__file__).resolve().parent / "data" / "instances" + + +class TestRefSolver(unittest.TestCase): + def test_solution_valid_on_fixed_instances(self): + for p in sorted(DATA.glob("instance_*.json")): + inst = load_instance(p) + cuts = ref_solver.solve(inst)["cuts"] + ok, reason = validate(inst, cuts) + self.assertTrue(ok, f"{p.name}: {reason}") + + def test_deterministic(self): + inst = load_instance(sorted(DATA.glob("instance_*.json"))[0]) + self.assertEqual(ref_solver.solve(inst), ref_solver.solve(inst)) + + def test_equals_or_beats_equal_split(self): + """参考解利用率应不劣于朴素等分（一般显著更优）。""" + import subprocess + import sys as _sys + from simulator import score as _score + + baseline = Path(__file__).resolve().parent.parent / "baseline" / "solver.py" + for p in sorted(DATA.glob("instance_*.json"))[:4]: + inst = load_instance(p) + ref_cuts = ref_solver.solve(inst)["cuts"] + out = subprocess.run( + [_sys.executable, str(baseline), str(p)], capture_output=True, text=True + ) + import json + base_cuts = json.loads(out.stdout)["cuts"] + _, r = score(inst, ref_cuts) + _, b = score(inst, base_cuts) + self.assertLessEqual(r["score"], b["score"], f"{p.name}: ref should be <= baseline") + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_simulator.py b/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_simulator.py new file mode 100644 index 00000000..a53ac2c7 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_simulator.py @@ -0,0 +1,124 @@ +"""simulator.py 单测：评分规则、校验逻辑、零废段处理。""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import generator # noqa: E402 +from simulator import ( # noqa: E402 + MAX_BASIC, + MIN_BASIC, + TARGET_PENALTY_WEIGHT, + boundaries, + piece_penalty, + piece_scrap, + score, + validate, +) + + +def _inst(seed: int = 1, difficulty: str = "medium"): + inst = generator.generate(seed, difficulty) + # 强制放入一个 0.8m 零废段，便于测试（若无则加上一个远离端部的） + if not inst["defects"]: + S = float(inst["billet"]["total_length"]) + inst["defects"] = [[round(S / 2.0, 1), round(S / 2.0 + 0.8, 1)]] + return inst + + +class TestPieceRules(unittest.TestCase): + def test_too_short_fully_scrapped(self): + inst = _inst() + self.assertEqual(piece_scrap(6.0, inst), 6.0) # < min_process(8.0) -> 整块报废 + + def test_in_window_zero_scrap(self): + inst = _inst() + tmin = float(inst["customer"]["target_min"]) + tmax = float(inst["customer"]["target_max"]) + self.assertEqual(piece_scrap((tmin + tmax) / 2.0, inst), 0.0) + + def test_over_target_extra_scrapped(self): + inst = _inst() + tmax = float(inst["customer"]["target_max"]) + self.assertAlmostEqual(piece_scrap(tmax + 1.6, inst), 1.6) + + def test_penalty_is_distance_to_target(self): + inst = _inst() + t = float(inst["customer"]["target"]) + tmax = float(inst["customer"]["target_max"]) + self.assertAlmostEqual(piece_penalty(t, inst), 0.0) + # 超出 target_max 的部分会报废；实际交付长度 = min(c, target_max)，惩罚 = |交付 - target| + self.assertAlmostEqual(piece_penalty(t + 1.0, inst), abs(min(t + 1.0, tmax) - t)) + + +class TestValidate(unittest.TestCase): + def test_valid_solution_passes(self): + import ref_solver + inst = _inst() + cuts = ref_solver.solve(inst)["cuts"] + ok, reason = validate(inst, cuts) + self.assertTrue(ok, reason) + + def test_sum_mismatch(self): + inst = _inst() + cuts = [5.0, 5.0] + ok, _ = validate(inst, cuts) + self.assertFalse(ok) + + def test_cut_out_of_range(self): + inst = _inst() + S = float(inst["billet"]["total_length"]) + # 全部切成越界长度（< min_basic），且不覆盖零废段端点 + cuts = [3.0] * int(round(S / 3.0)) + ok, _ = validate(inst, cuts) + self.assertFalse(ok) + + def test_defect_piece_allowed_below_min_basic(self): + import ref_solver + inst = _inst(seed=3, difficulty="medium") + self.assertTrue(inst["defects"]) + cuts = ref_solver.solve(inst)["cuts"] + ok, reason = validate(inst, cuts) + self.assertTrue(ok, reason) + # 参考解应包含一个 < min_basic 的零废段块（0.8m），且合法 + defect_len = round(inst["defects"][0][1] - inst["defects"][0][0], 4) + self.assertLess(defect_len, MIN_BASIC) + self.assertIn(defect_len, [round(c, 4) for c in cuts]) + + def test_crossing_defect_rejected(self): + inst = _inst() + a, b = inst["defects"][0] + # 找两个切口，一个在零废段内部、一个在其后，跨过零废段 -> 非法 + piece_in = a + 0.3 # 落在零废段内 + cuts = [piece_in, 5.0, 5.0, 5.0, 5.0] + # 需要校验 sum 与端点；直接断言 endpoints 之一不是边界时非法 + ok, reason = validate(inst, cuts) + # 这些 cuts 大概率 sum 不匹配也非法；重点确认至少被判非法 + self.assertFalse(ok) + + +class TestScore(unittest.TestCase): + def test_score_is_scrap_plus_lam_penalty(self): + import ref_solver + inst = _inst() + cuts = ref_solver.solve(inst)["cuts"] + ok, m = score(inst, cuts) + self.assertTrue(ok) + # score 四舍五入到 4 位，故用 3 位精度比较 + self.assertAlmostEqual( + m["score"], m["scrap"] + TARGET_PENALTY_WEIGHT * m["penalty"], places=3 + ) + + def test_invalid_solution_scores_invalid(self): + inst = _inst() + ok, m = score(inst, [1.0, 1.0]) + self.assertFalse(ok) + self.assertFalse(m["valid"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_validator.py b/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_validator.py new file mode 100644 index 00000000..1b80b6a9 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/test_validator.py @@ -0,0 +1,80 @@ +"""validator.py 单测：静态完整性检查、禁引用、绝对路径、硬编码、环境剥离。""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from validator import ( # noqa: E402 + candidate_env, + check_candidate, + static_check_source, +) + +BASELINE = Path(__file__).resolve().parent.parent / "baseline" / "solver.py" + +TEMPLATE = '''#!/usr/bin/env python3 +"""candidate""" +from __future__ import annotations +import json, sys +from pathlib import Path + +def solve(inst): + # EVOLVE-BLOCK-START + return [float(inst['billet']['total_length'])] + # EVOLVE-BLOCK-END +''' + +CHEAT_REF = TEMPLATE.replace('return [float(inst[\'billet\'][\'total_length\'])]', 'from ref_solver import solve\n return solve(inst)["cuts"]') +CHEAT_GEN = TEMPLATE.replace('return [float(inst[\'billet\'][\'total_length\'])]', 'import generator\n return generator.generate(1)["billet"]') +CHEAT_ABS = TEMPLATE.replace('return [float(inst[\'billet\'][\'total_length\'])]', 'open("C:\\\\Users\\\\x.txt")') +CHEAT_HARDCODE = TEMPLATE.replace('return [float(inst[\'billet\'][\'total_length\'])]', 'return [("instance_1": [1,2])]') +NO_MARKERS = TEMPLATE.replace('# EVOLVE-BLOCK-START\n ', '').replace('# EVOLVE-BLOCK-END\n', '') + + +class TestValidator(unittest.TestCase): + def test_baseline_passes(self): + self.assertEqual(check_candidate(BASELINE, baseline_path=BASELINE), []) + + def test_plain_candidate_passes(self): + self.assertEqual(static_check_source(TEMPLATE), []) + + def test_ref_solver_import_caught(self): + self.assertTrue(any("ref_solver" in m for m in static_check_source(CHEAT_REF))) + + def test_generator_import_caught(self): + self.assertTrue(any("generator" in m.lower() for m in static_check_source(CHEAT_GEN))) + + def test_absolute_path_caught(self): + self.assertTrue(any("absolute" in m for m in static_check_source(CHEAT_ABS))) + + def test_hardcode_caught(self): + self.assertTrue(any("hardcode" in m.lower() for m in static_check_source(CHEAT_HARDCODE))) + + def test_missing_markers_caught(self): + self.assertTrue(any("EVOLVE-BLOCK" in m for m in static_check_source(NO_MARKERS))) + + def test_fixed_region_vs_baseline(self): + # 修改了 EVOLVE-BLOCK 之外的代码（改 docstring）应被抓到 + base_src = BASELINE.read_text(encoding="utf-8") + mod = base_src.replace("朴素基线", "朴素基线（被修改）").replace("连铸切割 baseline 求解器", "连铸切割 baseline 求解器X") + self.assertTrue(any("outside EVOLVE-BLOCK" in m for m in static_check_source(mod, base_src))) + + def test_candidate_env_strips_vars(self): + import os + os.environ["CUTTING_EVAL_GENERATE_SEED"] = "42" + os.environ["FRONTIER_EVAL_SOMETHING"] = "x" + try: + env = candidate_env() + self.assertNotIn("CUTTING_EVAL_GENERATE_SEED", env) + self.assertNotIn("FRONTIER_EVAL_SOMETHING", env) + finally: + os.environ.pop("CUTTING_EVAL_GENERATE_SEED", None) + os.environ.pop("FRONTIER_EVAL_SOMETHING", None) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/verification/validator.py b/benchmarks/ContinuousCasting/CuttingOptimization/verification/validator.py new file mode 100644 index 00000000..cfb0b953 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimization/verification/validator.py @@ -0,0 +1,154 @@ +"""ContinuousCasting 候选完整性校验（借鉴 CVRP / TelecomBackup validator）。 + +在评分前以"可执行检查"强制任务约束（不只是自然语言）： + 1. EVOLVE-BLOCK 完整性：标记必须存在；标记外的代码必须与初始 baseline 逐字节一致。 + 2. 禁引用：候选不得 import verification 的评测/生成模块（evaluate/generator/ref_solver）， + 不得出现绝对路径，不得按实例名硬编码。 + 3. 确定性：同一实例跑两次必须输出一致。 + 4. candidate_env：候选子进程环境剥离 FRONTIER_*/CUTTING_EVAL_* 变量， + 封死"通过宿主环境变量定位评测基线"的侧信道。 + +候选允许 import `verification/simulator.py`（白盒计分器，任务有意暴露）， +但不允许 import 评测/生成/参考解逻辑。纯标准库。 +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +from pathlib import Path +from typing import Any + +EVOLVE_START = "EVOLVE-BLOCK-START" +EVOLVE_END = "EVOLVE-BLOCK-END" +PROJECT_PREFIX = "CUTTING_EVAL_" + +# 禁引用 token：强 token 子串匹配即可（正常求解器几乎不会出现）。 +STRONG_TOKENS = ( + "ref_solver", + "CUTTING_EVAL_", + "from generator", + "import generator", +) +# "verification" 只在与评测/生成模块构成 import/路径上下文时禁止 +# （候选被允许 import verification/simulator.py）。 +FORBIDDEN_RE = ( + re.compile(r"verification[\\/](?:evaluate|generator|ref_solver)"), + re.compile(r"verification\s*\.\s*(?:evaluate|generator|ref_solver)\b"), + re.compile(r"\b(?:from|import)\s+(?:evaluate|generator|ref_solver)\b"), +) +# Windows 盘符 / POSIX 家目录绝对路径。 +ABS_PATH_RE = re.compile(r"[A-Za-z]:[\\/]|/home/|/Users/") +# 按实例名硬编码，如 "instance_1": [...] 或 "gen_42_1": [...] +HARDCODE_RE = re.compile(r"[\"'](?:instance|gen)_\d+(?:_\d+)?[\"']\s*:") + + +def split_evolve_blocks(src: str) -> tuple[str, str, str] | None: + start = src.find(EVOLVE_START) + end = src.find(EVOLVE_END) + if start == -1 or end == -1 or end <= start: + return None + return ( + src[:start], + src[start + len(EVOLVE_START) : end], + src[end + len(EVOLVE_END) :], + ) + + +def fixed_region(parts: tuple[str, str, str]) -> str: + """EVOLVE-BLOCK 之外的只读部分（忽略 CRLF/LF 与结尾缺换行的无损差异）。""" + return (parts[0] + parts[2]).replace("\r\n", "\n").rstrip("\n") + + +def static_check_source(src: str, baseline_src: str | None = None) -> list[str]: + issues: list[str] = [] + parts = split_evolve_blocks(src) + if parts is None: + issues.append("missing EVOLVE-BLOCK-START / EVOLVE-BLOCK-END markers") + elif baseline_src is not None: + init_parts = split_evolve_blocks(baseline_src) + if init_parts is not None and fixed_region(init_parts) != fixed_region(parts): + issues.append("code outside EVOLVE-BLOCK differs from initial baseline") + + for token in STRONG_TOKENS: + if token in src: + issues.append(f"candidate references forbidden token {token!r}") + for pat in FORBIDDEN_RE: + if pat.search(src): + issues.append("candidate references forbidden evaluation/generation/ref module") + + if ABS_PATH_RE.search(src): + issues.append("candidate contains an absolute filesystem path") + if HARDCODE_RE.search(src): + issues.append("candidate hardcodes per-instance schedules by name") + + return issues + + +def check_candidate( + solver_path: Path | str, baseline_path: Path | str | None = None +) -> list[str]: + solver_path = Path(solver_path) + try: + src = solver_path.read_text(encoding="utf-8", errors="replace") + except Exception as exc: + return [f"cannot read candidate source: {exc}"] + + baseline_src = None + if baseline_path is not None: + try: + baseline_src = Path(baseline_path).read_text( + encoding="utf-8", errors="replace" + ) + except Exception: + baseline_src = None + return static_check_source(src, baseline_src) + + +def candidate_env() -> dict[str, str]: + """候选子进程环境：剥离宿主路径与评测相关变量。""" + env = os.environ.copy() + for key in list(env): + upper = key.upper() + if upper.startswith("FRONTIER") or upper.startswith(PROJECT_PREFIX): + del env[key] + return env + + +def check_determinism( + python: str, + solver_path: Path | str, + inst_path: Path | str, + timeout: float, +) -> tuple[bool, str]: + """同一实例跑两次，输出必须一致。""" + solver_path = Path(solver_path) + inst_path = Path(inst_path) + outputs: list[Any] = [] + for _ in range(2): + try: + proc = subprocess.run( + [python, str(solver_path), str(inst_path)], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=timeout, + cwd=str(solver_path.parent), + env=candidate_env(), + ) + except subprocess.TimeoutExpired: + return False, "timeout during determinism check" + except Exception as exc: + return False, f"error during determinism check: {exc}" + if proc.returncode != 0: + return False, f"candidate exited with code {proc.returncode}" + try: + outputs.append(json.loads(proc.stdout)) + except Exception as exc: + return False, f"cannot parse determinism output: {exc}" + if outputs[0] != outputs[1]: + return False, "candidate is not deterministic (output differs across two runs)" + return True, "" diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/.gitignore b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/.gitignore new file mode 100644 index 00000000..7a60b85e --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md new file mode 100644 index 00000000..6fb4d612 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md @@ -0,0 +1,151 @@ +# CuttingOptimizationOnline: Online Optimization of Continuous-Casting Cutting (Frontier-Eng Benchmark) + +An **original** Frontier-Engineering benchmark extending the offline +[CuttingOptimization](../CuttingOptimization/README.md) task with a genuinely **online** +(closed-loop) decision process, modeled on CUMCM 2021 Problem D. + +A continuously cast steel billet is drawn past a cutter. **Crystallizer anomalies create +0.8 m scrap segments, but the agent is only told about a segment when it is within +`reveal_lead` metres of the cut line** (default 8.0 m → hidden anomalies; `reveal_lead = 60` +is the "faithful" setting where the agent sees all relevant defects). The agent is invoked +*once per cut decision* with the **current visible state only** — it never sees future +defects — and must choose the next cut length. At the end the plan is scored against the +**full hidden defect set**: any piece overlapping a scrap segment is contaminated and fully +scrapped. + +The full rules and evaluation semantics are in [Task.md](./Task.md) (Chinese). + +## Layout + +``` +benchmarks/ContinuousCasting/CuttingOptimizationOnline/ +├── baseline/solver.py # Agent solver: decide(state)->piece_length (EVOLVE-BLOCK editable) +├── verification/ +│ ├── generator.py # Seeded instance generator (HIDDEN anomaly schedule + reveal_lead) +│ ├── simulator.py # Closed-loop simulator + contamination/scrap scoring +│ ├── evaluate.py # Per-decision closed-loop evaluation entry +│ ├── validator.py # Integrity checks (static + FRONTIER_* stripping + determinism) +│ ├── ref_solver.py # Clairvoyant reference DP (sees all defects) — the ceiling +│ ├── multiseed_stat.py # Multi-run mean±std tool +│ ├── test_simulator.py # Unit tests: scoring / validity +│ ├── test_generator.py # Unit tests: deterministic / hidden schedule / reveal_lead +│ ├── test_ref_solver.py # Unit tests: reference validity + beats baseline +│ ├── test_validator.py # Unit tests: static checks / env stripping +│ ├── test_evaluator.py # Unit tests: closed-loop eval / cheat rejection / generation +│ ├── test_frontier_eval_evaluator.py # Unit tests: sandbox evaluator entry +│ ├── data/instances/ # 8 fixed instances (seed-fixed) +│ ├── docker/Dockerfile # Minimal stdlib-only image +│ └── requirements.txt +├── frontier_eval/ # UnifiedTask metadata (openai-compatible LLM) +├── Task.md # Formal model, interface, scoring, reference scores +└── README.md +``` + +## Requirements + +- Python >= 3.10, standard library only. The benchmark itself needs no third-party deps. +- To *run the agent search* you also need the Frontier-Eng framework + an OpenAI-compatible + LLM endpoint (this task uses `deepseek-v4-flash` through a local reasoning-control proxy). + +## Run + +```powershell +# Score the baseline solver on the fixed 8-instance set (per-decision closed loop) +python verification/evaluate.py baseline/solver.py --reveal-lead 10 + +# Add runtime-generated instances (anti-hardcoding) +$env:ONLINE_CUT_EVAL_GENERATE_SEED = "" +python verification/evaluate.py baseline/solver.py + +# Multi-run stats (mean ± std) across a framework run dir +python verification/multiseed_stat.py --runs-dir runs/unified__ContinuousCasting__CuttingOptimizationOnline/openevolve +``` + +### Using an OpenAI-compatible proxy (required for the agent search) + +`deepseek-v4-flash` is a reasoning model that can exhaust the token budget and return empty +content unless reasoning is controlled. Two things are needed: + +1. **Route the LLM through a proxy** that raises `max_tokens` and injects a low + `reasoning_effort`, e.g. `PROXY_PORT=8765 REASONING_MODE=low MAX_TOKENS=32768 python deepseek_proxy.py`, + then `OPENAI_API_BASE=http://127.0.0.1:8765/v1`. +2. **ShinkaEvolve in particular loads `.env` with `override=True`**, which clobbers + `OPENAI_API_BASE` and makes it bypass the proxy. **Force it via the hydra override + `llm.api_base=http://127.0.0.1:8765/v1`** (config value, not env). openevolve / abmcts + read `OPENAI_API_BASE` directly and do not need this. + +## Tests + +```powershell +python -m unittest discover -s verification -p "test_*.py" +``` + +23 tests across simulator / generator / ref_solver / validator / evaluator / sandbox +evaluator: scoring & validity (contamination, short-tail-as-scrap), deterministic generation, +reference validity + beats-baseline, validator integrity (EVOLVE-BLOCK / forbidden refs / +absolute paths / `FRONTIER_*` stripping / determinism), closed-loop evaluation (cheating +candidate rejected, runtime generation), and sandbox-evaluator consistency. + +## Integrity / threat model + +- `verification/ref_solver.py` and `verification/generator.py` are **not** copied into the + sandbox and are additionally forbidden by the validator (`ref_solver`, `generator`, + `anomaly_seed` tokens). +- Candidate subprocesses get **all `FRONTIER_*`** and `ONLINE_CUT_EVAL_*` variables stripped + (`validator.candidate_env`), closing the host-env side channel. +- **Runtime generation** (`ONLINE_CUT_EVAL_GENERATE_SEED`) produces fresh instances at eval + time, so a candidate cannot pre-position answers. (A fixed seed is predictable if the + generator is public; use a fresh seed per runner for true anti-fingerprinting.) +- Determinism probe: the closed loop is run twice on a probe instance; the cut sequences must + match. +- Honest note: in process mode the candidate has host filesystem access (framework-wide + limitation); this benchmark relies on the layered defenses above. + +## Scoring + +- **Metric**: material utilization `util = 100 * (S - scrap) / S` averaged over instances + (0-100, higher better). `scrap` = contaminated pieces (any overlap with a defect → whole + piece scrapped) + clean pieces < 8.0 m (fully scrapped) + clean pieces above `target_max` + (excess scrapped), plus a tiny target-fit penalty (`1e-4 * Σ|delivered − target|`) to break + ties. +- **Contamination is unavoidable**: the minimum cut is 4.8 m, so a 0.8 m defect can never be + isolated; it always contaminates ≥ 4.8 m of product. This is the source of the + "online difficulty" and the informative asymmetry. +- **Reference scores (fixed 8 instances)**: baseline (fixed-target greedy) = **52.4**; + clairvoyant DP (`verification/ref_solver.py`, sees all defects) = **76.4**. The clairvoyant + is the theoretical ceiling — **an online agent that cannot see the future cannot reach it**. + +### Agent scores (final config: `reveal_lead = 10`; unified low reasoning, 15 generations each, 3 runs per framework) + +| Framework | run #1 | run #2 | run #3 | mean ± std | +|---|---|---|---|---| +| openevolve | 69.48 | 70.10 | 70.10 | **69.89 ± 0.36** | +| shinkaevolve | 70.06 | 71.18 | 70.10 | **70.45 ± 0.64** | +| abmcts | 72.31 | 70.10 | 70.45 | **70.95 ± 1.19** | + +All 9 runs combined: mean **70.43 ± 0.83**. Reference (clairvoyant, sees all defects) = **76.4** +(gap ≈ 6.0), baseline (fixed-target greedy) = **52.4**. + +Observed: **every run is below the clairvoyant ceiling (76.4) and above baseline (52.4)** — i.e. +the hidden-anomaly info asymmetry genuinely keeps online agents from the full-information optimum, +consistently across all three frameworks. This is the **key difference vs the offline** task, where +openevolve reaches the (offline) optimal exactly. The per-framework spread is small +(std ≈ 0.4–1.2), so agent scores settle around the "safe short-cut" plateau (~70) that limits how +far an online agent can get without full foresight. + +> Honest note: agents improve in step-jumps (stuck at baseline for several generations, then a +> single mutation cracks ~70), not gradual climbing — consistent with a hard constraint where +> "get the strategy right once" beats incremental search. AB-MCTS's single high run (72.31) vs its +> std (1.19) reflects run-to-run variance, not a systematic advantage. + +> Honest note: agents improve in step-jumps (stuck at baseline for several generations, then a +> single mutation cracks ~68-72), not gradual climbing — consistent with a hard constraint +> where "get the strategy right once" beats incremental search. + +## Docker + +A minimal `python:3.11-slim` image is provided (`verification/docker/Dockerfile`). Docker +isolation scoring depends on the shared Frontier-Eng framework's env-forwarding, which is a +known framework-level limitation (the reference path env may not reach the container). +`docker` isolation is therefore best verified under WSL/Linux with the unified runtime's +`isolation_mode=docker`; on Windows hosts it is limited by a framework path bug. diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README_zh-CN.md b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README_zh-CN.md new file mode 100644 index 00000000..9edbdfd2 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README_zh-CN.md @@ -0,0 +1,116 @@ +# CuttingOptimizationOnline：连铸切割的在线优化（Frontier-Eng 基准） + +一个**原创**的 Frontier-Engineering 基准，是离线版 +[CuttingOptimization](../CuttingOptimization/README_zh-CN.md) 的**在线（闭环）**扩展，题材取自 2021 全国大学生数学建模竞赛 D 题。 + +一根连续浇铸的钢坯被拉过切割机。**结晶器异常会产生 0.8m 报废段，但 agent 只在报废段距切割线 `reveal_lead` 米以内时才被告知**（默认 8m → 隐藏异常；`reveal_lead=60` 为"忠实版"，此时 agent 能看到所有相关异常）。agent **每个切割决策点被调用一次、只拿到当下可见状态**，永远看不到未来，并要选下一刀长度。最后按**完整隐藏报废表**评分：任何与报废段重叠的切块都被污染、整块报废。 + +完整规则与评测语义见 [Task.md](./Task.md)。 + +## 目录结构 + +``` +benchmarks/ContinuousCasting/CuttingOptimizationOnline/ +├── baseline/solver.py # agent 求解器：decide(state)->切长（EVOLVE-BLOCK 可改） +├── verification/ +│ ├── generator.py # 种子实例生成器（隐藏异常表 + reveal_lead） +│ ├── simulator.py # 闭环模拟器 + 污染/报废评分 +│ ├── evaluate.py # 每决策点闭环评测入口 +│ ├── validator.py # 完整性校验（静态 + FRONTIER_* 剥离 + 确定性） +│ ├── ref_solver.py # 全知参考解 DP（看全异常）—— 理论天花板 +│ ├── multiseed_stat.py # 多轮均值±std 工具 +│ ├── test_simulator.py # 单测：评分/合法性 +│ ├── test_generator.py # 单测：确定性/隐藏表/reveal_lead +│ ├── test_ref_solver.py # 单测：参考解合法性 + 优于基线 +│ ├── test_validator.py # 单测：静态检查/环境剥离 +│ ├── test_evaluator.py # 单测：闭环评测/作弊拒绝/生成 +│ ├── test_frontier_eval_evaluator.py # 单测：沙箱入口 +│ ├── data/instances/ # 8 个固定实例（种子固定） +│ ├── docker/Dockerfile # 最简 python:3.11-slim 镜像 +│ └── requirements.txt +├── frontier_eval/ # UnifiedTask 元数据（OpenAI-compatible LLM） +├── Task.md # 正式模型、接口、评分、参考分 +└── README_zh-CN.md +``` + +## 运行 + +```powershell +# 在固定 8 实例上给 baseline 打分（每决策点闭环） +python verification/evaluate.py baseline/solver.py --reveal-lead 10 + +# 加运行时生成实例（防硬编码） +$env:ONLINE_CUT_EVAL_GENERATE_SEED = "" +python verification/evaluate.py baseline/solver.py + +# 多轮统计（mean±std） +python verification/multiseed_stat.py --runs-dir runs/unified__ContinuousCasting__CuttingOptimizationOnline/openevolve +``` + +### 用 OpenAI-compatible proxy（agent 搜索必需） + +`deepseek-v4-flash` 是思考型模型，可能吃光 token 预算而返回空 content。需要： + +1. **让 LLM 走代理**：抬高 `max_tokens` 并注入低 `reasoning_effort`，如 + `PROXY_PORT=8765 REASONING_MODE=low MAX_TOKENS=32768 python deepseek_proxy.py`，再 `OPENAI_API_BASE=http://127.0.0.1:8765/v1`。 +2. **shinkaevolve 会用 `override=True` 加载 `.env`**，把 `OPENAI_API_BASE` 覆盖掉、从而绕过代理——**必须用 hydra 覆盖 `llm.api_base=http://127.0.0.1:8765/v1`**（配置值，非环境变量）。openevolve / abmcts 直接读 `OPENAI_API_BASE`，无需此步。 + +## 测试 + +```powershell +python -m unittest discover -s verification -p "test_*.py" +``` + +23 个单测（simulator / generator / ref_solver / validator / evaluator / 沙箱 evaluator）： +评分与合法性（污染、短尾当报废）、确定性生成、参考解合法且优于基线、validator 完整性 +（EVOLVE-BLOCK / 禁引用 / 绝对路径 / `FRONTIER_*` 剥离 / 确定性）、闭环评测（作弊候选被拒、 +运行时生成）、以及沙箱入口一致性。 + +## 完整性 / 威胁模型 + +- `verification/ref_solver.py` 与 `generator.py` **不**进沙箱，且被 validator 禁用 + （`ref_solver`、`generator`、`anomaly_seed` token）。 +- 候选子进程剥离**所有 `FRONTIER_*`** 与 `ONLINE_CUT_EVAL_*` 变量（`candidate_env`）， + 封宿主侧信道。 +- **运行时生成**（`ONLINE_CUT_EVAL_GENERATE_SEED`）评测时现场生成实例，候选无法预记忆。 + （固定种子可预测；要真正防指纹，runner 每轮用新种子。） +- 确定性探针：对探针实例跑两遍闭环，切段序列必须一致。 +- 诚实说明：process 模式下候选有主机文件系统访问（框架级限制）；本基准依赖上述分层防御。 + +## 评分 + +- **指标**：材料利用率 `util = 100*(S - scrap)/S`，多实例取平均（0~100，越高越好）。 + `scrap` = 污染块（与任一报废段重叠即整块报废）+ 干净块 <8m 整块报废 + 干净块超 `target_max` + 的余量报废，外加一个极小贴合度惩罚 `1e-4 * Σ|交付 - target|` 破平。 +- **污染不可避免**：最小切段 4.8m > 0.8m 报废段，报废段永远无法单独切出，必污染 ≥4.8m 成品。 + 这正是"在线难度"与信息不对称的来源。 +- **参考分（固定 8 实例）**：baseline（恒定目标贪心）= **52.4**；全知 DP（`ref_solver.py`， + 看全异常）= **76.4**。全知是**理论天花板**——看不到未来的在线 agent **无法达到它**。 + +### Agent 分（最终配置 `reveal_lead = 10`；统一 low 推理，各 15 代，每框架 3 次） + +| 框架 | 次1 | 次2 | 次3 | mean ± std | +|---|---|---|---|---| +| openevolve | 69.48 | 70.10 | 70.10 | **69.89 ± 0.36** | +| shinkaevolve | 70.06 | 71.18 | 70.10 | **70.45 ± 0.64** | +| abmcts | 72.31 | 70.10 | 70.45 | **70.95 ± 1.19** | + +9 次合计：mean **70.43 ± 0.83**；全知 reference（看全异常）= **76.4**（差 ≈6.0），baseline（恒定目标贪心）= **52.4**。 + +观察：**每一次都低于全知天花板（76.4）、高于 baseline（52.4）**——即隐藏异常的信息不对称 +**确实让在线 agent 无法达到全知最优**，三个框架一致。这是与离线版的关键差别（离线版 openevolve +能精确达到离线最优）。各框架 spread 小（std≈0.4–1.2），agent 分数聚在"安全短切"平台（~70）附近， +正是"看不到未来"限制住 agent 的地方。 + +> 诚实说明：agent 改进呈**阶梯式跳变**（卡 baseline 多代、某次突变到 ~70），不是渐进爬升——符合 +> "强约束下把策略一次想对"胜过增量搜索。AB-MCTS 那次高分（72.31）与其 std（1.19）反映的是 +> 运行间随机性，不是系统性优势。 + +> 诚实说明：agent 改进呈**阶梯式跳变**（卡 baseline 多代、某次突变到 ~68-72），不是渐进爬升 +> ——符合"强约束下把策略一次想对"胜过增量搜索。 + +## Docker + +提供最简 `python:3.11-slim` 镜像（`verification/docker/Dockerfile`）。Docker 隔离评分依赖共享 +Frontier-Eng 框架的 env 转发（已知框架级限制，参考路径 env 可能进不了容器）。`docker` 隔离最好在 +WSL/Linux 下用 unified 的 `isolation_mode=docker` 验证；Windows 宿主受框架路径 bug 限制。 diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/Task.md b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/Task.md new file mode 100644 index 00000000..74977f31 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/Task.md @@ -0,0 +1,80 @@ +# 连铸切割的在线优化（CuttingOptimizationOnline） + +## 0. 与离线版（CuttingOptimization）的区别一句话 + +离线版给**整个钢坯 + 全部零废段位置**，agent 一次性产出静态切段，可推演精确 DP → 对强 AI 太容易（openevolve 第 8 轮即打满最优 88.72）。 +在线版是**闭环交互**：异常在运行中**按揭示距离 R 逐步暴露**，agent 在**每个切割决策点只拿到当下可见状态、看不到未来**，必须在"切长块省料"与"怕撞未知报废"之间做真实权衡。**信息不对称**是难度的唯一来源。 + +## 1. 工艺参数（吞吐非约束，难点在信息不对称） + +- 拉坯速度 `v = 1.0` m/min；切一块 `tc = 3` min + 回程 `tr = 1` min；结晶器→切割机 `D = 60` m。 +- 报废段长 `scrap_len = 0.8` m。 +- 切长窗口：能运 `[min_basic, max_basic] = [4.8, 12.6]`；下道可接 `[min_process, max_process] = [8.0, 11.6]`；用户 `target` + 窗口 `[target_min, target_max]`。 +- **揭示距离 `reveal_dist = R`（默认 8.0 m）**：一段报废料仅在距离当前切割启动点 ≤ `R` 米时，才"揭示"给 agent。 +- 因 `v×(tc+tr)=4m < 4.8m`，切割机总能跟上 → **无吞吐瓶颈**。时间只在"揭示与到位"时差上起作用，不构成物理约束。 + +> 为什么 `R < max_basic`：若把折现距离设成原题的 60m 缓冲（远大于 12.6m 切段），agent 在承诺每块之前总能看见该块内所有异常，从而永远主动避开 → 退化为离线。取 `R` 小于最大切段，使"远端未知带"存在，才产生真实不确定性。这是本任务与离线版难度分界的核心旋钮。 + +## 2. 材料坐标与时序 + +材料用**流坐标** `x ∈ [0, S]`（S = v·浇铸时长 = 总材料长度）。横截面 `x` 到达切割点时间为 `x/v + D/v`。正被切割的横截面位置记作 `cut_pos`（也是下一段的起点），初始 0。 + +异常由隐藏种子确定：一段报废料占据流区间 `[x_a, x_a+0.8]`。它在**距切割点 ≤ R 时揭示**（即当 `x_a ≤ cut_pos + R`），并将其位置/到达时间加入 agent 可见列表。 + +## 3. 求解接口（闭环 REPL） + +agent 程序**启动一次**：`python solver.py `（instance 里**不含任何异常/异常种子**），进入循环： + +- 从 stdin 读一行**状态 JSON**： +```json +{"cut_pos": 12.6, "committed": [9.1, 11.4], "visible_defects": [{"x": 20.0}], "target": 9.5, + "target_min": 9.0, "target_max": 10.0, "limits": {"min_basic":4.8,"max_basic":12.6,"min_process":8.0,"max_process":11.6}, + "total_length": 106.2} +``` + - `cut_pos`：当前切割启动点（下一段的起点）。 + - `committed`：已承诺的切段长度列表（之和 = cut_pos）。 + - `visible_defects`：**已揭示**（距 cut_pos ≤ R）且尚未被切过/尚未过去的报废段流位置（只含 `x_a`，长度固定 0.8m）。 + - `reveal_dist`、`total_length`、process 等。 +- 向 stdout 写一行**决策 JSON**：`{"piece_length": L}`，`L ∈ [4.8, 12.6]`。 +- 评审器按流推进：`cut_pos += L`，重复喂状态/拿下一刀；当 `S - cut_pos ≤ 12.6` 时强制收尾（最后一段 = `S - cut_pos`，agent 无需作答）。 + +agent **永远看不到** `x_a > cut_pos + R` 的异常（未揭示）；也看不到完整报废表。`visible_defects` 只含已揭示的。 + +## 4. 污染判定（不可规避的代价，难度来源） + +由于最小切段 4.8m > 0.8m，0.8m 报废段无法被单独切出。因此： +- 任何**流区间与任一报废区间重叠**的已承诺切段 → **整块污染，全部报废**（报废长度 = 整块长度）。 +- 每个报废段必然污染 ≥1 块（无解）→ 报废有**下限**；agent 要尽量让被污染块短、并让其余干净块进目标窗口。 +- 若 agent 切长块 > R，而远端 `(cut_pos+R, cut_pos+L]` 内恰好有未揭示报废料 → **惊喜污染**，这是信息不对称的惩罚。 + +## 5. 评分（确定性） + +整根材料切完： +- `scrap` = 所有污染块长度之和 +（干净块）`< min_process` 整块报废、超 `target_max` 部分报废、窗口外偏短仅贴合度惩罚。 +- 贴合度惩罚 `penalty = Σ|min(块长, target_max) − target|`（只对干净块）。 +- 指标：`util = 100 × (S − scrap) / S`，多实例平均（0~100，越高越好），外加一个极小贴合度 + 惩罚破平。**不设调整次数惩罚**（原题并没说"调整多不好"，动态调整是正常响应，故不计入）。 + +seed 固定 → 报废表与揭示序列确定 → 模拟与评分可复现、纯标准库。 + +## 6. 三层参照（区分度结构） + +| 求解器 | 信息 | 预期 | +|---|---|---| +| baseline（贪心恒定/不看异常） | 无(或不理会) | 底部 | +| **agent（在线，限视 R）** | 只看到 R 内 | 中间：**低于全知参考解**（看不到未来） | +| reference（全知 clairvoyant DP） | 看全报废表 | 理论最优 → 天花板 | + +**关键**：agent 因限视 **打不满参考解**——这正是"在线比离线难"的体现（离线 agent 能推 DP +打满最优；在线 agent 做不到全知）。 + +## 7. 既定设置（已定稿） + +- `reveal_lead`（揭示提前量，米/分钟）是实例 `process` 参数。**默认取 `10.0`**（隐藏异常， + 制造信息不对称）；`60.0` 为"忠实版"（此时 agent 能看到所有相关异常，接近离线）。评估入口 + `verification/evaluate.py --reveal-lead ` 可覆盖。 +- **接口为"每决策点一次子进程调用"**：`python solver.py ` → 打印 `{"piece_length": L}` + （`state` 只含 `cut_pos`/`committed`/`visible_defects`/target/limits/`reveal_lead` 等，不含未来 + 异常；`visible_defects` 只含距 `cut_pos` ≤ `reveal_lead` 的已揭示报废段）。评估器在时间线上 + 每个决策点新建一次子进程喂状态。 +- metrics：`util` 为主 + 极小权重×贴合度。（不带调整次数惩罚。） diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/baseline/result_log.txt b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/baseline/result_log.txt new file mode 100644 index 00000000..db93b932 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/baseline/result_log.txt @@ -0,0 +1,28 @@ +CuttingOptimizationOnline — baseline & reference scores (8 fixed instances, online closed-loop). +Final config: reveal_lead = 10.0 (hidden anomalies → genuine online info asymmetry). + +Metric: material utilization = 100*(S - scrap)/S, averaged over instances (higher better). +Reference (clairvoyant DP, sees all defects) = theoretical ceiling. +Baseline (fixed-target greedy) = floor. + + baseline = 52.4 + reference (clairvoyant, all-foresight) = 76.4 <- ceiling an online agent cannot reach + +Agent scores (unified low-reasoning via proxy, 15 generations each, 3 runs per framework): + + Framework | run1 | run2 | run3 | mean ± std + -------------|-------|-------|-------|----------- + openevolve | 69.48 | 70.10 | 70.10 | 69.89 ± 0.36 + shinkaevolve | 70.06 | 71.18 | 70.10 | 70.45 ± 0.64 + abmcts | 72.31 | 70.10 | 70.45 | 70.95 ± 1.19 + + All 9 runs: mean 70.43 ± 0.83. + +Key finding: every run is below the clairvoyant ceiling (76.4) and above baseline (52.4) — +the hidden-anomaly info asymmetry keeps online agents from the full-information optimum +consistently across all three frameworks. This is the key difference vs the offline task +(where openevolve reaches the offline optimum exactly = 88.72). + +Frameworks were run with the OpenAI-compatible reasoning-control proxy at REASONING_MODE=low; +ShinkaEvolve additionally needs the hydra override `llm.api_base=http://127.0.0.1:8765/v1` +(its .env override otherwise makes it bypass the proxy and return empty content). diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/baseline/solver.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/baseline/solver.py new file mode 100644 index 00000000..9b924ceb --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/baseline/solver.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""在线切割 baseline 求解器（朴素：恒定切目标值，无视异常）。 + +用法：python solver.py +读取一个状态 JSON（决策点），向 stdout 打印 {"piece_length": L}（L ∈ [min_basic, max_basic]）。 +本 solver 由评估器在时间线上每个决策点调用一次；state 只含"已揭示"异常。 + +只允许修改 EVOLVE-BLOCK 区域内的代码；接口契约（main/JSON/JSON）必须保留。 + +基线是"恒定切目标值、不在意报废段"：从不根据 visible_defects 调整长度。 +在会出现异常的实例上，它常把报废段切进整块成品里（或切出超/欠目标）， +因此明显弱于"看到异常勤快避让"的 agent，更弱于全知参考解。 +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + + +def decide(state: dict) -> float: + """给定一个决策点状态，返回下一刀切段长度。""" + # EVOLVE-BLOCK-START + # 朴素策略：恒定切到用户目标值，并夹到 [min_basic, max_basic]。 + # 完全不看 state["visible_defects"]（不理会报废段），因此容易踩雷。 + t = float(state.get("target", 9.5)) + lo = float(state["limits"]["min_basic"]) + hi = float(state["limits"]["max_basic"]) + total = float(state["total_length"]) + cut_pos = float(state["cut_pos"]) + remaining = total - cut_pos + # 若剩余可直接作为最后一段，则切剩余（保证合法尾段）。 + if remaining <= hi: + return max(lo, remaining) + cand = max(lo, min(hi, t)) + # 避免留下 (0, lo) 的非法尾段：缩短当前块，让余段恰好为 lo。 + if 0 < remaining - cand < lo: + cand = remaining - lo + if cand < lo: + cand = remaining # 退化为整段（此时 remaining 应 >= lo） + return max(lo, min(hi, cand)) + # EVOLVE-BLOCK-END + + +def main() -> int: + if len(sys.argv) < 2: + print("usage: python solver.py ", file=sys.stderr) + return 2 + state = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) + print(json.dumps({"piece_length": decide(state)})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/agent_files.txt b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/agent_files.txt new file mode 100644 index 00000000..640607a1 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/agent_files.txt @@ -0,0 +1,4 @@ +README.md +Task.md +baseline/solver.py +frontier_eval/constraints.txt diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/artifact_files.txt b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/artifact_files.txt new file mode 100644 index 00000000..bc02d42d --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/artifact_files.txt @@ -0,0 +1,2 @@ +# No extra artifact files are auto-collected by default. +# metrics.json and artifacts.json are handled separately by UnifiedTask. diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/constraints.txt b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/constraints.txt new file mode 100644 index 00000000..a74a9dd1 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/constraints.txt @@ -0,0 +1,22 @@ +UnifiedTask constraints (online cutting, closed-loop): +1) Only modify `baseline/solver.py`, and only inside the EVOLVE-BLOCK-START / EVOLVE-BLOCK-END region. +2) Public contract: the evaluator drives you on a timeline. It calls `python baseline/solver.py ` + and expects a single JSON object {"piece_length": L} on stdout, where L is a cut length in + [limits.min_basic, limits.max_basic] (metres) for the NEXT piece starting at state.cut_pos. + You are invoked once per cutting decision, in order, and receive only the defects that have + ALREADY been revealed (state.visible_defects). You CANNOT see future defects — that is the + core of the online task. State fields: cut_pos, total_length, target/target_min/target_max, + limits, reveal_lead, visible_defects, committed (the lengths already cut). +3) Do not modify benchmark assets, documentation, verification code, instance data, or `frontier_eval/` metadata. +4) Every returned L must be a valid cut. Malformed output, non-numeric/infinite L, crashes, or timeouts + cause that decision to fall back and score worse. The final plan is validated against the FULL + (hidden) defect set: any piece overlapping a 0.8 m scrap segment is fully scrapped (contamination), + pieces < 8.0 m are fully scrapped, pieces above target_max incur excess scrap, and target-fit is + measured by |min(piece, target_max) - target|. +5) You may import `verification/simulator.py` (read-only) to evaluate candidate decisions while searching, + but the decision you print must come from your own algorithm. You may NOT import or read + `verification/generator.py`, `verification/ref_solver.py`, or `verification/evaluate.py`. +6) Objective: minimize total scrap first (contaminated + short + over-window), then make shipped pieces + close to the target. Score = mean material utilization 100*(total_length - scrap)/total_length over + instances (0-100, higher is better). Because you cannot see the future, a clairvoyant reference + solver (which sees all defects) is the theoretical ceiling and is unreachable by an optimal online agent. diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/copy_files.txt b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/copy_files.txt new file mode 100644 index 00000000..79f8b1a5 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/copy_files.txt @@ -0,0 +1,6 @@ +baseline +verification/evaluate.py +verification/simulator.py +verification/validator.py +verification/data/instances +frontier_eval diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/eval_command.txt b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/eval_command.txt new file mode 100644 index 00000000..3a31d525 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/eval_command.txt @@ -0,0 +1 @@ +FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR={benchmark_source} {python} frontier_eval/run_eval.py --candidate {candidate} --metrics-out metrics.json --artifacts-out artifacts.json diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/eval_cwd.txt b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/eval_cwd.txt new file mode 100644 index 00000000..9c558e35 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/eval_cwd.txt @@ -0,0 +1 @@ +. diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/evaluator.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/evaluator.py new file mode 100644 index 00000000..53deec30 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/evaluator.py @@ -0,0 +1,31 @@ +"""Unified evaluator entry point for the CuttingOptimizationOnline benchmark. + +Loaded by `frontier_eval/run_eval.py`; exposes top-level `evaluate(program_path, **kwargs)`. +Real implementation lives in `verification/evaluate.py`. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any + +TIME_BUDGET_S = 60.0 + + +def _load_verification_evaluator() -> Any: + evaluator_path = (Path(__file__).resolve().parent.parent / "verification" / "evaluate.py") + spec = importlib.util.spec_from_file_location("_oc_verification_evaluator", evaluator_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Failed to load verification evaluator from {evaluator_path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def evaluate(program_path: str, **kwargs: Any) -> Any: + module = _load_verification_evaluator() + result = module.evaluate(program_path, time_budget=TIME_BUDGET_S, **kwargs) + if isinstance(result, dict) and "metrics" in result: + return result + return {"metrics": result, "artifacts": {}} diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/initial_program.txt b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/initial_program.txt new file mode 100644 index 00000000..6645b02f --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/initial_program.txt @@ -0,0 +1 @@ +baseline/solver.py diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/readonly_files.txt b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/readonly_files.txt new file mode 100644 index 00000000..1205de75 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/readonly_files.txt @@ -0,0 +1,4 @@ +README.md +Task.md +verification +frontier_eval diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/run_eval.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/run_eval.py new file mode 100644 index 00000000..720d0e09 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/run_eval.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import argparse +import json +import sys +import traceback +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +from typing import Any + +INVALID_COMBINED_SCORE = -1e18 + + +def _write_json(path: Path, obj: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(obj, ensure_ascii=False, indent=2, default=str) + "\n", + encoding="utf-8", + ) + + +def _normalize_result(result: Any) -> tuple[dict[str, Any], dict[str, Any]]: + if hasattr(result, "metrics") and hasattr(result, "artifacts"): + return dict(getattr(result, "metrics")), dict(getattr(result, "artifacts")) + + if isinstance(result, dict): + raw_metrics = result.get("metrics") + raw_artifacts = result.get("artifacts") + if isinstance(raw_metrics, dict): + return dict(raw_metrics), dict(raw_artifacts or {}) + return dict(result), {} + + raise TypeError( + "Evaluator must return an EvaluationResult-like object or a dict of metrics." + ) + + +def _load_local_evaluator() -> Any: + evaluator_path = Path(__file__).with_name("evaluator.py").resolve() + spec = spec_from_file_location("_frontier_eval_local_evaluator", evaluator_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Failed to load local evaluator from {evaluator_path}") + module = module_from_spec(spec) + spec.loader.exec_module(module) + try: + return getattr(module, "evaluate") + except AttributeError as exc: + raise RuntimeError( + f"Local evaluator does not define evaluate(): {evaluator_path}" + ) from exc + + +def _find_repo_root() -> Path: + import os + + env_root = os.environ.get("FRONTIER_ENGINEERING_ROOT") + if env_root: + return Path(env_root).expanduser().resolve() + + here = Path(__file__).resolve() + for parent in [here.parent, *here.parents]: + if (parent / "frontier_eval").is_dir() and (parent / "benchmarks").is_dir(): + return parent + return Path.cwd().resolve() + + +def _build_kwargs(evaluate_fn: Any) -> dict[str, Any]: + kwargs: dict[str, Any] = {} + try: + parameters = inspect_signature(evaluate_fn) + except Exception: + return kwargs + + if "repo_root" in parameters: + kwargs["repo_root"] = _find_repo_root() + return kwargs + + +def inspect_signature(fn: Any) -> set[str]: + import inspect + + return set(inspect.signature(fn).parameters) + + +def _parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run a benchmark-local unified evaluator and export metrics/artifacts JSON." + ) + parser.add_argument("--candidate", required=True) + parser.add_argument("--metrics-out", default="metrics.json") + parser.add_argument("--artifacts-out", default="artifacts.json") + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = _parse_args(argv) + + candidate_path = Path(args.candidate).expanduser().resolve() + metrics_out = Path(args.metrics_out).expanduser().resolve() + artifacts_out = Path(args.artifacts_out).expanduser().resolve() + + metrics: dict[str, Any] = { + "combined_score": INVALID_COMBINED_SCORE, + "valid": 0.0, + } + artifacts: dict[str, Any] = { + "local_evaluator_path": str(Path(__file__).with_name("evaluator.py").resolve()), + "candidate_path": str(candidate_path), + } + + try: + evaluate_fn = _load_local_evaluator() + result = evaluate_fn(str(candidate_path), **_build_kwargs(evaluate_fn)) + metrics, evaluator_artifacts = _normalize_result(result) + artifacts.update(evaluator_artifacts) + except Exception as exc: + artifacts["error_message"] = str(exc) + artifacts["traceback"] = traceback.format_exc() + + _write_json(metrics_out, metrics) + _write_json(artifacts_out, artifacts) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_1.json b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_1.json new file mode 100644 index 00000000..8b6f57ba --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_1.json @@ -0,0 +1,33 @@ +{ + "seed": 1, + "anomaly_seed": 100001, + "process": { + "v": 1.0, + "tc": 3, + "tr": 1, + "buffer_len": 60.0, + "scrap_len": 0.8, + "reveal_lead": 10.0 + }, + "cast": { + "total_length": 75.77, + "t_cast": 195.77 + }, + "customer": { + "target": 8.5, + "target_min": 8.0, + "target_max": 9.0 + }, + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + }, + "defects": [ + [ + 28.96, + 29.76 + ] + ] +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_2.json b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_2.json new file mode 100644 index 00000000..b919e8e7 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_2.json @@ -0,0 +1,33 @@ +{ + "seed": 2, + "anomaly_seed": 100002, + "process": { + "v": 1.0, + "tc": 3, + "tr": 1, + "buffer_len": 60.0, + "scrap_len": 0.8, + "reveal_lead": 10.0 + }, + "cast": { + "total_length": 67.88, + "t_cast": 187.88 + }, + "customer": { + "target": 11.1, + "target_min": 10.6, + "target_max": 11.6 + }, + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + }, + "defects": [ + [ + 58.29, + 59.09 + ] + ] +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_3.json b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_3.json new file mode 100644 index 00000000..4f641a58 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_3.json @@ -0,0 +1,41 @@ +{ + "seed": 3, + "anomaly_seed": 100003, + "process": { + "v": 1.0, + "tc": 3, + "tr": 1, + "buffer_len": 60.0, + "scrap_len": 0.8, + "reveal_lead": 10.0 + }, + "cast": { + "total_length": 97.86, + "t_cast": 217.86 + }, + "customer": { + "target": 11.1, + "target_min": 10.6, + "target_max": 11.6 + }, + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + }, + "defects": [ + [ + 18.37, + 19.17 + ], + [ + 34.7, + 35.5 + ], + [ + 57.19, + 57.99 + ] + ] +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_4.json b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_4.json new file mode 100644 index 00000000..b4a1b5ad --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_4.json @@ -0,0 +1,53 @@ +{ + "seed": 4, + "anomaly_seed": 100004, + "process": { + "v": 1.0, + "tc": 3, + "tr": 1, + "buffer_len": 60.0, + "scrap_len": 0.8, + "reveal_lead": 10.0 + }, + "cast": { + "total_length": 118.25, + "t_cast": 238.25 + }, + "customer": { + "target": 8.5, + "target_min": 8.0, + "target_max": 9.0 + }, + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + }, + "defects": [ + [ + 9.89, + 10.69 + ], + [ + 24.89, + 25.69 + ], + [ + 31.28, + 32.08 + ], + [ + 39.15, + 39.95 + ], + [ + 63.81, + 64.61 + ], + [ + 108.41, + 109.21 + ] + ] +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_5.json b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_5.json new file mode 100644 index 00000000..67e24107 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_5.json @@ -0,0 +1,45 @@ +{ + "seed": 5, + "anomaly_seed": 100005, + "process": { + "v": 1.0, + "tc": 3, + "tr": 1, + "buffer_len": 60.0, + "scrap_len": 0.8, + "reveal_lead": 10.0 + }, + "cast": { + "total_length": 99.64, + "t_cast": 219.64 + }, + "customer": { + "target": 11.1, + "target_min": 10.6, + "target_max": 11.6 + }, + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + }, + "defects": [ + [ + 19.32, + 20.12 + ], + [ + 47.35, + 48.15 + ], + [ + 56.35, + 57.15 + ], + [ + 70.55, + 71.35 + ] + ] +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_6.json b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_6.json new file mode 100644 index 00000000..081ccdfe --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_6.json @@ -0,0 +1,77 @@ +{ + "seed": 6, + "anomaly_seed": 100006, + "process": { + "v": 1.0, + "tc": 3, + "tr": 1, + "buffer_len": 60.0, + "scrap_len": 0.8, + "reveal_lead": 10.0 + }, + "cast": { + "total_length": 154.74, + "t_cast": 274.74 + }, + "customer": { + "target": 11.1, + "target_min": 10.6, + "target_max": 11.6 + }, + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + }, + "defects": [ + [ + 8.86, + 9.66 + ], + [ + 15.1, + 15.9 + ], + [ + 20.72, + 21.52 + ], + [ + 33.13, + 33.93 + ], + [ + 45.85, + 46.65 + ], + [ + 53.46, + 54.26 + ], + [ + 71.21, + 72.01 + ], + [ + 91.21, + 92.01 + ], + [ + 105.29, + 106.09 + ], + [ + 119.26, + 120.06 + ], + [ + 138.01, + 138.81 + ], + [ + 148.16, + 148.96 + ] + ] +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_7.json b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_7.json new file mode 100644 index 00000000..db997b94 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_7.json @@ -0,0 +1,69 @@ +{ + "seed": 7, + "anomaly_seed": 100007, + "process": { + "v": 1.0, + "tc": 3, + "tr": 1, + "buffer_len": 60.0, + "scrap_len": 0.8, + "reveal_lead": 10.0 + }, + "cast": { + "total_length": 135.88, + "t_cast": 255.88 + }, + "customer": { + "target": 9.5, + "target_min": 9.0, + "target_max": 10.0 + }, + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + }, + "defects": [ + [ + 26.1, + 26.9 + ], + [ + 32.25, + 33.05 + ], + [ + 43.39, + 44.19 + ], + [ + 52.27, + 53.07 + ], + [ + 63.58, + 64.38 + ], + [ + 80.37, + 81.17 + ], + [ + 96.62, + 97.42 + ], + [ + 102.98, + 103.78 + ], + [ + 113.33, + 114.13 + ], + [ + 125.97, + 126.77 + ] + ] +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_8.json b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_8.json new file mode 100644 index 00000000..dae0b817 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/data/instances/instance_8.json @@ -0,0 +1,77 @@ +{ + "seed": 8, + "anomaly_seed": 100008, + "process": { + "v": 1.0, + "tc": 3, + "tr": 1, + "buffer_len": 60.0, + "scrap_len": 0.8, + "reveal_lead": 10.0 + }, + "cast": { + "total_length": 157.99, + "t_cast": 277.99 + }, + "customer": { + "target": 8.5, + "target_min": 8.0, + "target_max": 9.0 + }, + "limits": { + "min_basic": 4.8, + "max_basic": 12.6, + "min_process": 8.0, + "max_process": 11.6 + }, + "defects": [ + [ + 5.18, + 5.98 + ], + [ + 30.12, + 30.92 + ], + [ + 37.6, + 38.4 + ], + [ + 58.52, + 59.32 + ], + [ + 64.85, + 65.65 + ], + [ + 74.07, + 74.87 + ], + [ + 85.96, + 86.76 + ], + [ + 92.63, + 93.43 + ], + [ + 121.45, + 122.25 + ], + [ + 134.09, + 134.89 + ], + [ + 140.08, + 140.88 + ], + [ + 151.43, + 152.23 + ] + ] +} diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/docker/Dockerfile b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/docker/Dockerfile new file mode 100644 index 00000000..7c94d870 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/docker/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.11-slim + +WORKDIR /workspace + +# The evaluator, simulator, reference solver and baseline use only the Python +# standard library. The unified runtime mounts the benchmark sandbox into the +# container, so no benchmark files are baked into the image. The candidate may +# read `verification/simulator.py` (it is part of the scoring objective and is +# allowed by the constraints), but no instance data is baked in. + +ENV PYTHONUNBUFFERED=1 + +CMD ["python"] diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/evaluate.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/evaluate.py new file mode 100644 index 00000000..f8c1ec26 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/evaluate.py @@ -0,0 +1,223 @@ +"""在线切割评测入口（闭环 per-decision 驱动）。 + +每个实例：评测器在时间线上推进，每到一个切割决策点就： + 1) 用 simulator._make_state 构造"agent 可见状态"（只含已揭示异常，不含 future/hidden）； + 2) 把状态写入临时文件，`subprocess` 调用 `python `； + 3) 解析 {"piece_length": L} 作为下一刀长度，落账推进。 +直到整根材料切完；最后用完整 defects 表打分（污染/干净规则）。 + +评分 = 各实例材料利用率均值（0~100，越高越好）。 +防作弊/确定性/运行时生成 对齐离线版（见 validator.py）。 + +环境变量： + ONLINE_CUT_EVAL_GENERATE_SEED 设置后开启运行时生成实例（防硬编码） + ONLINE_CUT_EVAL_GENERATE_COUNT 生成实例数（默认 8） + +对外接口（供 frontier_eval/evaluator.py 包装）： + evaluate(program_path, *, time_budget=60.0) -> {"combined_score", "valid", "per_instance"} +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from simulator import load_instance, partition, score # noqa: E402 +from validator import candidate_env, check_candidate # noqa: E402 + +INVALID_SCORE = 0.0 +DATA_DIR = Path(__file__).resolve().parent / "data" / "instances" +DEFAULT_GENERATE_COUNT = 8 +GEN_DIFFS = ("easy", "medium", "hard", "medium", "hard", "medium", "hard", "hard") + + +def _source_benchmark_dir() -> Path | None: + raw = os.environ.get("FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR", "").strip() + if raw and Path(raw).is_dir(): + return Path(raw) + return None + + +class AgentClient: + """把 agent 子进程封装成 decision_fn(state)->length（每决策一次调用）。""" + + def __init__(self, program_path: Path, python: str, time_budget: float, tmp: Path): + self.program_path = program_path + self.python = python + self.time_budget = time_budget + self.tmp = tmp + + def decide(self, state: dict[str, Any]) -> float: + state_path = self.tmp / f"state_{id(state)}.json" + state_path.write_text(json.dumps(state, ensure_ascii=False), encoding="utf-8") + try: + proc = subprocess.run( + [self.python, str(self.program_path), str(state_path)], + capture_output=True, text=True, encoding="utf-8", errors="replace", + timeout=self.time_budget, cwd=str(self.program_path.parent), + env=candidate_env(), + ) + except subprocess.TimeoutExpired: + return float("nan") + if proc.returncode != 0: + return float("nan") + try: + obj = json.loads(proc.stdout) + return float(obj["piece_length"]) + except Exception: + return float("nan") + + +def _load_host_module(mod_name: str): + import importlib.util + src = os.environ.get("FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR", "").strip() + if src and Path(src).is_dir(): + p = Path(src) / "verification" / f"{mod_name}.py" + if p.is_file(): + spec = importlib.util.spec_from_file_location(f"_oc_{mod_name}", p) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + return None + + +def _generate_instances(base_seed: int, count: int, out_dir: Path, reveal_lead: float) -> list[Path]: + gen_mod = _load_host_module("generator") or __import__("generator") + paths: list[Path] = [] + i = 0 + attempts = 0 + while len(paths) < count and attempts < count * 64: + inst = gen_mod.generate(base_seed * 1000 + i, GEN_DIFFS[i % len(GEN_DIFFS)], reveal_lead) + i += 1 + attempts += 1 + if not gen_mod._drivable(inst): + continue + p = out_dir / f"gen_{base_seed}_{len(paths) + 1}.json" + p.write_text(json.dumps(inst, ensure_ascii=False) + "\n", encoding="utf-8") + paths.append(p) + return paths + + +def check_determinism(python: str, prog: Path, inst_path: Path, time_budget: float, + work: Path, reveal_lead: float | None = None) -> tuple[bool, str]: + """同一探针实例驱动两遍闭环，切段长度序列必须一致。""" + inst = load_instance(inst_path) + seqs: list[list[float]] = [] + for _ in range(2): + client = AgentClient(prog, python, time_budget, work) + seqs.append(partition(inst, client.decide)) + if seqs[0] != seqs[1]: + return False, "agent is not deterministic across two closed-loop runs" + return True, "" + + +def _run_one(prog: Path, inst_path: Path, time_budget: float, py: str, tmp: Path) -> tuple[float, Any]: + inst = load_instance(inst_path) + client = AgentClient(prog, py, time_budget, tmp) + cuts = partition(inst, client.decide) + ok, m = score(inst, cuts) + if not ok: + return 0.0, {"status": "invalid", "reason": m.get("reason")} + return m["util"], {"status": "ok", "util": m["util"], "scrap": m["scrap"], + "n_cuts": len(cuts)} + + +def evaluate(program_path: str, *, time_budget: float = 60.0, python: str | None = None, + data_dir: str | Path | None = None, reveal_lead: float | None = None) -> dict[str, Any]: + prog = Path(program_path).resolve() + if not prog.exists(): + return {"combined_score": 0.0, "valid": 0.0, "per_instance": {}, "error": "program not found"} + + violations = check_candidate(prog) + + inst_dir = Path(data_dir).resolve() if data_dir else DATA_DIR + instances = sorted(inst_dir.glob("instance_*.json")) if inst_dir.is_dir() else [] + if not instances: + # 若无固定实例集，现场生成一批（便于直接评测） + tmp = Path(tempfile.mkdtemp(prefix="oc_inst_")) + instances = _generate_instances(7, 6, tmp, reveal_lead or 10.0) + + gen_seed_raw = os.environ.get("ONLINE_CUT_EVAL_GENERATE_SEED", "").strip() + tmp: Path | None = None + if gen_seed_raw: + base_seed = 0 + try: + base_seed = int(gen_seed_raw) + except ValueError: + pass + count = DEFAULT_GENERATE_COUNT + try: + count = max(0, int(os.environ.get("ONLINE_CUT_EVAL_GENERATE_COUNT", "").strip())) + except ValueError: + pass + if count > 0: + tmp = Path(tempfile.mkdtemp(prefix="oc_gen_")) + instances = instances + _generate_instances(base_seed, count, tmp, + reveal_lead or 10.0) + + if not instances: + return {"combined_score": 0.0, "valid": 0.0, "per_instance": {}, + "error": "no instances"} + + py = python or sys.executable + + # 确定性探针：在一个探针实例上跑两遍闭合，切段序列必须一致 + if not violations: + probe = instances[0] + work = Path(tempfile.mkdtemp(prefix="oc_probe_")) + det_ok, det_note = check_determinism(py, prog, probe, time_budget, work) + shutil.rmtree(work, ignore_errors=True) + if not det_ok: + violations = [f"determinism check failed: {det_note}"] + + per_instance: dict[str, Any] = {} + total = 0.0 + all_valid = True + work = Path(tempfile.mkdtemp(prefix="oc_run_")) + try: + for inst_path in instances: + if violations: + per_instance[inst_path.name] = {"status": "preflight_failed", "reasons": violations} + continue + util, info = _run_one(prog, inst_path, time_budget, py, work) + per_instance[inst_path.name] = info + if util <= 0 and info.get("status") != "ok": + all_valid = False + total += util + finally: + shutil.rmtree(work, ignore_errors=True) + if tmp is not None: + shutil.rmtree(tmp, ignore_errors=True) + + combined = total / len(instances) if instances else 0.0 + return {"combined_score": round(combined, 2), "valid": 1.0 if all_valid and not violations else 0.0, + "per_instance": per_instance, "num_instances": len(instances), + "time_budget_s": time_budget, "generate_seed": gen_seed_raw or None} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="在线切割评测") + parser.add_argument("solver") + parser.add_argument("--time-budget", type=float, default=60.0) + parser.add_argument("--data-dir", type=str, default=None) + parser.add_argument("--reveal-lead", type=float, default=None, + help="覆盖实例的 reveal_lead 显式指定（便于多配置对比）") + args = parser.parse_args(argv) + result = evaluate(args.solver, time_budget=args.time_budget, + data_dir=args.data_dir, reveal_lead=args.reveal_lead) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/generator.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/generator.py new file mode 100644 index 00000000..72e484fd --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/generator.py @@ -0,0 +1,100 @@ +"""在线切割实例生成器：种子生成隐藏异常表（不进 agent 可见实例）。 + +参数语义： +- 浇铸时长 T_cast 决定总材料长度 S = v*T_cast（流坐标 [0, S]）。 +- 隐藏异常由 anomaly_seed 确定性生成（position、scrap_len 固定 0.8m）， + 在 process.reveal_lead 确定的提前量下"逐步揭示"给 agent。 +- 实例 JSON 含 `defects`（隐藏表，仅供 simulator/参考解使用）； + `strip_for_agent(inst)` 会去掉 defects/anomaly_seed，得到 agent 可见的版本。 +""" + +from __future__ import annotations + +import json +import random +import sys +from pathlib import Path +from typing import Any + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parent)) + +MIN_BASIC = 4.8 +MAX_BASIC = 12.6 +LIMITS = {"min_basic": 4.8, "max_basic": 12.6, "min_process": 8.0, "max_process": 11.6} + + +def generate(seed: int, difficulty: str = "medium", reveal_lead: float = 60.0, + n_anomaly: int | None = None) -> dict[str, Any]: + rng = random.Random(seed * 10007 + 11) + if difficulty == "easy": + s_lo, s_hi, a_lo, a_hi = 60.0, 90.0, 1, 3 + elif difficulty == "hard": + s_lo, s_hi, a_lo, a_hi = 120.0, 180.0, 6, 12 + else: + s_lo, s_hi, a_lo, a_hi = 80.0, 130.0, 3, 7 + + S = round(rng.uniform(s_lo, s_hi), 2) + n = n_anomaly if n_anomaly is not None else rng.randint(a_lo, a_hi) + T = float(rng.choice([8.5, 9.5, 11.1])) + t_min, t_max = T - 0.5, T + 0.5 + + scrap_len = 0.8 + defects: list[list[float]] = [] + # 生成异常位置：彼此与两端都留出 >= min_basic 的干净材料（保证每段可切、可污染） + low = MIN_BASIC + high = S - MIN_BASIC - scrap_len + attempts = 0 + while len(defects) < n and attempts < 800: + attempts += 1 + cand = round(rng.uniform(low, high), 2) + ok = True + for a, b in defects: + if cand < b + MIN_BASIC and a < cand + scrap_len + MIN_BASIC: + ok = False + break + if ok: + defects.append([cand, round(cand + scrap_len, 2)]) + defects.sort() + + # 每 1000 米诞生一个异常不现实——这里按"异常密度"换算为浇铸时长。 + # T_cast 取 S/v + 充分裕量，确保整根材料都在观测范围内。 + v = 1.0 + t_cast = round(S / v + 120.0, 2) + + inst: dict[str, Any] = { + "seed": seed, + "anomaly_seed": seed + 100000, + "process": {"v": v, "tc": 3, "tr": 1, "buffer_len": 60.0, + "scrap_len": scrap_len, "reveal_lead": reveal_lead}, + "cast": {"total_length": S, "t_cast": t_cast}, + "customer": {"target": T, "target_min": t_min, "target_max": t_max}, + "limits": LIMITS, + "defects": defects, + } + return inst + + +def strip_for_agent(inst: dict[str, Any]) -> dict[str, Any]: + """agent 可见版本：不含隐藏异常表/异常种子。""" + out = dict(inst) + out.pop("defects", None) + out.pop("anomaly_seed", None) + return out + + +def _drivable(inst: dict[str, Any]) -> bool: + return bool(inst["defects"]) and float(inst["cast"]["total_length"]) > 30.0 + + +if __name__ == "__main__": + import sys as _s + if len(_s.argv) < 2: + print("usage: python generator.py [difficulty] [reveal_lead]", file=_s.stderr) + raise SystemExit(2) + out = Path(_s.argv[1]) + diff = _s.argv[2] if len(_s.argv) > 2 else "medium" + rl = float(_s.argv[3]) if len(_s.argv) > 3 else 60.0 + inst = generate(1, diff, rl) + out.write_text(json.dumps(inst, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") + print(f"wrote {out} S={inst['cast']['total_length']} #defects={len(inst['defects'])} reveal_lead={rl}") diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/multiseed_stat.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/multiseed_stat.py new file mode 100644 index 00000000..e6df657a --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/multiseed_stat.py @@ -0,0 +1,87 @@ +"""多运行（多种子/多轮）agent 分数统计：对一组框架运行目录取 combined_score 的 mean±std。 + +用法： + python verification/multiseed_stat.py --runs-dir runs/unified__ContinuousCasting__CuttingOptimization/openevolve + python verification/multiseed_stat.py --runs-dir "runs/**/openevolve/deepseek-v4-flash" # glob + python verification/multiseed_stat.py --runs-dir --pattern "*openevolve*" + +从每个运行目录下读 `/best/best_program_info.json` 的 `metrics.combined_score` +（框架统一保存的 best 程序分数），对多次运行做 mean / std / min / max， +并给出"相对 reference 的余量"（gap = reference_util - mean）。 + +对齐 CVRP 评审点：agent 分数需多轮均值±std（而非单次），衡量稳定性与真实水平。 +纯标准库。 +""" + +from __future__ import annotations + +import argparse +import glob +import json +import math +import statistics +from pathlib import Path + +DEFAULT_REFERENCE = 88.72 # 全知参考解利用率（verification/ref_solver.py），可 --reference 覆盖 + + +def _best_score(run_dir: Path) -> float | None: + for rel in ("openevolve/best/best_program_info.json", + "shinkaevolve/best/best_program_info.json", + "abmcts/best/best_program_info.json", + "best/best_program_info.json"): + p = run_dir / rel + if p.is_file(): + try: + return float(json.loads(p.read_text(encoding="utf-8"))["metrics"]["combined_score"]) + except Exception: + return None + return None + + +def collect(runs_dir: str | Path, pattern: str | None = None) -> list[tuple[Path, float]]: + base = Path(runs_dir) + if pattern: + bests = sorted(Path(p) for p in glob.glob(str(base / pattern), recursive=True)) + else: + bests = sorted(base.rglob("best/best_program_info.json")) + out: list[tuple[Path, float]] = [] + seen: set[Path] = set() + for best in bests: + # best_program_info.json 位于 //best/ 下 + run_dir = best.parent.parent.parent + if run_dir in seen: + continue + score = _best_score(run_dir) + if score is not None: + out.append((run_dir, score)) + seen.add(run_dir) + return out + + +def main() -> int: + parser = argparse.ArgumentParser(description="多运行 agent 分数统计") + parser.add_argument("--runs-dir", required=True, help="运行目录或 glob 模式") + parser.add_argument("--pattern", default=None, help="追加的子 glob 模式") + parser.add_argument("--reference", type=float, default=DEFAULT_REFERENCE, + help="参考解利用率（默认 88.72）") + args = parser.parse_args() + + pairs = collect(args.runs_dir, args.pattern) + if not pairs: + print("no runs found under", args.runs_dir) + return 1 + + scores = [s for _, s in pairs] + mean = statistics.mean(scores) + std = statistics.stdev(scores) if len(scores) > 1 else 0.0 + print(f"runs: {len(scores)}") + for run_dir, s in pairs: + print(f" {run_dir.name}: {s}") + print(f"mean={mean:.2f} std={std:.2f} min={min(scores):.2f} max={max(scores):.2f}") + print(f"gap to reference ({args.reference}): {args.reference - mean:.2f}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/ref_solver.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/ref_solver.py new file mode 100644 index 00000000..05f71953 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/ref_solver.py @@ -0,0 +1,109 @@ +"""在线切割全知参考解（clairvoyant，看到全部报废表）。 + +静态一维划分 DP（网格化，纯标准库）： +- 流坐标 [0,S]，网格 GRID；每块长度 ∈ [min_basic, max_basic]。 +- 一块若与任一报废段区间重叠 -> 整块报废（cost = 块长，无贴合度惩罚）； + 否则按干净块规则（ bool: + for xa, xb in defects: + if x1 > xa + OVERLAP_TOL and x0 < xb - OVERLAP_TOL: + return True + return False + + +def _cost(inst: dict[str, Any], x0: float, c: float, defects: list[list[float]]) -> float: + if _piece_overlap(x0, x0 + c, defects): + return c # 污染块，整块报废（无贴合度惩罚） + t = float(inst["customer"]["target"]) + t_max = float(inst["customer"]["target_max"]) + lim_min_proc = _limits(inst)[2] + if c < lim_min_proc: + return c + scrap = max(0.0, c - t_max) + penalty = abs(min(c, t_max) - t) + return scrap + LAMBDA * penalty + + +def solve(inst: dict[str, Any]) -> dict[str, Any]: + """返回 {'cuts': [...]}：全知最优切段。""" + import math + + S = float(inst["cast"]["total_length"]) + defects = [list((a, b)) for a, b in inst.get("defects", []) or []] + min_basic, max_basic, _ = _limits(inst) + n = max(1, int(round(S / GRID))) + min_steps = int(round(min_basic / GRID)) + max_steps = int(round(max_basic / GRID)) + + INF = float("inf") + f = [INF] * (n + 1) + bp = [-1] * (n + 1) + f[0] = 0.0 + for pos in range(min_steps, n + 1): + best = INF + best_s = -1 + hi = min(max_steps, pos) + for s in range(min_steps, hi + 1): + prev = pos - s + if f[prev] >= INF: + continue + val = f[prev] + _cost(inst, prev * GRID, s * GRID, defects) + if val < best: + best = val + best_s = s + f[pos] = best + bp[pos] = best_s + + if f[n] >= INF: + # 兜底：均匀等分（保证产出合法切段） + k = max(1, math.ceil(S / max_basic)) + base = S / k + cuts = [round(base, 4)] * k + return {"cuts": cuts} + + cuts: list[float] = [] + pos = n + while pos > 0: + s = bp[pos] + cuts.append(round(s * GRID, 4)) + pos -= s + cuts.reverse() + # 消除浮点累计误差，使求和精确等于 S + drift = S - sum(cuts) + cuts[-1] = round(cuts[-1] + drift, 4) + return {"cuts": cuts} + + +def best_util(inst: dict[str, Any]) -> float: + """全知参考解的材料利用率。""" + cuts = solve(inst)["cuts"] + ok, m = score(inst, cuts) + return m.get("util", 0.0) if ok else 0.0 diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/requirements.txt b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/requirements.txt new file mode 100644 index 00000000..4e66700e --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/requirements.txt @@ -0,0 +1,2 @@ +# The evaluator uses only the Python standard library. +# Runtime requirement: Python >= 3.10 (no third-party dependencies). diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/simulator.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/simulator.py new file mode 100644 index 00000000..b204a48a --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/simulator.py @@ -0,0 +1,154 @@ +"""连铸切割在线模拟器（闭环、最小切段污染、揭示提前量）。 + +模型（详见 Task.md）： +- 材料用流坐标 x∈[0,S]，S = v*浇铸时长。横截面 x 到达切割点在 x/v + D/v。 +- 隐藏异常（由 anomaly_seed 决定）：报废段占流区间 [x_a, x_a+scrap_len]。 +- 揭示提前量 reveal_lead（分钟；v=1 时等于"上游米数"）：一段报废料在 + x_a <= cut_pos + reveal_lead 时"揭示"给 agent；否则不可见。reveal_lead=60 为忠实版 + （τ_a 即知，60m 提前量）；reveal_lead= min_basic 的 + 污染块里、整块报废（不允许像离线版那样把报废段当 0.8m 小块切出）。 +- 评分：scrap = 所有污染块长度 + 干净块超窗口报废 + 干净块 dict[str, Any]: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _p(inst: dict[str, Any], key: str): + proc = inst.get("process", {}) + return float(proc.get(key, DEFAULTS[key])) + + +def _limits(inst: dict[str, Any]) -> dict[str, float]: + lim = inst.get("limits", {}) + return { + "min_basic": float(lim.get("min_basic", DEFAULTS["min_basic"])), + "max_basic": float(lim.get("max_basic", DEFAULTS["max_basic"])), + "min_process": float(lim.get("min_process", DEFAULTS["min_process"])), + "max_process": float(lim.get("max_process", DEFAULTS["max_process"])), + } + + +def visible_defects(inst: dict[str, Any], cut_pos: float) -> list[dict[str, float]]: + """返回 `cut_pos` 处 agent 可见的报废段（已揭示且未完全过去）。""" + reveal = _p(inst, "reveal_lead") + out: list[dict[str, float]] = [] + for xa, xb in inst.get("defects", []) or []: + if xa <= cut_pos + reveal + OVERLAP_TOL and xb > cut_pos + OVERLAP_TOL: + out.append({"x": xa, "x_end": xb}) + return out + + +def _make_state(inst: dict[str, Any], cut_pos: float, committed: list[float]) -> dict[str, Any]: + cust = inst.get("customer", {}) + lim = _limits(inst) + return { + "cut_pos": round(cut_pos, 4), + "committed": committed, + "visible_defects": sorted(visible_defects(inst, cut_pos), key=lambda d: d["x"]), + "total_length": float(inst["cast"]["total_length"]), + "target": float(cust.get("target", 9.5)), + "target_min": float(cust.get("target_min", 9.0)), + "target_max": float(cust.get("target_max", 10.0)), + "limits": { + "min_basic": lim["min_basic"], "max_basic": lim["max_basic"], + "min_process": lim["min_process"], "max_process": lim["max_process"], + }, + "reveal_lead": _p(inst, "reveal_lead"), + } + + +def partition(inst: dict[str, Any], decision_fn: Callable[[dict[str, Any]], float]) -> list[float]: + """闭环推进：反复喂状态、取下一刀，直到切完；返回切段长度列表（之和 = S）。""" + S = float(inst["cast"]["total_length"]) + lim = _limits(inst) + committed: list[float] = [] + cut_pos = 0.0 + guard = 0 + while S - cut_pos > SUM_TOL and guard < 100000: + guard += 1 + remaining = S - cut_pos + if remaining <= lim["max_basic"] + SUM_TOL: + # 最后一段 = 剩余材料（agent 不决定，直接收尾） + piece = remaining + else: + L = decision_fn(_make_state(inst, cut_pos, list(committed))) + # 校验可接受：非法/越界则回退到目标值夹取（保证推进） + if not isinstance(L, (int, float)) or math.isnan(L) or math.isinf(L): + L = cust_target(inst) + L = float(L) + piece = max(lim["min_basic"], min(lim["max_basic"], L)) + if piece > remaining: + piece = remaining + committed.append(round(piece, 4)) + cut_pos += piece + return committed + + +def cust_target(inst: dict[str, Any]) -> float: + return float(inst.get("customer", {}).get("target", 9.5)) + + +def _piece_is_clean(piece_iv: tuple[float, float], defects: list[list[float]]) -> bool: + (x0, x1) = piece_iv + for xa, xb in defects: + if x1 > xa + OVERLAP_TOL and x0 < xb - OVERLAP_TOL: + return False # 与任一报废段重叠 -> 污染 + return True + + +def score(inst: dict[str, Any], cuts: list[float]) -> tuple[bool, dict[str, Any]]: + """给定整根切段，校验 + 评分。返回 (valid, metrics)。""" + S = float(inst["cast"]["total_length"]) + lim = _limits(inst) + if abs(sum(cuts) - S) > SUM_TOL: + return False, {"valid": False, "reason": f"sum {sum(cuts)} != S {S}"} + # 每块不得超过 [max_basic]；< min_basic 视为报废（尾段/短块物理上无法运走，作废） + for c in cuts: + if c > lim["max_basic"] + SUM_TOL: + return False, {"valid": False, "reason": f"cut {c} exceeds max_basic"} + + defects = [list((a, b)) for a, b in inst.get("defects", []) or []] + boundaries = [0.0] + for c in cuts: + boundaries.append(boundaries[-1] + c) + total_scrap = 0.0 + total_penalty = 0.0 + for i, c in enumerate(cuts): + if c < lim["min_basic"] - SUM_TOL: + total_scrap += c # 短块/尾段：物理无法运走，整块报废 + continue + iv = (boundaries[i], boundaries[i + 1]) + if not _piece_is_clean(iv, defects): + total_scrap += c # 污染块整块报废 + continue + # 干净块 + if c < lim["min_process"]: + total_scrap += c # <8.0 -> 整块报废 + else: + total_scrap += max(0.0, c - float(inst["customer"]["target_max"])) + total_penalty += abs(min(c, float(inst["customer"]["target_max"])) + - float(inst["customer"]["target"])) + util = max(0.0, 100.0 * (S - total_scrap) / S) + return True, {"valid": True, "scrap": round(total_scrap, 4), + "penalty": round(total_penalty, 4), "util": round(util, 2), + "cuts": cuts} diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_evaluator.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_evaluator.py new file mode 100644 index 00000000..d45afa3e --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_evaluator.py @@ -0,0 +1,63 @@ +"""online evaluate.py 单测：闭环驱动、作弊拒绝、运行时生成。""" + +from __future__ import annotations + +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import evaluate # noqa: E402 + +BASE = Path(__file__).resolve().parent.parent / "baseline" / "solver.py" +DATA = Path(__file__).resolve().parent / "data" / "instances" + +CHEAT = '''#!/usr/bin/env python3 +from __future__ import annotations +import json, sys + +def decide(state): + # EVOLVE-BLOCK-START + from ref_solver import solve, best_util + return float(state["target"]) + # EVOLVE-BLOCK-END + +def main(): + state = json.load(open(sys.argv[1])) + print(json.dumps({"piece_length": decide(state)})) + +if __name__ == "__main__": + main() +''' + + +class TestEvaluator(unittest.TestCase): + def test_baseline_valid(self): + res = evaluate.evaluate(str(BASE), time_budget=30, data_dir=str(DATA)) + self.assertEqual(res["valid"], 1.0) + self.assertGreater(res["combined_score"], 0.0) + + def test_cheat_rejected(self): + with tempfile.TemporaryDirectory() as td: + c = Path(td) / "cheat.py" + c.write_text(CHEAT, encoding="utf-8") + res = evaluate.evaluate(str(c), time_budget=30, data_dir=str(DATA)) + self.assertEqual(res["valid"], 0.0) + + def test_runtime_generation(self): + import os + os.environ["ONLINE_CUT_EVAL_GENERATE_SEED"] = "7" + os.environ["ONLINE_CUT_EVAL_GENERATE_COUNT"] = "3" + try: + res = evaluate.evaluate(str(BASE), time_budget=30, data_dir=str(DATA)) + finally: + os.environ.pop("ONLINE_CUT_EVAL_GENERATE_SEED", None) + os.environ.pop("ONLINE_CUT_EVAL_GENERATE_COUNT", None) + gen = [k for k in res["per_instance"] if k.startswith("gen_")] + self.assertEqual(len(gen), 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_frontier_eval_evaluator.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_frontier_eval_evaluator.py new file mode 100644 index 00000000..6a0e86a5 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_frontier_eval_evaluator.py @@ -0,0 +1,65 @@ +"""online frontier_eval/evaluator.py（沙箱入口）测试：与 verification 版一致、拒作弊。""" + +from __future__ import annotations + +import importlib.util +import sys +import tempfile +import unittest +from pathlib import Path + +TASK = Path(__file__).resolve().parent.parent + + +def _load_evaluator(): + p = TASK / "frontier_eval" / "evaluator.py" + spec = importlib.util.spec_from_file_location("_oc_sandbox_evaluator", p) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + return m + + +def _cheat(d: Path) -> Path: + p = d / "cheat.py" + p.write_text( + '#!/usr/bin/env python3\nfrom __future__ import annotations\nimport json, sys\n' + 'def decide(state):\n # EVOLVE-BLOCK-START\n from ref_solver import solve\n' + ' return float(state["target"])\n # EVOLVE-BLOCK-END\n' + 'def main():\n state=json.load(open(sys.argv[1]))\n' + ' print(json.dumps({"piece_length": decide(state)}))\n' + 'if __name__=="__main__": main()\n', + encoding="utf-8", + ) + return p + + +class TestSandboxEvaluator(unittest.TestCase): + def setUp(self): + sys.path.insert(0, str(TASK / "verification")) + self.ev = _load_evaluator() + self.data = str(TASK / "verification" / "data" / "instances") + + def test_baseline_valid(self): + import os + os.environ.pop("ONLINE_CUT_EVAL_GENERATE_SEED", None) + r = self.ev.evaluate(str(TASK / "baseline" / "solver.py"), data_dir=self.data) + r = r if isinstance(r, dict) and "combined_score" in r else r.get("metrics", r) + self.assertEqual(r.get("valid"), 1.0) + self.assertGreater(r.get("combined_score", 0.0), 0.0) + + def test_cheat_rejected(self): + with tempfile.TemporaryDirectory() as td: + r = self.ev.evaluate(str(_cheat(Path(td))), data_dir=self.data) + r = r if isinstance(r, dict) and "combined_score" in r else r.get("metrics", r) + self.assertEqual(r.get("valid"), 0.0) + + def test_consistent_with_verification(self): + from evaluate import evaluate as ver + r = self.ev.evaluate(str(TASK / "baseline" / "solver.py"), data_dir=self.data) + r = r if isinstance(r, dict) and "combined_score" in r else r.get("metrics", r) + v = ver(str(TASK / "baseline" / "solver.py"), data_dir=self.data) + self.assertAlmostEqual(r["combined_score"], v["combined_score"], places=2) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_generator.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_generator.py new file mode 100644 index 00000000..65bc4736 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_generator.py @@ -0,0 +1,41 @@ +"""online generator.py 单测：确定性、隐藏异常、可驱动性。""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import generator # noqa: E402 + + +class TestGenerate(unittest.TestCase): + def test_deterministic(self): + self.assertEqual(generator.generate(7, "medium", 10), generator.generate(7, "medium", 10)) + + def test_reveal_lead_set(self): + for rl in (10.0, 60.0): + inst = generator.generate(1, "medium", rl) + self.assertEqual(inst["process"]["reveal_lead"], rl) + + def test_defects_within_bounds_and_sorted(self): + inst = generator.generate(6, "hard", 10) + S = float(inst["cast"]["total_length"]) + prev = 0.0 + for a, b in inst["defects"]: + self.assertGreater(a, prev) + self.assertGreaterEqual(a, inst["limits"]["min_basic"] - 1e-6) + self.assertLessEqual(b, S) + prev = b + + def test_strip_for_agent_hides_defects(self): + inst = generator.generate(3, "medium", 10) + ag = generator.strip_for_agent(inst) + self.assertNotIn("defects", ag) + self.assertNotIn("anomaly_seed", ag) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_ref_solver.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_ref_solver.py new file mode 100644 index 00000000..c54d6b4e --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_ref_solver.py @@ -0,0 +1,42 @@ +"""online ref_solver.py 单测：全知参考解合法性、确定性、优于朴素基线。""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import ref_solver # noqa: E402 +from simulator import partition, score # noqa: E402 +import generator # noqa: E402 + + +class TestRefSolver(unittest.TestCase): + def test_solution_valid_and_sums(self): + for seed in (1, 2, 3): + inst = generator.generate(seed, "medium", 10) + cuts = ref_solver.solve(inst)["cuts"] + ok, m = score(inst, cuts) + self.assertTrue(ok) + self.assertAlmostEqual(sum(cuts), float(inst["cast"]["total_length"]), places=2) + + def test_deterministic(self): + inst = generator.generate(1, "hard", 10) + self.assertEqual(ref_solver.solve(inst), ref_solver.solve(inst)) + + def test_beats_naive_baseline(self): + """全知参考解的利用率应 > 朴素"恒切目标值"基线。""" + def naive(state): + return float(state["target"]) + for seed in (1, 2): + inst = generator.generate(seed, "medium", 10) + ref_u = ref_solver.best_util(inst) + ok, m = score(inst, partition(inst, naive)) + base_u = m["util"] if ok else 0.0 + self.assertGreater(ref_u, base_u, f"seed {seed}") + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_simulator.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_simulator.py new file mode 100644 index 00000000..c56427ed --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_simulator.py @@ -0,0 +1,67 @@ +"""online simulator.py 单测：评分合法性与校验规则。""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import generator # noqa: E402 +import ref_solver # noqa: E402 +from simulator import partition, score # noqa: E402 + + +def _inst(seed: int = 1, reveal: float = 10.0, diff: str = "medium"): + return generator.generate(seed, diff, reveal) + + +def _decide(state): + return float(state["target"]) + + +class TestScore(unittest.TestCase): + def test_ref_solution_scores_valid(self): + for seed in (1, 2, 3): + inst = _inst(seed) + cuts = ref_solver.solve(inst)["cuts"] + ok, m = score(inst, cuts) + self.assertTrue(ok, f"seed {seed}") + self.assertGreaterEqual(m["scrap"], 0.0) + self.assertGreaterEqual(m["util"], 0.0) + self.assertLessEqual(m["util"], 100.0) + + def test_short_tail_scrapped_not_invalid(self): + # 手搓小实例：最后一段 <4.8 视为报废，不判非法 + inst = {"seed": 0, + "process": {"v": 1.0, "tc": 3, "tr": 1, "buffer_len": 60.0, + "scrap_len": 0.8, "reveal_lead": 10.0}, + "cast": {"total_length": 20.0, "t_cast": 140.0}, + "customer": {"target": 9.0, "target_min": 8.5, "target_max": 9.5}, + "limits": {"min_basic": 4.8, "max_basic": 12.6, + "min_process": 8.0, "max_process": 11.6}, + "defects": []} + cuts = [8.0, 8.0, 4.0] # 尾段 4.0 < 4.8 + ok, m = score(inst, cuts) + self.assertTrue(ok) + # 尾段 4.0 报废；8.0 在 [min_process, target_min) 内 -> 0 报废 + self.assertAlmostEqual(m["scrap"], 4.0, places=2) + + +class TestPartition(unittest.TestCase): + def test_partition_sums_to_total(self): + inst = _inst(2, 10) + cuts = partition(inst, _decide) + self.assertAlmostEqual(sum(cuts), float(inst["cast"]["total_length"]), places=2) + + def test_partition_intermediate_in_window(self): + inst = _inst(3, 10) + cuts = partition(inst, _decide) + lo = inst["limits"]["min_basic"] + for c in cuts[:-1]: + self.assertGreaterEqual(c, lo - 1e-3) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_validator.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_validator.py new file mode 100644 index 00000000..3c2f9240 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/test_validator.py @@ -0,0 +1,73 @@ +"""online validator.py 单测：静态检查、禁引用、绝对路径、硬编码、环境剥离。""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from validator import candidate_env, static_check_source # noqa: E402 + +BASE = Path(__file__).resolve().parent.parent / "baseline" / "solver.py" + +TPL = '''#!/usr/bin/env python3 +from __future__ import annotations +import json, sys + +def decide(state): + # EVOLVE-BLOCK-START + return float(state["target"]) + # EVOLVE-BLOCK-END + +def main(): + import json as _j + state = _j.load(open(sys.argv[1])) + print(_j.dumps({"piece_length": decide(state)})) + +if __name__ == "__main__": + main() +''' + + +class TestValidator(unittest.TestCase): + def test_baseline_passes(self): + self.assertEqual(_check(BASE, BASE), []) + + def test_ref_import_caught(self): + src = TPL.replace('return float(state["target"])', + 'from ref_solver import solve\n return 0.0') + self.assertTrue(any("ref_solver" in m for m in static_check_source(src))) + + def test_generator_import_caught(self): + src = TPL.replace('return float(state["target"])', 'import generator\n return 0.0') + self.assertTrue(any("generator" in m.lower() for m in static_check_source(src))) + + def test_absolute_path_caught(self): + src = TPL.replace('return float(state["target"])', 'open("C:\\\\Users\\\\x")') + self.assertTrue(any("absolute" in m for m in static_check_source(src))) + + def test_missing_markers_caught(self): + src = TPL.replace("# EVOLVE-BLOCK-START\n ", "").replace("# EVOLVE-BLOCK-END\n", "") + self.assertTrue(any("EVOLVE-BLOCK" in m for m in static_check_source(src))) + + def test_env_strips_frontier(self): + import os + os.environ["ONLINE_CUT_EVAL_GENERATE_SEED"] = "42" + os.environ["FRONTIER_EVAL_SOMETHING"] = "x" + try: + env = candidate_env() + self.assertNotIn("ONLINE_CUT_EVAL_GENERATE_SEED", env) + self.assertNotIn("FRONTIER_EVAL_SOMETHING", env) + finally: + os.environ.pop("ONLINE_CUT_EVAL_GENERATE_SEED", None) + os.environ.pop("FRONTIER_EVAL_SOMETHING", None) + + +def _check(p, base): + return static_check_source(p.read_text(encoding="utf-8"), base.read_text(encoding="utf-8")) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/validator.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/validator.py new file mode 100644 index 00000000..31a7d778 --- /dev/null +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/validator.py @@ -0,0 +1,93 @@ +"""在线切割候选完整性校验（静态检查 + 环境剥离，借鉴离线/CVRP 惯例）。 + +评分前以可执行检查强制约束： + 1. EVOLVE-BLOCK 完整性 + 标记外代码与初始 baseline 逐字节一致； + 2. 禁引用评测/生成/参考解模块（evaluate/generator/ref_solver）、绝对路径、按实例名硬编码； + 3. candidate_env：候选子进程剥离 FRONTIER_*/ONLINE_CUT_EVAL_* 变量，封宿主侧信道。 + 4. 确定性探针（闭环跑两遍）在 evaluate.py 中执行。 + +候选允许 import verification/simulator.py（白盒计分器），但禁止评测/生成/参考解逻辑。 +""" + +from __future__ import annotations + +import os +import re +from pathlib import Path + +EVOLVE_START = "EVOLVE-BLOCK-START" +EVOLVE_END = "EVOLVE-BLOCK-END" +PROJECT_PREFIX = "ONLINE_CUT_EVAL_" + +STRONG_TOKENS = ( + "ref_solver", + "ONLINE_CUT_EVAL_", + "from generator", + "import generator", + "anomaly_seed", +) +FORBIDDEN_RE = ( + re.compile(r"verification[\\/](?:evaluate|generator|ref_solver)"), + re.compile(r"verification\s*\.\s*(?:evaluate|generator|ref_solver)\b"), + re.compile(r"\b(?:from|import)\s+(?:evaluate|generator|ref_solver)\b"), +) +ABS_PATH_RE = re.compile(r"[A-Za-z]:[\\/]|/home/|/Users/") +HARDCODE_RE = re.compile(r"[\"'](?:instance|gen)_\d+(?:_\d+)?[\"']\s*:") + + +def split_evolve_blocks(src: str) -> tuple[str, str, str] | None: + start = src.find(EVOLVE_START) + end = src.find(EVOLVE_END) + if start == -1 or end == -1 or end <= start: + return None + return (src[:start], src[start + len(EVOLVE_START):end], src[end + len(EVOLVE_END):]) + + +def fixed_region(parts: tuple[str, str, str]) -> str: + return (parts[0] + parts[2]).replace("\r\n", "\n").rstrip("\n") + + +def static_check_source(src: str, baseline_src: str | None = None) -> list[str]: + issues: list[str] = [] + parts = split_evolve_blocks(src) + if parts is None: + issues.append("missing EVOLVE-BLOCK-START / EVOLVE-BLOCK-END markers") + elif baseline_src is not None: + init_parts = split_evolve_blocks(baseline_src) + if init_parts is not None and fixed_region(init_parts) != fixed_region(parts): + issues.append("code outside EVOLVE-BLOCK differs from initial baseline") + for token in STRONG_TOKENS: + if token in src: + issues.append(f"candidate references forbidden token {token!r}") + for pat in FORBIDDEN_RE: + if pat.search(src): + issues.append("candidate references forbidden evaluation/generation/ref module") + if ABS_PATH_RE.search(src): + issues.append("candidate contains an absolute filesystem path") + if HARDCODE_RE.search(src): + issues.append("candidate hardcodes per-instance schedules by name") + return issues + + +def check_candidate(solver_path: Path | str, baseline_path: Path | str | None = None) -> list[str]: + solver_path = Path(solver_path) + try: + src = solver_path.read_text(encoding="utf-8", errors="replace") + except Exception as exc: + return [f"cannot read candidate source: {exc}"] + baseline_src = None + if baseline_path is not None: + try: + baseline_src = Path(baseline_path).read_text(encoding="utf-8", errors="replace") + except Exception: + baseline_src = None + return static_check_source(src, baseline_src) + + +def candidate_env() -> dict[str, str]: + env = os.environ.copy() + for key in list(env): + upper = key.upper() + if upper.startswith("FRONTIER") or upper.startswith(PROJECT_PREFIX): + del env[key] + return env diff --git a/benchmarks/ContinuousCasting/README.md b/benchmarks/ContinuousCasting/README.md new file mode 100644 index 00000000..a8b4e6d3 --- /dev/null +++ b/benchmarks/ContinuousCasting/README.md @@ -0,0 +1,23 @@ +# ContinuousCasting + +This domain collects deterministic engineering-optimization tasks for continuous steel casting +(连铸) and the adjacent industrial process-control problems. Current tasks emphasize realistic +operational constraints, a well-defined material-utilization objective, and executable, +standard-library-only verification. + +## Tasks + +- `CuttingOptimization` + - Unified benchmark: `task=unified task.benchmark=ContinuousCasting/CuttingOptimization` + - Quick run: `python -m frontier_eval task=unified task.benchmark=ContinuousCasting/CuttingOptimization algorithm.iterations=0` + - Description: cut a continuously cast steel billet (with fixed 0.8 m scrap segments) into + pieces to minimize total scrapped length, then make every shipped piece as close as possible + to a customer target length — inspired by the CUMCM 2021 Problem D. The optimum is reachable + (a strong agent can derive the exact partition DP). +- `CuttingOptimizationOnline` + - Unified benchmark: `task=unified task.benchmark=ContinuousCasting/CuttingOptimizationOnline` + - Quick run: `python -m frontier_eval task=unified task.benchmark=ContinuousCasting/CuttingOptimizationOnline algorithm.iterations=0` + - Description: the **online (closed-loop)** version — the agent is told about a 0.8 m scrap + segment only when it is within `reveal_lead` m of the cut line, decides each cut with only the + visible past, and is scored against the full hidden defect set. An online agent cannot reach + the clairvoyant optimum (info asymmetry), so it is genuinely harder than the offline task. diff --git a/benchmarks/ContinuousCasting/README_zh-CN.md b/benchmarks/ContinuousCasting/README_zh-CN.md new file mode 100644 index 00000000..54585901 --- /dev/null +++ b/benchmarks/ContinuousCasting/README_zh-CN.md @@ -0,0 +1,19 @@ +# ContinuousCasting + +本 domain 汇集"连铸（连续浇铸钢材）及其相邻工业过程控制"的**确定性工程优化**任务。 +当前任务强调真实工况约束、明确的材料利用率目标，以及可执行、纯标准库的验证。 + +## 任务 + +- `CuttingOptimization` + - Unified 基准：`task=unified task.benchmark=ContinuousCasting/CuttingOptimization` + - 快速运行：`python -m frontier_eval task=unified task.benchmark=ContinuousCasting/CuttingOptimization algorithm.iterations=0` + - 描述：把一根连续浇铸的钢坯（带固定的 0.8 m 零废段）切成成品段，先最小化报废总长度， + 再让每块成品尽量贴近客户目标值——灵感来自 2021 全国大学生数学建模竞赛 D 题。最优可达 + （强 agent 能推导出精确划分 DP）。 +- `CuttingOptimizationOnline` + - Unified 基准：`task=unified task.benchmark=ContinuousCasting/CuttingOptimizationOnline` + - 快速运行：`python -m frontier_eval task=unified task.benchmark=ContinuousCasting/CuttingOptimizationOnline algorithm.iterations=0` + - 描述：**在线（闭环）版**——agent 只在报废段距切割线 `reveal_lead` 米以内时才得知其存在， + 每个决策点只拿可见过去，最后按完整隐藏报废表评分。在线 agent 无法达到全知最优（信息不对称）， + 因此比离线版更难。 From 6c00d95afd841579d528923fa842d86e0fbb40df Mon Sep 17 00:00:00 2001 From: zzy <17092805+mz2007@user.noreply.gitee.com> Date: Sat, 5 Sep 2026 17:24:07 +0800 Subject: [PATCH 2/6] fix: resolve doc/vs-impl inconsistencies + online anti-cheat hole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address reviewer feedback on the new ContinuousCasting benchmarks: - online (threat model, high): fixed instances embedding the hidden defect schedule are no longer copied into the sandbox (copy_files.txt drops verification/data/instances); the evaluator loads them from the host source benchmark dir, so a candidate cannot read the hidden defects out of a data file. - online: unify naming/default to reveal_lead = 10.0 (generator default 60.0 -> 10.0; docs reveal_dist/8.0 -> reveal_lead/10.0). - online: scoring now applies the 1e-4 target-fit penalty in the utilization (matches the offline formula; the docs already claimed it), so equal-scrap plans are tie-broken. - online: Task.md interface section now matches the per-decision-call implementation (was stale REPL text with wrong state keys). - offline: titles disambiguate as offline/static ('连铸切割优化(离线/静态版)') so they are not confused with the online task. - online: remove a duplicated/stale README note; soften the 'cannot reach' claim to an empirical observation + note the optional online-oracle reference layer. --- .../CuttingOptimization/README.md | 2 +- .../CuttingOptimization/README_zh-CN.md | 2 +- .../CuttingOptimization/Task.md | 2 +- .../CuttingOptimizationOnline/README.md | 15 +++++++++-- .../CuttingOptimizationOnline/README_zh-CN.md | 7 ++--- .../CuttingOptimizationOnline/Task.md | 27 ++++++++++--------- .../frontier_eval/copy_files.txt | 1 - .../verification/evaluate.py | 19 +++++++++++-- .../verification/generator.py | 2 +- .../verification/simulator.py | 4 ++- 10 files changed, 55 insertions(+), 26 deletions(-) diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/README.md b/benchmarks/ContinuousCasting/CuttingOptimization/README.md index 84d631f1..a540988f 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimization/README.md +++ b/benchmarks/ContinuousCasting/CuttingOptimization/README.md @@ -1,4 +1,4 @@ -# CuttingOptimization: Online Optimization of Continuous-Casting Cutting (Frontier-Eng Benchmark) +# CuttingOptimization: Continuous-Casting Cutting Optimization — offline/static (Frontier-Eng Benchmark) An **original** Frontier-Engineering benchmark inspired by the CUMCM 2021 Problem D («连铸切割的在线优化»), formalized into a self-contained, deterministic optimization task. diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/README_zh-CN.md b/benchmarks/ContinuousCasting/CuttingOptimization/README_zh-CN.md index 1ae3af2b..86f4b89b 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimization/README_zh-CN.md +++ b/benchmarks/ContinuousCasting/CuttingOptimization/README_zh-CN.md @@ -1,4 +1,4 @@ -# CuttingOptimization：连铸切割的在线优化（Frontier-Eng 基准） +# CuttingOptimization：连铸切割优化（离线/静态版，Frontier-Eng 基准） 一个**原创**的 Frontier-Engineering 基准，灵感来自 2021 全国大学生数学建模竞赛 D 题 《连铸切割的在线优化》，并被形式化成**确定性、自包含**的切割优化任务。 diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/Task.md b/benchmarks/ContinuousCasting/CuttingOptimization/Task.md index 7192e868..bb0670ba 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimization/Task.md +++ b/benchmarks/ContinuousCasting/CuttingOptimization/Task.md @@ -1,4 +1,4 @@ -# 连铸切割的在线优化（CuttingOptimization） +# 连铸切割优化（离线/静态版，CuttingOptimization） ## 1. 背景 diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md index 6fb4d612..060e2d0a 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md @@ -6,8 +6,8 @@ An **original** Frontier-Engineering benchmark extending the offline A continuously cast steel billet is drawn past a cutter. **Crystallizer anomalies create 0.8 m scrap segments, but the agent is only told about a segment when it is within -`reveal_lead` metres of the cut line** (default 8.0 m → hidden anomalies; `reveal_lead = 60` -is the "faithful" setting where the agent sees all relevant defects). The agent is invoked +`reveal_lead` metres of the cut line** (final config `reveal_lead = 10` → hidden anomalies; +`reveal_lead = 60` is the "faithful" setting where the agent sees all relevant defects). The agent is invoked *once per cut decision* with the **current visible state only** — it never sees future defects — and must choose the next cut length. At the end the plan is scored against the **full hidden defect set**: any piece overlapping a scrap segment is contaminated and fully @@ -88,6 +88,12 @@ candidate rejected, runtime generation), and sandbox-evaluator consistency. ## Integrity / threat model +- **The instance data (which embeds the full hidden defect schedule) is NOT copied into the + sandbox.** `frontier_eval/copy_files.txt` excludes `verification/data/instances`; the + evaluator loads the fixed instances from the host source benchmark dir + (`FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR`), so a candidate can never read the hidden + defects out of a data file (it only ever gets the visible `state`). This is the core + anti-cheat for the online task. - `verification/ref_solver.py` and `verification/generator.py` are **not** copied into the sandbox and are additionally forbidden by the validator (`ref_solver`, `generator`, `anomaly_seed` tokens). @@ -133,6 +139,11 @@ openevolve reaches the (offline) optimal exactly. The per-framework spread is sm (std ≈ 0.4–1.2), so agent scores settle around the "safe short-cut" plateau (~70) that limits how far an online agent can get without full foresight. +> Methodological note: "online agents do not reach the clairvoyant optimum" is an **empirical** +> observation (3 frameworks × 3 runs each are all below 76.4), not a proven structural lower bound. +> To establish it rigorously one would add an "online oracle" reference layer — the best strategy +> that uses only the revealed information (respecting `reveal_lead`) — and show agents fall below it. + > Honest note: agents improve in step-jumps (stuck at baseline for several generations, then a > single mutation cracks ~70), not gradual climbing — consistent with a hard constraint where > "get the strategy right once" beats incremental search. AB-MCTS's single high run (72.31) vs its diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README_zh-CN.md b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README_zh-CN.md index 9edbdfd2..9cd37e81 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README_zh-CN.md +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README_zh-CN.md @@ -3,7 +3,7 @@ 一个**原创**的 Frontier-Engineering 基准，是离线版 [CuttingOptimization](../CuttingOptimization/README_zh-CN.md) 的**在线（闭环）**扩展，题材取自 2021 全国大学生数学建模竞赛 D 题。 -一根连续浇铸的钢坯被拉过切割机。**结晶器异常会产生 0.8m 报废段，但 agent 只在报废段距切割线 `reveal_lead` 米以内时才被告知**（默认 8m → 隐藏异常；`reveal_lead=60` 为"忠实版"，此时 agent 能看到所有相关异常）。agent **每个切割决策点被调用一次、只拿到当下可见状态**，永远看不到未来，并要选下一刀长度。最后按**完整隐藏报废表**评分：任何与报废段重叠的切块都被污染、整块报废。 +一根连续浇铸的钢坯被拉过切割机。**结晶器异常会产生 0.8m 报废段，但 agent 只在报废段距切割线 `reveal_lead` 米以内时才被告知**（本基准最终配置 `reveal_lead=10` → 隐藏异常；`reveal_lead=60` 为"忠实版"，此时 agent 能看到所有相关异常）。agent **每个切割决策点被调用一次、只拿到当下可见状态**，永远看不到未来，并要选下一刀长度。最后按**完整隐藏报废表**评分：任何与报废段重叠的切块都被污染、整块报废。 完整规则与评测语义见 [Task.md](./Task.md)。 @@ -106,8 +106,9 @@ python -m unittest discover -s verification -p "test_*.py" > "强约束下把策略一次想对"胜过增量搜索。AB-MCTS 那次高分（72.31）与其 std（1.19）反映的是 > 运行间随机性，不是系统性优势。 -> 诚实说明：agent 改进呈**阶梯式跳变**（卡 baseline 多代、某次突变到 ~68-72），不是渐进爬升 -> ——符合"强约束下把策略一次想对"胜过增量搜索。 +> 方法学说明：以上"在线 agent 无法达到全知最优"是**经验观察**（3 框架 × 3 次均在 ref 之下）， +> 尚无"在线策略最优下界"的严格证明；若需坐实，可另加一个"只限可见信息、按揭示窗口保守决策"的 +> 在线 oracle 层作为参照。 ## Docker diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/Task.md b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/Task.md index 74977f31..0f72047c 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/Task.md +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/Task.md @@ -10,10 +10,10 @@ - 拉坯速度 `v = 1.0` m/min；切一块 `tc = 3` min + 回程 `tr = 1` min；结晶器→切割机 `D = 60` m。 - 报废段长 `scrap_len = 0.8` m。 - 切长窗口：能运 `[min_basic, max_basic] = [4.8, 12.6]`；下道可接 `[min_process, max_process] = [8.0, 11.6]`；用户 `target` + 窗口 `[target_min, target_max]`。 -- **揭示距离 `reveal_dist = R`（默认 8.0 m）**：一段报废料仅在距离当前切割启动点 ≤ `R` 米时，才"揭示"给 agent。 +- **揭示距离 `reveal_lead = R`（最终配置默认 `10.0` m）**：一段报废料仅在距离当前切割启动点 ≤ `R` 米时，才"揭示"给 agent。（单位即"米"；v=1.0 时它也等于"提前的分钟数"，文档简称"米"。） - 因 `v×(tc+tr)=4m < 4.8m`，切割机总能跟上 → **无吞吐瓶颈**。时间只在"揭示与到位"时差上起作用，不构成物理约束。 -> 为什么 `R < max_basic`：若把折现距离设成原题的 60m 缓冲（远大于 12.6m 切段），agent 在承诺每块之前总能看见该块内所有异常，从而永远主动避开 → 退化为离线。取 `R` 小于最大切段，使"远端未知带"存在，才产生真实不确定性。这是本任务与离线版难度分界的核心旋钮。 +> 为什么 `R < max_basic`：若把揭示距离设成原题的 60m 缓冲（远大于 12.6m 切段），agent 在承诺每块之前总能看见该块内所有异常，从而永远主动避开 → 退化为离线。取 `R` 小于最大切段，使"远端未知带"存在，才产生真实不确定性。这是本任务与离线版难度分界的核心旋钮。 ## 2. 材料坐标与时序 @@ -21,24 +21,25 @@ 异常由隐藏种子确定：一段报废料占据流区间 `[x_a, x_a+0.8]`。它在**距切割点 ≤ R 时揭示**（即当 `x_a ≤ cut_pos + R`），并将其位置/到达时间加入 agent 可见列表。 -## 3. 求解接口（闭环 REPL） +## 3. 求解接口（每决策点一次调用） -agent 程序**启动一次**：`python solver.py `（instance 里**不含任何异常/异常种子**），进入循环： +agent 程序被评测器**每个切割决策点调用一次**：`python solver.py ` → stdout 打印决策。 -- 从 stdin 读一行**状态 JSON**： +- 从 `state.json`（不含任何异常/异常种子，且**不含完整报废表**）读**当前决策点状态**，字段如下： ```json -{"cut_pos": 12.6, "committed": [9.1, 11.4], "visible_defects": [{"x": 20.0}], "target": 9.5, - "target_min": 9.0, "target_max": 10.0, "limits": {"min_basic":4.8,"max_basic":12.6,"min_process":8.0,"max_process":11.6}, - "total_length": 106.2} +{"cut_pos": 12.6, "committed": [9.1, 11.4], "visible_defects": [{"x": 20.0, "x_end": 20.8}], + "target": 9.5, "target_min": 9.0, "target_max": 10.0, + "limits": {"min_basic":4.8,"max_basic":12.6,"min_process":8.0,"max_process":11.6}, + "total_length": 106.2, "reveal_lead": 10.0} ``` - `cut_pos`：当前切割启动点（下一段的起点）。 - `committed`：已承诺的切段长度列表（之和 = cut_pos）。 - - `visible_defects`：**已揭示**（距 cut_pos ≤ R）且尚未被切过/尚未过去的报废段流位置（只含 `x_a`，长度固定 0.8m）。 - - `reveal_dist`、`total_length`、process 等。 -- 向 stdout 写一行**决策 JSON**：`{"piece_length": L}`，`L ∈ [4.8, 12.6]`。 -- 评审器按流推进：`cut_pos += L`，重复喂状态/拿下一刀；当 `S - cut_pos ≤ 12.6` 时强制收尾（最后一段 = `S - cut_pos`，agent 无需作答）。 + - `visible_defects`：**已揭示**（距 cut_pos ≤ `${reveal_lead}`）且尚未被切过/尚未过去的报废段流位置（含 `x` 起点与 `x_end`，长度固定 0.8m）。 + - `reveal_lead`：本次实例的揭示距离（米）。 +- 向 stdout 打印一行**决策 JSON**：`{"piece_length": L}`，`L ∈ [4.8, 12.6]`。 +- 评测器按流推进：`cut_pos += L`，重复"喂状态 → 拿下一刀"；当 `S - cut_pos ≤ 12.6` 时强制收尾（最后一段 = `S - cut_pos`，agent 无需作答）。 -agent **永远看不到** `x_a > cut_pos + R` 的异常（未揭示）；也看不到完整报废表。`visible_defects` 只含已揭示的。 +agent **永远看不到** `x_a > cut_pos + reveal_lead` 的异常（未揭示）；也看不到完整报废表。`visible_defects` 只含已揭示的。 ## 4. 污染判定（不可规避的代价，难度来源） diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/copy_files.txt b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/copy_files.txt index 79f8b1a5..10228058 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/copy_files.txt +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/frontier_eval/copy_files.txt @@ -2,5 +2,4 @@ baseline verification/evaluate.py verification/simulator.py verification/validator.py -verification/data/instances frontier_eval diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/evaluate.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/evaluate.py index f8c1ec26..4044788e 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/evaluate.py +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/evaluate.py @@ -132,6 +132,21 @@ def _run_one(prog: Path, inst_path: Path, time_budget: float, py: str, tmp: Path "n_cuts": len(cuts)} +def _instances_dir() -> Path: + """在线版实例目录：只从宿主源 benchmark 目录加载（不复制进沙箱）。 + + 在线版的关键是"隐藏报废表"——实例文件含完整 defects/anomaly_seed，绝不能进候选可见 + 目录（否则候选可读 JSON 拿到全知解作弊）。因此在沙箱内（本地无 data/instances 时）改从 + FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR 加载；直跑（非沙箱）时回退到本地目录。 + """ + src = os.environ.get("FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR", "").strip() + if src: + host = Path(src) / "verification" / "data" / "instances" + if host.is_dir(): + return host + return DATA_DIR + + def evaluate(program_path: str, *, time_budget: float = 60.0, python: str | None = None, data_dir: str | Path | None = None, reveal_lead: float | None = None) -> dict[str, Any]: prog = Path(program_path).resolve() @@ -140,10 +155,10 @@ def evaluate(program_path: str, *, time_budget: float = 60.0, python: str | None violations = check_candidate(prog) - inst_dir = Path(data_dir).resolve() if data_dir else DATA_DIR + inst_dir = Path(data_dir).resolve() if data_dir else _instances_dir() instances = sorted(inst_dir.glob("instance_*.json")) if inst_dir.is_dir() else [] if not instances: - # 若无固定实例集，现场生成一批（便于直接评测） + # 若无固定实例集，现场生成一批（便于直接评测；仅宿主，不进沙箱） tmp = Path(tempfile.mkdtemp(prefix="oc_inst_")) instances = _generate_instances(7, 6, tmp, reveal_lead or 10.0) diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/generator.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/generator.py index 72e484fd..771dc7df 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/generator.py +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/generator.py @@ -24,7 +24,7 @@ LIMITS = {"min_basic": 4.8, "max_basic": 12.6, "min_process": 8.0, "max_process": 11.6} -def generate(seed: int, difficulty: str = "medium", reveal_lead: float = 60.0, +def generate(seed: int, difficulty: str = "medium", reveal_lead: float = 10.0, n_anomaly: int | None = None) -> dict[str, Any]: rng = random.Random(seed * 10007 + 11) if difficulty == "easy": diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/simulator.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/simulator.py index b204a48a..6ef23790 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/simulator.py +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/simulator.py @@ -148,7 +148,9 @@ def score(inst: dict[str, Any], cuts: list[float]) -> tuple[bool, dict[str, Any] total_scrap += max(0.0, c - float(inst["customer"]["target_max"])) total_penalty += abs(min(c, float(inst["customer"]["target_max"])) - float(inst["customer"]["target"])) - util = max(0.0, 100.0 * (S - total_scrap) / S) + # util 含极小贴合度惩罚（破平），与离线版口径一致：scrap 先、penalty 破平 + lam = float(inst.get("limits", {}).get("target_penalty_weight", 1e-4)) + util = max(0.0, 100.0 * (S - (total_scrap + lam * total_penalty)) / S) return True, {"valid": True, "scrap": round(total_scrap, 4), "penalty": round(total_penalty, 4), "util": round(util, 2), "cuts": cuts} From 3394acc0fa10cf41308369f38642e34f4bf83cbc Mon Sep 17 00:00:00 2001 From: zzy <17092805+mz2007@user.noreply.gitee.com> Date: Sat, 5 Sep 2026 17:37:28 +0800 Subject: [PATCH 3/6] fix: align online tooling/defaults with final reveal_lead=10 config Address second reviewer pass on CuttingOptimizationOnline: - multiseed_stat.py: DEFAULT_REFERENCE 88.72 (offline) -> 76.37 (online clairvoyant), and the docstring/usage example now point at the online run dir; README notes the tool aggregates every run in the dir (restrict via sub-glob for config-specific stats). - simulator.py: DEFAULTS reveal_lead 60.0 -> 10.0 (final config; no silent 'faithful' default), docstring updated for the penalty-inclusive util formula. - Tie-break is no longer rounded away: simulator keeps util at 4 decimals and evaluate rounds combined_score to 3 decimals, so the 1e-4 target-fit penalty (order ~1e-3 util) actually discriminates equal-scrap plans (matches the offline formula). --- .../CuttingOptimizationOnline/README.md | 2 ++ .../verification/evaluate.py | 3 ++- .../verification/multiseed_stat.py | 6 +++--- .../verification/simulator.py | 17 ++++++++++------- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md index 060e2d0a..2a0f32b9 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md @@ -58,6 +58,8 @@ $env:ONLINE_CUT_EVAL_GENERATE_SEED = "" python verification/evaluate.py baseline/solver.py # Multi-run stats (mean ± std) across a framework run dir +# Multi-run stats — note the tool aggregates every run in the dir; point it at the full run dir +# (it defaults to the online clairvoyant reference 76.37), or pass a sub-glob to restrict. python verification/multiseed_stat.py --runs-dir runs/unified__ContinuousCasting__CuttingOptimizationOnline/openevolve ``` diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/evaluate.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/evaluate.py index 4044788e..f5d4aa5b 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/evaluate.py +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/evaluate.py @@ -215,7 +215,8 @@ def evaluate(program_path: str, *, time_budget: float = 60.0, python: str | None shutil.rmtree(tmp, ignore_errors=True) combined = total / len(instances) if instances else 0.0 - return {"combined_score": round(combined, 2), "valid": 1.0 if all_valid and not violations else 0.0, + # 保留 3 位：贴合度惩罚量级 ~1e-3，2 位舍入会把破平项抹掉，故至少 3 位 + return {"combined_score": round(combined, 3), "valid": 1.0 if all_valid and not violations else 0.0, "per_instance": per_instance, "num_instances": len(instances), "time_budget_s": time_budget, "generate_seed": gen_seed_raw or None} diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/multiseed_stat.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/multiseed_stat.py index e6df657a..0e8cb18b 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/multiseed_stat.py +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/multiseed_stat.py @@ -1,9 +1,9 @@ """多运行（多种子/多轮）agent 分数统计：对一组框架运行目录取 combined_score 的 mean±std。 用法： - python verification/multiseed_stat.py --runs-dir runs/unified__ContinuousCasting__CuttingOptimization/openevolve + python verification/multiseed_stat.py --runs-dir runs/unified__ContinuousCasting__CuttingOptimizationOnline/openevolve python verification/multiseed_stat.py --runs-dir "runs/**/openevolve/deepseek-v4-flash" # glob - python verification/multiseed_stat.py --runs-dir --pattern "*openevolve*" + python verification/multiseed_stat.py --runs-dir --pattern "*openevolve*" --reference 76.37 从每个运行目录下读 `/best/best_program_info.json` 的 `metrics.combined_score` （框架统一保存的 best 程序分数），对多次运行做 mean / std / min / max， @@ -22,7 +22,7 @@ import statistics from pathlib import Path -DEFAULT_REFERENCE = 88.72 # 全知参考解利用率（verification/ref_solver.py），可 --reference 覆盖 +DEFAULT_REFERENCE = 76.37 # 在线版全知参考解利用率（verification/ref_solver.py），可 --reference 覆盖 def _best_score(run_dir: Path) -> float | None: diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/simulator.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/simulator.py index 6ef23790..96c660eb 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/simulator.py +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/simulator.py @@ -3,14 +3,16 @@ 模型（详见 Task.md）： - 材料用流坐标 x∈[0,S]，S = v*浇铸时长。横截面 x 到达切割点在 x/v + D/v。 - 隐藏异常（由 anomaly_seed 决定）：报废段占流区间 [x_a, x_a+scrap_len]。 -- 揭示提前量 reveal_lead（分钟；v=1 时等于"上游米数"）：一段报废料在 - x_a <= cut_pos + reveal_lead 时"揭示"给 agent；否则不可见。reveal_lead=60 为忠实版 - （τ_a 即知，60m 提前量）；reveal_lead= min_basic 的 污染块里、整块报废（不允许像离线版那样把报废段当 0.8m 小块切出）。 - 评分：scrap = 所有污染块长度 + 干净块超窗口报废 + 干净块 tuple[bool, dict[str, Any] total_scrap += max(0.0, c - float(inst["customer"]["target_max"])) total_penalty += abs(min(c, float(inst["customer"]["target_max"])) - float(inst["customer"]["target"])) - # util 含极小贴合度惩罚（破平），与离线版口径一致：scrap 先、penalty 破平 + # util 含极小贴合度惩罚（破平），与离线版口径一致：scrap 先、penalty 破平。 + # 保留到 4 位（不低于 penalty 量级），否则惩罚项会被 2 位舍入吞掉、破平失效。 lam = float(inst.get("limits", {}).get("target_penalty_weight", 1e-4)) util = max(0.0, 100.0 * (S - (total_scrap + lam * total_penalty)) / S) return True, {"valid": True, "scrap": round(total_scrap, 4), - "penalty": round(total_penalty, 4), "util": round(util, 2), + "penalty": round(total_penalty, 4), "util": round(util, 4), "cuts": cuts} From 24bc779ec9c26cb5bbf1066813df456827757ca3 Mon Sep 17 00:00:00 2001 From: zzy <17092805+mz2007@user.noreply.gitee.com> Date: Sat, 5 Sep 2026 17:48:26 +0800 Subject: [PATCH 4/6] docs: fix --reference help default text (88.72 -> 76.37) in online multiseed_stat --- .../CuttingOptimizationOnline/verification/multiseed_stat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/multiseed_stat.py b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/multiseed_stat.py index 0e8cb18b..416192c1 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/multiseed_stat.py +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/multiseed_stat.py @@ -64,7 +64,7 @@ def main() -> int: parser.add_argument("--runs-dir", required=True, help="运行目录或 glob 模式") parser.add_argument("--pattern", default=None, help="追加的子 glob 模式") parser.add_argument("--reference", type=float, default=DEFAULT_REFERENCE, - help="参考解利用率（默认 88.72）") + help="参考解利用率（默认 76.37）") args = parser.parse_args() pairs = collect(args.runs_dir, args.pattern) From 1881fd30416892ba4882679c22b8ce83a5af6bc8 Mon Sep 17 00:00:00 2001 From: zzy <17092805+mz2007@user.noreply.gitee.com> Date: Sat, 5 Sep 2026 18:06:02 +0800 Subject: [PATCH 5/6] docs: soften 'cannot reach clairvoyant' to empirical wording in TASK_DETAILS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align TASK_DETAILS rows with the Task/README framing: the online info-asymmetry claim is an observed effect (3 frameworks x 3 runs each below the clairvoyant ceiling), not a proven lower bound. EN/ZH rows now use 'empirically keeps below' / '(经验上)低于' wording. --- TASK_DETAILS.md | 2 +- TASK_DETAILS_zh-CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/TASK_DETAILS.md b/TASK_DETAILS.md index c2d2c98b..86ae747d 100644 --- a/TASK_DETAILS.md +++ b/TASK_DETAILS.md @@ -353,7 +353,7 @@ We welcome new engineering problem ideas — even without complete verification CuttingOptimizationOnline - Online (closed-loop) cutting where 0.8 m scrap segments are revealed only within a reveal horizon — info asymmetry keeps agents below the clairvoyant optimum + Online (closed-loop) cutting where 0.8 m scrap segments are revealed only within a reveal horizon — info asymmetry empirically keeps agents below the clairvoyant optimum (observed, not a proven lower bound) AdditiveManufacturing diff --git a/TASK_DETAILS_zh-CN.md b/TASK_DETAILS_zh-CN.md index 5e9f0559..e1a0dee7 100644 --- a/TASK_DETAILS_zh-CN.md +++ b/TASK_DETAILS_zh-CN.md @@ -353,7 +353,7 @@ Frontier-Eng 目前已覆盖以下领域的任务。每个任务均配有可运 CuttingOptimizationOnline - 在线（闭环）切割：0.8m 报废段只在揭示提前量内才告知 agent——信息不对称让 agent 达不到全知最优 + 在线（闭环）切割：0.8m 报废段只在揭示提前量内才告知 agent——信息不对称使 agent（经验上）低于全知最优（观察结果，非严格下界证明） AdditiveManufacturing From ab1bfc2d14a0ab0edfaadba7372099130e69c592 Mon Sep 17 00:00:00 2001 From: zzy <17092805+mz2007@user.noreply.gitee.com> Date: Sat, 5 Sep 2026 18:17:30 +0800 Subject: [PATCH 6/6] docs: record WSL docker-isolation verification (both tasks) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ran unified runtime in task.runtime.isolation_mode=docker under WSL for both benchmarks (FRONTIER_EVAL_UNIFIED_DOCKER_USER=1000:1000, no shell override, docker_image built from the task Dockerfile). Offline baseline scored 72.49 / valid=1.0 and online baseline 52.40 / valid=1.0, identical to process mode — so docker isolation scoring is verified (not just documented), and for the online task this also confirms the anti-cheat holds in-container (fixed instances with the hidden defect schedule stay out of the sandbox; the evaluator reads them from the container's mounted source repo). --- .../CuttingOptimization/README.md | 17 +++++++++++-- .../CuttingOptimization/README_zh-CN.md | 6 +++++ .../CuttingOptimizationOnline/README.md | 24 +++++++++++++++---- .../CuttingOptimizationOnline/README_zh-CN.md | 8 ++++--- 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/README.md b/benchmarks/ContinuousCasting/CuttingOptimization/README.md index a540988f..ccf5ae3f 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimization/README.md +++ b/benchmarks/ContinuousCasting/CuttingOptimization/README.md @@ -61,10 +61,23 @@ The evaluator is pure stdlib, so a minimal `python` image suffices. Build it and unified runtime's `isolation_mode=docker`: ```bash -# Build (inside the CuttingOptimization directory) -docker build -t cutting-opt-benchmark -f verification/docker/Dockerfile . +# Build (from the repo root) +docker build -t cutting-opt-benchmark -f benchmarks/ContinuousCasting/CuttingOptimization/verification/docker/Dockerfile benchmarks/ContinuousCasting/CuttingOptimization + +# Score the baseline under docker isolation (WSL/Linux; Windows hosts are limited by a +# framework path bug). docker mode must NOT set task.runtime.shell; set DOCKER_USER so the +# container can write the WSL /tmp sandbox. +FRONTIER_EVAL_UNIFIED_DOCKER_USER=1000:1000 \ + .venvs/frontier-eval-driver-wsl/bin/python -m frontier_eval task=unified \ + task.benchmark=ContinuousCasting/CuttingOptimization algorithm=openevolve algorithm.iterations=0 \ + llm.timeout=600 task.runtime.isolation_mode=docker task.runtime.docker_image=cutting-opt-benchmark ``` +**Verified (WSL, docker isolation)**: baseline `combined_score=72.49, valid=1.0, num_instances=8` +— identical to process mode. `docker` scoring works under WSL/Linux because `eval_command.txt` +injects `FRONTIER_EVAL_UNIFIED_SOURCE_BENCHMARK_DIR={benchmark_source}`, which the container +resolves to the mounted repo path (no framework env-forwarding needed). + ## Tests ```powershell diff --git a/benchmarks/ContinuousCasting/CuttingOptimization/README_zh-CN.md b/benchmarks/ContinuousCasting/CuttingOptimization/README_zh-CN.md index 86f4b89b..c4d27d9c 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimization/README_zh-CN.md +++ b/benchmarks/ContinuousCasting/CuttingOptimization/README_zh-CN.md @@ -47,6 +47,12 @@ python verification/evaluate.py baseline/solver.py --generate-seed python verification/evaluate.py baseline/solver.py --time-budget 10 ``` +## Docker + +提供最简 `python:3.11-slim` 镜像（`verification/docker/Dockerfile`）。构建后用 unified 的 +`isolation_mode=docker`（WSL/Linux；Windows 宿主受框架路径 bug 限制）。**已在 WSL 下实测通过**： +基线 `combined_score=72.49, valid=1.0, num_instances=8`，与 process 模式一致。 + ## 测试 ```powershell diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md index 2a0f32b9..8fe4b852 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README.md @@ -157,8 +157,22 @@ far an online agent can get without full foresight. ## Docker -A minimal `python:3.11-slim` image is provided (`verification/docker/Dockerfile`). Docker -isolation scoring depends on the shared Frontier-Eng framework's env-forwarding, which is a -known framework-level limitation (the reference path env may not reach the container). -`docker` isolation is therefore best verified under WSL/Linux with the unified runtime's -`isolation_mode=docker`; on Windows hosts it is limited by a framework path bug. +A minimal `python:3.11-slim` image is provided (`verification/docker/Dockerfile`). Build it +and run the unified runtime in `isolation_mode=docker` (WSL/Linux; Windows hosts are limited +by a framework path bug). docker mode must NOT set `task.runtime.shell`; set +`FRONTIER_EVAL_UNIFIED_DOCKER_USER=1000:1000` so the container can write the WSL `/tmp` +sandbox: + +```bash +docker build -t cutting-opt-online -f benchmarks/ContinuousCasting/CuttingOptimizationOnline/verification/docker/Dockerfile benchmarks/ContinuousCasting/CuttingOptimizationOnline +FRONTIER_EVAL_UNIFIED_DOCKER_USER=1000:1000 \ + .venvs/frontier-eval-driver-wsl/bin/python -m frontier_eval task=unified \ + task.benchmark=ContinuousCasting/CuttingOptimizationOnline algorithm=openevolve algorithm.iterations=0 \ + llm.timeout=600 task.runtime.isolation_mode=docker task.runtime.docker_image=cutting-opt-online +``` + +**Verified (WSL, docker isolation)**: baseline `combined_score=52.40, valid=1.0, num_instances=8` +— identical to process mode. This also confirms the online anti-cheat holds in docker: the fixed +instances (with the hidden defect schedule) are **not** copied into the sandbox; the evaluator +loads them from the host source benchmark dir (`{benchmark_source}` → the mounted repo path in the +container), so the candidate in the container never reads the hidden defects. diff --git a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README_zh-CN.md b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README_zh-CN.md index 9cd37e81..4a9b2b8f 100644 --- a/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README_zh-CN.md +++ b/benchmarks/ContinuousCasting/CuttingOptimizationOnline/README_zh-CN.md @@ -112,6 +112,8 @@ python -m unittest discover -s verification -p "test_*.py" ## Docker -提供最简 `python:3.11-slim` 镜像（`verification/docker/Dockerfile`）。Docker 隔离评分依赖共享 -Frontier-Eng 框架的 env 转发（已知框架级限制，参考路径 env 可能进不了容器）。`docker` 隔离最好在 -WSL/Linux 下用 unified 的 `isolation_mode=docker` 验证；Windows 宿主受框架路径 bug 限制。 +提供最简 `python:3.11-slim` 镜像（`verification/docker/Dockerfile`）。构建镜像后用 unified 的 +`isolation_mode=docker`（WSL/Linux；Windows 宿主受框架路径 bug 限制）。**已在 WSL 下实测通过**： +基线 `combined_score=52.40, valid=1.0, num_instances=8`，与 process 模式一致。这也再次验证了在线版 +防作弊在容器里同样成立——固定实例（含隐藏报废表）**不进沙箱**，评测器从宿主源 benchmark 目录 +（`{benchmark_source}` → 容器内挂载的 repo 路径）加载，候选在容器内读不到隐藏报废表。