From 48940e95b39cdd78e82637ee93be80315366bcf9 Mon Sep 17 00:00:00 2001 From: Mike <2025013664@nwafu.edu.cn> Date: Mon, 7 Sep 2026 19:24:20 +0800 Subject: [PATCH 1/2] feat: add EdgeServiceReplicaPlacement benchmark --- TASK_DETAILS.md | 6 +- TASK_DETAILS_zh-CN.md | 6 +- .../EdgeServiceReplicaPlacement/README.md | 56 +++ .../README_zh-CN.md | 45 ++ .../EdgeServiceReplicaPlacement/Task.md | 68 +++ .../EdgeServiceReplicaPlacement/Task_zh-CN.md | 63 +++ .../baseline/result_log.txt | 108 +++++ .../calibration/analyze_scoring.py | 388 +++++++++++++++ .../calibration/strong.py | 131 +++++ .../calibration/weak.py | 50 ++ .../docs/evaluator-threat-model.md | 75 +++ .../docs/parameter_assumptions.md | 137 ++++++ .../docs/scoring-calibration-report.md | 165 +++++++ .../docs/tiny_oracle.md | 123 +++++ .../frontier_eval/agent_files.txt | 7 + .../frontier_eval/artifact_files.txt | 2 + .../frontier_eval/candidate_destination.txt | 1 + .../frontier_eval/constraints.txt | 9 + .../frontier_eval/copy_files.txt | 13 + .../frontier_eval/eval_command.txt | 1 + .../frontier_eval/initial_program.txt | 1 + .../frontier_eval/readonly_files.txt | 22 + .../frontier_eval/run_eval.sh | 10 + .../references/config.json | 34 ++ .../references/design_notes.md | 24 + .../scripts/init.py | 149 ++++++ .../verification/evaluator.py | 236 +++++++++ .../verification/policy_runtime.py | 247 ++++++++++ .../verification/policy_worker.py | 95 ++++ .../verification/requirements.txt | 1 + .../verification/simulator.py | 451 ++++++++++++++++++ .../verification/test_evaluator.py | 58 +++ .../verification/test_policy_runtime.py | 133 ++++++ .../verification/test_simulator.py | 209 ++++++++ .../verification/test_tiny_oracle.py | 21 + .../verification/tiny_oracle.py | 232 +++++++++ benchmarks/ComputerSystems/README.md | 1 + benchmarks/ComputerSystems/README_zh-CN.md | 1 + 38 files changed, 3377 insertions(+), 2 deletions(-) create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/README.md create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/README_zh-CN.md create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/Task.md create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/Task_zh-CN.md create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/baseline/result_log.txt create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/calibration/analyze_scoring.py create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/calibration/strong.py create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/calibration/weak.py create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/evaluator-threat-model.md create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/parameter_assumptions.md create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/scoring-calibration-report.md create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/tiny_oracle.md create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/agent_files.txt create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/artifact_files.txt create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/candidate_destination.txt create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/constraints.txt create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/copy_files.txt create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/eval_command.txt create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/initial_program.txt create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/readonly_files.txt create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/run_eval.sh create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/references/config.json create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/references/design_notes.md create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/scripts/init.py create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/evaluator.py create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/policy_runtime.py create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/policy_worker.py create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/requirements.txt create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/simulator.py create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_evaluator.py create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_policy_runtime.py create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_simulator.py create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_tiny_oracle.py create mode 100644 benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/tiny_oracle.py diff --git a/TASK_DETAILS.md b/TASK_DETAILS.md index 7475c4cc..94cb94b4 100644 --- a/TASK_DETAILS.md +++ b/TASK_DETAILS.md @@ -204,7 +204,7 @@ We welcome new engineering problem ideas — even without complete verification Polarization-multiplexed holography - ComputerSystems + ComputerSystems MallocLab High-performance C memory allocator (utilization & throughput) @@ -212,6 +212,10 @@ We welcome new engineering problem ideas — even without complete verification DuckDBWorkloadOptimization Index / materialized-view selection and query rewriting on official DuckDB workloads + + EdgeServiceReplicaPlacement + Dynamic edge-service replica placement and routing under workload bursts, failures, and link degradation + EngDesign CY_03, WJ_01, XY_05, AM_02, AM_03, YJ_02, YJ_03 diff --git a/TASK_DETAILS_zh-CN.md b/TASK_DETAILS_zh-CN.md index e2a070a2..e9878960 100644 --- a/TASK_DETAILS_zh-CN.md +++ b/TASK_DETAILS_zh-CN.md @@ -204,7 +204,7 @@ Frontier-Eng 目前已覆盖以下领域的任务。每个任务均配有可运 偏振复用全息 - ComputerSystems + ComputerSystems MallocLab 高性能 C 动态内存分配器（utilization & throughput） @@ -212,6 +212,10 @@ Frontier-Eng 目前已覆盖以下领域的任务。每个任务均配有可运 DuckDBWorkloadOptimization 基于 DuckDB 官方 workload 的索引 / 物化视图选择与查询改写 + + EdgeServiceReplicaPlacement + 面向流量突发、节点故障和链路退化的动态边缘服务副本放置与请求路由 + EngDesign CY_03, WJ_01, XY_05, AM_02, AM_03, YJ_02, YJ_03 diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/README.md b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/README.md new file mode 100644 index 00000000..c5517b97 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/README.md @@ -0,0 +1,56 @@ +# EdgeServiceReplicaPlacement + +This is a CPU-only, deterministic benchmark for stateful edge-service replica +placement and traffic routing. A candidate implements a policy, not a one-shot allocation: + +```python +def decide(observation: dict) -> dict: + ... +``` + +The evaluator runs ten 24-period scenarios covering normal diurnal demand, regional +bursts, node-failure-plus-burst, cross-region link degradation, and post-failure traffic +migration. Scale-ups have a one-period cold start. The simulator independently computes +availability, P95/P99 latency estimates, SLA violations, compute cost, cross-region +traffic/cost, and recovery time. + +## Direct validation + +From this directory: + +```bash +python verification/evaluator.py scripts/init.py \ + --metrics-out metrics.json --artifacts-out artifacts.json +python -m unittest discover -s verification -p "test_*.py" -v +``` + +## Unified zero-iteration validation + +From repository root: + +```bash +python -m frontier_eval \ + task=unified \ + task.benchmark=ComputerSystems/EdgeServiceReplicaPlacement \ + algorithm=openevolve \ + algorithm.iterations=0 +``` + +On Windows, set `PYTHONUTF8=1` and `PYTHONIOENCODING=utf-8` if the OpenEvolve dependency +otherwise uses the system GBK codec. This is an environment workaround, not a benchmark +requirement. + +## Editable boundary + +Only the EVOLVE-BLOCK in `scripts/init.py` is agent-editable. The worker receives JSON +observations and returns JSON actions. It runs in a fresh temporary process per scenario, +with bounded decision time and output size. This process boundary prevents ordinary +shared-state and protocol coupling; it is not a substitute for an operating-system +security sandbox. + +See [Task.md](Task.md) for the interface and [references/design_notes.md](references/design_notes.md) +for model scope. Detailed review material is in +[docs/parameter_assumptions.md](docs/parameter_assumptions.md), +[docs/tiny_oracle.md](docs/tiny_oracle.md), +[docs/evaluator-threat-model.md](docs/evaluator-threat-model.md), and +[docs/scoring-calibration-report.md](docs/scoring-calibration-report.md). diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/README_zh-CN.md b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/README_zh-CN.md new file mode 100644 index 00000000..8f6d8a2c --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/README_zh-CN.md @@ -0,0 +1,45 @@ +# EdgeServiceReplicaPlacement + +这是一个 CPU-only、确定性的 edge service 副本放置与流量路由 benchmark。候选实现 +动态策略，而不是一次性 allocation： + +```python +def decide(observation: dict) -> dict: + ... +``` + +Evaluator 在 10 个、每个 24 时段的场景中运行策略，覆盖正常日内负载、区域突发、节点 +故障叠加突发、跨区链路降级和故障恢复后的流量迁移。扩容有一个时段 cold start。 +Simulator 独立计算 availability、P95/P99 估计、SLA 违规、计算成本、跨区流量/成本和 +恢复时长。 + +## 直接验证 + +在当前目录运行： + +```bash +python verification/evaluator.py scripts/init.py \ + --metrics-out metrics.json --artifacts-out artifacts.json +python -m unittest discover -s verification -p "test_*.py" -v +``` + +## Unified 零迭代验证 + +在仓库根目录运行： + +```bash +python -m frontier_eval \ + task=unified \ + task.benchmark=ComputerSystems/EdgeServiceReplicaPlacement \ + algorithm=openevolve \ + algorithm.iterations=0 +``` + +## 可编辑边界 + +Agent 只能修改 `scripts/init.py` 中的 EVOLVE-BLOCK。Worker 接收 JSON observation 并返回 +JSON action；每个 scenario 使用新的临时进程，并限制决定时间和输出大小。该进程边界可 +避免一般的共享状态和协议耦合，但不能替代操作系统级安全沙箱。 + +接口见 [Task_zh-CN.md](Task_zh-CN.md)，模型假设见 +[references/design_notes.md](references/design_notes.md)。 diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/Task.md b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/Task.md new file mode 100644 index 00000000..c4c5aa13 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/Task.md @@ -0,0 +1,68 @@ +# Task: Dynamic edge service replica placement and routing + +Improve the policy inside the EVOLVE-BLOCK in `scripts/init.py`. At each five-minute +period, choose desired service replicas and current request routes. The same policy is +evaluated across hidden deterministic variants of five operating regimes. Future workload, +failure, and link events are not visible. + +## Observation + +`decide(observation)` receives JSON-compatible data containing: + +- timestep and period duration; +- nodes with region, failure domain, CPU capacity, and current alive status; +- services with CPU/replica, service rate, base latency, P99 SLO, response size, and + reliability class; +- current regional service demand and up to four historical periods; +- active and pending replicas; +- region RTTs and current cross-region bandwidth limits; +- the prior action and coarse violation feedback. + +No future trace values or candidate-computed score components are supplied. + +## Action + +Return exactly: + +```json +{ + "replicas": [ + {"service_id": "api", "node_id": "a-1", "count": 2} + ], + "routes": [ + {"service_id": "api", "source_region": "edge-a", "node_id": "a-1", "fraction": 0.8} + ] +} +``` + +`replicas` is the complete desired placement for the next state. New replicas are pending +for one period. Removed replicas stop serving immediately. `routes` controls only the +current period and may target active replicas that the same action retains. Fractions may +sum to less than one; the remainder becomes unserved demand and receives a continuous +performance penalty. Fractions above one are invalid. + +## Hard-invalid conditions + +- malformed schema, unknown or duplicate IDs; +- booleans used as integers, negative/non-integer replica counts; +- NaN/Infinity or route fractions outside `[0, 1]`; +- placement above physical CPU capacity or on failed nodes; +- routing to failed, pending, absent, or immediately removed replicas; +- per-service/source route sum above one; +- import/runtime error, timeout, or response above 64 KiB. + +Utilization, under-routing, oversubscription, SLA misses, cross-region traffic, temporary +failure impact, slow recovery, and overprovisioning remain continuous performance effects. + +## Metrics and score + +The evaluator independently reports request availability, request-weighted P95/P99, +P99-SLO violation rate, compute cost, cross-region GB/cost, and failure recovery steps. +For valid policies, each scenario receives a bounded engineering loss. Reliability and SLA +components carry 70% of the current weight; latency, compute, bandwidth, and recovery form +the remainder. Tail latency is normalized continuously by the median configured service +P99 SLO. The aggregate is 75% mean scenario score plus 25% 20th-percentile score. + +Raw metrics remain visible so engineering trade-offs are not hidden by the combined score. +Normalization, caps, extreme-policy checks, and weight sensitivity are documented in +`docs/scoring-calibration-report.md`. diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/Task_zh-CN.md b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/Task_zh-CN.md new file mode 100644 index 00000000..2cbc80ef --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/Task_zh-CN.md @@ -0,0 +1,63 @@ +# 任务：动态 Edge Service 副本放置与路由 + +改进 `scripts/init.py` 的 EVOLVE-BLOCK。每个五分钟时段，策略需要决定目标服务副本和 +当前请求路由。同一个策略会在五类运行状态的隐藏确定性变体上评测，未来 workload、 +failure 和 link event 不可见。 + +## Observation + +`decide(observation)` 接收 JSON-compatible 数据： + +- 当前 timestep 和时段长度； +- 节点的 region、failure domain、CPU 容量和存活状态； +- 服务的单副本 CPU、服务率、基础延迟、P99 SLO、响应大小和可靠性等级； +- 当前各 region/service 的 demand，以及最多四个历史时段； +- active 和 pending replicas； +- region RTT 和当前跨区带宽上限； +- 上一步动作与粗粒度违规反馈。 + +Observation 不包含未来 trace，也不包含候选自行计算的得分分量。 + +## Action + +必须准确返回： + +```json +{ + "replicas": [ + {"service_id": "api", "node_id": "a-1", "count": 2} + ], + "routes": [ + {"service_id": "api", "source_region": "edge-a", "node_id": "a-1", "fraction": 0.8} + ] +} +``` + +`replicas` 是完整的 desired placement。新副本 pending 一个时段后才 active；被移除的副本 +立即停止服务。`routes` 只控制当前时段，且只能指向当前 active、同时被本动作保留的副本。 +Fraction 合计可以小于 1，剩余请求会成为未服务流量并得到连续惩罚；合计大于 1 为非法。 + +## Hard-invalid 条件 + +- schema 错误、未知或重复 ID； +- bool 冒充 int、负数或非整数副本数； +- NaN/Infinity 或 `[0, 1]` 外的 route fraction； +- placement 超过物理 CPU，或放在故障节点； +- 向故障、pending、不存在或被本动作移除的副本路由； +- 每个 service/source 的 route sum 大于 1； +- import/runtime error、超时或响应大于 64 KiB。 + +高利用率、未完全路由、超载、SLA miss、临时故障影响、恢复慢和过度配置属于连续性能 +后果，不直接判 invalid。 + +## Metrics 与评分 + +Evaluator 独立报告 request availability、request-weighted P95/P99、P99 SLO violation、 +compute cost、cross-region GB/cost 和 failure recovery steps。 + +有效策略的每个 scenario 先得到 bounded engineering loss。当前 reliability + SLA 权重为 +70%，其余由 latency、compute、bandwidth 和 recovery 构成；tail latency 按三个服务 P99 +SLO 的中位数连续归一化；跨 scenario 的聚合为 75% 平均分 + 25% P20 分位数。 + +Raw metrics 始终可见，避免 combined score 隐藏工程权衡。Normalization、cap、极端策略 +和权重敏感性分析见 `docs/scoring-calibration-report.md`。 diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/baseline/result_log.txt b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/baseline/result_log.txt new file mode 100644 index 00000000..0eeb9555 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/baseline/result_log.txt @@ -0,0 +1,108 @@ +EdgeServiceReplicaPlacement benchmark calibration +Date: 2026-09-07 +Environment: + direct/tests: Windows, Python 3.13.5, CPU-only + unified/wrapper: Git Bash 5.2.37 and Python 3.12.13 driver venv +Scenarios: 10 total (five families, two deterministic variants each), 24 periods each + +Scoring correction in this validation round: + Before: tail latency used max(0, P99 / SLO - 1), so every below-SLO policy + received a zero tail-latency loss even though raw P99 values differed. + After: tail latency is normalized continuously against the median configured + service P99 SLO (180 ms). Compute and bandwidth caps were raised from 2 to 3 + after an overprovision policy reached the cap and obtained free reliability gains. + Simulator behavior and raw metrics were not changed. + +Final calibration policies + weak.py + valid: true + combined_score: 74.0766 + + scripts/init.py (reasonable baseline) + valid: true + evaluated_scenarios: 10/10 + combined_score: 73.9434 + mean_request_availability: 0.9944750191 + mean_request_weighted_p95_ms: 39.5209897096 + mean_request_weighted_p99_ms: 47.4463666782 + mean_p99_slo_violation_rate: 0.0294749761 + mean_compute_cost: 4.36 + mean_cross_region_gb: 1.1677365842 + mean_failure_recovery_steps: 0.2 + + strong.py (calibration-only policy; not agent context) + valid: true + combined_score: 74.4628 + +The labels are implementation descriptions, not an enforced score ordering. +Final observed ordering: reasonable (73.9434) < weak (74.0766) < strong (74.4628). +The small weak/reasonable reversal reflects a bandwidth-versus-reliability trade-off; +all three remain valid and distinguishable. + +Extreme-policy check + sla-first overprovision: 73.1402 + fixed full capacity: 70.2510 + local routing only: 70.0369 + ignore failures: 62.1667 + zero replicas: 11.7809 + cost-first underprovision: 11.1039 + minimum replicas: 9.5307 + aggressive cross-region routing: 7.6351 + No obviously absurd policy exceeded the three calibration policies after the cap fix. + +Determinism check + Three complete reasonable-baseline runs produced byte-identical metrics JSON. + metrics SHA-256: E8A32CDF200775F4E038E022CA8F8CC04041D8C2300B5172545AC15F621C99B7 + artifacts SHA-256: BBCF4D34CF15B345D8B39162ADAD4FDC00E8227EB5CA6820B2FF263E9F715B4A + Run wall times: 1.1619 s, 1.1417 s, 1.1981 s + +Adversarial/unit suite + command: python -m pytest benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification -q + result: 36 passed in 9.95 s; exit 0; command wall time 10.78 s + +Multi-timestep tiny oracle + command: python benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/tiny_oracle.py + trajectories enumerated: 723 + best raw metrics: availability=1.0, p95=32.0 ms, p99=42.0 ms, + SLO violation=0.0, compute cost=0.55, cross-region=0.0, recovery=0.0 + normalized components: reliability=0.0, SLA=0.0, tail=0.2333333333, + compute=0.0916666667, bandwidth=0.0, recovery=0.0 + combined_score: 96.62493678284117 + independent oracle and production evaluator matched exactly + command wall time: 0.52 s; exit 0 + +Direct evaluator + valid=true; 10/10 scenarios; score=73.9434 + command wall time: 1.96 s; exit 0 + +frontier_eval/run_eval.sh + valid=true; 10/10 scenarios; score=73.9434 + command wall time: 2.22 s; exit 0 + +Unified zero-iteration + valid=1; benchmark_returncode=0; score=73.9434 + benchmark runtime_s=2.8343; complete command wall time=5.52 s; exit 0 + Windows-specific invocation used Git Bash plus an explicit existing Python 3.12 + executable. No machine-specific absolute path is stored in task metadata. + +Metadata/readonly audit + repository audit command exited 0 in non-strict mode. It printed warnings for existing + tasks elsewhere in the repository; EdgeServiceReplicaPlacement was not listed. + +Scoring sensitivity + Each component weight was varied independently by -5%, -2%, +2%, and +5%, with + the other weights proportionally renormalized. Strong remained top in every run. + Only the close fixed-full/local-only pair reversed under small reliability/compute + preference changes, consistent with their genuine cost/reliability trade-off. + +Agent optimization + command: python -m frontier_eval task=unified + task.benchmark=ComputerSystems/EdgeServiceReplicaPlacement + algorithm=openevolve algorithm.iterations=10 [Windows runtime overrides] + result: NOT EXECUTED; exit 1 before iteration 0; command wall time 2.75 s + reason: OPENAI_API_KEY (or llm.api_key) is unavailable in this environment. + No optimization trajectory or learnability claim is made. + +These values demonstrate a deterministic, executable benchmark and calibrated score. +They do not claim production validity. A real multi-iteration Agent run remains required +by the current PR-readiness gate. diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/calibration/analyze_scoring.py b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/calibration/analyze_scoring.py new file mode 100644 index 00000000..a85b2b29 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/calibration/analyze_scoring.py @@ -0,0 +1,388 @@ +"""Reproduce score-pipeline, extreme-policy, and weight-sensitivity checks.""" + +from __future__ import annotations + +import argparse +from collections import OrderedDict +import json +import math +from pathlib import Path +from statistics import fmean, median +import sys +import tempfile +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "verification")) + +from evaluator import _quantile, evaluate # noqa: E402 +from simulator import load_config # noqa: E402 + + +NOMINAL_WEIGHTS: dict[str, float] = { + "reliability": 0.40, + "sla": 0.30, + "tail_latency": 0.10, + "compute": 0.12, + "bandwidth": 0.04, + "recovery": 0.04, +} +RELATIVE_PERTURBATIONS = (-0.05, -0.02, 0.0, 0.02, 0.05) + + +POLICY_TEMPLATE = '''\ +from __future__ import annotations +import math + +MODE = {mode!r} + + +def reset_policy(): + return None + + +def _add(placement, cpu_used, nodes, services, service_id, node_id, count): + cpu = float(services[service_id]["cpu_per_replica"]) + capacity = float(nodes[node_id]["cpu_capacity"]) + feasible = min(count, int((capacity - cpu_used[node_id] + 1e-12) // cpu)) + if feasible > 0: + placement[(service_id, node_id)] = placement.get((service_id, node_id), 0) + feasible + cpu_used[node_id] += feasible * cpu + + +def decide(observation): + if MODE == "zero": + return {{"replicas": [], "routes": []}} + + nodes = {{node["id"]: node for node in observation["nodes"]}} + services = {{service["id"]: service for service in observation["services"]}} + alive = sorted(node_id for node_id, node in nodes.items() if node["alive"]) + regions = list(observation["workload_rps"]) + cpu_used = {{node_id: 0.0 for node_id in alive}} + placement = {{}} + + if MODE in {{"fixed_full", "aggressive_cross"}}: + mix = {{"api": 1, "search": 1, "media": 3}} + for node_id in alive: + for service_id, count in mix.items(): + _add(placement, cpu_used, nodes, services, service_id, node_id, count) + elif MODE == "sla_first": + mix = {{"api": 2, "search": 1, "media": 2}} + for node_id in alive: + for service_id, count in mix.items(): + _add(placement, cpu_used, nodes, services, service_id, node_id, count) + elif MODE == "ignore_failure": + # Fixed preselected nodes; a failed target is omitted, never replaced elsewhere. + for region_index, region in enumerate(regions): + fixed_nodes = [f"{{region[-1]}}-1", f"{{region[-1]}}-2"] + for service_index, service_id in enumerate(services): + node_id = fixed_nodes[service_index % len(fixed_nodes)] + if node_id in cpu_used: + _add(placement, cpu_used, nodes, services, service_id, node_id, 1) + elif MODE in {{"minimum", "cost_first"}}: + for service_index, service_id in enumerate(services): + if not alive: + break + # One global replica per service. Cost-first pins it to edge-a and refuses + # remote traffic; minimum allows all regions to compete for it. + preferred = [node_id for node_id in alive if nodes[node_id]["region"] == "edge-a"] + choices = preferred or alive + node_id = choices[service_index % len(choices)] + _add(placement, cpu_used, nodes, services, service_id, node_id, 1) + elif MODE == "local_only": + # Workload-responsive placement, but no cross-region recovery routing. + for service_id in sorted(services): + cpu = float(services[service_id]["cpu_per_replica"]) + rate = float(services[service_id]["service_rate_rps"]) + for region in regions: + needed = max( + 1, + math.ceil( + 1.20 * float(observation["workload_rps"][region][service_id]) / rate + ), + ) + local = sorted( + node_id for node_id in alive if nodes[node_id]["region"] == region + ) + for _ in range(needed): + feasible = [ + node_id + for node_id in local + if cpu_used[node_id] + cpu <= float(nodes[node_id]["cpu_capacity"]) + 1e-12 + ] + if not feasible: + break + node_id = min( + feasible, + key=lambda item: ( + placement.get((service_id, item), 0), cpu_used[item], item + ), + ) + _add(placement, cpu_used, nodes, services, service_id, node_id, 1) + else: + raise RuntimeError(f"unknown mode: {{MODE}}") + + replicas = [ + {{"service_id": service_id, "node_id": node_id, "count": count}} + for (service_id, node_id), count in sorted(placement.items()) + ] + active = {{ + (row["service_id"], row["node_id"]): int(row["count"]) + for row in observation["active_replicas"] + }} + retained = {{ + (service_id, node_id) + for (service_id, node_id), count in placement.items() + if count > 0 and active.get((service_id, node_id), 0) > 0 + }} + routes = [] + for source in regions: + for service_id in services: + targets = [ + node_id for candidate_service, node_id in retained + if candidate_service == service_id + ] + if MODE in {{"cost_first"}}: + targets = [node_id for node_id in targets if nodes[node_id]["region"] == source] + elif MODE == "aggressive_cross": + remote = [node_id for node_id in targets if nodes[node_id]["region"] != source] + if remote: + max_rtt = max( + observation["network_rtt_ms"][source][nodes[node_id]["region"]] + for node_id in remote + ) + targets = [ + node_id for node_id in remote + if observation["network_rtt_ms"][source][nodes[node_id]["region"]] == max_rtt + ] + else: + targets = [] + elif MODE in {{"fixed_full", "sla_first", "local_only", "ignore_failure"}}: + targets = [node_id for node_id in targets if nodes[node_id]["region"] == source] + else: + targets.sort( + key=lambda node_id: ( + nodes[node_id]["region"] != source, + observation["network_rtt_ms"][source][nodes[node_id]["region"]], + node_id, + ) + ) + targets = targets[:1] + if targets: + fraction = 1.0 / len(targets) + for node_id in sorted(targets): + routes.append( + {{ + "service_id": service_id, + "source_region": source, + "node_id": node_id, + "fraction": fraction, + }} + ) + return {{"replicas": replicas, "routes": routes}} +''' + + +def _weights_with_relative_change(component: str, relative_change: float) -> dict[str, float]: + original = NOMINAL_WEIGHTS[component] + target = original * (1.0 + relative_change) + other_scale = (1.0 - target) / (1.0 - original) + return { + name: target if name == component else weight * other_scale + for name, weight in NOMINAL_WEIGHTS.items() + } + + +def _score_rows(rows: list[dict[str, Any]], weights: dict[str, float]) -> float: + scenario_scores = [] + for row in rows: + components = row["normalized_loss_components"] + loss = sum(weights[name] * float(components[name]) for name in weights) + scenario_scores.append(100.0 * math.exp(-loss)) + return 0.75 * fmean(scenario_scores) + 0.25 * _quantile(scenario_scores, 0.20) + + +def _summarize(result: dict[str, Any]) -> dict[str, Any]: + rows = result["rows"] + metric_names = ( + "request_availability", + "unserved_rate", + "request_weighted_p95_ms", + "request_weighted_p99_ms", + "p99_slo_violation_rate", + "compute_cost", + "cross_region_gb", + "cross_region_cost", + "failure_recovery_steps", + ) + return { + "valid": bool(result["valid"]), + "combined_score": result["combined_score"], + "mean_raw_metrics": { + name: fmean(float(row["raw_metrics"][name]) for row in rows) + for name in metric_names + }, + "mean_normalized_components": { + name: fmean(float(row["normalized_loss_components"][name]) for row in rows) + for name in NOMINAL_WEIGHTS + }, + "mean_weighted_loss_contributions": { + name: NOMINAL_WEIGHTS[name] + * fmean(float(row["normalized_loss_components"][name]) for row in rows) + for name in NOMINAL_WEIGHTS + }, + "scenario_rows": [ + { + "scenario": row["scenario"], + "family": row["family"], + "score": row["score"], + "p95_ms": row["raw_metrics"]["request_weighted_p95_ms"], + "p99_ms": row["raw_metrics"]["request_weighted_p99_ms"], + "tail_latency_component": row["normalized_loss_components"]["tail_latency"], + } + for row in rows + ], + } + + +def _normalization_spec() -> dict[str, Any]: + config = load_config() + budgets = config["score_budgets"] + latency_reference = median(float(item["p99_slo_ms"]) for item in config["services"]) + return { + "reliability": { + "raw": "unserved_rate", + "unit": "ratio", + "reference": budgets["unserved_rate"], + "cap": 3.0, + "classification": "benchmark operational target / synthetic calibration constant", + }, + "sla": { + "raw": "p99_slo_violation_rate", + "unit": "ratio", + "reference": budgets["slo_violation_rate"], + "cap": 3.0, + "classification": "synthetic calibration constant", + }, + "tail_latency": { + "raw": "request_weighted_p99_ms", + "unit": "milliseconds", + "reference": latency_reference, + "reference_derivation": "median configured service P99 SLO", + "cap": 2.0, + "classification": "scenario/task-derived reference", + }, + "compute": { + "raw": "compute_cost", + "unit": "abstract cost units/episode", + "reference": budgets["compute_cost"], + "cap": 3.0, + "classification": "baseline-calibrated synthetic constant", + }, + "bandwidth": { + "raw": "cross_region_gb", + "unit": "GB/episode", + "reference": budgets["cross_region_gb"], + "cap": 3.0, + "classification": "baseline-calibrated synthetic constant", + }, + "recovery": { + "raw": "failure_recovery_steps", + "unit": "control periods", + "reference": budgets["recovery_steps"], + "cap": 2.0, + "classification": "scenario-derived failure-duration budget", + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path) + args = parser.parse_args() + + candidate_paths: OrderedDict[str, Path] = OrderedDict( + ( + ("weak_static", ROOT / "calibration" / "weak.py"), + ("reasonable", ROOT / "scripts" / "init.py"), + ("strong", ROOT / "calibration" / "strong.py"), + ) + ) + modes = OrderedDict( + ( + ("zero", "zero"), + ("minimum_replica", "minimum"), + ("cost_first_underprovision", "cost_first"), + ("local_routing_only", "local_only"), + ("fixed_full_capacity", "fixed_full"), + ("sla_first_overprovision", "sla_first"), + ("aggressive_cross_region", "aggressive_cross"), + ("ignore_failure", "ignore_failure"), + ) + ) + + with tempfile.TemporaryDirectory(prefix="edge_score_review_") as tempdir: + temp_root = Path(tempdir) + for label, mode in modes.items(): + path = temp_root / f"{label}.py" + path.write_text(POLICY_TEMPLATE.format(mode=mode), encoding="utf-8") + candidate_paths[label] = path + evaluations = {name: evaluate(path) for name, path in candidate_paths.items()} + + policies = {name: _summarize(result) for name, result in evaluations.items()} + nominal_ranking = sorted(policies, key=lambda name: policies[name]["combined_score"], reverse=True) + sensitivity: dict[str, Any] = {} + for component in NOMINAL_WEIGHTS: + component_rows = [] + for relative_change in RELATIVE_PERTURBATIONS: + weights = _weights_with_relative_change(component, relative_change) + scores = { + name: _score_rows(evaluations[name]["rows"], weights) + for name in evaluations + } + ranking = sorted(scores, key=scores.get, reverse=True) + component_rows.append( + { + "relative_change": relative_change, + "weights": weights, + "scores": scores, + "ranking_high_to_low": ranking, + "rank_reversal_vs_nominal": ranking != nominal_ranking, + } + ) + sensitivity[component] = component_rows + + # Retain the previously observed absolute 0.02 SLA-to-bandwidth trade-off as a + # separate engineering-preference test, not as the systematic relative sweep. + shifted = dict(NOMINAL_WEIGHTS) + shifted["sla"] -= 0.02 + shifted["bandwidth"] += 0.02 + pairwise_scores = { + name: _score_rows(evaluations[name]["rows"], shifted) for name in evaluations + } + payload = { + "normalization": _normalization_spec(), + "nominal_weights": NOMINAL_WEIGHTS, + "policies": policies, + "nominal_ranking_high_to_low": nominal_ranking, + "relative_weight_sensitivity": sensitivity, + "absolute_pairwise_shift": { + "change": "SLA -0.02, bandwidth +0.02", + "weights": shifted, + "scores": pairwise_scores, + "ranking_high_to_low": sorted(pairwise_scores, key=pairwise_scores.get, reverse=True), + }, + "service_p99_slo_ms": { + item["id"]: item["p99_slo_ms"] for item in load_config()["services"] + }, + } + rendered = json.dumps(payload, indent=2, sort_keys=True, allow_nan=False) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered + "\n", encoding="utf-8") + print(rendered) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/calibration/strong.py b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/calibration/strong.py new file mode 100644 index 00000000..746cc63d --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/calibration/strong.py @@ -0,0 +1,131 @@ +"""Stronger hidden calibration policy using history-aware demand estimates.""" + +from __future__ import annotations + +import math +from typing import Any + + +def reset_policy() -> None: + return None + + +def decide(observation: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + nodes = {node["id"]: node for node in observation["nodes"]} + services = {service["id"]: service for service in observation["services"]} + alive = [node_id for node_id, node in nodes.items() if node["alive"]] + active_now = { + (item["service_id"], item["node_id"]): int(item["count"]) + for item in observation["active_replicas"] + } + cpu_used = {node_id: 0.0 for node_id in alive} + placement: dict[tuple[str, str], int] = {} + history = observation["workload_history"] + + for service_id in sorted(services, key=lambda sid: services[sid]["reliability_class"] != "critical"): + service = services[service_id] + cpu = float(service["cpu_per_replica"]) + rate = float(service["service_rate_rps"]) + for source, current_by_service in observation["workload_rps"].items(): + current = float(current_by_service[service_id]) + recent = [float(period[source][service_id]) for period in history[-3:]] + trend = max(0.0, current - recent[-1]) if recent else 0.0 + forecast = max([current, *recent], default=current) + 1.5 * trend + headroom = 1.35 if service["reliability_class"] == "critical" else 1.22 + needed = max(1, math.ceil(headroom * forecast / rate)) + local_domains: set[str] = set() + candidates = sorted( + alive, + key=lambda node_id: ( + nodes[node_id]["region"] != source, + observation["network_rtt_ms"][source][nodes[node_id]["region"]], + cpu_used[node_id], + node_id, + ), + ) + for _ in range(needed): + feasible = [node_id for node_id in candidates if cpu_used[node_id] + cpu <= float(nodes[node_id]["cpu_capacity"])] + if not feasible: + break + node_id = min( + feasible, + key=lambda candidate: ( + nodes[candidate]["region"] != source, + observation["network_rtt_ms"][source][nodes[candidate]["region"]], + active_now.get((service_id, candidate), 0) <= 0, + nodes[candidate]["failure_domain"] in local_domains, + placement.get((service_id, candidate), 0), + cpu_used[candidate], + candidate, + ), + ) + placement[(service_id, node_id)] = placement.get((service_id, node_id), 0) + 1 + cpu_used[node_id] += cpu + local_domains.add(nodes[node_id]["failure_domain"]) + + replicas = [ + {"service_id": service_id, "node_id": node_id, "count": count} + for (service_id, node_id), count in sorted(placement.items()) + ] + active = active_now + routes: list[dict[str, Any]] = [] + remaining_capacity = { + (service_id, node_id): active.get((service_id, node_id), 0) + * float(services[service_id]["service_rate_rps"]) + for service_id in services + for node_id in alive + if active.get((service_id, node_id), 0) > 0 + and placement.get((service_id, node_id), 0) > 0 + } + unmet: list[tuple[str, str, float, float]] = [] + + # First reserve each target's capacity for demand originating in its own region. + for source, demand_by_service in observation["workload_rps"].items(): + for service_id, demand_value in demand_by_service.items(): + demand = max(float(demand_value), 1e-9) + remaining_demand = demand + local_targets = sorted( + node_id + for node_id in alive + if nodes[node_id]["region"] == source + and remaining_capacity.get((service_id, node_id), 0.0) > 0.0 + ) + for node_id in local_targets: + amount = min(remaining_demand, remaining_capacity[(service_id, node_id)]) + if amount > 1e-12: + routes.append({"service_id": service_id, "source_region": source, "node_id": node_id, "fraction": amount / demand}) + remaining_capacity[(service_id, node_id)] -= amount + remaining_demand -= amount + if remaining_demand > 1e-12: + unmet.append((source, service_id, demand, remaining_demand)) + + # Then use only spare remote capacity, bounded by the observed link budget. + link_remaining = dict(observation["cross_region_bandwidth_mb_per_period"]) + period_seconds = float(observation["period_minutes"]) * 60.0 + for source, service_id, demand, remaining_demand in unmet: + response_mb = float(services[service_id]["response_mb"]) + remote_targets = sorted( + ( + node_id + for node_id in alive + if nodes[node_id]["region"] != source + and remaining_capacity.get((service_id, node_id), 0.0) > 0.0 + ), + key=lambda node_id: ( + observation["network_rtt_ms"][source][nodes[node_id]["region"]], + node_id, + ), + ) + for node_id in remote_targets: + target_region = nodes[node_id]["region"] + link = f"{source}->{target_region}" + by_link = link_remaining.get(link, 0.0) / max(response_mb * period_seconds, 1e-12) + amount = min(remaining_demand, remaining_capacity[(service_id, node_id)], by_link) + if amount > 1e-12: + routes.append({"service_id": service_id, "source_region": source, "node_id": node_id, "fraction": amount / demand}) + remaining_capacity[(service_id, node_id)] -= amount + link_remaining[link] -= amount * response_mb * period_seconds + remaining_demand -= amount + if remaining_demand <= 1e-12: + break + return {"replicas": replicas, "routes": routes} diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/calibration/weak.py b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/calibration/weak.py new file mode 100644 index 00000000..37180437 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/calibration/weak.py @@ -0,0 +1,50 @@ +"""Weak static calibration policy; never exposed as an agent-editable file.""" + +from __future__ import annotations + +from typing import Any + + +def reset_policy() -> None: + return None + + +def decide(observation: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + nodes = {node["id"]: node for node in observation["nodes"]} + alive = {node_id for node_id, node in nodes.items() if node["alive"]} + services = {service["id"]: service for service in observation["services"]} + regions = list(observation["workload_rps"]) + placement: dict[tuple[str, str], int] = {} + for region in regions: + local = sorted(node_id for node_id in alive if nodes[node_id]["region"] == region) + for index, service_id in enumerate(services): + if local: + placement[(service_id, local[index % len(local)])] = 1 + replicas = [ + {"service_id": service_id, "node_id": node_id, "count": count} + for (service_id, node_id), count in sorted(placement.items()) + ] + active = { + (item["service_id"], item["node_id"]): int(item["count"]) + for item in observation["active_replicas"] + } + routes: list[dict[str, Any]] = [] + for source in regions: + for service_id in services: + targets = [ + node_id + for node_id in alive + if nodes[node_id]["region"] == source + and active.get((service_id, node_id), 0) > 0 + and placement.get((service_id, node_id), 0) > 0 + ] + if targets: + routes.append( + { + "service_id": service_id, + "source_region": source, + "node_id": targets[0], + "fraction": 1.0, + } + ) + return {"replicas": replicas, "routes": routes} diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/evaluator-threat-model.md b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/evaluator-threat-model.md new file mode 100644 index 00000000..1c603474 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/evaluator-threat-model.md @@ -0,0 +1,75 @@ +# Evaluator threat model + +## Scope + +The evaluator treats candidate output as untrusted and independently computes placement, +routing, latency, availability, cost, recovery, and score. This review focuses on wrong +results, resource abuse, protocol failure, and reproducibility. It is not a claim that a +local Python subprocess is a hostile-code security sandbox. + +## Attack surface review + +| Attack surface | Current defense | Test covering it | Remaining limitation | +| --- | --- | --- | --- | +| `NaN` route fraction | Explicit numeric check plus `math.isfinite`; JSON RPC encoding also disables non-finite request values | `test_nan_route_is_rejected` | Candidate-internal non-finite state is irrelevant until it crosses the action boundary. | +| Positive/negative infinity | Same finite-number check; values cannot enter scoring | `test_extreme_route_float_is_rejected` | None known at the action boundary. | +| `bool` used as replica integer | Exact exclusion of `bool` before accepting `int` | `test_bool_cannot_impersonate_integer` | Python subclasses or unusual objects are removed by the JSON boundary. | +| Negative zero | `-0.0` route is accepted as numerically identical to `0.0`; replica counts must be integers and non-negative | Covered indirectly by finite/range validation; reviewed manually | Canonicalizing `-0.0` would not change simulator behavior or score, so no extra rejection was added. | +| Extreme finite route float | Range `[0,1]` enforced after finite conversion | `test_extreme_route_float_is_rejected`, `test_route_fraction_out_of_range_is_rejected` | None known. | +| Duplicate placement IDs | Exact `(service_id,node_id)` uniqueness | `test_duplicate_placement_is_rejected` | None known. | +| Duplicate route IDs | Exact `(service_id,source_region,node_id)` uniqueness | `test_duplicate_route_is_rejected` | Multiple different destination nodes are intentionally allowed. | +| Unknown service/node/region IDs | All IDs resolved against evaluator-owned config | `test_unknown_id_is_rejected`; malformed route test exercises nested validation | Config authenticity depends on the framework readonly boundary, not this check alone. | +| Missing top-level IDs/objects | Action must contain exactly `replicas` and `routes`; nested objects require exact field sets | `test_exact_top_level_schema_is_enforced`, `test_malformed_nested_route_is_rejected` | Omitting a service from desired placement is intentionally legal and means scale to zero; omitting routes is a poor but valid policy with continuous unserved-demand loss. | +| Unexpected keys | Exact key sets at top level and in each nested item | `test_exact_top_level_schema_is_enforced`, `test_malformed_nested_route_is_rejected` | This deliberately rejects forward-compatible extensions until the schema changes. | +| Negative/non-integer/huge replica count | Exact integer, non-negative range, then CPU-capacity check | `test_negative_and_noninteger_replica_counts_are_rejected`, `test_huge_replica_count_is_rejected_by_capacity` | Python arbitrary-size integer parsing occurs before capacity rejection, but the 64 KiB response limit bounds its textual size. | +| Over-capacity placement | Evaluator recomputes per-node CPU use from trusted service footprints | `test_capacity_is_checked` | CPU is a reduced abstract capacity, not a scheduler model. | +| Route fraction rounding | Per-key sum uses a documented `1e-8` tolerance | `test_route_sum_tolerance_has_a_strict_boundary` | The tolerance is an engineering choice and should be reviewed if the action schema changes. | +| Route sum slightly above one | Any sum above `1 + 1e-8` is invalid | `test_route_sum_above_one_is_rejected`, tolerance-boundary test | A sum within tolerance can serve at most a negligible excess before downstream capacity, and is accepted intentionally. | +| Route to pending replica | Route target must be active now and retained by desired placement | `test_pending_replica_cannot_receive_traffic` | Candidate receives pending state, so the behavior is diagnosable. | +| Route to failed node | Alive-node check applies to placement and routing | `test_failed_node_cannot_receive_placement`, `test_failed_node_cannot_receive_route` | The MVP models individual node failures only. | +| Route to a replica removed in the same action | Desired placement must retain a positive count in addition to current activity | Existing inactive-route path is exercised by simulator validation tests | A dedicated named regression test would be useful only if this branch later changes; current coverage is adequate. | +| Giant JSON/action | Worker response line capped at 64 KiB; action list capped at 256 total items | `test_oversized_response_is_rejected`; action-size path covered through schema validation | Reading is line-oriented; up to the cap must still be parsed. | +| Non-JSON stdout | Strict one-line JSON RPC; decoding/parsing failure closes worker | `test_non_json_result_is_rejected`, `test_non_json_stdout_cannot_corrupt_protocol` | Candidate must not use stdout for logging; stderr is the diagnostic channel. | +| Stderr spam | Stderr is continuously drained to avoid deadlock; retained tail bounded to 16 KiB | `test_stderr_spam_is_drained_and_tail_is_bounded` | Spam can consume CPU/I/O until timeout; this is reliability control, not a byte-level OS quota. | +| Candidate exception | Worker returns typed error; evaluator fails closed and scores invalid candidate as zero | `test_candidate_exception_is_reported`, `test_invalid_action_cannot_compete` | Error text is diagnostic only and is not trusted as a metric. | +| Infinite loop | Per-call and per-scenario monotonic deadlines; process tree termination | `test_infinite_loop_times_out` | Terminating descendants is best-effort and platform-dependent without a container/job-object sandbox. | +| Abrupt process exit / nonzero exit | EOF and process status become a candidate error; remaining scenarios are not silently scored | `test_abrupt_process_exit_is_reported` | Native crashes provide only bounded stderr diagnostics. | +| Missing `decide` entry point | Candidate import/reset handshake fails closed | `test_missing_decide_fails_closed` | None known. | +| Malformed nested objects | Exact dict/list/field/type checks before simulation | `test_malformed_nested_route_is_rejected` and other schema tests | Deep nesting is bounded indirectly by response bytes; Python JSON decoder depth remains an implementation limit. | +| Candidate mutates observation | Observation crosses a JSON serialization boundary; evaluator retains its own objects | `test_candidate_mutates_only_its_json_copy_of_observation` | Candidate can mutate its private copy, which is harmless and intentional. | +| Candidate self-reports score or metrics | Unexpected output keys fail schema; evaluator computes every raw metric and score itself | `test_exact_top_level_schema_is_enforced`, `test_invalid_action_cannot_compete` | Candidate code can know the public formula, as expected for an optimization benchmark. | +| Candidate changes exogenous randomness | All workload/failure/link traces are generated from fixed seeds before candidate execution | `test_generation_is_deterministic` | Fixed public scenario structure can be overfit; only one of two variants per family is included in feedback artifacts. | +| Nondeterministic replay | Official baselines are compared across repeated evaluator runs | `test_reasonable_baseline_is_deterministic`; final validation repeats the reasonable baseline three times | An arbitrary candidate is not automatically run three times or rejected for nondeterminism. | +| Worker modifies copied candidate or temp working directory | Candidate runs from an isolated temporary copy and clean working directory | `test_reasonable_candidate_round_trip` exercises the boundary | This protects evaluator reliability, not host files outside the temp directory. | +| Worker reads/writes host files or opens network connections | No evaluator-level claim or defense | Not applicable | **Unmitigated at OS level.** A plain Python worker can attempt filesystem, process, or network operations allowed to the host account. A stronger sandbox would require framework/container/OS controls and maintainer agreement. | +| Evaluator/reference mutation in unified execution | Task metadata declares verifier/config/docs/runtime files readonly and copies a controlled set | Metadata/readonly audit; direct tests do not model framework mounts | Readonly effectiveness belongs to Frontier Eval execution. Running the evaluator directly does not create a readonly filesystem. | + +## Isolation boundaries + +### Process reliability isolation + +The runtime copies the candidate to a temporary working directory, launches a separate +isolated-mode Python process, speaks bounded JSON-lines RPC, drains stderr, imposes +timeouts, and terminates the process tree on failure. These controls prevent common hangs, +stdout corruption, shared-object mutation, and unbounded retained diagnostics. + +### Framework readonly boundary + +The unified metadata lists evaluator, simulator, runtime, worker, configuration, tests, +docs, and wrapper files as readonly. Frontier Eval is responsible for applying that +boundary in a unified run. It is distinct from candidate-process isolation and must be +checked with the repository audit plus an actual zero-iteration unified run. + +### OS-level sandboxing + +There is no benchmark-owned OS security sandbox. The subprocess is not proof against host +filesystem access, network access, subprocess creation, environment discovery, or every +resource-exhaustion technique. No Docker/Kubernetes layer was added in this review because +that would change scope and should follow maintainer guidance. + +## Review conclusion + +No new high-score exploit was found after adding targeted tests for extreme values, +malformed objects, route-tolerance boundaries, failed-node routing, abrupt exit, stdout +noise, stderr spam, and observation mutation. The most important remaining limitation is +the absence of OS-level hostile-code containment; documentation now states that explicitly. diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/parameter_assumptions.md b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/parameter_assumptions.md new file mode 100644 index 00000000..bed27e22 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/parameter_assumptions.md @@ -0,0 +1,137 @@ +# Parameter provenance and assumptions + +## Status and interpretation + +This document inventories the parameters that materially affect the MVP result. It is +not evidence that the current numbers describe a particular cloud provider or production +deployment. Unless a row explicitly says otherwise, values are local MVP assumptions +chosen to exercise the state transitions and engineering trade-offs. + +The provenance labels used in this document are: + +- `source-backed`: supported by a cited external source; +- `engineering simplification`: a deliberately reduced model or implementation guardrail; +- `scenario-derived budget`: derived from an explicit scenario duration or bound; +- `baseline-calibrated`: scaled against measured policies in this benchmark; +- `synthetic calibration parameter`: selected to produce auditable deterministic scenarios; +- `arbitrary placeholder requiring maintainer feedback`: provisional value whose suitability + should be discussed before a formal PR. + +There are currently **no numerical values claimed as source-backed**. Values without a +formal source are identified below as a **synthetic calibration assumption for the MVP**, +not presented as measured industry data. + +## Topology, time, and service model + +| Parameter | Current value/range | Unit | Used where | Type | Rationale/source | +| --- | ---: | --- | --- | --- | --- | +| Control period | 5 | minutes | `config.json`; observation; link-volume conversion | arbitrary placeholder requiring maintainer feedback | A short aggregate control interval that makes cold start and recovery visible without creating a large simulation. It is not a measured autoscaler interval. | +| Episode length | 24 | periods (120 minutes) | `make_scenario` | engineering simplification | Long enough to contain a baseline, event, and recovery phase while keeping CPU evaluation fast. | +| Regions | 3 | logical regions | `config.json` | engineering simplification | Minimum small topology that permits local, adjacent, and higher-latency routing choices. Region names are synthetic. | +| Nodes per region | 2 (6 total) | logical nodes | `config.json` | engineering simplification | Two failure domains per region permit a placement-diversity decision without modeling a cluster. | +| Failure domains | one rack label per node | logical label | node observations and placement reasoning | engineering simplification | Exposes correlated-placement structure; the MVP currently fails individual nodes, not entire racks. | +| Node CPU capacity | 10.0 | abstract CPU-capacity units/node | placement validation | synthetic calibration parameter | Synthetic calibration assumption for the MVP. Combined with per-replica footprints, it forces multi-service capacity choices but is not a vCPU claim. | +| API CPU footprint | 2.0 | CPU-capacity units/replica | placement validation | synthetic calibration parameter | Synthetic calibration assumption for the MVP; separates replica count from service throughput. | +| Search CPU footprint | 2.5 | CPU-capacity units/replica | placement validation | synthetic calibration parameter | Synthetic calibration assumption for the MVP; search is intentionally the largest footprint. | +| Media CPU footprint | 1.5 | CPU-capacity units/replica | placement validation | synthetic calibration parameter | Synthetic calibration assumption for the MVP; no claim about a real media service. | +| API nominal service rate | 90.0 | requests/second/active replica | service-capacity scaling | synthetic calibration parameter | Synthetic calibration assumption for the MVP. It leaves normal-load headroom but makes burst handling consequential. | +| Search nominal service rate | 65.0 | requests/second/active replica | service-capacity scaling | synthetic calibration parameter | Same role as API rate, with a lower synthetic throughput. | +| Media nominal service rate | 50.0 | requests/second/active replica | service-capacity scaling | synthetic calibration parameter | Same role as API rate, with a lower synthetic throughput. | +| Scale-up delay | 1 | control period (5 minutes) | `pending` to `active` transition | engineering simplification | Represents aggregate scheduling, startup, and readiness delay. It is explicitly not a measured container-start time. Scale-down is immediate in the MVP. | +| Bootstrap placement | 1 replica/service/region, round-robin over two nodes | replicas | simulator initialization | engineering simplification | Avoids a meaningless all-cold first period and provides a deterministic initial state. | + +## Workload, event, and network assumptions + +| Parameter | Current value/range | Unit | Used where | Type | Rationale/source | +| --- | ---: | --- | --- | --- | --- | +| Base request rate | API 54; search 34; media 25 | requests/second/region before modifiers | `_base_demand` | synthetic calibration parameter | Synthetic calibration assumption for the MVP. Values are sized against nominal service rates to create both spare capacity and overload regimes. | +| Region demand factors | 1.08, 0.94, 0.82 | multiplier | `_base_demand` | synthetic calibration parameter | Creates asymmetric regions so uniform placement is not automatically optimal. | +| Diurnal factor | 0.82 to 1.12 | multiplier | `_base_demand` | synthetic calibration parameter | A deterministic sinusoid supplies gradual demand change. It is not fitted to a production trace. | +| Per-scenario variant factor | 0.96 to 1.04 | multiplier | `make_scenario` | synthetic calibration parameter | Seeded variation prevents two variants from being byte-identical while keeping comparisons controlled. | +| Regional burst | API/search 1.85; media 1.55, steps 8–12 | multiplier for 5 periods (25 minutes) | regional-burst scenarios | synthetic calibration parameter | Creates a short overload that rewards headroom and timely scale-up. It is not an empirical burst distribution. | +| Recovery-migration load increase | 1.45, steps 7–14 | multiplier for 8 periods (40 minutes) | recovery-migration scenarios | synthetic calibration parameter | Makes post-failure restoration and later scale-down observable. | +| Node-failure duration | steps 8–13 | 6 periods (30 minutes) | node-failure-burst scenarios | synthetic calibration parameter | Long enough for a delayed replacement to become useful; not a production MTTR claim. | +| Recovery-migration failure duration | steps 6–10 | 5 periods (25 minutes) | recovery-migration scenarios | synthetic calibration parameter | Separates failure onset, recovery start, and workload normalization in a small episode. | +| Local RTT | 7, 8, 9 | milliseconds | latency model | synthetic calibration parameter | Gives a small local-path difference. Values are illustrative, not measured. | +| Cross-region RTT | 32, 35, 54 | milliseconds | latency model | synthetic calibration parameter | Creates near/far routing choices without modeling geography. Values are illustrative, not provider claims. | +| Cross-region capacity | 360 | megabytes/link/control period | link-capacity scaling | synthetic calibration parameter | Synthetic calibration assumption for the MVP. With a five-minute period this equals 1.2 MB/s (9.6 Mb/s), intentionally tight enough to expose routing trade-offs. | +| Link-degradation factor | 0.22, steps 9–15 | multiplier (79.2 MB/period) | link-degradation scenarios | synthetic calibration parameter | Creates a deterministic capacity incident on both directions of one region pair. | +| API response size | 0.025 | megabytes/request | link load and cross-region volume | synthetic calibration parameter | Synthetic calibration assumption for the MVP; only relative traffic size is intended. | +| Search response size | 0.04 | megabytes/request | link load and cross-region volume | synthetic calibration parameter | Synthetic calibration assumption for the MVP; only relative traffic size is intended. | +| Media response size | 0.18 | megabytes/request | link load and cross-region volume | synthetic calibration parameter | Makes media routing more bandwidth-sensitive. It is not a measured object-size distribution. | +| Scenario seeds | `4100 + 17*family_index + variant` | integer seed | scenario generation | engineering simplification | Fixed seeds make replay deterministic. All exogenous traces are generated before candidate execution. | +| Variants per family | 2; variant 0 exposes feedback, variant 1 is validation-only | scenarios | `SCENARIOS` and public artifacts | engineering simplification | Small MVP split supports review of generalization plumbing; ten scenarios are not enough to claim broad external validity. | + +## Latency, reliability, and cost semantics + +| Parameter | Current value/range | Unit | Used where | Type | Rationale/source | +| --- | ---: | --- | --- | --- | --- | +| Base service latency | API 12; search 20; media 28 | milliseconds | P95/P99 reduced-order model | synthetic calibration parameter | Synthetic calibration assumption for the MVP; represents service time before queue-pressure adjustment. | +| Queue-pressure function | `u^2 / max(0.08, 1-u)`; input utilization first capped at 0.995 for latency | dimensionless | P95/P99 calculation | synthetic calibration parameter | Smooth monotone penalty that becomes steep near saturation. It is a calibration curve, not a fitted queueing model. | +| Capacity utilization cap | incoming/capacity capped at 1.25; latency input capped at 0.995 | dimensionless | service scaling and latency | engineering simplification | Keeps overload state representable and prevents a singular/infinite latency value. | +| Tail multipliers | P95 0.85; P99 1.35 | dimensionless | latency model | synthetic calibration parameter | Separates P95 and P99 continuously. These are not empirical percentiles. | +| P99 SLO thresholds | API 120; search 180; media 250 | milliseconds | served-request SLO violation count | arbitrary placeholder requiring maintainer feedback | Provisional service-specific thresholds chosen to create differentiated tolerance. They are not contractual SLAs. | +| Unserved request semantics | requested rate not served by route, link, or replica capacity | requests/second aggregated by equal periods | availability and SLO violation | engineering simplification | Unserved demand counts as an SLO violation; the simulator never silently reroutes omitted/overflow traffic. | +| Availability | total served rate / total demand rate over equal-length periods | ratio | raw metric, reliability loss, recovery | engineering simplification | Equal period lengths make summing rates equivalent to request-weighted aggregation over the episode. | +| Recovery threshold | first post-recovery period with availability at least 0.99 | periods | `failure_recovery_steps` | arbitrary placeholder requiring maintainer feedback | A simple observable recovery definition. It does not model a multi-window reliability objective. | +| Compute price | 0.02 | abstract cost units/active-replica-period | compute cost | arbitrary placeholder requiring maintainer feedback | A relative cost coefficient only; it is not USD or a provider price. | +| Pending-replica price factor | 0.5 | multiplier | compute cost during cold start | arbitrary placeholder requiring maintainer feedback | Represents partial resource consumption before readiness. | +| Cross-region price | 0.08 | abstract cost units/gigabyte | cross-region cost | arbitrary placeholder requiring maintainer feedback | A relative coefficient only; it must not be described as a cloud-provider tariff. | +| Decimal GB conversion | 1024 MB/GB in current code | MB/GB conversion | cross-region cost | engineering simplification | The implementation uses 1024 for consistency with its current configuration; terminology should be clarified before merge. | + +## Validation and runtime guardrails + +| Parameter | Current value/range | Unit | Used where | Type | Rationale/source | +| --- | ---: | --- | --- | --- | --- | +| Route-sum tolerance | `1e-8` | fraction | hard route validation | engineering simplification | Allows harmless floating-point accumulation but rejects a sum more than `1 + 1e-8`. | +| Capacity tolerance | `1e-9` | CPU-capacity units | hard placement validation | engineering simplification | Avoids rejecting a mathematically equal sum due only to floating-point representation. | +| Maximum action items | 256 | placement plus route records/action | hard action validation | engineering simplification | Bounds parsing and per-step work well above the legitimate MVP action space. | +| Maximum response line | 64 KiB | bytes/RPC response | policy runtime | engineering simplification | Prevents unbounded candidate output; it is a reliability guardrail, not a security sandbox. | +| Retained stderr tail | 16 KiB | bytes | policy runtime diagnostics | engineering simplification | Keeps useful error context while bounding retained memory. The worker still consumes I/O until termination. | +| Startup timeout | 3.0 | seconds/scenario worker | evaluator/runtime | engineering simplification | Current CPU reliability budget; must be calibrated if dependencies or host assumptions change. | +| Decision timeout | 0.35 | seconds/call | evaluator/runtime | arbitrary placeholder requiring maintainer feedback | Current MVP limit chosen above baseline latency. It is not a repository-wide requirement. | +| Scenario total timeout | 12.0 | seconds/worker | evaluator/runtime | engineering simplification | Bounds candidate runtime over 24 decisions. The project-level personal goal remains an approximately 60-second full evaluation. | + +## Provisional scoring parameters + +| Parameter | Current value/range | Unit | Used where | Type | Rationale/source | +| --- | ---: | --- | --- | --- | --- | +| Reliability/SLA/tail/compute/bandwidth/recovery weights | 0.40 / 0.30 / 0.10 / 0.12 / 0.04 / 0.04 | share of normalized loss | `scenario_score` | arbitrary placeholder requiring maintainer feedback | Reliability plus SLA deliberately total 70%; exact weights remain provisional pending calibration/review. | +| Unserved-rate budget | 0.05 | ratio | reliability normalization | engineering simplification | Matches the benchmark's 5% availability-feedback threshold; not an external SLA. | +| SLO-violation budget | 0.10 | ratio | SLA normalization | arbitrary placeholder requiring maintainer feedback | Sets a scale for continuous loss; not an official SLA. | +| P99 latency reference | 180 | milliseconds | continuous tail-latency normalization | scenario-derived budget | Median of the three configured service P99 SLOs (120/180/250 ms); raw per-service SLO checks remain separate. | +| Compute budget | 6.0 | abstract cost units/episode | compute normalization | baseline-calibrated | Sized around current policies; no external economic interpretation. | +| Cross-region-volume budget | 1.0 | gigabytes/episode | bandwidth normalization | baseline-calibrated | Sized to make cross-region use visible after correct rate-to-volume conversion. | +| Recovery budget | 6.0 | periods | recovery normalization | scenario-derived budget | Matches the longest synthetic node-failure duration, not a real recovery target. | +| Per-component caps | 3.0 for reliability/SLA/compute/bandwidth; 2.0 for tail/recovery | normalized-loss units | `scenario_score` | engineering simplification | Bounds extreme loss while retaining observed compute/bandwidth differences; the earlier 2.0 compute/bandwidth cap was removed after an overprovision exploit test. | +| Score mapping | `100 * exp(-weighted_loss)` | score points | `scenario_score` | arbitrary placeholder requiring maintainer feedback | Monotone positive mapping. Raw metrics remain authoritative; formula is not frozen. | +| Cross-scenario aggregation | 75% mean + 25% P20 | score points | `evaluate` | arbitrary placeholder requiring maintainer feedback | Gives some weight to weak scenarios without making the single worst seed dominate. | + +## Dimensional consistency audit + +- Workload and service capacity are rates in requests/second. Fractions and capacity + scaling operate on rates. +- Cross-region link limits are volumes in megabytes/control period. A routed rate is + converted with `requests/second * MB/request * period_minutes * 60 seconds` before it + is compared with the link limit. Served cross-region volume uses the same conversion. +- A previous MVP implementation omitted the period-seconds multiplier. The implementation + and the analytical one-period oracle test now use the same dimensionally correct + conversion; the correction materially narrowed the baseline score gap and is disclosed + in the calibration report. +- Latency values are all milliseconds. The queue-pressure term and tail multipliers are + dimensionless. +- Compute and bandwidth prices are deliberately abstract cost units. They are never added + directly to milliseconds or ratios: each score component is normalized first. +- Because every simulated period has equal duration, episode availability and violation + rates can aggregate per-period request rates without changing the resulting ratio. + +## Maintainer decisions still needed + +1. Whether the five-minute control interval and one-period aggregate cold start are a + useful reduced-order abstraction. +2. Whether any workload, RTT, service-rate, response-size, failure-duration, or SLO value + needs a cited public source before formal submission. +3. Whether abstract cost units are acceptable or should be replaced by an explicitly + sourced economic model. +4. Whether the provisional score budgets, weights, and P20 aggregation provide appropriate + optimization pressure after stronger baseline calibration. diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/scoring-calibration-report.md b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/scoring-calibration-report.md new file mode 100644 index 00000000..e1744b89 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/scoring-calibration-report.md @@ -0,0 +1,165 @@ +# Scoring calibration report + +## Verdict + +The audit found two scoring defects and applied only the corresponding minimal fixes: + +1. below-reference P99 differences were clipped to zero, making the tail-latency component + inactive for every calibration policy; +2. compute and bandwidth losses stopped increasing at two budget units, which let a fixed + full-capacity policy consume 13.95 cost units while receiving no penalty beyond 12.0. + +No simulator, workload, scenario, metric, weight, or cross-scenario aggregation changed. +Raw metrics are therefore identical before and after; only normalized loss and score changed. + +Reproduce the complete policy and sensitivity matrix with: + +```text +python calibration/analyze_scoring.py --output scoring-analysis.json +``` + +## Scoring pipeline + +```text +per-period demand, served traffic, latency, replicas, bandwidth, recovery + -> scenario raw metrics + -> component normalization and cap + -> weighted scenario loss + -> scenario score = 100 * exp(-loss) + -> combined score = 0.75 * mean + 0.25 * P20 across 10 scenarios +``` + +| Component | Raw metric | Unit | Reference | Cap | Weight | Classification/rationale | +| --- | --- | --- | ---: | ---: | ---: | --- | +| reliability | unserved rate | ratio | 0.05 | 3 | 0.40 | benchmark operational target; same 5% threshold used for feedback | +| SLA | P99 SLO violation rate | ratio | 0.10 | 3 | 0.30 | synthetic calibration constant; service SLOs are checked before aggregation | +| tail latency | request-weighted P99 | ms | 180 | 2 | 0.10 | task-derived: median configured service P99 SLO (120/180/250 ms) | +| compute | active+pending replica cost | abstract cost/episode | 6.0 | 3 | 0.12 | baseline-calibrated synthetic budget | +| bandwidth | cross-region served volume | GB/episode | 1.0 | 3 | 0.04 | baseline-calibrated synthetic budget | +| recovery | periods until availability >= 0.99 | 5-minute periods | 6 | 2 | 0.04 | scenario-derived from the longest failure interval | + +All physical units are normalized before addition. No component can dominate merely because +it is measured in milliseconds, gigabytes, or cost units. + +## Tail-latency root cause + +### Before + +```text +excess = max(0, request_weighted_p99_ms / 180 ms - 1) +tail_component = min(2, excess / 0.50) +``` + +Observed P99 values were 28–66 ms for the relevant policies. Their raw P95/P99 values did +differ, the latency model responded to utilization/routing, and units were correct, but the +180 ms threshold clipped every value to exactly zero. This was a normalization/clipping bug, +not a congestion-model or unit bug. + +### Change + +```text +latency_reference_ms = median(service P99 SLOs) = 180 ms +tail_component = min(2, request_weighted_p99_ms / latency_reference_ms) +``` + +This preserves a small continuous incentive to improve latency while below an SLO. Actual +SLO breaches remain independently counted per service by the SLA component. + +### Per-scenario evidence after the fix + +Each policy cell is `P95/P99/tail_component`. Configured service P99 SLOs are API 120 ms, +search 180 ms, and media 250 ms. + +| Scenario | Weak | Reasonable | Strong | Fixed full | +| --- | --- | --- | --- | --- | +| normal_diurnal-v0 | 35.3/40.9/0.2270 | 35.3/40.9/0.2270 | 35.2/40.6/0.2257 | 27.3/28.1/0.1563 | +| normal_diurnal-v1 | 35.8/41.7/0.2316 | 35.8/41.7/0.2316 | 34.9/40.2/0.2231 | 27.4/28.2/0.1569 | +| regional_burst-v17 | 48.2/61.4/0.3409 | 48.5/61.8/0.3433 | 38.3/45.6/0.2531 | 27.8/28.9/0.1607 | +| regional_burst-v18 | 38.4/45.7/0.2540 | 43.2/53.2/0.2958 | 36.2/42.3/0.2352 | 27.5/28.4/0.1579 | +| node_failure_burst-v34 | 50.9/65.6/0.3647 | 41.0/49.5/0.2747 | 40.8/49.2/0.2734 | 43.4/53.8/0.2990 | +| node_failure_burst-v35 | 44.0/54.7/0.3038 | 40.6/49.2/0.2733 | 39.5/47.4/0.2634 | 38.0/45.2/0.2509 | +| link_degradation-v51 | 35.5/41.2/0.2291 | 35.5/41.2/0.2291 | 35.2/40.7/0.2262 | 27.3/28.2/0.1566 | +| link_degradation-v52 | 36.1/42.2/0.2343 | 36.1/42.2/0.2343 | 35.0/40.4/0.2244 | 27.4/28.3/0.1572 | +| recovery_migration-v68 | 36.9/43.4/0.2410 | 40.5/48.9/0.2714 | 36.9/43.2/0.2398 | 29.7/31.9/0.1774 | +| recovery_migration-v69 | 35.5/41.2/0.2289 | 38.6/46.0/0.2554 | 37.6/44.4/0.2468 | 28.1/29.4/0.1634 | + +## Compute/bandwidth cap root cause + +### Before + +Compute and bandwidth used a cap of 2.0. `fixed_full_capacity` and +`sla_first_overprovision` both cost 13.95 units, or 2.325 times the six-unit budget, but +were charged only 2.0. SLA-first consequently became the highest-scoring policy despite +occupying almost all node capacity throughout the episode. + +### Change + +The compute and bandwidth caps now use the evaluator's existing general cap of 3.0. This +keeps extreme loss bounded while preserving feedback through three budget units. Tail and +recovery retain cap 2.0 because no tested policy approaches those caps. + +## Final four-policy component table + +Values are mean per-scenario normalized component followed by weighted contribution. + +| Policy | Reliability | SLA | Tail | Compute | Bandwidth | Recovery | Final score | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| weak static | 0.1789 / 0.0716 | 0.3416 / 0.1025 | 0.2655 / 0.0266 | 0.7180 / 0.0862 | 0 / 0 | 0.0667 / 0.0027 | 74.0766 | +| reasonable | 0.1105 / 0.0442 | 0.2947 / 0.0884 | 0.2636 / 0.0264 | 0.7267 / 0.0872 | 0.9634 / 0.0385 | 0.0333 / 0.0013 | 73.9434 | +| strong | 0.1210 / 0.0484 | 0.1842 / 0.0553 | 0.2411 / 0.0241 | 0.7530 / 0.0904 | 1.0824 / 0.0433 | 0.0500 / 0.0020 | 74.4628 | +| fixed full | 0.0343 / 0.0137 | 0.1880 / 0.0564 | 0.1836 / 0.0184 | 2.3250 / 0.2790 | 0 / 0 | 0 / 0 | 70.2510 | + +Normalization is applied per scenario before aggregation. Therefore `mean(raw)/reference` +can differ from the mean normalized component when individual scenarios hit a cap. + +## Extreme-policy results + +| Policy | Score | Availability | P99 ms | SLA violation | Compute | Cross-region GB | Recovery | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| strong | 74.4628 | 0.993951 | 43.401 | 0.018423 | 4.518 | 1.588 | 0.3 | +| weak static | 74.0766 | 0.991055 | 47.795 | 0.034160 | 4.308 | 0 | 0.4 | +| reasonable | 73.9434 | 0.994475 | 47.446 | 0.029475 | 4.360 | 1.168 | 0.2 | +| SLA-first overprovision | 73.1402 | 0.999708 | 29.998 | 0.005975 | 13.950 | 0 | 0 | +| fixed full capacity | 70.2510 | 0.998283 | 33.053 | 0.018799 | 13.950 | 0 | 0 | +| local routing only | 70.0369 | 0.981444 | 44.327 | 0.028112 | 4.319 | 0 | 0.4 | +| ignore failure | 62.1667 | 0.972494 | 46.029 | 0.046809 | 4.248 | 0 | 0.4 | +| zero | 11.7809 | 0 | 0 | 1.0 | 0 | 0 | 4.6 | +| cost-first underprovision | 11.1039 | 0.371147 | 52.538 | 0.638148 | 1.436 | 0 | 4.6 | +| minimum replica | 9.5307 | 0.483400 | 113.116 | 0.617490 | 1.436 | 16.401 | 4.6 | +| aggressive cross-region | 7.6351 | 0.172783 | 65.637 | 0.827217 | 13.950 | 24.929 | 4.6 | + +No obvious extreme policy scores anomalously high after the cap fix. A zero-service policy +has a zero latency average because no requests are served, but reliability and SLA loss keep +its total score low; this does not provide an exploit. + +## Weight sensitivity + +Every major weight was independently tested at nominal -5%, -2%, nominal, +2%, and +5%. +The selected weight changes relatively; all other weights scale proportionally back to a +sum of one. + +- `strong` remains the top policy in every relative perturbation. +- The ordering among strong, weak, reasonable, and SLA-first remains stable. +- Only two close low-ranked policies (`fixed_full_capacity` 70.2510 and + `local_routing_only` 70.0369) swap under small compute/reliability changes. This is a + legitimate engineering trade-off: one spends much more compute for availability, while + the other accepts failures and uses no cross-region bandwidth. +- No relative perturbation produces a large score jump or elevates an obviously broken + policy. + +The separate absolute transfer `SLA -0.02, bandwidth +0.02` still changes the ordering of +weak/reasonable/strong. That is not numerical instability: it increases bandwidth's weight +by 50% and explicitly prefers the zero-cross-region weak policy. It should be described as +an engineering-preference change, not a two-percent relative perturbation. + +## Before / change / after summary + +| Item | Before | Root cause | Change | After | +| --- | --- | --- | --- | --- | +| tail signal | all calibration policies exactly 0 | below-180 values clipped | normalize P99 continuously by median configured SLO | nonzero 0.18–0.27 mean components; raw metrics unchanged | +| compute clipping | 13.95 cost normalized to 2.0 | cap below observed extreme | cap 3.0 | 13.95 normalizes to 2.325; full policies no longer win | +| bandwidth clipping | traffic above 2 GB/scenario free after cap | cap compressed remote-routing differences | cap 3.0 | more continuous penalty; aggressive remote policy remains low | +| calibration ranking | weak 76.1354 < reasonable 76.4882 < strong 77.4852 | old normalization | no ranking-targeted tuning | strong 74.4628 > weak 74.0766 > reasonable 73.9434, reflecting explicit trade-offs | + +The final ranking is not encoded as a required test. Tests require valid, deterministic, +distinct calibration policies rather than forcing a subjective preference ordering. diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/tiny_oracle.md b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/tiny_oracle.md new file mode 100644 index 00000000..33c5c7e8 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/tiny_oracle.md @@ -0,0 +1,123 @@ +# Multi-timestep tiny oracle + +## Purpose + +The tiny oracle checks the complete production state transition and score path on a case +small enough to enumerate. It is a validation fixture, not a new benchmark scenario and +not a baseline target. + +It covers: + +- desired placement; +- one-period cold start; +- pending-to-active transition; +- routing to active retained replicas; +- replica service capacity; +- latency and compute-cost accumulation; +- three timestep transitions; +- raw episode metrics, normalized loss components, and final score. + +## Finite oracle case + +- one service (`svc`); +- two one-replica nodes (`n1`, `n2`) in one region; +- three periods with demand rates `1, 3, 3` requests/second; +- `n1` initially active and `n2` absent; +- one replica per node maximum; +- route fractions on the declared finite grid `{0.0, 0.5, 1.0}`; +- no failures and no cross-region traffic. + +“All feasible trajectories” means every hard-valid action sequence on this explicitly +declared finite grid. It does not mean every real-valued route fraction. + +## Independence boundary + +`verification/tiny_oracle.py` enumerates the trajectories and independently transcribes +the tiny state transition, metrics, and current scoring equation. It does not call +`EdgeServiceSimulator` during enumeration or metric calculation. + +After choosing the best trajectory, the test passes exactly that action sequence through +the production `EdgeServiceSimulator` and production `scenario_score` via +`evaluate_action_sequence`. The test uses exact dictionary and floating-point equality; +there is no relaxed tolerance hiding a discrepancy. + +## Executed result + +Command: + +```text +python verification/tiny_oracle.py +``` + +Enumerated trajectories: `723` + +Best action sequence: + +```json +[ + { + "replicas": [ + {"service_id": "svc", "node_id": "n1", "count": 1}, + {"service_id": "svc", "node_id": "n2", "count": 1} + ], + "routes": [ + {"service_id": "svc", "source_region": "tiny", "node_id": "n1", "fraction": 1.0} + ] + }, + { + "replicas": [ + {"service_id": "svc", "node_id": "n1", "count": 1}, + {"service_id": "svc", "node_id": "n2", "count": 1} + ], + "routes": [ + {"service_id": "svc", "source_region": "tiny", "node_id": "n1", "fraction": 0.5}, + {"service_id": "svc", "source_region": "tiny", "node_id": "n2", "fraction": 0.5} + ] + }, + { + "replicas": [ + {"service_id": "svc", "node_id": "n1", "count": 1}, + {"service_id": "svc", "node_id": "n2", "count": 1} + ], + "routes": [ + {"service_id": "svc", "source_region": "tiny", "node_id": "n1", "fraction": 0.5}, + {"service_id": "svc", "source_region": "tiny", "node_id": "n2", "fraction": 0.5} + ] + } +] +``` + +Oracle raw metrics: + +| Metric | Value | +| --- | ---: | +| request availability | 1.0 | +| unserved rate | 0.0 | +| request-weighted P95 | 32.0 ms | +| request-weighted P99 | 42.0 ms | +| P99 SLO violation rate | 0.0 | +| compute cost | 0.55 abstract cost units | +| cross-region traffic/cost | 0.0 | +| failure recovery steps | 0.0 | + +Normalized components are zero except tail latency `42/180 = 0.23333333333333334` and +compute `0.55/6 = 0.09166666666666667`. + +Oracle combined score: `96.62493678284117` + +Production evaluator result: the raw metrics, all six normalized loss components, and +combined score were exactly equal. The covering test is +`test_bruteforce_oracle_matches_production_simulator_and_score_exactly`. + +## Additional one-period dimensional check + +The simulator suite also checks an analytical local-plus-cross-region routing case. With +150 requests/second split equally, local service can serve 75 requests/second. The remote +path is limited to: + +```text +360 MB / (0.025 MB/request * 300 seconds) = 48 requests/second +``` + +The correct combined upper bound is therefore `75 + 48 = 123 requests/second`. This test +would fail if the simulator again compared a rate directly with a per-period volume. diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/agent_files.txt b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/agent_files.txt new file mode 100644 index 00000000..e8a78865 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/agent_files.txt @@ -0,0 +1,7 @@ +README.md +README_zh-CN.md +Task.md +Task_zh-CN.md +scripts/init.py +references/design_notes.md +frontier_eval/constraints.txt diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/artifact_files.txt b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/artifact_files.txt new file mode 100644 index 00000000..76dc893a --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/artifact_files.txt @@ -0,0 +1,2 @@ +metrics.json +artifacts.json diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/candidate_destination.txt b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/candidate_destination.txt new file mode 100644 index 00000000..b9411b3d --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/candidate_destination.txt @@ -0,0 +1 @@ +scripts/init.py diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/constraints.txt b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/constraints.txt new file mode 100644 index 00000000..da5fdfef --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/constraints.txt @@ -0,0 +1,9 @@ +Edge service replica placement constraints: +1) Edit only `scripts/init.py` inside the EVOLVE-BLOCK markers. +2) Keep `decide(observation) -> dict` working; `reset_policy()` is optional. +3) Return exactly `replicas` and `routes`; use only IDs present in the observation. +4) Replica counts must be non-negative integers (booleans are not integers) and placements must respect live-node CPU capacity. +5) Route fractions must be finite numbers in [0, 1], sum to at most 1 per service/source region, and target active replicas retained by the current placement. +6) Keep the policy deterministic and use only current or historical observable state; future trace events are unavailable. +7) The worker allows 0.35 seconds per decision, 12 seconds per scenario, and at most 64 KiB per JSON response. +8) Do not read, write, or modify evaluator, simulator, reference, result, or environment-secret files. diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/copy_files.txt b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/copy_files.txt new file mode 100644 index 00000000..a056b212 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/copy_files.txt @@ -0,0 +1,13 @@ +README.md +README_zh-CN.md +Task.md +Task_zh-CN.md +scripts/init.py +verification/evaluator.py +verification/simulator.py +verification/policy_runtime.py +verification/policy_worker.py +verification/requirements.txt +references/config.json +references/design_notes.md +frontier_eval/run_eval.sh diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/eval_command.txt b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/eval_command.txt new file mode 100644 index 00000000..6682234f --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/eval_command.txt @@ -0,0 +1 @@ +bash frontier_eval/run_eval.sh {python} {benchmark} {candidate} diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/initial_program.txt b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/initial_program.txt new file mode 100644 index 00000000..b9411b3d --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/initial_program.txt @@ -0,0 +1 @@ +scripts/init.py diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/readonly_files.txt b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/readonly_files.txt new file mode 100644 index 00000000..2a13cdb1 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/readonly_files.txt @@ -0,0 +1,22 @@ +README.md +README_zh-CN.md +Task.md +Task_zh-CN.md +verification/evaluator.py +verification/simulator.py +verification/policy_runtime.py +verification/policy_worker.py +verification/test_simulator.py +verification/test_evaluator.py +verification/test_policy_runtime.py +verification/tiny_oracle.py +verification/test_tiny_oracle.py +verification/requirements.txt +references/config.json +references/design_notes.md +docs/parameter_assumptions.md +docs/tiny_oracle.md +docs/evaluator-threat-model.md +docs/scoring-calibration-report.md +frontier_eval/run_eval.sh +scripts/init.py diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/run_eval.sh b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/run_eval.sh new file mode 100644 index 00000000..9e5eaf2a --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/frontier_eval/run_eval.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +PYTHON_CMD="${1:?missing python command}" +BENCHMARK_DIR="${2:?missing benchmark directory}" +CANDIDATE_PATH="${3:?missing candidate path}" + +"${PYTHON_CMD}" "${BENCHMARK_DIR}/verification/evaluator.py" "${CANDIDATE_PATH}" \ + --metrics-out "${BENCHMARK_DIR}/metrics.json" \ + --artifacts-out "${BENCHMARK_DIR}/artifacts.json" diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/references/config.json b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/references/config.json new file mode 100644 index 00000000..e23a89b1 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/references/config.json @@ -0,0 +1,34 @@ +{ + "model_version": "mvp-v0.1", + "period_minutes": 5, + "regions": ["edge-a", "edge-b", "edge-c"], + "nodes": [ + {"id": "a-1", "region": "edge-a", "failure_domain": "a-rack-1", "cpu_capacity": 10.0}, + {"id": "a-2", "region": "edge-a", "failure_domain": "a-rack-2", "cpu_capacity": 10.0}, + {"id": "b-1", "region": "edge-b", "failure_domain": "b-rack-1", "cpu_capacity": 10.0}, + {"id": "b-2", "region": "edge-b", "failure_domain": "b-rack-2", "cpu_capacity": 10.0}, + {"id": "c-1", "region": "edge-c", "failure_domain": "c-rack-1", "cpu_capacity": 10.0}, + {"id": "c-2", "region": "edge-c", "failure_domain": "c-rack-2", "cpu_capacity": 10.0} + ], + "services": [ + {"id": "api", "cpu_per_replica": 2.0, "service_rate_rps": 90.0, "base_latency_ms": 12.0, "p99_slo_ms": 120.0, "response_mb": 0.025, "reliability_class": "critical"}, + {"id": "search", "cpu_per_replica": 2.5, "service_rate_rps": 65.0, "base_latency_ms": 20.0, "p99_slo_ms": 180.0, "response_mb": 0.04, "reliability_class": "critical"}, + {"id": "media", "cpu_per_replica": 1.5, "service_rate_rps": 50.0, "base_latency_ms": 28.0, "p99_slo_ms": 250.0, "response_mb": 0.18, "reliability_class": "standard"} + ], + "network_rtt_ms": { + "edge-a": {"edge-a": 7.0, "edge-b": 32.0, "edge-c": 54.0}, + "edge-b": {"edge-a": 32.0, "edge-b": 8.0, "edge-c": 35.0}, + "edge-c": {"edge-a": 54.0, "edge-b": 35.0, "edge-c": 9.0} + }, + "cross_region_bandwidth_mb_per_period": 360.0, + "compute_cost_per_replica_period": 0.02, + "pending_replica_cost_factor": 0.5, + "cross_region_cost_per_gb": 0.08, + "score_budgets": { + "unserved_rate": 0.05, + "slo_violation_rate": 0.10, + "compute_cost": 6.0, + "cross_region_gb": 1.0, + "recovery_steps": 6.0 + } +} diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/references/design_notes.md b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/references/design_notes.md new file mode 100644 index 00000000..609fec69 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/references/design_notes.md @@ -0,0 +1,24 @@ +# Model scope and assumptions + +This benchmark is a deterministic reduced-order control simulator, not a claim to +reproduce Kubernetes, a production data center, or a packet-level network. + +- One period represents five minutes; the candidate acts for 24 periods. +- A desired scale-up becomes active one period later, representing aggregate cold-start + and rollout delay. Scale-down is immediate. +- Each service replica has a fixed CPU footprint and nominal service rate. Queue pressure + grows nonlinearly with utilization and affects reduced-order P95/P99 estimates. +- A route consumes response-size-weighted cross-region capacity. When a link or service + is oversubscribed, requests are proportionally unserved rather than silently rerouted. +- Failed nodes immediately lose active and pending replicas. The candidate sees current + failures but not future events. +- All workload and failure traces are synthesized before candidate execution from fixed + seeds. Candidate behavior cannot alter the exogenous sequence. + +The numerical values in `config.json` are calibration assumptions, not measured production +parameters. Their provenance and limitations are listed in +`docs/parameter_assumptions.md`. + +Raw metrics are always emitted so score behavior remains auditable. Reliability plus SLA +loss carry 70% of the weight; latency, compute, bandwidth, and recovery account for the +remainder. See `docs/scoring-calibration-report.md` for normalization and exploit checks. diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/scripts/init.py b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/scripts/init.py new file mode 100644 index 00000000..1012d1fc --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/scripts/init.py @@ -0,0 +1,149 @@ +"""Reasonable CPU-only baseline for EdgeServiceReplicaPlacement.""" + +from __future__ import annotations + +import math +from typing import Any + + +# EVOLVE-BLOCK-START +def reset_policy() -> None: + """The baseline is stateless; the hook keeps scenario isolation explicit.""" + + +def decide(observation: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + """Choose desired replicas and current-period routes from observable state only.""" + + nodes = {node["id"]: node for node in observation["nodes"]} + services = {service["id"]: service for service in observation["services"]} + alive = [node_id for node_id, node in nodes.items() if node["alive"]] + cpu_used = {node_id: 0.0 for node_id in alive} + placement: dict[tuple[str, str], int] = {} + + # Critical services are allocated first. Each regional demand gets modest headroom; + # capacity spillover is placed on the nearest surviving region. + service_order = sorted( + services, + key=lambda service_id: services[service_id]["reliability_class"] != "critical", + ) + for service_id in service_order: + service = services[service_id] + cpu = float(service["cpu_per_replica"]) + rate = float(service["service_rate_rps"]) + for source_region, demand_by_service in observation["workload_rps"].items(): + needed = max(1, math.ceil(1.20 * float(demand_by_service[service_id]) / rate)) + candidates = sorted( + alive, + key=lambda node_id: ( + nodes[node_id]["region"] != source_region, + observation["network_rtt_ms"][source_region][nodes[node_id]["region"]], + cpu_used[node_id], + node_id, + ), + ) + for replica_index in range(needed): + feasible = [ + node_id + for node_id in candidates + if cpu_used[node_id] + cpu <= float(nodes[node_id]["cpu_capacity"]) + ] + if not feasible: + break + # Alternate failure domains when equivalent capacity is available. + node_id = min( + feasible, + key=lambda candidate: ( + nodes[candidate]["region"] != source_region, + observation["network_rtt_ms"][source_region][nodes[candidate]["region"]], + placement.get((service_id, candidate), 0), + cpu_used[candidate], + candidate, + ), + ) + placement[(service_id, node_id)] = placement.get((service_id, node_id), 0) + 1 + cpu_used[node_id] += cpu + + replicas = [ + {"service_id": service_id, "node_id": node_id, "count": count} + for (service_id, node_id), count in sorted(placement.items()) + if count > 0 + ] + active = { + (item["service_id"], item["node_id"]): int(item["count"]) + for item in observation["active_replicas"] + } + routes: list[dict[str, Any]] = [] + remaining_capacity = { + (service_id, node_id): active.get((service_id, node_id), 0) + * float(services[service_id]["service_rate_rps"]) + for service_id in services + for node_id in alive + if active.get((service_id, node_id), 0) > 0 + and placement.get((service_id, node_id), 0) > 0 + } + unmet: list[tuple[str, str, float, float]] = [] + for source_region, demand_by_service in observation["workload_rps"].items(): + for service_id, demand_value in demand_by_service.items(): + demand = max(float(demand_value), 1e-9) + remaining_demand = demand + targets = sorted( + node_id + for node_id in alive + if nodes[node_id]["region"] == source_region + and remaining_capacity.get((service_id, node_id), 0.0) > 0.0 + ) + for node_id in targets: + amount = min(remaining_demand, remaining_capacity[(service_id, node_id)]) + if amount > 1e-12: + routes.append( + { + "service_id": service_id, + "source_region": source_region, + "node_id": node_id, + "fraction": amount / demand, + } + ) + remaining_capacity[(service_id, node_id)] -= amount + remaining_demand -= amount + if remaining_demand <= 1e-12: + break + if remaining_demand > 1e-12: + unmet.append((source_region, service_id, demand, remaining_demand)) + + link_remaining = dict(observation["cross_region_bandwidth_mb_per_period"]) + period_seconds = float(observation["period_minutes"]) * 60.0 + for source_region, service_id, demand, remaining_demand in unmet: + response_mb = float(services[service_id]["response_mb"]) + targets = sorted( + ( + node_id + for node_id in alive + if nodes[node_id]["region"] != source_region + and remaining_capacity.get((service_id, node_id), 0.0) > 0.0 + ), + key=lambda node_id: ( + observation["network_rtt_ms"][source_region][nodes[node_id]["region"]], + node_id, + ), + ) + for node_id in targets: + target_region = nodes[node_id]["region"] + link = f"{source_region}->{target_region}" + by_link = link_remaining.get(link, 0.0) / max(response_mb * period_seconds, 1e-12) + amount = min(remaining_demand, remaining_capacity[(service_id, node_id)], by_link) + if amount > 1e-12: + routes.append( + { + "service_id": service_id, + "source_region": source_region, + "node_id": node_id, + "fraction": amount / demand, + } + ) + remaining_capacity[(service_id, node_id)] -= amount + link_remaining[link] -= amount * response_mb * period_seconds + remaining_demand -= amount + if remaining_demand <= 1e-12: + break + return {"replicas": replicas, "routes": routes} +# EVOLVE-BLOCK-END diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/evaluator.py b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/evaluator.py new file mode 100644 index 00000000..1b30dcce --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/evaluator.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from statistics import fmean, median +from typing import Any + +try: + from .policy_runtime import PolicyRuntime + from .simulator import SCENARIOS, EdgeServiceSimulator, load_config +except ImportError: + from policy_runtime import PolicyRuntime + from simulator import SCENARIOS, EdgeServiceSimulator, load_config + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_CANDIDATE = ROOT / "scripts" / "init.py" +POLICY_STARTUP_TIMEOUT_S = 3.0 +POLICY_CALL_TIMEOUT_S = 0.35 +POLICY_SCENARIO_TIMEOUT_S = 12.0 + + +def _run_candidate(candidate_path: Path, scenario: Any) -> dict[str, Any]: + with PolicyRuntime( + candidate_path, + startup_timeout_s=POLICY_STARTUP_TIMEOUT_S, + call_timeout_s=POLICY_CALL_TIMEOUT_S, + total_timeout_s=POLICY_SCENARIO_TIMEOUT_S, + ) as policy: + policy.reset_policy() + simulator = EdgeServiceSimulator(scenario) + simulator._activate_pending_and_apply_failures() + while not simulator.done: + simulator.step(policy.decide(simulator.observation())) + return simulator.metrics() + + +def _bounded_ratio(value: float, budget: float, maximum: float = 3.0) -> float: + return min(maximum, max(0.0, value) / max(budget, 1e-12)) + + +def scenario_score(metrics: dict[str, Any]) -> tuple[float, dict[str, float]]: + """Calibratable engineering loss; raw metrics remain the primary explanation.""" + + budgets = load_config()["score_budgets"] + # Normalize latency against a task-defined reference instead of clipping every + # below-SLO result to zero. The median service P99 SLO is 180 ms in the MVP. + latency_reference_ms = median( + float(service["p99_slo_ms"]) for service in load_config()["services"] + ) + components = { + "reliability": _bounded_ratio(float(metrics["unserved_rate"]), float(budgets["unserved_rate"])), + "sla": _bounded_ratio(float(metrics["p99_slo_violation_rate"]), float(budgets["slo_violation_rate"])), + "tail_latency": _bounded_ratio( + float(metrics["request_weighted_p99_ms"]), latency_reference_ms, 2.0 + ), + "compute": _bounded_ratio(float(metrics["compute_cost"]), float(budgets["compute_cost"])), + "bandwidth": _bounded_ratio(float(metrics["cross_region_gb"]), float(budgets["cross_region_gb"])), + "recovery": _bounded_ratio(float(metrics["failure_recovery_steps"]), float(budgets["recovery_steps"]), 2.0), + } + loss = ( + 0.40 * components["reliability"] + + 0.30 * components["sla"] + + 0.10 * components["tail_latency"] + + 0.12 * components["compute"] + + 0.04 * components["bandwidth"] + + 0.04 * components["recovery"] + ) + return 100.0 * math.exp(-loss), components + + +def evaluate_action_sequence( + actions: list[dict[str, Any]], scenario: Any, config: dict[str, Any] +) -> dict[str, Any]: + """Evaluate a fixed tiny action trajectory through the production state machine. + + This test hook deliberately reuses the real simulator and scoring function. The + independent oracle in ``tiny_oracle.py`` does not call this function when searching. + """ + + if len(actions) != len(scenario.workloads): + raise ValueError("action count must match scenario length") + simulator = EdgeServiceSimulator(scenario, config=config) + simulator._activate_pending_and_apply_failures() + for action in actions: + simulator.step(action) + raw_metrics = simulator.metrics() + score, components = scenario_score(raw_metrics) + return { + "raw_metrics": raw_metrics, + "combined_score": score, + "normalized_loss_components": components, + } + + +def _quantile(values: list[float], q: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + position = (len(ordered) - 1) * q + lower = int(math.floor(position)) + upper = int(math.ceil(position)) + if lower == upper: + return ordered[lower] + return ordered[lower] * (upper - position) + ordered[upper] * (position - lower) + + +def evaluate(candidate_path: Path) -> dict[str, Any]: + candidate_path = candidate_path.expanduser().resolve() + rows: list[dict[str, Any]] = [] + for scenario_index, scenario in enumerate(SCENARIOS): + try: + raw = _run_candidate(candidate_path, scenario) + score, components = scenario_score(raw) + rows.append( + { + "scenario": scenario.name, + "family": scenario.family, + "seed": scenario.seed, + "feedback": scenario.feedback, + "score": score, + "raw_metrics": raw, + "normalized_loss_components": components, + } + ) + except Exception as exc: + rows.append( + { + "scenario": scenario.name, + "family": scenario.family, + "seed": scenario.seed, + "feedback": scenario.feedback, + "score": 0.0, + "error": f"{type(exc).__name__}: {exc}", + } + ) + for remaining in SCENARIOS[scenario_index + 1 :]: + rows.append( + { + "scenario": remaining.name, + "family": remaining.family, + "seed": remaining.seed, + "feedback": remaining.feedback, + "score": 0.0, + "error": "not evaluated after prior candidate failure", + } + ) + break + + scores = [float(row["score"]) for row in rows] + diagnostic_score = 0.75 * fmean(scores) + 0.25 * _quantile(scores, 0.20) if scores else 0.0 + valid = all("error" not in row for row in rows) + successful = [row["raw_metrics"] for row in rows if "raw_metrics" in row] + + def mean_metric(name: str) -> float: + return fmean(float(metrics[name]) for metrics in successful) if successful else 0.0 + + return { + "combined_score": diagnostic_score if valid else 0.0, + "diagnostic_score": diagnostic_score, + "valid": float(valid), + "evaluated_scenarios": float(len(successful)), + "mean_request_availability": mean_metric("request_availability"), + "mean_request_weighted_p95_ms": mean_metric("request_weighted_p95_ms"), + "mean_request_weighted_p99_ms": mean_metric("request_weighted_p99_ms"), + "mean_p99_slo_violation_rate": mean_metric("p99_slo_violation_rate"), + "mean_compute_cost": mean_metric("compute_cost"), + "mean_cross_region_gb": mean_metric("cross_region_gb"), + "mean_failure_recovery_steps": mean_metric("failure_recovery_steps"), + "rows": rows, + } + + +def _write_json(path: str | None, payload: dict[str, Any]) -> None: + if not path: + return + output = Path(path).expanduser().resolve() + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(payload, indent=2, ensure_ascii=False, allow_nan=False), encoding="utf-8") + + +def _public_artifacts(result: dict[str, Any], candidate_label: str) -> dict[str, Any]: + feedback_rows = [row for row in result["rows"] if bool(row.get("feedback"))] + validation_rows = [row for row in result["rows"] if not bool(row.get("feedback"))] + return { + "candidate_path": candidate_label, + "feedback_rows": feedback_rows, + "validation_summary": { + "num_scenarios": len(validation_rows), + "mean_score": fmean(float(row["score"]) for row in validation_rows) if validation_rows else 0.0, + "failed_scenarios": sum("error" in row for row in validation_rows), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Evaluate an edge replica-placement and routing policy") + parser.add_argument("candidate", nargs="?", default=str(DEFAULT_CANDIDATE)) + parser.add_argument("--metrics-out", default=None) + parser.add_argument("--artifacts-out", default=None) + args = parser.parse_args() + candidate_path = Path(args.candidate).expanduser().resolve() + result = evaluate(candidate_path) + print("=== Edge Service Replica Placement MVP ===") + for row in result["rows"]: + if not row["feedback"]: + continue + if "error" in row: + print(f"scenario={row['scenario']} score=0.00 error={row['error']}") + else: + raw = row["raw_metrics"] + print( + f"scenario={row['scenario']} score={row['score']:.2f} " + f"availability={raw['request_availability']:.4f} " + f"p99_ms={raw['request_weighted_p99_ms']:.1f} " + f"sla_violation={raw['p99_slo_violation_rate']:.4f} " + f"compute_cost={raw['compute_cost']:.3f} cross_region_gb={raw['cross_region_gb']:.3f}" + ) + print("---") + print(f"valid: {bool(result['valid'])}") + print(f"diagnostic_score: {result['diagnostic_score']:.4f}") + print(f"combined_score: {result['combined_score']:.4f}") + + metrics = {key: value for key, value in result.items() if key != "rows"} + try: + candidate_label = candidate_path.relative_to(ROOT).as_posix() + except ValueError: + candidate_label = candidate_path.name + _write_json(args.metrics_out, metrics) + _write_json(args.artifacts_out, _public_artifacts(result, candidate_label)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/policy_runtime.py b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/policy_runtime.py new file mode 100644 index 00000000..de6efc55 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/policy_runtime.py @@ -0,0 +1,247 @@ +"""Cross-platform parent-side runtime for isolated edge-control candidates.""" + +from __future__ import annotations + +from collections import deque +import json +import os +from pathlib import Path +import queue +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +from typing import Any, Mapping + + +MAX_RESPONSE_BYTES = 64 * 1024 +_EOF = object() + + +class PolicyRuntimeError(RuntimeError): + pass + + +class PolicyTimeoutError(PolicyRuntimeError): + pass + + +class PolicyProtocolError(PolicyRuntimeError): + pass + + +class PolicyCandidateError(PolicyRuntimeError): + pass + + +def _clean_environment() -> dict[str, str]: + allowed = { + "COMSPEC", "LANG", "LC_ALL", "NUMBER_OF_PROCESSORS", "PATH", "PATHEXT", + "SYSTEMDRIVE", "SYSTEMROOT", "TEMP", "TMP", "WINDIR", + } + env = {key: value for key, value in os.environ.items() if key.upper() in allowed} + env["PYTHONUTF8"] = "1" + env["PYTHONIOENCODING"] = "utf-8" + return env + + +class PolicyRuntime: + """Persistent, bounded JSON RPC process for one scenario.""" + + def __init__( + self, + candidate_path: str | Path, + *, + startup_timeout_s: float = 3.0, + call_timeout_s: float = 0.35, + total_timeout_s: float = 12.0, + ) -> None: + if min(startup_timeout_s, call_timeout_s, total_timeout_s) <= 0: + raise ValueError("timeouts must be positive") + source = Path(candidate_path).expanduser().resolve() + if not source.is_file(): + raise FileNotFoundError(source) + self.call_timeout_s = float(call_timeout_s) + self._deadline = time.monotonic() + float(total_timeout_s) + self._responses: queue.Queue[Any] = queue.Queue() + self._stderr: deque[bytes] = deque() + self._stderr_size = 0 + self._next_id = 1 + self._closed = False + self._tempdir = Path(tempfile.mkdtemp(prefix="edge_policy_")) + self._process: subprocess.Popen[bytes] | None = None + try: + copied_candidate = self._tempdir / "candidate.py" + shutil.copy2(source, copied_candidate) + worker = Path(__file__).with_name("policy_worker.py").resolve() + kwargs: dict[str, Any] = { + "cwd": str(self._tempdir), + "env": _clean_environment(), + "stdin": subprocess.PIPE, + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "bufsize": 0, + } + if os.name == "nt": + kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP + else: + kwargs["start_new_session"] = True + self._process = subprocess.Popen([sys.executable, "-I", "-u", str(worker), str(copied_candidate)], **kwargs) + assert self._process.stdout is not None and self._process.stderr is not None + threading.Thread(target=self._read_stdout, args=(self._process.stdout,), daemon=True).start() + threading.Thread(target=self._drain_stderr, args=(self._process.stderr,), daemon=True).start() + ready = self._receive(startup_timeout_s, "candidate import") + if not isinstance(ready, dict) or not ready.get("ok") or not ready.get("ready"): + self._raise_response_error(ready, "candidate import") + except BaseException: + self.close(force=True) + raise + + @property + def stderr_tail(self) -> str: + return b"".join(self._stderr).decode("utf-8", errors="replace") + + def _read_stdout(self, stream: Any) -> None: + try: + while True: + line = stream.readline(MAX_RESPONSE_BYTES + 1) + if not line: + break + self._responses.put(line) + if len(line) > MAX_RESPONSE_BYTES or not line.endswith(b"\n"): + break + finally: + self._responses.put(_EOF) + + def _drain_stderr(self, stream: Any) -> None: + while True: + chunk = stream.read(4096) + if not chunk: + return + self._stderr.append(chunk) + self._stderr_size += len(chunk) + while self._stderr_size > 16 * 1024 and self._stderr: + self._stderr_size -= len(self._stderr.popleft()) + + def _receive(self, timeout_s: float, operation: str) -> dict[str, Any]: + timeout = min(float(timeout_s), self._deadline - time.monotonic()) + if timeout <= 0: + self.close(force=True) + raise PolicyTimeoutError("candidate total runtime budget exceeded") + try: + item = self._responses.get(timeout=timeout) + except queue.Empty as exc: + self.close(force=True) + raise PolicyTimeoutError(f"{operation} timed out after {timeout:.3f}s") from exc + if item is _EOF: + code = None if self._process is None else self._process.poll() + detail = self.stderr_tail[-1000:] + raise PolicyCandidateError(f"candidate exited unexpectedly (code={code})" + (f": {detail}" if detail else "")) + if not isinstance(item, bytes) or len(item) > MAX_RESPONSE_BYTES or not item.endswith(b"\n"): + self.close(force=True) + raise PolicyProtocolError("candidate response exceeds 64 KiB or lacks newline") + try: + response = json.loads(item.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + self.close(force=True) + raise PolicyProtocolError("candidate returned invalid UTF-8 JSON") from exc + if not isinstance(response, dict): + raise PolicyProtocolError("candidate response must be a JSON object") + return response + + @staticmethod + def _raise_response_error(response: Any, operation: str) -> None: + if isinstance(response, dict) and isinstance(response.get("error"), dict): + error = response["error"] + raise PolicyCandidateError(f"{operation} failed: {error.get('type', 'CandidateError')}: {error.get('message', '')}") + raise PolicyProtocolError(f"malformed response during {operation}") + + def _rpc(self, operation: str, **payload: Any) -> dict[str, Any]: + if self._closed or self._process is None or self._process.poll() is not None: + raise PolicyCandidateError("candidate worker is not running") + request_id = self._next_id + self._next_id += 1 + try: + encoded = (json.dumps({"id": request_id, "op": operation, **payload}, ensure_ascii=True, allow_nan=False, separators=(",", ":")) + "\n").encode("utf-8") + except (TypeError, ValueError) as exc: + raise PolicyProtocolError("request is not finite JSON data") from exc + try: + assert self._process.stdin is not None + self._process.stdin.write(encoded) + self._process.stdin.flush() + except (BrokenPipeError, OSError) as exc: + raise PolicyCandidateError("candidate closed its input") from exc + response = self._receive(self.call_timeout_s, operation) + if response.get("id") != request_id: + self.close(force=True) + raise PolicyProtocolError("response id does not match request") + if not response.get("ok"): + self._raise_response_error(response, operation) + return response + + def reset_policy(self) -> None: + self._rpc("reset") + + def decide(self, observation: Mapping[str, Any]) -> dict[str, Any]: + response = self._rpc("decide", observation=observation) + action = response.get("action") + if not isinstance(action, dict): + raise PolicyProtocolError("decide must return a JSON object") + return action + + def close(self, *, force: bool = False) -> None: + if self._closed: + return + process = self._process + if process is not None and process.poll() is None and not force: + try: + self._rpc("shutdown") + process.wait(timeout=0.25) + except Exception: + force = True + if process is not None and process.poll() is None: + self._kill_process_tree(process) + self._closed = True + if process is not None: + for stream in (process.stdin, process.stdout, process.stderr): + try: + if stream is not None: + stream.close() + except OSError: + pass + shutil.rmtree(self._tempdir, ignore_errors=True) + + @staticmethod + def _kill_process_tree(process: subprocess.Popen[bytes]) -> None: + if process.poll() is not None: + return + if os.name == "nt": + try: + subprocess.run(["taskkill", "/PID", str(process.pid), "/T", "/F"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=2.0, check=False) + except Exception: + process.kill() + else: + try: + os.killpg(process.pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + process.kill() + try: + process.wait(timeout=2.0) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=2.0) + + def __enter__(self) -> "PolicyRuntime": + return self + + def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> None: + self.close() + + def __del__(self) -> None: + try: + self.close(force=True) + except Exception: + pass diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/policy_worker.py b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/policy_worker.py new file mode 100644 index 00000000..61f94ed9 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/policy_worker.py @@ -0,0 +1,95 @@ +"""JSON-lines worker for a candidate policy. This is isolation, not an OS sandbox.""" + +from __future__ import annotations + +import importlib.util +import json +import os +from pathlib import Path +import sys +from typing import Any + + +_loads = json.loads +_dumps = json.dumps + + +def _streams() -> tuple[Any, Any]: + protocol_in = os.fdopen(os.dup(0), "r", encoding="utf-8", newline="\n") + protocol_out = os.fdopen(os.dup(1), "w", encoding="utf-8", newline="\n", buffering=1) + devnull = os.open(os.devnull, os.O_WRONLY) + try: + os.dup2(devnull, 1) + finally: + os.close(devnull) + return protocol_in, protocol_out + + +def _send(stream: Any, payload: dict[str, Any]) -> None: + stream.write(_dumps(payload, ensure_ascii=True, allow_nan=False) + "\n") + stream.flush() + + +def _error(request_id: Any, exc: BaseException) -> dict[str, Any]: + try: + message = str(exc)[:500] + except Exception: + message = "failed to format candidate exception" + return {"id": request_id, "ok": False, "error": {"type": type(exc).__name__, "message": message}} + + +def _load(path: Path) -> Any: + spec = importlib.util.spec_from_file_location("edge_candidate", path) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load candidate from {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + if not callable(getattr(module, "decide", None)): + raise AttributeError("candidate must define callable decide(observation)") + return module + + +def main() -> int: + protocol_in, protocol_out = _streams() + if len(sys.argv) != 2: + _send(protocol_out, _error(None, ValueError("expected candidate path"))) + return 2 + try: + candidate = _load(Path(sys.argv[1]).resolve()) + except BaseException as exc: + _send(protocol_out, _error(None, exc)) + return 1 + _send(protocol_out, {"id": None, "ok": True, "ready": True}) + for line in protocol_in: + request_id: Any = None + try: + request = _loads(line) + if not isinstance(request, dict): + raise TypeError("request must be an object") + request_id = request.get("id") + operation = request.get("op") + if operation == "reset": + reset = getattr(candidate, "reset_policy", None) + if reset is not None: + if not callable(reset): + raise TypeError("reset_policy must be callable") + reset() + response = {"id": request_id, "ok": True} + elif operation == "decide": + response = {"id": request_id, "ok": True, "action": candidate.decide(request.get("observation"))} + elif operation == "shutdown": + _send(protocol_out, {"id": request_id, "ok": True}) + return 0 + else: + raise ValueError(f"unknown operation {operation!r}") + _send(protocol_out, response) + except BaseException as exc: + try: + _send(protocol_out, _error(request_id, exc)) + except BaseException: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/requirements.txt b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/requirements.txt new file mode 100644 index 00000000..6fdde22a --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/requirements.txt @@ -0,0 +1 @@ +# Standard-library-only MVP evaluator. diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/simulator.py b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/simulator.py new file mode 100644 index 00000000..034ace8a --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/simulator.py @@ -0,0 +1,451 @@ +"""Deterministic reduced-order simulator for an edge service control policy.""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +import math +from pathlib import Path +import random +from typing import Any, Callable, Mapping + + +ROOT = Path(__file__).resolve().parents[1] +CONFIG_PATH = ROOT / "references" / "config.json" +ROUTE_TOLERANCE = 1e-8 +MAX_ACTION_ITEMS = 256 + + +class ActionValidationError(ValueError): + """Candidate action violates a structural or physical hard constraint.""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(f"{code}: {message}") + self.code = code + + +@dataclass(frozen=True) +class ScenarioTrace: + name: str + family: str + seed: int + feedback: bool + workloads: tuple[dict[str, dict[str, float]], ...] + failed_nodes: tuple[frozenset[str], ...] + link_factors: tuple[dict[str, float], ...] + recovery_start: int | None + + +def load_config(path: Path = CONFIG_PATH) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _pair(source: str, target: str) -> str: + return f"{source}->{target}" + + +def _base_demand(service_id: str, region_index: int, step: int, variant: float) -> float: + base = {"api": 54.0, "search": 34.0, "media": 25.0}[service_id] + region_factor = (1.08, 0.94, 0.82)[region_index] + phase = (step + 2 * region_index) % 24 + day_wave = 0.82 + 0.30 * (1.0 + math.sin(2.0 * math.pi * phase / 24.0)) / 2.0 + return base * region_factor * day_wave * variant + + +def make_scenario(family: str, seed: int, *, feedback: bool) -> ScenarioTrace: + """Generate every exogenous event before candidate execution.""" + + rng = random.Random(seed) + regions = ("edge-a", "edge-b", "edge-c") + services = ("api", "search", "media") + variant = 0.96 + 0.08 * rng.random() + burst_region = regions[seed % len(regions)] + failed_node = f"{burst_region[-1]}-{1 + seed % 2}" + workloads: list[dict[str, dict[str, float]]] = [] + failed_nodes: list[frozenset[str]] = [] + link_factors: list[dict[str, float]] = [] + recovery_start: int | None = None + + for step in range(24): + demand: dict[str, dict[str, float]] = {} + for region_index, region in enumerate(regions): + demand[region] = {} + for service in services: + value = _base_demand(service, region_index, step, variant) + if family in {"regional_burst", "node_failure_burst"} and region == burst_region and 8 <= step <= 12: + value *= 1.85 if service != "media" else 1.55 + if family == "recovery_migration" and region == burst_region and 7 <= step <= 14: + value *= 1.45 + demand[region][service] = round(value, 6) + workloads.append(demand) + + failed: set[str] = set() + if family == "node_failure_burst" and 8 <= step <= 13: + failed.add(failed_node) + recovery_start = 14 + elif family == "recovery_migration" and 6 <= step <= 10: + failed.add(failed_node) + recovery_start = 11 + failed_nodes.append(frozenset(failed)) + + factors: dict[str, float] = {} + if family == "link_degradation" and 9 <= step <= 15: + degraded_target = regions[(seed + 1) % len(regions)] + factors[_pair(burst_region, degraded_target)] = 0.22 + factors[_pair(degraded_target, burst_region)] = 0.22 + link_factors.append(factors) + + return ScenarioTrace( + name=f"{family}-v{seed % 100}", + family=family, + seed=seed, + feedback=feedback, + workloads=tuple(workloads), + failed_nodes=tuple(failed_nodes), + link_factors=tuple(link_factors), + recovery_start=recovery_start, + ) + + +SCENARIO_FAMILIES = ( + "normal_diurnal", + "regional_burst", + "node_failure_burst", + "link_degradation", + "recovery_migration", +) +SCENARIOS = tuple( + make_scenario(family, 4100 + family_index * 17 + variant, feedback=variant == 0) + for family_index, family in enumerate(SCENARIO_FAMILIES) + for variant in range(2) +) + + +def _is_number(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _finite_float(value: Any, field: str) -> float: + if not _is_number(value): + raise ActionValidationError("type", f"{field} must be a number, not {type(value).__name__}") + result = float(value) + if not math.isfinite(result): + raise ActionValidationError("non_finite", f"{field} must be finite") + return result + + +class EdgeServiceSimulator: + """Stateful placement and routing simulator with one-period replica cold starts.""" + + def __init__(self, scenario: ScenarioTrace, config: Mapping[str, Any] | None = None) -> None: + self.config = dict(load_config() if config is None else config) + self.scenario = scenario + self.nodes = {item["id"]: dict(item) for item in self.config["nodes"]} + self.services = {item["id"]: dict(item) for item in self.config["services"]} + self.regions = tuple(self.config["regions"]) + self.step_index = 0 + self.active: dict[tuple[str, str], int] = {} + self.pending: dict[tuple[str, str], int] = {} + self.history: list[dict[str, dict[str, float]]] = [] + self.last_action: dict[str, Any] | None = None + self.last_feedback: list[str] = [] + self.period_rows: list[dict[str, float]] = [] + self._recovered = scenario.recovery_start is None + self._recovery_steps = 0 + self._bootstrap_replicas() + + def _bootstrap_replicas(self) -> None: + for region in self.regions: + region_nodes = [node_id for node_id, node in self.nodes.items() if node["region"] == region] + for index, service_id in enumerate(self.services): + self.active[(service_id, region_nodes[index % len(region_nodes)])] = 1 + + @property + def done(self) -> bool: + return self.step_index >= len(self.scenario.workloads) + + def _alive_nodes(self) -> set[str]: + failed = self.scenario.failed_nodes[self.step_index] + return set(self.nodes) - set(failed) + + def _activate_pending_and_apply_failures(self) -> None: + alive = self._alive_nodes() + for key, count in list(self.pending.items()): + if key[1] in alive and count > 0: + self.active[key] = self.active.get(key, 0) + count + self.pending.clear() + self.active = {key: count for key, count in self.active.items() if key[1] in alive and count > 0} + + def observation(self) -> dict[str, Any]: + if self.done: + raise RuntimeError("scenario has finished") + alive = self._alive_nodes() + bandwidth = {} + default_limit = float(self.config["cross_region_bandwidth_mb_per_period"]) + factors = self.scenario.link_factors[self.step_index] + for source in self.regions: + for target in self.regions: + if source != target: + bandwidth[_pair(source, target)] = default_limit * factors.get(_pair(source, target), 1.0) + return { + "timestep": self.step_index, + "period_minutes": self.config["period_minutes"], + "nodes": [ + { + **node, + "alive": node_id in alive, + } + for node_id, node in self.nodes.items() + ], + "services": list(self.services.values()), + "workload_rps": self.scenario.workloads[self.step_index], + "workload_history": self.history[-4:], + "active_replicas": [ + {"service_id": service, "node_id": node, "count": count} + for (service, node), count in sorted(self.active.items()) + ], + "pending_replicas": [ + {"service_id": service, "node_id": node, "count": count, "ready_in_steps": 1} + for (service, node), count in sorted(self.pending.items()) + ], + "network_rtt_ms": self.config["network_rtt_ms"], + "cross_region_bandwidth_mb_per_period": bandwidth, + "last_action": self.last_action, + "last_feedback": self.last_feedback, + } + + def _validate_action(self, action: Any) -> tuple[dict[tuple[str, str], int], list[dict[str, Any]]]: + if not isinstance(action, dict): + raise ActionValidationError("schema", "action must be a JSON object") + if set(action) != {"replicas", "routes"}: + raise ActionValidationError("schema", "action must contain exactly replicas and routes") + replicas = action["replicas"] + routes = action["routes"] + if not isinstance(replicas, list) or not isinstance(routes, list): + raise ActionValidationError("type", "replicas and routes must be lists") + if len(replicas) + len(routes) > MAX_ACTION_ITEMS: + raise ActionValidationError("size", f"action exceeds {MAX_ACTION_ITEMS} items") + + alive = self._alive_nodes() + desired: dict[tuple[str, str], int] = {} + cpu_by_node = {node_id: 0.0 for node_id in self.nodes} + for index, item in enumerate(replicas): + if not isinstance(item, dict) or set(item) != {"service_id", "node_id", "count"}: + raise ActionValidationError("schema", f"replicas[{index}] has invalid fields") + service_id = item["service_id"] + node_id = item["node_id"] + count = item["count"] + if service_id not in self.services or node_id not in self.nodes: + raise ActionValidationError("unknown_id", f"unknown placement {service_id!r}/{node_id!r}") + if node_id not in alive: + raise ActionValidationError("failed_node", f"cannot place replicas on failed node {node_id}") + if isinstance(count, bool) or not isinstance(count, int): + raise ActionValidationError("type", f"replica count for {service_id}/{node_id} must be an integer") + if count < 0: + raise ActionValidationError("range", "replica count must be non-negative") + key = (service_id, node_id) + if key in desired: + raise ActionValidationError("duplicate", f"duplicate placement {service_id}/{node_id}") + desired[key] = count + cpu_by_node[node_id] += count * float(self.services[service_id]["cpu_per_replica"]) + for node_id, used in cpu_by_node.items(): + if used > float(self.nodes[node_id]["cpu_capacity"]) + 1e-9: + raise ActionValidationError("capacity", f"placement uses {used:g} CPU on {node_id}") + + cleaned_routes: list[dict[str, Any]] = [] + seen_routes: set[tuple[str, str, str]] = set() + route_sums: dict[tuple[str, str], float] = {} + for index, item in enumerate(routes): + required = {"service_id", "source_region", "node_id", "fraction"} + if not isinstance(item, dict) or set(item) != required: + raise ActionValidationError("schema", f"routes[{index}] has invalid fields") + service_id = item["service_id"] + source = item["source_region"] + node_id = item["node_id"] + if service_id not in self.services or source not in self.regions or node_id not in self.nodes: + raise ActionValidationError("unknown_id", f"unknown route {service_id!r}/{source!r}/{node_id!r}") + fraction = _finite_float(item["fraction"], f"routes[{index}].fraction") + if fraction < 0.0 or fraction > 1.0: + raise ActionValidationError("range", "route fraction must be in [0, 1]") + key = (service_id, source, node_id) + if key in seen_routes: + raise ActionValidationError("duplicate", f"duplicate route {key}") + seen_routes.add(key) + placement_key = (service_id, node_id) + if node_id not in alive: + raise ActionValidationError("failed_node", f"cannot route to failed node {node_id}") + if self.active.get(placement_key, 0) <= 0: + raise ActionValidationError("inactive_route", f"route targets no active replica at {service_id}/{node_id}") + if desired.get(placement_key, 0) <= 0: + raise ActionValidationError("inactive_route", f"route targets placement removed by this action: {service_id}/{node_id}") + route_key = (service_id, source) + route_sums[route_key] = route_sums.get(route_key, 0.0) + fraction + if route_sums[route_key] > 1.0 + ROUTE_TOLERANCE: + raise ActionValidationError("route_sum", f"route fractions exceed 1 for {service_id}/{source}") + cleaned_routes.append({**item, "fraction": fraction}) + return desired, cleaned_routes + + def _apply_desired(self, desired: Mapping[tuple[str, str], int]) -> None: + keys = set(self.active) | set(self.pending) | set(desired) + for key in keys: + target = int(desired.get(key, 0)) + active = self.active.get(key, 0) + pending = self.pending.get(key, 0) + total = active + pending + if target > total: + self.pending[key] = pending + target - total + elif target < total: + remove = total - target + pending_removed = min(pending, remove) + pending -= pending_removed + remove -= pending_removed + active = max(0, active - remove) + if pending: + self.pending[key] = pending + else: + self.pending.pop(key, None) + if active: + self.active[key] = active + else: + self.active.pop(key, None) + + def _simulate_period(self, routes: list[dict[str, Any]]) -> dict[str, float]: + workload = self.scenario.workloads[self.step_index] + period_seconds = float(self.config["period_minutes"]) * 60.0 + total_demand = sum(sum(values.values()) for values in workload.values()) + route_rows: list[dict[str, Any]] = [] + routed = 0.0 + for route in routes: + service_id = route["service_id"] + source = route["source_region"] + node_id = route["node_id"] + demand = float(workload[source][service_id]) + requested = demand * route["fraction"] + routed += requested + route_rows.append({**route, "requested": requested, "after_link": requested}) + + default_limit = float(self.config["cross_region_bandwidth_mb_per_period"]) + factors = self.scenario.link_factors[self.step_index] + by_link: dict[str, list[dict[str, Any]]] = {} + for row in route_rows: + target = self.nodes[row["node_id"]]["region"] + if row["source_region"] != target: + by_link.setdefault(_pair(row["source_region"], target), []).append(row) + for link, rows in by_link.items(): + requested_mb = sum( + row["requested"] + * float(self.services[row["service_id"]]["response_mb"]) + * period_seconds + for row in rows + ) + limit = default_limit * factors.get(link, 1.0) + scale = min(1.0, limit / requested_mb) if requested_mb > 0 else 1.0 + for row in rows: + row["after_link"] *= scale + + by_target: dict[tuple[str, str], list[dict[str, Any]]] = {} + for row in route_rows: + by_target.setdefault((row["service_id"], row["node_id"]), []).append(row) + for (service_id, node_id), rows in by_target.items(): + capacity = self.active.get((service_id, node_id), 0) * float(self.services[service_id]["service_rate_rps"]) + incoming = sum(row["after_link"] for row in rows) + scale = min(1.0, capacity / incoming) if incoming > 0 else 1.0 + utilization = min(1.25, incoming / capacity) if capacity > 0 else 1.25 + for row in rows: + row["served"] = row["after_link"] * scale + row["utilization"] = utilization + + served = sum(float(row.get("served", 0.0)) for row in route_rows) + weighted_p95 = 0.0 + weighted_p99 = 0.0 + slo_violations = max(0.0, total_demand - served) + cross_region_mb = 0.0 + for row in route_rows: + amount = float(row.get("served", 0.0)) + service = self.services[row["service_id"]] + target_region = self.nodes[row["node_id"]]["region"] + rtt = float(self.config["network_rtt_ms"][row["source_region"]][target_region]) + utilization = min(0.995, float(row.get("utilization", 1.25))) + queue_pressure = utilization * utilization / max(0.08, 1.0 - utilization) + p95 = rtt + float(service["base_latency_ms"]) * (1.0 + 0.85 * queue_pressure) + p99 = rtt + float(service["base_latency_ms"]) * (1.0 + 1.35 * queue_pressure) + weighted_p95 += amount * p95 + weighted_p99 += amount * p99 + if p99 > float(service["p99_slo_ms"]): + slo_violations += amount + if row["source_region"] != target_region: + cross_region_mb += amount * float(service["response_mb"]) * period_seconds + + compute_cost = float(self.config["compute_cost_per_replica_period"]) * sum(self.active.values()) + compute_cost += ( + float(self.config["compute_cost_per_replica_period"]) + * float(self.config["pending_replica_cost_factor"]) + * sum(self.pending.values()) + ) + availability = served / total_demand if total_demand else 1.0 + if self.scenario.recovery_start is not None and self.step_index >= self.scenario.recovery_start and not self._recovered: + if availability >= 0.99: + self._recovered = True + else: + self._recovery_steps += 1 + return { + "demand": total_demand, + "served": served, + "slo_violations": min(total_demand, slo_violations), + "p95_weighted": weighted_p95, + "p99_weighted": weighted_p99, + "compute_cost": compute_cost, + "cross_region_mb": cross_region_mb, + "cross_region_cost": cross_region_mb / 1024.0 * float(self.config["cross_region_cost_per_gb"]), + } + + def step(self, action: Any) -> dict[str, float]: + if self.done: + raise RuntimeError("scenario has finished") + desired, routes = self._validate_action(action) + self._apply_desired(desired) + row = self._simulate_period(routes) + self.period_rows.append(row) + self.last_action = action + feedback = [] + if row["served"] < 0.99 * row["demand"]: + feedback.append("availability_below_0.99") + if row["slo_violations"] > 0.05 * row["demand"]: + feedback.append("slo_violation_rate_above_0.05") + self.last_feedback = feedback + self.history.append(self.scenario.workloads[self.step_index]) + self.step_index += 1 + if not self.done: + self._activate_pending_and_apply_failures() + return row + + def metrics(self) -> dict[str, float | str]: + if not self.done: + raise RuntimeError("scenario is not complete") + total_demand = sum(row["demand"] for row in self.period_rows) + served = sum(row["served"] for row in self.period_rows) + violations = sum(row["slo_violations"] for row in self.period_rows) + return { + "scenario_family": self.scenario.family, + "request_availability": served / total_demand if total_demand else 1.0, + "unserved_rate": 1.0 - served / total_demand if total_demand else 0.0, + "request_weighted_p95_ms": sum(row["p95_weighted"] for row in self.period_rows) / max(served, 1e-12), + "request_weighted_p99_ms": sum(row["p99_weighted"] for row in self.period_rows) / max(served, 1e-12), + "p99_slo_violation_rate": violations / total_demand if total_demand else 0.0, + "compute_cost": sum(row["compute_cost"] for row in self.period_rows), + "cross_region_gb": sum(row["cross_region_mb"] for row in self.period_rows) / 1024.0, + "cross_region_cost": sum(row["cross_region_cost"] for row in self.period_rows), + "failure_recovery_steps": float(self._recovery_steps), + } + + +def run_policy(policy: Any, scenario: ScenarioTrace) -> dict[str, float | str]: + reset = getattr(policy, "reset_policy", None) + if reset is not None: + reset() + simulator = EdgeServiceSimulator(scenario) + simulator._activate_pending_and_apply_failures() + while not simulator.done: + action = policy.decide(simulator.observation()) + simulator.step(action) + return simulator.metrics() diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_evaluator.py b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_evaluator.py new file mode 100644 index 00000000..6a309f54 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_evaluator.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import unittest + +from evaluator import evaluate, scenario_score + + +ROOT = Path(__file__).resolve().parents[1] + + +class EvaluatorTests(unittest.TestCase): + def test_tail_latency_component_is_continuous_below_slo_reference(self) -> None: + metrics = { + "unserved_rate": 0.0, + "p99_slo_violation_rate": 0.0, + "request_weighted_p99_ms": 45.0, + "compute_cost": 0.0, + "cross_region_gb": 0.0, + "failure_recovery_steps": 0.0, + } + _, components = scenario_score(metrics) + self.assertEqual(components["tail_latency"], 0.25) + + def test_calibration_policies_are_valid_and_distinguishable(self) -> None: + weak = evaluate(ROOT / "calibration" / "weak.py") + reasonable = evaluate(ROOT / "scripts" / "init.py") + strong = evaluate(ROOT / "calibration" / "strong.py") + results = (weak, reasonable, strong) + self.assertTrue(all(result["valid"] == 1.0 for result in results)) + scores = [float(result["combined_score"]) for result in results] + self.assertEqual(len(set(scores)), 3) + self.assertGreater(max(scores) - min(scores), 0.25) + + def test_reasonable_baseline_is_deterministic(self) -> None: + first = evaluate(ROOT / "scripts" / "init.py") + second = evaluate(ROOT / "scripts" / "init.py") + self.assertEqual(first, second) + self.assertEqual(first["valid"], 1.0) + + def test_invalid_action_cannot_compete(self) -> None: + with tempfile.TemporaryDirectory() as directory: + candidate = Path(directory) / "invalid.py" + candidate.write_text( + "def decide(observation):\n" + " return {'replicas': [{'service_id': 'api', 'node_id': 'a-1', 'count': True}], 'routes': []}\n", + encoding="utf-8", + ) + result = evaluate(candidate) + self.assertEqual(result["valid"], 0.0) + self.assertEqual(result["combined_score"], 0.0) + self.assertIn("type", result["rows"][0]["error"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_policy_runtime.py b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_policy_runtime.py new file mode 100644 index 00000000..822ccb9a --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_policy_runtime.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from pathlib import Path +import tempfile +import textwrap +import unittest + +from policy_runtime import PolicyCandidateError, PolicyProtocolError, PolicyRuntime, PolicyTimeoutError + + +ROOT = Path(__file__).resolve().parents[1] + + +class PolicyRuntimeTests(unittest.TestCase): + def test_reasonable_candidate_round_trip(self) -> None: + from simulator import EdgeServiceSimulator, SCENARIOS + + simulator = EdgeServiceSimulator(SCENARIOS[0]) + simulator._activate_pending_and_apply_failures() + with PolicyRuntime(ROOT / "scripts" / "init.py") as policy: + policy.reset_policy() + action = policy.decide(simulator.observation()) + self.assertEqual(set(action), {"replicas", "routes"}) + + def _candidate(self, source: str) -> Path: + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + path = Path(directory.name) / "candidate.py" + path.write_text(textwrap.dedent(source), encoding="utf-8") + return path + + def test_missing_decide_fails_closed(self) -> None: + with self.assertRaises(PolicyCandidateError): + PolicyRuntime(self._candidate("x = 1\n")) + + def test_infinite_loop_times_out(self) -> None: + candidate = self._candidate( + """ + def decide(observation): + while True: + pass + """ + ) + with PolicyRuntime(candidate, call_timeout_s=0.1, total_timeout_s=1.0) as policy: + with self.assertRaises(PolicyTimeoutError): + policy.decide({}) + + def test_oversized_response_is_rejected(self) -> None: + candidate = self._candidate( + """ + def decide(observation): + return {"replicas": [], "routes": [], "padding": "x" * 70000} + """ + ) + with PolicyRuntime(candidate) as policy: + with self.assertRaises((PolicyProtocolError, PolicyCandidateError)): + policy.decide({}) + + def test_non_json_result_is_rejected(self) -> None: + candidate = self._candidate( + """ + def decide(observation): + return {"replicas": set(), "routes": []} + """ + ) + with PolicyRuntime(candidate) as policy: + with self.assertRaises(PolicyCandidateError): + policy.decide({}) + + def test_candidate_exception_is_reported(self) -> None: + candidate = self._candidate( + """ + def decide(observation): + raise RuntimeError("intentional failure") + """ + ) + with PolicyRuntime(candidate) as policy: + with self.assertRaisesRegex(PolicyCandidateError, "intentional failure"): + policy.decide({}) + + def test_abrupt_process_exit_is_reported(self) -> None: + candidate = self._candidate( + """ + import os + def decide(observation): + os._exit(7) + """ + ) + with PolicyRuntime(candidate) as policy: + with self.assertRaises(PolicyCandidateError): + policy.decide({}) + + def test_non_json_stdout_cannot_corrupt_protocol(self) -> None: + candidate = self._candidate( + """ + def decide(observation): + print("not-json protocol noise") + return {"replicas": [], "routes": []} + """ + ) + with PolicyRuntime(candidate) as policy: + self.assertEqual(policy.decide({}), {"replicas": [], "routes": []}) + + def test_stderr_spam_is_drained_and_tail_is_bounded(self) -> None: + candidate = self._candidate( + """ + import sys + def decide(observation): + sys.stderr.write("x" * 200000) + sys.stderr.flush() + return {"replicas": [], "routes": []} + """ + ) + with PolicyRuntime(candidate) as policy: + self.assertEqual(policy.decide({}), {"replicas": [], "routes": []}) + self.assertLessEqual(len(policy.stderr_tail.encode("utf-8")), 16 * 1024) + + def test_candidate_mutates_only_its_json_copy_of_observation(self) -> None: + candidate = self._candidate( + """ + def decide(observation): + observation["nested"]["value"] = 99 + return {"seen": observation["nested"]["value"]} + """ + ) + original = {"nested": {"value": 1}} + with PolicyRuntime(candidate) as policy: + self.assertEqual(policy.decide(original), {"seen": 99}) + self.assertEqual(original, {"nested": {"value": 1}}) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_simulator.py b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_simulator.py new file mode 100644 index 00000000..ca9e84a9 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_simulator.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import importlib.util +import math +from pathlib import Path +import unittest + +from simulator import ActionValidationError, EdgeServiceSimulator, SCENARIOS, ScenarioTrace, make_scenario, run_policy + + +ROOT = Path(__file__).resolve().parents[1] + + +def load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def preserve_action(simulator: EdgeServiceSimulator) -> dict: + replicas = [ + {"service_id": service, "node_id": node, "count": count} + for (service, node), count in sorted(simulator.active.items()) + ] + return {"replicas": replicas, "routes": []} + + +class ScenarioTests(unittest.TestCase): + def test_generation_is_deterministic(self) -> None: + first = make_scenario("regional_burst", 4118, feedback=True) + second = make_scenario("regional_burst", 4118, feedback=True) + self.assertEqual(first, second) + self.assertEqual(len(first.workloads), 24) + + def test_five_scenario_families_have_two_variants(self) -> None: + self.assertEqual(len(SCENARIOS), 10) + self.assertEqual(len({scenario.family for scenario in SCENARIOS}), 5) + + +class ValidationTests(unittest.TestCase): + def setUp(self) -> None: + self.simulator = EdgeServiceSimulator(SCENARIOS[0]) + self.simulator._activate_pending_and_apply_failures() + + def assert_code(self, action: dict, code: str) -> None: + with self.assertRaises(ActionValidationError) as caught: + self.simulator._validate_action(action) + self.assertEqual(caught.exception.code, code) + + def test_bool_cannot_impersonate_integer(self) -> None: + action = preserve_action(self.simulator) + action["replicas"][0]["count"] = True + self.assert_code(action, "type") + + def test_negative_and_noninteger_replica_counts_are_rejected(self) -> None: + for value, code in [(-1, "range"), (1.5, "type")]: + with self.subTest(value=value): + action = preserve_action(self.simulator) + action["replicas"][0]["count"] = value + self.assert_code(action, code) + + def test_unknown_id_is_rejected(self) -> None: + action = preserve_action(self.simulator) + action["replicas"][0]["node_id"] = "missing-node" + self.assert_code(action, "unknown_id") + + def test_exact_top_level_schema_is_enforced(self) -> None: + action = preserve_action(self.simulator) + action["reported_score"] = 100.0 + self.assert_code(action, "schema") + + def test_nan_route_is_rejected(self) -> None: + action = preserve_action(self.simulator) + action["routes"] = [{"service_id": "api", "source_region": "edge-a", "node_id": "a-1", "fraction": math.nan}] + self.assert_code(action, "non_finite") + + def test_route_fraction_out_of_range_is_rejected(self) -> None: + action = preserve_action(self.simulator) + action["routes"] = [{"service_id": "api", "source_region": "edge-a", "node_id": "a-1", "fraction": 1.01}] + self.assert_code(action, "range") + + def test_extreme_route_float_is_rejected(self) -> None: + action = preserve_action(self.simulator) + action["routes"] = [{"service_id": "api", "source_region": "edge-a", "node_id": "a-1", "fraction": 1e308}] + self.assert_code(action, "range") + + def test_huge_replica_count_is_rejected_by_capacity(self) -> None: + action = preserve_action(self.simulator) + action["replicas"] = [{"service_id": "api", "node_id": "a-1", "count": 10**100}] + self.assert_code(action, "capacity") + + def test_malformed_nested_route_is_rejected(self) -> None: + action = preserve_action(self.simulator) + action["routes"] = [{"service_id": "api", "source_region": "edge-a", "fraction": 1.0}] + self.assert_code(action, "schema") + + def test_capacity_is_checked(self) -> None: + action = preserve_action(self.simulator) + action["replicas"] = [{"service_id": "api", "node_id": "a-1", "count": 6}] + self.assert_code(action, "capacity") + + def test_duplicate_placement_is_rejected(self) -> None: + action = preserve_action(self.simulator) + action["replicas"].append(dict(action["replicas"][0])) + self.assert_code(action, "duplicate") + + def test_route_sum_above_one_is_rejected(self) -> None: + action = preserve_action(self.simulator) + action["routes"] = [ + {"service_id": "api", "source_region": "edge-a", "node_id": "a-1", "fraction": 0.6}, + {"service_id": "api", "source_region": "edge-a", "node_id": "b-1", "fraction": 0.5}, + ] + self.assert_code(action, "route_sum") + + def test_route_sum_tolerance_has_a_strict_boundary(self) -> None: + within = preserve_action(self.simulator) + within["routes"] = [ + {"service_id": "api", "source_region": "edge-a", "node_id": "a-1", "fraction": 0.5}, + {"service_id": "api", "source_region": "edge-a", "node_id": "b-1", "fraction": 0.500000005}, + ] + self.simulator._validate_action(within) + outside = preserve_action(self.simulator) + outside["routes"] = [ + {"service_id": "api", "source_region": "edge-a", "node_id": "a-1", "fraction": 0.5}, + {"service_id": "api", "source_region": "edge-a", "node_id": "b-1", "fraction": 0.50000002}, + ] + self.assert_code(outside, "route_sum") + + def test_duplicate_route_is_rejected(self) -> None: + action = preserve_action(self.simulator) + route = {"service_id": "api", "source_region": "edge-a", "node_id": "a-1", "fraction": 0.4} + action["routes"] = [route, dict(route)] + self.assert_code(action, "duplicate") + + def test_pending_replica_cannot_receive_traffic(self) -> None: + action = preserve_action(self.simulator) + action["replicas"].append({"service_id": "api", "node_id": "a-2", "count": 1}) + action["routes"] = [{"service_id": "api", "source_region": "edge-a", "node_id": "a-2", "fraction": 1.0}] + self.assert_code(action, "inactive_route") + + def test_failed_node_cannot_receive_placement(self) -> None: + scenario = next(item for item in SCENARIOS if item.family == "node_failure_burst") + simulator = EdgeServiceSimulator(scenario) + simulator.step_index = 8 + simulator._activate_pending_and_apply_failures() + failed_node = next(iter(scenario.failed_nodes[8])) + action = preserve_action(simulator) + action["replicas"].append({"service_id": "api", "node_id": failed_node, "count": 1}) + with self.assertRaises(ActionValidationError) as caught: + simulator._validate_action(action) + self.assertEqual(caught.exception.code, "failed_node") + + def test_failed_node_cannot_receive_route(self) -> None: + scenario = next(item for item in SCENARIOS if item.family == "node_failure_burst") + simulator = EdgeServiceSimulator(scenario) + simulator.step_index = 8 + simulator._activate_pending_and_apply_failures() + failed_node = next(iter(scenario.failed_nodes[8])) + action = preserve_action(simulator) + action["routes"] = [{"service_id": "api", "source_region": "edge-a", "node_id": failed_node, "fraction": 0.5}] + with self.assertRaises(ActionValidationError) as caught: + simulator._validate_action(action) + self.assertEqual(caught.exception.code, "failed_node") + + +class MetricTests(unittest.TestCase): + def test_static_policy_produces_finite_metrics(self) -> None: + weak = load_module("weak_policy", ROOT / "calibration" / "weak.py") + metrics = run_policy(weak, SCENARIOS[0]) + self.assertGreater(float(metrics["request_availability"]), 0.95) + for key, value in metrics.items(): + if key != "scenario_family": + self.assertTrue(math.isfinite(float(value)), key) + + def test_tiny_routing_case_matches_capacity_and_bandwidth_oracle(self) -> None: + zero = {region: {service: 0.0 for service in ("api", "search", "media")} for region in ("edge-a", "edge-b", "edge-c")} + zero["edge-a"]["api"] = 150.0 + scenario = ScenarioTrace( + name="tiny-oracle", + family="tiny-oracle", + seed=0, + feedback=True, + workloads=(zero,), + failed_nodes=(frozenset(),), + link_factors=({},), + recovery_start=None, + ) + best_served = 0.0 + for first in (0.0, 0.5, 1.0): + for second in (0.0, 0.5, 1.0): + if first + second > 1.0: + continue + simulator = EdgeServiceSimulator(scenario) + routes = [ + {"service_id": "api", "source_region": "edge-a", "node_id": "a-1", "fraction": first}, + {"service_id": "api", "source_region": "edge-a", "node_id": "b-1", "fraction": second}, + ] + best_served = max(best_served, simulator._simulate_period(routes)["served"]) + local_served = min(75.0, 90.0) + cross_region_link_rps = 360.0 / (0.025 * 5.0 * 60.0) + remote_served = min(75.0, 90.0, cross_region_link_rps) + exact_capacity_bound = local_served + remote_served + self.assertAlmostEqual(best_served, exact_capacity_bound, places=9) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_tiny_oracle.py b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_tiny_oracle.py new file mode 100644 index 00000000..15390251 --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/test_tiny_oracle.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import unittest + +from evaluator import evaluate_action_sequence +from tiny_oracle import TINY_CONFIG, TINY_SCENARIO, brute_force_oracle + + +class MultiTimestepTinyOracleTests(unittest.TestCase): + def test_bruteforce_oracle_matches_production_simulator_and_score_exactly(self) -> None: + oracle = brute_force_oracle() + evaluated = evaluate_action_sequence( + oracle["best_action_sequence"], TINY_SCENARIO, TINY_CONFIG + ) + self.assertEqual(oracle["raw_metrics"], evaluated["raw_metrics"]) + self.assertEqual(oracle["normalized_loss_components"], evaluated["normalized_loss_components"]) + self.assertEqual(oracle["combined_score"], evaluated["combined_score"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/tiny_oracle.py b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/tiny_oracle.py new file mode 100644 index 00000000..0d1d6bbb --- /dev/null +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/tiny_oracle.py @@ -0,0 +1,232 @@ +"""Independent brute-force oracle for a three-period, two-node control case.""" + +from __future__ import annotations + +from itertools import product +import json +import math +from typing import Any, Iterable + +try: + from .simulator import ScenarioTrace, load_config +except ImportError: + from simulator import ScenarioTrace, load_config + + +NODES = ("n1", "n2") +ROUTE_GRID = (0.0, 0.5, 1.0) +TINY_CONFIG: dict[str, Any] = { + "model_version": "tiny-oracle-v1", + "period_minutes": 5, + "regions": ["tiny"], + "nodes": [ + {"id": "n1", "region": "tiny", "failure_domain": "rack-1", "cpu_capacity": 1.0}, + {"id": "n2", "region": "tiny", "failure_domain": "rack-2", "cpu_capacity": 1.0}, + ], + "services": [ + { + "id": "svc", + "cpu_per_replica": 1.0, + "service_rate_rps": 2.0, + "base_latency_ms": 10.0, + "p99_slo_ms": 50.0, + "response_mb": 0.01, + "reliability_class": "critical", + } + ], + "network_rtt_ms": {"tiny": {"tiny": 5.0}}, + "cross_region_bandwidth_mb_per_period": 1000.0, + "compute_cost_per_replica_period": 0.1, + "pending_replica_cost_factor": 0.5, + "cross_region_cost_per_gb": 0.08, +} +TINY_SCENARIO = ScenarioTrace( + name="tiny-three-period-control", + family="tiny_oracle", + seed=0, + feedback=True, + workloads=( + {"tiny": {"svc": 1.0}}, + {"tiny": {"svc": 3.0}}, + {"tiny": {"svc": 3.0}}, + ), + failed_nodes=(frozenset(), frozenset(), frozenset()), + link_factors=({}, {}, {}), + recovery_start=None, +) + + +def _target_nodes(action: dict[str, Any]) -> frozenset[str]: + return frozenset(item["node_id"] for item in action["replicas"] if item["count"] == 1) + + +def _placement_options() -> Iterable[frozenset[str]]: + for mask in range(1 << len(NODES)): + yield frozenset(node for index, node in enumerate(NODES) if mask & (1 << index)) + + +def action_options(active: frozenset[str]) -> list[dict[str, Any]]: + """Enumerate the complete discrete action set used by this oracle. + + Placement is binary per node. Route fractions use the declared {0, 0.5, 1} grid; + therefore "all feasible" in this oracle means all hard-valid trajectories on that + finite grid, not every real-valued route fraction. + """ + + actions: list[dict[str, Any]] = [] + for target in _placement_options(): + retained = tuple(sorted(active & target)) + route_vectors = product(ROUTE_GRID, repeat=len(retained)) if retained else [()] + for vector in route_vectors: + if sum(vector) > 1.0: + continue + actions.append( + { + "replicas": [ + {"service_id": "svc", "node_id": node, "count": 1} + for node in sorted(target) + ], + "routes": [ + { + "service_id": "svc", + "source_region": "tiny", + "node_id": node, + "fraction": fraction, + } + for node, fraction in zip(retained, vector) + if fraction > 0.0 + ], + } + ) + return actions + + +def enumerate_action_sequences() -> list[list[dict[str, Any]]]: + sequences: list[list[dict[str, Any]]] = [] + + def visit(step: int, active: frozenset[str], prefix: list[dict[str, Any]]) -> None: + if step == len(TINY_SCENARIO.workloads): + sequences.append(prefix) + return + for action in action_options(active): + # With no failures and a one-period cold start, every desired replica is + # active at the beginning of the next period. + visit(step + 1, _target_nodes(action), [*prefix, action]) + + visit(0, frozenset({"n1"}), []) + return sequences + + +def independent_metrics(actions: list[dict[str, Any]]) -> dict[str, float | str]: + """Recompute the tiny trajectory without calling EdgeServiceSimulator.""" + + if len(actions) != 3: + raise ValueError("tiny oracle requires exactly three actions") + active = frozenset({"n1"}) + rows: list[dict[str, float]] = [] + demands = (1.0, 3.0, 3.0) + for demand, action in zip(demands, actions): + target = _target_nodes(action) + retained = active & target + pending = target - active + requested_by_node = {node: 0.0 for node in retained} + for route in action["routes"]: + node = route["node_id"] + if node not in retained: + raise ValueError("oracle sequence routes to a non-active target") + requested_by_node[node] += demand * float(route["fraction"]) + if sum(requested_by_node.values()) > demand + 1e-12: + raise ValueError("oracle route sum exceeds one") + + served_total = 0.0 + p95_weighted = 0.0 + p99_weighted = 0.0 + served_slo_violations = 0.0 + for requested in requested_by_node.values(): + capacity = 2.0 + served = min(requested, capacity) + utilization = min(0.995, min(1.25, requested / capacity)) + pressure = utilization * utilization / max(0.08, 1.0 - utilization) + p95 = 5.0 + 10.0 * (1.0 + 0.85 * pressure) + p99 = 5.0 + 10.0 * (1.0 + 1.35 * pressure) + served_total += served + p95_weighted += served * p95 + p99_weighted += served * p99 + if p99 > 50.0: + served_slo_violations += served + unserved = demand - served_total + rows.append( + { + "demand": demand, + "served": served_total, + "slo_violations": unserved + served_slo_violations, + "p95_weighted": p95_weighted, + "p99_weighted": p99_weighted, + "compute_cost": 0.1 * len(retained) + 0.05 * len(pending), + } + ) + active = target + + total_demand = sum(row["demand"] for row in rows) + total_served = sum(row["served"] for row in rows) + return { + "scenario_family": "tiny_oracle", + "request_availability": total_served / total_demand, + "unserved_rate": 1.0 - total_served / total_demand, + "request_weighted_p95_ms": sum(row["p95_weighted"] for row in rows) / max(total_served, 1e-12), + "request_weighted_p99_ms": sum(row["p99_weighted"] for row in rows) / max(total_served, 1e-12), + "p99_slo_violation_rate": sum(row["slo_violations"] for row in rows) / total_demand, + "compute_cost": sum(row["compute_cost"] for row in rows), + "cross_region_gb": 0.0, + "cross_region_cost": 0.0, + "failure_recovery_steps": 0.0, + } + + +def independent_score(metrics: dict[str, float | str]) -> tuple[float, dict[str, float]]: + """Independent transcription of the current production scoring equation.""" + + budgets = load_config()["score_budgets"] + + def ratio(value: float, budget: float, maximum: float = 3.0) -> float: + return min(maximum, max(0.0, value) / max(budget, 1e-12)) + + latency_reference_ms = 180.0 # median P99 SLO in the production task configuration + components = { + "reliability": ratio(float(metrics["unserved_rate"]), float(budgets["unserved_rate"])), + "sla": ratio(float(metrics["p99_slo_violation_rate"]), float(budgets["slo_violation_rate"])), + "tail_latency": ratio(float(metrics["request_weighted_p99_ms"]), latency_reference_ms, 2.0), + "compute": ratio(float(metrics["compute_cost"]), float(budgets["compute_cost"])), + "bandwidth": ratio(float(metrics["cross_region_gb"]), float(budgets["cross_region_gb"])), + "recovery": ratio(float(metrics["failure_recovery_steps"]), float(budgets["recovery_steps"]), 2.0), + } + loss = ( + 0.40 * components["reliability"] + + 0.30 * components["sla"] + + 0.10 * components["tail_latency"] + + 0.12 * components["compute"] + + 0.04 * components["bandwidth"] + + 0.04 * components["recovery"] + ) + return 100.0 * math.exp(-loss), components + + +def brute_force_oracle() -> dict[str, Any]: + rows: list[tuple[float, str, list[dict[str, Any]], dict[str, float | str], dict[str, float]]] = [] + for actions in enumerate_action_sequences(): + metrics = independent_metrics(actions) + score, components = independent_score(metrics) + canonical = json.dumps(actions, sort_keys=True, separators=(",", ":")) + rows.append((score, canonical, actions, metrics, components)) + best = min(rows, key=lambda row: (-row[0], row[1])) + return { + "enumerated_trajectories": len(rows), + "best_action_sequence": best[2], + "raw_metrics": best[3], + "combined_score": best[0], + "normalized_loss_components": best[4], + } + + +if __name__ == "__main__": + print(json.dumps(brute_force_oracle(), indent=2, ensure_ascii=False, allow_nan=False)) diff --git a/benchmarks/ComputerSystems/README.md b/benchmarks/ComputerSystems/README.md index 411962ac..0690c61c 100644 --- a/benchmarks/ComputerSystems/README.md +++ b/benchmarks/ComputerSystems/README.md @@ -3,5 +3,6 @@ Includes computer-systems engineering optimization tasks: - `MallocLab`: dynamic memory allocation. - `DuckDBWorkloadOptimization`: analytical SQL workload tuning (index/materialized-view selection + query rewrite). +- `EdgeServiceReplicaPlacement`: dynamic edge-service replica placement and traffic routing under bursts, failures, and link degradation. Note for contributors: ensure the evolved baseline source file contains `EVOLVE-BLOCK-START` / `EVOLVE-BLOCK-END` markers (use `// ...` in C/C++). diff --git a/benchmarks/ComputerSystems/README_zh-CN.md b/benchmarks/ComputerSystems/README_zh-CN.md index 0d12dfc1..15980e75 100644 --- a/benchmarks/ComputerSystems/README_zh-CN.md +++ b/benchmarks/ComputerSystems/README_zh-CN.md @@ -3,5 +3,6 @@ 包含以下计算机系统工程优化任务： - `MallocLab`：动态内存分配。 - `DuckDBWorkloadOptimization`：分析型 SQL 负载调优（索引/物化视图选择 + 查询改写）。 +- `EdgeServiceReplicaPlacement`：在流量突发、节点故障和链路退化下动态放置边缘服务副本并调度请求路由。 贡献提示：请确保被 evolve 的 baseline 源码文件包含 `EVOLVE-BLOCK-START` / `EVOLVE-BLOCK-END` 标记（C/C++ 中使用 `// ...`）。 From 7327570cd052f474362a0c5927594a1bc7208870 Mon Sep 17 00:00:00 2001 From: Mike <2025013664@nwafu.edu.cn> Date: Tue, 8 Sep 2026 20:51:41 +0800 Subject: [PATCH 2/2] docs: add optimization validation results --- .../baseline/result_log.txt | 34 +++++++++++++------ .../docs/scoring-calibration-report.md | 2 +- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/baseline/result_log.txt b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/baseline/result_log.txt index 0eeb9555..d4f6e16a 100644 --- a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/baseline/result_log.txt +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/baseline/result_log.txt @@ -58,7 +58,7 @@ Determinism check Adversarial/unit suite command: python -m pytest benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification -q - result: 36 passed in 9.95 s; exit 0; command wall time 10.78 s + result: 36 passed in 11.44 s; exit 0; command wall time 12.56 s Multi-timestep tiny oracle command: python benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/verification/tiny_oracle.py @@ -69,25 +69,27 @@ Multi-timestep tiny oracle compute=0.0916666667, bandwidth=0.0, recovery=0.0 combined_score: 96.62493678284117 independent oracle and production evaluator matched exactly - command wall time: 0.52 s; exit 0 + command wall time: 0.26 s; exit 0 Direct evaluator valid=true; 10/10 scenarios; score=73.9434 - command wall time: 1.96 s; exit 0 + command wall time: 1.31 s; exit 0 frontier_eval/run_eval.sh valid=true; 10/10 scenarios; score=73.9434 - command wall time: 2.22 s; exit 0 + command wall time: 1.68 s; exit 0 Unified zero-iteration valid=1; benchmark_returncode=0; score=73.9434 - benchmark runtime_s=2.8343; complete command wall time=5.52 s; exit 0 + benchmark runtime_s=3.0958; complete command wall time=6.17 s; exit 0 Windows-specific invocation used Git Bash plus an explicit existing Python 3.12 executable. No machine-specific absolute path is stored in task metadata. Metadata/readonly audit - repository audit command exited 0 in non-strict mode. It printed warnings for existing + repository audit command exited 0 in standard mode. It printed warnings for existing tasks elsewhere in the repository; EdgeServiceReplicaPlacement was not listed. + Strict mode additionally treats repository-wide optional recommendations as failures; + it is not used as the task pass criterion. Scoring sensitivity Each component weight was varied independently by -5%, -2%, +2%, and +5%, with @@ -99,10 +101,20 @@ Agent optimization command: python -m frontier_eval task=unified task.benchmark=ComputerSystems/EdgeServiceReplicaPlacement algorithm=openevolve algorithm.iterations=10 [Windows runtime overrides] - result: NOT EXECUTED; exit 1 before iteration 0; command wall time 2.75 s - reason: OPENAI_API_KEY (or llm.api_key) is unavailable in this environment. - No optimization trajectory or learnability claim is made. + purpose: generative-optimization sanity check using an OpenAI-compatible LLM backend + parser success: 10/10 model responses + evaluator-valid candidates: 4/10 + evaluator-invalid candidates: 6/10 due to candidate constraint violations + API timeouts: 0 + initial score: 73.9434 + handwritten strong calibration score: 74.4628 + best evolved score: 74.6663 + first and best improvement: iteration 6; remained best through iteration 10 + best raw metrics: availability=0.995251, P99=47.237 ms, + SLO violation=0.027664, compute cost=4.342, + cross-region=1.027 GB, recovery=0.2 steps + The benchmark itself does not depend on the provider used for this local experiment. + This run is a sanity check, not evidence of general model performance. These values demonstrate a deterministic, executable benchmark and calibrated score. -They do not claim production validity. A real multi-iteration Agent run remains required -by the current PR-readiness gate. +They do not claim production validity or comprehensive model performance. diff --git a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/scoring-calibration-report.md b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/scoring-calibration-report.md index e1744b89..1f4d1c44 100644 --- a/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/scoring-calibration-report.md +++ b/benchmarks/ComputerSystems/EdgeServiceReplicaPlacement/docs/scoring-calibration-report.md @@ -159,7 +159,7 @@ an engineering-preference change, not a two-percent relative perturbation. | tail signal | all calibration policies exactly 0 | below-180 values clipped | normalize P99 continuously by median configured SLO | nonzero 0.18–0.27 mean components; raw metrics unchanged | | compute clipping | 13.95 cost normalized to 2.0 | cap below observed extreme | cap 3.0 | 13.95 normalizes to 2.325; full policies no longer win | | bandwidth clipping | traffic above 2 GB/scenario free after cap | cap compressed remote-routing differences | cap 3.0 | more continuous penalty; aggressive remote policy remains low | -| calibration ranking | weak 76.1354 < reasonable 76.4882 < strong 77.4852 | old normalization | no ranking-targeted tuning | strong 74.4628 > weak 74.0766 > reasonable 73.9434, reflecting explicit trade-offs | +| calibration ranking | pre-correction scores (invalidated) | old normalization | no ranking-targeted tuning | strong 74.4628 > weak 74.0766 > reasonable 73.9434, reflecting explicit trade-offs | The final ranking is not encoded as a required test. Tests require valid, deterministic, distinct calibration policies rather than forcing a subjective preference ordering.