From ccf51f7b2ae9498539e151cc7feb5b1df06543ff Mon Sep 17 00:00:00 2001 From: ahydchh <1550666251@qq.com> Date: Tue, 15 Sep 2026 12:17:46 +0800 Subject: [PATCH] Isolate Malloc candidates in Wasm64 and refresh scores --- README.md | 4 +- README_zh-CN.md | 4 +- .../ComputerSystems/MallocLab/README.md | 9 + .../ComputerSystems/MallocLab/README_zh-CN.md | 9 + benchmarks/ComputerSystems/MallocLab/Task.md | 13 +- .../ComputerSystems/MallocLab/Task_zh-CN.md | 11 +- .../frontier_eval/artifact_files.txt | 6 +- .../MallocLab/frontier_eval/constraints.txt | 13 +- .../MallocLab/frontier_eval/copy_files.txt | 1 - .../frontier_eval/parse_mdriver_result.py | 120 ---- .../frontier_eval/readonly_files.txt | 1 - .../MallocLab/frontier_eval/run_eval.sh | 65 +-- .../MallocLab/malloclab-handout/mdriver.c | 91 +-- benchmarks/_shared/malloc_isolation.py | 195 +++++++ .../_shared/malloc_wasm/allowed_imports.txt | 5 + benchmarks/_shared/malloc_wasm/guest.c | 38 ++ benchmarks/_shared/malloc_wasm/host.c | 523 ++++++++++++++++++ .../_shared/malloc_wasm/include/assert.h | 8 + .../_shared/malloc_wasm/include/limits.h | 10 + .../_shared/malloc_wasm/include/stddef.h | 7 + .../_shared/malloc_wasm/include/stdint.h | 15 + .../_shared/malloc_wasm/include/stdio.h | 10 + .../_shared/malloc_wasm/include/stdlib.h | 10 + .../_shared/malloc_wasm/include/string.h | 9 + .../_shared/malloc_wasm/include/unistd.h | 5 + benchmarks/_shared/malloc_wasm/setup.py | 234 ++++++++ .../tasks/malloclab/evaluator/python.py | 307 +--------- leaderboard/README.md | 4 +- leaderboard/exp1_models_raw.csv | 2 +- leaderboard/medal_leaderboard.csv | 4 +- leaderboard/medal_podium.csv | 2 +- leaderboard/submission_example.csv | 2 +- 32 files changed, 1159 insertions(+), 578 deletions(-) delete mode 100644 benchmarks/ComputerSystems/MallocLab/frontier_eval/parse_mdriver_result.py create mode 100644 benchmarks/_shared/malloc_isolation.py create mode 100644 benchmarks/_shared/malloc_wasm/allowed_imports.txt create mode 100644 benchmarks/_shared/malloc_wasm/guest.c create mode 100644 benchmarks/_shared/malloc_wasm/host.c create mode 100644 benchmarks/_shared/malloc_wasm/include/assert.h create mode 100644 benchmarks/_shared/malloc_wasm/include/limits.h create mode 100644 benchmarks/_shared/malloc_wasm/include/stddef.h create mode 100644 benchmarks/_shared/malloc_wasm/include/stdint.h create mode 100644 benchmarks/_shared/malloc_wasm/include/stdio.h create mode 100644 benchmarks/_shared/malloc_wasm/include/stdlib.h create mode 100644 benchmarks/_shared/malloc_wasm/include/string.h create mode 100644 benchmarks/_shared/malloc_wasm/include/unistd.h create mode 100644 benchmarks/_shared/malloc_wasm/setup.py diff --git a/README.md b/README.md index aea45db1..460fe632 100644 --- a/README.md +++ b/README.md @@ -130,9 +130,9 @@ Detailed leaderboard (incl. average rank): [lab.einsia.ai/frontier-eng/leaderboa | 1 | Claude Opus 4.6 | 0.533 | 0.501 | 14 | 15 | 3 | | 2 | GPT-5.4 | 0.454 | 0.267 | 18 | 4 | 2 | | 3 | GLM-5 | 0.347 | 0.300 | 7 | 8 | 12 | -| 4 | Gemini 3.1 Pro Preview | 0.277 | 0.267 | 7 | 7 | 4 | +| 4 | Gemini 3.1 Pro Preview | 0.284 | 0.300 | 7 | 7 | 5 | | 5 | DeepSeek V3.2 | 0.269 | 0.299 | 6 | 6 | 8 | -| 6 | Grok 4.20 | 0.227 | 0.200 | 6 | 5 | 4 | +| 6 | Grok 4.20 | 0.220 | 0.167 | 6 | 5 | 3 | | 7 | Seed 2.0 Pro | 0.206 | 0.100 | 6 | 4 | 3 | | 8 | Qwen3 Coder Next | 0.170 | 0.066 | 5 | 3 | 3 | diff --git a/README_zh-CN.md b/README_zh-CN.md index bcd0c6b7..fdb21836 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -125,9 +125,9 @@ bash scripts/batch/validate_v1_task_envs.sh | 1 | Claude Opus 4.6 | 0.533 | 0.501 | 14 | 15 | 3 | | 2 | GPT-5.4 | 0.454 | 0.267 | 18 | 4 | 2 | | 3 | GLM-5 | 0.347 | 0.300 | 7 | 8 | 12 | -| 4 | Gemini 3.1 Pro Preview | 0.277 | 0.267 | 7 | 7 | 4 | +| 4 | Gemini 3.1 Pro Preview | 0.284 | 0.300 | 7 | 7 | 5 | | 5 | DeepSeek V3.2 | 0.269 | 0.299 | 6 | 6 | 8 | -| 6 | Grok 4.20 | 0.227 | 0.200 | 6 | 5 | 4 | +| 6 | Grok 4.20 | 0.220 | 0.167 | 6 | 5 | 3 | | 7 | Seed 2.0 Pro | 0.206 | 0.100 | 6 | 4 | 3 | | 8 | Qwen3 Coder Next | 0.170 | 0.066 | 5 | 3 | 3 | diff --git a/benchmarks/ComputerSystems/MallocLab/README.md b/benchmarks/ComputerSystems/MallocLab/README.md index 1aa95484..336d0458 100644 --- a/benchmarks/ComputerSystems/MallocLab/README.md +++ b/benchmarks/ComputerSystems/MallocLab/README.md @@ -5,3 +5,12 @@ The relevant files are located in `benchmarks/ComputerSystems/MallocLab/mallocla For more details, please see [Task](Task.md). Note: the evolved candidate file is `malloclab-handout/mm.c`. Keep function signatures unchanged, and keep `// EVOLVE-BLOCK-START` / `// EVOLVE-BLOCK-END` markers in place so evolution algorithms can safely apply diffs. + +Official scoring uses a Wasm64 allocator with a trusted host driver. Install the Linux x86-64 toolchain once from the repository root: + +```bash +python benchmarks/_shared/malloc_wasm/setup.py --install +bash benchmarks/ComputerSystems/MallocLab/frontier_eval/run_eval.sh python3 benchmarks/ComputerSystems/MallocLab +``` + +The score measures calls in the isolated runtime; native `make && ./mdriver -V` remains available for local debugging. The runtime keeps 64-bit pointers and the 20 MiB simulated heap. It requires Linux user namespaces, bubblewrap, and a native C compiler. `FRONTIER_MALLOC_TOOLCHAIN` selects an alternate toolchain installation directory. diff --git a/benchmarks/ComputerSystems/MallocLab/README_zh-CN.md b/benchmarks/ComputerSystems/MallocLab/README_zh-CN.md index 88c40f80..d0c19b64 100644 --- a/benchmarks/ComputerSystems/MallocLab/README_zh-CN.md +++ b/benchmarks/ComputerSystems/MallocLab/README_zh-CN.md @@ -5,3 +5,12 @@ 更多详细信息请查看 [Task](Task_zh-CN.md) 提示:被 evolve 的候选文件为 `malloclab-handout/mm.c`。请保持函数签名不变,并保留 `// EVOLVE-BLOCK-START` / `// EVOLVE-BLOCK-END` 标记,便于演化算法安全地应用 diff。 + +正式评分将分配器编译为 Wasm64,由模块外的可信驱动进行验证和计时。在仓库根目录安装一次 Linux x86-64 工具链: + +```bash +python benchmarks/_shared/malloc_wasm/setup.py --install +bash benchmarks/ComputerSystems/MallocLab/frontier_eval/run_eval.sh python3 benchmarks/ComputerSystems/MallocLab +``` + +分数使用隔离运行时中的调用耗时;原生 `make && ./mdriver -V` 可用于本地调试。运行时保留 64 位指针和 20 MiB 模拟堆,需要 Linux 用户命名空间、bubblewrap 和本机 C 编译器。可用 `FRONTIER_MALLOC_TOOLCHAIN` 指定工具链安装目录。 diff --git a/benchmarks/ComputerSystems/MallocLab/Task.md b/benchmarks/ComputerSystems/MallocLab/Task.md index 04b939ab..5ee7d57a 100644 --- a/benchmarks/ComputerSystems/MallocLab/Task.md +++ b/benchmarks/ComputerSystems/MallocLab/Task.md @@ -113,9 +113,9 @@ The `memlib.c` package simulates a memory system for the dynamic memory allocato * Interface functions in `mm.c` must not be modified. -* `mm.c` must not read standard input. +* Official scoring provides no file, network, process, standard-input, or clock interfaces. -* System library functions must not be called. +* System allocation functions (`malloc`, `free`, `realloc`, `sbrk`, `mmap`) must not be called; obtain heap space through `mem_sbrk`. The runtime supports `memcpy`, `memmove`, `memset`, `memcmp`, and `strlen`. * Global or static composite data structures, such as arrays, structures, trees, or lists, must not be defined in the `mm.c` program. However, global scalar variables, such as integers, floating-point numbers, and pointers, can be declared in `mm.c`. @@ -123,16 +123,17 @@ The `memlib.c` package simulates a memory system for the dynamic memory allocato ## Scoring Criteria -The evaluator reads the result file written by `mdriver`. The allocator and -driver execute in the same process and share an address space. +Official scoring uses a Wasm64 allocator and a trusted host driver. The host owns the traces, heap high-water mark, and score, and checks allocation bounds, alignment, overlap, and preserved data. Each passing trace is independently executed and validated ten times; throughput uses the median allocator-call time measured by the host. Compilation, runtime initialization, and host payload checks are excluded from call time. Failed traces contribute no utilization or completed operations. + +See the task README for the scoring command. Native `mdriver` remains a local debugging tool; its timings are not directly comparable to isolated-runtime scores. * Space Utilization: The ratio between the maximum amount of memory used by the program and the maximum heap size used by the allocator; the optimal ratio is 1. * Throughput: Kops (kilo operations per second) -* Scoring Formula: $$P = wU + (1 - w)\min(1, \frac{T}{T_{libc}})$$ +* Scoring Formula: $$P = 100\left(wU + (1-w)\min(1,T/T_{ref})\right)m/N$$ -* where w is space utilization, and $T_{libc}$ is throughput. $T_{libc}$ is the throughput of libc malloc tested by the teaching assistant on the course cluster. The specific value is based on `AVG_LIBC_THRUPUT` in `config.h`. A balance needs to be considered when optimizing space utilization and throughput. +* Here $w=0.6$, $N=11$, $m$ is the number of passing traces, and $U$ is mean utilization across all traces (zero for failed traces). $T$ is completed operations divided by the sum of median call times for passing traces. The fixed reference cap is $T_{ref}=10{,}000{,}000$ operations/second. It is a score normalization constant, not a fresh measurement of native libc performance. ## Some Suggestions diff --git a/benchmarks/ComputerSystems/MallocLab/Task_zh-CN.md b/benchmarks/ComputerSystems/MallocLab/Task_zh-CN.md index 8f55b9b6..f59c1649 100644 --- a/benchmarks/ComputerSystems/MallocLab/Task_zh-CN.md +++ b/benchmarks/ComputerSystems/MallocLab/Task_zh-CN.md @@ -85,19 +85,22 @@ void *mm_realloc(void *ptr, size_t size); * 使用方式可通过 `./mdriver -h` 查看。其中 `-V` 可用于定位报错出现的文件,`-f` 可用于指定 trace 进行测试。 ## 编程规则 -* `mm.c` 不得读取标准输入。 +* 正式评分不提供文件、网络、进程、标准输入或时钟接口。 * 不允许改变 `mm.c` 的接口函数 -* 不允许调用系统的库函数 +* 不允许调用系统内存分配函数(如 `malloc`、`free`、`realloc`、`sbrk`、`mmap`);分配堆空间必须使用 `mem_sbrk`。支持 `memcpy`、`memmove`、`memset`、`memcmp` 和 `strlen`。 * 不允许在 `mm.c` 程序中定义全局或静态的复合数据结构,如数组、结构、树或列表。但是可以在 `mm.c` 中声明全局标量变量,如整数、浮点数和指针。 * 返回的内存块应 16 字节对齐 ## 评分标准 -评测器读取 `mdriver` 写出的结果文件。分配器和驱动在同一进程中执行,共享地址空间。 +正式评分使用 Wasm64 分配器和模块外的可信驱动。驱动掌握测试序列、堆高水位和分数,验证返回内存的边界、对齐、重叠及数据保留。每条通过的 trace 独立执行并验证 10 次,吞吐量使用外部测得的分配器调用耗时中位数;编译、初始化运行时和宿主数据检查不计入调用耗时。失败的 trace 不贡献利用率或已完成操作数。 + +运行方式见本题 README。原生 `mdriver` 用于本地调试,其计时结果与隔离运行时分数不直接比较。 * 空间利用率:程序使用的最大内存量与分配器使用的最大堆大小之间的比率,最佳比率为 1。 * 吞吐量:Kops (kilo operations per second) -* 评分公式: $$P = wU + (1 - w)\min(1, \frac{T}{T_{libc}})$$ +* 评分公式: $$P = 100\left(wU + (1-w)\min(1,T/T_{ref})\right)m/N$$ +* 其中 $w=0.6$、$N=11$,$m$ 为通过的 trace 数,$U$ 为全部 trace 的平均利用率(失败项为零)。$T$ 为通过项的操作总数除以调用耗时中位数之和。固定归一化上限 $T_{ref}=10{,}000{,}000$ 次/秒,不代表本次重新测量了原生 libc 的性能。 * 为空间利用率, 为吞吐量 (throughput), $T_{libc}$ 是助教在课程集群上测试的 libc malloc 的吞吐量,具体值以 `config.h` 的 `AVG_LIBC_THRUPUT` 为准。,需要均衡地考虑空间利用率和吞吐量的优化。 ## 一些建议 diff --git a/benchmarks/ComputerSystems/MallocLab/frontier_eval/artifact_files.txt b/benchmarks/ComputerSystems/MallocLab/frontier_eval/artifact_files.txt index d032fc28..d8af6da2 100644 --- a/benchmarks/ComputerSystems/MallocLab/frontier_eval/artifact_files.txt +++ b/benchmarks/ComputerSystems/MallocLab/frontier_eval/artifact_files.txt @@ -1,5 +1 @@ -run_meta.txt -make_clean.log -make.log -mdriver.stdout.txt -mdriver.stderr.txt +malloc_details.json diff --git a/benchmarks/ComputerSystems/MallocLab/frontier_eval/constraints.txt b/benchmarks/ComputerSystems/MallocLab/frontier_eval/constraints.txt index e24ec82a..720436f6 100644 --- a/benchmarks/ComputerSystems/MallocLab/frontier_eval/constraints.txt +++ b/benchmarks/ComputerSystems/MallocLab/frontier_eval/constraints.txt @@ -8,9 +8,10 @@ MallocLab UnifiedTask constraints: 3) Do not modify benchmark runner files (`mdriver.c`, trace files, build scripts). These are enforced read-only and fingerprinted; changing one invalidates the run. 4) Candidate should target correctness first, then optimize throughput/utilization. -5) Evaluator compiles with `make` and runs `./mdriver -V -o `. -6) The score is read from the result file mdriver writes, NOT from its stdout. - The record only counts if it carries the per-run token the grader hands - mdriver on stdin. Printing a `Score = ... = N/100` line yourself does - nothing; consuming stdin before mdriver's main() reads it aborts the run - with a zero. mm.c must not read stdin. +5) Official evaluation compiles C to Wasm64 with 64-bit pointers and size_t. + The host owns the 20 MiB simulated heap limit, correctness checks and timers. +6) Obtain heap space through mem_sbrk. System malloc/free/realloc/sbrk/mmap + and file, process, network, standard-input or clock interfaces are unavailable. +7) Preserve data across realloc and do not overwrite other live allocations. + A trusted host computes scores from validated operations; candidate output + is not a scoring channel. Native mdriver is for local debugging only. diff --git a/benchmarks/ComputerSystems/MallocLab/frontier_eval/copy_files.txt b/benchmarks/ComputerSystems/MallocLab/frontier_eval/copy_files.txt index 15b45dc6..28216dba 100644 --- a/benchmarks/ComputerSystems/MallocLab/frontier_eval/copy_files.txt +++ b/benchmarks/ComputerSystems/MallocLab/frontier_eval/copy_files.txt @@ -1,6 +1,5 @@ malloclab-handout frontier_eval/run_eval.sh -frontier_eval/parse_mdriver_result.py frontier_eval/constraints.txt frontier_eval/artifact_files.txt Task_zh-CN.md diff --git a/benchmarks/ComputerSystems/MallocLab/frontier_eval/parse_mdriver_result.py b/benchmarks/ComputerSystems/MallocLab/frontier_eval/parse_mdriver_result.py deleted file mode 100644 index 6a2c72b3..00000000 --- a/benchmarks/ComputerSystems/MallocLab/frontier_eval/parse_mdriver_result.py +++ /dev/null @@ -1,120 +0,0 @@ -from __future__ import annotations - -import argparse -import hmac -import json -import math -from pathlib import Path -from typing import Any - - -def _read_text(path: Path) -> str: - if not path.is_file(): - return "" - return path.read_text(encoding="utf-8", errors="replace") - - -def _write_json(path: Path, obj: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(obj, ensure_ascii=False, indent=2, default=str) + "\n", encoding="utf-8") - - -def _parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description="Parse MallocLab mdriver output to metrics.json.") - p.add_argument("--result-file", type=str, required=True) - p.add_argument("--expected-token", type=str, required=True) - p.add_argument("--stdout-file", type=str, required=True) - p.add_argument("--stderr-file", type=str, required=True) - p.add_argument("--mdriver-returncode", type=int, required=True) - p.add_argument("--metrics-out", type=str, required=True) - return p.parse_args() - - -def _finite(value: Any) -> float | None: - """Accept only a real, finite number. Rejects bool, NaN and +-Inf.""" - if isinstance(value, bool) or not isinstance(value, (int, float)): - return None - out = float(value) - return out if math.isfinite(out) else None - - -def read_result(result_text: str, expected_token: str) -> tuple[dict[str, float] | None, str]: - """Validate the record mdriver wrote and return its fields. - - The score is never taken from stdout. mm.c is linked into mdriver, so it can - print whatever it likes there -- and the old parser scanned stdout for the - last "Score = ... = N/100" line, which made a single extra printf a perfect - score. A record only counts if it carries the token the grader handed - mdriver on stdin. - """ - if not result_text.strip(): - return None, "mdriver wrote no result record" - try: - record = json.loads(result_text) - except Exception as exc: - return None, f"result record is not valid JSON: {exc}" - if not isinstance(record, dict): - return None, "result record must be a JSON object" - - token = record.get("run_token") - if not isinstance(token, str) or not hmac.compare_digest(token, expected_token): - return None, "result record does not carry this run's token" - - score = _finite(record.get("score_100")) - if score is None: - return None, "result record has no finite score_100" - if not 0.0 <= score <= 100.0: - return None, f"score_100 out of range: {score}" - - passed = _finite(record.get("testcases_passed")) - total = _finite(record.get("testcases_total")) - errors = _finite(record.get("errors")) - if passed is None or total is None or total <= 0 or not 0.0 <= passed <= total: - return None, "result record has an implausible testcase count" - if errors is None or errors < 0: - return None, "result record has an implausible error count" - # A failing trace is a normal outcome, not an invalid run: mdriver already - # prices it in by scaling the score by numcorrect/num_tracefiles. The - # shipped baseline fails 5 of 11 and scores ~28. - - metrics = { - "score_100": score, - "score_ratio": score / 100.0, - "testcases_passed": passed, - "testcases_total": total, - "testcase_pass_rate": passed / total, - "errors": errors, - } - for key in ("util_points", "thru_points"): - value = _finite(record.get(key)) - if value is not None: - metrics[key] = value - return metrics, "" - - -def main() -> int: - args = _parse_args() - result_text = _read_text(Path(args.result_file).expanduser().resolve()) - - metrics: dict[str, float] = { - "combined_score": 0.0, - "valid": 0.0, - "mdriver_returncode": float(args.mdriver_returncode), - } - - parsed, error_message = read_result(result_text, args.expected_token) - if parsed is None: - metrics["error_message"] = error_message - elif int(args.mdriver_returncode) != 0: - metrics["error_message"] = f"mdriver exited {args.mdriver_returncode}" - else: - metrics.update(parsed) - metrics["valid"] = 1.0 - metrics["combined_score"] = parsed["score_100"] - - _write_json(Path(args.metrics_out).expanduser().resolve(), metrics) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/ComputerSystems/MallocLab/frontier_eval/readonly_files.txt b/benchmarks/ComputerSystems/MallocLab/frontier_eval/readonly_files.txt index 9fd6363f..5f89c437 100644 --- a/benchmarks/ComputerSystems/MallocLab/frontier_eval/readonly_files.txt +++ b/benchmarks/ComputerSystems/MallocLab/frontier_eval/readonly_files.txt @@ -25,4 +25,3 @@ README_zh-CN.md README.md frontier_eval/constraints.txt frontier_eval/run_eval.sh -frontier_eval/parse_mdriver_result.py diff --git a/benchmarks/ComputerSystems/MallocLab/frontier_eval/run_eval.sh b/benchmarks/ComputerSystems/MallocLab/frontier_eval/run_eval.sh index a9460d2d..0550be39 100644 --- a/benchmarks/ComputerSystems/MallocLab/frontier_eval/run_eval.sh +++ b/benchmarks/ComputerSystems/MallocLab/frontier_eval/run_eval.sh @@ -1,60 +1,15 @@ #!/usr/bin/env bash set -euo pipefail -PYTHON_CMD="${1:?missing python command}" -BENCHMARK_DIR="${2:?missing benchmark dir}" -CANDIDATE_PATH="${3:-}" - -HANDOUT_DIR="${BENCHMARK_DIR}/malloclab-handout" -MAKE_CLEAN_LOG="${BENCHMARK_DIR}/make_clean.log" -MAKE_LOG="${BENCHMARK_DIR}/make.log" -MDRIVER_STDOUT="${BENCHMARK_DIR}/mdriver.stdout.txt" -MDRIVER_STDERR="${BENCHMARK_DIR}/mdriver.stderr.txt" -MDRIVER_RESULT="${BENCHMARK_DIR}/mdriver_result.json" -METRICS_JSON="${BENCHMARK_DIR}/metrics.json" - -# Per-run token for the authenticated result channel. -# -# The candidate's mm.c is compiled into mdriver, so it can write anything it -# likes to mdriver's stdout -- and the score used to be parsed from there. It -# now travels in ${MDRIVER_RESULT}, which only counts if it carries this token. -# -# The token is a shell variable, never exported and never written to disk while -# mdriver runs, so it is not in mdriver's environ and not readable from the -# filesystem. It reaches mdriver on stdin, which mdriver consumes and closes -# before it calls into the allocator, and it reaches the parser on a command -# line that is only built after mdriver has already exited. -RUN_TOKEN="$(od -An -N32 -tx1 /dev/urandom | tr -d ' \n')" -if [[ -z "${RUN_TOKEN}" ]]; then - echo "ERROR: could not generate a run token" >&2 - exit 1 +PYTHON_CMD="${1:-python3}" +BENCHMARK_DIR="${2:?benchmark directory is required}" +CANDIDATE="${3:-${BENCHMARK_DIR}/malloclab-handout/mm.c}" +REPO_ROOT="${FRONTIER_ENGINEERING_ROOT:-}" +if [[ -z "${REPO_ROOT}" ]]; then + REPO_ROOT="$(cd "${BENCHMARK_DIR}/../../.." && pwd)" fi -rm -f "${MDRIVER_RESULT}" - -cd "${HANDOUT_DIR}" - -make clean >"${MAKE_CLEAN_LOG}" 2>&1 -make >"${MAKE_LOG}" 2>&1 - -set +e -printf '%s\n' "${RUN_TOKEN}" \ - | ./mdriver -V -o "${MDRIVER_RESULT}" >"${MDRIVER_STDOUT}" 2>"${MDRIVER_STDERR}" -MDRIVER_RC=$? -set -e - -{ - echo "candidate_path=${CANDIDATE_PATH}" - echo "mdriver_returncode=${MDRIVER_RC}" -} > "${BENCHMARK_DIR}/run_meta.txt" - -"${PYTHON_CMD}" "${BENCHMARK_DIR}/frontier_eval/parse_mdriver_result.py" \ - --result-file "${MDRIVER_RESULT}" \ - --expected-token "${RUN_TOKEN}" \ - --stdout-file "${MDRIVER_STDOUT}" \ - --stderr-file "${MDRIVER_STDERR}" \ - --mdriver-returncode "${MDRIVER_RC}" \ - --metrics-out "${METRICS_JSON}" - -# Always return 0 here: parsed `metrics.json` already encodes validity/score. -exit 0 +exec "${PYTHON_CMD}" "${REPO_ROOT}/benchmarks/_shared/malloc_isolation.py" \ + "${CANDIDATE}" --benchmark "${BENCHMARK_DIR}" \ + --metrics-out "${BENCHMARK_DIR}/metrics.json" \ + --details-out "${BENCHMARK_DIR}/malloc_details.json" diff --git a/benchmarks/ComputerSystems/MallocLab/malloclab-handout/mdriver.c b/benchmarks/ComputerSystems/MallocLab/malloclab-handout/mdriver.c index f6f5f751..6c2eba34 100644 --- a/benchmarks/ComputerSystems/MallocLab/malloclab-handout/mdriver.c +++ b/benchmarks/ComputerSystems/MallocLab/malloclab-handout/mdriver.c @@ -132,72 +132,6 @@ static void unix_error(char *msg); static void malloc_error(int tracenum, int opnum, char *msg); static void app_error(char *msg); -/******************************************************************* - * Authenticated result channel - * - * The score used to travel to the grader over stdout, which mm.c -- - * linked into this very binary -- can write to. A single extra - * printf("Score = ... = 100/100") after ours was a perfect score, - * because the parser takes the last matching line. - * - * So the grader now generates a per-run token, hands it to us on stdin, - * and reads the result from the file named by -o. Anything not carrying - * the token is not a result. main() consumes and closes stdin before it - * touches the allocator, so no code reachable from mm_init/mm_malloc/ - * mm_free/mm_realloc can obtain it. - * - * Known limit: a __attribute__((constructor)) in mm.c runs before main() - * and can read stdin first. read_run_token() then sees an empty stdin and - * aborts the run, so that attempt is loud rather than silent -- but a - * candidate that re-supplies the token on fd 0 defeats this. Closing that - * properly needs the allocator out of the driver's address space, which - * this benchmark's premise does not allow. See README. - *******************************************************************/ -static char run_token[128]; - -static void read_run_token(void) { - size_t n; - - if (fgets(run_token, (int)sizeof(run_token), stdin) == NULL) { - fprintf(stderr, - "ERROR: no run token on stdin. The grader supplies one; if it is " - "missing here it was consumed before main() ran.\n"); - exit(2); - } - n = strlen(run_token); - while (n > 0 && (run_token[n - 1] == '\n' || run_token[n - 1] == '\r')) - run_token[--n] = '\0'; - if (n == 0) { - fprintf(stderr, "ERROR: empty run token on stdin.\n"); - exit(2); - } - /* Nothing downstream needs stdin; take it away so mm.c cannot re-read it. */ - if (freopen("/dev/null", "r", stdin) == NULL) - fclose(stdin); -} - -static void write_result_file(const char *path, double p1, double p2, - double score, int numcorrect, int num_tracefiles, - int errors) { - FILE *f = fopen(path, "w"); - if (f == NULL) { - fprintf(stderr, "ERROR: cannot open result file %s: %s\n", path, - strerror(errno)); - exit(2); - } - fprintf(f, - "{\"run_token\": \"%s\", \"util_points\": %.6f, \"thru_points\": " - "%.6f, \"score_100\": %.6f, \"testcases_passed\": %d, " - "\"testcases_total\": %d, \"errors\": %d}\n", - run_token, p1 * 100.0, p2 * 100.0, score, numcorrect, num_tracefiles, - errors); - if (fclose(f) != 0) { - fprintf(stderr, "ERROR: cannot write result file %s: %s\n", path, - strerror(errno)); - exit(2); - } -} - /************** * Main routine **************/ @@ -215,26 +149,16 @@ int main(int argc, char **argv) { int team_check = 1; /* If set, check team structure (reset by -a) */ int run_libc = 0; /* If set, run libc malloc (set by -l) */ int autograder = 0; /* If set, emit summary info for autograder (-g) */ - char *result_path = NULL; /* -o: authenticated result file for the grader */ /* temporaries used to compute the performance index */ double secs, ops, util, avg_mm_util, avg_mm_throughput, p1, p2, score; int numcorrect; - /* - * Take the run token off stdin before anything else. This must stay the - * first statement in main(): everything after it may reach mm.c. - */ - read_run_token(); - /* * Read and interpret the command line arguments */ - while ((c = getopt(argc, argv, "f:t:o:hvVgal")) != EOF) { + while ((c = getopt(argc, argv, "f:t:hvVgal")) != EOF) { switch (c) { - case 'o': /* Write the authenticated result record here */ - result_path = optarg; - break; case 'g': /* Generate summary info for the autograder */ autograder = 1; break; @@ -476,14 +400,6 @@ int main(int argc, char **argv) { printf("score:%.0f\n", score); } - /* - * The number above is for humans. The grader reads this file, and scores - * nothing if it is absent or does not carry the run token. - */ - if (result_path != NULL) - write_result_file(result_path, p1, p2, score, numcorrect, num_tracefiles, - errors); - exit(0); } @@ -1082,16 +998,13 @@ void malloc_error(int tracenum, int opnum, char *msg) { * usage - Explain the command line arguments */ static void usage(void) { - fprintf(stderr, - "Usage: mdriver [-hvVal] [-f ] [-t ] [-o ]\n"); + fprintf(stderr, "Usage: mdriver [-hvVal] [-f ] [-t ]\n"); fprintf(stderr, "Options\n"); fprintf(stderr, "\t-a Don't check the team structure.\n"); fprintf(stderr, "\t-f Use as the trace file.\n"); fprintf(stderr, "\t-g Generate summary info for autograder.\n"); fprintf(stderr, "\t-h Print this message.\n"); fprintf(stderr, "\t-l Run libc malloc as well.\n"); - fprintf(stderr, - "\t-o Write the authenticated result record here.\n"); fprintf(stderr, "\t-t Directory to find default traces.\n"); fprintf(stderr, "\t-v Print per-trace performance breakdowns.\n"); fprintf(stderr, "\t-V Print additional debug info.\n"); diff --git a/benchmarks/_shared/malloc_isolation.py b/benchmarks/_shared/malloc_isolation.py new file mode 100644 index 00000000..f0064bb9 --- /dev/null +++ b/benchmarks/_shared/malloc_isolation.py @@ -0,0 +1,195 @@ +"""Compile the allocator to Wasm64 and score it with a separate trusted driver.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import math +import os +from pathlib import Path +import resource +import shutil +import signal +import subprocess +import sys +import tempfile +import time + +from candidate_sandbox import CANDIDATE_RUNTIME_ENV, namespace_command + +SUPPORT = Path(__file__).resolve().with_name("malloc_wasm") +TRACES = ( + "amptjp-bal.rep", "cccp-bal.rep", "cp-decl-bal.rep", "expr-bal.rep", + "coalescing-bal.rep", "random-bal.rep", "random2-bal.rep", "binary-bal.rep", + "binary2-bal.rep", "realloc-bal.rep", "realloc2-bal.rep", +) + + +def _setup_module(): + spec = importlib.util.spec_from_file_location("frontier_malloc_setup", SUPPORT / "setup.py") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _run(command, work: Path, readable, timeout: float): + env = {key: os.environ[key] for key in CANDIDATE_RUNTIME_ENV if key in os.environ} + env["OMP_NUM_THREADS"] = "1" + + def limits(): + os.setsid() + resource.setrlimit(resource.RLIMIT_FSIZE, (32 << 20, 32 << 20)) + resource.setrlimit(resource.RLIMIT_NOFILE, (128, 128)) + resource.setrlimit(resource.RLIMIT_CPU, (int(timeout) + 5, int(timeout) + 5)) + + with tempfile.TemporaryDirectory(prefix="malloc_logs_") as log_dir: + out = Path(log_dir) / "stdout" + err = Path(log_dir) / "stderr" + with out.open("wb") as stdout, err.open("wb") as stderr: + proc = subprocess.Popen( + namespace_command(command, work, readonly_paths=readable), + cwd=work, env=env, stdout=stdout, stderr=stderr, + stdin=subprocess.DEVNULL, preexec_fn=limits, + ) + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + raise TimeoutError(f"Malloc evaluation exceeded {timeout:g} seconds") from None + stdout_text = out.read_text(errors="replace") + stderr_text = err.read_text(errors="replace") + if proc.returncode: + if "wasm trap: interrupt" in stderr_text: + raise TimeoutError("allocator call exceeded the one-second execution deadline") + raise ValueError(f"process exited {proc.returncode}: {stderr_text[-12000:]}") + return stdout_text, stderr_text + + +def _metrics(report: dict) -> dict: + if report.get("schema") != "malloc_wasm64.v1": + raise ValueError("unexpected trusted driver result schema") + traces = report.get("traces") + if not isinstance(traces, list) or [t.get("name") for t in traces] != list(TRACES): + raise ValueError("incomplete trusted driver result") + passed, util, seconds, operations = 0, 0.0, 0.0, 0 + for trace in traces: + if trace.get("valid") not in (0, 1): + raise ValueError("invalid trace verdict") + if not trace["valid"]: + continue + u, t, n = trace["utilization"], trace["runtime_s"], trace["operations"] + if not (math.isfinite(u) and 0 <= u <= 1 and math.isfinite(t) and t > 0): + raise ValueError("invalid measured utilization or runtime") + if type(n) is not int or n <= 0: + raise ValueError("invalid measured operation count") + passed += 1 + util += u + seconds += t + operations += n + throughput = operations / seconds if seconds else 0.0 + util_points = 60.0 * util / len(TRACES) + thru_points = 40.0 * min(1.0, throughput / 10_000_000.0) + score = (util_points + thru_points) * passed / len(TRACES) + return { + "combined_score": score, "score_100": score, "score_ratio": score / 100, + "valid": float(passed > 0), "timeout": 0.0, + "testcases_passed": passed, "testcases_total": len(TRACES), + "testcase_pass_rate": passed / len(TRACES), "errors": len(TRACES) - passed, + "util_points": util_points, "thru_points": thru_points, + "throughput_ops_s": throughput, "allocator_runtime_s": seconds, + "isolated_candidate": 1.0, + } + + +def evaluate(program: Path, benchmark: Path) -> tuple[dict, dict]: + started = time.monotonic() + artifacts = {} + metrics = {"combined_score": 0.0, "valid": 0.0, "timeout": 0.0} + try: + program, benchmark = Path(program).resolve(), Path(benchmark).resolve() + if program.stat().st_size > 2 << 20: + raise ValueError("candidate C source exceeds 2 MiB") + source = program.read_bytes() + if not source or len(source) > 2 << 20: + raise ValueError("candidate C source must be between 1 byte and 2 MiB") + artifacts["candidate_sha256"] = hashlib.sha256(source).hexdigest() + setup = _setup_module() + toolchain = setup.resolve() + artifacts["toolchain"] = { + "compiler": setup.WASI_DIRECTORY, "runtime": setup.WASMTIME_DIRECTORY, + } + artifacts["driver_source_sha256"] = hashlib.sha256( + (SUPPORT / "host.c").read_bytes() + ).hexdigest() + artifacts["guest_runtime_sha256"] = hashlib.sha256( + (SUPPORT / "guest.c").read_bytes() + ).hexdigest() + handout = benchmark / "malloclab-handout" + with tempfile.TemporaryDirectory(prefix="malloc_wasm_") as directory: + work = Path(directory).resolve() + staged = work / "candidate.c" + staged.write_bytes(source) + headers = work / "headers" + headers.mkdir() + for name in ("mm.h", "memlib.h"): + shutil.copyfile(handout / name, headers / name) + wasm = work / "candidate.wasm" + _run( + setup.compile_command(staged, wasm, headers, toolchain=toolchain), + work, [toolchain.clang.parent.parent, SUPPORT], 60, + ) + artifacts["module_sha256"] = hashlib.sha256(wasm.read_bytes()).hexdigest() + cc = shutil.which("cc") or shutil.which("gcc") + if not cc: + raise ValueError("a native C compiler is required for the trusted driver") + host = work / "malloc_host" + _, warnings = _run( + [cc, "-O2", "-std=c11", "-Wall", "-Wextra", "-Werror", + "-I", str(toolchain.wasmtime / "include"), str(SUPPORT / "host.c"), + "-L", str(toolchain.wasmtime / "lib"), + "-Wl,-rpath," + str(toolchain.wasmtime / "lib"), + "-lwasmtime", "-lpthread", "-lm", "-o", str(host)], + work, [toolchain.wasmtime, SUPPORT], 60, + ) + if warnings: + artifacts["driver_build_log"] = warnings + output, diagnostics = _run( + [str(host), str(wasm), str(handout / "traces")], + work, [toolchain.wasmtime, handout / "traces"], 180, + ) + report = json.loads(output) + metrics = _metrics(report) + artifacts["trace_results"] = report + if diagnostics: + artifacts["driver_log"] = diagnostics + except TimeoutError as exc: + metrics.update(timeout=1.0, error_message=str(exc)) + except Exception as exc: + metrics["error_message"] = str(exc) + metrics["runtime_s"] = time.monotonic() - started + return metrics, artifacts + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("candidate", type=Path) + parser.add_argument("--benchmark", required=True, type=Path) + parser.add_argument("--metrics-out", required=True, type=Path) + parser.add_argument("--details-out", type=Path) + args = parser.parse_args() + metrics, artifacts = evaluate(args.candidate, args.benchmark) + args.metrics_out.write_text(json.dumps(metrics, indent=2) + "\n") + if args.details_out: + args.details_out.write_text(json.dumps(artifacts, indent=2) + "\n") + print(json.dumps(metrics)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/_shared/malloc_wasm/allowed_imports.txt b/benchmarks/_shared/malloc_wasm/allowed_imports.txt new file mode 100644 index 00000000..ab9e9011 --- /dev/null +++ b/benchmarks/_shared/malloc_wasm/allowed_imports.txt @@ -0,0 +1,5 @@ +mem_sbrk +mem_heap_lo +mem_heap_hi +mem_heapsize +mem_pagesize diff --git a/benchmarks/_shared/malloc_wasm/guest.c b/benchmarks/_shared/malloc_wasm/guest.c new file mode 100644 index 00000000..1be21154 --- /dev/null +++ b/benchmarks/_shared/malloc_wasm/guest.c @@ -0,0 +1,38 @@ +#include + +_Static_assert(sizeof(void *) == 8, "pointer width must be 64 bits"); +_Static_assert(sizeof(size_t) == 8, "size_t width must be 64 bits"); +_Static_assert(sizeof(long) == 8, "long width must be 64 bits"); + +void *memcpy(void *dst, const void *src, size_t n) { + return __builtin_memcpy(dst, src, n); +} + +void *memmove(void *dst, const void *src, size_t n) { + return __builtin_memmove(dst, src, n); +} + +void *memset(void *dst, int value, size_t n) { + return __builtin_memset(dst, value, n); +} + +int memcmp(const void *lhs, const void *rhs, size_t n) { + const unsigned char *a = lhs, *b = rhs; + for (size_t i = 0; i < n; ++i) + if (a[i] != b[i]) + return (int)a[i] - (int)b[i]; + return 0; +} + +size_t strlen(const char *text) { + size_t n = 0; + while (text[n]) + ++n; + return n; +} + +_Noreturn void abort(void) { __builtin_trap(); } +_Noreturn void exit(int status) { + (void)status; + __builtin_trap(); +} diff --git a/benchmarks/_shared/malloc_wasm/host.c b/benchmarks/_shared/malloc_wasm/host.c new file mode 100644 index 00000000..0eed61a4 --- /dev/null +++ b/benchmarks/_shared/malloc_wasm/host.c @@ -0,0 +1,523 @@ +/* Trusted Malloc Lab driver. Candidate code executes only in Wasm memory. */ +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HEAP_START (4ULL << 20) +#define MAX_HEAP (20ULL << 20) +#define MEMORY_SIZE (HEAP_START + MAX_HEAP) +#define REPEATS 10 +#define THROUGHPUT_CAP 10000000.0 +static const char *traces[] = { + "amptjp-bal.rep", "cccp-bal.rep", "cp-decl-bal.rep", + "expr-bal.rep", "coalescing-bal.rep", "random-bal.rep", + "random2-bal.rep", "binary-bal.rep", "binary2-bal.rep", + "realloc-bal.rep", "realloc2-bal.rep"}; +static wasm_engine_t *engine; +static atomic_bool watchdog_done; +static char failure[1024]; + +typedef struct { + char kind; + unsigned id; + uint64_t size; +} operation; +typedef struct { + size_t nids, nops; + operation *ops; +} trace; +typedef struct { + uint64_t offset, size; + uint8_t *expected; + int live; +} block; +typedef struct { + wasmtime_store_t *store; + wasmtime_context_t *context; + wasmtime_memory_t memory; + wasmtime_func_t init, alloc, release, resize; + uint64_t brk; + int ready; + double call_seconds; +} guest; + +static void die(const char *message) { + fprintf(stderr, "%s\n", message); + exit(2); +} +static void *checked_calloc(size_t n, size_t s) { + void *p = calloc(n, s); + if (!p) + die("host allocation failed"); + return p; +} +static double now(void) { + struct timespec t; + if (clock_gettime(CLOCK_MONOTONIC, &t)) + die("clock failed"); + return t.tv_sec + t.tv_nsec * 1e-9; +} +static void *watchdog(void *unused) { + (void)unused; + struct timespec delay = {0, 10000000}; + while (!atomic_load(&watchdog_done)) { + nanosleep(&delay, NULL); + wasmtime_engine_increment_epoch(engine); + } + return NULL; +} +static void error_text(wasmtime_error_t *error, wasm_trap_t *trap) { + wasm_name_t msg; + if (error) + wasmtime_error_message(error, &msg); + else + wasm_trap_message(trap, &msg); + size_t n = msg.size < sizeof(failure) - 1 ? msg.size : sizeof(failure) - 1; + memcpy(failure, msg.data, n); + failure[n] = 0; + wasm_name_delete(&msg); + if (error) + wasmtime_error_delete(error); + if (trap) + wasm_trap_delete(trap); +} +static void random_fill(uint8_t *p, size_t n) { + while (n) { + ssize_t count = getrandom(p, n, 0); + if (count < 0 && errno == EINTR) + continue; + if (count <= 0) + die("host entropy unavailable"); + p += count; + n -= count; + } +} +static wasm_trap_t *mem_call(void *env, wasmtime_caller_t *caller, + const wasmtime_val_t *args, size_t nargs, + wasmtime_val_t *results, size_t nresults) { + (void)caller; + (void)nargs; + (void)nresults; + uintptr_t packed = (uintptr_t)env; + unsigned kind = (unsigned)(packed & 7); + guest *g = (guest *)(packed & ~(uintptr_t)7); + uint64_t value = 0; + if (!g->ready) + return wasmtime_trap_new( + "memlib called during module initialization", + strlen("memlib called during module initialization")); + switch (kind) { + case 0: { + int32_t inc = args[0].of.i32; + if (inc < 0 || (uint64_t)inc > HEAP_START + MAX_HEAP - g->brk) + value = UINT64_MAX; + else { + value = g->brk; + g->brk += (uint32_t)inc; + } + break; + } + case 1: + value = HEAP_START; + break; + case 2: + value = g->brk - 1; + break; + case 3: + value = g->brk - HEAP_START; + break; + case 4: + value = 4096; + break; + } + results[0].kind = WASMTIME_I64; + results[0].of.i64 = (int64_t)value; + return NULL; +} +static int get_func(guest *g, wasmtime_instance_t *instance, const char *name, + unsigned n, int result64, wasmtime_func_t *out) { + wasmtime_extern_t item; + if (!wasmtime_instance_export_get(g->context, instance, name, strlen(name), + &item) || + item.kind != WASMTIME_EXTERN_FUNC) { + snprintf(failure, sizeof(failure), "missing function %s", name); + return 0; + } + wasm_functype_t *t = wasmtime_func_type(g->context, &item.of.func); + const wasm_valtype_vec_t *p = wasm_functype_params(t), + *r = wasm_functype_results(t); + int ok = p->size == n && r->size == (result64 < 0 ? 0 : 1); + for (size_t i = 0; i < p->size; i++) + if (wasm_valtype_kind(p->data[i]) != WASM_I64) + ok = 0; + if (r->size && + wasm_valtype_kind(r->data[0]) != (result64 ? WASM_I64 : WASM_I32)) + ok = 0; + wasm_functype_delete(t); + if (!ok) { + snprintf(failure, sizeof(failure), "incorrect signature for %s", name); + return 0; + } + *out = item.of.func; + return 1; +} +static int new_guest(guest *g, wasmtime_module_t *module) { + memset(g, 0, sizeof(*g)); + g->brk = HEAP_START; + g->store = wasmtime_store_new(engine, NULL, NULL); + g->context = wasmtime_store_context(g->store); + wasmtime_store_limiter(g->store, MEMORY_SIZE, 1000, 1, 1, 1); + wasmtime_context_set_epoch_deadline(g->context, 100); + wasmtime_linker_t *linker = wasmtime_linker_new(engine); + const char *names[] = {"mem_sbrk", "mem_heap_lo", "mem_heap_hi", + "mem_heapsize", "mem_pagesize"}; + for (unsigned i = 0; i < 5; i++) { + wasm_functype_t *type = i == 0 + ? wasm_functype_new_1_1(wasm_valtype_new_i32(), + wasm_valtype_new_i64()) + : wasm_functype_new_0_1(wasm_valtype_new_i64()); + wasmtime_error_t *e = wasmtime_linker_define_func( + linker, "env", 3, names[i], strlen(names[i]), type, mem_call, + (void *)((uintptr_t)g | i), NULL); + wasm_functype_delete(type); + if (e) { + error_text(e, NULL); + wasmtime_linker_delete(linker); + return 0; + } + } + wasmtime_instance_t instance; + wasm_trap_t *trap = NULL; + wasmtime_error_t *error = + wasmtime_linker_instantiate(linker, g->context, module, &instance, &trap); + wasmtime_linker_delete(linker); + if (error || trap) { + error_text(error, trap); + die(failure); + } + wasmtime_extern_t item; + if (!wasmtime_instance_export_get(g->context, &instance, "memory", 6, + &item) || + item.kind != WASMTIME_EXTERN_MEMORY) { + strcpy(failure, "missing memory"); + return 0; + } + g->memory = item.of.memory; + wasm_memorytype_t *mt = wasmtime_memory_type(g->context, &g->memory); + uint64_t maximum = 0; + int fixed = wasmtime_memorytype_is64(mt) && + !wasmtime_memorytype_isshared(mt) && + wasmtime_memorytype_maximum(mt, &maximum) && + maximum == MEMORY_SIZE / 65536 && + wasmtime_memorytype_minimum(mt) == maximum; + wasm_memorytype_delete(mt); + if (!fixed || + wasmtime_memory_data_size(g->context, &g->memory) != MEMORY_SIZE) { + strcpy(failure, "memory must be fixed 24 MiB, unshared, and 64-bit"); + return 0; + } + if (!wasmtime_instance_export_get(g->context, &instance, "__heap_base", 11, + &item) || + item.kind != WASMTIME_EXTERN_GLOBAL) { + strcpy(failure, "missing __heap_base"); + return 0; + } + wasmtime_val_t base; + wasmtime_global_get(g->context, &item.of.global, &base); + if (base.kind != WASMTIME_I64 || (uint64_t)base.of.i64 > HEAP_START) { + strcpy(failure, "candidate static storage exceeds 4 MiB"); + return 0; + } + if (!get_func(g, &instance, "mm_init", 0, 0, &g->init) || + !get_func(g, &instance, "mm_malloc", 1, 1, &g->alloc) || + !get_func(g, &instance, "mm_free", 1, -1, &g->release) || + !get_func(g, &instance, "mm_realloc", 2, 1, &g->resize)) + return 0; + g->ready = 1; + return 1; +} +static int call(guest *g, const wasmtime_func_t *func, uint64_t a, uint64_t b, + unsigned n, int result64, uint64_t *value) { + wasmtime_val_raw_t args[2] = {{0}, {0}}; + args[0].i64 = a; + args[1].i64 = b; + wasm_trap_t *trap = NULL; + wasmtime_context_set_epoch_deadline(g->context, 100); + double start = now(); + /* Signatures were checked in get_func. The last length is the buffer + capacity, including result space; mm_init needs one slot for its result. */ + wasmtime_error_t *error = wasmtime_func_call_unchecked( + g->context, func, args, n ? n : (result64 < 0 ? 0 : 1), &trap); + g->call_seconds += now() - start; + if (error || trap) { + error_text(error, trap); + die(failure); + } + if (result64 >= 0) + *value = result64 ? (uint64_t)args[0].i64 + : (uint64_t)(int64_t)(int32_t)args[0].i32; + return 1; +} +static int in_heap(guest *g, uint64_t ptr, uint64_t size) { + return ptr >= HEAP_START && ptr <= g->brk && size <= g->brk - ptr && + !(ptr % 16) && size > 0; +} +static int run_trace(wasmtime_module_t *module, const trace *t, double *secs, + double *util) { + guest *g = checked_calloc(1, sizeof(*g)); + block *blocks = checked_calloc(t->nids, sizeof(*blocks)); + int ok = 0; + if (!new_guest(g, module)) + die(failure); + uint64_t result = 0, live_bytes = 0, peak_bytes = 0; + if (!call(g, &g->init, 0, 0, 0, 0, &result) || (int64_t)result < 0) { + if (!failure[0]) + strcpy(failure, "mm_init failed"); + goto done; + } + uint8_t *memory = wasmtime_memory_data(g->context, &g->memory); + for (size_t k = 0; k < t->nops; k++) { + const operation *op = &t->ops[k]; + block *old = &blocks[op->id]; + if (op->kind != 'a' && + (!old->live || + memcmp(memory + old->offset, old->expected, old->size) != 0)) { + snprintf(failure, sizeof(failure), + "operation %zu: existing block contents damaged", k); + goto done; + } + if (op->kind == 'f') { + if (!call(g, &g->release, old->offset, 0, 1, -1, &result)) + goto done; + live_bytes -= old->size; + old->live = 0; + free(old->expected); + old->expected = NULL; + continue; + } + if (op->kind == 'a' && old->live) { + strcpy(failure, "trace allocates active ID"); + goto done; + } + if (op->kind == 'a') { + if (!call(g, &g->alloc, op->size, 0, 1, 1, &result)) + goto done; + } else { + if (!call(g, &g->resize, old->offset, op->size, 2, 1, &result)) + goto done; + } + if (!in_heap(g, result, op->size)) { + snprintf(failure, sizeof(failure), + "operation %zu: returned address is null, unaligned, or outside " + "requested heap", + k); + goto done; + } + for (size_t j = 0; j < t->nids; j++) { + block *other = &blocks[j]; + if (other->live && j != op->id && result < other->offset + other->size && + other->offset < result + op->size) { + snprintf(failure, sizeof(failure), + "operation %zu: overlapping allocation", k); + goto done; + } + } + if (op->kind == 'r') { + uint64_t preserved = old->size < op->size ? old->size : op->size; + if (memcmp(memory + result, old->expected, preserved) != 0) { + snprintf(failure, sizeof(failure), + "operation %zu: realloc did not preserve data", k); + goto done; + } + live_bytes -= old->size; + } + free(old->expected); + old->expected = checked_calloc(op->size, 1); + random_fill(old->expected, op->size); + old->offset = result; + old->size = op->size; + old->live = 1; + memcpy(memory + result, old->expected, op->size); + live_bytes += op->size; + if (live_bytes > peak_bytes) + peak_bytes = live_bytes; + } + for (size_t j = 0; j < t->nids; j++) + if (blocks[j].live && memcmp(memory + blocks[j].offset, blocks[j].expected, + blocks[j].size) != 0) { + strcpy(failure, "final live block contents damaged"); + goto done; + } + if (g->brk <= HEAP_START || peak_bytes > g->brk - HEAP_START) { + strcpy(failure, "invalid heap utilization"); + goto done; + } + *secs = g->call_seconds; + *util = (double)peak_bytes / (g->brk - HEAP_START); + ok = 1; +done: + if (g->store) + wasmtime_store_delete(g->store); + free(g); + for (size_t j = 0; j < t->nids; j++) + free(blocks[j].expected); + free(blocks); + return ok; +} +static trace load_trace(const char *directory, const char *name) { + char path[8192]; + if (snprintf(path, sizeof(path), "%s/%s", directory, name) >= + (int)sizeof(path)) + die("trace path too long"); + FILE *f = fopen(path, "r"); + if (!f) + die("cannot open trusted trace"); + size_t heap, nids, nops, weight; + if (fscanf(f, "%zu %zu %zu %zu", &heap, &nids, &nops, &weight) != 4 || + nids == 0 || nids > 1000000 || nops == 0 || nops > 10000000) + die("invalid trace header"); + trace t = {nids, nops, checked_calloc(nops, sizeof(operation))}; + unsigned char *active = checked_calloc(nids, 1); + for (size_t i = 0; i < nops; i++) { + operation *o = &t.ops[i]; + if (fscanf(f, " %c %u", &o->kind, &o->id) != 2 || o->id >= nids) + die("invalid trace ID"); + if (o->kind == 'a' || o->kind == 'r') { + unsigned long long size; + if (fscanf(f, "%llu", &size) != 1 || size == 0 || size > MAX_HEAP) + die("invalid trace size"); + o->size = size; + } else if (o->kind != 'f') + die("invalid trace operation"); + if ((o->kind == 'a' && active[o->id]) || (o->kind != 'a' && !active[o->id])) + die("invalid trace lifetime"); + active[o->id] = o->kind != 'f'; + } + char extra; + if (fscanf(f, " %c", &extra) == 1) + die("extra trace operations"); + fclose(f); + free(active); + return t; +} +static void json_string(const char *s) { + putchar('"'); + for (; *s; s++) { + unsigned char c = (unsigned char)*s; + if (c == '"' || c == '\\') + putchar('\\'); + if (c >= 32 && c < 127) + putchar(c); + else + printf("\\u%04x", c); + } + putchar('"'); +} +int main(int argc, char **argv) { + if (argc != 3) + die("usage: malloc_host candidate.wasm trusted-trace-directory"); + FILE *f = fopen(argv[1], "rb"); + if (!f) + die("cannot read candidate module"); + if (fseek(f, 0, SEEK_END)) + die("seek failed"); + long n = ftell(f); + if (n <= 0 || n > 16 * 1024 * 1024) + die("invalid module size"); + rewind(f); + wasm_byte_vec_t bytes; + wasm_byte_vec_new_uninitialized(&bytes, (size_t)n); + if (fread(bytes.data, 1, n, f) != (size_t)n) + die("read failed"); + fclose(f); + wasm_config_t *config = wasm_config_new(); + wasmtime_config_wasm_memory64_set(config, true); + wasmtime_config_wasm_threads_set(config, false); + wasmtime_config_epoch_interruption_set(config, true); + wasmtime_config_max_wasm_stack_set(config, 1024 * 1024); + engine = wasm_engine_new_with_config(config); + wasmtime_module_t *module; + wasmtime_error_t *error = + wasmtime_module_new(engine, (uint8_t *)bytes.data, bytes.size, &module); + wasm_byte_vec_delete(&bytes); + if (error) { + error_text(error, NULL); + die(failure); + } + pthread_t thread; + if (pthread_create(&thread, NULL, watchdog, NULL)) + die("watchdog failed"); + const size_t count = sizeof(traces) / sizeof(traces[0]); + size_t passed = 0; + double sum_util = 0, sum_seconds = 0, completed_ops = 0; + printf("{\"schema\":\"malloc_wasm64.v1\",\"traces\":["); + for (size_t i = 0; i < count; i++) { + trace t = load_trace(argv[2], traces[i]); + double secs[REPEATS] = {0}, util[REPEATS] = {0}; + int valid = 1; + for (unsigned r = 0; r < REPEATS; r++) { + failure[0] = 0; + if (!run_trace(module, &t, &secs[r], &util[r])) { + valid = 0; + break; + } + } + double seconds = 0, utilization = 0; + if (valid) { + /* Every timing pass is independently instantiated and fully validated. */ + for (unsigned a = 0; a < REPEATS; a++) + for (unsigned b = a + 1; b < REPEATS; b++) + if (secs[b] < secs[a]) { + double x = secs[a]; + secs[a] = secs[b]; + secs[b] = x; + } + seconds = (secs[REPEATS / 2 - 1] + secs[REPEATS / 2]) / 2; + utilization = util[0]; + for (unsigned r = 1; r < REPEATS; r++) + if (util[r] < utilization) + utilization = util[r]; + if (!isfinite(seconds) || seconds <= 0) + valid = 0; + } + if (valid) { + passed++; + sum_util += utilization; + sum_seconds += seconds; + completed_ops += t.nops; + } + if (i) + putchar(','); + printf("{\"name\":"); + json_string(traces[i]); + printf(",\"valid\":%d,\"operations\":%zu,\"runtime_s\":%.12g," + "\"utilization\":%.12g,\"error\":", + valid, t.nops, seconds, utilization); + json_string(valid ? "" : failure); + putchar('}'); + fflush(stdout); + free(t.ops); + } + double throughput = sum_seconds > 0 ? completed_ops / sum_seconds : 0; + double up = 60 * sum_util / count, + tp = 40 * fmin(1, throughput / THROUGHPUT_CAP); + double score = (up + tp) * passed / count; + printf("],\"testcases_passed\":%zu,\"testcases_total\":%zu,\"util_points\":%." + "12g,\"thru_points\":%.12g,\"throughput_ops_s\":%.12g,\"score_100\":%." + "12g}\n", + passed, count, up, tp, throughput, score); + atomic_store(&watchdog_done, true); + pthread_join(thread, NULL); + wasmtime_module_delete(module); + wasm_engine_delete(engine); + return 0; +} diff --git a/benchmarks/_shared/malloc_wasm/include/assert.h b/benchmarks/_shared/malloc_wasm/include/assert.h new file mode 100644 index 00000000..6c5c8117 --- /dev/null +++ b/benchmarks/_shared/malloc_wasm/include/assert.h @@ -0,0 +1,8 @@ +#ifndef MALLOC_GUEST_ASSERT_H +#define MALLOC_GUEST_ASSERT_H +#ifdef NDEBUG +#define assert(x) ((void)0) +#else +#define assert(x) ((x) ? (void)0 : __builtin_trap()) +#endif +#endif diff --git a/benchmarks/_shared/malloc_wasm/include/limits.h b/benchmarks/_shared/malloc_wasm/include/limits.h new file mode 100644 index 00000000..d182b5ad --- /dev/null +++ b/benchmarks/_shared/malloc_wasm/include/limits.h @@ -0,0 +1,10 @@ +#ifndef MALLOC_GUEST_LIMITS_H +#define MALLOC_GUEST_LIMITS_H +#define CHAR_BIT 8 +#define INT_MAX __INT_MAX__ +#define INT_MIN (-INT_MAX - 1) +#define UINT_MAX (__INT_MAX__ * 2U + 1U) +#define LONG_MAX __LONG_MAX__ +#define LONG_MIN (-LONG_MAX - 1L) +#define ULONG_MAX (__LONG_MAX__ * 2UL + 1UL) +#endif diff --git a/benchmarks/_shared/malloc_wasm/include/stddef.h b/benchmarks/_shared/malloc_wasm/include/stddef.h new file mode 100644 index 00000000..2262de80 --- /dev/null +++ b/benchmarks/_shared/malloc_wasm/include/stddef.h @@ -0,0 +1,7 @@ +#ifndef MALLOC_GUEST_STDDEF_H +#define MALLOC_GUEST_STDDEF_H +typedef __SIZE_TYPE__ size_t; +typedef __PTRDIFF_TYPE__ ptrdiff_t; +#define NULL ((void *)0) +#define offsetof(type, member) __builtin_offsetof(type, member) +#endif diff --git a/benchmarks/_shared/malloc_wasm/include/stdint.h b/benchmarks/_shared/malloc_wasm/include/stdint.h new file mode 100644 index 00000000..c176c7ac --- /dev/null +++ b/benchmarks/_shared/malloc_wasm/include/stdint.h @@ -0,0 +1,15 @@ +#ifndef MALLOC_GUEST_STDINT_H +#define MALLOC_GUEST_STDINT_H +typedef __INT8_TYPE__ int8_t; +typedef __INT16_TYPE__ int16_t; +typedef __INT32_TYPE__ int32_t; +typedef __INT64_TYPE__ int64_t; +typedef __UINT8_TYPE__ uint8_t; +typedef __UINT16_TYPE__ uint16_t; +typedef __UINT32_TYPE__ uint32_t; +typedef __UINT64_TYPE__ uint64_t; +typedef __INTPTR_TYPE__ intptr_t; +typedef __UINTPTR_TYPE__ uintptr_t; +#define SIZE_MAX __SIZE_MAX__ +#define UINTPTR_MAX __UINTPTR_MAX__ +#endif diff --git a/benchmarks/_shared/malloc_wasm/include/stdio.h b/benchmarks/_shared/malloc_wasm/include/stdio.h new file mode 100644 index 00000000..108f9c3f --- /dev/null +++ b/benchmarks/_shared/malloc_wasm/include/stdio.h @@ -0,0 +1,10 @@ +#ifndef MALLOC_GUEST_STDIO_H +#define MALLOC_GUEST_STDIO_H +#include +typedef struct guest_FILE FILE; +extern FILE *stdin, *stdout, *stderr; +int printf(const char *, ...); +int fprintf(FILE *, const char *, ...); +int snprintf(char *, size_t, const char *, ...); +int puts(const char *); +#endif diff --git a/benchmarks/_shared/malloc_wasm/include/stdlib.h b/benchmarks/_shared/malloc_wasm/include/stdlib.h new file mode 100644 index 00000000..95a4b2aa --- /dev/null +++ b/benchmarks/_shared/malloc_wasm/include/stdlib.h @@ -0,0 +1,10 @@ +#ifndef MALLOC_GUEST_STDLIB_H +#define MALLOC_GUEST_STDLIB_H +#include +_Noreturn void abort(void); +_Noreturn void exit(int); +void *malloc(size_t); +void free(void *); +void *calloc(size_t, size_t); +void *realloc(void *, size_t); +#endif diff --git a/benchmarks/_shared/malloc_wasm/include/string.h b/benchmarks/_shared/malloc_wasm/include/string.h new file mode 100644 index 00000000..8bb4b727 --- /dev/null +++ b/benchmarks/_shared/malloc_wasm/include/string.h @@ -0,0 +1,9 @@ +#ifndef MALLOC_GUEST_STRING_H +#define MALLOC_GUEST_STRING_H +#include +void *memcpy(void *, const void *, size_t); +void *memmove(void *, const void *, size_t); +void *memset(void *, int, size_t); +int memcmp(const void *, const void *, size_t); +size_t strlen(const char *); +#endif diff --git a/benchmarks/_shared/malloc_wasm/include/unistd.h b/benchmarks/_shared/malloc_wasm/include/unistd.h new file mode 100644 index 00000000..ae07364e --- /dev/null +++ b/benchmarks/_shared/malloc_wasm/include/unistd.h @@ -0,0 +1,5 @@ +#ifndef MALLOC_GUEST_UNISTD_H +#define MALLOC_GUEST_UNISTD_H +#include +typedef __PTRDIFF_TYPE__ ssize_t; +#endif diff --git a/benchmarks/_shared/malloc_wasm/setup.py b/benchmarks/_shared/malloc_wasm/setup.py new file mode 100644 index 00000000..d66a64c9 --- /dev/null +++ b/benchmarks/_shared/malloc_wasm/setup.py @@ -0,0 +1,234 @@ +"""Install the pinned Linux x86-64 compiler and Wasmtime C API. + +From the repository root: + python benchmarks/_shared/malloc_wasm/setup.py --install + +Without --install this command only checks local dependencies. Set +FRONTIER_MALLOC_TOOLCHAIN or pass --root to use a different installation root. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import hashlib +import os +from pathlib import Path, PurePosixPath +import platform +import shutil +import tarfile +import tempfile +import urllib.parse +import urllib.request + + +WASI_DIRECTORY = "wasi-sdk-34.0-x86_64-linux" +WASMTIME_DIRECTORY = "wasmtime-v48.0.2-x86_64-linux-c-api" +ASSETS = ( + ( + WASI_DIRECTORY, + "https://github.com/WebAssembly/wasi-sdk/releases/download/" + "wasi-sdk-34/wasi-sdk-34.0-x86_64-linux.tar.gz", + "b761e3a0721dbae9c09a0059e5fdb2bf917d1b4a8a7b430fb3b5aafb0984b2c4", + ), + ( + WASMTIME_DIRECTORY, + "https://github.com/bytecodealliance/wasmtime/releases/download/" + "v48.0.2/wasmtime-v48.0.2-x86_64-linux-c-api.tar.xz", + "d9a2b5dfaf688035f288a7ae81a4b96c3acdd3e849262c2ab577b61908c3f9f9", + ), +) + + +@dataclass(frozen=True) +class Toolchain: + root: Path + clang: Path + wasmtime: Path + + +def _root(root: str | Path | None = None) -> Path: + value = root or os.environ.get("FRONTIER_MALLOC_TOOLCHAIN") + return Path(value).expanduser().resolve() if value else ( + Path.home() / ".cache/frontier-eval/malloc-wasm" + ).resolve() + + +def resolve(root: str | Path | None = None) -> Toolchain: + """Return installed tool paths without network access.""" + root = _root(root) + result = Toolchain(root, root / WASI_DIRECTORY / "bin/clang", + root / WASMTIME_DIRECTORY) + required = ( + result.clang, + result.clang.parent / "wasm-ld", + result.wasmtime / "include/wasmtime.h", + result.wasmtime / "include/wasm.h", + result.wasmtime / "lib/libwasmtime.so", + ) + if not all(path.is_file() for path in required) or not os.access(result.clang, os.X_OK): + raise FileNotFoundError( + f"Malloc Wasm dependencies are incomplete at {root}. " + "Run python benchmarks/_shared/malloc_wasm/setup.py --install " + f"--root {root}" + ) + return result + + +def _digest(path: Path) -> str: + result = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + result.update(block) + return result.hexdigest() + + +def _check_download_url(url: str) -> None: + parsed = urllib.parse.urlsplit(url) + if parsed.scheme != "https" or parsed.hostname not in { + "github.com", "release-assets.githubusercontent.com", + "objects.githubusercontent.com", + } or parsed.username or parsed.password or parsed.port not in (None, 443): + raise ValueError("Toolchain download must use an official HTTPS release asset") + + +class _ReleaseRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + _check_download_url(newurl) + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def _download(url: str, destination: Path, expected: str) -> None: + if destination.is_file() and _digest(destination) == expected: + return + _check_download_url(url) + opener = urllib.request.build_opener(_ReleaseRedirect()) + temporary: Path | None = None + try: + with tempfile.NamedTemporaryFile(dir=destination.parent, delete=False) as output: + temporary = Path(output.name) + request = urllib.request.Request(url, headers={"User-Agent": "Frontier-Malloc-Setup"}) + with opener.open(request, timeout=120) as response: + _check_download_url(response.url) + total = 0 + while block := response.read(1024 * 1024): + total += len(block) + if total > 1024 * 1024 * 1024: + raise ValueError("Toolchain archive exceeds the download size limit") + output.write(block) + if _digest(temporary) != expected: + raise ValueError(f"SHA-256 mismatch for {destination.name}") + os.replace(temporary, destination) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def _extract(archive: Path, destination: Path, directory: str) -> None: + """Extract regular files first, then contained relative symbolic links.""" + destination = destination.resolve() + with tarfile.open(archive) as source: + members = source.getmembers() + names: set[str] = set() + total = 0 + for member in members: + name = PurePosixPath(member.name) + if (name.is_absolute() or ".." in name.parts or not name.parts + or name.parts[0] != directory or str(name) in names): + raise ValueError(f"Unsafe archive path: {member.name}") + names.add(str(name)) + if not (member.isfile() or member.isdir() or member.issym()): + raise ValueError(f"Unsupported archive entry: {member.name}") + if member.issym(): + target = PurePosixPath(member.linkname) + resolved = (destination / name.parent / target).resolve() + if target.is_absolute() or not resolved.is_relative_to(destination / directory): + raise ValueError(f"Unsafe archive link: {member.name}") + total += member.size + if total > 2 * 1024 * 1024 * 1024: + raise ValueError("Unpacked toolchain exceeds the size limit") + + for member in members: + if member.issym(): + continue + target = destination / member.name + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + else: + target.parent.mkdir(parents=True, exist_ok=True) + with source.extractfile(member) as input_file, target.open("xb") as output: + shutil.copyfileobj(input_file, output) + target.chmod(member.mode & 0o777) + + for member in members: + if not member.issym(): + continue + target = destination / member.name + if any(parent.is_symlink() for parent in target.parents): + raise ValueError(f"Archive link traverses another link: {member.name}") + target.parent.mkdir(parents=True, exist_ok=True) + target.symlink_to(member.linkname) + for member in members: + if member.issym() and not (destination / member.name).resolve().is_relative_to( + destination / directory + ): + raise ValueError(f"Archive link escapes extraction directory: {member.name}") + + +def install(root: str | Path | None = None) -> Toolchain: + """Explicitly download and install hash-verified release assets.""" + root = _root(root) + if platform.system() != "Linux" or platform.machine().lower() not in {"x86_64", "amd64"}: + raise RuntimeError("The pinned Malloc toolchain supports Linux x86-64") + root.mkdir(parents=True, exist_ok=True) + import fcntl + + with (root / ".install.lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + try: + return resolve(root) + except FileNotFoundError: + pass + downloads = root / "downloads" + downloads.mkdir(exist_ok=True) + for directory, url, expected in ASSETS: + target = root / directory + if target.exists(): + continue + archive = downloads / url.rsplit("/", 1)[1] + _download(url, archive, expected) + with tempfile.TemporaryDirectory(prefix=".extract-", dir=root) as stage: + _extract(archive, Path(stage), directory) + os.replace(Path(stage) / directory, target) + return resolve(root) + + +def compile_command(source: str | Path, output: str | Path, handout: str | Path, + toolchain: Toolchain | None = None) -> list[str]: + """Build a memory64 module with only the host-controlled memlib imports.""" + toolchain = toolchain or resolve() + support = Path(__file__).resolve().parent + return [ + str(toolchain.clang), "--no-default-config", "--target=wasm64-unknown-unknown", + "-O2", "-nostdlib", "-nostdinc", "-ffreestanding", "-fno-builtin", "-mbulk-memory", + "-I" + str(support / "include"), "-I" + str(Path(handout).resolve()), + str(Path(source).resolve()), str(support / "guest.c"), + "-Wl,--no-entry", "-Wl,--export=mm_init", "-Wl,--export=mm_malloc", + "-Wl,--export=mm_free", "-Wl,--export=mm_realloc", "-Wl,--export=__heap_base", + "-Wl,--export-memory", "-Wl,--initial-memory=25165824", "-Wl,--max-memory=25165824", + "-Wl,-z,stack-size=1048576", "-Wl,--allow-undefined-file=" + str(support / "allowed_imports.txt"), + "-o", str(Path(output).resolve()), + ] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--install", action="store_true", help="Download the pinned dependencies") + parser.add_argument("--root", type=Path, help="Override FRONTIER_MALLOC_TOOLCHAIN") + args = parser.parse_args() + paths = install(args.root) if args.install else resolve(args.root) + print(f"clang: {paths.clang}\nwasmtime: {paths.wasmtime}") + + +if __name__ == "__main__": + main() diff --git a/frontier_eval/tasks/malloclab/evaluator/python.py b/frontier_eval/tasks/malloclab/evaluator/python.py index 04637fed..fe64c3b0 100644 --- a/frontier_eval/tasks/malloclab/evaluator/python.py +++ b/frontier_eval/tasks/malloclab/evaluator/python.py @@ -1,289 +1,36 @@ -from __future__ import annotations +"""Legacy task entry point for the common isolated Malloc Lab evaluator.""" -import os -import re -import shutil -import subprocess -import tempfile -import time from pathlib import Path - - -def _is_repo_root(path: Path) -> bool: - if not (path / "frontier_eval").is_dir(): - return False - if (path / "benchmarks").is_dir(): - return True - return (path / "Astrodynamics").is_dir() and (path / "ElectronicDesignAutomation").is_dir() - - -def _find_repo_root() -> Path: - if "FRONTIER_ENGINEERING_ROOT" in os.environ: - return Path(os.environ["FRONTIER_ENGINEERING_ROOT"]).expanduser().resolve() - - here = Path(__file__).resolve() - for parent in [here.parent, *here.parents]: - if _is_repo_root(parent): - return parent - return Path.cwd().resolve() - - -def _tail(text: str, limit: int = 8000) -> str: - if len(text) <= limit: - return text - return text[-limit:] - - -def _truncate_middle(text: str, limit: int = 200_000) -> str: - if len(text) <= limit: - return text - keep = max(0, (limit - 128) // 2) - omitted = len(text) - (2 * keep) - return text[:keep] + f"\n\n[... truncated {omitted} chars ...]\n\n" + text[-keep:] - - -def _read_text(path: Path) -> str | None: - try: - return path.read_text(encoding="utf-8", errors="replace") - except Exception: - return None - - -def _remaining_timeout(deadline_s: float) -> float: - return max(1.0, float(deadline_s - time.time())) - - -def _parse_mdriver_output(text: str) -> tuple[dict[str, float], dict[str, str]]: - metrics: dict[str, float] = {} - artifacts: dict[str, str] = {} - - score_line = "" - for raw in (text or "").splitlines(): - line = raw.strip() - if line.startswith("Score =") or line.startswith("Perf index ="): - score_line = line - if score_line: - artifacts["score_line"] = score_line - - score_match = re.search(r"=\s*([0-9]+(?:\.[0-9]+)?)\s*/\s*100\b", score_line or text) - if score_match: - score = float(score_match.group(1)) - metrics["score_raw_100"] = score - metrics["score_raw_ratio"] = score / 100.0 - - util_thru_match = re.search( - r"\(\s*([0-9]+(?:\.[0-9]+)?)\s*\(util\)\s*\+\s*([0-9]+(?:\.[0-9]+)?)\s*\(thru\)\s*\)", - score_line or text, - ) - if util_thru_match: - metrics["util_points"] = float(util_thru_match.group(1)) - metrics["thru_points"] = float(util_thru_match.group(2)) - - testcase_match = re.search( - r"\*\s*([0-9]+)\s*/\s*([0-9]+)\s*\(testcase\)", - score_line or text, - ) - if testcase_match: - passed = float(testcase_match.group(1)) - total = float(testcase_match.group(2)) - metrics["testcases_passed"] = passed - metrics["testcases_total"] = total - if total > 0: - metrics["testcase_pass_rate"] = passed / total - - errors_match = re.search(r"(\d+)\s+errors?\s+occurred", text or "", flags=re.IGNORECASE) - if errors_match: - metrics["errors_count"] = float(errors_match.group(1)) - else: - metrics["errors_count"] = 0.0 - - # Guard against mdriver's unchecked util overflow on invalid allocators. - # Physics-aware bounds from config.h: - # - util contribution in printed score is in [0, 60] - # - throughput contribution is in [0, 40] - # Final score is (util_pts + thru_pts) * passed/total. - util_points = metrics.get("util_points") - thru_points = metrics.get("thru_points") - passed = metrics.get("testcases_passed") - total = metrics.get("testcases_total") - - guarded_score: float | None = None - if ( - util_points is not None - and thru_points is not None - and passed is not None - and total is not None - and total > 0 - ): - util_capped = max(0.0, min(float(util_points), 60.0)) - thru_capped = max(0.0, min(float(thru_points), 40.0)) - metrics["util_points_capped"] = util_capped - metrics["thru_points_capped"] = thru_capped - guarded_score = (util_capped + thru_capped) * (float(passed) / float(total)) - metrics["score_guarded_100"] = guarded_score - - if guarded_score is None and "score_raw_100" in metrics: - guarded_score = max(0.0, min(float(metrics["score_raw_100"]), 100.0)) - metrics["score_guarded_100"] = guarded_score - - final_score: float | None = None - if guarded_score is not None: - raw_score = metrics.get("score_raw_100") - if raw_score is not None: - # Keep mdriver's displayed score when it is within a safe bound. - # If mdriver score is inflated by util overflow, cap it by guarded score. - final_score = min(float(raw_score), float(guarded_score)) - else: - final_score = float(guarded_score) - - if final_score is not None: - metrics["combined_score"] = final_score - metrics["score_100"] = final_score - metrics["score_ratio"] = final_score / 100.0 - - return metrics, artifacts +import json +import os +import sys def evaluate(program_path: str, *, repo_root: Path | None = None): - """ - OpenEvolve evaluator for benchmarks/ComputerSystems/MallocLab. - - Contract for candidate program: - - Candidate file is copied to malloclab-handout/mm.c - - Evaluator runs `make` then `./mdriver -V` - - Final score is parsed from `Score = ... = X/100` - """ - start = time.time() - repo_root = _find_repo_root() if repo_root is None else repo_root.expanduser().resolve() - program_path_p = Path(program_path).expanduser().resolve() - - benchmark_dir = (repo_root / "benchmarks" / "ComputerSystems" / "MallocLab").resolve() - if not benchmark_dir.is_dir(): - benchmark_dir = (repo_root / "ComputerSystems" / "MallocLab").resolve() - handout_dir = (benchmark_dir / "malloclab-handout").resolve() - - artifacts: dict[str, str] = {} - metrics: dict[str, float] = { - "combined_score": 0.0, - "valid": 0.0, - "timeout": 0.0, - "runtime_s": 0.0, - } - artifacts["interface_contract"] = ( - "Hard requirements for candidate program (do NOT change these):\n" - "1) Candidate program is C source for malloclab-handout/mm.c.\n" - "2) Only mm.c should be modified.\n" - "3) Evaluator runs `make` then `./mdriver -V`.\n" - "4) Final score is parsed from the `Score = ... = X/100` line.\n" - "5) Keep function signatures in mm.c unchanged (mm_init/mm_malloc/mm_free/mm_realloc)." - ) - - task_spec_zh_cn_path = (benchmark_dir / "Task_zh-CN.md").resolve() - artifacts["task_spec_zh_cn_path"] = str(task_spec_zh_cn_path) - task_spec_zh_cn = _read_text(task_spec_zh_cn_path) - if task_spec_zh_cn: - artifacts["task_spec_zh_cn"] = _truncate_middle(task_spec_zh_cn) - - if not handout_dir.is_dir(): - artifacts["error_message"] = f"MallocLab benchmark folder missing: {handout_dir}" - metrics["runtime_s"] = float(time.time() - start) - return _wrap(metrics, artifacts) - if not program_path_p.is_file(): - artifacts["error_message"] = f"program not found: {program_path_p}" - metrics["runtime_s"] = float(time.time() - start) - return _wrap(metrics, artifacts) - - evaluator_timeout_s = float(os.environ.get("FRONTIER_EVAL_EVALUATOR_TIMEOUT_S", "300") or "300") - deadline_s = start + max(1.0, evaluator_timeout_s - 5.0) - - work_dir = Path(tempfile.mkdtemp(prefix="fe_malloclab_")).resolve() - try: - sandbox_dir = (work_dir / "malloclab-handout").resolve() - shutil.copytree(handout_dir, sandbox_dir) - - candidate_dst = (sandbox_dir / "mm.c").resolve() - shutil.copyfile(program_path_p, candidate_dst) - artifacts["candidate_program"] = str(candidate_dst) - - try: - proc_make = subprocess.run( - ["make"], - cwd=str(sandbox_dir), - capture_output=True, - text=True, - timeout=_remaining_timeout(deadline_s), - ) - except subprocess.TimeoutExpired as e: - metrics["timeout"] = 1.0 - metrics["runtime_s"] = float(time.time() - start) - artifacts["error_message"] = f"build timeout: {e}" - return _wrap(metrics, artifacts) - except FileNotFoundError as e: - metrics["runtime_s"] = float(time.time() - start) - artifacts["error_message"] = f"build tool unavailable: {e}" - return _wrap(metrics, artifacts) - - metrics["make_returncode"] = float(proc_make.returncode) - artifacts["make_stdout"] = _tail(proc_make.stdout) - artifacts["make_stderr"] = _tail(proc_make.stderr) - artifacts["make_stdout_full"] = _truncate_middle(proc_make.stdout) - artifacts["make_stderr_full"] = _truncate_middle(proc_make.stderr) - - if proc_make.returncode != 0: - artifacts["error_message"] = "build failed (make returned non-zero)" - metrics["runtime_s"] = float(time.time() - start) - return _wrap(metrics, artifacts) - - try: - proc_driver = subprocess.run( - ["./mdriver", "-V"], - cwd=str(sandbox_dir), - capture_output=True, - text=True, - timeout=_remaining_timeout(deadline_s), - ) - except subprocess.TimeoutExpired as e: - metrics["timeout"] = 1.0 - metrics["runtime_s"] = float(time.time() - start) - artifacts["error_message"] = f"mdriver timeout: {e}" - return _wrap(metrics, artifacts) - except FileNotFoundError as e: - metrics["runtime_s"] = float(time.time() - start) - artifacts["error_message"] = f"mdriver unavailable: {e}" - return _wrap(metrics, artifacts) - - metrics["mdriver_returncode"] = float(proc_driver.returncode) - artifacts["mdriver_stdout"] = _tail(proc_driver.stdout) - artifacts["mdriver_stderr"] = _tail(proc_driver.stderr) - artifacts["mdriver_stdout_full"] = _truncate_middle(proc_driver.stdout) - artifacts["mdriver_stderr_full"] = _truncate_middle(proc_driver.stderr) - - combined_output = (proc_driver.stdout or "") + "\n" + (proc_driver.stderr or "") - parsed_metrics, parsed_artifacts = _parse_mdriver_output(combined_output) - metrics.update(parsed_metrics) - artifacts.update(parsed_artifacts) - - if proc_driver.returncode != 0: - artifacts["error_message"] = ( - f"mdriver failed (returncode={proc_driver.returncode}), score may be incomplete" - ) - metrics["runtime_s"] = float(time.time() - start) - return _wrap(metrics, artifacts) - - if "combined_score" in metrics: - metrics["valid"] = 1.0 + if repo_root is None: + configured = os.environ.get("FRONTIER_ENGINEERING_ROOT") + if configured: + repo_root = Path(configured).resolve() else: - artifacts["error_message"] = "failed to parse final score from mdriver output" - - metrics["runtime_s"] = float(time.time() - start) - return _wrap(metrics, artifacts) - finally: - shutil.rmtree(work_dir, ignore_errors=True) - - -def _wrap(metrics: dict[str, float], artifacts: dict[str, str]): + repo_root = next( + p for p in Path(__file__).resolve().parents + if (p / "benchmarks" / "_shared" / "malloc_isolation.py").is_file() + ) + repo_root = Path(repo_root).resolve() + shared = repo_root / "benchmarks" / "_shared" + if str(shared) not in sys.path: + sys.path.insert(0, str(shared)) + from malloc_isolation import evaluate as evaluate_isolated + + metrics, artifacts = evaluate_isolated( + Path(program_path), repo_root / "benchmarks" / "ComputerSystems" / "MallocLab" + ) try: from openevolve.evaluation_result import EvaluationResult - except Exception: + except ModuleNotFoundError: return metrics - return EvaluationResult(metrics=metrics, artifacts=artifacts) + return EvaluationResult( + metrics=metrics, + artifacts={key: value if isinstance(value, str) else json.dumps(value) + for key, value in artifacts.items()}, + ) diff --git a/leaderboard/README.md b/leaderboard/README.md index d622c361..2b9957c5 100644 --- a/leaderboard/README.md +++ b/leaderboard/README.md @@ -36,9 +36,9 @@ diagnostics are on the [website leaderboard](https://lab.einsia.ai/frontier-eng/ | 1 | claude-opus-4.6 | 0.533 | 0.501 | 14 | 15 | 3 | | 2 | gpt-5.4 | 0.454 | 0.267 | 18 | 4 | 2 | | 3 | glm-5 | 0.347 | 0.300 | 7 | 8 | 12 | -| 4 | gemini-3.1-pro-preview | 0.277 | 0.267 | 7 | 7 | 4 | +| 4 | gemini-3.1-pro-preview | 0.284 | 0.300 | 7 | 7 | 5 | | 5 | deepseek-v3.2 | 0.269 | 0.299 | 6 | 6 | 8 | -| 6 | grok-4.20 | 0.227 | 0.200 | 6 | 5 | 4 | +| 6 | grok-4.20 | 0.220 | 0.167 | 6 | 5 | 3 | | 7 | seed-2.0-pro | 0.206 | 0.100 | 6 | 4 | 3 | | 8 | qwen3-coder-next | 0.170 | 0.066 | 5 | 3 | 3 | diff --git a/leaderboard/exp1_models_raw.csv b/leaderboard/exp1_models_raw.csv index 84a469b8..9abd68d4 100644 --- a/leaderboard/exp1_models_raw.csv +++ b/leaderboard/exp1_models_raw.csv @@ -1,7 +1,7 @@ Task,Baseline,claude-opus-4.6_best,deepseek-v3.2_best,gemini-3.1-pro-preview_best,glm-5_best,gpt-5.4_best,grok-4.20_best,qwen3-coder-next_best,seed-2.0-pro_best Aerodynamics_CarAerodynamicsSensing,0.9617,0.9624,0.9632,0.9632,0.9628,0.9630695838481188,0.9624,0.9632,0.9624 Astrodynamics_MannedLunarLanding,4577.437,6027.3126,6079.2455,4674.9462,6839.0331,6660.942428,4577.437,4577.437,4733.0435 -ComputerSystems_MallocLab,28,96.0,53.0,48.0,86.0,28.0,57.0,32.0,38.0 +ComputerSystems_MallocLab,28.145172775318017,84.05590296956399,49.25408810704752,50.18820374517181,68.13746400834182,28.145172775318017,50.06419665043305,,30.74003967183236 Cryptographic_AES-128,7.5209,11.8617,12.4591,10.2396,7.9669,39.824967043300866,10.8615,5.5501,7.9481 Cryptographic_SHA-256,9.8274,16.7955,9.718,9.942,15.1655,26.34045367870492,17.2504,9.8475,15.2838 Cryptographic_SHA3-256,16.0932,17.4003,17.0749,16.2255,17.5778,37.44512785396786,16.0594,16.5292,18.3478 diff --git a/leaderboard/medal_leaderboard.csv b/leaderboard/medal_leaderboard.csv index 063dd2a5..eb7399c4 100644 --- a/leaderboard/medal_leaderboard.csv +++ b/leaderboard/medal_leaderboard.csv @@ -2,8 +2,8 @@ Rank,Model,Medal_v1,Medal_v1lite,Gold,Silver,Bronze 1,claude-opus-4.6,0.533,0.501,14,15,3 2,gpt-5.4,0.454,0.267,18,4,2 3,glm-5,0.347,0.300,7,8,12 -4,gemini-3.1-pro-preview,0.277,0.267,7,7,4 +4,gemini-3.1-pro-preview,0.284,0.300,7,7,5 5,deepseek-v3.2,0.269,0.299,6,6,8 -6,grok-4.20,0.227,0.200,6,5,4 +6,grok-4.20,0.220,0.167,6,5,3 7,seed-2.0-pro,0.206,0.100,6,4,3 8,qwen3-coder-next,0.170,0.066,5,3,3 diff --git a/leaderboard/medal_podium.csv b/leaderboard/medal_podium.csv index 18b5fbd3..27cd5e70 100644 --- a/leaderboard/medal_podium.csv +++ b/leaderboard/medal_podium.csv @@ -1,7 +1,7 @@ Task,Baseline,Gold,Gold_model,Silver,Silver_model,Bronze,Bronze_model Aerodynamics_CarAerodynamicsSensing,0.9617,0.9632,deepseek-v3.2/gemini-3.1-pro-preview/qwen3-coder-next,0.9632,deepseek-v3.2/gemini-3.1-pro-preview/qwen3-coder-next,0.9632,deepseek-v3.2/gemini-3.1-pro-preview/qwen3-coder-next Astrodynamics_MannedLunarLanding,4577.437,6839.0331,glm-5,6660.942428,gpt-5.4,6079.2455,deepseek-v3.2 -ComputerSystems_MallocLab,28,96.0,claude-opus-4.6,86.0,glm-5,57.0,grok-4.20 +ComputerSystems_MallocLab,28.145172775318017,84.05590296956399,claude-opus-4.6,68.13746400834182,glm-5,50.18820374517181,gemini-3.1-pro-preview Cryptographic_AES-128,7.5209,39.824967043300866,gpt-5.4,12.4591,deepseek-v3.2,11.8617,claude-opus-4.6 Cryptographic_SHA-256,9.8274,26.34045367870492,gpt-5.4,17.2504,grok-4.20,16.7955,claude-opus-4.6 Cryptographic_SHA3-256,16.0932,37.44512785396786,gpt-5.4,18.3478,seed-2.0-pro,17.5778,glm-5 diff --git a/leaderboard/submission_example.csv b/leaderboard/submission_example.csv index 55b97b61..6cac1ec2 100644 --- a/leaderboard/submission_example.csv +++ b/leaderboard/submission_example.csv @@ -1,7 +1,7 @@ Task,Score Aerodynamics_CarAerodynamicsSensing,0.9624 Astrodynamics_MannedLunarLanding,6027.3126 -ComputerSystems_MallocLab,96.0 +ComputerSystems_MallocLab,84.05590296956399 Cryptographic_AES-128,11.8617 Cryptographic_SHA-256,16.7955 Cryptographic_SHA3-256,17.4003