diff --git a/benchmark/check_deepswe_v1.py b/benchmark/check_deepswe_v1.py new file mode 100644 index 0000000000..31f12ede4f --- /dev/null +++ b/benchmark/check_deepswe_v1.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Check the immutable v1 archive without launching a benchmark or model.""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +from pathlib import Path +import subprocess + + +ARCHIVE = Path(__file__).resolve().with_name("deepswe-gptxhigh-v1") +EXPECTED_TREE = "1bc5d2b3b74761a97d34ba3f3612e977fd610340" + + +def check_archive(archive: Path) -> dict: + tree = bytearray() + python_count = shell_count = embedded_count = 0 + for path in sorted(archive.iterdir(), key=lambda entry: entry.name.encode()): + if not path.is_file() or path.is_symlink(): + raise ValueError(f"unexpected archive entry: {path.name}") + raw = path.read_bytes() + blob = hashlib.sha1(b"blob " + str(len(raw)).encode() + b"\0" + raw).digest() + mode = b"100755" if path.stat().st_mode & 0o111 else b"100644" + tree.extend(mode + b" " + path.name.encode() + b"\0" + blob) + if path.suffix == ".py": + compile(raw, path.name, "exec") + python_count += 1 + for node in ast.parse(raw).body: + if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.Constant): + continue + names = {target.id for target in node.targets if isinstance(target, ast.Name)} + if names & {"_RUNNER", "_BOOTSTRAP"} and isinstance(node.value.value, str): + source = node.value.value + if "_RUNNER" in names: + source = source.format(remote_dir="/tmp/loopx-goal") + compile(source, f"{path.name}:embedded", "exec") + embedded_count += 1 + elif path.suffix == ".sh": + subprocess.run(["bash", "-n", str(path)], check=True, capture_output=True, timeout=10) + shell_count += 1 + actual_tree = hashlib.sha1(b"tree " + str(len(tree)).encode() + b"\0" + tree).hexdigest() + if actual_tree != EXPECTED_TREE: + raise ValueError("archive contents or executable modes differ from the original v1 snapshot") + return { + "archive_tree": actual_tree, + "original_snapshot_preserved": True, + "python_syntax_checks": python_count, + "embedded_python_checks": embedded_count, + "shell_syntax_checks": shell_count, + "benchmark_executed": False, + "standalone_runnable": False, + "runtime_validation": "not_performed", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--archive", type=Path, default=ARCHIVE) + args = parser.parse_args() + try: + report = check_archive(args.archive) + except (OSError, ValueError, SyntaxError, subprocess.SubprocessError) as exc: + print(json.dumps({"ok": False, "error": str(exc)})) + return 1 + print(json.dumps({"ok": True, **report}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/deepswe-gptxhigh-v1/README.md b/benchmark/deepswe-gptxhigh-v1/README.md new file mode 100644 index 0000000000..858513c0a3 --- /dev/null +++ b/benchmark/deepswe-gptxhigh-v1/README.md @@ -0,0 +1,44 @@ +# DeepSWE Five-Arm Benchmark Harness (v1) + +How we evaluate five agent configurations ("arms") on the DeepSWE task set +(113 SWE tasks). Runner/methodology code only — no API/gateway config, no trajectories. +Run against LoopX revision `2cef51d` (this branch's base). + +## The five arms +| Arm | Transport | Goal / LoopX | Continuation | +|---|---|---|---| +| `plain` | codex app-server | no Goal, no LoopX | single pass | +| `goal` | codex app-server | native Codex Goal | app-server continues while Goal active | +| `heartbeat` | `codex exec` (fresh, then `resume`) | LoopX Goal/Todo | recurring supervisor wakes | +| `codex-cli` | `codex exec` (CLI) | LoopX control plane | external `loopx turn run-once --host codex-cli`, multi-segment | +| `ssh-goal` | codex app-server | LoopX + native Goal (official full path) | same thread/Goal; LoopX clears blocked + restarts turn | + +- Arm dispatch: `pier_cn.py` (`MR_CODEX_ARM=plain|goal|loopx-native|loopx-native-codex-cli|loopx-native-heartbeat`) +- Arm classes: `goal_codex.py` (`PlainAppServerCodex`, `GoalCodex`, `LoopxCodex`) +- Runners: `loopx_wen_native_runner.py` (ssh-goal), `loopx_codex_cli_runner.py` (codex-cli), + `loopx_heartbeat_supervisor.py` (heartbeat) +- Delivery gate: `workspace_delivery.py` (recover agent work from linked git worktrees into + `/app` so the collected patch is non-empty) +- Admission: `preflight_loopx_rerun.py` (pins LoopX revision, delivery self-test) + +## Validity (strict) +`exception_info == null`, independent `verifier/reward.json` present & consistent, task +checksum matches, and a **non-empty committed patch** exists. Goal/Todo state is lifecycle +evidence only; the independent verifier is the sole correctness authority. `partial > 0` +alone does NOT count as valid delivery. + +## Results — v1 (113 tasks, per-task best valid) +| Rank | Arm | Solved | Solve rate | Partial | F2P | P2P | +|---|---|---:|---:|---:|---:|---:| +| 1 | heartbeat | 70/113 | 61.9% | 0.9739 | 0.886 | 0.993 | +| 2 | codex-cli | 66/113 | 58.4% | 0.9546 | 0.884 | 0.996 | +| 3 | goal | 60/113 | 53.1% | 0.9620 | 0.868 | 0.997 | +| 4 | ssh-goal | 58/113 | 51.3% | 0.9654 | 0.883 | 0.997 | +| 5 | plain | 54/113 | 47.8% | 0.9206 | 0.745 | 0.997 | + +P2P (regression) ≈ 1.0 for all arms; spread is driven by F2P and solve rate. +LoopX arms (heartbeat/codex-cli) and goal outperform the plain baseline. + +> Model & gateway endpoints are configured via `MR_*` env vars (not included). +> Internal hosts/paths replaced with placeholders (`127.0.0.1`, ``, ``). +> v2 (latest LoopX main) evaluation is in progress and will be published separately. diff --git a/benchmark/deepswe-gptxhigh-v1/codex_nosandbox_wrapper.py b/benchmark/deepswe-gptxhigh-v1/codex_nosandbox_wrapper.py new file mode 100755 index 0000000000..dc044d9104 --- /dev/null +++ b/benchmark/deepswe-gptxhigh-v1/codex_nosandbox_wrapper.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""A `codex` stand-in that drops LoopX's sandbox flags before running the real one. + +LoopX's codex-cli host is the path that works: driven through it, a Turn loop +ran four times on one task, committed a 22 KB patch and scored f2p 31/35. Its +one problem is the sandbox — it always passes `--sandbox ` (or +`-c sandbox_mode=...` when resuming), only permits read-only and +workspace-write, and both need bubblewrap, which needs unprivileged user +namespaces these containers do not have: + + bwrap: No permissions to create a new namespace + +Switching to `--host generic-cli` avoided that but bought a worse problem: the +generic host carries its own scheduler contract, and eleven of sixteen turns +died at "LoopX Turn route is not host executable" before any model work, with +no route recorded to explain why. + +So keep the working host and fix the flag instead. This sits earlier on PATH +than the real codex, strips the sandbox arguments, and substitutes the same +`--dangerously-bypass-approvals-and-sandbox` the other two arms already use — +which is also what keeps the three arms identical in permissions. LoopX's +contracts are untouched: it still believes it is driving codex-cli, because it +is. + +Set MR_REAL_CODEX to the real binary; defaults to /usr/local/bin/codex. +MR_LOOPX_CODEX_LOG names the log file; defaults to /tmp/loopx-goal/codex-wrapper.log. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +# Resolve the real binary rather than assuming /usr/local/bin/codex. Codex is +# installed into the image through nvm, so it lives under the Node version's +# bin directory and the hardcoded path does not exist — which made the wrapper +# die before it ever reached Codex, and LoopX report the indistinguishable +# `codex_cli_exit_nonzero`. The wrapper is invoked by absolute path through +# --codex-bin and is not itself on PATH, so a PATH lookup finds the real one. +REAL = ( + os.environ.get("MR_REAL_CODEX") + or shutil.which("codex") + or "/usr/local/bin/codex" +) +BYPASS = "--dangerously-bypass-approvals-and-sandbox" +REASONING_EFFORT = os.environ.get("MR_CODEX_REASONING_EFFORT", "").strip() +LOG = Path( + os.environ.get("MR_LOOPX_CODEX_LOG", "/tmp/loopx-goal/codex-wrapper.log") +) + + +def rewrite(argv: list[str]) -> list[str]: + out: list[str] = [] + skip_next = False + for i, arg in enumerate(argv): + if skip_next: + skip_next = False + continue + # `--sandbox ` — new-session form. + if arg == "--sandbox": + skip_next = True + continue + if arg.startswith("--sandbox="): + continue + # `-c sandbox_mode="..."` — resume form. The value is a separate argv + # item after -c, so both have to go, and only when it is that key: -c + # carries every other config override too. + if arg == "-c" and i + 1 < len(argv) and argv[i + 1].startswith("sandbox_mode="): + skip_next = True + continue + out.append(arg) + + # Insert the bypass right after the subcommand so it lands before `--`, + # which codex treats as the end of flags. + if out and out[0] == "exec": + out.insert(1, BYPASS) + if REASONING_EFFORT and not any( + value.startswith("model_reasoning_effort=") for value in out + ): + out[2:2] = ["-c", f"model_reasoning_effort={REASONING_EFFORT}"] + else: + out.insert(0, BYPASS) + return out + + +def _log(text: str) -> None: + try: + LOG.parent.mkdir(parents=True, exist_ok=True) + with LOG.open("a", encoding="utf-8") as handle: + handle.write(text.rstrip("\n") + "\n") + except OSError: + pass + + +def main() -> int: + argv = rewrite(sys.argv[1:]) + # Run the real codex as a child rather than execv'ing it, so its stderr can + # be recorded. LoopX reports a failed Turn as `codex_cli_exit_nonzero` and + # keeps neither the exit code's cause nor any output, and the container is + # gone by the time anyone looks — so an execv here means the only evidence + # of why Codex refused is destroyed at the moment it is produced. + # + # stdout stays inherited and untouched: LoopX parses Codex's `--json` + # stream off it, so anything written there would corrupt the Turn. + _log(f"--- argv in : {sys.argv[1:]}") + _log(f"--- argv out: {argv}") + _log(f"--- real : {REAL} (exists={os.path.exists(REAL)})") + try: + completed = subprocess.run( # noqa: S603 + [REAL, *argv], stderr=subprocess.PIPE, check=False + ) + except OSError as exc: + # Without this the wrapper's own failure to start Codex is reported by + # LoopX as `codex_cli_exit_nonzero`, which reads as "the model refused" + # rather than "the binary is not there". + _log(f"--- launch failed: {type(exc).__name__}: {exc}") + sys.stderr.write(f"codex wrapper could not launch {REAL}: {exc}\n") + return 127 + stderr = completed.stderr.decode("utf-8", "replace") if completed.stderr else "" + _log(f"--- exit {completed.returncode}") + if stderr: + _log(stderr) + sys.stderr.write(stderr) + return completed.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/deepswe-gptxhigh-v1/goal_claude.py b/benchmark/deepswe-gptxhigh-v1/goal_claude.py new file mode 100644 index 0000000000..cd4d1ecae6 --- /dev/null +++ b/benchmark/deepswe-gptxhigh-v1/goal_claude.py @@ -0,0 +1,218 @@ +"""Claude Code arms for the DeepSWE sweep: plain, and driven by LoopX. + +The Claude Code counterpart of `goal_codex.py`, and structured the same way for +the same reason: Pier's `ClaudeCode.run()` is one long method whose setup (auth +resolution, CLAUDE_CONFIG_DIR, skills/memory/MCP registration, session capture, +trajectory conversion) would have to be duplicated and kept in step with +upstream. Intercepting the single command that matters leaves all of that +untouched, and if Pier ever changes how it invokes Claude Code the marker below +stops matching and the run fails loudly instead of quietly degrading to plain +Claude Code — which would look like a successful LoopX run with no loop in it. + +There is no `GoalClaudeCode` here. Codex has a native Goal API (app-server +`thread/goal/set`) that can own a continuation loop; Claude Code's equivalent +is its native `/loop`, which is an interactive slash command with no headless +(`--print`) entry point, so it cannot be exercised inside Pier's one-shot +container invocation. On this harness the Claude Code contrast is therefore +plain vs LoopX-driven, not native-Goal vs LoopX. + +Usage: + + MR_AGENT=claude-code MR_CLAUDE_ARM=plain ./run.sh --all -i + MR_AGENT=claude-code MR_CLAUDE_ARM=loopx ./run.sh --all -i + +Environment: + + MR_LOOPX_ROOT path to the LoopX checkout on the host + MR_LOOPX_QUOTA max governed Turns per task (default 4) + MR_LOOPX_TURN_TIMEOUT per-Turn ceiling in seconds (default 1200) +""" + +from __future__ import annotations + +import os +import shlex +import subprocess +import tempfile +from pathlib import Path + +from pier.agents.installed.claude_code import ClaudeCode +from pier.models.trial.paths import EnvironmentPaths + +# Pier builds exactly one command containing this; see +# pier/agents/installed/claude_code.py, ClaudeCode.run(). +_CLAUDE_EXEC_MARKER = "--output-format=stream-json" + +_REMOTE_DIR = "/tmp/loopx-goal" +_LOOPX_MOUNT = "/opt/loopx" +_DEFAULT_LOOPX_ROOT = str( + Path(__file__).resolve().parents[1] / "loopx-official-latest" +) + +# Carried over verbatim from goal_codex.py. DeepSWE grades a committed patch, +# so committing belongs in the objective — a run that solves the task and never +# commits scores zero — and both arms must be prompted with the same words or +# "the effect of that wording" folds into the measured difference. +_OBJECTIVE = ( + "Complete the software engineering task described in the task file. " + "Work in the repository, keep existing behaviour intact, verify the change " + "against the repository's own tests, and commit the finished work to a new " + "branch off main. The goal is complete only once the change is committed." +) + +_OBJECTIVE_STAGED = ( + "Complete the software engineering task described in the task file, in " + "stages, and do not consider the goal complete until every stage is done.\n" + "Stage 1: make the target behaviour work and commit it.\n" + "Stage 2: re-read the task description and check your implementation " + "against every requirement it states, including ones you did not address " + "in stage 1. Fix what is missing and commit.\n" + "Stage 3: look for behaviour you may have broken elsewhere in the " + "repository, run the wider test suite, and fix any regression you find.\n" + "Stage 4: consider edge cases the tests may not cover — empty inputs, " + "concurrent use, error paths — and handle the ones the task implies.\n" + "The goal is complete only after stage 4." +) + + +def _objective() -> str: + return _OBJECTIVE_STAGED if os.environ.get("MR_GOAL_OBJECTIVE") == "staged" else _OBJECTIVE + + +class PlainClaudeCode(ClaudeCode): + """The control arm: stock Claude Code, plus the objective text. + + The objective is appended for the same reason `PlainCodex` appends it: the + LoopX arm necessarily carries that wording in its staged Todo, so + withholding it here would make the comparison partly about the prompt + instead of about the loop. + + What remains different is intrinsic and is the treatment: `claude --print` + runs until the model stops and then exits, while LoopX runs a Turn, requires + an independent validator to prove the postcondition, and only then asks for + the next one. + """ + + async def run(self, instruction, environment, context): # type: ignore[override] + return await super().run( + f"{instruction}\n\n{_objective()}", environment, context + ) + + +class LoopxClaudeCode(ClaudeCode): + """Claude Code driven by LoopX's governed Turn loop. + + The plain arm stops when the model says it is finished. LoopX is the arm + where something other than the model decides: it runs one Turn, requires an + independent validator to prove HEAD moved and the tree is clean, and only + then commits the Todo and spends quota. Turn two happens because the + controller asks for it, not because the model volunteered. + + Everything else is held to the plain arm — same model, same + `bypassPermissions`, same container, same task set, same objective wording. + + The validator is deliberately structural (HEAD moved, tree clean) and never + the hidden tests: the benchmark runs its verifier after the trial and + outside the controller, so a loop that could read the grade could steer on + it. What the validator proves is that the agent really committed work + rather than declaring success over an unchanged tree. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._loopx_instruction: str | None = None + self._loopx_swapped = False + + async def run(self, instruction, environment, context): # type: ignore[override] + self._loopx_instruction = instruction + self._loopx_swapped = False + try: + return await super().run(instruction, environment, context) + finally: + if not self._loopx_swapped: + raise RuntimeError( + "LoopxClaudeCode never intercepted a `claude --print` command " + "— this run would have been plain Claude Code with no LoopX loop." + ) + + async def exec_as_agent(self, environment, command: str = "", env=None, **kwargs): # type: ignore[override] + if _CLAUDE_EXEC_MARKER not in command: + return await super().exec_as_agent( + environment, command=command, env=env, **kwargs + ) + + self._loopx_swapped = True + + loopx_root = Path(os.environ.get("MR_LOOPX_ROOT", _DEFAULT_LOOPX_ROOT)) + if not (loopx_root / "loopx" / "__init__.py").is_file(): + raise FileNotFoundError(f"LoopX package not found under {loopx_root}") + here = Path(__file__).resolve().parent + + await super().exec_as_agent( + environment, command=f"mkdir -p {shlex.quote(_REMOTE_DIR)}", env=env + ) + + # LoopX arrives as one tarball rather than a bind mount: the container is + # created by Pier from its own compose file, so an agent cannot add a + # mount to it, and uploading 710 files one at a time is not a serious + # option. + with tempfile.TemporaryDirectory() as tmp: + tarball = Path(tmp) / "loopx.tar.gz" + subprocess.run( + ["tar", "czf", str(tarball), "-C", str(loopx_root), "loopx"], + check=True, + ) + task_path = Path(tmp) / "task.txt" + task_path.write_text(self._loopx_instruction or "", encoding="utf-8") + for local, remote in ( + (tarball, f"{_REMOTE_DIR}/loopx.tar.gz"), + (task_path, f"{_REMOTE_DIR}/task.txt"), + (here / "loopx_turn_runner.py", f"{_REMOTE_DIR}/loopx_turn_runner.py"), + (here / "loopx_claude_adapter.py", f"{_REMOTE_DIR}/loopx_claude_adapter.py"), + ): + await environment.upload_file(str(local), remote) + + if environment.default_user is not None: + await self.exec_as_root( + environment, + command=f"chown -R {environment.default_user} {shlex.quote(_REMOTE_DIR)}", + ) + await super().exec_as_agent( + environment, + command=( + f"mkdir -p {shlex.quote(_LOOPX_MOUNT)} && " + f"tar xzf {shlex.quote(_REMOTE_DIR)}/loopx.tar.gz " + f"-C {shlex.quote(_LOOPX_MOUNT)} && " + f"python3 -c 'import sys; sys.path.insert(0, \"{_LOOPX_MOUNT}\"); " + "import loopx.cli_commands.turn'" + ), + env=env, + ) + + args = [ + "python3", + f"{_REMOTE_DIR}/loopx_turn_runner.py", + "--project", "__PWD__", + "--task-file", f"{_REMOTE_DIR}/task.txt", + "--runtime-root", f"{_REMOTE_DIR}/runtime", + "--adapter", "loopx_claude_adapter.py", + # --model is required by the runner but the adapter resolves the + # real one from ANTHROPIC_MODEL, which Pier has already set in this + # container from the sweep's --model. Passing it twice would let + # the two drift. + "--model", (self.model_name or ""), + "--quota", os.environ.get("MR_LOOPX_QUOTA", "4"), + ] + rendered = shlex.join(args).replace("'__PWD__'", '"$(pwd)"').replace( + "__PWD__", '"$(pwd)"' + ) + output = (EnvironmentPaths.agent_dir / "loopx-turns.json").as_posix() + return await super().exec_as_agent( + environment, + command=( + 'export PATH="$HOME/.local/bin:$PATH"; ' + f"PYTHONPATH={shlex.quote(_LOOPX_MOUNT)} {rendered} " + f"2>&1 thread/start -> thread/goal/set(active) + -> turn/start -> observe continuation turns while the Goal stays active + +That transaction is not reimplemented here. `native_codex_goal.py` from LoopX +already owns it, is stdlib-only, and is the same code path the LoopX arm will +use later — sharing it is what keeps the two arms differing in LoopX alone +rather than in how each one talks to Codex. It is copied into the container at +run time rather than baked into the image so that the two arms cannot drift. + +The swap is done by intercepting `exec_as_agent` rather than by reimplementing +`run()`. Pier's `run()` is one long method whose setup (auth resolution, +ownership fixes, config blocks) would have to be duplicated and kept in step +with upstream; intercepting the one command that matters leaves that setup +untouched. If Pier ever changes how it invokes Codex, the marker below stops +matching and this fails loudly instead of silently reverting to plain +`codex exec` — which would look like a successful Goal run with no Goal in it. + +Usage: + + MR_AGENT=goal_codex:GoalCodex MR_MODEL=openai/gpt-5.5 ./run.sh --all -i + +Environment: + + MR_GOAL_PREFLIGHT=1 prove Goal attachment and stop before any model + turn — costs nothing, use it first on a new box + MR_GOAL_TIMEOUT_SEC ceiling for the continuation loop (default 3600, + under the 5400 s task budget so the loop stops + itself instead of being killed mid-turn) + MR_GOAL_TOKEN_BUDGET optional Goal token budget + MR_NATIVE_GOAL_MODULE path to LoopX's native_codex_goal.py on the host +""" + +from __future__ import annotations + +import os +import shlex +import subprocess +import tempfile +from pathlib import Path + +from pier.agents.installed.codex import Codex +from pier.models.trial.paths import EnvironmentPaths + +# Pier builds exactly one command containing this; see +# pier/agents/installed/codex.py, Codex.run(). +_CODEX_EXEC_MARKER = "codex exec " + +_REMOTE_DIR = "/tmp/loopx-goal" +_LOOPX_MOUNT = "/opt/loopx" +_DEFAULT_LOOPX_ROOT = str( + Path(__file__).resolve().parents[1] / "wen" / "loopx" +) +_DEFAULT_MODULE_RELATIVE = ( + "loopx/capabilities/benchmark_toolkit/native_codex_goal.py" +) + +# The Goal objective is fixed rather than derived from the task text. A Goal is +# meant to state the durable intent that survives across continuation turns, +# while the task file already carries the specifics; restating the task as the +# objective gave the model two copies of the same thing and nothing to hold on +# to between turns. DeepSWE grades a committed patch, so committing belongs in +# the objective — a run that solves the task and never commits scores zero. +_OBJECTIVE = ( + "Complete the software engineering task described in the task file. " + "Work in the repository, keep existing behaviour intact, verify the change " + "against the repository's own tests, and commit the finished work to a new " + "branch off main. The goal is complete only once the change is committed." +) + +# A Goal only continues while it is still active, so an objective that one turn +# can satisfy never exercises the continuation loop — and the objective above +# says outright that committing completes it. Across 53 runs the continuation +# count was zero every time, which makes the measured "Goal API has no effect" +# a statement about an objective that never needed the API, not about the API. +# +# This variant withholds completion until work that cannot plausibly finish in +# one turn is done: pass, then re-derive from the tests, then hunt regressions, +# then edge cases. Whether that actually keeps the Goal active is the thing +# being tested — if the continuation count is still zero, single-turn +# termination is Codex's behaviour here rather than an artefact of the wording. +_OBJECTIVE_STAGED = ( + "Complete the software engineering task described in the task file, in " + "stages, and do not consider the goal complete until every stage is done.\n" + "Stage 1: make the target behaviour work and commit it.\n" + "Stage 2: re-read the task description and check your implementation " + "against every requirement it states, including ones you did not address " + "in stage 1. Fix what is missing and commit.\n" + "Stage 3: look for behaviour you may have broken elsewhere in the " + "repository, run the wider test suite, and fix any regression you find.\n" + "Stage 4: consider edge cases the tests may not cover — empty inputs, " + "concurrent use, error paths — and handle the ones the task implies.\n" + "The goal is complete only after stage 4." +) + + +def _objective() -> str: + return _OBJECTIVE_STAGED if os.environ.get("MR_GOAL_OBJECTIVE") == "staged" else _OBJECTIVE + + +# All three arms keep Pier's own agent name. Overriding name() per arm looked +# tidy but fed straight into AgentInstallSpec.fingerprint(), whose first input is +# agent_name — so each arm produced a different PIER_AGENT_INSTALL_FINGERPRINT, +# invalidated the Docker layer cache, and rebuilt `nvm install 22` plus the npm +# install of Codex for every task in every arm: 162 builds where 54 would do. +# It also removed the only fallback for a network outage, since a cached layer +# needs no proxy. The arm is selected by pier_cn.py rebinding AgentName.CODEX, +# which needs no distinct name. + +_WEB_SEARCH_OFF = 'printf "\\nweb_search = \\"disabled\\"\\n" >> "$CODEX_HOME/config.toml"' + + +class PlainCodex(Codex): + """The control arm: same everything, no Goal attached. + + Exists so that Goal vs no-Goal differs in the Goal API and nothing else. + Two things have to be carried over from GoalCodex or the comparison measures + the wrong difference: + + * ``web_search`` off. Stock Codex leaves it at its default, and it is a + hosted tool the container's egress allowlist cannot block, so one arm + could look answers up. + * the objective text. A Goal cannot exist without an objective, so the Goal + arm is necessarily prompted with those three sentences. Withholding them + here would fold "the effect of that wording" into the measured difference. + Appending them leaves the API as the only variable. + + What remains different is intrinsic: `codex exec` runs one turn and exits, + while the app-server keeps serving continuations while the Goal stays + active. That *is* the treatment. + """ + + async def run(self, instruction, environment, context): # type: ignore[override] + return await super().run( + f"{instruction}\n\n{_objective()}", environment, context + ) + + async def exec_as_agent(self, environment, command: str = "", env=None, **kwargs): # type: ignore[override] + if _CODEX_EXEC_MARKER in command: + await super().exec_as_agent(environment, command=_WEB_SEARCH_OFF, env=env) + return await super().exec_as_agent( + environment, command=command, env=env, **kwargs + ) + + +class LoopxCodex(Codex): + """The third arm: Codex driven by LoopX's governed Turn loop. + + The other two arms both stop when the model says it is finished — `codex + exec` exits, and the Goal API marked every one of 53 runs complete on the + first turn. LoopX is the only arm where something other than the model + decides: it runs one Turn, requires an independent validator to prove the + postcondition, and only then commits and spends quota. Turn two happens + because the controller asks for it. + + Everything else is held to the other arms: same model, same disabled + web_search, same bypassed sandbox, same task set. The staged Todo text + mirrors the four-stage objective already tested on the Goal arm, where it + did not produce a single continuation — so any multi-turn behaviour here is + attributable to the loop rather than to the wording. + + LoopX is bind-mounted rather than installed: it declares no runtime + dependencies, and a read-only mount cannot drift between tasks the way 54 + separate installs could. + + Sandbox is ``workspace-write`` rather than the bypass the other two arms + use, because LoopX rejects anything else in two places — the argparse + choices and again in the driver ("Codex CLI sandbox must be read-only or + workspace-write"). Whether Codex can actually execute under it inside + these containers is the thing this arm has to establish first: the app- + server path could not, but that is a different code path from + ``codex exec --sandbox``, and assuming they behave alike is what a smoke + test is for. If it works, the other two arms should move to the same value + so permissions stop being a second difference between the arms. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._loopx_instruction: str | None = None + self._loopx_swapped = False + + async def run(self, instruction, environment, context): # type: ignore[override] + self._loopx_instruction = instruction + self._loopx_swapped = False + try: + return await super().run(instruction, environment, context) + finally: + if not self._loopx_swapped: + raise RuntimeError( + "LoopxCodex never intercepted a `codex exec` command — this " + "run would have been plain Codex with no LoopX loop." + ) + + async def exec_as_agent(self, environment, command: str = "", env=None, **kwargs): # type: ignore[override] + if _CODEX_EXEC_MARKER not in command: + return await super().exec_as_agent( + environment, command=command, env=env, **kwargs + ) + + self._loopx_swapped = True + model = self._command_model_name or (self.model_name or "").split("/")[-1] + + loopx_root = Path(os.environ.get("MR_LOOPX_ROOT", _DEFAULT_LOOPX_ROOT)) + if not (loopx_root / "loopx" / "__init__.py").is_file(): + raise FileNotFoundError(f"LoopX package not found under {loopx_root}") + runner_src = Path(__file__).resolve().parent / "loopx_turn_runner.py" + + await super().exec_as_agent( + environment, command=f"mkdir -p {shlex.quote(_REMOTE_DIR)}", env=env + ) + await super().exec_as_agent(environment, command=_WEB_SEARCH_OFF, env=env) + + # LoopX arrives as one tarball rather than a bind mount: the container is + # created by Pier from its own compose file, so an agent cannot add a + # mount to it, and uploading 710 files one at a time is not a serious + # option. Packed once per run rather than once per task would be nicer + # still, but the tar is ~13 MB and building it is far cheaper than the + # model turn that follows. + runner_source = getattr(self, "_goal_runner_source", None) + with tempfile.TemporaryDirectory() as tmp: + tarball = Path(tmp) / "loopx.tar.gz" + subprocess.run( + ["tar", "czf", str(tarball), "-C", str(loopx_root), "loopx"], + check=True, + ) + task_path = Path(tmp) / "task.txt" + task_path.write_text(self._loopx_instruction or "", encoding="utf-8") + for local, remote in ( + (tarball, f"{_REMOTE_DIR}/loopx.tar.gz"), + (task_path, f"{_REMOTE_DIR}/task.txt"), + (runner_src, f"{_REMOTE_DIR}/loopx_turn_runner.py"), + (runner_src.parent / "codex_nosandbox_wrapper.py", + f"{_REMOTE_DIR}/codex_nosandbox_wrapper.py"), + ): + await environment.upload_file(str(local), remote) + + if environment.default_user is not None: + await self.exec_as_root( + environment, + command=f"chown -R {environment.default_user} {shlex.quote(_REMOTE_DIR)} && chmod +x {shlex.quote(_REMOTE_DIR)}/codex_nosandbox_wrapper.py", + ) + await super().exec_as_agent( + environment, + command=( + f"mkdir -p {shlex.quote(_LOOPX_MOUNT)} && " + f"tar xzf {shlex.quote(_REMOTE_DIR)}/loopx.tar.gz " + f"-C {shlex.quote(_LOOPX_MOUNT)} && " + f"python3 -c 'import sys; sys.path.insert(0, \"{_LOOPX_MOUNT}\"); " + "import loopx.cli_commands.turn'" + ), + env=env, + ) + + args = [ + "python3", + f"{_REMOTE_DIR}/loopx_turn_runner.py", + "--project", "__PWD__", + "--task-file", f"{_REMOTE_DIR}/task.txt", + "--runtime-root", f"{_REMOTE_DIR}/runtime", + "--codex-bin", "codex", + "--model", model, + "--sandbox", os.environ.get("MR_LOOPX_SANDBOX", "workspace-write"), + "--quota", os.environ.get("MR_LOOPX_QUOTA", "4"), + ] + rendered = shlex.join(args).replace("'__PWD__'", '"$(pwd)"').replace( + "__PWD__", '"$(pwd)"' + ) + output = (EnvironmentPaths.agent_dir / "loopx-turns.json").as_posix() + return await super().exec_as_agent( + environment, + command=( + "if [ -s ~/.nvm/nvm.sh ]; then . ~/.nvm/nvm.sh; fi; " + f"PYTHONPATH={shlex.quote(_LOOPX_MOUNT)} {rendered} " + f"2>&1 {shlex.quote(_REMOTE_DIR)}/run_goal.py <<'LOOPX_EOF'\n{runner}\nLOOPX_EOF", + env=env, + ) + + timeout = os.environ.get("MR_GOAL_TIMEOUT_SEC", "3600") + budget = os.environ.get("MR_GOAL_TOKEN_BUDGET", "") + effort = os.environ.get("MR_REASONING_EFFORT", "").strip() + preflight = os.environ.get("MR_GOAL_PREFLIGHT", "") not in ("", "0") + + args = [ + "python3", + f"{_REMOTE_DIR}/run_goal.py", + "--cwd", + "__PWD__", + "--objective-file", + # The LoopX arm renders its objective with LoopX's own CLI and + # points here; a subclass cannot substitute it by rewriting the + # command, because this method calls `super().exec_as_agent` rather + # than `self.`, so an override never sees the built command at all. + os.environ.get("MR_GOAL_OBJECTIVE_FILE", f"{_REMOTE_DIR}/objective.txt"), + "--task-file", + f"{_REMOTE_DIR}/task.txt", + "--codex-bin", + "codex", + "--model", + model, + "--goal-timeout-seconds", + timeout, + # NativeGoalConfig defaults to sandbox="workspace-write", which on + # Linux is enforced with landlock/seccomp and cannot be set up + # inside these task containers. Codex then declines to run + # commands: the first attempt produced 431 assistant-message deltas, + # zero command_execution events, zero file_change events and an + # empty model.patch, which the verifier scored 0/24 — a harness + # failure that reads exactly like the model failing the task. + # `codex exec` avoids it with --dangerously-bypass-approvals-and- + # sandbox; this is the app-server equivalent, so both arms execute + # under the same permissions. + "--sandbox", + os.environ.get("MR_GOAL_SANDBOX", "danger-full-access"), + ] + if effort: + args += ["--effort", effort] + if budget: + args += ["--token-budget", budget] + # Empty on this arm. The LoopX arm sets it to the skill ids its + # installed profile materialized, which arms `skills/list` as a + # precondition of thread creation: without it a run in which Codex never + # discovered LoopX still finishes and still scores, and is then filed as + # a LoopX result. One such run has already happened. + skills = os.environ.get("MR_GOAL_REQUIRED_SKILL_IDS", "").strip() + if skills: + args += ["--required-skill-ids", skills] + runner_env = getattr(self, "_goal_runner_env", {}) + if runner_env: + args += [ + "--loopx-cli", runner_env["LOOPX_CLI"], + "--registry", runner_env["LOOPX_REGISTRY"], + "--runtime-root", runner_env["LOOPX_RUNTIME_ROOT"], + "--goal-id", runner_env["LOOPX_GOAL_ID"], + "--agent-id", runner_env["LOOPX_AGENT_ID"], + ] + if runner_env.get("LOOPX_MODE"): + args += ["--mode", runner_env["LOOPX_MODE"]] + if runner_env.get("LOOPX_RECOVER_BLOCKED") == "1": + args.append("--recover-blocked") + if runner_env.get("LOOPX_MODE") in {"codex-cli", "heartbeat"}: + args += [ + "--segment-timeout-seconds", + os.environ.get("MR_HEARTBEAT_SEGMENT_TIMEOUT_SEC", "7200"), + ] + if runner_env.get("LOOPX_MODE") == "ssh-goal": + args += [ + "--turn-idle-timeout-seconds", + os.environ.get("MR_LOOPX_TURN_IDLE_TIMEOUT_SEC", "7200"), + ] + if preflight: + args.append("--preflight-only") + + # No task declares a working directory, so `codex exec` would have run + # in whatever WORKDIR the image sets — different per repository. Resolve + # it in the container instead of guessing. Both spellings are replaced + # because shlex.join only quotes a token that needs it, and this one + # (letters and underscores) comes back bare: matching only the quoted + # form silently leaves the placeholder in the command, which surfaces as + # FileNotFoundError('__PWD__') from inside the runner. + rendered = shlex.join(args) + rendered = rendered.replace("'__PWD__'", '"$(pwd)"').replace( + "__PWD__", '"$(pwd)"' + ) + + output = (EnvironmentPaths.agent_dir / "codex-goal.json").as_posix() + # The LoopX arm points CODEX_HOME at its installed profile so app-server + # discovers the LoopX skills; this arm leaves it as Pier set it up. + codex_home = os.environ.get("MR_GOAL_CODEX_HOME", "").strip() + prefix = f"export CODEX_HOME={shlex.quote(codex_home)}; " if codex_home else "" + for key, value in runner_env.items(): + prefix += f"export {key}={shlex.quote(str(value))}; " + # The profile's `loopx` launcher was built on the host and records the + # host's own Python path, which does not exist in the container. This + # was fixed for the one-shot bootstrap script by exporting the variable + # in that command's own shell -- but bootstrap and this long-lived + # app-server process are separate `docker exec` invocations, each with + # its own shell, so the export did not carry over. Any `loopx` command + # Codex itself runs *during* a turn -- todo claim, quota should-run, + # heartbeat-prompt -- inherits app-server's process environment, not + # bootstrap's, and failed with the same "configured Python executable + # not found" every time until this export is repeated here. A session + # transcript showed exactly that: two failed `loopx` calls mid-turn. + if os.environ.get("MR_GOAL_ARM_LOOPX_PYTHON", "") not in ("", "0"): + prefix += 'export LOOPX_PYTHON="$(command -v python3)"; ' + return await super().exec_as_agent( + environment, + command=( + "if [ -s ~/.nvm/nvm.sh ]; then . ~/.nvm/nvm.sh; fi; " + f"{prefix}{rendered} 2>&1 dict[str, Any]: + for line in reversed(text.splitlines()): + try: + value = json.loads(line) + except ValueError: + continue + if isinstance(value, dict): + return value + try: + value = json.loads(text) + except ValueError: + return {} + return value if isinstance(value, dict) else {} + + +def _goal_terminal(todo_payload: dict[str, Any]) -> bool: + todos = todo_payload.get("todos") + if not isinstance(todos, list): + return False + statuses = [ + item.get("status") + for item in todos + if isinstance(item, dict) and item.get("priority") == "P0" + ] + return bool(statuses) and all(status in {"done", "deferred"} for status in statuses) + + +def _todo_state(args: argparse.Namespace, project: Path) -> dict[str, Any]: + completed = subprocess.run( + [ + args.loopx_cli, + "--registry", + args.registry, + "--runtime-root", + args.runtime_root, + "--format", + "json", + "todo", + "list", + "--goal-id", + args.goal_id, + ], + cwd=project, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + payload = _json_payload(completed.stdout) + return { + "returncode": completed.returncode, + "terminal": completed.returncode == 0 and _goal_terminal(payload), + "payload": payload, + "stdout_tail": completed.stdout[-800:], + "stderr_tail": completed.stderr[-800:], + } + + +def _claim_primary_p0( + args: argparse.Namespace, + project: Path, +) -> dict[str, Any]: + """Keep the benchmark P0 selected across validated-progress segments.""" + state = _todo_state(args, project) + if state["returncode"] != 0: + return { + "ok": False, + "reason": "todo_list_failed", + "returncode": state["returncode"], + "stdout_tail": state["stdout_tail"], + "stderr_tail": state["stderr_tail"], + } + if state["terminal"]: + return {"ok": True, "terminal": True, "claimed": False} + + todos = state["payload"].get("todos") + open_p0 = [ + item + for item in todos if isinstance(item, dict) + and item.get("priority") == "P0" + and item.get("role") == "agent" + and item.get("status") == "open" + ] if isinstance(todos, list) else [] + if len(open_p0) != 1: + return { + "ok": False, + "reason": f"expected_one_open_agent_p0_got_{len(open_p0)}", + } + + todo = open_p0[0] + todo_id = str(todo.get("todo_id") or "") + if not todo_id: + return {"ok": False, "reason": "open_agent_p0_missing_todo_id"} + if todo.get("claimed_by") == args.agent_id: + return { + "ok": True, + "terminal": False, + "claimed": False, + "todo_id": todo_id, + "claimed_by": args.agent_id, + } + if todo.get("claimed_by") not in {None, ""}: + return { + "ok": False, + "reason": "benchmark_p0_claimed_by_other_agent", + "todo_id": todo_id, + "claimed_by": todo.get("claimed_by"), + } + + completed = subprocess.run( + [ + args.loopx_cli, + "--registry", + args.registry, + "--runtime-root", + args.runtime_root, + "--format", + "json", + "todo", + "claim", + "--goal-id", + args.goal_id, + "--todo-id", + todo_id, + "--claimed-by", + args.agent_id, + "--agent-id", + args.agent_id, + ], + cwd=project, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + payload = _json_payload(completed.stdout) + ok = completed.returncode == 0 and payload.get("ok") is not False + return { + "ok": ok, + "terminal": False, + "claimed": ok, + "todo_id": todo_id, + "claimed_by": args.agent_id if ok else None, + "returncode": completed.returncode, + "stdout_tail": completed.stdout[-800:], + "stderr_tail": completed.stderr[-800:], + } + + +def _retry_delay( + payload: dict[str, Any], + args: argparse.Namespace, + failure_streak: int, +) -> float: + if not failure_streak: + return args.segment_interval_seconds + delay = args.retry_backoff_base_seconds * (2 ** (failure_streak - 1)) + host_failure = payload.get("host_failure") + if isinstance(host_failure, dict) and host_failure.get("retryable") is True: + retry = host_failure.get("retry") + if isinstance(retry, dict): + recommended = retry.get("backoff_seconds") + if isinstance(recommended, (int, float)) and recommended > 0: + delay = max(delay, float(recommended)) + return min(args.retry_backoff_cap_seconds, delay) + + +def run(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + project = Path(args.cwd).resolve() + base_sha = head_sha(project) + deadline = time.monotonic() + args.goal_timeout_seconds + delivery_path = Path("/logs/agent/delivery_receipt.json") + receipts: list[dict[str, Any]] = [] + failure_streak = 0 + wrapper = Path(__file__).with_name("codex_nosandbox_wrapper.py") + delivery_helper = Path(__file__).with_name("workspace_delivery.py") + + if args.preflight_only: + payload = { + "schema_version": "deepswe_codex_cli_runner_v1", + "host_surface": "codex_cli_turn_run_once", + "preflight": wrapper.is_file() and delivery_helper.is_file(), + } + return payload, 0 if payload["preflight"] else 2 + + for segment in range(1, args.max_segments + 1): + remaining = deadline - time.monotonic() + if remaining <= 0: + break + claim = _claim_primary_p0(args, project) + if claim.get("terminal"): + delivery = normalize_delivery(project, base_sha) + write_receipt(delivery_path, delivery) + if delivery["treatment_valid"]: + result = { + "schema_version": "deepswe_codex_cli_runner_v1", + "execution_mode": "loopx_turn_run_once", + "host_surface": "codex_cli", + "model": args.model, + "effort": args.effort, + "turn_status": "completed", + "segments": receipts, + "segment_count": len(receipts), + "delivery": delivery, + "treatment_valid": True, + "task_correctness_authority": "independent_verifier", + } + return result, 0 + if not claim.get("ok"): + receipts.append({ + "segment": segment, + "returncode": claim.get("returncode", 1), + "status": "claim_failed", + "result_kind": None, + "validation_status": None, + "p0_claim": claim, + }) + break + segment_timeout = max(60.0, min(args.segment_timeout_seconds, remaining)) + validation = [ + "python3", + str(delivery_helper), + "--project", + str(project), + "--base-sha", + base_sha, + "--receipt", + str(delivery_path), + "--require-valid", + ] + command = [ + args.loopx_cli, + "--registry", + args.registry, + "--runtime-root", + args.runtime_root, + "--format", + "json", + "turn", + "run-once", + "--goal-id", + args.goal_id, + "--agent-id", + args.agent_id, + "--turn-instance-id", + f"deepswe-codex-cli-{segment}", + "--host", + "codex-cli", + "--execution-mode", + "isolated-headless", + "--scheduler-owner", + "agent_cli_loop", + "--project", + str(project), + "--codex-bin", + str(wrapper), + "--codex-sandbox", + "workspace-write", + "--codex-model", + args.model or "", + "--validation-command-json", + json.dumps(validation), + "--validation-timeout-seconds", + "180", + "--timeout-seconds", + str(segment_timeout), + "--no-global-sync", + "--execute", + ] + env = os.environ.copy() + env["MR_REAL_CODEX"] = args.codex_bin + env["MR_LOOPX_PROJECT"] = str(project) + env["MR_CODEX_REASONING_EFFORT"] = args.effort or "" + completed = subprocess.run( + command, + cwd=project, + env=env, + capture_output=True, + text=True, + timeout=segment_timeout + 240, + check=False, + ) + payload = _json_payload(completed.stdout) + receipts.append( + { + "segment": segment, + "returncode": completed.returncode, + "status": payload.get("status"), + "result_kind": payload.get("result_kind"), + "validation_status": payload.get("validation_status"), + "p0_claim": claim, + "stdout_tail": completed.stdout[-1200:], + "stderr_tail": completed.stderr[-1200:], + } + ) + delivery = normalize_delivery(project, base_sha) + write_receipt(delivery_path, delivery) + todo_state = _todo_state(args, project) + receipts[-1]["goal_terminal"] = todo_state["terminal"] + receipts[-1]["todo_state_returncode"] = todo_state["returncode"] + if delivery["treatment_valid"] and ( + completed.returncode == 0 + and ( + payload.get("status") == "committed" + or payload.get("validation_status") == "passed" + ) + and todo_state["terminal"] + ): + result = { + "schema_version": "deepswe_codex_cli_runner_v1", + "execution_mode": "loopx_turn_run_once", + "host_surface": "codex_cli", + "model": args.model, + "effort": args.effort, + "turn_status": "completed", + "segments": receipts, + "segment_count": len(receipts), + "delivery": delivery, + "treatment_valid": True, + "task_correctness_authority": "independent_verifier", + } + return result, 0 + if completed.returncode != 0 and not payload: + break + failure_streak = failure_streak + 1 if completed.returncode else 0 + if segment < args.max_segments: + delay = _retry_delay(payload, args, failure_streak) + time.sleep(min(delay, max(0.0, deadline - time.monotonic()))) + + delivery = normalize_delivery(project, base_sha) + write_receipt(delivery_path, delivery) + result = { + "schema_version": "deepswe_codex_cli_runner_v1", + "execution_mode": "loopx_turn_run_once", + "host_surface": "codex_cli", + "model": args.model, + "effort": args.effort, + "turn_status": "failed", + "segments": receipts, + "segment_count": len(receipts), + "delivery": delivery, + "treatment_valid": False, + "task_correctness_authority": "independent_verifier", + "error": "codex_cli_did_not_reach_validated_delivery", + } + return result, 12 + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--cwd", required=True) + result.add_argument("--objective-file", required=True) + result.add_argument("--task-file", required=True) + result.add_argument("--codex-bin", default="codex") + result.add_argument("--model") + result.add_argument("--effort") + result.add_argument("--token-budget", type=int) + result.add_argument("--response-timeout-seconds", type=float, default=180) + result.add_argument("--goal-timeout-seconds", type=float, default=5400) + result.add_argument("--segment-timeout-seconds", type=float, default=7200) + result.add_argument("--max-segments", type=int, default=256) + result.add_argument("--segment-interval-seconds", type=float, default=5) + result.add_argument("--retry-backoff-base-seconds", type=float, default=10) + result.add_argument("--retry-backoff-cap-seconds", type=float, default=60) + result.add_argument("--sandbox", default="danger-full-access") + result.add_argument("--required-skill-ids", default="") + result.add_argument("--preflight-only", action="store_true") + result.add_argument("--recover-blocked", action="store_true") + result.add_argument("--max-unblocks", type=int, default=8) + result.add_argument("--loopx-cli", required=True) + result.add_argument("--registry", required=True) + result.add_argument("--runtime-root", required=True) + result.add_argument("--goal-id", required=True) + result.add_argument("--agent-id", required=True) + result.add_argument("--mode") + return result + + +def main() -> int: + payload, code = run(parser().parse_args()) + print(json.dumps(payload, indent=2, sort_keys=True)) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/deepswe-gptxhigh-v1/loopx_heartbeat_supervisor.py b/benchmark/deepswe-gptxhigh-v1/loopx_heartbeat_supervisor.py new file mode 100755 index 0000000000..9317447d81 --- /dev/null +++ b/benchmark/deepswe-gptxhigh-v1/loopx_heartbeat_supervisor.py @@ -0,0 +1,403 @@ +#!/usr/bin/env python3 +"""Runner-owned recurring heartbeat host for the DeepSWE LoopX arm.""" + +from __future__ import annotations + +import argparse +import json +import os +import signal +import subprocess +import tempfile +import time +from pathlib import Path +from typing import Any + +from workspace_delivery import head_sha, normalize_delivery, write_receipt + + +def _decode_tail(value: bytes | str | None) -> str: + if value is None: + return "" + if isinstance(value, bytes): + value = value.decode("utf-8", "replace") + return value[-800:] + + +def _run_codex_segment( + command: list[str], + *, + cwd: Path, + env: dict[str, str], + trace: Path, + timeout: float, +) -> dict[str, Any]: + """Run one heartbeat wake without letting its timeout kill the supervisor.""" + timed_out = False + with trace.open("wb") as output: + process = subprocess.Popen( + command, + cwd=cwd, + env=env, + stdout=output, + stderr=subprocess.PIPE, + start_new_session=True, + ) + try: + _, stderr = process.communicate(timeout=timeout) + returncode = process.returncode + stderr_tail = _decode_tail(stderr) + except subprocess.TimeoutExpired as exc: + returncode = 124 + stderr_tail = _decode_tail(exc.stderr) + timed_out = True + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + # The CLI launcher can exit before its native child. Kill the whole + # isolated group even when the direct child has already returned. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + _, final_stderr = process.communicate() + if final_stderr: + stderr_tail = _decode_tail(final_stderr) + return { + "returncode": returncode, + "stderr_tail": stderr_tail, + "timed_out": timed_out, + } + + +def _invoke_json(command: list[str], *, cwd: Path, timeout: float = 180) -> dict[str, Any]: + completed = subprocess.run( + command, cwd=cwd, capture_output=True, text=True, timeout=timeout, check=False + ) + try: + payload = json.loads(completed.stdout) + except ValueError: + payload = {} + return { + "returncode": completed.returncode, + "payload": payload if isinstance(payload, dict) else {}, + "stdout_tail": completed.stdout[-800:], + "stderr_tail": completed.stderr[-800:], + } + + +def _loopx(args: argparse.Namespace, *command: str, cwd: Path) -> dict[str, Any]: + return _invoke_json( + [ + args.loopx_cli, + "--registry", + args.registry, + "--runtime-root", + args.runtime_root, + "--format", + "json", + *command, + ], + cwd=cwd, + ) + + +def _goal_terminal(todo_payload: dict[str, Any]) -> bool: + todos = todo_payload.get("todos") + if not isinstance(todos, list): + return False + statuses = [ + item.get("status") + for item in todos + if isinstance(item, dict) and item.get("priority") == "P0" + ] + return bool(statuses) and all(status in {"done", "deferred"} for status in statuses) + + +def _trace_thread_id(trace: Path) -> str | None: + try: + with trace.open(encoding="utf-8", errors="replace") as handle: + for line in handle: + try: + event = json.loads(line) + except ValueError: + continue + thread_id = event.get("thread_id") + if event.get("type") == "thread.started" and isinstance(thread_id, str): + return thread_id + except OSError: + return None + return None + + +def _release_canonical_duplicate( + project: Path, base_sha: str, delivery: dict[str, Any] +) -> bool: + """Keep linked continuation work without leaving two changed checkouts.""" + if not delivery.get("treatment_valid") or not delivery.get( + "recovered_from_linked_worktree" + ): + return False + source = Path(str(delivery.get("source_worktree") or "")).resolve() + if source == project: + return False + completed = subprocess.run( + ["git", "-C", str(project), "reset", "--hard", base_sha], + capture_output=True, + check=False, + ) + if completed.returncode: + raise RuntimeError( + "could not release canonical continuation duplicate: " + + _decode_tail(completed.stderr) + ) + return True + + +def _repair_todo(args: argparse.Namespace, wake: int, cwd: Path) -> dict[str, Any]: + return _loopx( + args, + "todo", + "add", + "--goal-id", + args.goal_id, + "--role", + "agent", + "--todo-id", + f"deepswe-delivery-repair-{wake}", + "--text", + "Recover or implement the requested task in /app, run public tests, and leave a non-empty committed patch. Do not complete the Goal before this delivery exists.", + "--task-class", + "advancement_task", + "--status", + "open", + "--execute", + cwd=cwd, + ) + + +def run(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + project = Path(args.cwd).resolve() + base_sha = head_sha(project) + deadline = time.monotonic() + args.goal_timeout_seconds + task_text = Path(args.task_file).read_text(encoding="utf-8").strip() + logs_dir = Path(args.logs_dir) + delivery_path = logs_dir / "delivery_receipt.json" + wakes: list[dict[str, Any]] = [] + resume_session_id: str | None = None + continuation_worktree: str | None = None + + if args.preflight_only: + prompt = _loopx( + args, + "heartbeat-prompt", + "--thin", + "--goal-id", + args.goal_id, + "--agent-id", + args.agent_id, + "--available-capability", + "shell", + "--available-capability", + "filesystem_write", + "--runtime-profile", + "outer_controller", + cwd=project, + ) + payload = { + "schema_version": "deepswe_heartbeat_supervisor_v1", + "host_surface": "recurring_outer_controller_heartbeat", + "preflight": prompt["returncode"] == 0 and bool(prompt["payload"].get("task_body")), + } + return payload, 0 if payload["preflight"] else 2 + + for wake in range(1, args.max_wakes + 1): + remaining = deadline - time.monotonic() + if remaining <= 0: + break + prompt_result = _loopx( + args, + "heartbeat-prompt", + "--thin", + "--goal-id", + args.goal_id, + "--agent-id", + args.agent_id, + "--available-capability", + "shell", + "--available-capability", + "filesystem_write", + "--runtime-profile", + "outer_controller", + cwd=project, + ) + task_body = str(prompt_result["payload"].get("task_body") or "") + if prompt_result["returncode"] != 0 or not task_body: + wakes.append({"wake": wake, "prompt": prompt_result, "error": "heartbeat_prompt_failed"}) + break + prompt = ( + task_body + + "\n\nBenchmark delivery fence: work on the task below in /app (a linked " + "worktree is allowed but will be recovered), run the repository's public tests, " + "and do not mark the Goal/Todo complete until a non-empty committed patch exists.\n\n" + + task_text + ) + if continuation_worktree: + prompt += ( + "\n\nHeartbeat continuation fence: continue the existing implementation in " + f"{continuation_worktree}; do not create another worktree or edit the " + "canonical checkout directly. Finish validation and the LoopX lifecycle " + "settlement before starting unrelated work." + ) + segment_timeout = max(60.0, min(args.segment_timeout_seconds, remaining)) + with tempfile.TemporaryDirectory(prefix="deepswe-heartbeat-") as tmp: + last = Path(tmp) / "last.txt" + trace = logs_dir / f"heartbeat-wake-{wake}.jsonl" + trace.parent.mkdir(parents=True, exist_ok=True) + common = [ + "--dangerously-bypass-approvals-and-sandbox", + "--skip-git-repo-check", + "--model", + args.model or "", + "-c", + f"model_reasoning_effort={args.effort}", + "--output-last-message", + str(last), + "--json", + ] + if resume_session_id: + command = [ + args.codex_bin, + "exec", + "resume", + *common, + resume_session_id, + prompt, + ] + else: + command = [ + args.codex_bin, + "exec", + *common, + "-C", + str(project), + "--", + prompt, + ] + segment = _run_codex_segment( + command, + cwd=project, + env=os.environ.copy(), + trace=trace, + timeout=segment_timeout, + ) + observed_session_id = _trace_thread_id(trace) + if observed_session_id: + resume_session_id = observed_session_id + last_text = last.read_text(encoding="utf-8", errors="replace") if last.exists() else "" + delivery = normalize_delivery(project, base_sha) + write_receipt(delivery_path, delivery) + todos = _loopx(args, "todo", "list", "--goal-id", args.goal_id, cwd=project) + terminal = _goal_terminal(todos["payload"]) + wakes.append( + { + "wake": wake, + "host_process": "codex_exec", + "returncode": segment["returncode"], + "timed_out": segment["timed_out"], + "resumed": wake > 1 and resume_session_id is not None, + "session_id": resume_session_id, + "prompt_sha256": __import__("hashlib").sha256(prompt.encode()).hexdigest(), + "last_message_tail": last_text[-800:], + "stderr_tail": segment["stderr_tail"], + "delivery_status": delivery["status"], + "goal_terminal": terminal, + } + ) + if delivery["treatment_valid"] and terminal: + result = { + "schema_version": "deepswe_heartbeat_supervisor_v1", + "execution_mode": "recurring_heartbeat", + "host_surface": "outer_controller_heartbeat", + "model": args.model, + "effort": args.effort, + "continuation_owner": "benchmark_supervisor", + "turn_status": "completed", + "wake_count": len(wakes), + "wakes": wakes, + "delivery": delivery, + "treatment_valid": True, + "task_correctness_authority": "independent_verifier", + } + return result, 0 + if terminal and not delivery["treatment_valid"]: + _repair_todo(args, wake, project) + canonical_released = _release_canonical_duplicate(project, base_sha, delivery) + wakes[-1]["canonical_released_for_continuation"] = canonical_released + if canonical_released: + continuation_worktree = str(delivery["source_worktree"]) + if wake < args.max_wakes: + time.sleep(args.heartbeat_interval_seconds) + + delivery = normalize_delivery(project, base_sha) + write_receipt(delivery_path, delivery) + result = { + "schema_version": "deepswe_heartbeat_supervisor_v1", + "execution_mode": "recurring_heartbeat", + "host_surface": "outer_controller_heartbeat", + "model": args.model, + "effort": args.effort, + "continuation_owner": "benchmark_supervisor", + "turn_status": "failed", + "wake_count": len(wakes), + "wakes": wakes, + "delivery": delivery, + "treatment_valid": False, + "task_correctness_authority": "independent_verifier", + "error": "heartbeat_did_not_reach_terminal_validated_delivery", + } + return result, 12 + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--cwd", required=True) + result.add_argument("--logs-dir", default="/logs/agent") + result.add_argument("--objective-file", required=True) + result.add_argument("--task-file", required=True) + result.add_argument("--codex-bin", default="codex") + result.add_argument("--model") + result.add_argument("--effort") + result.add_argument("--token-budget", type=int) + result.add_argument("--response-timeout-seconds", type=float, default=180) + result.add_argument("--goal-timeout-seconds", type=float, default=14400) + result.add_argument("--segment-timeout-seconds", type=float, default=7200) + result.add_argument("--max-wakes", type=int, default=8) + result.add_argument("--heartbeat-interval-seconds", type=float, default=5) + result.add_argument("--sandbox", default="danger-full-access") + result.add_argument("--required-skill-ids", default="") + result.add_argument("--preflight-only", action="store_true") + result.add_argument("--recover-blocked", action="store_true") + result.add_argument("--max-unblocks", type=int, default=8) + result.add_argument("--loopx-cli", required=True) + result.add_argument("--registry", required=True) + result.add_argument("--runtime-root", required=True) + result.add_argument("--goal-id", required=True) + result.add_argument("--agent-id", required=True) + result.add_argument("--mode") + return result + + +def main() -> int: + payload, code = run(parser().parse_args()) + print(json.dumps(payload, indent=2, sort_keys=True)) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/deepswe-gptxhigh-v1/loopx_native_codex.py b/benchmark/deepswe-gptxhigh-v1/loopx_native_codex.py new file mode 100644 index 0000000000..fab369d365 --- /dev/null +++ b/benchmark/deepswe-gptxhigh-v1/loopx_native_codex.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python3 +"""The LoopX arm, driven through LoopX's own product path. + +The first LoopX arm measured the wrong thing. It called `loopx turn run-once` +from an outer loop of my own, four times, against a goal document I hand-wrote +with four stages in it. `benchmark/deepswe/README.md` rules that out in as +many words -- the treatment must not be "an outer polling loop labeled as Goal +mode" -- and it makes the distinction a formal gate: "a clean run can still be +uncountable when the treatment did not execute the preregistered LoopX path". +Under LoopX's own standard those 54 results are uncountable. They describe my +wrapper, not this product. + +The real path, all of it LoopX's: + + install_native_codex_profile scripts/install-local.sh builds a release + snapshot and installs the seven LoopX skills + into a profile-owned CODEX_HOME + loopx bootstrap LoopX writes .loopx/registry.json and + .codex/goals//ACTIVE_GOAL_STATE.md, + with its own execution profile + loopx configure-goal registers the peer identity + render_native_codex_goal_prompt the installed CLI renders the real Goal + body -- the thing I used to hand-write + run_native_goal_process_until_terminal + codex app-server owns continuation while + the Goal stays active; nothing here counts + turns + +`required_skill_ids` makes `skills/list` a precondition of thread creation, so a +run in which Codex never discovered the LoopX skills fails before model work +instead of quietly scoring as a LoopX result. + +Everything Pier does to stand Codex up is inherited from GoalCodex, which also +already owns the app-server transaction (it copies LoopX's own +`native_codex_goal.py` into the container). Only three things differ from the +Goal arm: where CODEX_HOME points, who wrote the objective, and whether the +skills gate is armed. That is the intended contrast -- the same host, the same +transaction, LoopX present or absent. + +The profile is built on the host and shipped, rather than installed in the +container: `install-local.sh` is offline (no pip, npm, curl, wget, git clone or +apt in 872 lines, so the sandbox is not the obstacle), but it verifies that its +source tree is a clean checkout, and the container receives a tarball of the +`loopx` package with no `.git` to verify. Building it once on the host keeps +`source_clean` a real claim. Both sides use the same absolute path so the +release snapshot's symlinks stay valid. + +Usage: + + MR_AGENT=loopx_native_codex:LoopxNativeCodex MR_MODEL=openai/gpt-5.5 \ + ./run.sh --all -i + +Environment: + + MR_LOOPX_PREFLIGHT=1 prove skills discovery and Goal attachment, then + stop before any model turn -- costs nothing + MR_LOOPX_PROFILE_ROOT where the profile lives on host and in container + (default /tmp/loopx-profile; must match on both) + MR_LOOPX_ROOT LoopX checkout to install from + MR_GOAL_TIMEOUT_SEC ceiling for the continuation loop +""" + +from __future__ import annotations + +import json +import os +import shlex +import subprocess +import sys +import tempfile +from pathlib import Path + +from goal_codex import GoalCodex, _CODEX_EXEC_MARKER, _REMOTE_DIR + +_PROFILE_ROOT = os.environ.get( + "MR_LOOPX_PROFILE_ROOT", "/tmp/loopx-profile" +) +_LOOPX_ROOT = os.path.expanduser( + os.environ.get( + "MR_LOOPX_ROOT", + str(Path(__file__).resolve().parents[1] / "wen" / "loopx"), + ) +) +_GOAL_ID = "deepswe-task" +_AGENT_ID = "deepswe-codex" +_PROJECT = "/app" +_RUNTIME_PROFILES = { + "ssh-goal": "codex_app_ssh_goal", + "codex-cli": "codex_cli", + "heartbeat": "outer_controller", +} +# Deliberately not GoalCodex's objective.txt: that file is written after this +# runs, so sharing the name would let the Goal arm's hand-written objective +# overwrite the one LoopX rendered. +_OBJECTIVE_FILE = f"{_REMOTE_DIR}/loopx_objective.txt" +_GOAL_DOC_FILE = f"{_PROFILE_ROOT}/goal-doc.md" +_WEN_COMPAT = os.environ.get("MR_LOOPX_WEN_COMPAT", "1") not in ("", "0") +_SOURCE_DIR = Path(__file__).resolve().parent +_RUNNER_SOURCES = { + "ssh-goal": _SOURCE_DIR / "loopx_wen_native_runner.py", + "codex-cli": _SOURCE_DIR / "loopx_codex_cli_runner.py", + "heartbeat": _SOURCE_DIR / "loopx_heartbeat_supervisor.py", +} +_SUPPORT_SOURCES = ( + _SOURCE_DIR / "workspace_delivery.py", + _SOURCE_DIR / "codex_nosandbox_wrapper.py", +) + + +def build_host_profile(loopx_root: str = _LOOPX_ROOT, + profile_root: str = _PROFILE_ROOT) -> dict: + """Install the formal release snapshot once, on the host. + + Reuses an existing profile: the installer refuses a non-empty target on + purpose, because mixing installation revisions would invalidate the + treatment, and re-installing per task would repeat that work 54 times. + """ + sys.path.insert(0, loopx_root) + from loopx.capabilities.benchmark_toolkit.native_codex_profile import ( + compact_native_codex_profile_receipt, + inspect_native_codex_profile, + install_native_codex_profile, + ) + + target = Path(profile_root) + if target.exists() and any(target.iterdir()): + profile = inspect_native_codex_profile(target, source_root=loopx_root) + else: + profile = install_native_codex_profile(loopx_root, target) + return compact_native_codex_profile_receipt(profile) + + +class LoopxNativeCodex(GoalCodex): + """Codex with LoopX installed, driven by LoopX's rendered Goal.""" + + loopx_mode = "ssh-goal" + + async def exec_as_agent(self, environment, command, env=None, **kwargs): # noqa: ANN001 + # Intercept the same marker GoalCodex does, not the command it builds. + # GoalCodex reaches the launch through `super().exec_as_agent`, so an + # override keyed on `run_goal.py` is never called: the first version of + # this arm silently shipped no profile, armed no skills gate, and still + # produced a receipt that looked like a LoopX run. Everything below is + # therefore handed over through the environment, which GoalCodex reads + # while building its own command. + if _CODEX_EXEC_MARKER not in str(command): + return await super().exec_as_agent(environment, command, env=env, **kwargs) + + profile_receipt = build_host_profile() + mode = self.loopx_mode + runtime_profile = _RUNTIME_PROFILES[mode] + # GoalCodex owns the common Pier setup and command interception, but the + # three treatments must not share one execution surface. ssh-goal uses + # native app-server Goal attachment, codex-cli uses LoopX's built-in + # ``turn run-once`` Codex host, and heartbeat uses a recurring external + # supervisor which launches one fresh host wake per cadence tick. + self._goal_runner_source = _RUNNER_SOURCES[mode] + self._goal_runner_env = { + "LOOPX_CLI": f"{_PROFILE_ROOT}/bin/loopx", + "LOOPX_REGISTRY": f"{_PROJECT}/.loopx/registry.json", + "LOOPX_RUNTIME_ROOT": f"{_PROJECT}/.loopx/runtime", + "LOOPX_GOAL_ID": _GOAL_ID, + "LOOPX_AGENT_ID": _AGENT_ID, + "LOOPX_RECOVER_BLOCKED": "1", + "LOOPX_MODE": mode, + } + treatment_env = dict(env or {}) + treatment_env["LOOPX_WEN_COMPAT"] = "1" if _WEN_COMPAT else "0" + parent = str(Path(_PROFILE_ROOT).parent) + name = Path(_PROFILE_ROOT).name + + await super().exec_as_agent( + environment, command=f"mkdir -p {shlex.quote(_REMOTE_DIR)}", env=treatment_env + ) + + # Ship the installed profile to the identical absolute path: the release + # snapshot's `bin/loopx` is a symlink into releases/, so a different path + # would leave a dangling CLI and no Goal body could be rendered at all. + with tempfile.TemporaryDirectory() as tmp: + tarball = Path(tmp) / "profile.tar.gz" + subprocess.run( + ["tar", "czf", str(tarball), "-C", parent, name], check=True + ) + receipt_path = Path(tmp) / "profile_receipt.json" + receipt_path.write_text( + json.dumps(profile_receipt, indent=2), encoding="utf-8" + ) + bootstrap_path = Path(tmp) / "loopx_product_bootstrap.py" + bootstrap_path.write_text(_BOOTSTRAP, encoding="utf-8") + for local, remote in ( + (tarball, f"{_REMOTE_DIR}/profile.tar.gz"), + (receipt_path, f"{_REMOTE_DIR}/profile_receipt.json"), + (bootstrap_path, f"{_REMOTE_DIR}/loopx_product_bootstrap.py"), + (self._goal_runner_source, f"{_REMOTE_DIR}/{self._goal_runner_source.name}"), + *( + (source, f"{_REMOTE_DIR}/{source.name}") + for source in _SUPPORT_SOURCES + ), + ): + await environment.upload_file(str(local), remote) + + await super().exec_as_agent( + environment, + command=( + f"mkdir -p {shlex.quote(parent)} && " + f"tar xzf {shlex.quote(_REMOTE_DIR)}/profile.tar.gz " + f"-C {shlex.quote(parent)} && " + f"test -x {shlex.quote(_PROFILE_ROOT)}/bin/loopx && " + f"chmod +x {shlex.quote(_REMOTE_DIR)}/codex_nosandbox_wrapper.py" + ), + env=treatment_env, + ) + + # Bootstrap renders the product-owned objective before GoalCodex reaches + # its normal upload phase. Seed the task file here so a fresh container + # does not fail before the later upload (which also sends objective.txt + # and the native module). + with tempfile.TemporaryDirectory() as tmp: + task_path = Path(tmp) / "task.txt" + task_path.write_text(self._goal_instruction or "", encoding="utf-8") + await environment.upload_file(str(task_path), f"{_REMOTE_DIR}/task.txt") + + # LoopX writes its own registry, goal state and Goal body. Nothing in + # this arm authors goal content: the hand-written four-stage document + # the previous arm used is exactly what made its results describe a + # prompt of mine rather than this product. + await super().exec_as_agent( + environment, + command=( + # The profile is installed on the host, so its launcher records + # the host's Python path -- a uv-managed interpreter that does + # not exist in the task image, which made every CLI call exit 2 + # with "configured Python executable not found". LOOPX_PYTHON + # redirects the launcher at the container's own interpreter + # without reinstalling, so the release snapshot stays the one + # whose cleanliness was proven on the host. + f"cd {shlex.quote(_PROJECT)} && " + "export LOOPX_PYTHON=\"$(command -v python3)\" && " + "python3 -c 'import sys; assert sys.version_info >= (3, 11), sys.version' && " + f"python3 {shlex.quote(_REMOTE_DIR)}/loopx_product_bootstrap.py " + f"--profile-root {shlex.quote(_PROFILE_ROOT)} " + f"--project {shlex.quote(_PROJECT)} " + f"--goal-id {_GOAL_ID} --agent-id {_AGENT_ID} " + f"--runtime-profile {runtime_profile} " + f"--goal-doc-file {shlex.quote(_GOAL_DOC_FILE)} " + f"--task-file {shlex.quote(f'{_REMOTE_DIR}/task.txt')} " + f"--objective-out {shlex.quote(_OBJECTIVE_FILE)} " + f"--receipt-out {shlex.quote(_REMOTE_DIR)}/loopx_product.json" + ), + env=treatment_env, + ) + + os.environ["MR_GOAL_OBJECTIVE_FILE"] = _OBJECTIVE_FILE + os.environ["MR_GOAL_CODEX_HOME"] = f"{_PROFILE_ROOT}/codex-home" + os.environ["MR_GOAL_REQUIRED_SKILL_IDS"] = ",".join( + profile_receipt["required_skill_ids"] + ) + # Arm GoalCodex's LOOPX_PYTHON export for the actual app-server process, + # not just the earlier bootstrap script -- Codex's own mid-turn `loopx` + # calls run inside app-server's environment and hit the same failure + # bootstrap did until this is set here too. + os.environ["MR_GOAL_ARM_LOOPX_PYTHON"] = "1" + if os.environ.get("MR_LOOPX_PREFLIGHT", "") not in ("", "0"): + os.environ["MR_GOAL_PREFLIGHT"] = "1" + + # Give the profile's CODEX_HOME the credentials and provider settings + # Pier wrote into its own. Pointing app-server at the formally + # installed CODEX_HOME is what makes `skills/list` find LoopX, but that + # directory ships only `skills/` -- no auth.json, no config.toml, so no + # API key and no gateway base_url. A run configured that way discovers + # every skill, starts, and then waits for a model it has no address + # for: one smoke hung for 85 minutes and the gateway's call count never + # moved. The preflight cannot catch it, because Goal attachment stops + # before the first model turn and needs no credentials. + # + # Only top-level files are copied. `cp -a` of the whole directory + # would merge Pier's own skills over the formal install, and which + # skills app-server discovers is the one thing this arm must not + # improvise. + profile_home = f"{_PROFILE_ROOT}/codex-home" + await super().exec_as_agent( + environment, + command=( + 'for f in "$CODEX_HOME"/*; do ' + f'[ -f "$f" ] && cp -f "$f" {shlex.quote(profile_home)}/; ' + "done; " + # GoalCodex disables web_search by appending to *its* CODEX_HOME + # after this runs, so the copy above would leave the profile + # without it and give this arm a hosted search tool the other + # two do not have -- an advantage no network isolation would + # reveal, since the provider runs the search. + f'grep -q "^web_search" {shlex.quote(profile_home)}/config.toml ' + f'|| printf "\\nweb_search = \\"disabled\\"\\n" ' + f'>> {shlex.quote(profile_home)}/config.toml; ' + f'test -s {shlex.quote(profile_home)}/config.toml ' + '|| { echo "no config.toml reached the LoopX profile" >&2; exit 1; }' + ), + env=env, + ) + + # Capture LoopX's own trace before the container is torn down. Pier's + # own teardown copies `$CODEX_HOME/sessions` into the job directory, + # but that is Pier's CODEX_HOME -- this arm points app-server at the + # profile's instead, so codex writes its session transcript to + # `{profile_home}/sessions` and Pier's copy finds nothing. The first + # smoke run's job directory logged "No Codex session directory found" + # for exactly this reason: the transcript existed, just one directory + # over from where anyone looked for it. + # + # This has to be a second, separate call rather than the launch command + # rewritten to append a capture step. `command` at this point still + # reads "codex exec ..." -- GoalCodex.exec_as_agent below only checks + # for that marker's presence and then throws the string away, building + # its own `run_goal.py` invocation from internal state. Appending shell + # onto a string GoalCodex never looks at silently does nothing, which is + # exactly the bug that made the CODEX_HOME swap above necessary: this + # arm keeps stumbling on places where a value must go through GoalCodex + # rather than through the string it happens to be holding. + capture_dir = "/logs/agent/loopx_trace" + try: + return await super().exec_as_agent( + environment, command=command, env=env, **kwargs + ) + finally: + await super().exec_as_agent( + environment, + command=( + f"mkdir -p {shlex.quote(capture_dir)}; " + f'if [ -d {shlex.quote(profile_home)}/sessions ]; then ' + f'cp -R {shlex.quote(profile_home)}/sessions ' + f'{shlex.quote(capture_dir)}/sessions; fi; ' + f'find /app/.codex/goals -name ACTIVE_GOAL_STATE.md ' + f'-exec cp {{}} {shlex.quote(capture_dir)}/ACTIVE_GOAL_STATE.md \\; ' + f'2>/dev/null; ' + # Same LOOPX_PYTHON fix as the app-server launch above: this + # CLI call is yet another separate shell, and without its own + # export it fails with the same "configured Python + # executable not found" that showed up twice in mid-turn + # calls before the launch-side fix existed. + 'export LOOPX_PYTHON="$(command -v python3)"; ' + f'{shlex.quote(_PROFILE_ROOT)}/bin/loopx ' + f'--registry {shlex.quote(_PROJECT)}/.loopx/registry.json ' + f'--runtime-root {shlex.quote(_PROJECT)}/.loopx/runtime ' + f'--format json todo list --goal-id {_GOAL_ID} ' + f'> {shlex.quote(capture_dir)}/todo_list.json 2>&1; ' + "true" + ), + env=env, + ) + + +class LoopxCodexCliCodex(LoopxNativeCodex): + """LoopX governed Turns executed by the real Codex CLI host.""" + + loopx_mode = "codex-cli" + + +class LoopxHeartbeatCodex(LoopxNativeCodex): + """LoopX generic CLI heartbeat executed by a recurring supervisor.""" + + loopx_mode = "heartbeat" + + +# Runs inside the container: bootstrap, register the peer, render the Goal body. +_BOOTSTRAP = '''\ +import argparse, json, os, subprocess, sys +from pathlib import Path + +p = argparse.ArgumentParser() +p.add_argument("--profile-root", required=True) +p.add_argument("--project", required=True) +p.add_argument("--goal-id", required=True) +p.add_argument("--agent-id", required=True) +p.add_argument("--goal-doc-file", required=True) +p.add_argument("--task-file", required=True) +p.add_argument("--runtime-profile", required=True) +p.add_argument("--objective-out", required=True) +p.add_argument("--receipt-out", required=True) +a = p.parse_args() + +cli = f"{a.profile_root}/bin/loopx" +registry = f"{a.project}/.loopx/registry.json" +runtime = f"{a.project}/.loopx/runtime" +base = [cli, "--registry", registry, "--runtime-root", runtime, "--format", "json"] +steps = {} + + +def run(name, args): + out = subprocess.run(base + args, capture_output=True, text=True) + try: + payload = json.loads(out.stdout) + except Exception: + payload = {} + # Keep returncode and both streams for every step. Pier reports a failed + # agent command as "Command failed" and discards its output, so a step that + # dies here is otherwise invisible: the first run of this arm failed exactly + # once and left nothing but that phrase in the log. + steps[name] = { + "returncode": out.returncode, + "ok": payload.get("ok"), + "error": payload.get("error"), + "stdout_tail": out.stdout[-400:] if not payload else None, + "stderr_tail": out.stderr[-400:], + } + if out.returncode or payload.get("ok") is not True: + detail = payload.get("error") or out.stderr[-240:] or out.stdout[-240:] + raise RuntimeError(f"{name}_failed:{detail}") + return payload + + +goal_doc = Path(a.goal_doc_file) +goal_doc.write_text( + Path(a.task_file).read_text(encoding="utf-8"), + encoding="utf-8", +) +task_text = goal_doc.read_text(encoding="utf-8") +bootstrap_args = [ + "bootstrap", "--project", a.project, "--goal-id", a.goal_id, + "--objective", "Complete the software engineering task described in the task file and commit the finished work.", + "--goal-doc", str(goal_doc), + "--adapter-kind", "read_only_project_map_v0", + "--adapter-status", "connected-read-only", +] +if os.environ.get("LOOPX_WEN_COMPAT", "1") not in ("", "0"): + bootstrap_args += [ + # Benchmark admission supplies one frozen task. Project-map onboarding + # todos otherwise outrank it and measure repository housekeeping rather + # than the requested software-engineering task. + "--no-onboarding-scan", + "--begin-autonomous-advance", + "--codex-app-heartbeat", + "yes" if a.runtime_profile == "codex_app_ssh_goal" else "no", + "--write-scope", a.project, + ] +else: + bootstrap_args += ["--no-onboarding-scan", "--codex-app-heartbeat", "ask"] +run("bootstrap", bootstrap_args) +run("configure_goal", ["configure-goal", "--goal-id", a.goal_id, + "--registered-agent", a.agent_id, "--execute"]) +# The gate selects work from the todo frontier. Keeping the task only in the +# turn input lets the model spend the whole budget on onboarding todos while +# the target repository remains untouched; wen's native driver explicitly +# inserts this P0 advancement todo for the same reason. +run("add_task_todo", ["todo", "add", "--goal-id", a.goal_id, + "--role", "agent", "--todo-id", "deepswe-task", + "--text", "[P0] " + task_text, "--task-class", "advancement_task", + "--action-kind", "implement", + "--required-capability", "filesystem_write", + "--status", "open", "--execute"]) +if os.environ.get("LOOPX_WEN_COMPAT", "1") not in ("", "0"): + run("clear_waiting_on", ["configure-goal", "--goal-id", a.goal_id, + "--clear-waiting-on", + "--agent-work-mode", f"{a.agent_id}=active", + "--write-scope", a.project, "--execute"]) +prompt = run("heartbeat_prompt", + ["heartbeat-prompt", "--thin", "--goal-id", a.goal_id, + "--agent-id", a.agent_id, "--available-capability", "shell", + "--available-capability", "filesystem_write", + "--runtime-profile", a.runtime_profile, "--cli-bin", cli]) + +body = prompt.get("task_body") +if body: + Path(a.objective_out).write_text(body, encoding="utf-8") + steps["goal_body_chars"] = len(body) +else: + steps["fatal"] = "heartbeat-prompt returned no task_body" +Path(a.receipt_out).write_text(json.dumps(steps, indent=2), encoding="utf-8") +print(json.dumps(steps, indent=2)) +raise SystemExit(0 if body else 1) +''' diff --git a/benchmark/deepswe-gptxhigh-v1/loopx_turn_runner.py b/benchmark/deepswe-gptxhigh-v1/loopx_turn_runner.py new file mode 100644 index 0000000000..d58f2d17ba --- /dev/null +++ b/benchmark/deepswe-gptxhigh-v1/loopx_turn_runner.py @@ -0,0 +1,623 @@ +#!/usr/bin/env python3 +"""Drive one DeepSWE task through LoopX's governed Turn loop, inside the container. + +Run by LoopxCodex after the LoopX package, the Codex profile and the goal state +have been staged. Kept as a standalone script for the same reason +native_codex_goal.py is: it must execute where the repository is, and the +repository is inside the task container. + +Why this calls ``handle_turn_command`` instead of the ``loopx turn run-once`` +CLI: the CLI restricts ``--codex-sandbox`` to read-only and workspace-write, +and neither can be set up inside these task images — the Linux sandbox needs +kernel features the container does not grant, and Codex responds by narrating +instead of executing (measured once: 431 assistant messages, zero command +executions, an empty patch scored 0/24). The other two arms run Codex with +approvals and sandbox bypassed, so the LoopX arm has to as well or the three +differ in permissions as well as in looping. Editing LoopX's argparse choices +would also have made the source tree dirty, and install_native_codex_profile +refuses an unclean source because mixing revisions invalidates a benchmark +treatment. Building the Namespace directly avoids both problems and touches +no file in the LoopX checkout. + +The loop is the point of the arm. LoopX runs exactly one governed Turn per +call: it selects a Todo, has the host adapter invoke Codex, requires an +independent validator to prove the postcondition, and only then commits the +result and spends quota. Multi-turn behaviour comes from calling it again -- +which is precisely what the other two arms never do, since `codex exec` and the +app-server both stop as soon as the model says it is finished. +""" + +from __future__ import annotations + +import argparse +import io +import json +import os +import subprocess +import sys +from contextlib import redirect_stdout +from pathlib import Path + +REMOTE_DIR = Path(__file__).resolve().parent +DEFAULT_QUOTA = int(os.environ.get("MR_LOOPX_QUOTA", "4")) +DEFAULT_TURN_TIMEOUT = float(os.environ.get("MR_LOOPX_TURN_TIMEOUT", "1200")) + +GOAL_ID = "deepswe-task" +AGENT_ID = "deepswe-codex" +TODO_ID = "deepswe-todo-1" + + +def hide_loopx_state_from_git(project: Path) -> None: + """Keep LoopX's own files out of git's view. + + The goal document and registry have to live at paths inside the project — + LoopX resolves ``state_file`` relative to the repo — but they are control- + plane state, not the agent's work. Left visible they break the run twice: + ``git status`` never comes back clean, so the validator rejects every Turn, + and they land in ``git diff base..HEAD``, which is exactly the patch the + benchmark grades. + + Written to ``.git/info/exclude`` rather than ``.gitignore`` because that + file is local to the clone and never becomes part of the diff itself. + """ + exclude = project / ".git" / "info" / "exclude" + if not exclude.parent.is_dir(): + return + existing = exclude.read_text(encoding="utf-8") if exclude.exists() else "" + additions = [p for p in (".codex/", ".loopx/") if p not in existing] + if additions: + with exclude.open("a", encoding="utf-8") as fh: + fh.write("\n# LoopX control-plane state (benchmark harness)\n") + fh.write("\n".join(additions) + "\n") + + +def stage_goal_state(project: Path, instruction: str) -> Path: + """Write the goal document LoopX reads Todos from. + + The Todo, not the objective, is what LoopX plans against: it selects one per + Turn and asks the host to advance it. Phrasing it as staged work is what + gives the loop somewhere to go on turn two — a Todo that one turn satisfies + ends the goal exactly the way the Goal-API arm already ends, and the arm + would measure nothing. + """ + state = project / ".codex" / "goals" / GOAL_ID / "ACTIVE_GOAL_STATE.md" + state.parent.mkdir(parents=True, exist_ok=True) + state.write_text( + "\n".join( + [ + "---", + "status: active", + "updated_at: 2026-01-01T00:00:00+00:00", + "---", + "", + "# DeepSWE task", + "", + "## Agent Todo", + "", + "- [ ] [P0] Advance the task below by exactly one stage per Turn, " + "and report which stage you completed and what remains. " + "Stage 1: implement the target behaviour and commit. " + "Stage 2: re-check the implementation against every requirement " + "in the task text, fix gaps, commit. " + "Stage 3: run the wider test suite and repair any regression. " + "Stage 4: handle edge cases the tests do not cover.", + f" ", + "", + "## How to report each Turn", + "", + # gpt-5.5 reached for path_delta_mode=material_replan on a plain + # first implementation Turn, which LoopX rejects four ways at + # once: that mode is reserved for Turns that overturn a prior + # assumption, and it then demands result_kind=replan_required + # plus a goal_path_delta_v0 vision packet the model had not + # produced. Every Turn failed validation, so nothing committed + # and the quota bought nothing. Advancing a stage is routine + # continuation, so say so explicitly rather than leaving the + # model to pick. + "This Todo is routine staged continuation, never a replan. In the", + "typed result set `path_delta_mode=unchanged`, leave", + "`agent_vision_json` empty, and give a one-line", + "`vision_unchanged_reason` such as \"routine stage advance\".", + "Set `delivery_batch_scale` to `implementation` when you changed", + "source, or `test_only` when you only touched tests.", + "Use `result_kind=validated_progress` when the stage advanced and", + "`repair_required` when it did not.", + "", + "## Task", + "", + instruction, + "", + ] + ), + encoding="utf-8", + ) + return state + + +def stage_registry(project: Path, runtime: Path, state: Path) -> Path: + """Write the registry LoopX plans against. + + Shape follows LoopX's own e2e fixture rather than a guess: a goal needs + ``state_file`` to find its Todos, ``quota`` for the scheduler to spend + against, and a ``coordination`` block with ``registered_agents`` — without + the last one every Turn fails at planning with "quota should-run + --agent-id requires coordination.registered_agents", before the host is + ever invoked. + + ``write_scope`` is the repository rather than the fixture's ``docs/**``: + the agent's whole job here is to change source. + """ + registry = project / ".loopx" / "registry.json" + registry.parent.mkdir(parents=True, exist_ok=True) + registry.write_text( + json.dumps( + { + "schema_version": 1, + "common_runtime_root": str(runtime), + "goals": [ + { + "id": GOAL_ID, + "domain": "deepswe-benchmark", + "status": "active", + "repo": str(project), + "state_file": str(state.relative_to(project)), + "adapter": { + "kind": "fixture_v0", + "status": "connected-delivery", + }, + "quota": {"compute": 1.0, "window_hours": 24}, + "coordination": { + "agent_model": "peer_v1", + "registered_agents": [AGENT_ID], + "agent_profiles": { + AGENT_ID: { + "schema_version": "agent_profile_v1", + "profile_role": "benchmark", + "scope": "deepswe task", + } + }, + "write_scope": ["**"], + }, + } + ], + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return registry + + +def validator_command(project: Path, base_sha: str) -> list[str]: + """Postcondition: the Turn committed something new and left a clean tree. + + Comparing against the base commit is the whole point. An earlier version + only asked for a clean tree and a non-empty history, which any run + satisfies without doing anything at all — four Turns passed validation + while producing an empty patch. A Turn that has not moved HEAD has not + advanced the Todo, whatever the model reports. + + Deliberately structural, never the hidden tests: the benchmark's rules put + verifier invocation after the run and outside the controller, so that a + loop cannot steer on the grade. What this proves is that the agent really + committed work rather than declaring success over an unchanged tree. + """ + program = ( + "import json,subprocess,sys;" + "json.load(sys.stdin);" + f"p={str(project)!r};b={base_sha!r};" + "st=subprocess.run(['git','-C',p,'status','--porcelain']," + "capture_output=True,text=True);" + "hd=subprocess.run(['git','-C',p,'rev-parse','HEAD']," + "capture_output=True,text=True);" + "head=hd.stdout.strip();" + "clean=st.returncode==0 and not st.stdout.strip();" + "raise SystemExit(0 if clean and head and head!=b else 9)" + ) + return [sys.executable, "-c", program] + + +def head_sha(project: Path) -> str: + out = subprocess.run( + ["git", "-C", str(project), "rev-parse", "HEAD"], + capture_output=True, text=True, + ) + return out.stdout.strip() + + +def run_turn(*, project: Path, registry: Path, runtime: Path, codex_bin: str, + model: str, sandbox: str, turn_index: int, base_sha: str, + ) -> dict: + from loopx.cli_commands.turn import handle_turn_command, register_turn_commands + + # Build the Namespace from LoopX's own parser rather than by hand. Hand- + # writing it meant discovering missing attributes one failed Turn at a time + # (`resume_turn_key` was the third), and every LoopX upgrade would restart + # that game. Parsing real argv fills every default the handler expects and + # fails loudly here if an option is ever renamed. + # + # Host is generic-cli, not codex-cli. LoopX's built-in Codex host launches + # `codex exec --sandbox `, and both modes it permits need bubblewrap, + # which needs unprivileged user namespaces that these containers do not + # grant — every Turn came back with "bwrap: No permissions to create a new + # namespace" and an empty patch. The generic-cli seam lets the adapter + # launch Codex with the same approvals-and-sandbox bypass the other two + # arms use, so the loop stays LoopX's and the permissions stay identical. + # + # Which adapter is the only thing that differs between the Codex arm and the + # Claude Code arm: both go through this same generic-cli seam, the same + # validator and the same quota, and only the CLI that executes a Turn + # changes. Defaulting to the Codex one keeps that arm byte-identical to the + # runs already recorded against it. + wrapper = str(Path(__file__).resolve().parent / "codex_nosandbox_wrapper.py") + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command") + register_turn_commands(sub, lambda p: p.add_argument("--format", default="json")) + args = parser.parse_args([ + "turn", "run-once", + "--goal-id", GOAL_ID, + "--agent-id", AGENT_ID, + "--turn-instance-id", f"{GOAL_ID}-turn-{turn_index}", + # codex-cli, not generic-cli. The generic host was an attempt to get + # around the sandbox flag, and it cost more than it saved: it carries + # its own scheduler contract, and eleven of sixteen turns died at + # "LoopX Turn route is not host executable" before any model work, with + # no route recorded to say why. The codex host is the one that + # demonstrably works — four turns, a 22 KB patch, f2p 31/35 — so keep it + # and neutralise the sandbox at the binary instead, via a wrapper that + # strips --sandbox and substitutes the approvals-and-sandbox bypass. + "--host", "codex-cli", + "--execution-mode", "isolated-headless", + "--project", str(project), + "--codex-bin", wrapper, + # An accepted value that the wrapper then removes; LoopX only permits + # read-only and workspace-write, and this argument has to satisfy its + # parser rather than the container. + "--codex-sandbox", "workspace-write", + "--codex-model", model, + "--validation-command-json", json.dumps(validator_command(project, base_sha)), + "--validation-timeout-seconds", "60", + # Without --execute LoopX plans the Turn and stops: every receipt comes + # back ok=True with dry_run=True, the host is never invoked, and four + # turns of nothing look exactly like four turns of success. + "--execute", + # Deliberately no --scan-root. It is not "where the work is"; LoopX + # documents it as "public files to scan for obvious private material", + # and pointing it at the task repository made LoopX run its public + # boundary scanner over the repository's own history. On the OPA task + # that produced + # public_boundary_violation CHANGELOG.md:3588: private_ip + # public_boundary_violation CHANGELOG.md:4859: credential + # which sets contract health to not-ok, which makes the quota decision + # `should_run=false / quota_skip`, which makes the route `wait`, which + # is what "LoopX Turn route is not host executable" finally reports — + # four layers away from the file it actually objected to. + # + # It also explains the shape of the failure: any repository whose text + # happens to contain something resembling an IP or a credential is + # rejected before a model runs, which is most of them, while the odd + # clean repository sails through and looks like proof the setup works. + # `turn plan` kept answering ready_for_host because the plan probe never + # passed --scan-root and so scanned LoopX's own directory instead. + # + # The default is LoopX's own public root, which is what the scanner is + # for. The task repository reaches the Turn through --project. + "--no-global-sync", + "--timeout-seconds", str(DEFAULT_TURN_TIMEOUT), + "--format", "json", + ]) + + # PrintPayload is (payload, fmt, renderer) -> None and FormatSelector is + # (...) -> str; passing single-argument lambdas made every Turn die with a + # TypeError before any model work, which the loop then dutifully repeated + # four times. Capture the payload instead of printing it, so the receipt + # survives whatever the CLI would have rendered. + captured: list[dict] = [] + + def _print_payload(payload, fmt=None, renderer=None): # noqa: ANN001 + if isinstance(payload, dict): + captured.append(payload) + + # `run-once` decided `route: wait`, which `_typed_route` returns only when + # the envelope says should_run is false. The decision behind it blamed + # "status or contract health is not ok" while reporting the quota itself as + # eligible with zero slots spent — so the gate is `goal_status_health_ok`, + # which reads `contract` and `global_registry` straight off the status + # payload. `turn plan`, run first in this same process, said + # ready_for_host, so the two calls are seeing different status. Spy on + # collect_status for both and record what differs; the two subcommands do + # not take the same options (`--scan-root` differs, `--no-global-sync` is + # run-once only), and that is the remaining candidate. + from loopx.cli_commands import turn as _turn_mod + _statuses: list[dict] = [] + _original_collect = _turn_mod.collect_status + + def _spy_collect(*a, **kw): # noqa: ANN002, ANN003 + result = _original_collect(*a, **kw) + if isinstance(result, dict): + contract = (result.get("contract") + if isinstance(result.get("contract"), dict) else {}) + registry = (result.get("global_registry") + if isinstance(result.get("global_registry"), dict) else {}) + _statuses.append({ + "scan_roots": [str(x) for x in (kw.get("scan_roots") or [])], + "status_ok": result.get("ok"), + "contract_ok": contract.get("ok"), + "has_error_diagnostics": "error_diagnostics" in contract, + "contract_errors": json.dumps( + contract.get("error_diagnostics"), ensure_ascii=False + )[:600], + "global_registry_ok": registry.get("ok"), + "global_registry_error": json.dumps( + {k: v for k, v in registry.items() if k != "goals"}, + ensure_ascii=False, + )[:400], + }) + return result + + _turn_mod.collect_status = _spy_collect + + # Plan first, and keep the result whatever happens next. When the planner + # declines to call the host, `run-once` raises "LoopX Turn route is not + # host executable" and the receipt it leaves carries only ok and effects — + # the route, the selected Todo and the scheduler context all vanish. Eleven + # turns failed exactly that way, and reproducing the call on the host proved + # only that the *arguments* were fine, so every explanation stayed a guess. + # `turn plan` runs the same decision without invoking a host or spending + # quota, so recording it costs nothing and makes the next failure legible. + plan_payload: dict = {} + try: + # Parse `turn plan` argv rather than copying the run-once Namespace and + # renaming the subcommand. The two subparsers do not define the same + # options, so the copy was missing `include_transaction_detail` and the + # probe failed on its own AttributeError — producing five nulls that + # said nothing about the route it was meant to explain. + plan_parser = argparse.ArgumentParser() + plan_sub = plan_parser.add_subparsers(dest="command") + register_turn_commands( + plan_sub, lambda p: p.add_argument("--format", default="json") + ) + plan_args = plan_parser.parse_args([ + "turn", "plan", + "--goal-id", GOAL_ID, + "--agent-id", AGENT_ID, + "--host", "codex-cli", + "--execution-mode", "isolated-headless", + "--format", "json", + ]) + with redirect_stdout(io.StringIO()): + handle_turn_command( + plan_args, + registry_path=registry, + runtime_root_arg=str(runtime), + output_format=lambda *_a, **_k: "json", + print_payload=_print_payload, + ) + if captured: + p = captured.pop() + route = p.get("route") or {} + ctx = p.get("scheduler_execution_context") or {} + plan_payload = { + "route_kind": route.get("kind"), + "would_invoke_host": route.get("would_invoke_host"), + "selected_todo": (route.get("selected_todo") or {}).get("todo_id"), + "context_valid": ctx.get("valid"), + "context_errors": ctx.get("errors"), + } + # Keep the raw shape when the expected keys are absent. Reusing the + # run-once Namespace for `plan` produced a payload without `route` + # at all, and five nulls said only "not what I expected" — which is + # the same dead end as having no diagnostics. + if route or ctx: + pass + else: + plan_payload["raw_keys"] = sorted(p) + plan_payload["raw"] = json.dumps(p, ensure_ascii=False)[:1200] + except Exception as exc: + plan_payload = {"plan_error": f"{type(exc).__name__}: {exc}"} + + # Record the plan `run-once` builds and then rejects. "LoopX Turn route is + # not host executable" is raised by build_loopx_turn_host_request after + # reading route.would_invoke_host off a payload run-once assembled moments + # earlier and never prints, so the one run whose route matters is the one + # nobody can see — and `turn plan`, which does print it, keeps answering + # ready_for_host. Both go through build_loopx_turn_plan, so wrapping it + # captures run-once's own payload, including the `session` block that only + # run-once populates. Reproducing this on the host was not possible: there + # both subcommands succeed, so the difference lives in the container. + from loopx.cli_commands import turn as _turn_mod + _built: list[dict] = [] + _original_build = _turn_mod.build_loopx_turn_plan + + def _spy_build(*a, **kw): # noqa: ANN002, ANN003 + result = _original_build(*a, **kw) + _built.append({"session_binding": kw.get("session_binding"), + "payload": result}) + return result + + _turn_mod.build_loopx_turn_plan = _spy_build + + # The route came back `wait`, which `_typed_route` only returns when the + # envelope says should_run is false and a quiet no-op is allowed — a + # scheduling verdict, not a contract error. `turn plan`, run moments + # earlier in this same process against this same registry, said + # ready_for_host. So capture the decision the envelope is projected from: + # should_run is set by build_live_quota_should_run_decision, and its + # rationale is the only thing that can say why the two disagree. + _decisions: list[dict] = [] + _original_decision = _turn_mod.build_live_quota_should_run_decision + + def _spy_decision(*a, **kw): # noqa: ANN002, ANN003 + result = _original_decision(*a, **kw) + if isinstance(result, dict): + _decisions.append(result) + return result + + _turn_mod.build_live_quota_should_run_decision = _spy_decision + + buffer = io.StringIO() + try: + with redirect_stdout(buffer): + code = handle_turn_command( + args, + registry_path=registry, + runtime_root_arg=str(runtime), + output_format=lambda *_a, **_k: "json", + print_payload=_print_payload, + ) + finally: + _turn_mod.build_loopx_turn_plan = _original_build + _turn_mod.build_live_quota_should_run_decision = _original_decision + _turn_mod.collect_status = _original_collect + built_payload: dict = {} + if _built: + record = _built[-1] + built = record["payload"] + route = built.get("route") if isinstance(built.get("route"), dict) else {} + session = built.get("session") if isinstance(built.get("session"), dict) else {} + ctx = built.get("scheduler_execution_context") + ctx = ctx if isinstance(ctx, dict) else {} + transaction = (built.get("transaction") + if isinstance(built.get("transaction"), dict) else {}) + built_payload = { + "route_kind": route.get("kind"), + "would_invoke_host": route.get("would_invoke_host"), + "route_reasons": route.get("reasons") or route.get("errors"), + "session_action": session.get("action"), + "session_binding_status": session.get("binding_status"), + "session_binding_arg": record["session_binding"], + "context_valid": ctx.get("valid"), + "context_errors": ctx.get("errors"), + "turn_key": transaction.get("turn_key"), + "builds": len(_built), + } + envelope = (built.get("turn_envelope") + if isinstance(built.get("turn_envelope"), dict) else {}) + action = (envelope.get("action") + if isinstance(envelope.get("action"), dict) else {}) + built_payload["envelope"] = { + "should_run": envelope.get("should_run"), + "effective_action": envelope.get("effective_action"), + "delivery_allowed": action.get("delivery_allowed"), + "must_attempt": action.get("must_attempt"), + "quiet_noop_allowed": action.get("quiet_noop_allowed"), + } + if _decisions: + decision = _decisions[-1] + built_payload["decision"] = { + key: decision.get(key) + for key in ("should_run", "effective_action", "reason", "reasons", + "blocked_reason", "quota", "quota_state", "cadence", + "schedule", "gates") + if key in decision + } + built_payload["decision_keys"] = sorted(decision)[:40] + built_payload["decisions"] = len(_decisions) + # Two entries: the `plan` probe's status, then run-once's. Whatever differs + # between them is what flipped goal_status_health_ok. + built_payload["statuses"] = _statuses[:4] + if captured: + payload = captured[-1] + else: + raw = buffer.getvalue().strip() + try: + payload = json.loads(raw.splitlines()[-1]) if raw else {} + except Exception: + payload = {"unparsed": raw[-2000:]} + payload["_exit_code"] = code + payload["_plan"] = plan_payload + payload["_built"] = built_payload + # Carry the wrapper's log into the receipt. LoopX reports a failed host as + # `codex_cli_exit_nonzero` and keeps nothing else, and the container that + # holds the log is deleted as soon as the task ends, so the receipt is the + # only artifact that outlives the evidence. + codex_log = Path( + os.environ.get("MR_LOOPX_CODEX_LOG", "/tmp/loopx-goal/codex-wrapper.log") + ) + try: + payload["_codex_log"] = codex_log.read_text(encoding="utf-8")[-4000:] + codex_log.unlink() + except OSError: + payload["_codex_log"] = None + return payload + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--project", required=True) + parser.add_argument("--task-file", required=True) + parser.add_argument("--runtime-root", required=True) + parser.add_argument("--codex-bin", default="codex") + parser.add_argument("--adapter", default="loopx_codex_adapter.py", + help="generic-cli host adapter that executes one Turn " + "(loopx_codex_adapter.py | loopx_claude_adapter.py)") + parser.add_argument("--model", required=True) + parser.add_argument("--sandbox", default="danger-full-access") + parser.add_argument("--quota", type=int, default=DEFAULT_QUOTA) + args = parser.parse_args() + + project = Path(args.project).resolve() + runtime = Path(args.runtime_root) + runtime.mkdir(parents=True, exist_ok=True) + instruction = Path(args.task_file).read_text(encoding="utf-8").strip() + + # The host adapter needs the task text too, and cannot get it from the Turn + # envelope: the envelope carries only the selected Todo, and LoopX compacts + # that to an 8 KB budget, so the staged Todo's wording survives truncated + # ("Advance the task below by exactly one stage per Turn, and report which + # s...") while the `## Task` section of the goal document is never included + # at all. An adapter working from the envelope alone therefore sees a + # staging meta-instruction with no target behaviour attached, and a + # well-behaved model correctly refuses to invent one — four Turns of + # `validation_failed` with an empty patch, which reads like a model that + # could not do the work rather than a prompt that never described it. + # Handing the adapter the same file the goal document was built from keeps + # one source of truth for the task text. + os.environ["MR_LOOPX_TASK_FILE"] = str(Path(args.task_file).resolve()) + + hide_loopx_state_from_git(project) + state = stage_goal_state(project, instruction) + registry = stage_registry(project, runtime, state) + base_sha = head_sha(project) + + receipts = [] + for turn_index in range(1, args.quota + 1): + try: + payload = run_turn( + project=project, registry=registry, runtime=runtime, + codex_bin=args.codex_bin, model=args.model, + sandbox=args.sandbox, turn_index=turn_index, base_sha=base_sha, + + ) + except Exception as exc: # keep the receipt; a dead turn is evidence too + payload = {"error": f"{type(exc).__name__}: {exc}"} + receipts.append(payload) + # A Turn that never reached the host is a defect in this runner, not a + # result: repeating it burns the whole quota on the same traceback, as + # four identical TypeErrors once did. Stop and let the receipt show why. + if "error" in payload: + break + if payload.get("dry_run"): + payload["_fatal"] = "dry_run: --execute was not honoured" + break + status = str(payload.get("status") or payload.get("result_kind") or "") + # Stop early only when LoopX says the work is settled; a failed or + # repair-required Turn is exactly the case the next Turn exists for. + if status in {"completed", "goal_complete", "done"}: + break + + print(json.dumps({ + "schema": "deepswe_loopx_turn_log_v0", + "quota": args.quota, + "turns_run": len(receipts), + "receipts": receipts, + }, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/deepswe-gptxhigh-v1/loopx_wen_native_runner.py b/benchmark/deepswe-gptxhigh-v1/loopx_wen_native_runner.py new file mode 100644 index 0000000000..9bc6b3157d --- /dev/null +++ b/benchmark/deepswe-gptxhigh-v1/loopx_wen_native_runner.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python3 +"""Run the ssh-goal treatment on the native Codex app-server Goal surface. + +The LoopX toolkit owns the JSON-RPC transaction and event reducer. This +runner only adds the benchmark-host action that wen performs when a Goal is +blocked: clear the waiting gate and start the next turn on the existing thread. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import time +from pathlib import Path +from typing import Any, Mapping + +from native_codex_goal import ( + NativeGoalConfig, + NativeGoalProtocolError, + StdioNativeGoalTransport, + compact_native_goal_receipt, + probe_native_goal_process, + observe_native_goal_event, + refresh_native_goal_status, + start_native_goal_turn, +) +from workspace_delivery import head_sha, normalize_delivery, write_receipt + + +def _restart_turn_same_thread(transport, config, turn, instruction=None): + result = transport.request( + "turn/start", + { + "threadId": turn.thread_id, + "input": [{"type": "text", "text": instruction or config.task_instruction}], + "cwd": config.cwd, + "approvalPolicy": config.approval_policy, + **({"model": config.model} if config.model else {}), + **({"effort": config.effort} if config.effort else {}), + **( + {"sandboxPolicy": dict(config.sandbox_policy)} + if config.sandbox_policy is not None + else {} + ), + }, + ) + turn.methods.append("turn/start") + nested = result.get("turn") + nested = nested if isinstance(nested, dict) else {} + turn_id = str(nested.get("id") or result.get("turnId") or "") + if not turn_id: + raise NativeGoalProtocolError("turn_start_id_missing") + turn.turn_id = turn_id + turn.response_turn_id = turn_id + turn.turn_status = str(nested.get("status") or "accepted") + return turn + + +def _reactivate_goal(transport, config, turn) -> None: + payload = { + "threadId": turn.thread_id, + "objective": config.objective, + "status": "active", + } + if config.token_budget is not None: + payload["tokenBudget"] = config.token_budget + transport.request("thread/goal/set", payload) + turn.methods.append("thread/goal/set") + turn.post_goal_status = "active" + + +def _clear_blocked( + *, + loopx_cli: str, + registry: str, + runtime_root: str, + goal_id: str, + agent_id: str, + cwd: str, +) -> None: + command = [ + loopx_cli, + "--registry", + registry, + "--runtime-root", + runtime_root, + "--format", + "json", + "configure-goal", + "--goal-id", + goal_id, + "--clear-waiting-on", + "--agent-work-mode", + f"{agent_id}=active", + "--write-scope", + cwd, + "--execute", + ] + result = subprocess.run(command, capture_output=True, text=True, timeout=180) + if result.returncode: + detail = (result.stderr or result.stdout)[-240:] + raise NativeGoalProtocolError(f"blocked_recovery_failed:{detail}") + + +def _terminal_error(event: Mapping[str, Any]) -> str | None: + method = str(event.get("method") or "") + params = event.get("params") if isinstance(event.get("params"), Mapping) else {} + event_type = str(event.get("type") or "") + payload = event.get("payload") if isinstance(event.get("payload"), Mapping) else {} + payload_type = str(payload.get("type") or "") + terminal = method == "turn/completed" or ( + event_type == "event_msg" and payload_type in {"task_complete", "task_completed", "turn_completed"} + ) + if not terminal: + return None + turn = params.get("turn") if isinstance(params.get("turn"), Mapping) else {} + for container in (payload, turn, params): + error = container.get("error") if isinstance(container, Mapping) else None + if error is None: + continue + if isinstance(error, Mapping): + return str(error.get("message") or error.get("type") or error) + return str(error) + return None + + +def _wait_turn( + transport, + turn, + *, + deadline: float, + completed_before: int, + idle_timeout_seconds: float, + interrupt_grace_seconds: float, +) -> str | None: + error: str | None = None + last_event_at = time.monotonic() + interrupt_deadline: float | None = None + while turn.turn_completed_count <= completed_before: + now = time.monotonic() + remaining = deadline - now + if remaining <= 0: + raise NativeGoalProtocolError("goal_timeout_before_terminal") + event = transport.next_event(timeout_sec=min(0.25, remaining)) + if event is not None: + last_event_at = time.monotonic() + error = _terminal_error(event) or error + observe_native_goal_event(turn, event) + continue + now = time.monotonic() + if interrupt_deadline is not None: + if now >= interrupt_deadline: + raise NativeGoalProtocolError("turn_interrupt_timeout") + continue + if now - last_event_at >= idle_timeout_seconds: + transport.request( + "turn/interrupt", + {"threadId": turn.thread_id, "turnId": turn.turn_id}, + ) + error = f"turn idle timeout after {idle_timeout_seconds:g} seconds" + interrupt_deadline = min(deadline, now + interrupt_grace_seconds) + return error + + +def _is_transient_model_error(message: str) -> bool: + lowered = message.lower() + return any( + marker in lowered + for marker in ( + "rate limit", + "overloaded", + "server error", + "timed out", + "timeout", + "deployment_disabled", + "insufficient quota available", + ) + ) + + +def _benchmark_todo_terminal(args: argparse.Namespace, project: Path) -> bool: + result = subprocess.run( + [ + args.loopx_cli, + "--registry", + args.registry, + "--runtime-root", + args.runtime_root, + "--format", + "json", + "todo", + "list", + "--goal-id", + args.goal_id, + ], + cwd=project, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + if result.returncode: + return False + try: + payload = json.loads(result.stdout) + except ValueError: + return False + todos = payload.get("todos") if isinstance(payload, dict) else None + if not isinstance(todos, list): + return False + statuses = [ + item.get("status") + for item in todos + if isinstance(item, dict) and item.get("priority") == "P0" + ] + return bool(statuses) and all(status in {"done", "deferred"} for status in statuses) + + +def run_goal(args: argparse.Namespace): + project = Path(args.cwd).resolve() + base_sha = head_sha(project) + delivery_path = Path("/logs/agent/delivery_receipt.json") + config = NativeGoalConfig( + cwd=args.cwd, + objective=Path(args.objective_file).read_text(encoding="utf-8").strip(), + task_instruction=Path(args.task_file).read_text(encoding="utf-8").strip(), + model=args.model, + effort=args.effort, + token_budget=args.token_budget, + sandbox=args.sandbox, + required_skill_ids=tuple( + item for item in (part.strip() for part in args.required_skill_ids.split(",")) if item + ), + ) + process_command = [ + args.codex_bin, + "app-server", + "--listen", + "stdio://", + "--enable", + "goals", + "--enable", + "unified_exec", + ] + process_env = os.environ.copy() + if args.preflight_only: + return probe_native_goal_process( + config, + codex_bin=args.codex_bin, + process_command=process_command, + process_env=process_env, + process_cwd=args.cwd, + response_timeout_sec=args.response_timeout_seconds, + ), 0 + + with StdioNativeGoalTransport.spawn( + process_command, + cwd=args.cwd, + env=process_env, + response_timeout_sec=args.response_timeout_seconds, + ) as transport: + turn = start_native_goal_turn(transport, config) + deadline = time.monotonic() + args.goal_timeout_seconds + completed_before = turn.turn_completed_count + unblocks = 0 + delivery_retries = 0 + transient_retries = 0 + delivery = None + benchmark_todo_terminal = False + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise NativeGoalProtocolError("goal_timeout_before_terminal") + turn_error = _wait_turn( + transport, + turn, + deadline=deadline, + completed_before=completed_before, + idle_timeout_seconds=args.turn_idle_timeout_seconds, + interrupt_grace_seconds=args.interrupt_grace_seconds, + ) + completed_before = turn.turn_completed_count + if turn_error is not None: + if not _is_transient_model_error(turn_error): + raise NativeGoalProtocolError(f"non_transient_model_error:{turn_error}") + if transient_retries >= args.max_transient_retries: + raise NativeGoalProtocolError( + f"transient_model_retries_exhausted:{turn_error}" + ) + transient_retries += 1 + delay = min( + args.retry_backoff_cap_seconds, + args.retry_backoff_base_seconds * (2 ** (transient_retries - 1)), + max(0.0, deadline - time.monotonic()), + ) + if delay <= 0: + raise NativeGoalProtocolError("goal_timeout_during_transient_backoff") + time.sleep(delay) + _reactivate_goal(transport, config, turn) + turn = _restart_turn_same_thread( + transport, + config, + turn, + "The previous model turn ended in a transient provider error. " + "Resume the active P0 task from the current Goal and workspace state; " + "do not repeat completed investigation.", + ) + completed_before = turn.turn_completed_count + continue + status = refresh_native_goal_status(transport, turn) + if status == "active": + continue + delivery = normalize_delivery(project, base_sha) + write_receipt(delivery_path, delivery) + benchmark_todo_terminal = _benchmark_todo_terminal(args, project) + if delivery["treatment_valid"] and benchmark_todo_terminal: + break + + can_recover_blocked = ( + status == "blocked" and args.recover_blocked and unblocks < args.max_unblocks + ) + can_recover_delivery = ( + (not delivery["treatment_valid"] or not benchmark_todo_terminal) + and delivery_retries < args.max_delivery_retries + ) + if not can_recover_blocked and not can_recover_delivery: + raise NativeGoalProtocolError( + f"goal_terminal_without_valid_delivery:status={status}:" + f"delivery={delivery.get('status')}" + ) + if can_recover_blocked: + unblocks += 1 + _clear_blocked( + loopx_cli=args.loopx_cli, + registry=args.registry, + runtime_root=args.runtime_root, + goal_id=args.goal_id, + agent_id=args.agent_id, + cwd=args.cwd, + ) + if can_recover_delivery: + delivery_retries += 1 + _reactivate_goal(transport, config, turn) + repair = ( + "Runner delivery check failed. Continue the active P0 task in /app, or recover your linked " + "worktree, and do not complete the Goal until git diff from the task base " + "is non-empty, committed, and the repository's public tests pass." + ) + turn = _restart_turn_same_thread(transport, config, turn, repair) + completed_before = turn.turn_completed_count + if delivery is None: + delivery = normalize_delivery(project, base_sha) + write_receipt(delivery_path, delivery) + if not delivery["treatment_valid"]: + raise NativeGoalProtocolError("native_goal_finished_without_valid_delivery") + return ( + turn, + unblocks, + delivery_retries, + transient_retries, + benchmark_todo_terminal, + delivery, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--cwd", required=True) + parser.add_argument("--objective-file", required=True) + parser.add_argument("--task-file", required=True) + parser.add_argument("--codex-bin", default="codex") + parser.add_argument("--model") + parser.add_argument("--effort") + parser.add_argument("--token-budget", type=int) + parser.add_argument("--response-timeout-seconds", type=float, default=30) + parser.add_argument("--goal-timeout-seconds", type=float, default=3600) + parser.add_argument("--sandbox", default="danger-full-access") + parser.add_argument("--required-skill-ids", default="") + parser.add_argument("--preflight-only", action="store_true") + parser.add_argument("--recover-blocked", action="store_true") + parser.add_argument("--max-unblocks", type=int, default=8) + parser.add_argument("--max-delivery-retries", type=int, default=3) + parser.add_argument("--max-transient-retries", type=int, default=12) + parser.add_argument("--retry-backoff-base-seconds", type=float, default=5) + parser.add_argument("--retry-backoff-cap-seconds", type=float, default=60) + parser.add_argument("--turn-idle-timeout-seconds", type=float, default=300) + parser.add_argument("--interrupt-grace-seconds", type=float, default=60) + parser.add_argument("--loopx-cli", default="loopx") + parser.add_argument("--registry", required=True) + parser.add_argument("--runtime-root", required=True) + parser.add_argument("--goal-id", required=True) + parser.add_argument("--agent-id", required=True) + parser.add_argument("--mode", choices=("ssh-goal",), default="ssh-goal") + args = parser.parse_args() + + result = run_goal(args) + if args.preflight_only: + turn, unblocks = result + delivery_retries = 0 + transient_retries = 0 + benchmark_todo_terminal = False + delivery = None + else: + ( + turn, + unblocks, + delivery_retries, + transient_retries, + benchmark_todo_terminal, + delivery, + ) = result + receipt = compact_native_goal_receipt(turn) + receipt["execution_mode"] = "goal_attachment_preflight" if args.preflight_only else "goal_until_terminal" + receipt["loopx_mode"] = args.mode + receipt["continuation_owner"] = "codex" + receipt["loopx_unblock_count"] = unblocks + receipt["loopx_blocked_recovery"] = bool(args.recover_blocked) + receipt["delivery_retry_count"] = delivery_retries + receipt["transient_retry_count"] = transient_retries + receipt["turn_idle_timeout_seconds"] = args.turn_idle_timeout_seconds + receipt["benchmark_todo_terminal"] = benchmark_todo_terminal + receipt["delivery"] = delivery + receipt["treatment_valid"] = bool(delivery and delivery.get("treatment_valid")) + receipt["host_surface"] = "native_goal_appserver" + receipt["model"] = args.model + receipt["effort"] = args.effort + receipt["task_correctness_authority"] = "independent_verifier" + print(json.dumps(receipt, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/deepswe-gptxhigh-v1/pier_cn.py b/benchmark/deepswe-gptxhigh-v1/pier_cn.py new file mode 100644 index 0000000000..00886ef2d2 --- /dev/null +++ b/benchmark/deepswe-gptxhigh-v1/pier_cn.py @@ -0,0 +1,395 @@ +#!/usr/bin/env python +"""Pier, with agent-image builds pointed at domestic mirrors. + +Pier bakes the agent harness into every task image. Each of DeepSWE's 113 tasks +ships its own base image, so this layer is rebuilt 113 times and never reused +between tasks — whatever it costs, it costs 113 times, per harness. From here +the upstream install path is slow or fatal at three points: + + * ``apt-get update`` hits deb.debian.org, which does not answer at all from + this host; a build sat there for 25 min before it was killed. + * ``curl -LsSf https://astral.sh/uv/... | sh`` (mini-swe-agent) pulls a 17 MB + binary from GitHub only to *downgrade* the uv 0.9.18 these images already + carry in /root/.local/bin. + * ``uv tool install`` resolves against pypi.org, unreachable here without the + proxy and timing out even with it; ``npm install -g`` against + registry.npmjs.org (claude-code, codex, opencode) works but at 0.8 MB/s. + +So: keep the image's own uv when it has one, and resolve everything from +mirrors.aliyun.com / registry.npmmirror.com, which answer directly in ~0.2-0.5 s +at 2.8-5.6 MB/s and need no proxy. Measured end to end on one task image for +mini-swe-agent: 366 s and then a failure, against 8.8 s. + +The proxy still applies to the build (run.sh points DOCKER_CONFIG at +mr_common/docker-config); this only removes the steps that depend on it being +both up and fast. The mirror hosts are in that file's ``noProxy`` on purpose — +routed through the proxy they come back 502, and they need no proxy anyway. + +run.sh calls this instead of .venv/bin/pier. A plain `pier` is untouched and +still behaves exactly as upstream ships it. +""" + +from __future__ import annotations + +import importlib +import os +import re +import sys + +PYPI_INDEX = "https://mirrors.aliyun.com/pypi/simple/" +NPM_REGISTRY = "https://registry.npmmirror.com" +APT_MIRROR_HOST = "mirrors.aliyun.com" + +# UV_DEFAULT_INDEX is what uv >= 0.6 reads; UV_INDEX_URL keeps older uv (and +# pier's pinned 0.7.13, should an image ever need it fetched) on the mirror too. +PYPI_ENV = { + "UV_DEFAULT_INDEX": PYPI_INDEX, + "UV_INDEX_URL": PYPI_INDEX, + "PIP_INDEX_URL": PYPI_INDEX, + "UV_HTTP_TIMEOUT": "120", +} + +# npm reads either spelling depending on version; setting both is free. +NPM_ENV = { + "NPM_CONFIG_REGISTRY": NPM_REGISTRY, + "npm_config_registry": NPM_REGISTRY, +} + +# Matched verbatim against pier's install script, so an upstream change to the +# pinned uv version makes the patch fall through loudly rather than silently +# stop applying. +UV_INSTALL_LINE = "curl -LsSf https://astral.sh/uv/0.7.13/install.sh | sh" + +UV_REUSE_BLOCK = f"""if ! command -v uv >/dev/null 2>&1 && [ ! -x "$HOME/.local/bin/uv" ]; then + {UV_INSTALL_LINE} +fi +mkdir -p "$HOME/.local/bin" +[ -f "$HOME/.local/bin/env" ] || : > "$HOME/.local/bin/env" """ + +# Runs before the package manager. Images that are not Debian match no files. +APT_MIRROR_PREFIX = ( + "for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.sources" + " /etc/apt/sources.list.d/*.list; do" + ' [ -f "$f" ] || continue;' + f' sed -i "s|deb.debian.org|{APT_MIRROR_HOST}|g;' + f' s|security.debian.org|{APT_MIRROR_HOST}|g" "$f";' + " done 2>/dev/null; true; " +) + +# Every harness Pier can install that we might sweep with. Listed explicitly +# rather than walked from __subclasses__: the subclasses only exist once their +# module is imported, and an agent that quietly missed the patch would show up +# as a 25-minute hang rather than an error. +HARNESSES = ( + ("pier.agents.installed.mini_swe_agent", "MiniSweAgent"), + ("pier.agents.installed.claude_code", "ClaudeCode"), + ("pier.agents.installed.codex", "Codex"), + ("pier.agents.installed.opencode", "OpenCode"), + ("pier.agents.installed.cursor_cli", "CursorCli"), + ("pier.agents.installed.gemini_cli", "GeminiCli"), +) + + +def rewrite_step(step) -> list[str]: + """Point one install step at the mirrors. Returns what it changed.""" + changed = [] + run = step.run + + if ("apt-get" in run or "apk add" in run) and APT_MIRROR_PREFIX not in run: + run = APT_MIRROR_PREFIX + run + changed.append("apt") + + if UV_INSTALL_LINE in run: + run = run.replace(UV_INSTALL_LINE, UV_REUSE_BLOCK) + changed.append("uv-reuse") + if "uv tool install" in run or "uv pip install" in run or "pip install" in run: + step.env = {**(step.env or {}), **PYPI_ENV} + changed.append("pypi") + + if "npm install" in run or "npm i " in run: + step.env = {**(step.env or {}), **NPM_ENV} + # Registry downloads occasionally reset the TLS connection on this host; + # make image builds retry instead of turning a transient reset into an + # invalid benchmark trial. + run = run.replace( + "npm install -g @openai/codex@latest", + "npm install --fetch-retries=5 --fetch-retry-mintimeout=5000 " + "--fetch-retry-maxtimeout=60000 -g @openai/codex@latest", + ) + # Debian task images already ship node/npm. Avoid the upstream nvm + # bootstrap from GitHub, which is unreachable when the host proxy is + # down; install the harness directly from the configured npm mirror. + if "nvm-sh/nvm" in run: + run = re.sub( + r"else\s+curl -o- https://raw\.githubusercontent\.com/nvm-sh/nvm/.*?; fi && codex --version", + "else npm install --fetch-retries=5 --fetch-retry-mintimeout=5000 " + "--fetch-retry-maxtimeout=60000 -g @openai/codex@latest; " + "fi && codex --version", + run, + flags=re.DOTALL, + ) + changed.append("nvm-bypass") + changed.append("npm") + + step.run = run + return changed + + +def patch_harnesses() -> None: + """Wrap install_spec on every known harness class.""" + patched_any = False + for module_path, class_name in HARNESSES: + try: + cls = getattr(importlib.import_module(module_path), class_name) + except (ImportError, AttributeError) as exc: + print(f"pier_cn: skipping {class_name} ({exc})", file=sys.stderr) + continue + + original = cls.install_spec + + def install_spec(self, _original=original, _name=class_name): + spec = _original(self) + changed = [] + for step in spec.steps: + changed += rewrite_step(step) + if not changed: + # Pier changed its install script out from under the patch: say + # so rather than let a sweep crawl or die one task at a time. + print( + f"pier_cn: WARNING — no install step of {_name} matched; its " + "build will use upstream's GitHub/PyPI/npm path and may hang.", + file=sys.stderr, + ) + return spec + + cls.install_spec = install_spec + patched_any = True + + if not patched_any: + raise SystemExit("pier_cn: could not patch any harness — refusing to run") + + +def patch_goal_mode() -> None: + """Point the ``codex`` agent name at one of the experiment's two arms. + + Pier's CLI validates ``--agent`` against the ``AgentName`` enum, so the + factory's import-path loader ('module:Class') cannot be reached from the + command line and a new name cannot simply be added. Rebinding the existing + name suits the experiment better anyway: both arms then run byte-for-byte + identical commands and differ only in this one environment variable, which + is what makes the comparison controlled. + + MR_CODEX_ARM=goal Goal attached over the app-server (treatment) + MR_CODEX_ARM=plain same objective text and web_search setting, but + app-server transport with no Goal (control) + MR_CODEX_ARM=loopx Codex driven by LoopX's governed Turn loop + MR_CODEX_ARM=loopx-native + LoopX through its own product path: formal release + install, LoopX-rendered Goal body, and continuation + owned by app-server. `loopx` predates it and drove + `turn run-once` from an outer loop with a + hand-written goal document, which LoopX's own + benchmark method rules out; its results describe + that wrapper rather than this product. + MR_CODEX_ARM=loopx-native-deepseek + Byte-identical to loopx-native; only the arm name + differs, so it lands in its own + deepswe--loopx-native-deepseek job + directory instead of overwriting the gpt-5.5 + native run already recorded there. Combine with + MR_MODEL to actually route to a different model. + MR_CODEX_ARM=loopx-native-deepseek-flash + Same as loopx-native-deepseek; a separate alias so + a deepseek-v4-flash sweep lands in its own job + directory instead of mixing into + deepswe--loopx-native-deepseek, which + already holds the deepseek-v4-pro results. + MR_CODEX_ARM=loopx-native-codex-cli + Official `codex_cli` profile through LoopX + `turn run-once`, launching real `codex exec` turns. + MR_CODEX_ARM=loopx-native-heartbeat + Official `generic_cli` heartbeat profile; recurring + wakes are owned by the benchmark supervisor. + + MR_GOAL_MODE=1 is still honoured as a synonym for the goal arm. Unset, an + ordinary sweep is untouched. + """ + arm = os.environ.get("MR_CODEX_ARM", "").strip().lower() + if not arm and os.environ.get("MR_GOAL_MODE", "") not in ("", "0"): + arm = "goal" + if not arm: + return + if arm not in ("goal", "plain", "loopx", "loopx-native", "loopx-native-deepseek", + "loopx-native-deepseek-flash", "loopx-native-codex-cli", + "loopx-native-heartbeat"): + raise SystemExit( + "pier_cn: MR_CODEX_ARM must be 'goal', 'plain', 'loopx', 'loopx-native', " + "'loopx-native-deepseek', 'loopx-native-deepseek-flash', " + "'loopx-native-codex-cli' or 'loopx-native-heartbeat', " + f"got {arm!r}" + ) + + from pier.agents.factory import AgentFactory + from pier.models.agent.name import AgentName + + import goal_codex + + if arm in ("loopx-native", "loopx-native-deepseek", "loopx-native-deepseek-flash", + "loopx-native-codex-cli", "loopx-native-heartbeat"): + os.environ["MR_LOOPX_MODE"] = { + "loopx-native": "ssh-goal", + "loopx-native-deepseek": "ssh-goal", + "loopx-native-deepseek-flash": "ssh-goal", + "loopx-native-codex-cli": "codex-cli", + "loopx-native-heartbeat": "heartbeat", + }[arm] + import loopx_native_codex + + agent = { + "loopx-native": loopx_native_codex.LoopxNativeCodex, + "loopx-native-deepseek": loopx_native_codex.LoopxNativeCodex, + "loopx-native-deepseek-flash": loopx_native_codex.LoopxNativeCodex, + "loopx-native-codex-cli": loopx_native_codex.LoopxCodexCliCodex, + "loopx-native-heartbeat": loopx_native_codex.LoopxHeartbeatCodex, + }[arm] + else: + agent = { + "goal": goal_codex.GoalCodex, + "plain": goal_codex.PlainAppServerCodex, + "loopx": goal_codex.LoopxCodex, + }[arm] + AgentFactory._AGENT_MAP[AgentName.CODEX] = agent + print( + f"pier_cn: MR_CODEX_ARM={arm} — 'codex' now runs as {agent.__qualname__}", + file=sys.stderr, + ) + + +def patch_claude_arm() -> None: + """Point the ``claude-code`` agent name at one of its two arms. + + The same rebinding trick as ``patch_goal_mode``, on a separate environment + variable so the two agent families stay independent: a sweep can run the + Codex arms and the Claude Code arms without either one's setting leaking + into the other. + + MR_CLAUDE_ARM=plain stock `claude --print`, plus the objective text + that the LoopX arm necessarily carries (control) + MR_CLAUDE_ARM=loopx Claude Code driven by LoopX's governed Turn loop + + There is no ``goal`` arm here: Claude Code's native `/loop` is interactive + and has no `--print` entry point, so it cannot be exercised inside Pier's + one-shot container invocation. Unset, an ordinary sweep is untouched. + """ + arm = os.environ.get("MR_CLAUDE_ARM", "").strip().lower() + if not arm: + return + if arm not in ("plain", "loopx"): + raise SystemExit( + f"pier_cn: MR_CLAUDE_ARM must be 'plain' or 'loopx', got {arm!r}" + ) + + from pier.agents.factory import AgentFactory + from pier.models.agent.name import AgentName + + import goal_claude + + agent = { + "plain": goal_claude.PlainClaudeCode, + "loopx": goal_claude.LoopxClaudeCode, + }[arm] + AgentFactory._AGENT_MAP[AgentName.CLAUDE_CODE] = agent + print( + f"pier_cn: MR_CLAUDE_ARM={arm} — 'claude-code' now runs as {agent.__qualname__}", + file=sys.stderr, + ) + + +def patch_modelonly_network() -> None: + """Attach Pier's task container to the internal model-only network. + + The compose overlay is appended after Pier's generated files, so the task + can reach the host gateway at 127.0.0.1 without relying on the egress + proxy or the container's loopback address. + """ + if os.environ.get("MR_MODELONLY_NET", "0") not in ("1", "true", "yes"): + return + from pathlib import Path + from pier.environments.docker.docker import DockerEnvironment + + local_agent_base = os.environ.get("MR_LOCAL_AGENT_BASE_IMAGE", "").strip() + if local_agent_base and not getattr( + DockerEnvironment._prepare_agent_build_context, "_deepswe_local_base", False + ): + original_prepare = DockerEnvironment._prepare_agent_build_context + + def prepare_agent_build_context(self): + # BuildKit performs a registry metadata HEAD even when the remote + # base image was already loaded. Use the explicit local alias only + # while Pier writes the generated agent Dockerfile; the task config + # is restored before Compose starts, so scoring metadata remains + # unchanged. + original_image = self.task_env_config.docker_image + self.task_env_config.docker_image = local_agent_base + try: + return original_prepare(self) + finally: + self.task_env_config.docker_image = original_image + + prepare_agent_build_context._deepswe_local_base = True + DockerEnvironment._prepare_agent_build_context = prepare_agent_build_context + print( + f"pier_cn: local agent base override={local_agent_base}", + file=sys.stderr, + ) + + overlay = Path(__file__).resolve().with_name("docker-compose-modelonly.yaml") + if not overlay.exists(): + raise SystemExit(f"pier_cn: missing model-only compose overlay: {overlay}") + original = DockerEnvironment._docker_compose_paths + if getattr(original, "_deepswe_modelonly", False): + return + + def paths(self): + # Verifier environments are deliberately no-network. Adding a + # `networks` key there conflicts with Compose's `network_mode: none`. + # Pier uses `no-network` for both agent and verifier task declarations; + # the verifier's environment directory is the reliable distinction. + if Path(self.environment_dir).name == "tests": + return list(original.fget(self)) + result = list(original.fget(self)) + result.append(overlay) + return result + + paths._deepswe_modelonly = True + DockerEnvironment._docker_compose_paths = property(paths) + original_agent_env = DockerEnvironment.agent_process_env + + def agent_process_env(self, env): + result = dict(original_agent_env(self, env) or {}) + for key in ("HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"): + result[key] = "" + no_proxy = result.get("NO_PROXY", "") + result["NO_PROXY"] = ",".join( + item for item in (no_proxy, "localhost", "127.0.0.1", "127.0.0.1") if item + ) + result["no_proxy"] = result["NO_PROXY"] + return result + + agent_process_env._deepswe_modelonly = True + DockerEnvironment.agent_process_env = agent_process_env + print(f"pier_cn: model-only network overlay={overlay}", file=sys.stderr) + + +def main() -> int: + patch_harnesses() + patch_goal_mode() + patch_claude_arm() + patch_modelonly_network() + from pier.cli.main import app + + return app() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmark/deepswe-gptxhigh-v1/preflight_loopx_rerun.py b/benchmark/deepswe-gptxhigh-v1/preflight_loopx_rerun.py new file mode 100644 index 0000000000..f562da6a99 --- /dev/null +++ b/benchmark/deepswe-gptxhigh-v1/preflight_loopx_rerun.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Fail-closed admission check run independently before every LoopX arm.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import socket +import subprocess +import sys +import tempfile +import tomllib +from datetime import datetime, timezone +from pathlib import Path + +from goal30_subset import SUBSET +from hard24_subset import HARD_SUBSET +from remaining4_subset import REMAINING_SUBSET +from workspace_delivery import head_sha, normalize_delivery + + +TASKS = tuple(dict.fromkeys((*SUBSET, *HARD_SUBSET, *REMAINING_SUBSET))) +EXPECTED_MODEL = "openai/gpt-5.6-sol" +EXPECTED_EFFORT = "xhigh" +EXPECTED_AGENT_TIMEOUT_MULTIPLIER = 3.0 +EXPECTED_GOAL_TIMEOUT_SECONDS = 14400.0 +EXPECTED_HEARTBEAT_SEGMENT_TIMEOUT_SECONDS = 7200.0 +EXPECTED_TURN_IDLE_TIMEOUT_SECONDS = 7200.0 +EXPECTED_LOOPX_REVISION = os.environ.get( + "MR_EXPECTED_LOOPX_REVISION", "2cef51d08b2a0103f4ba026bf47fd70dc8acee30" +) +RUNNERS = { + "ssh-goal": { + "file": "loopx_wen_native_runner.py", + "surface": "native_goal_appserver", + "required": ('"app-server"', '"turn/interrupt"', '"task_correctness_authority"', '"independent_verifier"'), + "forbidden": (), + }, + "codex-cli": { + "file": "loopx_codex_cli_runner.py", + "surface": "codex_cli", + "required": ('"run-once"', '"--host"', '"codex-cli"', '"task_correctness_authority"', '"independent_verifier"'), + "forbidden": ('"app-server"',), + }, + "heartbeat": { + "file": "loopx_heartbeat_supervisor.py", + "surface": "outer_controller_heartbeat", + "required": ('"heartbeat-prompt"', '"outer_controller"', '"codex_exec"', '"continuation_owner": "benchmark_supervisor"', '"task_correctness_authority": "independent_verifier"'), + "forbidden": ('"app-server"',), + }, +} + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _git(*args: str, cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ("git", *args), cwd=cwd, capture_output=True, text=True, check=False + ) + + +def delivery_self_test() -> tuple[bool, str]: + """Exercise the exact ignored-control-state failure seen in smoke.""" + with tempfile.TemporaryDirectory(prefix="deepswe-delivery-preflight-") as tmp: + project = Path(tmp) / "project" + project.mkdir() + commands = ( + ("init", "-q"), + ("config", "user.name", "DeepSWE preflight"), + ("config", "user.email", "preflight@deepswe.invalid"), + ) + for command in commands: + result = _git(*command, cwd=project) + if result.returncode: + return False, result.stderr[-300:] + (project / "source.txt").write_text("base\n", encoding="utf-8") + if _git("add", "source.txt", cwd=project).returncode: + return False, "could_not_stage_base" + if _git("commit", "-qm", "base", cwd=project).returncode: + return False, "could_not_commit_base" + base = head_sha(project) + for name in (".loopx", ".codex"): + (project / name).mkdir() + (project / name / "state.json").write_text("{}\n", encoding="utf-8") + (project / "delivered.txt").write_text("ok\n", encoding="utf-8") + receipt = normalize_delivery(project, base) + tracked = _git("ls-files", cwd=project).stdout.splitlines() + valid = ( + receipt.get("treatment_valid") is True + and receipt.get("patch_applies") is True + and "delivered.txt" in tracked + and not any(path.startswith((".loopx/", ".codex/")) for path in tracked) + ) + return valid, str(receipt.get("reason") or "unknown") + + +def port_is_free(port: int) -> bool: + for host in ("127.0.0.1", "127.0.0.1"): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.settimeout(0.1) + if probe.connect_ex((host, port)) == 0: + return False + return True + + +def task_manifest(task_root: Path) -> tuple[str, float, list[str]]: + entries = [] + timeouts = [] + missing = [] + for task in TASKS: + path = task_root / task / "task.toml" + if not path.is_file(): + missing.append(task) + continue + raw = path.read_bytes() + data = tomllib.loads(raw.decode("utf-8")) + base = str(data.get("metadata", {}).get("base_commit_hash") or "") + timeout = float(data.get("agent", {}).get("timeout_sec") or 0) + if re.fullmatch(r"[0-9a-fA-F]{7,40}", base) is None or timeout <= 0: + missing.append(task) + continue + timeouts.append(timeout) + entries.append(f"{task}\t{base}\t{hashlib.sha256(raw).hexdigest()}") + payload = "\n".join(entries).encode("utf-8") + return hashlib.sha256(payload).hexdigest(), min(timeouts, default=0), missing + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--arm", choices=tuple(RUNNERS), required=True) + parser.add_argument("--loopx-root", type=Path, required=True) + parser.add_argument("--port", type=int, required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--effort", required=True) + parser.add_argument("--goal-timeout", type=float, required=True) + parser.add_argument("--heartbeat-segment-timeout", type=float, required=True) + parser.add_argument("--turn-idle-timeout", type=float, required=True) + parser.add_argument("--agent-timeout-multiplier", type=float, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + root = Path(__file__).resolve().parent + errors: list[str] = [] + contract = RUNNERS[args.arm] + runner_name = str(contract["file"]) + surface = str(contract["surface"]) + runner = root / runner_name + delivery = root / "workspace_delivery.py" + adapter = root / "loopx_native_codex.py" + wrapper = root / "codex_nosandbox_wrapper.py" + harness = root / "goal_codex.py" + task_root = root / "upstream" / "tasks" + manifest_sha256, minimum_agent_timeout, malformed_tasks = task_manifest(task_root) + if args.port in {4141, 4250}: + errors.append("prohibited_gateway_port") + if not port_is_free(args.port): + errors.append("gateway_port_occupied") + if args.model != EXPECTED_MODEL: + errors.append("model_not_fair_control") + if args.effort != EXPECTED_EFFORT: + errors.append("effort_not_fair_control") + if args.agent_timeout_multiplier != EXPECTED_AGENT_TIMEOUT_MULTIPLIER: + errors.append("agent_timeout_multiplier_not_fair_control") + if args.goal_timeout != EXPECTED_GOAL_TIMEOUT_SECONDS: + errors.append("goal_timeout_not_pinned") + if args.heartbeat_segment_timeout != EXPECTED_HEARTBEAT_SEGMENT_TIMEOUT_SECONDS: + errors.append("heartbeat_segment_timeout_not_pinned") + if args.turn_idle_timeout != EXPECTED_TURN_IDLE_TIMEOUT_SECONDS: + errors.append("turn_idle_timeout_not_pinned") + if args.heartbeat_segment_timeout >= args.goal_timeout: + errors.append("heartbeat_segment_not_below_goal_timeout") + if minimum_agent_timeout <= 0 or args.goal_timeout >= minimum_agent_timeout * args.agent_timeout_multiplier: + errors.append("inner_timeout_not_below_agent_timeout") + source = runner.read_text(encoding="utf-8") if runner.is_file() else "" + if not runner.is_file(): + errors.append("runner_missing") + if any(marker not in source for marker in contract["required"]): + errors.append("runner_contract_marker_missing") + if any(marker in source for marker in contract["forbidden"]): + errors.append("runner_uses_wrong_surface") + if not delivery.is_file(): + errors.append("delivery_gate_missing") + adapter_source = adapter.read_text(encoding="utf-8") if adapter.is_file() else "" + if not adapter.is_file() or '"--accept-onboarding-agent-todos"' in adapter_source: + errors.append("benchmark_adapter_allows_onboarding_todos") + if '"--no-onboarding-scan"' not in adapter_source or '"--text", "[P0] " + task_text' not in adapter_source: + errors.append("benchmark_task_admission_contract_missing") + wrapper_source = wrapper.read_text(encoding="utf-8") if wrapper.is_file() else "" + if args.arm == "codex-cli" and ( + not wrapper.is_file() or "MR_CODEX_REASONING_EFFORT" not in wrapper_source + ): + errors.append("codex_cli_effort_injection_missing") + if args.arm == "heartbeat" and "model_reasoning_effort=" not in source: + errors.append("heartbeat_effort_injection_missing") + harness_source = harness.read_text(encoding="utf-8") if harness.is_file() else "" + if not harness.is_file() or 'args += ["--effort", effort]' not in harness_source: + errors.append("runner_effort_forwarding_missing") + if args.arm in {"codex-cli", "heartbeat"} and ( + 'runner_env.get("LOOPX_MODE") in {"codex-cli", "heartbeat"}' not in harness_source + or '"--segment-timeout-seconds"' not in harness_source + or 'MR_HEARTBEAT_SEGMENT_TIMEOUT_SEC' not in harness_source + ): + errors.append("runner_segment_timeout_forwarding_missing") + if args.arm == "ssh-goal" and ( + 'runner_env.get("LOOPX_MODE") == "ssh-goal"' not in harness_source + or '"--turn-idle-timeout-seconds"' not in harness_source + or "MR_LOOPX_TURN_IDLE_TIMEOUT_SEC" not in harness_source + ): + errors.append("runner_turn_idle_timeout_forwarding_missing") + compile_check = subprocess.run( + [sys.executable, "-m", "py_compile", str(runner), str(delivery)], + capture_output=True, + text=True, + check=False, + ) + if compile_check.returncode: + errors.append("runner_or_delivery_not_compilable") + delivery_probe_ok, delivery_probe_detail = delivery_self_test() + if not delivery_probe_ok: + errors.append("delivery_self_test_failed") + if len(TASKS) != 54 or len(set(TASKS)) != 54: + errors.append("task_set_not_54_unique") + if malformed_tasks: + errors.append("missing_or_malformed_task_definitions") + git = subprocess.run( + ["git", "-C", str(args.loopx_root), "status", "--porcelain"], + capture_output=True, + text=True, + check=False, + ) + if git.returncode or git.stdout.strip(): + errors.append("loopx_source_not_clean") + revision = subprocess.run( + ["git", "-C", str(args.loopx_root), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=False, + ).stdout.strip() + if len(revision) != 40: + errors.append("loopx_revision_missing") + elif revision != EXPECTED_LOOPX_REVISION: + errors.append("loopx_revision_not_pinned") + + receipt = { + "schema_version": "deepswe_loopx_arm_admission_v1", + "generated_at": datetime.now(timezone.utc).isoformat(), + "arm": args.arm, + "host_surface": surface, + "runner": runner_name, + "runner_sha256": digest(runner) if runner.is_file() else None, + "delivery_gate_sha256": digest(delivery) if delivery.is_file() else None, + "benchmark_adapter_sha256": digest(adapter) if adapter.is_file() else None, + "codex_wrapper_sha256": digest(wrapper) if wrapper.is_file() else None, + "goal_harness_sha256": digest(harness) if harness.is_file() else None, + "loopx_revision": revision or None, + "loopx_source_clean": "loopx_source_not_clean" not in errors, + "model": args.model, + "effort": args.effort, + "goal_timeout_seconds": args.goal_timeout, + "heartbeat_segment_timeout_seconds": args.heartbeat_segment_timeout, + "turn_idle_timeout_seconds": args.turn_idle_timeout, + "agent_timeout_multiplier": args.agent_timeout_multiplier, + "minimum_native_agent_timeout_seconds": minimum_agent_timeout, + "task_count": len(TASKS), + "task_manifest_sha256": manifest_sha256, + "missing_or_malformed_tasks": malformed_tasks, + "gateway_port": args.port, + "gateway_port_free": "gateway_port_occupied" not in errors, + "network_policy": "model_only_dedicated_gateway", + "sandbox_policy": "danger_full_access_equivalent", + "web_search": "disabled", + "delivery_self_test": delivery_probe_ok, + "delivery_self_test_detail": delivery_probe_detail, + "lifecycle_authority": "loopx_goal_or_todo", + "task_correctness_authority": "independent_verifier", + "admitted": not errors, + "errors": errors, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(receipt, sort_keys=True)) + return 0 if receipt["admitted"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/deepswe-gptxhigh-v1/run_five_arms_remaining59_20260910.sh b/benchmark/deepswe-gptxhigh-v1/run_five_arms_remaining59_20260910.sh new file mode 100755 index 0000000000..407a9ed685 --- /dev/null +++ b/benchmark/deepswe-gptxhigh-v1/run_five_arms_remaining59_20260910.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Full 5-arm launch over the remaining 59 tasks (113 total - frozen 54). +# 3 LoopX arms mirror run_loopx_rerun_54_20260908.sh; plain/goal mirror +# run_goal_plain_infra_complete.sh. Task set: remaining59.txt. +set -uo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PY="$DIR/.venv-user-395647/bin/python" +MODEL="${MR_MODEL:-openai/gpt-5.6-sol}"; EFFORT="${MR_EFFORT:-xhigh}" +LOOPX_ROOT="${MR_LOOPX_ROOT:-$DIR/../loopx-official-latest}" +SHA="$(git -C "$LOOPX_ROOT" rev-parse --verify HEAD | cut -c1-12)" +OUT_ROOT="$DIR/jobs/loopx-five-arms-remaining59-20260910" +LOG_ROOT="$DIR/logs/loopx-five-arms-remaining59-20260910" +mkdir -p "$OUT_ROOT" "$LOG_ROOT" +GOAL_TIMEOUT=14400; HB_SEG=7200; TURN_IDLE=7200; MULT=3.0; LOOPX_CC=2 +PG_TIMEOUT=3600; PG_CC=5 +mapfile -t TASKS < "$DIR/remaining59.txt" +INC=(); for t in "${TASKS[@]}"; do INC+=(-i "$t"); done +echo "$(date -Is) launching 5 arms x ${#TASKS[@]} tasks -> $OUT_ROOT" + +launch_loopx() { # arm_mode codex_arm port + local m="$1" arm="$2" port="$3" jobs="$OUT_ROOT/$1"; mkdir -p "$jobs" + echo "$(date -Is) preflight $m (port $port)" + "$PY" "$DIR/preflight_loopx_rerun.py" --arm "$m" --loopx-root "$LOOPX_ROOT" \ + --port "$port" --model "$MODEL" --effort "$EFFORT" --goal-timeout "$GOAL_TIMEOUT" \ + --heartbeat-segment-timeout "$HB_SEG" --turn-idle-timeout "$TURN_IDLE" \ + --agent-timeout-multiplier "$MULT" --output "$LOG_ROOT/admission-$m.json" \ + >"$LOG_ROOT/preflight-$m.log" 2>&1 || { echo "$(date -Is) PREFLIGHT FAILED $m"; return 1; } + echo "$(date -Is) start $m" + ( cd "$DIR" + PIER_CUSTOM_NETWORKS=1 MR_MODELONLY_NET=1 MR_MODELONLY_HOST=127.0.0.1 MR_MODELONLY_PORT="$port" \ + MR_API_BASE="http://127.0.0.1:$port/v1" MR_AGENT=codex MR_MODEL="$MODEL" MR_EFFORT="$EFFORT" MR_REASONING_EFFORT="$EFFORT" \ + MR_CODEX_ARM="$arm" MR_LOOPX_MODE="$m" MR_LOOPX_ROOT="$LOOPX_ROOT" \ + MR_LOOPX_PROFILE_ROOT="/tmp/loopx-profile-fair54-$SHA-$m" MR_LOOPX_PREFLIGHT=0 MR_LOOPX_WEN_COMPAT=1 \ + MR_GOAL_TIMEOUT_SEC="$GOAL_TIMEOUT" MR_UPSTREAM_TIMEOUT="$GOAL_TIMEOUT" \ + MR_HEARTBEAT_SEGMENT_TIMEOUT_SEC="$HB_SEG" MR_LOOPX_TURN_IDLE_TIMEOUT_SEC="$TURN_IDLE" \ + MR_RUN_LABEL="five-arms-rem59-$m" MR_GATEWAY_PORT="$port" MR_JOBS_DIR="$jobs" \ + MR_LOG_PATH="$LOG_ROOT/gateway_calls_$m.jsonl" MR_GATEWAY_OUT="$LOG_ROOT/gateway_$m.out" \ + MR_GATEWAY_MAX_RETRIES=12 MR_GATEWAY_RETRY_BASE_SEC=5 MR_GATEWAY_RETRY_CAP_SEC=60 \ + ./run.sh --all "${INC[@]}" -k 1 -n "$LOOPX_CC" --agent-timeout-multiplier "$MULT" + ) >"$LOG_ROOT/$m.log" 2>&1 & + echo "$(date -Is) $m launched pid=$!" +} +launch_pg() { # mode port + local m="$1" port="$2" jobs="$OUT_ROOT/$1"; mkdir -p "$jobs" + echo "$(date -Is) start $m (gateway $port)" + ( cd "$DIR" + PIER_CUSTOM_NETWORKS=1 MR_MODELONLY_NET=1 MR_MODELONLY_HOST=127.0.0.1 MR_MODELONLY_PORT=4250 \ + MR_API_BASE=http://127.0.0.1:4250/v1 MR_AGENT=codex MR_MODEL="$MODEL" MR_EFFORT="$EFFORT" MR_REASONING_EFFORT="$EFFORT" \ + MR_CODEX_ARM="$m" MR_LOOPX_MODE=x MR_LOOPX_ROOT="$LOOPX_ROOT" MR_LOOPX_PREFLIGHT=0 MR_LOOPX_WEN_COMPAT=1 \ + MR_GOAL_TIMEOUT_SEC="$PG_TIMEOUT" MR_UPSTREAM_TIMEOUT="$PG_TIMEOUT" \ + MR_RUN_LABEL="five-arms-rem59-$m" MR_GATEWAY_PORT="$port" MR_JOBS_DIR="$jobs" \ + MR_LOG_PATH="$LOG_ROOT/gateway_calls_$m.jsonl" MR_GATEWAY_OUT="$LOG_ROOT/gateway_$m.out" \ + ./run.sh --all "${INC[@]}" -k 1 -n "$PG_CC" + ) >"$LOG_ROOT/$m.log" 2>&1 & + echo "$(date -Is) $m launched pid=$!" +} +launch_loopx ssh-goal loopx-native 4411 +launch_loopx codex-cli loopx-native-codex-cli 4412 +launch_loopx heartbeat loopx-native-heartbeat 4413 +launch_pg goal 4393 +launch_pg plain 4394 +echo "$(date -Is) all 5 arms launched; logs -> $LOG_ROOT" +wait +echo "$(date -Is) all arms finished" diff --git a/benchmark/deepswe-gptxhigh-v1/run_loopx_rerun_54_20260908.sh b/benchmark/deepswe-gptxhigh-v1/run_loopx_rerun_54_20260908.sh new file mode 100755 index 0000000000..31f6e48817 --- /dev/null +++ b/benchmark/deepswe-gptxhigh-v1/run_loopx_rerun_54_20260908.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# Fair rerun of the three LoopX treatments after host/delivery fixes. +set -euo pipefail +trap 'exit 130' INT TERM + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MODE="${1:-all}" +shift $(( $# > 0 ? 1 : 0 )) + +case "$MODE" in + all|ssh-goal|codex-cli|heartbeat) ;; + *) echo "usage: $0 [all|ssh-goal|codex-cli|heartbeat] [task ...]" >&2; exit 2 ;; +esac + +MODEL="${MR_MODEL:-openai/gpt-5.6-sol}" +EFFORT="${MR_EFFORT:-xhigh}" +CONCURRENT="${MR_CONCURRENT:-2}" +GOAL_TIMEOUT="${MR_GOAL_TIMEOUT_SEC:-14400}" +HEARTBEAT_SEGMENT_TIMEOUT="${MR_HEARTBEAT_SEGMENT_TIMEOUT_SEC:-7200}" +TURN_IDLE_TIMEOUT="${MR_LOOPX_TURN_IDLE_TIMEOUT_SEC:-7200}" +AGENT_TIMEOUT_MULTIPLIER="${MR_AGENT_TIMEOUT_MULTIPLIER:-3.0}" +PORT_BASE="${MR_GATEWAY_BASE_PORT:-4411}" +OUT_ROOT="${MR_LOOPX_RERUN_ROOT:-$DIR/jobs/loopx-three-arms-54-fair-rerun-20260908}" +LOG_ROOT="${MR_LOOPX_RERUN_LOG_ROOT:-$DIR/logs/loopx-three-arms-54-fair-rerun-20260908}" +LOOPX_ROOT="${MR_LOOPX_ROOT:-$DIR/../loopx-official-latest}" +LOOPX_REVISION="$(git -C "$LOOPX_ROOT" rev-parse --verify HEAD)" +LOOPX_REVISION_SHORT="${LOOPX_REVISION:0:12}" + +mapfile -t ALL_TASKS < <( + cd "$DIR" + python3 - <<'PY' +from goal30_subset import SUBSET +from hard24_subset import HARD_SUBSET +from remaining4_subset import REMAINING_SUBSET + +tasks = list(dict.fromkeys([*SUBSET, *HARD_SUBSET, *REMAINING_SUBSET])) +if len(tasks) != 54: + raise SystemExit(f"expected 54 tasks, got {len(tasks)}") +print("\n".join(tasks)) +PY +) + +if (( $# )); then + TASKS=("$@") + for requested in "${TASKS[@]}"; do + found=0 + for allowed in "${ALL_TASKS[@]}"; do + [[ "$requested" == "$allowed" ]] && found=1 && break + done + (( found == 1 )) || { echo "task is outside the frozen 54: $requested" >&2; exit 2; } + done +else + TASKS=("${ALL_TASKS[@]}") +fi + +port_for() { + case "$1" in + ssh-goal) echo "$PORT_BASE" ;; + codex-cli) echo "$((PORT_BASE + 1))" ;; + heartbeat) echo "$((PORT_BASE + 2))" ;; + esac +} + +arm_for() { + case "$1" in + ssh-goal) echo loopx-native ;; + codex-cli) echo loopx-native-codex-cli ;; + heartbeat) echo loopx-native-heartbeat ;; + esac +} + +preflight_arm() { + local arm_mode="$1" + local port jobs + port="$(port_for "$arm_mode")" + jobs="$OUT_ROOT/$arm_mode" + [[ "$port" != 4141 && "$port" != 4250 ]] || { + echo "prohibited DeepSWE gateway port: $port" >&2 + return 2 + } + if ss -ltnH "sport = :$port" | grep -q .; then + echo "gateway port already occupied: $port" >&2 + return 1 + fi + if pgrep -af "[p]ier_cn.py.*--jobs-dir $jobs" >/dev/null; then + echo "$arm_mode already running: $jobs" >&2 + return 1 + fi + + mkdir -p "$jobs" "$LOG_ROOT" + "$DIR/.venv-user-395647/bin/python" "$DIR/preflight_loopx_rerun.py" \ + --arm "$arm_mode" --loopx-root "$LOOPX_ROOT" --port "$port" \ + --model "$MODEL" --effort "$EFFORT" --goal-timeout "$GOAL_TIMEOUT" \ + --heartbeat-segment-timeout "$HEARTBEAT_SEGMENT_TIMEOUT" \ + --turn-idle-timeout "$TURN_IDLE_TIMEOUT" \ + --agent-timeout-multiplier "$AGENT_TIMEOUT_MULTIPLIER" \ + --output "$LOG_ROOT/admission-$arm_mode.json" +} + +run_arm() { + local arm_mode="$1" + local port arm jobs label + port="$(port_for "$arm_mode")" + arm="$(arm_for "$arm_mode")" + jobs="$OUT_ROOT/$arm_mode" + label="loopx-fair54-20260908-$arm_mode" + local -a include=() + local task + for task in "${TASKS[@]}"; do include+=(-i "$task"); done + echo "$(date -Is) start $arm_mode tasks=${#TASKS[@]} concurrency=$CONCURRENT port=$port" + ( + cd "$DIR" + PIER_CUSTOM_NETWORKS=1 \ + MR_MODELONLY_NET=1 MR_MODELONLY_HOST=127.0.0.1 MR_MODELONLY_PORT="$port" \ + MR_API_BASE="http://127.0.0.1:$port/v1" \ + MR_AGENT=codex MR_MODEL="$MODEL" MR_EFFORT="$EFFORT" MR_REASONING_EFFORT="$EFFORT" \ + MR_CODEX_ARM="$arm" MR_LOOPX_MODE="$arm_mode" MR_LOOPX_ROOT="$LOOPX_ROOT" \ + MR_LOOPX_PROFILE_ROOT="/tmp/loopx-profile-fair54-$LOOPX_REVISION_SHORT-$arm_mode" \ + MR_LOOPX_PREFLIGHT=0 MR_LOOPX_WEN_COMPAT=1 \ + MR_GOAL_TIMEOUT_SEC="$GOAL_TIMEOUT" MR_UPSTREAM_TIMEOUT="$GOAL_TIMEOUT" \ + MR_HEARTBEAT_SEGMENT_TIMEOUT_SEC="$HEARTBEAT_SEGMENT_TIMEOUT" \ + MR_LOOPX_TURN_IDLE_TIMEOUT_SEC="$TURN_IDLE_TIMEOUT" \ + MR_RUN_LABEL="$label" MR_GATEWAY_PORT="$port" MR_JOBS_DIR="$jobs" \ + MR_LOG_PATH="$LOG_ROOT/gateway_calls_$arm_mode.jsonl" \ + MR_GATEWAY_OUT="$LOG_ROOT/gateway_$arm_mode.out" \ + MR_GATEWAY_MAX_RETRIES=12 MR_GATEWAY_RETRY_BASE_SEC=5 MR_GATEWAY_RETRY_CAP_SEC=60 \ + ./run.sh --all "${include[@]}" -k 1 -n "$CONCURRENT" \ + --agent-timeout-multiplier "$AGENT_TIMEOUT_MULTIPLIER" + ) >"$LOG_ROOT/$arm_mode.log" 2>&1 + echo "$(date -Is) finish $arm_mode" +} + +if [[ "$MODE" == all ]]; then + for arm_mode in ssh-goal codex-cli heartbeat; do + preflight_arm "$arm_mode" + done + "$DIR/.venv-user-395647/bin/python" - "$LOG_ROOT" <<'PY' +import json +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +receipts = [json.loads((root / f"admission-{arm}.json").read_text()) for arm in ("ssh-goal", "codex-cli", "heartbeat")] +fixed = ( + "model", + "effort", + "agent_timeout_multiplier", + "goal_timeout_seconds", + "heartbeat_segment_timeout_seconds", + "turn_idle_timeout_seconds", + "task_count", + "task_manifest_sha256", + "loopx_revision", + "network_policy", + "sandbox_policy", + "web_search", + "task_correctness_authority", +) +for key in fixed: + values = {json.dumps(receipt.get(key), sort_keys=True) for receipt in receipts} + if len(values) != 1: + raise SystemExit(f"cross-arm admission mismatch for {key}: {values}") +surfaces = {receipt["host_surface"] for receipt in receipts} +if len(surfaces) != 3: + raise SystemExit(f"host surfaces are not distinct: {surfaces}") +if not all(receipt.get("admitted") for receipt in receipts): + raise SystemExit("at least one arm was not admitted") +print("cross-arm admission: matched controls, three distinct execution surfaces") +PY + for arm_mode in ssh-goal codex-cli heartbeat; do + run_arm "$arm_mode" + done + exit 0 +fi + +preflight_arm "$MODE" +run_arm "$MODE" diff --git a/benchmark/deepswe-gptxhigh-v1/workspace_delivery.py b/benchmark/deepswe-gptxhigh-v1/workspace_delivery.py new file mode 100755 index 0000000000..d6f539b1db --- /dev/null +++ b/benchmark/deepswe-gptxhigh-v1/workspace_delivery.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +"""Normalize an agent's work into the canonical DeepSWE checkout. + +DeepSWE grades ``git diff HEAD`` from ``/app``. LoopX may legitimately +place an agent in a linked worktree, so a commit can exist while the collector +still sees an empty patch. This module makes that delivery boundary explicit: +it finds the one changed worktree, commits any remaining agent changes, copies +the resulting patch into the canonical checkout, and proves that the patch +applies to the task base before the verifier is allowed to run. + +LoopX control state is runner-owned and is never included in the submission. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +CONTROL_PATHS = (".loopx", ".codex", ".worktrees") + + +class DeliveryError(RuntimeError): + pass + + +@dataclass(frozen=True) +class Candidate: + path: Path + head: str + patch: bytes + + @property + def patch_sha256(self) -> str: + return hashlib.sha256(self.patch).hexdigest() + + +def _run( + argv: list[str], + *, + cwd: Path | None = None, + input_bytes: bytes | None = None, + check: bool = True, +) -> subprocess.CompletedProcess[bytes]: + result = subprocess.run( + argv, + cwd=cwd, + input=input_bytes, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if check and result.returncode: + detail = result.stderr.decode("utf-8", "replace")[-600:] + raise DeliveryError(f"command failed ({result.returncode}): {argv!r}: {detail}") + return result + + +def head_sha(project: Path) -> str: + return _run(["git", "-C", str(project), "rev-parse", "HEAD"]).stdout.decode().strip() + + +def _exclude_control_state(project: Path) -> None: + git_dir = _run( + ["git", "-C", str(project), "rev-parse", "--git-common-dir"] + ).stdout.decode().strip() + common = Path(git_dir) + if not common.is_absolute(): + common = (project / common).resolve() + exclude = common / "info" / "exclude" + exclude.parent.mkdir(parents=True, exist_ok=True) + existing = exclude.read_text(encoding="utf-8") if exclude.exists() else "" + additions = [f"/{name}/" for name in CONTROL_PATHS if f"/{name}/" not in existing] + if additions: + with exclude.open("a", encoding="utf-8") as handle: + handle.write("\n# DeepSWE runner control state\n") + handle.write("\n".join(additions) + "\n") + + +def worktrees(project: Path) -> list[Path]: + output = _run( + ["git", "-C", str(project), "worktree", "list", "--porcelain"] + ).stdout.decode("utf-8", "replace") + paths = [] + for line in output.splitlines(): + if line.startswith("worktree "): + paths.append(Path(line.removeprefix("worktree ")).resolve()) + canonical = project.resolve() + return sorted(set(paths), key=lambda path: (path != canonical, str(path))) + + +def _commit_pending(path: Path) -> None: + _exclude_control_state(path) + # The runner state is ignored through the repository's shared exclude file. + # Passing ignored paths as explicit negative pathspecs makes Git reject the + # entire add operation in some task repositories. + _run(["git", "-C", str(path), "add", "-A", "--", "."]) + staged = _run( + ["git", "-C", str(path), "diff", "--cached", "--quiet"], check=False + ) + if staged.returncode not in (0, 1): + raise DeliveryError(f"could not inspect staged changes in {path}") + if staged.returncode == 1: + env = os.environ.copy() + env.setdefault("GIT_AUTHOR_NAME", "DeepSWE delivery runner") + env.setdefault("GIT_AUTHOR_EMAIL", "runner@deepswe.invalid") + env.setdefault("GIT_COMMITTER_NAME", env["GIT_AUTHOR_NAME"]) + env.setdefault("GIT_COMMITTER_EMAIL", env["GIT_AUTHOR_EMAIL"]) + result = subprocess.run( + ["git", "-C", str(path), "commit", "-m", "chore: deliver agent work"], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if result.returncode: + raise DeliveryError( + "could not commit pending agent changes: " + + result.stderr.decode("utf-8", "replace")[-600:] + ) + + +def _candidate(path: Path, base_sha: str) -> Candidate | None: + _commit_pending(path) + head = head_sha(path) + patch = _run( + [ + "git", + "-C", + str(path), + "diff", + "--binary", + base_sha, + "HEAD", + "--", + ".", + *(f":(exclude){name}" for name in CONTROL_PATHS), + ] + ).stdout + return Candidate(path=path, head=head, patch=patch) if patch.strip() else None + + +def _prove_applies(project: Path, base_sha: str, patch: bytes) -> None: + with tempfile.TemporaryDirectory(prefix="deepswe-delivery-check-") as tmp: + checkout = Path(tmp) / "base" + _run(["git", "clone", "--shared", "--no-checkout", str(project), str(checkout)]) + _run(["git", "-C", str(checkout), "checkout", "--detach", base_sha]) + _run(["git", "-C", str(checkout), "apply", "--check", "--binary", "-"], input_bytes=patch) + + +def _install_patch(project: Path, base_sha: str, patch: bytes) -> str: + _run(["git", "-C", str(project), "reset", "--hard", base_sha]) + _run(["git", "-C", str(project), "apply", "--index", "--binary", "-"], input_bytes=patch) + env = os.environ.copy() + env.setdefault("GIT_AUTHOR_NAME", "DeepSWE delivery runner") + env.setdefault("GIT_AUTHOR_EMAIL", "runner@deepswe.invalid") + env.setdefault("GIT_COMMITTER_NAME", env["GIT_AUTHOR_NAME"]) + env.setdefault("GIT_COMMITTER_EMAIL", env["GIT_AUTHOR_EMAIL"]) + result = subprocess.run( + ["git", "-C", str(project), "commit", "-m", "chore: recover agent worktree delivery"], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if result.returncode: + raise DeliveryError( + "could not install recovered patch: " + + result.stderr.decode("utf-8", "replace")[-600:] + ) + return head_sha(project) + + +def normalize_delivery(project: Path, base_sha: str) -> dict[str, Any]: + project = project.resolve() + receipt: dict[str, Any] = { + "schema_version": "deepswe_delivery_receipt_v1", + "project": str(project), + "base_sha": base_sha, + "status": "error", + "treatment_valid": False, + } + try: + candidates = [ + item + for path in worktrees(project) + if (item := _candidate(path, base_sha)) is not None + ] + by_patch: dict[str, list[Candidate]] = {} + for item in candidates: + by_patch.setdefault(item.patch_sha256, []).append(item) + receipt["changed_worktree_count"] = len(candidates) + receipt["distinct_patch_count"] = len(by_patch) + receipt["changed_worktrees"] = [str(item.path) for item in candidates] + if not candidates: + receipt["status"] = "empty" + receipt["reason"] = "no_agent_delta_from_task_base" + return receipt + if len(by_patch) != 1: + receipt["status"] = "ambiguous" + receipt["reason"] = "multiple_distinct_agent_patches" + return receipt + + selected = next(iter(by_patch.values()))[0] + _prove_applies(project, base_sha, selected.patch) + canonical = project.resolve() + recovered = selected.path.resolve() != canonical + if recovered: + final_head = _install_patch(project, base_sha, selected.patch) + else: + final_head = selected.head + final_patch = _run( + [ + "git", + "-C", + str(project), + "diff", + "--binary", + base_sha, + "HEAD", + "--", + ".", + *(f":(exclude){name}" for name in CONTROL_PATHS), + ] + ).stdout + if not final_patch.strip(): + raise DeliveryError("canonical patch became empty after normalization") + if hashlib.sha256(final_patch).hexdigest() != selected.patch_sha256: + raise DeliveryError("canonical patch digest differs after normalization") + _prove_applies(project, base_sha, final_patch) + + receipt.update( + { + "status": "valid", + "reason": "canonical_patch_verified", + "treatment_valid": True, + "source_worktree": str(selected.path), + "recovered_from_linked_worktree": recovered, + "final_head": final_head, + "patch_bytes": len(final_patch), + "patch_sha256": selected.patch_sha256, + "patch_applies": True, + } + ) + except Exception as exc: + receipt["status"] = "error" + receipt["reason"] = f"{type(exc).__name__}: {exc}" + return receipt + + +def write_receipt(path: Path, receipt: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--project", type=Path, required=True) + parser.add_argument("--base-sha", required=True) + parser.add_argument("--receipt", type=Path) + parser.add_argument("--require-valid", action="store_true") + args = parser.parse_args() + + receipt = normalize_delivery(args.project, args.base_sha) + if args.receipt: + write_receipt(args.receipt, receipt) + print(json.dumps(receipt, sort_keys=True)) + return 0 if receipt["treatment_valid"] or not args.require_valid else 12 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/deepswe-gptxhigh-versions.md b/benchmark/deepswe-gptxhigh-versions.md new file mode 100644 index 0000000000..01f9f98151 --- /dev/null +++ b/benchmark/deepswe-gptxhigh-versions.md @@ -0,0 +1,94 @@ +# DeepSWE GPT xhigh v1 Archive + +This PR publishes only the original v1 code and historical result summary in +[deepswe-gptxhigh-v1](deepswe-gptxhigh-v1/README.md). The revised execution package +has been removed from this PR. No later admission, retry, gateway, concurrency, +or termination changes are applied to v1. + +## Snapshot Identity + +The 14 files come from commit `98c262a8487ff5688d38091f603a87f7e2a78c64`, +subtree `benchmark/deepswe-five-arm/`. Only the containing directory name +changes. Contents and executable modes, including the original README, match +Git tree `1bc5d2b3b74761a97d34ba3f3612e977fd610340`. + +The evaluated LoopX revision was `2cef51d`. The original README's references to +"this branch's base" and an in-progress v2 are historical text, not current +project status. No new benchmark results are included here. + +## Executability Checks + +From the repository root, with Python 3.11+ and Bash installed: + +```sh +python3 benchmark/check_deepswe_v1.py +``` + +This read-only check verifies the archive's actual file bytes and modes, parses +all Python files (including embedded runner/bootstrap programs), and runs +`bash -n` on the launchers. It makes no model calls and does not run task code. +Exit zero means those checks passed, **not that a complete benchmark ran**. +The report explicitly records `standalone_runnable: false`. + +An additional local smoke imported the original adapters and dispatched all +five arm selectors using Python 3.12.13 and `datacurve-pier` 0.3.1 with external +experiment modules available. No agents were instantiated and no task/model +execution was attempted. This is import/dispatch compatibility evidence only, +not a full dependency lock or an end-to-end reproduction. + +## Runtime Prerequisites + +The original snapshot is an export from an external experiment workspace, +not a standalone distribution. Running it requires all of the following: + +| Requirement | Original interface | +| --- | --- | +| Python environment | Python 3.11+, compatible `datacurve-pier`; archived launchers expect `.venv-user-395647/bin/python` | +| LoopX source | A separate clean checkout of evaluated revision `2cef51d08b2a0103f4ba026bf47fd70dc8acee30`, selected with `MR_LOOPX_ROOT` | +| Plain arm support | Original external `plain_appserver_runner.py` beside `goal_codex.py` | +| Launch environment | External `run.sh` and its gateway, host-environment, and reporting dependencies | +| Task selection | `goal30_subset.py`, `hard24_subset.py`, `remaining4_subset.py`, and `remaining59.txt` | +| Task definitions | DeepSWE task definitions under `upstream/tasks/`, including independent verifier environments | +| Container runtime | Docker, Compose, task images, and external `docker-compose-modelonly.yaml` | +| Provider access | Separately configured gateway reachable from the task containers; sanitized loopback placeholders are not a complete deployment configuration | + +Assemble these inputs in a **new, isolated experiment workspace**, then place +the archived files there without rewriting them. Keep the published archive +unchanged, and do not overlay files into an ongoing benchmark workspace. +Record the external dependency versions and hashes with that run. Credentials, +raw task text, model configuration, and trajectories are intentionally absent +from this public repository. + +The five documented selectors are `plain`, `goal`, `loopx-native`, +`loopx-native-codex-cli`, and `loopx-native-heartbeat`. Legacy Claude and +`MR_CODEX_ARM=loopx` paths are preserved as source history; they are not an +additional supported reproduction claim. + +## Known Limits + +Original defects remain visible: missing external support files, hard-coded +environment assumptions, the remaining-59 launcher's 54-task admission mismatch, +profile initialization races, and launcher/retry failure handling. +These defects prevent a claim that the unchanged archive is fully runnable or +that its admission receipts reliably validate every future run. +Guaranteeing a clean end-to-end run requires separately reviewed execution +changes and validation in the actual runtime environment; silently changing +v1 to achieve that would invalidate the immutable snapshot boundary. + +The former revised package remains available in the previous PR commit +`a8fa4f9cf842557ee11ba24056c46a1410763c8c`, outside the final PR file set. +It is not the currently running new benchmark and has no attributed results. + +No benchmark was rerun for this PR. The +[current SWE Marathon publication](swe-marathon/README.md) withdraws SSH Goal and +Codex CLI data and conclusions pending revalidation. The unchanged historical +v1 summary does not override that correction. + +## 中文说明 + +本 PR 仅保留旧 v1 原始代码及结果，已移出 `v1-revised`。 +原始 14 个文件的内容和权限不变；没有混入正在运行的新 benchmark 结果。 + +新增的归档检查命令只验证文件完整性、Python（含内嵌代码）及 Shell 语法， +不调用模型、不执行任务，也不代表完整实验已跑通。运行仍需上表中的外部依赖， +原始执行缺陷同样保留。不能在不改原逻辑、未验证真实环境的前提下保证端到端可执行。