Archive original DeepSWE GPT xhigh v1 code and historical results

benchmark/check_deepswe_v1.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+"""Check the immutable v1 archive without launching a benchmark or model."""
+
+from __future__ import annotations
+
+import argparse
+import ast
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+
+
+ARCHIVE = Path(__file__).resolve().with_name("deepswe-gptxhigh-v1")
+EXPECTED_TREE = "1bc5d2b3b74761a97d34ba3f3612e977fd610340"
+
+
+def check_archive(archive: Path) -> dict:
+    tree = bytearray()
+    python_count = shell_count = embedded_count = 0
+    for path in sorted(archive.iterdir(), key=lambda entry: entry.name.encode()):
+        if not path.is_file() or path.is_symlink():
+            raise ValueError(f"unexpected archive entry: {path.name}")
+        raw = path.read_bytes()
+        blob = hashlib.sha1(b"blob " + str(len(raw)).encode() + b"\0" + raw).digest()
+        mode = b"100755" if path.stat().st_mode & 0o111 else b"100644"
+        tree.extend(mode + b" " + path.name.encode() + b"\0" + blob)
+        if path.suffix == ".py":
+            compile(raw, path.name, "exec")
+            python_count += 1
+            for node in ast.parse(raw).body:
+                if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.Constant):
+                    continue
+                names = {target.id for target in node.targets if isinstance(target, ast.Name)}
+                if names & {"_RUNNER", "_BOOTSTRAP"} and isinstance(node.value.value, str):
+                    source = node.value.value
+                    if "_RUNNER" in names:
+                        source = source.format(remote_dir="/tmp/loopx-goal")
+                    compile(source, f"{path.name}:embedded", "exec")
+                    embedded_count += 1
+        elif path.suffix == ".sh":
+            subprocess.run(["bash", "-n", str(path)], check=True, capture_output=True, timeout=10)
+            shell_count += 1
+    actual_tree = hashlib.sha1(b"tree " + str(len(tree)).encode() + b"\0" + tree).hexdigest()
+    if actual_tree != EXPECTED_TREE:
+        raise ValueError("archive contents or executable modes differ from the original v1 snapshot")
+    return {
+        "archive_tree": actual_tree,
+        "original_snapshot_preserved": True,
+        "python_syntax_checks": python_count,
+        "embedded_python_checks": embedded_count,
+        "shell_syntax_checks": shell_count,
+        "benchmark_executed": False,
+        "standalone_runnable": False,
+        "runtime_validation": "not_performed",
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--archive", type=Path, default=ARCHIVE)
+    args = parser.parse_args()
+    try:
+        report = check_archive(args.archive)
+    except (OSError, ValueError, SyntaxError, subprocess.SubprocessError) as exc:
+        print(json.dumps({"ok": False, "error": str(exc)}))
+        return 1
+    print(json.dumps({"ok": True, **report}, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmark/deepswe-gptxhigh-v1/README.md

@@ -0,0 +1,44 @@
+# DeepSWE Five-Arm Benchmark Harness (v1)
+
+How we evaluate five agent configurations ("arms") on the DeepSWE task set
+(113 SWE tasks). Runner/methodology code only — no API/gateway config, no trajectories.
+Run against LoopX revision `2cef51d` (this branch's base).
+
+## The five arms
+| Arm | Transport | Goal / LoopX | Continuation |
+|---|---|---|---|
+| `plain` | codex app-server | no Goal, no LoopX | single pass |
+| `goal` | codex app-server | native Codex Goal | app-server continues while Goal active |
+| `heartbeat` | `codex exec` (fresh, then `resume`) | LoopX Goal/Todo | recurring supervisor wakes |
+| `codex-cli` | `codex exec` (CLI) | LoopX control plane | external `loopx turn run-once --host codex-cli`, multi-segment |
+| `ssh-goal` | codex app-server | LoopX + native Goal (official full path) | same thread/Goal; LoopX clears blocked + restarts turn |
+
+- Arm dispatch: `pier_cn.py` (`MR_CODEX_ARM=plain|goal|loopx-native|loopx-native-codex-cli|loopx-native-heartbeat`)
+- Arm classes: `goal_codex.py` (`PlainAppServerCodex`, `GoalCodex`, `LoopxCodex`)
+- Runners: `loopx_wen_native_runner.py` (ssh-goal), `loopx_codex_cli_runner.py` (codex-cli),
+  `loopx_heartbeat_supervisor.py` (heartbeat)
+- Delivery gate: `workspace_delivery.py` (recover agent work from linked git worktrees into
+  `/app` so the collected patch is non-empty)
+- Admission: `preflight_loopx_rerun.py` (pins LoopX revision, delivery self-test)
+
+## Validity (strict)
+`exception_info == null`, independent `verifier/reward.json` present & consistent, task
+checksum matches, and a **non-empty committed patch** exists. Goal/Todo state is lifecycle
+evidence only; the independent verifier is the sole correctness authority. `partial > 0`
+alone does NOT count as valid delivery.
+
+## Results — v1 (113 tasks, per-task best valid)
+| Rank | Arm | Solved | Solve rate | Partial | F2P | P2P |
+|---|---|---:|---:|---:|---:|---:|
+| 1 | heartbeat | 70/113 | 61.9% | 0.9739 | 0.886 | 0.993 |
+| 2 | codex-cli | 66/113 | 58.4% | 0.9546 | 0.884 | 0.996 |
+| 3 | goal | 60/113 | 53.1% | 0.9620 | 0.868 | 0.997 |
+| 4 | ssh-goal | 58/113 | 51.3% | 0.9654 | 0.883 | 0.997 |
+| 5 | plain | 54/113 | 47.8% | 0.9206 | 0.745 | 0.997 |
+
+P2P (regression) ≈ 1.0 for all arms; spread is driven by F2P and solve rate.
+LoopX arms (heartbeat/codex-cli) and goal outperform the plain baseline.
+
+> Model & gateway endpoints are configured via `MR_*` env vars (not included).
+> Internal hosts/paths replaced with placeholders (`127.0.0.1`, `<REPO_ROOT>`, `<HOME>`).
+> v2 (latest LoopX main) evaluation is in progress and will be published separately.

benchmark/deepswe-gptxhigh-v1/codex_nosandbox_wrapper.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""A `codex` stand-in that drops LoopX's sandbox flags before running the real one.
+
+LoopX's codex-cli host is the path that works: driven through it, a Turn loop
+ran four times on one task, committed a 22 KB patch and scored f2p 31/35.  Its
+one problem is the sandbox — it always passes `--sandbox <mode>` (or
+`-c sandbox_mode=...` when resuming), only permits read-only and
+workspace-write, and both need bubblewrap, which needs unprivileged user
+namespaces these containers do not have:
+
+    bwrap: No permissions to create a new namespace
+
+Switching to `--host generic-cli` avoided that but bought a worse problem: the
+generic host carries its own scheduler contract, and eleven of sixteen turns
+died at "LoopX Turn route is not host executable" before any model work, with
+no route recorded to explain why.
+
+So keep the working host and fix the flag instead.  This sits earlier on PATH
+than the real codex, strips the sandbox arguments, and substitutes the same
+`--dangerously-bypass-approvals-and-sandbox` the other two arms already use —
+which is also what keeps the three arms identical in permissions.  LoopX's
+contracts are untouched: it still believes it is driving codex-cli, because it
+is.
+
+Set MR_REAL_CODEX to the real binary; defaults to /usr/local/bin/codex.
+MR_LOOPX_CODEX_LOG names the log file; defaults to /tmp/loopx-goal/codex-wrapper.log.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+# Resolve the real binary rather than assuming /usr/local/bin/codex.  Codex is
+# installed into the image through nvm, so it lives under the Node version's
+# bin directory and the hardcoded path does not exist — which made the wrapper
+# die before it ever reached Codex, and LoopX report the indistinguishable
+# `codex_cli_exit_nonzero`.  The wrapper is invoked by absolute path through
+# --codex-bin and is not itself on PATH, so a PATH lookup finds the real one.
+REAL = (
+    os.environ.get("MR_REAL_CODEX")
+    or shutil.which("codex")
+    or "/usr/local/bin/codex"
+)
+BYPASS = "--dangerously-bypass-approvals-and-sandbox"
+REASONING_EFFORT = os.environ.get("MR_CODEX_REASONING_EFFORT", "").strip()
+LOG = Path(
+    os.environ.get("MR_LOOPX_CODEX_LOG", "/tmp/loopx-goal/codex-wrapper.log")
+)
+
+
+def rewrite(argv: list[str]) -> list[str]:
+    out: list[str] = []
+    skip_next = False
+    for i, arg in enumerate(argv):
+        if skip_next:
+            skip_next = False
+            continue
+        # `--sandbox <mode>` — new-session form.
+        if arg == "--sandbox":
+            skip_next = True
+            continue
+        if arg.startswith("--sandbox="):
+            continue
+        # `-c sandbox_mode="..."` — resume form.  The value is a separate argv
+        # item after -c, so both have to go, and only when it is that key: -c
+        # carries every other config override too.
+        if arg == "-c" and i + 1 < len(argv) and argv[i + 1].startswith("sandbox_mode="):
+            skip_next = True
+            continue
+        out.append(arg)
+
+    # Insert the bypass right after the subcommand so it lands before `--`,
+    # which codex treats as the end of flags.
+    if out and out[0] == "exec":
+        out.insert(1, BYPASS)
+        if REASONING_EFFORT and not any(
+            value.startswith("model_reasoning_effort=") for value in out
+        ):
+            out[2:2] = ["-c", f"model_reasoning_effort={REASONING_EFFORT}"]
+    else:
+        out.insert(0, BYPASS)
+    return out
+
+
+def _log(text: str) -> None:
+    try:
+        LOG.parent.mkdir(parents=True, exist_ok=True)
+        with LOG.open("a", encoding="utf-8") as handle:
+            handle.write(text.rstrip("\n") + "\n")
+    except OSError:
+        pass
+
+
+def main() -> int:
+    argv = rewrite(sys.argv[1:])
+    # Run the real codex as a child rather than execv'ing it, so its stderr can
+    # be recorded.  LoopX reports a failed Turn as `codex_cli_exit_nonzero` and
+    # keeps neither the exit code's cause nor any output, and the container is
+    # gone by the time anyone looks — so an execv here means the only evidence
+    # of why Codex refused is destroyed at the moment it is produced.
+    #
+    # stdout stays inherited and untouched: LoopX parses Codex's `--json`
+    # stream off it, so anything written there would corrupt the Turn.
+    _log(f"--- argv in : {sys.argv[1:]}")
+    _log(f"--- argv out: {argv}")
+    _log(f"--- real    : {REAL} (exists={os.path.exists(REAL)})")
+    try:
+        completed = subprocess.run(  # noqa: S603
+            [REAL, *argv], stderr=subprocess.PIPE, check=False
+        )
+    except OSError as exc:
+        # Without this the wrapper's own failure to start Codex is reported by
+        # LoopX as `codex_cli_exit_nonzero`, which reads as "the model refused"
+        # rather than "the binary is not there".
+        _log(f"--- launch failed: {type(exc).__name__}: {exc}")
+        sys.stderr.write(f"codex wrapper could not launch {REAL}: {exc}\n")
+        return 127
+    stderr = completed.stderr.decode("utf-8", "replace") if completed.stderr else ""
+    _log(f"--- exit {completed.returncode}")
+    if stderr:
+        _log(stderr)
+        sys.stderr.write(stderr)
+    return completed.returncode
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())