From 4c42447fb246b91b0b45358f829250d5f242f863 Mon Sep 17 00:00:00 2001 From: shangzh0 <97216392+shangzh0@users.noreply.github.com> Date: Wed, 16 Sep 2026 01:02:36 +0800 Subject: [PATCH 1/2] fix(scheduler): accept terminal packets without cadence detail Signed-off-by: shangzh0 <97216392+shangzh0@users.noreply.github.com> --- scripts/external_scheduler_worker.py | 26 ++++++++++++++++++++----- tests/test_external_scheduler_worker.py | 26 +++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/scripts/external_scheduler_worker.py b/scripts/external_scheduler_worker.py index c282e228d4..afa2282a28 100644 --- a/scripts/external_scheduler_worker.py +++ b/scripts/external_scheduler_worker.py @@ -92,6 +92,26 @@ def _extract_reset_token(payload: dict[str, Any]) -> str: def parse_tick(payload: dict[str, Any]) -> TickDecision: hint = _mapping(payload.get("scheduler_hint")) action = str(hint.get("action") or "").strip() + cadence_class = str(hint.get("cadence_class") or "").strip() + reason = str(hint.get("reason") or payload.get("state") or "").strip() + should_run = bool(payload.get("should_run")) + if action in TERMINAL_ACTIONS: + # Terminal packets intentionally omit cold-path cadence detail: no + # further wake is legal, so an interval cannot affect the decision. + return TickDecision( + should_run=should_run, + action=action, + cadence_class=cadence_class, + reason=reason, + interval_minutes=1, + progression=(1,), + unchanged_limit=None, + after_limit="stop_tick_loop", + final_probe_enabled=False, + final_probe_action="", + reset_token=_extract_reset_token(payload), + terminal=True, + ) local = _extract_local_scheduler(payload) progression_values = local.get("example_progression_minutes") @@ -125,10 +145,6 @@ def parse_tick(payload: dict[str, Any]) -> TickDecision: "action rerun_quota_should_run_once" ) - cadence_class = str(hint.get("cadence_class") or "").strip() - reason = str(hint.get("reason") or payload.get("state") or "").strip() - should_run = bool(payload.get("should_run")) - return TickDecision( should_run=should_run, action=action, @@ -141,7 +157,7 @@ def parse_tick(payload: dict[str, Any]) -> TickDecision: final_probe_enabled=bool(final_probe.get("enabled")), final_probe_action=str(final_probe.get("action") or "").strip(), reset_token=_extract_reset_token(payload), - terminal=action in TERMINAL_ACTIONS, + terminal=False, ) diff --git a/tests/test_external_scheduler_worker.py b/tests/test_external_scheduler_worker.py index e885ea568a..7b4766d64d 100644 --- a/tests/test_external_scheduler_worker.py +++ b/tests/test_external_scheduler_worker.py @@ -194,6 +194,32 @@ def test_quota_probe_timeout_enters_tick_error(tmp_path: Path) -> None: assert time.monotonic() - started < 2.0 +def test_terminal_packet_does_not_require_cold_path_scheduler_detail( + tmp_path: Path, +) -> None: + root = tmp_path / "terminal-without-cold-detail" + fake_cli = root / "fake-loopx" + payload = _hint_payload(should_run=False) + del payload["scheduler_hint"]["cold_path_detail"] # type: ignore[index] + _write_executable( + fake_cli, + "#!/usr/bin/env python3\n" + f"print({json.dumps(payload)!r})\n", + ) + + result = worker.run_worker( + _args( + fake_cli=fake_cli, + state_file=root / "state.json", + quota_timeout_seconds=1.0, + ) + ) + + assert result == 0 + state = json.loads((root / "state.json").read_text(encoding="utf-8")) + assert state == {"reset_token": "fixture-token", "unchanged_count": 0} + + @pytest.mark.skipif(os.name != "posix", reason="POSIX process-group regression") def test_wake_timeout_enters_failed_backoff_and_kills_descendants( tmp_path: Path, From f33a6f4c5b129886bf99bd2f48b74c09daaab126 Mon Sep 17 00:00:00 2001 From: shangzh0 <97216392+shangzh0@users.noreply.github.com> Date: Wed, 16 Sep 2026 01:03:29 +0800 Subject: [PATCH 2/2] feat(benchmark): add LHTB LoopX heartbeat runner Signed-off-by: shangzh0 <97216392+shangzh0@users.noreply.github.com> --- benchmark/LHTB/.env.example | 28 + benchmark/LHTB/.gitignore | 9 + benchmark/LHTB/README.md | 206 +++++++ .../LHTB/agents/codex_loopx_heartbeat.py | 579 ++++++++++++++++++ benchmark/LHTB/agents/codex_offline.py | 122 ++++ .../LHTB/configs/heartbeat-generic-cli.yaml | 76 +++ .../docker-compose-modelonly-plus.yaml | 9 + .../docker-compose-modelonly.yaml | 8 + .../harbor_patch/prepare_harbor_modelonly.py | 131 ++++ benchmark/LHTB/reports/.gitkeep | 0 benchmark/LHTB/run.sh | 213 +++++++ benchmark/LHTB/runs/.gitkeep | 0 benchmark/LHTB/runtime/wake_once.py | 289 +++++++++ .../LHTB/scripts/build_verifier_images.sh | 39 ++ benchmark/LHTB/scripts/preflight.py | 239 ++++++++ benchmark/LHTB/scripts/render_config.py | 62 ++ benchmark/LHTB/scripts/summarize_results.py | 51 ++ .../LHTB/verifier-images/langchain/Dockerfile | 16 + .../LHTB/verifier-images/nbody/Dockerfile | 10 + 19 files changed, 2087 insertions(+) create mode 100644 benchmark/LHTB/.env.example create mode 100644 benchmark/LHTB/.gitignore create mode 100644 benchmark/LHTB/README.md create mode 100644 benchmark/LHTB/agents/codex_loopx_heartbeat.py create mode 100644 benchmark/LHTB/agents/codex_offline.py create mode 100644 benchmark/LHTB/configs/heartbeat-generic-cli.yaml create mode 100644 benchmark/LHTB/harbor_patch/docker-compose-modelonly-plus.yaml create mode 100644 benchmark/LHTB/harbor_patch/docker-compose-modelonly.yaml create mode 100755 benchmark/LHTB/harbor_patch/prepare_harbor_modelonly.py create mode 100644 benchmark/LHTB/reports/.gitkeep create mode 100755 benchmark/LHTB/run.sh create mode 100644 benchmark/LHTB/runs/.gitkeep create mode 100755 benchmark/LHTB/runtime/wake_once.py create mode 100755 benchmark/LHTB/scripts/build_verifier_images.sh create mode 100755 benchmark/LHTB/scripts/preflight.py create mode 100755 benchmark/LHTB/scripts/render_config.py create mode 100755 benchmark/LHTB/scripts/summarize_results.py create mode 100644 benchmark/LHTB/verifier-images/langchain/Dockerfile create mode 100644 benchmark/LHTB/verifier-images/nbody/Dockerfile diff --git a/benchmark/LHTB/.env.example b/benchmark/LHTB/.env.example new file mode 100644 index 0000000000..75ddb9c516 --- /dev/null +++ b/benchmark/LHTB/.env.example @@ -0,0 +1,28 @@ +# Copy to .env and set paths for your local checkouts and runtimes. +LHTB_ROOT=/absolute/path/to/LHTB +OPENAI_BASE_URL=http://192.0.2.1:8000/v1 +OPENAI_API_KEY=gateway-injects-the-real-key +CODEX_WIRE_API=responses + +MODEL_NAME=openai/gpt-5.6-sol +REASONING_EFFORT=max +CONCURRENCY=4 +AGENT_TIMEOUT_SEC=5400 + +# The scheduler must finish before Harbor's outer agent timeout. +LOOPX_SCHEDULER_TIMEOUT_SEC=5080 +LOOPX_WAKE_TIMEOUT_SEC=4800 +LOOPX_CODEX_TURN_TIMEOUT_SEC=4700 + +CODEX_BIN=/absolute/path/to/vendor/x86_64-unknown-linux-musl/bin/codex +# LOOPX_SRC_DIR defaults to this LoopX checkout. Pin the commit optionally. +# LOOPX_SRC_DIR=/absolute/path/to/loopx +# LOOPX_EXPECTED_COMMIT= +LOOPX_PORTABLE_PYTHON=/absolute/path/to/python-3.12-portable-root +LOOPX_NODE_DIR=/absolute/path/to/node-v22-root + +LHTB_MODELONLY_NETWORK=lhtb-modelonly +LHTB_MODELONLY_SUBNET=192.0.2.0/24 +LHTB_MODELONLY_GATEWAY=192.0.2.1 +LHTB_MAX_RETRIES=2 +RUNNER_RESTARTS=2 diff --git a/benchmark/LHTB/.gitignore b/benchmark/LHTB/.gitignore new file mode 100644 index 0000000000..d1be39fdd8 --- /dev/null +++ b/benchmark/LHTB/.gitignore @@ -0,0 +1,9 @@ +.env +.generated/* +!.generated/.gitkeep +runs/* +!runs/.gitkeep +reports/* +!reports/.gitkeep +__pycache__/ +*.pyc diff --git a/benchmark/LHTB/README.md b/benchmark/LHTB/README.md new file mode 100644 index 0000000000..f13b3f4a97 --- /dev/null +++ b/benchmark/LHTB/README.md @@ -0,0 +1,206 @@ +# LHTB: LoopX generic_cli heartbeat + Codex exec + +This directory defines one LHTB treatment arm: + +```text +LoopX source from the current checkout (optionally pinned with LOOPX_EXPECTED_COMMIT) ++ runtime profile generic_cli ++ current LoopX external_scheduler_worker.py ++ one fresh codex exec per heartbeat (never exec resume) ++ openai/gpt-5.6-sol, reasoning=max ++ execution_profile.replan_after_completed_todos=3 +``` + +This runner includes one narrow scheduler compatibility fix in +`scripts/external_scheduler_worker.py`: terminal scheduler packets may omit +`cold_path_detail.local_scheduler`, so the worker must recognize terminal state +before parsing a cadence that will never be used. A focused upstream test in +`tests/test_external_scheduler_worker.py` covers this behavior. + +The runner keeps the LHTB task manifests authoritative for internet and +verifier policy. It adds an internal model-only network for offline task +containers and stages Codex and LoopX without downloading them in the task. +It does not use the app-server heartbeat agent. + +## Run + +Prerequisites are an LHTB checkout with its Harbor virtual environment, Docker, +the native Codex bundle, a portable Python 3.11+ tree, and Node 22.6+. Copy +`.env.example` to `.env`, set `LHTB_ROOT`, the model gateway, and local runtime +paths, then run from this directory. + +The example uses the RFC 5737 documentation range for the configurable +`lhtb-modelonly` bridge. Bind or proxy an OpenAI-compatible endpoint on the +configured bridge gateway, or change `OPENAI_BASE_URL`, +`LHTB_MODELONLY_SUBNET`, and +`LHTB_MODELONLY_GATEWAY` together. The internal bridge is deliberately not a +general internet route. + +```bash +cd loopx/benchmark/LHTB + +# No model call. Validate all contracts and render a 46-task Harbor config. +./run.sh preflight + +# One quick real trial before any full run. +./run.sh smoke tabular-data-feature-covshift + +# Full 46-task arm, default concurrency 4. +./run.sh full +``` + +Override concurrency without editing YAML: + +```bash +CONCURRENCY=10 ./run.sh full +``` + +The full run is intentionally not launched by setup or preflight. + +## Execution flow + +For every Harbor trial, the agent performs the following sequence inside that +task's container: + +```text +stage native Codex + current LoopX source/profile + -> expose staged Node 22 through BASH_ENV for Codex login shells + -> create trial-local registry/runtime/task document + -> bootstrap one trial-local Goal + -> Harbor adapter marks the connection provider-prevalidated + -> skip the generic repo-intake onboarding Todo + -> register lhtb-codex-heartbeat + -> configure replan_after_completed_todos=3 and read it back + -> add the current Harbor phase as a claimed advancement Todo + -> start LoopX external_scheduler_worker.py + -> quota should-run --runtime-profile generic_cli + -> when allowed, invoke wake_once.py + -> create a unique TURN_ID + -> heartbeat-prompt --thin --runtime-profile generic_cli + -> require ok=true and a non-empty current task_body + -> create a new CODEX_HOME + -> codex exec --json, with task_body on stdin + -> save JSONL/receipt and any emitted session files + -> delete the temporary CODEX_HOME + -> obey LoopX local_scheduler wait/stop hints + -> Harbor runs interim/final verifier and owns trial termination +``` + +There is no `codex exec resume` path. A new Codex model conversation starts on +each wake. Only the task workspace and the trial-local LoopX registry/runtime +persist across wakes. Each wake's `invocation.json` records +`fresh_codex_exec=true`, `resume=false`, and its unique Turn ID. + +Codex tool commands run through a login shell, which can replace the inherited +`PATH`. The adapter supplies a trial-local `BASH_ENV` that prepends the staged +Node 22 directory, so later `loopx refresh-state` and settlement commands use +the same qualified Effect runtime without replacing files in the task image. + +The exact task is not replaced by one shared objective. The objective is a +short control-plane statement; Harbor's current instruction is written to the +trial-local goal document, and the selected P0 Todo points the model to that +document. Registries are inside their own task containers, so no Goal, Todo, +or scheduler state is shared between the 46 trials. + +The Harbor adapter has already validated the project bridge and writes the +benchmark phase as an explicit P0 Todo. Bootstrap therefore uses +`--no-onboarding-scan --onboarding-connection-validation +provider-prevalidated`. This suppresses the unrelated generic repo-intake Todo; +it does not suppress successor Todos created while solving the LHTB task. + +## Replan cadence + +The runner applies and reads back: + +```bash +loopx configure-goal \ + --goal-id lhtb-heartbeat-goal \ + --execution-replan-after-todos 3 \ + --execute +``` + +This does not roll back the third Todo and does not force a plan change. After +three qualifying advancement Todos completed by this agent without a covering +outcome checkpoint, the next quota/frontier evaluation opens a review/replan +obligation. The review may keep a correct plan (`continue`/`no_change`) or +change it. Open Todos, protocol steps, work by another agent, and unqualified +refreshes do not increment or reset this counter. + +## LHTB and fairness settings + +- 46 tasks, one attempt each. +- 22 task manifests are offline and 24 allow internet. +- Offline task containers attach only to the internal `lhtb-modelonly` network, + which reaches the model gateway but has no public route. +- Online tasks retain the task-defined public network and also receive the + model-only interface. +- The task manifests remain authoritative for verifier mode: 44 shared tasks; + only `langchain-version-migration` and `nbody-accel-iterative` are separate. +- Harbor remains responsible for interim/final verifier calls, total timeout, + score collection, artifacts, and `result.json`. +- Verifier feedback is binary (`HB_VERIFIER_FEEDBACK_MODE=binary`). +- Codex native Goal is disabled so the measured continuation owner is LoopX. +- Server-side Codex Web Search is disabled for every task. +- Container isolation is the command sandbox; Codex is launched with the same + external-container trust assumption as the existing LHTB Codex runner. + +The shared verifier mode is intentionally the original LHTB policy. This +runner does not claim stronger hidden-test isolation than the underlying +Harbor/LHTB checkout provides. + +## Timeouts + +Defaults leave cleanup room between nested layers: + +| Layer | Default | +| --- | ---: | +| Harbor agent limit | 5400 s | +| LoopX scheduler process | 5080 s | +| One wake command | 4800 s | +| One Codex exec | 4700 s | + +The scheduler can perform multiple shorter wakes within 5080 seconds. A single +long Codex wake can consume most of that budget, which is expected; Harbor's +outer timeout and verifier still determine the trial outcome. + +## Outputs + +Each run writes to `runs//`, with the generated YAML under +`.generated/` and launcher logs/environment receipts under `reports/`. +Per-trial agent logs include: + +```text +agent/ + loopx-install.json + loopx-worker-phase-*.log + trajectory.json + wakes// + heartbeat.json + task-body.md + invocation.json + codex-events.jsonl + codex-stderr.log + receipt.json + sessions/.../*.jsonl # present only when this Codex build emits them +``` + +`trajectory.json` aggregates every fresh Codex wake's JSONL event stream into +one ATIF trajectory. Raw per-wake files remain available for auditing the +LoopX decision, exact task body, model invocation, token usage, and settlement. +If Codex also writes local session files under the temporary `CODEX_HOME`, the +runner copies them before deleting that home; `codex-events.jsonl` remains the +authoritative captured stream when it does not. + +## Implementation map + +- `run.sh`: environment, LHTB networking, preflight, Harbor launch. +- `configs/heartbeat-generic-cli.yaml`: immutable 46-task template. +- `agents/codex_loopx_heartbeat.py`: Harbor lifecycle and LoopX Goal setup. +- `runtime/wake_once.py`: unique Turn, thin heartbeat body, fresh Codex exec. +- `scripts/preflight.py`: fail-closed parity and safety checks. +- `harbor_patch/`: opt-in model-only Docker networking patch. +- `verifier-images/`: the two task-declared separate verifier images. + +Official Codex CLI documentation describes `codex exec` as the scripted/CI +entry point and `--json` as newline-delimited JSON events: +. diff --git a/benchmark/LHTB/agents/codex_loopx_heartbeat.py b/benchmark/LHTB/agents/codex_loopx_heartbeat.py new file mode 100644 index 0000000000..b0c1b9b950 --- /dev/null +++ b/benchmark/LHTB/agents/codex_loopx_heartbeat.py @@ -0,0 +1,579 @@ +"""Harbor agent for LoopX generic_cli heartbeat + fresh Codex exec wakes.""" + +from __future__ import annotations + +import json +import os +import re +import shlex +import subprocess +import tempfile +from pathlib import Path +from typing import Iterable + +from harbor.agents.installed.base import with_prompt_template +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext +from harbor.models.trajectories import FinalMetrics, Trajectory +from harbor.utils.trajectory_utils import format_trajectory_json + +from codex_offline import CodexOffline + + +_ROOT = "/opt/loopx-lhtb" +_SRC = f"{_ROOT}/source" +_PYTHON = f"{_ROOT}/python" +_NODE = f"{_ROOT}/node" +_PROFILE = f"{_ROOT}/profile" +_PROFILE_HOME = f"{_PROFILE}/home" +_SHARED_CODEX_HOME = f"{_PROFILE}/codex-home" +_SHARED_SKILLS = f"{_SHARED_CODEX_HOME}/skills" +_CLI = f"{_PROFILE}/bin/loopx" +_CONTROL = f"{_ROOT}/control" +_REGISTRY = f"{_CONTROL}/registry.json" +_LOOPX_RUNTIME = f"{_ROOT}/state/runtime" +_SCHEDULER_STATE = f"{_CONTROL}/scheduler-state.json" +_TASK_DOC = f"{_CONTROL}/task.md" +_BASH_ENV = f"{_CONTROL}/bash-env" +_TURN_ROOT = f"{_ROOT}/turns" +_WAKE_SCRIPT = f"{_ROOT}/runtime/wake_once.py" +_WAKE_LOG_DIR = "/logs/agent/wakes" +_GOAL_ID = "lhtb-heartbeat-goal" +_AGENT_ID = "lhtb-codex-heartbeat" +_REPLAN_AFTER_TODOS = 3 + + +class LoopxHeartbeatCodex(CodexOffline): + """One independent LoopX control plane per Harbor trial.""" + + _phase_number = 0 + + @staticmethod + def name() -> str: + return "loopx-generic-cli-heartbeat-codex" + + def _container_id(self, environment: BaseEnvironment) -> str: + from harbor.environments.docker.docker import ( + _sanitize_docker_compose_project_name, + ) + + project = _sanitize_docker_compose_project_name(environment.session_id) + completed = subprocess.run( + [ + "docker", "ps", "-q", + "--filter", f"label=com.docker.compose.project={project}", + "--filter", "label=com.docker.compose.service=main", + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=60, + check=False, + ) + ids = completed.stdout.split() + if len(ids) != 1: + raise RuntimeError( + f"expected one main container for compose project {project}, got {ids}" + ) + return ids[0] + + @staticmethod + def _copy_tree(container_id: str, source: Path, destination: str) -> None: + if not source.is_dir(): + raise FileNotFoundError(f"required directory is missing: {source}") + completed = subprocess.run( + ["docker", "cp", f"{source}/.", f"{container_id}:{destination}"], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=1200, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + f"docker cp {source} failed: {(completed.stderr or completed.stdout)[-500:]}" + ) + + @staticmethod + def _copy_git_snapshot(container_id: str, source: Path, destination: str) -> None: + """Stage only files tracked by the pinned LoopX commit. + + The LoopX checkout also hosts benchmark runs. Copying the working tree + would expose prior trajectories and artifacts inside the agent container. + """ + archive = subprocess.Popen( + ["git", "-C", str(source), "archive", "--format=tar", "HEAD"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if archive.stdout is None: + archive.kill() + raise RuntimeError("could not open LoopX git archive stream") + extract = subprocess.Popen( + ["docker", "exec", "-i", container_id, "tar", "-x", "-C", destination], + stdin=archive.stdout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=False, + ) + archive.stdout.close() + try: + extract_stdout, extract_stderr = extract.communicate(timeout=1200) + except subprocess.TimeoutExpired: + extract.kill() + archive.kill() + extract.communicate() + archive.communicate() + raise RuntimeError("timed out staging the LoopX git snapshot") + archive_stderr = archive.stderr.read() if archive.stderr is not None else b"" + archive_returncode = archive.wait(timeout=30) + if archive_returncode != 0 or extract.returncode != 0: + detail = archive_stderr or extract_stderr or extract_stdout + raise RuntimeError( + "failed to stage clean LoopX git snapshot: " + + detail.decode("utf-8", errors="replace")[-500:] + ) + + def _profile_env(self) -> dict[str, str]: + return { + "HOME": _PROFILE_HOME, + "CODEX_HOME": _SHARED_CODEX_HOME, + "PATH": f"{_NODE}/bin:{_PROFILE}/bin:/usr/local/bin:/usr/bin:/bin", + "LOOPX_PYTHON": f"{_PYTHON}/bin/python3", + "LOOPX_PROMOTE_DEFAULT": "1", + "LOOPX_INSTALL_CANARY": "0", + "LOOPX_BIN_DIR": f"{_PROFILE}/bin", + "LOOPX_RELEASES_DIR": f"{_PROFILE}/releases", + "LOOPX_RELEASE_ID": "lhtb-generic-cli-heartbeat", + "LOOPX_MAN_ROOT": f"{_PROFILE}/man", + "LOOPX_MAN_DIR": f"{_PROFILE}/man/man1", + "LOOPX_SHELL_PROFILE": f"{_PROFILE_HOME}/.profile", + "LOOPX_SKILLS_DIR": _SHARED_SKILLS, + "LOOPX_INSTALL_SLASH_COMMANDS": "0", + "LOOPX_INSTALL_OPENCODE": "0", + "LOOPX_INSTALL_CLAUDE": "0", + "LOOPX_SKILL_DEDUPE_OTHER_ROOT": "0", + # Codex tool calls use `bash -lc`, whose login profile may replace + # PATH. BASH_ENV restores the staged Node for LoopX subprocesses. + "BASH_ENV": _BASH_ENV, + } + + async def install(self, environment: BaseEnvironment) -> None: + await super().install(environment) + + loopx_src = Path(os.environ["LOOPX_SRC_DIR"]).resolve() + portable_python = Path(os.environ["LOOPX_PORTABLE_PYTHON"]).resolve() + node_root = Path(os.environ["LOOPX_NODE_DIR"]).resolve() + wake_source = Path(__file__).resolve().parent.parent / "runtime" / "wake_once.py" + expected_commit = os.environ.get("LOOPX_EXPECTED_COMMIT", "").strip() + + actual_commit = subprocess.run( + ["git", "-C", str(loopx_src), "rev-parse", "HEAD"], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=30, + check=False, + ).stdout.strip() + if expected_commit and actual_commit != expected_commit: + raise RuntimeError( + f"LoopX commit mismatch: expected {expected_commit}, got {actual_commit}" + ) + + await self.exec_as_root( + environment, + command=( + f"mkdir -p {_SRC} {_PYTHON} {_NODE} {_PROFILE_HOME} " + f"{_SHARED_CODEX_HOME} {_PROFILE}/bin {_PROFILE}/releases {_PROFILE}/man " + f"{_CONTROL} " + f"{_LOOPX_RUNTIME} {_TURN_ROOT} {os.path.dirname(_WAKE_SCRIPT)} " + f"{_WAKE_LOG_DIR}; chmod -R 0777 {_ROOT} {_WAKE_LOG_DIR}" + ), + timeout_sec=180, + ) + container_id = self._container_id(environment) + self._copy_git_snapshot(container_id, loopx_src, _SRC) + self._copy_tree(container_id, portable_python, _PYTHON) + self._copy_tree(container_id, node_root, _NODE) + await environment.upload_file(wake_source, _WAKE_SCRIPT) + await self.exec_as_root( + environment, + command=( + f"chmod 0755 {_WAKE_SCRIPT}; " + f"printf '%s\\n' 'export PATH={_NODE}/bin:$PATH' > {_BASH_ENV}; " + f"chmod 0644 {_BASH_ENV}; " + f"find {_SRC} -maxdepth 2 \\( -name '*.egg-info' -o " + f"-name '*.dist-info' \\) -exec rm -rf {{}} +; " + f"chmod -R a+rX {_SRC} {_PYTHON} {_NODE}; " + f"chmod -R a+rwX {_PROFILE} {_CONTROL} {_TURN_ROOT} {_WAKE_LOG_DIR}" + ), + timeout_sec=300, + ) + install = await self.exec_as_agent( + environment, + command=f"bash {_SRC}/scripts/install-local.sh", + env=self._profile_env(), + timeout_sec=1200, + ) + if "error" in (install.stderr or "").lower(): + self.logger.debug("LoopX installer stderr: %s", install.stderr[-1000:]) + + doctor = await self.exec_as_agent( + environment, + command=f"{_CLI} --format json doctor --agent-type codex-cli", + env=self._profile_env(), + timeout_sec=300, + ) + try: + doctor_payload = json.loads(doctor.stdout or "") + except json.JSONDecodeError as exc: + raise RuntimeError("LoopX doctor returned invalid JSON") from exc + if doctor_payload.get("ok") is not True: + raise RuntimeError(f"LoopX doctor failed: {doctor_payload}") + + receipt = { + "loopx_commit": actual_commit, + "runtime_profile": "generic_cli", + "codex_driver": "fresh_exec_per_wake", + "onboarding_connection_validation": "provider-prevalidated", + "login_shell_node_path": _BASH_ENV, + "scheduler_terminal_packet_compatibility": True, + "replan_after_completed_todos": _REPLAN_AFTER_TODOS, + } + await self.exec_as_agent( + environment, + command=( + f"printf %s {shlex.quote(json.dumps(receipt, sort_keys=True))} " + f"> /logs/agent/loopx-install.json" + ), + env=self._profile_env(), + ) + + async def _write_task_document( + self, environment: BaseEnvironment, instruction: str + ) -> None: + descriptor, name = tempfile.mkstemp(prefix="lhtb-loopx-task-", suffix=".md") + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write("# Current LHTB task\n\n") + handle.write(instruction.strip()) + handle.write("\n") + await environment.upload_file(Path(name), _TASK_DOC) + await self.exec_as_root( + environment, + command=f"chmod 0644 {_TASK_DOC}", + ) + finally: + Path(name).unlink(missing_ok=True) + + async def _loopx( + self, + environment: BaseEnvironment, + args: list[str], + *, + cwd: str, + require_ok: bool = True, + ) -> dict: + argv = [ + _CLI, + "--format", "json", + "--registry", _REGISTRY, + "--runtime-root", _LOOPX_RUNTIME, + *args, + ] + result = await self.exec_as_agent( + environment, + command=shlex.join(argv), + env=self._profile_env(), + cwd=cwd, + timeout_sec=300, + ) + text = (result.stdout or "").strip() + if text.startswith("```"): + text = text.split("\n", 1)[-1].rsplit("```", 1)[0].strip() + try: + payload = json.loads(text) + except json.JSONDecodeError as exc: + raise RuntimeError(f"LoopX command returned invalid JSON: {text[:300]}") from exc + if require_ok and payload.get("ok") is False: + raise RuntimeError(f"LoopX command failed: {payload.get('error')}") + return payload + + async def _registry_exists(self, environment: BaseEnvironment) -> bool: + result = await environment.exec(command=f"test -s {_REGISTRY}") + return result.return_code == 0 + + async def _prepare_phase( + self, environment: BaseEnvironment, instruction: str, *, cwd: str + ) -> None: + await self._write_task_document(environment, instruction) + if not await self._registry_exists(environment): + await self._loopx( + environment, + [ + "bootstrap", + "--project", ".", + "--goal-id", _GOAL_ID, + "--objective", + "Complete the current LHTB task through validated LoopX Todos.", + "--goal-doc", _TASK_DOC, + "--adapter-kind", "read_only_project_map_v0", + "--adapter-status", "connected-read-only", + "--write-scope", "**", + "--no-onboarding-scan", + "--onboarding-connection-validation", "provider-prevalidated", + "--begin-autonomous-advance", + "--codex-app-heartbeat", "no", + "--no-global-sync", + ], + cwd=cwd, + ) + await self._loopx( + environment, + [ + "configure-goal", + "--goal-id", _GOAL_ID, + "--registered-agent", _AGENT_ID, + "--execution-replan-after-todos", str(_REPLAN_AFTER_TODOS), + "--agent-work-mode", f"{_AGENT_ID}=active", + "--execute", + ], + cwd=cwd, + ) + else: + await self._loopx( + environment, + [ + "configure-goal", + "--goal-id", _GOAL_ID, + "--execution-replan-after-todos", str(_REPLAN_AFTER_TODOS), + "--clear-waiting-on", + "--agent-work-mode", f"{_AGENT_ID}=active", + "--execute", + ], + cwd=cwd, + ) + + todo_id = f"lhtb-task-phase-{self._phase_number:03d}" + await self._loopx( + environment, + [ + "todo", "add", + "--goal-id", _GOAL_ID, + "--role", "agent", + "--todo-id", todo_id, + "--text", + ( + f"[P0] Execute benchmark phase {self._phase_number}. Read the exact " + f"current task from {_TASK_DOC}; inspect the workspace, implement and " + "validate it, and create bounded successor Todos for remaining work." + ), + "--task-class", "advancement_task", + "--action-kind", "lhtb_benchmark_task", + "--claimed-by", _AGENT_ID, + "--status", "open", + "--execute", + ], + cwd=cwd, + ) + + cadence = await self._loopx( + environment, + ["configure-goal", "--goal-id", _GOAL_ID], + cwd=cwd, + ) + configured_state = cadence.get("after") or cadence.get("before") or {} + configured = configured_state.get("execution_profile", {}).get( + "replan_after_completed_todos" + ) + if configured != _REPLAN_AFTER_TODOS: + raise RuntimeError( + f"replan cadence readback mismatch: expected 3, got {configured!r}" + ) + + def _worker_env(self, *, cwd: str) -> dict[str, str]: + return { + **self._profile_env(), + "LOOPX_CLI": _CLI, + "LOOPX_REGISTRY": _REGISTRY, + "LOOPX_RUNTIME_ROOT": _LOOPX_RUNTIME, + "LOOPX_GOAL_ID": _GOAL_ID, + "LOOPX_AGENT_ID": _AGENT_ID, + "LOOPX_PROJECT": cwd, + "LOOPX_WAKE_LOG_DIR": _WAKE_LOG_DIR, + "LOOPX_TURN_ROOT": _TURN_ROOT, + "LOOPX_SHARED_SKILLS": _SHARED_SKILLS, + "LOOPX_CODEX_TURN_TIMEOUT_SEC": os.environ.get( + "LOOPX_CODEX_TURN_TIMEOUT_SEC", "4700" + ), + "CODEX_BIN": "/usr/local/bin/codex", + "MODEL_NAME": self.model_name or "", + "REASONING_EFFORT": str( + self._resolved_flags.get("reasoning_effort", "max") + ), + "OPENAI_BASE_URL": self._get_env("OPENAI_BASE_URL") or "", + "OPENAI_API_KEY": self._get_env("OPENAI_API_KEY") or "", + "CODEX_WIRE_API": self._get_env("CODEX_WIRE_API") or "responses", + } + + def _session_trajectories(self, roots: Iterable[Path]) -> list[Trajectory]: + parents: set[Path] = set() + for root in roots: + if root.is_dir(): + parents.update(path.parent for path in root.glob("sessions/**/*.jsonl")) + trajectories: list[Trajectory] = [] + for parent in sorted(parents): + try: + trajectory = self._convert_events_to_trajectory(parent) + except Exception: + self.logger.exception("failed to parse Codex session under %s", parent) + continue + if trajectory is not None: + trajectories.append(trajectory) + return trajectories + + @staticmethod + def _totals(trajectories: Iterable[Trajectory]) -> dict[str, int | float | None]: + prompt = completion = cached = 0 + costs: list[float] = [] + for trajectory in trajectories: + metrics = trajectory.final_metrics + if metrics is None: + continue + prompt += metrics.total_prompt_tokens or 0 + completion += metrics.total_completion_tokens or 0 + cached += metrics.total_cached_tokens or 0 + if metrics.total_cost_usd is not None: + costs.append(metrics.total_cost_usd) + return { + "prompt": prompt, + "completion": completion, + "cached": cached, + "cost": sum(costs) if costs else None, + } + + def _write_aggregate_trajectory(self) -> list[Trajectory]: + wake_root = self.logs_dir / "wakes" + wake_dirs = sorted(path for path in wake_root.iterdir() if path.is_dir()) if wake_root.is_dir() else [] + trajectories = self._session_trajectories(wake_dirs) + if not trajectories: + return [] + steps = [] + for trajectory in trajectories: + for step in trajectory.steps: + copied = step.model_copy(deep=True) + copied.step_id = len(steps) + 1 + steps.append(copied) + totals = self._totals(trajectories) + aggregate = Trajectory( + schema_version="ATIF-v1.5", + session_id=f"loopx-heartbeat-{self.logs_dir.parent.name}", + agent=trajectories[0].agent, + steps=steps, + final_metrics=FinalMetrics( + total_prompt_tokens=totals["prompt"] or None, + total_completion_tokens=totals["completion"] or None, + total_cached_tokens=totals["cached"] or None, + total_cost_usd=totals["cost"], + total_steps=len(steps), + extra={"heartbeat_wakes": len(trajectories)}, + ), + ) + (self.logs_dir / "trajectory.json").write_text( + format_trajectory_json(aggregate.to_json_dict()), encoding="utf-8" + ) + return trajectories + + def _populate_context(self, context: AgentContext, wake_dirs: list[Path]) -> None: + phase_trajectories = self._session_trajectories(wake_dirs) + totals = self._totals(phase_trajectories) + context.n_input_tokens = int(totals["prompt"] or 0) + context.n_output_tokens = int(totals["completion"] or 0) + context.n_cache_tokens = int(totals["cached"] or 0) + context.cost_usd = totals["cost"] + context.metadata = { + "loopx_runtime_profile": "generic_cli", + "codex_session_policy": "fresh_exec_per_wake", + "codex_resume_used": False, + "replan_after_completed_todos": _REPLAN_AFTER_TODOS, + "onboarding_connection_validation": "provider-prevalidated", + "login_shell_node_path": _BASH_ENV, + "scheduler_terminal_packet_compatibility": True, + "heartbeat_wakes": len(wake_dirs), + "benchmark_phase": self._phase_number, + } + self._write_aggregate_trajectory() + + def populate_context_post_run(self, context: AgentContext) -> None: + wake_root = self.logs_dir / "wakes" + wake_dirs = sorted(path for path in wake_root.iterdir() if path.is_dir()) if wake_root.is_dir() else [] + self._populate_context(context, wake_dirs) + + @with_prompt_template + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + if not self.model_name: + raise ValueError("model_name is required") + self._phase_number += 1 + pwd = await self.exec_as_agent(environment, command="pwd", timeout_sec=30) + cwd = (pwd.stdout or "").strip() + if not cwd.startswith("/"): + raise RuntimeError(f"could not resolve container working directory: {cwd!r}") + + wake_root = self.logs_dir / "wakes" + before = {path.name for path in wake_root.iterdir() if path.is_dir()} if wake_root.is_dir() else set() + try: + await self._prepare_phase(environment, instruction, cwd=cwd) + wake_command = shlex.join([f"{_PYTHON}/bin/python3", _WAKE_SCRIPT]) + worker_argv = [ + f"{_PYTHON}/bin/python3", + f"{_SRC}/scripts/external_scheduler_worker.py", + "--cli-bin", _CLI, + "--registry", _REGISTRY, + "--runtime-root", _LOOPX_RUNTIME, + "--runtime-profile", "generic_cli", + "--goal-id", _GOAL_ID, + "--agent-id", _AGENT_ID, + "--state-file", _SCHEDULER_STATE, + "--wake-cmd", wake_command, + "--wake-timeout-seconds", os.environ.get( + "LOOPX_WAKE_TIMEOUT_SEC", "4800" + ), + "--quota-timeout-seconds", "30", + "--error-backoff-seconds", "15", + ] + scheduler_timeout = int(os.environ.get("LOOPX_SCHEDULER_TIMEOUT_SEC", "5080")) + phase_log = f"/logs/agent/loopx-worker-phase-{self._phase_number:03d}.log" + shell = ( + "set +e; " + f"timeout --signal=TERM --kill-after=15 {scheduler_timeout}s " + f"{shlex.join(worker_argv)} >> {shlex.quote(phase_log)} 2>&1; " + "rc=$?; set -e; " + f"if [ \"$rc\" -eq 124 ]; then echo scheduler_timeout >> {shlex.quote(phase_log)}; exit 0; fi; " + "exit \"$rc\"" + ) + await self.exec_as_agent( + environment, + command=shell, + env=self._worker_env(cwd=cwd), + cwd=cwd, + timeout_sec=scheduler_timeout + 60, + ) + finally: + after_dirs = ( + sorted(path for path in wake_root.iterdir() if path.is_dir() and path.name not in before) + if wake_root.is_dir() + else [] + ) + self._populate_context(context, after_dirs) + + +def safe_trial_slug(value: str) -> str: + """Retained for receipts and tests that need a public-safe trial label.""" + + normalized = re.sub(r"[^A-Za-z0-9._-]", "-", value).strip("-._") + return normalized or "lhtb-trial" diff --git a/benchmark/LHTB/agents/codex_offline.py b/benchmark/LHTB/agents/codex_offline.py new file mode 100644 index 0000000000..1aa2ba2267 --- /dev/null +++ b/benchmark/LHTB/agents/codex_offline.py @@ -0,0 +1,122 @@ +"""Harbor Codex adapter that stages native binaries without container egress. + +LHTB contains offline task containers, so installing Codex from npm inside a +trial would violate the task network policy. This adapter copies the native +Codex bundle from ``CODEX_OFFLINE_DIR`` and otherwise retains Harbor's Codex +configuration and trajectory handling. +""" + +from __future__ import annotations + +import os +import tempfile +from pathlib import Path + +from harbor.agents.installed.base import CliFlag +from harbor.agents.installed.codex import Codex +from harbor.environments.base import BaseEnvironment + + +_DEFAULT_OFFLINE_DIR = str(Path(__file__).resolve().parent.parent / "codex") +_STAGE_DIR = os.path.join(tempfile.gettempdir(), "codex-offline") + +# These settings tolerate transient gateway failures. Persistent throttling +# should be addressed by reducing ``n_concurrent_trials``. +_RETRY_FLAGS = ( + "-c model_providers.harbor.name=harbor" + " -c model_providers.harbor.request_max_retries=8" + " -c model_providers.harbor.stream_max_retries=8" + " -c model_providers.harbor.stream_idle_timeout_ms=300000" +) + + +class CodexOffline(Codex): + """Codex with explicit Goal/Web Search flags and offline installation.""" + + CLI_FLAGS = Codex.CLI_FLAGS + [ + CliFlag( + "goals", + cli="-c", + type="enum", + choices=["true", "false"], + format="-c features.goals={value}", + ), + CliFlag( + "web_search", + cli="-c", + type="enum", + choices=["disabled", "cached", "live"], + format='-c web_search="{value}"', + ), + ] + + @staticmethod + def name() -> str: + return "codex-offline" + + def version(self) -> str | None: + return self._version or "offline" + + def get_version_command(self) -> str | None: + return "/usr/local/bin/codex --version" + + def build_cli_flags(self) -> str: + flags = super().build_cli_flags() + # Disable server-side Web Search independently of shell networking. + if "web_search=" not in flags: + flags = f'{flags} -c web_search="disabled"'.strip() + return f"{flags} {_RETRY_FLAGS}" if flags else _RETRY_FLAGS + + async def install(self, environment: BaseEnvironment) -> None: + offline_dir = Path(os.environ.get("CODEX_OFFLINE_DIR", _DEFAULT_OFFLINE_DIR)) + # Resolve symlinks because Harbor upload does not follow them. + codex_bin = (offline_dir / "codex").resolve() + rg_bin = (offline_dir / "rg").resolve() + sidecar_bin = (offline_dir / "codex-code-mode-host").resolve() + bwrap_bin = offline_dir / "codex-resources" / "bwrap" + + if not codex_bin.is_file(): + raise FileNotFoundError( + f"Codex binary not found: {codex_bin}. " + "Point CODEX_OFFLINE_DIR at the native Codex bundle." + ) + if not sidecar_bin.is_file(): + raise FileNotFoundError( + f"codex-code-mode-host not found: {sidecar_bin}. " + "Point CODEX_OFFLINE_DIR at the complete native Codex bundle." + ) + + await self.exec_as_root(environment, command=f"mkdir -p {_STAGE_DIR}") + await environment.upload_file(codex_bin, f"{_STAGE_DIR}/codex") + await environment.upload_file( + sidecar_bin, f"{_STAGE_DIR}/codex-code-mode-host" + ) + if rg_bin.is_file(): + await environment.upload_file(rg_bin, f"{_STAGE_DIR}/rg") + if bwrap_bin.is_file(): + await environment.upload_file(bwrap_bin, f"{_STAGE_DIR}/bwrap") + + # Install with explicit modes and retain a version/fingerprint receipt. + await self.exec_as_root( + environment, + command=( + "set -eu; " + f"install -m 0755 {_STAGE_DIR}/codex /usr/local/bin/codex; " + f"install -m 0755 {_STAGE_DIR}/codex-code-mode-host " + " /usr/local/bin/codex-code-mode-host; " + f"if [ -f {_STAGE_DIR}/rg ]; then " + f" install -m 0755 {_STAGE_DIR}/rg /usr/local/bin/rg; " + "fi; " + f"if [ -f {_STAGE_DIR}/bwrap ]; then " + f" install -m 0755 {_STAGE_DIR}/bwrap /usr/local/bin/bwrap; " + "fi; " + f"rm -rf {_STAGE_DIR}; " + "mkdir -p /logs/agent; " + "{ /usr/local/bin/codex --version; " + " md5sum /usr/local/bin/codex /usr/local/bin/codex-code-mode-host; " + "} > /logs/agent/codex_version.txt 2>&1; " + "cat /logs/agent/codex_version.txt" + ), + ) + + self.logger.info("staged Codex bundle from %s", offline_dir) diff --git a/benchmark/LHTB/configs/heartbeat-generic-cli.yaml b/benchmark/LHTB/configs/heartbeat-generic-cli.yaml new file mode 100644 index 0000000000..6f3743b78d --- /dev/null +++ b/benchmark/LHTB/configs/heartbeat-generic-cli.yaml @@ -0,0 +1,76 @@ +job_name: lhtb-loopx-generic-cli-heartbeat-gpt56sol-max-template +jobs_dir: ./benchmark/LHTB/runs +n_attempts: 1 +n_concurrent_trials: 4 +timeout_multiplier: 1.0 + +environment: + type: docker + force_build: false + delete: false + env: + HTTP_PROXY: "" + HTTPS_PROXY: "" + http_proxy: "" + https_proxy: "" + NO_PROXY: "localhost,127.0.0.1,game" + no_proxy: "localhost,127.0.0.1,game" + +agents: + - import_path: codex_loopx_heartbeat:LoopxHeartbeatCodex + model_name: openai/gpt-5.6-sol + override_timeout_sec: 5400 + kwargs: + reasoning_effort: max + goals: "false" + web_search: disabled + +datasets: + - path: ./upstream/tasks + task_names: + - "2048" + - alp-paper-reproduction + - apex-ib244-matter + - apex-investment-banking-matter + - apex-law433-matter + - apex-management-consulting-matter + - apex-openroad-ibex-signoff + - audio-visual-event-alignment + - chess-mate + - climate-netcdf-extreme-event-audit + - commit0-multilib-tdd + - dicom-radiology-audit + - document-table-layout-reconstruction + - duckdb-optimizer-closure + - epa-swmm-stormwater-regression-audit + - epidemic-inverse-control-audit + - foldseek-paper-reproduction + - gdal-proj-raster-regression + - generals-bot-arena + - grammar-fuzz-coverage-hunt + - great-expectations-audit + - langchain-version-migration + - materials-phase-diagram-audit + - matpower-opf-regression + - microscopy-cell-count-qc-audit + - modflow6-groundwater-regression-audit + - nbody-accel-iterative + - nrel-pysam-hybrid-renewables-audit + - opensees-seismic-structural-regression-audit + - poc-exploit-craft + - riscv-core-debug + - robotics-slam-benchmark-repair + - rush_hour_campaign + - satellite-flood-change-detection-audit + - scientific-figure-data-reconstruction + - snake_maze_campaign + - sokoban + - spice-ephemeris-regression + - spot-scheduler-traces + - su2-airfoil-regression + - sudoku-recovery + - super-mario + - tabular-data-feature-covshift + - unison-paper-reproduction + - unknown-config-semantics + - vector-db-iterative-build diff --git a/benchmark/LHTB/harbor_patch/docker-compose-modelonly-plus.yaml b/benchmark/LHTB/harbor_patch/docker-compose-modelonly-plus.yaml new file mode 100644 index 0000000000..c47090d272 --- /dev/null +++ b/benchmark/LHTB/harbor_patch/docker-compose-modelonly-plus.yaml @@ -0,0 +1,9 @@ +services: + main: + networks: + - default + - lhtb-modelonly + +networks: + lhtb-modelonly: + external: true diff --git a/benchmark/LHTB/harbor_patch/docker-compose-modelonly.yaml b/benchmark/LHTB/harbor_patch/docker-compose-modelonly.yaml new file mode 100644 index 0000000000..c3540744d2 --- /dev/null +++ b/benchmark/LHTB/harbor_patch/docker-compose-modelonly.yaml @@ -0,0 +1,8 @@ +services: + main: + networks: + - lhtb-modelonly + +networks: + lhtb-modelonly: + external: true diff --git a/benchmark/LHTB/harbor_patch/prepare_harbor_modelonly.py b/benchmark/LHTB/harbor_patch/prepare_harbor_modelonly.py new file mode 100755 index 0000000000..9c4453ad11 --- /dev/null +++ b/benchmark/LHTB/harbor_patch/prepare_harbor_modelonly.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Add an opt-in model-only network mode to Harbor's Docker provider. + +Codex runs inside the task container and needs a route to its model gateway. +For an LHTB task with ``allow_internet=false``, Harbor normally applies +``network_mode: none``, which blocks both public egress and model access. This +patch replaces that mode, only when ``LHTB_MODELONLY_NET`` is set, with an +internal Docker network that can reach a gateway bound to the bridge address +but has no public route. + +Without the environment flag, Harbor's behavior is unchanged. The patch is +idempotent and can be rerun after checking out a compatible Harbor revision. +""" + +import os +import shutil +import sys +import time +from pathlib import Path + +HARBOR = Path(os.environ.get("LHTB_HARBOR_SRC", str(Path(__file__).resolve().parents[2] / "upstream" / "harbor" / "src" / "harbor"))) +DOCKER_DIR = HARBOR / "environments" / "docker" +INIT_PY = DOCKER_DIR / "__init__.py" +DOCKER_PY = DOCKER_DIR / "docker.py" + +MARK = "LHTB_MODELONLY_NET" + +INIT_OLD = 'COMPOSE_NO_NETWORK_PATH = COMPOSE_DIR / "docker-compose-no-network.yaml"' +INIT_NEW = INIT_OLD + """ +# Optional overlays that separate model access from public internet access. +COMPOSE_MODELONLY_PATH = COMPOSE_DIR / "docker-compose-modelonly.yaml" +COMPOSE_MODELONLY_PLUS_PATH = COMPOSE_DIR / "docker-compose-modelonly-plus.yaml\"""" + +IMPORT_OLD = """from harbor.environments.docker import ( + COMPOSE_BASE_PATH, + COMPOSE_BUILD_PATH, + COMPOSE_NO_NETWORK_PATH, + COMPOSE_PREBUILT_PATH, + COMPOSE_WINDOWS_KEEPALIVE_PATH, + write_mounts_compose_file, +)""" +IMPORT_NEW = """from harbor.environments.docker import ( + COMPOSE_BASE_PATH, + COMPOSE_BUILD_PATH, + COMPOSE_MODELONLY_PATH, + COMPOSE_MODELONLY_PLUS_PATH, + COMPOSE_NO_NETWORK_PATH, + COMPOSE_PREBUILT_PATH, + COMPOSE_WINDOWS_KEEPALIVE_PATH, + write_mounts_compose_file, +)""" + +ATTR_OLD = " _DOCKER_COMPOSE_NO_NETWORK_PATH = COMPOSE_NO_NETWORK_PATH" +ATTR_NEW = ( + ATTR_OLD + + """ + _DOCKER_COMPOSE_MODELONLY_PATH = COMPOSE_MODELONLY_PATH + _DOCKER_COMPOSE_MODELONLY_PLUS_PATH = COMPOSE_MODELONLY_PLUS_PATH""" +) + +BRANCH_OLD = """ if not self.task_env_config.allow_internet: + paths.append(self._DOCKER_COMPOSE_NO_NETWORK_PATH) + + return paths""" +BRANCH_NEW = """ modelonly = bool(os.environ.get("LHTB_MODELONLY_NET")) + + # Network selection is task-scoped. Separate verifier mode is handled + # by Harbor's trial layer and must not replace the task's network. + if not self.task_env_config.allow_internet: + # The default remains network_mode:none. Opt-in offline tasks use + # the internal bridge and retain the no-public-egress constraint. + paths.append( + self._DOCKER_COMPOSE_MODELONLY_PATH + if modelonly + else self._DOCKER_COMPOSE_NO_NETWORK_PATH + ) + elif modelonly: + # Online tasks keep their default network and gain the model-only + # interface so every task can use one gateway URL. + paths.append(self._DOCKER_COMPOSE_MODELONLY_PLUS_PATH) + + return paths""" + + +def patch(path: Path, pairs: list[tuple[str, str]]) -> bool: + text = path.read_text() + if MARK in text or all(new in text for _, new in pairs): + print(f" {path.name}: already patched") + return False + for old, new in pairs: + if old not in text: + sys.exit(f"FATAL: compatible anchor not found in {path}\n---\n{old}\n---") + if text.count(old) != 1: + sys.exit(f"FATAL: anchor occurs {text.count(old)} times in {path}") + text = text.replace(old, new) + backup = path.with_suffix(path.suffix + f".bak-modelonly-{time.strftime('%H%M%S')}") + shutil.copy2(path, backup) + path.write_text(text) + print(f" {path.name}: patched (backup: {backup.name})") + return True + + +def main() -> None: + staging = Path(__file__).parent + for name in ("docker-compose-modelonly.yaml", "docker-compose-modelonly-plus.yaml"): + src = staging / name + if not src.exists(): + sys.exit(f"FATAL: missing {src}") + shutil.copy2(src, DOCKER_DIR / name) + print(f" installed {name}") + + patch(INIT_PY, [(INIT_OLD, INIT_NEW)]) + patch(DOCKER_PY, [(IMPORT_OLD, IMPORT_NEW), (ATTR_OLD, ATTR_NEW), (BRANCH_OLD, BRANCH_NEW)]) + + # Import smoke: the new compose constants must be available. + sys.path.insert(0, str(HARBOR.parent)) + import os as _os + + _os.environ.pop("LHTB_MODELONLY_NET", None) + from harbor.environments.docker import ( # noqa: F401 + COMPOSE_MODELONLY_PATH, + COMPOSE_MODELONLY_PLUS_PATH, + ) + + assert COMPOSE_MODELONLY_PATH.exists(), COMPOSE_MODELONLY_PATH + assert COMPOSE_MODELONLY_PLUS_PATH.exists(), COMPOSE_MODELONLY_PLUS_PATH + print(" import smoke passed") + + +if __name__ == "__main__": + main() diff --git a/benchmark/LHTB/reports/.gitkeep b/benchmark/LHTB/reports/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/benchmark/LHTB/run.sh b/benchmark/LHTB/run.sh new file mode 100755 index 0000000000..09518c2f95 --- /dev/null +++ b/benchmark/LHTB/run.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +CODE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LOOPX_ROOT="$(cd "$CODE_DIR/../.." && pwd)" + +if [[ -f "$CODE_DIR/.env" ]]; then + # shellcheck disable=SC1091 + source "$CODE_DIR/.env" +fi + +die() { echo "FATAL: $*" >&2; exit 1; } + +[[ -n "${LHTB_ROOT:-}" ]] || die "Set LHTB_ROOT to an LHTB checkout" +[[ -d "$LHTB_ROOT/upstream/tasks" ]] || die "LHTB tasks not found under: $LHTB_ROOT" +LHTB_ROOT="$(cd "$LHTB_ROOT" && pwd)" + +MODE="${1:-preflight}" +SMOKE_TASK="${2:-tabular-data-feature-covshift}" +case "$MODE" in + preflight|smoke|full) ;; + *) + echo "Usage: $0 {preflight|smoke [task-name]|full}" >&2 + exit 2 + ;; +esac + +VENV="${LHTB_VENV:-$LHTB_ROOT/.venv}" +[[ -x "$VENV/bin/harbor" ]] || die "Harbor not found: $VENV/bin/harbor" +[[ -x "$VENV/bin/python" ]] || die "Python not found: $VENV/bin/python" + +OPENAI_BASE_URL="${OPENAI_BASE_URL:-}" +OPENAI_API_KEY="${OPENAI_API_KEY:-}" +[[ -n "$OPENAI_BASE_URL" ]] || die "Set OPENAI_BASE_URL to the model gateway" +[[ -n "$OPENAI_API_KEY" ]] || die "Set OPENAI_API_KEY (a placeholder is sufficient for an injecting gateway)" +MODEL_NAME="${MODEL_NAME:-openai/gpt-5.6-sol}" +REASONING_EFFORT="${REASONING_EFFORT:-max}" +CONCURRENCY="${CONCURRENCY:-4}" +AGENT_TIMEOUT_SEC="${AGENT_TIMEOUT_SEC:-5400}" +LOOPX_SCHEDULER_TIMEOUT_SEC="${LOOPX_SCHEDULER_TIMEOUT_SEC:-5080}" +LOOPX_WAKE_TIMEOUT_SEC="${LOOPX_WAKE_TIMEOUT_SEC:-4800}" +LOOPX_CODEX_TURN_TIMEOUT_SEC="${LOOPX_CODEX_TURN_TIMEOUT_SEC:-4700}" +LHTB_MAX_RETRIES="${LHTB_MAX_RETRIES:-2}" +RUNNER_RESTARTS="${RUNNER_RESTARTS:-2}" +LHTB_MODELONLY_NETWORK="${LHTB_MODELONLY_NETWORK:-lhtb-modelonly}" +LHTB_MODELONLY_SUBNET="${LHTB_MODELONLY_SUBNET:-192.0.2.0/24}" +LHTB_MODELONLY_GATEWAY="${LHTB_MODELONLY_GATEWAY:-192.0.2.1}" +LOOPX_SRC_DIR="${LOOPX_SRC_DIR:-$LOOPX_ROOT}" +LOOPX_EXPECTED_COMMIT="${LOOPX_EXPECTED_COMMIT:-$(git -C "$LOOPX_SRC_DIR" rev-parse HEAD)}" + +if [[ -z "${CODEX_BIN:-}" ]]; then + npm_root="$(npm root -g 2>/dev/null || true)" + candidate="$npm_root/@openai/codex/node_modules/@openai/codex-linux-x64/vendor/x86_64-unknown-linux-musl/bin/codex" + [[ -x "$candidate" ]] && CODEX_BIN="$candidate" +fi +[[ -n "${CODEX_BIN:-}" && -x "$CODEX_BIN" ]] || die "Set CODEX_BIN to the native Codex binary" +CODEX_OFFLINE_DIR="${CODEX_OFFLINE_DIR:-$(dirname "$CODEX_BIN")}" + +LOOPX_PORTABLE_PYTHON="${LOOPX_PORTABLE_PYTHON:-}" +[[ -n "$LOOPX_PORTABLE_PYTHON" ]] || die "Set LOOPX_PORTABLE_PYTHON to a portable Python >=3.11 root" +[[ -x "$LOOPX_PORTABLE_PYTHON/bin/python3" ]] || die "Set LOOPX_PORTABLE_PYTHON to a portable Python >=3.11 root" + +if [[ -z "${LOOPX_NODE_DIR:-}" ]]; then + node_command="$(command -v node 2>/dev/null || true)" + node_binary="$(readlink -f "$node_command" 2>/dev/null || true)" + [[ -n "$node_binary" ]] && LOOPX_NODE_DIR="$(cd "$(dirname "$node_binary")/.." && pwd)" +fi +[[ -n "${LOOPX_NODE_DIR:-}" && -x "$LOOPX_NODE_DIR/bin/node" ]] || die "Set LOOPX_NODE_DIR to a Node >=22.6 root" + +for value in "$CONCURRENCY" "$AGENT_TIMEOUT_SEC" "$LOOPX_SCHEDULER_TIMEOUT_SEC" \ + "$LOOPX_WAKE_TIMEOUT_SEC" "$LOOPX_CODEX_TURN_TIMEOUT_SEC" "$LHTB_MAX_RETRIES" "$RUNNER_RESTARTS"; do + [[ "$value" =~ ^[0-9]+$ ]] || die "numeric configuration expected, got: $value" +done +(( LOOPX_CODEX_TURN_TIMEOUT_SEC < LOOPX_WAKE_TIMEOUT_SEC )) || die "Codex turn timeout must be below wake timeout" +(( LOOPX_WAKE_TIMEOUT_SEC < LOOPX_SCHEDULER_TIMEOUT_SEC )) || die "wake timeout must be below scheduler timeout" +(( LOOPX_SCHEDULER_TIMEOUT_SEC < AGENT_TIMEOUT_SEC )) || die "scheduler timeout must be below Harbor agent timeout" +[[ "$MODEL_NAME" == "openai/gpt-5.6-sol" ]] || die "this treatment is pinned to openai/gpt-5.6-sol" +[[ "$REASONING_EFFORT" == "max" ]] || die "this treatment is pinned to reasoning=max" + +gateway_host="$($VENV/bin/python -c 'from urllib.parse import urlsplit; import sys; print(urlsplit(sys.argv[1]).hostname or "")' "$OPENAI_BASE_URL")" +[[ -n "$gateway_host" ]] || die "Invalid OPENAI_BASE_URL: $OPENAI_BASE_URL" + +echo "=== prepare Harbor model-only networking ===" +HARBOR_DOCKER_DIR="$LHTB_ROOT/upstream/harbor/src/harbor/environments/docker" +if ! grep -q 'LHTB_MODELONLY_NET' "$HARBOR_DOCKER_DIR/docker.py" 2>/dev/null; then + LHTB_HARBOR_SRC="$LHTB_ROOT/upstream/harbor/src/harbor" \ + "$VENV/bin/python" "$CODE_DIR/harbor_patch/prepare_harbor_modelonly.py" +fi +if ! docker network inspect "$LHTB_MODELONLY_NETWORK" >/dev/null 2>&1; then + docker network create --driver bridge --internal \ + --subnet "$LHTB_MODELONLY_SUBNET" \ + --gateway "$LHTB_MODELONLY_GATEWAY" \ + "$LHTB_MODELONLY_NETWORK" >/dev/null +fi +network_internal="$(docker network inspect "$LHTB_MODELONLY_NETWORK" --format '{{.Internal}}')" +network_gateway="$(docker network inspect "$LHTB_MODELONLY_NETWORK" --format '{{(index .IPAM.Config 0).Gateway}}')" +[[ "$network_internal" == true ]] || die "$LHTB_MODELONLY_NETWORK must be internal" +[[ "$network_gateway" == "$LHTB_MODELONLY_GATEWAY" ]] || die "network gateway mismatch: $network_gateway" +[[ "$gateway_host" == "$network_gateway" ]] || die "offline tasks can only reach $network_gateway; gateway uses $gateway_host" + +run_stamp="$(date +%Y%m%d-%H%M%S)" +task_args=() +expected_task_count=46 +job_suffix="full46" +if [[ "$MODE" == smoke ]]; then + task_args=(--task "$SMOKE_TASK") + expected_task_count=1 + job_suffix="smoke-${SMOKE_TASK}" +fi +job_name="lhtb-loopx-hb-gpt56sol-max-${job_suffix}-${run_stamp}" +generated_config="$CODE_DIR/.generated/${job_name}.yaml" +jobs_dir="$CODE_DIR/runs" + +"$VENV/bin/python" "$CODE_DIR/scripts/render_config.py" \ + --template "$CODE_DIR/configs/heartbeat-generic-cli.yaml" \ + --output "$generated_config" \ + --job-name "$job_name" \ + --jobs-dir "$jobs_dir" \ + --concurrency "$CONCURRENCY" \ + --model "$MODEL_NAME" \ + --effort "$REASONING_EFFORT" \ + --timeout "$AGENT_TIMEOUT_SEC" \ + "${task_args[@]}" + +export OPENAI_BASE_URL OPENAI_API_KEY MODEL_NAME REASONING_EFFORT +export CODEX_BIN CODEX_OFFLINE_DIR CODEX_WIRE_API="${CODEX_WIRE_API:-responses}" +export LOOPX_SRC_DIR LOOPX_EXPECTED_COMMIT LOOPX_PORTABLE_PYTHON LOOPX_NODE_DIR +export LOOPX_SCHEDULER_TIMEOUT_SEC LOOPX_WAKE_TIMEOUT_SEC LOOPX_CODEX_TURN_TIMEOUT_SEC +export LHTB_MODELONLY_NETWORK CONCURRENCY AGENT_TIMEOUT_SEC +export LHTB_MODELONLY_NET=1 HB_VERIFIER_FEEDBACK_MODE=binary +export DOCKER_DEFAULT_PLATFORM="${DOCKER_DEFAULT_PLATFORM:-linux/amd64}" +export LITELLM_LOCAL_MODEL_COST_MAP="${LITELLM_LOCAL_MODEL_COST_MAP:-True}" +export PYTHONPATH="$CODE_DIR/agents:$LOOPX_SRC_DIR${PYTHONPATH:+:$PYTHONPATH}" +export NO_PROXY="127.0.0.1,localhost,$gateway_host,${NO_PROXY:-}" +export no_proxy="$NO_PROXY" + +# Preserve LHTB's task declarations: 44 shared verifier tasks, two separate. +unset HARBOR_FORCE_SEPARATE_VERIFIER HARBOR_AGENT_URL_FILTER_DIR HARBOR_AGENT_URL_FILTER_IMAGE +unset HB_CONTINUE_MODE HB_PROCESS_REWARD + +echo "=== fail-closed preflight ===" +"$VENV/bin/python" "$CODE_DIR/scripts/preflight.py" \ + --config "$generated_config" \ + --lhtb-root "$LHTB_ROOT" \ + --loopx-src "$LOOPX_SRC_DIR" \ + --codex-bin "$CODEX_BIN" \ + --portable-python "$LOOPX_PORTABLE_PYTHON" \ + --node-dir "$LOOPX_NODE_DIR" \ + --gateway "$OPENAI_BASE_URL" \ + --network "$LHTB_MODELONLY_NETWORK" \ + --expected-commit "$LOOPX_EXPECTED_COMMIT" \ + --expected-task-count "$expected_task_count" + +mkdir -p "$CODE_DIR/reports" "$CODE_DIR/runs" +receipt="$CODE_DIR/reports/${job_name}.env" +{ + printf 'job_name=%s\nmode=%s\nmodel=%s\nreasoning_effort=%s\n' "$job_name" "$MODE" "$MODEL_NAME" "$REASONING_EFFORT" + printf 'concurrency=%s\nagent_timeout_sec=%s\nscheduler_timeout_sec=%s\n' "$CONCURRENCY" "$AGENT_TIMEOUT_SEC" "$LOOPX_SCHEDULER_TIMEOUT_SEC" + printf 'gateway=%s\nwire_api=%s\nweb_search=disabled\n' "$OPENAI_BASE_URL" "$CODEX_WIRE_API" + printf 'runtime_profile=generic_cli\ncodex_driver=fresh_exec_per_wake\ncodex_resume=false\n' + printf 'onboarding_connection_validation=provider-prevalidated\n' + printf 'scheduler_terminal_packet_compatibility=true\n' + printf 'replan_after_completed_todos=3\nverifier_policy=44_shared_2_separate\n' + printf 'loopx_commit=%s\n' "$(git -C "$LOOPX_SRC_DIR" rev-parse HEAD)" + "$CODEX_BIN" --version 2>/dev/null | sed 's/^/codex_version=/' || true +} | tee "$receipt" + +if [[ "$MODE" == preflight ]]; then + echo "Preflight passed. Generated config: $generated_config" + exit 0 +fi + +if [[ "$MODE" == full ]]; then + echo "=== prepare separate verifier images ===" + LHTB_ROOT="$LHTB_ROOT" "$CODE_DIR/scripts/build_verifier_images.sh" +fi + +job_dir="$CODE_DIR/runs/$job_name" +job_complete() { + [[ -f "$job_dir/result.json" ]] || return 1 + "$VENV/bin/python" - "$job_dir/result.json" <<'PY' +import json, sys +try: + result = json.load(open(sys.argv[1], encoding="utf-8")) +except Exception: + raise SystemExit(1) +raise SystemExit(0 if result.get("finished_at") else 1) +PY +} + +echo "=== run $MODE ===" +attempt=0 +while :; do + set +e + ( + cd "$LHTB_ROOT" + "$VENV/bin/harbor" run --yes -c "$generated_config" --max-retries "$LHTB_MAX_RETRIES" + ) 2>&1 | tee -a "$CODE_DIR/reports/${job_name}.log" + rc=${PIPESTATUS[0]} + set -e + if job_complete; then + break + fi + if (( attempt >= RUNNER_RESTARTS )); then + die "Harbor exited rc=$rc without a finished result; partial job: $job_dir" + fi + attempt=$((attempt + 1)) + echo "Harbor exited rc=$rc; resuming the same job ($attempt/$RUNNER_RESTARTS)" + sleep 5 +done + +"$VENV/bin/python" "$CODE_DIR/scripts/summarize_results.py" "$job_dir" || true +echo "Completed: $job_dir" diff --git a/benchmark/LHTB/runs/.gitkeep b/benchmark/LHTB/runs/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/benchmark/LHTB/runtime/wake_once.py b/benchmark/LHTB/runtime/wake_once.py new file mode 100755 index 0000000000..881637f7f3 --- /dev/null +++ b/benchmark/LHTB/runtime/wake_once.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +"""Run one LoopX heartbeat as one fresh, non-resumed Codex exec session.""" + +from __future__ import annotations + +import json +import os +import shutil +import signal +import subprocess +import sys +import time +import uuid +from pathlib import Path +from typing import Any + + +GLOBAL_REGISTRY_TOKEN = "$HOME/.codex/loopx/registry.global.json" + + +def required_env(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + raise RuntimeError(f"missing required environment variable: {name}") + return value + + +def parse_json_output(text: str, *, command: str) -> dict[str, Any]: + value = text.strip() + if value.startswith("```"): + value = value.split("\n", 1)[-1].rsplit("```", 1)[0].strip() + try: + payload = json.loads(value) + except json.JSONDecodeError as exc: + raise RuntimeError(f"{command} returned invalid JSON: {value[:300]}") from exc + if not isinstance(payload, dict): + raise RuntimeError(f"{command} returned a non-object JSON value") + return payload + + +def run_loopx(argv: list[str], *, cwd: Path, env: dict[str, str]) -> dict[str, Any]: + completed = subprocess.run( + argv, + cwd=cwd, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=120, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + f"LoopX exited {completed.returncode}: " + f"{(completed.stderr or completed.stdout)[-500:]}" + ) + return parse_json_output(completed.stdout, command="heartbeat-prompt") + + +def build_heartbeat_argv( + *, cli: str, registry: str, runtime_root: str, goal_id: str, + agent_id: str, turn_id: str, +) -> list[str]: + return [ + cli, + "--format", "json", + "--registry", registry, + "--runtime-root", runtime_root, + "heartbeat-prompt", + "--thin", + "--runtime-profile", "generic_cli", + "--goal-id", goal_id, + "--agent-id", agent_id, + "--turn-instance-id", turn_id, + "--cli-bin", cli, + "--available-capability", "shell", + "--available-capability", "filesystem_write", + ] + + +def build_codex_argv(*, codex_bin: str, model: str, effort: str, cwd: str) -> list[str]: + # Deliberately contains no `resume`: every heartbeat is a new Codex session. + return [ + codex_bin, + "exec", + "--dangerously-bypass-approvals-and-sandbox", + "--skip-git-repo-check", + "--cd", cwd, + "--model", model, + "--json", + "--enable", "unified_exec", + "-c", f'model_reasoning_effort="{effort}"', + "-c", "features.goals=false", + "-c", 'web_search="disabled"', + "-c", "model_providers.harbor.request_max_retries=8", + "-c", "model_providers.harbor.stream_max_retries=8", + "-c", "model_providers.harbor.stream_idle_timeout_ms=300000", + "-", + ] + + +def write_codex_home( + path: Path, *, base_url: str, api_key: str, wire_api: str, + workspace: str, shared_skills: Path, +) -> None: + path.mkdir(parents=True, exist_ok=False) + (path / "auth.json").write_text( + json.dumps({"OPENAI_API_KEY": api_key}) + "\n", encoding="utf-8" + ) + (path / "auth.json").chmod(0o600) + quoted_workspace = json.dumps(workspace) + config = "\n".join( + [ + 'web_search = "disabled"', + 'sandbox_mode = "danger-full-access"', + 'model_provider = "harbor"', + f"[projects.{quoted_workspace}]", + 'trust_level = "trusted"', + "[model_providers.harbor]", + 'name = "harbor"', + f"base_url = {json.dumps(base_url)}", + f"wire_api = {json.dumps(wire_api)}", + 'env_key = "OPENAI_API_KEY"', + "request_max_retries = 8", + "stream_max_retries = 8", + "stream_idle_timeout_ms = 300000", + "", + ] + ) + (path / "config.toml").write_text(config, encoding="utf-8") + if shared_skills.is_dir(): + (path / "skills").symlink_to(shared_skills, target_is_directory=True) + + +def terminate_process_group(process: subprocess.Popen[bytes]) -> None: + try: + os.killpg(process.pid, signal.SIGTERM) + process.wait(timeout=10) + except (ProcessLookupError, subprocess.TimeoutExpired): + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def main() -> int: + cli = required_env("LOOPX_CLI") + registry = required_env("LOOPX_REGISTRY") + runtime_root = required_env("LOOPX_RUNTIME_ROOT") + goal_id = required_env("LOOPX_GOAL_ID") + agent_id = required_env("LOOPX_AGENT_ID") + workspace = required_env("LOOPX_PROJECT") + codex_bin = required_env("CODEX_BIN") + model = required_env("MODEL_NAME").split("/")[-1] + effort = required_env("REASONING_EFFORT") + base_url = required_env("OPENAI_BASE_URL") + api_key = required_env("OPENAI_API_KEY") + wire_api = os.environ.get("CODEX_WIRE_API", "responses").strip() or "responses" + output_root = Path(required_env("LOOPX_WAKE_LOG_DIR")) + turn_root = Path(required_env("LOOPX_TURN_ROOT")) + shared_skills = Path(required_env("LOOPX_SHARED_SKILLS")) + timeout_seconds = float(os.environ.get("LOOPX_CODEX_TURN_TIMEOUT_SEC", "4700")) + + turn_id = f"lhtb-{time.time_ns()}-{uuid.uuid4().hex[:16]}" + wake_dir = output_root / turn_id + wake_dir.mkdir(parents=True, exist_ok=False) + turn_dir = turn_root / turn_id + codex_home = turn_dir / "codex-home" + turn_dir.mkdir(parents=True, exist_ok=False) + + env = dict(os.environ) + env["LOOPX_TURN"] = turn_id + heartbeat_argv = build_heartbeat_argv( + cli=cli, + registry=registry, + runtime_root=runtime_root, + goal_id=goal_id, + agent_id=agent_id, + turn_id=turn_id, + ) + payload = run_loopx(heartbeat_argv, cwd=Path(workspace), env=env) + (wake_dir / "heartbeat.json").write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + if payload.get("ok") is not True: + raise RuntimeError(f"heartbeat-prompt not ready: {payload.get('error')}") + if payload.get("turn_instance_id") != turn_id: + raise RuntimeError( + "heartbeat Turn identity mismatch: " + f"expected {turn_id!r}, got {payload.get('turn_instance_id')!r}" + ) + body = payload.get("task_body") + if not isinstance(body, str) or not body.strip(): + raise RuntimeError("heartbeat-prompt returned no task_body") + if "--runtime-profile generic_cli" not in body: + raise RuntimeError("heartbeat task_body is not bound to generic_cli") + body = body.replace(GLOBAL_REGISTRY_TOKEN, registry) + if GLOBAL_REGISTRY_TOKEN in body: + raise RuntimeError("heartbeat task_body retained the global registry token") + (wake_dir / "task-body.md").write_text(body, encoding="utf-8") + + write_codex_home( + codex_home, + base_url=base_url, + api_key=api_key, + wire_api=wire_api, + workspace=workspace, + shared_skills=shared_skills, + ) + codex_env = dict(env) + codex_env["CODEX_HOME"] = str(codex_home) + codex_argv = build_codex_argv( + codex_bin=codex_bin, + model=model, + effort=effort, + cwd=workspace, + ) + (wake_dir / "invocation.json").write_text( + json.dumps( + { + "turn_id": turn_id, + "runtime_profile": "generic_cli", + "model": model, + "reasoning_effort": effort, + "fresh_codex_exec": True, + "resume": False, + "web_search": "disabled", + "argv": codex_argv, + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + timed_out = False + return_code = 1 + with (wake_dir / "codex-events.jsonl").open("wb") as stdout_handle, ( + wake_dir / "codex-stderr.log" + ).open("wb") as stderr_handle: + process = subprocess.Popen( + codex_argv, + cwd=workspace, + env=codex_env, + stdin=subprocess.PIPE, + stdout=stdout_handle, + stderr=stderr_handle, + start_new_session=True, + ) + try: + process.communicate(input=body.encode("utf-8"), timeout=timeout_seconds) + except subprocess.TimeoutExpired: + timed_out = True + terminate_process_group(process) + return_code = process.returncode if process.returncode is not None else 124 + + sessions = codex_home / "sessions" + if sessions.is_dir(): + shutil.copytree(sessions, wake_dir / "sessions", dirs_exist_ok=True) + receipt = { + "ok": return_code == 0 and not timed_out, + "turn_id": turn_id, + "codex_return_code": return_code, + "timed_out": timed_out, + "fresh_codex_exec": True, + "resume": False, + "task_body_sha256": __import__("hashlib").sha256(body.encode()).hexdigest(), + } + (wake_dir / "receipt.json").write_text( + json.dumps(receipt, indent=2) + "\n", encoding="utf-8" + ) + shutil.rmtree(turn_dir, ignore_errors=True) + print(json.dumps(receipt, separators=(",", ":")), flush=True) + return 0 if receipt["ok"] else (124 if timed_out else max(1, return_code)) + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: + print( + json.dumps( + {"ok": False, "error": f"{type(exc).__name__}: {exc}"}, + separators=(",", ":"), + ), + file=sys.stderr, + ) + raise diff --git a/benchmark/LHTB/scripts/build_verifier_images.sh b/benchmark/LHTB/scripts/build_verifier_images.sh new file mode 100755 index 0000000000..96f6383d42 --- /dev/null +++ b/benchmark/LHTB/scripts/build_verifier_images.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +CODE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [[ -z "${LHTB_ROOT:-}" ]]; then + echo "FATAL: Set LHTB_ROOT to an LHTB checkout" >&2 + exit 1 +fi +TASKS="$LHTB_ROOT/upstream/tasks" + +LANGCHAIN_IMAGE="${LHTB_LANGCHAIN_VERIFIER_IMAGE:-lhtb-local/langchain-version-migration-verifier:20260826}" +NBODY_IMAGE="${LHTB_NBODY_VERIFIER_IMAGE:-lhtb-local/nbody-accel-iterative-verifier:20260826}" + +build_if_missing() { + local image="$1" dockerfile="$2" context="$3" + if docker image inspect "$image" >/dev/null 2>&1; then + echo "verifier image present: $image" + return + fi + echo "building verifier image: $image" + docker build --pull=false --file "$dockerfile" --tag "$image" "$context" +} + +build_if_missing \ + "$LANGCHAIN_IMAGE" \ + "$CODE_DIR/verifier-images/langchain/Dockerfile" \ + "$TASKS/langchain-version-migration/tests" + +build_if_missing \ + "$NBODY_IMAGE" \ + "$CODE_DIR/verifier-images/nbody/Dockerfile" \ + "$TASKS/nbody-accel-iterative/tests" + +for image in "$LANGCHAIN_IMAGE" "$NBODY_IMAGE"; do + docker run --rm --entrypoint /bin/sh "$image" \ + -c 'test -x /tests/test.sh && test -d /tests' +done + +echo "verifier images ready" diff --git a/benchmark/LHTB/scripts/preflight.py b/benchmark/LHTB/scripts/preflight.py new file mode 100755 index 0000000000..e91dddb66f --- /dev/null +++ b/benchmark/LHTB/scripts/preflight.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Fail-closed preflight for the LHTB LoopX generic_cli heartbeat arm.""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import subprocess +import tomllib +from pathlib import Path +from urllib.parse import urlsplit + +import yaml + + +EXPECTED_SEPARATE = {"langchain-version-migration", "nbody-accel-iterative"} +EXPECTED_AGENT = "codex_loopx_heartbeat:LoopxHeartbeatCodex" + + +def command(argv: list[str], timeout: int = 30) -> tuple[int, str]: + try: + result = subprocess.run( + argv, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + return 127, str(exc) + return result.returncode, (result.stdout or "").strip() + + +def load_wake_module(path: Path): + spec = importlib.util.spec_from_file_location("lhtb_loopx_wake_once", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--lhtb-root", type=Path, required=True) + parser.add_argument("--loopx-src", type=Path, required=True) + parser.add_argument("--codex-bin", type=Path, required=True) + parser.add_argument("--portable-python", type=Path, required=True) + parser.add_argument("--node-dir", type=Path, required=True) + parser.add_argument("--gateway", required=True) + parser.add_argument("--network", required=True) + parser.add_argument("--expected-commit", required=True) + parser.add_argument("--expected-task-count", type=int, required=True) + args = parser.parse_args() + + failures: list[str] = [] + + def check(label: str, passed: bool, detail: str) -> None: + print(f"[{'OK' if passed else 'FAIL'}] {label}: {detail}") + if not passed: + failures.append(label) + + lhtb_root = args.lhtb_root.resolve() + tasks_root = lhtb_root / "upstream" / "tasks" + task_dirs = sorted(path for path in tasks_root.iterdir() if path.is_dir()) + check("LHTB task count", len(task_dirs) == 46, f"{len(task_dirs)}/46") + + separate: set[str] = set() + offline = 0 + for task in task_dirs: + manifest = tomllib.loads((task / "task.toml").read_text(encoding="utf-8")) + if manifest.get("environment", {}).get("allow_internet") is False: + offline += 1 + verifier = manifest.get("verifier", {}) + if verifier.get("environment_mode") == "separate" or verifier.get("environment") is not None: + separate.add(task.name) + check("LHTB internet policy", offline == 22, f"offline={offline}, online={46 - offline}") + check( + "LHTB verifier policy", + separate == EXPECTED_SEPARATE, + f"shared={46 - len(separate)}, separate={sorted(separate)}", + ) + + config = yaml.safe_load(args.config.read_text(encoding="utf-8")) + selected = config["datasets"][0]["task_names"] + check( + "selected task coverage", + len(selected) == args.expected_task_count and set(selected) <= {p.name for p in task_dirs}, + f"{len(selected)}/{args.expected_task_count}", + ) + agent = config["agents"][0] + kwargs = agent.get("kwargs", {}) + check("Harbor adapter", agent.get("import_path") == EXPECTED_AGENT, str(agent.get("import_path"))) + check("model", agent.get("model_name") == "openai/gpt-5.6-sol", str(agent.get("model_name"))) + check("reasoning", kwargs.get("reasoning_effort") == "max", str(kwargs.get("reasoning_effort"))) + check("native Goal disabled", kwargs.get("goals") == "false", str(kwargs.get("goals"))) + check("web search disabled", kwargs.get("web_search") == "disabled", str(kwargs.get("web_search"))) + + actual_commit = command(["git", "-C", str(args.loopx_src), "rev-parse", "HEAD"])[1] + check("LoopX commit", actual_commit == args.expected_commit, actual_commit or "unavailable") + check( + "LoopX external scheduler", + (args.loopx_src / "scripts" / "external_scheduler_worker.py").is_file(), + str(args.loopx_src / "scripts" / "external_scheduler_worker.py"), + ) + scheduler_source = ( + args.loopx_src / "scripts" / "external_scheduler_worker.py" + ).read_text(encoding="utf-8") + terminal_branch = scheduler_source.find("if action in TERMINAL_ACTIONS:") + cadence_parse = scheduler_source.find("local = _extract_local_scheduler(payload)") + check( + "terminal scheduler compatibility", + 0 <= terminal_branch < cadence_parse, + "terminal packets do not require omitted cold-path cadence detail", + ) + rc, help_text = command([str(args.loopx_src / "scripts" / "loopx"), "configure-goal", "--help"]) + check( + "Todo replan cadence CLI", + rc == 0 and "--execution-replan-after-todos {1,2,3,4,5}" in help_text, + "supports threshold 1..5", + ) + + wake_path = args.config.resolve().parents[1] / "runtime" / "wake_once.py" + try: + wake = load_wake_module(wake_path) + turn_id = "preflight-unique-turn" + heartbeat = wake.build_heartbeat_argv( + cli="/opt/loopx", + registry="/tmp/registry.json", + runtime_root="/tmp/runtime", + goal_id="goal", + agent_id="agent", + turn_id=turn_id, + ) + codex = wake.build_codex_argv( + codex_bin="codex", model="gpt-5.6-sol", effort="max", cwd="/app" + ) + check( + "generic_cli heartbeat contract", + "generic_cli" in heartbeat and "--turn-instance-id" in heartbeat and turn_id in heartbeat, + "runtime_profile=generic_cli + explicit TURN_ID", + ) + check( + "fresh Codex exec contract", + codex[:2] == ["codex", "exec"] and "resume" not in codex, + "codex exec; resume absent", + ) + check("Codex effort", 'model_reasoning_effort="max"' in codex, "max") + check("Codex web search", 'web_search="disabled"' in codex, "disabled") + check("Codex native Goal", "features.goals=false" in codex, "disabled") + except Exception as exc: + check("wake module", False, f"{type(exc).__name__}: {exc}") + + agent_source = args.config.resolve().parents[1] / "agents" / "codex_loopx_heartbeat.py" + source_text = agent_source.read_text(encoding="utf-8") + check( + "replan threshold pinned", + "_REPLAN_AFTER_TODOS = 3" in source_text + and '"--execution-replan-after-todos", str(_REPLAN_AFTER_TODOS)' in source_text, + "3 with readback gate", + ) + check( + "benchmark-owned onboarding", + '"--no-onboarding-scan"' in source_text + and '"--onboarding-connection-validation", "provider-prevalidated"' in source_text + and '"--accept-onboarding-agent-todos"' not in source_text, + "provider-prevalidated; no unrelated repo-intake Todo", + ) + check( + "LoopX install profile directories", + "_PROFILE_HOME" in source_text + and "_SHARED_CODEX_HOME" in source_text + and "{_PROFILE}/releases" in source_text, + "HOME, CODEX_HOME, bin, releases and man are pre-created", + ) + check( + "clean LoopX source staging", + "_copy_git_snapshot" in source_text + and "self._copy_git_snapshot(container_id, loopx_src, _SRC)" in source_text, + "pinned git snapshot excludes local benchmark runs and artifacts", + ) + check( + "Codex login-shell Node bridge", + '"BASH_ENV": _BASH_ENV' in source_text + and "export PATH={_NODE}/bin:$PATH" in source_text, + "portable Node survives bash -lc", + ) + + harbor = lhtb_root / ".venv" / "bin" / "harbor" + check("Harbor", harbor.is_file() and os.access(harbor, os.X_OK), str(harbor)) + check("Codex binary", args.codex_bin.is_file() and os.access(args.codex_bin, os.X_OK), str(args.codex_bin)) + check( + "Codex code-mode sidecar", + (args.codex_bin.parent / "codex-code-mode-host").is_file(), + str(args.codex_bin.parent / "codex-code-mode-host"), + ) + check( + "portable Python", + (args.portable_python / "bin" / "python3").is_file(), + str(args.portable_python), + ) + check("Node runtime", (args.node_dir / "bin" / "node").is_file(), str(args.node_dir)) + + docker_source = lhtb_root / "upstream/harbor/src/harbor/environments/docker/docker.py" + patched = docker_source.is_file() and "LHTB_MODELONLY_NET" in docker_source.read_text(encoding="utf-8") + check("model-only Harbor patch", patched, str(docker_source)) + rc, output = command(["docker", "info"]) + check("Docker daemon", rc == 0, output.splitlines()[0] if output else "unavailable") + rc, output = command(["docker", "network", "inspect", args.network, "--format", "{{.Internal}}"]) + check("internal model network", rc == 0 and output == "true", f"{args.network}: {output or 'missing'}") + + parsed = urlsplit(args.gateway) + health = f"{parsed.scheme}://{parsed.netloc}/health" + rc, output = command(["curl", "-sS", "--noproxy", "*", "-m", "5", health], timeout=10) + check("model gateway", rc == 0 and bool(output), f"{health}: {(output or 'unreachable')[:120]}") + + receipt = { + "ok": not failures, + "config": str(args.config.resolve()), + "task_count": len(selected), + "loopx_commit": actual_commit, + "runtime_profile": "generic_cli", + "codex_driver": "fresh_exec_per_wake", + "model": agent.get("model_name"), + "reasoning_effort": kwargs.get("reasoning_effort"), + "replan_after_completed_todos": 3, + "verifier_policy": {"shared": 44, "separate": sorted(EXPECTED_SEPARATE)}, + "failures": failures, + } + print(json.dumps(receipt, ensure_ascii=False, indent=2)) + return 0 if not failures else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/LHTB/scripts/render_config.py b/benchmark/LHTB/scripts/render_config.py new file mode 100755 index 0000000000..497989ed54 --- /dev/null +++ b/benchmark/LHTB/scripts/render_config.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Render the immutable 46-task template for full, smoke, or preflight use.""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +import yaml + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--template", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--job-name", required=True) + parser.add_argument("--jobs-dir", required=True) + parser.add_argument("--concurrency", type=int, required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--effort", required=True) + parser.add_argument("--timeout", type=int, required=True) + parser.add_argument("--task", action="append", default=[]) + args = parser.parse_args() + + if not 1 <= args.concurrency <= 64: + parser.error("concurrency must be in 1..64") + if args.timeout < 60: + parser.error("timeout must be at least 60 seconds") + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", args.job_name): + parser.error("job-name contains unsafe characters") + + payload = yaml.safe_load(args.template.read_text(encoding="utf-8")) + all_tasks = payload["datasets"][0]["task_names"] + if len(all_tasks) != 46 or len(set(all_tasks)) != 46: + parser.error(f"template must contain 46 unique tasks, got {len(all_tasks)}") + selected = args.task or all_tasks + unknown = sorted(set(selected) - set(all_tasks)) + if unknown: + parser.error(f"unknown task(s): {', '.join(unknown)}") + + payload["job_name"] = args.job_name + payload["jobs_dir"] = args.jobs_dir + payload["n_concurrent_trials"] = min(args.concurrency, len(selected)) + payload["datasets"][0]["task_names"] = selected + agent = payload["agents"][0] + agent["model_name"] = args.model + agent["override_timeout_sec"] = args.timeout + agent["kwargs"]["reasoning_effort"] = args.effort + agent["kwargs"]["goals"] = "false" + agent["kwargs"]["web_search"] = "disabled" + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/LHTB/scripts/summarize_results.py b/benchmark/LHTB/scripts/summarize_results.py new file mode 100755 index 0000000000..63f495f93a --- /dev/null +++ b/benchmark/LHTB/scripts/summarize_results.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Print a compact summary for one Harbor job.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + + +def extract_reward(payload: dict) -> int | float | None: + reward = payload.get("reward") + if isinstance(reward, dict): + reward = reward.get("reward") or reward.get("score") + if isinstance(reward, (int, float)): + return reward + + verifier_result = payload.get("verifier_result") + if not isinstance(verifier_result, dict): + return None + rewards = verifier_result.get("rewards") + if not isinstance(rewards, dict): + return None + reward = rewards.get("reward") + return reward if isinstance(reward, (int, float)) else None + + +def main() -> int: + job = Path(sys.argv[1]) + trials = sorted(path for path in job.glob("**/result.json") if path.parent != job) + rows = [] + for path in trials: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception: + continue + reward = extract_reward(payload) + rows.append((path.parent.name, reward, payload.get("exception_info"))) + numeric = [float(row[1]) for row in rows if isinstance(row[1], (int, float))] + print(json.dumps({ + "job": str(job), + "trials_found": len(rows), + "rewards_found": len(numeric), + "mean_reward": sum(numeric) / len(numeric) if numeric else None, + "rows": rows, + }, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/LHTB/verifier-images/langchain/Dockerfile b/benchmark/LHTB/verifier-images/langchain/Dockerfile new file mode 100644 index 0000000000..918409831e --- /dev/null +++ b/benchmark/LHTB/verifier-images/langchain/Dockerfile @@ -0,0 +1,16 @@ +FROM zli12321/lhtb-langchain-version-migration:20260615 + +COPY . /tests/ + +# The task image already carries the offline wheelhouse. Install the verifier +# runtime here without contacting a registry, and keep the baked test suite. +RUN python -m pip install --no-cache-dir --no-index \ + --find-links /opt/wheelhouse \ + "langchain==1.3.4" \ + "pyyaml==6.0.2" \ + "pytest==8.4.1" \ + "setuptools>=68" \ + "wheel" \ + && chmod +x /tests/test.sh + +CMD ["/bin/bash"] diff --git a/benchmark/LHTB/verifier-images/nbody/Dockerfile b/benchmark/LHTB/verifier-images/nbody/Dockerfile new file mode 100644 index 0000000000..6427b4ffab --- /dev/null +++ b/benchmark/LHTB/verifier-images/nbody/Dockerfile @@ -0,0 +1,10 @@ +FROM zli12321/lhtb-nbody-accel-iterative:20260615 + +COPY . /tests/ + +# Runtime verification uses the prebuilt numpy and does not need network +# access. The hardening tests are run by the publisher build on the canonical +# image; here we only ensure the shipped verifier entrypoints are executable. +RUN chmod +x /tests/test.sh /tests/submission_runner.py + +CMD ["/bin/bash"]