Make the coding-agent RL loop train on ROCm/MI355X

agent_run/local_multi_turn_smoke/Dockerfile.sympy-23950

@@ -0,0 +1,12 @@
+FROM python:3.10-bullseye
+
+# python:3.10-bullseye already ships git 2.30.2, so this image never needed an
+# apt install. Keeping one made the build fail outright once bullseye reached
+# EOL and deb.debian.org's bullseye-security Release file expired:
+#   E: Release file ... is expired (invalid since ...)
+RUN git clone https://github.com/sympy/sympy.git /workspace/sympy \
+    && cd /workspace/sympy \
+    && git checkout 88664e6e0b781d0a8b5347896af74b555e92891e \
+    && pip install --no-cache-dir -e . pytest
+
+WORKDIR /workspace/sympy

agent_run/local_multi_turn_smoke/README.md

@@ -0,0 +1,50 @@
+# Local coding-agent smoke test
+
+Runs ten SWE-bench Verified SymPy tasks with one local Docker sandbox per task.
+The tested configuration is Qwen3.5-27B-GPTQ-Int4, tensor parallel 8, a 256K
+context window, and a 16K response limit. This is rollout-only; it does not train.
+
+## Requirements
+
+- One 8x H200 host with Docker and `vllm/vime:latest`.
+- At least 35 GB free while downloading the model.
+- This Vime checkout at `/mnt/data/vime-agent-smoke/vime`.
+- `sympy-10.jsonl`, Node 22, and the Claude Code npm tarball under the paths below.
+
+## Prepare
+
+```bash
+ROOT=/mnt/data/vime-agent-smoke
+mkdir -p "$ROOT"/{assets,models,runs,tasks}
+
+cp agent_run/local_multi_turn_smoke/sympy-10.jsonl "$ROOT/tasks/"
+curl -L https://nodejs.org/dist/v22.20.0/node-v22.20.0-linux-x64.tar.xz \
+  -o "$ROOT/assets/node-v22.20.0-linux-x64.tar.xz"
+docker run --rm -v "$ROOT/assets:/out" -w /out node:22 \
+  npm pack @anthropic-ai/claude-code
+mv "$ROOT"/assets/anthropic-ai-claude-code-*.tgz \
+  "$ROOT/assets/anthropic-ai-claude-code.tgz"
+
+docker build -t vime-swe-sympy-23950:local \
+  -f agent_run/local_multi_turn_smoke/Dockerfile.sympy-23950 .
+docker run --rm -v "$ROOT/models:/models" vllm/vime:latest \
+  hf download Qwen/Qwen3.5-27B-GPTQ-Int4 \
+  --local-dir /models/Qwen3.5-27B-GPTQ-Int4
+```
+
+The ten task IDs are `23950`, `22714`, `22914`, `23534`, `24213`, `23824`,
+`23262`, `24066`, `24539`, and `23413`, all prefixed by `sympy__sympy-`.
+
+## Run
+
+```bash
+bash agent_run/local_multi_turn_smoke/run_h200.sh
+```
+
+Results are written to `$ROOT/runs/latest`: `run.log` contains the aggregate
+result, while `trace/<instance_id>/` contains the input, trajectory, source
+patch, and grading result for each task.
+
+The official evaluator can produce false positives. In the recorded run,
+`sympy__sympy-22714` passed its supplied test but incorrectly accepted
+`Point(1 + I, 2)`, so successful rewards still require patch review.

agent_run/local_multi_turn_smoke/generate.py

@@ -0,0 +1,88 @@
+import json
+import os
+from contextvars import ContextVar
+from pathlib import Path
+
+from examples.coding_agent_rl import generate as coding_generate
+from examples.coding_agent_rl import swe
+
+from .sandbox import LocalDockerSandbox
+
+coding_generate.E2BSandbox = LocalDockerSandbox
+swe.E2BSandbox = LocalDockerSandbox
+
+_git_diff = swe.git_diff
+_run_evaluation = swe.run_evaluation
+_instance_id: ContextVar[str] = ContextVar("instance_id", default="unknown")
+
+
+def _trace_dir() -> Path | None:
+    """Per-instance trace directory, or None when tracing is off.
+
+    sandbox.py already treats VIME_LOCAL_SANDBOX_TRACE_DIR as optional and skips
+    tracing when it is unset; this module required it and raised KeyError, so
+    running without it killed the rollout over a debugging aid.
+    """
+    root = os.environ.get("VIME_LOCAL_SANDBOX_TRACE_DIR")
+    if not root:
+        return None
+    path = Path(root) / _instance_id.get()
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+async def _traced_git_diff(sb, workdir: str) -> str:
+    diff = await _git_diff(sb, workdir)
+    td = _trace_dir()
+    if td is not None:
+        (td / "solution.patch").write_text(diff)
+        _, trajectory, _ = await sb.exec(f"cat {workdir}/.harness/trajectory.jsonl", user="agent")
+        (td / "trajectory.jsonl").write_text(trajectory)
+    return diff
+
+
+async def _traced_run_evaluation(md: dict, *, diff_text: str, timeout_sec: int):
+    result = await _run_evaluation(md, diff_text=diff_text, timeout_sec=timeout_sec)
+    td = _trace_dir()
+    if td is not None:
+        (td / "grading.json").write_text(
+            json.dumps(
+                {
+                    "instance_id": md["instance_id"],
+                    "reward": result.reward,
+                    "applied_cleanly": result.applied_cleanly,
+                    "eval_cmd": md["grading"].get("eval_cmd"),
+                },
+                indent=2,
+            )
+            + "\n"
+        )
+    return result
+
+
+swe.git_diff = _traced_git_diff
+swe.run_evaluation = _traced_run_evaluation
+
+
+async def generate(args, base_sample, sampling_params, evaluation: bool = False):
+    token = _instance_id.set(base_sample.metadata["instance_id"])
+    try:
+        td = _trace_dir()
+        if td is not None:
+            (td / "input.json").write_text(
+                json.dumps(
+                    {
+                        "prompt": base_sample.prompt,
+                        "label": base_sample.label,
+                        "metadata": base_sample.metadata,
+                        "sampling_params": sampling_params,
+                        "evaluation": evaluation,
+                    },
+                    indent=2,
+                    default=str,
+                )
+                + "\n"
+            )
+        return await coding_generate.generate(args, base_sample, sampling_params, evaluation)
+    finally:
+        _instance_id.reset(token)

agent_run/local_multi_turn_smoke/run_h200.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ROOT=/mnt/data/vime-agent-smoke
+mkdir -p "${ROOT}/runs"
+
+docker run --rm --gpus all --ipc=host --network host \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -v /usr/bin/docker:/usr/bin/docker:ro \
+  -v "${ROOT}/vime:/root/vime" \
+  -v "${ROOT}/models:/work/models" \
+  -v "${ROOT}/assets:/work/assets" \
+  -v "${ROOT}/tasks:/work/tasks:ro" \
+  -v "${ROOT}/runs:/work/runs" \
+  -w /root/vime \
+  vllm/vime:latest \
+  bash agent_run/local_multi_turn_smoke/run_inside.sh

agent_run/local_multi_turn_smoke/run_inside.sh

@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cd /root/vime
+
+RUN_ROOT="/work/runs/single-agent-$(date +%Y%m%d-%H%M%S)"
+mkdir -p "${RUN_ROOT}/rollout_dumps" "${RUN_ROOT}/trace"
+ln -sfn "${RUN_ROOT}" /work/runs/latest
+
+export PYTHONUNBUFFERED=1
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export MASTER_ADDR=127.0.0.1
+export SWE_AGENT=claude_code
+export SWE_TRAIN_PROTOCOL=scaleswe
+export ADAPTER_PUBLIC_HOST=127.0.0.1
+export ADAPTER_BIND_HOST=0.0.0.0
+export ADAPTER_PORT=18001
+export SWE_BOOT_CONCURRENCY=1
+export SWE_BOOT_RETRIES=1
+export SWE_AGENT_TIME_BUDGET_SEC=600
+export SWE_EVAL_TIMEOUT_SEC=300
+export SWE_ROLLOUT_GUARD_SEC=9000
+export VIME_AGENT_NODE_TARBALL=/work/assets/node-v22.20.0-linux-x64.tar.xz
+export VIME_AGENT_CC_TARBALL=/work/assets/anthropic-ai-claude-code.tgz
+export VIME_AGENT_CC_EXTRA_ARGS="--disable-slash-commands --disallowedTools Agent WebFetch WebSearch Write NotebookEdit"
+export VLLM_DEEP_GEMM_WARMUP=skip
+export SWE_CC_PROMPT="Complete the issue in PROBLEM_STATEMENT.md. Inspect the relevant source, actually edit the smallest possible source-only fix, and run a focused behavior check. Do not edit tests or commit, and do not merely describe a patch. Finish with a one-line summary."
+export VIME_LOCAL_SANDBOX_TRACE_DIR="${RUN_ROOT}/trace"
+export no_proxy=127.0.0.1
+export NO_PROXY=127.0.0.1
+
+source scripts/models/qwen3.5-27B.sh
+
+ray stop --force || true
+pkill -9 -f '[v]llm serve|VLL[M]::' || true
+ray start --head --node-ip-address 127.0.0.1 --num-gpus 8 --disable-usage-stats --dashboard-host=0.0.0.0
+
+RUNTIME_ENV_JSON=$(python - <<'PY'
+import json
+import os
+
+prefixes = ("ADAPTER_", "SWE_", "VIME_", "VLLM_")
+env = {
+    key: value
+    for key, value in os.environ.items()
+    if key.startswith(prefixes) or key in {"CUDA_VISIBLE_DEVICES", "MASTER_ADDR", "NO_PROXY", "no_proxy"}
+}
+env.update(
+    PYTHONUNBUFFERED="1",
+    PYTHONPATH="/root/vime:/root/Megatron-LM",
+    CUDA_DEVICE_MAX_CONNECTIONS="1",
+    NCCL_NVLS_ENABLE="0",
+)
+print(json.dumps({"env_vars": env}))
+PY
+)
+
+ray job submit --address=http://127.0.0.1:8265 \
+  --runtime-env-json="${RUNTIME_ENV_JSON}" \
+  -- python -u train.py \
+  "${MODEL_ARGS[@]}" \
+  --hf-checkpoint /work/models/Qwen3.5-27B-GPTQ-Int4 \
+  --ref-load /work/models/Qwen3.5-27B-GPTQ-Int4 \
+  --custom-generate-function-path agent_run.local_multi_turn_smoke.generate.generate \
+  --prompt-data /work/tasks/sympy-10.jsonl \
+  --input-key prompt \
+  --label-key label \
+  --metadata-key metadata \
+  --apply-chat-template \
+  --num-rollout 1 \
+  --rollout-batch-size 10 \
+  --n-samples-per-prompt 1 \
+  --rollout-max-context-len 262144 \
+  --rollout-max-response-len 16384 \
+  --rollout-stop-token-ids 248046 248044 \
+  --rollout-temperature 0.0 \
+  --num-steps-per-rollout 1 \
+  --global-batch-size 10 \
+  --micro-batch-size 1 \
+  --save-debug-rollout-data "${RUN_ROOT}/rollout_dumps/rollout_{rollout_id}.pt" \
+  --debug-rollout-only \
+  --tensor-model-parallel-size 8 \
+  --pipeline-model-parallel-size 1 \
+  --context-parallel-size 1 \
+  --expert-model-parallel-size 1 \
+  --expert-tensor-parallel-size 1 \
+  --recompute-granularity full \
+  --recompute-method uniform \
+  --recompute-num-layers 1 \
+  --use-dynamic-batch-size \
+  --max-tokens-per-gpu 262144 \
+  --log-probs-chunk-size 1024 \
+  --advantage-estimator grpo \
+  --kl-loss-coef 0.0 \
+  --kl-loss-type low_var_kl \
+  --kl-coef 0.0 \
+  --entropy-coef 0.0 \
+  --eps-clip 0.2 \
+  --eps-clip-high 0.28 \
+  --optimizer adam \
+  --lr 1e-6 \
+  --lr-decay-style constant \
+  --weight-decay 0.1 \
+  --adam-beta1 0.9 \
+  --adam-beta2 0.98 \
+  --rollout-num-gpus 8 \
+  --rollout-num-gpus-per-engine 8 \
+  --vllm-gpu-memory-utilization 0.80 \
+  --vllm-tool-call-parser qwen3_coder \
+  --vllm-reasoning-parser qwen3 \
+  --attention-dropout 0.0 \
+  --hidden-dropout 0.0 \
+  --accumulate-allreduce-grads-in-fp32 \
+  --attention-softmax-in-fp32 \
+  --attention-backend flash \
+  --actor-num-nodes 1 \
+  --actor-num-gpus-per-node 8 \
+  --colocate \
+  2>&1 | tee "${RUN_ROOT}/run.log"
+
+echo "RUN_ROOT=${RUN_ROOT}" | tee "${RUN_ROOT}/completed.txt"