diff --git a/AGENTS.md b/AGENTS.md index 134a189..6731eed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,8 +66,9 @@ PostTrainBench/ the harness to the latest npm release and writes its version to `cli_version.txt` (surfaced in the result dir). The helper (`src/utils/update_agent_cli.sh`, copied into the sandbox by `run_task.sh`) holds the binary→npm-package mapping; add a `case` entry there if the agent uses - a CLI not already covered (`claude`, `codex`, `gemini`, `opencode`). The update is best-effort — - a failure falls back to the container's pinned version and still records what actually ran. + a CLI not already covered (`claude`, `codex`, `gemini`, `mcode`, `opencode`). The update is + best-effort — a failure falls back to the container's pinned version and still records what + actually ran. Set `POST_TRAIN_BENCH_SKIP_CLI_UPDATE=1` in `.env` to disable the update globally and pin CLI versions to whatever the container ships; `cli_version.txt` still records what ran (`update: skipped`). @@ -88,7 +89,7 @@ below. With the allowlist in place there is **no** need to `unset`/blank keys in Currently supported agents include: `claude`, `claude_non_api`, `claude_non_api_max`, `codex`, `codex_non_api` (and `_high`, `_xhigh`, `_reprompt`, ...), `codexhigh`, `codexlow`, `cursor_cli`, -`gemini`, `glm5`, `grok_cli`, `opencode`, `qwen3max`. +`gemini`, `glm5`, `grok_cli`, `mcode`, `opencode`, `qwen3max`. `cursor_cli` uses the Cursor CLI (`agent`) with subscription auth: `solve.sh` installs the CLI via the official curl installer (`cursor.com/install`, drops `agent` + `cursor-agent` symlinks in diff --git a/README.md b/README.md index 8ef9bca..38b2513 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ The `.env` file contains API keys and configuration. See `example.env` for all a | `GEMINI_API_KEY` | Google Gemini API key | — | | `OPENCODE_API_KEY` | OpenCode API key (used by the `opencode` agent) | — | | `ZAI_API_KEY` | Z.AI API key (used by the `opencode` and `glm5` agents) | — | +| `MINIMAX_API_KEY` | MiniMax API key (used by the `mcode` agent) | — | | `HF_HOME` | HuggingFace cache directory | `$HOME/.cache/huggingface` | | `POST_TRAIN_BENCH_RESULTS_DIR` | Directory for results | `results` | | `POST_TRAIN_BENCH_CONTAINERS_DIR` | Directory for containers | `containers` | @@ -100,6 +101,24 @@ Currently, we only support the HTCondor job scheduler. [Harbor](https://github.c Most agents authenticate via API keys set as environment variables (e.g., `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GEMINI_API_KEY`). These are passed into the container automatically by `run_task.sh`. Set them in your environment before running `commit.sh`. +**MiniMax Code (`agents/mcode/`)** + +Set `MINIMAX_API_KEY`, then submit a job with the `mcode` agent and a provider-qualified model ID: + +```bash +condor_submit_bid 50 \ + -a "agent=mcode" \ + -a "agent_config=minimax/MiniMax-M3" \ + -a "eval=gsm8k" \ + -a "model_to_train=Qwen/Qwen3-1.7B" \ + -a "num_hours=10" \ + src/commit_utils/single_task.sub +``` + +The runner creates a mode-0700 temporary `MINIMAX_DATA_DIR`, seeds its per-run provider config +from `MINIMAX_API_KEY`, streams the prompt through `mcode exec`, and removes all temporary state +on exit so the key is not retained between benchmark runs. + #### Subscription-based agents (non-API) Some models are only available through CLI subscriptions rather than API keys (e.g., GPT-5.3-Codex via ChatGPT Pro). These agents require separate authentication setup. diff --git a/agents/mcode/api_keys.json b/agents/mcode/api_keys.json new file mode 100644 index 0000000..1b0f429 --- /dev/null +++ b/agents/mcode/api_keys.json @@ -0,0 +1,5 @@ +{ + "allowed_api_keys": [ + "MINIMAX_API_KEY" + ] +} diff --git a/agents/mcode/solve.sh b/agents/mcode/solve.sh new file mode 100755 index 0000000..fa1a294 --- /dev/null +++ b/agents/mcode/solve.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -euo pipefail + +# Keep provider credentials and run state isolated to this benchmark process. +MCODE_DATA_DIR="$(mktemp -d "${TMPDIR:-/tmp}/posttrainbench-mcode.XXXXXX")" +chmod 700 "$MCODE_DATA_DIR" +export MINIMAX_DATA_DIR="$MCODE_DATA_DIR" + +cleanup() { + rm -rf -- "$MCODE_DATA_DIR" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +# Auto-update the CLI harness to the latest release and record its version. +bash /home/ben/update_agent_cli.sh mcode + +# Seed the per-run provider config from the environment. The resulting config +# lives only inside the mode-0700 temporary directory removed by the EXIT trap. +mcode provider set-minimax-key --api-key-env MINIMAX_API_KEY + +printf '%s' "$PROMPT" | mcode exec \ + --input - \ + --cwd /home/ben/task \ + --permission full \ + --output-format stream-json \ + --model "$AGENT_CONFIG" diff --git a/containers/standard.def b/containers/standard.def index 1682512..9a40e8e 100644 --- a/containers/standard.def +++ b/containers/standard.def @@ -39,8 +39,8 @@ From: nvidia/cuda:12.9.1-cudnn-devel-ubuntu22.04 # 3) flash-attn (needs no-build-isolation) uv pip install --system --no-cache flash-attn==2.8.3 --no-build-isolation - # 4) AI CLI tools via npm (only for the judge, the other ones are installed in solve.sh) - npm install -g @openai/codex@0.137.0 + # 4) Pinned AI CLI fallbacks (solve.sh may best-effort update them per run) + npm install -g @openai/codex@0.137.0 @minimax-ai/code@0.1.2 # 5) inspect_evals from source (pinned commit) mkdir -p /opt diff --git a/example.env b/example.env index d1681f6..f94d78d 100644 --- a/example.env +++ b/example.env @@ -6,6 +6,7 @@ OPENCODE_API_KEY="your-opencode-key" OPENROUTER_API_KEY="your-openrouter-key" GLMX_API_KEY="your-zai-key" KIMI_API_KEY="your-moonshot-key" +MINIMAX_API_KEY="your-minimax-key" # Paths HF_HOME="$HOME/.cache/huggingface" diff --git a/src/trace_parsing/mcode_parser.py b/src/trace_parsing/mcode_parser.py new file mode 100644 index 0000000..5a6a1d0 --- /dev/null +++ b/src/trace_parsing/mcode_parser.py @@ -0,0 +1,163 @@ +"""Pretty-print MiniMax Code ``mcode exec --output-format stream-json`` traces.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from _common import TIMESTAMP_PREFIX_RE, pretty_format_json + + +TOOL_STATUSES = {0: "pending", 1: "running", 2: "completed", 3: "failed"} + + +def indent(text: str, level: int) -> str: + pad = " " * level + return "\n".join(pad + line if line else pad for line in text.splitlines()) + + +def add_text(lines: list[str], label: str, value: Any) -> None: + if value is None or value == "": + return + lines.append(indent(f"{label}:", 1)) + lines.append(indent(str(value).rstrip(), 2)) + + +def format_tool_call(tool_call: dict[str, Any]) -> list[str]: + name = tool_call.get("name", "unknown") + tool_id = tool_call.get("id", "") + raw_status = tool_call.get("status", "unknown") + status = TOOL_STATUSES.get(raw_status, str(raw_status)) + header = f"Tool: {name}" + if tool_id: + header += f" | id: {tool_id}" + header += f" | status: {status}" + lines = [indent(header, 1)] + + if "input" in tool_call: + lines.append(indent("Input:", 2)) + lines.append(indent(pretty_format_json(tool_call["input"]), 3)) + + output = tool_call.get("output") + if isinstance(output, dict): + content = output.get("content") + if isinstance(content, list): + texts = [ + item.get("text", "") + for item in content + if isinstance(item, dict) and item.get("type") == "text" + ] + if any(texts): + lines.append(indent("Output:", 2)) + lines.append(indent("\n".join(texts).rstrip(), 3)) + if "details" in output: + lines.append(indent("Details:", 2)) + lines.append(indent(pretty_format_json(output["details"]), 3)) + elif output not in (None, ""): + lines.append(indent("Output:", 2)) + lines.append(indent(str(output).rstrip(), 3)) + + return lines + + +def format_payload(payload: dict[str, Any]) -> list[str]: + lines: list[str] = [] + if role := payload.get("role"): + lines.append(indent(f"Role: {role}", 1)) + add_text(lines, "Thinking", payload.get("thinking")) + add_text(lines, "Content", payload.get("content")) + if finish_reason := payload.get("finishReason"): + lines.append(indent(f"Finish reason: {finish_reason}", 1)) + + for tool_call in payload.get("toolCalls") or []: + if isinstance(tool_call, dict): + lines.extend(format_tool_call(tool_call)) + + usage = payload.get("usage") + if isinstance(usage, dict): + ordered_keys = ( + "totalTokens", + "inputTokens", + "outputTokens", + "cacheReadTokens", + "cacheWriteTokens", + "requestDurationMs", + ) + bits = [f"{key}={usage[key]}" for key in ordered_keys if key in usage] + if bits: + lines.append(indent(f"Usage: {', '.join(bits)}", 1)) + return lines + + +def format_event(index: int, event: dict[str, Any], wall_ts: str | None = None) -> str: + event_type = str(event.get("type", "unknown")) + header_bits = [f"type: {event_type}"] + if status := event.get("status"): + header_bits.append(f"status: {status}") + if event_type == "generic" and (generic_type := event.get("eventType")): + header_bits.append(f"eventType: {generic_type}") + if wall_ts: + header_bits.append(f"ts: {wall_ts}") + lines = [f"=== Event {index} | {' | '.join(header_bits)} ==="] + + if event_type in {"delta", "message"}: + payload = event.get("message", event) + if isinstance(payload, dict): + lines.extend(format_payload(payload)) + elif event_type == "exec.result": + for key, label in (("sessionId", "Session"), ("turnId", "Turn"), ("model", "Model")): + if value := event.get(key): + lines.append(indent(f"{label}: {value}", 1)) + if "durationMs" in event: + lines.append(indent(f"Duration: {event['durationMs']} ms", 1)) + add_text(lines, "Answer", event.get("answer")) + add_text(lines, "Error", event.get("error")) + elif event_type == "generic": + if "data" in event: + lines.append(indent("Data:", 1)) + lines.append(indent(pretty_format_json(event["data"]), 2)) + else: + for key, label in (("sessionId", "Session"), ("turnId", "Turn"), ("messageId", "Message")): + if value := event.get(key): + lines.append(indent(f"{label}: {value}", 1)) + + return "\n".join(lines) + + +def format_unparsable_line(index: int, line: str, error: str) -> str: + return ( + f"=== Event {index} | NOT PARSABLE ===\n" + f" Error: {error}\n" + f" Raw:\n{indent(line, 2)}" + ) + + +def parse(input_path: Path, output_path: Path) -> None: + formatted_events: list[str] = [] + with input_path.open("r", encoding="utf-8") as stream: + for raw_line in stream: + stripped = raw_line.strip() + if not stripped: + continue + + wall_ts = None + if ts_match := TIMESTAMP_PREFIX_RE.match(stripped): + wall_ts = ts_match.group(1) + stripped = stripped[ts_match.end():] + + try: + event = json.loads(stripped) + if not isinstance(event, dict): + raise ValueError("Parsed JSON is not an object") + except (json.JSONDecodeError, ValueError) as exc: + formatted_events.append( + format_unparsable_line(len(formatted_events) + 1, stripped, str(exc)) + ) + continue + + formatted_events.append( + format_event(len(formatted_events) + 1, event, wall_ts) + ) + + output_path.write_text("\n\n".join(formatted_events) + "\n", encoding="utf-8") diff --git a/src/trace_parsing/parse_trace.py b/src/trace_parsing/parse_trace.py index 7728b63..94b5eb2 100644 --- a/src/trace_parsing/parse_trace.py +++ b/src/trace_parsing/parse_trace.py @@ -1,8 +1,8 @@ """Parse an agent or judge trace into a human-readable transcript. Picks the right per-agent parser by substring-matching the agent name against -{claude, codex, cursor, gemini, opencode}. If the name matches zero keys, the -input is copied verbatim (preserves the historical fallback for agents like +{claude, codex, cursor, gemini, mcode, opencode}. If the name matches zero keys, +the input is copied verbatim (preserves the historical fallback for agents like glm5 and qwen3max that don't produce a structured trace). If the name matches more than one key, the script errors out instead of guessing. @@ -21,6 +21,7 @@ import codex_parser import cursor_parser import gemini_parser +import mcode_parser import opencode_parser import sanitize_trace @@ -29,6 +30,7 @@ "codex": codex_parser.parse, "cursor": cursor_parser.parse, "gemini": gemini_parser.parse, + "mcode": mcode_parser.parse, "opencode": opencode_parser.parse, } diff --git a/src/trace_parsing/test_mcode_parser.py b/src/trace_parsing/test_mcode_parser.py new file mode 100644 index 0000000..1c124c1 --- /dev/null +++ b/src/trace_parsing/test_mcode_parser.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import mcode_parser +import parse_trace + + +class MCodeParserTest(unittest.TestCase): + def test_parse_stream_json_trace(self) -> None: + trace = """[2026-08-17T01:02:03Z] {\"type\":\"session-status\",\"status\":\"started\",\"turnId\":\"turn-test\"} +{\"type\":\"delta\",\"role\":\"assistant\",\"thinking\":\"checking\",\"content\":\"hello\",\"toolCalls\":[{\"id\":\"tool-test\",\"name\":\"bash\",\"status\":1,\"input\":{\"command\":\"pwd\"}}]} +{\"type\":\"delta\",\"role\":\"assistant\",\"toolCalls\":[{\"id\":\"tool-test\",\"name\":\"bash\",\"status\":2,\"input\":{\"command\":\"pwd\"},\"output\":{\"content\":[{\"type\":\"text\",\"text\":\"/home/ben/task\\n\"}],\"details\":{\"exitCode\":0}}}]} +{\"type\":\"message\",\"message\":{\"role\":\"assistant\",\"content\":\"done\",\"finishReason\":\"stop\",\"usage\":{\"totalTokens\":12,\"inputTokens\":8,\"outputTokens\":4,\"cacheReadTokens\":2}}} +{\"schemaVersion\":1,\"type\":\"exec.result\",\"sessionId\":\"session-test\",\"turnId\":\"turn-test\",\"status\":\"succeeded\",\"answer\":\"done\",\"durationMs\":321} +not-json +""" + with tempfile.TemporaryDirectory() as tmp: + input_path = Path(tmp) / "trace.jsonl" + output_path = Path(tmp) / "trace.txt" + input_path.write_text(trace, encoding="utf-8") + + mcode_parser.parse(input_path, output_path) + parsed = output_path.read_text(encoding="utf-8") + + self.assertIn("type: session-status | status: started", parsed) + self.assertIn("ts: 2026-08-17T01:02:03Z", parsed) + self.assertIn("Thinking:\n checking", parsed) + self.assertIn("Content:\n hello", parsed) + self.assertIn("Tool: bash | id: tool-test | status: running", parsed) + self.assertIn('"command": "pwd"', parsed) + self.assertIn("/home/ben/task", parsed) + self.assertIn("exitCode", parsed) + self.assertIn("totalTokens=12", parsed) + self.assertIn("Answer:\n done", parsed) + self.assertIn("Duration: 321 ms", parsed) + self.assertIn("NOT PARSABLE", parsed) + self.assertIn("not-json", parsed) + + def test_parse_trace_dispatches_mcode(self) -> None: + self.assertIs(parse_trace.select_parser("mcode"), mcode_parser.parse) + self.assertIs(parse_trace.select_parser("mcode-minimax-m3"), mcode_parser.parse) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/utils/update_agent_cli.sh b/src/utils/update_agent_cli.sh index 810a29a..8a8eaa5 100755 --- a/src/utils/update_agent_cli.sh +++ b/src/utils/update_agent_cli.sh @@ -27,6 +27,7 @@ case "$BIN" in claude) PKG="@anthropic-ai/claude-code" ;; codex) PKG="@openai/codex" ;; gemini) PKG="@google/gemini-cli" ;; + mcode) PKG="@minimax-ai/code" ;; opencode) PKG="opencode-ai" ;; *) echo "[update_agent_cli] ERROR: no npm package mapping for binary '$BIN'" >&2