From 05f052c158d9a0298a1824d741f253b82a19828e Mon Sep 17 00:00:00 2001
From: amd-mkarvir <272370325+amd-mkarvir@users.noreply.github.com>
Date: Mon, 31 Aug 2026 23:04:43 -0700
Subject: [PATCH 1/2] Add advisory Codex skill eval backend

---
 .github/workflows/codex-evals.yml | 115 +++++++++++++++
 docs/evals.md                     |  27 +++-
 eval/agent.py                     | 233 +++++++++++++++++++++++-------
 eval/codex_backend.py             | 183 +++++++++++++++++++++++
 eval/routing.py                   | 144 +++++++++++++-----
 eval/run_evals.py                 | 153 +++++++++++++++-----
 eval/test_evals.py                |  75 ++++++++++
 7 files changed, 802 insertions(+), 128 deletions(-)
 create mode 100644 .github/workflows/codex-evals.yml
 create mode 100644 eval/codex_backend.py

diff --git a/.github/workflows/codex-evals.yml b/.github/workflows/codex-evals.yml
new file mode 100644
index 0000000..65a9218
--- /dev/null
+++ b/.github/workflows/codex-evals.yml
@@ -0,0 +1,115 @@
+name: codex-evals
+
+# Codex is a second consumer of the same per-skill datasets used by evals.yml.
+# Keep this signal advisory while the repository establishes a baseline. It is
+# deliberately separate from the required `evals` aggregate gate.
+
+on:
+  pull_request:
+    paths:
+      - "skills/**"
+      - "eval/**"
+      - ".codex-plugin/plugin.json"
+      - ".agents/plugins/marketplace.json"
+      - ".github/workflows/codex-evals.yml"
+  workflow_dispatch:
+    inputs:
+      only:
+        description: "Comma-separated routing case ids (blank = all required cases)."
+        required: false
+        default: ""
+      model:
+        description: "Codex model override (blank = CLI default)."
+        required: false
+        default: ""
+      min_accuracy:
+        description: "Optional routing accuracy floor (0-1)."
+        required: false
+        default: "0"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: codex-evals-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  routing:
+    name: Codex routing (advisory)
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Check whether Codex credentials are available
+        id: auth
+        shell: bash
+        run: |
+          if [ -n "${OPENAI_API_KEY:-}" ]; then
+            echo "available=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "available=false" >> "$GITHUB_OUTPUT"
+            echo "::notice::OPENAI_API_KEY is unavailable; Codex evals are skipped."
+            echo "Fork pull requests do not receive repository secrets." >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Set up Python
+        if: steps.auth.outputs.available == 'true'
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        if: steps.auth.outputs.available == 'true'
+        uses: astral-sh/setup-uv@v7
+
+      - name: Set up Node
+        if: steps.auth.outputs.available == 'true'
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install Codex CLI
+        if: steps.auth.outputs.available == 'true'
+        run: npm install -g @openai/codex
+
+      - name: Record Codex CLI version
+        if: steps.auth.outputs.available == 'true'
+        run: codex --version
+
+      - name: Run Codex routing evals
+        if: steps.auth.outputs.available == 'true'
+        env:
+          ONLY: ${{ github.event.inputs.only }}
+          MODEL: ${{ github.event.inputs.model }}
+          MIN_ACCURACY: ${{ github.event.inputs.min_accuracy || '0' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          model_args=()
+          if [ -n "${MODEL:-}" ]; then
+            model_args=(--model "$MODEL")
+          fi
+          uv run --with pyyaml python eval/run_evals.py \
+            --agent codex \
+            --mode routing \
+            --no-extended \
+            --only "${ONLY:-}" \
+            --min-accuracy "$MIN_ACCURACY" \
+            "${model_args[@]}" \
+            --output codex-routing-report.json \
+            --keep-logs codex-routing-logs
+
+      - name: Upload Codex routing report
+        if: always() && steps.auth.outputs.available == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: codex-routing-report
+          path: |
+            codex-routing-report.json
+            codex-routing-logs/
+          if-no-files-found: warn
diff --git a/docs/evals.md b/docs/evals.md
index bde3aaa..76d8447 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -114,6 +114,31 @@ python eval/run_evals.py --mode routing          # the published bundle
 python eval/run_evals.py --only <case-id> --keep-logs logs   # one case, keeping the transcript
 ```
 
-Everything but `--validate` needs the `claude` CLI authenticated, plus whatever your own cases need. No `pip install`: the runner is standard library only.
+These commands use Claude by default. Everything but `--validate` needs the
+`claude` CLI authenticated, plus whatever the cases need. No `pip install` is
+needed for the runner itself.
+
+The same datasets can also evaluate Codex:
+
+```bash
+npm install -g @openai/codex
+export OPENAI_API_KEY="..."
+python eval/run_evals.py --agent codex --mode routing --no-extended
+python eval/run_evals.py --agent codex --mode behavior --skill <your-skill> --no-extended
+```
+
+For Codex, the runner builds a temporary plugin containing exactly the skills
+under test and installs it into a temporary `CODEX_HOME`. This prevents a
+developer's personal plugins, skills, and settings from changing the result.
+The model defaults to the Codex CLI default; pass `--model <model>` to pin one.
+Codex JSON reports use `codex-routing-*` and `codex-behavior-*` names so they
+can be kept beside Claude reports.
 
 In CI, the `evals` workflow runs routing when a change can move a routing decision (a published description, any dataset, or the bundle itself), and runs behavior for the skills a change touches.
+
+The separate `codex-evals` workflow runs Codex routing as an advisory signal.
+It is intentionally outside the required `evals` gate while a baseline is
+being established, and skips cleanly when the repository's `OPENAI_API_KEY`
+secret is unavailable (including pull requests from forks). Codex behavior
+evals remain opt-in because they may need the hardware and setup declared by
+each skill.
diff --git a/eval/agent.py b/eval/agent.py
index 816f3de..8b0275a 100644
--- a/eval/agent.py
+++ b/eval/agent.py
@@ -2,7 +2,7 @@
 #
 # See LICENSE for license information.
 
-"""Agent staging and grading for behavior-mode eval runs.
+"""Agent staging and grading for Claude and Codex behavior-mode eval runs.
 
 One skill is copied into an isolated temp workspace, one prompt is run to
 completion, and the result is graded against a case's expectations::
@@ -43,6 +43,7 @@
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 from datasets import SKILLS_DIR  # noqa: E402
+import codex_backend  # noqa: E402
 
 DEFAULT_MODEL = os.environ.get("EVAL_MODEL", "opus")
 DEFAULT_EFFORT = os.environ.get("EVAL_EFFORT", "high")
@@ -126,7 +127,7 @@ def check_api_reachable(model: str | None = DEFAULT_MODEL, timeout: int = 60) ->
     return True, "ok"
 
 
-def _stage_workspace(skill: str, seed: Path | None = None) -> Path:
+def _stage_workspace(skill: str, seed: Path | None = None, backend: str = "claude") -> Path:
     """Copy ``skill`` into an isolated temp workspace and return its path.
 
     ``seed`` is a directory of fixture files (a case's ``workspace``) whose
@@ -138,9 +139,10 @@ def _stage_workspace(skill: str, seed: Path | None = None) -> Path:
         raise FileNotFoundError(f"skill '{skill}' not found at {skill_src / 'SKILL.md'}")
 
     workspace = Path(tempfile.mkdtemp(prefix=f"behavior-{skill}-"))
-    dest = workspace / ".claude" / "skills" / skill
-    dest.parent.mkdir(parents=True, exist_ok=True)
-    shutil.copytree(skill_src, dest)
+    if backend == "claude":
+        dest = workspace / ".claude" / "skills" / skill
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copytree(skill_src, dest)
 
     if seed is not None:
         if not seed.is_dir():
@@ -150,27 +152,54 @@ def _stage_workspace(skill: str, seed: Path | None = None) -> Path:
     return workspace
 
 
-def _run_agent(prompt_text: str, workspace: Path, model: str | None, effort: str | None) -> list[dict]:
+def _run_agent(
+    prompt_text: str,
+    workspace: Path,
+    model: str | None,
+    effort: str | None,
+    *,
+    backend: str = "claude",
+    codex_home: Path | None = None,
+) -> list[dict]:
     """Run the agent once in ``workspace`` and return the stream-json events."""
-    claude_bin = shutil.which("claude")
-    if not claude_bin:
-        raise RuntimeError("'claude' CLI not found on PATH")
-
-    cmd = [
-        claude_bin, "-p",
-        "--output-format", "stream-json", "--verbose",
-        "--dangerously-skip-permissions",
-        "--add-dir", str(workspace),
-    ]
-    if model:
-        cmd += ["--model", model]
-    if effort:
-        cmd += ["--effort", effort]
+    if backend == "codex":
+        if codex_home is None:
+            raise RuntimeError("Codex behavior run requires an isolated Codex home")
+        cmd = codex_backend.exec_command(
+            workspace, model=model, effort=effort, sandbox="workspace-write"
+        )
+        proc = subprocess.run(
+            cmd,
+            cwd=str(workspace),
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            input=prompt_text,
+            env=codex_backend.codex_env(codex_home),
+        )
+        cli_name = "codex"
+    else:
+        claude_bin = shutil.which("claude")
+        if not claude_bin:
+            raise RuntimeError("'claude' CLI not found on PATH")
+
+        cmd = [
+            claude_bin, "-p",
+            "--output-format", "stream-json", "--verbose",
+            "--dangerously-skip-permissions",
+            "--add-dir", str(workspace),
+        ]
+        if model:
+            cmd += ["--model", model]
+        if effort:
+            cmd += ["--effort", effort]
 
-    proc = subprocess.run(
-        cmd, cwd=str(workspace), capture_output=True, text=True,
-        encoding="utf-8", input=prompt_text, env=claude_env(),
-    )
+        proc = subprocess.run(
+            cmd, cwd=str(workspace), capture_output=True, text=True,
+            encoding="utf-8", input=prompt_text, env=claude_env(),
+        )
+        cli_name = "claude"
 
     events: list[dict] = []
     for line in (proc.stdout or "").splitlines():
@@ -184,9 +213,12 @@ def _run_agent(prompt_text: str, workspace: Path, model: str | None, effort: str
 
     if not events:
         raise RuntimeError(
-            f"claude exited with code {proc.returncode} and produced no "
+            f"{cli_name} exited with code {proc.returncode} and produced no "
             f"parseable stream-json output. stderr:\n{proc.stderr}"
         )
+    if proc.returncode != 0:
+        detail = (proc.stderr or proc.stdout or f"exit code {proc.returncode}").strip()
+        raise RuntimeError(f"{cli_name} exited with code {proc.returncode}: {detail[:1000]}")
     return events
 
 
@@ -204,6 +236,17 @@ def _walk(obj, tool_uses, tool_results) -> None:
                 for c in content:
                     if isinstance(c, dict) and isinstance(c.get("text"), str):
                         tool_results.append(c["text"])
+        elif otype in {"command_execution", "mcp_tool_call", "file_change"}:
+            name = {
+                "command_execution": "command_execution",
+                "mcp_tool_call": str(obj.get("tool") or obj.get("name") or "mcp_tool_call"),
+                "file_change": "file_change",
+            }[otype]
+            value = obj.get("command") or obj.get("arguments") or obj.get("changes") or {}
+            tool_uses.append((name, json.dumps(value, ensure_ascii=False)))
+            output = obj.get("aggregated_output") or obj.get("output") or obj.get("result")
+            if isinstance(output, str):
+                tool_results.append(output)
         for v in obj.values():
             _walk(v, tool_uses, tool_results)
     elif isinstance(obj, list):
@@ -211,10 +254,28 @@ def _walk(obj, tool_uses, tool_results) -> None:
             _walk(v, tool_uses, tool_results)
 
 
+def _codex_agent_messages(stdout: str) -> str:
+    """Return agent-message text from Codex JSONL, ignoring non-JSON noise."""
+    messages: list[str] = []
+    for line in stdout.splitlines():
+        if not line.strip():
+            continue
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        item = event.get("item", {}) if isinstance(event, dict) else {}
+        if isinstance(item, dict) and item.get("type") == "agent_message":
+            text = item.get("text")
+            if isinstance(text, str):
+                messages.append(text)
+    return "\n".join(messages)
+
+
 def _list_workspace_files(workspace: Path) -> list[str]:
     files: list[str] = []
     for p in sorted(workspace.rglob("*")):
-        if ".claude" in p.relative_to(workspace).parts:
+        if any(part in {".claude", ".codex"} for part in p.relative_to(workspace).parts):
             continue
         if p.is_file():
             files.append(str(p.relative_to(workspace)).replace("\\", "/"))
@@ -258,10 +319,6 @@ def _grade_with_llm(
     The grader may read files in the workspace (e.g. open out.png), so the
     workspace is added and tool permissions are bypassed for the grader too.
     """
-    claude_bin = shutil.which("claude")
-    if not claude_bin:
-        return False, "llm_judge skipped: 'claude' CLI not on PATH"
-
     cmd_text = run.command_text
     if len(cmd_text) > 4000:
         cmd_text = cmd_text[:4000] + "\n...[truncated]..."
@@ -294,28 +351,43 @@ def _grade_with_llm(
         "Respond with ONLY a single-line JSON object and nothing else: "
         '{"pass": true|false, "reason": "<one short sentence, no braces>"}'
     )
-    cmd = [
-        claude_bin, "-p",
-        "--output-format", "json",
-        "--dangerously-skip-permissions",
-        "--add-dir", str(run.workspace),
-    ]
-    if judge_model:
-        cmd += ["--model", judge_model]
+    if run.backend == "codex":
+        if run.codex_home is None:
+            return False, "llm_judge skipped: isolated Codex home is unavailable"
+        cmd = codex_backend.exec_command(
+            run.workspace, model=judge_model, effort=None, sandbox="read-only"
+        )
+        env = codex_backend.codex_env(run.codex_home)
+    else:
+        claude_bin = shutil.which("claude")
+        if not claude_bin:
+            return False, "llm_judge skipped: 'claude' CLI not on PATH"
+        cmd = [
+            claude_bin, "-p",
+            "--output-format", "json",
+            "--dangerously-skip-permissions",
+            "--add-dir", str(run.workspace),
+        ]
+        if judge_model:
+            cmd += ["--model", judge_model]
+        env = claude_env()
 
     try:
         proc = subprocess.run(
             cmd, capture_output=True, text=True, encoding="utf-8",
-            input=prompt_text, timeout=180, env=claude_env(),
+            input=prompt_text, timeout=180, env=env,
         )
     except subprocess.TimeoutExpired:
         return False, "llm_judge timed out after 180s"
 
-    try:
-        payload = json.loads((proc.stdout or "").strip())
-        verdict_text = payload.get("result", "") if isinstance(payload, dict) else ""
-    except json.JSONDecodeError:
-        verdict_text = (proc.stdout or "").strip()
+    if run.backend == "codex":
+        verdict_text = _codex_agent_messages(proc.stdout or "")
+    else:
+        try:
+            payload = json.loads((proc.stdout or "").strip())
+            verdict_text = payload.get("result", "") if isinstance(payload, dict) else ""
+        except json.JSONDecodeError:
+            verdict_text = (proc.stdout or "").strip()
 
     # A chatty judge may wrap the verdict in prose, and its reason may itself
     # contain braces (a regex quantifier, a quoted JSON snippet), so let the
@@ -353,7 +425,15 @@ class Check:
 class Run:
     """The captured result of one agent run."""
 
-    def __init__(self, *, workspace: Path, events: list[dict], judge_model: str | None) -> None:
+    def __init__(
+        self,
+        *,
+        workspace: Path,
+        events: list[dict],
+        judge_model: str | None,
+        backend: str = "claude",
+        codex_home: Path | None = None,
+    ) -> None:
         tool_uses: list[tuple[str, str]] = []
         tool_results: list[str] = []
         for ev in events:
@@ -363,9 +443,18 @@ def __init__(self, *, workspace: Path, events: list[dict], judge_model: str | No
         for ev in events:
             if ev.get("type") == "result" and isinstance(ev.get("result"), str):
                 result_text = ev["result"]
+            item = ev.get("item")
+            if (
+                isinstance(item, dict)
+                and item.get("type") == "agent_message"
+                and isinstance(item.get("text"), str)
+            ):
+                result_text = item["text"]
 
         self.workspace = workspace
         self.judge_model = judge_model
+        self.backend = backend
+        self.codex_home = codex_home
         self.files = _list_workspace_files(workspace)
         self.tool_names = {name for name, _ in tool_uses if name}
         self.result_text = result_text
@@ -469,16 +558,20 @@ def __init__(
         skill: str,
         effort: str | None = DEFAULT_EFFORT,
         seed: Path | None = None,
+        backend: str = "claude",
+        codex_home: Path | None = None,
     ) -> None:
         # Coerce here so the agent run and the LLM judge share the capped model.
-        self.model = enforce_model_policy(model)
+        self.model = enforce_model_policy(model) if backend == "claude" else model
         self.skill = skill
         self.effort = effort
         self.seed = seed
+        self.backend = backend
+        self.codex_home = codex_home
         self.workspace: Path | None = None
 
     def __enter__(self) -> "Agent":
-        self.workspace = _stage_workspace(self.skill, self.seed)
+        self.workspace = _stage_workspace(self.skill, self.seed, self.backend)
         return self
 
     def __exit__(self, *exc) -> None:
@@ -491,9 +584,25 @@ def prompt(self, text: str) -> Run:
         if self.workspace is None:
             raise RuntimeError("Agent.prompt() must be called inside a 'with' block")
 
-        _safe_print(f"\n[behavior] skill='{self.skill}' model='{self.model}': {text}")
-        events = _run_agent(text, self.workspace, self.model, self.effort)
-        return Run(workspace=self.workspace, events=events, judge_model=self.model)
+        _safe_print(
+            f"\n[behavior] agent='{self.backend}' skill='{self.skill}' "
+            f"model='{self.model or 'default'}': {text}"
+        )
+        events = _run_agent(
+            text,
+            self.workspace,
+            self.model,
+            self.effort,
+            backend=self.backend,
+            codex_home=self.codex_home,
+        )
+        return Run(
+            workspace=self.workspace,
+            events=events,
+            judge_model=self.model,
+            backend=self.backend,
+            codex_home=self.codex_home,
+        )
 
 
 def claude(
@@ -503,5 +612,27 @@ def claude(
     effort: str | None = DEFAULT_EFFORT,
     seed: Path | None = None,
 ) -> Agent:
-    """Factory for a Claude-backed `Agent` (the only agent backend today)."""
+    """Factory for a Claude-backed `Agent`."""
     return Agent(model, skill=skill, effort=effort, seed=seed)
+
+
+def session(
+    backend: str,
+    model: str | None,
+    *,
+    skill: str,
+    effort: str | None,
+    seed: Path | None = None,
+    codex_home: Path | None = None,
+) -> Agent:
+    """Create a behavior session for the requested agent backend."""
+    if backend not in {"claude", "codex"}:
+        raise ValueError(f"unsupported agent backend: {backend}")
+    return Agent(
+        model,
+        skill=skill,
+        effort=effort,
+        seed=seed,
+        backend=backend,
+        codex_home=codex_home,
+    )
diff --git a/eval/codex_backend.py b/eval/codex_backend.py
new file mode 100644
index 0000000..cbf3928
--- /dev/null
+++ b/eval/codex_backend.py
@@ -0,0 +1,183 @@
+# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+
+"""Isolated Codex CLI setup shared by routing and behavior evals."""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+from contextlib import AbstractContextManager
+from pathlib import Path
+
+from datasets import SKILLS_DIR
+
+PLUGIN_NAME = "amd-skills-eval"
+
+
+def codex_env(home: Path) -> dict[str, str]:
+    """Return a subprocess environment with an isolated Codex home."""
+    env = dict(os.environ)
+    env["CODEX_HOME"] = str(home)
+    return env
+
+
+def _run_setup(cmd: list[str], env: dict[str, str]) -> None:
+    proc = subprocess.run(
+        cmd,
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        timeout=120,
+        env=env,
+    )
+    if proc.returncode != 0:
+        detail = (proc.stderr or proc.stdout or f"exit code {proc.returncode}").strip()
+        raise RuntimeError(f"Codex plugin setup failed: {detail[:1000]}")
+
+
+class CodexInstall(AbstractContextManager[Path]):
+    """Install exactly ``skills`` as a local plugin in a throwaway Codex home."""
+
+    def __init__(self, skills: list[str]) -> None:
+        self.skills = skills
+        self.root: Path | None = None
+        self.home: Path | None = None
+
+    def __enter__(self) -> Path:
+        codex_bin = shutil.which("codex")
+        if not codex_bin:
+            raise RuntimeError("'codex' CLI not found on PATH")
+        if not self.skills:
+            raise RuntimeError("cannot build a Codex eval plugin with no skills")
+
+        self.root = Path(tempfile.mkdtemp(prefix="codex-eval-"))
+        try:
+            self.home = self.root / "home"
+            source = self.root / "plugin"
+            (source / ".codex-plugin").mkdir(parents=True)
+            (source / ".agents" / "plugins").mkdir(parents=True)
+            (source / "skills").mkdir()
+
+            skill_paths: list[str] = []
+            for skill in self.skills:
+                src = SKILLS_DIR / skill
+                if not (src / "SKILL.md").is_file():
+                    raise FileNotFoundError(
+                        f"skill '{skill}' not found at {src / 'SKILL.md'}"
+                    )
+                shutil.copytree(src, source / "skills" / skill)
+                skill_paths.append(f"./skills/{skill}")
+
+            plugin = {
+                "name": PLUGIN_NAME,
+                "version": "0.0.0-eval",
+                "description": "Temporary isolated plugin for AMD skill evaluations.",
+                "skills": skill_paths,
+            }
+            marketplace = {
+                "name": PLUGIN_NAME,
+                "plugins": [
+                    {
+                        "name": PLUGIN_NAME,
+                        "source": {"source": "local", "path": "./"},
+                        "policy": {"installation": "AVAILABLE"},
+                    }
+                ],
+            }
+            (source / ".codex-plugin" / "plugin.json").write_text(
+                json.dumps(plugin, indent=2) + "\n", encoding="utf-8"
+            )
+            (source / ".agents" / "plugins" / "marketplace.json").write_text(
+                json.dumps(marketplace, indent=2) + "\n", encoding="utf-8"
+            )
+
+            self.home.mkdir()
+            env = codex_env(self.home)
+            _run_setup(
+                [codex_bin, "plugin", "marketplace", "add", str(source), "--json"],
+                env,
+            )
+            _run_setup(
+                [codex_bin, "plugin", "add", f"{PLUGIN_NAME}@{PLUGIN_NAME}", "--json"],
+                env,
+            )
+            return self.home
+        except Exception:
+            self.__exit__(None, None, None)
+            raise
+
+    def __exit__(self, *exc) -> None:
+        if self.root is not None:
+            shutil.rmtree(self.root, ignore_errors=True)
+        self.root = None
+        self.home = None
+
+
+def install(skills: list[str]) -> CodexInstall:
+    """Return a context manager for an isolated Codex plugin installation."""
+    return CodexInstall(skills)
+
+
+def exec_command(
+    workspace: Path,
+    *,
+    model: str | None,
+    effort: str | None,
+    sandbox: str,
+) -> list[str]:
+    """Build a non-interactive Codex JSONL command for one eval prompt."""
+    codex_bin = shutil.which("codex")
+    if not codex_bin:
+        raise RuntimeError("'codex' CLI not found on PATH")
+    cmd = [
+        codex_bin,
+        "exec",
+        "--json",
+        "--ephemeral",
+        "--ignore-rules",
+        "--skip-git-repo-check",
+        "--sandbox",
+        sandbox,
+        "--cd",
+        str(workspace),
+    ]
+    if sandbox == "workspace-write":
+        cmd.append("--approve-for-me")
+    if model:
+        cmd += ["--model", model]
+    if effort:
+        cmd += ["--config", f'model_reasoning_effort="{effort}"']
+    cmd.append("-")
+    return cmd
+
+
+def check_api_reachable(
+    home: Path, model: str | None = None, effort: str | None = None, timeout: int = 60
+) -> tuple[bool, str]:
+    """Confirm Codex can authenticate and reach its API from an isolated home."""
+    workspace = Path(tempfile.mkdtemp(prefix="codex-preflight-"))
+    try:
+        proc = subprocess.run(
+            exec_command(workspace, model=model, effort=effort, sandbox="read-only"),
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            input="Reply with the single word: ok",
+            timeout=timeout,
+            env=codex_env(home),
+        )
+    except subprocess.TimeoutExpired:
+        return False, f"API preflight timed out after {timeout}s (is the network reachable?)"
+    finally:
+        shutil.rmtree(workspace, ignore_errors=True)
+    if proc.returncode != 0:
+        detail = (proc.stderr or proc.stdout or f"exit code {proc.returncode}").strip()
+        return False, detail[:500]
+    return True, "ok"
diff --git a/eval/routing.py b/eval/routing.py
index 6aa8f66..49db4ab 100644
--- a/eval/routing.py
+++ b/eval/routing.py
@@ -50,6 +50,7 @@
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 from agent import claude_env  # noqa: E402
+import codex_backend  # noqa: E402
 from datasets import SKILLS_DIR, Case  # noqa: E402
 
 # Tools that carry no routing signal. An agent often opens with a todo list or
@@ -84,6 +85,8 @@ class RoutingConfig:
     keep_logs: str = ""
     available_flags: set[str] = field(default_factory=set)
     isolate_config: bool = False
+    agent: str = "claude"
+    codex_home: Path | None = None
 
 
 @dataclass
@@ -105,7 +108,7 @@ class Outcome:
     error: str | None = None
 
 
-def stage_workspace(skills: list[str]) -> Path:
+def stage_workspace(skills: list[str], agent: str = "claude") -> Path:
     """Install every catalog skill into a fresh temp workspace.
 
     Claude Code loads ``.claude/skills/`` from a directory passed with
@@ -115,14 +118,15 @@ def stage_workspace(skills: list[str]) -> Path:
     lets them run concurrently).
     """
     workspace = Path(tempfile.mkdtemp(prefix="routing-"))
-    dest_root = workspace / ".claude" / "skills"
-    dest_root.mkdir(parents=True, exist_ok=True)
-    for skill in skills:
-        shutil.copytree(SKILLS_DIR / skill, dest_root / skill)
+    if agent == "claude":
+        dest_root = workspace / ".claude" / "skills"
+        dest_root.mkdir(parents=True, exist_ok=True)
+        for skill in skills:
+            shutil.copytree(SKILLS_DIR / skill, dest_root / skill)
     return workspace
 
 
-def supported_flags(flags: list[str]) -> set[str]:
+def supported_flags(flags: list[str], agent: str = "claude") -> set[str]:
     """Which of `flags` the installed `claude` build advertises in --help.
 
     The two cost-control flags this eval likes to pass are recent additions. An
@@ -130,12 +134,12 @@ def supported_flags(flags: list[str]) -> set[str]:
     reads like a routing collapse rather than a flag problem -- so check once
     (free, no tokens) and drop what isn't there.
     """
-    claude_bin = shutil.which("claude")
-    if not claude_bin:
+    cli_bin = shutil.which(agent)
+    if not cli_bin:
         return set()
     try:
         proc = subprocess.run(
-            [claude_bin, "--help"], capture_output=True, text=True, encoding="utf-8", timeout=60
+            [cli_bin, "--help"], capture_output=True, text=True, encoding="utf-8", timeout=60
         )
     except (subprocess.SubprocessError, OSError):
         return set()
@@ -168,6 +172,11 @@ def walk(node) -> None:
                         json.dumps(node.get("input", {}), ensure_ascii=False),
                     )
                 )
+            elif node.get("type") in {"command_execution", "mcp_tool_call", "file_change"}:
+                kind = str(node.get("type"))
+                name = str(node.get("tool") or node.get("name") or kind)
+                value = node.get("command") or node.get("arguments") or node.get("changes") or {}
+                found.append((name, json.dumps(value, ensure_ascii=False)))
             for value in node.values():
                 walk(value)
         elif isinstance(node, list):
@@ -247,6 +256,33 @@ def detect_activation(event: dict, skills: list[str], allow_body_path: bool = Tr
     is a contaminated runner, not a routing result, and the report should say
     so rather than silently scoring it as a miss.
     """
+    def find_codex_invocation(node) -> str | None:
+        if isinstance(node, dict):
+            if node.get("type") == "skill_invocation":
+                invoked = str(
+                    node.get("skill_name")
+                    or node.get("name")
+                    or node.get("skill")
+                    or node.get("skill_id")
+                    or ""
+                ).strip()
+                hit = _match_skill(invoked, skills)
+                return hit or f"other:{invoked or 'unknown'}"
+            for value in node.values():
+                hit = find_codex_invocation(value)
+                if hit:
+                    return hit
+        elif isinstance(node, list):
+            for value in node:
+                hit = find_codex_invocation(value)
+                if hit:
+                    return hit
+        return None
+
+    codex_hit = find_codex_invocation(event)
+    if codex_hit:
+        return codex_hit
+
     for name, tool_input in _iter_tool_uses(event):
         lowered = name.lower()
         if lowered in SKILL_TOOLS:
@@ -394,35 +430,47 @@ def classify(expect: str | None, observed: str | None) -> str:
 
 def run_case(case: Case, skills: list[str], config: RoutingConfig) -> Outcome:
     """Run one prompt, stopping as soon as the routing decision is known."""
-    claude_bin = shutil.which("claude")
-    if not claude_bin:
-        raise SystemExit("error: 'claude' CLI not found on PATH")
+    cli_bin = shutil.which(config.agent)
+    if not cli_bin:
+        raise SystemExit(f"error: '{config.agent}' CLI not found on PATH")
+    if config.agent == "codex" and config.codex_home is None:
+        raise SystemExit("error: Codex routing requires an isolated plugin installation")
 
-    workspace = stage_workspace(skills)
+    workspace = stage_workspace(skills, config.agent)
     # Outside the workspace: the agent can list its own cwd, and a config dir
     # sitting in there would be one more thing for it to find.
     config_dir = (
-        Path(tempfile.mkdtemp(prefix="routing-config-")) if config.isolate_config else None
+        Path(tempfile.mkdtemp(prefix="routing-config-"))
+        if config.agent == "claude" and config.isolate_config
+        else None
     )
-    cmd = [
-        claude_bin,
-        "-p",
-        "--output-format",
-        "stream-json",
-        "--verbose",
-        "--dangerously-skip-permissions",
-        "--add-dir",
-        str(workspace),
-        "--model",
-        config.model,
-    ]
-    if config.effort:
-        cmd += ["--effort", config.effort]
-    # Dozens of throwaway sessions per run; don't leave them on disk.
-    if "--no-session-persistence" in config.available_flags:
-        cmd += ["--no-session-persistence"]
-    if config.max_budget_usd > 0 and "--max-budget-usd" in config.available_flags:
-        cmd += ["--max-budget-usd", str(config.max_budget_usd)]
+    if config.agent == "codex":
+        cmd = codex_backend.exec_command(
+            workspace,
+            model=config.model or None,
+            effort=config.effort,
+            sandbox="read-only",
+        )
+    else:
+        cmd = [
+            cli_bin,
+            "-p",
+            "--output-format",
+            "stream-json",
+            "--verbose",
+            "--dangerously-skip-permissions",
+            "--add-dir",
+            str(workspace),
+            "--model",
+            config.model,
+        ]
+        if config.effort:
+            cmd += ["--effort", config.effort]
+        # Dozens of throwaway sessions per run; don't leave them on disk.
+        if "--no-session-persistence" in config.available_flags:
+            cmd += ["--no-session-persistence"]
+        if config.max_budget_usd > 0 and "--max-budget-usd" in config.available_flags:
+            cmd += ["--max-budget-usd", str(config.max_budget_usd)]
 
     spawn: dict = {}
     if os.name == "nt":
@@ -430,13 +478,17 @@ def run_case(case: Case, skills: list[str], config: RoutingConfig) -> Outcome:
     else:
         spawn["start_new_session"] = True
 
-    env = claude_env()
-    if config_dir is not None:
-        env["CLAUDE_CONFIG_DIR"] = str(config_dir)
+    if config.agent == "codex":
+        assert config.codex_home is not None
+        env = codex_backend.codex_env(config.codex_home)
+    else:
+        env = claude_env()
+        if config_dir is not None:
+            env["CLAUDE_CONFIG_DIR"] = str(config_dir)
 
     events: list[dict] = []
     observed: str | None = None
-    visible: list[str] = []
+    visible: list[str] = list(skills) if config.agent == "codex" else []
     extra: list[str] = []
     stop_reason = "completed"
     tool_calls = 0
@@ -512,6 +564,13 @@ def run_case(case: Case, skills: list[str], config: RoutingConfig) -> Outcome:
                 if event.get("is_error"):
                     error = str(event.get("result") or "result event reported an error")[:400]
                 break
+            if event.get("type") == "turn.completed":
+                stop_reason = "result"
+                break
+            if event.get("type") == "error":
+                stop_reason = "result"
+                error = str(event.get("message") or "Codex reported an error")[:400]
+                break
 
             for name, tool_input in _iter_tool_uses(event):
                 if name.lower() in BOOKKEEPING_TOOLS:
@@ -541,7 +600,10 @@ def run_case(case: Case, skills: list[str], config: RoutingConfig) -> Outcome:
             shutil.rmtree(config_dir, ignore_errors=True)
 
     if not events:
-        error = ("".join(stderr_lines).strip() or "claude produced no stream-json output")[:400]
+        error = (
+            "".join(stderr_lines).strip()
+            or f"{config.agent} produced no stream-json output"
+        )[:400]
 
     # "no skill activated" is only a real finding when the run got far enough to
     # show a decision: the agent answered (`result`) or started doing the work
@@ -668,6 +730,8 @@ def render_markdown(summary: dict) -> str:
     totals = summary["totals"]
     verdicts = summary["verdicts"]
     meta = summary["meta"]
+    model = meta.get("model") or "CLI default"
+    agent = meta.get("agent", "claude")
     accuracy = totals["accuracy"]
     lines = [
         "## Skill routing",
@@ -675,7 +739,7 @@ def render_markdown(summary: dict) -> str:
         f"**{totals['passed']}/{totals['graded']} correct "
         f"({'n/a' if accuracy is None else f'{accuracy:.1%}'})** across "
         f"{totals['cases']} prompts with the {len(meta['skills'])} published "
-        f"skills installed together, on `{meta['model']}` "
+        f"skills installed together with `{agent}` on `{model}` "
         f"(effort `{meta['effort']}`).",
         "",
     ]
@@ -786,7 +850,7 @@ def render_markdown(summary: dict) -> str:
             "> **Not a valid result:** no skill activated in any case, including "
             f"the {totals['activations_expected']} that expected one. The skills "
             "were probably not installed for the session, or the activation "
-            "detector no longer matches this `claude` build. Re-run with "
+            f"detector no longer matches this `{agent}` build. Re-run with "
             "`--keep-logs` and inspect a transcript before trusting these numbers.",
         ]
     if summary["unexpected_skills"]:
diff --git a/eval/run_evals.py b/eval/run_evals.py
index 449cbdf..89ed6bc 100644
--- a/eval/run_evals.py
+++ b/eval/run_evals.py
@@ -39,7 +39,11 @@
     # one case, keeping the raw transcript
     python eval/run_evals.py --only qwen-on-mi300x --keep-logs eval-logs
 
-Reports go to stdout as markdown, to ``$GITHUB_STEP_SUMMARY`` under Actions,
+    # run the same routing dataset through Codex
+    python eval/run_evals.py --agent codex --mode routing --no-extended
+
+Claude is the default backend; ``--agent codex`` selects Codex. Reports go to
+stdout as markdown, to ``$GITHUB_STEP_SUMMARY`` under Actions,
 and to a JSON artifact under ``eval/runs/``.
 """
 
@@ -64,11 +68,13 @@
 
 import datasets  # noqa: E402
 import routing  # noqa: E402
+import codex_backend  # noqa: E402
 from agent import (  # noqa: E402
     Check,
     check_api_reachable,
     claude,
     enforce_model_policy,
+    session as agent_session,
 )
 from datasets import Case  # noqa: E402
 
@@ -135,7 +141,13 @@ def _expand(text: str, ctx: dict) -> str:
 
 
 def run_behavior_case(
-    case: Case, ctx: dict, hooks: ModuleType | None, model: str, effort: str
+    case: Case,
+    ctx: dict,
+    hooks: ModuleType | None,
+    model: str,
+    effort: str,
+    agent_name: str = "claude",
+    codex_home: Path | None = None,
 ) -> BehaviorOutcome:
     """Stage one skill, run the prompt to completion, grade what happened."""
     assert case.skill is not None
@@ -146,7 +158,15 @@ def run_behavior_case(
     error: str | None = None
 
     try:
-        with claude(model, skill=case.skill, effort=effort, seed=seed) as session:
+        factory = claude if agent_name == "claude" else agent_session
+        kwargs = {"skill": case.skill, "effort": effort, "seed": seed}
+        if agent_name == "codex":
+            session_ctx = factory(
+                agent_name, model, codex_home=codex_home, **kwargs
+            )
+        else:
+            session_ctx = factory(model, **kwargs)
+        with session_ctx as session:
             workspace = session.workspace
             assert workspace is not None
             if hooks is not None and hasattr(hooks, "setup"):
@@ -193,7 +213,13 @@ def run_behavior_case(
     )
 
 
-def run_behavior(skills: list[str], cases: list[Case], model: str, effort: str) -> list[BehaviorOutcome]:
+def run_behavior(
+    skills: list[str],
+    cases: list[Case],
+    model: str,
+    effort: str,
+    agent_name: str = "claude",
+) -> list[BehaviorOutcome]:
     """Run every behavior case, grouped by skill so session setup happens once."""
     outcomes: list[BehaviorOutcome] = []
     for skill in skills:
@@ -201,20 +227,36 @@ def run_behavior(skills: list[str], cases: list[Case], model: str, effort: str)
         if not skill_cases:
             continue
 
-        hooks = _load_hooks(skill)
-        ctx: dict = {}
-        cache_dir: Path | None = None
-        if hooks is not None and hasattr(hooks, "setup_session"):
-            cache_dir = Path(tempfile.mkdtemp(prefix=f"evalcache-{skill}-"))
-            print(f"[behavior] {skill}: running evals/hooks.py setup_session()", flush=True)
-            ctx.update(hooks.setup_session(cache_dir) or {})
+        codex_install = codex_backend.install([skill]) if agent_name == "codex" else None
+        codex_home = codex_install.__enter__() if codex_install is not None else None
         try:
-            print(f"[behavior] {skill}: {len(skill_cases)} case(s)", flush=True)
-            for case in skill_cases:
-                outcomes.append(run_behavior_case(case, ctx, hooks, model, effort))
+            hooks = _load_hooks(skill)
+            ctx: dict = {}
+            cache_dir: Path | None = None
+            if hooks is not None and hasattr(hooks, "setup_session"):
+                cache_dir = Path(tempfile.mkdtemp(prefix=f"evalcache-{skill}-"))
+                print(f"[behavior] {skill}: running evals/hooks.py setup_session()", flush=True)
+                ctx.update(hooks.setup_session(cache_dir) or {})
+            try:
+                print(f"[behavior] {skill}: {len(skill_cases)} case(s)", flush=True)
+                for case in skill_cases:
+                    outcomes.append(
+                        run_behavior_case(
+                            case,
+                            ctx,
+                            hooks,
+                            model,
+                            effort,
+                            agent_name,
+                            codex_home,
+                        )
+                    )
+            finally:
+                if cache_dir is not None:
+                    shutil.rmtree(cache_dir, ignore_errors=True)
         finally:
-            if cache_dir is not None:
-                shutil.rmtree(cache_dir, ignore_errors=True)
+            if codex_install is not None:
+                codex_install.__exit__(None, None, None)
     return outcomes
 
 
@@ -245,12 +287,14 @@ def summarize_behavior(outcomes: list[BehaviorOutcome], meta: dict) -> dict:
 def render_behavior_markdown(summary: dict) -> str:
     totals = summary["totals"]
     meta = summary["meta"]
+    model = meta.get("model") or "CLI default"
+    agent = meta.get("agent", "claude")
     lines = [
         "## Skill behavior",
         "",
         f"**{totals['passed']}/{totals['cases']} cases passed** "
         f"({totals['checks_passed']}/{totals['checks']} individual expectations) "
-        f"on `{meta['model']}` (effort `{meta['effort']}`).",
+        f"with `{agent}` on `{model}` (effort `{meta['effort']}`).",
         "",
         "| Skill | Cases | Passed | Expectations | Met |",
         "| --- | --- | --- | --- | --- |",
@@ -330,6 +374,12 @@ def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
     )
+    parser.add_argument(
+        "--agent",
+        default="claude",
+        choices=["claude", "codex"],
+        help="Agent CLI to evaluate. Default: claude.",
+    )
     parser.add_argument(
         "--mode",
         default="both",
@@ -358,7 +408,11 @@ def build_parser() -> argparse.ArgumentParser:
         help="Check every dataset structurally and exit. No agent, no tokens.",
     )
     parser.add_argument("--list-skills", action="store_true", help="Print skills that have a dataset, as JSON.")
-    parser.add_argument("--model", default="opus", help="Model alias. CI pins this to opus. Default: opus.")
+    parser.add_argument(
+        "--model",
+        default="",
+        help="Model alias. Default: opus for Claude; the Codex CLI default for Codex.",
+    )
     parser.add_argument(
         "--effort", default="high", choices=["low", "medium", "high", "max"],
         help="Reasoning effort. Default: high.",
@@ -395,7 +449,10 @@ def build_parser() -> argparse.ArgumentParser:
     )
     parser.add_argument(
         "--max-budget-usd", type=float, default=0.75,
-        help="Per-case routing spend cap enforced by the CLI. 0 disables. Default: 0.75.",
+        help=(
+            "Per-case Claude routing spend cap enforced by that CLI. Codex "
+            "relies on early termination instead. 0 disables. Default: 0.75."
+        ),
     )
     parser.add_argument("--output", default="", help="Write the JSON report here. Default: eval/runs/<mode>-<timestamp>.json.")
     parser.add_argument("--summary", default="", help="Write the markdown report here (defaults to $GITHUB_STEP_SUMMARY when set).")
@@ -434,14 +491,23 @@ def main(argv: list[str] | None = None) -> int:
         )
         return 0
 
-    args.model = enforce_model_policy(args.model) or args.model
+    if not args.model:
+        args.model = "opus" if args.agent == "claude" else os.environ.get("CODEX_EVAL_MODEL", "")
+    if args.agent == "claude":
+        args.model = enforce_model_policy(args.model) or args.model
     skills = _selected_skills(args)
     catalog = datasets.routing_catalog()
 
     if not args.skip_preflight:
-        ok, detail = check_api_reachable(args.model)
+        if args.agent == "codex":
+            with codex_backend.install([skills[0]]) as preflight_home:
+                ok, detail = codex_backend.check_api_reachable(
+                    preflight_home, args.model or None, args.effort
+                )
+        else:
+            ok, detail = check_api_reachable(args.model)
         if not ok:
-            raise SystemExit(f"error: claude API not reachable -- {detail}")
+            raise SystemExit(f"error: {args.agent} API not reachable -- {detail}")
 
     failed = False
     started = time.time()
@@ -473,20 +539,24 @@ def main(argv: list[str] | None = None) -> int:
                 "installed and so could never win them."
             )
 
+        codex_install = codex_backend.install(catalog) if args.agent == "codex" else None
+        codex_home = codex_install.__enter__() if codex_install is not None else None
         config = routing.RoutingConfig(
             model=args.model,
             effort=args.effort,
             timeout=args.timeout,
             max_tool_calls=args.max_tool_calls,
             max_inspection_calls=args.max_inspection_calls,
-            max_budget_usd=args.max_budget_usd,
+            max_budget_usd=args.max_budget_usd if args.agent == "claude" else 0,
             keep_logs=args.keep_logs,
             available_flags=routing.supported_flags(
-                ["--no-session-persistence", "--max-budget-usd"]
+                ["--no-session-persistence", "--max-budget-usd"], args.agent
             ),
-            isolate_config=routing.can_isolate_config(),
+            isolate_config=(args.agent == "codex" or routing.can_isolate_config()),
+            agent=args.agent,
+            codex_home=codex_home,
         )
-        if not config.isolate_config:
+        if args.agent == "claude" and not config.isolate_config:
             print(
                 "[routing] warning: ANTHROPIC_API_KEY is not set, so the runner's "
                 "own config dir is used and any user-level skill in it joins the "
@@ -499,18 +569,26 @@ def main(argv: list[str] | None = None) -> int:
                 f"[routing] unpublished, so their prompts are not graded here: "
                 f"{', '.join(held_out)}"
             )
-        print(f"[routing] {len(cases)} cases, model={args.model}, jobs={args.jobs}")
-        if args.jobs > 1 and len(cases) > 1:
-            with ThreadPoolExecutor(max_workers=args.jobs) as pool:
-                outcomes = list(pool.map(lambda c: routing.run_case(c, catalog, config), cases))
-        else:
-            outcomes = [routing.run_case(case, catalog, config) for case in cases]
+        print(
+            f"[routing] {len(cases)} cases, agent={args.agent}, "
+            f"model={args.model or 'default'}, jobs={args.jobs}"
+        )
+        try:
+            if args.jobs > 1 and len(cases) > 1:
+                with ThreadPoolExecutor(max_workers=args.jobs) as pool:
+                    outcomes = list(pool.map(lambda c: routing.run_case(c, catalog, config), cases))
+            else:
+                outcomes = [routing.run_case(case, catalog, config) for case in cases]
+        finally:
+            if codex_install is not None:
+                codex_install.__exit__(None, None, None)
 
         summary = routing.summarize(
             outcomes,
             catalog,
             {
                 "model": args.model,
+                "agent": args.agent,
                 "effort": args.effort,
                 "skills": catalog,
                 "held_out_skills": held_out,
@@ -519,12 +597,13 @@ def main(argv: list[str] | None = None) -> int:
                 "max_tool_calls": args.max_tool_calls,
                 "max_inspection_calls": args.max_inspection_calls,
                 "isolated_config_dir": config.isolate_config,
-                "max_budget_usd": args.max_budget_usd,
+                "max_budget_usd": config.max_budget_usd,
                 "optional_cli_flags_used": sorted(config.available_flags),
                 "github_run_id": os.environ.get("GITHUB_RUN_ID"),
             },
         )
-        _write_report(summary, routing.render_markdown(summary), args, "routing")
+        routing_label = "routing" if args.agent == "claude" else "codex-routing"
+        _write_report(summary, routing.render_markdown(summary), args, routing_label)
 
         totals = summary["totals"]
         if totals["graded"] == 0:
@@ -562,11 +641,12 @@ def main(argv: list[str] | None = None) -> int:
                 "`files_exist` to a triggering evaluation."
             )
         else:
-            outcomes = run_behavior(skills, gradable, args.model, args.effort)
+            outcomes = run_behavior(skills, gradable, args.model, args.effort, args.agent)
             summary = summarize_behavior(
                 outcomes,
                 {
                     "model": args.model,
+                    "agent": args.agent,
                     "effort": args.effort,
                     "skills": skills,
                     "extended": args.extended,
@@ -574,7 +654,8 @@ def main(argv: list[str] | None = None) -> int:
                     "github_run_id": os.environ.get("GITHUB_RUN_ID"),
                 },
             )
-            _write_report(summary, render_behavior_markdown(summary), args, "behavior")
+            behavior_label = "behavior" if args.agent == "claude" else "codex-behavior"
+            _write_report(summary, render_behavior_markdown(summary), args, behavior_label)
             if summary["totals"]["passed"] != summary["totals"]["cases"]:
                 failed = True
 
diff --git a/eval/test_evals.py b/eval/test_evals.py
index acd5665..876552d 100644
--- a/eval/test_evals.py
+++ b/eval/test_evals.py
@@ -19,12 +19,14 @@
 import sys
 import tempfile
 import unittest
+from unittest import mock
 from pathlib import Path
 
 EVAL_DIR = Path(__file__).resolve().parent
 sys.path.insert(0, str(EVAL_DIR))
 
 import agent  # noqa: E402
+import codex_backend  # noqa: E402
 import datasets  # noqa: E402
 import routing  # noqa: E402
 import run_evals  # noqa: E402
@@ -506,6 +508,28 @@ def test_skill_tool_call_is_an_activation(self) -> None:
         event = self.event("Skill", {"command": "local-ai-use"})
         self.assertEqual(routing.detect_activation(event, self.SKILLS), "local-ai-use")
 
+    def test_codex_skill_invocation_is_an_activation(self) -> None:
+        event = {
+            "type": "item.completed",
+            "item": {
+                "type": "skill_invocation",
+                "skill_name": "amd-skills:serving-llms-on-instinct",
+            },
+        }
+        self.assertEqual(
+            routing.detect_activation(event, self.SKILLS),
+            "serving-llms-on-instinct",
+        )
+
+    def test_codex_skill_outside_catalog_is_flagged(self) -> None:
+        event = {
+            "type": "item.completed",
+            "item": {"type": "skill_invocation", "skill_name": "other-plugin:demo"},
+        }
+        self.assertEqual(
+            routing.detect_activation(event, self.SKILLS), "other:other-plugin:demo"
+        )
+
     def test_longest_name_wins_when_one_is_a_prefix_of_another(self) -> None:
         event = self.event("Skill", {"command": "local-ai-app-integration"})
         self.assertEqual(
@@ -598,6 +622,31 @@ def test_transcript_and_tools_are_captured(self) -> None:
         self.assertIn("detect.py", run.logs)
         self.assertEqual(run.result_text, "done")
 
+    def test_codex_transcript_and_final_message_are_captured(self) -> None:
+        events = [
+            {
+                "type": "item.completed",
+                "item": {
+                    "type": "command_execution",
+                    "command": "python detect.py",
+                    "aggregated_output": "ok",
+                },
+            },
+            {
+                "type": "item.completed",
+                "item": {"type": "agent_message", "text": "done with Codex"},
+            },
+        ]
+        run = agent.Run(
+            workspace=self.workspace,
+            events=events,
+            judge_model=None,
+            backend="codex",
+        )
+        self.assertIn("command_execution", run.tool_names)
+        self.assertIn("detect.py", run.command_text)
+        self.assertEqual(run.result_text, "done with Codex")
+
     def test_logs_contain_is_case_insensitive(self) -> None:
         run = self.make_run(stream(("Bash", {"command": "python DETECT.py"})))
         checks = run.evaluate(logs_contain=["detect.py"])
@@ -660,6 +709,32 @@ def test_dot_claude_is_excluded_from_workspace_listing(self) -> None:
         self.assertEqual(self.make_run(stream()).files, ["out.png"])
 
 
+class TestCodexBackend(unittest.TestCase):
+    def test_exec_command_is_noninteractive_ephemeral_and_sandboxed(self) -> None:
+        with mock.patch.object(codex_backend.shutil, "which", return_value="codex"):
+            cmd = codex_backend.exec_command(
+                Path("workspace"),
+                model="gpt-test",
+                effort="high",
+                sandbox="read-only",
+            )
+        self.assertEqual(cmd[:2], ["codex", "exec"])
+        self.assertIn("--json", cmd)
+        self.assertIn("--ephemeral", cmd)
+        self.assertIn("read-only", cmd)
+        self.assertIn("gpt-test", cmd)
+        self.assertNotIn("--dangerously-bypass-approvals-and-sandbox", cmd)
+        self.assertEqual(cmd[-1], "-")
+
+    def test_behavior_command_uses_workspace_write_with_approval_review(self) -> None:
+        with mock.patch.object(codex_backend.shutil, "which", return_value="codex"):
+            cmd = codex_backend.exec_command(
+                Path("workspace"), model=None, effort=None, sandbox="workspace-write"
+            )
+        self.assertIn("workspace-write", cmd)
+        self.assertIn("--approve-for-me", cmd)
+
+
 class FakeAgent:
     """Stands in for a real agent session so the flow can be tested offline."""
 

From 7703ebf9d1be498e7507dbdeee168be05d2ad6e6 Mon Sep 17 00:00:00 2001
From: amd-mkarvir <272370325+amd-mkarvir@users.noreply.github.com>
Date: Mon, 31 Aug 2026 23:04:44 -0700
Subject: [PATCH 2/2] Validate Codex evals against live JSONL

---
 eval/fixtures/codex-exec-skill-read.jsonl |   7 +
 eval/routing.py                           |  44 +++---
 eval/run_evals.py                         |  34 ++++-
 eval/test_evals.py                        | 156 ++++++++++++++++------
 4 files changed, 167 insertions(+), 74 deletions(-)
 create mode 100644 eval/fixtures/codex-exec-skill-read.jsonl

diff --git a/eval/fixtures/codex-exec-skill-read.jsonl b/eval/fixtures/codex-exec-skill-read.jsonl
new file mode 100644
index 0000000..43a593a
--- /dev/null
+++ b/eval/fixtures/codex-exec-skill-read.jsonl
@@ -0,0 +1,7 @@
+{"type":"thread.started","thread_id":"<redacted>"}
+{"type":"turn.started"}
+{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I am using the local-ai-use skill and will read its instructions first."}}
+{"type":"item.started","item":{"id":"item_1","type":"command_execution","command":"Get-Content -Raw \"<CODEX_HOME>/plugins/cache/amd-skills/amd-skills/0.2.0/skills/local-ai-use/SKILL.md\"","aggregated_output":"","exit_code":null,"status":"in_progress"}}
+{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"Get-Content -Raw \"<CODEX_HOME>/plugins/cache/amd-skills/amd-skills/0.2.0/skills/local-ai-use/SKILL.md\"","aggregated_output":"---\nname: local-ai-use\n---","exit_code":0,"status":"completed"}}
+{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"The first step is to confirm the local service is installed and running."}}
+{"type":"turn.completed","usage":{"input_tokens":0,"cached_input_tokens":0,"cache_write_input_tokens":0,"output_tokens":0,"reasoning_output_tokens":0}}
diff --git a/eval/routing.py b/eval/routing.py
index 49db4ab..7854368 100644
--- a/eval/routing.py
+++ b/eval/routing.py
@@ -256,33 +256,6 @@ def detect_activation(event: dict, skills: list[str], allow_body_path: bool = Tr
     is a contaminated runner, not a routing result, and the report should say
     so rather than silently scoring it as a miss.
     """
-    def find_codex_invocation(node) -> str | None:
-        if isinstance(node, dict):
-            if node.get("type") == "skill_invocation":
-                invoked = str(
-                    node.get("skill_name")
-                    or node.get("name")
-                    or node.get("skill")
-                    or node.get("skill_id")
-                    or ""
-                ).strip()
-                hit = _match_skill(invoked, skills)
-                return hit or f"other:{invoked or 'unknown'}"
-            for value in node.values():
-                hit = find_codex_invocation(value)
-                if hit:
-                    return hit
-        elif isinstance(node, list):
-            for value in node:
-                hit = find_codex_invocation(value)
-                if hit:
-                    return hit
-        return None
-
-    codex_hit = find_codex_invocation(event)
-    if codex_hit:
-        return codex_hit
-
     for name, tool_input in _iter_tool_uses(event):
         lowered = name.lower()
         if lowered in SKILL_TOOLS:
@@ -428,6 +401,16 @@ def classify(expect: str | None, observed: str | None) -> str:
     return "correct_trigger" if observed == expect else "wrong_skill"
 
 
+def _terminal_failure(event: dict) -> str | None:
+    """Return a terminal Codex failure message; retry notices are non-terminal."""
+    if event.get("type") != "turn.failed":
+        return None
+    failure = event.get("error")
+    if isinstance(failure, dict):
+        failure = failure.get("message") or failure
+    return str(failure or "Codex turn failed")[:400]
+
+
 def run_case(case: Case, skills: list[str], config: RoutingConfig) -> Outcome:
     """Run one prompt, stopping as soon as the routing decision is known."""
     cli_bin = shutil.which(config.agent)
@@ -567,9 +550,12 @@ def run_case(case: Case, skills: list[str], config: RoutingConfig) -> Outcome:
             if event.get("type") == "turn.completed":
                 stop_reason = "result"
                 break
-            if event.get("type") == "error":
+            # Generic `error` events can be retry notices; Codex may recover
+            # and finish the turn. `turn.failed` is the terminal failure.
+            failure = _terminal_failure(event)
+            if failure is not None:
                 stop_reason = "result"
-                error = str(event.get("message") or "Codex reported an error")[:400]
+                error = failure
                 break
 
             for name, tool_input in _iter_tool_uses(event):
diff --git a/eval/run_evals.py b/eval/run_evals.py
index 89ed6bc..35c1466 100644
--- a/eval/run_evals.py
+++ b/eval/run_evals.py
@@ -497,13 +497,31 @@ def main(argv: list[str] | None = None) -> int:
         args.model = enforce_model_policy(args.model) or args.model
     skills = _selected_skills(args)
     catalog = datasets.routing_catalog()
+    routing_codex_install = None
+    routing_codex_home = None
 
     if not args.skip_preflight:
         if args.agent == "codex":
-            with codex_backend.install([skills[0]]) as preflight_home:
-                ok, detail = codex_backend.check_api_reachable(
-                    preflight_home, args.model or None, args.effort
-                )
+            if args.mode in ("routing", "both") and catalog:
+                # Reuse the catalog installation for the routing run below.
+                routing_codex_install = codex_backend.install(catalog)
+                try:
+                    routing_codex_home = routing_codex_install.__enter__()
+                    ok, detail = codex_backend.check_api_reachable(
+                        routing_codex_home, args.model or None, args.effort
+                    )
+                except BaseException:
+                    routing_codex_install.__exit__(None, None, None)
+                    raise
+                if not ok:
+                    routing_codex_install.__exit__(None, None, None)
+                    routing_codex_install = None
+                    routing_codex_home = None
+            else:
+                with codex_backend.install([skills[0]]) as preflight_home:
+                    ok, detail = codex_backend.check_api_reachable(
+                        preflight_home, args.model or None, args.effort
+                    )
         else:
             ok, detail = check_api_reachable(args.model)
         if not ok:
@@ -539,8 +557,12 @@ def main(argv: list[str] | None = None) -> int:
                 "installed and so could never win them."
             )
 
-        codex_install = codex_backend.install(catalog) if args.agent == "codex" else None
-        codex_home = codex_install.__enter__() if codex_install is not None else None
+        if args.agent == "codex":
+            codex_install = routing_codex_install or codex_backend.install(catalog)
+            codex_home = routing_codex_home or codex_install.__enter__()
+        else:
+            codex_install = None
+            codex_home = None
         config = routing.RoutingConfig(
             model=args.model,
             effort=args.effort,
diff --git a/eval/test_evals.py b/eval/test_evals.py
index 876552d..cfe298d 100644
--- a/eval/test_evals.py
+++ b/eval/test_evals.py
@@ -16,6 +16,8 @@
 from __future__ import annotations
 
 import json
+import os
+import subprocess
 import sys
 import tempfile
 import unittest
@@ -34,6 +36,16 @@
 
 TRIGGERING = "triggeringEvaluation"
 NON_TRIGGERING = "nonTriggeringEvaluation"
+CODEX_EXEC_FIXTURE = EVAL_DIR / "fixtures" / "codex-exec-skill-read.jsonl"
+
+
+def codex_exec_fixture() -> list[dict]:
+    """Load the sanitized JSONL captured from a live Codex CLI run."""
+    return [
+        json.loads(line)
+        for line in CODEX_EXEC_FIXTURE.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
 
 
 def parse(
@@ -494,6 +506,21 @@ def test_only_correct_and_true_negative_pass(self) -> None:
             routing.PASSING_VERDICTS, {"correct_trigger", "true_negative"}
         )
 
+    def test_codex_retry_notice_is_not_terminal(self) -> None:
+        self.assertIsNone(
+            routing._terminal_failure(
+                {"type": "error", "message": "Reconnecting... 2/5"}
+            )
+        )
+
+    def test_codex_turn_failure_is_terminal(self) -> None:
+        self.assertEqual(
+            routing._terminal_failure(
+                {"type": "turn.failed", "error": {"message": "request failed"}}
+            ),
+            "request failed",
+        )
+
 
 class TestActivationDetection(unittest.TestCase):
     SKILLS = ["local-ai-use", "local-ai-app-integration", "serving-llms-on-instinct"]
@@ -508,27 +535,12 @@ def test_skill_tool_call_is_an_activation(self) -> None:
         event = self.event("Skill", {"command": "local-ai-use"})
         self.assertEqual(routing.detect_activation(event, self.SKILLS), "local-ai-use")
 
-    def test_codex_skill_invocation_is_an_activation(self) -> None:
-        event = {
-            "type": "item.completed",
-            "item": {
-                "type": "skill_invocation",
-                "skill_name": "amd-skills:serving-llms-on-instinct",
-            },
-        }
-        self.assertEqual(
-            routing.detect_activation(event, self.SKILLS),
-            "serving-llms-on-instinct",
-        )
-
-    def test_codex_skill_outside_catalog_is_flagged(self) -> None:
-        event = {
-            "type": "item.completed",
-            "item": {"type": "skill_invocation", "skill_name": "other-plugin:demo"},
-        }
-        self.assertEqual(
-            routing.detect_activation(event, self.SKILLS), "other:other-plugin:demo"
-        )
+    def test_live_codex_skill_read_is_an_activation(self) -> None:
+        # Captured with Codex CLI 0.150.0-alpha.12.2. The CLI represented skill
+        # activation as command_execution reading the installed SKILL.md; IDs,
+        # paths, output, prose, and usage were sanitized in the fixture.
+        hits = [routing.detect_activation(event, self.SKILLS) for event in codex_exec_fixture()]
+        self.assertEqual([hit for hit in hits if hit], ["local-ai-use", "local-ai-use"])
 
     def test_longest_name_wins_when_one_is_a_prefix_of_another(self) -> None:
         event = self.event("Skill", {"command": "local-ai-app-integration"})
@@ -577,6 +589,28 @@ def test_catalog_inspection_is_recognized(self) -> None:
         self.assertFalse(routing._is_catalog_inspection('{"path": "src/main.py"}', self.SKILLS))
 
 
+class TestRoutingTermination(unittest.TestCase):
+    def test_terminate_stops_the_process_group(self) -> None:
+        spawn = (
+            {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP}
+            if os.name == "nt"
+            else {"start_new_session": True}
+        )
+        proc = subprocess.Popen(
+            [sys.executable, "-c", "import time; time.sleep(30)"],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            **spawn,
+        )
+        try:
+            routing._terminate(proc)
+            self.assertIsNotNone(proc.poll())
+        finally:
+            if proc.poll() is None:
+                proc.kill()
+                proc.wait(timeout=5)
+
+
 class TestPromptTemplating(unittest.TestCase):
     def test_placeholders_are_substituted(self) -> None:
         self.assertEqual(
@@ -622,30 +656,19 @@ def test_transcript_and_tools_are_captured(self) -> None:
         self.assertIn("detect.py", run.logs)
         self.assertEqual(run.result_text, "done")
 
-    def test_codex_transcript_and_final_message_are_captured(self) -> None:
-        events = [
-            {
-                "type": "item.completed",
-                "item": {
-                    "type": "command_execution",
-                    "command": "python detect.py",
-                    "aggregated_output": "ok",
-                },
-            },
-            {
-                "type": "item.completed",
-                "item": {"type": "agent_message", "text": "done with Codex"},
-            },
-        ]
+    def test_live_codex_transcript_and_final_message_are_captured(self) -> None:
         run = agent.Run(
             workspace=self.workspace,
-            events=events,
+            events=codex_exec_fixture(),
             judge_model=None,
             backend="codex",
         )
         self.assertIn("command_execution", run.tool_names)
-        self.assertIn("detect.py", run.command_text)
-        self.assertEqual(run.result_text, "done with Codex")
+        self.assertIn("skills/local-ai-use/SKILL.md", run.command_text)
+        self.assertEqual(
+            run.result_text,
+            "The first step is to confirm the local service is installed and running.",
+        )
 
     def test_logs_contain_is_case_insensitive(self) -> None:
         run = self.make_run(stream(("Bash", {"command": "python DETECT.py"})))
@@ -735,6 +758,61 @@ def test_behavior_command_uses_workspace_write_with_approval_review(self) -> Non
         self.assertIn("--approve-for-me", cmd)
 
 
+class TestCodexMainFlow(unittest.TestCase):
+    def test_routing_preflight_reuses_the_catalog_install(self) -> None:
+        cases, errors = parse(
+            triggers(id="one-case", prompt="route this"), skill="local-ai-use"
+        )
+        self.assertEqual(errors, [])
+        install_context = mock.MagicMock()
+        install_context.__enter__.return_value = Path("isolated-codex-home")
+        install = mock.Mock(return_value=install_context)
+        summary = {
+            "totals": {
+                "graded": 1,
+                "activations": 1,
+                "activations_expected": 1,
+                "accuracy": 1.0,
+            }
+        }
+
+        with (
+            mock.patch.object(datasets, "validate_all", return_value=[]),
+            mock.patch.object(
+                datasets, "skills_with_datasets", return_value=["local-ai-use"]
+            ),
+            mock.patch.object(
+                datasets, "routing_catalog", return_value=["local-ai-use"]
+            ),
+            mock.patch.object(datasets, "load_all_cases", return_value=cases),
+            mock.patch.object(codex_backend, "install", install),
+            mock.patch.object(
+                codex_backend, "check_api_reachable", return_value=(True, "ok")
+            ),
+            mock.patch.object(routing, "supported_flags", return_value=set()),
+            mock.patch.object(routing, "run_case", return_value=object()),
+            mock.patch.object(routing, "summarize", return_value=summary),
+            mock.patch.object(routing, "render_markdown", return_value=""),
+            mock.patch.object(run_evals, "_write_report"),
+        ):
+            result = run_evals.main(
+                [
+                    "--agent",
+                    "codex",
+                    "--mode",
+                    "routing",
+                    "--only",
+                    "one-case",
+                    "--jobs",
+                    "1",
+                ]
+            )
+
+        self.assertEqual(result, 0)
+        install.assert_called_once_with(["local-ai-use"])
+        install_context.__exit__.assert_called_once_with(None, None, None)
+
+
 class FakeAgent:
     """Stands in for a real agent session so the flow can be tested offline."""