feat(codex): add codex as a native benchmark agent

strands-infra-runner/ORCHESTRATOR.md

@@ -30,7 +30,7 @@ run-benchmark.sh <agent> <model> <dataset> [concurrency]
 
 | Arg | Options | Default |
 |-----|---------|---------|
-| `agent` | `claude-code`, `opencode`, or any dir under `strands-infra-runner/agents/` | required |
+| `agent` | `claude-code`, `opencode`, `codex`, or any dir under `strands-infra-runner/agents/` | required |
 | `model` | `sonnet-4.6`, `opus-4.6`, `opus-4.8`, `sonnet-5`, `kimi-k2.5`, or a raw model ID (e.g. `openai.gpt-5.6-sol` for GPT via Bedrock Mantle) | required |
 | `dataset` | Any Harbor dataset (e.g. `swe-bench/swe-bench-verified`, `terminal-bench/terminal-bench-2-1`, `gaia/gaia`) | required |
 | `concurrency` | Number of parallel EC2 instances | Dataset task count, capped at 2,000; 500 for unknown datasets |
@@ -172,25 +172,29 @@ it's a branch/tag, the SHA comes from `git ls-remote`; when it's already a SHA,
 it's used directly — `ls-remote` matches refs only and returns nothing for a raw
 commit, so it cannot be used to resolve or validate one.
 
-## Native Claude Code and OpenCode setup
+## Native Claude Code, OpenCode, and Codex setup
 
-The `claude-code` and `opencode` names select Harbor's built-in installed-agent
-adapters. No local wrapper directory is required.
+The `claude-code`, `opencode`, and `codex` names select Harbor's built-in
+installed-agent adapters. No local wrapper directory is required.
 
 | Agent | Default pinned version | Harbor model value |
 |-------|------------------------|--------------------|
 | `claude-code` | `2.1.220` | Raw Bedrock model ID |
 | `opencode` | `1.18.9` | `amazon-bedrock/<model-id>` or `openai/<model-id>` |
-
-Override the pins with `CLAUDE_CODE_VERSION` or `OPENCODE_VERSION`. Claude Code
-uses the fleet instance profile; its main, fast, and subagent model aliases are
-pinned to the selected benchmark model. Claude Code only supports Claude
-models. OpenCode uses Bedrock directly for Claude/Kimi and Bedrock Mantle for
-GPT/GLM. GPT uses Mantle's `/openai/v1` Responses route; GLM uses the `/v1`
-OpenAI-compatible route with the transport-only `openai.` prefix removed from
-its model ID. The launcher loads `bedrock_api_key` for both paths and pins
-OpenCode's small-model work to the selected benchmark model. Harbor stores
-environment references rather than secret values in job configuration.
+| `codex` | `0.147.0` | Bare model ID (e.g. `openai.gpt-5.6-sol`); auth flows via `OPENAI_API_KEY` / `OPENAI_BASE_URL` |
+
+Override the pins with `CLAUDE_CODE_VERSION`, `OPENCODE_VERSION`, or
+`CODEX_VERSION`. Claude Code uses the fleet instance profile; its main, fast,
+and subagent model aliases are all pinned to the selected benchmark model.
+Claude Code only supports Claude models. OpenCode uses Bedrock directly for
+Claude/Kimi and Bedrock Mantle for GPT/GLM. GPT uses Mantle's `/openai/v1`
+Responses route; GLM uses the `/v1` OpenAI-compatible route with the
+transport-only `openai.` prefix removed from its model ID. The launcher loads
+`bedrock_api_key` for both paths and pins OpenCode's small-model work to the
+selected benchmark model. Codex runs through Bedrock Mantle's `/openai/v1`
+Responses route using the same `bedrock_api_key` secret; it only supports
+OpenAI GPT model IDs and hard-rejects non-GPT models at the launcher. Harbor
+stores environment references rather than secret values in job configuration.
 
 No custom skills, MCP servers, memory, or web-search credentials are enabled.
 For TAU3 runs, the same `bedrock_api_key` secret also configures the simulated

strands-infra-runner/README.md

@@ -30,7 +30,7 @@ derives it from `STAN_BRANCH` when `VERSION_TAG` is unset.
 
 | Component | Source | Example |
 |-----------|--------|---------|
-| agent | First arg | `stan`, `claude-code`, or `opencode` |
+| agent | First arg | `stan`, `claude-code`, `opencode`, or `codex` |
 | version | Stan commit or native product version | `2c58790` or `2.1.220` |
 | model | Second arg (alias) | `opus-4.6` |
 | dataset-slug | Third arg with `/` → `-` | `terminal-bench-terminal-bench-2-1` |
@@ -50,7 +50,7 @@ run-benchmark.sh <agent> <model> <dataset> [concurrency]
 
 | Arg | What to pass | Default |
 |-----|-------------|---------|
-| `agent` | `claude-code`, `opencode`, or a directory under `strands-infra-runner/agents/` | required |
+| `agent` | `claude-code`, `opencode`, `codex`, or a directory under `strands-infra-runner/agents/` | required |
 | `model` | Alias or raw Bedrock model ID | required |
 | `dataset` | Harbor dataset path | required |
 | `concurrency` | Parallel EC2 instances | Dataset task count, capped at 2,000; 500 for unknown datasets |
@@ -86,6 +86,7 @@ Kimi runs through Bedrock Runtime in `us-east-1`. See the
 | `STAN_BRANCH` | Git ref to install Stan from | `main` |
 | `CLAUDE_CODE_VERSION` | Claude Code CLI version installed by Harbor | `2.1.220` |
 | `OPENCODE_VERSION` | OpenCode CLI version installed by Harbor | `1.18.9` |
+| `CODEX_VERSION` | Codex CLI version installed by Harbor | `0.147.0` |
 | `HARBOR_STRANDS_CHECKOUT` | Separate Harbor fork checkout used by the custom benchmark | `/home/ubuntu/harbor-strands-working` |
 | `INSTANCE_TYPE` | Fleet node instance type | `m7i.xlarge` |
 | `BEDROCK_API_KEY_SECRET_ID` | Bedrock bearer-token secret used by OpenCode and TAU3 | `bedrock_api_key` |
@@ -111,6 +112,9 @@ bash strands-infra-runner/run-benchmark.sh claude-code sonnet-4.6 swe-bench/swe-
 # OpenCode through Bedrock and Harbor's native adapter
 bash strands-infra-runner/run-benchmark.sh opencode sonnet-4.6 swe-bench/swe-bench-verified
 
+# Codex through Bedrock Mantle (OpenAI-compatible endpoint)
+bash strands-infra-runner/run-benchmark.sh codex openai.gpt-5.6-sol swe-bench/swe-bench-verified
+
 # Full supported pass@2 index matrix for both native products
 setsid bash strands-infra-runner/run-matrix.sh \
   -a claude-code,opencode -k 2 \
@@ -181,8 +185,8 @@ S3 versioning is enabled on the results bucket.
 ### What `run-benchmark.sh` does
 
 1. Resolves agent path, model ID, job name
-2. Selects Harbor's native adapter for `claude-code` and `opencode`; other
-   agents continue through the custom Strands adapter
+2. Selects Harbor's native adapter for `claude-code`, `opencode`, and `codex`;
+   other agents continue through the custom Strands adapter
 3. For `stan` agents: fetches PAT from Secrets Manager, pip installs Stan,
    copies `strands_stan/` into agent dir so containers can import it
 4. For TAU3: fetches `bedrock_api_key` and configures the simulated user and
@@ -225,6 +229,16 @@ construction and is not written into the launcher argv or persisted job
 configuration. TAU3 also uses the same secret for its simulated user and
 assertion grader through Bedrock Mantle. Claude Code does not receive it.
 
+Codex runs through Bedrock Mantle's OpenAI-compatible `/openai/v1` endpoint.
+The launcher loads the same `bedrock_api_key` secret used by OpenCode and TAU3
+and exposes it as `OPENAI_API_KEY` alongside `OPENAI_BASE_URL`; Harbor's Codex
+adapter reads both through its `ModelConnectionSpec(default_provider="openai")`
+and forwards them into the fleet container as `${OPENAI_API_KEY}` and
+`${CODEX_OPENAI_BASE_URL}` templates. Only OpenAI GPT models are supported;
+the launcher hard-rejects non-GPT model IDs. `CODEX_VERSION` pins the CLI
+version and gets baked into the job name (`codex@<version>--<model>--<dataset>`);
+default is `0.147.0`.
+
 ### Hard-won learnings
 
 1. **Run orchestrator on big box** — 500 SSH sessions need ≥16 cores. Laptop stalls.

strands-infra-runner/run-benchmark.sh

@@ -8,6 +8,7 @@
 #   run-benchmark stan sonnet-4.6 terminal-bench/terminal-bench-2-1
 #   run-benchmark claude-code sonnet-4.6 swe-bench/swe-bench-verified 500
 #   run-benchmark opencode sonnet-4.6 swe-bench/swe-bench-verified 500
+#   run-benchmark codex openai.gpt-5.6-sol swe-bench/swe-bench-verified 500
 #
 # Results upload to: s3://strands-benchmark-results/<agent>/<model>/<dataset-slug>/
 # Local results at:  jobs/<agent>@<commit-sha>--<model>--<dataset-slug>/
@@ -69,13 +70,23 @@ case "$AGENT" in
     VERSION_TAG="${VERSION_TAG:-$AGENT_VERSION}"
     AGENT_PATH=""
     ;;
+  codex)
+    AGENT="codex"
+    HARBOR_AGENT="codex"
+    # Pin an explicit default so back-to-back runs of the same command hit the
+    # same CLI. Codex publishes weekly; bump this alongside intentional matrix
+    # refreshes rather than letting `@latest` drift silently.
+    AGENT_VERSION="${CODEX_VERSION:-0.147.0}"
+    VERSION_TAG="${VERSION_TAG:-$AGENT_VERSION}"
+    AGENT_PATH=""
+    ;;
   *)
     if [ -d "${AGENTS_DIR}/${AGENT}" ]; then
       HARBOR_AGENT="strands_evals.benchmarks.harbor.installed.py:StrandsInstalledPyAgent"
       AGENT_PATH="${AGENTS_DIR}/${AGENT}"
     else
       echo "Agent not found: ${AGENTS_DIR}/${AGENT}" >&2
-      echo "Native agents: claude-code opencode" >&2
+      echo "Native agents: claude-code opencode codex" >&2
       echo "Custom agents: $(ls "${AGENTS_DIR}" 2>/dev/null | tr '\n' ' ')" >&2
       exit 1
     fi
@@ -125,6 +136,19 @@ case "$HARBOR_AGENT" in
       HARBOR_MODEL_NAME="amazon-bedrock/${MODEL_ID}"
     fi
     ;;
+  codex)
+    # Codex's adapter uses ModelConnectionSpec(default_provider="openai"), so
+    # it always resolves the OpenAI provider (OPENAI_API_KEY, OPENAI_BASE_URL).
+    # HARBOR_MODEL_NAME is the bare model ID the CLI passes to the endpoint;
+    # for Mantle it's the same "openai." prefix Strands uses.
+    if [[ "$MODEL_ID" == openai.gpt* ]]; then
+      HARBOR_MODEL_NAME="$MODEL_ID"
+      CODEX_MANTLE_BASE_URL="https://bedrock-mantle.${TAU3_MANTLE_REGION:-us-east-1}.api.aws/openai/v1"
+    else
+      echo "ERROR: Codex Bedrock runs require an OpenAI GPT model on Mantle, got: ${MODEL_ID}" >&2
+      exit 1
+    fi
+    ;;
 esac
 
 # --- Instance type (always xlarge to handle any task's resource requirements) ---
@@ -153,6 +177,9 @@ fi
 if [ -n "$OPENCODE_MANTLE_BASE_URL" ]; then
   echo "  Mantle URL:  $OPENCODE_MANTLE_BASE_URL"
 fi
+if [ -n "${CODEX_MANTLE_BASE_URL:-}" ]; then
+  echo "  Mantle URL:  $CODEX_MANTLE_BASE_URL"
+fi
 echo "  Dataset:     $DATASET"
 echo "  Harbor ref:  $HARBOR_REF"
 echo "  Code root:   $EVALS_DIR"
@@ -293,6 +320,18 @@ configure_opencode_model() {
   fi
 }
 
+configure_codex_mantle() {
+  # Codex authenticates via OPENAI_API_KEY and reads its endpoint from
+  # OPENAI_BASE_URL; the Harbor adapter forwards both when it resolves the
+  # openai provider. Same Secrets Manager entry that TAU3/OpenCode use, since
+  # Mantle re-uses the bearer token as the API key.
+  load_bedrock_api_key
+  export OPENAI_API_KEY="$BEDROCK_API_KEY_VALUE"
+  export CODEX_OPENAI_BASE_URL="$CODEX_MANTLE_BASE_URL"
+  export OPENAI_BASE_URL="$CODEX_MANTLE_BASE_URL"
+  echo "Configured Codex with the Bedrock Mantle OpenAI-compatible endpoint."
+}
+
 configure_tau3_mantle() {
   local mantle_region="${TAU3_MANTLE_REGION:-us-east-1}"
 
@@ -312,6 +351,10 @@ if [ "$HARBOR_AGENT" = "opencode" ]; then
   configure_opencode_model
 fi
 
+if [ "$HARBOR_AGENT" = "codex" ]; then
+  configure_codex_mantle
+fi
+
 if [[ "$DATASET" == sierra-research/tau3-bench* || "$DATASET" == "$STRANDS_HARNESS_DATASET" ]]; then
   configure_tau3_mantle
 fi

strands-infra-runner/run-matrix.sh

@@ -21,6 +21,7 @@
 #   run-matrix.sh -k 4                             # same matrix at pass@4
 #   run-matrix.sh -m opus-4.8,sonnet-4.6 -k 2      # just two models
 #   run-matrix.sh -a claude-code,opencode           # full supported native matrix
+#   run-matrix.sh -a codex -m gpt-5.6-sol           # single-cell Codex smoke
 #   run-matrix.sh -s 45fed43                       # pin an exact Stan commit
 #   run-matrix.sh -s my-feature-branch             # or a branch/tag
 #
@@ -131,7 +132,7 @@ for agent in "${RAW_AGENT_LIST[@]}"; do
     claude)
       agent="claude-code"
       ;;
-    claude-code|opencode)
+    claude-code|opencode|codex)
       HAS_NATIVE=1
       ;;
     stan*)
@@ -188,6 +189,11 @@ agent_model_incompatibility() {
         echo "Claude Code requires the Anthropic Claude protocol"
       fi
       ;;
+    codex)
+      if [[ "$model_id" != openai.gpt* ]]; then
+        echo "Codex requires an OpenAI GPT model on Bedrock Mantle"
+      fi
+      ;;
   esac
 }
 
@@ -363,6 +369,9 @@ for agent in "${AGENT_LIST[@]}"; do
     opencode)
       AGENT_STATE_PARTS+=("${agent}@${OPENCODE_VERSION:-1.18.9}")
       ;;
+    codex)
+      AGENT_STATE_PARTS+=("${agent}@${CODEX_VERSION:-0.147.0}")
+      ;;
     *)
       AGENT_STATE_PARTS+=("$agent")
       ;;

strands-infra-runner/run.py

@@ -142,7 +142,7 @@ def _resolve_aws_creds() -> dict[str, str]:
 # need to forward the orchestrator's creds (avoids token expiry + creds in argv).
 _aws_creds = {} if IAM_INSTANCE_PROFILE else _resolve_aws_creds()
 
-_native_agents = {"claude-code", "opencode"}
+_native_agents = {"claude-code", "opencode", "codex"}
 _is_native_agent = HARBOR_AGENT in _native_agents
 if _is_native_agent and not HARBOR_MODEL_NAME:
     raise SystemExit(
@@ -216,6 +216,25 @@ def _resolve_aws_creds() -> dict[str, str]:
         # Harbor resolves host-env templates at construction and persists the
         # templates, not the secret values, in the job configuration.
         _agent_env["AWS_BEARER_TOKEN_BEDROCK"] = "${AWS_BEARER_TOKEN_BEDROCK}"
+elif _is_native_agent and HARBOR_AGENT == "codex":
+    # Codex reads OPENAI_API_KEY and OPENAI_BASE_URL through its
+    # ModelConnectionSpec(default_provider="openai"). run-benchmark.sh sets
+    # both via configure_codex_mantle before invoking this script.
+    missing = [
+        key
+        for key in ("OPENAI_API_KEY", "CODEX_OPENAI_BASE_URL")
+        if not os.environ.get(key)
+    ]
+    if not DRY_RUN and missing:
+        raise SystemExit(
+            f"{', '.join(missing)} required for Codex Mantle runs."
+        )
+    _agent_env.update(
+        {
+            "OPENAI_API_KEY": "${OPENAI_API_KEY}",
+            "OPENAI_BASE_URL": "${CODEX_OPENAI_BASE_URL}",
+        }
+    )
 elif not _is_native_agent and os.environ.get("STRANDS_MODEL"):
     _agent_env["STRANDS_MODEL"] = os.environ["STRANDS_MODEL"]