chore(eval): pin all agents to Sonnet 5 and de-duplicate LLM response metrics

.amplifier/evaluation/.gitignore

@@ -2,6 +2,20 @@
 # Never source controlled (matches the existing per-harness runs/ convention).
 runs/
 
+# Personal analysis tooling and scratch. Useful locally, not part of the
+# harness contract, so it stays out of the tracked tree.
+local/
+
+# Scratch the AI User is instructed to write when a prompt contains a double
+# quote (see agents/*/invocation.md). It belongs at /workspace/eval-prompt.txt
+# INSIDE the DTU, but the AI User drives from this directory on the host, so a
+# missing path prefix drops it here instead. swe_bench_lifecycle.DRIVER_SCRATCH
+# already cleans these in-DTU; this keeps a host-side leak out of the repo.
+eval-prompt.txt
+eval-run.out
+eval-run.done
+answer.txt
+
 # Python / uv
 .venv/
 __pycache__/

.amplifier/evaluation/agents/amplifier-agent-local/install.yaml

@@ -23,9 +23,17 @@ setup_cmds:
   # Write the host-config JSON that every task invocation passes via --config.
   # It sets headless approval, pins the provider/model, and adds an extra skills
   # source location (/root/extra-skills) used by the skill-config-location task.
+  #
+  # `debug.rawLlmPayloads: true` folds into the provider config as `raw: true`,
+  # which makes the provider attach the FULL outbound request kwargs to
+  # `llm:request` and the accumulated final response to `llm:response`. Those
+  # events are written to the session events.jsonl that extract.yaml already
+  # pulls (glob **/events.jsonl), so no extraction change is needed. This is the
+  # analysis surface for the eval: the actual prompts and responses, not just
+  # token counts. Contract covered by tests/e2e/suites/raw_capture.
   - |
     cat > /root/host-config.json <<'JSON'
-    {"approval":{"mode":"yes"},"provider":{"module":"anthropic","config":{"default_model":"claude-sonnet-5"}},"skills":{"skills":["/root/extra-skills"]}}
+    {"approval":{"mode":"yes"},"provider":{"module":"anthropic","config":{"default_model":"claude-sonnet-5"}},"skills":{"skills":["/root/extra-skills"]},"debug":{"rawLlmPayloads":true}}
     JSON
   - mkdir -p /root/extra-skills
   # install setup_cmds run under a login shell. Writing this profile.d file

.amplifier/evaluation/agents/amplifier-foundation/install.yaml

@@ -13,9 +13,19 @@
 # `requires.env` is verified on the host before launch and merged into the task
 # profile's passthrough services, so ANTHROPIC_API_KEY reaches the agent.
 #
-# Opus 4.8 is pinned via the opus48 routing matrix (routes all model_role
-# mappings to claude-opus-4-8) so this variant runs the SAME model as the
-# opencode + amplifier-agent stack; the only difference is the agent stack.
+# Sonnet 5 is pinned via the provider's `default_model` so this variant runs the
+# SAME model as the opencode + amplifier-agent stack; the only difference is the
+# agent stack.
+#
+# NO `routing.matrix` is set, deliberately. A previous revision named a matrix
+# (`opus48`) that does not exist in the routing-matrix bundle; the reference
+# resolved to nothing and the pin silently came from `default_model` alone. That
+# was verified against a real run: every one of the 84 LLM requests across three
+# tasks -- including the delegated sub-agent sessions -- carried the
+# `default_model` value. Naming a matrix here would only re-introduce a
+# role-based fan-out (`reasoning`/`creative`/`writing` -> opus under the stock
+# `anthropic` matrix), which is precisely what a single-model pin must avoid.
+# `default_model` is the whole mechanism; keep it that way.
 requires:
   env:
     - ANTHROPIC_API_KEY
@@ -37,12 +47,10 @@ setup_cmds:
           source: git+https://github.com/microsoft/amplifier-module-provider-anthropic@main
           config:
             api_key: $ANTHROPIC_API_KEY
-            default_model: claude-opus-4-8
+            default_model: claude-sonnet-5
             enable_1m_context: 'true'
             enable_prompt_caching: 'true'
             priority: 1
-      routing:
-        matrix: opus48
     EOF
 
   # Compose the anchors bundle as an app bundle (same mechanism the foundation
@@ -56,8 +64,20 @@ setup_cmds:
   # `includes:` the FULL amplifier-foundation bundle, which would drag all of
   # foundation on top of the lean anchors variant and defeat its purpose. The
   # behaviors keep the composition lean: anchors stays the active bundle and only
-  # the two CI agents are added -- no full foundation, no design mode, no logging
-  # hook (that lives only in the separate -logging behavior).
+  # the two CI agents are added -- no full foundation, no design mode, and no
+  # ADDITIONAL logging hook (that lives in the separate -logging behavior, which
+  # we do not add here).
+  #
+  # To be precise about what IS logging, because it caught us out once: anchors
+  # itself already composes TWO loggers upstream --
+  # `foundation:behaviors/logging` (hooks-logging -> <session>/events.jsonl) and
+  # `context-intelligence:behaviors/context-intelligence-logging`
+  # (hook-context-intelligence -> <session>/context-intelligence/events.jsonl).
+  # So every LLM call in this variant is written to disk TWICE, in two different
+  # envelope shapes. That is anchors' own design and is deliberately NOT patched
+  # here -- this agent evaluates anchors as published. The harness is what has to
+  # cope: `metrics.parse_events` de-duplicates by response identity. Before that
+  # fix this variant reported exactly double its real calls, tokens and cost.
   #
   # We add BOTH behavior files explicitly and on purpose:
   #   - navigation (LAYER 1) registers session-navigator (reads local session

.amplifier/evaluation/agents/amplifier-foundation/invocation.md

@@ -98,10 +98,10 @@ took. Do NOT judge correctness yourself -- the grader does that.
 
 ## Model pinning
 
-Opus 4.8 is pinned at install time via the provider's `default_model:
-claude-opus-4-8` plus the `opus48` routing matrix in
-`/root/.amplifier/settings.yaml`. That is the single source of truth for which
-model runs, so `amplifier run` needs no model flag -- just invoke it as shown.
+Sonnet 5 is pinned at install time via the provider's `default_model:
+claude-sonnet-5` in `/root/.amplifier/settings.yaml`, with no routing matrix
+set. That is the single source of truth for which model runs, so `amplifier run`
+needs no model flag -- just invoke it as shown.
 
 ## Notes
 

.amplifier/evaluation/agents/amplifier-foundation/meta.yaml

@@ -12,8 +12,8 @@ id: amplifier-foundation
 description: >
   The Amplifier CLI with the anchors bundle (amplifier-foundation@main,
   bundles/anchors) plus the context-intelligence bundle (agents only, no server)
-  composed, driven via `amplifier run` on Claude Opus 4.8. A general-agent variant
+  composed, driven via `amplifier run` on Claude Sonnet 5. A general-agent variant
   under test, run on the same tasks as the opencode + amplifier-agent stack for
-  head-to-head comparison. Opus is pinned via the opus48 routing matrix so the
-  model matches the opencode agent.
-model: claude-opus-4-8
+  head-to-head comparison. Sonnet 5 is pinned via the provider's default_model
+  so the model matches every other agent-under-test.
+model: claude-sonnet-5

.amplifier/evaluation/agents/opencode-amplifier-agent/invocation.md

@@ -8,7 +8,7 @@ already on PATH once you export the two tool directories.
 Run each turn with a single command. Note the PATH export (opencode and uv
 tools live in `$HOME/.opencode/bin` and `$HOME/.local/bin`) and the model pin:
 
-    export PATH="$HOME/.opencode/bin:$HOME/.local/bin:$PATH"; cd /workspace && amplifier-opencode launch -- run --auto --model amplifier/claude-opus-4-8 "<your message>"
+    export PATH="$HOME/.opencode/bin:$HOME/.local/bin:$PATH"; cd /workspace && amplifier-opencode launch -- run --auto --model amplifier/claude-sonnet-5 "<your message>"
 
 The CLI prints the agent's final response to stdout and exits. Capture the
 response from stdout.
@@ -25,7 +25,7 @@ on every invocation.
 
 ## Model pinning
 
-opus-4-8 is pinned via opencode's `--model amplifier/claude-opus-4-8` flag. The
+sonnet-5 is pinned via opencode's `--model amplifier/claude-sonnet-5` flag. The
 amplifier adapter writes no default model, so the flag is the single source of
 truth for which model runs. Keep it on every invocation.
 

.amplifier/evaluation/agents/opencode-amplifier-agent/meta.yaml

@@ -5,7 +5,7 @@ id: opencode-amplifier-agent
 description: >
   opencode driven by amplifier-agent (OpenAI-compatible backend), running
   Anthropic models. The agent-under-test for evaluation runs.
-model: claude-opus-4-8
+model: claude-sonnet-5
 
 # Agent-owned timing hints for clean agent-only wall clock. The AI User drives
 # this agent by shelling `amplifier-digital-twin

.amplifier/evaluation/agents/opencode-vanilla/install.yaml

@@ -8,7 +8,7 @@
 # opencode's anthropic provider reads ANTHROPIC_API_KEY from the environment
 # automatically -- no `opencode auth login` is needed.
 #
-# Opus 4.8 (`claude-opus-4-8`) is not in opencode's models.dev catalog, so we
+# Sonnet 5 (`claude-sonnet-5`) is not in opencode's models.dev catalog, so we
 # register it explicitly under the built-in anthropic provider (@ai-sdk/anthropic)
 # in ~/.config/opencode/opencode.json. opencode sends the id to the Anthropic API
 # verbatim.
@@ -30,19 +30,19 @@ setup_cmds:
     export PATH="$HOME/.opencode/bin:$PATH"; opencode --version
   - echo 'export PATH="$HOME/.opencode/bin:$PATH"' > /etc/profile.d/opencode.sh
   - export PATH="$HOME/.opencode/bin:$PATH"; opencode --version
-  # Register Opus 4.8 on the anthropic provider and make it the default model.
+  # Register Sonnet 5 on the anthropic provider and make it the default model.
   - mkdir -p "$HOME/.config/opencode"
   - |
     cat > "$HOME/.config/opencode/opencode.json" <<'JSON'
     {
       "$schema": "https://opencode.ai/config.json",
-      "model": "anthropic/claude-opus-4-8",
+      "model": "anthropic/claude-sonnet-5",
       "provider": {
         "anthropic": {
           "npm": "@ai-sdk/anthropic",
           "models": {
-            "claude-opus-4-8": {
-              "name": "Claude Opus 4.8"
+            "claude-sonnet-5": {
+              "name": "Claude Sonnet 5"
             }
           }
         }
@@ -51,4 +51,4 @@ setup_cmds:
     JSON
   # Best-effort warm-up: exercises the full path once (auth + model resolution)
   # so the operator's first real run is fast. Bounded and non-fatal.
-  - 'export PATH="$HOME/.opencode/bin:$PATH"; timeout 240 opencode run --model anthropic/claude-opus-4-8 --auto "reply with exactly: ok" || true'
+  - 'export PATH="$HOME/.opencode/bin:$PATH"; timeout 240 opencode run --model anthropic/claude-sonnet-5 --auto "reply with exactly: ok" || true'

.amplifier/evaluation/agents/opencode-vanilla/invocation.md

@@ -8,7 +8,7 @@ a one-shot per turn.
 Run each turn with a single command. Note the PATH export (opencode lives in
 `$HOME/.opencode/bin`), the model pin, and `--auto`:
 
-    export PATH="$HOME/.opencode/bin:$PATH"; cd /workspace && opencode run --model anthropic/claude-opus-4-8 --auto "<your message>"
+    export PATH="$HOME/.opencode/bin:$PATH"; cd /workspace && opencode run --model anthropic/claude-sonnet-5 --auto "<your message>"
 
 The CLI prints the agent's final response to stdout and exits. Capture the
 response from stdout.
@@ -21,7 +21,7 @@ agent cannot complete web or file tasks. Always pass `--auto`.
 
 ## Model pinning
 
-opus-4-8 is pinned via `--model anthropic/claude-opus-4-8`. It is also the
+sonnet-5 is pinned via `--model anthropic/claude-sonnet-5`. It is also the
 default model in opencode's config, but keep the flag on every invocation as the
 single source of truth for which model runs.
 

.amplifier/evaluation/agents/opencode-vanilla/meta.yaml

@@ -4,9 +4,9 @@
 id: opencode-vanilla
 description: >
   Vanilla opencode talking directly to the Anthropic API (no amplifier-agent /
-  amplifier-app-opencode bridge), pinned to Claude Opus 4.8. A baseline
+  amplifier-app-opencode bridge), pinned to Claude Sonnet 5. A baseline
   agent-under-test for comparison against opencode-amplifier-agent.
-model: claude-opus-4-8
+model: claude-sonnet-5
 
 # Agent-owned timing hints for clean agent-only wall clock. The AI User drives
 # this agent by shelling `amplifier-digital-twin exec <dtu> -- bash -c '...

.amplifier/evaluation/pyproject.toml

@@ -40,6 +40,16 @@ amplifier-bundle-evaluation = { git = "https://github.com/microsoft/amplifier-bu
 amplifier-core = { git = "https://github.com/microsoft/amplifier-core", branch = "main" }
 amplifier-foundation = { git = "https://github.com/microsoft/amplifier-foundation", branch = "main" }
 
+# Test-only dependencies. pytest-asyncio is required not by these tests but by
+# amplifier-core's pytest plugin, which loads via a pytest11 entry point as soon
+# as pytest starts in this venv; without it collection fails before any test
+# runs. Install with `uv sync --group dev`.
+[dependency-groups]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
+]
+
 [tool.pyright]
 include = ["src"]
 extraPaths = ["src"]