fix(#880): resolve retro agent provider credential conflict

eval/README.md

@@ -1,8 +1,9 @@
 # Eval Harness
 
 Functional tests for fullsend agents. Each agent has its own eval
-directory (`triage/`, `review/`, `code/`) containing an `eval.yaml`
-config and a `cases/` directory with test case definitions.
+directory (`triage/`, `review/`, `code/`, `fix/`, `retro/`) containing
+an `eval.yaml` config and a `cases/` directory with test case
+definitions.
 
 ## Running evals
 

eval/retro/cases/001-smoke-sandbox/annotations.yaml

@@ -0,0 +1,7 @@
+# Smoke test: we only care that the sandbox created successfully.
+# The agent will fail its actual task (no real workflow run to analyze)
+# but that's expected — the sandbox_started judge checks metrics.json.
+state: open
+
+max_turns: 40
+max_cost_usd: 2.00

eval/retro/cases/001-smoke-sandbox/input.yaml

@@ -0,0 +1,19 @@
+forge: github
+fixture:
+  type: pull_request
+  title: "chore: update README formatting"
+  body: |
+    Minor formatting fix in the README.
+
+    This PR is a fixture for the retro agent smoke test. It exists only
+    to provide a valid ORIGINATING_URL (the PR URL) for the retro harness.
+  files:
+    - path: README.md
+      content: |
+        # Test Repository
+
+        This is a test repository for functional evaluation.
+
+        ## Overview
+
+        Updated formatting for consistency.

eval/retro/cases/001-smoke-sandbox/repo/README.md

@@ -0,0 +1,3 @@
+# Test Repository
+
+This is a test repository for functional evaluation.

eval/retro/eval.yaml

@@ -0,0 +1,144 @@
+name: retro-eval
+description: Smoke test — verifies sandbox creation succeeds for the retro agent
+
+skill: retro
+
+execution:
+  mode: case
+  timeout: 600  # 10 min — sandbox creation can stall in merge queue runners
+  parallelism: 1
+  env:
+    EVAL_TIMEOUT: "540"
+    EVAL_ORG: $EVAL_ORG
+    GH_TOKEN: $GH_TOKEN
+    FULLSEND_DIR: $FULLSEND_DIR
+    GOOGLE_APPLICATION_CREDENTIALS: $GOOGLE_APPLICATION_CREDENTIALS
+    ANTHROPIC_VERTEX_PROJECT_ID: $ANTHROPIC_VERTEX_PROJECT_ID
+    GOOGLE_CLOUD_PROJECT: $GOOGLE_CLOUD_PROJECT
+    CLOUD_ML_REGION: $CLOUD_ML_REGION
+
+hooks:
+  before_each:
+    - command: "setup-fixture.sh"
+      timeout: 120
+      description: "Create ephemeral repo and fixture"
+
+  after_each:
+    - command: "capture-fixture.sh"
+      timeout: 30
+      description: "Capture fixture state for judges"
+    - command: "teardown-fixture.sh"
+      timeout: 30
+      on_failure: continue
+      description: "Delete ephemeral repo"
+
+runner:
+  type: cli
+  command:
+    - "run-fullsend.sh"
+    - "{agent}"
+    - "{workspace}"
+    - "{output_dir}"
+
+models:
+  skill: claude-opus-4-6
+  judge: claude-opus-4-6
+
+dataset:
+  path: cases
+  schema: |
+    Each case directory contains:
+    - input.yaml: Fixture definition with forge, fixture type, title, body.
+    - annotations.yaml: Expected outcomes.
+    - repo/: Directory with base repo contents.
+
+outputs:
+  - path: output
+    schema: |
+      metrics.json: Agent run metrics. Presence indicates the sandbox
+      was created and the agent started.
+
+# ---------------------------------------------------------------------------
+# Judges — smoke test: did the sandbox start + behavioral guards
+# ---------------------------------------------------------------------------
+judges:
+  - name: sandbox_started
+    description: Sandbox creation succeeded and agent ran at least one turn
+    check: |
+      import json
+      raw = outputs["files"].get("output/metrics.json")
+      if not raw:
+          return False, "metrics.json not found — sandbox likely failed to create (credential conflict?)"
+      metrics = json.loads(raw)
+      turns = metrics.get("num_turns")
+      if turns is None:
+          return False, "num_turns not present in metrics.json"
+      if int(turns) < 1:
+          return False, f"Agent ran 0 turns — sandbox may have failed"
+      return True, f"Sandbox started, agent ran {turns} turn(s)"
+
+  - name: forbidden_labels
+    description: Labels listed in annotations.yaml forbidden list must NOT be present
+    check: |
+      import json
+      raw = outputs["files"].get("output/fixture-state.json")
+      if not raw:
+          return False, "fixture-state.json not found — capture-fixture.sh did not run or failed"
+      state = json.loads(raw)
+      actual = [l.lower() for l in state.get("labels", [])]
+      forbidden = outputs.get("annotations", {}).get("labels", {}).get("forbidden", [])
+      if not forbidden:
+          return True, "No forbidden labels specified"
+      present = [l for l in forbidden if l.lower() in actual]
+      if present:
+          return False, f"Forbidden labels present: {present} (actual: {actual})"
+      return True, f"No forbidden labels found (checked: {forbidden})"
+
+  - name: max_turns
+    description: Agent must complete within the declared turn budget
+    check: |
+      import json
+      raw = outputs["files"].get("output/metrics.json")
+      if not raw:
+          return False, "metrics.json not found"
+      metrics = json.loads(raw)
+      actual = metrics.get("num_turns")
+      if actual is None:
+          return False, "num_turns not present in metrics.json"
+      limit = outputs.get("annotations", {}).get("max_turns")
+      if limit is None:
+          return False, "max_turns not declared in annotations.yaml"
+      if int(actual) > int(limit):
+          return False, f"Exceeded max_turns: {actual} > {limit}"
+      return True, f"Turns OK: {actual} <= {limit}"
+
+  - name: max_cost
+    description: Agent must complete within the declared cost budget
+    check: |
+      import json
+      raw = outputs["files"].get("output/metrics.json")
+      if not raw:
+          return False, "metrics.json not found"
+      metrics = json.loads(raw)
+      actual = metrics.get("total_cost_usd")
+      if actual is None:
+          return False, "total_cost_usd not present in metrics.json"
+      limit = outputs.get("annotations", {}).get("max_cost_usd")
+      if limit is None:
+          return False, "max_cost_usd not declared in annotations.yaml"
+      if float(actual) > float(limit):
+          return False, f"Exceeded max_cost_usd: {actual} > {limit}"
+      return True, f"Cost OK: {actual} <= {limit}"
+
+# ---------------------------------------------------------------------------
+# Thresholds
+# ---------------------------------------------------------------------------
+thresholds:
+  sandbox_started:
+    min_pass_rate: 1.0
+  forbidden_labels:
+    min_pass_rate: 1.0
+  max_turns:
+    min_pass_rate: 1.0
+  max_cost:
+    min_pass_rate: 1.0

eval/scripts/run-fullsend.sh

@@ -203,6 +203,11 @@ install -m 0600 /dev/null "$ENV_FILE"
     emit_env "REVIEW_BODY_FILE" "${REVIEW_BODY_FILE}"
   fi
 
+  if [[ "$AGENT" == "retro" ]]; then
+    emit_env "ORIGINATING_URL" "${FIXTURE_URL}"
+    emit_env "RETRO_COMMENT" "${RETRO_COMMENT:-}"
+  fi
+
   # Review agent: both REVIEW_PROTECTED_PATHS and
   # REVIEW_FINDING_SEVERITY_THRESHOLD are literal defaults baked into
   # harness/review.yaml's env.runner/env.sandbox stanzas. Default here to

providers/github-artifacts.yaml

@@ -1,5 +1,13 @@
 ---
+# Credential-less provider: only network policy rules are needed here.
+# The initial API call to api.github.com is authenticated by the companion
+# github-ro provider (which injects GH_TOKEN). The 302 redirect targets
+# (Azure Blob Storage) use pre-signed SAS URLs — no Authorization header
+# required. The gh CLI (and OpenShell's proxy) drops the Authorization
+# header on cross-domain redirects, so GH_TOKEN must NOT be declared here
+# or it will conflict with github-ro's GH_TOKEN key and OpenShell will
+# reject the sandbox.
 name: github-artifacts
 type: fullsend-github-artifacts
 credentials:
-  GH_TOKEN: "${GH_TOKEN}"
+  _NOOP_GITHUB_ARTIFACTS: ""