fix(e2e): redact replay evidence and clean up compose resources

.github/workflows/e2e-harness.yml

@@ -11,6 +11,9 @@ on:
 permissions:
   contents: read
 
+env:
+  TRUFFLEHOG_IMAGE: ghcr.io/trufflesecurity/trufflehog:3.96.0
+
 concurrency:
   group: e2e-harness-${{ github.event.pull_request.number || github.ref }}-${{ github.sha }}
   cancel-in-progress: false
@@ -50,68 +53,36 @@ jobs:
           POWERCONTEXT_E2E_OUTPUT: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/acceptance
         run: make harness-compose-acceptance
 
-      - name: Upload replay evidence
+      - name: Scan replay evidence
+        id: evidence_scan
         if: always()
-        uses: actions/upload-artifact@v7
-        with:
-          name: e2e-acceptance-${{ matrix.database }}-${{ github.sha }}
-          path: .powercontext-e2e/${{ matrix.database }}/acceptance
-          if-no-files-found: error
-          retention-days: 14
-
-      - name: Stop harness environment
-        if: always()
-        env:
-          POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }}
-        run: e2e/bub/run.sh down
-
-  live-replay:
-    needs: acceptance
-    if: github.event_name != 'pull_request'
-    runs-on: ubuntu-latest
-    timeout-minutes: 50
-    strategy:
-      fail-fast: false
-      matrix:
-        database: [sqlite, oceanbase]
-    name: Live replay (${{ matrix.database }})
-    steps:
-      - name: Check out
-        uses: actions/checkout@v7
-
-      - name: Check provider configuration
-        id: provider
+        continue-on-error: true
         env:
-          API_KEY: ${{ secrets.POWERCONTEXT_E2E_API_KEY }}
-          MODEL: ${{ vars.POWERCONTEXT_E2E_MODEL }}
+          EVIDENCE_PATH: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/acceptance
         run: |
-          if [[ -n "${API_KEY}" && -n "${MODEL}" ]]; then
-            echo "configured=true" >> "${GITHUB_OUTPUT}"
-          else
-            echo "configured=false" >> "${GITHUB_OUTPUT}"
-            echo "Live replay skipped because provider credentials are not configured." >> "${GITHUB_STEP_SUMMARY}"
-          fi
-
-      - name: Run live replay
-        if: steps.provider.outputs.configured == 'true'
-        env:
-          BUB_API_BASE: ${{ vars.POWERCONTEXT_E2E_API_BASE }}
-          BUB_API_KEY: ${{ secrets.POWERCONTEXT_E2E_API_KEY }}
-          BUB_MODEL: ${{ vars.POWERCONTEXT_E2E_MODEL }}
-          POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }}
-          POWERCONTEXT_E2E_OUTPUT: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/live
-        run: make harness-compose-live
-
-      - name: Publish replay summary
-        if: always() && steps.provider.outputs.configured == 'true'
-        run: cat ".powercontext-e2e/${{ matrix.database }}/live/report.md" >> "${GITHUB_STEP_SUMMARY}"
+          test -d "${EVIDENCE_PATH}"
+          docker run --rm \
+            --volume "${EVIDENCE_PATH}:/evidence:ro" \
+            "${TRUFFLEHOG_IMAGE}" \
+            filesystem /evidence \
+            --no-verification \
+            --results=verified,unknown,unverified \
+            --fail \
+            --fail-on-scan-errors \
+            --no-update \
+            --json > "${RUNNER_TEMP}/trufflehog-acceptance-${{ matrix.database }}.jsonl"
+
+      - name: Report suppressed replay evidence
+        if: always() && steps.evidence_scan.outcome != 'success'
+        run: echo "::warning::Replay evidence was not published because secret scanning did not complete cleanly."
 
       - name: Upload replay evidence
-        if: always() && steps.provider.outputs.configured == 'true'
+        if: always() && steps.evidence_scan.outcome == 'success'
         uses: actions/upload-artifact@v7
         with:
-          name: e2e-live-${{ matrix.database }}-${{ github.sha }}
-          path: .powercontext-e2e/${{ matrix.database }}/live
+          name: e2e-acceptance-${{ matrix.database }}-${{ github.sha }}
+          path: .powercontext-e2e/${{ matrix.database }}/acceptance
+          include-hidden-files: true
           if-no-files-found: error
           retention-days: 14
 
@@ -120,3 +91,91 @@ jobs:
         env:
           POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }}
         run: e2e/bub/run.sh down
+
+  # live-replay:
+  #   needs: acceptance
+  #   if: github.event_name != 'pull_request'
+  #   runs-on: ubuntu-latest
+  #   timeout-minutes: 50
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       database: [sqlite, oceanbase]
+  #   name: Live replay (${{ matrix.database }})
+  #   steps:
+  #     - name: Check out
+  #       uses: actions/checkout@v7
+
+  #     - name: Check provider configuration
+  #       id: provider
+  #       env:
+  #         API_KEY: ${{ secrets.POWERCONTEXT_E2E_API_KEY }}
+  #         MODEL: ${{ vars.POWERCONTEXT_E2E_MODEL }}
+  #       run: |
+  #         if [[ -n "${API_KEY}" && -n "${MODEL}" ]]; then
+  #           echo "configured=true" >> "${GITHUB_OUTPUT}"
+  #         else
+  #           echo "configured=false" >> "${GITHUB_OUTPUT}"
+  #           echo "Live replay skipped because provider credentials are not configured." >> "${GITHUB_STEP_SUMMARY}"
+  #         fi
+
+  #     - name: Run live replay
+  #       if: steps.provider.outputs.configured == 'true'
+  #       env:
+  #         BUB_API_BASE: ${{ vars.POWERCONTEXT_E2E_API_BASE }}
+  #         BUB_API_KEY: ${{ secrets.POWERCONTEXT_E2E_API_KEY }}
+  #         BUB_MODEL: ${{ vars.POWERCONTEXT_E2E_MODEL }}
+  #         POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }}
+  #         POWERCONTEXT_E2E_OUTPUT: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/live
+  #       run: make harness-compose-live
+
+  #     - name: Scan replay evidence
+  #       id: evidence_scan
+  #       if: always() && steps.provider.outputs.configured == 'true'
+  #       continue-on-error: true
+  #       env:
+  #         EVIDENCE_PATH: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/live
+  #       run: |
+  #         test -d "${EVIDENCE_PATH}"
+  #         docker run --rm \
+  #           --volume "${EVIDENCE_PATH}:/evidence:ro" \
+  #           "${TRUFFLEHOG_IMAGE}" \
+  #           filesystem /evidence \
+  #           --no-verification \
+  #           --results=verified,unknown,unverified \
+  #           --fail \
+  #           --fail-on-scan-errors \
+  #           --no-update \
+  #           --json > "${RUNNER_TEMP}/trufflehog-live-${{ matrix.database }}.jsonl"
+
+  #     - name: Report suppressed replay evidence
+  #       if: >-
+  #         always() &&
+  #         steps.provider.outputs.configured == 'true' &&
+  #         steps.evidence_scan.outcome != 'success'
+  #       run: echo "::warning::Replay evidence was not published because secret scanning did not complete cleanly."
+
+  #     - name: Publish replay summary
+  #       if: >-
+  #         always() &&
+  #         steps.provider.outputs.configured == 'true' &&
+  #         steps.evidence_scan.outcome == 'success'
+  #       run: cat ".powercontext-e2e/${{ matrix.database }}/live/report.md" >> "${GITHUB_STEP_SUMMARY}"
+
+  #     - name: Upload replay evidence
+  #       if: >-
+  #         always() &&
+  #         steps.provider.outputs.configured == 'true' &&
+  #         steps.evidence_scan.outcome == 'success'
+  #       uses: actions/upload-artifact@v7
+  #       with:
+  #         name: e2e-live-${{ matrix.database }}-${{ github.sha }}
+  #         path: .powercontext-e2e/${{ matrix.database }}/live
+  #         if-no-files-found: error
+  #         retention-days: 14
+
+  #     - name: Stop harness environment
+  #       if: always()
+  #       env:
+  #         POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }}
+  #       run: e2e/bub/run.sh down

Makefile

@@ -48,6 +48,7 @@ harness-check: ## Validate the Bub replay harness and committed scenarios.
 	@uv run ruff check e2e/bub
 	@uv run ruff format --check e2e/bub
 	@uv run ty check --project e2e/bub --python e2e/bub/.venv e2e/bub/src integrations/bub/src
+	@uv run --project e2e/bub python -m pytest e2e/bub/tests
 	@uv run --project e2e/bub powercontext-e2e --help >/dev/null
 
 .PHONY: harness-acceptance

e2e/bub/README.md

@@ -12,8 +12,10 @@ It supports three evidence modes:
 - `rescore` reads `replay.json` and runs the same Pydantic Evals oracle without rerunning Bub or PowerContext.
 
 Each run writes `replay.json`, `eval-report.json`, and `report.md`. The replay is self-contained and contains the
-scenario, public Memory snapshots, prepared context, outputs, and Pydantic Evals-compatible spans. It never records
-API keys, authorization headers, or database URLs.
+scenario, public Memory snapshots, prepared context, outputs, and Pydantic Evals-compatible spans. Known runtime
+secrets are redacted when these files are written. CI also scans the complete evidence directory with TruffleHog
+before publishing a summary or artifact; evidence is not published when that scan does not complete cleanly. Treat
+local evidence as potentially sensitive until it has been inspected.
 
 ## Run against an existing Server
 
@@ -51,4 +53,5 @@ POWERCONTEXT_E2E_DATABASE=oceanbase make harness-compose-acceptance
 ```
 
 `make harness-compose-live` uses the provider variables above. Evidence is written below `.powercontext-e2e/bub/`;
-set `POWERCONTEXT_E2E_OUTPUT` to keep it elsewhere. `make harness-compose-down` removes containers and database volumes.
+set `POWERCONTEXT_E2E_OUTPUT` to keep it elsewhere. Compose containers, networks, and volumes are removed after both
+successful and failed runs. `make harness-compose-down` remains available as an idempotent manual cleanup.

e2e/bub/pyproject.toml

@@ -13,6 +13,9 @@ dependencies = [
     "pyyaml>=6,<7",
 ]
 
+[dependency-groups]
+dev = ["pytest>=9.0.2"]
+
 [project.scripts]
 powercontext-e2e = "powercontext_e2e.__main__:main"
 
@@ -38,5 +41,8 @@ line-length = 120
 select = ["A", "B", "C4", "C90", "E", "F", "I", "PGH", "RUF", "S", "SIM", "T10", "TRY", "UP", "W", "YTT"]
 ignore = ["E501"]
 
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = ["S101"]
+
 [tool.ruff.format]
 preview = true

e2e/bub/run.sh

@@ -68,6 +68,26 @@ if [ -z "${GITHUB_SHA:-}" ]; then
     export GITHUB_SHA
 fi
 
+cleanup() {
+    status=$?
+    trap - EXIT INT TERM
+    set +e
+
+    docker compose $compose_files down --volumes --remove-orphans
+    cleanup_status=$?
+    if [ "$cleanup_status" -ne 0 ]; then
+        echo "Compose cleanup failed with exit code $cleanup_status" >&2
+        if [ "$status" -eq 0 ]; then
+            status=$cleanup_status
+        fi
+    fi
+    exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
 docker compose $compose_files build powercontext harness
 docker compose $compose_files up --detach --wait powercontext
 
@@ -81,5 +101,3 @@ else
     scenario=${POWERCONTEXT_E2E_SCENARIO:-e2e/bub/scenarios/project-database-decision.yaml}
     docker compose $compose_files run --rm harness live "$scenario" --output /evidence
 fi
-
-docker compose $compose_files down --volumes --remove-orphans

e2e/bub/src/powercontext_e2e/runner.py

@@ -40,6 +40,17 @@
 Mode = Literal["acceptance", "live", "offline-rescore"]
 Report = EvaluationReport[ScenarioSpec, ReplayObservation, dict[str, str]]
 Context = EvaluatorContext[ScenarioSpec, ReplayObservation, dict[str, str]]
+_EVIDENCE_SECRET_ENVIRONMENT_NAMES = (
+    "ANTHROPIC_API_KEY",
+    "BUB_API_KEY",
+    "DEEPSEEK_API_KEY",
+    "OPENAI_API_KEY",
+    "OPENROUTER_API_KEY",
+    "POWERCONTEXT_CLIENT_API_TOKEN",
+    "POWERCONTEXT_SERVER_AUTH_TOKEN",
+    "POWERCONTEXT_SERVER_DATABASE_URL",
+)
+_REDACTED = "[REDACTED]"
 
 
 @dataclass
@@ -223,7 +234,7 @@ async def _run_replay(
                 except Exception as exc:
                     output = ""
                     status = "failed"
-                    error = _redact(f"{type(exc).__name__}: {exc}")
+                    error = f"{type(exc).__name__}: {exc}"
                     errors.append(f"Session {session.id}: {error}")
                 memory_after_session = await _memory_snapshot(client, scope_id)
                 observations.append(
@@ -233,15 +244,15 @@ async def _run_replay(
                         status=status,
                         error=error,
                         prepared_context=prepared,
-                        output=_redact(output),
+                        output=output,
                         memory_after=memory_after_session,
                     )
                 )
                 if status == "failed":
                     break
             memory_after = await _memory_snapshot(client, scope_id)
     except Exception as exc:
-        errors.append(_redact(f"{type(exc).__name__}: {exc}"))
+        errors.append(f"{type(exc).__name__}: {exc}")
         memory_after = observations[-1].memory_after if observations else memory_before
 
     return ReplayObservation(
@@ -390,16 +401,14 @@ def _commit() -> str:
     return completed.stdout.strip() if completed.returncode == 0 else "unknown"
 
 
-def _redact(value: str) -> str:
-    secret = os.getenv("BUB_API_KEY")
-    return value.replace(secret, "[REDACTED]") if secret else value
-
-
 def write_artifacts(observation: ReplayObservation, report: Report, output_dir: Path) -> None:
+    secrets = _runtime_secrets()
     output_dir.mkdir(parents=True, exist_ok=True)
-    (output_dir / "replay.json").write_text(
-        observation.model_dump_json(by_alias=True, indent=2) + "\n",
-        encoding="utf-8",
+    replay_payload = observation.model_dump(mode="json", by_alias=True)
+    _write_evidence(
+        output_dir / "replay.json",
+        json.dumps(replay_payload, indent=2, ensure_ascii=False) + "\n",
+        secrets,
     )
 
     cases = [
@@ -428,17 +437,31 @@ def write_artifacts(observation: ReplayObservation, report: Report, output_dir:
         "cases": cases,
         "failures": [{"name": failure.name, "error": failure.error_message} for failure in report.failures],
     }
-    (output_dir / "eval-report.json").write_text(
-        json.dumps(report_payload, indent=2, sort_keys=True) + "\n",
-        encoding="utf-8",
+    _write_evidence(
+        output_dir / "eval-report.json",
+        json.dumps(report_payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n",
+        secrets,
     )
-    (output_dir / "report.md").write_text(
+    _write_evidence(
+        output_dir / "report.md",
         "# PowerContext session replay\n\n"
         f"- Scenario: `{observation.scenario.id}`\n"
         f"- Mode: `{observation.environment.mode}`\n"
         f"- Database: `{observation.environment.database}`\n"
         f"- Status: `{observation.status}`\n\n"
         "## Evaluation\n\n"
         f"```text\n{report.render(include_reasons=True)}\n```\n",
-        encoding="utf-8",
+        secrets,
     )
+
+
+def _runtime_secrets() -> tuple[str, ...]:
+    secrets = {value for name in _EVIDENCE_SECRET_ENVIRONMENT_NAMES if (value := os.getenv(name))}
+    return tuple(sorted(secrets, key=lambda value: (-len(value), value)))
+
+
+def _write_evidence(path: Path, content: str, secrets: tuple[str, ...]) -> None:
+    for secret in secrets:
+        content = content.replace(secret, _REDACTED)
+        content = content.replace(json.dumps(secret, ensure_ascii=False)[1:-1], _REDACTED)
+    path.write_text(content, encoding="utf-8")