From 658547eb9b5e61dc0bd2749a6e0d30153ecd319f Mon Sep 17 00:00:00 2001 From: ThunGuo Date: Tue, 11 Aug 2026 11:55:48 +0800 Subject: [PATCH 1/7] fix(e2e): redact replay evidence and clean up compose resources - redact credentials across all replay evidence sinks - preserve the normal live replay schema - clean up Compose resources after success or failure - preserve original build, startup, and harness exit codes --- Makefile | 1 + e2e/bub/README.md | 3 +- e2e/bub/pyproject.toml | 6 + e2e/bub/run.sh | 22 +- e2e/bub/src/powercontext_e2e/redaction.py | 190 +++++++++++++++++ e2e/bub/src/powercontext_e2e/runner.py | 29 ++- e2e/bub/tests/test_evidence_redaction.py | 236 ++++++++++++++++++++++ e2e/bub/tests/test_run_script.py | 153 ++++++++++++++ e2e/bub/uv.lock | 42 ++++ 9 files changed, 669 insertions(+), 13 deletions(-) create mode 100644 e2e/bub/src/powercontext_e2e/redaction.py create mode 100644 e2e/bub/tests/test_evidence_redaction.py create mode 100644 e2e/bub/tests/test_run_script.py diff --git a/Makefile b/Makefile index 9927712e8..7383df385 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,7 @@ harness-check: ## Validate the Bub replay harness and committed scenarios. @uv run ruff check e2e/bub @uv run ruff format --check e2e/bub @uv run ty check --project e2e/bub --python e2e/bub/.venv e2e/bub/src integrations/bub/src + @uv run --project e2e/bub python -m pytest e2e/bub/tests @uv run --project e2e/bub powercontext-e2e --help >/dev/null .PHONY: harness-acceptance diff --git a/e2e/bub/README.md b/e2e/bub/README.md index 805e7e093..af0afbe94 100644 --- a/e2e/bub/README.md +++ b/e2e/bub/README.md @@ -51,4 +51,5 @@ POWERCONTEXT_E2E_DATABASE=oceanbase make harness-compose-acceptance ``` `make harness-compose-live` uses the provider variables above. Evidence is written below `.powercontext-e2e/bub/`; -set `POWERCONTEXT_E2E_OUTPUT` to keep it elsewhere. `make harness-compose-down` removes containers and database volumes. +set `POWERCONTEXT_E2E_OUTPUT` to keep it elsewhere. Compose containers, networks, and volumes are removed after both +successful and failed runs. `make harness-compose-down` remains available as an idempotent manual cleanup. diff --git a/e2e/bub/pyproject.toml b/e2e/bub/pyproject.toml index 8647e8e35..c9f0f8829 100644 --- a/e2e/bub/pyproject.toml +++ b/e2e/bub/pyproject.toml @@ -13,6 +13,9 @@ dependencies = [ "pyyaml>=6,<7", ] +[dependency-groups] +dev = ["pytest>=9.0.2"] + [project.scripts] powercontext-e2e = "powercontext_e2e.__main__:main" @@ -38,5 +41,8 @@ line-length = 120 select = ["A", "B", "C4", "C90", "E", "F", "I", "PGH", "RUF", "S", "SIM", "T10", "TRY", "UP", "W", "YTT"] ignore = ["E501"] +[tool.ruff.lint.per-file-ignores] +"tests/**/*.py" = ["S101"] + [tool.ruff.format] preview = true diff --git a/e2e/bub/run.sh b/e2e/bub/run.sh index 933e88c04..fcd9a6f39 100755 --- a/e2e/bub/run.sh +++ b/e2e/bub/run.sh @@ -68,6 +68,26 @@ if [ -z "${GITHUB_SHA:-}" ]; then export GITHUB_SHA fi +cleanup() { + status=$? + trap - EXIT INT TERM + set +e + + docker compose $compose_files down --volumes --remove-orphans + cleanup_status=$? + if [ "$cleanup_status" -ne 0 ]; then + echo "Compose cleanup failed with exit code $cleanup_status" >&2 + if [ "$status" -eq 0 ]; then + status=$cleanup_status + fi + fi + exit "$status" +} + +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + docker compose $compose_files build powercontext harness docker compose $compose_files up --detach --wait powercontext @@ -81,5 +101,3 @@ else scenario=${POWERCONTEXT_E2E_SCENARIO:-e2e/bub/scenarios/project-database-decision.yaml} docker compose $compose_files run --rm harness live "$scenario" --output /evidence fi - -docker compose $compose_files down --volumes --remove-orphans diff --git a/e2e/bub/src/powercontext_e2e/redaction.py b/e2e/bub/src/powercontext_e2e/redaction.py new file mode 100644 index 000000000..1ce3cea2f --- /dev/null +++ b/e2e/bub/src/powercontext_e2e/redaction.py @@ -0,0 +1,190 @@ +"""Credential redaction at the replay evidence boundary.""" + +from __future__ import annotations + +import os +import re +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any +from urllib.parse import parse_qsl, quote, unquote, urlsplit + +REDACTED = "[REDACTED]" + +_SENSITIVE_ENVIRONMENT_NAMES = frozenset({ + "api_key", + "authorization", + "credentials", + "database_url", + "mysql_pwd", + "password", + "pgpassword", + "private_key", + "secret", + "token", +}) +_SENSITIVE_ENVIRONMENT_SUFFIXES = ( + "_access_key_id", + "_api_key", + "_authorization", + "_client_secret", + "_credentials", + "_database_url", + "_dsn", + "_password", + "_private_key", + "_secret", + "_secret_access_key", + "_secret_key", + "_token", +) +_SENSITIVE_FIELD_NAMES = frozenset({ + "access_token", + "api_key", + "auth_token", + "authorization", + "client_secret", + "connection_string", + "credentials", + "database_url", + "dsn", + "password", + "private_key", + "proxy_authorization", + "refresh_token", + "secret", + "secret_key", + "token", +}) +_SENSITIVE_FIELD_SUFFIXES = ( + "_access_token", + "_api_key", + "_auth_token", + "_authorization", + "_client_secret", + "_connection_string", + "_credentials", + "_database_url", + "_dsn", + "_password", + "_private_key", + "_refresh_token", + "_secret", + "_secret_key", +) +_DATABASE_URL = re.compile( + r"(?i)\b(?:cockroachdb|mariadb|mssql|mysql|oceanbase|oracle|postgres|postgresql|sqlite)" + r"(?:\+[a-z0-9_.-]+)?://[^\s\"'`<>{}\[\](),;\\]+" +) +_URL_PASSWORD = re.compile(r"(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@]+:)[^@/\s\"'`<>]+(@)") +_URL_QUERY_CREDENTIAL = re.compile( + r"(?i)([?&](?:access[_-]?token|api[_-]?key|auth[_-]?token|client[_-]?secret|password|secret|token)=)" + r"[^&#\s\"'`]+" +) +_CREDENTIAL_ASSIGNMENT = re.compile( + r"(?i)(\b(?:access[_-]?token|api[_-]?key|auth[_-]?token|authorization|client[_-]?secret|password|secret)" + r"\b\s*[:=]\s*)(?:(?:basic|bearer)\s+)?[^\s,;&\"'`]+" +) +_CREDENTIAL_OPTION = re.compile( + r"(?i)(--(?:access-token|api-key|auth-token|client-secret|password|token)\s+)" + r"[^\s,;\"'`]+" +) +_BEARER_CREDENTIAL = re.compile(r"(?i)\bbearer\s+[a-z0-9._~+/=-]{16,}") +_PROVIDER_CREDENTIAL = re.compile(r"(? EvidenceRedactor: + secrets: set[str] = set() + for name, value in os.environ.items(): + if value and _is_sensitive_environment_name(name): + secrets.update(_secret_variants(value)) + return cls(tuple(sorted(secrets, key=lambda item: (-len(item), item)))) + + def redact(self, value: Any) -> Any: + """Recursively redact JSON-compatible evidence without changing its normal shape.""" + return self._redact(value, sensitive=False) + + def redact_text(self, value: str) -> str: + """Redact one final serialized evidence value.""" + for secret in self.secrets: + if len(secret) >= 8: + value = value.replace(secret, REDACTED) + else: + value = re.sub(rf"(? Any: + if isinstance(value, str): + return REDACTED if sensitive and value else self.redact_text(value) + if isinstance(value, Mapping): + return { + self.redact_text(key) if isinstance(key, str) else key: self._redact( + item, + sensitive=sensitive or (isinstance(key, str) and _is_sensitive_field_name(key)), + ) + for key, item in value.items() + } + if isinstance(value, list): + return [self._redact(item, sensitive=sensitive) for item in value] + if isinstance(value, tuple): + return tuple(self._redact(item, sensitive=sensitive) for item in value) + return value + + +def _is_sensitive_environment_name(name: str) -> bool: + normalized = _normalize_name(name) + return normalized in _SENSITIVE_ENVIRONMENT_NAMES or normalized.endswith(_SENSITIVE_ENVIRONMENT_SUFFIXES) + + +def _is_sensitive_field_name(name: str) -> bool: + normalized = _normalize_name(name) + return normalized in _SENSITIVE_FIELD_NAMES or normalized.endswith(_SENSITIVE_FIELD_SUFFIXES) + + +def _normalize_name(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "_", value.casefold()).strip("_") + + +def _secret_variants(value: str) -> set[str]: + variants = {value} + stripped = value.strip() + if stripped: + variants.add(stripped) + lowered = stripped.casefold() + for prefix in ("basic ", "bearer "): + if lowered.startswith(prefix): + variants.add(stripped[len(prefix) :]) + variants.update(_url_secret_variants(stripped)) + for item in tuple(variants): + if item: + variants.add(quote(item, safe="")) + return {item for item in variants if item} + + +def _url_secret_variants(value: str) -> set[str]: + try: + parsed = urlsplit(value) + except ValueError: + return set() + if not parsed.netloc: + return set() + variants = set() + if parsed.password: + variants.update((parsed.password, unquote(parsed.password))) + for name, item in parse_qsl(parsed.query, keep_blank_values=True): + if item and _is_sensitive_field_name(name): + variants.update((item, unquote(item))) + return variants diff --git a/e2e/bub/src/powercontext_e2e/runner.py b/e2e/bub/src/powercontext_e2e/runner.py index 8f5352ef0..62e4ecefb 100644 --- a/e2e/bub/src/powercontext_e2e/runner.py +++ b/e2e/bub/src/powercontext_e2e/runner.py @@ -36,6 +36,7 @@ ScenarioSpec, SessionObservation, ) +from .redaction import EvidenceRedactor Mode = Literal["acceptance", "live", "offline-rescore"] Report = EvaluationReport[ScenarioSpec, ReplayObservation, dict[str, str]] @@ -391,15 +392,17 @@ def _commit() -> str: def _redact(value: str) -> str: - secret = os.getenv("BUB_API_KEY") - return value.replace(secret, "[REDACTED]") if secret else value + return EvidenceRedactor.from_environment().redact_text(value) def write_artifacts(observation: ReplayObservation, report: Report, output_dir: Path) -> None: + redactor = EvidenceRedactor.from_environment() output_dir.mkdir(parents=True, exist_ok=True) - (output_dir / "replay.json").write_text( - observation.model_dump_json(by_alias=True, indent=2) + "\n", - encoding="utf-8", + replay_payload = json.loads(observation.model_dump_json(by_alias=True)) + _write_evidence( + output_dir / "replay.json", + json.dumps(redactor.redact(replay_payload), indent=2, ensure_ascii=False) + "\n", + redactor, ) cases = [ @@ -428,11 +431,13 @@ def write_artifacts(observation: ReplayObservation, report: Report, output_dir: "cases": cases, "failures": [{"name": failure.name, "error": failure.error_message} for failure in report.failures], } - (output_dir / "eval-report.json").write_text( - json.dumps(report_payload, indent=2, sort_keys=True) + "\n", - encoding="utf-8", + _write_evidence( + output_dir / "eval-report.json", + json.dumps(redactor.redact(report_payload), indent=2, sort_keys=True, ensure_ascii=False) + "\n", + redactor, ) - (output_dir / "report.md").write_text( + _write_evidence( + output_dir / "report.md", "# PowerContext session replay\n\n" f"- Scenario: `{observation.scenario.id}`\n" f"- Mode: `{observation.environment.mode}`\n" @@ -440,5 +445,9 @@ def write_artifacts(observation: ReplayObservation, report: Report, output_dir: f"- Status: `{observation.status}`\n\n" "## Evaluation\n\n" f"```text\n{report.render(include_reasons=True)}\n```\n", - encoding="utf-8", + redactor, ) + + +def _write_evidence(path: Path, content: str, redactor: EvidenceRedactor) -> None: + path.write_text(redactor.redact_text(content), encoding="utf-8") diff --git a/e2e/bub/tests/test_evidence_redaction.py b/e2e/bub/tests/test_evidence_redaction.py new file mode 100644 index 000000000..4a35603de --- /dev/null +++ b/e2e/bub/tests/test_evidence_redaction.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +from pydantic_evals.otel import SpanNode + +from powercontext_e2e.models import ( + MemoryEntrySnapshot, + MemorySnapshot, + PreparedContextSnapshot, + ReplayObservation, + RunEnvironment, + ScenarioSpec, + SessionObservation, +) +from powercontext_e2e.runner import write_artifacts + + +def test_write_artifacts_redacts_every_evidence_sink(monkeypatch, tmp_path: Path) -> None: + credentials = { + "BUB_API_KEY": "sk-bub-evidence-sentinel", + "OPENAI_API_KEY": "sk-openai-evidence-sentinel", + "POWERCONTEXT_SERVER_AUTH_TOKEN": "server-auth-evidence-sentinel", + "POWERCONTEXT_CODEX_AUTHORIZATION": "Bearer codex-auth-evidence-sentinel", + "POWERCONTEXT_SERVER_DATABASE_URL": ( + "mysql+aoceanbase://root:database-evidence-sentinel@db.example/powercontext" + ), + } + for name, value in credentials.items(): + monkeypatch.setenv(name, value) + + extra_sentinels = ( + "header-evidence-sentinel", + "span-database-evidence-sentinel", + "query-evidence-sentinel", + "attribute-evidence-sentinel", + "failure-evidence-sentinel", + "basic-evidence-sentinel", + "sk-unregistered-evidence-sentinel", + ) + observation = _observation( + session_input=f"Question containing {credentials['BUB_API_KEY']}", + memory_text=f"Memory containing {credentials['OPENAI_API_KEY']}", + context=f"Context containing {credentials['POWERCONTEXT_SERVER_AUTH_TOKEN']}", + output=f"Output containing {credentials['POWERCONTEXT_CODEX_AUTHORIZATION']}", + error=f"Database failed at {credentials['POWERCONTEXT_SERVER_DATABASE_URL']}", + span_attributes={ + "http.request.header.authorization": f"Bearer {extra_sentinels[0]}", + "db.connection_string": (f"postgresql://user:{extra_sentinels[1]}@db.example/powercontext"), + "http.url": f"https://provider.example/v1?access_token={extra_sentinels[2]}&mode=live", + }, + ) + report = _report( + assertion_reason=f"Judge echoed {credentials['OPENAI_API_KEY']} and {extra_sentinels[6]}", + score_reason=f"Authorization: {credentials['POWERCONTEXT_CODEX_AUTHORIZATION']}", + label_reason=f"Provider URL used api_key={extra_sentinels[2]}", + attributes={"api_key": extra_sentinels[3]}, + failure=f"password={extra_sentinels[4]}", + rendered=f"Authorization: Basic {extra_sentinels[5]} and {credentials['BUB_API_KEY']}", + ) + + write_artifacts(observation, report, tmp_path) + + artifacts = {path.name: path.read_text(encoding="utf-8") for path in tmp_path.iterdir()} + assert set(artifacts) == {"eval-report.json", "replay.json", "report.md"} + sentinels = ( + credentials["BUB_API_KEY"], + credentials["OPENAI_API_KEY"], + credentials["POWERCONTEXT_SERVER_AUTH_TOKEN"], + "codex-auth-evidence-sentinel", + "database-evidence-sentinel", + *extra_sentinels, + ) + for artifact_name, content in artifacts.items(): + for sentinel in sentinels: + assert sentinel not in content, f"{sentinel!r} leaked into {artifact_name}" + assert "[REDACTED]" in content + + assert "mysql+aoceanbase://" not in artifacts["replay.json"] + assert "postgresql://" not in artifacts["replay.json"] + json.loads(artifacts["replay.json"]) + json.loads(artifacts["eval-report.json"]) + + +def test_write_artifacts_preserves_normal_live_replay_schema(tmp_path: Path) -> None: + observation = _observation( + session_input="Which database was selected?", + memory_text="The project selected OceanBase; Bearer authentication is unrelated.", + context="OceanBase supports shared persistent context.", + output="The project selected OceanBase.", + error=None, + span_attributes={ + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "deepseek-v4-flash", + "gen_ai.usage.input_tokens": 21, + "http.url": "https://provider.example/v1?mode=live", + }, + ) + report = _report( + assertion_reason="The run completed.", + score_reason="The expected fact was present.", + label_reason="pass", + attributes={"mode": "live"}, + failure=None, + rendered="All checks passed.", + ) + expected_replay = json.loads(observation.model_dump_json(by_alias=True)) + expected_evaluation = { + "schema": "powercontext.session-replay-evaluation/v1", + "experiment": "live:database-decision", + "cases": [ + { + "name": "database-decision", + "assertions": {"run_completed": {"value": True, "reason": "The run completed."}}, + "scores": {"answer": {"value": 1.0, "reason": "The expected fact was present."}}, + "labels": {"judge": {"value": "pass", "reason": "pass"}}, + "metrics": {"model_calls": 1}, + "attributes": {"mode": "live"}, + "task_duration": 0.25, + "total_duration": 0.5, + } + ], + "failures": [], + } + + write_artifacts(observation, report, tmp_path) + + actual_replay = json.loads((tmp_path / "replay.json").read_text(encoding="utf-8")) + actual_evaluation = json.loads((tmp_path / "eval-report.json").read_text(encoding="utf-8")) + assert actual_replay == expected_replay + assert actual_evaluation == expected_evaluation + assert ReplayObservation.model_validate(actual_replay) == observation + + +def _observation( + *, + session_input: str, + memory_text: str, + context: str, + output: str, + error: str | None, + span_attributes: dict[str, Any], +) -> ReplayObservation: + scenario = ScenarioSpec.model_validate({ + "schema": "powercontext.session-replay/v1", + "id": "database-decision", + "sessions": [ + { + "id": "recall", + "input": session_input, + "expected_answer": "The project selected OceanBase.", + } + ], + }) + memory = MemorySnapshot( + entries=( + MemoryEntrySnapshot( + entry_id="memory-1", + entry_version_id="memory-version-1", + version=1, + kind="fact", + text=memory_text, + state="committed", + ), + ) + ) + session = SessionObservation( + id="recall", + agent_session_id="session-1", + status="failed" if error else "completed", + error=error, + prepared_context=PreparedContextSnapshot(status="ready", content=context), + output=output, + memory_after=memory, + ) + timestamp = datetime(2026, 8, 11, 1, 2, 3, tzinfo=UTC) + span = SpanNode( + name="bub.model", + trace_id=1, + span_id=2, + parent_span_id=None, + start_timestamp=timestamp, + end_timestamp=timestamp, + attributes=span_attributes, + ) + return ReplayObservation( + run_id="database-decision-run", + environment=RunEnvironment( + mode="live", + commit="abcdef0", + database="sqlite", + agent_model="deepseek:deepseek-v4-flash", + generation_model="deepseek:deepseek-v4-flash", + judge_model="deepseek:deepseek-v4-flash", + started_at=timestamp, + ), + scenario=scenario, + status="failed" if error else "completed", + errors=(error,) if error else (), + memory_before=memory, + memory_after=memory, + sessions=(session,), + spans=(span,), + ) + + +def _report( + *, + assertion_reason: str, + score_reason: str, + label_reason: str, + attributes: dict[str, Any], + failure: str | None, + rendered: str, +) -> Any: + case = SimpleNamespace( + name="database-decision", + assertions={"run_completed": SimpleNamespace(value=True, reason=assertion_reason)}, + scores={"answer": SimpleNamespace(value=1.0, reason=score_reason)}, + labels={"judge": SimpleNamespace(value="pass", reason=label_reason)}, + metrics={"model_calls": 1}, + attributes=attributes, + task_duration=0.25, + total_duration=0.5, + ) + failures = [] if failure is None else [SimpleNamespace(name="judge", error_message=failure)] + return SimpleNamespace( + name="live:database-decision", + cases=[case], + failures=failures, + render=lambda include_reasons: rendered, + ) diff --git a/e2e/bub/tests/test_run_script.py b/e2e/bub/tests/test_run_script.py new file mode 100644 index 000000000..2beb726e2 --- /dev/null +++ b/e2e/bub/tests/test_run_script.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest + +_REPOSITORY_ROOT = Path(__file__).parents[3] +_RUN_SCRIPT = _REPOSITORY_ROOT / "e2e" / "bub" / "run.sh" + + +@pytest.mark.parametrize( + ("failed_command", "exit_code", "expected_commands"), + [ + ("build", 31, ["build", "down"]), + ("up", 32, ["build", "up", "down"]), + ("run", 33, ["build", "up", "run", "down"]), + ], +) +def test_failure_cleans_compose_resources_and_preserves_exit_code( + tmp_path: Path, + failed_command: str, + exit_code: int, + expected_commands: list[str], +) -> None: + result, state, commands = _run_with_fake_docker( + tmp_path, + failed_command=failed_command, + exit_code=exit_code, + ) + + assert result.returncode == exit_code + assert list(state.iterdir()) == [] + assert commands == expected_commands + + +def test_cleanup_failure_does_not_mask_harness_exit_code(tmp_path: Path) -> None: + result, state, commands = _run_with_fake_docker( + tmp_path, + failed_command="run", + exit_code=33, + cleanup_exit_code=71, + ) + + assert result.returncode == 33 + assert list(state.iterdir()) == [] + assert commands == ["build", "up", "run", "down"] + assert "Compose cleanup failed with exit code 71" in result.stderr + + +def test_success_uses_the_same_cleanup_path(tmp_path: Path) -> None: + result, state, commands = _run_with_fake_docker( + tmp_path, + failed_command="none", + exit_code=0, + ) + + assert result.returncode == 0 + assert list(state.iterdir()) == [] + assert commands == ["build", "up", "run", "down"] + + +def test_cleanup_failure_makes_a_successful_run_fail(tmp_path: Path) -> None: + result, state, commands = _run_with_fake_docker( + tmp_path, + failed_command="none", + exit_code=0, + cleanup_exit_code=71, + ) + + assert result.returncode == 71 + assert list(state.iterdir()) == [] + assert commands == ["build", "up", "run", "down"] + assert "Compose cleanup failed with exit code 71" in result.stderr + + +def _run_with_fake_docker( + tmp_path: Path, + *, + failed_command: str, + exit_code: int, + cleanup_exit_code: int = 0, +) -> tuple[subprocess.CompletedProcess[str], Path, list[str]]: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + state = tmp_path / "state" + state.mkdir() + log = tmp_path / "docker.log" + docker = fake_bin / "docker" + docker.write_text( + """#!/bin/sh +set -eu + +command= +for argument in "$@"; do + case "$argument" in + build | config | down | run | up) + command=$argument + break + ;; + esac +done +printf '%s\n' "$command" >> "$FAKE_DOCKER_LOG" + +case "$command" in + build) + if [ "$FAKE_DOCKER_FAIL" = build ]; then + exit "$FAKE_DOCKER_EXIT" + fi + ;; + up) + touch "$FAKE_DOCKER_STATE/container" "$FAKE_DOCKER_STATE/network" "$FAKE_DOCKER_STATE/volume" + if [ "$FAKE_DOCKER_FAIL" = up ]; then + exit "$FAKE_DOCKER_EXIT" + fi + ;; + run) + if [ "$FAKE_DOCKER_FAIL" = run ]; then + exit "$FAKE_DOCKER_EXIT" + fi + ;; + down) + rm -f "$FAKE_DOCKER_STATE"/* + exit "$FAKE_DOCKER_CLEANUP_EXIT" + ;; +esac +""", + encoding="utf-8", + ) + docker.chmod(0o755) + environment = os.environ.copy() + environment.update({ + "FAKE_DOCKER_CLEANUP_EXIT": str(cleanup_exit_code), + "FAKE_DOCKER_EXIT": str(exit_code), + "FAKE_DOCKER_FAIL": failed_command, + "FAKE_DOCKER_LOG": str(log), + "FAKE_DOCKER_STATE": str(state), + "GITHUB_SHA": "test-revision", + "PATH": f"{fake_bin}{os.pathsep}{environment['PATH']}", + "POWERCONTEXT_E2E_DATABASE": "sqlite", + "POWERCONTEXT_E2E_OUTPUT": str(tmp_path / "evidence"), + }) + result = subprocess.run( # noqa: S603 - executes the repository script with an isolated fake Docker binary. + ["/bin/sh", str(_RUN_SCRIPT), "acceptance"], + cwd=_REPOSITORY_ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + commands = log.read_text(encoding="utf-8").splitlines() + return result, state, commands diff --git a/e2e/bub/uv.lock b/e2e/bub/uv.lock index 595a5ab07..ae2e3bc71 100644 --- a/e2e/bub/uv.lock +++ b/e2e/bub/uv.lock @@ -607,6 +607,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "inquirer-textual" version = "0.6.1" @@ -940,6 +949,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, ] +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + [[package]] name = "platformdirs" version = "4.11.0" @@ -1049,6 +1067,11 @@ dependencies = [ { name = "pyyaml" }, ] +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + [package.metadata] requires-dist = [ { name = "bub", specifier = ">=0.4.0,<0.5.0" }, @@ -1060,6 +1083,9 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6,<7" }, ] +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=9.0.2" }] + [[package]] name = "prompt-toolkit" version = "3.0.53" @@ -1340,6 +1366,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.2" From e8de89edacbfdb0a6785833c4cf024585c0e76b6 Mon Sep 17 00:00:00 2001 From: ThunGuo Date: Tue, 11 Aug 2026 13:33:57 +0800 Subject: [PATCH 2/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- e2e/bub/src/powercontext_e2e/redaction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/bub/src/powercontext_e2e/redaction.py b/e2e/bub/src/powercontext_e2e/redaction.py index 1ce3cea2f..ecde20b65 100644 --- a/e2e/bub/src/powercontext_e2e/redaction.py +++ b/e2e/bub/src/powercontext_e2e/redaction.py @@ -78,7 +78,7 @@ ) _URL_PASSWORD = re.compile(r"(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@]+:)[^@/\s\"'`<>]+(@)") _URL_QUERY_CREDENTIAL = re.compile( - r"(?i)([?&](?:access[_-]?token|api[_-]?key|auth[_-]?token|client[_-]?secret|password|secret|token)=)" + r"(?i)([?&](?:access[_-]?token|api[_-]?key|auth[_-]?token|client[_-]?secret|password|refresh[_-]?token|secret|token)=)" r"[^&#\s\"'`]+" ) _CREDENTIAL_ASSIGNMENT = re.compile( From e92194a7148f870b4d265aab0ea32529e1b9c19a Mon Sep 17 00:00:00 2001 From: ThunGuo Date: Tue, 11 Aug 2026 14:20:32 +0800 Subject: [PATCH 3/7] fix: copilot comment --- e2e/bub/src/powercontext_e2e/runner.py | 32 +++++++++++++++----------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/e2e/bub/src/powercontext_e2e/runner.py b/e2e/bub/src/powercontext_e2e/runner.py index 62e4ecefb..9c043752a 100644 --- a/e2e/bub/src/powercontext_e2e/runner.py +++ b/e2e/bub/src/powercontext_e2e/runner.py @@ -151,6 +151,7 @@ async def evaluate_scenario( ) -> bool: _configure_tracing() judge = _judge_model() if mode == "live" else None + redactor = EvidenceRedactor.from_environment() evaluator = ReplayEvaluator(judge_model=judge) dataset = Dataset[ScenarioSpec, ReplayObservation, dict[str, str]]( name="powercontext-session-replay", @@ -159,16 +160,17 @@ async def evaluate_scenario( ) async def run(inputs: ScenarioSpec) -> ReplayObservation: - return await _run_replay(inputs, mode=mode, judge_model=_judge_model_name(judge)) + return await _run_replay(inputs, mode=mode, judge_model=_judge_model_name(judge), redactor=redactor) report = await dataset.evaluate(run, name=f"{mode}:{scenario.id}", max_concurrency=1, progress=False) observation = report.cases[0].output - write_artifacts(observation, report, output_dir) + write_artifacts(observation, report, output_dir, redactor=redactor) return not report.failures and all(result.value for result in report.cases[0].assertions.values()) async def rescore_replay(replay_path: Path, output_dir: Path) -> bool: _configure_tracing() + redactor = EvidenceRedactor.from_environment() observation = ReplayObservation.model_validate_json(replay_path.read_text(encoding="utf-8")) environment = observation.environment.model_copy(update={"mode": "offline-rescore"}) observation = observation.model_copy(update={"environment": environment}) @@ -183,7 +185,7 @@ async def recorded(_: ScenarioSpec) -> ReplayObservation: return observation report = await dataset.evaluate(recorded, name=f"offline:{observation.scenario.id}", progress=False) - write_artifacts(report.cases[0].output, report, output_dir) + write_artifacts(report.cases[0].output, report, output_dir, redactor=redactor) return not report.failures and all(result.value for result in report.cases[0].assertions.values()) @@ -192,6 +194,7 @@ async def _run_replay( *, mode: Literal["acceptance", "live"], judge_model: str | None, + redactor: EvidenceRedactor, ) -> ReplayObservation: run_id = f"{scenario.id}-{uuid4().hex[:12]}" scope_id = f"e2e:{run_id}" @@ -224,7 +227,7 @@ async def _run_replay( except Exception as exc: output = "" status = "failed" - error = _redact(f"{type(exc).__name__}: {exc}") + error = redactor.redact_text(f"{type(exc).__name__}: {exc}") errors.append(f"Session {session.id}: {error}") memory_after_session = await _memory_snapshot(client, scope_id) observations.append( @@ -234,7 +237,7 @@ async def _run_replay( status=status, error=error, prepared_context=prepared, - output=_redact(output), + output=redactor.redact_text(output), memory_after=memory_after_session, ) ) @@ -242,7 +245,7 @@ async def _run_replay( break memory_after = await _memory_snapshot(client, scope_id) except Exception as exc: - errors.append(_redact(f"{type(exc).__name__}: {exc}")) + errors.append(redactor.redact_text(f"{type(exc).__name__}: {exc}")) memory_after = observations[-1].memory_after if observations else memory_before return ReplayObservation( @@ -391,14 +394,17 @@ def _commit() -> str: return completed.stdout.strip() if completed.returncode == 0 else "unknown" -def _redact(value: str) -> str: - return EvidenceRedactor.from_environment().redact_text(value) - - -def write_artifacts(observation: ReplayObservation, report: Report, output_dir: Path) -> None: - redactor = EvidenceRedactor.from_environment() +def write_artifacts( + observation: ReplayObservation, + report: Report, + output_dir: Path, + *, + redactor: EvidenceRedactor | None = None, +) -> None: + if redactor is None: + redactor = EvidenceRedactor.from_environment() output_dir.mkdir(parents=True, exist_ok=True) - replay_payload = json.loads(observation.model_dump_json(by_alias=True)) + replay_payload = observation.model_dump(mode="json", by_alias=True) _write_evidence( output_dir / "replay.json", json.dumps(redactor.redact(replay_payload), indent=2, ensure_ascii=False) + "\n", From e9e3dfee4af231e148c3b98e64255659f00d5dc8 Mon Sep 17 00:00:00 2001 From: ThunGuo Date: Tue, 11 Aug 2026 15:47:06 +0800 Subject: [PATCH 4/7] =?UTF-8?q?improve=EF=BC=9Asimplify=20the=20method=20f?= =?UTF-8?q?or=20handling=20evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/e2e-harness.yml | 64 +++++++- e2e/bub/README.md | 6 +- e2e/bub/src/powercontext_e2e/redaction.py | 190 ---------------------- e2e/bub/src/powercontext_e2e/runner.py | 59 ++++--- e2e/bub/tests/test_evidence_redaction.py | 69 +++----- e2e/bub/tests/test_run_script.py | 43 ++--- 6 files changed, 129 insertions(+), 302 deletions(-) delete mode 100644 e2e/bub/src/powercontext_e2e/redaction.py diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index abca0be89..a71482e47 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -11,6 +11,9 @@ on: permissions: contents: read +env: + TRUFFLEHOG_IMAGE: ghcr.io/trufflesecurity/trufflehog:3.96.0 + concurrency: group: e2e-harness-${{ github.event.pull_request.number || github.ref }}-${{ github.sha }} cancel-in-progress: false @@ -50,8 +53,31 @@ jobs: POWERCONTEXT_E2E_OUTPUT: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/acceptance run: make harness-compose-acceptance - - name: Upload replay evidence + - name: Scan replay evidence + id: evidence_scan if: always() + continue-on-error: true + env: + EVIDENCE_PATH: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/acceptance + run: | + test -d "${EVIDENCE_PATH}" + docker run --rm \ + --volume "${EVIDENCE_PATH}:/evidence:ro" \ + "${TRUFFLEHOG_IMAGE}" \ + filesystem /evidence \ + --no-verification \ + --results=verified,unknown,unverified \ + --fail \ + --fail-on-scan-errors \ + --no-update \ + --json > "${RUNNER_TEMP}/trufflehog-acceptance-${{ matrix.database }}.jsonl" + + - name: Report suppressed replay evidence + if: always() && steps.evidence_scan.outcome != 'success' + run: echo "::warning::Replay evidence was not published because secret scanning did not complete cleanly." + + - name: Upload replay evidence + if: always() && steps.evidence_scan.outcome == 'success' uses: actions/upload-artifact@v7 with: name: e2e-acceptance-${{ matrix.database }}-${{ github.sha }} @@ -102,12 +128,44 @@ jobs: POWERCONTEXT_E2E_OUTPUT: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/live run: make harness-compose-live - - name: Publish replay summary + - name: Scan replay evidence + id: evidence_scan if: always() && steps.provider.outputs.configured == 'true' + continue-on-error: true + env: + EVIDENCE_PATH: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/live + run: | + test -d "${EVIDENCE_PATH}" + docker run --rm \ + --volume "${EVIDENCE_PATH}:/evidence:ro" \ + "${TRUFFLEHOG_IMAGE}" \ + filesystem /evidence \ + --no-verification \ + --results=verified,unknown,unverified \ + --fail \ + --fail-on-scan-errors \ + --no-update \ + --json > "${RUNNER_TEMP}/trufflehog-live-${{ matrix.database }}.jsonl" + + - name: Report suppressed replay evidence + if: >- + always() && + steps.provider.outputs.configured == 'true' && + steps.evidence_scan.outcome != 'success' + run: echo "::warning::Replay evidence was not published because secret scanning did not complete cleanly." + + - name: Publish replay summary + if: >- + always() && + steps.provider.outputs.configured == 'true' && + steps.evidence_scan.outcome == 'success' run: cat ".powercontext-e2e/${{ matrix.database }}/live/report.md" >> "${GITHUB_STEP_SUMMARY}" - name: Upload replay evidence - if: always() && steps.provider.outputs.configured == 'true' + if: >- + always() && + steps.provider.outputs.configured == 'true' && + steps.evidence_scan.outcome == 'success' uses: actions/upload-artifact@v7 with: name: e2e-live-${{ matrix.database }}-${{ github.sha }} diff --git a/e2e/bub/README.md b/e2e/bub/README.md index af0afbe94..5cb7f6929 100644 --- a/e2e/bub/README.md +++ b/e2e/bub/README.md @@ -12,8 +12,10 @@ It supports three evidence modes: - `rescore` reads `replay.json` and runs the same Pydantic Evals oracle without rerunning Bub or PowerContext. Each run writes `replay.json`, `eval-report.json`, and `report.md`. The replay is self-contained and contains the -scenario, public Memory snapshots, prepared context, outputs, and Pydantic Evals-compatible spans. It never records -API keys, authorization headers, or database URLs. +scenario, public Memory snapshots, prepared context, outputs, and Pydantic Evals-compatible spans. Known runtime +secrets are redacted when these files are written. CI also scans the complete evidence directory with TruffleHog +before publishing a summary or artifact; evidence is not published when that scan does not complete cleanly. Treat +local evidence as potentially sensitive until it has been inspected. ## Run against an existing Server diff --git a/e2e/bub/src/powercontext_e2e/redaction.py b/e2e/bub/src/powercontext_e2e/redaction.py deleted file mode 100644 index ecde20b65..000000000 --- a/e2e/bub/src/powercontext_e2e/redaction.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Credential redaction at the replay evidence boundary.""" - -from __future__ import annotations - -import os -import re -from collections.abc import Mapping -from dataclasses import dataclass -from typing import Any -from urllib.parse import parse_qsl, quote, unquote, urlsplit - -REDACTED = "[REDACTED]" - -_SENSITIVE_ENVIRONMENT_NAMES = frozenset({ - "api_key", - "authorization", - "credentials", - "database_url", - "mysql_pwd", - "password", - "pgpassword", - "private_key", - "secret", - "token", -}) -_SENSITIVE_ENVIRONMENT_SUFFIXES = ( - "_access_key_id", - "_api_key", - "_authorization", - "_client_secret", - "_credentials", - "_database_url", - "_dsn", - "_password", - "_private_key", - "_secret", - "_secret_access_key", - "_secret_key", - "_token", -) -_SENSITIVE_FIELD_NAMES = frozenset({ - "access_token", - "api_key", - "auth_token", - "authorization", - "client_secret", - "connection_string", - "credentials", - "database_url", - "dsn", - "password", - "private_key", - "proxy_authorization", - "refresh_token", - "secret", - "secret_key", - "token", -}) -_SENSITIVE_FIELD_SUFFIXES = ( - "_access_token", - "_api_key", - "_auth_token", - "_authorization", - "_client_secret", - "_connection_string", - "_credentials", - "_database_url", - "_dsn", - "_password", - "_private_key", - "_refresh_token", - "_secret", - "_secret_key", -) -_DATABASE_URL = re.compile( - r"(?i)\b(?:cockroachdb|mariadb|mssql|mysql|oceanbase|oracle|postgres|postgresql|sqlite)" - r"(?:\+[a-z0-9_.-]+)?://[^\s\"'`<>{}\[\](),;\\]+" -) -_URL_PASSWORD = re.compile(r"(?i)(\b[a-z][a-z0-9+.-]*://[^/\s:@]+:)[^@/\s\"'`<>]+(@)") -_URL_QUERY_CREDENTIAL = re.compile( - r"(?i)([?&](?:access[_-]?token|api[_-]?key|auth[_-]?token|client[_-]?secret|password|refresh[_-]?token|secret|token)=)" - r"[^&#\s\"'`]+" -) -_CREDENTIAL_ASSIGNMENT = re.compile( - r"(?i)(\b(?:access[_-]?token|api[_-]?key|auth[_-]?token|authorization|client[_-]?secret|password|secret)" - r"\b\s*[:=]\s*)(?:(?:basic|bearer)\s+)?[^\s,;&\"'`]+" -) -_CREDENTIAL_OPTION = re.compile( - r"(?i)(--(?:access-token|api-key|auth-token|client-secret|password|token)\s+)" - r"[^\s,;\"'`]+" -) -_BEARER_CREDENTIAL = re.compile(r"(?i)\bbearer\s+[a-z0-9._~+/=-]{16,}") -_PROVIDER_CREDENTIAL = re.compile(r"(? EvidenceRedactor: - secrets: set[str] = set() - for name, value in os.environ.items(): - if value and _is_sensitive_environment_name(name): - secrets.update(_secret_variants(value)) - return cls(tuple(sorted(secrets, key=lambda item: (-len(item), item)))) - - def redact(self, value: Any) -> Any: - """Recursively redact JSON-compatible evidence without changing its normal shape.""" - return self._redact(value, sensitive=False) - - def redact_text(self, value: str) -> str: - """Redact one final serialized evidence value.""" - for secret in self.secrets: - if len(secret) >= 8: - value = value.replace(secret, REDACTED) - else: - value = re.sub(rf"(? Any: - if isinstance(value, str): - return REDACTED if sensitive and value else self.redact_text(value) - if isinstance(value, Mapping): - return { - self.redact_text(key) if isinstance(key, str) else key: self._redact( - item, - sensitive=sensitive or (isinstance(key, str) and _is_sensitive_field_name(key)), - ) - for key, item in value.items() - } - if isinstance(value, list): - return [self._redact(item, sensitive=sensitive) for item in value] - if isinstance(value, tuple): - return tuple(self._redact(item, sensitive=sensitive) for item in value) - return value - - -def _is_sensitive_environment_name(name: str) -> bool: - normalized = _normalize_name(name) - return normalized in _SENSITIVE_ENVIRONMENT_NAMES or normalized.endswith(_SENSITIVE_ENVIRONMENT_SUFFIXES) - - -def _is_sensitive_field_name(name: str) -> bool: - normalized = _normalize_name(name) - return normalized in _SENSITIVE_FIELD_NAMES or normalized.endswith(_SENSITIVE_FIELD_SUFFIXES) - - -def _normalize_name(value: str) -> str: - return re.sub(r"[^a-z0-9]+", "_", value.casefold()).strip("_") - - -def _secret_variants(value: str) -> set[str]: - variants = {value} - stripped = value.strip() - if stripped: - variants.add(stripped) - lowered = stripped.casefold() - for prefix in ("basic ", "bearer "): - if lowered.startswith(prefix): - variants.add(stripped[len(prefix) :]) - variants.update(_url_secret_variants(stripped)) - for item in tuple(variants): - if item: - variants.add(quote(item, safe="")) - return {item for item in variants if item} - - -def _url_secret_variants(value: str) -> set[str]: - try: - parsed = urlsplit(value) - except ValueError: - return set() - if not parsed.netloc: - return set() - variants = set() - if parsed.password: - variants.update((parsed.password, unquote(parsed.password))) - for name, item in parse_qsl(parsed.query, keep_blank_values=True): - if item and _is_sensitive_field_name(name): - variants.update((item, unquote(item))) - return variants diff --git a/e2e/bub/src/powercontext_e2e/runner.py b/e2e/bub/src/powercontext_e2e/runner.py index 9c043752a..55b7d7c67 100644 --- a/e2e/bub/src/powercontext_e2e/runner.py +++ b/e2e/bub/src/powercontext_e2e/runner.py @@ -36,11 +36,21 @@ ScenarioSpec, SessionObservation, ) -from .redaction import EvidenceRedactor Mode = Literal["acceptance", "live", "offline-rescore"] Report = EvaluationReport[ScenarioSpec, ReplayObservation, dict[str, str]] Context = EvaluatorContext[ScenarioSpec, ReplayObservation, dict[str, str]] +_EVIDENCE_SECRET_ENVIRONMENT_NAMES = ( + "ANTHROPIC_API_KEY", + "BUB_API_KEY", + "DEEPSEEK_API_KEY", + "OPENAI_API_KEY", + "OPENROUTER_API_KEY", + "POWERCONTEXT_CLIENT_API_TOKEN", + "POWERCONTEXT_SERVER_AUTH_TOKEN", + "POWERCONTEXT_SERVER_DATABASE_URL", +) +_REDACTED = "[REDACTED]" @dataclass @@ -151,7 +161,6 @@ async def evaluate_scenario( ) -> bool: _configure_tracing() judge = _judge_model() if mode == "live" else None - redactor = EvidenceRedactor.from_environment() evaluator = ReplayEvaluator(judge_model=judge) dataset = Dataset[ScenarioSpec, ReplayObservation, dict[str, str]]( name="powercontext-session-replay", @@ -160,17 +169,16 @@ async def evaluate_scenario( ) async def run(inputs: ScenarioSpec) -> ReplayObservation: - return await _run_replay(inputs, mode=mode, judge_model=_judge_model_name(judge), redactor=redactor) + return await _run_replay(inputs, mode=mode, judge_model=_judge_model_name(judge)) report = await dataset.evaluate(run, name=f"{mode}:{scenario.id}", max_concurrency=1, progress=False) observation = report.cases[0].output - write_artifacts(observation, report, output_dir, redactor=redactor) + write_artifacts(observation, report, output_dir) return not report.failures and all(result.value for result in report.cases[0].assertions.values()) async def rescore_replay(replay_path: Path, output_dir: Path) -> bool: _configure_tracing() - redactor = EvidenceRedactor.from_environment() observation = ReplayObservation.model_validate_json(replay_path.read_text(encoding="utf-8")) environment = observation.environment.model_copy(update={"mode": "offline-rescore"}) observation = observation.model_copy(update={"environment": environment}) @@ -185,7 +193,7 @@ async def recorded(_: ScenarioSpec) -> ReplayObservation: return observation report = await dataset.evaluate(recorded, name=f"offline:{observation.scenario.id}", progress=False) - write_artifacts(report.cases[0].output, report, output_dir, redactor=redactor) + write_artifacts(report.cases[0].output, report, output_dir) return not report.failures and all(result.value for result in report.cases[0].assertions.values()) @@ -194,7 +202,6 @@ async def _run_replay( *, mode: Literal["acceptance", "live"], judge_model: str | None, - redactor: EvidenceRedactor, ) -> ReplayObservation: run_id = f"{scenario.id}-{uuid4().hex[:12]}" scope_id = f"e2e:{run_id}" @@ -227,7 +234,7 @@ async def _run_replay( except Exception as exc: output = "" status = "failed" - error = redactor.redact_text(f"{type(exc).__name__}: {exc}") + error = f"{type(exc).__name__}: {exc}" errors.append(f"Session {session.id}: {error}") memory_after_session = await _memory_snapshot(client, scope_id) observations.append( @@ -237,7 +244,7 @@ async def _run_replay( status=status, error=error, prepared_context=prepared, - output=redactor.redact_text(output), + output=output, memory_after=memory_after_session, ) ) @@ -245,7 +252,7 @@ async def _run_replay( break memory_after = await _memory_snapshot(client, scope_id) except Exception as exc: - errors.append(redactor.redact_text(f"{type(exc).__name__}: {exc}")) + errors.append(f"{type(exc).__name__}: {exc}") memory_after = observations[-1].memory_after if observations else memory_before return ReplayObservation( @@ -394,21 +401,14 @@ def _commit() -> str: return completed.stdout.strip() if completed.returncode == 0 else "unknown" -def write_artifacts( - observation: ReplayObservation, - report: Report, - output_dir: Path, - *, - redactor: EvidenceRedactor | None = None, -) -> None: - if redactor is None: - redactor = EvidenceRedactor.from_environment() +def write_artifacts(observation: ReplayObservation, report: Report, output_dir: Path) -> None: + secrets = _runtime_secrets() output_dir.mkdir(parents=True, exist_ok=True) replay_payload = observation.model_dump(mode="json", by_alias=True) _write_evidence( output_dir / "replay.json", - json.dumps(redactor.redact(replay_payload), indent=2, ensure_ascii=False) + "\n", - redactor, + json.dumps(replay_payload, indent=2, ensure_ascii=False) + "\n", + secrets, ) cases = [ @@ -439,8 +439,8 @@ def write_artifacts( } _write_evidence( output_dir / "eval-report.json", - json.dumps(redactor.redact(report_payload), indent=2, sort_keys=True, ensure_ascii=False) + "\n", - redactor, + json.dumps(report_payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n", + secrets, ) _write_evidence( output_dir / "report.md", @@ -451,9 +451,16 @@ def write_artifacts( f"- Status: `{observation.status}`\n\n" "## Evaluation\n\n" f"```text\n{report.render(include_reasons=True)}\n```\n", - redactor, + secrets, ) -def _write_evidence(path: Path, content: str, redactor: EvidenceRedactor) -> None: - path.write_text(redactor.redact_text(content), encoding="utf-8") +def _runtime_secrets() -> tuple[str, ...]: + secrets = {value for name in _EVIDENCE_SECRET_ENVIRONMENT_NAMES if (value := os.getenv(name))} + return tuple(sorted(secrets, key=lambda value: (-len(value), value))) + + +def _write_evidence(path: Path, content: str, secrets: tuple[str, ...]) -> None: + for secret in secrets: + content = content.replace(secret, _REDACTED) + path.write_text(content, encoding="utf-8") diff --git a/e2e/bub/tests/test_evidence_redaction.py b/e2e/bub/tests/test_evidence_redaction.py index 4a35603de..209402d5a 100644 --- a/e2e/bub/tests/test_evidence_redaction.py +++ b/e2e/bub/tests/test_evidence_redaction.py @@ -20,68 +20,37 @@ from powercontext_e2e.runner import write_artifacts -def test_write_artifacts_redacts_every_evidence_sink(monkeypatch, tmp_path: Path) -> None: - credentials = { - "BUB_API_KEY": "sk-bub-evidence-sentinel", - "OPENAI_API_KEY": "sk-openai-evidence-sentinel", - "POWERCONTEXT_SERVER_AUTH_TOKEN": "server-auth-evidence-sentinel", - "POWERCONTEXT_CODEX_AUTHORIZATION": "Bearer codex-auth-evidence-sentinel", - "POWERCONTEXT_SERVER_DATABASE_URL": ( - "mysql+aoceanbase://root:database-evidence-sentinel@db.example/powercontext" - ), - } - for name, value in credentials.items(): - monkeypatch.setenv(name, value) - - extra_sentinels = ( - "header-evidence-sentinel", - "span-database-evidence-sentinel", - "query-evidence-sentinel", - "attribute-evidence-sentinel", - "failure-evidence-sentinel", - "basic-evidence-sentinel", - "sk-unregistered-evidence-sentinel", - ) +def test_write_artifacts_redacts_known_runtime_secrets_at_every_sink(monkeypatch, tmp_path: Path) -> None: + runtime_secrets = ("provider-runtime-secret-sentinel", "server-runtime-secret-sentinel") + monkeypatch.setenv("BUB_API_KEY", runtime_secrets[0]) + monkeypatch.setenv("POWERCONTEXT_SERVER_AUTH_TOKEN", runtime_secrets[1]) + observation = _observation( - session_input=f"Question containing {credentials['BUB_API_KEY']}", - memory_text=f"Memory containing {credentials['OPENAI_API_KEY']}", - context=f"Context containing {credentials['POWERCONTEXT_SERVER_AUTH_TOKEN']}", - output=f"Output containing {credentials['POWERCONTEXT_CODEX_AUTHORIZATION']}", - error=f"Database failed at {credentials['POWERCONTEXT_SERVER_DATABASE_URL']}", - span_attributes={ - "http.request.header.authorization": f"Bearer {extra_sentinels[0]}", - "db.connection_string": (f"postgresql://user:{extra_sentinels[1]}@db.example/powercontext"), - "http.url": f"https://provider.example/v1?access_token={extra_sentinels[2]}&mode=live", - }, + session_input=f"Question containing {runtime_secrets[0]}", + memory_text="The project selected OceanBase.", + context="OceanBase supports shared persistent context.", + output=f"Output containing {runtime_secrets[0]}", + error=f"Request failed with {runtime_secrets[1]}", + span_attributes={"gen_ai.operation.name": "chat"}, ) report = _report( - assertion_reason=f"Judge echoed {credentials['OPENAI_API_KEY']} and {extra_sentinels[6]}", - score_reason=f"Authorization: {credentials['POWERCONTEXT_CODEX_AUTHORIZATION']}", - label_reason=f"Provider URL used api_key={extra_sentinels[2]}", - attributes={"api_key": extra_sentinels[3]}, - failure=f"password={extra_sentinels[4]}", - rendered=f"Authorization: Basic {extra_sentinels[5]} and {credentials['BUB_API_KEY']}", + assertion_reason=f"Judge echoed {runtime_secrets[0]}", + score_reason="The expected fact was present.", + label_reason="pass", + attributes={"mode": "live"}, + failure=f"Request failed with {runtime_secrets[1]}", + rendered=f"Run contained {runtime_secrets[0]} and {runtime_secrets[1]}", ) write_artifacts(observation, report, tmp_path) artifacts = {path.name: path.read_text(encoding="utf-8") for path in tmp_path.iterdir()} assert set(artifacts) == {"eval-report.json", "replay.json", "report.md"} - sentinels = ( - credentials["BUB_API_KEY"], - credentials["OPENAI_API_KEY"], - credentials["POWERCONTEXT_SERVER_AUTH_TOKEN"], - "codex-auth-evidence-sentinel", - "database-evidence-sentinel", - *extra_sentinels, - ) for artifact_name, content in artifacts.items(): - for sentinel in sentinels: - assert sentinel not in content, f"{sentinel!r} leaked into {artifact_name}" + for secret in runtime_secrets: + assert secret not in content, f"{secret!r} leaked into {artifact_name}" assert "[REDACTED]" in content - assert "mysql+aoceanbase://" not in artifacts["replay.json"] - assert "postgresql://" not in artifacts["replay.json"] json.loads(artifacts["replay.json"]) json.loads(artifacts["eval-report.json"]) diff --git a/e2e/bub/tests/test_run_script.py b/e2e/bub/tests/test_run_script.py index 2beb726e2..c4f796e00 100644 --- a/e2e/bub/tests/test_run_script.py +++ b/e2e/bub/tests/test_run_script.py @@ -11,20 +11,19 @@ @pytest.mark.parametrize( - ("failed_command", "exit_code", "expected_commands"), + ("failed_command", "exit_code"), [ - ("build", 31, ["build", "down"]), - ("up", 32, ["build", "up", "down"]), - ("run", 33, ["build", "up", "run", "down"]), + ("build", 31), + ("up", 32), + ("run", 33), ], ) def test_failure_cleans_compose_resources_and_preserves_exit_code( tmp_path: Path, failed_command: str, exit_code: int, - expected_commands: list[str], ) -> None: - result, state, commands = _run_with_fake_docker( + result, state = _run_with_fake_docker( tmp_path, failed_command=failed_command, exit_code=exit_code, @@ -32,11 +31,10 @@ def test_failure_cleans_compose_resources_and_preserves_exit_code( assert result.returncode == exit_code assert list(state.iterdir()) == [] - assert commands == expected_commands def test_cleanup_failure_does_not_mask_harness_exit_code(tmp_path: Path) -> None: - result, state, commands = _run_with_fake_docker( + result, state = _run_with_fake_docker( tmp_path, failed_command="run", exit_code=33, @@ -45,12 +43,11 @@ def test_cleanup_failure_does_not_mask_harness_exit_code(tmp_path: Path) -> None assert result.returncode == 33 assert list(state.iterdir()) == [] - assert commands == ["build", "up", "run", "down"] assert "Compose cleanup failed with exit code 71" in result.stderr def test_success_uses_the_same_cleanup_path(tmp_path: Path) -> None: - result, state, commands = _run_with_fake_docker( + result, state = _run_with_fake_docker( tmp_path, failed_command="none", exit_code=0, @@ -58,11 +55,10 @@ def test_success_uses_the_same_cleanup_path(tmp_path: Path) -> None: assert result.returncode == 0 assert list(state.iterdir()) == [] - assert commands == ["build", "up", "run", "down"] def test_cleanup_failure_makes_a_successful_run_fail(tmp_path: Path) -> None: - result, state, commands = _run_with_fake_docker( + result, state = _run_with_fake_docker( tmp_path, failed_command="none", exit_code=0, @@ -71,7 +67,6 @@ def test_cleanup_failure_makes_a_successful_run_fail(tmp_path: Path) -> None: assert result.returncode == 71 assert list(state.iterdir()) == [] - assert commands == ["build", "up", "run", "down"] assert "Compose cleanup failed with exit code 71" in result.stderr @@ -81,12 +76,11 @@ def _run_with_fake_docker( failed_command: str, exit_code: int, cleanup_exit_code: int = 0, -) -> tuple[subprocess.CompletedProcess[str], Path, list[str]]: +) -> tuple[subprocess.CompletedProcess[str], Path]: fake_bin = tmp_path / "bin" fake_bin.mkdir() state = tmp_path / "state" state.mkdir() - log = tmp_path / "docker.log" docker = fake_bin / "docker" docker.write_text( """#!/bin/sh @@ -101,22 +95,11 @@ def _run_with_fake_docker( ;; esac done -printf '%s\n' "$command" >> "$FAKE_DOCKER_LOG" case "$command" in - build) - if [ "$FAKE_DOCKER_FAIL" = build ]; then - exit "$FAKE_DOCKER_EXIT" - fi - ;; - up) + build | run | up) touch "$FAKE_DOCKER_STATE/container" "$FAKE_DOCKER_STATE/network" "$FAKE_DOCKER_STATE/volume" - if [ "$FAKE_DOCKER_FAIL" = up ]; then - exit "$FAKE_DOCKER_EXIT" - fi - ;; - run) - if [ "$FAKE_DOCKER_FAIL" = run ]; then + if [ "$FAKE_DOCKER_FAIL" = "$command" ]; then exit "$FAKE_DOCKER_EXIT" fi ;; @@ -134,7 +117,6 @@ def _run_with_fake_docker( "FAKE_DOCKER_CLEANUP_EXIT": str(cleanup_exit_code), "FAKE_DOCKER_EXIT": str(exit_code), "FAKE_DOCKER_FAIL": failed_command, - "FAKE_DOCKER_LOG": str(log), "FAKE_DOCKER_STATE": str(state), "GITHUB_SHA": "test-revision", "PATH": f"{fake_bin}{os.pathsep}{environment['PATH']}", @@ -149,5 +131,4 @@ def _run_with_fake_docker( capture_output=True, text=True, ) - commands = log.read_text(encoding="utf-8").splitlines() - return result, state, commands + return result, state From 2614cd7272204fb7d6a130fa5ffd568bf75ed18a Mon Sep 17 00:00:00 2001 From: ThunGuo Date: Tue, 11 Aug 2026 15:53:15 +0800 Subject: [PATCH 5/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- e2e/bub/src/powercontext_e2e/runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/bub/src/powercontext_e2e/runner.py b/e2e/bub/src/powercontext_e2e/runner.py index 55b7d7c67..bb0d460c9 100644 --- a/e2e/bub/src/powercontext_e2e/runner.py +++ b/e2e/bub/src/powercontext_e2e/runner.py @@ -463,4 +463,5 @@ def _runtime_secrets() -> tuple[str, ...]: def _write_evidence(path: Path, content: str, secrets: tuple[str, ...]) -> None: for secret in secrets: content = content.replace(secret, _REDACTED) + content = content.replace(json.dumps(secret, ensure_ascii=False)[1:-1], _REDACTED) path.write_text(content, encoding="utf-8") From 2abf351637ea2b8c62e028cb07e3a7150cbfab90 Mon Sep 17 00:00:00 2001 From: ThunGuo Date: Tue, 11 Aug 2026 20:20:29 +0800 Subject: [PATCH 6/7] Enable inclusion of hidden files in artifact upload --- .github/workflows/e2e-harness.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index a71482e47..fa8238ed3 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -82,6 +82,7 @@ jobs: with: name: e2e-acceptance-${{ matrix.database }}-${{ github.sha }} path: .powercontext-e2e/${{ matrix.database }}/acceptance + include-hidden-files: true if-no-files-found: error retention-days: 14 From 7136a68c8adb04b9eade71558701f6ac4c1e7df1 Mon Sep 17 00:00:00 2001 From: ThunGuo Date: Tue, 11 Aug 2026 20:49:31 +0800 Subject: [PATCH 7/7] Comment out live replay job in e2e-harness.yml Comment out the live replay job and its associated steps in the e2e-harness workflow. --- .github/workflows/e2e-harness.yml | 174 +++++++++++++++--------------- 1 file changed, 87 insertions(+), 87 deletions(-) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index fa8238ed3..44ebc698c 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -92,90 +92,90 @@ jobs: POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }} run: e2e/bub/run.sh down - live-replay: - needs: acceptance - if: github.event_name != 'pull_request' - runs-on: ubuntu-latest - timeout-minutes: 50 - strategy: - fail-fast: false - matrix: - database: [sqlite, oceanbase] - name: Live replay (${{ matrix.database }}) - steps: - - name: Check out - uses: actions/checkout@v7 - - - name: Check provider configuration - id: provider - env: - API_KEY: ${{ secrets.POWERCONTEXT_E2E_API_KEY }} - MODEL: ${{ vars.POWERCONTEXT_E2E_MODEL }} - run: | - if [[ -n "${API_KEY}" && -n "${MODEL}" ]]; then - echo "configured=true" >> "${GITHUB_OUTPUT}" - else - echo "configured=false" >> "${GITHUB_OUTPUT}" - echo "Live replay skipped because provider credentials are not configured." >> "${GITHUB_STEP_SUMMARY}" - fi - - - name: Run live replay - if: steps.provider.outputs.configured == 'true' - env: - BUB_API_BASE: ${{ vars.POWERCONTEXT_E2E_API_BASE }} - BUB_API_KEY: ${{ secrets.POWERCONTEXT_E2E_API_KEY }} - BUB_MODEL: ${{ vars.POWERCONTEXT_E2E_MODEL }} - POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }} - POWERCONTEXT_E2E_OUTPUT: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/live - run: make harness-compose-live - - - name: Scan replay evidence - id: evidence_scan - if: always() && steps.provider.outputs.configured == 'true' - continue-on-error: true - env: - EVIDENCE_PATH: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/live - run: | - test -d "${EVIDENCE_PATH}" - docker run --rm \ - --volume "${EVIDENCE_PATH}:/evidence:ro" \ - "${TRUFFLEHOG_IMAGE}" \ - filesystem /evidence \ - --no-verification \ - --results=verified,unknown,unverified \ - --fail \ - --fail-on-scan-errors \ - --no-update \ - --json > "${RUNNER_TEMP}/trufflehog-live-${{ matrix.database }}.jsonl" - - - name: Report suppressed replay evidence - if: >- - always() && - steps.provider.outputs.configured == 'true' && - steps.evidence_scan.outcome != 'success' - run: echo "::warning::Replay evidence was not published because secret scanning did not complete cleanly." - - - name: Publish replay summary - if: >- - always() && - steps.provider.outputs.configured == 'true' && - steps.evidence_scan.outcome == 'success' - run: cat ".powercontext-e2e/${{ matrix.database }}/live/report.md" >> "${GITHUB_STEP_SUMMARY}" - - - name: Upload replay evidence - if: >- - always() && - steps.provider.outputs.configured == 'true' && - steps.evidence_scan.outcome == 'success' - uses: actions/upload-artifact@v7 - with: - name: e2e-live-${{ matrix.database }}-${{ github.sha }} - path: .powercontext-e2e/${{ matrix.database }}/live - if-no-files-found: error - retention-days: 14 - - - name: Stop harness environment - if: always() - env: - POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }} - run: e2e/bub/run.sh down + # live-replay: + # needs: acceptance + # if: github.event_name != 'pull_request' + # runs-on: ubuntu-latest + # timeout-minutes: 50 + # strategy: + # fail-fast: false + # matrix: + # database: [sqlite, oceanbase] + # name: Live replay (${{ matrix.database }}) + # steps: + # - name: Check out + # uses: actions/checkout@v7 + + # - name: Check provider configuration + # id: provider + # env: + # API_KEY: ${{ secrets.POWERCONTEXT_E2E_API_KEY }} + # MODEL: ${{ vars.POWERCONTEXT_E2E_MODEL }} + # run: | + # if [[ -n "${API_KEY}" && -n "${MODEL}" ]]; then + # echo "configured=true" >> "${GITHUB_OUTPUT}" + # else + # echo "configured=false" >> "${GITHUB_OUTPUT}" + # echo "Live replay skipped because provider credentials are not configured." >> "${GITHUB_STEP_SUMMARY}" + # fi + + # - name: Run live replay + # if: steps.provider.outputs.configured == 'true' + # env: + # BUB_API_BASE: ${{ vars.POWERCONTEXT_E2E_API_BASE }} + # BUB_API_KEY: ${{ secrets.POWERCONTEXT_E2E_API_KEY }} + # BUB_MODEL: ${{ vars.POWERCONTEXT_E2E_MODEL }} + # POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }} + # POWERCONTEXT_E2E_OUTPUT: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/live + # run: make harness-compose-live + + # - name: Scan replay evidence + # id: evidence_scan + # if: always() && steps.provider.outputs.configured == 'true' + # continue-on-error: true + # env: + # EVIDENCE_PATH: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/live + # run: | + # test -d "${EVIDENCE_PATH}" + # docker run --rm \ + # --volume "${EVIDENCE_PATH}:/evidence:ro" \ + # "${TRUFFLEHOG_IMAGE}" \ + # filesystem /evidence \ + # --no-verification \ + # --results=verified,unknown,unverified \ + # --fail \ + # --fail-on-scan-errors \ + # --no-update \ + # --json > "${RUNNER_TEMP}/trufflehog-live-${{ matrix.database }}.jsonl" + + # - name: Report suppressed replay evidence + # if: >- + # always() && + # steps.provider.outputs.configured == 'true' && + # steps.evidence_scan.outcome != 'success' + # run: echo "::warning::Replay evidence was not published because secret scanning did not complete cleanly." + + # - name: Publish replay summary + # if: >- + # always() && + # steps.provider.outputs.configured == 'true' && + # steps.evidence_scan.outcome == 'success' + # run: cat ".powercontext-e2e/${{ matrix.database }}/live/report.md" >> "${GITHUB_STEP_SUMMARY}" + + # - name: Upload replay evidence + # if: >- + # always() && + # steps.provider.outputs.configured == 'true' && + # steps.evidence_scan.outcome == 'success' + # uses: actions/upload-artifact@v7 + # with: + # name: e2e-live-${{ matrix.database }}-${{ github.sha }} + # path: .powercontext-e2e/${{ matrix.database }}/live + # if-no-files-found: error + # retention-days: 14 + + # - name: Stop harness environment + # if: always() + # env: + # POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }} + # run: e2e/bub/run.sh down