diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index abca0be89..44ebc698c 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -11,6 +11,9 @@ on: permissions: contents: read +env: + TRUFFLEHOG_IMAGE: ghcr.io/trufflesecurity/trufflehog:3.96.0 + concurrency: group: e2e-harness-${{ github.event.pull_request.number || github.ref }}-${{ github.sha }} cancel-in-progress: false @@ -50,68 +53,36 @@ jobs: POWERCONTEXT_E2E_OUTPUT: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/acceptance run: make harness-compose-acceptance - - name: Upload replay evidence + - name: Scan replay evidence + id: evidence_scan if: always() - uses: actions/upload-artifact@v7 - with: - name: e2e-acceptance-${{ matrix.database }}-${{ github.sha }} - path: .powercontext-e2e/${{ matrix.database }}/acceptance - if-no-files-found: error - retention-days: 14 - - - name: Stop harness environment - if: always() - env: - POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }} - run: e2e/bub/run.sh down - - live-replay: - needs: acceptance - if: github.event_name != 'pull_request' - runs-on: ubuntu-latest - timeout-minutes: 50 - strategy: - fail-fast: false - matrix: - database: [sqlite, oceanbase] - name: Live replay (${{ matrix.database }}) - steps: - - name: Check out - uses: actions/checkout@v7 - - - name: Check provider configuration - id: provider + continue-on-error: true env: - API_KEY: ${{ secrets.POWERCONTEXT_E2E_API_KEY }} - MODEL: ${{ vars.POWERCONTEXT_E2E_MODEL }} + EVIDENCE_PATH: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/acceptance run: | - if [[ -n "${API_KEY}" && -n "${MODEL}" ]]; then - echo "configured=true" >> "${GITHUB_OUTPUT}" - else - echo "configured=false" >> "${GITHUB_OUTPUT}" - echo "Live replay skipped because provider credentials are not configured." >> "${GITHUB_STEP_SUMMARY}" - fi - - - name: Run live replay - if: steps.provider.outputs.configured == 'true' - env: - BUB_API_BASE: ${{ vars.POWERCONTEXT_E2E_API_BASE }} - BUB_API_KEY: ${{ secrets.POWERCONTEXT_E2E_API_KEY }} - BUB_MODEL: ${{ vars.POWERCONTEXT_E2E_MODEL }} - POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }} - POWERCONTEXT_E2E_OUTPUT: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/live - run: make harness-compose-live - - - name: Publish replay summary - if: always() && steps.provider.outputs.configured == 'true' - run: cat ".powercontext-e2e/${{ matrix.database }}/live/report.md" >> "${GITHUB_STEP_SUMMARY}" + test -d "${EVIDENCE_PATH}" + docker run --rm \ + --volume "${EVIDENCE_PATH}:/evidence:ro" \ + "${TRUFFLEHOG_IMAGE}" \ + filesystem /evidence \ + --no-verification \ + --results=verified,unknown,unverified \ + --fail \ + --fail-on-scan-errors \ + --no-update \ + --json > "${RUNNER_TEMP}/trufflehog-acceptance-${{ matrix.database }}.jsonl" + + - name: Report suppressed replay evidence + if: always() && steps.evidence_scan.outcome != 'success' + run: echo "::warning::Replay evidence was not published because secret scanning did not complete cleanly." - name: Upload replay evidence - if: always() && steps.provider.outputs.configured == 'true' + if: always() && steps.evidence_scan.outcome == 'success' uses: actions/upload-artifact@v7 with: - name: e2e-live-${{ matrix.database }}-${{ github.sha }} - path: .powercontext-e2e/${{ matrix.database }}/live + name: e2e-acceptance-${{ matrix.database }}-${{ github.sha }} + path: .powercontext-e2e/${{ matrix.database }}/acceptance + include-hidden-files: true if-no-files-found: error retention-days: 14 @@ -120,3 +91,91 @@ jobs: env: POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }} run: e2e/bub/run.sh down + + # live-replay: + # needs: acceptance + # if: github.event_name != 'pull_request' + # runs-on: ubuntu-latest + # timeout-minutes: 50 + # strategy: + # fail-fast: false + # matrix: + # database: [sqlite, oceanbase] + # name: Live replay (${{ matrix.database }}) + # steps: + # - name: Check out + # uses: actions/checkout@v7 + + # - name: Check provider configuration + # id: provider + # env: + # API_KEY: ${{ secrets.POWERCONTEXT_E2E_API_KEY }} + # MODEL: ${{ vars.POWERCONTEXT_E2E_MODEL }} + # run: | + # if [[ -n "${API_KEY}" && -n "${MODEL}" ]]; then + # echo "configured=true" >> "${GITHUB_OUTPUT}" + # else + # echo "configured=false" >> "${GITHUB_OUTPUT}" + # echo "Live replay skipped because provider credentials are not configured." >> "${GITHUB_STEP_SUMMARY}" + # fi + + # - name: Run live replay + # if: steps.provider.outputs.configured == 'true' + # env: + # BUB_API_BASE: ${{ vars.POWERCONTEXT_E2E_API_BASE }} + # BUB_API_KEY: ${{ secrets.POWERCONTEXT_E2E_API_KEY }} + # BUB_MODEL: ${{ vars.POWERCONTEXT_E2E_MODEL }} + # POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }} + # POWERCONTEXT_E2E_OUTPUT: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/live + # run: make harness-compose-live + + # - name: Scan replay evidence + # id: evidence_scan + # if: always() && steps.provider.outputs.configured == 'true' + # continue-on-error: true + # env: + # EVIDENCE_PATH: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/live + # run: | + # test -d "${EVIDENCE_PATH}" + # docker run --rm \ + # --volume "${EVIDENCE_PATH}:/evidence:ro" \ + # "${TRUFFLEHOG_IMAGE}" \ + # filesystem /evidence \ + # --no-verification \ + # --results=verified,unknown,unverified \ + # --fail \ + # --fail-on-scan-errors \ + # --no-update \ + # --json > "${RUNNER_TEMP}/trufflehog-live-${{ matrix.database }}.jsonl" + + # - name: Report suppressed replay evidence + # if: >- + # always() && + # steps.provider.outputs.configured == 'true' && + # steps.evidence_scan.outcome != 'success' + # run: echo "::warning::Replay evidence was not published because secret scanning did not complete cleanly." + + # - name: Publish replay summary + # if: >- + # always() && + # steps.provider.outputs.configured == 'true' && + # steps.evidence_scan.outcome == 'success' + # run: cat ".powercontext-e2e/${{ matrix.database }}/live/report.md" >> "${GITHUB_STEP_SUMMARY}" + + # - name: Upload replay evidence + # if: >- + # always() && + # steps.provider.outputs.configured == 'true' && + # steps.evidence_scan.outcome == 'success' + # uses: actions/upload-artifact@v7 + # with: + # name: e2e-live-${{ matrix.database }}-${{ github.sha }} + # path: .powercontext-e2e/${{ matrix.database }}/live + # if-no-files-found: error + # retention-days: 14 + + # - name: Stop harness environment + # if: always() + # env: + # POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }} + # run: e2e/bub/run.sh down diff --git a/Makefile b/Makefile index 9927712e8..7383df385 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,7 @@ harness-check: ## Validate the Bub replay harness and committed scenarios. @uv run ruff check e2e/bub @uv run ruff format --check e2e/bub @uv run ty check --project e2e/bub --python e2e/bub/.venv e2e/bub/src integrations/bub/src + @uv run --project e2e/bub python -m pytest e2e/bub/tests @uv run --project e2e/bub powercontext-e2e --help >/dev/null .PHONY: harness-acceptance diff --git a/e2e/bub/README.md b/e2e/bub/README.md index 805e7e093..5cb7f6929 100644 --- a/e2e/bub/README.md +++ b/e2e/bub/README.md @@ -12,8 +12,10 @@ It supports three evidence modes: - `rescore` reads `replay.json` and runs the same Pydantic Evals oracle without rerunning Bub or PowerContext. Each run writes `replay.json`, `eval-report.json`, and `report.md`. The replay is self-contained and contains the -scenario, public Memory snapshots, prepared context, outputs, and Pydantic Evals-compatible spans. It never records -API keys, authorization headers, or database URLs. +scenario, public Memory snapshots, prepared context, outputs, and Pydantic Evals-compatible spans. Known runtime +secrets are redacted when these files are written. CI also scans the complete evidence directory with TruffleHog +before publishing a summary or artifact; evidence is not published when that scan does not complete cleanly. Treat +local evidence as potentially sensitive until it has been inspected. ## Run against an existing Server @@ -51,4 +53,5 @@ POWERCONTEXT_E2E_DATABASE=oceanbase make harness-compose-acceptance ``` `make harness-compose-live` uses the provider variables above. Evidence is written below `.powercontext-e2e/bub/`; -set `POWERCONTEXT_E2E_OUTPUT` to keep it elsewhere. `make harness-compose-down` removes containers and database volumes. +set `POWERCONTEXT_E2E_OUTPUT` to keep it elsewhere. Compose containers, networks, and volumes are removed after both +successful and failed runs. `make harness-compose-down` remains available as an idempotent manual cleanup. diff --git a/e2e/bub/pyproject.toml b/e2e/bub/pyproject.toml index 8647e8e35..c9f0f8829 100644 --- a/e2e/bub/pyproject.toml +++ b/e2e/bub/pyproject.toml @@ -13,6 +13,9 @@ dependencies = [ "pyyaml>=6,<7", ] +[dependency-groups] +dev = ["pytest>=9.0.2"] + [project.scripts] powercontext-e2e = "powercontext_e2e.__main__:main" @@ -38,5 +41,8 @@ line-length = 120 select = ["A", "B", "C4", "C90", "E", "F", "I", "PGH", "RUF", "S", "SIM", "T10", "TRY", "UP", "W", "YTT"] ignore = ["E501"] +[tool.ruff.lint.per-file-ignores] +"tests/**/*.py" = ["S101"] + [tool.ruff.format] preview = true diff --git a/e2e/bub/run.sh b/e2e/bub/run.sh index 933e88c04..fcd9a6f39 100755 --- a/e2e/bub/run.sh +++ b/e2e/bub/run.sh @@ -68,6 +68,26 @@ if [ -z "${GITHUB_SHA:-}" ]; then export GITHUB_SHA fi +cleanup() { + status=$? + trap - EXIT INT TERM + set +e + + docker compose $compose_files down --volumes --remove-orphans + cleanup_status=$? + if [ "$cleanup_status" -ne 0 ]; then + echo "Compose cleanup failed with exit code $cleanup_status" >&2 + if [ "$status" -eq 0 ]; then + status=$cleanup_status + fi + fi + exit "$status" +} + +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + docker compose $compose_files build powercontext harness docker compose $compose_files up --detach --wait powercontext @@ -81,5 +101,3 @@ else scenario=${POWERCONTEXT_E2E_SCENARIO:-e2e/bub/scenarios/project-database-decision.yaml} docker compose $compose_files run --rm harness live "$scenario" --output /evidence fi - -docker compose $compose_files down --volumes --remove-orphans diff --git a/e2e/bub/src/powercontext_e2e/runner.py b/e2e/bub/src/powercontext_e2e/runner.py index 8f5352ef0..bb0d460c9 100644 --- a/e2e/bub/src/powercontext_e2e/runner.py +++ b/e2e/bub/src/powercontext_e2e/runner.py @@ -40,6 +40,17 @@ Mode = Literal["acceptance", "live", "offline-rescore"] Report = EvaluationReport[ScenarioSpec, ReplayObservation, dict[str, str]] Context = EvaluatorContext[ScenarioSpec, ReplayObservation, dict[str, str]] +_EVIDENCE_SECRET_ENVIRONMENT_NAMES = ( + "ANTHROPIC_API_KEY", + "BUB_API_KEY", + "DEEPSEEK_API_KEY", + "OPENAI_API_KEY", + "OPENROUTER_API_KEY", + "POWERCONTEXT_CLIENT_API_TOKEN", + "POWERCONTEXT_SERVER_AUTH_TOKEN", + "POWERCONTEXT_SERVER_DATABASE_URL", +) +_REDACTED = "[REDACTED]" @dataclass @@ -223,7 +234,7 @@ async def _run_replay( except Exception as exc: output = "" status = "failed" - error = _redact(f"{type(exc).__name__}: {exc}") + error = f"{type(exc).__name__}: {exc}" errors.append(f"Session {session.id}: {error}") memory_after_session = await _memory_snapshot(client, scope_id) observations.append( @@ -233,7 +244,7 @@ async def _run_replay( status=status, error=error, prepared_context=prepared, - output=_redact(output), + output=output, memory_after=memory_after_session, ) ) @@ -241,7 +252,7 @@ async def _run_replay( break memory_after = await _memory_snapshot(client, scope_id) except Exception as exc: - errors.append(_redact(f"{type(exc).__name__}: {exc}")) + errors.append(f"{type(exc).__name__}: {exc}") memory_after = observations[-1].memory_after if observations else memory_before return ReplayObservation( @@ -390,16 +401,14 @@ def _commit() -> str: return completed.stdout.strip() if completed.returncode == 0 else "unknown" -def _redact(value: str) -> str: - secret = os.getenv("BUB_API_KEY") - return value.replace(secret, "[REDACTED]") if secret else value - - def write_artifacts(observation: ReplayObservation, report: Report, output_dir: Path) -> None: + secrets = _runtime_secrets() output_dir.mkdir(parents=True, exist_ok=True) - (output_dir / "replay.json").write_text( - observation.model_dump_json(by_alias=True, indent=2) + "\n", - encoding="utf-8", + replay_payload = observation.model_dump(mode="json", by_alias=True) + _write_evidence( + output_dir / "replay.json", + json.dumps(replay_payload, indent=2, ensure_ascii=False) + "\n", + secrets, ) cases = [ @@ -428,11 +437,13 @@ def write_artifacts(observation: ReplayObservation, report: Report, output_dir: "cases": cases, "failures": [{"name": failure.name, "error": failure.error_message} for failure in report.failures], } - (output_dir / "eval-report.json").write_text( - json.dumps(report_payload, indent=2, sort_keys=True) + "\n", - encoding="utf-8", + _write_evidence( + output_dir / "eval-report.json", + json.dumps(report_payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n", + secrets, ) - (output_dir / "report.md").write_text( + _write_evidence( + output_dir / "report.md", "# PowerContext session replay\n\n" f"- Scenario: `{observation.scenario.id}`\n" f"- Mode: `{observation.environment.mode}`\n" @@ -440,5 +451,17 @@ def write_artifacts(observation: ReplayObservation, report: Report, output_dir: f"- Status: `{observation.status}`\n\n" "## Evaluation\n\n" f"```text\n{report.render(include_reasons=True)}\n```\n", - encoding="utf-8", + secrets, ) + + +def _runtime_secrets() -> tuple[str, ...]: + secrets = {value for name in _EVIDENCE_SECRET_ENVIRONMENT_NAMES if (value := os.getenv(name))} + return tuple(sorted(secrets, key=lambda value: (-len(value), value))) + + +def _write_evidence(path: Path, content: str, secrets: tuple[str, ...]) -> None: + for secret in secrets: + content = content.replace(secret, _REDACTED) + content = content.replace(json.dumps(secret, ensure_ascii=False)[1:-1], _REDACTED) + path.write_text(content, encoding="utf-8") diff --git a/e2e/bub/tests/test_evidence_redaction.py b/e2e/bub/tests/test_evidence_redaction.py new file mode 100644 index 000000000..209402d5a --- /dev/null +++ b/e2e/bub/tests/test_evidence_redaction.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +from pydantic_evals.otel import SpanNode + +from powercontext_e2e.models import ( + MemoryEntrySnapshot, + MemorySnapshot, + PreparedContextSnapshot, + ReplayObservation, + RunEnvironment, + ScenarioSpec, + SessionObservation, +) +from powercontext_e2e.runner import write_artifacts + + +def test_write_artifacts_redacts_known_runtime_secrets_at_every_sink(monkeypatch, tmp_path: Path) -> None: + runtime_secrets = ("provider-runtime-secret-sentinel", "server-runtime-secret-sentinel") + monkeypatch.setenv("BUB_API_KEY", runtime_secrets[0]) + monkeypatch.setenv("POWERCONTEXT_SERVER_AUTH_TOKEN", runtime_secrets[1]) + + observation = _observation( + session_input=f"Question containing {runtime_secrets[0]}", + memory_text="The project selected OceanBase.", + context="OceanBase supports shared persistent context.", + output=f"Output containing {runtime_secrets[0]}", + error=f"Request failed with {runtime_secrets[1]}", + span_attributes={"gen_ai.operation.name": "chat"}, + ) + report = _report( + assertion_reason=f"Judge echoed {runtime_secrets[0]}", + score_reason="The expected fact was present.", + label_reason="pass", + attributes={"mode": "live"}, + failure=f"Request failed with {runtime_secrets[1]}", + rendered=f"Run contained {runtime_secrets[0]} and {runtime_secrets[1]}", + ) + + write_artifacts(observation, report, tmp_path) + + artifacts = {path.name: path.read_text(encoding="utf-8") for path in tmp_path.iterdir()} + assert set(artifacts) == {"eval-report.json", "replay.json", "report.md"} + for artifact_name, content in artifacts.items(): + for secret in runtime_secrets: + assert secret not in content, f"{secret!r} leaked into {artifact_name}" + assert "[REDACTED]" in content + + json.loads(artifacts["replay.json"]) + json.loads(artifacts["eval-report.json"]) + + +def test_write_artifacts_preserves_normal_live_replay_schema(tmp_path: Path) -> None: + observation = _observation( + session_input="Which database was selected?", + memory_text="The project selected OceanBase; Bearer authentication is unrelated.", + context="OceanBase supports shared persistent context.", + output="The project selected OceanBase.", + error=None, + span_attributes={ + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "deepseek-v4-flash", + "gen_ai.usage.input_tokens": 21, + "http.url": "https://provider.example/v1?mode=live", + }, + ) + report = _report( + assertion_reason="The run completed.", + score_reason="The expected fact was present.", + label_reason="pass", + attributes={"mode": "live"}, + failure=None, + rendered="All checks passed.", + ) + expected_replay = json.loads(observation.model_dump_json(by_alias=True)) + expected_evaluation = { + "schema": "powercontext.session-replay-evaluation/v1", + "experiment": "live:database-decision", + "cases": [ + { + "name": "database-decision", + "assertions": {"run_completed": {"value": True, "reason": "The run completed."}}, + "scores": {"answer": {"value": 1.0, "reason": "The expected fact was present."}}, + "labels": {"judge": {"value": "pass", "reason": "pass"}}, + "metrics": {"model_calls": 1}, + "attributes": {"mode": "live"}, + "task_duration": 0.25, + "total_duration": 0.5, + } + ], + "failures": [], + } + + write_artifacts(observation, report, tmp_path) + + actual_replay = json.loads((tmp_path / "replay.json").read_text(encoding="utf-8")) + actual_evaluation = json.loads((tmp_path / "eval-report.json").read_text(encoding="utf-8")) + assert actual_replay == expected_replay + assert actual_evaluation == expected_evaluation + assert ReplayObservation.model_validate(actual_replay) == observation + + +def _observation( + *, + session_input: str, + memory_text: str, + context: str, + output: str, + error: str | None, + span_attributes: dict[str, Any], +) -> ReplayObservation: + scenario = ScenarioSpec.model_validate({ + "schema": "powercontext.session-replay/v1", + "id": "database-decision", + "sessions": [ + { + "id": "recall", + "input": session_input, + "expected_answer": "The project selected OceanBase.", + } + ], + }) + memory = MemorySnapshot( + entries=( + MemoryEntrySnapshot( + entry_id="memory-1", + entry_version_id="memory-version-1", + version=1, + kind="fact", + text=memory_text, + state="committed", + ), + ) + ) + session = SessionObservation( + id="recall", + agent_session_id="session-1", + status="failed" if error else "completed", + error=error, + prepared_context=PreparedContextSnapshot(status="ready", content=context), + output=output, + memory_after=memory, + ) + timestamp = datetime(2026, 8, 11, 1, 2, 3, tzinfo=UTC) + span = SpanNode( + name="bub.model", + trace_id=1, + span_id=2, + parent_span_id=None, + start_timestamp=timestamp, + end_timestamp=timestamp, + attributes=span_attributes, + ) + return ReplayObservation( + run_id="database-decision-run", + environment=RunEnvironment( + mode="live", + commit="abcdef0", + database="sqlite", + agent_model="deepseek:deepseek-v4-flash", + generation_model="deepseek:deepseek-v4-flash", + judge_model="deepseek:deepseek-v4-flash", + started_at=timestamp, + ), + scenario=scenario, + status="failed" if error else "completed", + errors=(error,) if error else (), + memory_before=memory, + memory_after=memory, + sessions=(session,), + spans=(span,), + ) + + +def _report( + *, + assertion_reason: str, + score_reason: str, + label_reason: str, + attributes: dict[str, Any], + failure: str | None, + rendered: str, +) -> Any: + case = SimpleNamespace( + name="database-decision", + assertions={"run_completed": SimpleNamespace(value=True, reason=assertion_reason)}, + scores={"answer": SimpleNamespace(value=1.0, reason=score_reason)}, + labels={"judge": SimpleNamespace(value="pass", reason=label_reason)}, + metrics={"model_calls": 1}, + attributes=attributes, + task_duration=0.25, + total_duration=0.5, + ) + failures = [] if failure is None else [SimpleNamespace(name="judge", error_message=failure)] + return SimpleNamespace( + name="live:database-decision", + cases=[case], + failures=failures, + render=lambda include_reasons: rendered, + ) diff --git a/e2e/bub/tests/test_run_script.py b/e2e/bub/tests/test_run_script.py new file mode 100644 index 000000000..c4f796e00 --- /dev/null +++ b/e2e/bub/tests/test_run_script.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest + +_REPOSITORY_ROOT = Path(__file__).parents[3] +_RUN_SCRIPT = _REPOSITORY_ROOT / "e2e" / "bub" / "run.sh" + + +@pytest.mark.parametrize( + ("failed_command", "exit_code"), + [ + ("build", 31), + ("up", 32), + ("run", 33), + ], +) +def test_failure_cleans_compose_resources_and_preserves_exit_code( + tmp_path: Path, + failed_command: str, + exit_code: int, +) -> None: + result, state = _run_with_fake_docker( + tmp_path, + failed_command=failed_command, + exit_code=exit_code, + ) + + assert result.returncode == exit_code + assert list(state.iterdir()) == [] + + +def test_cleanup_failure_does_not_mask_harness_exit_code(tmp_path: Path) -> None: + result, state = _run_with_fake_docker( + tmp_path, + failed_command="run", + exit_code=33, + cleanup_exit_code=71, + ) + + assert result.returncode == 33 + assert list(state.iterdir()) == [] + assert "Compose cleanup failed with exit code 71" in result.stderr + + +def test_success_uses_the_same_cleanup_path(tmp_path: Path) -> None: + result, state = _run_with_fake_docker( + tmp_path, + failed_command="none", + exit_code=0, + ) + + assert result.returncode == 0 + assert list(state.iterdir()) == [] + + +def test_cleanup_failure_makes_a_successful_run_fail(tmp_path: Path) -> None: + result, state = _run_with_fake_docker( + tmp_path, + failed_command="none", + exit_code=0, + cleanup_exit_code=71, + ) + + assert result.returncode == 71 + assert list(state.iterdir()) == [] + assert "Compose cleanup failed with exit code 71" in result.stderr + + +def _run_with_fake_docker( + tmp_path: Path, + *, + failed_command: str, + exit_code: int, + cleanup_exit_code: int = 0, +) -> tuple[subprocess.CompletedProcess[str], Path]: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + state = tmp_path / "state" + state.mkdir() + docker = fake_bin / "docker" + docker.write_text( + """#!/bin/sh +set -eu + +command= +for argument in "$@"; do + case "$argument" in + build | config | down | run | up) + command=$argument + break + ;; + esac +done + +case "$command" in + build | run | up) + touch "$FAKE_DOCKER_STATE/container" "$FAKE_DOCKER_STATE/network" "$FAKE_DOCKER_STATE/volume" + if [ "$FAKE_DOCKER_FAIL" = "$command" ]; then + exit "$FAKE_DOCKER_EXIT" + fi + ;; + down) + rm -f "$FAKE_DOCKER_STATE"/* + exit "$FAKE_DOCKER_CLEANUP_EXIT" + ;; +esac +""", + encoding="utf-8", + ) + docker.chmod(0o755) + environment = os.environ.copy() + environment.update({ + "FAKE_DOCKER_CLEANUP_EXIT": str(cleanup_exit_code), + "FAKE_DOCKER_EXIT": str(exit_code), + "FAKE_DOCKER_FAIL": failed_command, + "FAKE_DOCKER_STATE": str(state), + "GITHUB_SHA": "test-revision", + "PATH": f"{fake_bin}{os.pathsep}{environment['PATH']}", + "POWERCONTEXT_E2E_DATABASE": "sqlite", + "POWERCONTEXT_E2E_OUTPUT": str(tmp_path / "evidence"), + }) + result = subprocess.run( # noqa: S603 - executes the repository script with an isolated fake Docker binary. + ["/bin/sh", str(_RUN_SCRIPT), "acceptance"], + cwd=_REPOSITORY_ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + return result, state diff --git a/e2e/bub/uv.lock b/e2e/bub/uv.lock index 595a5ab07..ae2e3bc71 100644 --- a/e2e/bub/uv.lock +++ b/e2e/bub/uv.lock @@ -607,6 +607,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "inquirer-textual" version = "0.6.1" @@ -940,6 +949,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, ] +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + [[package]] name = "platformdirs" version = "4.11.0" @@ -1049,6 +1067,11 @@ dependencies = [ { name = "pyyaml" }, ] +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + [package.metadata] requires-dist = [ { name = "bub", specifier = ">=0.4.0,<0.5.0" }, @@ -1060,6 +1083,9 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6,<7" }, ] +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=9.0.2" }] + [[package]] name = "prompt-toolkit" version = "3.0.53" @@ -1340,6 +1366,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.2"