diff --git a/NOTES.md b/NOTES.md index 3df77b0..0ff7bda 100644 --- a/NOTES.md +++ b/NOTES.md @@ -153,7 +153,7 @@ English or any other language. The `eval` command (documented in [README.md's Evaluating accuracy section](README.md#evaluating-accuracy)) runs -the tracer against `tests/fixtures/eval_cases.json`, a set of advisories with +the tracer against `src/code_audit/eval_cases.json`, a set of advisories with known-correct answers, and reports how often the agent's output matches. **Sourcing ground truth.** Every non-null expected value in the fixture carries diff --git a/README.md b/README.md index d7e205f..c9b920a 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,15 @@ Fetch an advisory and print it as JSON: uv run python -m code_audit advisory GHSA-jfh8-c2jp-5v3q ``` +Add `--pretty` to print a short human-readable summary (the GHSA id, summary, severity, +and source location, one per line) instead of the full JSON: + +```sh +uv run python -m code_audit advisory GHSA-jfh8-c2jp-5v3q --pretty +``` + +The default (no `--pretty`) output is unchanged. + Trace an advisory to its introducing and fixing changes. Deterministic tracing from the advisory references runs first; a Claude agent then investigates only the fields that remain unknown, and every finding is verified against the GitHub API: @@ -125,7 +134,7 @@ Expected output (trimmed): }, "introducing_pull_request": null, "fixing_commit": { - "sha": "c77b3cb39312b83b053d23a2158b6e528f9a6ab9", + "sha": "c77b3cb39312b83b053d23a2158b99ac7de44dd3", "message": "Restrict LDAP access via JNDI (#608)", "files": ["..."] }, @@ -141,6 +150,25 @@ Expected output (trimmed): Fields stay `null` when no confirmed answer exists, for example an introducing change that predates pull requests. +Add `--pretty` to print a short human-readable summary instead of the full JSON: one +line per field (`introducing_commit`, `introducing_pull_request`, `fixing_commit`, +`fixing_pull_request`) with a short SHA or PR number, the first line of the commit +message or PR title, and a file or state count, or `not found` when a field is null: + +```sh +uv run python -m code_audit trace GHSA-jfh8-c2jp-5v3q --pretty +``` + +``` +introducing_commit: f1a0cac LOG4J2-313 - Add JNDILookup (1 file) +introducing_pull_request: not found +fixing_commit: c77b3cb Restrict LDAP access via JNDI (#608) (1 file) +fixing_pull_request: #608 Restrict LDAP access via JNDI (closed, merged) +``` + +The default (no `--pretty`) output is unchanged: full JSON, byte for byte the same as +before, so anything scripted against it keeps working without passing `--pretty`. + Add `--metrics` to print execution metrics as a second JSON document after the trace result. It reports timings for the deterministic and LLM phases, API request counts, token usage, and the estimated API cost: @@ -188,8 +216,9 @@ uv run python -m code_audit trace-many --from-file advisories.txt `--concurrency` (default 5) bounds how many advisories are traced at once, since running too many in parallel would trip the GitHub and Anthropic rate limits and their retry -backoff rather than finishing faster. `--debug` and `--metrics` work as they do for -`trace`, writing one transcript and one metrics block per advisory. With `--metrics`, +backoff rather than finishing faster. `--debug`, `--metrics`, and `--pretty` work as they +do for `trace`, writing one transcript, one metrics block, or one pretty summary per +advisory in place of its JSON block. With `--metrics`, each advisory's block also reports `started_at_seconds` and `ended_at_seconds` as offsets from the batch start, so overlapping windows show the advisories ran concurrently rather than one after another. The GitHub and Anthropic clients are shared across threads, so @@ -210,7 +239,7 @@ honest `null`: uv run python -m code_audit eval --runs 3 ``` -The cases live in `tests/fixtures/eval_cases.json`; each entry records the expected +The cases live in `src/code_audit/eval_cases.json`; each entry records the expected fields and a source URL for every confirmed value. A field's expected value may be a list when several identifiers are acceptable (for example the same fix cherry-picked to different branches with different SHAs). A case marked `"disputed": true` (a diff --git a/src/code_audit/agent.py b/src/code_audit/agent.py index 8414973..f3c504d 100644 --- a/src/code_audit/agent.py +++ b/src/code_audit/agent.py @@ -1,4 +1,6 @@ import re +import sys +import threading from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from typing import Any, cast @@ -17,6 +19,15 @@ MAX_TOKENS = 16000 MAX_TURNS = 20 +# MAX_TURNS bounds how many requests a run can make, but nothing bounds how long +# a single turn's generation takes. Adaptive thinking has been observed, rarely, +# to spend far longer than usual before landing on the same inconclusive answer +# a quick run would have reached anyway (one observed case: 285 seconds and over +# 16000 output tokens across just 3 turns, versus a typical 25-75 seconds for the +# same advisory). This caps the whole run's wall-clock time well above any normal +# case but well below that kind of outlier. +MAX_RUN_SECONDS = 180.0 + # GitHub owner and repository names are limited to this ASCII set, so a # reported repository outside it is malformed rather than another language. REPOSITORY_PATTERN = re.compile(r"[A-Za-z0-9._-]+/[A-Za-z0-9._-]+") @@ -62,6 +73,7 @@ def complete_trace( anthropic_client: anthropic.Anthropic | None = None, on_usage: Callable[[Usage], None] | None = None, on_turn: Callable[[dict[str, Any]], None] | None = None, + on_progress: Callable[[str], None] | None = None, ) -> TraceResult: """Fill the unknown fields of a deterministic trace result using one agent. @@ -70,7 +82,11 @@ def complete_trace( client, so the returned TraceResult contains nothing the API did not confirm. When on_usage is given it observes the token usage of every model response. When on_turn is given it receives one debug record per - model turn. + model turn. When on_progress is given it receives one short human-readable + line per turn, unconditionally rather than only under --debug, since a run + can otherwise take up to MAX_RUN_SECONDS with no output at all. The run is + capped at MAX_RUN_SECONDS wall-clock time; exceeding it is treated like any + other inconclusive outcome and falls back to deterministic_result. """ unknown_fields = [ field for field in TRACE_FIELDS if getattr(deterministic_result, field) is None @@ -80,7 +96,7 @@ def complete_trace( if anthropic_client is None: anthropic_client = anthropic.Anthropic() - findings = _run_agent( + findings = _run_agent_with_timeout( advisory, deterministic_result, unknown_fields, @@ -88,6 +104,7 @@ def complete_trace( anthropic_client, on_usage, on_turn, + on_progress, ) if findings is None or not _repository_is_trustworthy(findings.repository, advisory): # A garbled generation can yield syntactically valid JSON with a @@ -113,6 +130,74 @@ def _repository_is_trustworthy(repository: str | None, advisory: Advisory) -> bo return known is None or repository.lower() == known +class _AgentRunOutcome: + """Holds whatever the agent thread produced, so it can cross back to the caller.""" + + def __init__(self) -> None: + self.findings: AgentFindings | None = None + self.exc: BaseException | None = None + + +def _run_agent_with_timeout( + advisory: Advisory, + result: TraceResult, + unknown_fields: list[str], + github_client: GitHubClient, + anthropic_client: anthropic.Anthropic, + on_usage: Callable[[Usage], None] | None, + on_turn: Callable[[dict[str, Any]], None] | None, + on_progress: Callable[[str], None] | None, +) -> AgentFindings | None: + """Run the agent on a daemon thread and give up after MAX_RUN_SECONDS. + + Giving up is reported to stderr and then treated exactly like any other + inconclusive outcome: the caller falls back to the deterministic result. + + This deliberately does not use ThreadPoolExecutor. Its worker threads are + registered in concurrent.futures.thread's module-level _threads_queues and + joined by an atexit hook at interpreter shutdown, regardless of whether the + executor itself was shut down with wait=False; a run that times out here + would still block process exit until the abandoned call finally finishes or + the Anthropic SDK's own request timeout elapses, silently defeating the cap + (confirmed by timing an actual process's exit, not just when this function + returns: shutdown(wait=False) returned in ~1s while the process did not + actually exit for another ~4s, matching the abandoned call's own duration). + A plain daemon thread is not tracked by that registry, so the interpreter + does not wait for it, and the process exits as soon as its own work is + done. + """ + outcome = _AgentRunOutcome() + + def target() -> None: + try: + outcome.findings = _run_agent( + advisory, + result, + unknown_fields, + github_client, + anthropic_client, + on_usage, + on_turn, + on_progress, + ) + except BaseException as exc: + outcome.exc = exc + + thread = threading.Thread(target=target, daemon=True) + thread.start() + thread.join(timeout=MAX_RUN_SECONDS) + if thread.is_alive(): + print( + f"Warning: tracing {advisory.ghsa_id} exceeded {MAX_RUN_SECONDS:.0f}s; " + "falling back to the deterministic result.", + file=sys.stderr, + ) + return None + if outcome.exc is not None: + raise outcome.exc + return outcome.findings + + def _run_agent( advisory: Advisory, result: TraceResult, @@ -121,12 +206,15 @@ def _run_agent( anthropic_client: anthropic.Anthropic, on_usage: Callable[[Usage], None] | None, on_turn: Callable[[dict[str, Any]], None] | None, + on_progress: Callable[[str], None] | None, ) -> AgentFindings | None: """Run the tool loop and return the agent's findings, or None if it stalls.""" messages: list[MessageParam] = [ {"role": "user", "content": build_task(advisory, result, unknown_fields)} ] for turn in range(1, MAX_TURNS + 1): + if on_progress is not None: + on_progress(f"Turn {turn}/{MAX_TURNS}: waiting for the model...") # The conversation only grows at the end, so caching the request # prefix means each turn pays full price only for the blocks added # since the previous turn. diff --git a/src/code_audit/cli.py b/src/code_audit/cli.py index 06b1bd0..54d8409 100644 --- a/src/code_audit/cli.py +++ b/src/code_audit/cli.py @@ -13,7 +13,7 @@ from code_audit.config import Config, ConfigError from code_audit.github_client import GitHubClient, GitHubClientError from code_audit.instrumentation import MetricsCollector -from code_audit.models import TraceResult +from code_audit.models import Advisory, Commit, PullRequest, TraceResult from code_audit.tracing import trace_advisory app = typer.Typer(no_args_is_help=True) @@ -31,6 +31,10 @@ def main() -> None: @app.command() def advisory( ghsa_id: Annotated[str, typer.Argument(help="Advisory ID, e.g. GHSA-jfh8-c2jp-5v3q")], + pretty: Annotated[ + bool, + typer.Option("--pretty", help="Print a short human-readable summary instead of JSON."), + ] = False, ) -> None: """Fetch a GitHub security advisory and print it as JSON.""" try: @@ -41,7 +45,10 @@ def advisory( typer.echo(f"Error: {exc}", err=True) raise typer.Exit(1) from exc - typer.echo(result.model_dump_json(indent=2)) + if pretty: + typer.echo(_pretty_advisory(result)) + else: + typer.echo(result.model_dump_json(indent=2)) @app.command() @@ -54,6 +61,10 @@ def trace( bool, typer.Option("--debug", help="Write a per-turn JSON Lines transcript under debug/."), ] = False, + pretty: Annotated[ + bool, + typer.Option("--pretty", help="Print a short human-readable summary instead of JSON."), + ] = False, ) -> None: """Trace an advisory to its introducing and fixing commits and pull requests.""" collector = MetricsCollector() @@ -61,10 +72,10 @@ def trace( on_turn: Callable[[dict[str, Any]], None] | None = None debug_file: TextIO | None = None debug_path: Path | None = None - if debug: - debug_path, debug_file, on_turn = _debug_recorder(ghsa_id) try: + if debug: + debug_path, debug_file, on_turn = _debug_recorder(ghsa_id) config = Config.from_env() anthropic_client = anthropic.Anthropic( api_key=config.anthropic_api_key, @@ -85,16 +96,20 @@ def trace( anthropic_client, on_usage=collector.record_usage, on_turn=on_turn, + on_progress=_print_progress, ) llm_seconds = time.perf_counter() - llm_start - except (ConfigError, GitHubClientError, anthropic.AnthropicError) as exc: + except (ConfigError, GitHubClientError, anthropic.AnthropicError, OSError) as exc: typer.echo(f"Error: {exc}", err=True) raise typer.Exit(1) from exc finally: if debug_file is not None: debug_file.close() - typer.echo(result.model_dump_json(indent=2)) + if pretty: + typer.echo(_pretty_trace_result(result)) + else: + typer.echo(result.model_dump_json(indent=2)) if debug_path is not None: typer.echo(f"Debug transcript written to {debug_path}", err=True) if metrics: @@ -128,6 +143,12 @@ def trace_many( bool, typer.Option("--debug", help="Write a per-turn JSON Lines transcript per advisory."), ] = False, + pretty: Annotated[ + bool, + typer.Option( + "--pretty", help="Print a short human-readable summary instead of JSON per advisory." + ), + ] = False, ) -> None: """Trace several advisories concurrently. @@ -139,7 +160,11 @@ def trace_many( cannot be attributed to a single advisory when shared, --metrics reports the request totals once in the final summary rather than per advisory. """ - ghsa_ids_to_trace = _collect_ids(ghsa_ids, from_file) + try: + ghsa_ids_to_trace = _collect_ids(ghsa_ids, from_file) + except OSError as exc: + typer.echo(f"Error: could not read {from_file}: {exc}", err=True) + raise typer.Exit(1) from exc if not ghsa_ids_to_trace: typer.echo("Error: no advisory IDs were given.", err=True) raise typer.Exit(1) @@ -178,7 +203,7 @@ def trace_many( ] for future in as_completed(futures): outcome = future.result() - _print_outcome(outcome, metrics) + _print_outcome(outcome, metrics, pretty) if outcome.error is None: succeeded += 1 else: @@ -218,7 +243,14 @@ def run_eval( per-field success rate is reported instead of a single pass or fail. Cases marked "disputed" are still reported but excluded from the exit code. """ - eval_cases = json.loads(cases.read_text(encoding="utf-8")) + try: + eval_cases = json.loads(cases.read_text(encoding="utf-8")) + except OSError as exc: + typer.echo(f"Error: could not read {cases}: {exc}", err=True) + raise typer.Exit(1) from exc + except json.JSONDecodeError as exc: + typer.echo(f"Error: {cases} is not valid JSON: {exc}", err=True) + raise typer.Exit(1) from exc correct_field_runs = 0 total_field_runs = 0 wrong_values = 0 @@ -246,7 +278,7 @@ def run_eval( if not disputed: wrong_values += wrong typer.echo(f" {field}: {correct}/{len(results)} correct{_detail(results)}") - except (ConfigError, GitHubClientError, anthropic.AnthropicError) as exc: + except (ConfigError, GitHubClientError, anthropic.AnthropicError, OSError) as exc: typer.echo(f"Error: {exc}", err=True) raise typer.Exit(1) from exc @@ -304,9 +336,9 @@ def on_usage(usage: Any) -> None: on_turn: Callable[[dict[str, Any]], None] | None = None debug_file: TextIO | None = None debug_path: Path | None = None - if debug: - debug_path, debug_file, on_turn = _debug_recorder(ghsa_id) try: + if debug: + debug_path, debug_file, on_turn = _debug_recorder(ghsa_id) advisory = client.fetch_advisory(ghsa_id) deterministic_start = time.perf_counter() deterministic_result = trace_advisory(advisory, client) @@ -319,6 +351,7 @@ def on_usage(usage: Any) -> None: anthropic_client, on_usage=on_usage, on_turn=on_turn, + on_progress=lambda message: _print_progress(f"{ghsa_id}: {message}"), ) llm_seconds = time.perf_counter() - llm_start except Exception as exc: @@ -346,18 +379,25 @@ def on_usage(usage: Any) -> None: return _BatchOutcome(ghsa_id, result, metrics_block, debug_path, None) -def _print_outcome(outcome: _BatchOutcome, metrics: bool) -> None: +def _print_outcome(outcome: _BatchOutcome, metrics: bool, pretty: bool) -> None: typer.echo(f"=== {outcome.ghsa_id} ===") if outcome.error is not None or outcome.result is None: typer.echo(f"Error: {outcome.error}", err=True) return - typer.echo(outcome.result.model_dump_json(indent=2)) + if pretty: + typer.echo(_pretty_trace_result(outcome.result)) + else: + typer.echo(outcome.result.model_dump_json(indent=2)) if outcome.debug_path is not None: typer.echo(f"Debug transcript written to {outcome.debug_path}", err=True) if metrics and outcome.metrics is not None: typer.echo(json.dumps(outcome.metrics, indent=2)) +def _print_progress(message: str) -> None: + typer.echo(message, err=True) + + def _run_case( ghsa_id: str, client: GitHubClient, @@ -370,9 +410,9 @@ def _run_case( on_turn: Callable[[dict[str, Any]], None] | None = None debug_file: TextIO | None = None debug_path: Path | None = None - if debug: - debug_path, debug_file, on_turn = _debug_recorder(ghsa_id, f"run{run_index}") try: + if debug: + debug_path, debug_file, on_turn = _debug_recorder(ghsa_id, f"run{run_index}") result = complete_trace( advisory, deterministic_result, client, anthropic_client, on_turn=on_turn ) @@ -440,3 +480,40 @@ def _classify(expected: object, actual: object) -> str: if actual is None: return "null_wrong" return "wrong_value" + + +def _pretty_advisory(advisory: Advisory) -> str: + """Format an advisory as a short human-readable summary.""" + severity = advisory.severity if advisory.severity is not None else "not found" + source = ( + advisory.source_code_location if advisory.source_code_location is not None else "not found" + ) + return f"{advisory.ghsa_id}\n{advisory.summary}\nSeverity: {severity}\nSource: {source}" + + +def _pretty_trace_result(result: TraceResult) -> str: + """Format a trace result as a short human-readable summary.""" + return "\n".join( + [ + f"introducing_commit: {_pretty_commit(result.introducing_commit)}", + f"introducing_pull_request: {_pretty_pull_request(result.introducing_pull_request)}", + f"fixing_commit: {_pretty_commit(result.fixing_commit)}", + f"fixing_pull_request: {_pretty_pull_request(result.fixing_pull_request)}", + ] + ) + + +def _pretty_commit(commit: Commit | None) -> str: + if commit is None: + return "not found" + subject = commit.message.partition("\n")[0] + file_word = "file" if len(commit.files) == 1 else "files" + return f"{commit.sha[:7]} {subject} ({len(commit.files)} {file_word})" + + +def _pretty_pull_request(pull_request: PullRequest | None) -> str: + if pull_request is None: + return "not found" + title = pull_request.title.partition("\n")[0] + status = f"{pull_request.state}, merged" if pull_request.merged else pull_request.state + return f"#{pull_request.number} {title} ({status})" diff --git a/tests/test_agent.py b/tests/test_agent.py index 3f3710a..8b10b37 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -1,12 +1,15 @@ import base64 import json +import time from types import SimpleNamespace from typing import Any, cast import anthropic import httpx +import pytest from anthropic.types import ToolUseBlock, Usage +import code_audit.agent from code_audit.agent import DEBUG_PREVIEW_CHARS, MAX_TURNS, complete_trace from code_audit.github_client import GitHubClient from code_audit.github_tools import ( @@ -507,6 +510,25 @@ def test_debug_callback_receives_one_record_per_turn() -> None: assert second["final_findings"] == findings.content[0].text +def test_on_progress_receives_one_call_per_turn() -> None: + result = TraceResult(ghsa_id=ADVISORY.ghsa_id) + tool_call = tool_use_response("fetch_commit", {"repository": REPOSITORY, "sha": FIXING_SHA}) + findings = findings_response(repository=None) + anthropic_client, _ = fake_anthropic([tool_call, findings]) + responses = {f"/repos/{REPOSITORY}/commits/{FIXING_SHA}": commit_response(FIXING_SHA)} + messages: list[str] = [] + + with make_github_client(responses) as github_client: + complete_trace( + ADVISORY, result, github_client, anthropic_client, on_progress=messages.append + ) + + assert messages == [ + f"Turn 1/{MAX_TURNS}: waiting for the model...", + f"Turn 2/{MAX_TURNS}: waiting for the model...", + ] + + def test_debug_callback_truncates_large_tool_results() -> None: result = TraceResult(ghsa_id=ADVISORY.ghsa_id) tool_block = SimpleNamespace( @@ -548,3 +570,40 @@ def test_refusal_falls_back_to_the_deterministic_result() -> None: assert completed == result assert len(fake.requests) == 1 + + +def test_run_exceeding_the_time_limit_falls_back_to_the_deterministic_result( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + # A tiny limit paired with a deliberately slow response reliably triggers + # the timeout without the test itself waiting anywhere close to real usage. + # This only exercises the fallback and reporting contract: the abandoned + # call still returns a value on this daemon thread, but nothing here waits + # for it or observes it. The reason a daemon thread is used instead of + # ThreadPoolExecutor (its worker threads are joined by an atexit hook at + # interpreter shutdown regardless of shutdown(wait=False), which would + # silently block real process exit until the abandoned call finishes) is + # not something this in-process test can observe, since pytest never exits + # the process between tests; that was verified separately by timing an + # actual subprocess's exit from outside it. + monkeypatch.setattr(code_audit.agent, "MAX_RUN_SECONDS", 0.05) + result = TraceResult(ghsa_id=ADVISORY.ghsa_id) + + class SlowAnthropicClient: + def __init__(self) -> None: + self.messages = SimpleNamespace(create=self._create) + + def _create(self, **kwargs: Any) -> Any: + time.sleep(0.3) + return findings_response(repository=None) + + anthropic_client = cast(anthropic.Anthropic, SlowAnthropicClient()) + + with make_github_client({}) as github_client: + completed = complete_trace(ADVISORY, result, github_client, anthropic_client) + + assert completed == result + stderr = capsys.readouterr().err + assert ADVISORY.ghsa_id in stderr + assert "exceeded" in stderr + assert "falling back to the deterministic result" in stderr diff --git a/tests/test_cli.py b/tests/test_cli.py index f494a4a..b285f16 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,12 +9,92 @@ from code_audit.cli import app from code_audit.config import Config, ConfigError from code_audit.github_client import GitHubClient, GitHubClientError -from code_audit.models import Advisory, Commit, PullRequest, TraceResult +from code_audit.models import Advisory, Commit, CommitFile, PullRequest, TraceResult runner = CliRunner() ADVISORY = Advisory(ghsa_id="GHSA-jfh8-c2jp-5v3q", summary="Remote code injection in Log4j") +INTRODUCING_COMMIT = Commit( + sha="a" * 40, + message="Add JNDI lookup\n\nLonger explanation.", + files=[CommitFile(filename="JndiLookup.java", status="added")], +) +FIXING_COMMIT = Commit( + sha="b" * 40, + message="Restrict LDAP access via JNDI (#608)", + files=[ + CommitFile(filename="JndiManager.java", status="modified"), + CommitFile(filename="JndiLookup.java", status="modified"), + ], +) +INTRODUCING_PR = PullRequest( + number=41, title="Add JNDI lookup support", state="closed", merged=True +) +FIXING_PR = PullRequest( + number=608, title="Restrict LDAP access via JNDI", state="closed", merged=True +) + + +def test_advisory_pretty_prints_summary_with_all_fields(monkeypatch: pytest.MonkeyPatch) -> None: + full_advisory = Advisory( + ghsa_id="GHSA-jfh8-c2jp-5v3q", + summary="Remote code injection in Log4j", + severity="critical", + source_code_location="https://github.com/apache/logging-log4j2", + ) + monkeypatch.setattr( + Config, + "from_env", + classmethod(lambda cls: Config(github_token="t", anthropic_api_key="k")), + ) + monkeypatch.setattr(GitHubClient, "fetch_advisory", lambda self, ghsa_id: full_advisory) + + result = runner.invoke(app, ["advisory", full_advisory.ghsa_id, "--pretty"]) + + assert result.exit_code == 0, result.output + assert result.stdout == ( + "GHSA-jfh8-c2jp-5v3q\n" + "Remote code injection in Log4j\n" + "Severity: critical\n" + "Source: https://github.com/apache/logging-log4j2\n" + ) + + +def test_advisory_pretty_prints_not_found_for_missing_fields( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sparse_advisory = Advisory(ghsa_id="GHSA-sparse", summary="Something vague") + monkeypatch.setattr( + Config, + "from_env", + classmethod(lambda cls: Config(github_token="t", anthropic_api_key="k")), + ) + monkeypatch.setattr(GitHubClient, "fetch_advisory", lambda self, ghsa_id: sparse_advisory) + + result = runner.invoke(app, ["advisory", sparse_advisory.ghsa_id, "--pretty"]) + + assert result.exit_code == 0, result.output + assert result.stdout == ( + "GHSA-sparse\nSomething vague\nSeverity: not found\nSource: not found\n" + ) + + +def test_advisory_default_output_is_unaffected_by_pretty_flag( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + Config, + "from_env", + classmethod(lambda cls: Config(github_token="t", anthropic_api_key="k")), + ) + monkeypatch.setattr(GitHubClient, "fetch_advisory", lambda self, ghsa_id: ADVISORY) + + result = runner.invoke(app, ["advisory", ADVISORY.ghsa_id]) + + assert result.exit_code == 0, result.output + assert json.loads(result.stdout) == ADVISORY.model_dump(mode="json") + def test_trace_runs_the_full_pipeline(monkeypatch: pytest.MonkeyPatch) -> None: deterministic = TraceResult(ghsa_id=ADVISORY.ghsa_id) @@ -29,6 +109,7 @@ def fake_complete_trace( anthropic_client: object, on_usage: object, on_turn: object, + on_progress: object, ) -> TraceResult: assert advisory == ADVISORY assert deterministic_result == deterministic @@ -95,6 +176,89 @@ def fail() -> Config: assert "GITHUB_TOKEN" in result.stderr +def test_trace_pretty_prints_summary_with_all_fields(monkeypatch: pytest.MonkeyPatch) -> None: + completed = TraceResult( + ghsa_id=ADVISORY.ghsa_id, + introducing_commit=INTRODUCING_COMMIT, + introducing_pull_request=INTRODUCING_PR, + fixing_commit=FIXING_COMMIT, + fixing_pull_request=FIXING_PR, + ) + monkeypatch.setattr( + Config, + "from_env", + classmethod(lambda cls: Config(github_token="t", anthropic_api_key="k")), + ) + monkeypatch.setattr(GitHubClient, "fetch_advisory", lambda self, ghsa_id: ADVISORY) + monkeypatch.setattr( + code_audit.cli, + "trace_advisory", + lambda advisory, client: TraceResult(ghsa_id=ADVISORY.ghsa_id), + ) + monkeypatch.setattr(code_audit.cli, "complete_trace", lambda *args, **kwargs: completed) + + result = runner.invoke(app, ["trace", ADVISORY.ghsa_id, "--pretty"]) + + assert result.exit_code == 0, result.output + assert result.stdout == ( + "introducing_commit: aaaaaaa Add JNDI lookup (1 file)\n" + "introducing_pull_request: #41 Add JNDI lookup support (closed, merged)\n" + "fixing_commit: bbbbbbb Restrict LDAP access via JNDI (#608) (2 files)\n" + "fixing_pull_request: #608 Restrict LDAP access via JNDI (closed, merged)\n" + ) + + +def test_trace_pretty_prints_not_found_for_missing_fields(monkeypatch: pytest.MonkeyPatch) -> None: + completed = TraceResult( + ghsa_id=ADVISORY.ghsa_id, fixing_commit=Commit(sha="c" * 40, message="Fix") + ) + monkeypatch.setattr( + Config, + "from_env", + classmethod(lambda cls: Config(github_token="t", anthropic_api_key="k")), + ) + monkeypatch.setattr(GitHubClient, "fetch_advisory", lambda self, ghsa_id: ADVISORY) + monkeypatch.setattr( + code_audit.cli, + "trace_advisory", + lambda advisory, client: TraceResult(ghsa_id=ADVISORY.ghsa_id), + ) + monkeypatch.setattr(code_audit.cli, "complete_trace", lambda *args, **kwargs: completed) + + result = runner.invoke(app, ["trace", ADVISORY.ghsa_id, "--pretty"]) + + assert result.exit_code == 0, result.output + assert result.stdout == ( + "introducing_commit: not found\n" + "introducing_pull_request: not found\n" + "fixing_commit: ccccccc Fix (0 files)\n" + "fixing_pull_request: not found\n" + ) + + +def test_trace_default_output_is_unaffected_by_pretty_flag(monkeypatch: pytest.MonkeyPatch) -> None: + completed = TraceResult( + ghsa_id=ADVISORY.ghsa_id, fixing_commit=Commit(sha="a" * 40, message="Fix") + ) + monkeypatch.setattr( + Config, + "from_env", + classmethod(lambda cls: Config(github_token="t", anthropic_api_key="k")), + ) + monkeypatch.setattr(GitHubClient, "fetch_advisory", lambda self, ghsa_id: ADVISORY) + monkeypatch.setattr( + code_audit.cli, + "trace_advisory", + lambda advisory, client: TraceResult(ghsa_id=ADVISORY.ghsa_id), + ) + monkeypatch.setattr(code_audit.cli, "complete_trace", lambda *args, **kwargs: completed) + + result = runner.invoke(app, ["trace", ADVISORY.ghsa_id]) + + assert result.exit_code == 0, result.output + assert json.loads(result.stdout) == completed.model_dump(mode="json") + + def test_trace_many_completes_all_even_when_one_raises(monkeypatch: pytest.MonkeyPatch) -> None: ids = ["GHSA-aaaa", "GHSA-bbbb", "GHSA-cccc"] @@ -105,6 +269,7 @@ def fake_complete_trace( anthropic_client: object, on_usage: object = None, on_turn: object = None, + on_progress: object = None, ) -> TraceResult: if advisory.ghsa_id == "GHSA-bbbb": raise GitHubClientError("boom") @@ -167,6 +332,7 @@ def fake_complete_trace( anthropic_client: object, on_usage: object = None, on_turn: object = None, + on_progress: object = None, ) -> TraceResult: if callable(on_usage): on_usage(Usage(input_tokens=5, output_tokens=1)) @@ -208,6 +374,7 @@ def fake_complete_trace( anthropic_client: object, on_usage: object = None, on_turn: object = None, + on_progress: object = None, ) -> TraceResult: traced.append(advisory.ghsa_id) return TraceResult( @@ -239,6 +406,98 @@ def fake_complete_trace( assert "2 advisories: 2 succeeded, 0 failed" in result.stdout +def test_trace_many_pretty_prints_summary_per_advisory(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_complete_trace( + advisory: Advisory, + deterministic_result: TraceResult, + client: GitHubClient, + anthropic_client: object, + on_usage: object = None, + on_turn: object = None, + on_progress: object = None, + ) -> TraceResult: + if advisory.ghsa_id == "GHSA-full": + return TraceResult( + ghsa_id=advisory.ghsa_id, + introducing_commit=INTRODUCING_COMMIT, + introducing_pull_request=INTRODUCING_PR, + fixing_commit=FIXING_COMMIT, + fixing_pull_request=FIXING_PR, + ) + return TraceResult( + ghsa_id=advisory.ghsa_id, fixing_commit=Commit(sha="c" * 40, message="Fix") + ) + + monkeypatch.setattr( + Config, + "from_env", + classmethod(lambda cls: Config(github_token="t", anthropic_api_key="k")), + ) + monkeypatch.setattr( + GitHubClient, "fetch_advisory", lambda self, ghsa_id: Advisory(ghsa_id=ghsa_id, summary="s") + ) + monkeypatch.setattr( + code_audit.cli, + "trace_advisory", + lambda advisory, client: TraceResult(ghsa_id=advisory.ghsa_id), + ) + monkeypatch.setattr(code_audit.cli, "complete_trace", fake_complete_trace) + + result = runner.invoke(app, ["trace-many", "GHSA-full", "GHSA-sparse", "--pretty"]) + + assert result.exit_code == 0, result.output + assert "=== GHSA-full ===" in result.stdout + assert "=== GHSA-sparse ===" in result.stdout + assert "introducing_commit: aaaaaaa Add JNDI lookup (1 file)" in result.stdout + assert "introducing_pull_request: #41 Add JNDI lookup support (closed, merged)" in result.stdout + assert "fixing_commit: bbbbbbb Restrict LDAP access via JNDI (#608) (2 files)" in result.stdout + assert ( + "fixing_pull_request: #608 Restrict LDAP access via JNDI (closed, merged)" in result.stdout + ) + assert "introducing_commit: not found" in result.stdout + assert "introducing_pull_request: not found" in result.stdout + assert "fixing_commit: ccccccc Fix (0 files)" in result.stdout + assert '"ghsa_id"' not in result.stdout + + +def test_trace_many_default_output_is_unaffected_by_pretty_flag( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_complete_trace( + advisory: Advisory, + deterministic_result: TraceResult, + client: GitHubClient, + anthropic_client: object, + on_usage: object = None, + on_turn: object = None, + on_progress: object = None, + ) -> TraceResult: + return TraceResult( + ghsa_id=advisory.ghsa_id, fixing_commit=Commit(sha="a" * 40, message="Fix") + ) + + monkeypatch.setattr( + Config, + "from_env", + classmethod(lambda cls: Config(github_token="t", anthropic_api_key="k")), + ) + monkeypatch.setattr( + GitHubClient, "fetch_advisory", lambda self, ghsa_id: Advisory(ghsa_id=ghsa_id, summary="s") + ) + monkeypatch.setattr( + code_audit.cli, + "trace_advisory", + lambda advisory, client: TraceResult(ghsa_id=advisory.ghsa_id), + ) + monkeypatch.setattr(code_audit.cli, "complete_trace", fake_complete_trace) + + result = runner.invoke(app, ["trace-many", "GHSA-1"]) + + assert result.exit_code == 0, result.output + assert '"ghsa_id": "GHSA-1"' in result.stdout + assert f'"sha": "{"a" * 40}"' in result.stdout + + def _prepare_eval(monkeypatch: pytest.MonkeyPatch, results: dict[str, TraceResult]) -> None: monkeypatch.setattr( Config, diff --git a/tests/test_references.py b/tests/test_references.py index d37ca8d..4deb405 100644 --- a/tests/test_references.py +++ b/tests/test_references.py @@ -1,5 +1,22 @@ from code_audit.models import CommitReference, CompareReference, PullRequestReference -from code_audit.references import extract_references, parse_reference +from code_audit.references import advisory_repository, extract_references, parse_reference + + +def test_advisory_repository_extracts_owner_and_name_lowercased() -> None: + repository = advisory_repository("https://github.com/Apache/Logging-Log4j2") + assert repository == "apache/logging-log4j2" + + +def test_advisory_repository_ignores_non_github_hosts() -> None: + assert advisory_repository("https://gitlab.com/owner/repo") is None + + +def test_advisory_repository_ignores_incomplete_paths() -> None: + assert advisory_repository("https://github.com/apache") is None + + +def test_advisory_repository_returns_none_when_missing() -> None: + assert advisory_repository(None) is None def test_parse_commit_url() -> None: