diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c646ad1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: CI + +on: + pull_request: + push: + branches: [main, master] + +permissions: + contents: read + +jobs: + test: + name: test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # The scanner targets Linux and macOS, so both get exercised. jq is + # preinstalled on both runner images, as it is on the CodeBuild + # standard images the buildspec documents. + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + - name: Tool versions + run: | + bash --version | head -1 + jq --version + - name: Run the scanner test suite + run: bash test/run-tests.sh diff --git a/test/fixtures/clean.json b/test/fixtures/clean.json new file mode 100644 index 0000000..762b70b --- /dev/null +++ b/test/fixtures/clean.json @@ -0,0 +1,99 @@ +{ + "scan_id": "scan_6aeb2293a4cb95a4", + "repo": "testdata/clean", + "languages": [ + "python" + ], + "sdks": null, + "has_shell_invocations": true, + "manifest": { + "repo_root": "/src/testdata/clean", + "is_remote": false, + "python_files": [ + "agent.py" + ], + "yaml_files": [], + "has_claude_sdk_dependency": false, + "has_openshell_artifact": false, + "components": [ + { + "kind": "dependency_manifest", + "path": "requirements.txt", + "language": "python" + } + ] + }, + "tools": [ + { + "name": "run_shell", + "kind": "shell_invocation", + "language": "python", + "file_path": "agent.py", + "start_line": 6, + "end_line": 8, + "description": "Run a shell command and return its output.", + "has_typed_params": true, + "param_names": [ + "cmd" + ], + "facts": { + "shells_out": "true" + } + } + ], + "agents": null, + "hosted_tools": null, + "mcp_servers": null, + "subagents": null, + "skills": null, + "dependencies": [ + { + "name": "anthropic", + "ecosystem": "pypi", + "source": "requirements.txt", + "start_line": 1, + "end_line": 1 + }, + { + "name": "requests", + "ecosystem": "pypi", + "source": "requirements.txt", + "start_line": 2, + "end_line": 2 + } + ], + "slash_commands": null, + "plugin_manifests": null, + "claude_settings": null, + "findings": null, + "surfaces": [ + { + "kind": "tool", + "name": "run_shell", + "file_path": "agent.py", + "score": 1, + "finding_count": 0, + "weighted_severity": 0 + } + ], + "overall_score": 1, + "projected_scores": { + "fix_critical": 1, + "fix_high": 1, + "fix_medium": 1, + "fix_low": 1, + "fix_all": 1 + }, + "rules_source": "https://github.com/trustabl/trustabl-rules", + "rules_version": "3dcf1e9c8796bdb8408e91707d7799388c291bac4b17321030872a0b7fa742bb", + "rules_from_cache": false, + "rules_schema_version": 14, + "rules_origin": { + "signed": true, + "channel": "production" + }, + "coverage": { + "files_parsed": 1, + "files_skipped": 0 + } +} diff --git a/test/fixtures/findings.json b/test/fixtures/findings.json new file mode 100644 index 0000000..774160f --- /dev/null +++ b/test/fixtures/findings.json @@ -0,0 +1,455 @@ +{ + "scan_id": "scan_8432e6168f06aba5", + "repo": "testdata/handoffs", + "languages": [ + "python" + ], + "sdks": [ + "claude_agent_sdk", + "openai_agents" + ], + "has_shell_invocations": false, + "manifest": { + "repo_root": "/src/testdata/handoffs", + "is_remote": false, + "python_files": [ + "message_filter.py", + "message_filter_streaming.py" + ], + "yaml_files": [], + "has_claude_sdk_dependency": false, + "has_openshell_artifact": false + }, + "tools": [ + { + "name": "random_number_tool", + "kind": "claude_sdk_tool", + "language": "python", + "file_path": "message_filter.py", + "start_line": 19, + "end_line": 21, + "description": "Return a random integer between 0 and the given maximum.", + "has_typed_params": true, + "param_names": [ + "max" + ] + }, + { + "name": "random_number_tool", + "kind": "claude_sdk_tool", + "language": "python", + "file_path": "message_filter_streaming.py", + "start_line": 19, + "end_line": 21, + "description": "Return a random integer between 0 and the given maximum.", + "has_typed_params": true, + "param_names": [ + "max" + ] + } + ], + "agents": [ + { + "sdk": "openai_agents", + "class": "Agent", + "language": "python", + "file_path": "message_filter.py", + "start_line": 52, + "end_line": 56, + "name": "Assistant", + "kwargs": { + "children": { + "instructions": { + "value": { + "kind": "literal_string", + "text": "\"Be extremely concise.\"" + } + }, + "name": { + "value": { + "kind": "literal_string", + "text": "\"Assistant\"" + } + }, + "tools": { + "value": { + "kind": "list", + "text": "[random_number_tool]", + "list": [ + { + "kind": "name_ref", + "text": "random_number_tool" + } + ] + } + } + } + }, + "tool_refs": [ + { + "name": "random_number_tool", + "external": false + } + ], + "hosted_tool_refs": null, + "mcp_server_refs": null, + "handoff_refs": null, + "input_guards": null, + "output_guards": null, + "opaque": false + }, + { + "sdk": "openai_agents", + "class": "Agent", + "language": "python", + "file_path": "message_filter.py", + "start_line": 58, + "end_line": 62, + "name": "Spanish Assistant", + "kwargs": { + "children": { + "handoff_description": { + "value": { + "kind": "literal_string", + "text": "\"A Spanish-speaking assistant.\"" + } + }, + "instructions": { + "value": { + "kind": "literal_string", + "text": "\"You only speak Spanish and are extremely concise.\"" + } + }, + "name": { + "value": { + "kind": "literal_string", + "text": "\"Spanish Assistant\"" + } + } + } + }, + "tool_refs": null, + "hosted_tool_refs": null, + "mcp_server_refs": null, + "handoff_refs": null, + "input_guards": null, + "output_guards": null, + "opaque": false + }, + { + "sdk": "openai_agents", + "class": "Agent", + "language": "python", + "file_path": "message_filter.py", + "start_line": 64, + "end_line": 70, + "name": "Assistant", + "kwargs": { + "children": { + "handoffs": { + "value": { + "kind": "list", + "text": "[handoff(spanish_agent, input_filter=spanish_handoff_message_filter)]", + "list": [ + { + "kind": "call", + "text": "handoff(spanish_agent, input_filter=spanish_handoff_message_filter)" + } + ] + } + }, + "instructions": { + "value": { + "kind": "unknown", + "text": "(\n \"Be a helpful assistant. If the user speaks Spanish, handoff to the Spanish assistant.\"\n )" + } + }, + "name": { + "value": { + "kind": "literal_string", + "text": "\"Assistant\"" + } + } + } + }, + "tool_refs": null, + "hosted_tool_refs": null, + "mcp_server_refs": null, + "handoff_refs": [ + { + "name": "handoff(spanish_agent, input_filter=spanish_handoff_message_filter)", + "external": true + } + ], + "input_guards": null, + "output_guards": null, + "opaque": false + }, + { + "sdk": "openai_agents", + "class": "Agent", + "language": "python", + "file_path": "message_filter_streaming.py", + "start_line": 52, + "end_line": 56, + "name": "Assistant", + "kwargs": { + "children": { + "instructions": { + "value": { + "kind": "literal_string", + "text": "\"Be extremely concise.\"" + } + }, + "name": { + "value": { + "kind": "literal_string", + "text": "\"Assistant\"" + } + }, + "tools": { + "value": { + "kind": "list", + "text": "[random_number_tool]", + "list": [ + { + "kind": "name_ref", + "text": "random_number_tool" + } + ] + } + } + } + }, + "tool_refs": [ + { + "name": "random_number_tool", + "external": false + } + ], + "hosted_tool_refs": null, + "mcp_server_refs": null, + "handoff_refs": null, + "input_guards": null, + "output_guards": null, + "opaque": false + }, + { + "sdk": "openai_agents", + "class": "Agent", + "language": "python", + "file_path": "message_filter_streaming.py", + "start_line": 58, + "end_line": 62, + "name": "Spanish Assistant", + "kwargs": { + "children": { + "handoff_description": { + "value": { + "kind": "literal_string", + "text": "\"A Spanish-speaking assistant.\"" + } + }, + "instructions": { + "value": { + "kind": "literal_string", + "text": "\"You only speak Spanish and are extremely concise.\"" + } + }, + "name": { + "value": { + "kind": "literal_string", + "text": "\"Spanish Assistant\"" + } + } + } + }, + "tool_refs": null, + "hosted_tool_refs": null, + "mcp_server_refs": null, + "handoff_refs": null, + "input_guards": null, + "output_guards": null, + "opaque": false + }, + { + "sdk": "openai_agents", + "class": "Agent", + "language": "python", + "file_path": "message_filter_streaming.py", + "start_line": 64, + "end_line": 70, + "name": "Assistant", + "kwargs": { + "children": { + "handoffs": { + "value": { + "kind": "list", + "text": "[handoff(spanish_agent, input_filter=spanish_handoff_message_filter)]", + "list": [ + { + "kind": "call", + "text": "handoff(spanish_agent, input_filter=spanish_handoff_message_filter)" + } + ] + } + }, + "instructions": { + "value": { + "kind": "unknown", + "text": "(\n \"Be a helpful assistant. If the user speaks Spanish, handoff to the Spanish assistant.\"\n )" + } + }, + "name": { + "value": { + "kind": "literal_string", + "text": "\"Assistant\"" + } + } + } + }, + "tool_refs": null, + "hosted_tool_refs": null, + "mcp_server_refs": null, + "handoff_refs": [ + { + "name": "handoff(spanish_agent, input_filter=spanish_handoff_message_filter)", + "external": true + } + ], + "input_guards": null, + "output_guards": null, + "opaque": false + } + ], + "hosted_tools": null, + "mcp_servers": null, + "subagents": null, + "skills": null, + "dependencies": null, + "slash_commands": null, + "plugin_manifests": null, + "claude_settings": null, + "findings": [ + { + "rule_id": "CSDK-203", + "category": "claude_sdk", + "scope": "repo", + "severity": "low", + "tool_name": "", + "file_path": "", + "start_line": 0, + "end_line": 0, + "title": "Repo ships Claude Agent SDK code without an agent-guidance doc (AGENTS.md/CLAUDE.md)", + "explanation": "A repository that builds on the Claude Agent SDK but ships no agent-guidance doc — CLAUDE.md (the Claude Code convention) or AGENTS.md (the cross-vendor one) — leaves any coding agent that opens the repo with no project-specific guidance. The conventions, build commands, test runners, lint scripts, and \"do not touch\" boundaries a human maintainer would describe in onboarding are absent, so the agent has to infer them from the source on every session and frequently guesses wrong — bypassing the project's lint, picking the wrong test command, or violating commit conventions that were never written down anywhere it could read. The cost compounds: each new contributor (human or agent) reinvents the same wrong assumptions.\n", + "suggested_fix": "Add a CLAUDE.md (or an AGENTS.md) at the repository root documenting how to build, test, and lint the project, the coding conventions the agent must respect, the directories or files it must not modify, and any project-specific safety guardrails (e.g. \"never run migrations\", \"never push to main\"). Treat it as the project's agent-facing README and keep it under version control so reviewers see drift.\n", + "confidence": 0.9 + }, + { + "rule_id": "OAI-201", + "category": "openai_sdk", + "scope": "repo", + "severity": "medium", + "tool_name": "", + "file_path": "", + "start_line": 0, + "end_line": 0, + "title": "Project uses default OpenAI tracing", + "explanation": "The project uses the OpenAI Agents SDK with default tracing enabled. By default, inputs, tool calls, tool outputs, and agent responses are sent to OpenAI's hosted tracing backend. For projects handling sensitive data (PII, credentials, internal documents), this is a data egress channel that is easy to miss.\n", + "suggested_fix": "Either disable tracing entirely (set OPENAI_AGENTS_DISABLE_TRACING=1 in the environment) OR register a custom trace processor via agents.tracing.add_trace_processor(...) that redacts sensitive fields before they leave the process.\n", + "confidence": 0.8 + }, + { + "rule_id": "OAI-202", + "category": "openai_sdk", + "scope": "repo", + "severity": "low", + "tool_name": "", + "file_path": "", + "start_line": 0, + "end_line": 0, + "title": "OpenAI Agents project ships no agent-guidance doc (AGENTS.md/CLAUDE.md)", + "explanation": "The project uses the OpenAI Agents SDK in code but ships no agent-guidance doc (AGENTS.md or CLAUDE.md) at the repo root. AGENTS.md is the cross-vendor convention an editing coding agent reads before it acts; with neither file present, any agent that opens this repo has no project-specific guidance on Agent vs SandboxAgent choice, handoff topology, required input/output guardrails, tool_choice settings, or the local test and build commands. The likely consequence is generated code that bypasses the project's safety contracts because nothing in-tree teaches the agent the local rules.\n", + "suggested_fix": "Add an AGENTS.md at the repo root (the cross-vendor standard; a CLAUDE.md also satisfies this rule). State whether the project uses Agent or SandboxAgent, list required guardrails (input_guardrails, output_guardrails) and tool_choice conventions, note any handoff or tracing policy, and give the exact test, lint, and build commands. Keep it short and concrete so an editing agent can act on it without re-deriving the conventions.\n", + "confidence": 0.9 + } + ], + "surfaces": [ + { + "kind": "repo", + "name": "", + "file_path": "", + "score": 0.8033333333333333, + "finding_count": 3, + "weighted_severity": 0.5900000000000001 + }, + { + "kind": "agent", + "name": "Assistant", + "file_path": "message_filter.py", + "score": 1, + "finding_count": 0, + "weighted_severity": 0 + }, + { + "kind": "agent", + "name": "Assistant", + "file_path": "message_filter_streaming.py", + "score": 1, + "finding_count": 0, + "weighted_severity": 0 + }, + { + "kind": "agent", + "name": "Spanish Assistant", + "file_path": "message_filter.py", + "score": 1, + "finding_count": 0, + "weighted_severity": 0 + }, + { + "kind": "agent", + "name": "Spanish Assistant", + "file_path": "message_filter_streaming.py", + "score": 1, + "finding_count": 0, + "weighted_severity": 0 + }, + { + "kind": "tool", + "name": "random_number_tool", + "file_path": "message_filter.py", + "score": 1, + "finding_count": 0, + "weighted_severity": 0 + }, + { + "kind": "tool", + "name": "random_number_tool", + "file_path": "message_filter_streaming.py", + "score": 1, + "finding_count": 0, + "weighted_severity": 0 + } + ], + "overall_score": 0.9588010540184454, + "projected_scores": { + "fix_critical": 0.9588010540184454, + "fix_high": 0.9588010540184454, + "fix_medium": 0.9842778541953232, + "fix_low": 1, + "fix_all": 1 + }, + "rules_source": "https://github.com/trustabl/trustabl-rules", + "rules_version": "3dcf1e9c8796bdb8408e91707d7799388c291bac4b17321030872a0b7fa742bb", + "rules_from_cache": false, + "rules_schema_version": 14, + "rules_origin": { + "signed": true, + "channel": "production" + }, + "coverage": { + "files_parsed": 2, + "files_skipped": 0 + } +} diff --git a/test/fixtures/findings.sarif b/test/fixtures/findings.sarif new file mode 100644 index 0000000..3cc5040 --- /dev/null +++ b/test/fixtures/findings.sarif @@ -0,0 +1,156 @@ +{ + "version": "2.1.0", + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "runs": [ + { + "tool": { + "driver": { + "name": "trustabl", + "fullName": "Trustabl — static analyzer for agent reliability", + "informationUri": "https://github.com/trustabl/trustabl", + "version": "0.1.7", + "semanticVersion": "0.1.7", + "rules": [ + { + "id": "CSDK-203", + "shortDescription": { + "text": "Repo ships Claude Agent SDK code without an agent-guidance doc (AGENTS.md/CLAUDE.md)" + }, + "fullDescription": { + "text": "A repository that builds on the Claude Agent SDK but ships no agent-guidance doc — CLAUDE.md (the Claude Code convention) or AGENTS.md (the cross-vendor one) — leaves any coding agent that opens the repo with no project-specific guidance. The conventions, build commands, test runners, lint scripts, and \"do not touch\" boundaries a human maintainer would describe in onboarding are absent, so the agent has to infer them from the source on every session and frequently guesses wrong — bypassing the project's lint, picking the wrong test command, or violating commit conventions that were never written down anywhere it could read. The cost compounds: each new contributor (human or agent) reinvents the same wrong assumptions.\n" + }, + "help": { + "text": "Add a CLAUDE.md (or an AGENTS.md) at the repository root documenting how to build, test, and lint the project, the coding conventions the agent must respect, the directories or files it must not modify, and any project-specific safety guardrails (e.g. \"never run migrations\", \"never push to main\"). Treat it as the project's agent-facing README and keep it under version control so reviewers see drift.\n" + }, + "defaultConfiguration": { + "level": "note" + }, + "properties": { + "confidence": 0.9, + "security-severity": "3.0", + "tags": [ + "claude_sdk", + "repo", + "python" + ] + } + }, + { + "id": "OAI-201", + "shortDescription": { + "text": "Project uses default OpenAI tracing" + }, + "fullDescription": { + "text": "The project uses the OpenAI Agents SDK with default tracing enabled. By default, inputs, tool calls, tool outputs, and agent responses are sent to OpenAI's hosted tracing backend. For projects handling sensitive data (PII, credentials, internal documents), this is a data egress channel that is easy to miss.\n" + }, + "help": { + "text": "Either disable tracing entirely (set OPENAI_AGENTS_DISABLE_TRACING=1 in the environment) OR register a custom trace processor via agents.tracing.add_trace_processor(...) that redacts sensitive fields before they leave the process.\n" + }, + "defaultConfiguration": { + "level": "warning" + }, + "properties": { + "confidence": 0.8, + "security-severity": "5.5", + "tags": [ + "openai_sdk", + "repo", + "python" + ] + } + }, + { + "id": "OAI-202", + "shortDescription": { + "text": "OpenAI Agents project ships no agent-guidance doc (AGENTS.md/CLAUDE.md)" + }, + "fullDescription": { + "text": "The project uses the OpenAI Agents SDK in code but ships no agent-guidance doc (AGENTS.md or CLAUDE.md) at the repo root. AGENTS.md is the cross-vendor convention an editing coding agent reads before it acts; with neither file present, any agent that opens this repo has no project-specific guidance on Agent vs SandboxAgent choice, handoff topology, required input/output guardrails, tool_choice settings, or the local test and build commands. The likely consequence is generated code that bypasses the project's safety contracts because nothing in-tree teaches the agent the local rules.\n" + }, + "help": { + "text": "Add an AGENTS.md at the repo root (the cross-vendor standard; a CLAUDE.md also satisfies this rule). State whether the project uses Agent or SandboxAgent, list required guardrails (input_guardrails, output_guardrails) and tool_choice conventions, note any handoff or tracing policy, and give the exact test, lint, and build commands. Keep it short and concrete so an editing agent can act on it without re-deriving the conventions.\n" + }, + "defaultConfiguration": { + "level": "note" + }, + "properties": { + "confidence": 0.9, + "security-severity": "3.0", + "tags": [ + "openai_sdk", + "repo", + "python" + ] + } + } + ], + "properties": { + "rules_from_cache": false, + "rules_source": "https://github.com/trustabl/trustabl-rules", + "rules_version": "3dcf1e9c8796bdb8408e91707d7799388c291bac4b17321030872a0b7fa742bb" + } + } + }, + "invocations": [ + { + "executionSuccessful": true + } + ], + "results": [ + { + "ruleId": "CSDK-203", + "ruleIndex": 0, + "kind": "informational", + "message": { + "text": "A repository that builds on the Claude Agent SDK but ships no agent-guidance doc — CLAUDE.md (the Claude Code convention) or AGENTS.md (the cross-vendor one) — leaves any coding agent that opens the repo with no project-specific guidance. The conventions, build commands, test runners, lint scripts, and \"do not touch\" boundaries a human maintainer would describe in onboarding are absent, so the agent has to infer them from the source on every session and frequently guesses wrong — bypassing the project's lint, picking the wrong test command, or violating commit conventions that were never written down anywhere it could read. The cost compounds: each new contributor (human or agent) reinvents the same wrong assumptions.\n" + }, + "rank": 90, + "partialFingerprints": { + "primaryLocationLineHash": "8c927f0b1b0fb63358ed8144fb13e050c197ca78e3086aa6eb128d97cc47085a" + }, + "properties": { + "confidence": 0.9 + } + }, + { + "ruleId": "OAI-201", + "ruleIndex": 1, + "kind": "informational", + "message": { + "text": "The project uses the OpenAI Agents SDK with default tracing enabled. By default, inputs, tool calls, tool outputs, and agent responses are sent to OpenAI's hosted tracing backend. For projects handling sensitive data (PII, credentials, internal documents), this is a data egress channel that is easy to miss.\n" + }, + "rank": 80, + "partialFingerprints": { + "primaryLocationLineHash": "909f2973bbb976bb4bca098a37428ad29f25c0b2587171b86430a13f1367e5d4" + }, + "properties": { + "confidence": 0.8 + } + }, + { + "ruleId": "OAI-202", + "ruleIndex": 2, + "kind": "informational", + "message": { + "text": "The project uses the OpenAI Agents SDK in code but ships no agent-guidance doc (AGENTS.md or CLAUDE.md) at the repo root. AGENTS.md is the cross-vendor convention an editing coding agent reads before it acts; with neither file present, any agent that opens this repo has no project-specific guidance on Agent vs SandboxAgent choice, handoff topology, required input/output guardrails, tool_choice settings, or the local test and build commands. The likely consequence is generated code that bypasses the project's safety contracts because nothing in-tree teaches the agent the local rules.\n" + }, + "rank": 90, + "partialFingerprints": { + "primaryLocationLineHash": "26a8e7422c51ce85da5436d8ee503243e0a68a42e73ccb4ac96ab9be0ea776e1" + }, + "properties": { + "confidence": 0.9 + } + } + ], + "automationDetails": { + "id": "scan_8432e6168f06aba5" + }, + "originalUriBaseIds": { + "REPO_ROOT": { + "uri": "file:///src/testdata/handoffs/" + } + } + } + ] +} diff --git a/test/lib/harness.sh b/test/lib/harness.sh new file mode 100644 index 0000000..4d82023 --- /dev/null +++ b/test/lib/harness.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Test harness for scan/trustabl-scan.sh. +# +# Each test gets a throwaway workspace holding a locally built "release" — a real +# gzipped tarball containing the stub engine, plus a real checksums.txt over it. +# The stub curl serves that directory by URL basename, so the scanner's download, +# sha256 verification, extraction and invocation all run for real; only the +# network and the engine itself are substituted. + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +STUB_DIR="$REPO_ROOT/test/stubs" +FIXTURE_DIR="$REPO_ROOT/test/fixtures" +SCANNER="$REPO_ROOT/scan/trustabl-scan.sh" + +# The version every test pins, so no test needs the latest-release lookup. +TEST_VERSION="v9.9.9" + +TESTS_RUN=0 +TESTS_FAILED=0 +CURRENT_TEST="" +FAILURES=() + +# sha256_of prints the sha256 of a file. Written portably because the harness has +# to run wherever the scanner does. +sha256_of() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +# asset_name mirrors the scanner's own OS/arch asset naming, so the release the +# harness builds is the one the scanner asks for. +asset_name() { + local os arch + case "$(uname -s)" in + Linux) os=linux ;; + Darwin) os=darwin ;; + *) echo "unsupported OS $(uname -s)" >&2; return 1 ;; + esac + case "$(uname -m)" in + x86_64|amd64) arch=amd64 ;; + aarch64|arm64) arch=arm64 ;; + *) echo "unsupported arch $(uname -m)" >&2; return 1 ;; + esac + echo "trustabl_${TEST_VERSION#v}_${os}_${arch}.tar.gz" +} + +# new_workspace prints the path to a fresh workspace with a built release in +# ./release. Callers cd into it (the scanner writes its artifacts to $PWD). +new_workspace() { + local ws asset staging + ws="$(mktemp -d "${TMPDIR:-/tmp}/trustabl-test.XXXXXX")" + mkdir -p "$ws/release" "$ws/work" + + asset="$(asset_name)" + staging="$ws/staging" + mkdir -p "$staging" + cp "$STUB_DIR/trustabl" "$staging/trustabl" + chmod +x "$staging/trustabl" + tar -czf "$ws/release/$asset" -C "$staging" trustabl + + printf '%s %s\n' "$(sha256_of "$ws/release/$asset")" "$asset" > "$ws/release/checksums.txt" + printf '{"tag_name":"%s"}\n' "$TEST_VERSION" > "$ws/release/latest" + + echo "$ws" +} + +# run_scan [KEY=VALUE ...] runs the scanner in $workspace/work with +# the stubs on PATH. It sets SCAN_EXIT and SCAN_OUT; it never aborts the caller. +run_scan() { + local ws="$1"; shift + SCAN_OUT="$( + cd "$ws/work" && env \ + PATH="$STUB_DIR:$PATH" \ + STUB_RELEASE_DIR="$ws/release" \ + STUB_JSON="${STUB_JSON:-$FIXTURE_DIR/findings.json}" \ + STUB_SARIF="${STUB_SARIF:-$FIXTURE_DIR/findings.sarif}" \ + VERSION="$TEST_VERSION" \ + BRANCH="test-branch" \ + "$@" \ + bash "$SCANNER" 2>&1 + )" + SCAN_EXIT=$? + return 0 +} + +# env_var prints the value the scanner wrote to trustabl.env. +env_var() { + sed -n "s/^$2=//p" "$1/work/trustabl.env" +} + +# ---- assertions ---- + +fail() { + TESTS_FAILED=$((TESTS_FAILED + 1)) + FAILURES+=("$CURRENT_TEST: $1") + printf ' not ok — %s\n' "$1" +} + +assert_eq() { + if [ "$2" = "$3" ]; then return 0; fi + fail "$1: expected '$3', got '$2'" +} + +assert_contains() { + case "$2" in + *"$3"*) return 0 ;; + esac + fail "$1: output did not contain '$3'" +} + +assert_not_contains() { + case "$2" in + *"$3"*) fail "$1: output unexpectedly contained '$3'"; return 0 ;; + esac +} + +assert_file() { + [ -f "$1" ] && return 0 + fail "expected file $1 to exist" +} + +# it runs one test and reports the result. +it() { + CURRENT_TEST="$1" + TESTS_RUN=$((TESTS_RUN + 1)) + local before=$TESTS_FAILED + "$2" + if [ "$TESTS_FAILED" -eq "$before" ]; then printf 'ok %d — %s\n' "$TESTS_RUN" "$1"; else printf 'NOT OK %d — %s\n' "$TESTS_RUN" "$1"; fi +} + +summarize() { + echo + echo "1..$TESTS_RUN" + if [ "$TESTS_FAILED" -gt 0 ]; then + echo "$TESTS_FAILED failure(s):" + printf ' - %s\n' "${FAILURES[@]}" + return 1 + fi + echo "all $TESTS_RUN test(s) passed" +} diff --git a/test/run-tests.sh b/test/run-tests.sh new file mode 100755 index 0000000..3aa97bb --- /dev/null +++ b/test/run-tests.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# End-to-end tests for scan/trustabl-scan.sh. Run with: bash test/run-tests.sh +# +# The fixtures under test/fixtures are unmodified ScanResult / SARIF output from +# a real engine run, so the assertions below pin the scanner against the shape +# the engine actually emits rather than against a hand-written approximation. +# +# clean.json 0 findings overall_score 1.0 +# findings.json 3 findings overall_score 0.9588… (2 low, 1 medium) + +set -uo pipefail + +# shellcheck source=test/lib/harness.sh +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/harness.sh" + +WORKSPACES=() +workspace() { + local ws + ws="$(new_workspace)" + WORKSPACES+=("$ws") + echo "$ws" +} +cleanup() { + local ws + for ws in ${WORKSPACES+"${WORKSPACES[@]}"}; do rm -rf "$ws"; done +} +trap cleanup EXIT + +# ---- tests ---- + +test_clean_scan_passes() { + local ws; ws="$(workspace)" + run_scan "$ws" STUB_JSON="$FIXTURE_DIR/clean.json" STUB_SARIF="$FIXTURE_DIR/findings.sarif" + assert_eq "exit code" "$SCAN_EXIT" 0 + assert_contains "report" "$SCAN_OUT" "Successfully passed scanning" + assert_eq "readiness" "$(env_var "$ws" TRUSTABL_READINESS_SCORE)" 100 + assert_eq "risk" "$(env_var "$ws" TRUSTABL_RISK_SCORE)" 0 + assert_eq "findings count" "$(env_var "$ws" TRUSTABL_FINDINGS_COUNT)" 0 + assert_eq "native exit" "$(env_var "$ws" TRUSTABL_EXIT_CODE)" 0 +} + +test_findings_scan_reports_engine_score() { + local ws; ws="$(workspace)" + run_scan "$ws" + # overall_score 0.9588… scales to 96; risk is its complement. + assert_eq "readiness" "$(env_var "$ws" TRUSTABL_READINESS_SCORE)" 96 + assert_eq "risk" "$(env_var "$ws" TRUSTABL_RISK_SCORE)" 4 + assert_eq "findings count" "$(env_var "$ws" TRUSTABL_FINDINGS_COUNT)" 3 + assert_eq "max severity" "$(env_var "$ws" TRUSTABL_MAX_SEVERITY)" medium +} + +test_engine_gate_fails_the_build() { + local ws; ws="$(workspace)" + run_scan "$ws" STUB_EXIT=1 + assert_eq "exit code" "$SCAN_EXIT" 1 + assert_contains "reason" "$SCAN_OUT" "trustabl gated" + assert_eq "native exit" "$(env_var "$ws" TRUSTABL_EXIT_CODE)" 1 +} + +test_engine_error_fails_the_build() { + local ws; ws="$(workspace)" + run_scan "$ws" STUB_EXIT=2 + # Asserted as "non-zero" rather than a specific code: a scanner error and a + # gate failure are different things and which code carries that distinction + # out to CodeBuild is a separate question from whether the build fails. + [ "$SCAN_EXIT" -ne 0 ] || fail "exit code: expected non-zero, got $SCAN_EXIT" + assert_contains "reason" "$SCAN_OUT" "scanner error (exit 2)" + assert_eq "native exit" "$(env_var "$ws" TRUSTABL_EXIT_CODE)" 2 +} + +test_severity_threshold_gates_at_the_max_severity() { + local ws; ws="$(workspace)" + run_scan "$ws" SEVERITY_THRESHOLD=medium + assert_eq "exit code" "$SCAN_EXIT" 1 + assert_contains "reason" "$SCAN_OUT" "max severity medium >= threshold medium" +} + +test_severity_threshold_above_the_max_severity_passes() { + local ws; ws="$(workspace)" + run_scan "$ws" SEVERITY_THRESHOLD=high + assert_eq "exit code" "$SCAN_EXIT" 0 + assert_contains "report" "$SCAN_OUT" "Successfully passed scanning" +} + +test_risk_threshold_gates_when_met() { + local ws; ws="$(workspace)" + run_scan "$ws" RISK_SCORE_THRESHOLD=4 + assert_eq "exit code" "$SCAN_EXIT" 1 + assert_contains "reason" "$SCAN_OUT" "risk 4 >= threshold 4" +} + +test_risk_threshold_above_the_risk_passes() { + local ws; ws="$(workspace)" + run_scan "$ws" RISK_SCORE_THRESHOLD=50 + assert_eq "exit code" "$SCAN_EXIT" 0 +} + +test_artifacts_are_written() { + local ws; ws="$(workspace)" + run_scan "$ws" + assert_file "$ws/work/trustabl.json" + assert_file "$ws/work/trustabl.sarif" + assert_file "$ws/work/trustabl-summary.md" + assert_file "$ws/work/trustabl.env" + # The JSON artifact must be the engine's ScanResult verbatim. + assert_eq "json artifact" "$(jq -r '.repo' "$ws/work/trustabl.json")" "testdata/handoffs" + assert_eq "sarif artifact" "$(jq -r '.runs[0].results | length' "$ws/work/trustabl.sarif")" 3 +} + +test_output_paths_are_configurable() { + local ws; ws="$(workspace)" + run_scan "$ws" JSON_FILE=custom.json SARIF_FILE=custom.sarif + assert_file "$ws/work/custom.json" + assert_file "$ws/work/custom.sarif" +} + +test_summary_records_the_severity_breakdown() { + local ws; ws="$(workspace)" + run_scan "$ws" + local summary; summary="$(cat "$ws/work/trustabl-summary.md")" + assert_contains "summary" "$summary" "| medium | 1 |" + assert_contains "summary" "$summary" "| low | 2 |" + assert_contains "summary" "$summary" "| critical | 0 |" + assert_contains "summary" "$summary" "Successfully passed scanning" +} + +test_scan_flags_reach_the_engine() { + local ws; ws="$(workspace)" + run_scan "$ws" STUB_ARGS_LOG="$ws/argv.log" \ + DETECTORS=claude_sdk,mcp STRICT=true RULES_REF=abc123 + local argv; argv="$(cat "$ws/argv.log")" + assert_contains "argv" "$argv" "--detectors claude_sdk,mcp" + assert_contains "argv" "$argv" "--strict" + assert_contains "argv" "$argv" "--rules-ref abc123" +} + +test_a_tampered_release_aborts_before_the_engine_runs() { + local ws; ws="$(workspace)" + # Rewrite the archive after checksums.txt was generated over the original. + printf 'tampered' >> "$ws/release/$(asset_name)" + run_scan "$ws" STUB_ARGS_LOG="$ws/argv.log" + assert_eq "exit code" "$SCAN_EXIT" 1 + assert_contains "reason" "$SCAN_OUT" "Checksum mismatch" + [ -f "$ws/argv.log" ] && fail "the engine ran despite a checksum mismatch" +} + +test_a_pinned_version_skips_the_latest_lookup() { + local ws; ws="$(workspace)" + run_scan "$ws" STUB_CURL_LOG="$ws/curl.log" + assert_not_contains "curl log" "$(cat "$ws/curl.log")" "releases/latest" + assert_contains "report" "$SCAN_OUT" "Trustabl version: $TEST_VERSION" +} + +test_the_branch_label_is_reported() { + local ws; ws="$(workspace)" + run_scan "$ws" BRANCH=release/1.2 + assert_contains "report" "$SCAN_OUT" "Trustabl scanning branch: release/1.2" +} + +# scaled_readiness runs one scan whose ScanResult carries the given +# overall_score and prints the readiness the scanner derived from it. +scaled_readiness() { + local ws="$1" raw="$2" + jq --argjson s "$raw" '.overall_score = $s' "$FIXTURE_DIR/findings.json" > "$ws/scaled.json" + run_scan "$ws" STUB_JSON="$ws/scaled.json" >/dev/null + env_var "$ws" TRUSTABL_READINESS_SCORE +} + +test_readiness_scaling_is_correct_across_the_range() { + local ws; ws="$(workspace)" + # overall_score is a float in [0,1]; readiness is it scaled to an integer + # percent, rounded half up. Boundaries and both rounding directions. + local cases="0:0 0.004:0 0.005:1 0.5:50 0.75:75 0.994:99 0.995:100 1:100" + local c raw want got + for c in $cases; do + raw="${c%%:*}"; want="${c##*:}" + got="$(scaled_readiness "$ws" "$raw")" + assert_eq "overall_score $raw" "$got" "$want" + done +} + +test_readiness_is_clamped_to_0_100() { + local ws; ws="$(workspace)" + # The engine owns the [0,1] contract, but a clamp that silently inverts or + # overflows would publish a nonsense score, so both ends are pinned. + assert_eq "negative score clamps to 0" "$(scaled_readiness "$ws" -0.5)" 0 + assert_eq "score above 1 clamps to 100" "$(scaled_readiness "$ws" 1.5)" 100 +} + +test_risk_is_always_the_complement_of_readiness() { + local ws; ws="$(workspace)" + local raw readiness risk + for raw in 0 0.005 0.331 0.5 0.789 0.995 1; do + scaled_readiness "$ws" "$raw" >/dev/null + readiness="$(env_var "$ws" TRUSTABL_READINESS_SCORE)" + risk="$(env_var "$ws" TRUSTABL_RISK_SCORE)" + assert_eq "readiness+risk at $raw" "$(( readiness + risk ))" 100 + if [ "$readiness" -lt 0 ] || [ "$readiness" -gt 100 ]; then + fail "readiness $readiness out of range at overall_score $raw" + fi + done +} + +# ---- run ---- + +it "a clean scan passes and reports a perfect readiness" test_clean_scan_passes +it "a scan with findings reports the engine's readiness" test_findings_scan_reports_engine_score +it "the engine's gate exit fails the build" test_engine_gate_fails_the_build +it "the engine's error exit fails the build" test_engine_error_fails_the_build +it "SEVERITY_THRESHOLD gates at the max severity" test_severity_threshold_gates_at_the_max_severity +it "SEVERITY_THRESHOLD above the max severity passes" test_severity_threshold_above_the_max_severity_passes +it "RISK_SCORE_THRESHOLD gates when the risk meets it" test_risk_threshold_gates_when_met +it "RISK_SCORE_THRESHOLD above the risk passes" test_risk_threshold_above_the_risk_passes +it "the scan artifacts are written" test_artifacts_are_written +it "JSON_FILE and SARIF_FILE relocate the artifacts" test_output_paths_are_configurable +it "the summary records the severity breakdown" test_summary_records_the_severity_breakdown +it "DETECTORS, STRICT and RULES_REF reach the engine" test_scan_flags_reach_the_engine +it "a tampered release aborts before the engine runs" test_a_tampered_release_aborts_before_the_engine_runs +it "a pinned VERSION skips the latest-release lookup" test_a_pinned_version_skips_the_latest_lookup +it "the branch label is reported" test_the_branch_label_is_reported +it "readiness scaling is correct across the range" test_readiness_scaling_is_correct_across_the_range +it "readiness is clamped to 0-100" test_readiness_is_clamped_to_0_100 +it "risk is always the complement of readiness" test_risk_is_always_the_complement_of_readiness + +summarize diff --git a/test/stubs/curl b/test/stubs/curl new file mode 100755 index 0000000..1711c9a --- /dev/null +++ b/test/stubs/curl @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Stub curl. Serves the files in $STUB_RELEASE_DIR by URL basename, so the +# scanner's real download / checksum / extract path runs unchanged against a +# locally built release. An unknown URL exits 22, the code real curl uses for an +# HTTP error under -f. + +set -uo pipefail + +out="" +url="" +prev="" +for arg in "$@"; do + case "$prev" in + -o) out="$arg" ;; + esac + case "$arg" in + -*) ;; + *) [[ "$arg" == http*://* ]] && url="$arg" ;; + esac + prev="$arg" +done + +[ -n "${STUB_CURL_LOG:-}" ] && printf '%s\n' "$url" >> "$STUB_CURL_LOG" + +src="$STUB_RELEASE_DIR/$(basename "$url")" +[ -f "$src" ] || exit 22 + +if [ -n "$out" ]; then cp "$src" "$out"; else cat "$src"; fi diff --git a/test/stubs/trustabl b/test/stubs/trustabl new file mode 100755 index 0000000..8353718 --- /dev/null +++ b/test/stubs/trustabl @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Stub trustabl engine. Shipped inside the fake release tarball the stub curl +# serves, so the scanner installs and invokes it exactly as it would the real +# binary. +# +# Behavior is driven by the environment, so a test can pick the ScanResult the +# engine "returns" and the exit code it returns it with: +# STUB_JSON file whose contents answer --format json (required) +# STUB_SARIF file whose contents answer --format sarif (required) +# STUB_EXIT exit code for a scan (default 0) +# STUB_ARGS_LOG append the argv of every invocation here (optional) + +set -euo pipefail + +[ -n "${STUB_ARGS_LOG:-}" ] && printf '%s\n' "$*" >> "$STUB_ARGS_LOG" + +case "${1:-}" in + version) echo "Trustabl 0.0.0-stub"; exit 0 ;; + scan) ;; + *) echo "stub trustabl: unsupported command ${1:-}" >&2; exit 2 ;; +esac + +format=human +for i in "$@"; do + if [ "${prev:-}" = "--format" ]; then format="$i"; fi + prev="$i" +done + +case "$format" in + json) cat "$STUB_JSON" ;; + sarif) cat "$STUB_SARIF" ;; + *) echo "stub trustabl: human report" ;; +esac + +exit "${STUB_EXIT:-0}"