From b7202b21aa224e4311e026d580d553d11d809052 Mon Sep 17 00:00:00 2001 From: bradAGI <46579244+bradAGI@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:17:28 -0400 Subject: [PATCH 1/2] test: add an end-to-end test suite and CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scanner is the whole product here — it resolves a release, verifies it, runs the engine, scales the score, and decides whether the build fails — and until now nothing checked any of it. A regression in the gate logic or in the jq that reads the engine's ScanResult would ship silently. The suite drives the real scan/trustabl-scan.sh. Each test builds an actual gzipped release tarball containing a stub engine and a real checksums.txt over it, then puts a stub curl on PATH that serves that directory by URL basename. The download, sha256 verification, extraction and invocation therefore all run unmodified; only the network and the engine binary are substituted. The fixtures are unmodified output from a real `trustabl scan` (engine v0.1.7), so the assertions pin the scanner against the ScanResult shape the engine actually emits rather than a hand-written approximation of it. --- .github/workflows/ci.yml | 29 +++ test/fixtures/clean.json | 99 ++++++++ test/fixtures/findings.json | 455 +++++++++++++++++++++++++++++++++++ test/fixtures/findings.sarif | 156 ++++++++++++ test/lib/harness.sh | 143 +++++++++++ test/run-tests.sh | 178 ++++++++++++++ test/stubs/curl | 28 +++ test/stubs/trustabl | 35 +++ 8 files changed, 1123 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 test/fixtures/clean.json create mode 100644 test/fixtures/findings.json create mode 100644 test/fixtures/findings.sarif create mode 100644 test/lib/harness.sh create mode 100755 test/run-tests.sh create mode 100755 test/stubs/curl create mode 100755 test/stubs/trustabl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c646ad1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: CI + +on: + pull_request: + push: + branches: [main, master] + +permissions: + contents: read + +jobs: + test: + name: test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # The scanner targets Linux and macOS, so both get exercised. jq is + # preinstalled on both runner images, as it is on the CodeBuild + # standard images the buildspec documents. + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + - name: Tool versions + run: | + bash --version | head -1 + jq --version + - name: Run the scanner test suite + run: bash test/run-tests.sh diff --git a/test/fixtures/clean.json b/test/fixtures/clean.json new file mode 100644 index 0000000..762b70b --- /dev/null +++ b/test/fixtures/clean.json @@ -0,0 +1,99 @@ +{ + "scan_id": "scan_6aeb2293a4cb95a4", + "repo": "testdata/clean", + "languages": [ + "python" + ], + "sdks": null, + "has_shell_invocations": true, + "manifest": { + "repo_root": "/src/testdata/clean", + "is_remote": false, + "python_files": [ + "agent.py" + ], + "yaml_files": [], + "has_claude_sdk_dependency": false, + "has_openshell_artifact": false, + "components": [ + { + "kind": "dependency_manifest", + "path": "requirements.txt", + "language": "python" + } + ] + }, + "tools": [ + { + "name": "run_shell", + "kind": "shell_invocation", + "language": "python", + "file_path": "agent.py", + "start_line": 6, + "end_line": 8, + "description": "Run a shell command and return its output.", + "has_typed_params": true, + "param_names": [ + "cmd" + ], + "facts": { + "shells_out": "true" + } + } + ], + "agents": null, + "hosted_tools": null, + "mcp_servers": null, + "subagents": null, + "skills": null, + "dependencies": [ + { + "name": "anthropic", + "ecosystem": "pypi", + "source": "requirements.txt", + "start_line": 1, + "end_line": 1 + }, + { + "name": "requests", + "ecosystem": "pypi", + "source": "requirements.txt", + "start_line": 2, + "end_line": 2 + } + ], + "slash_commands": null, + "plugin_manifests": null, + "claude_settings": null, + "findings": null, + "surfaces": [ + { + "kind": "tool", + "name": "run_shell", + "file_path": "agent.py", + "score": 1, + "finding_count": 0, + "weighted_severity": 0 + } + ], + "overall_score": 1, + "projected_scores": { + "fix_critical": 1, + "fix_high": 1, + "fix_medium": 1, + "fix_low": 1, + "fix_all": 1 + }, + "rules_source": "https://github.com/trustabl/trustabl-rules", + "rules_version": "3dcf1e9c8796bdb8408e91707d7799388c291bac4b17321030872a0b7fa742bb", + "rules_from_cache": false, + "rules_schema_version": 14, + "rules_origin": { + "signed": true, + "channel": "production" + }, + "coverage": { + "files_parsed": 1, + "files_skipped": 0 + } +} diff --git a/test/fixtures/findings.json b/test/fixtures/findings.json new file mode 100644 index 0000000..774160f --- /dev/null +++ b/test/fixtures/findings.json @@ -0,0 +1,455 @@ +{ + "scan_id": "scan_8432e6168f06aba5", + "repo": "testdata/handoffs", + "languages": [ + "python" + ], + "sdks": [ + "claude_agent_sdk", + "openai_agents" + ], + "has_shell_invocations": false, + "manifest": { + "repo_root": "/src/testdata/handoffs", + "is_remote": false, + "python_files": [ + "message_filter.py", + "message_filter_streaming.py" + ], + "yaml_files": [], + "has_claude_sdk_dependency": false, + "has_openshell_artifact": false + }, + "tools": [ + { + "name": "random_number_tool", + "kind": "claude_sdk_tool", + "language": "python", + "file_path": "message_filter.py", + "start_line": 19, + "end_line": 21, + "description": "Return a random integer between 0 and the given maximum.", + "has_typed_params": true, + "param_names": [ + "max" + ] + }, + { + "name": "random_number_tool", + "kind": "claude_sdk_tool", + "language": "python", + "file_path": "message_filter_streaming.py", + "start_line": 19, + "end_line": 21, + "description": "Return a random integer between 0 and the given maximum.", + "has_typed_params": true, + "param_names": [ + "max" + ] + } + ], + "agents": [ + { + "sdk": "openai_agents", + "class": "Agent", + "language": "python", + "file_path": "message_filter.py", + "start_line": 52, + "end_line": 56, + "name": "Assistant", + "kwargs": { + "children": { + "instructions": { + "value": { + "kind": "literal_string", + "text": "\"Be extremely concise.\"" + } + }, + "name": { + "value": { + "kind": "literal_string", + "text": "\"Assistant\"" + } + }, + "tools": { + "value": { + "kind": "list", + "text": "[random_number_tool]", + "list": [ + { + "kind": "name_ref", + "text": "random_number_tool" + } + ] + } + } + } + }, + "tool_refs": [ + { + "name": "random_number_tool", + "external": false + } + ], + "hosted_tool_refs": null, + "mcp_server_refs": null, + "handoff_refs": null, + "input_guards": null, + "output_guards": null, + "opaque": false + }, + { + "sdk": "openai_agents", + "class": "Agent", + "language": "python", + "file_path": "message_filter.py", + "start_line": 58, + "end_line": 62, + "name": "Spanish Assistant", + "kwargs": { + "children": { + "handoff_description": { + "value": { + "kind": "literal_string", + "text": "\"A Spanish-speaking assistant.\"" + } + }, + "instructions": { + "value": { + "kind": "literal_string", + "text": "\"You only speak Spanish and are extremely concise.\"" + } + }, + "name": { + "value": { + "kind": "literal_string", + "text": "\"Spanish Assistant\"" + } + } + } + }, + "tool_refs": null, + "hosted_tool_refs": null, + "mcp_server_refs": null, + "handoff_refs": null, + "input_guards": null, + "output_guards": null, + "opaque": false + }, + { + "sdk": "openai_agents", + "class": "Agent", + "language": "python", + "file_path": "message_filter.py", + "start_line": 64, + "end_line": 70, + "name": "Assistant", + "kwargs": { + "children": { + "handoffs": { + "value": { + "kind": "list", + "text": "[handoff(spanish_agent, input_filter=spanish_handoff_message_filter)]", + "list": [ + { + "kind": "call", + "text": "handoff(spanish_agent, input_filter=spanish_handoff_message_filter)" + } + ] + } + }, + "instructions": { + "value": { + "kind": "unknown", + "text": "(\n \"Be a helpful assistant. If the user speaks Spanish, handoff to the Spanish assistant.\"\n )" + } + }, + "name": { + "value": { + "kind": "literal_string", + "text": "\"Assistant\"" + } + } + } + }, + "tool_refs": null, + "hosted_tool_refs": null, + "mcp_server_refs": null, + "handoff_refs": [ + { + "name": "handoff(spanish_agent, input_filter=spanish_handoff_message_filter)", + "external": true + } + ], + "input_guards": null, + "output_guards": null, + "opaque": false + }, + { + "sdk": "openai_agents", + "class": "Agent", + "language": "python", + "file_path": "message_filter_streaming.py", + "start_line": 52, + "end_line": 56, + "name": "Assistant", + "kwargs": { + "children": { + "instructions": { + "value": { + "kind": "literal_string", + "text": "\"Be extremely concise.\"" + } + }, + "name": { + "value": { + "kind": "literal_string", + "text": "\"Assistant\"" + } + }, + "tools": { + "value": { + "kind": "list", + "text": "[random_number_tool]", + "list": [ + { + "kind": "name_ref", + "text": "random_number_tool" + } + ] + } + } + } + }, + "tool_refs": [ + { + "name": "random_number_tool", + "external": false + } + ], + "hosted_tool_refs": null, + "mcp_server_refs": null, + "handoff_refs": null, + "input_guards": null, + "output_guards": null, + "opaque": false + }, + { + "sdk": "openai_agents", + "class": "Agent", + "language": "python", + "file_path": "message_filter_streaming.py", + "start_line": 58, + "end_line": 62, + "name": "Spanish Assistant", + "kwargs": { + "children": { + "handoff_description": { + "value": { + "kind": "literal_string", + "text": "\"A Spanish-speaking assistant.\"" + } + }, + "instructions": { + "value": { + "kind": "literal_string", + "text": "\"You only speak Spanish and are extremely concise.\"" + } + }, + "name": { + "value": { + "kind": "literal_string", + "text": "\"Spanish Assistant\"" + } + } + } + }, + "tool_refs": null, + "hosted_tool_refs": null, + "mcp_server_refs": null, + "handoff_refs": null, + "input_guards": null, + "output_guards": null, + "opaque": false + }, + { + "sdk": "openai_agents", + "class": "Agent", + "language": "python", + "file_path": "message_filter_streaming.py", + "start_line": 64, + "end_line": 70, + "name": "Assistant", + "kwargs": { + "children": { + "handoffs": { + "value": { + "kind": "list", + "text": "[handoff(spanish_agent, input_filter=spanish_handoff_message_filter)]", + "list": [ + { + "kind": "call", + "text": "handoff(spanish_agent, input_filter=spanish_handoff_message_filter)" + } + ] + } + }, + "instructions": { + "value": { + "kind": "unknown", + "text": "(\n \"Be a helpful assistant. If the user speaks Spanish, handoff to the Spanish assistant.\"\n )" + } + }, + "name": { + "value": { + "kind": "literal_string", + "text": "\"Assistant\"" + } + } + } + }, + "tool_refs": null, + "hosted_tool_refs": null, + "mcp_server_refs": null, + "handoff_refs": [ + { + "name": "handoff(spanish_agent, input_filter=spanish_handoff_message_filter)", + "external": true + } + ], + "input_guards": null, + "output_guards": null, + "opaque": false + } + ], + "hosted_tools": null, + "mcp_servers": null, + "subagents": null, + "skills": null, + "dependencies": null, + "slash_commands": null, + "plugin_manifests": null, + "claude_settings": null, + "findings": [ + { + "rule_id": "CSDK-203", + "category": "claude_sdk", + "scope": "repo", + "severity": "low", + "tool_name": "", + "file_path": "", + "start_line": 0, + "end_line": 0, + "title": "Repo ships Claude Agent SDK code without an agent-guidance doc (AGENTS.md/CLAUDE.md)", + "explanation": "A repository that builds on the Claude Agent SDK but ships no agent-guidance doc — CLAUDE.md (the Claude Code convention) or AGENTS.md (the cross-vendor one) — leaves any coding agent that opens the repo with no project-specific guidance. The conventions, build commands, test runners, lint scripts, and \"do not touch\" boundaries a human maintainer would describe in onboarding are absent, so the agent has to infer them from the source on every session and frequently guesses wrong — bypassing the project's lint, picking the wrong test command, or violating commit conventions that were never written down anywhere it could read. The cost compounds: each new contributor (human or agent) reinvents the same wrong assumptions.\n", + "suggested_fix": "Add a CLAUDE.md (or an AGENTS.md) at the repository root documenting how to build, test, and lint the project, the coding conventions the agent must respect, the directories or files it must not modify, and any project-specific safety guardrails (e.g. \"never run migrations\", \"never push to main\"). Treat it as the project's agent-facing README and keep it under version control so reviewers see drift.\n", + "confidence": 0.9 + }, + { + "rule_id": "OAI-201", + "category": "openai_sdk", + "scope": "repo", + "severity": "medium", + "tool_name": "", + "file_path": "", + "start_line": 0, + "end_line": 0, + "title": "Project uses default OpenAI tracing", + "explanation": "The project uses the OpenAI Agents SDK with default tracing enabled. By default, inputs, tool calls, tool outputs, and agent responses are sent to OpenAI's hosted tracing backend. For projects handling sensitive data (PII, credentials, internal documents), this is a data egress channel that is easy to miss.\n", + "suggested_fix": "Either disable tracing entirely (set OPENAI_AGENTS_DISABLE_TRACING=1 in the environment) OR register a custom trace processor via agents.tracing.add_trace_processor(...) that redacts sensitive fields before they leave the process.\n", + "confidence": 0.8 + }, + { + "rule_id": "OAI-202", + "category": "openai_sdk", + "scope": "repo", + "severity": "low", + "tool_name": "", + "file_path": "", + "start_line": 0, + "end_line": 0, + "title": "OpenAI Agents project ships no agent-guidance doc (AGENTS.md/CLAUDE.md)", + "explanation": "The project uses the OpenAI Agents SDK in code but ships no agent-guidance doc (AGENTS.md or CLAUDE.md) at the repo root. AGENTS.md is the cross-vendor convention an editing coding agent reads before it acts; with neither file present, any agent that opens this repo has no project-specific guidance on Agent vs SandboxAgent choice, handoff topology, required input/output guardrails, tool_choice settings, or the local test and build commands. The likely consequence is generated code that bypasses the project's safety contracts because nothing in-tree teaches the agent the local rules.\n", + "suggested_fix": "Add an AGENTS.md at the repo root (the cross-vendor standard; a CLAUDE.md also satisfies this rule). State whether the project uses Agent or SandboxAgent, list required guardrails (input_guardrails, output_guardrails) and tool_choice conventions, note any handoff or tracing policy, and give the exact test, lint, and build commands. Keep it short and concrete so an editing agent can act on it without re-deriving the conventions.\n", + "confidence": 0.9 + } + ], + "surfaces": [ + { + "kind": "repo", + "name": "", + "file_path": "", + "score": 0.8033333333333333, + "finding_count": 3, + "weighted_severity": 0.5900000000000001 + }, + { + "kind": "agent", + "name": "Assistant", + "file_path": "message_filter.py", + "score": 1, + "finding_count": 0, + "weighted_severity": 0 + }, + { + "kind": "agent", + "name": "Assistant", + "file_path": "message_filter_streaming.py", + "score": 1, + "finding_count": 0, + "weighted_severity": 0 + }, + { + "kind": "agent", + "name": "Spanish Assistant", + "file_path": "message_filter.py", + "score": 1, + "finding_count": 0, + "weighted_severity": 0 + }, + { + "kind": "agent", + "name": "Spanish Assistant", + "file_path": "message_filter_streaming.py", + "score": 1, + "finding_count": 0, + "weighted_severity": 0 + }, + { + "kind": "tool", + "name": "random_number_tool", + "file_path": "message_filter.py", + "score": 1, + "finding_count": 0, + "weighted_severity": 0 + }, + { + "kind": "tool", + "name": "random_number_tool", + "file_path": "message_filter_streaming.py", + "score": 1, + "finding_count": 0, + "weighted_severity": 0 + } + ], + "overall_score": 0.9588010540184454, + "projected_scores": { + "fix_critical": 0.9588010540184454, + "fix_high": 0.9588010540184454, + "fix_medium": 0.9842778541953232, + "fix_low": 1, + "fix_all": 1 + }, + "rules_source": "https://github.com/trustabl/trustabl-rules", + "rules_version": "3dcf1e9c8796bdb8408e91707d7799388c291bac4b17321030872a0b7fa742bb", + "rules_from_cache": false, + "rules_schema_version": 14, + "rules_origin": { + "signed": true, + "channel": "production" + }, + "coverage": { + "files_parsed": 2, + "files_skipped": 0 + } +} diff --git a/test/fixtures/findings.sarif b/test/fixtures/findings.sarif new file mode 100644 index 0000000..3cc5040 --- /dev/null +++ b/test/fixtures/findings.sarif @@ -0,0 +1,156 @@ +{ + "version": "2.1.0", + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "runs": [ + { + "tool": { + "driver": { + "name": "trustabl", + "fullName": "Trustabl — static analyzer for agent reliability", + "informationUri": "https://github.com/trustabl/trustabl", + "version": "0.1.7", + "semanticVersion": "0.1.7", + "rules": [ + { + "id": "CSDK-203", + "shortDescription": { + "text": "Repo ships Claude Agent SDK code without an agent-guidance doc (AGENTS.md/CLAUDE.md)" + }, + "fullDescription": { + "text": "A repository that builds on the Claude Agent SDK but ships no agent-guidance doc — CLAUDE.md (the Claude Code convention) or AGENTS.md (the cross-vendor one) — leaves any coding agent that opens the repo with no project-specific guidance. The conventions, build commands, test runners, lint scripts, and \"do not touch\" boundaries a human maintainer would describe in onboarding are absent, so the agent has to infer them from the source on every session and frequently guesses wrong — bypassing the project's lint, picking the wrong test command, or violating commit conventions that were never written down anywhere it could read. The cost compounds: each new contributor (human or agent) reinvents the same wrong assumptions.\n" + }, + "help": { + "text": "Add a CLAUDE.md (or an AGENTS.md) at the repository root documenting how to build, test, and lint the project, the coding conventions the agent must respect, the directories or files it must not modify, and any project-specific safety guardrails (e.g. \"never run migrations\", \"never push to main\"). Treat it as the project's agent-facing README and keep it under version control so reviewers see drift.\n" + }, + "defaultConfiguration": { + "level": "note" + }, + "properties": { + "confidence": 0.9, + "security-severity": "3.0", + "tags": [ + "claude_sdk", + "repo", + "python" + ] + } + }, + { + "id": "OAI-201", + "shortDescription": { + "text": "Project uses default OpenAI tracing" + }, + "fullDescription": { + "text": "The project uses the OpenAI Agents SDK with default tracing enabled. By default, inputs, tool calls, tool outputs, and agent responses are sent to OpenAI's hosted tracing backend. For projects handling sensitive data (PII, credentials, internal documents), this is a data egress channel that is easy to miss.\n" + }, + "help": { + "text": "Either disable tracing entirely (set OPENAI_AGENTS_DISABLE_TRACING=1 in the environment) OR register a custom trace processor via agents.tracing.add_trace_processor(...) that redacts sensitive fields before they leave the process.\n" + }, + "defaultConfiguration": { + "level": "warning" + }, + "properties": { + "confidence": 0.8, + "security-severity": "5.5", + "tags": [ + "openai_sdk", + "repo", + "python" + ] + } + }, + { + "id": "OAI-202", + "shortDescription": { + "text": "OpenAI Agents project ships no agent-guidance doc (AGENTS.md/CLAUDE.md)" + }, + "fullDescription": { + "text": "The project uses the OpenAI Agents SDK in code but ships no agent-guidance doc (AGENTS.md or CLAUDE.md) at the repo root. AGENTS.md is the cross-vendor convention an editing coding agent reads before it acts; with neither file present, any agent that opens this repo has no project-specific guidance on Agent vs SandboxAgent choice, handoff topology, required input/output guardrails, tool_choice settings, or the local test and build commands. The likely consequence is generated code that bypasses the project's safety contracts because nothing in-tree teaches the agent the local rules.\n" + }, + "help": { + "text": "Add an AGENTS.md at the repo root (the cross-vendor standard; a CLAUDE.md also satisfies this rule). State whether the project uses Agent or SandboxAgent, list required guardrails (input_guardrails, output_guardrails) and tool_choice conventions, note any handoff or tracing policy, and give the exact test, lint, and build commands. Keep it short and concrete so an editing agent can act on it without re-deriving the conventions.\n" + }, + "defaultConfiguration": { + "level": "note" + }, + "properties": { + "confidence": 0.9, + "security-severity": "3.0", + "tags": [ + "openai_sdk", + "repo", + "python" + ] + } + } + ], + "properties": { + "rules_from_cache": false, + "rules_source": "https://github.com/trustabl/trustabl-rules", + "rules_version": "3dcf1e9c8796bdb8408e91707d7799388c291bac4b17321030872a0b7fa742bb" + } + } + }, + "invocations": [ + { + "executionSuccessful": true + } + ], + "results": [ + { + "ruleId": "CSDK-203", + "ruleIndex": 0, + "kind": "informational", + "message": { + "text": "A repository that builds on the Claude Agent SDK but ships no agent-guidance doc — CLAUDE.md (the Claude Code convention) or AGENTS.md (the cross-vendor one) — leaves any coding agent that opens the repo with no project-specific guidance. The conventions, build commands, test runners, lint scripts, and \"do not touch\" boundaries a human maintainer would describe in onboarding are absent, so the agent has to infer them from the source on every session and frequently guesses wrong — bypassing the project's lint, picking the wrong test command, or violating commit conventions that were never written down anywhere it could read. The cost compounds: each new contributor (human or agent) reinvents the same wrong assumptions.\n" + }, + "rank": 90, + "partialFingerprints": { + "primaryLocationLineHash": "8c927f0b1b0fb63358ed8144fb13e050c197ca78e3086aa6eb128d97cc47085a" + }, + "properties": { + "confidence": 0.9 + } + }, + { + "ruleId": "OAI-201", + "ruleIndex": 1, + "kind": "informational", + "message": { + "text": "The project uses the OpenAI Agents SDK with default tracing enabled. By default, inputs, tool calls, tool outputs, and agent responses are sent to OpenAI's hosted tracing backend. For projects handling sensitive data (PII, credentials, internal documents), this is a data egress channel that is easy to miss.\n" + }, + "rank": 80, + "partialFingerprints": { + "primaryLocationLineHash": "909f2973bbb976bb4bca098a37428ad29f25c0b2587171b86430a13f1367e5d4" + }, + "properties": { + "confidence": 0.8 + } + }, + { + "ruleId": "OAI-202", + "ruleIndex": 2, + "kind": "informational", + "message": { + "text": "The project uses the OpenAI Agents SDK in code but ships no agent-guidance doc (AGENTS.md or CLAUDE.md) at the repo root. AGENTS.md is the cross-vendor convention an editing coding agent reads before it acts; with neither file present, any agent that opens this repo has no project-specific guidance on Agent vs SandboxAgent choice, handoff topology, required input/output guardrails, tool_choice settings, or the local test and build commands. The likely consequence is generated code that bypasses the project's safety contracts because nothing in-tree teaches the agent the local rules.\n" + }, + "rank": 90, + "partialFingerprints": { + "primaryLocationLineHash": "26a8e7422c51ce85da5436d8ee503243e0a68a42e73ccb4ac96ab9be0ea776e1" + }, + "properties": { + "confidence": 0.9 + } + } + ], + "automationDetails": { + "id": "scan_8432e6168f06aba5" + }, + "originalUriBaseIds": { + "REPO_ROOT": { + "uri": "file:///src/testdata/handoffs/" + } + } + } + ] +} diff --git a/test/lib/harness.sh b/test/lib/harness.sh new file mode 100644 index 0000000..4d82023 --- /dev/null +++ b/test/lib/harness.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Test harness for scan/trustabl-scan.sh. +# +# Each test gets a throwaway workspace holding a locally built "release" — a real +# gzipped tarball containing the stub engine, plus a real checksums.txt over it. +# The stub curl serves that directory by URL basename, so the scanner's download, +# sha256 verification, extraction and invocation all run for real; only the +# network and the engine itself are substituted. + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +STUB_DIR="$REPO_ROOT/test/stubs" +FIXTURE_DIR="$REPO_ROOT/test/fixtures" +SCANNER="$REPO_ROOT/scan/trustabl-scan.sh" + +# The version every test pins, so no test needs the latest-release lookup. +TEST_VERSION="v9.9.9" + +TESTS_RUN=0 +TESTS_FAILED=0 +CURRENT_TEST="" +FAILURES=() + +# sha256_of prints the sha256 of a file. Written portably because the harness has +# to run wherever the scanner does. +sha256_of() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +# asset_name mirrors the scanner's own OS/arch asset naming, so the release the +# harness builds is the one the scanner asks for. +asset_name() { + local os arch + case "$(uname -s)" in + Linux) os=linux ;; + Darwin) os=darwin ;; + *) echo "unsupported OS $(uname -s)" >&2; return 1 ;; + esac + case "$(uname -m)" in + x86_64|amd64) arch=amd64 ;; + aarch64|arm64) arch=arm64 ;; + *) echo "unsupported arch $(uname -m)" >&2; return 1 ;; + esac + echo "trustabl_${TEST_VERSION#v}_${os}_${arch}.tar.gz" +} + +# new_workspace prints the path to a fresh workspace with a built release in +# ./release. Callers cd into it (the scanner writes its artifacts to $PWD). +new_workspace() { + local ws asset staging + ws="$(mktemp -d "${TMPDIR:-/tmp}/trustabl-test.XXXXXX")" + mkdir -p "$ws/release" "$ws/work" + + asset="$(asset_name)" + staging="$ws/staging" + mkdir -p "$staging" + cp "$STUB_DIR/trustabl" "$staging/trustabl" + chmod +x "$staging/trustabl" + tar -czf "$ws/release/$asset" -C "$staging" trustabl + + printf '%s %s\n' "$(sha256_of "$ws/release/$asset")" "$asset" > "$ws/release/checksums.txt" + printf '{"tag_name":"%s"}\n' "$TEST_VERSION" > "$ws/release/latest" + + echo "$ws" +} + +# run_scan [KEY=VALUE ...] runs the scanner in $workspace/work with +# the stubs on PATH. It sets SCAN_EXIT and SCAN_OUT; it never aborts the caller. +run_scan() { + local ws="$1"; shift + SCAN_OUT="$( + cd "$ws/work" && env \ + PATH="$STUB_DIR:$PATH" \ + STUB_RELEASE_DIR="$ws/release" \ + STUB_JSON="${STUB_JSON:-$FIXTURE_DIR/findings.json}" \ + STUB_SARIF="${STUB_SARIF:-$FIXTURE_DIR/findings.sarif}" \ + VERSION="$TEST_VERSION" \ + BRANCH="test-branch" \ + "$@" \ + bash "$SCANNER" 2>&1 + )" + SCAN_EXIT=$? + return 0 +} + +# env_var prints the value the scanner wrote to trustabl.env. +env_var() { + sed -n "s/^$2=//p" "$1/work/trustabl.env" +} + +# ---- assertions ---- + +fail() { + TESTS_FAILED=$((TESTS_FAILED + 1)) + FAILURES+=("$CURRENT_TEST: $1") + printf ' not ok — %s\n' "$1" +} + +assert_eq() { + if [ "$2" = "$3" ]; then return 0; fi + fail "$1: expected '$3', got '$2'" +} + +assert_contains() { + case "$2" in + *"$3"*) return 0 ;; + esac + fail "$1: output did not contain '$3'" +} + +assert_not_contains() { + case "$2" in + *"$3"*) fail "$1: output unexpectedly contained '$3'"; return 0 ;; + esac +} + +assert_file() { + [ -f "$1" ] && return 0 + fail "expected file $1 to exist" +} + +# it runs one test and reports the result. +it() { + CURRENT_TEST="$1" + TESTS_RUN=$((TESTS_RUN + 1)) + local before=$TESTS_FAILED + "$2" + if [ "$TESTS_FAILED" -eq "$before" ]; then printf 'ok %d — %s\n' "$TESTS_RUN" "$1"; else printf 'NOT OK %d — %s\n' "$TESTS_RUN" "$1"; fi +} + +summarize() { + echo + echo "1..$TESTS_RUN" + if [ "$TESTS_FAILED" -gt 0 ]; then + echo "$TESTS_FAILED failure(s):" + printf ' - %s\n' "${FAILURES[@]}" + return 1 + fi + echo "all $TESTS_RUN test(s) passed" +} diff --git a/test/run-tests.sh b/test/run-tests.sh new file mode 100755 index 0000000..c4fc574 --- /dev/null +++ b/test/run-tests.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# End-to-end tests for scan/trustabl-scan.sh. Run with: bash test/run-tests.sh +# +# The fixtures under test/fixtures are unmodified ScanResult / SARIF output from +# a real engine run, so the assertions below pin the scanner against the shape +# the engine actually emits rather than against a hand-written approximation. +# +# clean.json 0 findings overall_score 1.0 +# findings.json 3 findings overall_score 0.9588… (2 low, 1 medium) + +set -uo pipefail + +# shellcheck source=test/lib/harness.sh +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/harness.sh" + +WORKSPACES=() +workspace() { + local ws + ws="$(new_workspace)" + WORKSPACES+=("$ws") + echo "$ws" +} +cleanup() { + local ws + for ws in ${WORKSPACES+"${WORKSPACES[@]}"}; do rm -rf "$ws"; done +} +trap cleanup EXIT + +# ---- tests ---- + +test_clean_scan_passes() { + local ws; ws="$(workspace)" + run_scan "$ws" STUB_JSON="$FIXTURE_DIR/clean.json" STUB_SARIF="$FIXTURE_DIR/findings.sarif" + assert_eq "exit code" "$SCAN_EXIT" 0 + assert_contains "report" "$SCAN_OUT" "Successfully passed scanning" + assert_eq "readiness" "$(env_var "$ws" TRUSTABL_READINESS_SCORE)" 100 + assert_eq "risk" "$(env_var "$ws" TRUSTABL_RISK_SCORE)" 0 + assert_eq "findings count" "$(env_var "$ws" TRUSTABL_FINDINGS_COUNT)" 0 + assert_eq "native exit" "$(env_var "$ws" TRUSTABL_EXIT_CODE)" 0 +} + +test_findings_scan_reports_engine_score() { + local ws; ws="$(workspace)" + run_scan "$ws" + # overall_score 0.9588… scales to 96; risk is its complement. + assert_eq "readiness" "$(env_var "$ws" TRUSTABL_READINESS_SCORE)" 96 + assert_eq "risk" "$(env_var "$ws" TRUSTABL_RISK_SCORE)" 4 + assert_eq "findings count" "$(env_var "$ws" TRUSTABL_FINDINGS_COUNT)" 3 + assert_eq "max severity" "$(env_var "$ws" TRUSTABL_MAX_SEVERITY)" medium +} + +test_engine_gate_fails_the_build() { + local ws; ws="$(workspace)" + run_scan "$ws" STUB_EXIT=1 + assert_eq "exit code" "$SCAN_EXIT" 1 + assert_contains "reason" "$SCAN_OUT" "trustabl gated" + assert_eq "native exit" "$(env_var "$ws" TRUSTABL_EXIT_CODE)" 1 +} + +test_engine_error_fails_the_build() { + local ws; ws="$(workspace)" + run_scan "$ws" STUB_EXIT=2 + # Asserted as "non-zero" rather than a specific code: a scanner error and a + # gate failure are different things and which code carries that distinction + # out to CodeBuild is a separate question from whether the build fails. + [ "$SCAN_EXIT" -ne 0 ] || fail "exit code: expected non-zero, got $SCAN_EXIT" + assert_contains "reason" "$SCAN_OUT" "scanner error (exit 2)" + assert_eq "native exit" "$(env_var "$ws" TRUSTABL_EXIT_CODE)" 2 +} + +test_severity_threshold_gates_at_the_max_severity() { + local ws; ws="$(workspace)" + run_scan "$ws" SEVERITY_THRESHOLD=medium + assert_eq "exit code" "$SCAN_EXIT" 1 + assert_contains "reason" "$SCAN_OUT" "max severity medium >= threshold medium" +} + +test_severity_threshold_above_the_max_severity_passes() { + local ws; ws="$(workspace)" + run_scan "$ws" SEVERITY_THRESHOLD=high + assert_eq "exit code" "$SCAN_EXIT" 0 + assert_contains "report" "$SCAN_OUT" "Successfully passed scanning" +} + +test_risk_threshold_gates_when_met() { + local ws; ws="$(workspace)" + run_scan "$ws" RISK_SCORE_THRESHOLD=4 + assert_eq "exit code" "$SCAN_EXIT" 1 + assert_contains "reason" "$SCAN_OUT" "risk 4 >= threshold 4" +} + +test_risk_threshold_above_the_risk_passes() { + local ws; ws="$(workspace)" + run_scan "$ws" RISK_SCORE_THRESHOLD=50 + assert_eq "exit code" "$SCAN_EXIT" 0 +} + +test_artifacts_are_written() { + local ws; ws="$(workspace)" + run_scan "$ws" + assert_file "$ws/work/trustabl.json" + assert_file "$ws/work/trustabl.sarif" + assert_file "$ws/work/trustabl-summary.md" + assert_file "$ws/work/trustabl.env" + # The JSON artifact must be the engine's ScanResult verbatim. + assert_eq "json artifact" "$(jq -r '.repo' "$ws/work/trustabl.json")" "testdata/handoffs" + assert_eq "sarif artifact" "$(jq -r '.runs[0].results | length' "$ws/work/trustabl.sarif")" 3 +} + +test_output_paths_are_configurable() { + local ws; ws="$(workspace)" + run_scan "$ws" JSON_FILE=custom.json SARIF_FILE=custom.sarif + assert_file "$ws/work/custom.json" + assert_file "$ws/work/custom.sarif" +} + +test_summary_records_the_severity_breakdown() { + local ws; ws="$(workspace)" + run_scan "$ws" + local summary; summary="$(cat "$ws/work/trustabl-summary.md")" + assert_contains "summary" "$summary" "| medium | 1 |" + assert_contains "summary" "$summary" "| low | 2 |" + assert_contains "summary" "$summary" "| critical | 0 |" + assert_contains "summary" "$summary" "Successfully passed scanning" +} + +test_scan_flags_reach_the_engine() { + local ws; ws="$(workspace)" + run_scan "$ws" STUB_ARGS_LOG="$ws/argv.log" \ + DETECTORS=claude_sdk,mcp STRICT=true RULES_REF=abc123 + local argv; argv="$(cat "$ws/argv.log")" + assert_contains "argv" "$argv" "--detectors claude_sdk,mcp" + assert_contains "argv" "$argv" "--strict" + assert_contains "argv" "$argv" "--rules-ref abc123" +} + +test_a_tampered_release_aborts_before_the_engine_runs() { + local ws; ws="$(workspace)" + # Rewrite the archive after checksums.txt was generated over the original. + printf 'tampered' >> "$ws/release/$(asset_name)" + run_scan "$ws" STUB_ARGS_LOG="$ws/argv.log" + assert_eq "exit code" "$SCAN_EXIT" 1 + assert_contains "reason" "$SCAN_OUT" "Checksum mismatch" + [ -f "$ws/argv.log" ] && fail "the engine ran despite a checksum mismatch" +} + +test_a_pinned_version_skips_the_latest_lookup() { + local ws; ws="$(workspace)" + run_scan "$ws" STUB_CURL_LOG="$ws/curl.log" + assert_not_contains "curl log" "$(cat "$ws/curl.log")" "releases/latest" + assert_contains "report" "$SCAN_OUT" "Trustabl version: $TEST_VERSION" +} + +test_the_branch_label_is_reported() { + local ws; ws="$(workspace)" + run_scan "$ws" BRANCH=release/1.2 + assert_contains "report" "$SCAN_OUT" "Trustabl scanning branch: release/1.2" +} + +# ---- run ---- + +it "a clean scan passes and reports a perfect readiness" test_clean_scan_passes +it "a scan with findings reports the engine's readiness" test_findings_scan_reports_engine_score +it "the engine's gate exit fails the build" test_engine_gate_fails_the_build +it "the engine's error exit fails the build" test_engine_error_fails_the_build +it "SEVERITY_THRESHOLD gates at the max severity" test_severity_threshold_gates_at_the_max_severity +it "SEVERITY_THRESHOLD above the max severity passes" test_severity_threshold_above_the_max_severity_passes +it "RISK_SCORE_THRESHOLD gates when the risk meets it" test_risk_threshold_gates_when_met +it "RISK_SCORE_THRESHOLD above the risk passes" test_risk_threshold_above_the_risk_passes +it "the scan artifacts are written" test_artifacts_are_written +it "JSON_FILE and SARIF_FILE relocate the artifacts" test_output_paths_are_configurable +it "the summary records the severity breakdown" test_summary_records_the_severity_breakdown +it "DETECTORS, STRICT and RULES_REF reach the engine" test_scan_flags_reach_the_engine +it "a tampered release aborts before the engine runs" test_a_tampered_release_aborts_before_the_engine_runs +it "a pinned VERSION skips the latest-release lookup" test_a_pinned_version_skips_the_latest_lookup +it "the branch label is reported" test_the_branch_label_is_reported + +summarize diff --git a/test/stubs/curl b/test/stubs/curl new file mode 100755 index 0000000..1711c9a --- /dev/null +++ b/test/stubs/curl @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Stub curl. Serves the files in $STUB_RELEASE_DIR by URL basename, so the +# scanner's real download / checksum / extract path runs unchanged against a +# locally built release. An unknown URL exits 22, the code real curl uses for an +# HTTP error under -f. + +set -uo pipefail + +out="" +url="" +prev="" +for arg in "$@"; do + case "$prev" in + -o) out="$arg" ;; + esac + case "$arg" in + -*) ;; + *) [[ "$arg" == http*://* ]] && url="$arg" ;; + esac + prev="$arg" +done + +[ -n "${STUB_CURL_LOG:-}" ] && printf '%s\n' "$url" >> "$STUB_CURL_LOG" + +src="$STUB_RELEASE_DIR/$(basename "$url")" +[ -f "$src" ] || exit 22 + +if [ -n "$out" ]; then cp "$src" "$out"; else cat "$src"; fi diff --git a/test/stubs/trustabl b/test/stubs/trustabl new file mode 100755 index 0000000..8353718 --- /dev/null +++ b/test/stubs/trustabl @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Stub trustabl engine. Shipped inside the fake release tarball the stub curl +# serves, so the scanner installs and invokes it exactly as it would the real +# binary. +# +# Behavior is driven by the environment, so a test can pick the ScanResult the +# engine "returns" and the exit code it returns it with: +# STUB_JSON file whose contents answer --format json (required) +# STUB_SARIF file whose contents answer --format sarif (required) +# STUB_EXIT exit code for a scan (default 0) +# STUB_ARGS_LOG append the argv of every invocation here (optional) + +set -euo pipefail + +[ -n "${STUB_ARGS_LOG:-}" ] && printf '%s\n' "$*" >> "$STUB_ARGS_LOG" + +case "${1:-}" in + version) echo "Trustabl 0.0.0-stub"; exit 0 ;; + scan) ;; + *) echo "stub trustabl: unsupported command ${1:-}" >&2; exit 2 ;; +esac + +format=human +for i in "$@"; do + if [ "${prev:-}" = "--format" ]; then format="$i"; fi + prev="$i" +done + +case "$format" in + json) cat "$STUB_JSON" ;; + sarif) cat "$STUB_SARIF" ;; + *) echo "stub trustabl: human report" ;; +esac + +exit "${STUB_EXIT:-0}" From e66b7f63066e77cf0202e20ad7d5a0a06322d4f7 Mon Sep 17 00:00:00 2001 From: bradAGI <46579244+bradAGI@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:21:13 -0400 Subject: [PATCH 2/2] test: pin the score-scaling invariants, not one example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Readiness is the engine's overall_score — a float in [0,1] — scaled to an integer percent, and risk is its complement. Those two lines decide what every gate, every artifact and every downstream consumer sees, and until now they were covered by a single example: 0.9588 -> 96. One example cannot distinguish rounding from truncation, and cannot see a clamp at all. Three tests replace it: - scaling across the range, including both rounding directions and the half-way boundaries (0.004 -> 0, 0.005 -> 1, 0.994 -> 99, 0.995 -> 100) - the clamp at both ends, for inputs outside the engine's stated contract - readiness + risk == 100 for every input, with readiness in range Confirmed non-vacuous by mutation. Truncating instead of rounding half up: not ok — readiness: expected '96', got '95' Dropping the upper clamp: not ok — score above 1 clamps to 100: expected '100', got '150' No production change; scan/trustabl-scan.sh is untouched. --- test/run-tests.sh | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/test/run-tests.sh b/test/run-tests.sh index c4fc574..3aa97bb 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -157,6 +157,50 @@ test_the_branch_label_is_reported() { assert_contains "report" "$SCAN_OUT" "Trustabl scanning branch: release/1.2" } +# scaled_readiness runs one scan whose ScanResult carries the given +# overall_score and prints the readiness the scanner derived from it. +scaled_readiness() { + local ws="$1" raw="$2" + jq --argjson s "$raw" '.overall_score = $s' "$FIXTURE_DIR/findings.json" > "$ws/scaled.json" + run_scan "$ws" STUB_JSON="$ws/scaled.json" >/dev/null + env_var "$ws" TRUSTABL_READINESS_SCORE +} + +test_readiness_scaling_is_correct_across_the_range() { + local ws; ws="$(workspace)" + # overall_score is a float in [0,1]; readiness is it scaled to an integer + # percent, rounded half up. Boundaries and both rounding directions. + local cases="0:0 0.004:0 0.005:1 0.5:50 0.75:75 0.994:99 0.995:100 1:100" + local c raw want got + for c in $cases; do + raw="${c%%:*}"; want="${c##*:}" + got="$(scaled_readiness "$ws" "$raw")" + assert_eq "overall_score $raw" "$got" "$want" + done +} + +test_readiness_is_clamped_to_0_100() { + local ws; ws="$(workspace)" + # The engine owns the [0,1] contract, but a clamp that silently inverts or + # overflows would publish a nonsense score, so both ends are pinned. + assert_eq "negative score clamps to 0" "$(scaled_readiness "$ws" -0.5)" 0 + assert_eq "score above 1 clamps to 100" "$(scaled_readiness "$ws" 1.5)" 100 +} + +test_risk_is_always_the_complement_of_readiness() { + local ws; ws="$(workspace)" + local raw readiness risk + for raw in 0 0.005 0.331 0.5 0.789 0.995 1; do + scaled_readiness "$ws" "$raw" >/dev/null + readiness="$(env_var "$ws" TRUSTABL_READINESS_SCORE)" + risk="$(env_var "$ws" TRUSTABL_RISK_SCORE)" + assert_eq "readiness+risk at $raw" "$(( readiness + risk ))" 100 + if [ "$readiness" -lt 0 ] || [ "$readiness" -gt 100 ]; then + fail "readiness $readiness out of range at overall_score $raw" + fi + done +} + # ---- run ---- it "a clean scan passes and reports a perfect readiness" test_clean_scan_passes @@ -174,5 +218,8 @@ it "DETECTORS, STRICT and RULES_REF reach the engine" test_scan_flags_ it "a tampered release aborts before the engine runs" test_a_tampered_release_aborts_before_the_engine_runs it "a pinned VERSION skips the latest-release lookup" test_a_pinned_version_skips_the_latest_lookup it "the branch label is reported" test_the_branch_label_is_reported +it "readiness scaling is correct across the range" test_readiness_scaling_is_correct_across_the_range +it "readiness is clamped to 0-100" test_readiness_is_clamped_to_0_100 +it "risk is always the complement of readiness" test_risk_is_always_the_complement_of_readiness summarize