diff --git a/cmd/clawscan/main.go b/cmd/clawscan/main.go index dfcf661..cefb420 100644 --- a/cmd/clawscan/main.go +++ b/cmd/clawscan/main.go @@ -260,7 +260,7 @@ func printProfileCatalog(w io.Writer, catalog profiles.ProfileCatalog, cwd strin if info.Profile.Judge != nil { judge = "configured" } - fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", info.ID, displayProfileSource(info.Source, cwd), strings.Join(info.Profile.Scanners, ", "), judge) + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", info.ID, displayProfileSource(info.Source, cwd), strings.Join(info.Profile.ScannerIDs(), ", "), judge) } _ = tw.Flush() } @@ -337,6 +337,15 @@ func printRunSummary(w io.Writer, result runner.RunTargetsResult, outputPath str fmt.Fprintf(w, "scanner_other: %d\n", summary.ScannerOther) } fmt.Fprintf(w, "issues_found: %d\n", summary.IssuesFound) + fmt.Fprintf(w, "gate: %s", summary.Gate) + if len(summary.GateRules) > 0 { + details := make([]string, 0, len(summary.GateRules)) + for _, rule := range summary.GateRules { + details = append(details, fmt.Sprintf("%s exit %d -> %s", rule.Scanner, rule.ExitCode, rule.Action)) + } + fmt.Fprintf(w, " (%s)", strings.Join(details, ", ")) + } + fmt.Fprintln(w) if summary.HasJudge { fmt.Fprintf(w, "judge_completed: %d\n", summary.JudgeCompleted) fmt.Fprintf(w, "judge_failed: %d\n", summary.JudgeFailed) @@ -404,6 +413,8 @@ type runSummary struct { ScannerSkipped int ScannerOther int IssuesFound int + Gate string + GateRules []runner.FiredGateRule HasJudge bool JudgeCompleted int JudgeFailed int @@ -415,7 +426,7 @@ type runSummary struct { } func summarizeRunTargets(result runner.RunTargetsResult) runSummary { - var summary runSummary + summary := runSummary{Gate: "pass"} if result.Batch != nil { summary.Profile = result.Batch.Profile summary.Profiles = result.Batch.Summary.ProfileCount @@ -436,6 +447,10 @@ func (summary *runSummary) addArtifact(artifact runner.Artifact) { summary.Profile = artifact.Profile } summary.Targets++ + summary.GateRules = append(summary.GateRules, artifact.GateRules...) + if artifact.Gate == "block" || (artifact.Gate == "warn" && summary.Gate == "pass") { + summary.Gate = artifact.Gate + } for _, result := range artifact.Scanners { switch result.Status { case "completed": diff --git a/cmd/clawscan/main_test.go b/cmd/clawscan/main_test.go index 37d2e2f..8dc3cd5 100644 --- a/cmd/clawscan/main_test.go +++ b/cmd/clawscan/main_test.go @@ -7,6 +7,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/openclaw/clawscan/internal/runner" ) func TestRunCommandPrintsHelp(t *testing.T) { @@ -511,6 +513,21 @@ func TestRunCommandWritesDefaultOutputAndPrintsKeyValueSummary(t *testing.T) { } } +func TestPrintRunSummaryIncludesGateVerdictAndFiredRule(t *testing.T) { + artifact := runner.Artifact{ + Gate: "block", + GateRules: []runner.FiredGateRule{ + {Scanner: "my-scanner", Rule: "blockOnExitCode", ExitCode: 3, Action: "block"}, + }, + Scanners: map[string]runner.ScannerResult{}, + } + var output strings.Builder + printRunSummary(&output, runner.RunTargetsResult{Single: &artifact}, "") + if !strings.Contains(output.String(), "gate: block (my-scanner exit 3 -> block)") { + t.Fatalf("summary missing gate rule:\n%s", output.String()) + } +} + func TestRunCommandJSONDoesNotWriteDefaultOutput(t *testing.T) { dir := t.TempDir() target := filepath.Join(dir, "skill") diff --git a/docs/scanners.md b/docs/scanners.md index 8666a87..85bbd6b 100644 --- a/docs/scanners.md +++ b/docs/scanners.md @@ -17,6 +17,74 @@ clawscan scanners clawscan scanners skillspector ``` +## User-defined scanners + +A trusted config can mix built-in scanner IDs with user-defined command +scanners. The config schema uses the existing `profiles..scanners` list: + +```yaml +version: 1 + +profiles: + review: + scanners: + - clawscan-static + - id: my-scanner + command: my-scanner --json {{target}} + env: + - MY_SCANNER_TOKEN + targets: + - skill + - plugin + gate: + blockOnExitCode: nonzero +``` + +String entries select built-in scanners. Object entries define a scanner for +that config-backed run and accept these fields: + +| Field | Required | Meaning | +| --- | --- | --- | +| `id` | yes | Scanner ID using letters, digits, `_`, and `-`, starting with a letter or digit. It must not match a built-in scanner ID. | +| `command` | yes | Shell command to execute. Unquoted `{{target}}` is replaced with the safely passed resolved target; do not wrap the placeholder in shell quotes. | +| `env` | no | Required environment variable names. Values stay in the process environment and are never stored in the config or artifact. | +| `targets` | no | Supported target kinds: `skill`, `plugin`, and/or `url`. Defaults to `skill` and `url`. | +| `gate` | no | Exit-code policy with optional `blockOnExitCode` and `warnOnExitCode` rules. | + +Each exit-code rule accepts one non-negative integer, a list such as `[1, 2, +3]`, or the string `nonzero`. The block and warning rules may not claim the +same exit code. For example: + +```yaml +gate: + blockOnExitCode: [2, 3] + warnOnExitCode: 1 +``` + +After every selected scanner finishes, ClawScan records the strongest fired +action as the top-level artifact `gate`: `block` wins over `warn`, and an +artifact with no fired rules records `"gate": "pass"`. Each fired rule is also +listed in `gateRules` with its scanner ID, rule name, exit code, and action. +Skipped scanners do not fire gate rules. A scanner result with status `failed` +also does not fire an exit-code rule; a nonzero command that still returned +valid JSON has status `completed` and can fire one. + +The command must write JSON to stdout. ClawScan preserves valid stdout as the +scanner's raw evidence; empty or non-JSON stdout produces a failed scanner +result. Required environment variables are checked before any scanner starts. +Artifacts record each requirement as only `present` or `missing`. + +User-defined scanners use the same execution path as built-in command-backed +scanners. They run in the Docker sandbox by default, and declared `env` names +are added to its environment allowlist. Use `--sandbox off` only when you +intentionally want the command to run on the host. User-defined scanners are +local to the resolved config and do not appear in the built-in `clawscan +scanners` catalog. + +> **Trust boundary:** only load user-defined scanners from config files you +> control. A scanner entry is executable code. The default sandbox limits its +> host access, but does not make an untrusted command safe to run. + ## Target kinds Clawscan classifies each explicit target before dispatching scanners and records diff --git a/internal/profiles/registry.go b/internal/profiles/registry.go index 1ed0721..0bcaa56 100644 --- a/internal/profiles/registry.go +++ b/internal/profiles/registry.go @@ -28,8 +28,8 @@ func NewProfileRegistry(profiles map[string]resolvedProfile) (ProfileRegistry, e return ProfileRegistry{}, err } for _, scanner := range profile.profile.Scanners { - if !runner.DefaultScannerRegistry().Contains(scanner) { - return ProfileRegistry{}, unknownScannerInProfileError(id, scanner) + if !scanner.custom && !runner.DefaultScannerRegistry().Contains(scanner.ID) { + return ProfileRegistry{}, unknownScannerInProfileError(id, scanner.ID) } } registry.profiles[id] = profile diff --git a/internal/profiles/registry_test.go b/internal/profiles/registry_test.go index 24e5714..bcfc0a6 100644 --- a/internal/profiles/registry_test.go +++ b/internal/profiles/registry_test.go @@ -7,8 +7,8 @@ import ( func TestProfileRegistryReturnsSortedIDs(t *testing.T) { registry, err := NewProfileRegistry(map[string]resolvedProfile{ - "review": {profile: Profile{Scanners: []string{"snyk"}}}, - "clawhub": {profile: Profile{Scanners: []string{"skillspector"}}}, + "review": {profile: Profile{Scanners: []ProfileScanner{{ID: "snyk"}}}}, + "clawhub": {profile: Profile{Scanners: []ProfileScanner{{ID: "skillspector"}}}}, }) if err != nil { t.Fatal(err) @@ -26,7 +26,7 @@ func TestDefaultProfileRegistryContainsEmbeddedBuiltIns(t *testing.T) { if !ok { t.Fatal("missing clawhub profile") } - if got := strings.Join(clawhub.profile.Scanners, ","); got != "skillspector,clawscan-static" { + if got := strings.Join(profileScannerIDs(clawhub.profile.Scanners), ","); got != "skillspector,clawscan-static" { t.Fatalf("clawhub scanners = %q", got) } if clawhub.configDir != "clawhub" { @@ -46,7 +46,7 @@ func TestDefaultProfileRegistryContainsEmbeddedBuiltIns(t *testing.T) { if !ok { t.Fatal("missing clawhub-aig profile") } - if got := strings.Join(candidate.profile.Scanners, ","); got != "skillspector,aig" { + if got := strings.Join(profileScannerIDs(candidate.profile.Scanners), ","); got != "skillspector,aig" { t.Fatalf("clawhub-aig scanners = %q", got) } if candidate.configDir != "clawhub" { @@ -71,7 +71,7 @@ func TestDefaultProfileRegistryContainsEmbeddedBuiltIns(t *testing.T) { func TestProfileRegistryRejectsUnknownScannerReferences(t *testing.T) { _, err := NewProfileRegistry(map[string]resolvedProfile{ - "bad": {profile: Profile{Scanners: []string{"missing-scanner"}}}, + "bad": {profile: Profile{Scanners: []ProfileScanner{{ID: "missing-scanner"}}}}, }) if err == nil || err.Error() != "Profile bad references unknown scanner: missing-scanner" { t.Fatalf("err = %v", err) @@ -91,7 +91,7 @@ func TestInspectProfilesReturnsBuiltIns(t *testing.T) { if !ok { t.Fatal("missing clawhub profile") } - if got := strings.Join(clawhub.Profile.Scanners, ","); got != "skillspector,clawscan-static" { + if got := strings.Join(profileScannerIDs(clawhub.Profile.Scanners), ","); got != "skillspector,clawscan-static" { t.Fatalf("clawhub scanners = %q", got) } if clawhub.Source != "built-in" { diff --git a/internal/profiles/resolver.go b/internal/profiles/resolver.go index 9afadc5..462113e 100644 --- a/internal/profiles/resolver.go +++ b/internal/profiles/resolver.go @@ -30,7 +30,7 @@ type Config struct { } type Profile struct { - Scanners []string `yaml:"scanners"` + Scanners []ProfileScanner `yaml:"scanners"` ScannerResults map[string]string `yaml:"scannerResults,omitempty"` Output string `yaml:"output,omitempty"` JSON bool `yaml:"json,omitempty"` @@ -38,6 +38,199 @@ type Profile struct { Judge *Judge `yaml:"judge,omitempty"` } +func (profile Profile) ScannerIDs() []string { + return profileScannerIDs(profile.Scanners) +} + +type ProfileScanner struct { + ID string + Command string + Env []string + Targets []string + Gate *ProfileScannerGate + custom bool +} + +type ProfileScannerGate struct { + BlockOnExitCode *profileExitCodeRule `yaml:"blockOnExitCode,omitempty"` + WarnOnExitCode *profileExitCodeRule `yaml:"warnOnExitCode,omitempty"` +} + +type profileExitCodeRule struct { + Codes []int + Nonzero bool +} + +func (rule *profileExitCodeRule) UnmarshalYAML(node *yaml.Node) error { + switch node.Kind { + case yaml.ScalarNode: + if node.Tag == "!!str" && node.Value == "nonzero" { + rule.Nonzero = true + return nil + } + if node.Tag == "!!int" { + var code int + if err := node.Decode(&code); err == nil && code >= 0 { + rule.Codes = []int{code} + return nil + } + return errors.New("exit-code gate rule must contain only non-negative integers") + } + case yaml.SequenceNode: + if len(node.Content) == 0 { + return errors.New("exit-code gate rule must not be an empty list") + } + codes := make([]int, 0, len(node.Content)) + for _, item := range node.Content { + if item.Kind != yaml.ScalarNode || item.Tag != "!!int" { + return errors.New("exit-code gate rule must contain only non-negative integers") + } + var code int + if err := item.Decode(&code); err != nil || code < 0 { + return errors.New("exit-code gate rule must contain only non-negative integers") + } + codes = append(codes, code) + } + rule.Codes = codes + return nil + } + return errors.New(`exit-code gate rule must be a non-negative integer, a list of non-negative integers, or "nonzero"`) +} + +func (rule profileExitCodeRule) MarshalYAML() (interface{}, error) { + if rule.Nonzero { + return "nonzero", nil + } + switch len(rule.Codes) { + case 0: + return nil, errors.New("exit-code gate rule must include at least one exit code") + case 1: + return rule.Codes[0], nil + default: + return append([]int(nil), rule.Codes...), nil + } +} + +func (gate *ProfileScannerGate) UnmarshalYAML(node *yaml.Node) error { + if node.Kind != yaml.MappingNode { + return errors.New("scanner gate must be an object") + } + for index := 0; index < len(node.Content); index += 2 { + switch node.Content[index].Value { + case "blockOnExitCode", "warnOnExitCode": + default: + return fmt.Errorf("field %s not found in type profiles.ProfileScannerGate", node.Content[index].Value) + } + } + type plainGate ProfileScannerGate + return node.Decode((*plainGate)(gate)) +} + +func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error { + switch node.Kind { + case yaml.ScalarNode: + if err := node.Decode(&scanner.ID); err != nil { + return err + } + return nil + case yaml.MappingNode: + for index := 0; index < len(node.Content); index += 2 { + switch node.Content[index].Value { + case "id", "command", "env", "targets", "gate": + default: + return fmt.Errorf("field %s not found in type profiles.ProfileScanner", node.Content[index].Value) + } + } + var value struct { + ID string `yaml:"id"` + Command string `yaml:"command"` + Env []string `yaml:"env,omitempty"` + Targets []string `yaml:"targets,omitempty"` + Gate *ProfileScannerGate `yaml:"gate,omitempty"` + } + if err := node.Decode(&value); err != nil { + return err + } + scanner.ID = value.ID + scanner.Command = value.Command + scanner.Env = value.Env + scanner.Targets = value.Targets + scanner.Gate = value.Gate + scanner.custom = true + return nil + default: + return fmt.Errorf("scanner entry must be a string or object") + } +} + +func (scanner ProfileScanner) MarshalYAML() (interface{}, error) { + if !scanner.custom { + return scanner.ID, nil + } + return struct { + ID string `yaml:"id"` + Command string `yaml:"command"` + Env []string `yaml:"env,omitempty"` + Targets []string `yaml:"targets,omitempty"` + Gate *ProfileScannerGate `yaml:"gate,omitempty"` + }{scanner.ID, scanner.Command, scanner.Env, scanner.Targets, scanner.Gate}, nil +} + +func profileScannerIDs(scanners []ProfileScanner) []string { + ids := make([]string, 0, len(scanners)) + for _, scanner := range scanners { + ids = append(ids, scanner.ID) + } + return ids +} + +func profileScannerRegistry(scanners []ProfileScanner) (runner.ScannerRegistry, error) { + registry := runner.DefaultScannerRegistry() + for _, scanner := range scanners { + if !scanner.custom { + continue + } + targets := append([]string(nil), scanner.Targets...) + if len(targets) == 0 { + targets = []string{"skill", "url"} + } + adapter := runner.NewUserDefinedScanner(runner.UserDefinedScannerConfig{ + ID: scanner.ID, Command: scanner.Command, Env: scanner.Env, Targets: targets, + }) + var err error + registry, err = registry.WithAdapters(adapter) + if err != nil { + return runner.ScannerRegistry{}, err + } + } + return registry, nil +} + +func profileGateRules(scanners []ProfileScanner) map[string]runner.ScannerGatePolicy { + rules := map[string]runner.ScannerGatePolicy{} + for _, scanner := range scanners { + if scanner.Gate == nil { + continue + } + policy := runner.ScannerGatePolicy{} + if scanner.Gate.BlockOnExitCode != nil { + policy.BlockOnExitCode = &runner.ExitCodeRule{ + Codes: append([]int(nil), scanner.Gate.BlockOnExitCode.Codes...), Nonzero: scanner.Gate.BlockOnExitCode.Nonzero, + } + } + if scanner.Gate.WarnOnExitCode != nil { + policy.WarnOnExitCode = &runner.ExitCodeRule{ + Codes: append([]int(nil), scanner.Gate.WarnOnExitCode.Codes...), Nonzero: scanner.Gate.WarnOnExitCode.Nonzero, + } + } + if policy.BlockOnExitCode == nil && policy.WarnOnExitCode == nil { + continue + } + rules[scanner.ID] = policy + } + return rules +} + type Sandbox struct { Mode string `yaml:"mode,omitempty"` Image string `yaml:"image,omitempty"` @@ -91,6 +284,16 @@ type cliIntent struct { var judgePathPlaceholderPattern = regexp.MustCompile(`\{\{\s*(prompt|output_schema):([^}]+)\}\}`) +// scannerIDPattern restricts user-defined scanner IDs to lowercase. IDs are +// lowercased when used as evidence file names, so allowing uppercase would let +// two case-distinct IDs (Foo and foo) collide on the same output file. +var scannerIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`) +var scannerTargetPlaceholderPattern = regexp.MustCompile(`\{\{\s*target\s*\}\}`) + +// maxScannerIDLength bounds user-defined scanner IDs so .json evidence file +// names stay within filesystem limits. +const maxScannerIDLength = 64 + type ResolvedRunSet struct { Options []runner.Options OutputPath string @@ -182,11 +385,16 @@ func resolveRunSetIntent(intent cliIntent, cwd string) (ResolvedRunSet, error) { if err != nil { return ResolvedRunSet{}, err } - opts, err := runner.ParseArgs(finalArgs) + scannerRegistry, err := profileScannerRegistry(selected.profile.Scanners) + if err != nil { + return ResolvedRunSet{}, err + } + opts, err := runner.ParseArgsWithRegistry(finalArgs, scannerRegistry) if err != nil { return ResolvedRunSet{}, err } opts.Profile = profileName + opts.GateRules = profileGateRules(selected.profile.Scanners) opts.ConfigSource = configSource opts.DiscoverConfig = intent.discoverConfig if opts.Judge != nil { @@ -279,11 +487,16 @@ func resolveAllConfigProfiles(intent cliIntent, cwd string) (ResolvedRunSet, err if err != nil { return ResolvedRunSet{}, err } - opts, err := runner.ParseArgs(finalArgs) + scannerRegistry, err := profileScannerRegistry(selected.profile.Scanners) + if err != nil { + return ResolvedRunSet{}, err + } + opts, err := runner.ParseArgsWithRegistry(finalArgs, scannerRegistry) if err != nil { return ResolvedRunSet{}, err } opts.Profile = profileName + opts.GateRules = profileGateRules(selected.profile.Scanners) opts.ConfigSource = filepath.Clean(projectPath) opts.OutputPath = "" opts.JSON = false @@ -610,7 +823,7 @@ func buildRunnerArgs(intent cliIntent, selected resolvedProfile, profileName str } profile := selected.profile - scanners := append([]string{}, profile.Scanners...) + scanners := profileScannerIDs(profile.Scanners) if len(intent.scanners) > 0 { scanners = append([]string{}, intent.scanners...) } @@ -716,14 +929,137 @@ func resolveJudgePaths(command string, configDir string) string { func validateProfile(name string, profile Profile) error { seen := map[string]bool{} for _, scanner := range profile.Scanners { - if seen[scanner] { - return fmt.Errorf("Duplicate scanner in profile %s: %s", name, scanner) + if scanner.custom && strings.TrimSpace(scanner.ID) == "" { + return fmt.Errorf("User-defined scanner in profile %s must include a non-empty id", name) + } + if scanner.custom && !scannerIDPattern.MatchString(scanner.ID) { + return fmt.Errorf("User-defined scanner %s in profile %s has invalid id; use lowercase letters, digits, underscores, and hyphens, starting with a letter or digit", scanner.ID, name) + } + if scanner.custom && len(scanner.ID) > maxScannerIDLength { + return fmt.Errorf("User-defined scanner id in profile %s is %d characters; scanner IDs are used as file names and must be at most %d characters", name, len(scanner.ID), maxScannerIDLength) } - seen[scanner] = true + if scanner.custom && strings.TrimSpace(scanner.Command) == "" { + return fmt.Errorf("User-defined scanner %s in profile %s must include a non-empty command", scanner.ID, name) + } + if scanner.custom && !scannerTargetPlaceholdersAreUnquoted(scanner.Command) { + return fmt.Errorf("User-defined scanner %s in profile %s must use {{target}} outside shell quotes", scanner.ID, name) + } + if scanner.custom && runner.DefaultScannerRegistry().Contains(scanner.ID) { + return fmt.Errorf("User-defined scanner %s collides with a built-in scanner ID", scanner.ID) + } + if scanner.Gate != nil { + if code, overlaps := overlappingExitCodeRules(scanner.Gate.BlockOnExitCode, scanner.Gate.WarnOnExitCode); overlaps { + return fmt.Errorf("User-defined scanner %s in profile %s gate blockOnExitCode and warnOnExitCode both claim exit code %d", scanner.ID, name, code) + } + } + for _, target := range scanner.Targets { + switch target { + case "skill", "plugin", "url": + default: + return fmt.Errorf("User-defined scanner %s in profile %s has unsupported target kind: %s", scanner.ID, name, target) + } + } + if seen[scanner.ID] { + return fmt.Errorf("Duplicate scanner in profile %s: %s", name, scanner.ID) + } + seen[scanner.ID] = true } return nil } +func overlappingExitCodeRules(block *profileExitCodeRule, warn *profileExitCodeRule) (int, bool) { + if block == nil || warn == nil { + return 0, false + } + if block.Nonzero && warn.Nonzero { + return 1, true + } + if block.Nonzero { + for _, code := range warn.Codes { + if code != 0 { + return code, true + } + } + return 0, false + } + if warn.Nonzero { + for _, code := range block.Codes { + if code != 0 { + return code, true + } + } + return 0, false + } + warnCodes := make(map[int]bool, len(warn.Codes)) + for _, code := range warn.Codes { + warnCodes[code] = true + } + for _, code := range block.Codes { + if warnCodes[code] { + return code, true + } + } + return 0, false +} + +func scannerTargetPlaceholdersAreUnquoted(command string) bool { + matches := scannerTargetPlaceholderPattern.FindAllStringIndex(command, -1) + matchIndex := 0 + quote := byte(0) + escaped := false + comment := false + for index := 0; index < len(command); index++ { + if matchIndex < len(matches) && index == matches[matchIndex][0] { + if !comment && quote != 0 { + return false + } + index = matches[matchIndex][1] - 1 + matchIndex++ + continue + } + character := command[index] + if comment { + if character == '\n' { + comment = false + } + continue + } + if escaped { + escaped = false + continue + } + if character == '\\' && quote != '\'' { + escaped = true + continue + } + if quote == 0 { + if character == '#' && shellCommentCanStart(command, index) { + comment = true + continue + } + switch character { + case '\'', '"', '`': + quote = character + } + } else if character == quote { + quote = 0 + } + } + return true +} + +func shellCommentCanStart(command string, index int) bool { + if index == 0 { + return true + } + switch command[index-1] { + case ' ', '\t', '\r', '\n', ';', '|', '&', '(', ')': + return true + default: + return false + } +} + func unknownProfileError(profile string, available []string) error { return fmt.Errorf("Unknown profile: %s (available: %s)", profile, strings.Join(available, ", ")) } diff --git a/internal/profiles/resolver_test.go b/internal/profiles/resolver_test.go index 55e3b48..afccde4 100644 --- a/internal/profiles/resolver_test.go +++ b/internal/profiles/resolver_test.go @@ -1,12 +1,17 @@ package profiles import ( + "bytes" + "encoding/json" "os" "path/filepath" + "reflect" "strings" "testing" + "time" "github.com/openclaw/clawscan/internal/runner" + "gopkg.in/yaml.v3" ) func TestResolveArgsUsesEmbeddedClawHubProfile(t *testing.T) { @@ -572,6 +577,12 @@ func TestResolveArgsRejectsUnsupportedVersionAndUnknownFields(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "field defaultProfile not found") { t.Fatalf("err = %v", err) } + + writeFile(t, config, "version: 1\nprofiles:\n review:\n scanners:\n - id: custom\n command: custom {{target}}\n token: example\n") + _, err = ResolveArgs([]string{"./skill", "--config", config}, dir) + if err == nil || !strings.Contains(err.Error(), "field token not found") { + t.Fatalf("err = %v", err) + } } func TestResolveArgsUnknownProfileListsAvailableProfiles(t *testing.T) { @@ -738,6 +749,519 @@ profiles: } } +func TestResolveArgsParsesUserDefinedScannerAlongsideBuiltIn(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - clawscan-static + - id: my-scanner + command: my-scanner --json {{target}} + env: + - MY_SCANNER_TOKEN + targets: + - plugin +`) + + opts, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(opts.Scanners, ","); got != "clawscan-static,my-scanner" { + t.Fatalf("scanners = %q", got) + } + adapter, ok := opts.ScannerRegistry.Adapter("my-scanner") + if !ok { + t.Fatal("custom scanner missing from run registry") + } + if adapter.SupportsTargetKind("skill") { + t.Fatal("plugin-only custom scanner unexpectedly supports skill targets") + } + if !adapter.SupportsTargetKind("plugin") { + t.Fatal("plugin-only custom scanner does not support plugin targets") + } +} + +func TestResolveArgsParsesUserDefinedScannerExitCodeGateRules(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: blocker + command: blocker {{target}} + gate: + blockOnExitCode: nonzero + - id: warner + command: warner {{target}} + gate: + warnOnExitCode: [1, 2, 3] +`) + + opts, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + block := opts.GateRules["blocker"].BlockOnExitCode + if block == nil || !block.Nonzero { + t.Fatalf("block rule = %#v", block) + } + warn := opts.GateRules["warner"].WarnOnExitCode + if warn == nil || !reflect.DeepEqual(warn.Codes, []int{1, 2, 3}) { + t.Fatalf("warn rule = %#v", warn) + } +} + +func TestResolveArgsAcceptsSingleExitCodeGateRule(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: blocker + command: blocker {{target}} + gate: + blockOnExitCode: 7 +`) + opts, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + if got := opts.GateRules["blocker"].BlockOnExitCode.Codes; !reflect.DeepEqual(got, []int{7}) { + t.Fatalf("codes = %#v", got) + } +} + +func TestProfileScannerExitCodeGateRulesRoundTripYAML(t *testing.T) { + scanner := ProfileScanner{ + ID: "demo", Command: "demo {{target}}", custom: true, + Gate: &ProfileScannerGate{ + BlockOnExitCode: &profileExitCodeRule{Nonzero: true}, + WarnOnExitCode: &profileExitCodeRule{Codes: []int{1, 2, 3}}, + }, + } + encoded, err := yaml.Marshal(scanner) + if err != nil { + t.Fatal(err) + } + var decoded ProfileScanner + if err := yaml.Unmarshal(encoded, &decoded); err != nil { + t.Fatalf("round trip failed for %s: %v", encoded, err) + } + if decoded.Gate == nil || !decoded.Gate.BlockOnExitCode.Nonzero || !reflect.DeepEqual(decoded.Gate.WarnOnExitCode.Codes, []int{1, 2, 3}) { + t.Fatalf("decoded scanner = %#v from %s", decoded, encoded) + } +} + +func TestResolveArgsRejectsInvalidExitCodeGateRules(t *testing.T) { + tests := []struct { + name string + rules string + want string + }{ + {name: "negative", rules: "blockOnExitCode: -1", want: "must contain only non-negative integers"}, + {name: "non integer", rules: "blockOnExitCode: nope", want: `must be a non-negative integer, a list of non-negative integers, or "nonzero"`}, + {name: "empty list", rules: "blockOnExitCode: []", want: "must not be an empty list"}, + {name: "overlap", rules: "blockOnExitCode: nonzero\n warnOnExitCode: [0, 2]", want: "blockOnExitCode and warnOnExitCode both claim exit code 2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, "version: 1\nprofiles:\n review:\n scanners:\n - id: demo\n command: demo {{target}}\n gate:\n "+test.rules+"\n") + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("err = %v", err) + } + }) + } +} + +func TestGateRuleForProfileScannerExcludedByCLIOverrideFailsBeforeScanning(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: absent-scanner + command: absent {{target}} + gate: + blockOnExitCode: nonzero +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review", "--scanner", "clawscan-static", "--sandbox", "off"}, dir) + if err != nil { + t.Fatal(err) + } + commandRunner := &profileCommandRunner{stdout: `{}`} + _, err = runner.Run(opts, runner.RunContext{Env: map[string]string{}, CommandRunner: commandRunner}) + if err == nil || err.Error() != "gate rule references scanner absent-scanner, but it was not requested" { + t.Fatalf("err = %v", err) + } + if commandRunner.command != "" { + t.Fatalf("scanner executed: %q", commandRunner.command) + } +} + +func TestResolveArgsRejectsUserDefinedScannerIDCollision(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: clawscan-static + command: custom-static {{target}} +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner clawscan-static collides with a built-in scanner ID" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsRejectsIncompleteUserDefinedScanner(t *testing.T) { + for _, test := range []struct { + name string + entry string + wantErr string + }{ + {name: "missing id", entry: "command: scanner {{target}}", wantErr: "User-defined scanner in profile review must include a non-empty id"}, + {name: "missing command", entry: "id: my-scanner", wantErr: "User-defined scanner my-scanner in profile review must include a non-empty command"}, + } { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, "version: 1\nprofiles:\n review:\n scanners:\n - "+test.entry+"\n") + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != test.wantErr { + t.Fatalf("err = %v", err) + } + }) + } +} + +func TestResolveArgsRejectsUnknownUserDefinedScannerTarget(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: my-scanner + command: my-scanner {{target}} + targets: + - package +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner my-scanner in profile review has unsupported target kind: package" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsRejectsInvalidUserDefinedScannerID(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: foo=bar + command: scanner {{target}} +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner foo=bar in profile review has invalid id; use lowercase letters, digits, underscores, and hyphens, starting with a letter or digit" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsRejectsUppercaseUserDefinedScannerID(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: Foo + command: scanner {{target}} +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner Foo in profile review has invalid id; use lowercase letters, digits, underscores, and hyphens, starting with a letter or digit" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsRejectsOversizedUserDefinedScannerID(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + longID := strings.Repeat("a", 65) + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: `+longID+` + command: scanner {{target}} +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner id in profile review is 65 characters; scanner IDs are used as file names and must be at most 64 characters" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsRejectsQuotedUserDefinedScannerTargetPlaceholder(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: my-scanner + command: my-scanner "{{target}}" +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner my-scanner in profile review must use {{target}} outside shell quotes" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsAllowsApostropheInShellCommentBeforeTargetPlaceholder(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: my-scanner + command: |- + # don't quote the target here + my-scanner {{target}} +`) + + if _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir); err != nil { + t.Fatal(err) + } +} + +func TestResolveArgsSupportsAliasedScannerEntries(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + templates: + scanners: + - &builtin clawscan-static + - &custom + id: my-scanner + command: my-scanner {{target}} + review: + scanners: + - *builtin + - *custom +`) + + opts, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(opts.Scanners, ","); got != "clawscan-static,my-scanner" { + t.Fatalf("scanners = %q", got) + } +} + +func TestUserDefinedScannerRunsThroughDockerAndPreservesRawJSON(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: fixture-scanner + command: fixture-scan --json {{target}} +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + adapter, ok := opts.ScannerRegistry.Adapter("fixture-scanner") + if !ok || !adapter.SupportsTargetKind("skill") || !adapter.SupportsTargetKind("url") || adapter.SupportsTargetKind("plugin") { + t.Fatalf("default target support is incorrect: adapter=%#v ok=%v", adapter, ok) + } + commandRunner := &profileCommandRunner{stdout: `{"scanner":"fixture","findings":[]}`} + artifact, err := runner.Run(opts, runner.RunContext{ + Env: map[string]string{}, + HostCommandRunner: commandRunner, + DockerAvailability: func() error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + result := artifact.Scanners["fixture-scanner"] + if result.Status != "completed" { + t.Fatalf("result = %#v", result) + } + if !bytes.Equal(result.Raw, []byte(`{"scanner":"fixture","findings":[]}`)) { + t.Fatalf("raw = %s", result.Raw) + } + if commandRunner.command != "docker" { + t.Fatalf("command = %q", commandRunner.command) + } + joined := strings.Join(commandRunner.args, " ") + if !strings.Contains(joined, "fixture-scan --json") || !strings.Contains(joined, target) || strings.Contains(joined, "{{target}}") { + t.Fatalf("docker args = %#v", commandRunner.args) + } +} + +func TestUserDefinedScannerMissingEnvFailsBeforeExecution(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: credentialed-scanner + command: credentialed-scan {{target}} + env: + - MY_SCANNER_TOKEN +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + commandRunner := &profileCommandRunner{stdout: `{}`} + _, err = runner.Run(opts, runner.RunContext{ + Env: map[string]string{}, + HostCommandRunner: commandRunner, + DockerAvailability: func() error { return nil }, + }) + if err == nil || !strings.Contains(err.Error(), "MY_SCANNER_TOKEN") { + t.Fatalf("err = %v", err) + } + if commandRunner.command != "" { + t.Fatalf("scanner executed before env validation: %q %#v", commandRunner.command, commandRunner.args) + } +} + +func TestUserDefinedScannerEnvIsAllowlistedAndRecordedByPresenceOnly(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: credentialed-scanner + command: credentialed-scan {{target}} + env: + - MY_SCANNER_TOKEN +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + missingArtifact := runner.NewArtifact(opts, target, "start", "complete", map[string]string{}) + if missingArtifact.Env["MY_SCANNER_TOKEN"] != "missing" { + t.Fatalf("missing env = %#v", missingArtifact.Env) + } + + const envValue = "test-value-123" + commandRunner := &profileCommandRunner{stdout: `{}`} + artifact, err := runner.Run(opts, runner.RunContext{ + Env: map[string]string{"MY_SCANNER_TOKEN": envValue}, + HostCommandRunner: commandRunner, + DockerAvailability: func() error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + if artifact.Env["MY_SCANNER_TOKEN"] != "present" { + t.Fatalf("env = %#v", artifact.Env) + } + encoded, err := json.Marshal(artifact) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(encoded, []byte(envValue)) { + t.Fatalf("artifact leaked env value: %s", encoded) + } + joined := strings.Join(commandRunner.args, "\x00") + if !strings.Contains(joined, "\x00-e\x00MY_SCANNER_TOKEN\x00") { + t.Fatalf("docker args missing env allowlist name: %#v", commandRunner.args) + } + if strings.Contains(joined, envValue) { + t.Fatalf("docker args leaked env value: %#v", commandRunner.args) + } +} + +func TestUserDefinedPluginScannerSkipsSkillTarget(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: plugin-scanner + command: plugin-scan {{target}} + env: + - PLUGIN_SCANNER_TOKEN + targets: + - plugin + gate: + blockOnExitCode: nonzero +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + commandRunner := &profileCommandRunner{stdout: `{}`} + artifact, err := runner.Run(opts, runner.RunContext{ + Env: map[string]string{}, + HostCommandRunner: commandRunner, + DockerAvailability: func() error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + result := artifact.Scanners["plugin-scanner"] + if result.Status != "skipped" || !strings.Contains(result.Error, "does not support skill targets") { + t.Fatalf("result = %#v", result) + } + if artifact.Gate != "pass" || len(artifact.GateRules) != 0 { + t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules) + } + if commandRunner.command != "" { + t.Fatalf("unsupported scanner executed: %q %#v", commandRunner.command, commandRunner.args) + } +} + func TestResolveArgsRejectsProfileWithoutScannersUnlessCLIOverrides(t *testing.T) { dir := t.TempDir() config := filepath.Join(dir, ".clawscan.yml") @@ -770,3 +1294,15 @@ func writeFile(t *testing.T, path string, content string) { t.Fatal(err) } } + +type profileCommandRunner struct { + command string + args []string + stdout string +} + +func (commandRunner *profileCommandRunner) Run(command string, args []string, _ string, _ time.Duration) (runner.CommandOutput, error) { + commandRunner.command = command + commandRunner.args = append([]string(nil), args...) + return runner.CommandOutput{Stdout: commandRunner.stdout}, nil +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c8b735a..54926d8 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -33,11 +33,35 @@ type Options struct { ContextPath string Benchmark *BenchmarkOptions Scanners []string + ScannerRegistry ScannerRegistry ScannerResultPaths map[string]string OutputPath string JSON bool Judge *JudgeOptions Sandbox SandboxOptions + GateRules map[string]ScannerGatePolicy +} + +type ExitCodeRule struct { + Codes []int + Nonzero bool +} + +func (rule ExitCodeRule) Matches(exitCode int) bool { + if rule.Nonzero { + return exitCode != 0 + } + for _, code := range rule.Codes { + if code == exitCode { + return true + } + } + return false +} + +type ScannerGatePolicy struct { + BlockOnExitCode *ExitCodeRule + WarnOnExitCode *ExitCodeRule } type BenchmarkOptions struct { @@ -82,8 +106,9 @@ type CommandRunner interface { } type CommandOutput struct { - Stdout string - Stderr string + Stdout string + Stderr string + ExitCode *int } type Artifact struct { @@ -99,9 +124,18 @@ type Artifact struct { Env map[string]string `json:"env"` Sandbox SandboxMetadata `json:"sandbox"` Scanners map[string]ScannerResult `json:"scanners"` + Gate string `json:"gate"` + GateRules []FiredGateRule `json:"gateRules"` Judge *JudgeResult `json:"judge"` } +type FiredGateRule struct { + Scanner string `json:"scanner"` + Rule string `json:"rule"` + ExitCode int `json:"exitCode"` + Action string `json:"action"` +} + type RunTargetsResult struct { Single *Artifact Batch *BatchArtifact @@ -158,6 +192,7 @@ type ScannerResult struct { Command []string `json:"command"` Error string `json:"error"` OutputPath string `json:"outputPath,omitempty"` + ExitCode *int `json:"exitCode,omitempty"` Raw json.RawMessage `json:"raw"` } @@ -236,7 +271,12 @@ func NewBenchmarkOptions(id string, split string, limit int, offset int, predict } func ParseArgs(args []string) (Options, error) { + return ParseArgsWithRegistry(args, DefaultScannerRegistry()) +} + +func ParseArgsWithRegistry(args []string, registry ScannerRegistry) (Options, error) { opts := Options{ScannerResultPaths: map[string]string{}} + opts.ScannerRegistry = registry start := 0 if len(args) > 0 && !strings.HasPrefix(args[0], "--") { opts.Target = args[0] @@ -251,7 +291,7 @@ func ParseArgs(args []string) (Options, error) { if err != nil { return Options{}, err } - if !DefaultScannerRegistry().Contains(value) { + if !registry.Contains(value) { return Options{}, fmt.Errorf("Unknown scanner: %s", value) } opts.Scanners = append(opts.Scanners, value) @@ -279,7 +319,7 @@ func ParseArgs(args []string) (Options, error) { if !ok || scanner == "" || path == "" { return Options{}, errors.New("Expected --scanner-result value as scanner=path") } - if !DefaultScannerRegistry().Contains(scanner) { + if !registry.Contains(scanner) { return Options{}, fmt.Errorf("Unknown scanner: %s", scanner) } opts.ScannerResultPaths[scanner] = path @@ -365,6 +405,9 @@ func ValidateRequirements(opts Options, env map[string]string) error { } func Run(opts Options, ctx RunContext) (Artifact, error) { + if err := validateGateRuleScanners(opts); err != nil { + return Artifact{}, err + } env := ctx.Env if env == nil { env = EnvMap(os.Environ()) @@ -403,6 +446,7 @@ func Run(opts Options, ctx RunContext) (Artifact, error) { Profile: opts.Profile, TargetKind: target.kind, TargetID: target.id, + Registry: registryForOptions(opts), SandboxMode: scannerSandboxMode, SkillSpectorCommand: ctx.SkillSpectorCommand, VirusTotalHTTPClient: ctx.VirusTotalHTTPClient, @@ -433,6 +477,7 @@ func Run(opts Options, ctx RunContext) (Artifact, error) { result.DurationMs = time.Since(scannerTimerStarted).Milliseconds() artifact.Scanners[scanner] = result } + evaluateGate(&artifact, opts) if opts.Judge != nil { result, err := RunJudge(*opts.Judge, artifact, commandRunner, 20*time.Minute, env, sandbox.Mode) if err != nil { @@ -826,7 +871,7 @@ func scannerResult(opts Options, scanner string, target resolvedTarget, startedA Raw: json.RawMessage(raw), }, nil } - if !scannerSupportsTargetKind(scanner, target.kind) { + if !scannerSupportsTargetKindInRegistry(registryForOptions(opts), scanner, target.kind) { return unsupportedTargetKindResult(scanner, target.kind, startedAt), nil } return scannerRunner.RunScanner(scanner, target.resolvedPath, startedAt) @@ -2104,7 +2149,14 @@ func (runner defaultCommandRunner) Run(command string, args []string, cwd string if ctx.Err() == context.DeadlineExceeded { err = fmt.Errorf("command timed out after %s", timeout) } - return CommandOutput{Stdout: stdout.String(), Stderr: stderr.String()}, err + output := CommandOutput{Stdout: stdout.String(), Stderr: stderr.String()} + if cmd.ProcessState != nil { + exitCode := cmd.ProcessState.ExitCode() + if exitCode >= 0 { + output.ExitCode = &exitCode + } + } + return output, err } func NewArtifact(opts Options, resolvedPath string, startedAt string, completedAt string, env map[string]string) Artifact { @@ -2138,10 +2190,54 @@ func NewArtifact(opts Options, resolvedPath string, startedAt string, completedA Env: envPresence(opts, env), Sandbox: mustSandboxMetadata(opts, env), Scanners: scanners, + Gate: "pass", + GateRules: []FiredGateRule{}, Judge: nil, } } +func validateGateRuleScanners(opts Options) error { + requested := make(map[string]bool, len(opts.Scanners)) + for _, scanner := range opts.Scanners { + requested[scanner] = true + } + var unrequested []string + for scanner := range opts.GateRules { + if !requested[scanner] { + unrequested = append(unrequested, scanner) + } + } + if len(unrequested) == 0 { + return nil + } + sort.Strings(unrequested) + return fmt.Errorf("gate rule references scanner %s, but it was not requested", unrequested[0]) +} + +func evaluateGate(artifact *Artifact, opts Options) { + for _, scanner := range opts.Scanners { + result := artifact.Scanners[scanner] + if result.Status != "completed" || result.ExitCode == nil { + continue + } + policy := opts.GateRules[scanner] + if policy.BlockOnExitCode != nil && policy.BlockOnExitCode.Matches(*result.ExitCode) { + artifact.GateRules = append(artifact.GateRules, FiredGateRule{ + Scanner: scanner, Rule: "blockOnExitCode", ExitCode: *result.ExitCode, Action: "block", + }) + artifact.Gate = "block" + } + if policy.WarnOnExitCode != nil && policy.WarnOnExitCode.Matches(*result.ExitCode) { + artifact.GateRules = append(artifact.GateRules, FiredGateRule{ + Scanner: scanner, Rule: "warnOnExitCode", ExitCode: *result.ExitCode, Action: "warn", + }) + if artifact.Gate == "pass" { + artifact.Gate = "warn" + } + } + } +} + func WriteJSON(w io.Writer, value interface{}) error { encoder := json.NewEncoder(w) encoder.SetIndent("", " ") @@ -2237,13 +2333,20 @@ func requirements(opts Options, env map[string]string) []EnvRequirement { if opts.ScannerResultPaths[scanner] != "" { continue } - if adapter, ok := DefaultScannerRegistry().Adapter(scanner); ok { + if adapter, ok := registryForOptions(opts).Adapter(scanner); ok { reqs = append(reqs, adapter.Requirements(env)...) } } return dedupe(reqs) } +func registryForOptions(opts Options) ScannerRegistry { + if opts.ScannerRegistry.isZero() { + return DefaultScannerRegistry() + } + return opts.ScannerRegistry +} + func envPresence(opts Options, env map[string]string) map[string]string { out := map[string]string{} for _, req := range requirements(opts, env) { diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 66b0726..4f2bb0e 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "reflect" "strings" "testing" "time" @@ -1121,6 +1122,25 @@ func TestArtifactConfigSourceField_FlagsOnly(t *testing.T) { } } +func TestNewArtifactAlwaysIncludesPassingGate(t *testing.T) { + opts, err := ParseArgs([]string{"./my-skill", "--scanner", "clawscan-static"}) + if err != nil { + t.Fatal(err) + } + + artifact := NewArtifact(opts, "/tmp/my-skill", "start", "complete", map[string]string{}) + if artifact.Gate != "pass" { + t.Fatalf("gate = %q", artifact.Gate) + } + raw, err := json.Marshal(artifact) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(raw, []byte(`"gate":"pass"`)) { + t.Fatalf("artifact omitted gate: %s", raw) + } +} + func TestRunWritesScannerOnlyArtifact(t *testing.T) { dir := t.TempDir() target := filepath.Join(dir, "skill") @@ -1179,6 +1199,119 @@ func TestRunIncludesDurationMsForScannerResults(t *testing.T) { assertScannerDurationJSON(t, artifact, "skillspector") } +func TestRunBlocksWhenNonzeroExitCodeRuleFires(t *testing.T) { + target := t.TempDir() + exitCode := 2 + opts := Options{ + Target: target, Scanners: []string{"clawscan-static"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, + GateRules: map[string]ScannerGatePolicy{ + "clawscan-static": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}, + }, + } + artifact, err := Run(opts, RunContext{Env: map[string]string{}, ScannerRunner: &gateScannerRunner{ + results: map[string]ScannerResult{"clawscan-static": {Status: "completed", ExitCode: &exitCode}}, + }}) + if err != nil { + t.Fatal(err) + } + if artifact.Gate != "block" { + t.Fatalf("gate = %q", artifact.Gate) + } + want := []FiredGateRule{{Scanner: "clawscan-static", Rule: "blockOnExitCode", ExitCode: 2, Action: "block"}} + if !reflect.DeepEqual(artifact.GateRules, want) { + t.Fatalf("gate rules = %#v", artifact.GateRules) + } +} + +func TestRunExitCodeGateActionsAndPrecedence(t *testing.T) { + tests := []struct { + name string + results map[string]ScannerResult + rules map[string]ScannerGatePolicy + want string + fired int + }{ + { + name: "zero does not fire nonzero", results: gateResults(0), + rules: map[string]ScannerGatePolicy{"clawscan-static": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}}, + want: "pass", + }, + { + name: "warning fires", results: gateResults(3), + rules: map[string]ScannerGatePolicy{"clawscan-static": {WarnOnExitCode: &ExitCodeRule{Codes: []int{3}}}}, + want: "warn", fired: 1, + }, + { + name: "listed code fires", results: gateResults(2), + rules: map[string]ScannerGatePolicy{"clawscan-static": {BlockOnExitCode: &ExitCodeRule{Codes: []int{1, 2, 3}}}}, + want: "block", fired: 1, + }, + { + name: "unlisted code passes", results: gateResults(4), + rules: map[string]ScannerGatePolicy{"clawscan-static": {BlockOnExitCode: &ExitCodeRule{Codes: []int{1, 2, 3}}}}, + want: "pass", + }, + { + name: "skipped scanner does not fire", results: map[string]ScannerResult{"clawscan-static": {Status: "skipped", ExitCode: intPointer(2)}}, + rules: map[string]ScannerGatePolicy{"clawscan-static": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}}, + want: "pass", + }, + { + name: "failed scanner does not fire", results: map[string]ScannerResult{"clawscan-static": {Status: "failed", ExitCode: intPointer(2)}}, + rules: map[string]ScannerGatePolicy{"clawscan-static": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}}, + want: "pass", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + target := t.TempDir() + artifact, err := Run(Options{ + Target: target, Scanners: []string{"clawscan-static"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, GateRules: test.rules, + }, RunContext{Env: map[string]string{}, ScannerRunner: &gateScannerRunner{results: test.results}}) + if err != nil { + t.Fatal(err) + } + if artifact.Gate != test.want || len(artifact.GateRules) != test.fired { + t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules) + } + }) + } +} + +func TestRunBlockGateBeatsWarnAcrossScanners(t *testing.T) { + target := t.TempDir() + artifact, err := Run(Options{ + Target: target, Scanners: []string{"clawscan-static", "skillspector"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, + GateRules: map[string]ScannerGatePolicy{ + "clawscan-static": {WarnOnExitCode: &ExitCodeRule{Codes: []int{1}}}, + "skillspector": {BlockOnExitCode: &ExitCodeRule{Codes: []int{2}}}, + }, + }, RunContext{Env: map[string]string{}, ScannerRunner: &gateScannerRunner{results: map[string]ScannerResult{ + "clawscan-static": {Status: "completed", ExitCode: intPointer(1)}, + "skillspector": {Status: "completed", ExitCode: intPointer(2)}, + }}}) + if err != nil { + t.Fatal(err) + } + if artifact.Gate != "block" || len(artifact.GateRules) != 2 { + t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules) + } +} + +func TestRunRejectsGateRuleForUnrequestedScannerBeforeScanning(t *testing.T) { + scannerRunner := &gateScannerRunner{results: gateResults(0)} + _, err := Run(Options{ + Target: t.TempDir(), Scanners: []string{"clawscan-static"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, + GateRules: map[string]ScannerGatePolicy{"absent-scanner": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}}, + }, RunContext{Env: map[string]string{}, ScannerRunner: scannerRunner}) + if err == nil || err.Error() != "gate rule references scanner absent-scanner, but it was not requested" { + t.Fatalf("err = %v", err) + } + if scannerRunner.calls != 0 { + t.Fatalf("scanner ran %d times", scannerRunner.calls) + } +} + func TestRunIncludesDurationMsForFixtureScannerResults(t *testing.T) { dir := t.TempDir() target := filepath.Join(dir, "skill") @@ -4093,6 +4226,27 @@ func (skippedScannerRunner) RunScanner(name string, target string, startedAt str }, nil } +type gateScannerRunner struct { + results map[string]ScannerResult + calls int +} + +func (runner *gateScannerRunner) RunScanner(name string, _ string, startedAt string) (ScannerResult, error) { + runner.calls++ + result := runner.results[name] + result.StartedAt = startedAt + result.CompletedAt = startedAt + return result, nil +} + +func gateResults(exitCode int) map[string]ScannerResult { + return map[string]ScannerResult{"clawscan-static": {Status: "completed", ExitCode: intPointer(exitCode)}} +} + +func intPointer(value int) *int { + return &value +} + type staticScannerRunner struct { results map[string]ScannerResult } @@ -4337,6 +4491,7 @@ type recordingCommandRunner struct { stdout string stderr string err error + exitCode *int runHook func(command string, args []string, cwd string) error } @@ -4377,7 +4532,7 @@ func (r *recordingCommandRunner) Run(command string, args []string, cwd string, if stdout == "" { stdout = "ok" } - return CommandOutput{Stdout: stdout, Stderr: r.stderr}, r.err + return CommandOutput{Stdout: stdout, Stderr: r.stderr, ExitCode: r.exitCode}, r.err } var errCommandFailed = errors.New("exit status 1") diff --git a/internal/runner/sandbox.go b/internal/runner/sandbox.go index ee96e8a..13e088f 100644 --- a/internal/runner/sandbox.go +++ b/internal/runner/sandbox.go @@ -227,7 +227,7 @@ func sandboxEnvNames(opts Options, env map[string]string) []string { if opts.ScannerResultPaths[scanner] != "" { continue } - adapter, ok := DefaultScannerRegistry().Adapter(scanner) + adapter, ok := registryForOptions(opts).Adapter(scanner) if !ok { continue } @@ -257,7 +257,7 @@ func requiresCommandExecution(opts Options) bool { if opts.ScannerResultPaths[scanner] != "" { continue } - adapter, ok := DefaultScannerRegistry().Adapter(scanner) + adapter, ok := registryForOptions(opts).Adapter(scanner) if !ok { continue } diff --git a/internal/runner/scanner_registry.go b/internal/runner/scanner_registry.go index 1df7776..1cce62a 100644 --- a/internal/runner/scanner_registry.go +++ b/internal/runner/scanner_registry.go @@ -33,6 +33,16 @@ type ScannerRegistry struct { adapters map[string]ScannerAdapter } +func (registry ScannerRegistry) WithAdapters(adapters ...ScannerAdapter) (ScannerRegistry, error) { + all := make([]ScannerAdapter, 0, len(registry.adapters)+len(adapters)) + for _, id := range registry.IDs() { + adapter, _ := registry.Adapter(id) + all = append(all, adapter) + } + all = append(all, adapters...) + return NewScannerRegistry(all...) +} + func NewScannerRegistry(adapters ...ScannerAdapter) (ScannerRegistry, error) { registry := ScannerRegistry{adapters: map[string]ScannerAdapter{}} for _, adapter := range adapters { diff --git a/internal/runner/scanner_registry_test.go b/internal/runner/scanner_registry_test.go index b44c921..9eee4f6 100644 --- a/internal/runner/scanner_registry_test.go +++ b/internal/runner/scanner_registry_test.go @@ -3,6 +3,7 @@ package runner import ( "encoding/json" "errors" + "path/filepath" "strings" "testing" ) @@ -147,6 +148,110 @@ func TestExternalScannerRunnerDispatchesThroughRegistry(t *testing.T) { } } +func TestUserDefinedScannerPreservesValidJSONOnCommandFailure(t *testing.T) { + adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"}, + }) + registry, err := NewScannerRegistry(adapter) + if err != nil { + t.Fatal(err) + } + commandRunner := &recordingCommandRunner{stdout: `{"findings":["detected"]}`, stderr: "findings detected", err: errCommandFailed} + result, err := (ExternalScannerRunner{ + Registry: registry, CommandRunner: commandRunner, Env: map[string]string{}, SandboxMode: SandboxModeOff, + }).RunScanner("demo", t.TempDir(), "2026-07-21T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if result.Status != "completed" || string(result.Raw) != `{"findings":["detected"]}` { + t.Fatalf("result = %#v", result) + } + if !strings.Contains(result.Error, "findings detected") { + t.Fatalf("error = %q", result.Error) + } +} + +func TestUserDefinedScannerRecordsExitCode(t *testing.T) { + exitCode := 2 + adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"}, + }) + registry, err := NewScannerRegistry(adapter) + if err != nil { + t.Fatal(err) + } + commandRunner := &recordingCommandRunner{stdout: `{}`, err: errCommandFailed, exitCode: &exitCode} + result, err := (ExternalScannerRunner{ + Registry: registry, CommandRunner: commandRunner, Env: map[string]string{}, SandboxMode: SandboxModeOff, + }).RunScanner("demo", t.TempDir(), "2026-07-21T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if result.ExitCode == nil || *result.ExitCode != 2 { + t.Fatalf("exit code = %#v", result.ExitCode) + } +} + +func TestUserDefinedScannerOmitsAbnormalExitCode(t *testing.T) { + exitCode := -1 + adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"}, + }) + registry, err := NewScannerRegistry(adapter) + if err != nil { + t.Fatal(err) + } + commandRunner := &recordingCommandRunner{stdout: `{}`, err: errCommandFailed, exitCode: &exitCode} + result, err := (ExternalScannerRunner{ + Registry: registry, CommandRunner: commandRunner, Env: map[string]string{}, SandboxMode: SandboxModeOff, + }).RunScanner("demo", t.TempDir(), "2026-07-21T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if result.ExitCode != nil { + t.Fatalf("abnormal exit code recorded = %d", *result.ExitCode) + } +} + +func TestUserDefinedScannerInterpolatesDollarTargetLiterally(t *testing.T) { + adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"}, + }) + registry, err := NewScannerRegistry(adapter) + if err != nil { + t.Fatal(err) + } + commandRunner := &recordingCommandRunner{stdout: `{}`} + target := filepath.Join(t.TempDir(), "skill-$USER") + result, err := (ExternalScannerRunner{ + Registry: registry, CommandRunner: commandRunner, Env: map[string]string{}, SandboxMode: SandboxModeOff, + }).RunScanner("demo", target, "2026-07-21T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if result.Status != "completed" { + t.Fatalf("result = %#v", result) + } + if len(commandRunner.calls) != 1 { + t.Fatalf("calls = %#v", commandRunner.calls) + } + args := commandRunner.calls[0].args + if len(args) != 4 || !strings.Contains(args[1], `"$1"`) || strings.Contains(args[1], target) || args[3] != target { + t.Fatalf("target was not passed as a separate shell argument: %#v", args) + } +} + +func TestUserDefinedScannerUsesContainerShellInDockerMode(t *testing.T) { + dockerShell := userDefinedScannerShell("windows", SandboxModeDocker) + if dockerShell.command != "/bin/sh" || strings.Join(dockerShell.args, " ") != "-c" { + t.Fatalf("docker shell = %#v", dockerShell) + } + hostShell := userDefinedScannerShell("windows", SandboxModeOff) + if hostShell.command != "cmd.exe" || strings.Join(hostShell.args, " ") != "/C" { + t.Fatalf("host shell = %#v", hostShell) + } +} + func TestScannerAdapterRequirementsFeedValidation(t *testing.T) { requirements := stubScannerAdapter{ id: "demo", diff --git a/internal/runner/target.go b/internal/runner/target.go index b817bf6..d800d51 100644 --- a/internal/runner/target.go +++ b/internal/runner/target.go @@ -253,7 +253,7 @@ func isURLTarget(input string) bool { func runnableScanners(opts Options, kind string) []string { var out []string for _, scanner := range opts.Scanners { - if opts.ScannerResultPaths[scanner] == "" && !scannerSupportsTargetKind(scanner, kind) { + if opts.ScannerResultPaths[scanner] == "" && !scannerSupportsTargetKindInRegistry(registryForOptions(opts), scanner, kind) { continue } out = append(out, scanner) @@ -265,7 +265,11 @@ func runnableScanners(opts Options, kind string) []string { // target of the given kind. Unknown scanner IDs are permitted here so the // scanner runner can still emit its own skipped result for them. func scannerSupportsTargetKind(scanner string, kind string) bool { - adapter, ok := DefaultScannerRegistry().Adapter(scanner) + return scannerSupportsTargetKindInRegistry(DefaultScannerRegistry(), scanner, kind) +} + +func scannerSupportsTargetKindInRegistry(registry ScannerRegistry, scanner string, kind string) bool { + adapter, ok := registry.Adapter(scanner) if !ok { return true } diff --git a/internal/runner/user_defined_scanner.go b/internal/runner/user_defined_scanner.go new file mode 100644 index 0000000..ceef300 --- /dev/null +++ b/internal/runner/user_defined_scanner.go @@ -0,0 +1,133 @@ +package runner + +import ( + "encoding/json" + "fmt" + "net/url" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" +) + +type UserDefinedScannerConfig struct { + ID string + Command string + Env []string + Targets []string +} + +func NewUserDefinedScanner(config UserDefinedScannerConfig) ScannerAdapter { + targets := make(map[string]bool, len(config.Targets)) + for _, target := range config.Targets { + targets[target] = true + } + return userDefinedScannerAdapter{config: config, targets: targets} +} + +type userDefinedScannerAdapter struct { + config UserDefinedScannerConfig + targets map[string]bool +} + +func (adapter userDefinedScannerAdapter) ID() string { return adapter.config.ID } + +func (adapter userDefinedScannerAdapter) Requirements(_ map[string]string) []EnvRequirement { + requirements := make([]EnvRequirement, 0, len(adapter.config.Env)) + for _, name := range adapter.config.Env { + requirements = append(requirements, EnvRequirement{EnvVar: name, Reason: adapter.config.ID + " scanner"}) + } + return requirements +} + +func (adapter userDefinedScannerAdapter) Info() ScannerInfo { + return ScannerInfo{ID: adapter.config.ID, DisplayName: adapter.config.ID, RequiredEnv: append([]string(nil), adapter.config.Env...)} +} + +func (adapter userDefinedScannerAdapter) InstallPlan() InstallPlan { + return InstallPlan{ScannerID: adapter.config.ID, InstallUnsupportedReason: "user-defined scanner"} +} + +func (adapter userDefinedScannerAdapter) SupportsTargetKind(kind string) bool { + return adapter.targets[kind] +} + +func (adapter userDefinedScannerAdapter) CommandBacked() bool { return true } + +func (adapter userDefinedScannerAdapter) Run(runner ExternalScannerRunner, target string, startedAt string) (ScannerResult, error) { + shell := userDefinedScannerShell(runtime.GOOS, runner.SandboxMode) + targetReplacement := shell.quote(target) + usePositionalTarget := shell.command == "/bin/sh" + if usePositionalTarget { + targetReplacement = `"$1"` + } + rendered := targetPlaceholderPattern.ReplaceAllStringFunc(adapter.config.Command, func(string) string { + return targetReplacement + }) + args := append(append([]string(nil), shell.args...), rendered) + if usePositionalTarget { + args = append(args, "clawscan-target", target) + } + fullCommand := append([]string{shell.command}, args...) + timeout := runner.Timeout + if timeout == 0 { + timeout = 20 * time.Minute + } + output, runErr := runner.CommandRunner.Run(shell.command, args, userDefinedScannerCWD(target), timeout) + exitCode := gateEligibleExitCode(output.ExitCode) + completedAt := time.Now().UTC().Format(time.RFC3339Nano) + raw := strings.TrimSpace(output.Stdout) + if runErr != nil { + message := commandError(runErr, output.Stderr, runner.Env) + if json.Valid([]byte(raw)) { + return ScannerResult{ + Status: "completed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, + Error: message, ExitCode: exitCode, Raw: json.RawMessage(raw), + }, nil + } + return ScannerResult{ + Status: "failed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, + Error: message, ExitCode: exitCode, + }, nil + } + if !json.Valid([]byte(raw)) { + return ScannerResult{ + Status: "failed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, + Error: fmt.Sprintf("User-defined scanner %s returned invalid JSON", adapter.config.ID), ExitCode: exitCode, + }, nil + } + return ScannerResult{ + Status: "completed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, + ExitCode: exitCode, Raw: json.RawMessage(raw), + }, nil +} + +func gateEligibleExitCode(exitCode *int) *int { + if exitCode == nil || *exitCode < 0 { + return nil + } + return exitCode +} + +func userDefinedScannerShell(goos string, sandboxMode string) judgeShellSpec { + if sandboxMode == SandboxModeDocker { + return judgeShellForGOOS("linux") + } + return judgeShellForGOOS(goos) +} + +var targetPlaceholderPattern = regexp.MustCompile(`\{\{\s*target\s*\}\}`) + +func userDefinedScannerCWD(target string) string { + parsed, err := url.Parse(target) + if err == nil && parsed.Scheme != "" && parsed.Host != "" { + return "" + } + info, err := os.Stat(target) + if err == nil && info.IsDir() { + return target + } + return filepath.Dir(target) +}