From f7deb57e495ff4a72a14bd0519278ab2a192d767 Mon Sep 17 00:00:00 2001 From: Jesse Merhi <79823012+jesse-merhi@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:42:40 +1000 Subject: [PATCH 1/3] feat(runner): support user-defined scanners in profile config Scanner entries in a profile may now be objects declaring id, command, env (required variable names), and targets. Custom scanners register into a run-local registry and execute through the standard sandboxed command path with {{target}} passed positionally to prevent shell injection. Raw stdout is preserved as artifact evidence; env presence is recorded as present/missing only. --- cmd/clawscan/main.go | 2 +- docs/scanners.md | 47 +++ internal/profiles/registry.go | 4 +- internal/profiles/registry_test.go | 12 +- internal/profiles/resolver.go | 193 +++++++++++- internal/profiles/resolver_test.go | 367 +++++++++++++++++++++++ internal/runner/runner.go | 22 +- internal/runner/sandbox.go | 4 +- internal/runner/scanner_registry.go | 10 + internal/runner/scanner_registry_test.go | 63 ++++ internal/runner/target.go | 8 +- internal/runner/user_defined_scanner.go | 125 ++++++++ 12 files changed, 833 insertions(+), 24 deletions(-) create mode 100644 internal/runner/user_defined_scanner.go diff --git a/cmd/clawscan/main.go b/cmd/clawscan/main.go index dfcf661..97ae10b 100644 --- a/cmd/clawscan/main.go +++ b/cmd/clawscan/main.go @@ -260,7 +260,7 @@ func printProfileCatalog(w io.Writer, catalog profiles.ProfileCatalog, cwd strin if info.Profile.Judge != nil { judge = "configured" } - fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", info.ID, displayProfileSource(info.Source, cwd), strings.Join(info.Profile.Scanners, ", "), judge) + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", info.ID, displayProfileSource(info.Source, cwd), strings.Join(info.Profile.ScannerIDs(), ", "), judge) } _ = tw.Flush() } diff --git a/docs/scanners.md b/docs/scanners.md index 8666a87..cebe5d8 100644 --- a/docs/scanners.md +++ b/docs/scanners.md @@ -17,6 +17,53 @@ clawscan scanners clawscan scanners skillspector ``` +## User-defined scanners + +A trusted config can mix built-in scanner IDs with user-defined command +scanners. The config schema uses the existing `profiles..scanners` list: + +```yaml +version: 1 + +profiles: + review: + scanners: + - clawscan-static + - id: my-scanner + command: my-scanner --json {{target}} + env: + - MY_SCANNER_TOKEN + targets: + - skill + - plugin +``` + +String entries select built-in scanners. Object entries define a scanner for +that config-backed run and accept these fields: + +| Field | Required | Meaning | +| --- | --- | --- | +| `id` | yes | Scanner ID using letters, digits, `_`, and `-`, starting with a letter or digit. It must not match a built-in scanner ID. | +| `command` | yes | Shell command to execute. Unquoted `{{target}}` is replaced with the safely passed resolved target; do not wrap the placeholder in shell quotes. | +| `env` | no | Required environment variable names. Values stay in the process environment and are never stored in the config or artifact. | +| `targets` | no | Supported target kinds: `skill`, `plugin`, and/or `url`. Defaults to `skill` and `url`. | + +The command must write JSON to stdout. ClawScan preserves valid stdout as the +scanner's raw evidence; empty or non-JSON stdout produces a failed scanner +result. Required environment variables are checked before any scanner starts. +Artifacts record each requirement as only `present` or `missing`. + +User-defined scanners use the same execution path as built-in command-backed +scanners. They run in the Docker sandbox by default, and declared `env` names +are added to its environment allowlist. Use `--sandbox off` only when you +intentionally want the command to run on the host. User-defined scanners are +local to the resolved config and do not appear in the built-in `clawscan +scanners` catalog. + +> **Trust boundary:** only load user-defined scanners from config files you +> control. A scanner entry is executable code. The default sandbox limits its +> host access, but does not make an untrusted command safe to run. + ## Target kinds Clawscan classifies each explicit target before dispatching scanners and records diff --git a/internal/profiles/registry.go b/internal/profiles/registry.go index 1ed0721..0bcaa56 100644 --- a/internal/profiles/registry.go +++ b/internal/profiles/registry.go @@ -28,8 +28,8 @@ func NewProfileRegistry(profiles map[string]resolvedProfile) (ProfileRegistry, e return ProfileRegistry{}, err } for _, scanner := range profile.profile.Scanners { - if !runner.DefaultScannerRegistry().Contains(scanner) { - return ProfileRegistry{}, unknownScannerInProfileError(id, scanner) + if !scanner.custom && !runner.DefaultScannerRegistry().Contains(scanner.ID) { + return ProfileRegistry{}, unknownScannerInProfileError(id, scanner.ID) } } registry.profiles[id] = profile diff --git a/internal/profiles/registry_test.go b/internal/profiles/registry_test.go index 24e5714..bcfc0a6 100644 --- a/internal/profiles/registry_test.go +++ b/internal/profiles/registry_test.go @@ -7,8 +7,8 @@ import ( func TestProfileRegistryReturnsSortedIDs(t *testing.T) { registry, err := NewProfileRegistry(map[string]resolvedProfile{ - "review": {profile: Profile{Scanners: []string{"snyk"}}}, - "clawhub": {profile: Profile{Scanners: []string{"skillspector"}}}, + "review": {profile: Profile{Scanners: []ProfileScanner{{ID: "snyk"}}}}, + "clawhub": {profile: Profile{Scanners: []ProfileScanner{{ID: "skillspector"}}}}, }) if err != nil { t.Fatal(err) @@ -26,7 +26,7 @@ func TestDefaultProfileRegistryContainsEmbeddedBuiltIns(t *testing.T) { if !ok { t.Fatal("missing clawhub profile") } - if got := strings.Join(clawhub.profile.Scanners, ","); got != "skillspector,clawscan-static" { + if got := strings.Join(profileScannerIDs(clawhub.profile.Scanners), ","); got != "skillspector,clawscan-static" { t.Fatalf("clawhub scanners = %q", got) } if clawhub.configDir != "clawhub" { @@ -46,7 +46,7 @@ func TestDefaultProfileRegistryContainsEmbeddedBuiltIns(t *testing.T) { if !ok { t.Fatal("missing clawhub-aig profile") } - if got := strings.Join(candidate.profile.Scanners, ","); got != "skillspector,aig" { + if got := strings.Join(profileScannerIDs(candidate.profile.Scanners), ","); got != "skillspector,aig" { t.Fatalf("clawhub-aig scanners = %q", got) } if candidate.configDir != "clawhub" { @@ -71,7 +71,7 @@ func TestDefaultProfileRegistryContainsEmbeddedBuiltIns(t *testing.T) { func TestProfileRegistryRejectsUnknownScannerReferences(t *testing.T) { _, err := NewProfileRegistry(map[string]resolvedProfile{ - "bad": {profile: Profile{Scanners: []string{"missing-scanner"}}}, + "bad": {profile: Profile{Scanners: []ProfileScanner{{ID: "missing-scanner"}}}}, }) if err == nil || err.Error() != "Profile bad references unknown scanner: missing-scanner" { t.Fatalf("err = %v", err) @@ -91,7 +91,7 @@ func TestInspectProfilesReturnsBuiltIns(t *testing.T) { if !ok { t.Fatal("missing clawhub profile") } - if got := strings.Join(clawhub.Profile.Scanners, ","); got != "skillspector,clawscan-static" { + if got := strings.Join(profileScannerIDs(clawhub.Profile.Scanners), ","); got != "skillspector,clawscan-static" { t.Fatalf("clawhub scanners = %q", got) } if clawhub.Source != "built-in" { diff --git a/internal/profiles/resolver.go b/internal/profiles/resolver.go index 9afadc5..0c11add 100644 --- a/internal/profiles/resolver.go +++ b/internal/profiles/resolver.go @@ -30,7 +30,7 @@ type Config struct { } type Profile struct { - Scanners []string `yaml:"scanners"` + Scanners []ProfileScanner `yaml:"scanners"` ScannerResults map[string]string `yaml:"scannerResults,omitempty"` Output string `yaml:"output,omitempty"` JSON bool `yaml:"json,omitempty"` @@ -38,6 +38,95 @@ type Profile struct { Judge *Judge `yaml:"judge,omitempty"` } +func (profile Profile) ScannerIDs() []string { + return profileScannerIDs(profile.Scanners) +} + +type ProfileScanner struct { + ID string + Command string + Env []string + Targets []string + custom bool +} + +func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error { + switch node.Kind { + case yaml.ScalarNode: + if err := node.Decode(&scanner.ID); err != nil { + return err + } + return nil + case yaml.MappingNode: + for index := 0; index < len(node.Content); index += 2 { + switch node.Content[index].Value { + case "id", "command", "env", "targets": + default: + return fmt.Errorf("field %s not found in type profiles.ProfileScanner", node.Content[index].Value) + } + } + var value struct { + ID string `yaml:"id"` + Command string `yaml:"command"` + Env []string `yaml:"env,omitempty"` + Targets []string `yaml:"targets,omitempty"` + } + if err := node.Decode(&value); err != nil { + return err + } + scanner.ID = value.ID + scanner.Command = value.Command + scanner.Env = value.Env + scanner.Targets = value.Targets + scanner.custom = true + return nil + default: + return fmt.Errorf("scanner entry must be a string or object") + } +} + +func (scanner ProfileScanner) MarshalYAML() (interface{}, error) { + if !scanner.custom { + return scanner.ID, nil + } + return struct { + ID string `yaml:"id"` + Command string `yaml:"command"` + Env []string `yaml:"env,omitempty"` + Targets []string `yaml:"targets,omitempty"` + }{scanner.ID, scanner.Command, scanner.Env, scanner.Targets}, nil +} + +func profileScannerIDs(scanners []ProfileScanner) []string { + ids := make([]string, 0, len(scanners)) + for _, scanner := range scanners { + ids = append(ids, scanner.ID) + } + return ids +} + +func profileScannerRegistry(scanners []ProfileScanner) (runner.ScannerRegistry, error) { + registry := runner.DefaultScannerRegistry() + for _, scanner := range scanners { + if !scanner.custom { + continue + } + targets := append([]string(nil), scanner.Targets...) + if len(targets) == 0 { + targets = []string{"skill", "url"} + } + adapter := runner.NewUserDefinedScanner(runner.UserDefinedScannerConfig{ + ID: scanner.ID, Command: scanner.Command, Env: scanner.Env, Targets: targets, + }) + var err error + registry, err = registry.WithAdapters(adapter) + if err != nil { + return runner.ScannerRegistry{}, err + } + } + return registry, nil +} + type Sandbox struct { Mode string `yaml:"mode,omitempty"` Image string `yaml:"image,omitempty"` @@ -90,6 +179,8 @@ type cliIntent struct { } var judgePathPlaceholderPattern = regexp.MustCompile(`\{\{\s*(prompt|output_schema):([^}]+)\}\}`) +var scannerIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]*$`) +var scannerTargetPlaceholderPattern = regexp.MustCompile(`\{\{\s*target\s*\}\}`) type ResolvedRunSet struct { Options []runner.Options @@ -182,7 +273,11 @@ func resolveRunSetIntent(intent cliIntent, cwd string) (ResolvedRunSet, error) { if err != nil { return ResolvedRunSet{}, err } - opts, err := runner.ParseArgs(finalArgs) + scannerRegistry, err := profileScannerRegistry(selected.profile.Scanners) + if err != nil { + return ResolvedRunSet{}, err + } + opts, err := runner.ParseArgsWithRegistry(finalArgs, scannerRegistry) if err != nil { return ResolvedRunSet{}, err } @@ -279,7 +374,11 @@ func resolveAllConfigProfiles(intent cliIntent, cwd string) (ResolvedRunSet, err if err != nil { return ResolvedRunSet{}, err } - opts, err := runner.ParseArgs(finalArgs) + scannerRegistry, err := profileScannerRegistry(selected.profile.Scanners) + if err != nil { + return ResolvedRunSet{}, err + } + opts, err := runner.ParseArgsWithRegistry(finalArgs, scannerRegistry) if err != nil { return ResolvedRunSet{}, err } @@ -610,7 +709,7 @@ func buildRunnerArgs(intent cliIntent, selected resolvedProfile, profileName str } profile := selected.profile - scanners := append([]string{}, profile.Scanners...) + scanners := profileScannerIDs(profile.Scanners) if len(intent.scanners) > 0 { scanners = append([]string{}, intent.scanners...) } @@ -716,14 +815,94 @@ func resolveJudgePaths(command string, configDir string) string { func validateProfile(name string, profile Profile) error { seen := map[string]bool{} for _, scanner := range profile.Scanners { - if seen[scanner] { - return fmt.Errorf("Duplicate scanner in profile %s: %s", name, scanner) + if scanner.custom && strings.TrimSpace(scanner.ID) == "" { + return fmt.Errorf("User-defined scanner in profile %s must include a non-empty id", name) + } + if scanner.custom && !scannerIDPattern.MatchString(scanner.ID) { + return fmt.Errorf("User-defined scanner %s in profile %s has invalid id; use letters, digits, underscores, and hyphens, starting with a letter or digit", scanner.ID, name) + } + if scanner.custom && strings.TrimSpace(scanner.Command) == "" { + return fmt.Errorf("User-defined scanner %s in profile %s must include a non-empty command", scanner.ID, name) + } + if scanner.custom && !scannerTargetPlaceholdersAreUnquoted(scanner.Command) { + return fmt.Errorf("User-defined scanner %s in profile %s must use {{target}} outside shell quotes", scanner.ID, name) + } + if scanner.custom && runner.DefaultScannerRegistry().Contains(scanner.ID) { + return fmt.Errorf("User-defined scanner %s collides with a built-in scanner ID", scanner.ID) + } + for _, target := range scanner.Targets { + switch target { + case "skill", "plugin", "url": + default: + return fmt.Errorf("User-defined scanner %s in profile %s has unsupported target kind: %s", scanner.ID, name, target) + } + } + if seen[scanner.ID] { + return fmt.Errorf("Duplicate scanner in profile %s: %s", name, scanner.ID) } - seen[scanner] = true + seen[scanner.ID] = true } return nil } +func scannerTargetPlaceholdersAreUnquoted(command string) bool { + matches := scannerTargetPlaceholderPattern.FindAllStringIndex(command, -1) + matchIndex := 0 + quote := byte(0) + escaped := false + comment := false + for index := 0; index < len(command); index++ { + if matchIndex < len(matches) && index == matches[matchIndex][0] { + if !comment && quote != 0 { + return false + } + index = matches[matchIndex][1] - 1 + matchIndex++ + continue + } + character := command[index] + if comment { + if character == '\n' { + comment = false + } + continue + } + if escaped { + escaped = false + continue + } + if character == '\\' && quote != '\'' { + escaped = true + continue + } + if quote == 0 { + if character == '#' && shellCommentCanStart(command, index) { + comment = true + continue + } + switch character { + case '\'', '"', '`': + quote = character + } + } else if character == quote { + quote = 0 + } + } + return true +} + +func shellCommentCanStart(command string, index int) bool { + if index == 0 { + return true + } + switch command[index-1] { + case ' ', '\t', '\r', '\n', ';', '|', '&', '(', ')': + return true + default: + return false + } +} + func unknownProfileError(profile string, available []string) error { return fmt.Errorf("Unknown profile: %s (available: %s)", profile, strings.Join(available, ", ")) } diff --git a/internal/profiles/resolver_test.go b/internal/profiles/resolver_test.go index 55e3b48..41d30a4 100644 --- a/internal/profiles/resolver_test.go +++ b/internal/profiles/resolver_test.go @@ -1,10 +1,13 @@ package profiles import ( + "bytes" + "encoding/json" "os" "path/filepath" "strings" "testing" + "time" "github.com/openclaw/clawscan/internal/runner" ) @@ -572,6 +575,12 @@ func TestResolveArgsRejectsUnsupportedVersionAndUnknownFields(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "field defaultProfile not found") { t.Fatalf("err = %v", err) } + + writeFile(t, config, "version: 1\nprofiles:\n review:\n scanners:\n - id: custom\n command: custom {{target}}\n token: example\n") + _, err = ResolveArgs([]string{"./skill", "--config", config}, dir) + if err == nil || !strings.Contains(err.Error(), "field token not found") { + t.Fatalf("err = %v", err) + } } func TestResolveArgsUnknownProfileListsAvailableProfiles(t *testing.T) { @@ -738,6 +747,352 @@ profiles: } } +func TestResolveArgsParsesUserDefinedScannerAlongsideBuiltIn(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - clawscan-static + - id: my-scanner + command: my-scanner --json {{target}} + env: + - MY_SCANNER_TOKEN + targets: + - plugin +`) + + opts, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(opts.Scanners, ","); got != "clawscan-static,my-scanner" { + t.Fatalf("scanners = %q", got) + } + adapter, ok := opts.ScannerRegistry.Adapter("my-scanner") + if !ok { + t.Fatal("custom scanner missing from run registry") + } + if adapter.SupportsTargetKind("skill") { + t.Fatal("plugin-only custom scanner unexpectedly supports skill targets") + } + if !adapter.SupportsTargetKind("plugin") { + t.Fatal("plugin-only custom scanner does not support plugin targets") + } +} + +func TestResolveArgsRejectsUserDefinedScannerIDCollision(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: clawscan-static + command: custom-static {{target}} +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner clawscan-static collides with a built-in scanner ID" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsRejectsIncompleteUserDefinedScanner(t *testing.T) { + for _, test := range []struct { + name string + entry string + wantErr string + }{ + {name: "missing id", entry: "command: scanner {{target}}", wantErr: "User-defined scanner in profile review must include a non-empty id"}, + {name: "missing command", entry: "id: my-scanner", wantErr: "User-defined scanner my-scanner in profile review must include a non-empty command"}, + } { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, "version: 1\nprofiles:\n review:\n scanners:\n - "+test.entry+"\n") + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != test.wantErr { + t.Fatalf("err = %v", err) + } + }) + } +} + +func TestResolveArgsRejectsUnknownUserDefinedScannerTarget(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: my-scanner + command: my-scanner {{target}} + targets: + - package +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner my-scanner in profile review has unsupported target kind: package" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsRejectsInvalidUserDefinedScannerID(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: foo=bar + command: scanner {{target}} +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner foo=bar in profile review has invalid id; use letters, digits, underscores, and hyphens, starting with a letter or digit" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsRejectsQuotedUserDefinedScannerTargetPlaceholder(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: my-scanner + command: my-scanner "{{target}}" +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner my-scanner in profile review must use {{target}} outside shell quotes" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsAllowsApostropheInShellCommentBeforeTargetPlaceholder(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: my-scanner + command: |- + # don't quote the target here + my-scanner {{target}} +`) + + if _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir); err != nil { + t.Fatal(err) + } +} + +func TestResolveArgsSupportsAliasedScannerEntries(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + templates: + scanners: + - &builtin clawscan-static + - &custom + id: my-scanner + command: my-scanner {{target}} + review: + scanners: + - *builtin + - *custom +`) + + opts, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(opts.Scanners, ","); got != "clawscan-static,my-scanner" { + t.Fatalf("scanners = %q", got) + } +} + +func TestUserDefinedScannerRunsThroughDockerAndPreservesRawJSON(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: fixture-scanner + command: fixture-scan --json {{target}} +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + adapter, ok := opts.ScannerRegistry.Adapter("fixture-scanner") + if !ok || !adapter.SupportsTargetKind("skill") || !adapter.SupportsTargetKind("url") || adapter.SupportsTargetKind("plugin") { + t.Fatalf("default target support is incorrect: adapter=%#v ok=%v", adapter, ok) + } + commandRunner := &profileCommandRunner{stdout: `{"scanner":"fixture","findings":[]}`} + artifact, err := runner.Run(opts, runner.RunContext{ + Env: map[string]string{}, + HostCommandRunner: commandRunner, + DockerAvailability: func() error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + result := artifact.Scanners["fixture-scanner"] + if result.Status != "completed" { + t.Fatalf("result = %#v", result) + } + if !bytes.Equal(result.Raw, []byte(`{"scanner":"fixture","findings":[]}`)) { + t.Fatalf("raw = %s", result.Raw) + } + if commandRunner.command != "docker" { + t.Fatalf("command = %q", commandRunner.command) + } + joined := strings.Join(commandRunner.args, " ") + if !strings.Contains(joined, "fixture-scan --json") || !strings.Contains(joined, target) || strings.Contains(joined, "{{target}}") { + t.Fatalf("docker args = %#v", commandRunner.args) + } +} + +func TestUserDefinedScannerMissingEnvFailsBeforeExecution(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: credentialed-scanner + command: credentialed-scan {{target}} + env: + - MY_SCANNER_TOKEN +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + commandRunner := &profileCommandRunner{stdout: `{}`} + _, err = runner.Run(opts, runner.RunContext{ + Env: map[string]string{}, + HostCommandRunner: commandRunner, + DockerAvailability: func() error { return nil }, + }) + if err == nil || !strings.Contains(err.Error(), "MY_SCANNER_TOKEN") { + t.Fatalf("err = %v", err) + } + if commandRunner.command != "" { + t.Fatalf("scanner executed before env validation: %q %#v", commandRunner.command, commandRunner.args) + } +} + +func TestUserDefinedScannerEnvIsAllowlistedAndRecordedByPresenceOnly(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: credentialed-scanner + command: credentialed-scan {{target}} + env: + - MY_SCANNER_TOKEN +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + missingArtifact := runner.NewArtifact(opts, target, "start", "complete", map[string]string{}) + if missingArtifact.Env["MY_SCANNER_TOKEN"] != "missing" { + t.Fatalf("missing env = %#v", missingArtifact.Env) + } + + const envValue = "test-value-123" + commandRunner := &profileCommandRunner{stdout: `{}`} + artifact, err := runner.Run(opts, runner.RunContext{ + Env: map[string]string{"MY_SCANNER_TOKEN": envValue}, + HostCommandRunner: commandRunner, + DockerAvailability: func() error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + if artifact.Env["MY_SCANNER_TOKEN"] != "present" { + t.Fatalf("env = %#v", artifact.Env) + } + encoded, err := json.Marshal(artifact) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(encoded, []byte(envValue)) { + t.Fatalf("artifact leaked env value: %s", encoded) + } + joined := strings.Join(commandRunner.args, "\x00") + if !strings.Contains(joined, "\x00-e\x00MY_SCANNER_TOKEN\x00") { + t.Fatalf("docker args missing env allowlist name: %#v", commandRunner.args) + } + if strings.Contains(joined, envValue) { + t.Fatalf("docker args leaked env value: %#v", commandRunner.args) + } +} + +func TestUserDefinedPluginScannerSkipsSkillTarget(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: plugin-scanner + command: plugin-scan {{target}} + env: + - PLUGIN_SCANNER_TOKEN + targets: + - plugin +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + commandRunner := &profileCommandRunner{stdout: `{}`} + artifact, err := runner.Run(opts, runner.RunContext{ + Env: map[string]string{}, + HostCommandRunner: commandRunner, + DockerAvailability: func() error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + result := artifact.Scanners["plugin-scanner"] + if result.Status != "skipped" || !strings.Contains(result.Error, "does not support skill targets") { + t.Fatalf("result = %#v", result) + } + if commandRunner.command != "" { + t.Fatalf("unsupported scanner executed: %q %#v", commandRunner.command, commandRunner.args) + } +} + func TestResolveArgsRejectsProfileWithoutScannersUnlessCLIOverrides(t *testing.T) { dir := t.TempDir() config := filepath.Join(dir, ".clawscan.yml") @@ -770,3 +1125,15 @@ func writeFile(t *testing.T, path string, content string) { t.Fatal(err) } } + +type profileCommandRunner struct { + command string + args []string + stdout string +} + +func (commandRunner *profileCommandRunner) Run(command string, args []string, _ string, _ time.Duration) (runner.CommandOutput, error) { + commandRunner.command = command + commandRunner.args = append([]string(nil), args...) + return runner.CommandOutput{Stdout: commandRunner.stdout}, nil +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c8b735a..1f40745 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -33,6 +33,7 @@ type Options struct { ContextPath string Benchmark *BenchmarkOptions Scanners []string + ScannerRegistry ScannerRegistry ScannerResultPaths map[string]string OutputPath string JSON bool @@ -236,7 +237,12 @@ func NewBenchmarkOptions(id string, split string, limit int, offset int, predict } func ParseArgs(args []string) (Options, error) { + return ParseArgsWithRegistry(args, DefaultScannerRegistry()) +} + +func ParseArgsWithRegistry(args []string, registry ScannerRegistry) (Options, error) { opts := Options{ScannerResultPaths: map[string]string{}} + opts.ScannerRegistry = registry start := 0 if len(args) > 0 && !strings.HasPrefix(args[0], "--") { opts.Target = args[0] @@ -251,7 +257,7 @@ func ParseArgs(args []string) (Options, error) { if err != nil { return Options{}, err } - if !DefaultScannerRegistry().Contains(value) { + if !registry.Contains(value) { return Options{}, fmt.Errorf("Unknown scanner: %s", value) } opts.Scanners = append(opts.Scanners, value) @@ -279,7 +285,7 @@ func ParseArgs(args []string) (Options, error) { if !ok || scanner == "" || path == "" { return Options{}, errors.New("Expected --scanner-result value as scanner=path") } - if !DefaultScannerRegistry().Contains(scanner) { + if !registry.Contains(scanner) { return Options{}, fmt.Errorf("Unknown scanner: %s", scanner) } opts.ScannerResultPaths[scanner] = path @@ -403,6 +409,7 @@ func Run(opts Options, ctx RunContext) (Artifact, error) { Profile: opts.Profile, TargetKind: target.kind, TargetID: target.id, + Registry: registryForOptions(opts), SandboxMode: scannerSandboxMode, SkillSpectorCommand: ctx.SkillSpectorCommand, VirusTotalHTTPClient: ctx.VirusTotalHTTPClient, @@ -826,7 +833,7 @@ func scannerResult(opts Options, scanner string, target resolvedTarget, startedA Raw: json.RawMessage(raw), }, nil } - if !scannerSupportsTargetKind(scanner, target.kind) { + if !scannerSupportsTargetKindInRegistry(registryForOptions(opts), scanner, target.kind) { return unsupportedTargetKindResult(scanner, target.kind, startedAt), nil } return scannerRunner.RunScanner(scanner, target.resolvedPath, startedAt) @@ -2237,13 +2244,20 @@ func requirements(opts Options, env map[string]string) []EnvRequirement { if opts.ScannerResultPaths[scanner] != "" { continue } - if adapter, ok := DefaultScannerRegistry().Adapter(scanner); ok { + if adapter, ok := registryForOptions(opts).Adapter(scanner); ok { reqs = append(reqs, adapter.Requirements(env)...) } } return dedupe(reqs) } +func registryForOptions(opts Options) ScannerRegistry { + if opts.ScannerRegistry.isZero() { + return DefaultScannerRegistry() + } + return opts.ScannerRegistry +} + func envPresence(opts Options, env map[string]string) map[string]string { out := map[string]string{} for _, req := range requirements(opts, env) { diff --git a/internal/runner/sandbox.go b/internal/runner/sandbox.go index ee96e8a..13e088f 100644 --- a/internal/runner/sandbox.go +++ b/internal/runner/sandbox.go @@ -227,7 +227,7 @@ func sandboxEnvNames(opts Options, env map[string]string) []string { if opts.ScannerResultPaths[scanner] != "" { continue } - adapter, ok := DefaultScannerRegistry().Adapter(scanner) + adapter, ok := registryForOptions(opts).Adapter(scanner) if !ok { continue } @@ -257,7 +257,7 @@ func requiresCommandExecution(opts Options) bool { if opts.ScannerResultPaths[scanner] != "" { continue } - adapter, ok := DefaultScannerRegistry().Adapter(scanner) + adapter, ok := registryForOptions(opts).Adapter(scanner) if !ok { continue } diff --git a/internal/runner/scanner_registry.go b/internal/runner/scanner_registry.go index 1df7776..1cce62a 100644 --- a/internal/runner/scanner_registry.go +++ b/internal/runner/scanner_registry.go @@ -33,6 +33,16 @@ type ScannerRegistry struct { adapters map[string]ScannerAdapter } +func (registry ScannerRegistry) WithAdapters(adapters ...ScannerAdapter) (ScannerRegistry, error) { + all := make([]ScannerAdapter, 0, len(registry.adapters)+len(adapters)) + for _, id := range registry.IDs() { + adapter, _ := registry.Adapter(id) + all = append(all, adapter) + } + all = append(all, adapters...) + return NewScannerRegistry(all...) +} + func NewScannerRegistry(adapters ...ScannerAdapter) (ScannerRegistry, error) { registry := ScannerRegistry{adapters: map[string]ScannerAdapter{}} for _, adapter := range adapters { diff --git a/internal/runner/scanner_registry_test.go b/internal/runner/scanner_registry_test.go index b44c921..42d50bd 100644 --- a/internal/runner/scanner_registry_test.go +++ b/internal/runner/scanner_registry_test.go @@ -3,6 +3,7 @@ package runner import ( "encoding/json" "errors" + "path/filepath" "strings" "testing" ) @@ -147,6 +148,68 @@ func TestExternalScannerRunnerDispatchesThroughRegistry(t *testing.T) { } } +func TestUserDefinedScannerPreservesValidJSONOnCommandFailure(t *testing.T) { + adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"}, + }) + registry, err := NewScannerRegistry(adapter) + if err != nil { + t.Fatal(err) + } + commandRunner := &recordingCommandRunner{stdout: `{"findings":["detected"]}`, stderr: "findings detected", err: errCommandFailed} + result, err := (ExternalScannerRunner{ + Registry: registry, CommandRunner: commandRunner, Env: map[string]string{}, SandboxMode: SandboxModeOff, + }).RunScanner("demo", t.TempDir(), "2026-07-21T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if result.Status != "completed" || string(result.Raw) != `{"findings":["detected"]}` { + t.Fatalf("result = %#v", result) + } + if !strings.Contains(result.Error, "findings detected") { + t.Fatalf("error = %q", result.Error) + } +} + +func TestUserDefinedScannerInterpolatesDollarTargetLiterally(t *testing.T) { + adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"}, + }) + registry, err := NewScannerRegistry(adapter) + if err != nil { + t.Fatal(err) + } + commandRunner := &recordingCommandRunner{stdout: `{}`} + target := filepath.Join(t.TempDir(), "skill-$USER") + result, err := (ExternalScannerRunner{ + Registry: registry, CommandRunner: commandRunner, Env: map[string]string{}, SandboxMode: SandboxModeOff, + }).RunScanner("demo", target, "2026-07-21T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if result.Status != "completed" { + t.Fatalf("result = %#v", result) + } + if len(commandRunner.calls) != 1 { + t.Fatalf("calls = %#v", commandRunner.calls) + } + args := commandRunner.calls[0].args + if len(args) != 4 || !strings.Contains(args[1], `"$1"`) || strings.Contains(args[1], target) || args[3] != target { + t.Fatalf("target was not passed as a separate shell argument: %#v", args) + } +} + +func TestUserDefinedScannerUsesContainerShellInDockerMode(t *testing.T) { + dockerShell := userDefinedScannerShell("windows", SandboxModeDocker) + if dockerShell.command != "/bin/sh" || strings.Join(dockerShell.args, " ") != "-c" { + t.Fatalf("docker shell = %#v", dockerShell) + } + hostShell := userDefinedScannerShell("windows", SandboxModeOff) + if hostShell.command != "cmd.exe" || strings.Join(hostShell.args, " ") != "/C" { + t.Fatalf("host shell = %#v", hostShell) + } +} + func TestScannerAdapterRequirementsFeedValidation(t *testing.T) { requirements := stubScannerAdapter{ id: "demo", diff --git a/internal/runner/target.go b/internal/runner/target.go index b817bf6..d800d51 100644 --- a/internal/runner/target.go +++ b/internal/runner/target.go @@ -253,7 +253,7 @@ func isURLTarget(input string) bool { func runnableScanners(opts Options, kind string) []string { var out []string for _, scanner := range opts.Scanners { - if opts.ScannerResultPaths[scanner] == "" && !scannerSupportsTargetKind(scanner, kind) { + if opts.ScannerResultPaths[scanner] == "" && !scannerSupportsTargetKindInRegistry(registryForOptions(opts), scanner, kind) { continue } out = append(out, scanner) @@ -265,7 +265,11 @@ func runnableScanners(opts Options, kind string) []string { // target of the given kind. Unknown scanner IDs are permitted here so the // scanner runner can still emit its own skipped result for them. func scannerSupportsTargetKind(scanner string, kind string) bool { - adapter, ok := DefaultScannerRegistry().Adapter(scanner) + return scannerSupportsTargetKindInRegistry(DefaultScannerRegistry(), scanner, kind) +} + +func scannerSupportsTargetKindInRegistry(registry ScannerRegistry, scanner string, kind string) bool { + adapter, ok := registry.Adapter(scanner) if !ok { return true } diff --git a/internal/runner/user_defined_scanner.go b/internal/runner/user_defined_scanner.go new file mode 100644 index 0000000..b30e38e --- /dev/null +++ b/internal/runner/user_defined_scanner.go @@ -0,0 +1,125 @@ +package runner + +import ( + "encoding/json" + "fmt" + "net/url" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" +) + +type UserDefinedScannerConfig struct { + ID string + Command string + Env []string + Targets []string +} + +func NewUserDefinedScanner(config UserDefinedScannerConfig) ScannerAdapter { + targets := make(map[string]bool, len(config.Targets)) + for _, target := range config.Targets { + targets[target] = true + } + return userDefinedScannerAdapter{config: config, targets: targets} +} + +type userDefinedScannerAdapter struct { + config UserDefinedScannerConfig + targets map[string]bool +} + +func (adapter userDefinedScannerAdapter) ID() string { return adapter.config.ID } + +func (adapter userDefinedScannerAdapter) Requirements(_ map[string]string) []EnvRequirement { + requirements := make([]EnvRequirement, 0, len(adapter.config.Env)) + for _, name := range adapter.config.Env { + requirements = append(requirements, EnvRequirement{EnvVar: name, Reason: adapter.config.ID + " scanner"}) + } + return requirements +} + +func (adapter userDefinedScannerAdapter) Info() ScannerInfo { + return ScannerInfo{ID: adapter.config.ID, DisplayName: adapter.config.ID, RequiredEnv: append([]string(nil), adapter.config.Env...)} +} + +func (adapter userDefinedScannerAdapter) InstallPlan() InstallPlan { + return InstallPlan{ScannerID: adapter.config.ID, InstallUnsupportedReason: "user-defined scanner"} +} + +func (adapter userDefinedScannerAdapter) SupportsTargetKind(kind string) bool { + return adapter.targets[kind] +} + +func (adapter userDefinedScannerAdapter) CommandBacked() bool { return true } + +func (adapter userDefinedScannerAdapter) Run(runner ExternalScannerRunner, target string, startedAt string) (ScannerResult, error) { + shell := userDefinedScannerShell(runtime.GOOS, runner.SandboxMode) + targetReplacement := shell.quote(target) + usePositionalTarget := shell.command == "/bin/sh" + if usePositionalTarget { + targetReplacement = `"$1"` + } + rendered := targetPlaceholderPattern.ReplaceAllStringFunc(adapter.config.Command, func(string) string { + return targetReplacement + }) + args := append(append([]string(nil), shell.args...), rendered) + if usePositionalTarget { + args = append(args, "clawscan-target", target) + } + fullCommand := append([]string{shell.command}, args...) + timeout := runner.Timeout + if timeout == 0 { + timeout = 20 * time.Minute + } + output, runErr := runner.CommandRunner.Run(shell.command, args, userDefinedScannerCWD(target), timeout) + completedAt := time.Now().UTC().Format(time.RFC3339Nano) + raw := strings.TrimSpace(output.Stdout) + if runErr != nil { + message := commandError(runErr, output.Stderr, runner.Env) + if json.Valid([]byte(raw)) { + return ScannerResult{ + Status: "completed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, + Error: message, Raw: json.RawMessage(raw), + }, nil + } + return ScannerResult{ + Status: "failed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, + Error: message, + }, nil + } + if !json.Valid([]byte(raw)) { + return ScannerResult{ + Status: "failed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, + Error: fmt.Sprintf("User-defined scanner %s returned invalid JSON", adapter.config.ID), + }, nil + } + return ScannerResult{ + Status: "completed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, + Raw: json.RawMessage(raw), + }, nil +} + +func userDefinedScannerShell(goos string, sandboxMode string) judgeShellSpec { + if sandboxMode == SandboxModeDocker { + return judgeShellForGOOS("linux") + } + return judgeShellForGOOS(goos) +} + +var targetPlaceholderPattern = regexp.MustCompile(`\{\{\s*target\s*\}\}`) + +func userDefinedScannerCWD(target string) string { + parsed, err := url.Parse(target) + if err == nil && parsed.Scheme != "" && parsed.Host != "" { + return "" + } + info, err := os.Stat(target) + if err == nil && info.IsDir() { + return target + } + return filepath.Dir(target) +} From 92048ce2265c4f68067cef3d632aad28800d97b4 Mon Sep 17 00:00:00 2001 From: Jesse Merhi <79823012+jesse-merhi@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:14:03 +1000 Subject: [PATCH 2/3] feat(runner): evaluate exit-code gate policy into artifact gate verdict Object-form scanner entries accept a nested gate with blockOnExitCode / warnOnExitCode (int, int list, or "nonzero"). The runner captures user-defined scanner exit codes, evaluates rules after all scanners complete, and records gate: pass|warn|block plus fired gateRules in the artifact. Rules referencing unrequested scanners fail before scanning. Abnormal termination (signal/timeout) never fires exit-code rules. --- cmd/clawscan/main.go | 17 ++- cmd/clawscan/main_test.go | 17 +++ docs/scanners.md | 21 +++ internal/profiles/resolver.go | 166 +++++++++++++++++++++-- internal/profiles/resolver_test.go | 134 ++++++++++++++++++ internal/runner/runner.go | 95 ++++++++++++- internal/runner/runner_test.go | 157 ++++++++++++++++++++- internal/runner/scanner_registry_test.go | 42 ++++++ internal/runner/user_defined_scanner.go | 16 ++- 9 files changed, 646 insertions(+), 19 deletions(-) diff --git a/cmd/clawscan/main.go b/cmd/clawscan/main.go index 97ae10b..cefb420 100644 --- a/cmd/clawscan/main.go +++ b/cmd/clawscan/main.go @@ -337,6 +337,15 @@ func printRunSummary(w io.Writer, result runner.RunTargetsResult, outputPath str fmt.Fprintf(w, "scanner_other: %d\n", summary.ScannerOther) } fmt.Fprintf(w, "issues_found: %d\n", summary.IssuesFound) + fmt.Fprintf(w, "gate: %s", summary.Gate) + if len(summary.GateRules) > 0 { + details := make([]string, 0, len(summary.GateRules)) + for _, rule := range summary.GateRules { + details = append(details, fmt.Sprintf("%s exit %d -> %s", rule.Scanner, rule.ExitCode, rule.Action)) + } + fmt.Fprintf(w, " (%s)", strings.Join(details, ", ")) + } + fmt.Fprintln(w) if summary.HasJudge { fmt.Fprintf(w, "judge_completed: %d\n", summary.JudgeCompleted) fmt.Fprintf(w, "judge_failed: %d\n", summary.JudgeFailed) @@ -404,6 +413,8 @@ type runSummary struct { ScannerSkipped int ScannerOther int IssuesFound int + Gate string + GateRules []runner.FiredGateRule HasJudge bool JudgeCompleted int JudgeFailed int @@ -415,7 +426,7 @@ type runSummary struct { } func summarizeRunTargets(result runner.RunTargetsResult) runSummary { - var summary runSummary + summary := runSummary{Gate: "pass"} if result.Batch != nil { summary.Profile = result.Batch.Profile summary.Profiles = result.Batch.Summary.ProfileCount @@ -436,6 +447,10 @@ func (summary *runSummary) addArtifact(artifact runner.Artifact) { summary.Profile = artifact.Profile } summary.Targets++ + summary.GateRules = append(summary.GateRules, artifact.GateRules...) + if artifact.Gate == "block" || (artifact.Gate == "warn" && summary.Gate == "pass") { + summary.Gate = artifact.Gate + } for _, result := range artifact.Scanners { switch result.Status { case "completed": diff --git a/cmd/clawscan/main_test.go b/cmd/clawscan/main_test.go index 37d2e2f..8dc3cd5 100644 --- a/cmd/clawscan/main_test.go +++ b/cmd/clawscan/main_test.go @@ -7,6 +7,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/openclaw/clawscan/internal/runner" ) func TestRunCommandPrintsHelp(t *testing.T) { @@ -511,6 +513,21 @@ func TestRunCommandWritesDefaultOutputAndPrintsKeyValueSummary(t *testing.T) { } } +func TestPrintRunSummaryIncludesGateVerdictAndFiredRule(t *testing.T) { + artifact := runner.Artifact{ + Gate: "block", + GateRules: []runner.FiredGateRule{ + {Scanner: "my-scanner", Rule: "blockOnExitCode", ExitCode: 3, Action: "block"}, + }, + Scanners: map[string]runner.ScannerResult{}, + } + var output strings.Builder + printRunSummary(&output, runner.RunTargetsResult{Single: &artifact}, "") + if !strings.Contains(output.String(), "gate: block (my-scanner exit 3 -> block)") { + t.Fatalf("summary missing gate rule:\n%s", output.String()) + } +} + func TestRunCommandJSONDoesNotWriteDefaultOutput(t *testing.T) { dir := t.TempDir() target := filepath.Join(dir, "skill") diff --git a/docs/scanners.md b/docs/scanners.md index cebe5d8..85bbd6b 100644 --- a/docs/scanners.md +++ b/docs/scanners.md @@ -36,6 +36,8 @@ profiles: targets: - skill - plugin + gate: + blockOnExitCode: nonzero ``` String entries select built-in scanners. Object entries define a scanner for @@ -47,6 +49,25 @@ that config-backed run and accept these fields: | `command` | yes | Shell command to execute. Unquoted `{{target}}` is replaced with the safely passed resolved target; do not wrap the placeholder in shell quotes. | | `env` | no | Required environment variable names. Values stay in the process environment and are never stored in the config or artifact. | | `targets` | no | Supported target kinds: `skill`, `plugin`, and/or `url`. Defaults to `skill` and `url`. | +| `gate` | no | Exit-code policy with optional `blockOnExitCode` and `warnOnExitCode` rules. | + +Each exit-code rule accepts one non-negative integer, a list such as `[1, 2, +3]`, or the string `nonzero`. The block and warning rules may not claim the +same exit code. For example: + +```yaml +gate: + blockOnExitCode: [2, 3] + warnOnExitCode: 1 +``` + +After every selected scanner finishes, ClawScan records the strongest fired +action as the top-level artifact `gate`: `block` wins over `warn`, and an +artifact with no fired rules records `"gate": "pass"`. Each fired rule is also +listed in `gateRules` with its scanner ID, rule name, exit code, and action. +Skipped scanners do not fire gate rules. A scanner result with status `failed` +also does not fire an exit-code rule; a nonzero command that still returned +valid JSON has status `completed` and can fire one. The command must write JSON to stdout. ClawScan preserves valid stdout as the scanner's raw evidence; empty or non-JSON stdout produces a failed scanner diff --git a/internal/profiles/resolver.go b/internal/profiles/resolver.go index 0c11add..9cb0d54 100644 --- a/internal/profiles/resolver.go +++ b/internal/profiles/resolver.go @@ -47,9 +47,85 @@ type ProfileScanner struct { Command string Env []string Targets []string + Gate *ProfileScannerGate custom bool } +type ProfileScannerGate struct { + BlockOnExitCode *profileExitCodeRule `yaml:"blockOnExitCode,omitempty"` + WarnOnExitCode *profileExitCodeRule `yaml:"warnOnExitCode,omitempty"` +} + +type profileExitCodeRule struct { + Codes []int + Nonzero bool +} + +func (rule *profileExitCodeRule) UnmarshalYAML(node *yaml.Node) error { + switch node.Kind { + case yaml.ScalarNode: + if node.Tag == "!!str" && node.Value == "nonzero" { + rule.Nonzero = true + return nil + } + if node.Tag == "!!int" { + var code int + if err := node.Decode(&code); err == nil && code >= 0 { + rule.Codes = []int{code} + return nil + } + return errors.New("exit-code gate rule must contain only non-negative integers") + } + case yaml.SequenceNode: + if len(node.Content) == 0 { + return errors.New("exit-code gate rule must not be an empty list") + } + codes := make([]int, 0, len(node.Content)) + for _, item := range node.Content { + if item.Kind != yaml.ScalarNode || item.Tag != "!!int" { + return errors.New("exit-code gate rule must contain only non-negative integers") + } + var code int + if err := item.Decode(&code); err != nil || code < 0 { + return errors.New("exit-code gate rule must contain only non-negative integers") + } + codes = append(codes, code) + } + rule.Codes = codes + return nil + } + return errors.New(`exit-code gate rule must be a non-negative integer, a list of non-negative integers, or "nonzero"`) +} + +func (rule profileExitCodeRule) MarshalYAML() (interface{}, error) { + if rule.Nonzero { + return "nonzero", nil + } + switch len(rule.Codes) { + case 0: + return nil, errors.New("exit-code gate rule must include at least one exit code") + case 1: + return rule.Codes[0], nil + default: + return append([]int(nil), rule.Codes...), nil + } +} + +func (gate *ProfileScannerGate) UnmarshalYAML(node *yaml.Node) error { + if node.Kind != yaml.MappingNode { + return errors.New("scanner gate must be an object") + } + for index := 0; index < len(node.Content); index += 2 { + switch node.Content[index].Value { + case "blockOnExitCode", "warnOnExitCode": + default: + return fmt.Errorf("field %s not found in type profiles.ProfileScannerGate", node.Content[index].Value) + } + } + type plainGate ProfileScannerGate + return node.Decode((*plainGate)(gate)) +} + func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error { switch node.Kind { case yaml.ScalarNode: @@ -60,16 +136,17 @@ func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error { case yaml.MappingNode: for index := 0; index < len(node.Content); index += 2 { switch node.Content[index].Value { - case "id", "command", "env", "targets": + case "id", "command", "env", "targets", "gate": default: return fmt.Errorf("field %s not found in type profiles.ProfileScanner", node.Content[index].Value) } } var value struct { - ID string `yaml:"id"` - Command string `yaml:"command"` - Env []string `yaml:"env,omitempty"` - Targets []string `yaml:"targets,omitempty"` + ID string `yaml:"id"` + Command string `yaml:"command"` + Env []string `yaml:"env,omitempty"` + Targets []string `yaml:"targets,omitempty"` + Gate *ProfileScannerGate `yaml:"gate,omitempty"` } if err := node.Decode(&value); err != nil { return err @@ -78,6 +155,7 @@ func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error { scanner.Command = value.Command scanner.Env = value.Env scanner.Targets = value.Targets + scanner.Gate = value.Gate scanner.custom = true return nil default: @@ -90,11 +168,12 @@ func (scanner ProfileScanner) MarshalYAML() (interface{}, error) { return scanner.ID, nil } return struct { - ID string `yaml:"id"` - Command string `yaml:"command"` - Env []string `yaml:"env,omitempty"` - Targets []string `yaml:"targets,omitempty"` - }{scanner.ID, scanner.Command, scanner.Env, scanner.Targets}, nil + ID string `yaml:"id"` + Command string `yaml:"command"` + Env []string `yaml:"env,omitempty"` + Targets []string `yaml:"targets,omitempty"` + Gate *ProfileScannerGate `yaml:"gate,omitempty"` + }{scanner.ID, scanner.Command, scanner.Env, scanner.Targets, scanner.Gate}, nil } func profileScannerIDs(scanners []ProfileScanner) []string { @@ -127,6 +206,31 @@ func profileScannerRegistry(scanners []ProfileScanner) (runner.ScannerRegistry, return registry, nil } +func profileGateRules(scanners []ProfileScanner) map[string]runner.ScannerGatePolicy { + rules := map[string]runner.ScannerGatePolicy{} + for _, scanner := range scanners { + if scanner.Gate == nil { + continue + } + policy := runner.ScannerGatePolicy{} + if scanner.Gate.BlockOnExitCode != nil { + policy.BlockOnExitCode = &runner.ExitCodeRule{ + Codes: append([]int(nil), scanner.Gate.BlockOnExitCode.Codes...), Nonzero: scanner.Gate.BlockOnExitCode.Nonzero, + } + } + if scanner.Gate.WarnOnExitCode != nil { + policy.WarnOnExitCode = &runner.ExitCodeRule{ + Codes: append([]int(nil), scanner.Gate.WarnOnExitCode.Codes...), Nonzero: scanner.Gate.WarnOnExitCode.Nonzero, + } + } + if policy.BlockOnExitCode == nil && policy.WarnOnExitCode == nil { + continue + } + rules[scanner.ID] = policy + } + return rules +} + type Sandbox struct { Mode string `yaml:"mode,omitempty"` Image string `yaml:"image,omitempty"` @@ -282,6 +386,7 @@ func resolveRunSetIntent(intent cliIntent, cwd string) (ResolvedRunSet, error) { return ResolvedRunSet{}, err } opts.Profile = profileName + opts.GateRules = profileGateRules(selected.profile.Scanners) opts.ConfigSource = configSource opts.DiscoverConfig = intent.discoverConfig if opts.Judge != nil { @@ -383,6 +488,7 @@ func resolveAllConfigProfiles(intent cliIntent, cwd string) (ResolvedRunSet, err return ResolvedRunSet{}, err } opts.Profile = profileName + opts.GateRules = profileGateRules(selected.profile.Scanners) opts.ConfigSource = filepath.Clean(projectPath) opts.OutputPath = "" opts.JSON = false @@ -830,6 +936,11 @@ func validateProfile(name string, profile Profile) error { if scanner.custom && runner.DefaultScannerRegistry().Contains(scanner.ID) { return fmt.Errorf("User-defined scanner %s collides with a built-in scanner ID", scanner.ID) } + if scanner.Gate != nil { + if code, overlaps := overlappingExitCodeRules(scanner.Gate.BlockOnExitCode, scanner.Gate.WarnOnExitCode); overlaps { + return fmt.Errorf("User-defined scanner %s in profile %s gate blockOnExitCode and warnOnExitCode both claim exit code %d", scanner.ID, name, code) + } + } for _, target := range scanner.Targets { switch target { case "skill", "plugin", "url": @@ -845,6 +956,41 @@ func validateProfile(name string, profile Profile) error { return nil } +func overlappingExitCodeRules(block *profileExitCodeRule, warn *profileExitCodeRule) (int, bool) { + if block == nil || warn == nil { + return 0, false + } + if block.Nonzero && warn.Nonzero { + return 1, true + } + if block.Nonzero { + for _, code := range warn.Codes { + if code != 0 { + return code, true + } + } + return 0, false + } + if warn.Nonzero { + for _, code := range block.Codes { + if code != 0 { + return code, true + } + } + return 0, false + } + warnCodes := make(map[int]bool, len(warn.Codes)) + for _, code := range warn.Codes { + warnCodes[code] = true + } + for _, code := range block.Codes { + if warnCodes[code] { + return code, true + } + } + return 0, false +} + func scannerTargetPlaceholdersAreUnquoted(command string) bool { matches := scannerTargetPlaceholderPattern.FindAllStringIndex(command, -1) matchIndex := 0 diff --git a/internal/profiles/resolver_test.go b/internal/profiles/resolver_test.go index 41d30a4..8d03219 100644 --- a/internal/profiles/resolver_test.go +++ b/internal/profiles/resolver_test.go @@ -5,11 +5,13 @@ import ( "encoding/json" "os" "path/filepath" + "reflect" "strings" "testing" "time" "github.com/openclaw/clawscan/internal/runner" + "gopkg.in/yaml.v3" ) func TestResolveArgsUsesEmbeddedClawHubProfile(t *testing.T) { @@ -782,6 +784,133 @@ profiles: } } +func TestResolveArgsParsesUserDefinedScannerExitCodeGateRules(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: blocker + command: blocker {{target}} + gate: + blockOnExitCode: nonzero + - id: warner + command: warner {{target}} + gate: + warnOnExitCode: [1, 2, 3] +`) + + opts, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + block := opts.GateRules["blocker"].BlockOnExitCode + if block == nil || !block.Nonzero { + t.Fatalf("block rule = %#v", block) + } + warn := opts.GateRules["warner"].WarnOnExitCode + if warn == nil || !reflect.DeepEqual(warn.Codes, []int{1, 2, 3}) { + t.Fatalf("warn rule = %#v", warn) + } +} + +func TestResolveArgsAcceptsSingleExitCodeGateRule(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: blocker + command: blocker {{target}} + gate: + blockOnExitCode: 7 +`) + opts, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + if got := opts.GateRules["blocker"].BlockOnExitCode.Codes; !reflect.DeepEqual(got, []int{7}) { + t.Fatalf("codes = %#v", got) + } +} + +func TestProfileScannerExitCodeGateRulesRoundTripYAML(t *testing.T) { + scanner := ProfileScanner{ + ID: "demo", Command: "demo {{target}}", custom: true, + Gate: &ProfileScannerGate{ + BlockOnExitCode: &profileExitCodeRule{Nonzero: true}, + WarnOnExitCode: &profileExitCodeRule{Codes: []int{1, 2, 3}}, + }, + } + encoded, err := yaml.Marshal(scanner) + if err != nil { + t.Fatal(err) + } + var decoded ProfileScanner + if err := yaml.Unmarshal(encoded, &decoded); err != nil { + t.Fatalf("round trip failed for %s: %v", encoded, err) + } + if decoded.Gate == nil || !decoded.Gate.BlockOnExitCode.Nonzero || !reflect.DeepEqual(decoded.Gate.WarnOnExitCode.Codes, []int{1, 2, 3}) { + t.Fatalf("decoded scanner = %#v from %s", decoded, encoded) + } +} + +func TestResolveArgsRejectsInvalidExitCodeGateRules(t *testing.T) { + tests := []struct { + name string + rules string + want string + }{ + {name: "negative", rules: "blockOnExitCode: -1", want: "must contain only non-negative integers"}, + {name: "non integer", rules: "blockOnExitCode: nope", want: `must be a non-negative integer, a list of non-negative integers, or "nonzero"`}, + {name: "empty list", rules: "blockOnExitCode: []", want: "must not be an empty list"}, + {name: "overlap", rules: "blockOnExitCode: nonzero\n warnOnExitCode: [0, 2]", want: "blockOnExitCode and warnOnExitCode both claim exit code 2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, "version: 1\nprofiles:\n review:\n scanners:\n - id: demo\n command: demo {{target}}\n gate:\n "+test.rules+"\n") + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("err = %v", err) + } + }) + } +} + +func TestGateRuleForProfileScannerExcludedByCLIOverrideFailsBeforeScanning(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: absent-scanner + command: absent {{target}} + gate: + blockOnExitCode: nonzero +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review", "--scanner", "clawscan-static", "--sandbox", "off"}, dir) + if err != nil { + t.Fatal(err) + } + commandRunner := &profileCommandRunner{stdout: `{}`} + _, err = runner.Run(opts, runner.RunContext{Env: map[string]string{}, CommandRunner: commandRunner}) + if err == nil || err.Error() != "gate rule references scanner absent-scanner, but it was not requested" { + t.Fatalf("err = %v", err) + } + if commandRunner.command != "" { + t.Fatalf("scanner executed: %q", commandRunner.command) + } +} + func TestResolveArgsRejectsUserDefinedScannerIDCollision(t *testing.T) { dir := t.TempDir() config := filepath.Join(dir, ".clawscan.yml") @@ -1070,6 +1199,8 @@ profiles: - PLUGIN_SCANNER_TOKEN targets: - plugin + gate: + blockOnExitCode: nonzero `) opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review"}, dir) if err != nil { @@ -1088,6 +1219,9 @@ profiles: if result.Status != "skipped" || !strings.Contains(result.Error, "does not support skill targets") { t.Fatalf("result = %#v", result) } + if artifact.Gate != "pass" || len(artifact.GateRules) != 0 { + t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules) + } if commandRunner.command != "" { t.Fatalf("unsupported scanner executed: %q %#v", commandRunner.command, commandRunner.args) } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 1f40745..54926d8 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -39,6 +39,29 @@ type Options struct { JSON bool Judge *JudgeOptions Sandbox SandboxOptions + GateRules map[string]ScannerGatePolicy +} + +type ExitCodeRule struct { + Codes []int + Nonzero bool +} + +func (rule ExitCodeRule) Matches(exitCode int) bool { + if rule.Nonzero { + return exitCode != 0 + } + for _, code := range rule.Codes { + if code == exitCode { + return true + } + } + return false +} + +type ScannerGatePolicy struct { + BlockOnExitCode *ExitCodeRule + WarnOnExitCode *ExitCodeRule } type BenchmarkOptions struct { @@ -83,8 +106,9 @@ type CommandRunner interface { } type CommandOutput struct { - Stdout string - Stderr string + Stdout string + Stderr string + ExitCode *int } type Artifact struct { @@ -100,9 +124,18 @@ type Artifact struct { Env map[string]string `json:"env"` Sandbox SandboxMetadata `json:"sandbox"` Scanners map[string]ScannerResult `json:"scanners"` + Gate string `json:"gate"` + GateRules []FiredGateRule `json:"gateRules"` Judge *JudgeResult `json:"judge"` } +type FiredGateRule struct { + Scanner string `json:"scanner"` + Rule string `json:"rule"` + ExitCode int `json:"exitCode"` + Action string `json:"action"` +} + type RunTargetsResult struct { Single *Artifact Batch *BatchArtifact @@ -159,6 +192,7 @@ type ScannerResult struct { Command []string `json:"command"` Error string `json:"error"` OutputPath string `json:"outputPath,omitempty"` + ExitCode *int `json:"exitCode,omitempty"` Raw json.RawMessage `json:"raw"` } @@ -371,6 +405,9 @@ func ValidateRequirements(opts Options, env map[string]string) error { } func Run(opts Options, ctx RunContext) (Artifact, error) { + if err := validateGateRuleScanners(opts); err != nil { + return Artifact{}, err + } env := ctx.Env if env == nil { env = EnvMap(os.Environ()) @@ -440,6 +477,7 @@ func Run(opts Options, ctx RunContext) (Artifact, error) { result.DurationMs = time.Since(scannerTimerStarted).Milliseconds() artifact.Scanners[scanner] = result } + evaluateGate(&artifact, opts) if opts.Judge != nil { result, err := RunJudge(*opts.Judge, artifact, commandRunner, 20*time.Minute, env, sandbox.Mode) if err != nil { @@ -2111,7 +2149,14 @@ func (runner defaultCommandRunner) Run(command string, args []string, cwd string if ctx.Err() == context.DeadlineExceeded { err = fmt.Errorf("command timed out after %s", timeout) } - return CommandOutput{Stdout: stdout.String(), Stderr: stderr.String()}, err + output := CommandOutput{Stdout: stdout.String(), Stderr: stderr.String()} + if cmd.ProcessState != nil { + exitCode := cmd.ProcessState.ExitCode() + if exitCode >= 0 { + output.ExitCode = &exitCode + } + } + return output, err } func NewArtifact(opts Options, resolvedPath string, startedAt string, completedAt string, env map[string]string) Artifact { @@ -2145,10 +2190,54 @@ func NewArtifact(opts Options, resolvedPath string, startedAt string, completedA Env: envPresence(opts, env), Sandbox: mustSandboxMetadata(opts, env), Scanners: scanners, + Gate: "pass", + GateRules: []FiredGateRule{}, Judge: nil, } } +func validateGateRuleScanners(opts Options) error { + requested := make(map[string]bool, len(opts.Scanners)) + for _, scanner := range opts.Scanners { + requested[scanner] = true + } + var unrequested []string + for scanner := range opts.GateRules { + if !requested[scanner] { + unrequested = append(unrequested, scanner) + } + } + if len(unrequested) == 0 { + return nil + } + sort.Strings(unrequested) + return fmt.Errorf("gate rule references scanner %s, but it was not requested", unrequested[0]) +} + +func evaluateGate(artifact *Artifact, opts Options) { + for _, scanner := range opts.Scanners { + result := artifact.Scanners[scanner] + if result.Status != "completed" || result.ExitCode == nil { + continue + } + policy := opts.GateRules[scanner] + if policy.BlockOnExitCode != nil && policy.BlockOnExitCode.Matches(*result.ExitCode) { + artifact.GateRules = append(artifact.GateRules, FiredGateRule{ + Scanner: scanner, Rule: "blockOnExitCode", ExitCode: *result.ExitCode, Action: "block", + }) + artifact.Gate = "block" + } + if policy.WarnOnExitCode != nil && policy.WarnOnExitCode.Matches(*result.ExitCode) { + artifact.GateRules = append(artifact.GateRules, FiredGateRule{ + Scanner: scanner, Rule: "warnOnExitCode", ExitCode: *result.ExitCode, Action: "warn", + }) + if artifact.Gate == "pass" { + artifact.Gate = "warn" + } + } + } +} + func WriteJSON(w io.Writer, value interface{}) error { encoder := json.NewEncoder(w) encoder.SetIndent("", " ") diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 66b0726..4f2bb0e 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "reflect" "strings" "testing" "time" @@ -1121,6 +1122,25 @@ func TestArtifactConfigSourceField_FlagsOnly(t *testing.T) { } } +func TestNewArtifactAlwaysIncludesPassingGate(t *testing.T) { + opts, err := ParseArgs([]string{"./my-skill", "--scanner", "clawscan-static"}) + if err != nil { + t.Fatal(err) + } + + artifact := NewArtifact(opts, "/tmp/my-skill", "start", "complete", map[string]string{}) + if artifact.Gate != "pass" { + t.Fatalf("gate = %q", artifact.Gate) + } + raw, err := json.Marshal(artifact) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(raw, []byte(`"gate":"pass"`)) { + t.Fatalf("artifact omitted gate: %s", raw) + } +} + func TestRunWritesScannerOnlyArtifact(t *testing.T) { dir := t.TempDir() target := filepath.Join(dir, "skill") @@ -1179,6 +1199,119 @@ func TestRunIncludesDurationMsForScannerResults(t *testing.T) { assertScannerDurationJSON(t, artifact, "skillspector") } +func TestRunBlocksWhenNonzeroExitCodeRuleFires(t *testing.T) { + target := t.TempDir() + exitCode := 2 + opts := Options{ + Target: target, Scanners: []string{"clawscan-static"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, + GateRules: map[string]ScannerGatePolicy{ + "clawscan-static": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}, + }, + } + artifact, err := Run(opts, RunContext{Env: map[string]string{}, ScannerRunner: &gateScannerRunner{ + results: map[string]ScannerResult{"clawscan-static": {Status: "completed", ExitCode: &exitCode}}, + }}) + if err != nil { + t.Fatal(err) + } + if artifact.Gate != "block" { + t.Fatalf("gate = %q", artifact.Gate) + } + want := []FiredGateRule{{Scanner: "clawscan-static", Rule: "blockOnExitCode", ExitCode: 2, Action: "block"}} + if !reflect.DeepEqual(artifact.GateRules, want) { + t.Fatalf("gate rules = %#v", artifact.GateRules) + } +} + +func TestRunExitCodeGateActionsAndPrecedence(t *testing.T) { + tests := []struct { + name string + results map[string]ScannerResult + rules map[string]ScannerGatePolicy + want string + fired int + }{ + { + name: "zero does not fire nonzero", results: gateResults(0), + rules: map[string]ScannerGatePolicy{"clawscan-static": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}}, + want: "pass", + }, + { + name: "warning fires", results: gateResults(3), + rules: map[string]ScannerGatePolicy{"clawscan-static": {WarnOnExitCode: &ExitCodeRule{Codes: []int{3}}}}, + want: "warn", fired: 1, + }, + { + name: "listed code fires", results: gateResults(2), + rules: map[string]ScannerGatePolicy{"clawscan-static": {BlockOnExitCode: &ExitCodeRule{Codes: []int{1, 2, 3}}}}, + want: "block", fired: 1, + }, + { + name: "unlisted code passes", results: gateResults(4), + rules: map[string]ScannerGatePolicy{"clawscan-static": {BlockOnExitCode: &ExitCodeRule{Codes: []int{1, 2, 3}}}}, + want: "pass", + }, + { + name: "skipped scanner does not fire", results: map[string]ScannerResult{"clawscan-static": {Status: "skipped", ExitCode: intPointer(2)}}, + rules: map[string]ScannerGatePolicy{"clawscan-static": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}}, + want: "pass", + }, + { + name: "failed scanner does not fire", results: map[string]ScannerResult{"clawscan-static": {Status: "failed", ExitCode: intPointer(2)}}, + rules: map[string]ScannerGatePolicy{"clawscan-static": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}}, + want: "pass", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + target := t.TempDir() + artifact, err := Run(Options{ + Target: target, Scanners: []string{"clawscan-static"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, GateRules: test.rules, + }, RunContext{Env: map[string]string{}, ScannerRunner: &gateScannerRunner{results: test.results}}) + if err != nil { + t.Fatal(err) + } + if artifact.Gate != test.want || len(artifact.GateRules) != test.fired { + t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules) + } + }) + } +} + +func TestRunBlockGateBeatsWarnAcrossScanners(t *testing.T) { + target := t.TempDir() + artifact, err := Run(Options{ + Target: target, Scanners: []string{"clawscan-static", "skillspector"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, + GateRules: map[string]ScannerGatePolicy{ + "clawscan-static": {WarnOnExitCode: &ExitCodeRule{Codes: []int{1}}}, + "skillspector": {BlockOnExitCode: &ExitCodeRule{Codes: []int{2}}}, + }, + }, RunContext{Env: map[string]string{}, ScannerRunner: &gateScannerRunner{results: map[string]ScannerResult{ + "clawscan-static": {Status: "completed", ExitCode: intPointer(1)}, + "skillspector": {Status: "completed", ExitCode: intPointer(2)}, + }}}) + if err != nil { + t.Fatal(err) + } + if artifact.Gate != "block" || len(artifact.GateRules) != 2 { + t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules) + } +} + +func TestRunRejectsGateRuleForUnrequestedScannerBeforeScanning(t *testing.T) { + scannerRunner := &gateScannerRunner{results: gateResults(0)} + _, err := Run(Options{ + Target: t.TempDir(), Scanners: []string{"clawscan-static"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, + GateRules: map[string]ScannerGatePolicy{"absent-scanner": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}}, + }, RunContext{Env: map[string]string{}, ScannerRunner: scannerRunner}) + if err == nil || err.Error() != "gate rule references scanner absent-scanner, but it was not requested" { + t.Fatalf("err = %v", err) + } + if scannerRunner.calls != 0 { + t.Fatalf("scanner ran %d times", scannerRunner.calls) + } +} + func TestRunIncludesDurationMsForFixtureScannerResults(t *testing.T) { dir := t.TempDir() target := filepath.Join(dir, "skill") @@ -4093,6 +4226,27 @@ func (skippedScannerRunner) RunScanner(name string, target string, startedAt str }, nil } +type gateScannerRunner struct { + results map[string]ScannerResult + calls int +} + +func (runner *gateScannerRunner) RunScanner(name string, _ string, startedAt string) (ScannerResult, error) { + runner.calls++ + result := runner.results[name] + result.StartedAt = startedAt + result.CompletedAt = startedAt + return result, nil +} + +func gateResults(exitCode int) map[string]ScannerResult { + return map[string]ScannerResult{"clawscan-static": {Status: "completed", ExitCode: intPointer(exitCode)}} +} + +func intPointer(value int) *int { + return &value +} + type staticScannerRunner struct { results map[string]ScannerResult } @@ -4337,6 +4491,7 @@ type recordingCommandRunner struct { stdout string stderr string err error + exitCode *int runHook func(command string, args []string, cwd string) error } @@ -4377,7 +4532,7 @@ func (r *recordingCommandRunner) Run(command string, args []string, cwd string, if stdout == "" { stdout = "ok" } - return CommandOutput{Stdout: stdout, Stderr: r.stderr}, r.err + return CommandOutput{Stdout: stdout, Stderr: r.stderr, ExitCode: r.exitCode}, r.err } var errCommandFailed = errors.New("exit status 1") diff --git a/internal/runner/scanner_registry_test.go b/internal/runner/scanner_registry_test.go index 42d50bd..9eee4f6 100644 --- a/internal/runner/scanner_registry_test.go +++ b/internal/runner/scanner_registry_test.go @@ -171,6 +171,48 @@ func TestUserDefinedScannerPreservesValidJSONOnCommandFailure(t *testing.T) { } } +func TestUserDefinedScannerRecordsExitCode(t *testing.T) { + exitCode := 2 + adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"}, + }) + registry, err := NewScannerRegistry(adapter) + if err != nil { + t.Fatal(err) + } + commandRunner := &recordingCommandRunner{stdout: `{}`, err: errCommandFailed, exitCode: &exitCode} + result, err := (ExternalScannerRunner{ + Registry: registry, CommandRunner: commandRunner, Env: map[string]string{}, SandboxMode: SandboxModeOff, + }).RunScanner("demo", t.TempDir(), "2026-07-21T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if result.ExitCode == nil || *result.ExitCode != 2 { + t.Fatalf("exit code = %#v", result.ExitCode) + } +} + +func TestUserDefinedScannerOmitsAbnormalExitCode(t *testing.T) { + exitCode := -1 + adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"}, + }) + registry, err := NewScannerRegistry(adapter) + if err != nil { + t.Fatal(err) + } + commandRunner := &recordingCommandRunner{stdout: `{}`, err: errCommandFailed, exitCode: &exitCode} + result, err := (ExternalScannerRunner{ + Registry: registry, CommandRunner: commandRunner, Env: map[string]string{}, SandboxMode: SandboxModeOff, + }).RunScanner("demo", t.TempDir(), "2026-07-21T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if result.ExitCode != nil { + t.Fatalf("abnormal exit code recorded = %d", *result.ExitCode) + } +} + func TestUserDefinedScannerInterpolatesDollarTargetLiterally(t *testing.T) { adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"}, diff --git a/internal/runner/user_defined_scanner.go b/internal/runner/user_defined_scanner.go index b30e38e..ceef300 100644 --- a/internal/runner/user_defined_scanner.go +++ b/internal/runner/user_defined_scanner.go @@ -76,6 +76,7 @@ func (adapter userDefinedScannerAdapter) Run(runner ExternalScannerRunner, targe timeout = 20 * time.Minute } output, runErr := runner.CommandRunner.Run(shell.command, args, userDefinedScannerCWD(target), timeout) + exitCode := gateEligibleExitCode(output.ExitCode) completedAt := time.Now().UTC().Format(time.RFC3339Nano) raw := strings.TrimSpace(output.Stdout) if runErr != nil { @@ -83,26 +84,33 @@ func (adapter userDefinedScannerAdapter) Run(runner ExternalScannerRunner, targe if json.Valid([]byte(raw)) { return ScannerResult{ Status: "completed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, - Error: message, Raw: json.RawMessage(raw), + Error: message, ExitCode: exitCode, Raw: json.RawMessage(raw), }, nil } return ScannerResult{ Status: "failed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, - Error: message, + Error: message, ExitCode: exitCode, }, nil } if !json.Valid([]byte(raw)) { return ScannerResult{ Status: "failed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, - Error: fmt.Sprintf("User-defined scanner %s returned invalid JSON", adapter.config.ID), + Error: fmt.Sprintf("User-defined scanner %s returned invalid JSON", adapter.config.ID), ExitCode: exitCode, }, nil } return ScannerResult{ Status: "completed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, - Raw: json.RawMessage(raw), + ExitCode: exitCode, Raw: json.RawMessage(raw), }, nil } +func gateEligibleExitCode(exitCode *int) *int { + if exitCode == nil || *exitCode < 0 { + return nil + } + return exitCode +} + func userDefinedScannerShell(goos string, sandboxMode string) judgeShellSpec { if sandboxMode == SandboxModeDocker { return judgeShellForGOOS("linux") From bc6be59acb80f95beb5a72f3bb035bdfc87e445c Mon Sep 17 00:00:00 2001 From: Jesse Merhi <79823012+jesse-merhi@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:31:06 +1000 Subject: [PATCH 3/3] fix(profiles): restrict user-defined scanner IDs to lowercase Scanner evidence is written to a file named after the scanner ID, and that name is lowercased. Allowing uppercase IDs let two case-distinct IDs (Foo and foo) pass validation yet collide on the same evidence file, silently overwriting each other. Restrict IDs to lowercase and cap length at 64 characters, since IDs are used as file names. --- internal/profiles/resolver.go | 15 ++++++++++-- internal/profiles/resolver_test.go | 37 +++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/internal/profiles/resolver.go b/internal/profiles/resolver.go index 9cb0d54..462113e 100644 --- a/internal/profiles/resolver.go +++ b/internal/profiles/resolver.go @@ -283,9 +283,17 @@ type cliIntent struct { } var judgePathPlaceholderPattern = regexp.MustCompile(`\{\{\s*(prompt|output_schema):([^}]+)\}\}`) -var scannerIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]*$`) + +// scannerIDPattern restricts user-defined scanner IDs to lowercase. IDs are +// lowercased when used as evidence file names, so allowing uppercase would let +// two case-distinct IDs (Foo and foo) collide on the same output file. +var scannerIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`) var scannerTargetPlaceholderPattern = regexp.MustCompile(`\{\{\s*target\s*\}\}`) +// maxScannerIDLength bounds user-defined scanner IDs so .json evidence file +// names stay within filesystem limits. +const maxScannerIDLength = 64 + type ResolvedRunSet struct { Options []runner.Options OutputPath string @@ -925,7 +933,10 @@ func validateProfile(name string, profile Profile) error { return fmt.Errorf("User-defined scanner in profile %s must include a non-empty id", name) } if scanner.custom && !scannerIDPattern.MatchString(scanner.ID) { - return fmt.Errorf("User-defined scanner %s in profile %s has invalid id; use letters, digits, underscores, and hyphens, starting with a letter or digit", scanner.ID, name) + return fmt.Errorf("User-defined scanner %s in profile %s has invalid id; use lowercase letters, digits, underscores, and hyphens, starting with a letter or digit", scanner.ID, name) + } + if scanner.custom && len(scanner.ID) > maxScannerIDLength { + return fmt.Errorf("User-defined scanner id in profile %s is %d characters; scanner IDs are used as file names and must be at most %d characters", name, len(scanner.ID), maxScannerIDLength) } if scanner.custom && strings.TrimSpace(scanner.Command) == "" { return fmt.Errorf("User-defined scanner %s in profile %s must include a non-empty command", scanner.ID, name) diff --git a/internal/profiles/resolver_test.go b/internal/profiles/resolver_test.go index 8d03219..afccde4 100644 --- a/internal/profiles/resolver_test.go +++ b/internal/profiles/resolver_test.go @@ -981,7 +981,42 @@ profiles: `) _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) - if err == nil || err.Error() != "User-defined scanner foo=bar in profile review has invalid id; use letters, digits, underscores, and hyphens, starting with a letter or digit" { + if err == nil || err.Error() != "User-defined scanner foo=bar in profile review has invalid id; use lowercase letters, digits, underscores, and hyphens, starting with a letter or digit" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsRejectsUppercaseUserDefinedScannerID(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: Foo + command: scanner {{target}} +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner Foo in profile review has invalid id; use lowercase letters, digits, underscores, and hyphens, starting with a letter or digit" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsRejectsOversizedUserDefinedScannerID(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + longID := strings.Repeat("a", 65) + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: `+longID+` + command: scanner {{target}} +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner id in profile review is 65 characters; scanner IDs are used as file names and must be at most 64 characters" { t.Fatalf("err = %v", err) } }