From f7deb57e495ff4a72a14bd0519278ab2a192d767 Mon Sep 17 00:00:00 2001 From: Jesse Merhi <79823012+jesse-merhi@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:42:40 +1000 Subject: [PATCH 1/2] feat(runner): support user-defined scanners in profile config Scanner entries in a profile may now be objects declaring id, command, env (required variable names), and targets. Custom scanners register into a run-local registry and execute through the standard sandboxed command path with {{target}} passed positionally to prevent shell injection. Raw stdout is preserved as artifact evidence; env presence is recorded as present/missing only. --- cmd/clawscan/main.go | 2 +- docs/scanners.md | 47 +++ internal/profiles/registry.go | 4 +- internal/profiles/registry_test.go | 12 +- internal/profiles/resolver.go | 193 +++++++++++- internal/profiles/resolver_test.go | 367 +++++++++++++++++++++++ internal/runner/runner.go | 22 +- internal/runner/sandbox.go | 4 +- internal/runner/scanner_registry.go | 10 + internal/runner/scanner_registry_test.go | 63 ++++ internal/runner/target.go | 8 +- internal/runner/user_defined_scanner.go | 125 ++++++++ 12 files changed, 833 insertions(+), 24 deletions(-) create mode 100644 internal/runner/user_defined_scanner.go diff --git a/cmd/clawscan/main.go b/cmd/clawscan/main.go index dfcf661..97ae10b 100644 --- a/cmd/clawscan/main.go +++ b/cmd/clawscan/main.go @@ -260,7 +260,7 @@ func printProfileCatalog(w io.Writer, catalog profiles.ProfileCatalog, cwd strin if info.Profile.Judge != nil { judge = "configured" } - fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", info.ID, displayProfileSource(info.Source, cwd), strings.Join(info.Profile.Scanners, ", "), judge) + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", info.ID, displayProfileSource(info.Source, cwd), strings.Join(info.Profile.ScannerIDs(), ", "), judge) } _ = tw.Flush() } diff --git a/docs/scanners.md b/docs/scanners.md index 8666a87..cebe5d8 100644 --- a/docs/scanners.md +++ b/docs/scanners.md @@ -17,6 +17,53 @@ clawscan scanners clawscan scanners skillspector ``` +## User-defined scanners + +A trusted config can mix built-in scanner IDs with user-defined command +scanners. The config schema uses the existing `profiles..scanners` list: + +```yaml +version: 1 + +profiles: + review: + scanners: + - clawscan-static + - id: my-scanner + command: my-scanner --json {{target}} + env: + - MY_SCANNER_TOKEN + targets: + - skill + - plugin +``` + +String entries select built-in scanners. Object entries define a scanner for +that config-backed run and accept these fields: + +| Field | Required | Meaning | +| --- | --- | --- | +| `id` | yes | Scanner ID using letters, digits, `_`, and `-`, starting with a letter or digit. It must not match a built-in scanner ID. | +| `command` | yes | Shell command to execute. Unquoted `{{target}}` is replaced with the safely passed resolved target; do not wrap the placeholder in shell quotes. | +| `env` | no | Required environment variable names. Values stay in the process environment and are never stored in the config or artifact. | +| `targets` | no | Supported target kinds: `skill`, `plugin`, and/or `url`. Defaults to `skill` and `url`. | + +The command must write JSON to stdout. ClawScan preserves valid stdout as the +scanner's raw evidence; empty or non-JSON stdout produces a failed scanner +result. Required environment variables are checked before any scanner starts. +Artifacts record each requirement as only `present` or `missing`. + +User-defined scanners use the same execution path as built-in command-backed +scanners. They run in the Docker sandbox by default, and declared `env` names +are added to its environment allowlist. Use `--sandbox off` only when you +intentionally want the command to run on the host. User-defined scanners are +local to the resolved config and do not appear in the built-in `clawscan +scanners` catalog. + +> **Trust boundary:** only load user-defined scanners from config files you +> control. A scanner entry is executable code. The default sandbox limits its +> host access, but does not make an untrusted command safe to run. + ## Target kinds Clawscan classifies each explicit target before dispatching scanners and records diff --git a/internal/profiles/registry.go b/internal/profiles/registry.go index 1ed0721..0bcaa56 100644 --- a/internal/profiles/registry.go +++ b/internal/profiles/registry.go @@ -28,8 +28,8 @@ func NewProfileRegistry(profiles map[string]resolvedProfile) (ProfileRegistry, e return ProfileRegistry{}, err } for _, scanner := range profile.profile.Scanners { - if !runner.DefaultScannerRegistry().Contains(scanner) { - return ProfileRegistry{}, unknownScannerInProfileError(id, scanner) + if !scanner.custom && !runner.DefaultScannerRegistry().Contains(scanner.ID) { + return ProfileRegistry{}, unknownScannerInProfileError(id, scanner.ID) } } registry.profiles[id] = profile diff --git a/internal/profiles/registry_test.go b/internal/profiles/registry_test.go index 24e5714..bcfc0a6 100644 --- a/internal/profiles/registry_test.go +++ b/internal/profiles/registry_test.go @@ -7,8 +7,8 @@ import ( func TestProfileRegistryReturnsSortedIDs(t *testing.T) { registry, err := NewProfileRegistry(map[string]resolvedProfile{ - "review": {profile: Profile{Scanners: []string{"snyk"}}}, - "clawhub": {profile: Profile{Scanners: []string{"skillspector"}}}, + "review": {profile: Profile{Scanners: []ProfileScanner{{ID: "snyk"}}}}, + "clawhub": {profile: Profile{Scanners: []ProfileScanner{{ID: "skillspector"}}}}, }) if err != nil { t.Fatal(err) @@ -26,7 +26,7 @@ func TestDefaultProfileRegistryContainsEmbeddedBuiltIns(t *testing.T) { if !ok { t.Fatal("missing clawhub profile") } - if got := strings.Join(clawhub.profile.Scanners, ","); got != "skillspector,clawscan-static" { + if got := strings.Join(profileScannerIDs(clawhub.profile.Scanners), ","); got != "skillspector,clawscan-static" { t.Fatalf("clawhub scanners = %q", got) } if clawhub.configDir != "clawhub" { @@ -46,7 +46,7 @@ func TestDefaultProfileRegistryContainsEmbeddedBuiltIns(t *testing.T) { if !ok { t.Fatal("missing clawhub-aig profile") } - if got := strings.Join(candidate.profile.Scanners, ","); got != "skillspector,aig" { + if got := strings.Join(profileScannerIDs(candidate.profile.Scanners), ","); got != "skillspector,aig" { t.Fatalf("clawhub-aig scanners = %q", got) } if candidate.configDir != "clawhub" { @@ -71,7 +71,7 @@ func TestDefaultProfileRegistryContainsEmbeddedBuiltIns(t *testing.T) { func TestProfileRegistryRejectsUnknownScannerReferences(t *testing.T) { _, err := NewProfileRegistry(map[string]resolvedProfile{ - "bad": {profile: Profile{Scanners: []string{"missing-scanner"}}}, + "bad": {profile: Profile{Scanners: []ProfileScanner{{ID: "missing-scanner"}}}}, }) if err == nil || err.Error() != "Profile bad references unknown scanner: missing-scanner" { t.Fatalf("err = %v", err) @@ -91,7 +91,7 @@ func TestInspectProfilesReturnsBuiltIns(t *testing.T) { if !ok { t.Fatal("missing clawhub profile") } - if got := strings.Join(clawhub.Profile.Scanners, ","); got != "skillspector,clawscan-static" { + if got := strings.Join(profileScannerIDs(clawhub.Profile.Scanners), ","); got != "skillspector,clawscan-static" { t.Fatalf("clawhub scanners = %q", got) } if clawhub.Source != "built-in" { diff --git a/internal/profiles/resolver.go b/internal/profiles/resolver.go index 9afadc5..0c11add 100644 --- a/internal/profiles/resolver.go +++ b/internal/profiles/resolver.go @@ -30,7 +30,7 @@ type Config struct { } type Profile struct { - Scanners []string `yaml:"scanners"` + Scanners []ProfileScanner `yaml:"scanners"` ScannerResults map[string]string `yaml:"scannerResults,omitempty"` Output string `yaml:"output,omitempty"` JSON bool `yaml:"json,omitempty"` @@ -38,6 +38,95 @@ type Profile struct { Judge *Judge `yaml:"judge,omitempty"` } +func (profile Profile) ScannerIDs() []string { + return profileScannerIDs(profile.Scanners) +} + +type ProfileScanner struct { + ID string + Command string + Env []string + Targets []string + custom bool +} + +func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error { + switch node.Kind { + case yaml.ScalarNode: + if err := node.Decode(&scanner.ID); err != nil { + return err + } + return nil + case yaml.MappingNode: + for index := 0; index < len(node.Content); index += 2 { + switch node.Content[index].Value { + case "id", "command", "env", "targets": + default: + return fmt.Errorf("field %s not found in type profiles.ProfileScanner", node.Content[index].Value) + } + } + var value struct { + ID string `yaml:"id"` + Command string `yaml:"command"` + Env []string `yaml:"env,omitempty"` + Targets []string `yaml:"targets,omitempty"` + } + if err := node.Decode(&value); err != nil { + return err + } + scanner.ID = value.ID + scanner.Command = value.Command + scanner.Env = value.Env + scanner.Targets = value.Targets + scanner.custom = true + return nil + default: + return fmt.Errorf("scanner entry must be a string or object") + } +} + +func (scanner ProfileScanner) MarshalYAML() (interface{}, error) { + if !scanner.custom { + return scanner.ID, nil + } + return struct { + ID string `yaml:"id"` + Command string `yaml:"command"` + Env []string `yaml:"env,omitempty"` + Targets []string `yaml:"targets,omitempty"` + }{scanner.ID, scanner.Command, scanner.Env, scanner.Targets}, nil +} + +func profileScannerIDs(scanners []ProfileScanner) []string { + ids := make([]string, 0, len(scanners)) + for _, scanner := range scanners { + ids = append(ids, scanner.ID) + } + return ids +} + +func profileScannerRegistry(scanners []ProfileScanner) (runner.ScannerRegistry, error) { + registry := runner.DefaultScannerRegistry() + for _, scanner := range scanners { + if !scanner.custom { + continue + } + targets := append([]string(nil), scanner.Targets...) + if len(targets) == 0 { + targets = []string{"skill", "url"} + } + adapter := runner.NewUserDefinedScanner(runner.UserDefinedScannerConfig{ + ID: scanner.ID, Command: scanner.Command, Env: scanner.Env, Targets: targets, + }) + var err error + registry, err = registry.WithAdapters(adapter) + if err != nil { + return runner.ScannerRegistry{}, err + } + } + return registry, nil +} + type Sandbox struct { Mode string `yaml:"mode,omitempty"` Image string `yaml:"image,omitempty"` @@ -90,6 +179,8 @@ type cliIntent struct { } var judgePathPlaceholderPattern = regexp.MustCompile(`\{\{\s*(prompt|output_schema):([^}]+)\}\}`) +var scannerIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]*$`) +var scannerTargetPlaceholderPattern = regexp.MustCompile(`\{\{\s*target\s*\}\}`) type ResolvedRunSet struct { Options []runner.Options @@ -182,7 +273,11 @@ func resolveRunSetIntent(intent cliIntent, cwd string) (ResolvedRunSet, error) { if err != nil { return ResolvedRunSet{}, err } - opts, err := runner.ParseArgs(finalArgs) + scannerRegistry, err := profileScannerRegistry(selected.profile.Scanners) + if err != nil { + return ResolvedRunSet{}, err + } + opts, err := runner.ParseArgsWithRegistry(finalArgs, scannerRegistry) if err != nil { return ResolvedRunSet{}, err } @@ -279,7 +374,11 @@ func resolveAllConfigProfiles(intent cliIntent, cwd string) (ResolvedRunSet, err if err != nil { return ResolvedRunSet{}, err } - opts, err := runner.ParseArgs(finalArgs) + scannerRegistry, err := profileScannerRegistry(selected.profile.Scanners) + if err != nil { + return ResolvedRunSet{}, err + } + opts, err := runner.ParseArgsWithRegistry(finalArgs, scannerRegistry) if err != nil { return ResolvedRunSet{}, err } @@ -610,7 +709,7 @@ func buildRunnerArgs(intent cliIntent, selected resolvedProfile, profileName str } profile := selected.profile - scanners := append([]string{}, profile.Scanners...) + scanners := profileScannerIDs(profile.Scanners) if len(intent.scanners) > 0 { scanners = append([]string{}, intent.scanners...) } @@ -716,14 +815,94 @@ func resolveJudgePaths(command string, configDir string) string { func validateProfile(name string, profile Profile) error { seen := map[string]bool{} for _, scanner := range profile.Scanners { - if seen[scanner] { - return fmt.Errorf("Duplicate scanner in profile %s: %s", name, scanner) + if scanner.custom && strings.TrimSpace(scanner.ID) == "" { + return fmt.Errorf("User-defined scanner in profile %s must include a non-empty id", name) + } + if scanner.custom && !scannerIDPattern.MatchString(scanner.ID) { + return fmt.Errorf("User-defined scanner %s in profile %s has invalid id; use letters, digits, underscores, and hyphens, starting with a letter or digit", scanner.ID, name) + } + if scanner.custom && strings.TrimSpace(scanner.Command) == "" { + return fmt.Errorf("User-defined scanner %s in profile %s must include a non-empty command", scanner.ID, name) + } + if scanner.custom && !scannerTargetPlaceholdersAreUnquoted(scanner.Command) { + return fmt.Errorf("User-defined scanner %s in profile %s must use {{target}} outside shell quotes", scanner.ID, name) + } + if scanner.custom && runner.DefaultScannerRegistry().Contains(scanner.ID) { + return fmt.Errorf("User-defined scanner %s collides with a built-in scanner ID", scanner.ID) + } + for _, target := range scanner.Targets { + switch target { + case "skill", "plugin", "url": + default: + return fmt.Errorf("User-defined scanner %s in profile %s has unsupported target kind: %s", scanner.ID, name, target) + } + } + if seen[scanner.ID] { + return fmt.Errorf("Duplicate scanner in profile %s: %s", name, scanner.ID) } - seen[scanner] = true + seen[scanner.ID] = true } return nil } +func scannerTargetPlaceholdersAreUnquoted(command string) bool { + matches := scannerTargetPlaceholderPattern.FindAllStringIndex(command, -1) + matchIndex := 0 + quote := byte(0) + escaped := false + comment := false + for index := 0; index < len(command); index++ { + if matchIndex < len(matches) && index == matches[matchIndex][0] { + if !comment && quote != 0 { + return false + } + index = matches[matchIndex][1] - 1 + matchIndex++ + continue + } + character := command[index] + if comment { + if character == '\n' { + comment = false + } + continue + } + if escaped { + escaped = false + continue + } + if character == '\\' && quote != '\'' { + escaped = true + continue + } + if quote == 0 { + if character == '#' && shellCommentCanStart(command, index) { + comment = true + continue + } + switch character { + case '\'', '"', '`': + quote = character + } + } else if character == quote { + quote = 0 + } + } + return true +} + +func shellCommentCanStart(command string, index int) bool { + if index == 0 { + return true + } + switch command[index-1] { + case ' ', '\t', '\r', '\n', ';', '|', '&', '(', ')': + return true + default: + return false + } +} + func unknownProfileError(profile string, available []string) error { return fmt.Errorf("Unknown profile: %s (available: %s)", profile, strings.Join(available, ", ")) } diff --git a/internal/profiles/resolver_test.go b/internal/profiles/resolver_test.go index 55e3b48..41d30a4 100644 --- a/internal/profiles/resolver_test.go +++ b/internal/profiles/resolver_test.go @@ -1,10 +1,13 @@ package profiles import ( + "bytes" + "encoding/json" "os" "path/filepath" "strings" "testing" + "time" "github.com/openclaw/clawscan/internal/runner" ) @@ -572,6 +575,12 @@ func TestResolveArgsRejectsUnsupportedVersionAndUnknownFields(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "field defaultProfile not found") { t.Fatalf("err = %v", err) } + + writeFile(t, config, "version: 1\nprofiles:\n review:\n scanners:\n - id: custom\n command: custom {{target}}\n token: example\n") + _, err = ResolveArgs([]string{"./skill", "--config", config}, dir) + if err == nil || !strings.Contains(err.Error(), "field token not found") { + t.Fatalf("err = %v", err) + } } func TestResolveArgsUnknownProfileListsAvailableProfiles(t *testing.T) { @@ -738,6 +747,352 @@ profiles: } } +func TestResolveArgsParsesUserDefinedScannerAlongsideBuiltIn(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - clawscan-static + - id: my-scanner + command: my-scanner --json {{target}} + env: + - MY_SCANNER_TOKEN + targets: + - plugin +`) + + opts, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(opts.Scanners, ","); got != "clawscan-static,my-scanner" { + t.Fatalf("scanners = %q", got) + } + adapter, ok := opts.ScannerRegistry.Adapter("my-scanner") + if !ok { + t.Fatal("custom scanner missing from run registry") + } + if adapter.SupportsTargetKind("skill") { + t.Fatal("plugin-only custom scanner unexpectedly supports skill targets") + } + if !adapter.SupportsTargetKind("plugin") { + t.Fatal("plugin-only custom scanner does not support plugin targets") + } +} + +func TestResolveArgsRejectsUserDefinedScannerIDCollision(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: clawscan-static + command: custom-static {{target}} +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner clawscan-static collides with a built-in scanner ID" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsRejectsIncompleteUserDefinedScanner(t *testing.T) { + for _, test := range []struct { + name string + entry string + wantErr string + }{ + {name: "missing id", entry: "command: scanner {{target}}", wantErr: "User-defined scanner in profile review must include a non-empty id"}, + {name: "missing command", entry: "id: my-scanner", wantErr: "User-defined scanner my-scanner in profile review must include a non-empty command"}, + } { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, "version: 1\nprofiles:\n review:\n scanners:\n - "+test.entry+"\n") + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != test.wantErr { + t.Fatalf("err = %v", err) + } + }) + } +} + +func TestResolveArgsRejectsUnknownUserDefinedScannerTarget(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: my-scanner + command: my-scanner {{target}} + targets: + - package +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner my-scanner in profile review has unsupported target kind: package" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsRejectsInvalidUserDefinedScannerID(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: foo=bar + command: scanner {{target}} +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner foo=bar in profile review has invalid id; use letters, digits, underscores, and hyphens, starting with a letter or digit" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsRejectsQuotedUserDefinedScannerTargetPlaceholder(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: my-scanner + command: my-scanner "{{target}}" +`) + + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || err.Error() != "User-defined scanner my-scanner in profile review must use {{target}} outside shell quotes" { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsAllowsApostropheInShellCommentBeforeTargetPlaceholder(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: my-scanner + command: |- + # don't quote the target here + my-scanner {{target}} +`) + + if _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir); err != nil { + t.Fatal(err) + } +} + +func TestResolveArgsSupportsAliasedScannerEntries(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + templates: + scanners: + - &builtin clawscan-static + - &custom + id: my-scanner + command: my-scanner {{target}} + review: + scanners: + - *builtin + - *custom +`) + + opts, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(opts.Scanners, ","); got != "clawscan-static,my-scanner" { + t.Fatalf("scanners = %q", got) + } +} + +func TestUserDefinedScannerRunsThroughDockerAndPreservesRawJSON(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: fixture-scanner + command: fixture-scan --json {{target}} +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + adapter, ok := opts.ScannerRegistry.Adapter("fixture-scanner") + if !ok || !adapter.SupportsTargetKind("skill") || !adapter.SupportsTargetKind("url") || adapter.SupportsTargetKind("plugin") { + t.Fatalf("default target support is incorrect: adapter=%#v ok=%v", adapter, ok) + } + commandRunner := &profileCommandRunner{stdout: `{"scanner":"fixture","findings":[]}`} + artifact, err := runner.Run(opts, runner.RunContext{ + Env: map[string]string{}, + HostCommandRunner: commandRunner, + DockerAvailability: func() error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + result := artifact.Scanners["fixture-scanner"] + if result.Status != "completed" { + t.Fatalf("result = %#v", result) + } + if !bytes.Equal(result.Raw, []byte(`{"scanner":"fixture","findings":[]}`)) { + t.Fatalf("raw = %s", result.Raw) + } + if commandRunner.command != "docker" { + t.Fatalf("command = %q", commandRunner.command) + } + joined := strings.Join(commandRunner.args, " ") + if !strings.Contains(joined, "fixture-scan --json") || !strings.Contains(joined, target) || strings.Contains(joined, "{{target}}") { + t.Fatalf("docker args = %#v", commandRunner.args) + } +} + +func TestUserDefinedScannerMissingEnvFailsBeforeExecution(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: credentialed-scanner + command: credentialed-scan {{target}} + env: + - MY_SCANNER_TOKEN +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + commandRunner := &profileCommandRunner{stdout: `{}`} + _, err = runner.Run(opts, runner.RunContext{ + Env: map[string]string{}, + HostCommandRunner: commandRunner, + DockerAvailability: func() error { return nil }, + }) + if err == nil || !strings.Contains(err.Error(), "MY_SCANNER_TOKEN") { + t.Fatalf("err = %v", err) + } + if commandRunner.command != "" { + t.Fatalf("scanner executed before env validation: %q %#v", commandRunner.command, commandRunner.args) + } +} + +func TestUserDefinedScannerEnvIsAllowlistedAndRecordedByPresenceOnly(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: credentialed-scanner + command: credentialed-scan {{target}} + env: + - MY_SCANNER_TOKEN +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + missingArtifact := runner.NewArtifact(opts, target, "start", "complete", map[string]string{}) + if missingArtifact.Env["MY_SCANNER_TOKEN"] != "missing" { + t.Fatalf("missing env = %#v", missingArtifact.Env) + } + + const envValue = "test-value-123" + commandRunner := &profileCommandRunner{stdout: `{}`} + artifact, err := runner.Run(opts, runner.RunContext{ + Env: map[string]string{"MY_SCANNER_TOKEN": envValue}, + HostCommandRunner: commandRunner, + DockerAvailability: func() error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + if artifact.Env["MY_SCANNER_TOKEN"] != "present" { + t.Fatalf("env = %#v", artifact.Env) + } + encoded, err := json.Marshal(artifact) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(encoded, []byte(envValue)) { + t.Fatalf("artifact leaked env value: %s", encoded) + } + joined := strings.Join(commandRunner.args, "\x00") + if !strings.Contains(joined, "\x00-e\x00MY_SCANNER_TOKEN\x00") { + t.Fatalf("docker args missing env allowlist name: %#v", commandRunner.args) + } + if strings.Contains(joined, envValue) { + t.Fatalf("docker args leaked env value: %#v", commandRunner.args) + } +} + +func TestUserDefinedPluginScannerSkipsSkillTarget(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: plugin-scanner + command: plugin-scan {{target}} + env: + - PLUGIN_SCANNER_TOKEN + targets: + - plugin +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + commandRunner := &profileCommandRunner{stdout: `{}`} + artifact, err := runner.Run(opts, runner.RunContext{ + Env: map[string]string{}, + HostCommandRunner: commandRunner, + DockerAvailability: func() error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + result := artifact.Scanners["plugin-scanner"] + if result.Status != "skipped" || !strings.Contains(result.Error, "does not support skill targets") { + t.Fatalf("result = %#v", result) + } + if commandRunner.command != "" { + t.Fatalf("unsupported scanner executed: %q %#v", commandRunner.command, commandRunner.args) + } +} + func TestResolveArgsRejectsProfileWithoutScannersUnlessCLIOverrides(t *testing.T) { dir := t.TempDir() config := filepath.Join(dir, ".clawscan.yml") @@ -770,3 +1125,15 @@ func writeFile(t *testing.T, path string, content string) { t.Fatal(err) } } + +type profileCommandRunner struct { + command string + args []string + stdout string +} + +func (commandRunner *profileCommandRunner) Run(command string, args []string, _ string, _ time.Duration) (runner.CommandOutput, error) { + commandRunner.command = command + commandRunner.args = append([]string(nil), args...) + return runner.CommandOutput{Stdout: commandRunner.stdout}, nil +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c8b735a..1f40745 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -33,6 +33,7 @@ type Options struct { ContextPath string Benchmark *BenchmarkOptions Scanners []string + ScannerRegistry ScannerRegistry ScannerResultPaths map[string]string OutputPath string JSON bool @@ -236,7 +237,12 @@ func NewBenchmarkOptions(id string, split string, limit int, offset int, predict } func ParseArgs(args []string) (Options, error) { + return ParseArgsWithRegistry(args, DefaultScannerRegistry()) +} + +func ParseArgsWithRegistry(args []string, registry ScannerRegistry) (Options, error) { opts := Options{ScannerResultPaths: map[string]string{}} + opts.ScannerRegistry = registry start := 0 if len(args) > 0 && !strings.HasPrefix(args[0], "--") { opts.Target = args[0] @@ -251,7 +257,7 @@ func ParseArgs(args []string) (Options, error) { if err != nil { return Options{}, err } - if !DefaultScannerRegistry().Contains(value) { + if !registry.Contains(value) { return Options{}, fmt.Errorf("Unknown scanner: %s", value) } opts.Scanners = append(opts.Scanners, value) @@ -279,7 +285,7 @@ func ParseArgs(args []string) (Options, error) { if !ok || scanner == "" || path == "" { return Options{}, errors.New("Expected --scanner-result value as scanner=path") } - if !DefaultScannerRegistry().Contains(scanner) { + if !registry.Contains(scanner) { return Options{}, fmt.Errorf("Unknown scanner: %s", scanner) } opts.ScannerResultPaths[scanner] = path @@ -403,6 +409,7 @@ func Run(opts Options, ctx RunContext) (Artifact, error) { Profile: opts.Profile, TargetKind: target.kind, TargetID: target.id, + Registry: registryForOptions(opts), SandboxMode: scannerSandboxMode, SkillSpectorCommand: ctx.SkillSpectorCommand, VirusTotalHTTPClient: ctx.VirusTotalHTTPClient, @@ -826,7 +833,7 @@ func scannerResult(opts Options, scanner string, target resolvedTarget, startedA Raw: json.RawMessage(raw), }, nil } - if !scannerSupportsTargetKind(scanner, target.kind) { + if !scannerSupportsTargetKindInRegistry(registryForOptions(opts), scanner, target.kind) { return unsupportedTargetKindResult(scanner, target.kind, startedAt), nil } return scannerRunner.RunScanner(scanner, target.resolvedPath, startedAt) @@ -2237,13 +2244,20 @@ func requirements(opts Options, env map[string]string) []EnvRequirement { if opts.ScannerResultPaths[scanner] != "" { continue } - if adapter, ok := DefaultScannerRegistry().Adapter(scanner); ok { + if adapter, ok := registryForOptions(opts).Adapter(scanner); ok { reqs = append(reqs, adapter.Requirements(env)...) } } return dedupe(reqs) } +func registryForOptions(opts Options) ScannerRegistry { + if opts.ScannerRegistry.isZero() { + return DefaultScannerRegistry() + } + return opts.ScannerRegistry +} + func envPresence(opts Options, env map[string]string) map[string]string { out := map[string]string{} for _, req := range requirements(opts, env) { diff --git a/internal/runner/sandbox.go b/internal/runner/sandbox.go index ee96e8a..13e088f 100644 --- a/internal/runner/sandbox.go +++ b/internal/runner/sandbox.go @@ -227,7 +227,7 @@ func sandboxEnvNames(opts Options, env map[string]string) []string { if opts.ScannerResultPaths[scanner] != "" { continue } - adapter, ok := DefaultScannerRegistry().Adapter(scanner) + adapter, ok := registryForOptions(opts).Adapter(scanner) if !ok { continue } @@ -257,7 +257,7 @@ func requiresCommandExecution(opts Options) bool { if opts.ScannerResultPaths[scanner] != "" { continue } - adapter, ok := DefaultScannerRegistry().Adapter(scanner) + adapter, ok := registryForOptions(opts).Adapter(scanner) if !ok { continue } diff --git a/internal/runner/scanner_registry.go b/internal/runner/scanner_registry.go index 1df7776..1cce62a 100644 --- a/internal/runner/scanner_registry.go +++ b/internal/runner/scanner_registry.go @@ -33,6 +33,16 @@ type ScannerRegistry struct { adapters map[string]ScannerAdapter } +func (registry ScannerRegistry) WithAdapters(adapters ...ScannerAdapter) (ScannerRegistry, error) { + all := make([]ScannerAdapter, 0, len(registry.adapters)+len(adapters)) + for _, id := range registry.IDs() { + adapter, _ := registry.Adapter(id) + all = append(all, adapter) + } + all = append(all, adapters...) + return NewScannerRegistry(all...) +} + func NewScannerRegistry(adapters ...ScannerAdapter) (ScannerRegistry, error) { registry := ScannerRegistry{adapters: map[string]ScannerAdapter{}} for _, adapter := range adapters { diff --git a/internal/runner/scanner_registry_test.go b/internal/runner/scanner_registry_test.go index b44c921..42d50bd 100644 --- a/internal/runner/scanner_registry_test.go +++ b/internal/runner/scanner_registry_test.go @@ -3,6 +3,7 @@ package runner import ( "encoding/json" "errors" + "path/filepath" "strings" "testing" ) @@ -147,6 +148,68 @@ func TestExternalScannerRunnerDispatchesThroughRegistry(t *testing.T) { } } +func TestUserDefinedScannerPreservesValidJSONOnCommandFailure(t *testing.T) { + adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"}, + }) + registry, err := NewScannerRegistry(adapter) + if err != nil { + t.Fatal(err) + } + commandRunner := &recordingCommandRunner{stdout: `{"findings":["detected"]}`, stderr: "findings detected", err: errCommandFailed} + result, err := (ExternalScannerRunner{ + Registry: registry, CommandRunner: commandRunner, Env: map[string]string{}, SandboxMode: SandboxModeOff, + }).RunScanner("demo", t.TempDir(), "2026-07-21T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if result.Status != "completed" || string(result.Raw) != `{"findings":["detected"]}` { + t.Fatalf("result = %#v", result) + } + if !strings.Contains(result.Error, "findings detected") { + t.Fatalf("error = %q", result.Error) + } +} + +func TestUserDefinedScannerInterpolatesDollarTargetLiterally(t *testing.T) { + adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"}, + }) + registry, err := NewScannerRegistry(adapter) + if err != nil { + t.Fatal(err) + } + commandRunner := &recordingCommandRunner{stdout: `{}`} + target := filepath.Join(t.TempDir(), "skill-$USER") + result, err := (ExternalScannerRunner{ + Registry: registry, CommandRunner: commandRunner, Env: map[string]string{}, SandboxMode: SandboxModeOff, + }).RunScanner("demo", target, "2026-07-21T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if result.Status != "completed" { + t.Fatalf("result = %#v", result) + } + if len(commandRunner.calls) != 1 { + t.Fatalf("calls = %#v", commandRunner.calls) + } + args := commandRunner.calls[0].args + if len(args) != 4 || !strings.Contains(args[1], `"$1"`) || strings.Contains(args[1], target) || args[3] != target { + t.Fatalf("target was not passed as a separate shell argument: %#v", args) + } +} + +func TestUserDefinedScannerUsesContainerShellInDockerMode(t *testing.T) { + dockerShell := userDefinedScannerShell("windows", SandboxModeDocker) + if dockerShell.command != "/bin/sh" || strings.Join(dockerShell.args, " ") != "-c" { + t.Fatalf("docker shell = %#v", dockerShell) + } + hostShell := userDefinedScannerShell("windows", SandboxModeOff) + if hostShell.command != "cmd.exe" || strings.Join(hostShell.args, " ") != "/C" { + t.Fatalf("host shell = %#v", hostShell) + } +} + func TestScannerAdapterRequirementsFeedValidation(t *testing.T) { requirements := stubScannerAdapter{ id: "demo", diff --git a/internal/runner/target.go b/internal/runner/target.go index b817bf6..d800d51 100644 --- a/internal/runner/target.go +++ b/internal/runner/target.go @@ -253,7 +253,7 @@ func isURLTarget(input string) bool { func runnableScanners(opts Options, kind string) []string { var out []string for _, scanner := range opts.Scanners { - if opts.ScannerResultPaths[scanner] == "" && !scannerSupportsTargetKind(scanner, kind) { + if opts.ScannerResultPaths[scanner] == "" && !scannerSupportsTargetKindInRegistry(registryForOptions(opts), scanner, kind) { continue } out = append(out, scanner) @@ -265,7 +265,11 @@ func runnableScanners(opts Options, kind string) []string { // target of the given kind. Unknown scanner IDs are permitted here so the // scanner runner can still emit its own skipped result for them. func scannerSupportsTargetKind(scanner string, kind string) bool { - adapter, ok := DefaultScannerRegistry().Adapter(scanner) + return scannerSupportsTargetKindInRegistry(DefaultScannerRegistry(), scanner, kind) +} + +func scannerSupportsTargetKindInRegistry(registry ScannerRegistry, scanner string, kind string) bool { + adapter, ok := registry.Adapter(scanner) if !ok { return true } diff --git a/internal/runner/user_defined_scanner.go b/internal/runner/user_defined_scanner.go new file mode 100644 index 0000000..b30e38e --- /dev/null +++ b/internal/runner/user_defined_scanner.go @@ -0,0 +1,125 @@ +package runner + +import ( + "encoding/json" + "fmt" + "net/url" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" +) + +type UserDefinedScannerConfig struct { + ID string + Command string + Env []string + Targets []string +} + +func NewUserDefinedScanner(config UserDefinedScannerConfig) ScannerAdapter { + targets := make(map[string]bool, len(config.Targets)) + for _, target := range config.Targets { + targets[target] = true + } + return userDefinedScannerAdapter{config: config, targets: targets} +} + +type userDefinedScannerAdapter struct { + config UserDefinedScannerConfig + targets map[string]bool +} + +func (adapter userDefinedScannerAdapter) ID() string { return adapter.config.ID } + +func (adapter userDefinedScannerAdapter) Requirements(_ map[string]string) []EnvRequirement { + requirements := make([]EnvRequirement, 0, len(adapter.config.Env)) + for _, name := range adapter.config.Env { + requirements = append(requirements, EnvRequirement{EnvVar: name, Reason: adapter.config.ID + " scanner"}) + } + return requirements +} + +func (adapter userDefinedScannerAdapter) Info() ScannerInfo { + return ScannerInfo{ID: adapter.config.ID, DisplayName: adapter.config.ID, RequiredEnv: append([]string(nil), adapter.config.Env...)} +} + +func (adapter userDefinedScannerAdapter) InstallPlan() InstallPlan { + return InstallPlan{ScannerID: adapter.config.ID, InstallUnsupportedReason: "user-defined scanner"} +} + +func (adapter userDefinedScannerAdapter) SupportsTargetKind(kind string) bool { + return adapter.targets[kind] +} + +func (adapter userDefinedScannerAdapter) CommandBacked() bool { return true } + +func (adapter userDefinedScannerAdapter) Run(runner ExternalScannerRunner, target string, startedAt string) (ScannerResult, error) { + shell := userDefinedScannerShell(runtime.GOOS, runner.SandboxMode) + targetReplacement := shell.quote(target) + usePositionalTarget := shell.command == "/bin/sh" + if usePositionalTarget { + targetReplacement = `"$1"` + } + rendered := targetPlaceholderPattern.ReplaceAllStringFunc(adapter.config.Command, func(string) string { + return targetReplacement + }) + args := append(append([]string(nil), shell.args...), rendered) + if usePositionalTarget { + args = append(args, "clawscan-target", target) + } + fullCommand := append([]string{shell.command}, args...) + timeout := runner.Timeout + if timeout == 0 { + timeout = 20 * time.Minute + } + output, runErr := runner.CommandRunner.Run(shell.command, args, userDefinedScannerCWD(target), timeout) + completedAt := time.Now().UTC().Format(time.RFC3339Nano) + raw := strings.TrimSpace(output.Stdout) + if runErr != nil { + message := commandError(runErr, output.Stderr, runner.Env) + if json.Valid([]byte(raw)) { + return ScannerResult{ + Status: "completed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, + Error: message, Raw: json.RawMessage(raw), + }, nil + } + return ScannerResult{ + Status: "failed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, + Error: message, + }, nil + } + if !json.Valid([]byte(raw)) { + return ScannerResult{ + Status: "failed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, + Error: fmt.Sprintf("User-defined scanner %s returned invalid JSON", adapter.config.ID), + }, nil + } + return ScannerResult{ + Status: "completed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, + Raw: json.RawMessage(raw), + }, nil +} + +func userDefinedScannerShell(goos string, sandboxMode string) judgeShellSpec { + if sandboxMode == SandboxModeDocker { + return judgeShellForGOOS("linux") + } + return judgeShellForGOOS(goos) +} + +var targetPlaceholderPattern = regexp.MustCompile(`\{\{\s*target\s*\}\}`) + +func userDefinedScannerCWD(target string) string { + parsed, err := url.Parse(target) + if err == nil && parsed.Scheme != "" && parsed.Host != "" { + return "" + } + info, err := os.Stat(target) + if err == nil && info.IsDir() { + return target + } + return filepath.Dir(target) +} From 3e62e868b93806e7fae3ff5028bbb100e9fbccdf Mon Sep 17 00:00:00 2001 From: Jesse Merhi <79823012+jesse-merhi@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:22:04 +1000 Subject: [PATCH 2/2] test: strengthen user-defined scanner coverage --- internal/profiles/resolver_test.go | 144 ++++++++++++++++++++- internal/runner/benchmark_registry_test.go | 31 +++++ internal/runner/scanner_registry_test.go | 30 +++++ 3 files changed, 198 insertions(+), 7 deletions(-) diff --git a/internal/profiles/resolver_test.go b/internal/profiles/resolver_test.go index 41d30a4..c7b8a8a 100644 --- a/internal/profiles/resolver_test.go +++ b/internal/profiles/resolver_test.go @@ -3,6 +3,7 @@ package profiles import ( "bytes" "encoding/json" + "errors" "os" "path/filepath" "strings" @@ -965,6 +966,110 @@ profiles: } } +func TestUserDefinedScannerRequiresDockerBeforeExecution(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: fixture-scanner + command: fixture-scan {{target}} +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + commandRunner := &profileCommandRunner{stdout: `{}`} + dockerErr := errors.New("docker probe failed") + _, err = runner.Run(opts, runner.RunContext{ + Env: map[string]string{}, HostCommandRunner: commandRunner, + DockerAvailability: func() error { return dockerErr }, + }) + if !errors.Is(err, dockerErr) { + t.Fatalf("err = %v", err) + } + if len(commandRunner.calls) != 0 { + t.Fatalf("scanner executed before Docker gating: %#v", commandRunner.calls) + } +} + +func TestUserDefinedScannerRegistrySurvivesDiscoveredTargetBatch(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "skills", "alpha", "SKILL.md"), "# Alpha\n") + writeFile(t, filepath.Join(dir, "skills", "beta", "SKILL.md"), "# Beta\n") + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: fixture-scanner + command: fixture-scan {{target}} +`) + opts, err := ResolveArgs([]string{"--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + commandRunner := &profileCommandRunner{stdout: `{}`} + result, err := runner.RunTargets(opts, runner.RunContext{ + Env: map[string]string{}, CommandRunner: commandRunner, + }, dir) + if err != nil { + t.Fatal(err) + } + if result.Batch == nil || len(result.Batch.Runs) != 2 || len(commandRunner.calls) != 2 { + t.Fatalf("batch = %#v calls = %#v", result.Batch, commandRunner.calls) + } + for _, run := range result.Batch.Runs { + if run.Scanners["fixture-scanner"].Status != "completed" { + t.Fatalf("run = %#v", run) + } + } +} + +func TestUserDefinedScannerRegistriesStayIsolatedAcrossProfileBatch(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + writeFile(t, filepath.Join(target, "SKILL.md"), "# Demo\n") + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + alpha: + scanners: + - id: fixture-scanner + command: alpha-scan {{target}} + beta: + scanners: + - id: fixture-scanner + command: beta-scan {{target}} +`) + resolved, err := ResolveRunSet([]string{target, "--config", config}, dir) + if err != nil { + t.Fatal(err) + } + commandRunner := &profileCommandRunner{stdout: `{}`} + batch, err := runner.RunProfileBatch(resolved.Options, runner.RunContext{ + Env: map[string]string{}, CommandRunner: commandRunner, + }, dir) + if err != nil { + t.Fatal(err) + } + if len(batch.Runs) != 2 || len(commandRunner.calls) != 2 { + t.Fatalf("batch = %#v calls = %#v", batch, commandRunner.calls) + } + joined := strings.Join([]string{ + strings.Join(commandRunner.calls[0].args, " "), + strings.Join(commandRunner.calls[1].args, " "), + }, "\n") + if !strings.Contains(joined, "alpha-scan") || !strings.Contains(joined, "beta-scan") { + t.Fatalf("profile registries were not isolated: %s", joined) + } +} + func TestUserDefinedScannerMissingEnvFailsBeforeExecution(t *testing.T) { dir := t.TempDir() target := filepath.Join(dir, "skill") @@ -1019,13 +1124,21 @@ profiles: if err != nil { t.Fatal(err) } - missingArtifact := runner.NewArtifact(opts, target, "start", "complete", map[string]string{}) - if missingArtifact.Env["MY_SCANNER_TOKEN"] != "missing" { - t.Fatalf("missing env = %#v", missingArtifact.Env) + commandRunner := &profileCommandRunner{stdout: `{}`} + missingBatch, err := runner.RunProfileBatch([]runner.Options{opts}, runner.RunContext{ + Env: map[string]string{}, CommandRunner: commandRunner, + }, dir) + if err != nil { + t.Fatal(err) + } + if missingBatch.Env["MY_SCANNER_TOKEN"] != "missing" || len(missingBatch.Errors) != 1 { + t.Fatalf("missing batch = %#v", missingBatch) + } + if len(commandRunner.calls) != 0 { + t.Fatalf("scanner executed with missing env: %#v", commandRunner.calls) } const envValue = "test-value-123" - commandRunner := &profileCommandRunner{stdout: `{}`} artifact, err := runner.Run(opts, runner.RunContext{ Env: map[string]string{"MY_SCANNER_TOKEN": envValue}, HostCommandRunner: commandRunner, @@ -1076,10 +1189,14 @@ profiles: t.Fatal(err) } commandRunner := &profileCommandRunner{stdout: `{}`} + dockerChecked := false artifact, err := runner.Run(opts, runner.RunContext{ - Env: map[string]string{}, - HostCommandRunner: commandRunner, - DockerAvailability: func() error { return nil }, + Env: map[string]string{}, + HostCommandRunner: commandRunner, + DockerAvailability: func() error { + dockerChecked = true + return errors.New("Docker should not be required") + }, }) if err != nil { t.Fatal(err) @@ -1091,6 +1208,9 @@ profiles: if commandRunner.command != "" { t.Fatalf("unsupported scanner executed: %q %#v", commandRunner.command, commandRunner.args) } + if dockerChecked { + t.Fatal("Docker availability was checked for a scanner that can only skip") + } } func TestResolveArgsRejectsProfileWithoutScannersUnlessCLIOverrides(t *testing.T) { @@ -1129,11 +1249,21 @@ func writeFile(t *testing.T, path string, content string) { type profileCommandRunner struct { command string args []string + calls []profileCommandCall stdout string } +type profileCommandCall struct { + command string + args []string +} + func (commandRunner *profileCommandRunner) Run(command string, args []string, _ string, _ time.Duration) (runner.CommandOutput, error) { commandRunner.command = command commandRunner.args = append([]string(nil), args...) + commandRunner.calls = append(commandRunner.calls, profileCommandCall{ + command: command, + args: append([]string(nil), args...), + }) return runner.CommandOutput{Stdout: commandRunner.stdout}, nil } diff --git a/internal/runner/benchmark_registry_test.go b/internal/runner/benchmark_registry_test.go index aa664ce..1342765 100644 --- a/internal/runner/benchmark_registry_test.go +++ b/internal/runner/benchmark_registry_test.go @@ -58,6 +58,37 @@ func TestRunBenchmarkRejectsUnsupportedBenchmarkThroughRegistry(t *testing.T) { } } +func TestRunBenchmarkExecutesUserDefinedScannerFromRunRegistry(t *testing.T) { + custom := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "fixture-scanner", Command: "fixture-scan {{target}}", Targets: []string{targetKindSkill}, + }) + registry, err := DefaultScannerRegistry().WithAdapters(custom) + if err != nil { + t.Fatal(err) + } + opts := benchmarkTestOptions(t, "clawhub-security-signals", "eval_holdout", 0, 0, "") + opts.Scanners = []string{"fixture-scanner"} + opts.ScannerRegistry = registry + commandRunner := &recordingCommandRunner{stdout: `{"verdict":"clean"}`} + artifact, err := RunBenchmark(opts, RunContext{ + Env: map[string]string{}, CommandRunner: commandRunner, + BenchmarkClient: staticBenchmarkClient{rows: []OpenClawBenchmarkRow{{ + ID: "row-1", SkillSlug: "owner/demo", SkillVersion: "1.0.0", + SkillMDContent: "# Demo\n", ClawScanVerdict: "clean", + }}}, + }) + if err != nil { + t.Fatal(err) + } + if len(commandRunner.calls) != 1 || !strings.Contains(commandRunner.calls[0].args[1], "fixture-scan") { + t.Fatalf("calls = %#v", commandRunner.calls) + } + result := artifact.Cases[0].Run.Scanners["fixture-scanner"] + if result.Status != "completed" || string(result.Raw) != `{"verdict":"clean"}` { + t.Fatalf("result = %#v", result) + } +} + type stubBenchmarkAdapter struct { id string aliases []string diff --git a/internal/runner/scanner_registry_test.go b/internal/runner/scanner_registry_test.go index 42d50bd..ed892f3 100644 --- a/internal/runner/scanner_registry_test.go +++ b/internal/runner/scanner_registry_test.go @@ -171,6 +171,36 @@ func TestUserDefinedScannerPreservesValidJSONOnCommandFailure(t *testing.T) { } } +func TestUserDefinedScannerRejectsMissingOrInvalidJSON(t *testing.T) { + for _, test := range []struct { + name string + stdout string + }{ + {name: "empty"}, + {name: "invalid", stdout: "not json"}, + } { + t.Run(test.name, func(t *testing.T) { + adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"}, + }) + registry, err := NewScannerRegistry(adapter) + if err != nil { + t.Fatal(err) + } + result, err := (ExternalScannerRunner{ + Registry: registry, CommandRunner: &recordingCommandRunner{stdout: test.stdout}, + Env: map[string]string{}, SandboxMode: SandboxModeOff, + }).RunScanner("demo", t.TempDir(), "2026-07-21T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if result.Status != "failed" || len(result.Raw) != 0 || !strings.Contains(result.Error, "returned invalid JSON") { + t.Fatalf("result = %#v", result) + } + }) + } +} + func TestUserDefinedScannerInterpolatesDollarTargetLiterally(t *testing.T) { adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"},