diff --git a/internal/runner/benchmark.go b/internal/runner/benchmark.go index 2f6dcc6..73bc613 100644 --- a/internal/runner/benchmark.go +++ b/internal/runner/benchmark.go @@ -191,6 +191,20 @@ type huggingFaceRow struct { Row OpenClawBenchmarkRow `json:"row"` } +// runnableBenchmarkScanners filters requested scanners to those supporting the +// benchmark's per-case target kind, so requirement validation does not abort on +// a scanner (e.g. url-only) that would never run against these cases anyway. +func runnableBenchmarkScanners(opts Options, kind string) []string { + registry := registryForOptions(opts) + runnable := make([]string, 0, len(opts.Scanners)) + for _, id := range opts.Scanners { + if scannerSupportsTargetKindInRegistry(registry, id, kind) { + runnable = append(runnable, id) + } + } + return runnable +} + func RunBenchmark(opts Options, ctx RunContext) (BenchmarkArtifact, error) { if opts.Benchmark == nil { return BenchmarkArtifact{}, errors.New("missing benchmark options") @@ -210,7 +224,9 @@ func RunBenchmark(opts Options, ctx RunContext) (BenchmarkArtifact, error) { env = EnvMap(os.Environ()) } applyRuntimeEnvDefaults(opts, env) - if err := ValidateRequirements(opts, env); err != nil { + requirementOpts := opts + requirementOpts.Scanners = runnableBenchmarkScanners(opts, "skill") + if err := ValidateRequirements(requirementOpts, env); err != nil { return BenchmarkArtifact{}, err } if opts.Benchmark.IDsSource != "" { diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 4f2bb0e..0d8315e 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -849,6 +849,27 @@ func TestValidateRequirements(t *testing.T) { } } +func TestRunnableBenchmarkScannersFiltersUnsupportedTargetKind(t *testing.T) { + skillScanner := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "skillscan", Command: "run {{target}}", Targets: []string{"skill"}, + }) + urlScanner := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "urlscan", Command: "run {{target}}", Targets: []string{"url"}, + }) + registry, err := NewScannerRegistry(skillScanner, urlScanner) + if err != nil { + t.Fatal(err) + } + opts := Options{ + Scanners: []string{"skillscan", "urlscan"}, + ScannerRegistry: registry, + } + runnable := runnableBenchmarkScanners(opts, "skill") + if len(runnable) != 1 || runnable[0] != "skillscan" { + t.Fatalf("runnable = %#v, want [skillscan]", runnable) + } +} + func TestValidateRequirementsSkipsScannerResultCredentials(t *testing.T) { opts, err := ParseArgs([]string{ "./my-skill",