From 13c6c6db48c677c61dcc139fe210aa161d57ddd0 Mon Sep 17 00:00:00 2001 From: Jesse Merhi <79823012+jesse-merhi@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:45:08 +1000 Subject: [PATCH 1/6] feat(profiles)!: require explicit config; stop auto-discovering .clawscan.yml Auto-discovery of .clawscan.yml via upward directory walk is now opt-in (--discover-config). Discovered-but-ignored configs produce a stderr notice, and run artifacts record their config source. Groundwork for config-defined scanners: discovered config must never gain the power to execute commands chosen by a scan target. --- cmd/clawscan/main.go | 10 ++ cmd/clawscan/main_test.go | 203 ++++++++++++++++++++++++++++- docs/profiles.md | 26 ++++ internal/profiles/registry.go | 2 +- internal/profiles/resolver.go | 91 ++++++++++--- internal/profiles/resolver_test.go | 152 +++++++++++++++++++-- internal/runner/runner.go | 5 + internal/runner/runner_test.go | 30 +++++ 8 files changed, 486 insertions(+), 33 deletions(-) diff --git a/cmd/clawscan/main.go b/cmd/clawscan/main.go index 5cbebe2..c130122 100644 --- a/cmd/clawscan/main.go +++ b/cmd/clawscan/main.go @@ -61,6 +61,7 @@ func run(args []string, environ []string) error { if err != nil { return err } + printIgnoredConfigNotice(os.Stderr, resolved.IgnoredConfig) if resolved.AllProfiles { return runAllProfiles(resolved, environ, cwd) } @@ -124,6 +125,7 @@ func runBenchmarkCommand(args []string, environ []string) error { if err != nil { return err } + printIgnoredConfigNotice(os.Stderr, resolved.IgnoredConfig) if resolved.AllProfiles { return errors.New("clawscan benchmark does not support --config without --profile") } @@ -153,6 +155,13 @@ func runBenchmarkCommand(args []string, environ []string) error { return nil } +func printIgnoredConfigNotice(w io.Writer, configPath string) { + if configPath == "" { + return + } + fmt.Fprintf(w, "warning: discovered %s at %s but not loading it; use --config %s or --discover-config to load discovered config\n", filepath.Base(configPath), configPath, configPath) +} + func runAllProfiles(resolved profiles.ResolvedRunSet, environ []string, cwd string) error { if !resolved.JSON && resolved.OutputPath == "" { resolved.OutputPath = defaultOutputPath @@ -566,6 +575,7 @@ Usage: Core flags: --profile Profile to run. Use --profile clawhub for ClawHub parity. --config Load profiles from a specific .clawscan.yml file; omit --profile to run them all. + --discover-config Find and load the nearest .clawscan.yml/.clawscan.yaml in the current directory or a parent. --scanner Scanner to run. Repeat for multiple scanners. --scanner-result Use a JSON fixture instead of running that scanner. --context Load profile runtime context from a JSON file. diff --git a/cmd/clawscan/main_test.go b/cmd/clawscan/main_test.go index 220ca23..27bb397 100644 --- a/cmd/clawscan/main_test.go +++ b/cmd/clawscan/main_test.go @@ -32,6 +32,7 @@ func TestRunCommandPrintsHelp(t *testing.T) { "Install scanner dependencies without running scans.", "--profile ", "--config ", + "--discover-config", "Benchmark command flags:", "--split ", "--ids ", @@ -675,7 +676,7 @@ profiles: t.Chdir(dir) stdout := captureStdout(t, func() { - if err := run([]string{target, "--profile", "local"}, []string{}); err != nil { + if err := run([]string{target, "--profile", "local", "--discover-config"}, []string{}); err != nil { t.Fatal(err) } }) @@ -695,7 +696,150 @@ profiles: } } -func TestRunCommandConfigWithoutProfileRunsEveryConfigProfile(t *testing.T) { +func TestRunDefaultModeIgnoresConfigAndPrintsNotice(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + writeSkill(t, target, "# Ignored config\n") + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, "version: [\n") + t.Chdir(dir) + + stdout, stderr := captureOutput(t, func() { + if err := run([]string{target, "--scanner", "clawscan-static", "--json"}, []string{}); err != nil { + t.Fatal(err) + } + }) + + var artifact struct { + ConfigSource string `json:"configSource"` + } + if err := json.Unmarshal([]byte(stdout), &artifact); err != nil { + t.Fatalf("stdout is not artifact JSON: %v\n%s", err, stdout) + } + if artifact.ConfigSource != "flags-only" { + t.Fatalf("config source = %q", artifact.ConfigSource) + } + want := "warning: discovered .clawscan.yml at " + config + " but not loading it; use --config " + config + " or --discover-config to load discovered config\n" + if stderr != want { + t.Fatalf("stderr = %q, want %q", stderr, want) + } + if strings.Contains(stdout, "warning: discovered") { + t.Fatalf("notice leaked to stdout: %s", stdout) + } +} + +func TestRunDefaultModeIgnoresParentConfigAndPrintsNotice(t *testing.T) { + parent := t.TempDir() + config := filepath.Join(parent, ".clawscan.yml") + writeFile(t, config, "version: [\n") + child := filepath.Join(parent, "child") + target := filepath.Join(child, "skill") + writeSkill(t, target, "# Parent ignored config\n") + t.Chdir(child) + + _, stderr := captureOutput(t, func() { + if err := run([]string{target, "--scanner", "clawscan-static", "--json"}, []string{}); err != nil { + t.Fatal(err) + } + }) + + if !strings.Contains(stderr, config) || !strings.Contains(stderr, "--discover-config") { + t.Fatalf("stderr = %q", stderr) + } +} + +func TestRunWithDiscoverConfigLoadsConfig_NoNotice(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + writeSkill(t, target, "# Discovered config\n") + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + custom: + scanners: + - clawscan-static +`) + t.Chdir(dir) + + stdout, stderr := captureOutput(t, func() { + if err := run([]string{target, "--profile", "custom", "--discover-config", "--json"}, []string{}); err != nil { + t.Fatal(err) + } + }) + + var artifact struct { + ConfigSource string `json:"configSource"` + } + if err := json.Unmarshal([]byte(stdout), &artifact); err != nil { + t.Fatal(err) + } + if artifact.ConfigSource != config { + t.Fatalf("config source = %q, want %q", artifact.ConfigSource, config) + } + if stderr != "" { + t.Fatalf("stderr = %q", stderr) + } +} + +func TestRunWithExplicitConfigLoadsConfig_NoNotice(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + writeSkill(t, target, "# Explicit config\n") + writeFile(t, filepath.Join(dir, ".clawscan.yml"), "version: [\n") + explicit := filepath.Join(dir, "explicit.yml") + writeFile(t, explicit, `version: 1 +profiles: + p: + scanners: + - clawscan-static +`) + t.Chdir(dir) + + stdout, stderr := captureOutput(t, func() { + if err := run([]string{target, "--config", explicit, "--profile", "p", "--json"}, []string{}); err != nil { + t.Fatal(err) + } + }) + + var artifact struct { + ConfigSource string `json:"configSource"` + } + if err := json.Unmarshal([]byte(stdout), &artifact); err != nil { + t.Fatal(err) + } + if artifact.ConfigSource != explicit { + t.Fatalf("config source = %q, want %q", artifact.ConfigSource, explicit) + } + if stderr != "" { + t.Fatalf("stderr = %q", stderr) + } +} + +func TestRunJSONFlagDoesNotIncludeNoticeInStdout(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + writeSkill(t, target, "# JSON notice separation\n") + writeFile(t, filepath.Join(dir, ".clawscan.yml"), "version: 1\nprofiles: {}\n") + t.Chdir(dir) + + stdout, stderr := captureOutput(t, func() { + if err := run([]string{target, "--scanner", "clawscan-static", "--json"}, []string{}); err != nil { + t.Fatal(err) + } + }) + + if !json.Valid([]byte(stdout)) { + t.Fatalf("stdout is not valid JSON: %s", stdout) + } + if strings.Contains(stdout, "warning: discovered") { + t.Fatalf("notice leaked to stdout: %s", stdout) + } + if !strings.Contains(stderr, "warning: discovered") { + t.Fatalf("stderr = %q", stderr) + } +} + +func TestBatchArtifactRuns_EachHasConfigSource(t *testing.T) { dir := t.TempDir() writeSkill(t, filepath.Join(dir, "skills", "foo"), "# Foo\n") writeSkill(t, filepath.Join(dir, "skills", "bar"), "# Bar\n") @@ -720,8 +864,9 @@ profiles: var artifact struct { SchemaVersion string `json:"schemaVersion"` Runs []struct { - Profile string `json:"profile"` - Scanners map[string]interface{} `json:"scanners"` + Profile string `json:"profile"` + ConfigSource string `json:"configSource"` + Scanners map[string]interface{} `json:"scanners"` } `json:"runs"` } if err := json.Unmarshal([]byte(stdout), &artifact); err != nil { @@ -737,6 +882,9 @@ profiles: t.Fatalf("profiles = %q", got) } for _, run := range artifact.Runs { + if run.ConfigSource != config { + t.Fatalf("config source = %q, want %q", run.ConfigSource, config) + } if _, ok := run.Scanners["clawscan-static"]; !ok { t.Fatalf("missing clawscan-static scanner for %s: %#v", run.Profile, run.Scanners) } @@ -864,3 +1012,50 @@ func captureStdout(t *testing.T, fn func()) string { os.Stdout = original return string(out) } + +func captureOutput(t *testing.T, fn func()) (string, string) { + t.Helper() + + originalStdout := os.Stdout + originalStderr := os.Stderr + stdoutRead, stdoutWrite, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + stderrRead, stderrWrite, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stdout = stdoutWrite + os.Stderr = stderrWrite + t.Cleanup(func() { + os.Stdout = originalStdout + os.Stderr = originalStderr + }) + + fn() + + if err := stdoutWrite.Close(); err != nil { + t.Fatal(err) + } + if err := stderrWrite.Close(); err != nil { + t.Fatal(err) + } + stdout, err := io.ReadAll(stdoutRead) + if err != nil { + t.Fatal(err) + } + stderr, err := io.ReadAll(stderrRead) + if err != nil { + t.Fatal(err) + } + if err := stdoutRead.Close(); err != nil { + t.Fatal(err) + } + if err := stderrRead.Close(); err != nil { + t.Fatal(err) + } + os.Stdout = originalStdout + os.Stderr = originalStderr + return string(stdout), string(stderr) +} diff --git a/docs/profiles.md b/docs/profiles.md index d332559..3ea35f1 100644 --- a/docs/profiles.md +++ b/docs/profiles.md @@ -12,6 +12,32 @@ The same profile accepts an explicit OpenClaw plugin directory (or its `openclaw.plugin.json` manifest), runs both scanners, and renders the bundled judge prompt with `packageRelease` target context. +## Config discovery + +By default, ClawScan does not auto-discover `.clawscan.yml` or `.clawscan.yaml` +files from the current directory or parent directories. If your run detects a +config file that could have been loaded, ClawScan prints a notice to stderr +suggesting you use `--config ` or `--discover-config`. + +To load a discovered config file, use one of these flags: + +- `--config ` - Explicitly specify a config file path +- `--discover-config` - Search upward from the current directory and load the nearest `.clawscan.yml` or `.clawscan.yaml` + +Mixing `--config` and `--discover-config` is an error. + +```bash +clawscan ./my-skill --config ./security/clawscan.yml --profile review +clawscan ./my-skill --profile review --discover-config +``` + +Without either flag, ClawScan uses built-in profiles and CLI flags only. The +warning is a single stderr line and never changes JSON stdout. The +`clawscan profiles` catalog command still includes the nearest discovered +project config. + +## Inspect available profiles + Inspect the resolved profile catalog, including the nearest project `.clawscan.yml` / `.clawscan.yaml` when present: diff --git a/internal/profiles/registry.go b/internal/profiles/registry.go index 1d022fe..35671c4 100644 --- a/internal/profiles/registry.go +++ b/internal/profiles/registry.go @@ -54,7 +54,7 @@ func ProfileIDs() []string { } func InspectProfiles(cwd string) (ProfileCatalog, error) { - registry, err := loadConfigs(cwd, "") + registry, _, err := loadConfigs(cwd, "", true) if err != nil { return ProfileCatalog{}, err } diff --git a/internal/profiles/resolver.go b/internal/profiles/resolver.go index cf4353b..2fb19b9 100644 --- a/internal/profiles/resolver.go +++ b/internal/profiles/resolver.go @@ -61,6 +61,7 @@ type cliIntent struct { profile string profileSet bool configPath string + discoverConfig bool contextPath string scanners []string scannerResultPaths map[string]string @@ -91,10 +92,11 @@ type cliIntent struct { var judgePathPlaceholderPattern = regexp.MustCompile(`\{\{\s*(prompt|output_schema):([^}]+)\}\}`) type ResolvedRunSet struct { - Options []runner.Options - OutputPath string - JSON bool - AllProfiles bool + Options []runner.Options + OutputPath string + JSON bool + AllProfiles bool + IgnoredConfig string } func ResolveArgs(args []string, cwd string) (runner.Options, error) { @@ -145,17 +147,26 @@ func resolveRunSetIntent(intent cliIntent, cwd string) (ResolvedRunSet, error) { } profileName := "" + configSource := "" + if intent.configPath == "" && !intent.discoverConfig { + configSource = "flags-only" + } selected := resolvedProfile{profile: Profile{}} - if intent.profileSet || intent.configPath != "" { - registry, err := loadConfigs(cwd, intent.configPath) + if intent.profileSet || intent.configPath != "" || intent.discoverConfig { + registry, loadedConfig, err := loadConfigs(cwd, intent.configPath, intent.discoverConfig) if err != nil { return ResolvedRunSet{}, err } - profileName = intent.profile - var ok bool - selected, ok = registry.Profile(profileName) - if !ok { - return ResolvedRunSet{}, unknownProfileError(profileName, registry.IDs()) + if loadedConfig != "" { + configSource = loadedConfig + } + if intent.profileSet { + profileName = intent.profile + var ok bool + selected, ok = registry.Profile(profileName) + if !ok { + return ResolvedRunSet{}, unknownProfileError(profileName, registry.IDs()) + } } } @@ -168,6 +179,14 @@ func resolveRunSetIntent(intent cliIntent, cwd string) (ResolvedRunSet, error) { return ResolvedRunSet{}, err } opts.Profile = profileName + opts.ConfigSource = configSource + opts.DiscoverConfig = intent.discoverConfig + if intent.configPath == "" && !intent.discoverConfig { + opts.IgnoredConfig, err = findIgnoredConfig(cwd) + if err != nil { + return ResolvedRunSet{}, err + } + } if opts.Judge != nil { opts.Judge.Files = files } @@ -178,9 +197,10 @@ func resolveRunSetIntent(intent cliIntent, cwd string) (ResolvedRunSet, error) { } } return ResolvedRunSet{ - Options: []runner.Options{opts}, - OutputPath: opts.OutputPath, - JSON: opts.JSON, + Options: []runner.Options{opts}, + OutputPath: opts.OutputPath, + JSON: opts.JSON, + IgnoredConfig: opts.IgnoredConfig, }, nil } @@ -192,7 +212,7 @@ func explicitRunSelectionError() error { return errors.New("No scanner, profile, or config selected. Pass --scanner, --profile, or --config. Use `clawscan benchmark ` for benchmark runs.") } -func loadConfigs(cwd string, explicitConfig string) (ProfileRegistry, error) { +func loadConfigs(cwd string, explicitConfig string, discover bool) (ProfileRegistry, string, error) { registry := DefaultProfileRegistry() var projectPath string @@ -202,20 +222,25 @@ func loadConfigs(cwd string, explicitConfig string) (ProfileRegistry, error) { if !filepath.IsAbs(projectPath) { projectPath = filepath.Join(cwd, projectPath) } - } else { + projectPath = filepath.Clean(projectPath) + } else if discover { projectPath, err = discoverConfig(cwd) if err != nil { - return ProfileRegistry{}, err + return ProfileRegistry{}, "", err } } if projectPath == "" { - return registry, nil + return registry, "", nil } projectProfiles, err := loadProjectProfiles(projectPath) if err != nil { - return ProfileRegistry{}, err + return ProfileRegistry{}, "", err + } + registry, err = registry.Merge(projectProfiles) + if err != nil { + return ProfileRegistry{}, "", err } - return registry.Merge(projectProfiles) + return registry, projectPath, nil } func resolveAllConfigProfiles(intent cliIntent, cwd string) (ResolvedRunSet, error) { @@ -258,6 +283,7 @@ func resolveAllConfigProfiles(intent cliIntent, cwd string) (ResolvedRunSet, err return ResolvedRunSet{}, err } opts.Profile = profileName + opts.ConfigSource = filepath.Clean(projectPath) opts.OutputPath = "" opts.JSON = false if opts.Judge != nil { @@ -404,6 +430,26 @@ func discoverConfig(cwd string) (string, error) { } } +func findIgnoredConfig(cwd string) (string, error) { + current, err := filepath.Abs(cwd) + if err != nil { + return "", err + } + for { + for _, name := range []string{".clawscan.yml", ".clawscan.yaml"} { + path := filepath.Join(current, name) + if fileExists(path) { + return path, nil + } + } + parent := filepath.Dir(current) + if parent == current { + return "", nil + } + current = parent + } +} + func fileExists(path string) bool { info, err := os.Stat(path) return err == nil && !info.IsDir() @@ -434,6 +480,8 @@ func parseCLIIntent(args []string) (cliIntent, error) { } intent.configPath = value i = next + case "--discover-config": + intent.discoverConfig = true case "--context": value, next, err := readValue(args, i, arg) if err != nil { @@ -552,6 +600,9 @@ func parseCLIIntent(args []string) (cliIntent, error) { return cliIntent{}, fmt.Errorf("Unknown argument: %s", arg) } } + if intent.configPath != "" && intent.discoverConfig { + return cliIntent{}, errors.New("--config and --discover-config are mutually exclusive") + } return intent, nil } diff --git a/internal/profiles/resolver_test.go b/internal/profiles/resolver_test.go index 1ae9f7d..70a632b 100644 --- a/internal/profiles/resolver_test.go +++ b/internal/profiles/resolver_test.go @@ -140,6 +140,142 @@ func TestResolveArgsTreatsScannerOnlyCommandAsAdHocWithoutDefaultJudge(t *testin } } +func TestResolveArgsDefaultRunDoesNotAutoDiscover_WithCwdConfig(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, ".clawscan.yml"), `version: 1 +profiles: + custom: + scanners: + - virustotal +`) + + opts, err := ResolveArgs([]string{"./skill", "--scanner", "clawscan-static"}, dir) + if err != nil { + t.Fatal(err) + } + + if got := strings.Join(opts.Scanners, ","); got != "clawscan-static" { + t.Fatalf("scanners = %q", got) + } + if opts.ConfigSource != "flags-only" { + t.Fatalf("config source = %q, want flags-only", opts.ConfigSource) + } + if opts.IgnoredConfig != filepath.Join(dir, ".clawscan.yml") { + t.Fatalf("ignored config = %q, want %s", opts.IgnoredConfig, filepath.Join(dir, ".clawscan.yml")) + } +} + +func TestResolveArgsDefaultRunDoesNotAutoDiscover_WithParentConfig(t *testing.T) { + parent := t.TempDir() + config := filepath.Join(parent, ".clawscan.yml") + writeFile(t, config, "version: 1\nprofiles: {}\n") + child := filepath.Join(parent, "child") + if err := os.Mkdir(child, 0o755); err != nil { + t.Fatal(err) + } + + opts, err := ResolveArgs([]string{"./skill", "--scanner", "clawscan-static"}, child) + if err != nil { + t.Fatal(err) + } + + if opts.IgnoredConfig != config { + t.Fatalf("ignored config = %q, want %q", opts.IgnoredConfig, config) + } +} + +func TestResolveArgsDiscoverConfigFlag_RestoresOldBehavior(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + custom: + scanners: + - clawscan-static +`) + + opts, err := ResolveArgs([]string{"./skill", "--profile", "custom", "--discover-config"}, dir) + if err != nil { + t.Fatal(err) + } + + if opts.Profile != "custom" { + t.Fatalf("profile = %q", opts.Profile) + } + if opts.ConfigSource != config { + t.Fatalf("config source = %q, want %q", opts.ConfigSource, config) + } + if !opts.DiscoverConfig { + t.Fatal("DiscoverConfig = false, want true") + } +} + +func TestResolveArgsDiscoverConfigWithoutConfigLeavesSourceEmpty(t *testing.T) { + opts, err := ResolveArgs([]string{"./skill", "--scanner", "clawscan-static", "--discover-config"}, t.TempDir()) + if err != nil { + t.Fatal(err) + } + if opts.ConfigSource != "" { + t.Fatalf("config source = %q, want empty", opts.ConfigSource) + } +} + +func TestResolveArgsConfigAndDiscoverConfigFlagsAreMutuallyExclusive(t *testing.T) { + dir := t.TempDir() + _, err := ResolveArgs([]string{"./skill", "--config", "config.yml", "--discover-config", "--scanner", "clawscan-static"}, dir) + if err == nil { + t.Fatal("expected error for --config and --discover-config together") + } + if !strings.Contains(err.Error(), "mutually exclusive") { + t.Fatalf("err = %v", err) + } +} + +func TestResolveArgsExplicitConfigSkipsDiscoveryAndPrintsNoNotice(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, ".clawscan.yml"), `version: 1 +profiles: + builtin: + scanners: + - clawscan-static +`) + explicit := filepath.Join(dir, "custom.yml") + writeFile(t, explicit, `version: 1 +profiles: + custom: + scanners: + - virustotal +`) + + opts, err := ResolveArgs([]string{"./skill", "--config", explicit, "--profile", "custom"}, dir) + if err != nil { + t.Fatal(err) + } + + if opts.ConfigSource != explicit { + t.Fatalf("config source = %q", opts.ConfigSource) + } + if opts.IgnoredConfig != "" { + t.Fatalf("ignored config = %q, want empty", opts.IgnoredConfig) + } +} + +func TestResolveArgsNoConfigAnywherePrintsNoNotice(t *testing.T) { + dir := t.TempDir() + + opts, err := ResolveArgs([]string{"./skill", "--scanner", "clawscan-static"}, dir) + if err != nil { + t.Fatal(err) + } + + if opts.ConfigSource != "flags-only" { + t.Fatalf("config source = %q", opts.ConfigSource) + } + if opts.IgnoredConfig != "" { + t.Fatalf("ignored config = %q, want empty", opts.IgnoredConfig) + } +} + func TestResolveArgsAllowsExplicitProfileWithoutTarget(t *testing.T) { dir := t.TempDir() @@ -319,7 +455,7 @@ profiles: json: true `) - opts, err := ResolveArgs([]string{"./skill", "--profile", "clawhub"}, child) + opts, err := ResolveArgs([]string{"./skill", "--profile", "clawhub", "--discover-config"}, child) if err != nil { t.Fatal(err) } @@ -388,7 +524,7 @@ func TestResolveArgsRejectsAmbiguousDiscoveredConfig(t *testing.T) { writeFile(t, filepath.Join(dir, ".clawscan.yml"), "version: 1\nprofiles: {}\n") writeFile(t, filepath.Join(dir, ".clawscan.yaml"), "version: 1\nprofiles: {}\n") - _, err := ResolveArgs([]string{"./skill", "--profile", "clawhub"}, dir) + _, err := ResolveArgs([]string{"./skill", "--profile", "clawhub", "--discover-config"}, dir) if err == nil || !strings.Contains(err.Error(), "Ambiguous ClawScan config files") { t.Fatalf("err = %v", err) } @@ -401,7 +537,7 @@ func TestResolveArgsRejectsMalformedYAML(t *testing.T) { dir := t.TempDir() writeFile(t, filepath.Join(dir, ".clawscan.yml"), "version: [\n") - _, err := ResolveArgs([]string{"./skill", "--profile", "clawhub"}, dir) + _, err := ResolveArgs([]string{"./skill", "--profile", "clawhub", "--discover-config"}, dir) if err == nil || !strings.Contains(err.Error(), "parse ClawScan config") { t.Fatalf("err = %v", err) } @@ -502,7 +638,7 @@ profiles: - ANTHROPIC_API_KEY `) - opts, err := ResolveArgs([]string{"./skill", "--profile", "review"}, dir) + opts, err := ResolveArgs([]string{"./skill", "--profile", "review", "--discover-config"}, dir) if err != nil { t.Fatal(err) } @@ -565,7 +701,7 @@ profiles: OPENAI_API_KEY: secret `) - _, err := ResolveArgs([]string{"./skill", "--profile", "bad"}, dir) + _, err := ResolveArgs([]string{"./skill", "--profile", "bad", "--discover-config"}, dir) if err == nil || !strings.Contains(err.Error(), "field env not found") { t.Fatalf("err = %v", err) } @@ -582,7 +718,7 @@ profiles: - clawscan-static `) - _, err := ResolveArgs([]string{"./skill", "--profile", "dup"}, dir) + _, err := ResolveArgs([]string{"./skill", "--profile", "dup", "--discover-config"}, dir) if err == nil || err.Error() != "Duplicate scanner in profile dup: clawscan-static" { t.Fatalf("err = %v", err) } @@ -597,12 +733,12 @@ profiles: json: true `) - _, err := ResolveArgs([]string{"./skill", "--profile", "empty"}, dir) + _, err := ResolveArgs([]string{"./skill", "--profile", "empty", "--discover-config"}, dir) if err == nil || err.Error() != "Profile empty must include at least one scanner or use --scanner" { t.Fatalf("err = %v", err) } - opts, err := ResolveArgs([]string{"./skill", "--profile", "empty", "--scanner", "clawscan-static"}, dir) + opts, err := ResolveArgs([]string{"./skill", "--profile", "empty", "--scanner", "clawscan-static", "--discover-config"}, dir) if err != nil { t.Fatal(err) } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 11e36d1..f5b3a94 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -28,6 +28,9 @@ import ( type Options struct { Target string Profile string + ConfigSource string + DiscoverConfig bool + IgnoredConfig string ContextPath string Benchmark *BenchmarkOptions Scanners []string @@ -87,6 +90,7 @@ type CommandOutput struct { type Artifact struct { SchemaVersion string `json:"schemaVersion"` Profile string `json:"profile,omitempty"` + ConfigSource string `json:"configSource,omitempty"` Context json.RawMessage `json:"context,omitempty"` Target Target `json:"target"` StartedAt string `json:"startedAt"` @@ -2117,6 +2121,7 @@ func NewArtifact(opts Options, resolvedPath string, startedAt string, completedA return Artifact{ SchemaVersion: "clawscan-run-v1", Profile: opts.Profile, + ConfigSource: opts.ConfigSource, Target: Target{ Kind: "skill", Input: opts.Target, diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index d466641..bdf12c3 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -1100,6 +1100,36 @@ func TestArtifactRedactsEnvValues(t *testing.T) { } } +func TestArtifactConfigSourceField_FlagsOnly(t *testing.T) { + opts, err := ParseArgs([]string{"./my-skill", "--scanner", "clawscan-static"}) + if err != nil { + t.Fatal(err) + } + opts.ConfigSource = "flags-only" + + artifact := NewArtifact(opts, "/tmp/my-skill", "2026-06-03T00:00:00Z", "2026-06-03T00:00:01Z", map[string]string{}) + + if artifact.ConfigSource != "flags-only" { + t.Fatalf("config source = %q", artifact.ConfigSource) + } +} + +func TestArtifactConfigSourceField_OmitsEmptySource(t *testing.T) { + opts, err := ParseArgs([]string{"./my-skill", "--scanner", "clawscan-static"}) + if err != nil { + t.Fatal(err) + } + + artifact := NewArtifact(opts, "/tmp/my-skill", "2026-06-03T00:00:00Z", "2026-06-03T00:00:01Z", map[string]string{}) + raw, err := json.Marshal(artifact) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(raw, []byte(`"configSource"`)) { + t.Fatalf("empty config source was serialized: %s", raw) + } +} + func TestRunWritesScannerOnlyArtifact(t *testing.T) { dir := t.TempDir() target := filepath.Join(dir, "skill") From caf06103222ad518b3e4add8a0b3673480c72c72 Mon Sep 17 00:00:00 2001 From: Jesse Merhi <79823012+jesse-merhi@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:43:50 +1000 Subject: [PATCH 2/6] fix(profiles): surface ignored-config hint on unknown profile; update bundled skill Review findings from ticket-1 code review: the migration hint was skipped exactly when the requested profile lived in an unloaded discovered config, and skills/clawscan-cli/SKILL.md still documented automatic discovery. --- internal/profiles/resolver.go | 10 +++++++++- internal/profiles/resolver_test.go | 22 ++++++++++++++++++++++ skills/clawscan-cli/SKILL.md | 12 +++++++----- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/internal/profiles/resolver.go b/internal/profiles/resolver.go index 2fb19b9..dd7091d 100644 --- a/internal/profiles/resolver.go +++ b/internal/profiles/resolver.go @@ -165,7 +165,15 @@ func resolveRunSetIntent(intent cliIntent, cwd string) (ResolvedRunSet, error) { var ok bool selected, ok = registry.Profile(profileName) if !ok { - return ResolvedRunSet{}, unknownProfileError(profileName, registry.IDs()) + err := unknownProfileError(profileName, registry.IDs()) + // The profile may live in a discovered-but-unloaded config; + // surface the migration hint here or the user never sees it. + if intent.configPath == "" && !intent.discoverConfig { + if ignored, findErr := findIgnoredConfig(cwd); findErr == nil && ignored != "" { + err = fmt.Errorf("%w (found %s but did not load it; pass --config %s or --discover-config)", err, ignored, ignored) + } + } + return ResolvedRunSet{}, err } } } diff --git a/internal/profiles/resolver_test.go b/internal/profiles/resolver_test.go index 70a632b..f8848c2 100644 --- a/internal/profiles/resolver_test.go +++ b/internal/profiles/resolver_test.go @@ -165,6 +165,28 @@ profiles: } } +func TestResolveArgsUnknownProfileMentionsIgnoredConfig(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + custom: + scanners: + - virustotal +`) + + _, err := ResolveArgs([]string{"./skill", "--profile", "custom"}, dir) + if err == nil { + t.Fatal("expected unknown profile error") + } + if !strings.Contains(err.Error(), "Unknown profile: custom") { + t.Fatalf("error = %q, want unknown profile", err) + } + if !strings.Contains(err.Error(), config) || !strings.Contains(err.Error(), "--discover-config") { + t.Fatalf("error = %q, want ignored-config hint naming %s and --discover-config", err, config) + } +} + func TestResolveArgsDefaultRunDoesNotAutoDiscover_WithParentConfig(t *testing.T) { parent := t.TempDir() config := filepath.Join(parent, ".clawscan.yml") diff --git a/skills/clawscan-cli/SKILL.md b/skills/clawscan-cli/SKILL.md index f03c83d..204c251 100644 --- a/skills/clawscan-cli/SKILL.md +++ b/skills/clawscan-cli/SKILL.md @@ -83,11 +83,13 @@ Built-in profiles: | --- | --- | --- | | `clawhub` | `skillspector`, `clawscan-static` | bundled Codex judge with ClawHub prompt/schema | -Profiles are loaded from embedded built-ins plus the nearest `.clawscan.yml` or -`.clawscan.yaml` discovered upward from the current directory. A project profile -with the same name shadows the built-in whole profile. `--config ` loads a -specific config; without `--profile`, it runs every profile in that config and -emits a `clawscan-batch-v1` artifact. +Profiles are loaded from embedded built-ins. Project `.clawscan.yml` / +`.clawscan.yaml` files are NOT loaded automatically: pass `--config ` to +load a specific config, or `--discover-config` to load the nearest one found +upward from the current directory. A project profile with the same name shadows +the built-in whole profile. `--config ` without `--profile` runs every +profile in that config and emits a `clawscan-batch-v1` artifact. When a run +skips a discovered config, clawscan prints a stderr notice naming the file. Use `clawscan profiles` to inspect the resolved built-in plus nearest local profile catalog. Use `clawscan profiles -v` to print the merged catalog as From 8e128c02b482ee8bd4ad0b9454a8ab782516ce1e Mon Sep 17 00:00:00 2001 From: Jesse Merhi <79823012+jesse-merhi@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:52:22 +1000 Subject: [PATCH 3/6] fix(profiles): record built-in provenance for embedded profiles An embedded profile run recorded configSource: flags-only (or an unrelated discovered config path). selected.source is authoritative: built-in profiles record built-in provenance regardless of what other config was loaded alongside them. --- internal/profiles/resolver.go | 6 ++++++ internal/profiles/resolver_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/internal/profiles/resolver.go b/internal/profiles/resolver.go index dd7091d..bacf0f0 100644 --- a/internal/profiles/resolver.go +++ b/internal/profiles/resolver.go @@ -175,6 +175,12 @@ func resolveRunSetIntent(intent cliIntent, cwd string) (ResolvedRunSet, error) { } return ResolvedRunSet{}, err } + // selected.source is authoritative: a profile served from + // embedded YAML is "built-in" even when an unrelated project + // config was loaded alongside it. + if selected.source == "built-in" { + configSource = "built-in" + } } } diff --git a/internal/profiles/resolver_test.go b/internal/profiles/resolver_test.go index f8848c2..874d2bc 100644 --- a/internal/profiles/resolver_test.go +++ b/internal/profiles/resolver_test.go @@ -20,6 +20,9 @@ func TestResolveArgsUsesEmbeddedClawHubProfile(t *testing.T) { if opts.Target != "./skill" { t.Fatalf("target = %q", opts.Target) } + if opts.ConfigSource != "built-in" { + t.Fatalf("config source = %q, want built-in", opts.ConfigSource) + } if got := strings.Join(opts.Scanners, ","); got != "skillspector,clawscan-static" { t.Fatalf("scanners = %q", got) } @@ -165,6 +168,32 @@ profiles: } } +func TestResolveArgsBuiltInProfileProvenanceSurvivesDiscoveredConfig(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, ".clawscan.yml"), `version: 1 +profiles: + unrelated: + scanners: + - clawscan-static +`) + + opts, err := ResolveArgs([]string{"./skill", "--profile", "clawhub", "--discover-config"}, dir) + if err != nil { + t.Fatal(err) + } + if opts.ConfigSource != "built-in" { + t.Fatalf("config source = %q, want built-in (unrelated discovered config must not claim provenance)", opts.ConfigSource) + } + + opts, err = ResolveArgs([]string{"./skill", "--profile", "clawhub", "--discover-config"}, t.TempDir()) + if err != nil { + t.Fatal(err) + } + if opts.ConfigSource != "built-in" { + t.Fatalf("config source = %q, want built-in when discovery finds nothing", opts.ConfigSource) + } +} + func TestResolveArgsUnknownProfileMentionsIgnoredConfig(t *testing.T) { dir := t.TempDir() config := filepath.Join(dir, ".clawscan.yml") From e8f79db49d77472b3e30eabfa304c7ffe1b753b9 Mon Sep 17 00:00:00 2001 From: Jesse Merhi <79823012+jesse-merhi@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:03:30 +1000 Subject: [PATCH 4/6] fix(profiles): require --profile with --discover-config Discovery without a profile recorded the discovered file as the run's configSource while applying none of its settings (sandbox mode, image, env). Reject the combination with guidance instead of silently claiming provenance. --- cmd/clawscan/main.go | 2 +- docs/profiles.md | 5 ++++- internal/profiles/resolver.go | 6 ++++++ internal/profiles/resolver_test.go | 15 +++++++++------ 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/cmd/clawscan/main.go b/cmd/clawscan/main.go index c130122..c9e5125 100644 --- a/cmd/clawscan/main.go +++ b/cmd/clawscan/main.go @@ -575,7 +575,7 @@ Usage: Core flags: --profile Profile to run. Use --profile clawhub for ClawHub parity. --config Load profiles from a specific .clawscan.yml file; omit --profile to run them all. - --discover-config Find and load the nearest .clawscan.yml/.clawscan.yaml in the current directory or a parent. + --discover-config Find and load the nearest .clawscan.yml/.clawscan.yaml in the current directory or a parent. Requires --profile. --scanner Scanner to run. Repeat for multiple scanners. --scanner-result Use a JSON fixture instead of running that scanner. --context Load profile runtime context from a JSON file. diff --git a/docs/profiles.md b/docs/profiles.md index 3ea35f1..d19a411 100644 --- a/docs/profiles.md +++ b/docs/profiles.md @@ -24,7 +24,10 @@ To load a discovered config file, use one of these flags: - `--config ` - Explicitly specify a config file path - `--discover-config` - Search upward from the current directory and load the nearest `.clawscan.yml` or `.clawscan.yaml` -Mixing `--config` and `--discover-config` is an error. +Mixing `--config` and `--discover-config` is an error. `--discover-config` +also requires `--profile`: without a profile the run would record the +discovered file as its config source while applying none of its settings. +Use `--config ` without `--profile` to run every profile in a config. ```bash clawscan ./my-skill --config ./security/clawscan.yml --profile review diff --git a/internal/profiles/resolver.go b/internal/profiles/resolver.go index bacf0f0..bcc7ecf 100644 --- a/internal/profiles/resolver.go +++ b/internal/profiles/resolver.go @@ -145,6 +145,12 @@ func resolveRunSetIntent(intent cliIntent, cwd string) (ResolvedRunSet, error) { if intent.configPath != "" && !intent.profileSet { return resolveAllConfigProfiles(intent, cwd) } + // Discovery without a profile would record the discovered file as the + // run's ConfigSource while applying none of its settings — a provenance + // claim the run does not honor. Reject instead of misleading. + if intent.discoverConfig && !intent.profileSet { + return ResolvedRunSet{}, errors.New("--discover-config requires --profile; use --config to run every profile in a config") + } profileName := "" configSource := "" diff --git a/internal/profiles/resolver_test.go b/internal/profiles/resolver_test.go index 874d2bc..c820422 100644 --- a/internal/profiles/resolver_test.go +++ b/internal/profiles/resolver_test.go @@ -261,13 +261,16 @@ profiles: } } -func TestResolveArgsDiscoverConfigWithoutConfigLeavesSourceEmpty(t *testing.T) { - opts, err := ResolveArgs([]string{"./skill", "--scanner", "clawscan-static", "--discover-config"}, t.TempDir()) - if err != nil { - t.Fatal(err) +func TestResolveArgsDiscoverConfigRequiresProfile(t *testing.T) { + // Without a profile the run would record the discovered file as its + // ConfigSource while applying none of its settings (sandbox mode/image/ + // env); rejecting is honest, silently ignoring the config is not. + _, err := ResolveArgs([]string{"./skill", "--scanner", "clawscan-static", "--discover-config"}, t.TempDir()) + if err == nil { + t.Fatal("expected error for --discover-config without --profile") } - if opts.ConfigSource != "" { - t.Fatalf("config source = %q, want empty", opts.ConfigSource) + if !strings.Contains(err.Error(), "--discover-config requires --profile") { + t.Fatalf("err = %v", err) } } From 641e81fb2284f9335b80320620268e7011264836 Mon Sep 17 00:00:00 2001 From: Jesse Merhi <79823012+jesse-merhi@users.noreply.github.com> Date: Wed, 22 Jul 2026 01:41:13 +1000 Subject: [PATCH 5/6] feat(profiles)!: never traverse for config without --discover-config Flags-only runs no longer walk parent directories looking for .clawscan.yml, and the discovered-but-not-loaded stderr notice is gone: if we deliberately do not trust a config, we do not spend I/O finding it or mention it. Discovery lives solely in discoverConfig, reached only via --discover-config. clawscan profiles now lists built-ins only for the same reason. Help and docs explain the security rationale: a config can define scanners that run arbitrary commands, so ClawScan never loads config it was not explicitly told to trust. --- README.md | 5 +-- cmd/clawscan/main.go | 15 ++----- cmd/clawscan/main_test.go | 72 ++++++------------------------ docs/index.md | 2 +- docs/profiles.md | 18 ++++---- internal/profiles/registry.go | 7 +-- internal/profiles/registry_test.go | 27 +++-------- internal/profiles/resolver.go | 52 ++++----------------- internal/profiles/resolver_test.go | 66 ++++++--------------------- internal/runner/runner.go | 1 - skills/clawscan-cli/SKILL.md | 10 ++--- 11 files changed, 62 insertions(+), 213 deletions(-) diff --git a/README.md b/README.md index 9b5cbff..4d84106 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ ClawScan turns that approach into a repeatable CLI. It includes a built-in `claw | --- | --- | | `clawscan --scanner ` | Run one or more scanners against an explicit skill or OpenClaw plugin target. Omit `` to scan child skill directories under `./skills`; plugins are never auto-discovered. | | `clawscan scanners [list\|]` | Discover supported scanner IDs, required env vars, upstream links, descriptions, and install guidance. | -| `clawscan profiles [-v]` | Inspect built-in plus nearest project-local profiles; `-v` prints the resolved profile catalog as YAML. | +| `clawscan profiles [-v]` | Inspect built-in profiles; `-v` prints the catalog as YAML. | | `clawscan benchmark [list\|]` | Discover or run supported benchmarks through a selected scanner/profile/judge setup. | | `clawscan install [...]` | Install or verify local scanner dependencies where ClawScan has registry-backed install plans. | @@ -173,8 +173,7 @@ The same profile accepts an explicit OpenClaw plugin directory (or its `openclaw.plugin.json` manifest), runs both scanners, and renders the bundled judge prompt with `packageRelease` target context. -Inspect the resolved profile catalog, including the nearest project -`.clawscan.yml` / `.clawscan.yaml` when present: +Inspect the built-in profile catalog: ```bash clawscan profiles diff --git a/cmd/clawscan/main.go b/cmd/clawscan/main.go index c9e5125..dfcf661 100644 --- a/cmd/clawscan/main.go +++ b/cmd/clawscan/main.go @@ -61,7 +61,6 @@ func run(args []string, environ []string) error { if err != nil { return err } - printIgnoredConfigNotice(os.Stderr, resolved.IgnoredConfig) if resolved.AllProfiles { return runAllProfiles(resolved, environ, cwd) } @@ -125,7 +124,6 @@ func runBenchmarkCommand(args []string, environ []string) error { if err != nil { return err } - printIgnoredConfigNotice(os.Stderr, resolved.IgnoredConfig) if resolved.AllProfiles { return errors.New("clawscan benchmark does not support --config without --profile") } @@ -155,13 +153,6 @@ func runBenchmarkCommand(args []string, environ []string) error { return nil } -func printIgnoredConfigNotice(w io.Writer, configPath string) { - if configPath == "" { - return - } - fmt.Fprintf(w, "warning: discovered %s at %s but not loading it; use --config %s or --discover-config to load discovered config\n", filepath.Base(configPath), configPath, configPath) -} - func runAllProfiles(resolved profiles.ResolvedRunSet, environ []string, cwd string) error { if !resolved.JSON && resolved.OutputPath == "" { resolved.OutputPath = defaultOutputPath @@ -244,7 +235,7 @@ func runProfiles(args []string) error { if err != nil { return err } - catalog, err := profiles.InspectProfiles(cwd) + catalog, err := profiles.InspectProfiles() if err != nil { return err } @@ -575,7 +566,7 @@ Usage: Core flags: --profile Profile to run. Use --profile clawhub for ClawHub parity. --config Load profiles from a specific .clawscan.yml file; omit --profile to run them all. - --discover-config Find and load the nearest .clawscan.yml/.clawscan.yaml in the current directory or a parent. Requires --profile. + --discover-config Find and load the nearest .clawscan.yml/.clawscan.yaml in the current directory or a parent. Requires --profile. Off by default: config files can define scanners that run arbitrary commands, so ClawScan never loads config it was not explicitly told to trust. --scanner Scanner to run. Repeat for multiple scanners. --scanner-result Use a JSON fixture instead of running that scanner. --context Load profile runtime context from a JSON file. @@ -606,7 +597,7 @@ Catalog commands: clawscan scanners List supported scanners with required env vars. clawscan scanners list Alias for clawscan scanners. clawscan scanners Show scanner repository, description, env vars, and install guidance. - clawscan profiles List built-in plus nearest project .clawscan.yml/.clawscan.yaml profiles. + clawscan profiles List built-in profiles. clawscan profiles -v Print the resolved profile catalog as pasteable YAML. clawscan benchmark list List supported benchmarks with source datasets and splits. diff --git a/cmd/clawscan/main_test.go b/cmd/clawscan/main_test.go index 27bb397..7034acf 100644 --- a/cmd/clawscan/main_test.go +++ b/cmd/clawscan/main_test.go @@ -151,7 +151,7 @@ func TestRunCommandScannerDetailPrintsHumanReadableInfo(t *testing.T) { } } -func TestRunCommandProfilesPrintsMergedDiscoveredProfiles(t *testing.T) { +func TestRunCommandProfilesPrintsBuiltInProfilesOnly(t *testing.T) { dir := t.TempDir() removedProfile := "skills" + "-sh" writeFile(t, filepath.Join(dir, ".clawscan.yml"), `version: 1 @@ -178,12 +178,10 @@ profiles: "Source", "Scanners", "clawhub", - ".clawscan.yml", - "clawscan-static", + "built-in", + "skillspector, clawscan-static", "clawhub-aig", "skillspector, aig", - "local-review", - "snyk", } { if !strings.Contains(stdout, want) { t.Fatalf("profiles output missing %q:\n%s", want, stdout) @@ -192,6 +190,9 @@ profiles: if strings.Contains(stdout, removedProfile) { t.Fatalf("profiles output should not include removed profile %q:\n%s", removedProfile, stdout) } + if strings.Contains(stdout, "local-review") { + t.Fatalf("profiles output should not include project profile:\n%s", stdout) + } } func TestRunCommandProfilesVerbosePrintsResolvedYAML(t *testing.T) { @@ -217,10 +218,8 @@ profiles: "profiles:", "clawhub:", "clawhub-aig:", - "local-review:", - "- clawscan-static", + "- skillspector", "- aig", - "json: true", } { if !strings.Contains(stdout, want) { t.Fatalf("verbose profiles output missing %q:\n%s", want, stdout) @@ -229,6 +228,9 @@ profiles: if strings.Contains(stdout, removedProfile+":") { t.Fatalf("verbose profiles output should not include removed profile %q:\n%s", removedProfile, stdout) } + if strings.Contains(stdout, "local-review:") { + t.Fatalf("verbose profiles output should not include project profile:\n%s", stdout) + } } func TestRunCommandBenchmarkListPrintsCatalogTable(t *testing.T) { @@ -696,7 +698,7 @@ profiles: } } -func TestRunDefaultModeIgnoresConfigAndPrintsNotice(t *testing.T) { +func TestRunDefaultModeIgnoresConfigWithoutNotice(t *testing.T) { dir := t.TempDir() target := filepath.Join(dir, "skill") writeSkill(t, target, "# Ignored config\n") @@ -719,32 +721,8 @@ func TestRunDefaultModeIgnoresConfigAndPrintsNotice(t *testing.T) { if artifact.ConfigSource != "flags-only" { t.Fatalf("config source = %q", artifact.ConfigSource) } - want := "warning: discovered .clawscan.yml at " + config + " but not loading it; use --config " + config + " or --discover-config to load discovered config\n" - if stderr != want { - t.Fatalf("stderr = %q, want %q", stderr, want) - } - if strings.Contains(stdout, "warning: discovered") { - t.Fatalf("notice leaked to stdout: %s", stdout) - } -} - -func TestRunDefaultModeIgnoresParentConfigAndPrintsNotice(t *testing.T) { - parent := t.TempDir() - config := filepath.Join(parent, ".clawscan.yml") - writeFile(t, config, "version: [\n") - child := filepath.Join(parent, "child") - target := filepath.Join(child, "skill") - writeSkill(t, target, "# Parent ignored config\n") - t.Chdir(child) - - _, stderr := captureOutput(t, func() { - if err := run([]string{target, "--scanner", "clawscan-static", "--json"}, []string{}); err != nil { - t.Fatal(err) - } - }) - - if !strings.Contains(stderr, config) || !strings.Contains(stderr, "--discover-config") { - t.Fatalf("stderr = %q", stderr) + if stderr != "" { + t.Fatalf("stderr = %q, want empty", stderr) } } @@ -815,30 +793,6 @@ profiles: } } -func TestRunJSONFlagDoesNotIncludeNoticeInStdout(t *testing.T) { - dir := t.TempDir() - target := filepath.Join(dir, "skill") - writeSkill(t, target, "# JSON notice separation\n") - writeFile(t, filepath.Join(dir, ".clawscan.yml"), "version: 1\nprofiles: {}\n") - t.Chdir(dir) - - stdout, stderr := captureOutput(t, func() { - if err := run([]string{target, "--scanner", "clawscan-static", "--json"}, []string{}); err != nil { - t.Fatal(err) - } - }) - - if !json.Valid([]byte(stdout)) { - t.Fatalf("stdout is not valid JSON: %s", stdout) - } - if strings.Contains(stdout, "warning: discovered") { - t.Fatalf("notice leaked to stdout: %s", stdout) - } - if !strings.Contains(stderr, "warning: discovered") { - t.Fatalf("stderr = %q", stderr) - } -} - func TestBatchArtifactRuns_EachHasConfigSource(t *testing.T) { dir := t.TempDir() writeSkill(t, filepath.Join(dir, "skills", "foo"), "# Foo\n") diff --git a/docs/index.md b/docs/index.md index 36f5afd..5eacfcf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -107,6 +107,6 @@ ClawScan turns that approach into a repeatable CLI. It includes a built-in `claw | --- | --- | | `clawscan --scanner ` | Run one or more scanners against an explicit target. Omit `` to scan child skill directories under `./skills`. | | `clawscan scanners [list\|]` | Discover supported scanner IDs, required env vars, upstream links, descriptions, and install guidance. | -| `clawscan profiles [-v]` | Inspect built-in plus nearest project-local profiles; `-v` prints the resolved profile catalog as YAML. | +| `clawscan profiles [-v]` | Inspect built-in profiles; `-v` prints the catalog as YAML. | | `clawscan benchmark [list\|]` | Discover or run supported benchmarks through a selected scanner/profile/judge setup. | | `clawscan install [...]` | Install or verify local scanner dependencies where ClawScan has registry-backed install plans. | diff --git a/docs/profiles.md b/docs/profiles.md index d19a411..21e605c 100644 --- a/docs/profiles.md +++ b/docs/profiles.md @@ -15,9 +15,14 @@ bundled judge prompt with `packageRelease` target context. ## Config discovery By default, ClawScan does not auto-discover `.clawscan.yml` or `.clawscan.yaml` -files from the current directory or parent directories. If your run detects a -config file that could have been loaded, ClawScan prints a notice to stderr -suggesting you use `--config ` or `--discover-config`. +files from the current directory or parent directories. + +ClawScan never loads a config file it was not explicitly pointed at. A +`.clawscan.yml` can define user-defined scanners whose commands execute with +your credentials in the environment, so silently loading one from the current +directory or a parent (for example, from inside a repository you are scanning) +would let an untrusted target execute commands. Pass `--config ` or opt in +with `--discover-config`. To load a discovered config file, use one of these flags: @@ -35,14 +40,11 @@ clawscan ./my-skill --profile review --discover-config ``` Without either flag, ClawScan uses built-in profiles and CLI flags only. The -warning is a single stderr line and never changes JSON stdout. The -`clawscan profiles` catalog command still includes the nearest discovered -project config. +`clawscan profiles` catalog command lists built-in profiles only. ## Inspect available profiles -Inspect the resolved profile catalog, including the nearest project -`.clawscan.yml` / `.clawscan.yaml` when present: +Inspect the built-in profile catalog: ```bash clawscan profiles diff --git a/internal/profiles/registry.go b/internal/profiles/registry.go index 35671c4..1ed0721 100644 --- a/internal/profiles/registry.go +++ b/internal/profiles/registry.go @@ -53,11 +53,8 @@ func ProfileIDs() []string { return DefaultProfileRegistry().IDs() } -func InspectProfiles(cwd string) (ProfileCatalog, error) { - registry, _, err := loadConfigs(cwd, "", true) - if err != nil { - return ProfileCatalog{}, err - } +func InspectProfiles() (ProfileCatalog, error) { + registry := DefaultProfileRegistry() catalog := ProfileCatalog{profiles: map[string]ProfileInfo{}} for _, id := range registry.IDs() { resolved, _ := registry.Profile(id) diff --git a/internal/profiles/registry_test.go b/internal/profiles/registry_test.go index 82c306a..24e5714 100644 --- a/internal/profiles/registry_test.go +++ b/internal/profiles/registry_test.go @@ -1,8 +1,6 @@ package profiles import ( - "os" - "path/filepath" "strings" "testing" ) @@ -80,27 +78,12 @@ func TestProfileRegistryRejectsUnknownScannerReferences(t *testing.T) { } } -func TestInspectProfilesDiscoversNearestConfigAndShadowsBuiltIns(t *testing.T) { - dir := t.TempDir() - writeFile(t, filepath.Join(dir, ".clawscan.yml"), `version: 1 -profiles: - clawhub: - scanners: - - clawscan-static - local: - scanners: - - socket -`) - nested := filepath.Join(dir, "nested") - if err := os.Mkdir(nested, 0o755); err != nil { - t.Fatal(err) - } - - catalog, err := InspectProfiles(nested) +func TestInspectProfilesReturnsBuiltIns(t *testing.T) { + catalog, err := InspectProfiles() if err != nil { t.Fatal(err) } - if got := strings.Join(catalog.IDs(), ","); got != "clawhub,clawhub-aig,local" { + if got := strings.Join(catalog.IDs(), ","); got != "clawhub,clawhub-aig" { t.Fatalf("profile ids = %q", got) } @@ -108,10 +91,10 @@ profiles: if !ok { t.Fatal("missing clawhub profile") } - if got := strings.Join(clawhub.Profile.Scanners, ","); got != "clawscan-static" { + if got := strings.Join(clawhub.Profile.Scanners, ","); got != "skillspector,clawscan-static" { t.Fatalf("clawhub scanners = %q", got) } - if !strings.HasSuffix(clawhub.Source, ".clawscan.yml") { + if clawhub.Source != "built-in" { t.Fatalf("clawhub source = %q", clawhub.Source) } diff --git a/internal/profiles/resolver.go b/internal/profiles/resolver.go index bcc7ecf..0309b9b 100644 --- a/internal/profiles/resolver.go +++ b/internal/profiles/resolver.go @@ -92,11 +92,10 @@ type cliIntent struct { var judgePathPlaceholderPattern = regexp.MustCompile(`\{\{\s*(prompt|output_schema):([^}]+)\}\}`) type ResolvedRunSet struct { - Options []runner.Options - OutputPath string - JSON bool - AllProfiles bool - IgnoredConfig string + Options []runner.Options + OutputPath string + JSON bool + AllProfiles bool } func ResolveArgs(args []string, cwd string) (runner.Options, error) { @@ -171,15 +170,7 @@ func resolveRunSetIntent(intent cliIntent, cwd string) (ResolvedRunSet, error) { var ok bool selected, ok = registry.Profile(profileName) if !ok { - err := unknownProfileError(profileName, registry.IDs()) - // The profile may live in a discovered-but-unloaded config; - // surface the migration hint here or the user never sees it. - if intent.configPath == "" && !intent.discoverConfig { - if ignored, findErr := findIgnoredConfig(cwd); findErr == nil && ignored != "" { - err = fmt.Errorf("%w (found %s but did not load it; pass --config %s or --discover-config)", err, ignored, ignored) - } - } - return ResolvedRunSet{}, err + return ResolvedRunSet{}, unknownProfileError(profileName, registry.IDs()) } // selected.source is authoritative: a profile served from // embedded YAML is "built-in" even when an unrelated project @@ -201,12 +192,6 @@ func resolveRunSetIntent(intent cliIntent, cwd string) (ResolvedRunSet, error) { opts.Profile = profileName opts.ConfigSource = configSource opts.DiscoverConfig = intent.discoverConfig - if intent.configPath == "" && !intent.discoverConfig { - opts.IgnoredConfig, err = findIgnoredConfig(cwd) - if err != nil { - return ResolvedRunSet{}, err - } - } if opts.Judge != nil { opts.Judge.Files = files } @@ -217,10 +202,9 @@ func resolveRunSetIntent(intent cliIntent, cwd string) (ResolvedRunSet, error) { } } return ResolvedRunSet{ - Options: []runner.Options{opts}, - OutputPath: opts.OutputPath, - JSON: opts.JSON, - IgnoredConfig: opts.IgnoredConfig, + Options: []runner.Options{opts}, + OutputPath: opts.OutputPath, + JSON: opts.JSON, }, nil } @@ -450,26 +434,6 @@ func discoverConfig(cwd string) (string, error) { } } -func findIgnoredConfig(cwd string) (string, error) { - current, err := filepath.Abs(cwd) - if err != nil { - return "", err - } - for { - for _, name := range []string{".clawscan.yml", ".clawscan.yaml"} { - path := filepath.Join(current, name) - if fileExists(path) { - return path, nil - } - } - parent := filepath.Dir(current) - if parent == current { - return "", nil - } - current = parent - } -} - func fileExists(path string) bool { info, err := os.Stat(path) return err == nil && !info.IsDir() diff --git a/internal/profiles/resolver_test.go b/internal/profiles/resolver_test.go index c820422..654af3e 100644 --- a/internal/profiles/resolver_test.go +++ b/internal/profiles/resolver_test.go @@ -143,14 +143,18 @@ func TestResolveArgsTreatsScannerOnlyCommandAsAdHocWithoutDefaultJudge(t *testin } } -func TestResolveArgsDefaultRunDoesNotAutoDiscover_WithCwdConfig(t *testing.T) { - dir := t.TempDir() - writeFile(t, filepath.Join(dir, ".clawscan.yml"), `version: 1 +func TestResolveArgsFlagsOnlyRunIgnoresAncestorConfig(t *testing.T) { + parent := t.TempDir() + writeFile(t, filepath.Join(parent, ".clawscan.yml"), `version: 1 profiles: custom: scanners: - virustotal `) + dir := filepath.Join(parent, "child") + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } opts, err := ResolveArgs([]string{"./skill", "--scanner", "clawscan-static"}, dir) if err != nil { @@ -163,9 +167,6 @@ profiles: if opts.ConfigSource != "flags-only" { t.Fatalf("config source = %q, want flags-only", opts.ConfigSource) } - if opts.IgnoredConfig != filepath.Join(dir, ".clawscan.yml") { - t.Fatalf("ignored config = %q, want %s", opts.IgnoredConfig, filepath.Join(dir, ".clawscan.yml")) - } } func TestResolveArgsBuiltInProfileProvenanceSurvivesDiscoveredConfig(t *testing.T) { @@ -194,10 +195,9 @@ profiles: } } -func TestResolveArgsUnknownProfileMentionsIgnoredConfig(t *testing.T) { +func TestResolveArgsUnknownProfileDoesNotDiscoverConfig(t *testing.T) { dir := t.TempDir() - config := filepath.Join(dir, ".clawscan.yml") - writeFile(t, config, `version: 1 + writeFile(t, filepath.Join(dir, ".clawscan.yml"), `version: 1 profiles: custom: scanners: @@ -208,30 +208,9 @@ profiles: if err == nil { t.Fatal("expected unknown profile error") } - if !strings.Contains(err.Error(), "Unknown profile: custom") { - t.Fatalf("error = %q, want unknown profile", err) - } - if !strings.Contains(err.Error(), config) || !strings.Contains(err.Error(), "--discover-config") { - t.Fatalf("error = %q, want ignored-config hint naming %s and --discover-config", err, config) - } -} - -func TestResolveArgsDefaultRunDoesNotAutoDiscover_WithParentConfig(t *testing.T) { - parent := t.TempDir() - config := filepath.Join(parent, ".clawscan.yml") - writeFile(t, config, "version: 1\nprofiles: {}\n") - child := filepath.Join(parent, "child") - if err := os.Mkdir(child, 0o755); err != nil { - t.Fatal(err) - } - - opts, err := ResolveArgs([]string{"./skill", "--scanner", "clawscan-static"}, child) - if err != nil { - t.Fatal(err) - } - - if opts.IgnoredConfig != config { - t.Fatalf("ignored config = %q, want %q", opts.IgnoredConfig, config) + want := "Unknown profile: custom (available: clawhub, clawhub-aig)" + if err.Error() != want { + t.Fatalf("error = %q, want %q", err, want) } } @@ -285,7 +264,7 @@ func TestResolveArgsConfigAndDiscoverConfigFlagsAreMutuallyExclusive(t *testing. } } -func TestResolveArgsExplicitConfigSkipsDiscoveryAndPrintsNoNotice(t *testing.T) { +func TestResolveArgsExplicitConfigUsesExplicitSource(t *testing.T) { dir := t.TempDir() writeFile(t, filepath.Join(dir, ".clawscan.yml"), `version: 1 profiles: @@ -309,25 +288,6 @@ profiles: if opts.ConfigSource != explicit { t.Fatalf("config source = %q", opts.ConfigSource) } - if opts.IgnoredConfig != "" { - t.Fatalf("ignored config = %q, want empty", opts.IgnoredConfig) - } -} - -func TestResolveArgsNoConfigAnywherePrintsNoNotice(t *testing.T) { - dir := t.TempDir() - - opts, err := ResolveArgs([]string{"./skill", "--scanner", "clawscan-static"}, dir) - if err != nil { - t.Fatal(err) - } - - if opts.ConfigSource != "flags-only" { - t.Fatalf("config source = %q", opts.ConfigSource) - } - if opts.IgnoredConfig != "" { - t.Fatalf("ignored config = %q, want empty", opts.IgnoredConfig) - } } func TestResolveArgsAllowsExplicitProfileWithoutTarget(t *testing.T) { diff --git a/internal/runner/runner.go b/internal/runner/runner.go index f5b3a94..56470e7 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -30,7 +30,6 @@ type Options struct { Profile string ConfigSource string DiscoverConfig bool - IgnoredConfig string ContextPath string Benchmark *BenchmarkOptions Scanners []string diff --git a/skills/clawscan-cli/SKILL.md b/skills/clawscan-cli/SKILL.md index 204c251..c8a742e 100644 --- a/skills/clawscan-cli/SKILL.md +++ b/skills/clawscan-cli/SKILL.md @@ -88,12 +88,12 @@ Profiles are loaded from embedded built-ins. Project `.clawscan.yml` / load a specific config, or `--discover-config` to load the nearest one found upward from the current directory. A project profile with the same name shadows the built-in whole profile. `--config ` without `--profile` runs every -profile in that config and emits a `clawscan-batch-v1` artifact. When a run -skips a discovered config, clawscan prints a stderr notice naming the file. +profile in that config and emits a `clawscan-batch-v1` artifact. Discovery is +off by default because project configs can define scanner commands that execute +with the caller's environment and credentials. -Use `clawscan profiles` to inspect the resolved built-in plus nearest local -profile catalog. Use `clawscan profiles -v` to print the merged catalog as -pasteable YAML. +Use `clawscan profiles` to inspect the built-in profile catalog. Use +`clawscan profiles -v` to print it as pasteable YAML. CLI flags override the selected profile for one run. Passing `--scanner` without `--profile` creates an ad hoc scanner-only run, so profile judges are From 570f5410920e168d4a699283adcdbc7e62aa3452 Mon Sep 17 00:00:00 2001 From: Jesse Merhi <79823012+jesse-merhi@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:57:25 +1000 Subject: [PATCH 6/6] feat(runner)!: emit configSource as explicit null for flags-only runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flags-only sentinel read like an absence dressed up as a value. configSource is now a path, "built-in", or JSON null — and the field is deliberately never omitted: an explicit null is the artifact's positive claim that no config file or embedded profile influenced the run, distinct from an older schema that simply lacked the field. --- cmd/clawscan/main_test.go | 36 ++++++++++++++++++++---------- internal/profiles/resolver.go | 3 --- internal/profiles/resolver_test.go | 4 ++-- internal/runner/runner.go | 31 +++++++++++++++---------- internal/runner/runner_test.go | 19 +++++----------- 5 files changed, 50 insertions(+), 43 deletions(-) diff --git a/cmd/clawscan/main_test.go b/cmd/clawscan/main_test.go index 7034acf..2390c39 100644 --- a/cmd/clawscan/main_test.go +++ b/cmd/clawscan/main_test.go @@ -713,13 +713,16 @@ func TestRunDefaultModeIgnoresConfigWithoutNotice(t *testing.T) { }) var artifact struct { - ConfigSource string `json:"configSource"` + ConfigSource *string `json:"configSource"` } if err := json.Unmarshal([]byte(stdout), &artifact); err != nil { t.Fatalf("stdout is not artifact JSON: %v\n%s", err, stdout) } - if artifact.ConfigSource != "flags-only" { - t.Fatalf("config source = %q", artifact.ConfigSource) + if artifact.ConfigSource != nil { + t.Fatalf("config source = %q, want nil", *artifact.ConfigSource) + } + if !strings.Contains(stdout, `"configSource": null`) { + t.Fatalf("stdout lacks explicit null config source: %s", stdout) } if stderr != "" { t.Fatalf("stderr = %q, want empty", stderr) @@ -746,13 +749,16 @@ profiles: }) var artifact struct { - ConfigSource string `json:"configSource"` + ConfigSource *string `json:"configSource"` } if err := json.Unmarshal([]byte(stdout), &artifact); err != nil { t.Fatal(err) } - if artifact.ConfigSource != config { - t.Fatalf("config source = %q, want %q", artifact.ConfigSource, config) + if artifact.ConfigSource == nil { + t.Fatalf("config source = nil, want %q", config) + } + if *artifact.ConfigSource != config { + t.Fatalf("config source = %q, want %q", *artifact.ConfigSource, config) } if stderr != "" { t.Fatalf("stderr = %q", stderr) @@ -780,13 +786,16 @@ profiles: }) var artifact struct { - ConfigSource string `json:"configSource"` + ConfigSource *string `json:"configSource"` } if err := json.Unmarshal([]byte(stdout), &artifact); err != nil { t.Fatal(err) } - if artifact.ConfigSource != explicit { - t.Fatalf("config source = %q, want %q", artifact.ConfigSource, explicit) + if artifact.ConfigSource == nil { + t.Fatalf("config source = nil, want %q", explicit) + } + if *artifact.ConfigSource != explicit { + t.Fatalf("config source = %q, want %q", *artifact.ConfigSource, explicit) } if stderr != "" { t.Fatalf("stderr = %q", stderr) @@ -819,7 +828,7 @@ profiles: SchemaVersion string `json:"schemaVersion"` Runs []struct { Profile string `json:"profile"` - ConfigSource string `json:"configSource"` + ConfigSource *string `json:"configSource"` Scanners map[string]interface{} `json:"scanners"` } `json:"runs"` } @@ -836,8 +845,11 @@ profiles: t.Fatalf("profiles = %q", got) } for _, run := range artifact.Runs { - if run.ConfigSource != config { - t.Fatalf("config source = %q, want %q", run.ConfigSource, config) + if run.ConfigSource == nil { + t.Fatalf("config source = nil, want %q", config) + } + if *run.ConfigSource != config { + t.Fatalf("config source = %q, want %q", *run.ConfigSource, config) } if _, ok := run.Scanners["clawscan-static"]; !ok { t.Fatalf("missing clawscan-static scanner for %s: %#v", run.Profile, run.Scanners) diff --git a/internal/profiles/resolver.go b/internal/profiles/resolver.go index 0309b9b..9afadc5 100644 --- a/internal/profiles/resolver.go +++ b/internal/profiles/resolver.go @@ -153,9 +153,6 @@ func resolveRunSetIntent(intent cliIntent, cwd string) (ResolvedRunSet, error) { profileName := "" configSource := "" - if intent.configPath == "" && !intent.discoverConfig { - configSource = "flags-only" - } selected := resolvedProfile{profile: Profile{}} if intent.profileSet || intent.configPath != "" || intent.discoverConfig { registry, loadedConfig, err := loadConfigs(cwd, intent.configPath, intent.discoverConfig) diff --git a/internal/profiles/resolver_test.go b/internal/profiles/resolver_test.go index 654af3e..55e3b48 100644 --- a/internal/profiles/resolver_test.go +++ b/internal/profiles/resolver_test.go @@ -164,8 +164,8 @@ profiles: if got := strings.Join(opts.Scanners, ","); got != "clawscan-static" { t.Fatalf("scanners = %q", got) } - if opts.ConfigSource != "flags-only" { - t.Fatalf("config source = %q, want flags-only", opts.ConfigSource) + if opts.ConfigSource != "" { + t.Fatalf("config source = %q, want empty for a flags-only run", opts.ConfigSource) } } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 56470e7..c8b735a 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -87,17 +87,19 @@ type CommandOutput struct { } type Artifact struct { - SchemaVersion string `json:"schemaVersion"` - Profile string `json:"profile,omitempty"` - ConfigSource string `json:"configSource,omitempty"` - Context json.RawMessage `json:"context,omitempty"` - Target Target `json:"target"` - StartedAt string `json:"startedAt"` - CompletedAt string `json:"completedAt"` - Env map[string]string `json:"env"` - Sandbox SandboxMetadata `json:"sandbox"` - Scanners map[string]ScannerResult `json:"scanners"` - Judge *JudgeResult `json:"judge"` + SchemaVersion string `json:"schemaVersion"` + Profile string `json:"profile,omitempty"` + // ConfigSource deliberately lacks omitempty: null positively claims no config + // influenced the run, unlike older artifacts that omitted the field. + ConfigSource *string `json:"configSource"` + Context json.RawMessage `json:"context,omitempty"` + Target Target `json:"target"` + StartedAt string `json:"startedAt"` + CompletedAt string `json:"completedAt"` + Env map[string]string `json:"env"` + Sandbox SandboxMetadata `json:"sandbox"` + Scanners map[string]ScannerResult `json:"scanners"` + Judge *JudgeResult `json:"judge"` } type RunTargetsResult struct { @@ -2117,10 +2119,15 @@ func NewArtifact(opts Options, resolvedPath string, startedAt string, completedA Raw: nil, } } + var configSource *string + if opts.ConfigSource != "" { + source := opts.ConfigSource + configSource = &source + } return Artifact{ SchemaVersion: "clawscan-run-v1", Profile: opts.Profile, - ConfigSource: opts.ConfigSource, + ConfigSource: configSource, Target: Target{ Kind: "skill", Input: opts.Target, diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index bdf12c3..66b0726 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -1105,28 +1105,19 @@ func TestArtifactConfigSourceField_FlagsOnly(t *testing.T) { if err != nil { t.Fatal(err) } - opts.ConfigSource = "flags-only" + opts.ConfigSource = "" artifact := NewArtifact(opts, "/tmp/my-skill", "2026-06-03T00:00:00Z", "2026-06-03T00:00:01Z", map[string]string{}) - if artifact.ConfigSource != "flags-only" { - t.Fatalf("config source = %q", artifact.ConfigSource) + if artifact.ConfigSource != nil { + t.Fatalf("config source = %q, want nil", *artifact.ConfigSource) } -} - -func TestArtifactConfigSourceField_OmitsEmptySource(t *testing.T) { - opts, err := ParseArgs([]string{"./my-skill", "--scanner", "clawscan-static"}) - if err != nil { - t.Fatal(err) - } - - artifact := NewArtifact(opts, "/tmp/my-skill", "2026-06-03T00:00:00Z", "2026-06-03T00:00:01Z", map[string]string{}) raw, err := json.Marshal(artifact) if err != nil { t.Fatal(err) } - if bytes.Contains(raw, []byte(`"configSource"`)) { - t.Fatalf("empty config source was serialized: %s", raw) + if !strings.Contains(string(raw), `"configSource":null`) { + t.Fatalf("artifact lacks explicit null config source: %s", raw) } }