Skip to content

fix(profiles): remove VirusTotal from clawhub #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Patrick-Erichsen merged 1 commit into from
Jul 17, 2026
Merged

fix(profiles): remove VirusTotal from clawhub #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ clawscan profiles -v

| Profile | Scanners | Judge |
| --- | --- | --- |
| `clawhub` | `skillspector`, `virustotal`, `clawscan-static` | Codex `gpt-5.5`, high reasoning, bundled ClawHub prompt/schema |
| `clawhub` | `skillspector`, `clawscan-static` | Codex `gpt-5.5`, high reasoning, bundled ClawHub prompt/schema |

### Build a custom profile with `.clawscan.yml`

Expand Down
6 changes: 0 additions & 6 deletions cmd/clawscan/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -498,15 +498,12 @@ func TestRunCommandUsesBuiltInProfile(t *testing.T) {
writeSkill(t, target, "# Profile\n")
skillSpectorFixture := filepath.Join(dir, "skillspector.json")
writeFile(t, skillSpectorFixture, `{"status":"clean","findings":[]}`)
virusTotalFixture := filepath.Join(dir, "virustotal.json")
writeFile(t, virusTotalFixture, `{"status":"not_found","data":null}`)

stdout := captureStdout(t, func() {
if err := run([]string{
target,
"--profile", "clawhub",
"--scanner-result", "skillspector=" + skillSpectorFixture,
"--scanner-result", "virustotal=" + virusTotalFixture,
"--judge", clawHubReceiptJudgeCommand(),
"--sandbox", "off",
"--json",
Expand Down Expand Up @@ -542,15 +539,12 @@ func TestRunCommandDiscoversSkillsWithExplicitProfile(t *testing.T) {
writeSkill(t, filepath.Join(dir, "skills", "bar"), "# Bar\n")
skillSpectorFixture := filepath.Join(dir, "skillspector.json")
writeFile(t, skillSpectorFixture, `{"status":"clean","findings":[]}`)
virusTotalFixture := filepath.Join(dir, "virustotal.json")
writeFile(t, virusTotalFixture, `{"status":"not_found","data":null}`)
t.Chdir(dir)

stdout := captureStdout(t, func() {
if err := run([]string{
"--profile", "clawhub",
"--scanner-result", "skillspector=" + skillSpectorFixture,
"--scanner-result", "virustotal=" + virusTotalFixture,
"--judge", clawHubReceiptJudgeCommand(),
"--sandbox", "off",
"--json",
Expand Down
141 changes: 35 additions & 106 deletions cmd/verify-clawhub-prompt/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"regexp"
"strings"

"github.com/openclaw/clawscan/internal/clawhubprompt"
"github.com/openclaw/clawscan/internal/profiles"
"github.com/openclaw/clawscan/internal/runner"
)
Expand All @@ -34,7 +33,6 @@ func run() error {
outClawScanPrompt := flag.String("out-clawscan-prompt", "", "Optional exact ClawScan-rendered prompt path")
outOutputSchema := flag.String("out-output-schema", "", "Optional exported output schema path")
outSkillSpectorResult := flag.String("out-skillspector-result", "", "Optional exported SkillSpector fixture JSON path")
outVirusTotalResult := flag.String("out-virustotal-result", "", "Optional exported VirusTotal fixture JSON path")
fixturePath := flag.String("fixture", "", "Optional production context and scanner fixture JSON path")
flag.Parse()
if *clawhubDir == "" {
Expand All @@ -60,7 +58,7 @@ func run() error {
)
}
systemPrompt := rendered.SystemPrompt
expected, actual, vtInput, skillSpectorInput, fixtureLabel, err := renderParityInputs(
expected, actual, skillSpectorInput, fixtureLabel, err := renderParityInputs(
resolvedClawHubDir,
*fixturePath,
rendered,
Expand All @@ -87,15 +85,11 @@ func run() error {
if err != nil {
return err
}
vtFixture, err := prettyJSON(vtInput)
if err != nil {
return err
}
skillSpectorFixture, err := prettyJSON(skillSpectorInput)
if err != nil {
return err
}
promptTemplate, err := buildPromptTemplate(actual, vtFixture, skillSpectorFixture)
promptTemplate, err := buildPromptTemplate(actual, skillSpectorFixture)
if err != nil {
promptTemplate, err = buildPromptTemplateFromRendered(actual)
if err != nil {
Expand Down Expand Up @@ -142,9 +136,6 @@ func run() error {
if err := writeOptionalFile(*outSkillSpectorResult, []byte(skillSpectorFixture)); err != nil {
return err
}
if err := writeOptionalFile(*outVirusTotalResult, []byte(vtFixture)); err != nil {
return err
}
if err := writeOptionalFile(*outOutputSchema, schema); err != nil {
return err
}
Expand Down Expand Up @@ -174,7 +165,6 @@ func firstPromptDifference(expected string, actual string) string {

type parityFixture struct {
Context json.RawMessage `json:"context"`
VirusTotal json.RawMessage `json:"virustotal"`
SkillSpector json.RawMessage `json:"skillspector"`
}

Expand All @@ -183,9 +173,8 @@ func renderParityInputs(
fixturePath string,
rendered clawHubPromptRender,
clawScanPromptSource string,
) (expected string, actual string, vtInput any, skillSpectorInput any, fixtureLabel string, err error) {
) (expected string, actual string, skillSpectorInput any, fixtureLabel string, err error) {
if fixturePath == "" {
job := proofJob()
skillSpectorAnalysis := proofSkillSpectorAnalysis()
actual, err := runner.RenderClawHubPrompt(clawScanPromptSource, runner.Artifact{
Profile: "clawhub",
Expand All @@ -198,64 +187,56 @@ func renderParityInputs(
"skillSpectorCheckedAt":123
}`),
Scanners: map[string]runner.ScannerResult{
"virustotal": {
Status: "completed",
Raw: mustJSON(job.Target.Version.VTAnalysis),
},
"skillspector": {
Status: "completed",
Raw: mustJSON(skillSpectorAnalysis),
},
},
})
if err != nil {
return "", "", nil, nil, "", err
return "", "", nil, "", err
}
return rendered.Prompt, actual, job.Target.Version.VTAnalysis, skillSpectorAnalysis, "synthetic", nil
return rendered.Prompt, actual, skillSpectorAnalysis, "synthetic", nil
}

resolvedFixturePath, err := filepath.Abs(fixturePath)
if err != nil {
return "", "", nil, nil, "", err
return "", "", nil, "", err
}
raw, err := os.ReadFile(resolvedFixturePath)
if err != nil {
return "", "", nil, nil, "", err
return "", "", nil, "", err
}
var fixture parityFixture
if err := json.Unmarshal(raw, &fixture); err != nil {
return "", "", nil, nil, "", fmt.Errorf("parse parity fixture: %w", err)
return "", "", nil, "", fmt.Errorf("parse parity fixture: %w", err)
}
if !json.Valid(fixture.Context) {
return "", "", nil, nil, "", fmt.Errorf("parity fixture context is not valid JSON")
return "", "", nil, "", fmt.Errorf("parity fixture context is not valid JSON")
}
context, err := parityFixtureContext(fixture)
if err != nil {
return "", "", nil, nil, "", err
return "", "", nil, "", err
}
expected, err = renderClawHubFixturePrompt(clawhubDir, resolvedFixturePath)
if err != nil {
return "", "", nil, nil, "", err
return "", "", nil, "", err
}
artifact := runner.Artifact{
Profile: "clawhub",
Context: context,
Scanners: map[string]runner.ScannerResult{
"virustotal": {Status: "completed", Raw: fixture.VirusTotal},
"skillspector": {Status: "completed", Raw: fixture.SkillSpector},
},
}
actual, err = runner.RenderClawHubPrompt(clawScanPromptSource, artifact)
if err != nil {
return "", "", nil, nil, "", err
}
if err := json.Unmarshal(fixture.VirusTotal, &vtInput); err != nil {
return "", "", nil, nil, "", fmt.Errorf("parse VirusTotal fixture: %w", err)
return "", "", nil, "", err
}
if err := json.Unmarshal(fixture.SkillSpector, &skillSpectorInput); err != nil {
return "", "", nil, nil, "", fmt.Errorf("parse SkillSpector fixture: %w", err)
return "", "", nil, "", fmt.Errorf("parse SkillSpector fixture: %w", err)
}
return expected, actual, vtInput, skillSpectorInput, resolvedFixturePath, nil
return expected, actual, skillSpectorInput, resolvedFixturePath, nil
}

func parityFixtureContext(fixture parityFixture) (json.RawMessage, error) {
Expand Down Expand Up @@ -369,46 +350,32 @@ func writeOptionalFile(path string, content []byte) error {
return os.WriteFile(path, content, 0o644)
}

func buildPromptTemplate(prompt string, vtFixture string, skillSpectorFixture string) (string, error) {
replacements := []struct {
fixture string
placeholder string
}{
{fixture: vtFixture, placeholder: "{{ scanners.virustotal }}"},
{fixture: skillSpectorFixture, placeholder: "{{ scanners.skillspector }}"},
}
for _, replacement := range replacements {
prompt = strings.Replace(prompt, replacement.fixture, replacement.placeholder, 1)
if !strings.Contains(prompt, replacement.placeholder) {
return "", fmt.Errorf("failed to build exported prompt template with scanner placeholder %s", replacement.placeholder)
}
func buildPromptTemplate(prompt string, skillSpectorFixture string) (string, error) {
const placeholder = "{{ scanners.skillspector }}"
prompt = strings.Replace(prompt, skillSpectorFixture, placeholder, 1)
if !strings.Contains(prompt, placeholder) {
return "", fmt.Errorf("failed to build exported prompt template with scanner placeholder %s", placeholder)
}
return prompt, nil
}

func buildPromptTemplateFromRendered(prompt string) (string, error) {
replacements := []struct {
label string
placeholder string
}{
{label: "VirusTotal telemetry supplied to Codex:", placeholder: "{{ scanners.virustotal }}"},
{label: "SkillSpector findings supplied to Codex:", placeholder: "{{ scanners.skillspector }}"},
}
for _, replacement := range replacements {
prefix := replacement.label + "\n```json\n"
start := strings.Index(prompt, prefix)
if start == -1 {
return "", fmt.Errorf("rendered prompt missing %s block", replacement.label)
}
contentStart := start + len(prefix)
contentEndOffset := strings.Index(prompt[contentStart:], "\n```")
if contentEndOffset == -1 {
return "", fmt.Errorf("rendered prompt has unterminated %s block", replacement.label)
}
contentEnd := contentStart + contentEndOffset
prompt = prompt[:contentStart] + replacement.placeholder + prompt[contentEnd:]
}
return prompt, nil
const (
label = "SkillSpector findings supplied to Codex:"
placeholder = "{{ scanners.skillspector }}"
)
prefix := label + "\n```json\n"
start := strings.Index(prompt, prefix)
if start == -1 {
return "", fmt.Errorf("rendered prompt missing %s block", label)
}
contentStart := start + len(prefix)
contentEndOffset := strings.Index(prompt[contentStart:], "\n```")
if contentEndOffset == -1 {
return "", fmt.Errorf("rendered prompt has unterminated %s block", label)
}
contentEnd := contentStart + contentEndOffset
return prompt[:contentStart] + placeholder + prompt[contentEnd:], nil
}

func prettyJSON(value any) (string, error) {
Expand Down Expand Up @@ -516,28 +483,6 @@ func clawHubWorkerOutputSchemaRelPath(workerSource string) (string, error) {
return matches[1], nil
}

func proofJob() clawhubprompt.Job {
return clawhubprompt.Job{
Job: clawhubprompt.JobMetadata{
TargetKind: "skillVersion",
Source: "publish",
HasMaliciousSignal: true,
},
Target: clawhubprompt.Target{
TrustedOpenClawPlugin: true,
Version: &clawhubprompt.Version{
VTAnalysis: vtAnalysis{
Status: "suspicious",
Source: "engines",
Metadata: vtMetadata{
Stats: vtStats{Malicious: 1, Suspicious: 0, Harmless: 12},
},
},
},
},
}
}

func proofSkillSpectorAnalysis() skillSpectorAnalysis {
return skillSpectorAnalysis{
Status: "suspicious",
Expand All @@ -553,22 +498,6 @@ func proofSkillSpectorAnalysis() skillSpectorAnalysis {
}
}

type vtAnalysis struct {
Status string `json:"status"`
Source string `json:"source"`
Metadata vtMetadata `json:"metadata"`
}

type vtMetadata struct {
Stats vtStats `json:"stats"`
}

type vtStats struct {
Malicious int `json:"malicious"`
Suspicious int `json:"suspicious"`
Harmless int `json:"harmless"`
}

type skillSpectorAnalysis struct {
Status string `json:"status"`
Score int `json:"score"`
Expand Down
43 changes: 11 additions & 32 deletions cmd/verify-clawhub-prompt/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,51 +25,34 @@ func TestResolveClawHubDirReturnsAbsolutePath(t *testing.T) {
}
}

func TestBuildPromptTemplateReplacesScannerFixtures(t *testing.T) {
vtFixture := `{
"status": "suspicious",
"source": "engines"
}`
func TestBuildPromptTemplateReplacesSkillSpectorFixture(t *testing.T) {
skillSpectorFixture := `{
"status": "suspicious",
"score": 55
}`
prompt := strings.Join([]string{
"VirusTotal telemetry supplied to Codex:",
"```json",
vtFixture,
"```",
"SkillSpector findings supplied to Codex:",
"```json",
skillSpectorFixture,
"```",
}, "\n")

template, err := buildPromptTemplate(prompt, vtFixture, skillSpectorFixture)
template, err := buildPromptTemplate(prompt, skillSpectorFixture)
if err != nil {
t.Fatal(err)
}

for _, placeholder := range []string{"{{ scanners.virustotal }}", "{{ scanners.skillspector }}"} {
if !strings.Contains(template, placeholder) {
t.Fatalf("template missing placeholder %s:\n%s", placeholder, template)
}
if !strings.Contains(template, "{{ scanners.skillspector }}") {
t.Fatalf("template missing SkillSpector placeholder:\n%s", template)
}
for _, fixture := range []string{vtFixture, skillSpectorFixture} {
if strings.Contains(template, fixture) {
t.Fatalf("template still contains raw fixture %q:\n%s", fixture, template)
}
if strings.Contains(template, skillSpectorFixture) {
t.Fatalf("template still contains raw fixture:\n%s", template)
}
}

func TestBuildPromptTemplateFromRenderedReplacesScannerBlocks(t *testing.T) {
func TestBuildPromptTemplateFromRenderedReplacesSkillSpectorBlock(t *testing.T) {
prompt := strings.Join([]string{
"Before",
"VirusTotal telemetry supplied to Codex:",
"```json",
`{"status":"clean"}`,
"```",
"Middle",
"SkillSpector findings supplied to Codex:",
"```json",
`{"status":"suspicious"}`,
Expand All @@ -81,15 +64,11 @@ func TestBuildPromptTemplateFromRenderedReplacesScannerBlocks(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, placeholder := range []string{"{{ scanners.virustotal }}", "{{ scanners.skillspector }}"} {
if !strings.Contains(template, placeholder) {
t.Fatalf("template missing placeholder %s:\n%s", placeholder, template)
}
if !strings.Contains(template, "{{ scanners.skillspector }}") {
t.Fatalf("template missing SkillSpector placeholder:\n%s", template)
}
for _, fixture := range []string{`{"status":"clean"}`, `{"status":"suspicious"}`} {
if strings.Contains(template, fixture) {
t.Fatalf("template still contains raw fixture %q:\n%s", fixture, template)
}
if strings.Contains(template, `{"status":"suspicious"}`) {
t.Fatalf("template still contains raw fixture:\n%s", template)
}
}

Expand Down
2 changes: 1 addition & 1 deletion docs/profiles.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ clawscan profiles -v

| Profile | Scanners | Judge |
| --- | --- | --- |
| `clawhub` | `skillspector`, `virustotal`, `clawscan-static` | Codex `gpt-5.5`, high reasoning, bundled ClawHub prompt/schema |
| `clawhub` | `skillspector`, `clawscan-static` | Codex `gpt-5.5`, high reasoning, bundled ClawHub prompt/schema |

## Build a custom profile with `.clawscan.yml`

Expand Down
Loading
Loading