From 304a6ea15ee1e9920adcb7f701cb5d7d24e3f3b8 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 07:02:11 +0000 Subject: [PATCH] ci(monitor): bump budget + Anthropic deep-dive on the ambiguous band Enable targeted escalation in the scheduled sweep now that the pipeline is reliably committing: - --max-scans 3000 -> 4000 (rules-only is fetch-bound at ~2 req/s; 4000 still fits the window and shortens the ~66k-skill baseline build). - --escalate-backend anthropic --escalate-budget 200: the ~150 skills/run the rules flag short of a confident verdict (the ambiguous band) get an LLM deep-dive second opinion; confident-clean and confident-malicious are never escalated, so spend tracks the uncertain middle. - AnthropicBackend no longer live-crawls URLs during escalation (use_urls off): removes external-fetch latency/flakiness at escalation scale; the deep dive's value is the LLM's read of intent (rules + threat intel still run). HF classifier stays validated (protectai/deberta-v3-base-prompt-injection-v2, apache-2.0, ungated) and available for local "--escalate-backend hf|tiered", but is kept off the CI critical path to avoid a ~2GB torch install on the run that must reliably commit. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01DNoTXU8k3pfSBzR7aJubqL --- .github/workflows/registry-monitor.yml | 37 +++++++++++++++----------- docs/crawl.md | 2 +- src/malwar/monitor/escalation.py | 5 +++- tests/unit/test_escalation.py | 2 +- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/.github/workflows/registry-monitor.yml b/.github/workflows/registry-monitor.yml index 553634a..61d106c 100644 --- a/.github/workflows/registry-monitor.yml +++ b/.github/workflows/registry-monitor.yml @@ -4,10 +4,11 @@ # the `registry-snapshots` branch so `git log -p registry-snapshots` is a # permanent, auditable record of what changed day over day. # -# Scans with the rule engine + threat intel only (no per-skill LLM calls) so a -# whole-registry sweep is fast and cheap; the rule engine alone catches 100% of -# malicious samples in the benchmark. Use `malwar crawl scan ` for an LLM -# deep-dive on an individual flagged skill. +# Scans the whole registry with the rule engine + threat intel (fast, free, 100% +# recall on the benchmark), and escalates ONLY the ambiguous band — skills the +# rules flagged short of a confident verdict — to the Anthropic LLM for a +# budget-capped deep-dive second opinion. Confident-clean and confident-malicious +# skills are never escalated, so LLM spend tracks the small uncertain middle. # # Two cadences: # * Daily (06:00 UTC) — INCREMENTAL. Only skills whose version/updated_at @@ -97,23 +98,27 @@ jobs: else echo "Mode: incremental (only changed skills)." fi - # Rules + threat-intel only (--no-escalate): the rule engine alone - # detects 100% of malicious samples in the benchmark, and skipping - # per-skill LLM calls keeps a whole-registry sweep fast and cheap. - # (Run `malwar crawl scan ` for an LLM deep-dive on any one - # flagged skill.) + # The whole-registry sweep runs the rule engine + threat intel (fast, + # free, 100% recall on the benchmark). Only the AMBIGUOUS band — skills + # the rules flagged short of a confident verdict — is escalated to the + # Anthropic LLM for a deep-dive second opinion (which can raise a sneaky + # skill or clear a false positive). Confident-clean and confident- + # malicious skills are never escalated, and --escalate-budget caps the + # per-run LLM spend. The free local HF classifier tier is available for + # local runs (`--escalate-backend hf|tiered`, needs the [hf] extra) but + # is kept out of CI to avoid a heavy torch install on the critical path. # - # ClawHub is rate-limited (~2 req/s), so even so a full ~6k-skill - # registry can't be swept in one CI run. Cap scans per run; the - # overflow is deferred and picked up on subsequent runs, so the - # baseline builds up over a couple of runs and then daily runs only - # touch what changed. + # ClawHub is rate-limited (~2 req/s) and the registry is ~66k skills, so + # a full sweep can't run in one CI window. --max-scans caps scans per + # run; the overflow is deferred and picked up next run, so the baseline + # builds up over successive runs, then daily runs only touch what changed. malwar crawl monitor \ --snapshot-dir data/registry-snapshots \ --format json \ --output data/registry-snapshots/latest-diff.json \ - --no-escalate \ - --max-scans 3000 \ + --escalate-backend anthropic \ + --escalate-budget 200 \ + --max-scans 4000 \ $FULL - name: Publish snapshot to data branch diff --git a/docs/crawl.md b/docs/crawl.md index e171ad7..c15207c 100644 --- a/docs/crawl.md +++ b/docs/crawl.md @@ -129,7 +129,7 @@ malwar crawl monitor --fail-on-malicious # exit non-zero when newly-flagged **Options:** `--snapshot-dir`, `--full`, `--max-scans`, `--max`, `--escalate-backend` (`none`|`hf`|`anthropic`|`tiered`), `--escalate-budget`, `--no-escalate`, `--concurrency`, `--format`/`-f` (`console`|`json`), `--output`/`-o`, `--no-save`, `--digest`, `--publish`, `--fail-on-malicious`. Publishing to X requires the `MALWAR_X_*` credentials (see [Configuration](deployment/configuration.md#x-twitter-publishing)). -The bundled GitHub Actions workflow (`.github/workflows/registry-monitor.yml`) runs this on two cadences: **daily incremental** and a **weekly `--full`** re-scan, committing each snapshot to the `registry-snapshots` branch. It runs **rules-only (`--no-escalate`)** so a whole-registry sweep stays fast and cheap — the rule engine alone catches every malicious sample in the benchmark; use `malwar crawl scan ` for an LLM deep-dive on an individual flagged skill. +The bundled GitHub Actions workflow (`.github/workflows/registry-monitor.yml`) runs this on two cadences — **daily incremental** and a **weekly `--full`** re-scan — committing each snapshot to the `registry-snapshots` branch. The whole-registry sweep is rules + threat-intel; only the **ambiguous band** is escalated to the **Anthropic** LLM (`--escalate-backend anthropic --escalate-budget 200`), so LLM spend tracks the small uncertain middle, not the whole registry. The free local HF tier (`--escalate-backend hf|tiered`) is kept out of CI to avoid a heavy `torch` install on the critical path — use it for local runs. --- diff --git a/src/malwar/monitor/escalation.py b/src/malwar/monitor/escalation.py index 6aadd38..b5c6d30 100644 --- a/src/malwar/monitor/escalation.py +++ b/src/malwar/monitor/escalation.py @@ -142,7 +142,10 @@ class AnthropicBackend: async def assess(self, content: str, *, file_name: str) -> EscalationResult: from malwar.sdk import scan - result = await scan(content, file_name=file_name, use_llm=True, use_urls=True) + # LLM semantic analysis + rules + threat intel. Live URL crawling is + # left off: it adds latency and external-fetch flakiness at escalation + # scale, and the deep dive's value here is the LLM's read of intent. + result = await scan(content, file_name=file_name, use_llm=True, use_urls=False) return EscalationResult( backend=self.name, flagged=result.verdict != "CLEAN", diff --git a/tests/unit/test_escalation.py b/tests/unit/test_escalation.py index 05f4db8..f597a8d 100644 --- a/tests/unit/test_escalation.py +++ b/tests/unit/test_escalation.py @@ -197,7 +197,7 @@ class _Result: risk_score = 55 async def fake_scan(content, *, file_name, use_llm, use_urls): - assert use_llm and use_urls + assert use_llm and not use_urls # LLM on, live URL crawl off return _Result() monkeypatch.setattr("malwar.sdk.scan", fake_scan)