From da71e1851ada92aba49c7f6ba69c14020188374e Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Mon, 6 Jul 2026 02:41:02 +0000
Subject: [PATCH] fix(crawl): retry transient connection timeouts during
 registry sweeps
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Three registry-monitor runs died on httpx ConnectTimeout — the ClawHub
client created a fresh AsyncClient per request with no retries, so a single
transient connection blip (during enumeration paging or a per-skill fetch)
propagated and aborted the whole crawl.

- ClawHubClient now uses AsyncHTTPTransport(retries=3), which retries
  connection-level failures (ConnectError/ConnectTimeout) with backoff.
  Retry count is configurable.
- _enumerate_skills now degrades gracefully on any residual error (after
  retries) instead of only catching ClawHubError, so a hard failure stops
  paging rather than crashing the run.

Tests: client retry configuration + enumeration surviving a raw timeout.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DNoTXU8k3pfSBzR7aJubqL
---
 src/malwar/crawl/client.py      |  9 +++++++++
 src/malwar/monitor/snapshot.py  |  6 +++---
 tests/unit/crawl/test_client.py | 15 +++++++++++++++
 tests/unit/test_monitor.py      | 12 ++++++++++++
 4 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/src/malwar/crawl/client.py b/src/malwar/crawl/client.py
index 0cef544..35330e8 100644
--- a/src/malwar/crawl/client.py
+++ b/src/malwar/crawl/client.py
@@ -18,6 +18,10 @@
 
 BASE_URL = "https://clawhub.ai/api/v1"
 _TIMEOUT = 15.0
+# Transient connection failures (ConnectTimeout/ConnectError) are common when
+# sweeping thousands of skills from CI; retry them at the transport level so a
+# single blip does not abort a long registry crawl.
+_RETRIES = 3
 _USER_AGENT = f"malwar/{__version__}"
 
 
@@ -76,13 +80,18 @@ def __init__(
         self,
         base_url: str = BASE_URL,
         timeout: float = _TIMEOUT,
+        retries: int = _RETRIES,
     ) -> None:
         self.base_url = base_url.rstrip("/")
         self.timeout = timeout
+        self.retries = retries
 
     def _client(self) -> httpx.AsyncClient:
+        # AsyncHTTPTransport retries connection-level failures — ConnectError
+        # and ConnectTimeout (a subclass) — with exponential backoff.
         return httpx.AsyncClient(
             timeout=httpx.Timeout(self.timeout),
+            transport=httpx.AsyncHTTPTransport(retries=self.retries),
             headers={"User-Agent": _USER_AGENT},
         )
 
diff --git a/src/malwar/monitor/snapshot.py b/src/malwar/monitor/snapshot.py
index 8cdd851..f87e156 100644
--- a/src/malwar/monitor/snapshot.py
+++ b/src/malwar/monitor/snapshot.py
@@ -26,7 +26,7 @@
 from collections.abc import Callable
 from pathlib import Path
 
-from malwar.crawl.client import ClawHubClient, ClawHubError
+from malwar.crawl.client import ClawHubClient
 from malwar.monitor.models import RegistrySnapshot, SkillRecord
 from malwar.sdk import scan
 
@@ -65,7 +65,7 @@ async def _enumerate_skills(
     while True:
         try:
             items, cursor = await client.list_skills(limit=page_size, cursor=cursor)
-        except ClawHubError as exc:
+        except Exception as exc:  # ClawHubError, httpx timeouts after retries, etc.
             logger.warning("list_skills failed: %s", exc)
             break
         for item in items:
@@ -85,7 +85,7 @@ async def _enumerate_skills(
     for term in _SEED_TERMS:
         try:
             results = await client.search(term, limit=page_size)
-        except ClawHubError as exc:
+        except Exception as exc:  # ClawHubError, httpx timeouts after retries, etc.
             logger.warning("search %r failed: %s", term, exc)
             continue
         for r in results:
diff --git a/tests/unit/crawl/test_client.py b/tests/unit/crawl/test_client.py
index e82c732..f8dbdc7 100644
--- a/tests/unit/crawl/test_client.py
+++ b/tests/unit/crawl/test_client.py
@@ -266,3 +266,18 @@ async def test_http_error(self):
         with patch("malwar.crawl.client.httpx.AsyncClient", return_value=mc), \
              pytest.raises(ClawHubError):
             await fetch_url("https://example.com/missing.md")
+
+
+class TestConnectionResilience:
+    """The client retries transient connection failures at the transport level."""
+
+    def test_defaults_to_retrying_transport(self):
+        client = ClawHubClient()
+        assert client.retries == 3
+        # _client() wires the retrying transport without error.
+        real = client._client()
+        assert isinstance(real, httpx.AsyncClient)
+
+    def test_retries_configurable(self):
+        assert ClawHubClient(retries=0).retries == 0
+        assert ClawHubClient(retries=5).retries == 5
diff --git a/tests/unit/test_monitor.py b/tests/unit/test_monitor.py
index 574c400..ee282e4 100644
--- a/tests/unit/test_monitor.py
+++ b/tests/unit/test_monitor.py
@@ -142,6 +142,18 @@ async def _boom(slug, path="SKILL.md", version=None):
         assert snap.skills["good"].verdict == "CLEAN"
         assert snap.skills["bad"].error is not None
 
+    async def test_enumeration_timeout_does_not_crash(self):
+        # A raw timeout while listing the registry must not abort the run.
+        client = FakeClawHubClient({"a": BENIGN_BODY})
+
+        async def _boom(limit=20, cursor=None):
+            raise TimeoutError("connect timeout")
+
+        client.list_skills = _boom  # type: ignore[assignment]
+        # search fallback returns [] in the fake → empty snapshot, no exception.
+        snap = await build_snapshot(client, escalate=False)
+        assert snap.skill_count == 0
+
 
 class TestIncrementalSweep:
     async def test_unchanged_skills_are_not_refetched(self):