From c6e1a4ddad0634d4d6442ffa02c0e5683d2fd95e Mon Sep 17 00:00:00 2001 From: Todd Hoffman Date: Thu, 9 Jul 2026 16:04:54 -0700 Subject: [PATCH] fix: surface HTTP errors in fetch_url_to_workspace fetch_url_to_workspace never checked the HTTP status, so a 403/404 block page was saved to the workspace as if it were real content and the tool returned "Fetched URL successfully". Downstream, an analyzer agent reads the junk, reports the file empty/invalid, and the searcher retries with new queries and URLs until it exhausts its web_search quota and loop-detection aborts it. A first-try-correct answer thus degrades into a delegation loop. Two changes: - Call resp.raise_for_status() and return a clear, actionable error ("HTTP N, page NOT saved, do not retry, use a different source") instead of writing the error page as content. - Use a descriptive User-Agent derived from APP_NAME. The spoofed 2021 Chrome UA was itself triggering bot-blocking; sites like Wikipedia return 200 with a descriptive UA where the old one got 403. Verified in a generated agent: Wikipedia now saves 160KB+ of real markdown (previously an empty 403 page); a genuinely blocked source (Britannica) returns the error message and writes no file. Found while debugging an agent generated by this skill (kyuz0/deep-research-agent); this upstreams the same fix. Co-Authored-By: Claude Fable 5 --- .../examples/basic-tui-agent/src/tools/web.py | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/skills/local-agent-builder/examples/basic-tui-agent/src/tools/web.py b/skills/local-agent-builder/examples/basic-tui-agent/src/tools/web.py index b85396a..70bae96 100644 --- a/skills/local-agent-builder/examples/basic-tui-agent/src/tools/web.py +++ b/skills/local-agent-builder/examples/basic-tui-agent/src/tools/web.py @@ -29,8 +29,18 @@ def get_ddgs_client(): async def fetch_url_to_workspace(url: str, filename: str, convert_to_md: bool = True) -> str: """Fetch external web content and save it directly to the workspace. If convert_to_md is True, parses to Markdown.""" def _fetch(): - headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"} + import config as app_config + # A descriptive User-Agent gets 200s from sites (e.g. Wikipedia) that + # block spoofed browser UAs as bot traffic. + headers = { + "User-Agent": f"{app_config.APP_NAME}/0.1 (research agent) httpx", + "Accept": "text/html,application/xhtml+xml,application/pdf,*/*", + } resp = httpx.get(url, headers=headers, timeout=30, follow_redirects=True) + # Surface HTTP failures instead of saving the error page as "content". + # Without this, a 403/404 block page is written to the workspace and + # reported as success, so the agent re-fetches/re-searches in a loop. + resp.raise_for_status() if not convert_to_md: return resp.content # Raw bytes @@ -126,9 +136,15 @@ def _fetch(): else: _IN_MEMORY_FS[path] = chunk return f"Fetched URL successfully to '{filename}' in memory." + except httpx.HTTPStatusError as e: + # Clear, actionable message: this URL is unusable — try a different + # source rather than re-fetching or re-searching the same thing. + return (f"Error: Could not fetch '{url}' — the server returned HTTP " + f"{e.response.status_code}. The page was NOT saved. This URL is " + f"likely blocked or unavailable; do not retry it. Use a different source.") except Exception as e: - import traceback - return f"Failed: {e}\n\nTraceback:\n{traceback.format_exc()}" + return (f"Error: Could not fetch '{url}' ({type(e).__name__}: {e}). " + f"The page was NOT saved. Do not retry this URL; use a different source.") @tool async def web_search(