From 05a4e275872f25567228d0fe617c86351df806a2 Mon Sep 17 00:00:00 2001 From: doris2026 <314609042+Doris2026-0@users.noreply.github.com> Date: Tue, 1 Sep 2026 15:03:41 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat(http=5Fget):=20=E4=BD=93=E9=AA=8C?= =?UTF-8?q?=E4=B8=8E=E5=81=A5=E5=A3=AE=E6=80=A7=E5=85=A8=E9=9D=A2=E5=8D=87?= =?UTF-8?q?=E7=BA=A7=20=E2=80=94=20=E7=BC=96=E7=A0=81=E5=97=85=E6=8E=A2/gz?= =?UTF-8?q?ip=E8=A7=A3=E5=8E=8B/=E9=87=8D=E8=AF=95=E9=80=80=E9=81=BF/?= =?UTF-8?q?=E4=BA=8C=E8=BF=9B=E5=88=B6=E5=88=86=E6=B5=81/=E7=BF=BB?= =?UTF-8?q?=E9=A1=B5=E7=BC=93=E5=AD=98=E5=89=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 编码嗅探: Content-Type charset → charset_normalizer/chardet → utf-8，修复 GBK 站点静默乱码 - gzip/deflate 自动解压 + Accept-Encoding 协商，解压后再截断防压缩炸弹 - GET 对 5xx/连接错误指数退避重试(默认2次)，4xx 与 SSRF 拦截不重试，POST 非幂等不重试 - 二进制 Content-Type (PDF/图片/音视频等) 明确报错并 hint 改用 http_download - 返回新增 final_url/content_type；错误按状态码附排查 hint，区分 DNS/超时/拒连，透传 retries - 修复: 翻页缓存前置（原缓存在下载后检查，翻页整页重下且对端故障时缓存失效） - 修复: markdownify strip 只去标签不去文本导致内联 script/style 源码泄漏进正文 - 修复: _read_limited 恰好 5MB 时截断误报（多读 1 字节确认） - SPA 检测: 转换后近乎无文本且原始 HTML 含大量脚本时附 JS 渲染提示 - get/post timeout clamp 1~600s；新增 30 条测试；版本 2.6.5 --- CHANGELOG.md | 15 ++ metadata.yaml | 2 +- pyproject.toml | 2 +- tests/test_http_get.py | 514 ++++++++++++++++++++++++++++++++++++++++- tools/_registry.py | 2 +- tools/http_get.py | 315 +++++++++++++++++++++---- 6 files changed, 804 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61bcdf5..20f7707 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## v2.6.5 — http_get 健壮性：编码嗅探 / 指数退避重试 / 二进制分流 / final_url + +- **http_get 编码嗅探**: 响应体解码从硬编码 UTF-8 改为三级嗅探（Content-Type charset → charset_normalizer → chardet，均为可选降级，兜底 UTF-8），修复 GBK/GB18030 中文站点静默乱码（`errors="replace"` 产生的 `` 属于"成功但内容全毁"的静默错误）。 +- **http_get 指数退避重试**: 新增 `_open_with_retry`，对 5xx 与连接错误（超时/断连）自动重试，默认 2 次、0.5s 起指数退避；4xx 与 SSRF 重定向拦截（安全决策）立即失败不重试。POST 非幂等，明确不重试以避免重复提交。 +- **http_get 二进制分流**: 响应 Content-Type 为 PDF/图片/音视频/压缩包等二进制类型时，不再把字节流当文本 decode 成乱码返回，改为明确报错并附 `hint` 引导改用 http_download。 +- **http_get 可观测性**: 返回新增 `final_url`（重定向后的落地地址，短链展开/跳转调试可用）与 `content_type` 字段，分页缓存条目同步携带。 +- **http_get 截断误报修复**: `_read_limited` 原 `total >= max_bytes` 在恰好等于上限时误报截断，改为多读 1 字节确认。 +- **http_get 翻页缓存前置（正确性）**: 原实现缓存检查在 HTTP 下载之后，翻页时整页重下再丢弃（缓存只省转换不省下载），且翻页瞬间对端宕机/限流时即使缓存完好也直接报错；现翻页请求先查缓存，命中零网络请求直接切片返回。 +- **http_get gzip/deflate 解压**: 自动广告 `Accept-Encoding: gzip, deflate` 并解压响应（服务器强制压缩时原会 decode 出乱码）；解压后再过一次 5MB 截断防压缩炸弹。 +- **http_get 脚本内容剔除**: markdownify 的 `strip` 只去标签不去文本，内联 `" +) + + +class TestCompression: + def test_gzip_body_decompressed(self, monkeypatch): + """Content-Encoding: gzip 的响应应自动解压。""" + import gzip as gz + from tools import http_get as hg + + payload = gz.compress(SAMPLE_HTML.encode("utf-8")) + + def fake_open(self, req, timeout=None): + return FakeResponse(payload, headers={ + "Content-Type": "text/html; charset=utf-8", + "Content-Encoding": "gzip", + }) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", format="html") + assert r["ok"] is True + assert "

Hello World

" in r["body"] + + def test_deflate_body_decompressed(self, monkeypatch): + """Content-Encoding: deflate 的响应应自动解压。""" + import zlib + from tools import http_get as hg + + payload = zlib.compress(SAMPLE_HTML.encode("utf-8")) + + def fake_open(self, req, timeout=None): + return FakeResponse(payload, headers={ + "Content-Type": "text/html; charset=utf-8", + "Content-Encoding": "deflate", + }) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", format="html") + assert r["ok"] is True + assert "

Hello World

" in r["body"] + + def test_gzip_requested_via_accept_encoding(self, monkeypatch): + """未自定义 headers 时应自动广告 Accept-Encoding: gzip。""" + from tools import http_get as hg + captured = {} + + def fake_open(self, req, timeout=None): + captured["ae"] = req.headers.get("Accept-encoding") # urllib 首字母大写 + return FakeResponse(SAMPLE_HTML.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + get("http://example.com", format="html") + assert captured["ae"] is not None + assert "gzip" in captured["ae"] + + +class TestErrorHints: + def test_404_hint(self, monkeypatch): + from tools import http_get as hg + + def fake_open(self, req, timeout=None): + raise urllib.error.HTTPError( + "http://example.com", 404, "Not Found", None, None + ) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com/missing") + assert r["ok"] is False + assert r["status"] == 404 + assert "不存在" in r["hint"] + assert "retries" not in r # 4xx 不重试 + + def test_403_hint_mentions_antibot(self, monkeypatch): + from tools import http_get as hg + + def fake_open(self, req, timeout=None): + raise urllib.error.HTTPError( + "http://example.com", 403, "Forbidden", None, None + ) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com") + assert r["ok"] is False + assert "反爬" in r["hint"] + + def test_5xx_exhausted_reports_retries(self, monkeypatch): + from tools import http_get as hg + monkeypatch.setattr(hg, "_RETRY_BACKOFF", 0) + + def fake_open(self, req, timeout=None): + raise urllib.error.HTTPError( + "http://example.com", 503, "Service Unavailable", None, None + ) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com") + assert r["ok"] is False + assert r["retries"] == hg._MAX_RETRIES + assert "已自动重试" in r["error"] + assert "服务端错误" in r["hint"] + + def test_timeout_hint(self, monkeypatch): + from tools import http_get as hg + monkeypatch.setattr(hg, "_RETRY_BACKOFF", 0) + + def fake_open(self, req, timeout=None): + raise urllib.error.URLError(TimeoutError("timed out")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", timeout=7) + assert r["ok"] is False + assert "timeout=7" in r["hint"] + assert r["retries"] == hg._MAX_RETRIES + + def test_dns_hint(self, monkeypatch): + from tools import http_get as hg + monkeypatch.setattr(hg, "_RETRY_BACKOFF", 0) + + def fake_open(self, req, timeout=None): + raise urllib.error.URLError( + socket.gaierror(-2, "Name or service not known") + ) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com") + assert r["ok"] is False + assert "域名解析失败" in r["hint"] + + def test_ssrf_redirect_message_not_disguised(self, monkeypatch): + """SSRF 拦截不应伪装成'连接失败'。""" + from tools import http_get as hg + + def fake_open(self, req, timeout=None): + raise urllib.error.URLError( + "重定向目标被拦截: 禁止访问内网地址: 127.0.0.1" + ) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com") + assert r["ok"] is False + assert r["error"].startswith("重定向目标被拦截") + assert "连接失败" not in r["error"] + + +class TestJsRenderedHint: + def test_script_content_not_leaked(self, monkeypatch): + """内联脚本源码不应泄漏进转换结果（markdownify strip 只去标签）。""" + _clear_cache() + from tools import http_get as hg + html = ( + "

真正的正文内容在这里。

" + "" + ) + + def fake_open(self, req, timeout=None): + return FakeResponse(html.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", format="markdown") + assert r["ok"] is True + assert "真正的正文内容" in r["content"] + assert "tracker" not in r["content"] + + def test_spa_page_gets_hint(self, monkeypatch): + """大量脚本 + 几乎无文本的页面应提示疑似 JS 渲染。""" + from tools import http_get as hg + + def fake_open(self, req, timeout=None): + return FakeResponse(SPA_HTML.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", format="markdown") + assert r["ok"] is True + assert "hint" in r + assert "JS" in r["hint"] + + def test_normal_page_no_hint(self, monkeypatch): + """正常文本页面不应误报。""" + _clear_cache() + from tools import http_get as hg + + def fake_open(self, req, timeout=None): + return FakeResponse(SAMPLE_HTML.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", format="markdown") + assert r["ok"] is True + assert "hint" not in r + + +class TestTimeoutClamp: + def test_timeout_clamped_to_600(self, monkeypatch): + from tools import http_get as hg + captured = {} + + def fake_open(self, req, timeout=None): + captured["timeout"] = timeout + return FakeResponse(SAMPLE_HTML.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + get("http://example.com", timeout=9999) + assert captured["timeout"] == 600 + + def test_timeout_floor_is_1(self, monkeypatch): + from tools import http_get as hg + captured = {} + + def fake_open(self, req, timeout=None): + captured["timeout"] = timeout + return FakeResponse(SAMPLE_HTML.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + get("http://example.com", timeout=-3) + assert captured["timeout"] == 1 + + def test_timeout_invalid_uses_default(self, monkeypatch): + from tools import http_get as hg + captured = {} + + def fake_open(self, req, timeout=None): + captured["timeout"] = timeout + return FakeResponse(SAMPLE_HTML.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + get("http://example.com", timeout="abc") + assert captured["timeout"] == 15 diff --git a/tools/_registry.py b/tools/_registry.py index 4dab243..ad8c5fc 100644 --- a/tools/_registry.py +++ b/tools/_registry.py @@ -605,7 +605,7 @@ def _port_check_w(host: str = "127.0.0.1", ports: list | None = None) -> dict: HttpGetTool = make_tool( "http_get", - "【HTTP GET 唯一选择】HTTP GET 请求 + 智能内容转换。不要用 astrbot_execute_shell 跑 curl——它无 SSRF（内网 IP）防护。默认 format='markdown' 转为 LLM 友好 Markdown 并支持 offset 分页翻页；format='text' 提取纯文本正文；format='html' 返回原始 HTML（5000 字符截断，无分页，仅调试用）。extract=true 先提取正文再转换（trafilatura 去广告/导航/页脚），默认 false 全页转换。15s 超时。", + "【HTTP GET 唯一选择】HTTP GET 请求 + 智能内容转换。不要用 astrbot_execute_shell 跑 curl——它无 SSRF（内网 IP）防护。默认 format='markdown' 转为 LLM 友好 Markdown 并支持 offset 分页翻页；format='text' 提取纯文本正文；format='html' 返回原始 HTML（5000 字符截断，无分页，仅调试用）。extract=true 先提取正文再转换（trafilatura 去广告/导航/页脚），默认 false 全页转换。15s 超时。自动编码嗅探（GBK 等老站不乱码）、gzip/deflate 自动解压；5xx/连接错误自动指数退避重试；返回含 final_url（重定向后落地地址）与 content_type；错误附排查 hint（如 403 反爬、超时、DNS）；目标为 PDF/图片等二进制内容时返回错误并提示改用 http_download。", { "type": "object", "properties": { diff --git a/tools/http_get.py b/tools/http_get.py index 7c48b19..fcce542 100644 --- a/tools/http_get.py +++ b/tools/http_get.py @@ -9,10 +9,13 @@ import threading import time +import gzip +import json import re -import urllib.request +import socket import urllib.error -import json +import urllib.request +import zlib from typing import Any from ._http_utils import check_url, make_opener @@ -24,6 +27,9 @@ _PAGE_CACHE_SIZE = 10 # 最多缓存的页面数（LRU 淘汰） _PAGE_CACHE_TTL = 300 # 页面缓存有效期（秒），过期重新请求 _DEFAULT_TIMEOUT = 15 # 默认超时秒数 +_MAX_RETRIES = 2 # GET 失败重试次数（仅 5xx / 连接错误；POST 非幂等不重试） +_RETRY_BACKOFF = 0.5 # 重试退避基数（秒），按 2^attempt 指数增长 + # ── 翻页缓存：key=(url, format, extract) → 完整转换结果 ── _page_cache: OrderedDict = OrderedDict() @@ -59,8 +65,78 @@ def _set_cache(key: tuple, entry: dict) -> None: _page_cache[key] = entry -def _read_limited(resp, max_bytes: int = _MAX_RESPONSE_SIZE) -> str: - """分块读取响应体，超过 max_bytes 时截断。""" +_BINARY_CT_EXACT = frozenset({ + "application/pdf", + "application/octet-stream", + "application/zip", + "application/gzip", + "application/x-tar", + "application/x-bzip2", + "application/x-xz", + "application/x-7z-compressed", + "application/x-rar-compressed", + "application/msword", + "application/wasm", +}) +_BINARY_CT_PREFIXES = ("image/", "audio/", "video/", "font/", "application/vnd.") + + +def _resp_content_type(resp) -> str: + """取响应 Content-Type 主类型（小写、去参数），取不到返回空串。""" + headers = getattr(resp, "headers", None) + if headers is None: + return "" + ct = headers.get("Content-Type", "") or "" + return ct.split(";")[0].strip().lower() + + +def _resp_final_url(resp, url: str) -> str: + """取重定向后的落地 URL，取不到回退为原始 url。""" + geturl = getattr(resp, "geturl", None) + if callable(geturl): + try: + return geturl() or url + except Exception: + pass + return url + + +def _is_binary_content_type(content_type: str) -> bool: + """判断 Content-Type 是否为不可文本化的二进制内容。""" + if not content_type: + return False + if content_type in _BINARY_CT_EXACT: + return True + return any(content_type.startswith(p) for p in _BINARY_CT_PREFIXES) + + +def _detect_charset(resp, raw: bytes) -> str: + """编码嗅探：Content-Type charset → charset_normalizer/chardet（可选依赖）→ utf-8。""" + headers = getattr(resp, "headers", None) + if headers is not None: + ct = headers.get("Content-Type", "") or "" + m = re.search(r'charset=["\']?([\w.\-]+)', ct, re.IGNORECASE) + if m: + return m.group(1) + try: + from charset_normalizer import from_bytes + best = from_bytes(raw).best() + if best is not None: + return best.encoding + except Exception: + pass + try: + import chardet + result = chardet.detect(raw) + if result and result.get("encoding"): + return result["encoding"] + except Exception: + pass + return "utf-8" + + +def _read_limited(resp, max_bytes: int = _MAX_RESPONSE_SIZE) -> tuple: + """分块读取响应体（上限 max_bytes），按嗅探到的编码解码。返回 (body, truncated)。""" chunks = [] total = 0 while True: @@ -71,11 +147,42 @@ def _read_limited(resp, max_bytes: int = _MAX_RESPONSE_SIZE) -> str: chunks.append(chunk) if total >= max_bytes: break - body = b"".join(chunks).decode("utf-8", errors="replace") - return body, total >= max_bytes + # 多读 1 字节确认是否真截断（避免恰好 max_bytes 时误报） + truncated = total >= max_bytes and bool(resp.read(1)) + raw = b"".join(chunks) + + # 解压：Accept-Encoding 协商或服务器强制压缩的结果 + headers = getattr(resp, "headers", None) + content_encoding = "" + if headers is not None: + content_encoding = (headers.get("Content-Encoding", "") or "").strip().lower() + if content_encoding in ("gzip", "x-gzip"): + try: + raw = gzip.decompress(raw) + except Exception: + pass + elif content_encoding == "deflate": + try: + raw = zlib.decompress(raw) + except Exception: + try: + raw = zlib.decompress(raw, -zlib.MAX_WBITS) + except Exception: + pass + # 解压后可能远超下载上限（压缩比），再截断一次防内存膨胀 + if len(raw) > max_bytes: + raw = raw[:max_bytes] + truncated = True + + charset = _detect_charset(resp, raw) + try: + body = raw.decode(charset, errors="replace") + except (LookupError, ValueError): + body = raw.decode("utf-8", errors="replace") + return body, truncated -def _build_response(resp) -> dict: +def _build_response(resp, url: str = "") -> dict: body, was_truncated = _read_limited(resp) return { "ok": True, @@ -83,9 +190,88 @@ def _build_response(resp) -> dict: "size": len(body), "body": body, "truncated": was_truncated, + "final_url": _resp_final_url(resp, url), + "content_type": _resp_content_type(resp), } +def _open_with_retry(req, timeout: int, max_retries: int = _MAX_RETRIES): + """带指数退避的请求执行。仅对 5xx 与连接错误重试；SSRF 拦截与 4xx 立即抛出。 + + 抛出时给异常挂上 _retries 属性（已重试次数），供错误消息透传。 + """ + for attempt in range(max_retries + 1): + try: + return make_opener().open(req, timeout=timeout) + except urllib.error.HTTPError as e: + if e.code < 500 or attempt >= max_retries: + e._retries = attempt + raise + except urllib.error.URLError as e: + reason = str(getattr(e, "reason", "") or e) + if "重定向目标被拦截" in reason or attempt >= max_retries: + e._retries = attempt + raise + time.sleep(_RETRY_BACKOFF * (2 ** attempt)) + + +_STATUS_HINTS = { + 400: "请求格式有误——检查 URL 是否完整", + 401: "需要认证——如需登录态，请在 headers 中携带 Authorization/Cookie", + 403: "访问被拒——可能是反爬/WAF 拦截（本工具无浏览器指纹），可换地址、稍后重试，或寻找官方 API", + 404: "页面不存在——检查 URL 拼写，链接可能已失效", + 405: "方法不允许——该地址可能只接受 POST（http_post）等方法", + 429: "请求过于频繁（限流）——请稍后重试", +} + + +def _http_error_dict(e, retried: int = 0) -> dict: + """HTTP 错误结构化：状态码 + 可操作 hint + 重试次数。""" + body = "" + try: + body = e.read().decode("utf-8", errors="replace")[:500] + except Exception: + pass + error = f"HTTP {e.code}: {e.reason}" + if retried: + error += f"（已自动重试 {retried} 次）" + if e.code >= 500: + hint = "服务端错误——自动重试后仍失败，可稍后再试" + else: + hint = _STATUS_HINTS.get(e.code, "") + result = {"ok": False, "error": error, "status": e.code, "body": body} + if retried: + result["retries"] = retried + if hint: + result["hint"] = hint + return result + + +def _url_error_dict(e, timeout: int, retried: int = 0) -> dict: + """连接层错误结构化：区分 DNS/超时/拒连，附可操作 hint。""" + reason = getattr(e, "reason", e) + reason_s = str(reason) + if "重定向目标被拦截" in reason_s: + # SSRF 安全拦截：原样透出，不伪装成网络故障 + return {"ok": False, "error": reason_s} + error = f"连接失败: {reason_s}" + if retried: + error += f"（已自动重试 {retried} 次）" + hint = "" + if isinstance(reason, socket.gaierror) or "Name or service not known" in reason_s or "Temporary failure in name resolution" in reason_s: + hint = "域名解析失败——检查 URL 拼写，或本机 DNS/网络是否正常" + elif isinstance(reason, (TimeoutError, socket.timeout)) or "timed out" in reason_s.lower(): + hint = f"连接超时（当前 timeout={timeout}s）——可调大 timeout 后重试" + elif "refused" in reason_s.lower(): + hint = "连接被拒——目标服务可能未启动或端口不通" + result = {"ok": False, "error": error} + if retried: + result["retries"] = retried + if hint: + result["hint"] = hint + return result + + def _extract_metadata(html: str) -> dict: """从 HTML 中提取 title 和 meta description。""" title = "" @@ -121,6 +307,9 @@ def _markdown_convert(h): """markdownify 转换 + format=text 时去标记，失败返回 None。""" try: from markdownify import markdownify as md + # 先剔除 script/style 整块（含内容）——markdownify 的 strip 只去标签不去文本 + h = re.sub(r"", "", h, flags=re.IGNORECASE) + h = re.sub(r"", "", h, flags=re.IGNORECASE) text = md(h, heading_style="ATX", strip=["script", "style"]) if format == "text": text = re.sub(r"^#{1,6}\s+", "", text, flags=re.MULTILINE) @@ -195,10 +384,12 @@ def _paginate(entry: dict, offset: int, url: str, format: str, extract: bool) -> "换个更精确的 URL 或减小范围", ] - return { + result = { "ok": True, "status": entry["status"], "url": url, + "final_url": entry.get("final_url", url), + "content_type": entry.get("content_type", ""), "title": entry["title"], "description": entry["description"], "content": page, @@ -214,11 +405,17 @@ def _paginate(entry: dict, offset: int, url: str, format: str, extract: bool) -> "truncation_reason": f"共 {total} 字符，当前展示 offset={offset}~{offset + len(page)}" if has_more else "", "size": entry["size"], } + if entry.get("hint"): + result["hint"] = entry["hint"] + return result def _add_ua(req, headers: dict | None): if not headers or "User-Agent" not in headers: req.add_header("User-Agent", "IrmiaDevKit/2.3") + if not headers or "Accept-Encoding" not in headers: + # 只广告内置可解压的编码；服务器强制压缩时也能正确处理 + req.add_header("Accept-Encoding", "gzip, deflate, identity") def get( @@ -240,10 +437,15 @@ def get( offset: 分页偏移量（字符数），0=从头读取。首次调用不传，后续通过 next_call 透传 Returns: - format="html" 时返回 {"ok", "status", "body", "truncated", "size", "converter"} - format≠"html" 时返回 {"ok", "status", "url", "title", "description", - "content", "content_length", "format", "extracted", "converter", - "offset", "has_more", "next_call", "options", "truncated", "size"} + format="html" 时返回 {"ok", "status", "body", "truncated", "size", "converter", + "final_url", "content_type"} + format≠"html" 时返回 {"ok", "status", "url", "final_url", "content_type", + "title", "description", "content", "content_length", "format", + "extracted", "converter", "offset", "has_more", "next_call", + "options", "truncated", "size"} + 目标为 PDF/图片等二进制内容时返回 {"ok": False, "error", "status", + "content_type", "final_url", "hint"}，提示改用 http_download + 5xx/连接错误自动指数退避重试（最多 _MAX_RETRIES 次） """ err = check_url(url) if err: @@ -257,11 +459,39 @@ def get( if offset < 0: return {"ok": False, "error": f"offset 不能为负数: {offset}"} + try: + timeout = max(1, min(int(timeout), 600)) + except (TypeError, ValueError): + timeout = _DEFAULT_TIMEOUT + + # markdown/text 翻页：先查翻页缓存，命中直接切片返回——不重复下载， + # 对端故障/限流时缓存内容依然可用 + key = None + if format in ("markdown", "text"): + key = _cache_key(url, format, extract) + if offset > 0: + cached = _get_cached(key) + if cached is not None and offset < len(cached["content"]): + return _paginate(cached, offset, url, format, extract) + # 缓存未命中、已过期或 offset 越界：回退为首次请求 + offset = 0 + try: req = urllib.request.Request(url, headers=headers or {}) _add_ua(req, headers) - with make_opener().open(req, timeout=timeout) as resp: - raw = _build_response(resp) + with _open_with_retry(req, timeout) as resp: + content_type = _resp_content_type(resp) + if _is_binary_content_type(content_type): + return { + "ok": False, + "error": f"目标是二进制内容 ({content_type})，无法作为文本读取", + "status": resp.status, + "content_type": content_type, + "final_url": _resp_final_url(resp, url), + "hint": "如需保存该文件，请改用 http_download 工具", + } + + raw = _build_response(resp, url) if not raw["ok"]: return raw @@ -275,39 +505,37 @@ def get( raw["hint"] = "HTML 已截断且无分页；改用 format='markdown' 或 'text' 可分页读取全文" return raw - # format ∈ {"markdown", "text"} - key = _cache_key(url, format, extract) - - # offset > 0：尝试从缓存翻页 - if offset > 0: - cached = _get_cached(key) - if cached is not None and offset < len(cached["content"]): - return _paginate(cached, offset, url, format, extract) - # 缓存未命中、已过期或 offset 越界：从头下载 - offset = 0 - - # 首次请求：下载 + 转换 + 缓存 + # format ∈ {"markdown", "text"}：首次请求 = 下载 + 转换 + 缓存 meta = _extract_metadata(raw["body"]) content, converter = _convert_html(raw["body"], format, extract) + # SPA 检测：原始 HTML 很大且含脚本，但转换后几乎无文本 → 疑似 JS 渲染 + hint = "" + if ( + len(content) < 50 + and len(raw["body"]) > 2000 + and re.search(r"]", raw["body"], re.IGNORECASE) + ): + hint = ( + "页面几乎无文本内容但包含脚本，疑似 JS 动态渲染（SPA）——" + "本工具不执行 JS；建议寻找该站的 API/RSS/打印版，或确认 URL 是否正确" + ) entry = { "status": raw["status"], "size": raw["size"], + "final_url": raw["final_url"], + "content_type": raw["content_type"], "title": meta["title"], "description": meta["description"], "content": content, "converter": converter, + "hint": hint, } _set_cache(key, entry) return _paginate(entry, offset, url, format, extract) except urllib.error.HTTPError as e: - body = "" - try: - body = e.read().decode("utf-8", errors="replace")[:500] - except Exception: - pass - return {"ok": False, "error": f"HTTP {e.code}: {e.reason}", "status": e.code, "body": body} + return _http_error_dict(e, getattr(e, "_retries", 0)) except urllib.error.URLError as e: - return {"ok": False, "error": f"连接失败: {e.reason}"} + return _url_error_dict(e, timeout, getattr(e, "_retries", 0)) except Exception as e: return {"ok": False, "error": str(e)} @@ -315,7 +543,10 @@ def get( def post( url: str, data: Any = None, headers: dict | None = None, timeout: int = 10 ) -> dict: - """HTTP POST 请求。data 可以是 dict（自动 JSON）或 str。""" + """HTTP POST 请求。data 可以是 dict（自动 JSON）或 str。 + + 注意：POST 非幂等，不做自动重试，避免重复提交。 + """ err = check_url(url) if err: return err @@ -323,6 +554,11 @@ def post( if data is None: return {"ok": False, "error": "POST 请求必须提供 data 参数"} + try: + timeout = max(1, min(int(timeout), 600)) + except (TypeError, ValueError): + timeout = 10 + try: if isinstance(data, dict): data = json.dumps(data, ensure_ascii=False).encode("utf-8") @@ -336,15 +572,10 @@ def post( req = urllib.request.Request(url, data=data, headers=headers or {}) _add_ua(req, headers) with make_opener().open(req, timeout=timeout) as resp: - return _build_response(resp) + return _build_response(resp, url) except urllib.error.HTTPError as e: - body = "" - try: - body = e.read().decode("utf-8", errors="replace")[:500] - except Exception: - pass - return {"ok": False, "error": f"HTTP {e.code}: {e.reason}", "status": e.code, "body": body} + return _http_error_dict(e) except urllib.error.URLError as e: - return {"ok": False, "error": f"连接失败: {e.reason}"} + return _url_error_dict(e, timeout) except Exception as e: return {"ok": False, "error": str(e)} From 610243e44b4fa9b3a470dab730a17cb07bb18fc2 Mon Sep 17 00:00:00 2001 From: doris2026 <314609042+Doris2026-0@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:26:48 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix(http=5Fget):=20=E7=8B=AC=E7=AB=8B=20rev?= =?UTF-8?q?iew=20=E4=BF=AE=E5=A4=8D=20=E2=80=94=20=E5=8E=8B=E7=BC=A9?= =?UTF-8?q?=E7=82=B8=E5=BC=B9/=E8=B7=A8=E5=87=AD=E6=8D=AE=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E6=B3=84=E9=9C=B2/ReDoS/=E9=87=8D=E8=AF=95=E7=9B=B2?= =?UTF-8?q?=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 安全: - 解压改 zlib.decompressobj 流式限长(原全量解压后才截断,1MB gzip 实测撑 2GB;现 1GB 炸弹 RSS 增量 ~0) - 翻页缓存 key 纳入 headers 规范化指纹,修复跨凭据缓存命中泄露机密内容;next_call 透传 headers - script/style 剔除改线性扫描(原正则在大量无闭合标签时 O(n·m) DoS) - SSRF 拦截改专用异常 SsrfBlocked(原字符串匹配耦合文案) 健壮性: - RemoteDisconnected/ConnectionReset 等非 URLError 连接错误纳入重试 - offset 非整型返回错误(原 TypeError 泄漏);截断压缩流返回部分内容并标记(原静默返回压缩垃圾) - br/zstd 等不支持编码明确报错;HTTP 错误体读取限长 4KB;重试总预算 cap 180s;重试前 close 失败响应 - 429 hint 与不重试行为对齐;UA/Accept-Encoding 大小写不敏感;post 补二进制分流;编码嗅探限前 64KB 测试: - 分页衔接断言假阳性修复(原 startswith("") 恒真,改为与缓存全文比对) - FakeResponse headers 大小写不敏感;GBK skip 守卫修正;新增 20 条回归(共 82 passed) --- CHANGELOG.md | 3 + tests/test_http_get.py | 275 +++++++++++++++++++++++++++++++++++++-- tests/test_http_utils.py | 15 ++- tools/_http_utils.py | 7 +- tools/http_get.py | 233 +++++++++++++++++++++++++-------- 5 files changed, 467 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20f7707..96d9b4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ - **http_get SPA 检测**: 转换后几乎无文本但原始 HTML 含大量脚本时，返回附 `hint` 提示疑似 JS 动态渲染（本工具不执行 JS）并给出替代建议，不再让 LLM 面对"成功但空白"误判。 - **http_get/http_post 错误可操作化**: HTTP 错误按状态码附排查 hint（401 认证 / 403 反爬 WAF / 404 失效 / 405 方法 / 429 限流 / 5xx 服务端），连接层错误区分 DNS 解析失败/超时（含当前 timeout 值）/拒连；自动重试后仍失败时透传 `retries` 次数；SSRF 拦截不再被"连接失败"前缀伪装成网络故障。 - **http_get/http_post timeout clamp**: 直接调用也钳制到 1~600s（原仅注册表层对 post 生效），非法值回退默认。 +- **http_get review 修复（安全）**: ① 解压改 `zlib.decompressobj` 流式限长，压缩炸弹内存有界（原全量解压后才截断，1MB gzip 实测可撑 2GB）；② 翻页缓存 key 纳入 headers 规范化指纹，修复跨凭据缓存命中导致的机密内容泄露，`next_call` 同步透传 headers；③ script/style 剔除从正则改为线性扫描，修复大量无闭合标签构造的 O(n·m) DoS；④ SSRF 拦截改用专用异常 `SsrfBlocked`（原靠消息文案字符串匹配，脆弱耦合）。 +- **http_get review 修复（健壮性）**: `RemoteDisconnected`/`ConnectionResetError` 等非 URLError 连接错误纳入重试；offset 非整型返回错误而非 TypeError 泄漏；下载截断的压缩流返回已解压部分内容并标记（原静默返回压缩垃圾）；br/zstd 等不支持编码明确报错；HTTP 错误体读取限长 4KB；重试加总耗时预算（cap 180s，防大 timeout×重试放大）；重试前 close 失败响应；429 hint 与不重试行为对齐；自定义 UA/Accept-Encoding 大小写不敏感匹配；http_post 补二进制分流；编码嗅探只取前 64KB。 +- **测试修复**: 分页衔接断言原切片恒空（`startswith("")` 恒真）的假阳性改为与缓存全文比对；FakeResponse headers 改大小写不敏感（对齐真实 HTTPMessage）；GBK 嗅探 skip 守卫修正为任一嗅探库。 - **db_query**: params schema 补 `items`（数组类型缺 items 导致 Gemini function calling 400 INVALID_ARGUMENT，43004d6）。 ## v2.6.4 — 全量 code review 修复：响应协议 / 备份生命周期 / 行号寻址编辑 / CI 日志 diff --git a/tests/test_http_get.py b/tests/test_http_get.py index b3ecb0e..35ac1e2 100644 --- a/tests/test_http_get.py +++ b/tests/test_http_get.py @@ -36,6 +36,17 @@ def _clear_cache(): _page_cache.clear() +class _CIDict(dict): + """模拟 email.message.Message 的大小写不敏感 get（真实响应头行为）。""" + + def get(self, key, default=None): + key = key.lower() + for k, v in self.items(): + if k.lower() == key: + return v + return default + + class FakeResponse: """模拟 urllib response 对象。""" @@ -43,7 +54,7 @@ def __init__(self, body: bytes, status: int = 200, headers: dict | None = None, url: str = ""): self._body = BytesIO(body) self.status = status - self.headers = headers if headers is not None else {} + self.headers = _CIDict(headers or {}) self._url = url def read(self, n=-1): @@ -239,7 +250,11 @@ def fake_open(self, req, timeout=None): assert r2["ok"] is True page2 = r2["content"] assert page1 != page2 # 内容不应相同 - assert page2.startswith(page1[next_offset:next_offset + 20]) # 衔接正确 + # 与缓存中的完整转换结果比对验证衔接（原断言切片恒为空串，是假阳性） + key = hg._cache_key("http://example.com", "markdown", False, None) + full = hg._page_cache[key]["content"] + assert page1 == full[:len(page1)] + assert page2 == full[next_offset:next_offset + len(page2)] def test_format_markdown_no_has_more(self, monkeypatch): """短内容不应标记 has_more。""" @@ -343,7 +358,10 @@ def fake_open(self, req, timeout=None): def test_gbk_sniffed_without_charset_header(self, monkeypatch): """无 charset 声明时由 chardet/charset_normalizer 嗅探兜底。""" - pytest.importorskip("chardet") + import importlib.util + if not (importlib.util.find_spec("charset_normalizer") + or importlib.util.find_spec("chardet")): + pytest.skip("需要 charset_normalizer 或 chardet") from tools import http_get as hg def fake_open(self, req, timeout=None): @@ -527,14 +545,13 @@ def fake_open(self, req, timeout=None): def test_no_retry_on_ssrf_redirect_block(self, monkeypatch): """SSRF 重定向拦截是安全决策，绝不能重试。""" from tools import http_get as hg + from tools._http_utils import SsrfBlocked self._no_sleep(monkeypatch, hg) calls = {"n": 0} def fake_open(self, req, timeout=None): calls["n"] += 1 - raise urllib.error.URLError( - "重定向目标被拦截: 禁止访问内网地址: 127.0.0.1" - ) + raise SsrfBlocked("重定向目标被拦截: 禁止访问内网地址: 127.0.0.1") monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) r = get("http://example.com") @@ -721,11 +738,10 @@ def fake_open(self, req, timeout=None): def test_ssrf_redirect_message_not_disguised(self, monkeypatch): """SSRF 拦截不应伪装成'连接失败'。""" from tools import http_get as hg + from tools._http_utils import SsrfBlocked def fake_open(self, req, timeout=None): - raise urllib.error.URLError( - "重定向目标被拦截: 禁止访问内网地址: 127.0.0.1" - ) + raise SsrfBlocked("重定向目标被拦截: 禁止访问内网地址: 127.0.0.1") monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) r = get("http://example.com") @@ -816,3 +832,244 @@ def fake_open(self, req, timeout=None): monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) get("http://example.com", timeout="abc") assert captured["timeout"] == 15 + + +class TestRetryConnectionErrors: + def test_retry_on_remote_disconnected(self, monkeypatch): + """RemoteDisconnected（非 URLError）属瞬时连接错误，应纳入重试。""" + import http.client + from tools import http_get as hg + monkeypatch.setattr(hg, "_RETRY_BACKOFF", 0) + calls = {"n": 0} + + def fake_open(self, req, timeout=None): + calls["n"] += 1 + if calls["n"] <= 2: + raise http.client.RemoteDisconnected("Remote end closed connection") + return FakeResponse(SAMPLE_HTML.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com") + assert r["ok"] is True + assert calls["n"] == 3 + + def test_connection_reset_exhausted_reports_retries(self, monkeypatch): + """持续连接重置：重试耗尽后报错并透传 retries。""" + from tools import http_get as hg + monkeypatch.setattr(hg, "_RETRY_BACKOFF", 0) + + def fake_open(self, req, timeout=None): + raise ConnectionResetError(104, "Connection reset by peer") + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com") + assert r["ok"] is False + assert r["retries"] == hg._MAX_RETRIES + assert "连接中断" in r["error"] + + +class TestOffsetCoercion: + def test_offset_string_coerced(self, monkeypatch): + """LLM 传字符串 offset 应被容错转换，而非抛 TypeError。""" + _clear_cache() + from tools import http_get as hg + + def fake_open(self, req, timeout=None): + return FakeResponse(SAMPLE_HTML_LONG.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r1 = get("http://example.com", format="markdown") + nxt = r1["next_call"]["params"]["offset"] + r2 = get("http://example.com", format="markdown", offset=str(nxt)) + assert r2["ok"] is True + assert r2["offset"] == nxt + + def test_offset_invalid_string_returns_error(self): + r = get("http://example.com", offset="abc") + assert r["ok"] is False + assert "offset" in r["error"] + + +class TestDecompressionSafety: + def test_gzip_bomb_bounded(self, monkeypatch): + """压缩炸弹：10MB 全同字符（gzip ~10KB），解压必须流式限长。""" + import gzip as gz + from tools import http_get as hg + + payload = gz.compress(b"A" * (10 * 1024 * 1024)) + + def fake_open(self, req, timeout=None): + return FakeResponse(payload, headers={ + "Content-Type": "text/html", "Content-Encoding": "gzip"}) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", format="html") + assert r["ok"] is True + assert r["truncated"] is True + assert r["size"] == hg._MAX_RESPONSE_SIZE # 解压输出被限长在 5MB + + def test_truncated_gzip_returns_partial_not_garbage(self, monkeypatch): + """下载截断的 gzip 流：返回已解压部分内容并标记 truncated，而非压缩垃圾。""" + import gzip as gz + import os + from tools import http_get as hg + + payload = gz.compress(os.urandom(6 * 1024 * 1024)) # 不可压缩 → >5MB + + def fake_open(self, req, timeout=None): + return FakeResponse(payload, headers={ + "Content-Type": "text/plain", "Content-Encoding": "gzip"}) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", format="html") + assert r["ok"] is True + assert r["truncated"] is True + assert len(r["body"]) > 0 + + def test_unsupported_content_encoding_br(self, monkeypatch): + """服务器强制 br 编码：明确报错而非返回乱码。""" + from tools import http_get as hg + + def fake_open(self, req, timeout=None): + return FakeResponse(b"whatever", headers={ + "Content-Type": "text/html", "Content-Encoding": "br"}) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com") + assert r["ok"] is False + assert "br" in r["error"] + + +class TestCacheCredentialsIsolation: + AUTH = {"Authorization": "Bearer SECRET-TOKEN"} + + def test_next_call_carries_headers(self, monkeypatch): + """带 headers 的翻页：next_call 必须透传 headers，否则缓存指纹对不上。""" + _clear_cache() + from tools import http_get as hg + + def fake_open(self, req, timeout=None): + return FakeResponse(SAMPLE_HTML_LONG.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r1 = get("http://example.com", format="markdown", headers=self.AUTH) + assert r1["has_more"] is True + assert r1["next_call"]["params"].get("headers") == self.AUTH + + def test_authed_pagination_hits_own_cache(self, monkeypatch): + """同一凭据的翻页命中自己的缓存，不重复下载。""" + _clear_cache() + from tools import http_get as hg + calls = {"n": 0} + + def fake_open(self, req, timeout=None): + calls["n"] += 1 + return FakeResponse(SAMPLE_HTML_LONG.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r1 = get("http://example.com", format="markdown", headers=self.AUTH) + nxt = r1["next_call"]["params"]["offset"] + r2 = get("http://example.com", format="markdown", offset=nxt, headers=self.AUTH) + assert r2["ok"] is True + assert calls["n"] == 1 + + def test_anonymous_cannot_hit_credentialed_cache(self, monkeypatch): + """无凭据方翻页不得命中带凭据缓存（跨凭据泄露回归）。""" + _clear_cache() + from tools import http_get as hg + calls = {"n": 0} + + def fake_open(self, req, timeout=None): + calls["n"] += 1 + return FakeResponse(SAMPLE_HTML_LONG.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r1 = get("http://example.com", format="markdown", headers=self.AUTH) + nxt = r1["next_call"]["params"]["offset"] + get("http://example.com", format="markdown", offset=nxt) # 无凭据 + assert calls["n"] == 2 # 未命中带凭据缓存，重新下载 + + +class TestStripTagBlocks: + def test_closed_block_removed(self): + from tools.http_get import _strip_tag_blocks + out = _strip_tag_blocks("

1

2

", "script") + assert out == "

1

2

" + + def test_unclosed_keeps_content(self): + """无闭合标签：只丢开标签、保留后续内容，不误删正文。""" + from tools.http_get import _strip_tag_blocks + assert _strip_tag_blocks("

1