diff --git a/CHANGELOG.md b/CHANGELOG.md index 61bcdf5..96d9b4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## v2.6.5 — http_get 健壮性:编码嗅探 / 指数退避重试 / 二进制分流 / final_url + +- **http_get 编码嗅探**: 响应体解码从硬编码 UTF-8 改为三级嗅探(Content-Type charset → charset_normalizer → chardet,均为可选降级,兜底 UTF-8),修复 GBK/GB18030 中文站点静默乱码(`errors="replace"` 产生的 `` 属于"成功但内容全毁"的静默错误)。 +- **http_get 指数退避重试**: 新增 `_open_with_retry`,对 5xx 与连接错误(超时/断连)自动重试,默认 2 次、0.5s 起指数退避;4xx 与 SSRF 重定向拦截(安全决策)立即失败不重试。POST 非幂等,明确不重试以避免重复提交。 +- **http_get 二进制分流**: 响应 Content-Type 为 PDF/图片/音视频/压缩包等二进制类型时,不再把字节流当文本 decode 成乱码返回,改为明确报错并附 `hint` 引导改用 http_download。 +- **http_get 可观测性**: 返回新增 `final_url`(重定向后的落地地址,短链展开/跳转调试可用)与 `content_type` 字段,分页缓存条目同步携带。 +- **http_get 截断误报修复**: `_read_limited` 原 `total >= max_bytes` 在恰好等于上限时误报截断,改为多读 1 字节确认。 +- **http_get 翻页缓存前置(正确性)**: 原实现缓存检查在 HTTP 下载之后,翻页时整页重下再丢弃(缓存只省转换不省下载),且翻页瞬间对端宕机/限流时即使缓存完好也直接报错;现翻页请求先查缓存,命中零网络请求直接切片返回。 +- **http_get gzip/deflate 解压**: 自动广告 `Accept-Encoding: gzip, deflate` 并解压响应(服务器强制压缩时原会 decode 出乱码);解压后再过一次 5MB 截断防压缩炸弹。 +- **http_get 脚本内容剔除**: markdownify 的 `strip` 只去标签不去文本,内联 `" +) + + +class TestCompression: + def test_gzip_body_decompressed(self, monkeypatch): + """Content-Encoding: gzip 的响应应自动解压。""" + import gzip as gz + from tools import http_get as hg + + payload = gz.compress(SAMPLE_HTML.encode("utf-8")) + + def fake_open(self, req, timeout=None): + return FakeResponse(payload, headers={ + "Content-Type": "text/html; charset=utf-8", + "Content-Encoding": "gzip", + }) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", format="html") + assert r["ok"] is True + assert "

Hello World

" in r["body"] + + def test_deflate_body_decompressed(self, monkeypatch): + """Content-Encoding: deflate 的响应应自动解压。""" + import zlib + from tools import http_get as hg + + payload = zlib.compress(SAMPLE_HTML.encode("utf-8")) + + def fake_open(self, req, timeout=None): + return FakeResponse(payload, headers={ + "Content-Type": "text/html; charset=utf-8", + "Content-Encoding": "deflate", + }) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", format="html") + assert r["ok"] is True + assert "

Hello World

" in r["body"] + + def test_gzip_requested_via_accept_encoding(self, monkeypatch): + """未自定义 headers 时应自动广告 Accept-Encoding: gzip。""" + from tools import http_get as hg + captured = {} + + def fake_open(self, req, timeout=None): + captured["ae"] = req.headers.get("Accept-encoding") # urllib 首字母大写 + return FakeResponse(SAMPLE_HTML.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + get("http://example.com", format="html") + assert captured["ae"] is not None + assert "gzip" in captured["ae"] + + +class TestErrorHints: + def test_404_hint(self, monkeypatch): + from tools import http_get as hg + + def fake_open(self, req, timeout=None): + raise urllib.error.HTTPError( + "http://example.com", 404, "Not Found", None, None + ) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com/missing") + assert r["ok"] is False + assert r["status"] == 404 + assert "不存在" in r["hint"] + assert "retries" not in r # 4xx 不重试 + + def test_403_hint_mentions_antibot(self, monkeypatch): + from tools import http_get as hg + + def fake_open(self, req, timeout=None): + raise urllib.error.HTTPError( + "http://example.com", 403, "Forbidden", None, None + ) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com") + assert r["ok"] is False + assert "反爬" in r["hint"] + + def test_5xx_exhausted_reports_retries(self, monkeypatch): + from tools import http_get as hg + monkeypatch.setattr(hg, "_RETRY_BACKOFF", 0) + + def fake_open(self, req, timeout=None): + raise urllib.error.HTTPError( + "http://example.com", 503, "Service Unavailable", None, None + ) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com") + assert r["ok"] is False + assert r["retries"] == hg._MAX_RETRIES + assert "已自动重试" in r["error"] + assert "服务端错误" in r["hint"] + + def test_timeout_hint(self, monkeypatch): + from tools import http_get as hg + monkeypatch.setattr(hg, "_RETRY_BACKOFF", 0) + + def fake_open(self, req, timeout=None): + raise urllib.error.URLError(TimeoutError("timed out")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", timeout=7) + assert r["ok"] is False + assert "timeout=7" in r["hint"] + assert r["retries"] == hg._MAX_RETRIES + + def test_dns_hint(self, monkeypatch): + from tools import http_get as hg + monkeypatch.setattr(hg, "_RETRY_BACKOFF", 0) + + def fake_open(self, req, timeout=None): + raise urllib.error.URLError( + socket.gaierror(-2, "Name or service not known") + ) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com") + assert r["ok"] is False + assert "域名解析失败" in r["hint"] + + def test_ssrf_redirect_message_not_disguised(self, monkeypatch): + """SSRF 拦截不应伪装成'连接失败'。""" + from tools import http_get as hg + from tools._http_utils import SsrfBlocked + + def fake_open(self, req, timeout=None): + raise SsrfBlocked("重定向目标被拦截: 禁止访问内网地址: 127.0.0.1") + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com") + assert r["ok"] is False + assert r["error"].startswith("重定向目标被拦截") + assert "连接失败" not in r["error"] + + +class TestJsRenderedHint: + def test_script_content_not_leaked(self, monkeypatch): + """内联脚本源码不应泄漏进转换结果(markdownify strip 只去标签)。""" + _clear_cache() + from tools import http_get as hg + html = ( + "

真正的正文内容在这里。

" + "" + ) + + def fake_open(self, req, timeout=None): + return FakeResponse(html.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", format="markdown") + assert r["ok"] is True + assert "真正的正文内容" in r["content"] + assert "tracker" not in r["content"] + + def test_spa_page_gets_hint(self, monkeypatch): + """大量脚本 + 几乎无文本的页面应提示疑似 JS 渲染。""" + from tools import http_get as hg + + def fake_open(self, req, timeout=None): + return FakeResponse(SPA_HTML.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", format="markdown") + assert r["ok"] is True + assert "hint" in r + assert "JS" in r["hint"] + + def test_normal_page_no_hint(self, monkeypatch): + """正常文本页面不应误报。""" + _clear_cache() + from tools import http_get as hg + + def fake_open(self, req, timeout=None): + return FakeResponse(SAMPLE_HTML.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", format="markdown") + assert r["ok"] is True + assert "hint" not in r + + +class TestTimeoutClamp: + def test_timeout_clamped_to_600(self, monkeypatch): + from tools import http_get as hg + captured = {} + + def fake_open(self, req, timeout=None): + captured["timeout"] = timeout + return FakeResponse(SAMPLE_HTML.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + get("http://example.com", timeout=9999) + assert captured["timeout"] == 600 + + def test_timeout_floor_is_1(self, monkeypatch): + from tools import http_get as hg + captured = {} + + def fake_open(self, req, timeout=None): + captured["timeout"] = timeout + return FakeResponse(SAMPLE_HTML.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + get("http://example.com", timeout=-3) + assert captured["timeout"] == 1 + + def test_timeout_invalid_uses_default(self, monkeypatch): + from tools import http_get as hg + captured = {} + + def fake_open(self, req, timeout=None): + captured["timeout"] = timeout + return FakeResponse(SAMPLE_HTML.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + get("http://example.com", timeout="abc") + assert captured["timeout"] == 15 + + +class TestRetryConnectionErrors: + def test_retry_on_remote_disconnected(self, monkeypatch): + """RemoteDisconnected(非 URLError)属瞬时连接错误,应纳入重试。""" + import http.client + from tools import http_get as hg + monkeypatch.setattr(hg, "_RETRY_BACKOFF", 0) + calls = {"n": 0} + + def fake_open(self, req, timeout=None): + calls["n"] += 1 + if calls["n"] <= 2: + raise http.client.RemoteDisconnected("Remote end closed connection") + return FakeResponse(SAMPLE_HTML.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com") + assert r["ok"] is True + assert calls["n"] == 3 + + def test_connection_reset_exhausted_reports_retries(self, monkeypatch): + """持续连接重置:重试耗尽后报错并透传 retries。""" + from tools import http_get as hg + monkeypatch.setattr(hg, "_RETRY_BACKOFF", 0) + + def fake_open(self, req, timeout=None): + raise ConnectionResetError(104, "Connection reset by peer") + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com") + assert r["ok"] is False + assert r["retries"] == hg._MAX_RETRIES + assert "连接中断" in r["error"] + + +class TestOffsetCoercion: + def test_offset_string_coerced(self, monkeypatch): + """LLM 传字符串 offset 应被容错转换,而非抛 TypeError。""" + _clear_cache() + from tools import http_get as hg + + def fake_open(self, req, timeout=None): + return FakeResponse(SAMPLE_HTML_LONG.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r1 = get("http://example.com", format="markdown") + nxt = r1["next_call"]["params"]["offset"] + r2 = get("http://example.com", format="markdown", offset=str(nxt)) + assert r2["ok"] is True + assert r2["offset"] == nxt + + def test_offset_invalid_string_returns_error(self): + r = get("http://example.com", offset="abc") + assert r["ok"] is False + assert "offset" in r["error"] + + +class TestDecompressionSafety: + def test_gzip_bomb_bounded(self, monkeypatch): + """压缩炸弹:10MB 全同字符(gzip ~10KB),解压必须流式限长。""" + import gzip as gz + from tools import http_get as hg + + payload = gz.compress(b"A" * (10 * 1024 * 1024)) + + def fake_open(self, req, timeout=None): + return FakeResponse(payload, headers={ + "Content-Type": "text/html", "Content-Encoding": "gzip"}) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", format="html") + assert r["ok"] is True + assert r["truncated"] is True + assert r["size"] == hg._MAX_RESPONSE_SIZE # 解压输出被限长在 5MB + + def test_truncated_gzip_returns_partial_not_garbage(self, monkeypatch): + """下载截断的 gzip 流:返回已解压部分内容并标记 truncated,而非压缩垃圾。""" + import gzip as gz + import os + from tools import http_get as hg + + payload = gz.compress(os.urandom(6 * 1024 * 1024)) # 不可压缩 → >5MB + + def fake_open(self, req, timeout=None): + return FakeResponse(payload, headers={ + "Content-Type": "text/plain", "Content-Encoding": "gzip"}) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com", format="html") + assert r["ok"] is True + assert r["truncated"] is True + assert len(r["body"]) > 0 + + def test_unsupported_content_encoding_br(self, monkeypatch): + """服务器强制 br 编码:明确报错而非返回乱码。""" + from tools import http_get as hg + + def fake_open(self, req, timeout=None): + return FakeResponse(b"whatever", headers={ + "Content-Type": "text/html", "Content-Encoding": "br"}) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r = get("http://example.com") + assert r["ok"] is False + assert "br" in r["error"] + + +class TestCacheCredentialsIsolation: + AUTH = {"Authorization": "Bearer SECRET-TOKEN"} + + def test_next_call_carries_headers(self, monkeypatch): + """带 headers 的翻页:next_call 必须透传 headers,否则缓存指纹对不上。""" + _clear_cache() + from tools import http_get as hg + + def fake_open(self, req, timeout=None): + return FakeResponse(SAMPLE_HTML_LONG.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r1 = get("http://example.com", format="markdown", headers=self.AUTH) + assert r1["has_more"] is True + assert r1["next_call"]["params"].get("headers") == self.AUTH + + def test_authed_pagination_hits_own_cache(self, monkeypatch): + """同一凭据的翻页命中自己的缓存,不重复下载。""" + _clear_cache() + from tools import http_get as hg + calls = {"n": 0} + + def fake_open(self, req, timeout=None): + calls["n"] += 1 + return FakeResponse(SAMPLE_HTML_LONG.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r1 = get("http://example.com", format="markdown", headers=self.AUTH) + nxt = r1["next_call"]["params"]["offset"] + r2 = get("http://example.com", format="markdown", offset=nxt, headers=self.AUTH) + assert r2["ok"] is True + assert calls["n"] == 1 + + def test_anonymous_cannot_hit_credentialed_cache(self, monkeypatch): + """无凭据方翻页不得命中带凭据缓存(跨凭据泄露回归)。""" + _clear_cache() + from tools import http_get as hg + calls = {"n": 0} + + def fake_open(self, req, timeout=None): + calls["n"] += 1 + return FakeResponse(SAMPLE_HTML_LONG.encode("utf-8")) + + monkeypatch.setattr(hg, "make_opener", lambda: _fake_opener(fake_open)) + r1 = get("http://example.com", format="markdown", headers=self.AUTH) + nxt = r1["next_call"]["params"]["offset"] + get("http://example.com", format="markdown", offset=nxt) # 无凭据 + assert calls["n"] == 2 # 未命中带凭据缓存,重新下载 + + +class TestStripTagBlocks: + def test_closed_block_removed(self): + from tools.http_get import _strip_tag_blocks + out = _strip_tag_blocks("

1

2

", "script") + assert out == "

1

2

" + + def test_unclosed_keeps_content(self): + """无闭合标签:只丢开标签、保留后续内容,不误删正文。""" + from tools.http_get import _strip_tag_blocks + assert _strip_tag_blocks("

1