From bf673b2d7bf53381846e040cad46a1234c4073af Mon Sep 17 00:00:00 2001 From: jpirnay Date: Tue, 14 Jul 2026 16:35:52 +0200 Subject: [PATCH 1/7] Parse the URL scheme case-insensitively MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit URL schemes are case-insensitive (RFC 3986 §3.1). begin("HTTPS://...") — from user input or an upper-cased Location header — failed URL validation outright because the scheme was compared verbatim against "http"/"https". Lowercase it before comparing. --- libs/network/SecureNet/include/SecureHttpClient.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libs/network/SecureNet/include/SecureHttpClient.h b/libs/network/SecureNet/include/SecureHttpClient.h index 5c1c2c36..d56c5747 100644 --- a/libs/network/SecureNet/include/SecureHttpClient.h +++ b/libs/network/SecureNet/include/SecureHttpClient.h @@ -237,7 +237,11 @@ class SecureHttpClient { uint16_t& port) { const size_t schemeEnd = url.find("://"); if (schemeEnd == std::string::npos) return false; + // URL schemes are case-insensitive (RFC 3986 §3.1): "HTTPS://..." from a + // server's Location header or user input must parse like "https://...". scheme = url.substr(0, schemeEnd); + std::transform(scheme.begin(), scheme.end(), scheme.begin(), + [](unsigned char c) { return static_cast(tolower(c)); }); const size_t hostStart = schemeEnd + 3; const size_t pathStart = url.find('/', hostStart); const std::string hostPort = From efc8f1c858298c2154a3a6ce7a7513006a12927e Mon Sep 17 00:00:00 2001 From: jpirnay Date: Tue, 14 Jul 2026 16:41:07 +0200 Subject: [PATCH 2/7] Keep the connection alive across requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every request previously opened a fresh connection and forced Connection: close. Over wolfSSL that means a full TLS handshake per request — seconds of latency and the ECC/RSA bignum heap spike each time — which dominates bursty workloads like an OPDS catalog crawl, a KOSync exchange, or a multi-file download from one server. The client now holds its transports as members and reuses the open connection when the next request targets the same scheme://host:port, sending Connection: keep-alive. Server-side reality is handled explicitly: - Connection: close responses and HTTP/1.0 peers (which default to connection-per-request) are honored by closing after the body. - A keep-alive server may close the socket between requests at any time; the race surfaces as a failed write or a missing status line. That failure belongs to the reused connection, not the request, so it earns exactly one transparent retry on a fresh connection. Fresh- connection failures are real errors and are not retried. Request writes are now checked at all (they were fire-and-forget before). - Only a provably clean connection is reused: if the body was aborted, truncated, or close-delimited (no framing), the socket is closed rather than letting the next request parse leftover body bytes as its status line. end() now actually closes the held connection (it was a no-op kept for HTTPClient symmetry), the destructor does likewise, and the class is non-copyable since it owns a live connection. setReuse(false) restores the old connection-per-request behavior. --- .../SecureNet/include/SecureHttpClient.h | 227 ++++++++++++------ 1 file changed, 159 insertions(+), 68 deletions(-) diff --git a/libs/network/SecureNet/include/SecureHttpClient.h b/libs/network/SecureNet/include/SecureHttpClient.h index d56c5747..0a89ee38 100644 --- a/libs/network/SecureNet/include/SecureHttpClient.h +++ b/libs/network/SecureNet/include/SecureHttpClient.h @@ -8,7 +8,10 @@ // that API. Instead this implements the small slice of HTTP/1.1 that firmware // needs — GET/POST/PUT with custom headers and a buffered response body — // directly over SecureClient, handling Content-Length, chunked, and -// connection-close-delimited responses. +// connection-close-delimited responses. Connections are kept alive and reused +// across requests to the same scheme://host:port (see setReuse), so a burst of +// requests — an OPDS crawl, a sync exchange, a multi-file download — pays for +// one TLS handshake instead of one per request. // // Usage: // SecureHttpClient http; @@ -47,6 +50,12 @@ class SecureHttpClient { using DataCallback = std::function; using AbortCallback = std::function; + SecureHttpClient() = default; + ~SecureHttpClient() { end(); } + // Non-copyable: owns a live connection and a Client* into its own members. + SecureHttpClient(const SecureHttpClient&) = delete; + SecureHttpClient& operator=(const SecureHttpClient&) = delete; + // Skip peer verification (SecureClient does likewise). Required today because // the wolfSSL transport has no CA bundle wired up; see setCACert(). void setInsecure() { _insecure = true; } @@ -56,6 +65,11 @@ class SecureHttpClient { _insecure = false; } void setTimeout(uint32_t ms) { _timeoutMs = ms; } + // Keep the connection open between requests to the same scheme://host:port + // (the default). Reusing the TLS session skips a full handshake per request — + // seconds of latency plus the ECC/RSA heap spike on PSRAM-less boards. + // setReuse(false) restores connection-per-request behavior. + void setReuse(bool reuse) { _reuse = reuse; } // Parse the URL and reset per-request state. Returns false on a malformed // URL. @@ -65,9 +79,9 @@ class SecureHttpClient { _status = 0; return parseUrl(url, _scheme, _host, _path, _port); } - // Each request uses Connection: close, so there is no socket to release here; - // present for symmetry with HTTPClient call sites. - void end() {} + // Closes the kept-alive connection (if any). Call when done with a server; + // the next request transparently reconnects. + void end() { closeConnection(); } void addHeader(const std::string& name, const std::string& value) { _headers.push_back(name + ": " + value + "\r\n"); @@ -110,78 +124,89 @@ class SecureHttpClient { _callbackAborted = false; _aborted = false; - WiFiClient plain; - SecureClient secure; - Client* client; - if (_scheme == "https") { - if (_insecure) { - secure.setInsecure(); - } else if (_rootCA) { - secure.setCACert(_rootCA); + // One transparent retry: a keep-alive server may close the connection + // between requests at any time, and the race surfaces as a write failure + // or a missing status line on a socket that looked connected. That is a + // property of the reused connection, not of the request, so it earns one + // attempt on a fresh connection. A request that failed on a fresh + // connection is a real error and is not retried. + for (int attempt = 0; attempt < 2; ++attempt) { + const bool reusing = connectionMatches(); + if (isAborted(shouldAbort)) return -1; + if (!ensureConnected()) return -1; + + if (!writeRequest(method, payload, payloadLen)) { + closeConnection(); + if (reusing && attempt == 0) continue; + return -1; } - client = &secure; - } else { - client = &plain; - } - client->setTimeout(_timeoutMs / 1000); - if (isAborted(shouldAbort)) return -1; - if (!client->connect(_host.c_str(), _port)) return -1; - std::string req = - std::string(method) + " " + _path + " HTTP/1.1\r\nHost: " + hostHeader() + "\r\nConnection: close\r\n"; - for (const std::string& h : _headers) req += h; - if (payload && payloadLen) req += "Content-Length: " + std::to_string(payloadLen) + "\r\n"; - req += "\r\n"; - client->write(reinterpret_cast(req.data()), req.size()); - if (payload && payloadLen) client->write(payload, payloadLen); - - const unsigned long headerDeadline = millis() + _timeoutMs; - std::string line; - if (!readLine(*client, line, headerDeadline, shouldAbort)) { - client->stop(); - return -1; - } - // "HTTP/1.1 200 OK" — the status code starts at offset 9. - _status = line.size() >= 12 ? atoi(line.c_str() + 9) : 0; - - std::string transferEncoding; - while (readLine(*client, line, headerDeadline, shouldAbort)) { - if (line.empty()) break; // end of headers - const size_t colon = line.find(':'); - if (colon == std::string::npos) continue; - std::string name = line.substr(0, colon); - std::string value = line.substr(colon + 1); - while (!value.empty() && value.front() == ' ') value.erase(value.begin()); - std::transform(name.begin(), name.end(), name.begin(), - [](unsigned char c) { return static_cast(tolower(c)); }); - _responseHeaders.push_back(Header{name, value}); - if (name == "content-length") { - _contentLength = static_cast(strtoul(value.c_str(), nullptr, 10)); - _haveContentLength = true; - } else if (name == "transfer-encoding") { - std::transform(value.begin(), value.end(), value.begin(), + const unsigned long headerDeadline = millis() + _timeoutMs; + std::string line; + if (!readLine(*_conn, line, headerDeadline, shouldAbort)) { + closeConnection(); + if (reusing && attempt == 0 && !_aborted) continue; + return -1; + } + // "HTTP/1.1 200 OK" — the status code starts at offset 9. + _status = line.size() >= 12 ? atoi(line.c_str() + 9) : 0; + // HTTP/1.0 peers default to connection-per-request; only an explicit + // Connection: keep-alive header (below) overrides that. + bool keepAlive = line.compare(0, 9, "HTTP/1.0 ") != 0; + + std::string transferEncoding; + while (readLine(*_conn, line, headerDeadline, shouldAbort)) { + if (line.empty()) break; // end of headers + const size_t colon = line.find(':'); + if (colon == std::string::npos) continue; + std::string name = line.substr(0, colon); + std::string value = line.substr(colon + 1); + while (!value.empty() && value.front() == ' ') value.erase(value.begin()); + std::transform(name.begin(), name.end(), name.begin(), [](unsigned char c) { return static_cast(tolower(c)); }); - transferEncoding = value; + _responseHeaders.push_back(Header{name, value}); + if (name == "content-length") { + _contentLength = static_cast(strtoul(value.c_str(), nullptr, 10)); + _haveContentLength = true; + } else if (name == "transfer-encoding") { + std::transform(value.begin(), value.end(), value.begin(), + [](unsigned char c) { return static_cast(tolower(c)); }); + transferEncoding = value; + } else if (name == "connection") { + std::string v = value; + std::transform(v.begin(), v.end(), v.begin(), [](unsigned char c) { return static_cast(tolower(c)); }); + if (v.find("close") != std::string::npos) keepAlive = false; + else if (v.find("keep-alive") != std::string::npos) keepAlive = true; + } + } + if (_aborted) { + closeConnection(); + return -1; } - } - if (_aborted) { - client->stop(); - return -1; - } - if (transferEncoding.find("chunked") != std::string::npos) { - _bodyComplete = readChunked(*client, onData, shouldAbort); - } else if (transferEncoding.empty() || transferEncoding == "identity") { - if (_haveContentLength) { - _bodyComplete = readFixed(*client, _contentLength, onData, shouldAbort); + // A close-delimited body (no framing) ends WITH the connection, so it can + // never leave a reusable socket behind. + bool reusableFraming = true; + if (transferEncoding.find("chunked") != std::string::npos) { + _bodyComplete = readChunked(*_conn, onData, shouldAbort); + } else if (transferEncoding.empty() || transferEncoding == "identity") { + if (_haveContentLength) { + _bodyComplete = readFixed(*_conn, _contentLength, onData, shouldAbort); + } else { + _bodyComplete = readUntilClose(*_conn, onData, shouldAbort); + reusableFraming = false; + } } else { - _bodyComplete = readUntilClose(*client, onData, shouldAbort); + _bodyComplete = false; } - } else { - _bodyComplete = false; + + // Reuse only a provably clean connection. An aborted/truncated body + // leaves undrained bytes on the socket, and the next request would parse + // leftover body data as its status line. + if (!_reuse || !keepAlive || !_bodyComplete || !reusableFraming) closeConnection(); + return _status; } - client->stop(); - return _status; + return -1; } const std::string& getString() const { return _body; } @@ -268,6 +293,62 @@ class SecureHttpClient { return host + ":" + std::to_string(port); } + // True when the kept-alive connection matches the target of the next request + // and still looks alive. "Looks" is best-effort: the server may have closed + // it already, which the send path handles with its one-shot retry. + bool connectionMatches() const { + return _conn && _conn->connected() && _connHttps == (_scheme == "https") && _connHost == _host && + _connPort == _port; + } + + // Reuse the kept-alive connection when it matches, else (re)connect. + bool ensureConnected() { + if (connectionMatches()) return true; + closeConnection(); + if (_scheme == "https") { + if (_insecure) { + _secure.setInsecure(); + } else if (_rootCA) { + _secure.setCACert(_rootCA); + } + _secure.setTimeout(_timeoutMs / 1000); + if (!_secure.connect(_host.c_str(), _port)) return false; + _conn = &_secure; + _connHttps = true; + } else { + _plain.setTimeout(_timeoutMs / 1000); + if (!_plain.connect(_host.c_str(), _port)) return false; + _conn = &_plain; + _connHttps = false; + } + _connHost = _host; + _connPort = _port; + return true; + } + + void closeConnection() { + if (_conn) { + _conn->stop(); + _conn = nullptr; + } + _connHost.clear(); + _connPort = 0; + _connHttps = false; + } + + // Request line + headers (+ body). Write results are checked: a stale + // keep-alive socket often surfaces first as a failed write. + bool writeRequest(const char* method, const uint8_t* payload, size_t payloadLen) { + std::string req = std::string(method) + " " + _path + " HTTP/1.1\r\nHost: " + hostHeader() + "\r\n"; + req += _reuse ? "Connection: keep-alive\r\n" : "Connection: close\r\n"; + for (const std::string& h : _headers) req += h; + if (payload && payloadLen) req += "Content-Length: " + std::to_string(payloadLen) + "\r\n"; + req += "\r\n"; + if (_conn->write(reinterpret_cast(req.data()), req.size()) != req.size()) return false; + if (payload && payloadLen && _conn->write(payload, payloadLen) != payloadLen) return false; + return true; + } + bool isAborted(const AbortCallback& shouldAbort) { if (shouldAbort && shouldAbort()) { _aborted = true; @@ -370,6 +451,16 @@ class SecureHttpClient { static constexpr size_t READ_CHUNK = 1024; // body read buffer; keep TLS downloads moving static constexpr size_t MAX_LINE = 4096; // header / chunk-size line cap + // Kept-alive connection state. _conn points at _secure or _plain while a + // connection is held open, and null otherwise. + WiFiClient _plain; + SecureClient _secure; + Client* _conn = nullptr; + std::string _connHost; + uint16_t _connPort = 0; + bool _connHttps = false; + bool _reuse = true; + std::string _scheme; std::string _host; std::string _path; From 68f58da1092f7171781baadc2197eaeb071f91bf Mon Sep 17 00:00:00 2001 From: jpirnay Date: Tue, 14 Jul 2026 16:41:42 +0200 Subject: [PATCH 3/7] Send a User-Agent header on every request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Requests carried no User-Agent at all. Several CDNs treat UA-less requests as bots and answer 403 — Cloudflare in front of public OPDS catalogs is the case that bites e-reader firmware in practice. Send a default of "FreeInk-ESP32"; setUserAgent() lets firmware identify itself properly (name/version). --- libs/network/SecureNet/include/SecureHttpClient.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libs/network/SecureNet/include/SecureHttpClient.h b/libs/network/SecureNet/include/SecureHttpClient.h index 0a89ee38..a16a4bcd 100644 --- a/libs/network/SecureNet/include/SecureHttpClient.h +++ b/libs/network/SecureNet/include/SecureHttpClient.h @@ -65,6 +65,9 @@ class SecureHttpClient { _insecure = false; } void setTimeout(uint32_t ms) { _timeoutMs = ms; } + // Identify the client. Sent on every request: several CDNs (notably + // Cloudflare in front of OPDS catalogs) answer UA-less requests with 403. + void setUserAgent(const std::string& ua) { _userAgent = ua; } // Keep the connection open between requests to the same scheme://host:port // (the default). Reusing the TLS session skips a full handshake per request — // seconds of latency plus the ECC/RSA heap spike on PSRAM-less boards. @@ -340,6 +343,7 @@ class SecureHttpClient { // keep-alive socket often surfaces first as a failed write. bool writeRequest(const char* method, const uint8_t* payload, size_t payloadLen) { std::string req = std::string(method) + " " + _path + " HTTP/1.1\r\nHost: " + hostHeader() + "\r\n"; + req += "User-Agent: " + _userAgent + "\r\n"; req += _reuse ? "Connection: keep-alive\r\n" : "Connection: close\r\n"; for (const std::string& h : _headers) req += h; if (payload && payloadLen) req += "Content-Length: " + std::to_string(payloadLen) + "\r\n"; @@ -470,6 +474,7 @@ class SecureHttpClient { int _status = 0; size_t _contentLength = 0; const char* _rootCA = nullptr; + std::string _userAgent = "FreeInk-ESP32"; bool _insecure = false; bool _haveContentLength = false; bool _bodyComplete = false; From 558284c06169c3d2a149d829ff48e8052c1057dc Mon Sep 17 00:00:00 2001 From: jpirnay Date: Tue, 14 Jul 2026 16:42:21 +0200 Subject: [PATCH 4/7] Add HTTP Basic authentication OPDS catalogs are commonly password-protected with Basic auth. setBasicAuth(user, pass) sends the Authorization header on every request while set; an empty password with a non-empty user is valid per RFC 7617. Credentials go through the ESP32 core's base64. --- .../SecureNet/include/SecureHttpClient.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/libs/network/SecureNet/include/SecureHttpClient.h b/libs/network/SecureNet/include/SecureHttpClient.h index a16a4bcd..c5cb896b 100644 --- a/libs/network/SecureNet/include/SecureHttpClient.h +++ b/libs/network/SecureNet/include/SecureHttpClient.h @@ -32,6 +32,7 @@ #include #include +#include #include #include @@ -65,6 +66,17 @@ class SecureHttpClient { _insecure = false; } void setTimeout(uint32_t ms) { _timeoutMs = ms; } + // HTTP Basic authentication, sent on every request while set (OPDS servers + // commonly protect their catalogs this way). An empty user disables it; an + // empty password with a non-empty user is valid per RFC 7617. + void setBasicAuth(const std::string& user, const std::string& pass) { + _authUser = user; + _authPass = pass; + } + void clearBasicAuth() { + _authUser.clear(); + _authPass.clear(); + } // Identify the client. Sent on every request: several CDNs (notably // Cloudflare in front of OPDS catalogs) answer UA-less requests with 403. void setUserAgent(const std::string& ua) { _userAgent = ua; } @@ -345,6 +357,10 @@ class SecureHttpClient { std::string req = std::string(method) + " " + _path + " HTTP/1.1\r\nHost: " + hostHeader() + "\r\n"; req += "User-Agent: " + _userAgent + "\r\n"; req += _reuse ? "Connection: keep-alive\r\n" : "Connection: close\r\n"; + if (!_authUser.empty()) { + const std::string creds = _authUser + ":" + _authPass; + req += "Authorization: Basic " + std::string(base64::encode(creds.c_str()).c_str()) + "\r\n"; + } for (const std::string& h : _headers) req += h; if (payload && payloadLen) req += "Content-Length: " + std::to_string(payloadLen) + "\r\n"; req += "\r\n"; @@ -475,6 +491,8 @@ class SecureHttpClient { size_t _contentLength = 0; const char* _rootCA = nullptr; std::string _userAgent = "FreeInk-ESP32"; + std::string _authUser; + std::string _authPass; bool _insecure = false; bool _haveContentLength = false; bool _bodyComplete = false; From e7360fd42957a35b61bdebf8086d840cdc74a0c3 Mon Sep 17 00:00:00 2001 From: jpirnay Date: Tue, 14 Jul 2026 16:51:37 +0200 Subject: [PATCH 5/7] Follow HTTP redirects (opt-in) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Book downloads behind OPDS catalogs routinely bounce through one or more redirects (catalog -> storage host -> signed CDN URL), and every consumer was hand-rolling the follow loop around resolveUrl(). setFollowRedirects(maxHops) moves that loop into the client. Default is 0 — 3xx responses are returned to the caller exactly as before. When following: - Location is resolved against the current URL (absolute, host-relative and path-relative forms) via the existing resolveUrl(). - Intermediate 3xx bodies are drained — keeping the connection reusable for the next hop, which with keep-alive makes a same-host redirect chain cost zero extra handshakes — but never delivered to the caller's sink; only the final response is. - 303 (and, per long-standing convention, 301/302 after POST) continue as GET without the request body; 307/308 preserve method and body. - A https -> http downgrade stops the chain and surfaces the 3xx unless explicitly allowed via setAllowRedirectDowngrade(true), so transport security is never dropped silently. --- .../SecureNet/include/SecureHttpClient.h | 74 ++++++++++++++++++- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/libs/network/SecureNet/include/SecureHttpClient.h b/libs/network/SecureNet/include/SecureHttpClient.h index c5cb896b..e3fbdcbc 100644 --- a/libs/network/SecureNet/include/SecureHttpClient.h +++ b/libs/network/SecureNet/include/SecureHttpClient.h @@ -23,7 +23,8 @@ // http.end(); // } // -// Header-only. Callers must pass a final URL (no redirect following here). +// Header-only. Redirects are returned to the caller by default; opt in to +// following them with setFollowRedirects(). // // OPT-IN: requires -DFREEINK_NET_WOLFSSL=1 for TLS. With the flag off, // SecureClient is an inert stub and https requests fail at connect() @@ -66,6 +67,17 @@ class SecureHttpClient { _insecure = false; } void setTimeout(uint32_t ms) { _timeoutMs = ms; } + // Follow up to maxHops redirect hops (default 0: 3xx responses are returned + // to the caller, matching the previous behavior). While following, the + // intermediate 3xx bodies are drained and discarded; only the final + // response reaches the caller. 303 — and, per long-standing convention, + // 301/302 after a POST — continue as GET without the request body; 307/308 + // preserve method and body. + void setFollowRedirects(int maxHops) { _followRedirects = maxHops < 0 ? 0 : maxHops; } + // Allow a redirect to step down from https to http. Off by default, because + // the downgrade silently drops transport security; when refused, following + // stops and the caller sees the 3xx. + void setAllowRedirectDowngrade(bool allow) { _allowRedirectDowngrade = allow; } // HTTP Basic authentication, sent on every request while set (OPDS servers // commonly protect their catalogs this way). An empty user disables it; an // empty password with a non-empty user is valid per RFC 7617. @@ -130,6 +142,47 @@ class SecureHttpClient { // an incomplete body from a caller-initiated stop. int sendRequest(const char* method, const uint8_t* payload, size_t payloadLen, const DataCallback& onData, const AbortCallback& shouldAbort = nullptr) { + std::string activeMethod = method; + const uint8_t* activePayload = payload; + size_t activePayloadLen = payloadLen; + for (int hop = 0;; ++hop) { + const int status = sendRequestOnce(activeMethod.c_str(), activePayload, activePayloadLen, onData, shouldAbort); + if (status < 0 || hop >= _followRedirects || !isRedirectStatus(status)) return status; + + const std::string location = getHeader("location"); + if (location.empty()) return status; + const std::string base = _scheme + "://" + hostHeader() + _path; + std::string next; + if (!resolveUrl(base, location, next)) return status; + std::string scheme; + std::string host; + std::string path; + uint16_t port = 0; + if (!parseUrl(next, scheme, host, path, port)) return status; + // Refuse to silently drop transport security: a https -> http redirect + // stops here (the caller sees the 3xx) unless explicitly allowed. + if (_scheme == "https" && scheme == "http" && !_allowRedirectDowngrade) return status; + _scheme = scheme; + _host = host; + _path = path; + _port = port; + + // 303 always continues as GET; after 301/302, long-standing convention + // downgrades POST to GET too. 307/308 preserve method and body. + if ((status == 301 || status == 302 || status == 303) && activeMethod != "GET" && activeMethod != "HEAD") { + activeMethod = "GET"; + activePayload = nullptr; + activePayloadLen = 0; + } + } + } + + // One request/response transaction against the URL state — never follows + // redirects. When following is enabled and the response is a 3xx, its body + // is drained but NOT delivered to onData (only the final response's body + // reaches the caller's sink). + int sendRequestOnce(const char* method, const uint8_t* payload, size_t payloadLen, const DataCallback& onData, + const AbortCallback& shouldAbort = nullptr) { _status = 0; _body.clear(); _responseHeaders.clear(); @@ -201,14 +254,21 @@ class SecureHttpClient { // A close-delimited body (no framing) ends WITH the connection, so it can // never leave a reusable socket behind. + // A 3xx body that is about to be followed is protocol plumbing, not + // payload: drain it (keeping the connection reusable for the next hop) + // without touching the caller's sink. + const bool discardBody = _followRedirects > 0 && isRedirectStatus(_status); + const DataCallback discard = [](const uint8_t*, size_t) { return true; }; + const DataCallback& bodySink = discardBody ? discard : onData; + bool reusableFraming = true; if (transferEncoding.find("chunked") != std::string::npos) { - _bodyComplete = readChunked(*_conn, onData, shouldAbort); + _bodyComplete = readChunked(*_conn, bodySink, shouldAbort); } else if (transferEncoding.empty() || transferEncoding == "identity") { if (_haveContentLength) { - _bodyComplete = readFixed(*_conn, _contentLength, onData, shouldAbort); + _bodyComplete = readFixed(*_conn, _contentLength, bodySink, shouldAbort); } else { - _bodyComplete = readUntilClose(*_conn, onData, shouldAbort); + _bodyComplete = readUntilClose(*_conn, bodySink, shouldAbort); reusableFraming = false; } } else { @@ -273,6 +333,10 @@ class SecureHttpClient { std::string value; }; + static bool isRedirectStatus(int status) { + return status == 301 || status == 302 || status == 303 || status == 307 || status == 308; + } + static bool parseUrl(const std::string& url, std::string& scheme, std::string& host, std::string& path, uint16_t& port) { const size_t schemeEnd = url.find("://"); @@ -494,6 +558,8 @@ class SecureHttpClient { std::string _authUser; std::string _authPass; bool _insecure = false; + int _followRedirects = 0; + bool _allowRedirectDowngrade = false; bool _haveContentLength = false; bool _bodyComplete = false; bool _callbackAborted = false; From 3af96c51adc9cb17d3bb9eeeeb97bc604eba69fd Mon Sep 17 00:00:00 2001 From: jpirnay Date: Tue, 14 Jul 2026 16:53:58 +0200 Subject: [PATCH 6/7] Report download progress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Book downloads on an e-reader want a progress meter, and the data callback alone forces every consumer to duplicate the byte counting and Content-Length plumbing. setProgressCallback() is invoked after each delivered chunk with (bytes so far, total) — total is the Content-Length, or 0 when the response has no length (chunked/close-delimited). Drained redirect bodies never report. Returning false aborts the transfer, reported via callbackAborted(), so a UI cancel button needs no separate abort callback. --- .../SecureNet/include/SecureHttpClient.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/libs/network/SecureNet/include/SecureHttpClient.h b/libs/network/SecureNet/include/SecureHttpClient.h index e3fbdcbc..13c1ee3c 100644 --- a/libs/network/SecureNet/include/SecureHttpClient.h +++ b/libs/network/SecureNet/include/SecureHttpClient.h @@ -51,6 +51,9 @@ class SecureHttpClient { public: using DataCallback = std::function; using AbortCallback = std::function; + // (bytes so far, total from Content-Length or 0 when unknown). + // Return false to abort the transfer. + using ProgressCallback = std::function; SecureHttpClient() = default; ~SecureHttpClient() { end(); } @@ -67,6 +70,11 @@ class SecureHttpClient { _insecure = false; } void setTimeout(uint32_t ms) { _timeoutMs = ms; } + // Progress reporting for response bodies (download meters). Called after + // each delivered chunk with the running byte count; never called for + // drained redirect bodies. Returning false aborts the transfer (reported + // via callbackAborted()). Clear with nullptr. + void setProgressCallback(const ProgressCallback& progress) { _progress = progress; } // Follow up to maxHops redirect hops (default 0: 3xx responses are returned // to the caller, matching the previous behavior). While following, the // intermediate 3xx bodies are drained and discarded; only the final @@ -260,6 +268,8 @@ class SecureHttpClient { const bool discardBody = _followRedirects > 0 && isRedirectStatus(_status); const DataCallback discard = [](const uint8_t*, size_t) { return true; }; const DataCallback& bodySink = discardBody ? discard : onData; + _downloaded = 0; + _reportProgress = !discardBody && static_cast(_progress); bool reusableFraming = true; if (transferEncoding.find("chunked") != std::string::npos) { @@ -446,6 +456,11 @@ class SecureHttpClient { _callbackAborted = true; return false; } + _downloaded += len; + if (_reportProgress && !_progress(_downloaded, _haveContentLength ? _contentLength : 0)) { + _callbackAborted = true; + return false; + } return true; } @@ -560,6 +575,9 @@ class SecureHttpClient { bool _insecure = false; int _followRedirects = 0; bool _allowRedirectDowngrade = false; + ProgressCallback _progress; + size_t _downloaded = 0; + bool _reportProgress = false; bool _haveContentLength = false; bool _bodyComplete = false; bool _callbackAborted = false; From cecb7777f4eec8484bd08dbbc7f5b53e61ccd21e Mon Sep 17 00:00:00 2001 From: jpirnay Date: Tue, 14 Jul 2026 16:54:53 +0200 Subject: [PATCH 7/7] Grow the body read buffer to 2 KB The body readers drain wolfSSL's decrypted records through a stack-allocated buffer. At small sizes the per-read overhead dominates: a consuming firmware measured ~30 KB/s at 512 B, slow enough that Cloudflare dropped large downloads mid-stream. 2 KB (three readers never live at once; still modest for a 16 KB task stack) removed the stall in field testing on an ESP32-C3. --- libs/network/SecureNet/include/SecureHttpClient.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/libs/network/SecureNet/include/SecureHttpClient.h b/libs/network/SecureNet/include/SecureHttpClient.h index 13c1ee3c..916726f4 100644 --- a/libs/network/SecureNet/include/SecureHttpClient.h +++ b/libs/network/SecureNet/include/SecureHttpClient.h @@ -547,8 +547,13 @@ class SecureHttpClient { } } - static constexpr size_t READ_CHUNK = 1024; // body read buffer; keep TLS downloads moving - static constexpr size_t MAX_LINE = 4096; // header / chunk-size line cap + // Body read buffer. 2 KB drains wolfSSL's decrypted TLS records in few + // enough read() calls to keep large downloads moving: at 512 B a consuming + // firmware measured ~30 KB/s and slow CDNs (Cloudflare) dropped the + // connection mid-stream; 2 KB removed the stall. Stack-allocated in the + // body readers, so kept modest. + static constexpr size_t READ_CHUNK = 2048; + static constexpr size_t MAX_LINE = 4096; // header / chunk-size line cap // Kept-alive connection state. _conn points at _secure or _plain while a // connection is held open, and null otherwise.