fix: Propagate API token to custom HTTP clients (#956)

`ApifyClient.with_custom_http_client(token=...)` (and the async twin)
stored the token on the `ApifyClient` instance but never passed it to
the injected HTTP client, so no request carried an `Authorization`
header and every call failed with 401. The documented custom HTTP client
example inherited the bug.

- `with_custom_http_client` now sets `Authorization: Bearer <token>` on
the injected client's default headers, unless the client already has an
auth header configured (checked case-insensitively).
- `HttpClientBase._prepare_request_call` now merges the client's default
headers under the per-request headers, so any custom client using the
helper (including a pre-built `ImpitHttpClient` passed as the custom
client) actually sends them. For the default client the wire behavior is
unchanged, since impit request-level headers replace the identical
client-level ones.
- The HTTPX guide examples now merge `self._headers` before delegating,
and the `HttpClient` ABC docstring states that implementations must send
the default headers with every request.

Regression tests cover the token reaching the wire (custom sync/async
clients and a pre-built tokenless `ImpitHttpClient`) and the no-clobber
semantics for client-configured auth headers.

Author: vdusek

docs/03_guides/code/05_custom_http_client_async.py

@@ -35,6 +35,10 @@ async def call(
     ) -> HttpResponse:
         timeout_secs = self._compute_timeout(timeout, attempt=1) or 0
 
+        # Merge the client's default headers (including authorization)
+        # with the per-request ones.
+        headers = self._merge_headers(self._headers, headers)
+
         # httpx.Response satisfies the HttpResponse protocol,
         # so it can be returned directly.
         return await self._client.request(

docs/03_guides/code/05_custom_http_client_sync.py

@@ -34,6 +34,10 @@ def call(
     ) -> HttpResponse:
         timeout_secs = self._compute_timeout(timeout, attempt=1) or 0
 
+        # Merge the client's default headers (including authorization)
+        # with the per-request ones.
+        headers = self._merge_headers(self._headers, headers)
+
         # httpx.Response satisfies the HttpResponse protocol,
         # so it can be returned directly.
         return self._client.request(

src/apify_client/_apify_client.py

@@ -227,7 +227,7 @@ def with_custom_http_client(
 
         Use this alternative constructor when you want to provide your own HTTP client implementation
         instead of the default one. The custom client is responsible for its own configuration
-        (retries, timeouts, headers, etc.).
+        (retries, timeouts, etc.); only the token is applied to it, as described below.
 
         ### Usage
 
@@ -246,12 +246,15 @@ def call(self, *, method, url, **kwargs) -> HttpResponse:
         ```
 
         Args:
-            token: The Apify API token.
+            token: The Apify API token. It is set as the `Authorization` header on the custom client,
+                unless the client already has one configured.
             api_url: The URL of the Apify API server to connect to. Defaults to https://api.apify.com.
             api_public_url: The globally accessible URL of the Apify API server. Defaults to https://api.apify.com.
             http_client: A custom HTTP client instance extending `HttpClient`.
         """
         instance = cls(token=token, api_url=api_url, api_public_url=api_public_url)
+        if token is not None:
+            http_client.set_default_authorization(token)
         instance._http_client = http_client
         return instance
 
@@ -586,7 +589,7 @@ def with_custom_http_client(
 
         Use this alternative constructor when you want to provide your own HTTP client implementation
         instead of the default one. The custom client is responsible for its own configuration
-        (retries, timeouts, headers, etc.).
+        (retries, timeouts, etc.); only the token is applied to it, as described below.
 
         ### Usage
 
@@ -605,12 +608,15 @@ async def call(self, *, method, url, **kwargs) -> HttpResponse:
         ```
 
         Args:
-            token: The Apify API token.
+            token: The Apify API token. It is set as the `Authorization` header on the custom client,
+                unless the client already has one configured.
             api_url: The URL of the Apify API server to connect to. Defaults to https://api.apify.com.
             api_public_url: The globally accessible URL of the Apify API server. Defaults to https://api.apify.com.
             http_client: A custom HTTP client instance extending `HttpClientAsync`.
         """
         instance = cls(token=token, api_url=api_url, api_public_url=api_public_url)
+        if token is not None:
+            http_client.set_default_authorization(token)
         instance._http_client = http_client
         return instance
 

src/apify_client/http_clients/_base.py

@@ -143,7 +143,30 @@ def __init__(
         if token is not None:
             default_headers['Authorization'] = f'Bearer {token}'
 
-        self._headers = {**default_headers, **(headers or {})}
+        self._headers = self._merge_headers(default_headers, headers)
+
+    def set_default_authorization(self, token: str) -> None:
+        """Set the `Authorization` header from the token, unless an authorization header is already configured.
+
+        Args:
+            token: The Apify API token to set as the `Bearer` authorization.
+        """
+        if not any(key.lower() == 'authorization' for key in self._headers):
+            self._headers['Authorization'] = f'Bearer {token}'
+
+    @staticmethod
+    def _merge_headers(base: dict[str, str] | None, override: dict[str, str] | None) -> dict[str, str]:
+        """Merge two header dicts, treating header names case-insensitively.
+
+        A header from `override` replaces a same-named header in `base` regardless of the casing
+        of either name, and keeps the casing it was passed with.
+        """
+        merged = dict(base) if base else {}
+        for key, value in (override or {}).items():
+            for existing_key in [k for k in merged if k.lower() == key.lower()]:
+                del merged[existing_key]
+            merged[key] = value
+        return merged
 
     @staticmethod
     def _parse_params(params: dict[str, Any] | None) -> dict[str, Any] | None:
@@ -207,24 +230,31 @@ def _prepare_request_call(
         data: str | bytes | bytearray | None = None,
         json: JsonSerializable | None = None,
     ) -> tuple[dict[str, str], dict[str, Any] | None, bytes | None]:
-        """Prepare headers, params, and body for an HTTP request. Serializes JSON and compresses the body."""
+        """Prepare headers, params, and body for an HTTP request.
+
+        Merges the client's default headers (including authorization) with per-request headers,
+        serializes JSON and compresses the body. Header names are treated case-insensitively and
+        per-request values win over the client defaults. For JSON bodies, a `Content-Type` header
+        is set unless the caller supplied one.
+        """
         if json is not None and data is not None:
             raise ValueError('Cannot pass both "json" and "data" parameters at the same time!')
 
-        headers = dict(headers) if headers else {}
+        headers = self._merge_headers(self._headers, headers)
 
         # Dump JSON data to string so it can be compressed.
         if json is not None:
             data = jsonlib.dumps(json, ensure_ascii=False, allow_nan=False, default=str).encode('utf-8')
-            headers['Content-Type'] = 'application/json'
+            if not any(key.lower() == 'content-type' for key in headers):
+                headers['Content-Type'] = 'application/json'
 
         if isinstance(data, (str, bytes, bytearray)):
             if isinstance(data, str):
                 data = data.encode('utf-8')
             elif isinstance(data, bytearray):
                 data = bytes(data)
             data = self._http_compressor.compress(data)
-            headers['Content-Encoding'] = self._http_compressor.content_encoding
+            headers = self._merge_headers(headers, {'Content-Encoding': self._http_compressor.content_encoding})
 
         return (headers, self._parse_params(params), data)
 
@@ -252,6 +282,10 @@ class HttpClient(HttpClientBase, ABC):
     Extend this class to create a custom synchronous HTTP client. Override the `call` method
     with your implementation. Helper methods from the base class are available for request
     preparation, URL building, and parameter parsing.
+
+    Implementations must send the client's default headers from `self._headers` with every request,
+    otherwise the `Authorization` header never reaches the API. The `_prepare_request_call` helper
+    merges them into the per-request headers automatically.
     """
 
     @abstractmethod

src/apify_client/http_clients/_impit.py

@@ -104,7 +104,6 @@ def __init__(
         )
 
         self._impit_client = impit.Client(
-            headers=self._headers,
             follow_redirects=True,
         )
 
@@ -354,7 +353,6 @@ def __init__(
         )
 
         self._impit_async_client = impit.AsyncClient(
-            headers=self._headers,
             follow_redirects=True,
         )
 

tests/unit/test_client_headers.py

@@ -124,3 +124,33 @@ def test_headers_sync(httpserver: HTTPServer) -> None:
     }
     assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers
     assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'}
+
+
+async def test_per_request_headers_override_defaults_async(httpserver: HTTPServer) -> None:
+    """Test that a per-request header overrides a same-named default header on the wire, without duplication."""
+    client = ImpitHttpClientAsync(token='placeholder_token')
+    httpserver.expect_request('/').respond_with_handler(_header_handler)
+    api_url = httpserver.url_for('/').removesuffix('/')
+
+    response = await client.call(method='GET', url=f'{api_url}/', headers={'authorization': 'Bearer per-request'})
+
+    request_headers = json.loads(response.text)['received_headers']
+
+    # WSGI joins duplicate headers into one comma-separated value, so exact equality
+    # also proves the authorization header was sent only once.
+    assert request_headers['Authorization'] == 'Bearer per-request'
+
+
+def test_per_request_headers_override_defaults_sync(httpserver: HTTPServer) -> None:
+    """Test that a per-request header overrides a same-named default header on the wire, without duplication."""
+    client = ImpitHttpClient(token='placeholder_token')
+    httpserver.expect_request('/').respond_with_handler(_header_handler)
+    api_url = httpserver.url_for('/').removesuffix('/')
+
+    response = client.call(method='GET', url=f'{api_url}/', headers={'authorization': 'Bearer per-request'})
+
+    request_headers = json.loads(response.text)['received_headers']
+
+    # WSGI joins duplicate headers into one comma-separated value, so exact equality
+    # also proves the authorization header was sent only once.
+    assert request_headers['Authorization'] == 'Bearer per-request'

tests/unit/test_http_clients.py

@@ -170,6 +170,53 @@ def test_base_http_client_initialization() -> None:
     assert isinstance(client2._statistics, ClientStatistics)
 
 
+def test_http_client_init_headers_override_defaults_case_insensitively() -> None:
+    """Constructor headers replace same-named default headers even when their casings differ."""
+    client = _ConcreteHttpClient(token='default_token', headers={'authorization': 'Bearer custom'})
+
+    auth_headers = {key: value for key, value in client._headers.items() if key.lower() == 'authorization'}
+    assert auth_headers == {'authorization': 'Bearer custom'}
+
+
+def test_http_client_init_workflow_key_header(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The X-Apify-Workflow-Key default header is set from the APIFY_WORKFLOW_KEY env var."""
+    monkeypatch.setenv('APIFY_WORKFLOW_KEY', 'workflow_key_123')
+
+    client = _ConcreteHttpClient()
+
+    assert client._headers['X-Apify-Workflow-Key'] == 'workflow_key_123'
+
+
+def test_set_default_authorization_sets_token_when_missing() -> None:
+    """set_default_authorization sets the Bearer token when no authorization header is configured."""
+    client = _ConcreteHttpClient()
+
+    client.set_default_authorization('test_token')
+
+    assert client._headers['Authorization'] == 'Bearer test_token'
+
+
+@pytest.mark.parametrize(
+    ('base', 'override', 'expected'),
+    [
+        pytest.param(None, None, {}, id='both none'),
+        pytest.param({'Accept': 'a'}, None, {'Accept': 'a'}, id='override none'),
+        pytest.param(None, {'Accept': 'a'}, {'Accept': 'a'}, id='base none'),
+        pytest.param({'Accept': 'a'}, {'X-Custom': 'b'}, {'Accept': 'a', 'X-Custom': 'b'}, id='disjoint names'),
+        pytest.param({'Authorization': 'x'}, {'Authorization': 'y'}, {'Authorization': 'y'}, id='same casing'),
+        pytest.param({'Authorization': 'x'}, {'authorization': 'y'}, {'authorization': 'y'}, id='lowercase override'),
+        pytest.param({'authorization': 'x'}, {'AUTHORIZATION': 'y'}, {'AUTHORIZATION': 'y'}, id='uppercase override'),
+    ],
+)
+def test_merge_headers(
+    base: dict[str, str] | None,
+    override: dict[str, str] | None,
+    expected: dict[str, str],
+) -> None:
+    """_merge_headers merges case-insensitively, override values win and keep their casing."""
+    assert HttpClient._merge_headers(base, override) == expected
+
+
 def test_http_client_creates_sync_impit_client() -> None:
     """Test that ImpitHttpClient creates sync impit client correctly."""
     client = ImpitHttpClient(token='test_token_123')
@@ -282,11 +329,13 @@ def compressor_case(request: pytest.FixtureRequest) -> tuple:
 
 
 def test_prepare_request_call_basic() -> None:
-    """Test _prepare_request_call with basic parameters."""
-    client = _ConcreteHttpClient()
+    """Test _prepare_request_call returns the client default headers when no per-request values are given."""
+    client = _ConcreteHttpClient(token='test_token')
 
     headers, params, data = client._prepare_request_call()
-    assert headers == {}
+    assert headers == client._headers
+    assert headers is not client._headers
+    assert headers['Authorization'] == 'Bearer test_token'
     assert params is None
     assert data is None
 
@@ -447,6 +496,39 @@ def test_prepare_request_call_does_not_mutate_caller_headers() -> None:
     assert caller_headers == original
 
 
+def test_prepare_request_call_per_request_headers_override_defaults_case_insensitively() -> None:
+    """A per-request header replaces a same-named default header even when their casings differ."""
+    client = _ConcreteHttpClient(token='default_token')
+
+    headers, _params, _data = client._prepare_request_call(headers={'authorization': 'Bearer per-request'})
+
+    auth_headers = {key: value for key, value in headers.items() if key.lower() == 'authorization'}
+    assert auth_headers == {'authorization': 'Bearer per-request'}
+
+
+def test_prepare_request_call_json_keeps_caller_content_type() -> None:
+    """A caller-supplied content type is not overwritten by the JSON default, regardless of casing."""
+    client = _ConcreteHttpClient()
+
+    headers, _params, _data = client._prepare_request_call(
+        headers={'content-type': 'application/json; charset=utf-8'},
+        json={'key': 'value'},
+    )
+
+    content_type_headers = {key: value for key, value in headers.items() if key.lower() == 'content-type'}
+    assert content_type_headers == {'content-type': 'application/json; charset=utf-8'}
+
+
+def test_prepare_request_call_replaces_caller_content_encoding() -> None:
+    """The Content-Encoding header always reflects the compressor actually applied, replacing any caller value."""
+    client = _ConcreteHttpClient(http_compressor=GzipHttpCompressor())
+
+    headers, _params, _data = client._prepare_request_call(headers={'content-encoding': 'br'}, data='payload')
+
+    encoding_headers = {key: value for key, value in headers.items() if key.lower() == 'content-encoding'}
+    assert encoding_headers == {'Content-Encoding': 'gzip'}
+
+
 def test_build_url_with_params_none() -> None:
     """Test _build_url_with_params with None params."""
     client = _ConcreteHttpClient()