fix: CamelCase undeclared request fields in add_request and batch_add_requests

Author: vdusek

src/apify_client/_resource_clients/request_queue.py

@@ -7,6 +7,7 @@
 from typing import TYPE_CHECKING, Any, Literal
 
 from more_itertools import constrained_batches
+from pydantic.alias_generators import to_camel
 
 from apify_client._docs import docs_group
 from apify_client._models import (
@@ -61,6 +62,21 @@
 _SAFETY_BUFFER_PERCENT = 0.01 / 100  # 0.01%
 
 
+def _dump_request_draft(request: RequestDraft) -> dict:
+    """Dump the request draft with camelCase aliases, including extra (undeclared) fields.
+
+    `RequestDraft` declares only a subset of the request fields; the rest (e.g. `user_data`) are captured as
+    Pydantic extras, which bypass the `to_camel` alias generator on dump, so their keys have to be camelized here.
+    An extra whose camelCase form is already present in the dump is kept as-is, so it never overwrites another field.
+    """
+    dumped = request.model_dump(by_alias=True, exclude_none=True)
+    for key in request.model_extra or {}:
+        camel_key = to_camel(key)
+        if key in dumped and camel_key not in dumped:
+            dumped[camel_key] = dumped.pop(key)
+    return dumped
+
+
 @docs_group('Resource clients')
 class RequestQueueClient(ResourceClient):
     """Sub-client for managing a specific request queue.
@@ -223,7 +239,7 @@ def add_request(
         response = self._http_client.call(
             url=self._build_url('requests'),
             method='POST',
-            json=request.model_dump(by_alias=True, exclude_none=True),
+            json=_dump_request_draft(request),
             params=request_params,
             timeout=timeout,
         )
@@ -401,10 +417,7 @@ def batch_add_requests(
             raise NotImplementedError('max_parallel is only supported in async client')
 
         requests_as_dicts = [
-            (r if isinstance(r, RequestDraft) else RequestDraft.model_validate(r)).model_dump(
-                by_alias=True, exclude_none=True
-            )
-            for r in requests
+            _dump_request_draft(r if isinstance(r, RequestDraft) else RequestDraft.model_validate(r)) for r in requests
         ]
 
         request_params = self._build_params(clientKey=self.client_key, forefront=forefront)
@@ -750,7 +763,7 @@ async def add_request(
         response = await self._http_client.call(
             url=self._build_url('requests'),
             method='POST',
-            json=request.model_dump(by_alias=True, exclude_none=True),
+            json=_dump_request_draft(request),
             params=request_params,
             timeout=timeout,
         )
@@ -970,16 +983,7 @@ async def batch_add_requests(
             Result containing lists of processed and unprocessed requests.
         """
         requests_as_dicts = [
-            (
-                request
-                if isinstance(request, RequestDraft)
-                else RequestDraft.model_validate(
-                    request,
-                )
-            ).model_dump(
-                by_alias=True,
-                exclude_none=True,
-            )
+            _dump_request_draft(request if isinstance(request, RequestDraft) else RequestDraft.model_validate(request))
             for request in requests
         ]
 

tests/unit/test_client_request_queue.py

@@ -1,14 +1,20 @@
 from __future__ import annotations
 
+import gzip
+import json
 import re
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
+import brotli
 import pytest
+from werkzeug import Request, Response
 
 from apify_client import ApifyClient, ApifyClientAsync
 from apify_client.errors import ApifyApiError
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
+
     from pytest_httpserver import HTTPServer
 
     from apify_client._typeddicts import RequestDraftDict
@@ -117,3 +123,110 @@ def test_batch_processed_partially_sync(httpserver: HTTPServer) -> None:
     assert requests[0]['unique_key'] in {request.unique_key for request in batch_response.processed_requests}
     assert len(batch_response.unprocessed_requests) == 1
     assert batch_response.unprocessed_requests[0].unique_key == requests[1]['unique_key']
+
+
+_ADD_REQUEST_RESPONSE_CONTENT = """{
+  "data": {
+    "requestId": "YiKoxjkaS9gjGTqhF",
+    "wasAlreadyPresent": false,
+    "wasAlreadyHandled": false
+  }
+}"""
+
+_FULLY_ADDED_BATCH_RESPONSE_CONTENT = """{
+  "data": {
+    "processedRequests": [
+      {
+        "requestId": "YiKoxjkaS9gjGTqhF",
+        "uniqueKey": "http://example.com/1",
+        "wasAlreadyPresent": false,
+        "wasAlreadyHandled": false
+      }
+    ],
+    "unprocessedRequests": []
+  }
+}"""
+
+_SNAKE_CASE_REQUEST: dict[str, Any] = {
+    'unique_key': 'http://example.com/1',
+    'url': 'http://example.com/1',
+    'user_data': {'label': 'DETAIL'},
+    'no_retry': True,
+}
+
+_EXPECTED_CAMEL_CASE_REQUEST = {
+    'uniqueKey': 'http://example.com/1',
+    'url': 'http://example.com/1',
+    'userData': {'label': 'DETAIL'},
+    'noRetry': True,
+}
+
+
+def _make_json_capture_handler(received_bodies: list[Any], response_content: str) -> Callable[[Request], Response]:
+    def handler(request: Request) -> Response:
+        body = request.get_data()
+        encoding = request.headers.get('Content-Encoding')
+        if encoding == 'br':
+            body = brotli.decompress(body)
+        elif encoding == 'gzip':
+            body = gzip.decompress(body)
+        received_bodies.append(json.loads(body))
+        return Response(status=201, response=response_content, content_type='application/json')
+
+    return handler
+
+
+def test_add_request_camel_cases_fields_undeclared_on_model_sync(httpserver: HTTPServer) -> None:
+    """Snake_case fields not declared on `RequestDraft` (e.g. `user_data`) are camelCased in the API payload."""
+    server_url = httpserver.url_for('/').removesuffix('/')
+    client = ApifyClient(token='placeholder_token', api_url=server_url, api_public_url=server_url)
+
+    received_bodies: list[Any] = []
+    httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_handler(
+        _make_json_capture_handler(received_bodies, _ADD_REQUEST_RESPONSE_CONTENT)
+    )
+
+    client.request_queue(request_queue_id='whatever').add_request(_SNAKE_CASE_REQUEST)  # ty: ignore[invalid-argument-type]
+    assert received_bodies == [_EXPECTED_CAMEL_CASE_REQUEST]
+
+
+async def test_add_request_camel_cases_fields_undeclared_on_model_async(httpserver: HTTPServer) -> None:
+    """Snake_case fields not declared on `RequestDraft` (e.g. `user_data`) are camelCased in the API payload."""
+    server_url = httpserver.url_for('/').removesuffix('/')
+    client = ApifyClientAsync(token='placeholder_token', api_url=server_url, api_public_url=server_url)
+
+    received_bodies: list[Any] = []
+    httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_handler(
+        _make_json_capture_handler(received_bodies, _ADD_REQUEST_RESPONSE_CONTENT)
+    )
+
+    await client.request_queue(request_queue_id='whatever').add_request(_SNAKE_CASE_REQUEST)  # ty: ignore[invalid-argument-type]
+    assert received_bodies == [_EXPECTED_CAMEL_CASE_REQUEST]
+
+
+def test_batch_add_requests_camel_cases_fields_undeclared_on_model_sync(httpserver: HTTPServer) -> None:
+    """Snake_case fields not declared on `RequestDraft` (e.g. `user_data`) are camelCased in the API payload."""
+    server_url = httpserver.url_for('/').removesuffix('/')
+    client = ApifyClient(token='placeholder_token', api_url=server_url, api_public_url=server_url)
+
+    received_bodies: list[Any] = []
+    httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_handler(
+        _make_json_capture_handler(received_bodies, _FULLY_ADDED_BATCH_RESPONSE_CONTENT)
+    )
+
+    client.request_queue(request_queue_id='whatever').batch_add_requests(requests=[_SNAKE_CASE_REQUEST])  # ty: ignore[invalid-argument-type]
+    assert received_bodies == [[_EXPECTED_CAMEL_CASE_REQUEST]]
+
+
+async def test_batch_add_requests_camel_cases_fields_undeclared_on_model_async(httpserver: HTTPServer) -> None:
+    """Snake_case fields not declared on `RequestDraft` (e.g. `user_data`) are camelCased in the API payload."""
+    server_url = httpserver.url_for('/').removesuffix('/')
+    client = ApifyClientAsync(token='placeholder_token', api_url=server_url, api_public_url=server_url)
+
+    received_bodies: list[Any] = []
+    httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_handler(
+        _make_json_capture_handler(received_bodies, _FULLY_ADDED_BATCH_RESPONSE_CONTENT)
+    )
+
+    await client.request_queue(request_queue_id='whatever').batch_add_requests(requests=[_SNAKE_CASE_REQUEST])  # ty: ignore[invalid-argument-type]
+    assert received_bodies == [[_EXPECTED_CAMEL_CASE_REQUEST]]