chore(scrapy): serialization module docstring, drop test dividers

Author: vdusek

src/apify/scrapy/_serialization.py

@@ -1,3 +1,22 @@
+"""JSON serialization of Scrapy requests and cached responses for storage on the Apify platform.
+
+Scrapy requests and cached responses are stored in the Apify request queue and key-value store,
+which hold JSON, so they are serialized as JSON here rather than pickled.
+
+Only `body` (`bytes`) and `headers` (`{bytes: [bytes]}`) are not natively JSON-serializable; both sit at
+fixed keys and are base64-encoded in place. A `str` `body` is encoded as its UTF-8 bytes and comes back as
+`bytes`, matching Scrapy, which always stores `body` as `bytes`. Pydantic models such as Crawlee's
+`UserData` are dumped via `model_dump()`. Everything else, notably `meta` and `cb_kwargs`, must already be
+JSON-serializable, otherwise serialization fails with a clear error naming the offending value. No in-band
+sentinel is used, so no user value can collide with the encoding.
+
+Known limitations of the pickle -> JSON switch (a documented breaking change): JSON has fewer types than
+pickle, so values in `meta`/`cb_kwargs` are subject to JSON's coercions. A `tuple` round-trips as a `list`
+and non-string `dict` keys round-trip as strings (e.g. `{1: 'a'}` becomes `{'1': 'a'}`). Values JSON cannot
+represent at all (`datetime`, `set`, `Decimal`, arbitrary objects, ...) are not coerced silently:
+serialization raises and the request is skipped loudly rather than stored in a corrupted form.
+"""
+
 from __future__ import annotations
 
 import base64
@@ -6,22 +25,6 @@
 
 from pydantic import BaseModel
 
-# Scrapy requests and cached responses are stored in the Apify request queue and key-value store,
-# which hold JSON, so they are serialized as JSON here rather than pickled.
-#
-# Only `body` (`bytes`) and `headers` (`{bytes: [bytes]}`) are not natively JSON-serializable; both sit at
-# fixed keys and are base64-encoded in place. A `str` `body` is encoded as its UTF-8 bytes and comes back as
-# `bytes`, matching Scrapy, which always stores `body` as `bytes`. Pydantic models such as Crawlee's
-# `UserData` are dumped via `model_dump()`. Everything else, notably `meta` and `cb_kwargs`, must already be
-# JSON-serializable, otherwise serialization fails with a clear error naming the offending value. No in-band
-# sentinel is used, so no user value can collide with the encoding.
-#
-# Known limitations of the pickle -> JSON switch (a documented breaking change): JSON has fewer types than
-# pickle, so values in `meta`/`cb_kwargs` are subject to JSON's coercions. A `tuple` round-trips as a `list`
-# and non-string `dict` keys round-trip as strings (e.g. `{1: 'a'}` becomes `{'1': 'a'}`). Values JSON cannot
-# represent at all (`datetime`, `set`, `Decimal`, arbitrary objects, ...) are not coerced silently:
-# serialization raises and the request is skipped loudly rather than stored in a corrupted form.
-
 # Cap the offending value's repr in a serialization error message so a huge value cannot bloat the log.
 _MAX_ERROR_VALUE_REPR_LEN = 200
 

tests/unit/scrapy/test_serialization.py

@@ -15,9 +15,6 @@ def _round_trip(data: dict) -> dict:
     return decoded
 
 
-# --- body encoding (bytes/str symmetry) ---
-
-
 def test_bytes_body_round_trips() -> None:
     assert _round_trip({'body': b'\x00\x01\xff binary'})['body'] == b'\x00\x01\xff binary'
 
@@ -35,9 +32,6 @@ def test_empty_str_body_round_trips() -> None:
     assert _round_trip({'body': ''})['body'] == b''
 
 
-# --- headers ---
-
-
 def test_bytes_headers_round_trip() -> None:
     data = {'headers': {b'Content-Type': [b'text/html'], b'X-Bin': [b'\x00\xff']}}
     assert _round_trip(data)['headers'] == {b'Content-Type': [b'text/html'], b'X-Bin': [b'\x00\xff']}
@@ -53,9 +47,6 @@ def test_bare_header_value_is_normalized_to_list() -> None:
     assert _round_trip({'headers': {b'X-Single': b'one'}})['headers'] == {b'X-Single': [b'one']}
 
 
-# --- non-ASCII text is kept as UTF-8 (ensure_ascii=False) ---
-
-
 def test_non_ascii_is_not_escaped() -> None:
     """Non-ASCII text stays in its UTF-8 form instead of ASCII escape sequences, which would bloat storage."""
     encoded = encode_to_json({'meta': {'name': 'Ñoño café 日本語'}})
@@ -64,9 +55,6 @@ def test_non_ascii_is_not_escaped() -> None:
     assert decode_from_json(encoded)['meta']['name'] == 'Ñoño café 日本語'
 
 
-# --- pydantic models are dumped ---
-
-
 def test_pydantic_model_is_dumped_by_alias() -> None:
     class Model(BaseModel):
         first: int = Field(serialization_alias='First')
@@ -75,9 +63,6 @@ class Model(BaseModel):
     assert decode_from_json(encoded)['meta']['m'] == {'First': 1}
 
 
-# --- documented JSON coercions (a breaking change vs. pickle) ---
-
-
 def test_tuple_is_coerced_to_list() -> None:
     """Documented limitation: JSON has no tuple type, so a tuple round-trips as a list."""
     assert _round_trip({'meta': {'coords': (1, 2, 3)}})['meta']['coords'] == [1, 2, 3]
@@ -88,9 +73,6 @@ def test_non_string_dict_keys_are_coerced_to_strings() -> None:
     assert _round_trip({'cb_kwargs': {'m': {1: 'a'}}})['cb_kwargs']['m'] == {'1': 'a'}
 
 
-# --- values JSON cannot represent fail loudly with a useful message ---
-
-
 def test_non_serializable_value_raises_with_type_and_repr() -> None:
     """A value JSON cannot represent raises a `TypeError` naming the offending type and value."""
     when = datetime(2020, 1, 2, 3, 4, 5, tzinfo=UTC)