Skip to content

Commit 9cea9a4

Browse files
committed
docs: clarify in-flight add markers coordinate only in-process concurrent calls
1 parent 72f459c commit 9cea9a4

3 files changed

Lines changed: 19 additions & 9 deletions

File tree

src/apify/storage_clients/_apify/_request_queue_shared_client.py

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -79,9 +79,14 @@ def __init__(
7979
self._requests_being_added: dict[str, asyncio.Future[bool]] = {}
8080
"""In-flight `add_batch_of_requests` markers, keyed by request ID.
8181
82+
Coordinates only concurrent `add_batch_of_requests` calls sharing this one client instance (e.g. several
83+
producer coroutines adding requests in the same process). It does not coordinate separate client instances
84+
or processes, which each keep their own markers; deduplication across clients still relies on the platform.
85+
8286
Each future resolves once the platform call that is adding the request settles: `True` if the request was
83-
committed, `False` otherwise. Concurrent producers of the same request await it instead of re-sending,
84-
which preserves deduplication while still avoiding false success when the original add fails.
87+
committed, `False` otherwise. A concurrent call adding the same request awaits the future instead of
88+
re-sending it, which avoids a duplicate platform write while still avoiding false success when the original
89+
add fails.
8590
"""
8691

8792
self._queue_has_locked_requests: bool | None = None
@@ -128,7 +133,7 @@ async def add_batch_of_requests(
128133
awaited_in_flight.append((request, self._requests_being_added[request_id]))
129134

130135
else:
131-
# Register an in-flight marker so concurrent producers dedupe against it; caching is deferred
136+
# Register an in-flight marker so a concurrent call dedupes against it; caching is deferred
132137
# until the platform confirms the request was accepted (see below).
133138
new_requests.append(request)
134139
self._requests_being_added[request_id] = loop.create_future()
@@ -170,7 +175,7 @@ async def add_batch_of_requests(
170175
# Add the locally known already present processed requests based on the local cache.
171176
api_response.processed_requests.extend(already_present_requests)
172177
finally:
173-
# Release the in-flight markers we registered. Committed requests tell concurrent producers the
178+
# Release the in-flight markers we registered. Committed requests tell concurrent callers the
174179
# request reached the platform; everything else (unprocessed, API error, cancellation) tells them
175180
# it did not, so they retry instead of reporting false success.
176181
for request in new_requests:

src/apify/storage_clients/_apify/_request_queue_single_client.py

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -99,9 +99,14 @@ def __init__(
9999
self._requests_being_added: dict[str, asyncio.Future[bool]] = {}
100100
"""In-flight `add_batch_of_requests` markers, keyed by request ID.
101101
102+
Coordinates only concurrent `add_batch_of_requests` calls sharing this one client instance (e.g. several
103+
producer coroutines adding requests in the same process). It does not coordinate separate client instances
104+
or processes, which each keep their own markers; deduplication across clients still relies on the platform.
105+
102106
Each future resolves once the platform call that is adding the request settles: `True` if the request was
103-
committed, `False` otherwise. Concurrent producers of the same request await it instead of re-sending,
104-
which preserves deduplication while still avoiding false success when the original add fails.
107+
committed, `False` otherwise. A concurrent call adding the same request awaits the future instead of
108+
re-sending it, which avoids a duplicate platform write while still avoiding false success when the original
109+
add fails.
105110
"""
106111

107112
self._initialized_caches = False
@@ -159,7 +164,7 @@ async def add_batch_of_requests(
159164
awaited_in_flight.append((request, self._requests_being_added[request_id]))
160165
else:
161166
# Push the request to the platform. Probably not there, or we are not aware of it. Register an
162-
# in-flight marker so concurrent producers dedupe against it; caching is deferred until the
167+
# in-flight marker so a concurrent call dedupes against it; caching is deferred until the
163168
# platform confirms the request was accepted (see below).
164169
new_requests.append(request)
165170
self._requests_being_added[request_id] = loop.create_future()
@@ -193,7 +198,7 @@ async def add_batch_of_requests(
193198
# Add the locally known already present processed requests based on the local cache.
194199
api_response.processed_requests.extend(already_present_requests)
195200
finally:
196-
# Release the in-flight markers we registered. Committed requests tell concurrent producers the
201+
# Release the in-flight markers we registered. Committed requests tell concurrent callers the
197202
# request reached the platform; everything else (unprocessed, API error, cancellation) tells them
198203
# it did not, so they retry instead of reporting false success.
199204
for request in new_requests:

src/apify/storage_clients/_apify/_utils.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ def settle_pending_addition(
7373
*,
7474
committed: bool,
7575
) -> None:
76-
"""Resolve the in-flight add marker for a request, unblocking any concurrent producers awaiting it.
76+
"""Resolve the in-flight add marker for a request, unblocking any concurrent call awaiting it.
7777
7878
Args:
7979
requests_being_added: The client's map of in-flight `add_batch_of_requests` markers.

0 commit comments

Comments
 (0)