fix: Stop cursor-paginated iterators on the cursor, not on empty pages

Author: vdusek

src/apify_client/_pagination.py

@@ -129,8 +129,11 @@ def get_cursor_iterator(
     """Yield individual items from cursor-paginated API responses.
 
     Cursor pagination is restricted to the two API responses that expose it: `ListOfKeys` (for key-value store keys) and
-    `ListOfRequests` (for request queue requests). Iteration ends when a page scans no items, the next cursor is `None`,
-    or the user-requested `limit` is reached.
+    `ListOfRequests` (for request queue requests). Iteration ends when the next cursor is `None` or the user-requested
+    `limit` is reached. Emptiness alone does not stop iteration: server-side filters (such as the request-queue state
+    `filter`) can drop every item on a page while a live cursor still points at more data, so termination relies on the
+    cursor, not on whether a page returned items. Unlike offset responses, cursor responses expose no scanned-item
+    `count`, so `count` cannot be used to detect a fully-filtered page here.
 
     Args:
         callback: Function returning a single page of items. Receives `cursor` and `limit` kwargs.
@@ -149,13 +152,12 @@ def get_cursor_iterator(
         )
         yield from current_page.items
 
-        page_scanned = max(getattr(current_page, 'count', 0), len(current_page.items))
-        fetched_items += page_scanned
+        fetched_items += len(current_page.items)
         cursor = (
             current_page.next_exclusive_start_key if isinstance(current_page, ListOfKeys) else current_page.next_cursor
         )
 
-        if not page_scanned or cursor is None or (initial_limit and fetched_items >= initial_limit):
+        if cursor is None or (initial_limit and fetched_items >= initial_limit):
             break
 
 
@@ -195,13 +197,12 @@ async def get_cursor_iterator_async(
         for item in current_page.items:
             yield item
 
-        page_scanned = max(getattr(current_page, 'count', 0), len(current_page.items))
-        fetched_items += page_scanned
+        fetched_items += len(current_page.items)
         cursor = (
             current_page.next_exclusive_start_key if isinstance(current_page, ListOfKeys) else current_page.next_cursor
         )
 
-        if not page_scanned or cursor is None or (initial_limit and fetched_items >= initial_limit):
+        if cursor is None or (initial_limit and fetched_items >= initial_limit):
             break
 
 

tests/unit/test_client_pagination.py

@@ -634,11 +634,10 @@ def __init__(self, items: list[dict[str, int]], count: int) -> None:
 
 
 class _FakeCursorPage:
-    """Cursor-paginated page whose `count` (items scanned) may exceed `len(items)` when filters drop items."""
+    """Cursor-paginated page mirroring `ListOfRequests`: no scanned-`count`, so a filtered page is just `items=[]`."""
 
-    def __init__(self, items: list[dict[str, int]], count: int, next_cursor: str | None) -> None:
+    def __init__(self, items: list[dict[str, int]], next_cursor: str | None) -> None:
         self.items = items
-        self.count = count
         self.next_cursor = next_cursor
 
 
@@ -669,10 +668,10 @@ async def _callback(*, limit: int | None = None, offset: int | None = None) -> _
 
 
 def test_cursor_iterator_continues_past_fully_filtered_page() -> None:
-    """A fully-filtered page (`items=[]`, `count>0`) with a live cursor must not stop the cursor iterator."""
+    """A fully-filtered page (`items=[]`) with a live cursor must not stop the cursor iterator."""
     pages = {
-        None: _FakeCursorPage(items=[], count=1000, next_cursor='c1'),
-        'c1': _FakeCursorPage(items=[{'id': 1}, {'id': 2}], count=2, next_cursor=None),
+        None: _FakeCursorPage(items=[], next_cursor='c1'),
+        'c1': _FakeCursorPage(items=[{'id': 1}, {'id': 2}], next_cursor=None),
     }
 
     def _callback(*, limit: int | None = None, cursor: str | None = None) -> _FakeCursorPage:  # noqa: ARG001
@@ -682,10 +681,10 @@ def _callback(*, limit: int | None = None, cursor: str | None = None) -> _FakeCu
 
 
 async def test_cursor_iterator_async_continues_past_fully_filtered_page() -> None:
-    """A fully-filtered page (`items=[]`, `count>0`) with a live cursor must not stop the async cursor iterator."""
+    """A fully-filtered page (`items=[]`) with a live cursor must not stop the async cursor iterator."""
     pages = {
-        None: _FakeCursorPage(items=[], count=1000, next_cursor='c1'),
-        'c1': _FakeCursorPage(items=[{'id': 1}, {'id': 2}], count=2, next_cursor=None),
+        None: _FakeCursorPage(items=[], next_cursor='c1'),
+        'c1': _FakeCursorPage(items=[{'id': 1}, {'id': 2}], next_cursor=None),
     }
 
     async def _callback(*, limit: int | None = None, cursor: str | None = None) -> _FakeCursorPage:  # noqa: ARG001