linear: cursor pagination + set entry from createdAt

bugwarrior/services/linear.py

@@ -90,6 +90,7 @@ def get(v, k, default=None):
                 or None
             ),
             "priority": self.config.default_priority,
+            "entry": created,
             "annotations": get(self.extra, "annotations", []),
             "tags": self.get_tags(),
             self.URL: self.record["url"],
@@ -149,8 +150,8 @@ def __init__(self, *args, **kwargs):
             )
 
         self.query = """
-            query Issues($filter: IssueFilter!) {
-              issues(filter: $filter) {
+            query Issues($filter: IssueFilter!, $after: String) {
+              issues(filter: $filter, first: 250, after: $after) {
                 nodes {
                   url
                   title
@@ -181,6 +182,10 @@ def __init__(self, *args, **kwargs):
                     name
                   }
                 }
+                pageInfo {
+                  hasNextPage
+                  endCursor
+                }
               }
             }
             """
@@ -195,19 +200,35 @@ def issues(self):
 
     def get_issues(self):
         """
-        Make a Linear API request, using the query defined in the constructor.
-        """
-        data = {
-            "query": self.query,
-            "variables": {"filter": {"and": self.filter} if self.filter else {}},
-        }
-        response = self.session.post(self.config.host, data=json.dumps(data))
-        res = self.json_response(response)
+        Make Linear API requests, paginating with cursors until exhausted.
 
-        if "errors" in res:
-            messages = [
-                error.get("message", "Unknown error") for error in res['errors']
-            ]
-            raise ValueError("; ".join(messages))
-
-        return res.get("data", {}).get("issues", {}).get("nodes", [])
+        Linear's GraphQL API uses Relay-style cursor pagination on the
+        ``issues`` connection. Without an explicit ``first`` argument, the
+        server returns its default page size (50) and we silently lose any
+        remaining issues. We request the maximum page size (250) and follow
+        ``pageInfo.endCursor`` until ``hasNextPage`` is false.
+        """
+        cursor = None
+        filter_arg = {"and": self.filter} if self.filter else {}
+        while True:
+            data = {
+                "query": self.query,
+                "variables": {"filter": filter_arg, "after": cursor},
+            }
+            response = self.session.post(self.config.host, data=json.dumps(data))
+            res = self.json_response(response)
+
+            if "errors" in res:
+                messages = [
+                    error.get("message", "Unknown error") for error in res['errors']
+                ]
+                raise ValueError("; ".join(messages))
+
+            issues = res.get("data", {}).get("issues", {})
+            for node in issues.get("nodes", []):
+                yield node
+
+            page_info = issues.get("pageInfo", {})
+            if not page_info.get("hasNextPage"):
+                return
+            cursor = page_info.get("endCursor")

tests/test_linear.py

@@ -144,6 +144,7 @@ def test_to_taskwarrior(self):
         expected_output = {
             "project": "prj",
             "priority": "M",
+            "entry": created_timestamp,
             "annotations": [],
             "tags": [],
             "linearurl": "https://linear.app/dustins-doings/issue/DUS-5/do-stuff",
@@ -170,6 +171,7 @@ def test_to_taskwarrior(self):
         expected_output = {
             "project": None,
             "priority": "M",
+            "entry": created_timestamp,
             "annotations": [],
             "tags": ["Improvement", "Feature"],
             "linearurl": "https://linear.app/dustins-doings/issue/DUS-1/bugwarrior",
@@ -198,6 +200,7 @@ def test_issues(self):
             "annotations": [],
             "description": "(bw)#DUS-5 - DO STUFF .. "
             "https://linear.app/dustins-doings/issue/DUS-5/do-stuff",
+            "entry": created_timestamp,
             "linearassignee": "djmitche@gmail.com",
             "linearclosed": closed_timestamp,
             "linearcreated": created_timestamp,
@@ -214,3 +217,39 @@ def test_issues(self):
             "tags": [],
         }
         self.assertEqual(TaskConstructor(issue).get_taskwarrior_record(), expected)
+
+    @responses.activate
+    def test_issues_paginates(self):
+        """Drains every page when Linear signals ``hasNextPage``."""
+        # Reset the default mock registered in setUp so we can control page order.
+        responses.reset()
+
+        page_one = {
+            "data": {
+                "issues": {
+                    "nodes": [RESPONSE["data"]["issues"]["nodes"][0]],
+                    "pageInfo": {"hasNextPage": True, "endCursor": "cursor-page-2"},
+                }
+            }
+        }
+        page_two = {
+            "data": {
+                "issues": {
+                    "nodes": [RESPONSE["data"]["issues"]["nodes"][1]],
+                    "pageInfo": {"hasNextPage": False, "endCursor": None},
+                }
+            }
+        }
+        responses.add(responses.POST, "https://api.linear.app/graphql", json=page_one)
+        responses.add(responses.POST, "https://api.linear.app/graphql", json=page_two)
+
+        identifiers = [issue.record["identifier"] for issue in self.service.issues()]
+        self.assertEqual(identifiers, ["DUS-5", "DUS-1"])
+
+        # Two HTTP calls were made, and the second one carried the cursor
+        # returned by the first.
+        self.assertEqual(len(responses.calls), 2)
+        first_body = json.loads(responses.calls[0].request.body)
+        second_body = json.loads(responses.calls[1].request.body)
+        self.assertIsNone(first_body["variables"]["after"])
+        self.assertEqual(second_body["variables"]["after"], "cursor-page-2")