Retry then raise when __len__ fails loading a page

arkindex/pagination.py

@@ -81,6 +81,14 @@ class ResponsePaginator(Sized, Iterator):
         ), "allow_missing_data must be a boolean"
 
     def _fetch_page(self):
+        """
+        Retrieve the next page and store its results
+
+        Returns None in case of a failure
+        Returns True in case the page returned results
+        Returns False in case the page returned an empty result
+        Raises a StopIteration in case there are no pages left to iterate on
+        """
         # Filter out pages with no retries
         # Transform as a list of tuples for simpler output
         remaining = sorted([(m, v) for m, v in self.pages.items() if v > 0])
@@ -136,7 +144,7 @@ class ResponsePaginator(Sized, Iterator):
                 if self.count == 0:
                     # Pagination has retrieved 0 results
                     self.pages = {}
-                    return
+                    return False
                 self.pages_count = math.ceil(self.count / len(self.results))
                 logger.info(
                     f"Pagination will load a total of {self.pages_count} pages."
@@ -161,7 +169,7 @@ class ResponsePaginator(Sized, Iterator):
 
             # Stop happy path here, we don't need to process errors
             self.pages_loaded += 1
-            return self.data
+            return True
 
         except apistar.exceptions.ErrorResponse as e:
             logger.warning(f"API Error {e.status_code} on pagination: {e.content}")
@@ -194,6 +202,9 @@ class ResponsePaginator(Sized, Iterator):
             else:
                 raise Exception("Stopping pagination as data will be incomplete")
 
+        # No data could be fetch, return None
+        return None
+
     def __iter__(self):
         return self
 
@@ -216,7 +227,12 @@ class ResponsePaginator(Sized, Iterator):
     def __len__(self):
         # Handle calls to len when no requests have been made yet
         if not self.pages_loaded and self.count is None:
-            self._fetch_page()
+            # Continuously try to fetch a page until there are no retries left
+            while self._fetch_page() is None:
+                pass
+        # Count may be null in case of a StopIteration
+        if self.count is None:
+            raise Exception("An error occurred fetching items total count")
         return self.count
 
     def __repr__(self):

tests/test_pagination.py

@@ -216,6 +216,30 @@ def test_cursor_pagination_zero_results(mock_schema):
     assert len(responses.calls) == 2
 
 
+def test_cursor_pagination_len_errors(mock_schema, mocker):
+    """
+    Errors must be raised when calling the __len__ method of an iterator
+    """
+    base_url = "https://dummy.test/api/v1/process/process_id/elements/"
+    responses.add(
+        responses.GET,
+        f"{base_url}?with_count=true",
+        status=500,
+    )
+
+    # Prevent waiting for a random amount of time between retries
+    mocker.patch("arkindex.pagination.random.random", return_value=0.0001)
+
+    cli = ArkindexClient("t0k3n", base_url="https://dummy.test")
+    paginator = cli.paginate("ListProcessElements", id="process_id", retries=5)
+
+    with pytest.raises(
+        Exception, match="Stopping pagination as data will be incomplete"
+    ):
+        len(paginator)
+    assert len(responses.calls) == 6
+
+
 def test_paginate_x_paginated(mock_schema):
     responses.add(
         responses.GET,

tox.ini

@@ -9,3 +9,4 @@ commands =
 deps =
 	pytest
 	pytest-responses
+	pytest-mock