Skip to content
Snippets Groups Projects
Commit fe535f20 authored by Valentin Rigal's avatar Valentin Rigal Committed by Bastien Abadie
Browse files

Retry then raise when __len__ fails loading a page

parent f6512684
No related branches found
No related tags found
1 merge request!215Retry then raise when __len__ fails loading a page
Pipeline #29492 passed
......@@ -81,6 +81,14 @@ class ResponsePaginator(Sized, Iterator):
), "allow_missing_data must be a boolean"
def _fetch_page(self):
"""
Retrieve the next page and store its results
Returns None in case of a failure
Returns True in case the page returned results
Returns False in case the page returned an empty result
Raises a StopIteration in case there are no pages left to iterate on
"""
# Filter out pages with no retries
# Transform as a list of tuples for simpler output
remaining = sorted([(m, v) for m, v in self.pages.items() if v > 0])
......@@ -136,7 +144,7 @@ class ResponsePaginator(Sized, Iterator):
if self.count == 0:
# Pagination has retrieved 0 results
self.pages = {}
return
return False
self.pages_count = math.ceil(self.count / len(self.results))
logger.info(
f"Pagination will load a total of {self.pages_count} pages."
......@@ -161,7 +169,7 @@ class ResponsePaginator(Sized, Iterator):
# Stop happy path here, we don't need to process errors
self.pages_loaded += 1
return self.data
return True
except apistar.exceptions.ErrorResponse as e:
logger.warning(f"API Error {e.status_code} on pagination: {e.content}")
......@@ -194,6 +202,9 @@ class ResponsePaginator(Sized, Iterator):
else:
raise Exception("Stopping pagination as data will be incomplete")
# No data could be fetch, return None
return None
def __iter__(self):
return self
......@@ -216,7 +227,12 @@ class ResponsePaginator(Sized, Iterator):
def __len__(self):
# Handle calls to len when no requests have been made yet
if not self.pages_loaded and self.count is None:
self._fetch_page()
# Continuously try to fetch a page until there are no retries left
while self._fetch_page() is None:
pass
# Count may be null in case of a StopIteration
if self.count is None:
raise Exception("An error occurred fetching items total count")
return self.count
def __repr__(self):
......
......@@ -216,6 +216,30 @@ def test_cursor_pagination_zero_results(mock_schema):
assert len(responses.calls) == 2
def test_cursor_pagination_len_errors(mock_schema, mocker):
"""
Errors must be raised when calling the __len__ method of an iterator
"""
base_url = "https://dummy.test/api/v1/process/process_id/elements/"
responses.add(
responses.GET,
f"{base_url}?with_count=true",
status=500,
)
# Prevent waiting for a random amount of time between retries
mocker.patch("arkindex.pagination.random.random", return_value=0.0001)
cli = ArkindexClient("t0k3n", base_url="https://dummy.test")
paginator = cli.paginate("ListProcessElements", id="process_id", retries=5)
with pytest.raises(
Exception, match="Stopping pagination as data will be incomplete"
):
len(paginator)
assert len(responses.calls) == 6
def test_paginate_x_paginated(mock_schema):
responses.add(
responses.GET,
......
......@@ -9,3 +9,4 @@ commands =
deps =
pytest
pytest-responses
pytest-mock
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment