Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
A
api-client
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package Registry
Container Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Arkindex
api-client
Commits
fe535f20
Commit
fe535f20
authored
2 years ago
by
Valentin Rigal
Committed by
Bastien Abadie
2 years ago
Browse files
Options
Downloads
Patches
Plain Diff
Retry then raise when __len__ fails loading a page
parent
f6512684
No related branches found
No related tags found
1 merge request
!215
Retry then raise when __len__ fails loading a page
Pipeline
#29492
passed
2 years ago
Stage: release
Changes
3
Pipelines
10
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
arkindex/pagination.py
+19
-3
19 additions, 3 deletions
arkindex/pagination.py
tests/test_pagination.py
+24
-0
24 additions, 0 deletions
tests/test_pagination.py
tox.ini
+1
-0
1 addition, 0 deletions
tox.ini
with
44 additions
and
3 deletions
arkindex/pagination.py
+
19
−
3
View file @
fe535f20
...
...
@@ -81,6 +81,14 @@ class ResponsePaginator(Sized, Iterator):
),
"
allow_missing_data must be a boolean
"
def
_fetch_page
(
self
):
"""
Retrieve the next page and store its results
Returns None in case of a failure
Returns True in case the page returned results
Returns False in case the page returned an empty result
Raises a StopIteration in case there are no pages left to iterate on
"""
# Filter out pages with no retries
# Transform as a list of tuples for simpler output
remaining
=
sorted
([(
m
,
v
)
for
m
,
v
in
self
.
pages
.
items
()
if
v
>
0
])
...
...
@@ -136,7 +144,7 @@ class ResponsePaginator(Sized, Iterator):
if
self
.
count
==
0
:
# Pagination has retrieved 0 results
self
.
pages
=
{}
return
return
False
self
.
pages_count
=
math
.
ceil
(
self
.
count
/
len
(
self
.
results
))
logger
.
info
(
f
"
Pagination will load a total of
{
self
.
pages_count
}
pages.
"
...
...
@@ -161,7 +169,7 @@ class ResponsePaginator(Sized, Iterator):
# Stop happy path here, we don't need to process errors
self
.
pages_loaded
+=
1
return
self
.
data
return
True
except
apistar
.
exceptions
.
ErrorResponse
as
e
:
logger
.
warning
(
f
"
API Error
{
e
.
status_code
}
on pagination:
{
e
.
content
}
"
)
...
...
@@ -194,6 +202,9 @@ class ResponsePaginator(Sized, Iterator):
else
:
raise
Exception
(
"
Stopping pagination as data will be incomplete
"
)
# No data could be fetch, return None
return
None
def
__iter__
(
self
):
return
self
...
...
@@ -216,7 +227,12 @@ class ResponsePaginator(Sized, Iterator):
def
__len__
(
self
):
# Handle calls to len when no requests have been made yet
if
not
self
.
pages_loaded
and
self
.
count
is
None
:
self
.
_fetch_page
()
# Continuously try to fetch a page until there are no retries left
while
self
.
_fetch_page
()
is
None
:
pass
# Count may be null in case of a StopIteration
if
self
.
count
is
None
:
raise
Exception
(
"
An error occurred fetching items total count
"
)
return
self
.
count
def
__repr__
(
self
):
...
...
This diff is collapsed.
Click to expand it.
tests/test_pagination.py
+
24
−
0
View file @
fe535f20
...
...
@@ -216,6 +216,30 @@ def test_cursor_pagination_zero_results(mock_schema):
assert
len
(
responses
.
calls
)
==
2
def
test_cursor_pagination_len_errors
(
mock_schema
,
mocker
):
"""
Errors must be raised when calling the __len__ method of an iterator
"""
base_url
=
"
https://dummy.test/api/v1/process/process_id/elements/
"
responses
.
add
(
responses
.
GET
,
f
"
{
base_url
}
?with_count=true
"
,
status
=
500
,
)
# Prevent waiting for a random amount of time between retries
mocker
.
patch
(
"
arkindex.pagination.random.random
"
,
return_value
=
0.0001
)
cli
=
ArkindexClient
(
"
t0k3n
"
,
base_url
=
"
https://dummy.test
"
)
paginator
=
cli
.
paginate
(
"
ListProcessElements
"
,
id
=
"
process_id
"
,
retries
=
5
)
with
pytest
.
raises
(
Exception
,
match
=
"
Stopping pagination as data will be incomplete
"
):
len
(
paginator
)
assert
len
(
responses
.
calls
)
==
6
def
test_paginate_x_paginated
(
mock_schema
):
responses
.
add
(
responses
.
GET
,
...
...
This diff is collapsed.
Click to expand it.
tox.ini
+
1
−
0
View file @
fe535f20
...
...
@@ -9,3 +9,4 @@ commands =
deps
=
pytest
pytest-responses
pytest-mock
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment