Move retried_request from the toolbox

Closes #38 (closed)

I added the retry mechanism directly on the request function so that the migration is transparent (we don't have to change all the client.request to client.retried_request), as we use this system everywhere (callico, base worker, cli, scrapers etc.).

arkindex/client.py

@@ -11,6 +11,14 @@ from urllib.parse import urljoin, urlsplit, urlunsplit
 import apistar
 import requests
 import yaml
+from apistar.exceptions import ErrorResponse
+from tenacity import (
+    before_sleep_log,
+    retry,
+    retry_if_exception,
+    stop_after_attempt,
+    wait_exponential,
+)
 
 from arkindex.auth import TokenSessionAuthentication
 from arkindex.exceptions import SchemaError
@@ -27,6 +35,17 @@ SCHEMA_ENDPOINT = "/api/v1/openapi/?format=json"
 logger = logging.getLogger(__name__)
 
 
+def _is_500_error(exc: Exception) -> bool:
+    """
+    Check if an Arkindex API error is a 50x
+    This is used to retry most API calls implemented here
+    """
+    if not isinstance(exc, ErrorResponse):
+        return False
+
+    return 500 <= exc.status_code < 600
+
+
 def options_from_env():
     """
     Get API client keyword arguments from environment variables.
@@ -84,7+103,7 @@
         schema_url=None,
         csrf_cookie=None,
         sleep=0,
         verify=True,
         **kwargs,
     ):
         r"""
@@ -245,7+264,7 @@
             self.transport.session.auth.token = resp["auth_token"]
         return resp
 
-    def request(self, operation_id, *args, **kwargs):
+    def single_request(self, operation_id, *args, **kwargs):
         """
         Perform an API request.
         :param args: Arguments passed to the APIStar client.
@@ -274,3 +293,23 @@ class ArkindexClient(apistar.Client):
             )
             sleep(self.sleep_duration)
         return super().request(operation_id, *args, **kwargs)
+
+    @retry(
+        retry=retry_if_exception(_is_500_error),
+        wait=wait_exponential(multiplier=2, min=3),
+        reraise=True,
+        stop=stop_after_attempt(5),
+        before_sleep=before_sleep_log(logger, logging.INFO),
+    )
+    def request(self, operation_id, *args, **kwargs):
+        """
+        Proxy all Arkindex API requests with a retry mechanism in case of 50X errors.
+        The same API call will be retried 5 times, with an exponential sleep time
+        going through 3, 4, 8 and 16 seconds of wait between call.
+        If the 5th call still gives a 50x, the exception is re-raised and the caller should catch it.
+        Log messages are displayed before sleeping (when at least one exception occurred).
+
+        :param args: Arguments passed to the APIStar client.
+        :param kwargs: Keyword arguments passed to the APIStar client.
+        """
+        return self.single_request(operation_id, *args, **kwargs)