From 32600e7c165afca2bc33cfcf255735a5474769f4 Mon Sep 17 00:00:00 2001
From: Erwan Rouchet <rouchet@teklia.com>
Date: Thu, 28 Nov 2024 10:00:00 +0100
Subject: [PATCH] Add an option for maximum retries on all S3 requests
---
arkindex/project/aws.py | 4 ++++
arkindex/project/config.py | 1 +
arkindex/project/settings.py | 1 +
arkindex/project/tests/config_samples/defaults.yaml | 1 +
arkindex/project/tests/config_samples/errors.yaml | 1 +
arkindex/project/tests/config_samples/expected_errors.yaml | 2 ++
arkindex/project/tests/config_samples/override.yaml | 1 +
7 files changed, 11 insertions(+)
diff --git a/arkindex/project/aws.py b/arkindex/project/aws.py
index 1def44b850..3245ed056e 100644
--- a/arkindex/project/aws.py
+++ b/arkindex/project/aws.py
@@ -30,6 +30,10 @@ def get_s3_resource(
signature_version="s3v4",
s3={
"addressing_style": "auto" if endpoint else "virtual",
+ },
+ retries={
+ "mode": "standard",
+ "max_attempts": settings.AWS_MAX_RETRIES,
}
)
diff --git a/arkindex/project/config.py b/arkindex/project/config.py
index 47a70f3801..edcb4e3a4f 100644
--- a/arkindex/project/config.py
+++ b/arkindex/project/config.py
@@ -209,6 +209,7 @@ def get_settings_parser(base_dir):
parser.add_option("cache", default={}, type=cache_validator)
s3_parser = add_s3_parser(parser, "s3")
+ s3_parser.add_option("max_retries", type=int, default=5)
s3_parser.add_option("thumbnails_bucket", type=str, default="thumbnails")
s3_parser.add_option("staging_bucket", type=str, default="staging")
s3_parser.add_option("export_bucket", type=str, default="export")
diff --git a/arkindex/project/settings.py b/arkindex/project/settings.py
index 1b2d950cbc..d3173a6245 100644
--- a/arkindex/project/settings.py
+++ b/arkindex/project/settings.py
@@ -528,6 +528,7 @@ AWS_ACCESS_KEY = conf["s3"]["access_key_id"]
AWS_SECRET_KEY = conf["s3"]["secret_access_key"]
AWS_ENDPOINT = conf["s3"]["endpoint"]
AWS_REGION = conf["s3"]["region"]
+AWS_MAX_RETRIES = conf["s3"]["max_retries"]
PONOS_S3_LOGS_BUCKET = conf["s3"]["ponos_logs_bucket"]
PONOS_S3_ARTIFACTS_BUCKET = conf["s3"]["ponos_artifacts_bucket"]
diff --git a/arkindex/project/tests/config_samples/defaults.yaml b/arkindex/project/tests/config_samples/defaults.yaml
index d2e108d675..24aa0d51d4 100644
--- a/arkindex/project/tests/config_samples/defaults.yaml
+++ b/arkindex/project/tests/config_samples/defaults.yaml
@@ -81,6 +81,7 @@ s3:
access_key_id: null
endpoint: null
export_bucket: export
+ max_retries: 5
ponos_artifacts_bucket: ponos-artifacts
ponos_logs_bucket: ponos-logs
region: null
diff --git a/arkindex/project/tests/config_samples/errors.yaml b/arkindex/project/tests/config_samples/errors.yaml
index 5925b919da..fd795eded5 100644
--- a/arkindex/project/tests/config_samples/errors.yaml
+++ b/arkindex/project/tests/config_samples/errors.yaml
@@ -62,6 +62,7 @@ redis:
timeout: sauce
s3:
endpoint: null
+ max_retries: [1, 2]
ponos_artifacts_bucket: {}
ponos_logs_bucket: null
region: null
diff --git a/arkindex/project/tests/config_samples/expected_errors.yaml b/arkindex/project/tests/config_samples/expected_errors.yaml
index b9ec8b030b..97cd216bd5 100644
--- a/arkindex/project/tests/config_samples/expected_errors.yaml
+++ b/arkindex/project/tests/config_samples/expected_errors.yaml
@@ -40,6 +40,8 @@ redis:
port: "invalid literal for int() with base 10: 'over nine thousand'"
db: "invalid literal for int() with base 10: 'idk'"
timeout: "invalid literal for int() with base 10: 'sauce'"
+s3:
+ max_retries: "int() argument must be a string, a bytes-like object or a real number, not 'list'"
session:
cookie_samesite: "'foo' is not a valid CookieSameSiteOption"
signup_default_group: "badly formed hexadecimal UUID string"
diff --git a/arkindex/project/tests/config_samples/override.yaml b/arkindex/project/tests/config_samples/override.yaml
index 7a807a1fef..1de12bd3e8 100644
--- a/arkindex/project/tests/config_samples/override.yaml
+++ b/arkindex/project/tests/config_samples/override.yaml
@@ -98,6 +98,7 @@ s3:
access_key_id: abcd
endpoint: http://somewhere
export_bucket: exbort
+ max_retries: 99
ponos_artifacts_bucket: zstandardland
ponos_logs_bucket: plaintexttown
region: middle-earth-1
--
GitLab