diff --git a/arkindex/project/aws.py b/arkindex/project/aws.py index 1def44b850a7fa57be8306dbbae99502ffa12200..3245ed056e0a6cb50ae67a26e7a3a79eeff2c377 100644 --- a/arkindex/project/aws.py +++ b/arkindex/project/aws.py @@ -30,6 +30,10 @@ def get_s3_resource( signature_version="s3v4", s3={ "addressing_style": "auto" if endpoint else "virtual", + }, + retries={ + "mode": "standard", + "max_attempts": settings.AWS_MAX_RETRIES, } ) diff --git a/arkindex/project/config.py b/arkindex/project/config.py index 47a70f380101a86397a550a4312a98e49eb46260..edcb4e3a4f736c881cd14a091047bf0707502569 100644 --- a/arkindex/project/config.py +++ b/arkindex/project/config.py @@ -209,6 +209,7 @@ def get_settings_parser(base_dir): parser.add_option("cache", default={}, type=cache_validator) s3_parser = add_s3_parser(parser, "s3") + s3_parser.add_option("max_retries", type=int, default=5) s3_parser.add_option("thumbnails_bucket", type=str, default="thumbnails") s3_parser.add_option("staging_bucket", type=str, default="staging") s3_parser.add_option("export_bucket", type=str, default="export") diff --git a/arkindex/project/settings.py b/arkindex/project/settings.py index 1b2d950cbca60e711c9e2d0427f38fe3c00e1075..d3173a6245291737df01f330537e19716a2d166f 100644 --- a/arkindex/project/settings.py +++ b/arkindex/project/settings.py @@ -528,6 +528,7 @@ AWS_ACCESS_KEY = conf["s3"]["access_key_id"] AWS_SECRET_KEY = conf["s3"]["secret_access_key"] AWS_ENDPOINT = conf["s3"]["endpoint"] AWS_REGION = conf["s3"]["region"] +AWS_MAX_RETRIES = conf["s3"]["max_retries"] PONOS_S3_LOGS_BUCKET = conf["s3"]["ponos_logs_bucket"] PONOS_S3_ARTIFACTS_BUCKET = conf["s3"]["ponos_artifacts_bucket"] diff --git a/arkindex/project/tests/config_samples/defaults.yaml b/arkindex/project/tests/config_samples/defaults.yaml index d2e108d675a9157f0a4040e4126485c1fd72e4d6..24aa0d51d4b66cab220df1ad131648ab26039348 100644 --- a/arkindex/project/tests/config_samples/defaults.yaml +++ b/arkindex/project/tests/config_samples/defaults.yaml @@ -81,6 +81,7 @@ s3: access_key_id: null endpoint: null export_bucket: export + max_retries: 5 ponos_artifacts_bucket: ponos-artifacts ponos_logs_bucket: ponos-logs region: null diff --git a/arkindex/project/tests/config_samples/errors.yaml b/arkindex/project/tests/config_samples/errors.yaml index 5925b919da4f933f8c94df033bc368dc81a6e325..fd795eded5eb97f408d21b5263afabbb62e8f64e 100644 --- a/arkindex/project/tests/config_samples/errors.yaml +++ b/arkindex/project/tests/config_samples/errors.yaml @@ -62,6 +62,7 @@ redis: timeout: sauce s3: endpoint: null + max_retries: [1, 2] ponos_artifacts_bucket: {} ponos_logs_bucket: null region: null diff --git a/arkindex/project/tests/config_samples/expected_errors.yaml b/arkindex/project/tests/config_samples/expected_errors.yaml index b9ec8b030b4281343607231044d324cfa23d5dc3..97cd216bd5e9e28b0c3db8db72e49faa75379c7c 100644 --- a/arkindex/project/tests/config_samples/expected_errors.yaml +++ b/arkindex/project/tests/config_samples/expected_errors.yaml @@ -40,6 +40,8 @@ redis: port: "invalid literal for int() with base 10: 'over nine thousand'" db: "invalid literal for int() with base 10: 'idk'" timeout: "invalid literal for int() with base 10: 'sauce'" +s3: + max_retries: "int() argument must be a string, a bytes-like object or a real number, not 'list'" session: cookie_samesite: "'foo' is not a valid CookieSameSiteOption" signup_default_group: "badly formed hexadecimal UUID string" diff --git a/arkindex/project/tests/config_samples/override.yaml b/arkindex/project/tests/config_samples/override.yaml index 7a807a1fef29b69b3b84ce3a76a4bddb4d682e34..1de12bd3e8d5ab4153378b78a765ef46c79f682b 100644 --- a/arkindex/project/tests/config_samples/override.yaml +++ b/arkindex/project/tests/config_samples/override.yaml @@ -98,6 +98,7 @@ s3: access_key_id: abcd endpoint: http://somewhere export_bucket: exbort + max_retries: 99 ponos_artifacts_bucket: zstandardland ponos_logs_bucket: plaintexttown region: middle-earth-1