Make the Artifact maximum size configurable

arkindex/project/config.py

@@ -178,6 +178,7 @@ def get_settings_parser(base_dir):
     ponos_parser.add_option('private_key', type=Path, default=(base_dir / 'ponos.key').resolve())
     ponos_parser.add_option('default_env', type=dict, default={})
     ponos_parser.add_option('default_farm', type=uuid.UUID, default=None)
+    ponos_parser.add_option('artifact_max_size', type=int, default=5 * 1024**3)
 
     docker_parser = parser.add_subparser('docker', default={})
     docker_parser.add_option('tasks_image', type=str, default='registry.gitlab.com/teklia/arkindex/tasks')

arkindex/project/settings.py

@@ -502,6 +502,7 @@ PONOS_RECIPE = {
 }
 PONOS_PRIVATE_KEY = conf['ponos']['private_key']
 PONOS_DEFAULT_FARM = conf['ponos']['default_farm']
+PONOS_ARTIFACT_MAX_SIZE = conf['ponos']['artifact_max_size']
 
 # Docker images used by our ponos workflow
 ARKINDEX_TASKS_IMAGE = conf['docker']['tasks_image']

arkindex/project/tests/config_samples/defaults.yaml

@@ -61,6 +61,7 @@ job_timeouts:
 jwt_signing_key: null
 local_imageserver_id: 1
 ponos:
+  artifact_max_size: 5368709120
   default_env: {}
   default_farm: null
   private_key: /somewhere/backend/arkindex/ponos.key

arkindex/project/tests/config_samples/errors.yaml

@@ -44,6 +44,7 @@ job_timeouts:
 jwt_signing_key: null
 local_imageserver_id: 1
 ponos:
+  artifact_max_size: .nan
   default_env: {}
   private_key: /dev/zero
 public_hostname: darkindex.lol

arkindex/project/tests/config_samples/expected_errors.yaml

@@ -22,6 +22,8 @@ job_timeouts:
   export_corpus: "int() argument must be a string, a bytes-like object or a number, not 'list'"
   move_element: "int() argument must be a string, a bytes-like object or a number, not 'dict'"
   worker_results_delete: "int() argument must be a string, a bytes-like object or a number, not 'NoneType'"
+ponos:
+  artifact_max_size: cannot convert float NaN to integer
 public_hostname: The hostname must include an HTTP or HTTPS scheme.
 redis:
   port: "invalid literal for int() with base 10: 'over nine thousand'"

arkindex/project/tests/config_samples/override.yaml

@@ -75,6 +75,7 @@ job_timeouts:
 jwt_signing_key: deadbeef
 local_imageserver_id: 45
 ponos:
+  artifact_max_size: 12345678901234567890
   default_env:
     A: B
   default_farm: aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa