Add setting to prefix or not image paths with bucket name in s3 import

arkindex/process/builder.py

@@ -319,6 +319,8 @@ class ProcessBuilder(object):
             f' --page-type={shlex.quote(self.process.element_type.slug)}'
             f' --iiif-base-url={shlex.quote(ImageServer.objects.ingest.url)}'
         )
+        if settings.INGEST_PREFIX_BY_BUCKET_NAME:
+            command += ' --bucket-prefix'
         if self.process.prefix:
             command += f' --prefix={shlex.quote(self.process.prefix)}'
         if self.process.element:

arkindex/process/tests/test_create_s3_import.py

@@ -124,6 +124,7 @@ class TestCreateS3Import(FixtureTestCase):
         INGEST_S3_REGION=None,
         INGEST_S3_ACCESS_KEY='🔑',
         INGEST_S3_SECRET_KEY='its-secret-i-wont-tell-you',
+        INGEST_PREFIX_BY_BUCKET_NAME=True,
     )
     def test_create(self):
         self.user.user_scopes.create(scope=Scope.S3Ingest)
@@ -166,8 +167,8 @@ class TestCreateS3Import(FixtureTestCase):
         self.assertEqual(task.image, 'arkindex-tasks-image')
         self.assertEqual(task.command, f'python -m arkindex_tasks.import_s3 --corpus={self.corpus.id} '
                                        '--bucket=blah --folder-type=volume --page-type=page '
-                                       '--iiif-base-url=https://dev.null.teklia.com --prefix=a/b/c '
-                                       f'--element={element.id}')
+                                       '--iiif-base-url=https://dev.null.teklia.com --bucket-prefix '
+                                       f'--prefix=a/b/c --element={element.id}')
         self.assertDictEqual(task.env, {
             'ARKINDEX_CORPUS_ID': str(self.corpus.id),
             'ARKINDEX_PROCESS_ID': str(process.id),
@@ -187,6 +188,7 @@ class TestCreateS3Import(FixtureTestCase):
         INGEST_S3_ACCESS_KEY='🔑',
         INGEST_S3_SECRET_KEY='its-secret-i-wont-tell-you',
         IMPORTS_WORKER_VERSION='aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa',
+        INGEST_PREFIX_BY_BUCKET_NAME=True,
     )
     def test_defaults(self):
         self.user.user_scopes.create(scope=Scope.S3Ingest)
@@ -225,7 +227,8 @@ class TestCreateS3Import(FixtureTestCase):
         self.assertEqual(task.image, 'arkindex-tasks-image')
         self.assertEqual(task.command, f'python -m arkindex_tasks.import_s3 --corpus={self.corpus.id} '
                                        '--bucket=blah --folder-type=folder --page-type=page '
-                                       '--iiif-base-url=https://dev.null.teklia.com')
+                                       '--iiif-base-url=https://dev.null.teklia.com '
+                                       '--bucket-prefix')
         self.assertDictEqual(task.env, {
             'ARKINDEX_CORPUS_ID': str(self.corpus.id),
             'ARKINDEX_PROCESS_ID': str(process.id),

arkindex/project/config.py

@@ -224,6 +224,7 @@ def get_settings_parser(base_dir):
     ingest_parser = add_s3_parser(parser, 'ingest')
     ingest_parser.add_option('imageserver_id', type=int, default=None)
     ingest_parser.add_option('extra_buckets', type=str, many=True, default=[])
+    ingest_parser.add_option('prefix_by_bucket_name', type=bool, default=True)
 
     license_parser = parser.add_subparser('license', default={})
     license_parser.add_option('key', type=str, default=None)

arkindex/project/settings.py

@@ -497,6 +497,7 @@ INGEST_S3_ENDPOINT = conf['ingest']['endpoint']
 INGEST_S3_REGION = conf['ingest']['region']
 INGEST_IMAGESERVER_ID = conf['ingest']['imageserver_id']
 INGEST_EXTRA_BUCKETS = conf['ingest']['extra_buckets']
+INGEST_PREFIX_BY_BUCKET_NAME = conf['ingest']['prefix_by_bucket_name']
 
 # Ponos integration
 _ponos_env = {

arkindex/project/tests/config_samples/defaults.yaml

@@ -48,6 +48,7 @@ ingest:
   endpoint: null
   extra_buckets: []
   imageserver_id: null
+  prefix_by_bucket_name: true
   region: null
   secret_access_key: null
 internal_group_id: 2

arkindex/project/tests/config_samples/override.yaml

@@ -62,6 +62,7 @@ ingest:
   - a
   - b
   imageserver_id: 999
+  prefix_by_bucket_name: false
   region: middle-earth-1
   secret_access_key: hunter2
 internal_group_id: 4