diff --git a/arkindex/process/builder.py b/arkindex/process/builder.py index 376d4b3d5b22502b853d51626436f1cc86f6541a..76458142ce7590f0e6909debe990c8989110a28a 100644 --- a/arkindex/process/builder.py +++ b/arkindex/process/builder.py @@ -319,6 +319,8 @@ class ProcessBuilder(object): f' --page-type={shlex.quote(self.process.element_type.slug)}' f' --iiif-base-url={shlex.quote(ImageServer.objects.ingest.url)}' ) + if settings.INGEST_PREFIX_BY_BUCKET_NAME: + command += ' --bucket-prefix' if self.process.prefix: command += f' --prefix={shlex.quote(self.process.prefix)}' if self.process.element: diff --git a/arkindex/process/tests/test_create_s3_import.py b/arkindex/process/tests/test_create_s3_import.py index b8c9565e40d7e30e3878d2490270e4e3f11a778b..1230c71934a995928684533b2abf0c9eccba10db 100644 --- a/arkindex/process/tests/test_create_s3_import.py +++ b/arkindex/process/tests/test_create_s3_import.py @@ -124,6 +124,7 @@ class TestCreateS3Import(FixtureTestCase): INGEST_S3_REGION=None, INGEST_S3_ACCESS_KEY='🔑', INGEST_S3_SECRET_KEY='its-secret-i-wont-tell-you', + INGEST_PREFIX_BY_BUCKET_NAME=True, ) def test_create(self): self.user.user_scopes.create(scope=Scope.S3Ingest) @@ -166,8 +167,8 @@ class TestCreateS3Import(FixtureTestCase): self.assertEqual(task.image, 'arkindex-tasks-image') self.assertEqual(task.command, f'python -m arkindex_tasks.import_s3 --corpus={self.corpus.id} ' '--bucket=blah --folder-type=volume --page-type=page ' - '--iiif-base-url=https://dev.null.teklia.com --prefix=a/b/c ' - f'--element={element.id}') + '--iiif-base-url=https://dev.null.teklia.com --bucket-prefix ' + f'--prefix=a/b/c --element={element.id}') self.assertDictEqual(task.env, { 'ARKINDEX_CORPUS_ID': str(self.corpus.id), 'ARKINDEX_PROCESS_ID': str(process.id), @@ -187,6 +188,7 @@ class TestCreateS3Import(FixtureTestCase): INGEST_S3_ACCESS_KEY='🔑', INGEST_S3_SECRET_KEY='its-secret-i-wont-tell-you', IMPORTS_WORKER_VERSION='aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', + INGEST_PREFIX_BY_BUCKET_NAME=True, ) def test_defaults(self): self.user.user_scopes.create(scope=Scope.S3Ingest) @@ -225,7 +227,8 @@ class TestCreateS3Import(FixtureTestCase): self.assertEqual(task.image, 'arkindex-tasks-image') self.assertEqual(task.command, f'python -m arkindex_tasks.import_s3 --corpus={self.corpus.id} ' '--bucket=blah --folder-type=folder --page-type=page ' - '--iiif-base-url=https://dev.null.teklia.com') + '--iiif-base-url=https://dev.null.teklia.com ' + '--bucket-prefix') self.assertDictEqual(task.env, { 'ARKINDEX_CORPUS_ID': str(self.corpus.id), 'ARKINDEX_PROCESS_ID': str(process.id), diff --git a/arkindex/project/config.py b/arkindex/project/config.py index 9d58928a7d547b9254ae85581367bcdcf09dc543..528cc9ccfc68ddbb82364f2c69b0ad5fee688fa3 100644 --- a/arkindex/project/config.py +++ b/arkindex/project/config.py @@ -224,6 +224,7 @@ def get_settings_parser(base_dir): ingest_parser = add_s3_parser(parser, 'ingest') ingest_parser.add_option('imageserver_id', type=int, default=None) ingest_parser.add_option('extra_buckets', type=str, many=True, default=[]) + ingest_parser.add_option('prefix_by_bucket_name', type=bool, default=True) license_parser = parser.add_subparser('license', default={}) license_parser.add_option('key', type=str, default=None) diff --git a/arkindex/project/settings.py b/arkindex/project/settings.py index c1706deb6032279bb6224c582673ba05ec30fcf5..7339c279353647f3f2c4f12977705a64dc34f997 100644 --- a/arkindex/project/settings.py +++ b/arkindex/project/settings.py @@ -497,6 +497,7 @@ INGEST_S3_ENDPOINT = conf['ingest']['endpoint'] INGEST_S3_REGION = conf['ingest']['region'] INGEST_IMAGESERVER_ID = conf['ingest']['imageserver_id'] INGEST_EXTRA_BUCKETS = conf['ingest']['extra_buckets'] +INGEST_PREFIX_BY_BUCKET_NAME = conf['ingest']['prefix_by_bucket_name'] # Ponos integration _ponos_env = { diff --git a/arkindex/project/tests/config_samples/defaults.yaml b/arkindex/project/tests/config_samples/defaults.yaml index 152282022bcf32f5f15390d73ab47bf29fcd227a..aee19f5bcfa4a3c09ae9c5fbf2aeab7a04a10bbe 100644 --- a/arkindex/project/tests/config_samples/defaults.yaml +++ b/arkindex/project/tests/config_samples/defaults.yaml @@ -48,6 +48,7 @@ ingest: endpoint: null extra_buckets: [] imageserver_id: null + prefix_by_bucket_name: true region: null secret_access_key: null internal_group_id: 2 diff --git a/arkindex/project/tests/config_samples/override.yaml b/arkindex/project/tests/config_samples/override.yaml index db5fb2a2e82cec1966fa33b0bfa70516b978f388..fd8c571f31373534e31395e165b33b9161168a66 100644 --- a/arkindex/project/tests/config_samples/override.yaml +++ b/arkindex/project/tests/config_samples/override.yaml @@ -62,6 +62,7 @@ ingest: - a - b imageserver_id: 999 + prefix_by_bucket_name: false region: middle-earth-1 secret_access_key: hunter2 internal_group_id: 4