diff --git a/arkindex/dataimport/models.py b/arkindex/dataimport/models.py index c95b4915bba057fa4560dc6d0d147bb73b0f1321..5e0a19dd5e294be07d6198d18395868b32d1758a 100644 --- a/arkindex/dataimport/models.py +++ b/arkindex/dataimport/models.py @@ -218,8 +218,7 @@ class DataImport(IndexableModel): elif self.mode == DataImportMode.Workers: if chunks is not None: - assert isinstance(chunks, int) and chunks > 0, 'Chunks count should be an positive integer' - assert chunks < 11, 'Import distribution is limited to 10 chunks' + assert chunks <= settings.MAX_CHUNKS, f'Import distribution is limited to {settings.MAX_CHUNKS} chunks' ml_workflow_chunks = chunks args = [ diff --git a/arkindex/dataimport/serializers/imports.py b/arkindex/dataimport/serializers/imports.py index 0d029b338d575e1ec589bd3dd6963f46b8dafdd6..3f3b231ac80aaba94ff1ed138431cd6f2f9ef64a 100644 --- a/arkindex/dataimport/serializers/imports.py +++ b/arkindex/dataimport/serializers/imports.py @@ -195,7 +195,7 @@ class DataImportFromFilesSerializer(serializers.Serializer): class StartProcessSerializer(serializers.Serializer): - chunks = serializers.IntegerField(min_value=1, max_value=10, default=1) + chunks = serializers.IntegerField(min_value=1, max_value=settings.MAX_CHUNKS, default=1) thumbnails = serializers.BooleanField(default=False) farm = serializers.PrimaryKeyRelatedField(queryset=Farm.objects.all(), required=False, allow_null=True) diff --git a/arkindex/dataimport/tests/test_imports.py b/arkindex/dataimport/tests/test_imports.py index 9781ef323bd3a174b0a8ba24c7e90620e889c509..edb2061d89ce6927e1da4aa2a2f6c3b08c06d5b8 100644 --- a/arkindex/dataimport/tests/test_imports.py +++ b/arkindex/dataimport/tests/test_imports.py @@ -1,3 +1,5 @@ + +import importlib import uuid from unittest.mock import MagicMock, patch @@ -6,6 +8,7 @@ from django.conf import settings from django.test import override_settings from django.urls import reverse from rest_framework import status +from rest_framework.exceptions import ValidationError from arkindex.dataimport.models import DataImport, DataImportMode, RepositoryType from arkindex.dataimport.utils import get_default_farm_id @@ -1028,6 +1031,27 @@ class TestImports(FixtureAPITestCase): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertDictEqual(response.json(), check) + @override_settings(MAX_CHUNKS=42) + def test_start_process_serializer_chunks_config(self): + """ + Assert the start process serializer use configuration max chunks + """ + # Reloading serializer classes is required to update fields attributes from settings + from arkindex.dataimport.serializers import imports + imports = importlib.reload(imports) + + params_checks = [ + ({'chunks': 42}, True, None), + ({'chunks': 43}, False, {'chunks': ['Ensure this value is less than or equal to 42.']}), + ] + for (params, is_valid, message) in params_checks: + serializer = imports.StartProcessSerializer(data=params) + if not is_valid: + with self.assertRaisesRegex(ValidationError, 'Ensure this value is less than or equal to 42.'): + self.assertEqual(serializer.is_valid(raise_exception=True)) + else: + self.assertEqual(serializer.is_valid(), True) + @patch('arkindex.dataimport.models.DataImport.worker_runs') def test_start_process_parameters(self, worker_runs_mock): """ diff --git a/arkindex/project/config.py b/arkindex/project/config.py index fb0a49b3981db7aa77936a3f512c6d596cd43991..f87b1de1ec364e27bec376549f334f009310db25 100644 --- a/arkindex/project/config.py +++ b/arkindex/project/config.py @@ -36,6 +36,7 @@ def get_settings_parser(base_dir): parser.add_option('local_imageserver_id', type=int, default=1) parser.add_option('allowed_hosts', type=str, many=True, default=[]) parser.add_option('imports_worker_version', type=uuid.UUID, default=None) + parser.add_option('workers_max_chunks', type=int, default=10) # SECURITY WARNING: keep the secret key used in production secret! parser.add_option('secret_key', type=str, default='jf0w^y&ml(caax8f&a1mub)(js9(l5mhbbhosz3gi+m01ex+lo') diff --git a/arkindex/project/settings.py b/arkindex/project/settings.py index e8eb5ad49c401d59f7205255e60b0119febf8cb3..c80afbb1a8b155139a24566a3a66b0e4094246b0 100644 --- a/arkindex/project/settings.py +++ b/arkindex/project/settings.py @@ -511,7 +511,10 @@ PONOS_DEFAULT_FARM = conf['ponos']['default_farm'] # Docker images used by our ponos workflow ARKINDEX_TASKS_IMAGE = conf['docker']['tasks_image'] + +# Processes options IMPORTS_WORKER_VERSION = conf['imports_worker_version'] +MAX_CHUNKS = conf['workers_max_chunks'] # User groups with special permissions # Deprecated, left there only to run the users.0004 migration diff --git a/arkindex/project/tests/config_samples/defaults.yaml b/arkindex/project/tests/config_samples/defaults.yaml index 932a6dfe505707648644984eacb93fb48be2a862..23f9dc18285a09b8794fab9b44e4bcceb02ec982 100644 --- a/arkindex/project/tests/config_samples/defaults.yaml +++ b/arkindex/project/tests/config_samples/defaults.yaml @@ -85,3 +85,4 @@ transkribus: email: null password: null worker_version: null +workers_max_chunks: 10 diff --git a/arkindex/project/tests/config_samples/override.yaml b/arkindex/project/tests/config_samples/override.yaml index d9a00de8568d0606eb1f441bd91f55de45cdbd9d..896cb7a0c7401c85881256307d11c74e196dad56 100644 --- a/arkindex/project/tests/config_samples/override.yaml +++ b/arkindex/project/tests/config_samples/override.yaml @@ -100,3 +100,4 @@ transkribus: email: nope@nope password: superSecret worker_version: aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa +workers_max_chunks: 42