Merge branch 'remove-ml-tools' into 'master'

Remove ML tools from files import

See merge request !792

arkindex/dataimport/api.py

@@ -187,7 +187,6 @@ class DataImportFromFiles(CreateAPIView):
         folder_name = serializer.validated_data.get('folder_name')
         folder_type_slug = serializer.validated_data.get('folder_type')
         element_type_slug = serializer.validated_data.get('element_type')
-        ml_tools = serializer.validated_data.get('ml_tools')
 
         if folder:
             # The files' corpus is already validated as writable
@@ -209,8 +208,6 @@ class DataImportFromFiles(CreateAPIView):
         payload = {
             'folder_type': folder_type_slug,
             'element_type': element_type_slug,
-            # Re-serialize ML tools as JSON
-            'ml_tools': MLToolTaskSerializer(ml_tools, many=True).data,
         }
         if folder:
             payload['folder_id'] = str(folder.id)

arkindex/dataimport/serializers/imports.py

@@ -96,7 +96,6 @@ class DataImportFromFilesSerializer(serializers.Serializer):
     folder_name = serializers.CharField(max_length=250, required=False, allow_null=True)
     folder_type = serializers.SlugField(required=False, allow_null=True)
     element_type = serializers.SlugField()
-    ml_tools = MLToolTaskSerializer(many=True, required=False, allow_null=True)
 
     default_error_messages = {
         'mode_not_allowed': 'This mode is not allowed when importing from files',

arkindex/dataimport/tests/test_imports.py

-from unittest.mock import patch, call, MagicMock
 from django.urls import reverse
 from rest_framework import status
-from arkindex_common.ml_tool import MLToolType
 from arkindex_common.enums import DataImportMode
 from arkindex.dataimport.models import DataImport, DataFile
 from arkindex.documents.models import Element, Corpus
@@ -289,7 +287,6 @@ class TestImports(FixtureAPITestCase):
         self.assertDictEqual(dataimport.payload, {
             'folder_type': 'volume',
             'element_type': 'page',
-            'ml_tools': [],
         })
 
     def test_from_files_invalid_mode(self):
@@ -389,48 +386,3 @@ class TestImports(FixtureAPITestCase):
             'element_type': 'page',
         }, format='json')
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
-    @patch('arkindex_common.ml_tool.MLTool.get')
-    def test_from_files_set_tools(self, ml_get_mock):
-        classifier_mock, recognizer_mock = MagicMock(), MagicMock()
-        classifier_mock.type, classifier_mock.slug = MLToolType.Classifier, 'someclassifier'
-        recognizer_mock.type, recognizer_mock.slug = MLToolType.Recognizer, 'somerecognizer'
-        classifier_mock.get_task_recipe.return_value = {'image': 'some_image1', 'command': 'some_command1'}
-        recognizer_mock.get_task_recipe.return_value = {'image': 'some_image2', 'command': 'some_command2'}
-        tools = {'someclassifier': classifier_mock, 'somerecognizer': recognizer_mock}
-        ml_get_mock.side_effect = lambda directory, type, slug: tools[slug]
-
-        self.client.force_login(self.user)
-
-        with self.settings(ML_CLASSIFIERS_DIR='/somewhere'):
-            response = self.client.post(reverse('api:import-from-files'), {
-                'files': [str(self.img_df.id)],
-                'folder_type': 'volume',
-                'element_type': 'page',
-                'ml_tools': [
-                    {'type': 'classifier', 'slug': 'someclassifier'},
-                    {
-                        'type': 'recognizer',
-                        'slug': 'somerecognizer',
-                        'parents': [{'type': 'classifier', 'slug': 'someclassifier'}],
-                    },
-                ],
-            }, format='json')
-
-        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
-        data = response.json()
-        dataimport = DataImport.objects.get(id=data['id'])
-        self.assertListEqual(dataimport.payload.get('ml_tools'), [
-            {'type': 'classifier', 'slug': 'someclassifier', 'parents': []},
-            {
-                'type': 'recognizer',
-                'slug': 'somerecognizer',
-                'parents': [{'type': 'classifier', 'slug': 'someclassifier'}],
-            },
-        ])
-        self.assertEqual(ml_get_mock.call_count, 6)
-        self.assertListEqual(ml_get_mock.call_args_list, [
-            call('/somewhere', MLToolType.Classifier, 'someclassifier'),
-            call('/somewhere', MLToolType.Classifier, 'someclassifier'),
-            call('/somewhere', MLToolType.Recognizer, 'somerecognizer'),
-        ] * 2)