Support feature worker declaration from gitlab.teklia.com

Closes #1883 (closed)

• Requires tests

arkindex/process/tests/commands/test_update_system_workers.py

@@ -2,13 +2,16 @@ from io import StringIO
 from textwrap import dedent
 from unittest.mock import patch
 
+import responses
 from django.core.management import CommandError, call_command
 from django.utils import timezone
 
 from arkindex.process.models import ArkindexFeature, FeatureUsage, Worker, WorkerType, WorkerVersion, WorkerVersionState
 from arkindex.project.tests import ArkindexTestCase
 
-MOCK_CONFIG = {
+
+def mock_config():
+    return {
     "features": {
         "file_import": {"image": "registry.example.com/file-import:latest", "command": None},
         "init_elements":
@@ -19,6 +22,19 @@ MOCK_CONFIG = {
     }
 }
 
+def mock_teklia_worker_config():
+    return {
+        "features": {
+            "file_import": {
+                "teklia_worker": {
+                    "name": "repository/import/file",
+                    "version": "0.1.0",
+                    "slug": "file-import",
+                }
+            }
+        }
+    }
+
 
 class TestUpdateSystemWorkers(ArkindexTestCase):
 
@@ -41,8 +57,8 @@ class TestUpdateSystemWorkers(ArkindexTestCase):
         Assert that a WorkerVersion has the expected attributes to provide a given feature
         """
         self.assertEqual(worker_version.feature, feature)
-        self.assertEqual(worker_version.docker_image_iid, MOCK_CONFIG["features"][feature.value]["image"])
-        self.assertEqual(worker_version.docker_command, MOCK_CONFIG["features"][feature.value]["command"])
+        self.assertEqual(worker_version.docker_image_iid, mock_config()["features"][feature.value]["image"])
+        self.assertEqual(worker_version.docker_command, mock_config()["features"][feature.value]["command"])
         self.assertEqual(worker_version.state, WorkerVersionState.Available)
         self.assertEqual(worker_version.required_user_configuration_fields, set())
         self.assertEqual(worker_version.gpu_usage, FeatureUsage.Disabled)
@@ -51,7 +67,7 @@ class TestUpdateSystemWorkers(ArkindexTestCase):
         self.assertIsNone(worker_version.worker.archived)
         self.assertTrue(worker_version.worker.public)
 
-    @patch("arkindex.process.management.commands.update_system_workers.parse_config", lambda: MOCK_CONFIG)
+    @patch("arkindex.process.management.commands.update_system_workers.parse_config", mock_config)
     def test_from_scratch(self):
         output = self.update_system_workers()
 
@@ -122,7 +138,7 @@ class TestUpdateSystemWorkers(ArkindexTestCase):
             """
         ).strip())
 
-    @patch("arkindex.process.management.commands.update_system_workers.parse_config", lambda: MOCK_CONFIG)
+    @patch("arkindex.process.management.commands.update_system_workers.parse_config", mock_config)
     def test_existing_worker_type(self):
         worker_type = WorkerType.objects.create(slug="system", display_name="Système")
 
@@ -189,7 +205,7 @@ class TestUpdateSystemWorkers(ArkindexTestCase):
             """
         ).strip())
 
-    @patch("arkindex.process.management.commands.update_system_workers.parse_config", lambda: MOCK_CONFIG)
+    @patch("arkindex.process.management.commands.update_system_workers.parse_config", mock_config)
     def test_existing_system_workers(self):
         worker_type = WorkerType.objects.create(slug="system", display_name="Système")
         file_import_worker = Worker.objects.create(
@@ -283,7 +299,7 @@ class TestUpdateSystemWorkers(ArkindexTestCase):
             """
         ).strip())
 
-    @patch("arkindex.process.management.commands.update_system_workers.parse_config", lambda: MOCK_CONFIG)
+    @patch("arkindex.process.management.commands.update_system_workers.parse_config", mock_config)
     def test_worker_slug_conflict(self):
         worker_type = WorkerType.objects.create(slug="systemnt", display_name="Systemn't")
 
@@ -311,7 +327,7 @@ class TestUpdateSystemWorkers(ArkindexTestCase):
             """
         ).strip())
 
-    @patch("arkindex.process.management.commands.update_system_workers.parse_config", lambda: MOCK_CONFIG)
+    @patch("arkindex.process.management.commands.update_system_workers.parse_config", mock_config)
     def test_update_existing_system_workers(self):
         worker_type = WorkerType.objects.create(slug="system", display_name="Système")
         file_import_worker = Worker.objects.create(
@@ -413,7 +429,7 @@ class TestUpdateSystemWorkers(ArkindexTestCase):
             """
         ).strip())
 
-    @patch("arkindex.process.management.commands.update_system_workers.parse_config", lambda: MOCK_CONFIG)
+    @patch("arkindex.process.management.commands.update_system_workers.parse_config", mock_config)
     def test_assigns_existing_compatible_versions(self):
         """
         The command should assign the feature to an existing WorkerVersion compatible with it,
@@ -576,7 +592,7 @@ class TestUpdateSystemWorkers(ArkindexTestCase):
         pagexml_export_version.refresh_from_db()
         self.check_feature_version(pagexml_export_version, ArkindexFeature.ExportPageXML)
 
-    @patch("arkindex.process.management.commands.update_system_workers.parse_config", lambda: MOCK_CONFIG)
+    @patch("arkindex.process.management.commands.update_system_workers.parse_config", mock_config)
     def test_unassigns_incompatible_versions(self):
         worker_type = WorkerType.objects.create(slug="systemnt", display_name="Systemn't")
         file_import_worker = Worker.objects.create(
@@ -698,7 +714,7 @@ class TestUpdateSystemWorkers(ArkindexTestCase):
             """
         ).strip())
 
-    @patch("arkindex.process.management.commands.update_system_workers.parse_config", lambda: MOCK_CONFIG)
+    @patch("arkindex.process.management.commands.update_system_workers.parse_config", mock_config)
     def test_noop(self):
         worker_type = WorkerType.objects.create(slug="system", display_name="Système")
         worker = Worker.objects.create(type=worker_type, slug="worker", name="Worker", public=True)
@@ -790,3 +806,137 @@ class TestUpdateSystemWorkers(ArkindexTestCase):
         self.check_feature_version(pdf_export_version, ArkindexFeature.ExportPDF)
         pagexml_export_version.refresh_from_db()
         self.check_feature_version(pagexml_export_version, ArkindexFeature.ExportPageXML)
+
+    @responses.activate
+    @patch("arkindex.process.management.commands.update_system_workers.parse_config", mock_teklia_worker_config)
+    def test_teklia_worker_invalid_slug(self):
+        responses.add(
+            responses.GET,
+            "https://gitlab.teklia.com/repository/import/file/-/raw/0.1.0/.arkindex.yml",
+            json={"version": 2, "workers": [{"slug": "no"}]},
+          )
+        with self.assertRaisesMessage(CommandError, (
+            "No worker with slug file-import in .arkindex.yml at "
+            "https://gitlab.teklia.com/repository/import/file/-/raw/0.1.0/.arkindex.yml."
+        )):
+            self.update_system_workers()
+
+    @responses.activate
+    @patch("arkindex.process.management.commands.update_system_workers.parse_config", mock_teklia_worker_config)
+    def test_teklia_worker_invalid_yaml(self):
+        responses.add(
+            responses.GET,
+            "https://gitlab.teklia.com/repository/import/file/-/raw/0.1.0/.arkindex.yml",
+            json={"version": 42}
+          )
+        with self.assertRaisesMessage(CommandError, (
+            "Error retrieving configuration at "
+            "https://gitlab.teklia.com/repository/import/file/-/raw/0.1.0/.arkindex.yml: "
+            "invalid YAML configuration"
+        )):
+            self.update_system_workers()
+
+    @responses.activate
+    @patch("arkindex.process.management.commands.update_system_workers.parse_config", mock_teklia_worker_config)
+    def test_teklia_worker_from_scratch(self):
+        responses.add(
+            responses.GET,
+            "https://gitlab.teklia.com/repository/import/file/-/raw/0.1.0/.arkindex.yml",
+            json={
+                "version": 2,
+                "workers": [{
+                    "slug": "file-import",
+                    "name": "File Import",
+                    "type": "import_type",
+                    "description": "test",
+                }],
+            },
+          )
+
+        with self.assertRaises(WorkerVersion.DoesNotExist):
+            WorkerVersion.objects.get_by_feature(ArkindexFeature.FileImport)
+
+        output = self.update_system_workers()
+        worker_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.FileImport)
+        self.assertEqual(output, dedent(
+            f"""
+            ────────────────────────────────── FileImport ──────────────────────────────────
+            Configuring feature Fileimport from https://gitlab.teklia.com/repository/import/file
+            Using registry.gitlab.teklia.com/repository/import/file/0.1.0 to provide FileImport
+            Created new System worker type ({worker_version.worker.type_id})
+            Created new FileImport system worker
+            Creating new worker version
+            Using new worker version {worker_version.id}
+            """
+        ).strip())
+        self.assertEqual(worker_version.configuration, {
+            "description": "test",
+            "name": "File Import",
+            "slug": "file-import",
+            "type": "import_type",
+        })
+        self.assertEqual(worker_version.docker_image_iid, "registry.gitlab.teklia.com/repository/import/file/0.1.0")
+        self.assertEqual(worker_version.feature, ArkindexFeature.FileImport)
+        self.assertEqual(worker_version.state, WorkerVersionState.Available)
+        self.assertEqual(worker_version.version, 1)
+        self.assertEqual(worker_version.worker.name, "FileImport")
+        self.assertEqual(worker_version.worker.repository_url, "https://gitlab.teklia.com/repository/import/file")
+
+    @responses.activate
+    @patch("arkindex.process.management.commands.update_system_workers.parse_config", mock_teklia_worker_config)
+    def test_teklia_worker_unassign(self):
+        worker_type = WorkerType.objects.create(slug="system", display_name="Système")
+        file_import_worker = Worker.objects.create(
+            type=worker_type,
+            slug="file_import",
+            name="Sir File-a-Lot",
+            public=True,
+        )
+        version = file_import_worker.versions.create(
+            version=1,
+            docker_image_iid="test",
+            state=WorkerVersionState.Available,
+            feature=ArkindexFeature.FileImport,
+        )
+        responses.add(
+            responses.GET,
+            "https://gitlab.teklia.com/repository/import/file/-/raw/0.1.0/.arkindex.yml",
+            json={
+                "version": 2,
+                "workers": [{
+                    "slug": "file-import",
+                    "name": "File Import",
+                    "type": "import_type",
+                    "description": "test",
+                }],
+            },
+          )
+
+        output = self.update_system_workers()
+        worker_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.FileImport)
+        self.assertEqual(output, dedent(
+            f"""
+            ────────────────────────────────── FileImport ──────────────────────────────────
+            Configuring feature Fileimport from https://gitlab.teklia.com/repository/import/file
+            Using registry.gitlab.teklia.com/repository/import/file/0.1.0 to provide FileImport
+            Current worker version: {version.id} (test)
+            This version has an invalid Docker image.
+            This version uses a custom configuration which could interfere with the feature.
+            Unassigning feature from the current version
+            Worker is up to date
+            Creating new worker version
+            Using new worker version {worker_version.id}
+            """
+        ).strip())
+        self.assertEqual(worker_version.configuration, {
+            "description": "test",
+            "name": "File Import",
+            "slug": "file-import",
+            "type": "import_type",
+        })
+        self.assertEqual(worker_version.docker_image_iid, "registry.gitlab.teklia.com/repository/import/file/0.1.0")
+        self.assertEqual(worker_version.feature, ArkindexFeature.FileImport)
+        self.assertEqual(worker_version.state, WorkerVersionState.Available)
+        self.assertEqual(worker_version.version, 2)
+        self.assertEqual(worker_version.worker.name, "Sir File-a-Lot")
+        self.assertEqual(worker_version.worker.repository_url, None)