README.md

@@ -11,8 +11,9 @@ Backend for Historical Manuscripts Indexing
 * Python 3.10+
 * pip
 * [virtualenvwrapper](https://virtualenvwrapper.readthedocs.io/en/latest/)
-* Docker 24+
+* [Docker 24+](https://docs.docker.com/engine/install/#supported-platforms)
 * [mkcert](https://github.com/FiloSottile/mkcert?tab=readme-ov-file#installation)
+* [GeoDjango system dependencies](https://docs.djangoproject.com/en/3.1/ref/contrib/gis/install/geolibs/): `sudo apt install binutils libproj-dev gdal-bin`
 
 ## Setup for developers
 

VERSION

-1.6.0-alpha4
+1.6.0-beta1

arkindex/documents/management/commands/build_fixtures.py

 #!/usr/bin/env python3
-from datetime import datetime, timezone
 from unittest.mock import patch
 
 from django.contrib.gis.geos import LinearRing
@@ -8,7 +7,7 @@ from django.utils import timezone as DjangoTimeZone
 
 from arkindex.documents.models import Corpus, Element, MetaData, MetaType
 from arkindex.images.models import Image, ImageServer
-from arkindex.ponos.models import Farm, State
+from arkindex.ponos.models import Farm
 from arkindex.process.models import (
     FeatureUsage,
     Process,
@@ -104,23 +103,7 @@ class Command(BaseCommand):
         farm = Farm.objects.create(name="Wheat farm")
         farm.memberships.create(user=user, level=Role.Guest.value)
 
-        # Create a fake docker build with a docker image task
-        build_process = Process.objects.create(
-            farm=farm,
-            creator=superuser,
-            mode=ProcessMode.Repository,
-        )
-        build_task = build_process.tasks.create(
-            run=0,
-            depth=0,
-            slug="docker_build",
-            state=State.Completed,
-            # Use an expiry very far away so that task is never expired
-            expiry=datetime(2100, 12, 31, 23, 59, 59, 999999, timezone.utc),
-        )
-        docker_image = build_task.artifacts.create(size=42_000, path="/path/to/docker_build")
-
-        # Create some workers for the repository with their available version
+        # Create some workers with available versions
         recognizer_worker = WorkerVersion.objects.create(
             worker=worker_repo.workers.create(
                 name="Recognizer",
@@ -131,7 +114,7 @@ class Command(BaseCommand):
             configuration={"test": 42},
             state=WorkerVersionState.Available,
             model_usage=FeatureUsage.Disabled,
-            docker_image=docker_image
+            docker_image_iid="registry.somewhere.com/something:latest"
         )
         dla_worker = WorkerVersion.objects.create(
             worker=worker_repo.workers.create(
@@ -143,7 +126,7 @@ class Command(BaseCommand):
             configuration={"test": 42},
             state=WorkerVersionState.Available,
             model_usage=FeatureUsage.Disabled,
-            docker_image=docker_image
+            docker_image_iid="registry.somewhere.com/something:latest"
         )
 
         WorkerVersion.objects.create(
@@ -156,7 +139,7 @@ class Command(BaseCommand):
             configuration={},
             state=WorkerVersionState.Available,
             model_usage=FeatureUsage.Disabled,
-            docker_image=docker_image,
+            docker_image_iid="registry.somewhere.com/something:latest"
         )
 
         WorkerVersion.objects.create(
@@ -169,7 +152,7 @@ class Command(BaseCommand):
             configuration={"test": 42},
             state=WorkerVersionState.Available,
             model_usage=FeatureUsage.Disabled,
-            docker_image=docker_image,
+            docker_image_iid="registry.somewhere.com/something:latest",
             gpu_usage=FeatureUsage.Required
         )
 
@@ -185,7 +168,7 @@ class Command(BaseCommand):
             state=WorkerVersionState.Available,
             gpu_usage=FeatureUsage.Disabled,
             model_usage=FeatureUsage.Required,
-            docker_image=docker_image
+            docker_image_iid="registry.somewhere.com/something:latest"
         )
 
         # Create a custom worker version that is not linked to a Git repository/revision

arkindex/documents/tests/tasks/test_corpus_delete.py

@@ -3,7 +3,7 @@ from django.db.models.signals import pre_delete
 from arkindex.documents.models import Corpus, Element, EntityType, MetaType, Transcription
 from arkindex.documents.tasks import corpus_delete
 from arkindex.ponos.models import Farm, State, Task
-from arkindex.process.models import CorpusWorkerVersion, ProcessDataset, ProcessMode, Repository, WorkerVersion
+from arkindex.process.models import CorpusWorkerVersion, Process, ProcessDataset, ProcessMode, Repository, WorkerVersion
 from arkindex.project.tests import FixtureTestCase, force_constraints_immediate
 from arkindex.training.models import Dataset
 
@@ -132,7 +132,7 @@ class TestDeleteCorpus(FixtureTestCase):
             message="oh",
             author="me",
         )
-        cls.process = cls.rev.processes.create(
+        cls.process = Process.objects.create(
             creator=cls.user,
             corpus=cls.corpus2,
             mode=ProcessMode.Files,
@@ -204,7 +204,6 @@ class TestDeleteCorpus(FixtureTestCase):
         self.dataset_process3.refresh_from_db()
 
         self.assertTrue(self.repo.revisions.filter(id=self.rev.id).exists())
-        self.assertEqual(self.process.revision, self.rev)
         self.assertEqual(self.process.files.get(), self.df)
         self.assertTrue(Element.objects.get_descending(self.vol.id).filter(id=self.page.id).exists())
         self.assertTrue(self.corpus2.datasets.filter(id=self.dataset2.id).exists())

arkindex/documents/tests/tasks/test_export.py

@@ -22,7 +22,6 @@ from arkindex.documents.models import (
     TranscriptionEntity,
 )
 from arkindex.images.models import Image, ImageServer
-from arkindex.ponos.models import Artifact
 from arkindex.process.models import Repository, WorkerType, WorkerVersion, WorkerVersionState
 from arkindex.project.tests import FixtureTestCase
 
@@ -65,7 +64,7 @@ class TestExport(FixtureTestCase):
             worker=self.repo.workers.create(slug=str(uuid4()), type=self.worker_type),
             configuration={},
             state=WorkerVersionState.Available,
-            docker_image=Artifact.objects.first(),
+            docker_image_iid="registry.somewhere.com/something:latest",
         )
 
     @override_settings(PUBLIC_HOSTNAME="https://darkindex.lol")

arkindex/documents/tests/test_retrieve_elements.py

@@ -121,14 +121,14 @@ class TestRetrieveElements(FixtureAPITestCase):
         running the thumbnails generation command on a folder element
         """
         process = Process.objects.create(
-            mode=ProcessMode.Repository,
-            revision=self.worker_version.revision,
+            mode=ProcessMode.Workers,
             creator=self.user,
             generate_thumbnails=True,
             farm=Farm.objects.first(),
+            corpus=self.corpus
         )
         process.run()
-        task = process.tasks.get()
+        task = process.tasks.first()
         task.image = "task_image"
         task.command = "python generate_thumbnails"
 
@@ -154,13 +154,13 @@ class TestRetrieveElements(FixtureAPITestCase):
         can retrieve the thumbnails PUT URL.
         """
         process = Process.objects.create(
-            mode=ProcessMode.Repository,
-            revision=self.worker_version.revision,
+            mode=ProcessMode.Workers,
             creator=self.user,
             farm=Farm.objects.first(),
+            corpus=self.corpus
         )
         process.run()
-        task = process.tasks.get()
+        task = process.tasks.first()
         self.assertTrue(self.vol.type.folder)
         response = self.client.get(
             reverse("api:element-retrieve", kwargs={"pk": str(self.vol.id)}),

arkindex/images/tests/test_image_api.py

@@ -8,7 +8,7 @@ from rest_framework import status
 
 from arkindex.images.models import Image, ImageServer
 from arkindex.ponos.models import Farm
-from arkindex.process.models import Process, ProcessMode, Revision
+from arkindex.process.models import Process, ProcessMode
 from arkindex.project.aws import S3FileStatus
 from arkindex.project.tests import FixtureAPITestCase
 from arkindex.users.models import Scope
@@ -608,13 +608,13 @@ class TestImageApi(FixtureAPITestCase):
         IIIF images created by a Ponos task are immediately checked
         """
         process = Process.objects.create(
-            mode=ProcessMode.Repository,
-            revision=Revision.objects.first(),
+            mode=ProcessMode.Workers,
             creator=self.user,
+            corpus=self.corpus,
             farm=Farm.objects.first(),
         )
         process.run()
-        task = process.tasks.get()
+        task = process.tasks.first()
         # The user scope should not be necessary with Ponos task authentication
         self.assertFalse(self.user.user_scopes.filter(scope=Scope.CreateIIIFImage).exists())
 
@@ -655,13 +655,13 @@ class TestImageApi(FixtureAPITestCase):
             height=100,
         )
         process = Process.objects.create(
-            mode=ProcessMode.Repository,
-            revision=Revision.objects.first(),
+            mode=ProcessMode.Workers,
             creator=self.user,
+            corpus=self.corpus,
             farm=Farm.objects.first(),
         )
         process.run()
-        task = process.tasks.get()
+        task = process.tasks.first()
         # The user scope should not be necessary with Ponos task authentication
         self.assertFalse(self.user.user_scopes.filter(scope=Scope.CreateIIIFImage).exists())
 

arkindex/ponos/api.py

@@ -52,8 +52,7 @@ class TaskDetailsFromAgent(RetrieveUpdateAPIView):
         "agent__farm",
         "gpu",
         # Used for permission checks
-        "process__corpus",
-        "process__revision__repo",
+        "process__corpus"
     )
     permission_classes = (
         # On all HTTP methods, require either any Ponos agent, an instance admin, the task itself, or guest access to the process' task
@@ -102,7 +101,7 @@ class TaskArtifacts(ListCreateAPIView):
     def task(self):
         task = get_object_or_404(
             # Select the required tables for permissions checking
-            Task.objects.select_related("process__corpus", "process__revision__repo"),
+            Task.objects.select_related("process__corpus"),
             pk=self.kwargs["pk"],
         )
         self.check_object_permissions(self.request, task)
@@ -126,7 +125,7 @@ class TaskArtifactDownload(APIView):
     def get_object(self, pk, path):
         artifact = get_object_or_404(
             # Select the required tables for permissions checking
-            Artifact.objects.select_related("task__process__corpus", "task__process__revision__repo"),
+            Artifact.objects.select_related("task__process__corpus"),
             task_id=pk,
             path=path,
         )
@@ -167,5 +166,5 @@ class TaskUpdate(UpdateAPIView):
     authentication_classes = (TokenAuthentication, SessionAuthentication)
     # Only allow regular users that have admin access to the task's process
     permission_classes = (IsTaskAdmin, )
-    queryset = Task.objects.select_related("process__corpus", "process__revision__repo")
+    queryset = Task.objects.select_related("process__corpus")
     serializer_class = TaskTinySerializer

arkindex/ponos/migrations/0007_remove_task_has_docker_socket.py

+# Generated by Django 4.1.7 on 2024-03-20 08:45
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("ponos", "0006_task_worker_run"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="task",
+            name="has_docker_socket",
+        ),
+    ]

arkindex/ponos/models.py

@@ -273,7 +273,6 @@ class Task(models.Model):
     shm_size = models.CharField(max_length=80, blank=True, null=True, editable=False)
     command = models.TextField(blank=True, null=True)
     env = HStoreField(default=dict)
-    has_docker_socket = models.BooleanField(default=False)
     image_artifact = models.ForeignKey(
         "ponos.Artifact",
         related_name="tasks_using_image",

arkindex/ponos/tests/test_api.py

@@ -174,60 +174,6 @@ class TestAPI(FixtureAPITestCase):
                     call("get_object", Params={"Bucket": "ponos", "Key": "somelog"}),
                 )
 
-    @patch("arkindex.project.aws.s3")
-    def test_task_details_process_level_repo(self, s3_mock):
-        s3_mock.Object.return_value.bucket_name = "ponos"
-        s3_mock.Object.return_value.key = "somelog"
-        s3_mock.meta.client.generate_presigned_url.return_value = "http://somewhere"
-
-        self.client.force_login(self.user)
-        self.process.mode = ProcessMode.Repository
-        self.process.corpus = None
-        self.process.revision = self.rev
-        self.process.save()
-        membership = self.rev.repo.memberships.create(user=self.user, level=Role.Guest.value)
-
-        for role in Role:
-            s3_mock.reset_mock()
-            s3_mock.reset_mock()
-            # Recreate the BytesIO each time, because its contents get consumed each time the API is called
-            s3_mock.Object.return_value.get.return_value = {"Body": BytesIO(b"Failed successfully")}
-
-            with self.subTest(role=role):
-                membership.level = role.value
-                membership.save()
-
-                with self.assertNumQueries(4):
-                    resp = self.client.get(reverse("api:task-details", args=[self.task1.id]))
-                    self.assertEqual(resp.status_code, status.HTTP_200_OK)
-
-                self.assertDictEqual(
-                    resp.json(),
-                    {
-                        "id": str(self.task1.id),
-                        "run": 0,
-                        "depth": 0,
-                        "slug": "initialisation",
-                        "state": "unscheduled",
-                        "parents": [],
-                        "logs": "Failed successfully",
-                        "full_log": "http://somewhere",
-                        "extra_files": {},
-                        "agent": None,
-                        "gpu": None,
-                        "shm_size": None,
-                    },
-                )
-
-                self.assertEqual(s3_mock.Object.call_count, 2)
-                self.assertEqual(s3_mock.Object().get.call_count, 1)
-                self.assertEqual(s3_mock.Object().get.call_args, call(Range="bytes=-42"))
-                self.assertEqual(s3_mock.meta.client.generate_presigned_url.call_count, 1)
-                self.assertEqual(
-                    s3_mock.meta.client.generate_presigned_url.call_args,
-                    call("get_object", Params={"Bucket": "ponos", "Key": "somelog"}),
-                )
-
     @patch("arkindex.project.aws.s3")
     def test_task_details(self, s3_mock):
         s3_mock.Object.return_value.bucket_name = "ponos"
@@ -319,28 +265,6 @@ class TestAPI(FixtureAPITestCase):
                     )
                     self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
 
-    @expectedFailure
-    def test_update_task_requires_process_admin_repo(self):
-        self.process.mode = ProcessMode.Repository
-        self.process.corpus = None
-        self.process.revision = self.rev
-        self.process.creator = self.superuser
-        self.process.save()
-        self.client.force_login(self.user)
-
-        for role in [None, Role.Guest, Role.Contributor]:
-            with self.subTest(role=role):
-                self.rev.repo.memberships.filter(user=self.user).delete()
-                if role:
-                    self.rev.repo.memberships.create(user=self.user, level=role.value)
-
-                with self.assertNumQueries(5):
-                    resp = self.client.put(
-                        reverse("api:task-update", args=[self.task1.id]),
-                        data={"state": State.Stopping.value},
-                    )
-                    self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
-
     def test_update_running_task_state_stopping(self):
         self.task1.state = State.Running
         self.task1.save()
@@ -500,28 +424,6 @@ class TestAPI(FixtureAPITestCase):
                     )
                     self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
 
-    @expectedFailure
-    def test_partial_update_task_requires_process_admin_repo(self):
-        self.process.mode = ProcessMode.Repository
-        self.process.corpus = None
-        self.process.revision = self.rev
-        self.process.creator = self.superuser
-        self.process.save()
-        self.client.force_login(self.user)
-
-        for role in [None, Role.Guest, Role.Contributor]:
-            with self.subTest(role=role):
-                self.rev.repo.memberships.filter(user=self.user).delete()
-                if role:
-                    self.rev.repo.memberships.create(user=self.user, level=role.value)
-
-                with self.assertNumQueries(5):
-                    resp = self.client.patch(
-                        reverse("api:task-update", args=[self.task1.id]),
-                        data={"state": State.Stopping.value},
-                    )
-                    self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
-
     def test_partial_update_running_task_state_stopping(self):
         self.task1.state = State.Running
         self.task1.save()

arkindex/ponos/tests/test_artifacts_api.py

@@ -6,7 +6,7 @@ from django.urls import reverse
 from rest_framework import status
 
 from arkindex.documents.models import Corpus
-from arkindex.process.models import Process, ProcessMode, Repository
+from arkindex.process.models import Process, ProcessMode
 from arkindex.project.tests import FixtureAPITestCase
 from arkindex.users.models import Right, Role, User
 
@@ -17,14 +17,6 @@ class TestAPI(FixtureAPITestCase):
     @classmethod
     def setUpTestData(cls):
         super().setUpTestData()
-
-        cls.repo = Repository.objects.first()
-        cls.repository_process = Process.objects.create(
-            mode=ProcessMode.Repository,
-            creator=cls.superuser,
-            revision=cls.repo.revisions.first(),
-        )
-
         # Make corpus private
         cls.corpus.public = False
         cls.corpus.save()
@@ -119,43 +111,6 @@ class TestAPI(FixtureAPITestCase):
                     ],
                 )
 
-    def test_list_process_level_repo(self):
-        self.client.force_login(self.user)
-        membership = self.repo.memberships.create(user=self.user, level=Role.Guest.value)
-
-        for role in Role:
-            with self.subTest(role=role):
-                membership.level = role.value
-                membership.save()
-
-                with self.assertNumQueries(4):
-                    response = self.client.get(reverse("api:task-artifacts", args=[self.task1.id]))
-                    self.assertEqual(response.status_code, status.HTTP_200_OK)
-
-                self.assertListEqual(
-                    response.json(),
-                    [
-                        {
-                            "content_type": "application/json",
-                            "created": self.artifact1.created.isoformat().replace("+00:00", "Z"),
-                            "id": str(self.artifact1.id),
-                            "path": "path/to/file.json",
-                            "s3_put_url": None,
-                            "size": 42,
-                            "updated": self.artifact1.updated.isoformat().replace("+00:00", "Z"),
-                        },
-                        {
-                            "content_type": "text/plain",
-                            "created": self.artifact2.created.isoformat().replace("+00:00", "Z"),
-                            "id": str(self.artifact2.id),
-                            "path": "some/text.txt",
-                            "s3_put_url": None,
-                            "size": 1337,
-                            "updated": self.artifact2.updated.isoformat().replace("+00:00", "Z"),
-                        },
-                    ],
-                )
-
     def test_list_admin(self):
         self.client.force_login(self.superuser)
         with self.assertNumQueries(4):
@@ -423,31 +378,6 @@ class TestAPI(FixtureAPITestCase):
                     )
                 )
 
-    def test_download_process_level_repo(self):
-        self.client.force_login(self.user)
-        membership = self.repo.memberships.create(user=self.user, level=Role.Guest.value)
-
-        task = self.repository_process.tasks.create(run=0, depth=0, slug="a")
-        task.artifacts.create(path="path/to/file.json", content_type="application/json", size=42)
-
-        for role in Role:
-            with self.subTest(role=role):
-                membership.level = role.value
-                membership.save()
-
-                with self.assertNumQueries(3):
-                    response = self.client.get(
-                        reverse("api:task-artifact-download", args=[task.id, "path/to/file.json"]),
-                    )
-                    self.assertEqual(response.status_code, status.HTTP_302_FOUND)
-
-                self.assertTrue(response.has_header("Location"))
-                self.assertTrue(
-                    response["Location"].startswith(
-                        f"http://s3/ponos-artifacts/{task.id}/path/to/file.json"
-                    )
-                )
-
     def test_download_task(self):
         with self.assertNumQueries(2):
             response = self.client.get(

arkindex/process/api.py

@@ -306,8 +306,6 @@ class ProcessQuerysetMixin(object):
                     # Element and folder types are serialized as their slugs
                     "element_type",
                     "folder_type",
-                    # The revision is serialized with its commit URL, which also requires the repository
-                    "revision__repo",
                 )
                 # Files and tasks are also listed
                 .prefetch_related("files", "tasks__parents")
@@ -441,10 +439,6 @@ class ProcessRetry(ProcessACLMixin, ProcessQuerysetMixin, GenericAPIView):
         elif state == State.Stopping:
             raise ValidationError({"__all__": ["This process is stopping"]})
 
-        if process.mode == ProcessMode.Repository:
-            if not process.revision:
-                raise ValidationError({"__all__": ["Git repository imports must have a revision set"]})
-
     @extend_schema(
         operation_id="RetryProcess",
         tags=["process"],
@@ -701,7 +695,7 @@ class ProcessDatasets(ProcessACLMixin, ListAPIView):
     @cached_property
     def process(self):
         process = get_object_or_404(
-            Process.objects.using("default").select_related("corpus", "revision__repo"),
+            Process.objects.using("default").select_related("corpus"),
             Q(pk=self.kwargs["pk"])
         )
         if not self.process_access_level(process):
@@ -986,7 +980,7 @@ class WorkerTypesList(ListAPIView):
     get=extend_schema(
         description=(
             "List versions for a given worker ID with their revision and associated git references.\n\n"
-            "Requires an **execution** access to the worker or its repository."
+            "Requires an **execution** access to the worker."
         ),
         parameters=[
             OpenApiParameter(
@@ -1001,13 +995,7 @@ class WorkerTypesList(ListAPIView):
         description=dedent("""
             Create a new version for a worker.
 
-            Authentication can be done:
-            * Using a user authentication (via a cookie or token).
             The user must have an administrator access to the worker.
-            * Using a ponos task authentication.
-            The worker must be linked to a repository.
-
-            The `revision_id` parameter must be set for workers linked to a repository only.
         """)
     )
 )
@@ -1030,8 +1018,7 @@ class WorkerVersionList(WorkerACLMixin, ListCreateAPIView):
             raise PermissionDenied(detail="You do not have an execution access to this worker.")
         if (
             self.request.method not in permissions.SAFE_METHODS
-            # Either a task authentication or an admin access is required for creation
-            and not isinstance(self.request.auth, Task)
+            # An admin access is required for creation
             and not self.has_admin_access(worker)
         ):
             raise PermissionDenied(detail="You do not have an admin access to this worker.")

arkindex/process/builder.py

@@ -88,7 +88,6 @@ class ProcessBuilder(object):
         env={},
         image=None,
         artifact=None,
-        has_docker_socket=False,
         extra_files={},
         requires_gpu=False,
         shm_size=None,
@@ -112,7 +111,6 @@ class ProcessBuilder(object):
                 image=image,
                 requires_gpu=requires_gpu,
                 shm_size=shm_size,
-                has_docker_socket=has_docker_socket,
                 extra_files=extra_files,
                 image_artifact_id=artifact,
                 worker_run=worker_run,
@@ -191,10 +189,6 @@ class ProcessBuilder(object):
         ):
             raise ValidationError("Some model versions are on archived models and cannot be executed.")
 
-    def validate_repository(self) -> None:
-        if self.process.revision is None:
-            raise ValidationError("A revision is required to create an import workflow from GitLab repository")
-
     def validate_s3(self) -> None:
         if not self.process.bucket_name:
             raise ValidationError("Missing S3 bucket name")
@@ -251,13 +245,6 @@ class ProcessBuilder(object):
         )
         self._create_worker_versions_cache([(settings.IMPORTS_WORKER_VERSION, None, None)])
 
-    def build_repository(self):
-        self._build_task(
-            command=f"python -m arkindex_tasks.import_git {self.process.revision.id}",
-            slug="import_git",
-            env=self.base_env,
-        )
-
     def build_iiif(self):
         from arkindex.process.models import WorkerVersion
 

arkindex/process/migrations/0003_initial.py

@@ -3,12 +3,18 @@
 import django.db.models.deletion
 from django.conf import settings
 from django.db import migrations, models
+from enumfields import Enum, fields
 
 import arkindex.process.models
 import pgtrigger.compiler
 import pgtrigger.migrations
 
 
+class TmpProcessMode(Enum):
+    Repository = "repository"
+    Local = "local"
+
+
 class Migration(migrations.Migration):
 
     initial = True
@@ -189,15 +195,25 @@ class Migration(migrations.Migration):
         ),
         migrations.AddConstraint(
             model_name="process",
-            constraint=models.CheckConstraint(check=models.Q(models.Q(("mode", arkindex.process.models.ProcessMode["Local"]), _negated=True), ("workflow", None), _connector="OR"), name="local_process_no_workflow", violation_error_message="Local processes cannot be started."),
+            constraint=models.UniqueConstraint(models.F("creator"), condition=models.Q(("mode", arkindex.process.models.ProcessMode["Local"])), name="unique_local_process", violation_error_message="Only one local process is allowed per user."),
         ),
         migrations.AddConstraint(
             model_name="process",
-            constraint=models.CheckConstraint(check=models.Q(("mode__in", (arkindex.process.models.ProcessMode["Local"], arkindex.process.models.ProcessMode["Repository"])), models.Q(("corpus", None), _negated=True), _connector="XOR"), name="check_process_corpus", violation_error_message="Local and repository processes cannot have a corpus, and other modes must have one set."),
+            constraint=models.CheckConstraint(check=models.Q(models.Q(("mode", arkindex.process.models.ProcessMode["Local"]), _negated=True), ("workflow", None), _connector="OR"), name="local_process_no_workflow", violation_error_message="Local processes cannot be started."),
+        ),
+        migrations.AlterField(
+            model_name="process",
+            name="mode",
+            field=fields.EnumField(enum=TmpProcessMode, max_length=30)
         ),
         migrations.AddConstraint(
             model_name="process",
-            constraint=models.UniqueConstraint(models.F("creator"), condition=models.Q(("mode", arkindex.process.models.ProcessMode["Local"])), name="unique_local_process", violation_error_message="Only one local process is allowed per user."),
+            constraint=models.CheckConstraint(check=models.Q(("mode__in", (TmpProcessMode["Local"], TmpProcessMode["Repository"])), models.Q(("corpus", None), _negated=True), _connector="XOR"), name="check_process_corpus", violation_error_message="Local and repository processes cannot have a corpus, and other modes must have one set."),
+        ),
+        migrations.AlterField(
+            model_name="process",
+            name="mode",
+            field=fields.EnumField(enum=arkindex.process.models.ProcessMode, max_length=30)
         ),
         migrations.AlterUniqueTogether(
             name="gitref",

arkindex/process/migrations/0030_remove_repository_processes.py

+# Generated by Django 4.1.7 on 2024-02-29 13:28
+
+from django.db import migrations
+from enumfields import Enum
+
+
+class TmpProcessMode(Enum):
+    Repository = "repository"
+    Workers = "workers"
+
+
+def migrate_repository_processes(apps, schema):
+    """
+    If there are Repository processes on the instances, update their mode to Workers
+    and attach them to a "Repository processes" corpus
+    """
+    Process = apps.get_model("process", "Process")
+    Process.mode.field.enum = TmpProcessMode
+    # Do not do anything if there are no Repository processes to migrate
+    if not Process.objects.filter(mode=TmpProcessMode.Repository).exists():
+        return
+    # Create a project to which all the old repository process will be linked,
+    # as non-Repository process modes require a link to a corpus
+    Corpus = apps.get_model("documents", "Corpus")
+    archival_corpus, _ = Corpus.objects.get_or_create(
+        name="Repository processes",
+        description="A project created to re-home the migrated Gitlab repository import processes.",
+    )
+    Process.objects.filter(mode=TmpProcessMode.Repository).update(corpus=archival_corpus, mode=TmpProcessMode.Workers)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("process", "0029_processdataset_sets"),
+    ]
+
+    operations = [
+        migrations.RemoveConstraint(
+            model_name="process",
+            name="check_process_corpus",
+        ),
+        migrations.RunPython(
+            migrate_repository_processes,
+            reverse_code=migrations.RunPython.noop,
+            elidable=True,
+        ),
+    ]

arkindex/process/migrations/0031_process_corpus_check_and_remove_revision_field.py

+from django.db import migrations, models
+
+import arkindex.process.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("process", "0030_remove_repository_processes"),
+    ]
+
+    operations = [
+        migrations.AddConstraint(
+            model_name="process",
+            constraint=models.CheckConstraint(check=models.Q(("mode", arkindex.process.models.ProcessMode["Local"]), models.Q(("corpus", None), _negated=True), _connector="XOR"), name="check_process_corpus", violation_error_message="Local processes cannot have a corpus, and other modes must have one set."),
+        ),
+        migrations.RemoveField(
+            model_name="process",
+            name="revision",
+        ),
+    ]

arkindex/process/models.py

@@ -54,7 +54,6 @@ class ActivityState(Enum):
 
 class ProcessMode(Enum):
     Files = "files"
-    Repository = "repository"
     IIIF = "iiif"
     Workers = "workers"
     Template = "template"
@@ -71,8 +70,6 @@ class Process(IndexableModel):
     mode = EnumField(ProcessMode, max_length=30)
     files = models.ManyToManyField("process.DataFile", related_name="processes")
     datasets = models.ManyToManyField("training.Dataset", related_name="processes", through="process.ProcessDataset")
-    revision = models.ForeignKey(
-        "process.Revision", related_name="processes", on_delete=models.CASCADE, blank=True, null=True)
     versions = models.ManyToManyField("process.WorkerVersion", through="process.WorkerRun", related_name="processes")
     activity_state = EnumField(ActivityState, max_length=32, default=ActivityState.Disabled)
 
@@ -173,11 +170,11 @@ class Process(IndexableModel):
         verbose_name_plural = "processes"
         constraints = [
             models.CheckConstraint(
-                # Either the process mode is local or repository and the process does not have a corpus,
+                # Either the process mode is local and the process does not have a corpus,
                 # or it has another mode and has a corpus set.
-                check=Q(mode__in=(ProcessMode.Local, ProcessMode.Repository)) ^ ~Q(corpus=None),
+                check=Q(mode=ProcessMode.Local) ^ ~Q(corpus=None),
                 name="check_process_corpus",
-                violation_error_message="Local and repository processes cannot have a corpus, and other modes must have one set.",
+                violation_error_message="Local processes cannot have a corpus, and other modes must have one set.",
             ),
             models.UniqueConstraint(
                 "creator",

arkindex/process/serializers/imports.py

@@ -18,7 +18,6 @@ from arkindex.process.models import (
     WorkerRun,
     WorkerVersionState,
 )
-from arkindex.process.serializers.git import RevisionSerializer
 from arkindex.project.mixins import ProcessACLMixin
 from arkindex.project.serializer_fields import EnumField, LinearRingField
 from arkindex.project.validators import MaxValueValidator
@@ -78,7 +77,6 @@ class ProcessSerializer(ProcessLightSerializer):
     Serialize a process with its settings
     """
     from arkindex.documents.serializers.elements import ElementSlimSerializer
-    revision = RevisionSerializer(read_only=True)
     element = ElementSlimSerializer(read_only=True)
     element_id = serializers.PrimaryKeyRelatedField(
         queryset=Element.objects.none(),
@@ -119,7 +117,6 @@ class ProcessSerializer(ProcessLightSerializer):
     class Meta(ProcessLightSerializer.Meta):
         fields = ProcessLightSerializer.Meta.fields + (
             "files",
-            "revision",
             "element",
             "element_id",
             "folder_type",
@@ -132,7 +129,6 @@ class ProcessSerializer(ProcessLightSerializer):
         )
         read_only_fields = ProcessLightSerializer.Meta.read_only_fields + (
             "files",
-            "revision",
             "element",
             "folder_type",
             "use_gpu",
@@ -522,7 +518,6 @@ class CreateProcessTemplateSerializer(serializers.ModelSerializer):
     def validate(self, data):
         data = super().validate(data)
         template = self.context["template"]
-        data["revision"] = template.revision
         data["mode"] = ProcessMode.Template
         data["creator"] = self.context["request"].user
         data["corpus"] = template.corpus