VERSION

-1.6.0-alpha1
+1.6.0-alpha2

arkindex/documents/deletion.py

@@ -70,35 +70,6 @@ def delete_element(element_id: UUID) -> None:
         """, {"id": element_id})
         logger.info(f"Deleted {cursor.rowcount} usage from process as element")
 
-        # Set folders references on training processes to None
-        cursor.execute("""
-        UPDATE process_process
-        SET train_folder_id = NULL
-        WHERE train_folder_id = %(id)s
-        OR train_folder_id IN (
-            SELECT element_id FROM documents_elementpath WHERE path && ARRAY[%(id)s]
-        )
-        """, {"id": element_id})
-        logger.info(f"Deleted {cursor.rowcount} usage from process as train folder")
-        cursor.execute("""
-        UPDATE process_process
-        SET validation_folder_id = NULL
-        WHERE validation_folder_id = %(id)s
-        OR validation_folder_id IN (
-            SELECT element_id FROM documents_elementpath WHERE path && ARRAY[%(id)s]
-        )
-        """, {"id": element_id})
-        logger.info(f"Deleted {cursor.rowcount} usage from process as validation folder")
-        cursor.execute("""
-        UPDATE process_process
-        SET test_folder_id = NULL
-        WHERE test_folder_id = %(id)s
-        OR test_folder_id IN (
-            SELECT element_id FROM documents_elementpath WHERE path && ARRAY[%(id)s]
-        )
-        """, {"id": element_id})
-        logger.info(f"Deleted {cursor.rowcount} usage from process as test folder")
-
         # Remove user selections
         cursor.execute("""
         DELETE FROM documents_selection selection

arkindex/documents/managers.py

@@ -81,11 +81,6 @@ class ElementQuerySet(models.QuerySet):
         # This may cause workers processes to target a different set of elements
         Process.objects.filter(element_id__in=ids).update(element_id=None)
 
-        # Set folder references on training processes to None
-        Process.objects.filter(train_folder_id__in=ids).update(train_folder_id=None)
-        Process.objects.filter(validation_folder_id__in=ids).update(validation_folder_id=None)
-        Process.objects.filter(test_folder_id__in=ids).update(test_folder_id=None)
-
         # This is where it gets really ugly: in order to delete all the children
         # without losing their reference, we need to inject the generated SQL query
         # directly into an SQL DELETE statement for paths

arkindex/process/api.py

@@ -88,6 +88,7 @@ from arkindex.process.serializers.imports import (
 from arkindex.process.serializers.ingest import BucketSerializer, S3ImportSerializer
 from arkindex.process.serializers.training import ProcessDatasetSerializer
 from arkindex.process.serializers.worker_runs import (
+    CorpusWorkerRunSerializer,
     UserWorkerRunSerializer,
     WorkerRunEditSerializer,
     WorkerRunSerializer,
@@ -1085,6 +1086,56 @@ class CorpusWorkerVersionList(CorpusACLMixin, ListAPIView):
         )
 
 
+@extend_schema_view(
+    get=extend_schema(
+        operation_id="ListCorpusWorkerRuns",
+        tags=["ml"],
+    )
+)
+class CorpusWorkerRunList(CorpusACLMixin, ListAPIView):
+    """
+    List worker runs used by any ML result in a given corpus.
+
+    The results also include all of the current user's worker runs that have results on any corpora, and not
+    just the request's corpus, as user worker runs are independent from corpora.
+    The results do **not** include other users' worker runs, even if they have produced results on the
+    request's corpus.
+
+    Guest access is required on private corpora. No specific rights are required on the workers.
+    """
+
+    permission_classes = (IsVerifiedOrReadOnly, )
+    serializer_class = CorpusWorkerRunSerializer
+    # For OpenAPI type discovery
+    queryset = WorkerRun.objects.none()
+
+    @cached_property
+    def corpus(self):
+        return get_object_or_404(self.readable_corpora, pk=self.kwargs["pk"])
+
+    def get_queryset(self):
+        return (
+            WorkerRun.objects.select_related("process__corpus")
+            .select_related(
+                "version__worker__type",
+                "version__worker__repository",
+                "version__revision__repo",
+                "configuration",
+                "model_version__model",
+            )
+            .prefetch_related(
+                Prefetch(
+                    "version__revision__refs",
+                    queryset=GitRef.objects.select_related("repository")
+                ),
+                "process__tasks",
+            )
+            .filter(Q(process__corpus_id=self.corpus.id) | Q(process__creator_id=self.user.id, process__mode=ProcessMode.Local), has_results=True)
+            .annotate(process_element_count=Count("process__elements"))
+            .order_by("summary")
+        )
+
+
 @extend_schema(tags=["repos"])
 @extend_schema_view(
     get=extend_schema(

arkindex/process/management/commands/cache_worker_runs.py

+from django.core.management.base import BaseCommand
+
+from arkindex.process.models import WorkerRun
+
+
+class Command(BaseCommand):
+    help = "Update the has_results attribute on worker runs"
+
+    def handle(self, *args, **options):
+        self.stdout.write("Updating worker runs cache…")
+        WorkerRun.objects.all().set_has_results()
+        self.stdout.write("Worker runs cache successfully updated.")

arkindex/process/managers.py

 import logging
+import operator
+from functools import reduce
 
 from django.conf import settings
 from django.db import connections
-from django.db.models import Manager, ManyToOneRel, Q
+from django.db.models import Exists, Manager, ManyToOneRel, OuterRef, Q
 from django.db.models.functions import Coalesce
 from django.db.models.query import QuerySet
 from django.utils.functional import cached_property
@@ -171,23 +173,42 @@ class WorkerManager(BaseACLManager):
 
 
 class WorkerRunQuerySet(QuerySet):
-    def in_use(self):
-        """
-        Check if any data is linked to worker runs
-        """
-        ids = self.values_list("id", flat=True)
-        related_models = [
+    def worker_results_models(self):
+        return [
             field.related_model
             for field in self.model._meta.get_fields()
             if isinstance(field, ManyToOneRel)
             # Ignore reverse links to processes tasks
             and field.related_model is not Task
         ]
-        for field in related_models:
+
+    def in_use(self):
+        """
+        Check if any data is linked to worker runs
+        """
+        ids = self.values_list("id", flat=True)
+        for field in self.worker_results_models():
             if field.objects.filter(worker_run__in=ids).exists():
                 return True
         return False
 
+    def set_has_results(self):
+        """
+        Set the has_results field on worker runs to True if these workers runs are linked to existing machine learning results
+        """
+        queryset = list(self.annotate(new_has_results=self.has_results_expression()).only("id"))
+        for run in queryset:
+            run.has_results = run.new_has_results
+        self.bulk_update(queryset, ["has_results"])
+
+    def has_results_expression(self):
+        return reduce(operator.or_, [
+            Exists(
+                model.objects.filter(worker_run_id=OuterRef("pk"))
+            )
+            for model in self.worker_results_models()
+        ])
+
 
 class WorkerRunManager(Manager):
     def get_queryset(self):

arkindex/process/migrations/0027_workerrun_has_results.py

+# Generated by Django 4.1.7 on 2024-01-17 16:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("process", "0026_remove_repository_unique_repository_hook_token_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="workerrun",
+            name="has_results",
+            field=models.BooleanField(default=False),
+        ),
+    ]

arkindex/process/migrations/0028_remove_process_model_remove_process_test_folder_and_more.py

+# Generated by Django 4.1.7 on 2024-01-23 09:59
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("process", "0027_workerrun_has_results"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="process",
+            name="model",
+        ),
+        migrations.RemoveField(
+            model_name="process",
+            name="test_folder",
+        ),
+        migrations.RemoveField(
+            model_name="process",
+            name="train_folder",
+        ),
+        migrations.RemoveField(
+            model_name="process",
+            name="validation_folder",
+        ),
+        migrations.RunSQL(
+            [
+                """
+                UPDATE process_process
+                SET mode = 'workers', name = CONCAT('(Old Training) ', name)
+                WHERE mode = 'training'
+                """
+            ],
+            reverse_sql=migrations.RunSQL.noop,
+            elidable=True,
+        )
+    ]

arkindex/process/models.py

@@ -58,7 +58,6 @@ class ProcessMode(Enum):
     Workers = "workers"
     Template = "template"
     S3 = "s3"
-    Training = "training"
     Local = "local"
     Dataset = "dataset"
 
@@ -168,36 +167,6 @@ class Process(IndexableModel):
     # Object keys are up to 1024 characters long
     prefix = models.CharField(max_length=1024, blank=True, null=True)
 
-    # Training specific fields
-    model = models.ForeignKey(
-        "training.Model",
-        related_name="training_processes",
-        on_delete=models.CASCADE,
-        null=True,
-        blank=True,
-    )
-    train_folder = models.ForeignKey(
-        "documents.Element",
-        related_name="train_folder_processes",
-        on_delete=models.SET_NULL,
-        null=True,
-        blank=True,
-    )
-    validation_folder = models.ForeignKey(
-        "documents.Element",
-        related_name="validation_folder_processes",
-        on_delete=models.SET_NULL,
-        null=True,
-        blank=True,
-    )
-    test_folder = models.ForeignKey(
-        "documents.Element",
-        related_name="test_folder_processes",
-        on_delete=models.SET_NULL,
-        null=True,
-        blank=True,
-    )
-
     class Meta:
         ordering = ["corpus", "-created"]
         verbose_name_plural = "processes"
@@ -890,6 +859,7 @@ class WorkerRun(models.Model):
     summary = models.TextField()
     created = models.DateTimeField(auto_now_add=True)
     updated = models.DateTimeField(auto_now=True)
+    has_results = models.BooleanField(default=False)
 
     objects = WorkerRunManager()
 

arkindex/process/serializers/imports.py

 from collections import defaultdict
+from textwrap import dedent
 
 from django.conf import settings
 from rest_framework import serializers
@@ -46,34 +47,30 @@ class ProcessLightSerializer(serializers.ModelSerializer):
             "corpus",
             "creator",
             "activity_state",
+            "use_cache",
         )
-        read_only_fields = ("id", "state", "mode", "corpus", "creator", "activity_state")
+        read_only_fields = ("id", "state", "mode", "corpus", "creator", "activity_state", "use_cache")
 
 
-class ProcessTrainingSerializer(ProcessLightSerializer):
-    model_id = serializers.PrimaryKeyRelatedField(read_only=True)
-    train_folder_id = serializers.PrimaryKeyRelatedField(read_only=True)
-    validation_folder_id = serializers.PrimaryKeyRelatedField(read_only=True)
-    test_folder_id = serializers.PrimaryKeyRelatedField(read_only=True)
+class CorpusWorkerRunProcessSerializer(ProcessLightSerializer):
+    element_id = serializers.PrimaryKeyRelatedField(read_only=True, allow_null=True)
+    element_count = serializers.IntegerField(
+        read_only=True,
+        allow_null=True,
+        help_text=dedent("""
+            This count is only set for processes created from a selection of elements,
+            and does not account for either process filters nor the recursive option.
+        """)
+    )
 
     class Meta(ProcessLightSerializer.Meta):
         fields = ProcessLightSerializer.Meta.fields + (
-            "model_id",
-            "train_folder_id",
-            "validation_folder_id",
-            "test_folder_id",
-            "use_cache",
-        )
-        read_only_fields = ProcessLightSerializer.Meta.read_only_fields + (
-            "model_id",
-            "train_folder_id",
-            "validation_folder_id",
-            "test_folder_id",
-            "use_cache",
+            "element_id",
+            "element_count"
         )
 
 
-class ProcessSerializer(ProcessTrainingSerializer):
+class ProcessSerializer(ProcessLightSerializer):
     """
     Serialize a process with its settings
     """
@@ -116,8 +113,8 @@ class ProcessSerializer(ProcessTrainingSerializer):
         style={"base_template": "input.html"}
     )
 
-    class Meta(ProcessTrainingSerializer.Meta):
-        fields = ProcessTrainingSerializer.Meta.fields + (
+    class Meta(ProcessLightSerializer.Meta):
+        fields = ProcessLightSerializer.Meta.fields + (
             "files",
             "revision",
             "element",
@@ -130,7 +127,7 @@ class ProcessSerializer(ProcessTrainingSerializer):
             "use_gpu",
             "template_id",
         )
-        read_only_fields = ProcessTrainingSerializer.Meta.read_only_fields + (
+        read_only_fields = ProcessLightSerializer.Meta.read_only_fields + (
             "files",
             "revision",
             "element",

arkindex/process/serializers/worker_runs.py

@@ -15,7 +15,7 @@ from arkindex.process.models import (
     WorkerVersion,
     WorkerVersionState,
 )
-from arkindex.process.serializers.imports import ProcessTrainingSerializer
+from arkindex.process.serializers.imports import CorpusWorkerRunProcessSerializer, ProcessLightSerializer
 from arkindex.process.serializers.workers import WorkerConfigurationSerializer, WorkerVersionSerializer
 from arkindex.project.mixins import WorkerACLMixin
 from arkindex.training.models import ModelVersion, ModelVersionState
@@ -94,7 +94,7 @@ class WorkerRunSerializer(WorkerACLMixin, serializers.ModelSerializer):
                   "Only a configuration of the WorkerVersion's worker may be set.",
     )
 
-    process = ProcessTrainingSerializer(read_only=True)
+    process = ProcessLightSerializer(read_only=True)
 
     class Meta:
         model = WorkerRun
@@ -296,3 +296,13 @@ class UserWorkerRunSerializer(serializers.ModelSerializer):
     class Meta:
         model = WorkerRun
         fields = ("worker_version_id", "model_version_id", "configuration_id")
+
+
+class CorpusWorkerRunSerializer(WorkerRunSerializer):
+    process = CorpusWorkerRunProcessSerializer(read_only=True)
+
+    def to_representation(self, obj):
+        # Move the process_element_count annotation to the process object so it
+        # can be accessed by its serializer.
+        obj.process.element_count = obj.process_element_count
+        return super().to_representation(obj)

arkindex/process/tests/commands/test_cache_worker_runs.py

+from django.core.management import call_command
+
+from arkindex.documents.models import (
+    Classification,
+    Element,
+    ElementType,
+    Entity,
+    MetaData,
+    MetaType,
+    Transcription,
+    TranscriptionEntity,
+)
+from arkindex.process.models import WorkerRun
+from arkindex.project.tests import FixtureTestCase
+
+
+class TestCacheWorkerRuns(FixtureTestCase):
+
+    @classmethod
+    def setUpTestData(cls):
+        super().setUpTestData()
+        cls.run_1, cls.run_2, cls.run_3, cls.run_4 = WorkerRun.objects.all().order_by("summary")
+        # Set has_results to True for run_3 and run_4
+        cls.run_3.has_results = True
+        cls.run_4.has_results = True
+        cls.run_3.save()
+        cls.run_4.save()
+        # Remove all Element, Classifications, MetaData, Entity, TranscriptionEntity and Transcription objects
+        # to start clean and be sure of which runs have and do not have ml results
+        for model in [TranscriptionEntity, MetaData, Entity, Transcription, Classification, Element]:
+            model.objects.all().delete()
+        # Create objects created by run_2 and run_4
+        page = Element.objects.create(
+            corpus=cls.corpus,
+            type=ElementType.objects.get(slug="page"),
+            name="Fox Mulder",
+            worker_run=cls.run_2,
+            worker_version=cls.run_2.version
+        )
+        MetaData.objects.create(
+            name="partner",
+            type=MetaType.Text,
+            value="Dana Scully",
+            element=page,
+            worker_run=cls.run_4,
+            worker_version=cls.run_4.version
+        )
+
+    def test_run(self):
+        self.assertQuerysetEqual(
+            WorkerRun.objects.order_by("summary").values_list("id", "has_results"),
+            [(self.run_1.id, False), (self.run_2.id, False), (self.run_3.id, True), (self.run_4.id, True)]
+        )
+        with self.assertNumQueries(2):
+            call_command("cache_worker_runs")
+        self.assertQuerysetEqual(
+            WorkerRun.objects.order_by("summary").values_list("id", "has_results"),
+            [(self.run_1.id, False), (self.run_2.id, True), (self.run_3.id, False), (self.run_4.id, True)]
+        )

arkindex/process/tests/test_corpus_worker_runs.py

+from django.urls import reverse
+from rest_framework import status
+
+from arkindex.documents.models import Corpus, Element
+from arkindex.ponos.models import Farm
+from arkindex.process.models import FeatureUsage, Process, ProcessMode, WorkerRun, WorkerVersion
+from arkindex.project.tests import FixtureAPITestCase
+from arkindex.users.models import Role
+
+
+class TestCorpusWorkerRuns(FixtureAPITestCase):
+    """
+    Test corpus worker run endpoints and methods
+    """
+
+    @classmethod
+    def setUpTestData(cls):
+        super().setUpTestData()
+        cls.process = Process.objects.create(
+            farm=Farm.objects.first(),
+            creator=cls.user,
+            mode=ProcessMode.Workers,
+            corpus=cls.corpus
+        )
+        cls.dla_worker_version = WorkerVersion.objects.get(worker__slug="dla")
+        cls.reco_worker_version = WorkerVersion.objects.get(worker__slug="reco")
+        cls.run_1 = WorkerRun.objects.create(
+            process=cls.process,
+            version=cls.dla_worker_version,
+            parents=[],
+            has_results=True
+        )
+
+        cls.private_corpus = Corpus.objects.create(name="private")
+        page_type = cls.private_corpus.types.create(
+            slug="page",
+            display_name="Page"
+        )
+        Element.objects.bulk_create(
+            Element(
+                corpus=cls.private_corpus,
+                name=f"page {i}",
+                type=page_type
+            )
+            for i in range(1, 11)
+        )
+        cls.private_process = Process.objects.create(
+            farm=Farm.objects.first(),
+            creator=cls.user,
+            mode=ProcessMode.Workers,
+            corpus=cls.private_corpus
+        )
+        cls.private_process.elements.set(Element.objects.filter(corpus=cls.private_corpus, type=page_type))
+        cls.private_process.save()
+        cls.run_2 = WorkerRun.objects.create(
+            process=cls.private_process,
+            version=cls.reco_worker_version,
+            parents=[],
+        )
+        cls.run_3 = WorkerRun.objects.create(
+            process=cls.private_process,
+            version=cls.dla_worker_version,
+            parents=[],
+            has_results=True
+        )
+
+        cls.local_process = Process.objects.get(mode=ProcessMode.Local, creator=cls.user)
+        cls.local_process.element = Element.objects.create(
+            corpus=cls.private_corpus,
+            type=cls.private_corpus.types.create(
+                slug="book",
+                display_name="Book",
+                folder=True,
+            ),
+            name="Book One",
+        )
+        cls.local_process.save()
+        cls.local_worker_version = WorkerVersion.objects.get(worker__slug="custom")
+        cls.user_local_run = WorkerRun.objects.get(process__mode=ProcessMode.Local, process__creator=cls.user)
+        cls.user_local_run.has_results = True
+        cls.user_local_run.save()
+
+    def test_list_requires_read_access(self):
+        self.client.force_login(self.user)
+        with self.assertNumQueries(5):
+            response = self.client.get(reverse("api:corpus-runs", kwargs={"pk": self.private_corpus.id}))
+            self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
+
+    def test_list(self):
+        self.private_corpus.memberships.create(user=self.user, level=Role.Guest.value)
+        self.client.force_login(self.user)
+        with self.assertNumQueries(8):
+            response = self.client.get(reverse("api:corpus-runs", kwargs={"pk": self.private_corpus.id}))
+            self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.json()["results"], [
+            {
+                "configuration": None,
+                "id": str(self.user_local_run.id),
+                "model_version": None,
+                "parents": [],
+                "process": {
+                    "activity_state": "disabled",
+                    "corpus": None,
+                    "element_id": str(self.local_process.element.id),
+                    "element_count": 0,
+                    "id": str(self.local_process.id),
+                    "mode": "local",
+                    "name": None,
+                    "state": "unscheduled",
+                    "use_cache": False,
+                },
+                "summary": "Worker Custom worker @ version 1",
+                "worker_version": {
+                    "configuration": {
+                        "custom": "value"
+                    },
+                    "created": self.local_worker_version.created.isoformat().replace("+00:00", "Z"),
+                    "docker_image": None,
+                    "docker_image_iid": None,
+                    "docker_image_name": None,
+                    "gpu_usage": "disabled",
+                    "id": str(self.local_worker_version.id),
+                    "model_usage": FeatureUsage.Disabled.value,
+                    "revision": None,
+                    "state": "created",
+                    "version": 1,
+                    "worker": {
+                        "id": str(self.local_worker_version.worker.id),
+                        "name": "Custom worker",
+                        "slug": "custom",
+                        "type": "custom"
+                    }
+                }
+            },
+            {
+                "id": str(self.run_3.id),
+                "worker_version": {
+                    "id": str(self.dla_worker_version.id),
+                    "configuration": {"test": 42},
+                    "docker_image": str(self.dla_worker_version.docker_image.id),
+                    "docker_image_iid": None,
+                    "docker_image_name": f"my_repo.fake/workers/worker/dla:{self.dla_worker_version.id}",
+                    "gpu_usage": "disabled",
+                    "model_usage": FeatureUsage.Disabled.value,
+                    "revision": {
+                        "id": str(self.dla_worker_version.revision.id),
+                        "author": "Test user",
+                        "commit_url": "http://my_repo.fake/workers/worker/commit/1337",
+                        "created": self.dla_worker_version.revision.created.isoformat().replace("+00:00", "Z"),
+                        "hash": "1337",
+                        "message": "My w0rk3r",
+                        "refs": []
+                    },
+                    "version": None,
+                    "created": self.dla_worker_version.created.isoformat().replace("+00:00", "Z"),
+                    "state": "available",
+                    "worker": {
+                        "id": str(self.dla_worker_version.worker.id),
+                        "name": "Document layout analyser",
+                        "slug": "dla",
+                        "type": "dla"
+                    }
+                },
+                "parents": [],
+                "model_version": None,
+                "configuration": None,
+                "process": {
+                    "id": str(self.private_process.id),
+                    "activity_state": "disabled",
+                    "corpus": str(self.private_corpus.id),
+                    "element_id": None,
+                    "element_count": 10,
+                    "mode": "workers",
+                    "name": None,
+                    "state": "unscheduled",
+                    "use_cache": False,
+                },
+                "summary": f"Worker Document layout analyser @ {str(self.dla_worker_version.id)[:6]}",
+            }
+        ])

arkindex/process/tests/test_create_process.py

@@ -97,10 +97,6 @@ class TestCreateProcess(FixtureAPITestCase):
             "use_gpu": False,
             "activity_state": ActivityState.Disabled.value,
             "element_name_contains": None,
-            "model_id": None,
-            "train_folder_id": None,
-            "validation_folder_id": None,
-            "test_folder_id": None,
         })
 
         # The process needs to be started to produce a workflow
@@ -153,10 +149,6 @@ class TestCreateProcess(FixtureAPITestCase):
             "use_gpu": False,
             "activity_state": ActivityState.Disabled.value,
             "element_name_contains": None,
-            "model_id": None,
-            "train_folder_id": None,
-            "validation_folder_id": None,
-            "test_folder_id": None,
         })
 
     def test_single_page_process(self):
@@ -374,10 +366,6 @@ class TestCreateProcess(FixtureAPITestCase):
             "activity_state": ActivityState.Disabled.value,
             "element_name_contains": None,
             "ml_class_id": str(self.ml_class.id),
-            "model_id": None,
-            "train_folder_id": None,
-            "validation_folder_id": None,
-            "test_folder_id": None,
         })
 
     def test_create_process_non_folder_no_zone(self):
@@ -430,10 +418,6 @@ class TestCreateProcess(FixtureAPITestCase):
             "activity_state": ActivityState.Disabled.value,
             "element_name_contains": None,
             "ml_class_id": None,
-            "model_id": None,
-            "train_folder_id": None,
-            "validation_folder_id": None,
-            "test_folder_id": None,
         })
 
     def test_create_process_mixed_selection_filters(self):
@@ -935,10 +919,6 @@ class TestCreateProcess(FixtureAPITestCase):
             "activity_state": ActivityState.Disabled.value,
             "element_name_contains": None,
             "ml_class_id": None,
-            "model_id": None,
-            "train_folder_id": None,
-            "validation_folder_id": None,
-            "test_folder_id": None,
         })
 
         task = process.tasks.get(slug__startswith="custom_")
@@ -982,10 +962,6 @@ class TestCreateProcess(FixtureAPITestCase):
                 "activity_state": ActivityState.Disabled.value,
                 "element_name_contains": None,
                 "ml_class_id": None,
-                "model_id": None,
-                "train_folder_id": None,
-                "validation_folder_id": None,
-                "test_folder_id": None,
             })
 
     def test_create_process_dataset_mode_no_selection(self):

arkindex/process/tests/test_processes.py

@@ -159,6 +159,7 @@ class TestProcesses(FixtureAPITestCase):
                     "updated": self.repository_process.updated.isoformat().replace("+00:00", "Z"),
                     "started": None,
                     "finished": None,
+                    "use_cache": False,
                 },
                 {
                     "id": str(self.user_img_process.id),
@@ -172,6 +173,7 @@ class TestProcesses(FixtureAPITestCase):
                     "updated": self.user_img_process.updated.isoformat().replace("+00:00", "Z"),
                     "started": None,
                     "finished": None,
+                    "use_cache": False,
                 },
                 {
                     "id": str(self.elts_process.id),
@@ -185,6 +187,7 @@ class TestProcesses(FixtureAPITestCase):
                     "updated": self.elts_process.updated.isoformat().replace("+00:00", "Z"),
                     "started": self.elts_process.started.isoformat().replace("+00:00", "Z"),
                     "finished": None,
+                    "use_cache": False,
                 },
                 {
                     "id": str(self.workers_process.id),
@@ -198,6 +201,7 @@ class TestProcesses(FixtureAPITestCase):
                     "updated": self.workers_process.updated.isoformat().replace("+00:00", "Z"),
                     "started": None,
                     "finished": None,
+                    "use_cache": False,
                 },
             ]
         })
@@ -236,6 +240,7 @@ class TestProcesses(FixtureAPITestCase):
                     "updated": self.user_img_process.updated.isoformat().replace("+00:00", "Z"),
                     "started": self.user_img_process.started.isoformat().replace("+00:00", "Z"),
                     "finished": None,
+                    "use_cache": False
                 },
             ],
         })
@@ -270,6 +275,7 @@ class TestProcesses(FixtureAPITestCase):
                     "updated": self.user_img_process.updated.isoformat().replace("+00:00", "Z"),
                     "started": None,
                     "finished": None,
+                    "use_cache": False
                 },
                 {
                     "id": str(self.workers_process.id),
@@ -283,6 +289,7 @@ class TestProcesses(FixtureAPITestCase):
                     "updated": self.workers_process.updated.isoformat().replace("+00:00", "Z"),
                     "started": None,
                     "finished": None,
+                    "use_cache": False
                 },
             ]
         })
@@ -349,6 +356,7 @@ class TestProcesses(FixtureAPITestCase):
                     "updated": self.local_process.updated.isoformat().replace("+00:00", "Z"),
                     "started": None,
                     "finished": None,
+                    "use_cache": False
                 },
             ]
         })
@@ -390,6 +398,7 @@ class TestProcesses(FixtureAPITestCase):
                     "updated": self.user_img_process.updated.isoformat().replace("+00:00", "Z"),
                     "started": None,
                     "finished": None,
+                    "use_cache": False
                 },
                 {
                     "id": str(self.workers_process.id),
@@ -403,6 +412,7 @@ class TestProcesses(FixtureAPITestCase):
                     "updated": self.workers_process.updated.isoformat().replace("+00:00", "Z"),
                     "started": None,
                     "finished": None,
+                    "use_cache": False
                 },
             ]
         })
@@ -582,6 +592,7 @@ class TestProcesses(FixtureAPITestCase):
                     "updated": unscheduled_with_tasks.updated.isoformat().replace("+00:00", "Z"),
                     "started": unscheduled_with_tasks.started.isoformat().replace("+00:00", "Z"),
                     "finished": None,
+                    "use_cache": False
                 },
             ]
         })
@@ -608,6 +619,7 @@ class TestProcesses(FixtureAPITestCase):
                     "updated": self.elts_process.updated.isoformat().replace("+00:00", "Z"),
                     "started": None,
                     "finished": None,
+                    "use_cache": False
                 },
                 {
                     "id": str(self.repository_process.id),
@@ -621,6 +633,7 @@ class TestProcesses(FixtureAPITestCase):
                     "updated": self.repository_process.updated.isoformat().replace("+00:00", "Z"),
                     "started": None,
                     "finished": None,
+                    "use_cache": False
                 },
                 {
                     "id": str(self.user_img_process.id),
@@ -634,6 +647,7 @@ class TestProcesses(FixtureAPITestCase):
                     "updated": self.user_img_process.updated.isoformat().replace("+00:00", "Z"),
                     "started": None,
                     "finished": None,
+                    "use_cache": False
                 },
                 {
                     "id": str(self.workers_process.id),
@@ -647,6 +661,7 @@ class TestProcesses(FixtureAPITestCase):
                     "updated": self.workers_process.updated.isoformat().replace("+00:00", "Z"),
                     "started": None,
                     "finished": None,
+                    "use_cache": False
                 },
             ]
         })
@@ -686,7 +701,6 @@ class TestProcesses(FixtureAPITestCase):
             "load_children": False,
             "ml_class_id": None,
             "mode": "files",
-            "model_id": None,
             "name": None,
             "revision": None,
             "state": "unscheduled",
@@ -702,11 +716,8 @@ class TestProcesses(FixtureAPITestCase):
                 }
             ],
             "template_id": None,
-            "test_folder_id": None,
-            "train_folder_id": None,
             "use_cache": False,
             "use_gpu": False,
-            "validation_folder_id": None,
             "created": self.user_img_process.created.isoformat().replace("+00:00", "Z"),
             "updated": self.user_img_process.updated.isoformat().replace("+00:00", "Z"),
             "started": self.user_img_process.started.isoformat().replace("+00:00", "Z"),
@@ -1354,10 +1365,6 @@ class TestProcesses(FixtureAPITestCase):
             "revision": None,
             "state": "unscheduled",
             "template_id": None,
-            "model_id": None,
-            "train_folder_id": None,
-            "validation_folder_id": None,
-            "test_folder_id": None,
             "farm": None,
             "tasks": [],
             "created": process.created.isoformat().replace("+00:00", "Z"),
@@ -1594,10 +1601,6 @@ class TestProcesses(FixtureAPITestCase):
             "revision": None,
             "state": "unscheduled",
             "template_id": None,
-            "model_id": None,
-            "train_folder_id": None,
-            "validation_folder_id": None,
-            "test_folder_id": None,
             "farm": None,
             "tasks": [],
             "created": process.created.isoformat().replace("+00:00", "Z"),

arkindex/process/tests/test_user_workerruns.py

@@ -85,13 +85,9 @@ class TestUserWorkerRuns(FixtureAPITestCase):
                 "corpus": None,
                 "id": str(self.local_process.id),
                 "mode": "local",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None
             },
             "summary": f"Worker Recognizer @ {str(self.version_1.id)[:6]}",
             "worker_version": {
@@ -133,13 +129,9 @@ class TestUserWorkerRuns(FixtureAPITestCase):
                 "corpus": None,
                 "id": str(self.local_process.id),
                 "mode": "local",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None
             },
             "summary": "Worker Custom worker @ version 1",
             "worker_version": {
@@ -251,13 +243,9 @@ class TestUserWorkerRuns(FixtureAPITestCase):
                 "corpus": None,
                 "id": str(self.local_process.id),
                 "mode": "local",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None
             }
         })
 
@@ -449,13 +437,9 @@ class TestUserWorkerRuns(FixtureAPITestCase):
                 "corpus": None,
                 "id": str(self.local_process.id),
                 "mode": "local",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None
             }
         })
 

arkindex/process/tests/test_workerruns.py

@@ -161,13 +161,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None
             },
             "summary": f"Worker Recognizer @ {str(self.version_1.id)[:6]}",
         }])
@@ -442,13 +438,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                         "activity_state": "disabled",
                         "corpus": str(self.corpus.id),
                         "mode": mode.value,
-                        "model_id": None,
                         "name": None,
                         "state": "unscheduled",
-                        "test_folder_id": None,
-                        "train_folder_id": None,
                         "use_cache": False,
-                        "validation_folder_id": None,
                     },
                     "summary": f"Worker Recognizer @ {str(self.version_1.id)[:6]}",
                 })
@@ -544,13 +536,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "summary": f"Worker Recognizer @ {str(self.version_1.id)[:6]} using configuration 'My config'",
         })
@@ -648,13 +636,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "summary": "Worker Recognizer @ main, develop, trunk, master, patate, pouet",
         })
@@ -690,13 +674,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "worker_version": {
                 "id": str(self.version_1.id),
@@ -754,13 +734,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "worker_version": {
                 "id": str(self.version_1.id),
@@ -808,13 +784,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": None,
                 "mode": "local",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "worker_version": {
                 "id": str(self.version_custom.id),
@@ -965,13 +937,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": None,
                 "mode": "local",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "summary": f"Worker Recognizer @ {str(self.version_1.id)[:6]}",
         })
@@ -1039,13 +1007,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None
             },
             "summary": f"Worker Recognizer @ {str(self.version_1.id)[:6]}",
         })
@@ -1245,13 +1209,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "worker_version": {
                 "id": str(self.version_1.id),
@@ -1311,13 +1271,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "worker_version": {
                 "id": str(self.version_1.id),
@@ -1385,13 +1341,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "worker_version": {
                 "id": str(self.version_1.id),
@@ -1771,13 +1723,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                         "activity_state": "disabled",
                         "corpus": str(self.corpus.id),
                         "mode": "workers",
-                        "model_id": None,
                         "name": None,
                         "state": "unscheduled",
-                        "test_folder_id": None,
-                        "train_folder_id": None,
                         "use_cache": False,
-                        "validation_folder_id": None,
                     },
                     "worker_version": {
                         "id": str(version_with_model.id),
@@ -1874,13 +1822,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "worker_version": {
                 "id": str(version_with_model.id),
@@ -1953,13 +1897,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "worker_version": {
                 "id": str(self.version_1.id),
@@ -2184,13 +2124,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "worker_version": {
                 "id": str(self.version_1.id),
@@ -2250,13 +2186,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "worker_version": {
                 "id": str(self.version_1.id),
@@ -2321,13 +2253,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "worker_version": {
                 "id": str(self.version_1.id),
@@ -2694,13 +2622,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                         "activity_state": "disabled",
                         "corpus": str(self.corpus.id),
                         "mode": "workers",
-                        "model_id": None,
                         "name": None,
                         "state": "unscheduled",
-                        "test_folder_id": None,
-                        "train_folder_id": None,
                         "use_cache": False,
-                        "validation_folder_id": None,
                     },
                     "worker_version": {
                         "id": str(version_with_model.id),
@@ -2794,13 +2718,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "worker_version": {
                 "id": str(version_with_model.id),
@@ -2872,13 +2792,9 @@ class TestWorkerRuns(FixtureAPITestCase):
                 "activity_state": "disabled",
                 "corpus": str(self.corpus.id),
                 "mode": "workers",
-                "model_id": None,
                 "name": None,
                 "state": "unscheduled",
-                "test_folder_id": None,
-                "train_folder_id": None,
                 "use_cache": False,
-                "validation_folder_id": None,
             },
             "worker_version": {
                 "id": str(self.version_1.id),

arkindex/project/api_v1.py

@@ -80,6 +80,7 @@ from arkindex.process.api import (
     BucketList,
     ClearProcess,
     CorpusProcess,
+    CorpusWorkerRunList,
     CorpusWorkersActivity,
     CorpusWorkerVersionList,
     CreateDockerWorkerVersion,
@@ -191,6 +192,7 @@ api = [
     path("corpus/<uuid:pk>/allowed-metadata/", CorpusAllowedMetaData.as_view(), name="corpus-allowed-metadata"),
     path("corpus/<uuid:corpus>/allowed-metadata/<uuid:pk>/", AllowedMetaDataEdit.as_view(), name="allowed-metadata-edit"),
     path("corpus/<uuid:pk>/versions/", CorpusWorkerVersionList.as_view(), name="corpus-versions"),
+    path("corpus/<uuid:pk>/worker-runs/", CorpusWorkerRunList.as_view(), name="corpus-runs"),
     path("corpus/<uuid:pk>/selection/", CorpusSelectionDestroy.as_view(), name="corpus-delete-selection"),
     path("corpus/<uuid:pk>/search/", CorpusSearch.as_view(), name="corpus-search"),
     path("corpus/<uuid:pk>/export/", CorpusExportAPIView.as_view(), name="corpus-export"),

arkindex/sql_validation/element_dot_delete.sql

@@ -35,30 +35,6 @@ OR element_id IN (
     SELECT element_id FROM documents_elementpath WHERE path && ARRAY['{id}'::uuid]
 ) ;
 
-UPDATE process_process
-SET train_folder_id = NULL
-WHERE train_folder_id = '{id}'::uuid
-    OR train_folder_id IN
-        (SELECT element_id
-         FROM documents_elementpath
-         WHERE path && ARRAY['{id}'::uuid] ) ;
-
-UPDATE process_process
-SET validation_folder_id = NULL
-WHERE validation_folder_id = '{id}'::uuid
-    OR validation_folder_id IN
-        (SELECT element_id
-         FROM documents_elementpath
-         WHERE path && ARRAY['{id}'::uuid] ) ;
-
-UPDATE process_process
-SET test_folder_id = NULL
-WHERE test_folder_id = '{id}'::uuid
-    OR test_folder_id IN
-        (SELECT element_id
-         FROM documents_elementpath
-         WHERE path && ARRAY['{id}'::uuid] ) ;
-
 DELETE FROM documents_selection selection
 WHERE element_id = '{id}'::uuid
 OR element_id IN (