Merge branch 'list-worker-activity' into 'master'

Add ListWorkerActivity

Closes #892

See merge request !1542

arkindex/dataimport/api.py

@@ -1477,3 +1477,75 @@ class ProcessWorkersActivity(ProcessACLMixin, ListAPIView):
             status=status.HTTP_200_OK,
             data=WorkerStatisticsSerializer(stats, many=True).data
         )
+
+
+@extend_schema_view(
+    get=extend_schema(
+        operation_id='ListWorkerActivities',
+        tags=['imports'],
+        parameters=[
+            OpenApiParameter(
+                'process_id',
+                type=UUID,
+                description='Filter worker activities by process ID',
+                required=False,
+            ),
+            OpenApiParameter(
+                'state',
+                enum=[state.value for state in WorkerActivityState],
+                description='Filter worker activities by state',
+                required=False,
+            ),
+        ]
+    )
+)
+class WorkerActivityList(CorpusACLMixin, ProcessACLMixin, ListAPIView):
+    """
+    List worker activities in a corpus.
+
+    Requires guest access to the corpus.
+    """
+    permission_classes = (IsVerified, )
+    serializer_class = WorkerActivitySerializer
+
+    @cached_property
+    def corpus(self):
+        return self.get_corpus(self.kwargs['corpus'])
+
+    def get_queryset(self):
+        return WorkerActivity.objects.filter(element__corpus=self.corpus).order_by('id')
+
+    def filter_queryset(self, queryset):
+        errors = {}
+
+        process_id = None
+        if 'process_id' in self.request.query_params:
+            try:
+                process_id = UUID(self.request.query_params['process_id'])
+            except (TypeError, ValueError):
+                errors['process_id'] = ['Process ID should be an UUID.']
+
+        if process_id:
+            try:
+                # This uses a select_related because the process access level check looks for the corpus
+                process = DataImport.objects.select_related('corpus').get(id=process_id, corpus=self.corpus)
+            except DataImport.DoesNotExist:
+                errors['process_id'] = ['This process does not exist.']
+            else:
+                access_level = self.process_access_level(process)
+                if not access_level or access_level < Role.Guest.value:
+                    errors['process_id'] = ['You do not have a guest access to this process.']
+                queryset = queryset.filter(process=process)
+
+        if 'state' in self.request.query_params:
+            try:
+                state = WorkerActivityState(self.request.query_params['state'])
+            except (TypeError, ValueError):
+                errors['state'] = [f"{self.request.query_params['state']!r} is not a valid WorkerActivity state"]
+            else:
+                queryset = queryset.filter(state=state)
+
+        if errors:
+            raise ValidationError(errors)
+
+        return queryset

arkindex/dataimport/serializers/workers.py

@@ -167,15 +167,31 @@ class WorkerActivitySerializer(serializers.ModelSerializer):
     """
     state = EnumField(WorkerActivityState)
     element_id = serializers.UUIDField()
-    process_id = serializers.PrimaryKeyRelatedField(queryset=DataImport.objects.all())
+    process_id = serializers.PrimaryKeyRelatedField(
+        queryset=DataImport.objects.all(),
+        # Avoid loading up to 1000 processes when opening this endpoint in a browser
+        style={'base_template': 'input.html'},
+    )
+    configuration_id = serializers.UUIDField(read_only=True, allow_null=True, required=False)
+    worker_version_id = serializers.UUIDField(read_only=True)
 
     class Meta:
         model = WorkerActivity
         fields = (
             'element_id',
             'process_id',
+            'configuration_id',
+            'worker_version_id',
+            'created',
+            'updated',
             'state',
         )
+        read_only_fields = (
+            'created',
+            'updated',
+            'configuration_id',
+            'worker_version_id',
+        )
 
 
 class WorkerConfigurationSerializer(serializers.ModelSerializer):

arkindex/dataimport/tests/test_workeractivity.py

@@ -14,7 +14,7 @@ from arkindex.dataimport.models import (
     WorkerVersion,
 )
 from arkindex.dataimport.tasks import initialize_activity
-from arkindex.documents.models import Classification, ClassificationState, Element, MLClass
+from arkindex.documents.models import Classification, ClassificationState, Corpus, Element, MLClass
 from arkindex.project.tests import FixtureTestCase
 from arkindex.users.models import User
 
@@ -328,6 +328,7 @@ class TestWorkerActivity(FixtureTestCase):
             )
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertDictEqual(response.json(), {
+            'configuration_id': None,
             'element_id': str(self.element.id),
             'process_id': str(process2.id),
             'state': WorkerActivityState.Started.value,
@@ -374,3 +375,173 @@ class TestWorkerActivity(FixtureTestCase):
         process.refresh_from_db()
         self.assertEqual(process.activity_state, ActivityState.Error)
         self.assertEqual(WorkerActivity.objects.filter(element__type__slug='volume').count(), 0)
+
+    def test_list_requires_login(self):
+        with self.assertNumQueries(0):
+            response = self.client.get(reverse('api:corpus-activity', kwargs={'corpus': self.corpus.id}))
+            self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+
+    def test_list_requires_verified(self):
+        self.user.verified_email = False
+        self.user.save()
+        self.client.force_login(self.user)
+        with self.assertNumQueries(2):
+            response = self.client.get(reverse('api:corpus-activity', kwargs={'corpus': self.corpus.id}))
+            self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+
+    def test_list_corpus_acl(self):
+        private_corpus = Corpus.objects.create(name="can't touch this")
+        self.client.force_login(self.user)
+        with self.assertNumQueries(6):
+            response = self.client.get(reverse('api:corpus-activity', kwargs={'corpus': private_corpus.id}))
+            self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+        self.assertDictEqual(response.json(), {'detail': 'You do not have guest access to this corpus.'})
+
+    def test_list_corpus_not_found(self):
+        corpus_id = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'
+        self.assertNotEqual(str(self.corpus.id), corpus_id)
+        self.client.force_login(self.user)
+        with self.assertNumQueries(3):
+            response = self.client.get(reverse('api:corpus-activity', kwargs={'corpus': corpus_id}))
+            self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
+
+    def test_list(self):
+        self.client.force_login(self.user)
+        with self.assertNumQueries(5):
+            response = self.client.get(reverse('api:corpus-activity', kwargs={'corpus': self.corpus.id}))
+            self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertDictEqual(response.json(), {
+            'count': 1,
+            'number': 1,
+            'previous': None,
+            'next': None,
+            'results': [
+                {
+                    'created': self.activity.created.isoformat().replace('+00:00', 'Z'),
+                    'updated': self.activity.updated.isoformat().replace('+00:00', 'Z'),
+                    'element_id': str(self.element.id),
+                    'process_id': str(self.process.id),
+                    'worker_version_id': str(self.worker_version.id),
+                    'configuration_id': str(self.configuration.id),
+                    'state': 'queued'
+                }
+            ]
+        })
+
+    def test_list_filter_process_wrong_corpus(self):
+        private_corpus = Corpus.objects.create(name="can't touch this")
+        process = private_corpus.imports.create(
+            mode=DataImportMode.Workers,
+            creator=self.superuser,
+        )
+        self.client.force_login(self.user)
+        with self.assertNumQueries(4):
+            response = self.client.get(
+                reverse('api:corpus-activity', kwargs={'corpus': self.corpus.id}),
+                {'process_id': str(process.id)}
+            )
+            self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertDictEqual(response.json(), {'process_id': ['This process does not exist.']})
+
+    def test_list_filter_process(self):
+        process = self.corpus.imports.create(
+            mode=DataImportMode.Workers,
+            creator=self.user,
+        )
+        activity = process.activities.create(
+            element=self.element,
+            worker_version=self.worker_version,
+        )
+        self.client.force_login(self.user)
+
+        with self.assertNumQueries(9):
+            response = self.client.get(
+                reverse('api:corpus-activity', kwargs={'corpus': self.corpus.id}),
+                {'process_id': str(process.id)}
+            )
+            self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        self.assertDictEqual(response.json(), {
+            'count': 1,
+            'number': 1,
+            'previous': None,
+            'next': None,
+            'results': [
+                {
+                    'created': activity.created.isoformat().replace('+00:00', 'Z'),
+                    'updated': activity.updated.isoformat().replace('+00:00', 'Z'),
+                    'element_id': str(self.element.id),
+                    'process_id': str(process.id),
+                    'worker_version_id': str(self.worker_version.id),
+                    'configuration_id': None,
+                    'state': 'queued'
+                }
+            ]
+        })
+
+    def test_list_filter_state(self):
+        element = Element.objects.get(name='Volume 1')
+        activity = element.activities.create(
+            process=self.process,
+            worker_version=self.worker_version,
+            state=WorkerActivityState.Processed,
+        )
+        self.assertEqual(WorkerActivity.objects.filter(state=WorkerActivityState.Processed).count(), 1)
+        self.client.force_login(self.user)
+
+        with self.assertNumQueries(5):
+            response = self.client.get(
+                reverse('api:corpus-activity', kwargs={'corpus': self.corpus.id}),
+                {'state': 'processed'}
+            )
+            self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        self.assertDictEqual(response.json(), {
+            'count': 1,
+            'number': 1,
+            'previous': None,
+            'next': None,
+            'results': [
+                {
+                    'created': activity.created.isoformat().replace('+00:00', 'Z'),
+                    'updated': activity.updated.isoformat().replace('+00:00', 'Z'),
+                    'element_id': str(element.id),
+                    'process_id': str(self.process.id),
+                    'worker_version_id': str(self.worker_version.id),
+                    'configuration_id': None,
+                    'state': 'processed'
+                }
+            ]
+        })
+
+    def test_list_invalid_filters(self):
+        self.client.force_login(self.superuser)
+        cases = [
+            (
+                {'process_id': 'a'},
+                {'process_id': ['Process ID should be an UUID.']},
+            ),
+            (
+                {'process_id': 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'},
+                {'process_id': ['This process does not exist.']},
+            ),
+            (
+                {'state': 'lol'},
+                {'state': ["'lol' is not a valid WorkerActivity state"]},
+            ),
+            (
+                {'process_id': 'a', 'state': 'lol'},
+                {
+                    'process_id': ['Process ID should be an UUID.'],
+                    'state': ["'lol' is not a valid WorkerActivity state"],
+                },
+            ),
+        ]
+        for filters, expected_errors in cases:
+            with self.subTest(**filters):
+                response = self.client.get(
+                    reverse('api:corpus-activity', kwargs={'corpus': self.corpus.id}),
+                    filters
+                )
+                self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+                self.assertDictEqual(response.json(), expected_errors)

arkindex/project/api_v1.py

@@ -22,6 +22,7 @@ from arkindex.dataimport.api import (
     RevisionRetrieve,
     StartProcess,
     UpdateWorkerActivity,
+    WorkerActivityList,
     WorkerConfigurationList,
     WorkerConfigurationRetrieve,
     WorkerList,
@@ -152,6 +153,7 @@ api = [
     path('corpus/<uuid:pk>/export/', CorpusExportAPIView.as_view(), name='corpus-export'),
     path('corpus/<uuid:corpus>/workerversion/<uuid:version>/results/', WorkerResultsDestroy.as_view(), name='worker-delete-results'),
     path('corpus/<uuid:corpus>/activity-stats/', CorpusWorkersActivity.as_view(), name='corpus-activity-stats'),
+    path('corpus/<uuid:corpus>/activity/', WorkerActivityList.as_view(), name='corpus-activity'),
     path('export/<uuid:pk>/', DownloadExport.as_view(), name='download-export'),
 
     # Moderation