From 281fc11968c9827476b8e19212dd4c7714dfe977 Mon Sep 17 00:00:00 2001
From: Erwan Rouchet <rouchet@teklia.com>
Date: Fri, 20 Nov 2020 11:50:12 +0100
Subject: [PATCH] Add ListCorpusWorkerVersions endpoint
---
arkindex/dataimport/api.py | 23 ++++++++-
arkindex/dataimport/serializers/workers.py | 2 +
arkindex/dataimport/tests/test_workers.py | 57 ++++++++++++++++++++++
arkindex/project/api_v1.py | 2 +
4 files changed, 83 insertions(+), 1 deletion(-)
diff --git a/arkindex/dataimport/api.py b/arkindex/dataimport/api.py
index bff4120c4d..c2e78e385d 100644
--- a/arkindex/dataimport/api.py
+++ b/arkindex/dataimport/api.py
@@ -4,7 +4,7 @@ from uuid import UUID
from django.conf import settings
from django.core.exceptions import PermissionDenied
from django.db import transaction
-from django.db.models import F, Max, Q
+from django.db.models import Count, F, Max, Q
from django.http.response import Http404
from django.shortcuts import get_object_or_404
from rest_framework import status
@@ -743,6 +743,27 @@ class WorkerVersionList(ListCreateAPIView):
return Response(WorkerVersionSerializer(version).data, status=reponse_status)
+class CorpusWorkerVersionList(CorpusACLMixin, ListAPIView):
+ """
+ List worker versions used by elements of a given corpus.
+ """
+ permission_classes = (IsVerified, )
+ pagination_class = None
+ serializer_class = WorkerVersionSerializer
+ openapi_overrides = {
+ 'operationId': 'ListCorpusWorkerVersions',
+ 'tags': ['ml'],
+ }
+
+ def get_queryset(self):
+ return WorkerVersion.objects \
+ .filter(elements__corpus_id=self.kwargs['pk']) \
+ .select_related('revision__repo', 'worker__repository') \
+ .prefetch_related('revision__refs', 'revision__versions') \
+ .order_by('-revision__created') \
+ .annotate(element_count=Count('id'))
+
+
class WorkerVersionRetrieve(CorpusACLMixin, RetrieveUpdateAPIView):
"""
Retrieve a specific worker version
diff --git a/arkindex/dataimport/serializers/workers.py b/arkindex/dataimport/serializers/workers.py
index 8033ca88c2..4f3031a56b 100644
--- a/arkindex/dataimport/serializers/workers.py
+++ b/arkindex/dataimport/serializers/workers.py
@@ -34,6 +34,7 @@ class WorkerVersionSerializer(serializers.ModelSerializer):
# ModelSerializer does not yet support Django 3.1's JSONField
# https://github.com/encode/django-rest-framework/pull/7467
configuration = serializers.JSONField()
+ element_count = serializers.IntegerField(read_only=True)
class Meta:
model = WorkerVersion
@@ -46,6 +47,7 @@ class WorkerVersionSerializer(serializers.ModelSerializer):
'docker_image_name',
'state',
'worker',
+ 'element_count',
)
read_only_fields = ('docker_image_name',)
# Avoid loading all revisions and all Ponos artifacts when opening this endpoint in a browser
diff --git a/arkindex/dataimport/tests/test_workers.py b/arkindex/dataimport/tests/test_workers.py
index 47754685d4..5f21c58a96 100644
--- a/arkindex/dataimport/tests/test_workers.py
+++ b/arkindex/dataimport/tests/test_workers.py
@@ -1,3 +1,5 @@
+from unittest import expectedFailure
+
from django.urls import reverse
from rest_framework import status
@@ -489,3 +491,58 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
self.version_1.configuration = {'docker': {'command': 'mysupercommand'}}
self.assertEqual(self.version_1.docker_command, 'mysupercommand')
self.version_1.configuration = {"test": "test1"}
+
+ def test_corpus_worker_version_requires_login(self):
+ with self.assertNumQueries(0):
+ response = self.client.get(reverse('api:corpus-versions', kwargs={'pk': self.corpus.id}))
+ self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+
+ @expectedFailure
+ def test_corpus_worker_version_requires_verified(self):
+ """
+ This test fails due to a bug in the IsVerified permission class.
+ https://gitlab.com/arkindex/backend/-/issues/554
+ """
+ self.user.verified_email = False
+ self.user.save()
+ self.client.force_login(self.user)
+ with self.assertNumQueries(2):
+ response = self.client.get(reverse('api:corpus-versions', kwargs={'pk': self.corpus.id}))
+ self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+
+ def test_corpus_worker_version_list(self):
+ self.client.force_login(self.user)
+
+ self.corpus.elements.filter(type__slug='word').update(worker_version=self.version_1)
+
+ with self.assertNumQueries(5):
+ response = self.client.get(reverse('api:corpus-versions', kwargs={'pk': self.corpus.id}))
+ self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+ self.assertListEqual(response.json(), [
+ {
+ 'id': str(self.version_1.id),
+ 'configuration': {'test': 42},
+ 'revision': {
+ 'id': str(self.version_1.revision_id),
+ 'state': 'available',
+ 'hash': '1337',
+ 'author': 'Test user',
+ 'message': 'My w0rk3r',
+ 'created': '2020-02-02T01:23:45.678000Z',
+ 'commit_url': 'http://my_repo.fake/workers/worker/commit/1337',
+ 'refs': []
+ },
+ 'docker_image': str(self.version_1.docker_image_id),
+ 'docker_image_iid': None,
+ 'docker_image_name': self.version_1.docker_image_name,
+ 'state': 'available',
+ 'worker': {
+ 'id': str(self.worker_1.id),
+ 'name': 'Recognizer',
+ 'slug': 'reco',
+ 'type': 'recognizer'
+ },
+ 'element_count': 9
+ }
+ ])
diff --git a/arkindex/project/api_v1.py b/arkindex/project/api_v1.py
index be8ade7bc3..1dd403b540 100644
--- a/arkindex/project/api_v1.py
+++ b/arkindex/project/api_v1.py
@@ -6,6 +6,7 @@ from rest_framework.schemas import get_schema_view
from arkindex.dataimport.api import (
AvailableRepositoriesList,
+ CorpusWorkerVersionList,
CorpusWorkflow,
DataFileCreate,
DataFileList,
@@ -139,6 +140,7 @@ api = [
path('corpus/<uuid:pk>/roles/', CorpusRoles.as_view(), name='corpus-roles'),
path('corpus/<uuid:pk>/ml-stats/', CorpusMLStats.as_view(), name='corpus-ml-stats'),
path('corpus/<uuid:pk>/allowed-metadata/', CorpusAllowedMetaData.as_view(), name='corpus-allowed-metadata'),
+ path('corpus/<uuid:pk>/versions/', CorpusWorkerVersionList.as_view(), name='corpus-versions'),
# Moderation
path('ml-classes/', MLClassList.as_view(), name='mlclass-list'),
--
GitLab