Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • arkindex/backend
1 result
Show changes
Commits on Source (8)
Hide whitespace changes
Inline Side-by-side
Showing
with 669 additions and 53 deletions
VERSION
View file @ e4b0fd82
0.14.4-hotfix1
0.14.5-beta2
arkindex/documents/api/elements.py
View file @ e4b0fd82
......@@ -10,7 +10,7 @@ from django.db.models.functions import Cast
from django.shortcuts import get_object_or_404
from django.utils.functional import cached_property
from psycopg2.extras import execute_values
from rest_framework import serializers, status
from rest_framework import permissions, serializers, status
from rest_framework.exceptions import NotFound, ValidationError
from rest_framework.generics import (
CreateAPIView,
......@@ -21,6 +21,7 @@ from rest_framework.generics import (
RetrieveUpdateDestroyAPIView,
UpdateAPIView,
)
from rest_framework.mixins import DestroyModelMixin
from rest_framework.response import Response
from arkindex.dataimport.models import WorkerVersion
......@@ -56,7 +57,7 @@ from arkindex.project.openapi import AutoSchema
from arkindex.project.pagination import PageNumberPagination
from arkindex.project.permissions import IsAuthenticated, IsVerified, IsVerifiedOrReadOnly
from arkindex.project.tools import BulkMap
from arkindex.project.triggers import corpus_delete
from arkindex.project.triggers import corpus_delete, element_trash
from arkindex_common.enums import TranscriptionType
classifications_queryset = Classification.objects.select_related('ml_class', 'source').order_by('-confidence')
......@@ -124,6 +125,7 @@ class ElementsListMixin(object):
"""
serializer_class = ElementListSerializer
queryset = Element.objects.all()
permission_classes = (IsVerifiedOrReadOnly, )
openapi_overrides = {
'security': [],
'tags': ['elements'],
......@@ -135,7 +137,7 @@ class ElementsListMixin(object):
'required': False,
'schema': {
'type': 'string',
}
},
},
{
'name': 'name',
......@@ -153,7 +155,8 @@ class ElementsListMixin(object):
'schema': {'anyOf': [
{'type': 'string', 'format': 'uuid'},
{'type': 'boolean'},
]}
]},
'x-methods': ['get'],
},
{
'name': 'folder',
......@@ -185,7 +188,8 @@ class ElementsListMixin(object):
'schema': {
'type': 'boolean',
'default': False,
}
},
'x-methods': ['get'],
},
{
'name': 'with_has_children',
......@@ -198,7 +202,8 @@ class ElementsListMixin(object):
'schema': {
'type': 'boolean',
'default': False
}
},
'x-methods': ['get'],
},
{
'name': 'If-Modified-Since',
......@@ -214,20 +219,66 @@ class ElementsListMixin(object):
'schema': {
# Cannot use format: date-time here as HTTP headers do not use ISO 8601
'type': 'string'
}
},
'x-methods': ['get'],
},
{
'name': 'delete_children',
'required': False,
'in': 'query',
'description': 'Delete all child elements of those elements recursively.',
'schema': {
'type': 'boolean',
'default': True
},
'x-methods': ['delete'],
}
]
}
def initial(self, *args, **kwargs):
"""
Adds a `self.clean_params` attribute that holds any query params or headers,
filtered by their allowed HTTP method.
Raise HTTP 400 early if any filter is not allowed in this method.
Does not perform validation on the filter values.
"""
super().initial(*args, **kwargs)
self.clean_params = {}
errors = {}
for param in self.openapi_overrides['parameters']:
if param['in'] == 'header':
origin_dict = self.request.headers
elif param['in'] == 'query':
origin_dict = self.request.query_params
else:
raise NotImplementedError
if param['name'] not in origin_dict:
continue
if param.get('x-methods') and self.request.method.lower() not in param['x-methods']:
errors[param['name']] = [f'This parameter is not allowed in {self.request.method} requests.']
else:
self.clean_params[param['name']] = origin_dict[param['name']]
if errors:
raise ValidationError(errors)
@cached_property
def selected_corpus(self):
corpus_id = self.kwargs.get('corpus')
if corpus_id is None:
return
# When a corpus is specified, check independently that it's readable
# by the current user. This prevents comparing the corpus of every element
return self.get_corpus(corpus_id)
# When a corpus is specified, check independently that it's readable/writable
# by the current user. This prevents comparing the corpus of every element.
# Require write rights for 'unsafe' methods (POST/PUT/PATCH/DELETE)
return self.get_corpus(
corpus_id,
right=Right.Read if self.request.method in permissions.SAFE_METHODS else Right.Write,
)
def get_filters(self):
filters = {}
......@@ -237,16 +288,16 @@ class ElementsListMixin(object):
filters['corpus__in'] = Corpus.objects.readable(self.request.user)
if 'name' in self.request.query_params:
filters['name__icontains'] = self.request.query_params['name']
filters['name__icontains'] = self.clean_params['name']
if 'type' in self.request.query_params:
filters['type__slug'] = self.request.query_params['type']
filters['type__slug'] = self.clean_params['type']
only_folder = self.request.query_params.get('folder')
only_folder = self.clean_params.get('folder')
if only_folder is not None:
filters['type__folder'] = only_folder.lower() not in ('false', '0')
if 'worker_version' in self.request.query_params:
if 'worker_version' in self.clean_params:
try:
worker_version_id = uuid.UUID(self.request.query_params['worker_version'])
except (TypeError, ValueError):
......@@ -267,7 +318,7 @@ class ElementsListMixin(object):
- elements with any best classes
- elements with a specific best class
"""
class_filter = self.request.query_params.get('best_class')
class_filter = self.clean_params.get('best_class')
if class_filter is None:
return
......@@ -292,7 +343,7 @@ class ElementsListMixin(object):
def get_prefetch(self):
prefetch = {'corpus', 'zone__image__server', 'type'}
with_best_classes = self.request.query_params.get('with_best_classes')
with_best_classes = self.clean_params.get('with_best_classes')
if with_best_classes and with_best_classes.lower() not in ('false', '0'):
prefetch.add(best_classifications_prefetch)
......@@ -313,7 +364,7 @@ class ElementsListMixin(object):
# Use queryset.distinct() whenever best_class is defined
queryset = queryset.filter(class_filters).distinct()
with_has_children = self.request.query_params.get('with_has_children')
with_has_children = self.clean_params.get('with_has_children')
if with_has_children and with_has_children.lower() not in ('false', '0'):
queryset = BulkMap(_fetch_has_children, queryset)
......@@ -328,7 +379,7 @@ class ElementsListMixin(object):
queryset = queryset.iterable
assert isinstance(queryset, QuerySet), 'A Django QuerySet is required to check for modified elements'
modified_since_string = self.request.headers.get('If-Modified-Since')
modified_since_string = self.clean_params.get('If-Modified-Since')
if not modified_since_string:
return False
......@@ -358,6 +409,16 @@ class ElementsListMixin(object):
serializer = self.get_serializer(queryset, many=True)
return Response(serializer.data)
def delete(self, request, *args, **kwargs):
queryset = self.filter_queryset(self.get_queryset())
delete_children = self.clean_params.get('delete_children', '').lower() not in ('false', '0')
if not queryset.exists():
raise NotFound
element_trash(queryset, delete_children=delete_children)
return Response(status=status.HTTP_204_NO_CONTENT)
class DeprecatedElementsList(DeprecatedMixin, RetrieveAPIView):
"""
......@@ -376,7 +437,7 @@ class DeprecatedElementsList(DeprecatedMixin, RetrieveAPIView):
)
class CorpusElements(ElementsListMixin, CorpusACLMixin, ListAPIView):
class CorpusElements(ElementsListMixin, CorpusACLMixin, DestroyModelMixin, ListAPIView):
"""
List elements in a corpus and filter by type, name, ML class
"""
......@@ -396,7 +457,7 @@ class CorpusElements(ElementsListMixin, CorpusACLMixin, ListAPIView):
@property
def is_top_level(self):
return self.request.query_params.get('top_level') not in (None, 'false', '0')
return self.clean_params.get('top_level') not in (None, 'false', '0')
def get_queryset(self):
# Should not be possible due to the URL
......@@ -421,7 +482,7 @@ class CorpusElements(ElementsListMixin, CorpusACLMixin, ListAPIView):
return super().get_order_by()
class ElementParents(ElementsListMixin, ListAPIView):
class ElementParents(ElementsListMixin, DestroyModelMixin, ListAPIView):
"""
List all parents of an element
"""
......@@ -445,14 +506,22 @@ class ElementParents(ElementsListMixin, ListAPIView):
@property
def is_recursive(self):
recursive_param = self.request.query_params.get('recursive')
recursive_param = self.clean_params.get('recursive')
return recursive_param is not None and recursive_param.lower() not in ('false', '0')
def get_queryset(self):
if self.request.method in permissions.SAFE_METHODS:
corpora = Corpus.objects.readable(self.request.user)
else:
corpora = Corpus.objects.writable(self.request.user)
if not Element.objects.filter(id=self.kwargs['pk'], corpus__in=corpora).exists():
raise NotFound
return Element.objects.get_ascending(self.kwargs['pk'], recursive=self.is_recursive)
class ElementChildren(ElementsListMixin, ListAPIView):
class ElementChildren(ElementsListMixin, DestroyModelMixin, ListAPIView):
"""
List all children of an element
"""
......@@ -476,7 +545,7 @@ class ElementChildren(ElementsListMixin, ListAPIView):
def get_filters(self):
filters = super().get_filters()
recursive_param = self.request.query_params.get('recursive')
recursive_param = self.clean_params.get('recursive')
if recursive_param is None or recursive_param.lower() in ('false', '0'):
# Only list direct children
......@@ -492,6 +561,14 @@ class ElementChildren(ElementsListMixin, ListAPIView):
return ('paths__ordering', )
def get_queryset(self):
if self.request.method in permissions.SAFE_METHODS:
corpora = Corpus.objects.readable(self.request.user)
else:
corpora = Corpus.objects.writable(self.request.user)
if not Element.objects.filter(id=self.kwargs['pk'], corpus__in=corpora).exists():
raise NotFound
return Element.objects.get_descending(self.kwargs['pk'])
......
arkindex/documents/api/ml.py
View file @ e4b0fd82
......@@ -271,7 +271,10 @@ class ElementTranscriptionsBulk(CreateAPIView):
.only('id', 'zone_id')
}
# Load the paths immediately to avoid iterating over them for each element
paths = list(self.element.paths.all())
paths = list(self.element.paths.values_list('path', flat=True))
if not paths:
# Support top level elements, by adding an empty initial path to trigger loops below
paths = [[]]
next_path_ordering = self.element.get_next_order(elt_type)
for annotation in annotations:
# Look for a direct children with the right type and zone
......@@ -289,7 +292,7 @@ class ElementTranscriptionsBulk(CreateAPIView):
children[annotation['zone_id']] = annotation['element']
missing_elements.append(annotation['element'])
for parent_path in paths:
new_path = parent_path.path + [self.element.id]
new_path = parent_path + [self.element.id]
# Add the children to all of its parent paths
missing_paths.append(ElementPath(
element=annotation['element'],
......
arkindex/documents/managers.py
View file @ e4b0fd82
import uuid
from itertools import chain, groupby
from django.db import connections, models
import django
from django.db import DJANGO_VERSION_PICKLE_KEY, connections, models
from arkindex.project.fields import Unnest
class ElementQuerySet(models.QuerySet):
def __getstate__(self):
"""
Django's QuerySets can be pickled, but when they are, they first call self._fetch_all()
to fetch every row into the result cache and store it along with the pickled data,
because one could expect to 'save' a queryset that way.
We only pickle querysets for RQ tasks that will call .trash() later;
this will never use the result cache as it does not load rows into memory, so we ignore that step.
"""
return {**self.__dict__, DJANGO_VERSION_PICKLE_KEY: django.__version__}
def trash(self, delete_children=True):
"""Performant deletion of any element queryset"""
from arkindex.documents.models import Element, MetaData, Transcription, TranscriptionEntity, Classification, Selection
......
arkindex/documents/tasks.py
View file @ e4b0fd82
......@@ -8,6 +8,7 @@ from django_rq import job
from arkindex.dataimport.models import DataImport, DataImportElement, WorkerRun
from arkindex.documents.indexer import Indexer
from arkindex.documents.managers import ElementQuerySet
from arkindex.documents.models import (
Classification,
Corpus,
......@@ -193,3 +194,8 @@ def corpus_delete(corpus_id: str) -> None:
logger.info(f'Deleted {deleted_count} {queryset.model.__name__}')
logger.info(f'Deleted corpus {corpus_id}')
@job('high')
def element_trash(queryset: ElementQuerySet, delete_children: bool) -> None:
queryset.trash(delete_children=delete_children)
arkindex/documents/tests/test_bulk_element_transcriptions.py
View file @ e4b0fd82
......@@ -319,3 +319,68 @@ class TestBulkElementTranscriptions(FixtureAPITestCase):
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(response.json(), {'element': ['Cannot create transcriptions on an element without a zone.']})
@patch('arkindex.project.triggers.tasks.reindex_start.delay')
def test_top_level_element(self, delay_mock):
"""
Create transcriptions on a top level element
"""
# Create a top level page
top_level = self.corpus.elements.create(
type=self.page.type,
name='Top level page',
zone=self.page.zone,
)
transcriptions = [
([[13, 37], [133, 37], [133, 137], [13, 137], [13, 37]], 'Hello world !', 0.1337),
([[24, 42], [64, 42], [64, 142], [24, 142], [24, 42]], 'I <3 JavaScript', 0.42),
]
data = {
'element_type': 'text_line',
'transcription_type': 'line',
'worker_version': str(self.worker_version.id),
'transcriptions': [{
'polygon': poly,
'text': text,
'score': score
} for poly, text, score in transcriptions]
}
self.client.force_login(self.user)
response = self.client.post(
reverse('api:element-transcriptions-bulk', kwargs={'pk': top_level.id}),
format='json',
data=data
)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
created_elts = Element.objects.get_descending(top_level.id)
self.assertEqual(created_elts.count(), 2)
self.assertTrue(all(map(lambda elt: elt.zone.image == self.page.zone.image, created_elts)))
self.assertListEqual(
[
(elt.paths.first().ordering, elt.name, elt.zone.polygon.coords)
for elt in created_elts
],
[
(0, '1', ((13, 37), (13, 137), (133, 137), (133, 37), (13, 37))),
(1, '2', ((24, 42), (24, 142), (64, 142), (64, 42), (24, 42)))
]
)
self.assertCountEqual(
created_elts.values_list('transcriptions__type', 'transcriptions__text', 'transcriptions__source', 'transcriptions__worker_version'),
[
(TranscriptionType.Line, ('Hello world !'), None, self.worker_version.id),
(TranscriptionType.Line, ('I <3 JavaScript'), None, self.worker_version.id)
]
)
self.assertEqual(delay_mock.call_count, 1)
self.assertEqual(delay_mock.call_args, call(
element_id=str(top_level.id),
corpus_id=None,
entity_id=None,
transcriptions=True,
elements=True,
entities=False,
drop=False,
))
arkindex/documents/tests/test_children_elements.py
View file @ e4b0fd82
......@@ -104,7 +104,7 @@ class TestChildrenElements(FixtureAPITestCase):
def test_element_children_worker_version(self):
self.corpus.elements.filter(name__contains='page 1r').update(worker_version=self.worker_version)
with self.assertNumQueries(8):
with self.assertNumQueries(9):
response = self.client.get(
reverse('api:elements-children', kwargs={'pk': str(self.vol.id)}),
data={'worker_version': str(self.worker_version.id)}
......@@ -118,7 +118,7 @@ class TestChildrenElements(FixtureAPITestCase):
)
def test_element_children_worker_version_validation(self):
with self.assertNumQueries(0):
with self.assertNumQueries(1):
response = self.client.get(
reverse('api:elements-children', kwargs={'pk': str(self.vol.id)}),
data={'worker_version': 'blah'},
......@@ -126,7 +126,7 @@ class TestChildrenElements(FixtureAPITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(response.json(), {'worker_version': ['Invalid UUID']})
with self.assertNumQueries(1):
with self.assertNumQueries(2):
response = self.client.get(
reverse('api:elements-children', kwargs={'pk': str(self.vol.id)}),
data={'worker_version': uuid.uuid4()},
......@@ -163,7 +163,7 @@ class TestChildrenElements(FixtureAPITestCase):
)
element.add_parent(self.vol)
with self.assertNumQueries(9):
with self.assertNumQueries(10):
response = self.client.get(
reverse('api:elements-children', kwargs={'pk': str(self.vol.id)}),
data={'with_has_children': True},
......@@ -185,7 +185,7 @@ class TestChildrenElements(FixtureAPITestCase):
)
def test_children_modified_since_bad_format(self):
with self.assertNumQueries(0):
with self.assertNumQueries(1):
response = self.client.get(
reverse('api:elements-children', kwargs={'pk': str(self.vol.id)}),
HTTP_IF_MODIFIED_SINCE='AAAAAAAAAAAAAAAAA',
......@@ -194,7 +194,7 @@ class TestChildrenElements(FixtureAPITestCase):
self.assertDictEqual(response.json(), {'If-Modified-Since': ['Bad date format']})
def test_children_modified_since_not_modified(self):
with self.assertNumQueries(1):
with self.assertNumQueries(2):
response = self.client.get(
reverse('api:elements-children', kwargs={'pk': str(self.vol.id)}),
HTTP_IF_MODIFIED_SINCE='Thu, 02 Apr 2099 13:37:42 GMT',
......@@ -203,7 +203,7 @@ class TestChildrenElements(FixtureAPITestCase):
def test_children_modified_since(self):
self.corpus.elements.filter(name='Volume 1, page 1r').update(updated='2099-04-02T13:37:43Z')
with self.assertNumQueries(8):
with self.assertNumQueries(9):
response = self.client.get(
reverse('api:elements-children', kwargs={'pk': str(self.vol.id)}),
HTTP_IF_MODIFIED_SINCE='Thu, 02 Apr 2099 13:37:42 GMT',
......
arkindex/documents/tests/test_classes.py
View file @ e4b0fd82
......@@ -265,7 +265,7 @@ class TestClasses(FixtureAPITestCase):
def test_element_parents_best_classes(self):
self.populate_classified_elements()
with self.assertNumQueries(6):
with self.assertNumQueries(7):
response = self.client.get(
reverse('api:elements-parents', kwargs={'pk': str(self.common_children.id)}),
data={'type': self.classified.slug, 'with_best_classes': 1}
......@@ -281,7 +281,7 @@ class TestClasses(FixtureAPITestCase):
def test_element_children_best_classes(self):
self.populate_classified_elements()
with self.assertNumQueries(6):
with self.assertNumQueries(7):
response = self.client.get(
reverse('api:elements-children', kwargs={'pk': str(self.parent.id)}),
data={'type': self.classified.slug, 'with_best_classes': 'yes'}
......@@ -302,7 +302,7 @@ class TestClasses(FixtureAPITestCase):
self.populate_classified_elements()
child = Element.objects.filter(type=self.classified.id).first()
child.classifications.all().update(state=ClassificationState.Rejected)
with self.assertNumQueries(6):
with self.assertNumQueries(7):
response = self.client.get(
reverse('api:elements-children', kwargs={'pk': str(self.parent.id)}),
data={'type': self.classified.slug, 'with_best_classes': 'yes'}
......@@ -405,7 +405,7 @@ class TestClasses(FixtureAPITestCase):
self.populate_classified_elements()
parent = Element.objects.get_ascending(self.common_children.id).last()
parent.classifications.all().filter(confidence=.7).update(state=ClassificationState.Validated)
with self.assertNumQueries(5):
with self.assertNumQueries(6):
response = self.client.get(
reverse('api:elements-parents', kwargs={'pk': str(self.common_children.id)}),
data={'type': self.classified.slug, 'best_class': str(self.text.id)}
......@@ -419,7 +419,7 @@ class TestClasses(FixtureAPITestCase):
self.populate_classified_elements()
child = Element.objects.filter(type=self.classified.id).first()
child.classifications.all().filter(confidence=.7).update(state=ClassificationState.Validated)
with self.assertNumQueries(5):
with self.assertNumQueries(6):
response = self.client.get(
reverse('api:elements-children', kwargs={'pk': str(self.parent.id)}),
data={'type': self.classified.slug, 'best_class': str(self.text.id)}
......@@ -449,7 +449,7 @@ class TestClasses(FixtureAPITestCase):
self.populate_classified_elements()
self.assertEqual(Classification.objects.filter(high_confidence=True).count(), 24)
self.assertEqual(Classification.objects.filter(high_confidence=True).distinct('element_id').count(), 12)
with self.assertNumQueries(5):
with self.assertNumQueries(6):
response = self.client.get(
reverse('api:elements-parents', kwargs={'pk': str(self.common_children.id)}),
data={'type': self.classified.slug, 'best_class': str(self.cover.id)}
......@@ -465,7 +465,7 @@ class TestClasses(FixtureAPITestCase):
self.populate_classified_elements()
self.assertEqual(Classification.objects.filter(high_confidence=True).count(), 24)
self.assertEqual(Classification.objects.filter(high_confidence=True).distinct('element_id').count(), 12)
with self.assertNumQueries(5):
with self.assertNumQueries(6):
response = self.client.get(
reverse('api:elements-children', kwargs={'pk': str(self.parent.id)}),
data={'type': self.classified.slug, 'best_class': str(self.cover.id)}
......
arkindex/documents/tests/test_destroy_elements.py
View file @ e4b0fd82
from unittest.mock import patch
from django.urls import reverse
from rest_framework import status
......@@ -13,6 +15,7 @@ class TestDestroyElements(FixtureAPITestCase):
super().setUpTestData()
cls.volume_type = cls.corpus.types.get(slug='volume')
cls.vol = cls.corpus.elements.get(name='Volume 1')
cls.surface = cls.corpus.elements.get(name='Surface A')
cls.private_corpus = Corpus.objects.create(name='private', public=False)
def test_element_destroy_verified_user(self):
......@@ -160,3 +163,258 @@ class TestDestroyElements(FixtureAPITestCase):
Element.objects.filter(id=elements['A'].id).trash()
self.assertFalse(Element.objects.filter(id__in=[e.id for e in elements.values()]).exists())
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_corpus_elements_requires_login(self, delay_mock):
with self.assertNumQueries(0):
response = self.client.delete(reverse('api:corpus-elements', kwargs={'corpus': self.corpus.id}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertFalse(delay_mock.called)
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_corpus_elements_requires_writable(self, delay_mock):
self.user.corpus_right.filter(corpus=self.corpus).update(can_write=False, can_admin=False)
self.client.force_login(self.user)
with self.assertNumQueries(4):
response = self.client.delete(reverse('api:corpus-elements', kwargs={'corpus': self.corpus.id}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertFalse(delay_mock.called)
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_corpus_elements_empty(self, delay_mock):
self.client.force_login(self.user)
self.assertFalse(self.corpus.elements.filter(name='blablablabla').exists())
with self.assertNumQueries(5):
response = self.client.delete(
reverse('api:corpus-elements', kwargs={'corpus': self.corpus.id}),
QUERY_STRING='name=blablablabla',
)
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
self.assertFalse(delay_mock.called)
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_corpus_elements(self, delay_mock):
self.client.force_login(self.user)
with self.assertNumQueries(5):
response = self.client.delete(reverse('api:corpus-elements', kwargs={'corpus': self.corpus.id}))
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
self.assertEqual(delay_mock.call_count, 1)
args, kwargs = delay_mock.call_args
self.assertEqual(len(args), 0)
self.assertCountEqual(list(kwargs.pop('queryset')), list(self.corpus.elements.all()))
self.assertDictEqual(kwargs, {'delete_children': True})
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_corpus_elements_delete_children(self, delay_mock):
self.client.force_login(self.user)
with self.assertNumQueries(5):
response = self.client.delete(
reverse('api:corpus-elements', kwargs={'corpus': self.corpus.id}),
QUERY_STRING='delete_children=false'
)
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
self.assertEqual(delay_mock.call_count, 1)
args, kwargs = delay_mock.call_args
self.assertEqual(len(args), 0)
self.assertCountEqual(list(kwargs.pop('queryset')), list(self.corpus.elements.all()))
self.assertDictEqual(kwargs, {'delete_children': False})
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_corpus_elements_rejected_filters(self, delay_mock):
self.client.force_login(self.user)
with self.assertNumQueries(2):
response = self.client.delete(
reverse('api:corpus-elements', kwargs={'corpus': self.corpus.id}),
QUERY_STRING='with_best_classes=True&with_has_children=True&best_class=True',
HTTP_IF_MODIFIED_SINCE='Thu, 02 Apr 2099 13:37:42 GMT',
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertDictEqual(response.json(), {
'If-Modified-Since': ['This parameter is not allowed in DELETE requests.'],
'best_class': ['This parameter is not allowed in DELETE requests.'],
'with_best_classes': ['This parameter is not allowed in DELETE requests.'],
'with_has_children': ['This parameter is not allowed in DELETE requests.'],
})
self.assertFalse(delay_mock.called)
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_element_children_requires_login(self, delay_mock):
with self.assertNumQueries(0):
response = self.client.delete(reverse('api:elements-children', kwargs={'pk': self.vol.id}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertFalse(delay_mock.called)
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_element_children_requires_writable(self, delay_mock):
self.user.corpus_right.filter(corpus=self.corpus).update(can_write=False, can_admin=False)
self.client.force_login(self.user)
with self.assertNumQueries(3):
response = self.client.delete(reverse('api:elements-children', kwargs={'pk': self.vol.id}))
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
self.assertFalse(delay_mock.called)
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_element_children_empty(self, delay_mock):
self.client.force_login(self.user)
element = self.corpus.elements.create(type=self.volume_type, name='Lonely element')
with self.assertNumQueries(4):
response = self.client.delete(reverse('api:elements-children', kwargs={'pk': element.id}))
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
self.assertFalse(delay_mock.called)
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_element_children(self, delay_mock):
self.client.force_login(self.user)
with self.assertNumQueries(4):
response = self.client.delete(reverse('api:elements-children', kwargs={'pk': self.vol.id}))
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
self.assertEqual(delay_mock.call_count, 1)
args, kwargs = delay_mock.call_args
self.assertEqual(len(args), 0)
self.assertCountEqual(
list(kwargs.pop('queryset')),
# Direct children of the volume
list(Element.objects.get_descending(self.vol.id).filter(paths__path__last=self.vol.id)),
)
self.assertDictEqual(kwargs, {'delete_children': True})
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_element_children_delete_children(self, delay_mock):
self.client.force_login(self.user)
with self.assertNumQueries(4):
response = self.client.delete(
reverse('api:elements-children', kwargs={'pk': self.vol.id}),
QUERY_STRING='delete_children=false'
)
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
self.assertEqual(delay_mock.call_count, 1)
args, kwargs = delay_mock.call_args
self.assertEqual(len(args), 0)
self.assertCountEqual(
list(kwargs.pop('queryset')),
# Direct children of the volume
list(Element.objects.get_descending(self.vol.id).filter(paths__path__last=self.vol.id)),
)
self.assertDictEqual(kwargs, {'delete_children': False})
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_element_children_rejected_filters(self, delay_mock):
self.client.force_login(self.user)
with self.assertNumQueries(2):
response = self.client.delete(
reverse('api:elements-children', kwargs={'pk': self.vol.id}),
QUERY_STRING='with_best_classes=True&with_has_children=True&best_class=True',
HTTP_IF_MODIFIED_SINCE='Thu, 02 Apr 2099 13:37:42 GMT',
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertDictEqual(response.json(), {
'If-Modified-Since': ['This parameter is not allowed in DELETE requests.'],
'best_class': ['This parameter is not allowed in DELETE requests.'],
'with_best_classes': ['This parameter is not allowed in DELETE requests.'],
'with_has_children': ['This parameter is not allowed in DELETE requests.'],
})
self.assertFalse(delay_mock.called)
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_element_parents_requires_login(self, delay_mock):
with self.assertNumQueries(0):
response = self.client.delete(reverse('api:elements-parents', kwargs={'pk': self.surface.id}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertFalse(delay_mock.called)
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_element_parents_requires_writable(self, delay_mock):
self.user.corpus_right.filter(corpus=self.corpus).update(can_write=False, can_admin=False)
self.client.force_login(self.user)
with self.assertNumQueries(3):
response = self.client.delete(reverse('api:elements-parents', kwargs={'pk': self.surface.id}))
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
self.assertFalse(delay_mock.called)
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_element_parents_empty(self, delay_mock):
self.client.force_login(self.user)
element = self.corpus.elements.create(type=self.volume_type, name='Lonely element')
with self.assertNumQueries(4):
response = self.client.delete(reverse('api:elements-parents', kwargs={'pk': element.id}))
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
self.assertFalse(delay_mock.called)
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_element_parents(self, delay_mock):
self.client.force_login(self.user)
with self.assertNumQueries(4):
response = self.client.delete(reverse('api:elements-parents', kwargs={'pk': self.surface.id}))
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
self.assertEqual(delay_mock.call_count, 1)
args, kwargs = delay_mock.call_args
self.assertEqual(len(args), 0)
self.assertCountEqual(
list(kwargs.pop('queryset')),
# Direct parents of the surface
list(Element.objects.get_ascending(self.surface.id, recursive=False)),
)
self.assertDictEqual(kwargs, {'delete_children': True})
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_element_parents_delete_parents(self, delay_mock):
self.client.force_login(self.user)
with self.assertNumQueries(4):
response = self.client.delete(
reverse('api:elements-parents', kwargs={'pk': self.surface.id}),
QUERY_STRING='delete_children=false'
)
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
self.assertEqual(delay_mock.call_count, 1)
args, kwargs = delay_mock.call_args
self.assertEqual(len(args), 0)
self.assertCountEqual(
list(kwargs.pop('queryset')),
# Direct parents of the surface
list(Element.objects.get_ascending(self.surface.id, recursive=False)),
)
self.assertDictEqual(kwargs, {'delete_children': False})
@patch('arkindex.project.triggers.tasks.element_trash.delay')
def test_destroy_element_parents_rejected_filters(self, delay_mock):
self.client.force_login(self.user)
with self.assertNumQueries(2):
response = self.client.delete(
reverse('api:elements-parents', kwargs={'pk': self.surface.id}),
QUERY_STRING='with_best_classes=True&with_has_children=True&best_class=True',
HTTP_IF_MODIFIED_SINCE='Thu, 02 Apr 2099 13:37:42 GMT',
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertDictEqual(response.json(), {
'If-Modified-Since': ['This parameter is not allowed in DELETE requests.'],
'best_class': ['This parameter is not allowed in DELETE requests.'],
'with_best_classes': ['This parameter is not allowed in DELETE requests.'],
'with_has_children': ['This parameter is not allowed in DELETE requests.'],
})
self.assertFalse(delay_mock.called)
arkindex/documents/tests/test_parents_elements.py
View file @ e4b0fd82
......@@ -76,7 +76,7 @@ class TestParentsElements(FixtureAPITestCase):
"""
self.corpus.elements.filter(name__contains='Volume 1').update(worker_version=self.worker_version)
with self.assertNumQueries(6):
with self.assertNumQueries(7):
response = self.client.get(
reverse('api:elements-parents', kwargs={'pk': str(self.page.id)}),
data={'worker_version': str(self.worker_version.id)},
......@@ -89,7 +89,7 @@ class TestParentsElements(FixtureAPITestCase):
)
def test_parents_worker_version_validation(self):
with self.assertNumQueries(0):
with self.assertNumQueries(1):
response = self.client.get(
reverse('api:elements-parents', kwargs={'pk': str(self.page.id)}),
data={'worker_version': 'blah'},
......@@ -97,7 +97,7 @@ class TestParentsElements(FixtureAPITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(response.json(), {'worker_version': ['Invalid UUID']})
with self.assertNumQueries(1):
with self.assertNumQueries(2):
response = self.client.get(
reverse('api:elements-parents', kwargs={'pk': str(self.page.id)}),
data={'worker_version': uuid.uuid4()},
......@@ -107,7 +107,7 @@ class TestParentsElements(FixtureAPITestCase):
def test_parents_with_has_children(self):
surface = Element.objects.get(name='Surface A')
with self.assertNumQueries(7):
with self.assertNumQueries(8):
response = self.client.get(
reverse('api:elements-parents', kwargs={'pk': str(surface.id)}),
data={'recursive': True, 'with_has_children': True},
......@@ -122,7 +122,7 @@ class TestParentsElements(FixtureAPITestCase):
)
def test_parents_modified_since_bad_format(self):
with self.assertNumQueries(0):
with self.assertNumQueries(1):
response = self.client.get(
reverse('api:elements-parents', kwargs={'pk': str(self.page.id)}),
HTTP_IF_MODIFIED_SINCE='AAAAAAAAAAAAAAAAA',
......@@ -131,7 +131,7 @@ class TestParentsElements(FixtureAPITestCase):
self.assertDictEqual(response.json(), {'If-Modified-Since': ['Bad date format']})
def test_parents_modified_since_not_modified(self):
with self.assertNumQueries(1):
with self.assertNumQueries(2):
response = self.client.get(
reverse('api:elements-parents', kwargs={'pk': str(self.page.id)}),
HTTP_IF_MODIFIED_SINCE='Thu, 02 Apr 2099 13:37:42 GMT',
......@@ -140,7 +140,7 @@ class TestParentsElements(FixtureAPITestCase):
def test_parents_modified_since(self):
self.corpus.elements.filter(name='Volume 1').update(updated='2099-04-02T13:37:43Z')
with self.assertNumQueries(6):
with self.assertNumQueries(7):
response = self.client.get(
reverse('api:elements-parents', kwargs={'pk': str(self.page.id)}),
HTTP_IF_MODIFIED_SINCE='Thu, 02 Apr 2099 13:37:42 GMT',
......
arkindex/project/openapi/patch.yml
View file @ e4b0fd82
......@@ -49,6 +49,10 @@ paths:
security: []
post:
description: Create a new corpus
/api/v1/corpus/{corpus}/elements/:
delete:
operationId: DestroyElements
description: Delete elements in bulk
/api/v1/corpus/{id}/:
get:
description: Retrieve a single corpus
......@@ -116,6 +120,14 @@ paths:
# Will need https://gitlab.com/arkindex/backend/-/issues/86 to be removed
operationId: DestroyElementMLResults
description: Delete machine learning results on an element and its direct children.
/api/v1/elements/{id}/children/:
delete:
operationId: DestroyElementChildren
description: Delete child elements in bulk
/api/v1/elements/{id}/parents/:
delete:
operationId: DestroyElementParents
description: Delete parent elements in bulk
/api/v1/elements/selection/:
delete:
operationId: RemoveSelection
......@@ -421,3 +433,17 @@ paths:
security: []
tags:
- ponos
/ponos/v1/agents/:
get:
description: List the state of all Ponos agents
operationId: ListAgentStates
security: []
tags:
- ponos
/ponos/v1/agent/{id}/:
get:
description: Retrieve details of a Ponos agent with a list of its running tasks
operationId: RetrieveAgent
security: []
tags:
- ponos
arkindex/project/openapi/schemas.py
View file @ e4b0fd82
......@@ -82,8 +82,13 @@ class AutoSchema(BaseAutoSchema):
# Avoid removing the parameter overrides with .pop()
overrides = self.view.openapi_overrides.copy()
if 'parameters' in overrides:
# Filter parameters by HTTP method
allowed_parameters = [
parameter for parameter in overrides.pop('parameters')
if 'x-methods' not in parameter or method.lower() in parameter['x-methods']
]
# Append parameters instead of replacing
operation.setdefault('parameters', []).extend(overrides.pop('parameters'))
operation.setdefault('parameters', []).extend(allowed_parameters)
operation.update(overrides)
return operation
......
arkindex/project/tests/openapi/test_schemas.py
View file @ e4b0fd82
......@@ -155,6 +155,107 @@ class TestAutoSchema(TestCase):
}
)
def test_overrides_method_filter(self):
"""
Test the `x-methods` filter for parameters on openapi_overrides
"""
class ThingView(APIView):
action = 'Retrieve'
openapi_overrides = {
'parameters': [
{
'description': 'Some extra parameter',
'in': 'query',
'name': 'something_get',
'required': False,
'schema': {'type': 'integer'},
'x-methods': ['get'],
},
{
'description': 'Some extra parameter',
'in': 'query',
'name': 'something_post',
'required': False,
'schema': {'type': 'integer'},
'x-methods': ['post', 'delete'],
},
{
'description': 'Some extra parameter',
'in': 'query',
'name': 'something',
'required': False,
'schema': {'type': 'integer'},
}
],
}
def get(self, *args, **kwargs):
pass
def post(self, *args, **kwargs):
pass
inspector = AutoSchema()
inspector.view = create_view(ThingView, 'GET', create_request('/test/{id}/'))
self.assertListEqual(
inspector.get_operation('/test/{id}/', 'GET')['parameters'],
[
{
'description': '',
'in': 'path',
'name': 'id',
'required': True,
'schema': {'type': 'string'},
},
{
'description': 'Some extra parameter',
'in': 'query',
'name': 'something_get',
'required': False,
'schema': {'type': 'integer'},
'x-methods': ['get'],
},
{
'description': 'Some extra parameter',
'in': 'query',
'name': 'something',
'required': False,
'schema': {'type': 'integer'},
}
],
)
inspector = AutoSchema()
inspector.view = create_view(ThingView, 'POST', create_request('/test/{id}/'))
self.assertListEqual(
inspector.get_operation('/test/{id}/', 'POST')['parameters'],
[
{
'description': '',
'in': 'path',
'name': 'id',
'required': True,
'schema': {'type': 'string'},
},
{
'description': 'Some extra parameter',
'in': 'query',
'name': 'something_post',
'required': False,
'schema': {'type': 'integer'},
'x-methods': ['post', 'delete'],
},
{
'description': 'Some extra parameter',
'in': 'query',
'name': 'something',
'required': False,
'schema': {'type': 'integer'},
}
],
)
def test_bugfix_list_uppercase(self):
"""
Test list API views have title-cased endpoint names
......
arkindex/project/tests/test_ponos_view.py
View file @ e4b0fd82
......@@ -6,7 +6,7 @@ from django.urls import reverse
from rest_framework import status
from arkindex.users.models import User
from ponos.models import Secret, encrypt
from ponos.models import Agent, Farm, Secret, encrypt
@override_settings(PONOS_PRIVATE_KEY='staging')
......@@ -48,3 +48,34 @@ class TestPonosView(TestCase):
self.client.force_login(user)
response = self.client.get(reverse('secret-details', kwargs={"name": "secret/name"}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_list_agents(self):
"""
Only authenticated users should have the ability to list agents
"""
response = self.client.get(reverse('ponos-agents'))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.client.force_login(User.objects.create())
response = self.client.get(reverse('ponos-agents'))
self.assertEqual(response.status_code, status.HTTP_200_OK)
def test_retrieve_agent(self):
"""
Only authenticated users should have the ability to retrieve details of an agent
"""
agent = Agent.objects.create(
cpu_cores=3,
cpu_frequency=3e9,
gpu_count=0,
farm_id=Farm.objects.create().id,
ram_total=2e9,
last_ping='1999-09-09'
)
response = self.client.get(reverse('ponos-agent-details', kwargs={'pk': str(agent.id)}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.client.force_login(User.objects.create())
response = self.client.get(reverse('ponos-agent-details', kwargs={'pk': str(agent.id)}))
self.assertEqual(response.status_code, status.HTTP_200_OK)
arkindex/project/triggers.py
View file @ e4b0fd82
......@@ -7,6 +7,7 @@ from uuid import UUID
from django.conf import settings
from arkindex.documents import tasks
from arkindex.documents.managers import ElementQuerySet
from arkindex.documents.models import Corpus, Element, Entity
......@@ -102,3 +103,11 @@ def corpus_delete(corpus: Union[Corpus, UUID, str]) -> None:
corpus_id = str(corpus)
tasks.corpus_delete.delay(corpus_id=corpus_id)
def element_trash(queryset: ElementQuerySet, delete_children: bool = True) -> None:
"""
Run ElementQuerySet.trash to delete a batch of elements.
"""
assert isinstance(queryset, ElementQuerySet), 'Only Element querysets can be trashed'
tasks.element_trash.delay(queryset=queryset, delete_children=delete_children)
arkindex/project/urls.py
View file @ e4b0fd82
......@@ -4,7 +4,14 @@ from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.urls import include, path, re_path
from arkindex.project.api_v1 import api
from arkindex.project.views import CdnHome, FrontendView, OpenAPIDocsView, PonosSecretDetails
from arkindex.project.views import (
CdnHome,
FrontendView,
OpenAPIDocsView,
PonosAgentDetails,
PonosAgentsState,
PonosSecretDetails,
)
# Fallback to the dummy frontend view when CDN_ASSETS_URL is not set
frontend_view = FrontendView if settings.CDN_ASSETS_URL is None else CdnHome
......@@ -14,6 +21,8 @@ urlpatterns = [
path('api-docs/', OpenAPIDocsView.as_view(), name='openapi-docs'),
# Override Ponos endpoints
path('ponos/v1/secret/<path:name>', PonosSecretDetails.as_view(), name='secret-details'),
path('ponos/v1/agents/', PonosAgentsState.as_view(), name='ponos-agents'),
path('ponos/v1/agent/<uuid:pk>/', PonosAgentDetails.as_view(), name='ponos-agent-details'),
path('ponos/', include('ponos.urls')),
path('admin/', admin.site.urls),
path('rq/', include('django_rq.urls')),
......
arkindex/project/views.py
View file @ e4b0fd82
......@@ -3,7 +3,8 @@ from django.views.generic import TemplateView, View
from rest_framework import permissions
from arkindex.project.mixins import CachedViewMixin
from ponos.api import SecretDetails
from arkindex.project.permissions import IsVerified
from ponos.api import AgentDetails, AgentsState, SecretDetails
class FrontendView(View):
......@@ -51,3 +52,17 @@ class PonosSecretDetails(SecretDetails):
return request.user.is_authenticated and request.user.is_internal
permission_classes = (IsInternalOnly, )
class PonosAgentsState(AgentsState):
"""
Allow any verified user to see the state of Ponos agents on this instance
"""
permission_classes = (IsVerified, )
class PonosAgentDetails(AgentDetails):
"""
Allow any verified user to see the details of an agent including all its running tasks
"""
permission_classes = (IsVerified, )