Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • arkindex/backend
1 result
Show changes
Commits on Source (30)
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 1212 additions and 830 deletions
.gitlab-ci.yml
View file @ c7f1c938
......@@ -18,8 +18,8 @@ include:
- .cache/pip
before_script:
- pip install -r tests-requirements.txt
- "echo 'database: {host: postgres, port: 5432}\npublic_hostname: http://ci.arkindex.localhost' > $CONFIG_PATH"
- "echo database: {host: postgres, port: 5432} > $CONFIG_PATH"
- pip install -e .[test]
# Those jobs require the base image; they might fail if the image is not up to date.
# Allow them to fail when building a new base image, to prevent them from blocking a new base image build
......@@ -58,7 +58,7 @@ backend-tests:
- test-report.xml
script:
- python3 setup.py test
- arkindex/manage.py test
backend-lint:
image: python:3.10
......@@ -91,7 +91,6 @@ backend-migrations:
alias: postgres
script:
- pip install -e .
- arkindex/manage.py makemigrations --check --noinput --dry-run -v 3
backend-openapi:
......@@ -154,29 +153,7 @@ backend-build:
- when: never
script:
- ci/build.sh Dockerfile
backend-build-binary-docker:
stage: build
image: docker:19.03.1
services:
- docker:dind
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375/
# Run this on master and tags except base tags and schedules
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule"'
when: never
- if: '$CI_COMMIT_BRANCH == "master"'
when: on_success
- if: '$CI_COMMIT_TAG && $CI_COMMIT_TAG !~ /^base-.*/'
when: on_success
- when: never
script:
- ci/build.sh Dockerfile.binary "-binary"
- ci/build.sh
# Make sure arkindex is always compatible with Nuitka
backend-build-binary:
......
Dockerfile
View file @ c7f1c938
......@@ -6,8 +6,6 @@ ADD . build
RUN cd build && python3 setup.py sdist
FROM registry.gitlab.teklia.com/arkindex/backend/base:gitlab-teklia
# Auth token expires on 01/07/2024
ARG GITLAB_TOKEN="glpat-3sBZPFgkZbqJxfSqjcAa"
# Install arkindex and its deps
# Uses a source archive instead of full local copy to speedup docker build
......
Dockerfile.binary deleted 100644 → 0
View file @ b77279e3
# syntax=docker/dockerfile:1
FROM python:3.10-slim-bookworm AS compilation
RUN apt-get update && apt-get install --no-install-recommends -y build-essential wget
RUN pip install nuitka
# Auth token expires on 01/07/2024
ARG GITLAB_TOKEN="glpat-3sBZPFgkZbqJxfSqjcAa"
# We build in /usr/share because Django will try to load some files relative to that path
# once executed in the binary (management commands, ...)
WORKDIR /usr/share
# Add our own source code
ADD arkindex /usr/share/arkindex
ADD base/requirements.txt /tmp/requirements-base-arkindex.txt
ADD requirements.txt /tmp/requirements-arkindex.txt
# Build full requirements, removing relative or remote references to arkindex projects
RUN cat /tmp/requirements-*arkindex.txt | sort | uniq | grep -v -E '^arkindex|^#' > /requirements.txt
# List all management commands
RUN find /usr/share/arkindex/*/management -name '*.py' -not -name '__init__.py' > /commands.txt
# Remove arkindex unit tests
RUN find /usr/share/arkindex -type d -name tests | xargs rm -rf
# This configuration is needed to avoid a compilation crash at linking stage
# It only seems to happen on recent gcc
# See https://github.com/Nuitka/Nuitka/issues/959
ENV NUITKA_RESOURCE_MODE=linker
# Compile all our python source code
# Do not use the -O or -OO python flags here as it removes assert statements (see backend#432)
RUN python -m nuitka \
--nofollow-imports \
--include-package=arkindex \
--show-progress \
--lto=yes \
--output-dir=/build \
arkindex/manage.py
# Start over from a clean setup
FROM registry.gitlab.teklia.com/arkindex/backend/base:gitlab-teklia as build
# Import files from compilation
RUN mkdir /usr/share/arkindex
COPY --from=compilation /build/manage.bin /usr/bin/arkindex
COPY --from=compilation /requirements.txt /usr/share/arkindex
COPY --from=compilation /commands.txt /usr/share/arkindex
# Install open source Python dependencies
# We also add gunicorn, to be able to run `arkindex gunicorn`
RUN pip install -r /usr/share/arkindex/requirements.txt gunicorn
# Setup Arkindex VERSION
COPY VERSION /etc/arkindex.version
# Copy templates in base dir for binary
ENV BASE_DIR=/usr/share/arkindex
COPY arkindex/templates /usr/share/arkindex/templates
COPY arkindex/documents/export/*.sql /usr/share/arkindex/documents/export/
# Touch python files for needed management commands
# Otherwise Django will not load the compiled module
RUN for cmd in $(cat /usr/share/arkindex/commands.txt); do mkdir -p $(dirname $cmd); touch $cmd; done
HEALTHCHECK --start-period=1m --start-interval=1s --interval=1m --timeout=5s \
CMD wget --spider --quiet http://localhost/api/v1/public-key/ || exit 1
# Run gunicorn server
ENV PORT=80
EXPOSE $PORT
CMD arkindex gunicorn --host=0.0.0.0 --port $PORT
README.md
View file @ c7f1c938
......@@ -2,6 +2,7 @@ Backend for Historical Manuscripts Indexing
===========================================
[![pipeline status](https://gitlab.teklia.com/arkindex/backend/badges/master/pipeline.svg)](https://gitlab.teklia.com/arkindex/backend/commits/master)
[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
## Requirements
......
VERSION
View file @ c7f1c938
1.5.3
1.5.4-rc3
arkindex/documents/api/elements.py
View file @ c7f1c938
......@@ -1197,7 +1197,7 @@ class ElementChildren(ElementsListBase):
@extend_schema(tags=["elements"])
@extend_schema_view(
get=extend_schema(description="Retrieve a single element's information and metadata"),
patch=extend_schema(description="Rename an element"),
patch=extend_schema(description="Edit an element's attributes. Requires a write access on the corpus."),
put=extend_schema(description="Edit an element's attributes. Requires a write access on the corpus."),
delete=extend_schema(
description=dedent("""
......@@ -2193,6 +2193,14 @@ class CorpusSelectionDestroy(CorpusACLMixin, SelectionMixin, DestroyAPIView):
delete=extend_schema(
operation_id="DestroyWorkerResults",
parameters=[
OpenApiParameter(
"worker_run_id",
type=UUID,
required=False,
description="Only delete Worker Results produced by a specific worker run. "
"If this parameter is set, any `worker_version_id`, `model_version_id` "
"or `configuration_id` parameters will be ignored.",
),
OpenApiParameter(
"worker_version_id",
type=UUID,
......@@ -2233,38 +2241,30 @@ class CorpusSelectionDestroy(CorpusACLMixin, SelectionMixin, DestroyAPIView):
)
class WorkerResultsDestroy(CorpusACLMixin, DestroyAPIView):
"""
Delete all Worker Results from all WorkerVersions or a specific one
on a Corpus or under a specified parent element (parent element included)
Delete all Worker Results, or Worker Results produced by specific WorkerRuns or WorkerVersions
(the results to delete can also be filtered by ModelVersion and Configuration)
on a Corpus, the selection, or under a specified parent element (parent element included)
"""
permission_classes = (IsVerified, )
# https://github.com/tfranzel/drf-spectacular/issues/308
@extend_schema(responses={204: None})
def delete(self, request, *args, **kwargs):
corpus = self.get_corpus(self.kwargs["corpus"], role=Role.Admin)
def results_filters(self):
errors = defaultdict(list)
use_selection = self.request.query_params.get("use_selection", "false").lower() not in ("false", "0")
if use_selection:
# Only check for selected elements if the selection feature is enabled
if settings.ARKINDEX_FEATURES["selection"]:
if "element_id" in self.request.query_params:
errors["use_selection"].append("use_selection and element_id cannot be used simultaneously.")
if not self.request.user.selected_elements.filter(corpus=corpus).exists():
errors["use_selection"].append("No elements of the specified corpus have been selected.")
else:
errors["use_selection"].append("Selection is not available on this instance.")
element_id = None
if "element_id" in self.request.query_params:
if "worker_run_id" in self.request.query_params:
try:
element_id = UUID(self.request.query_params["element_id"])
worker_run_id = UUID(self.request.query_params["worker_run_id"])
except (TypeError, ValueError):
errors["element_id"].append("Invalid UUID.")
raise ValidationError({"worker_run_id": ["Invalid UUID."]})
else:
if not corpus.elements.filter(id=element_id).exists():
errors["element_id"].append("This element does not exist in the specified corpus.")
try:
worker_run = WorkerRun.objects.get(id=worker_run_id)
except WorkerRun.DoesNotExist:
raise ValidationError({"worker_run_id": ["This worker run does not exist."]})
# Ignore the other parameters when a worker run ID is set
return {
"worker_run": worker_run,
}
worker_version = None
if "worker_version_id" in self.request.query_params:
......@@ -2314,22 +2314,60 @@ class WorkerResultsDestroy(CorpusACLMixin, DestroyAPIView):
if errors:
raise ValidationError(errors)
return {
"version": worker_version,
"model_version": model_version,
"configuration": configuration
}
# https://github.com/tfranzel/drf-spectacular/issues/308
@extend_schema(responses={204: None})
def delete(self, request, *args, **kwargs):
corpus = self.get_corpus(self.kwargs["corpus"], role=Role.Admin)
errors = defaultdict(list)
use_selection = self.request.query_params.get("use_selection", "false").lower() not in ("false", "0")
if use_selection:
# Only check for selected elements if the selection feature is enabled
if settings.ARKINDEX_FEATURES["selection"]:
if "element_id" in self.request.query_params:
errors["use_selection"].append("use_selection and element_id cannot be used simultaneously.")
if not self.request.user.selected_elements.filter(corpus=corpus).exists():
errors["use_selection"].append("No elements of the specified corpus have been selected.")
else:
errors["use_selection"].append("Selection is not available on this instance.")
element_id = None
if "element_id" in self.request.query_params:
try:
element_id = UUID(self.request.query_params["element_id"])
except (TypeError, ValueError):
errors["element_id"].append("Invalid UUID.")
else:
if not corpus.elements.filter(id=element_id).exists():
errors["element_id"].append("This element does not exist in the specified corpus.")
try:
filters = self.results_filters()
except ValidationError as errs:
errors = errors | errs.detail
if errors:
raise ValidationError(errors)
if use_selection:
selection_worker_results_delete(
corpus=corpus,
version=worker_version,
model_version=model_version,
configuration=configuration,
user_id=self.request.user.id,
**filters
)
else:
worker_results_delete(
corpus_id=corpus.id,
version=worker_version,
element_id=element_id,
model_version=model_version,
configuration=configuration,
user_id=self.request.user.id,
**filters
)
return Response(status=status.HTTP_204_NO_CONTENT)
......
arkindex/documents/api/export.py
View file @ c7f1c938
......@@ -4,9 +4,9 @@ from textwrap import dedent
from django.conf import settings
from django.utils import timezone
from drf_spectacular.utils import extend_schema, extend_schema_view
from rest_framework import serializers, status
from rest_framework.exceptions import ValidationError
from rest_framework.generics import ListCreateAPIView, RetrieveAPIView
from rest_framework import permissions, serializers, status
from rest_framework.exceptions import PermissionDenied, ValidationError
from rest_framework.generics import ListCreateAPIView, RetrieveDestroyAPIView
from rest_framework.response import Response
from arkindex.documents.models import Corpus, CorpusExport, CorpusExportState
......@@ -70,25 +70,56 @@ class CorpusExportAPIView(CorpusACLMixin, ListCreateAPIView):
return Response(CorpusExportSerializer(export).data, status=status.HTTP_201_CREATED)
class DownloadExport(RetrieveAPIView):
"""
Download a corpus export.
@extend_schema(
tags=["exports"],
)
@extend_schema_view(
get=extend_schema(
operation_id="DownloadExport",
description=dedent(
"""
Download a corpus export.
Guest access is required on private corpora.
"""
Guest access is required on private corpora.
"""
),
responses={302: serializers.Serializer},
),
delete=extend_schema(
operation_id="DestroyExport",
description=dedent(
"""
Delete a corpus export.
Requires either an admin access to the corpus, or for the user to be
the export's creator and have contributor access to the corpus.
"""
)
)
)
class ManageExport(RetrieveDestroyAPIView):
queryset = CorpusExport.objects.none()
permission_classes = (IsVerified, )
serializer_class = CorpusExportSerializer
def get_queryset(self):
return CorpusExport.objects.filter(
states = {CorpusExportState.Done}
if self.request.method not in permissions.SAFE_METHODS:
states.add(CorpusExportState.Running)
return CorpusExport.objects.select_related("corpus").filter(
corpus__in=Corpus.objects.readable(self.request.user),
state=CorpusExportState.Done
).only("id")
state__in=states
)
@extend_schema(
operation_id="DownloadExport",
tags=["exports"],
responses={302: serializers.Serializer},
)
def get(self, *args, **kwargs):
return Response(status=status.HTTP_302_FOUND, headers={"Location": self.get_object().s3_url})
def check_object_permissions(self, request, obj):
super().check_object_permissions(request, obj)
if request.method in permissions.SAFE_METHODS:
return
if not obj.is_deletable(request.user):
raise PermissionDenied(detail="You do not have sufficient rights to delete this export.")
# Allow deleting running exports if they have not been updated in longer than EXPORT_TTL_SECONDS (not actually still running)
if obj.state == CorpusExportState.Running and obj.updated + timedelta(seconds=settings.EXPORT_TTL_SECONDS) > timezone.now():
raise ValidationError("You cannot delete an export that is still running.")
arkindex/documents/deletion.py
View file @ c7f1c938
......@@ -70,35 +70,6 @@ def delete_element(element_id: UUID) -> None:
""", {"id": element_id})
logger.info(f"Deleted {cursor.rowcount} usage from process as element")
# Set folders references on training processes to None
cursor.execute("""
UPDATE process_process
SET train_folder_id = NULL
WHERE train_folder_id = %(id)s
OR train_folder_id IN (
SELECT element_id FROM documents_elementpath WHERE path && ARRAY[%(id)s]
)
""", {"id": element_id})
logger.info(f"Deleted {cursor.rowcount} usage from process as train folder")
cursor.execute("""
UPDATE process_process
SET validation_folder_id = NULL
WHERE validation_folder_id = %(id)s
OR validation_folder_id IN (
SELECT element_id FROM documents_elementpath WHERE path && ARRAY[%(id)s]
)
""", {"id": element_id})
logger.info(f"Deleted {cursor.rowcount} usage from process as validation folder")
cursor.execute("""
UPDATE process_process
SET test_folder_id = NULL
WHERE test_folder_id = %(id)s
OR test_folder_id IN (
SELECT element_id FROM documents_elementpath WHERE path && ARRAY[%(id)s]
)
""", {"id": element_id})
logger.info(f"Deleted {cursor.rowcount} usage from process as test folder")
# Remove user selections
cursor.execute("""
DELETE FROM documents_selection selection
......
arkindex/documents/fixtures/data.json
View file @ c7f1c938
This diff is collapsed.
arkindex/documents/managers.py
View file @ c7f1c938
......@@ -81,11 +81,6 @@ class ElementQuerySet(models.QuerySet):
# This may cause workers processes to target a different set of elements
Process.objects.filter(element_id__in=ids).update(element_id=None)
# Set folder references on training processes to None
Process.objects.filter(train_folder_id__in=ids).update(train_folder_id=None)
Process.objects.filter(validation_folder_id__in=ids).update(validation_folder_id=None)
Process.objects.filter(test_folder_id__in=ids).update(test_folder_id=None)
# This is where it gets really ugly: in order to delete all the children
# without losing their reference, we need to inject the generated SQL query
# directly into an SQL DELETE statement for paths
......
arkindex/documents/models.py
View file @ c7f1c938
......@@ -25,6 +25,7 @@ from arkindex.project.aws import S3FileMixin
from arkindex.project.default_corpus import DEFAULT_CORPUS_TYPES
from arkindex.project.fields import ArrayConcat, ArrayField, LinearRingField
from arkindex.project.models import IndexableModel
from arkindex.users.models import Role
class Corpus(IndexableModel):
......@@ -656,9 +657,11 @@ class Element(IndexableModel):
if self.mirrored:
rotation_param = f"!{rotation_param}"
# Do no attempt to upsize small images for thumbnails
thumbnail_height = min(400, height)
return urljoin(
self.image.url + "/",
f"{x},{y},{width},{height}/,400/{rotation_param}/default.jpg"
f"{x},{y},{width},{height}/,{thumbnail_height}/{rotation_param}/default.jpg"
)
def __str__(self):
......@@ -1189,6 +1192,21 @@ class CorpusExport(S3FileMixin, IndexableModel):
def s3_key(self) -> str:
return str(self.id)
def is_deletable(self, user) -> bool:
"""
Whether or not the request user can delete this export
"""
if user.is_anonymous or getattr(user, "is_agent", False):
return False
if user.is_admin:
return True
from arkindex.users.utils import get_max_level
level = get_max_level(user, self.corpus)
return level is not None and ((level >= Role.Admin.value) or (level >= Role.Contributor.value and self.user_id == user.id))
def start(self):
from arkindex.project.triggers import export_corpus
assert self.state == CorpusExportState.Created, "This export has already been started"
......
arkindex/documents/tasks.py
View file @ c7f1c938
......@@ -104,6 +104,7 @@ def element_trash(queryset: ElementQuerySet, delete_children: bool) -> None:
@job("high", timeout=settings.RQ_TIMEOUTS["worker_results_delete"])
def selection_worker_results_delete(
corpus_id: str,
worker_run_id: Optional[str] = None,
model_version_id: Optional[str] = None,
configuration_id: Optional[str | Literal[False]] = None,
version_id: Optional[str] = None,
......@@ -123,6 +124,7 @@ def selection_worker_results_delete(
worker_results_delete(
corpus_id=corpus_id,
element_id=element_id,
worker_run_id=worker_run_id,
version_id=version_id,
model_version_id=model_version_id,
configuration_id=configuration_id,
......@@ -132,6 +134,7 @@ def selection_worker_results_delete(
@job("high", timeout=settings.RQ_TIMEOUTS["worker_results_delete"])
def worker_results_delete(
corpus_id: str,
worker_run_id: Optional[str] = None,
version_id: Optional[str] = None,
element_id: Optional[str] = None,
model_version_id: Optional[str] = None,
......@@ -142,6 +145,8 @@ def worker_results_delete(
whole corpus, under a specified parent element (parent element included), or on a single element.
Results can be filtered depending on a specific model version and a specific or unset configuration.
"""
assert (not worker_run_id or not version_id), "The worker_run_id and version_id parameters are mutually exclusive."
elements = Element.objects.filter(corpus_id=corpus_id)
classifications = Classification.objects.filter(element__corpus_id=corpus_id)
transcriptions = Transcription.objects.filter(element__corpus_id=corpus_id)
......@@ -155,8 +160,19 @@ def worker_results_delete(
metadata = MetaData.objects.filter(element__corpus_id=corpus_id)
worker_activities = WorkerActivity.objects.filter(element__corpus_id=corpus_id)
# When a worker run ID is defined, filter by that worker run ID
if worker_run_id:
elements = elements.filter(worker_run_id=worker_run_id)
classifications = classifications.filter(worker_run_id=worker_run_id)
transcriptions = transcriptions.filter(worker_run_id=worker_run_id)
transcription_entities = transcription_entities.filter(transcription__worker_run_id=worker_run_id)
worker_transcription_entities = worker_transcription_entities.filter(worker_run_id=worker_run_id)
metadata = metadata.filter(worker_run_id=worker_run_id)
# There is no worker_run_id on Worker Activities so the best thing we can do is delete the worker activities
# attached to the elements produced with that worker run, and they are already being deleted by elements.trash()
worker_activities = worker_activities.none()
# When a version ID is defined, filter by the exact version ID
if version_id:
elif version_id:
elements = elements.filter(worker_version_id=version_id)
classifications = classifications.filter(worker_version_id=version_id)
transcriptions = transcriptions.filter(worker_version_id=version_id)
......@@ -164,7 +180,9 @@ def worker_results_delete(
worker_transcription_entities = worker_transcription_entities.filter(worker_version_id=version_id)
metadata = metadata.filter(worker_version_id=version_id)
worker_activities = worker_activities.filter(worker_version_id=version_id)
# Otherwise, select everything that has any worker version ID.
# Otherwise, select everything that has any worker version ID. (When something has been created
# by a worker run, it always has a worker version; however we have things that were created with
# a worker version but without a worker run.)
# We use worker_version_id != None and not worker_version_id__isnull=False,
# because isnull would cause an unnecessary LEFT JOIN query.
else:
......@@ -202,7 +220,7 @@ def worker_results_delete(
metadata = metadata.filter(element_id=element_id)
worker_activities = worker_activities.filter(element_id=element_id)
if model_version_id:
if not worker_run_id and model_version_id:
elements = elements.filter(worker_run__model_version_id=model_version_id)
classifications = classifications.filter(worker_run__model_version_id=model_version_id)
transcriptions = transcriptions.filter(worker_run__model_version_id=model_version_id)
......@@ -212,7 +230,7 @@ def worker_results_delete(
# Activities are not linked to a worker run and cannot be filtered by model version
worker_activities = worker_activities.none()
if configuration_id is not None:
if not worker_run_id and configuration_id is not None:
if configuration_id is False:
# Only delete results generated on a worker run with no configuration
elements = elements.filter(worker_run__configuration_id=None)
......@@ -247,6 +265,7 @@ def worker_results_delete(
# we were supposed to delete worker results on.
worker_results_delete(
corpus_id=corpus_id,
worker_run_id=worker_run_id,
version_id=version_id,
element_id=element_id,
model_version_id=model_version_id,
......
arkindex/documents/tests/tasks/test_corpus_delete.py
View file @ c7f1c938
......@@ -3,7 +3,7 @@ from django.db.models.signals import pre_delete
from arkindex.documents.models import Corpus, Element, EntityType, MetaType, Transcription
from arkindex.documents.tasks import corpus_delete
from arkindex.ponos.models import Farm, State, Task
from arkindex.process.models import CorpusWorkerVersion, ProcessMode, Repository, WorkerVersion
from arkindex.process.models import CorpusWorkerVersion, ProcessDataset, ProcessMode, Repository, WorkerVersion
from arkindex.project.tests import FixtureTestCase, force_constraints_immediate
from arkindex.training.models import Dataset
......@@ -118,13 +118,14 @@ class TestDeleteCorpus(FixtureTestCase):
cls.dataset2 = Dataset.objects.create(name="Dead Sea Scrolls", description="How to trigger a Third Impact", creator=cls.user, corpus=cls.corpus2)
# Process on cls.corpus and with a dataset from cls.corpus
dataset_process1 = cls.corpus.processes.create(creator=cls.user, mode=ProcessMode.Dataset)
dataset_process1.datasets.set([dataset1])
ProcessDataset.objects.create(process=dataset_process1, dataset=dataset1, sets=dataset1.sets)
# Process on cls.corpus with a dataset from another corpus
dataset_process2 = cls.corpus.processes.create(creator=cls.user, mode=ProcessMode.Dataset)
dataset_process2.datasets.set([dataset1, cls.dataset2])
ProcessDataset.objects.create(process=dataset_process2, dataset=dataset1, sets=dataset1.sets)
ProcessDataset.objects.create(process=dataset_process2, dataset=cls.dataset2, sets=cls.dataset2.sets)
# Process on another corpus with a dataset from another corpus and none from cls.corpus
cls.dataset_process2 = cls.corpus2.processes.create(creator=cls.user, mode=ProcessMode.Dataset)
cls.dataset_process2.datasets.set([cls.dataset2])
cls.dataset_process3 = cls.corpus2.processes.create(creator=cls.user, mode=ProcessMode.Dataset)
ProcessDataset.objects.create(process=cls.dataset_process3, dataset=cls.dataset2, sets=cls.dataset2.sets)
cls.rev = cls.repo.revisions.create(
hash="42",
......@@ -200,14 +201,14 @@ class TestDeleteCorpus(FixtureTestCase):
self.df.refresh_from_db()
self.vol.refresh_from_db()
self.page.refresh_from_db()
self.dataset_process2.refresh_from_db()
self.dataset_process3.refresh_from_db()
self.assertTrue(self.repo.revisions.filter(id=self.rev.id).exists())
self.assertEqual(self.process.revision, self.rev)
self.assertEqual(self.process.files.get(), self.df)
self.assertTrue(Element.objects.get_descending(self.vol.id).filter(id=self.page.id).exists())
self.assertTrue(self.corpus2.datasets.filter(id=self.dataset2.id).exists())
self.assertTrue(self.corpus2.processes.filter(id=self.dataset_process2.id).exists())
self.assertTrue(self.corpus2.processes.filter(id=self.dataset_process3.id).exists())
md = self.vol.metadatas.get()
self.assertEqual(md.name, "meta")
......
arkindex/documents/tests/tasks/test_selection_worker_results_delete.py
View file @ c7f1c938
......@@ -4,7 +4,7 @@ from django.db import connections
from django.db.utils import IntegrityError
from arkindex.documents.tasks import selection_worker_results_delete
from arkindex.process.models import Worker, WorkerVersion
from arkindex.process.models import Worker, WorkerRun, WorkerVersion
from arkindex.project.tests import FixtureTestCase
from arkindex.training.models import Dataset, Model, ModelVersionState
......@@ -18,6 +18,7 @@ class TestDeleteSelectionWorkerResults(FixtureTestCase):
cls.page2 = cls.corpus.elements.get(name="Volume 1, page 1v")
cls.page3 = cls.corpus.elements.get(name="Volume 1, page 2r")
cls.version = WorkerVersion.objects.first()
cls.worker_run = WorkerRun.objects.first()
cls.model = Model.objects.create(name="Generic model", public=False)
cls.model_version = cls.model.versions.create(
state=ModelVersionState.Available,
......@@ -63,6 +64,7 @@ class TestDeleteSelectionWorkerResults(FixtureTestCase):
self.assertCountEqual(worker_results_delete_mock.call_args_list, [
call(
corpus_id=self.corpus.id,
worker_run_id=None,
version_id=None,
model_version_id=None,
configuration_id=None,
......@@ -70,6 +72,7 @@ class TestDeleteSelectionWorkerResults(FixtureTestCase):
),
call(
corpus_id=self.corpus.id,
worker_run_id=None,
version_id=None,
model_version_id=None,
configuration_id=None,
......@@ -102,6 +105,7 @@ class TestDeleteSelectionWorkerResults(FixtureTestCase):
self.assertCountEqual(worker_results_delete_mock.call_args_list, [
call(
corpus_id=self.corpus.id,
worker_run_id=None,
version_id=self.version.id,
model_version_id=self.model_version.id,
configuration_id=self.configuration.id,
......@@ -109,6 +113,7 @@ class TestDeleteSelectionWorkerResults(FixtureTestCase):
),
call(
corpus_id=self.corpus.id,
worker_run_id=None,
version_id=self.version.id,
model_version_id=self.model_version.id,
configuration_id=self.configuration.id,
......@@ -116,6 +121,60 @@ class TestDeleteSelectionWorkerResults(FixtureTestCase):
),
])
@patch("arkindex.documents.tasks.get_current_job")
def test_run_worker_run_or_version(self, job_mock):
self.user.selected_elements.set([self.page1, self.page2])
self.superuser.selected_elements.set([self.page3])
job_mock.return_value.user_id = self.user.id
with self.assertRaisesMessage(AssertionError, "The worker_run_id and version_id parameters are mutually exclusive."):
selection_worker_results_delete(
corpus_id=self.corpus.id,
worker_run_id=self.worker_run.id,
version_id=self.version.id,
model_version_id=self.model_version.id,
configuration_id=self.configuration.id,
)
@patch("arkindex.documents.tasks.get_current_job")
@patch("arkindex.documents.tasks.worker_results_delete")
def test_run_worker_run_filter(self, worker_results_delete_mock, job_mock):
self.user.selected_elements.set([self.page1, self.page2])
self.superuser.selected_elements.set([self.page3])
job_mock.return_value.user_id = self.user.id
selection_worker_results_delete(
corpus_id=self.corpus.id,
worker_run_id=self.worker_run.id
)
self.assertEqual(job_mock.call_count, 1)
self.assertEqual(job_mock().set_progress.call_count, 2)
self.assertListEqual(job_mock().set_progress.call_args_list, [
call(.0),
call(.5),
])
self.assertEqual(worker_results_delete_mock.call_count, 2)
self.assertCountEqual(worker_results_delete_mock.call_args_list, [
call(
corpus_id=self.corpus.id,
worker_run_id=self.worker_run.id,
version_id=None,
model_version_id=None,
configuration_id=None,
element_id=self.page1.id,
),
call(
corpus_id=self.corpus.id,
worker_run_id=self.worker_run.id,
version_id=None,
model_version_id=None,
configuration_id=None,
element_id=self.page2.id,
),
])
@patch("arkindex.documents.tasks.get_current_job")
def test_run_dataset_failure(self, job_mock):
"""
......
arkindex/documents/tests/tasks/test_worker_results_delete.py
View file @ c7f1c938
......@@ -157,7 +157,7 @@ class TestDeleteWorkerResults(FixtureTestCase):
"version_id": str(self.version_1.id),
"element_id": str(self.page1.id),
}):
worker_results_delete(self.corpus.id, self.version_1.id, self.page1.id)
worker_results_delete(corpus_id=self.corpus.id, version_id=self.version_1.id, element_id=self.page1.id)
self.check_deleted(
self.classification2,
self.transcription1,
......@@ -173,7 +173,7 @@ class TestDeleteWorkerResults(FixtureTestCase):
"version_id": str(self.version_1.id),
"element_id": str(self.page2.id),
}):
worker_results_delete(self.corpus.id, self.version_1.id, self.page2.id)
worker_results_delete(corpus_id=self.corpus.id, version_id=self.version_1.id, element_id=self.page2.id)
self.check_deleted(
self.classification3,
self.transcription2,
......@@ -278,3 +278,76 @@ class TestDeleteWorkerResults(FixtureTestCase):
# https://code.djangoproject.com/ticket/11665
with self.assertRaises(IntegrityError):
connections["default"].check_constraints()
def test_run_worker_run_or_version(self):
with self.assertRaisesMessage(AssertionError, "The worker_run_id and version_id parameters are mutually exclusive."):
worker_results_delete(
corpus_id=self.corpus.id,
version_id=self.version_1.id,
worker_run_id=self.worker_run_1.id
)
def test_run_worker_run_on_corpus(self):
with self.assertExactQueries("worker_results_delete_in_corpus_worker_run.sql", params={
"corpus_id": str(self.corpus.id),
"worker_run_id": str(self.worker_run_1.id),
}):
worker_results_delete(
corpus_id=self.corpus.id,
worker_run_id=self.worker_run_1.id,
)
self.check_deleted(
self.classification1,
self.classification2,
self.classification3
)
def test_run_worker_run_on_parent(self):
with self.assertExactQueries("worker_results_delete_under_parent_worker_run.sql", params={
"corpus_id": str(self.corpus.id),
"worker_run_id": str(self.worker_run_2.id),
"element_id": str(self.page1.id),
}):
worker_results_delete(corpus_id=self.corpus.id, worker_run_id=self.worker_run_2.id, element_id=self.page1.id)
self.check_deleted(
self.transcription1,
self.transcription_entity1,
)
def test_run_worker_run_on_parent_delete_element(self):
"""
The element itself is deleted after its related results from the same worker run
"""
self.page1.worker_run = self.worker_run_2
self.page1.worker_version = self.version_2
self.page1.save()
with self.assertExactQueries("worker_results_delete_under_parent_included_worker_run.sql", params={
"corpus_id": str(self.corpus.id),
"worker_run_id": str(self.worker_run_2.id),
"element_id": str(self.page1.id),
}):
worker_results_delete(corpus_id=self.corpus.id, worker_run_id=self.worker_run_2.id, element_id=self.page1.id)
self.check_deleted(
self.transcription1,
self.transcription_entity1,
self.page1,
# self.classifications2 is deleted as well since it's on self.page1
self.classification2
)
def test_run_worker_run_ignore_filters(self):
with self.assertExactQueries("worker_results_delete_in_corpus_worker_run.sql", params={
"corpus_id": str(self.corpus.id),
"worker_run_id": str(self.worker_run_1.id)
}):
worker_results_delete(
corpus_id=self.corpus.id,
worker_run_id=self.worker_run_1.id,
model_version_id=self.model_version.id,
configuration_id=self.configuration.id
)
self.check_deleted(
self.classification1,
self.classification2,
self.classification3
)
arkindex/documents/tests/test_destroy_worker_results.py
View file @ c7f1c938
......@@ -5,7 +5,7 @@ from django.urls import reverse
from rest_framework import status
from arkindex.documents.models import Corpus
from arkindex.process.models import Worker, WorkerVersion
from arkindex.process.models import Worker, WorkerRun, WorkerVersion
from arkindex.project.tests import FixtureAPITestCase
from arkindex.training.models import Model, ModelVersionState
......@@ -16,6 +16,7 @@ class TestDestroyWorkerResults(FixtureAPITestCase):
def setUpTestData(cls):
super().setUpTestData()
cls.version = WorkerVersion.objects.get(worker__slug="reco")
cls.worker_run = WorkerRun.objects.get(version=cls.version)
cls.page = cls.corpus.elements.get(name="Volume 1, page 2r")
cls.private_corpus = Corpus.objects.create(name="private", public=False)
cls.model = Model.objects.create(name="Generic model", public=False)
......@@ -72,6 +73,7 @@ class TestDestroyWorkerResults(FixtureAPITestCase):
self.assertEqual(delay_mock.call_count, 1)
self.assertEqual(delay_mock.call_args, call(
corpus_id=self.corpus.id,
worker_run_id=None,
version_id=None,
element_id=None,
model_version_id=None,
......@@ -93,6 +95,7 @@ class TestDestroyWorkerResults(FixtureAPITestCase):
self.assertEqual(delay_mock.call_count, 1)
self.assertEqual(delay_mock.call_args, call(
corpus_id=self.corpus.id,
worker_run_id=None,
element_id=None,
version_id=self.version.id,
model_version_id=None,
......@@ -101,6 +104,59 @@ class TestDestroyWorkerResults(FixtureAPITestCase):
description=f"Deletion of worker results produced by {self.version}",
))
@patch("arkindex.project.triggers.documents_tasks.worker_results_delete.delay")
def test_filter_worker_run_ignore_filters(self, delay_mock):
"""
When worker_run_id is passed, worker_version_id, model_version_id and configuration_id
are ignored
"""
self.client.force_login(self.user)
with self.assertNumQueries(7):
response = self.client.delete(
reverse("api:worker-delete-results", kwargs={"corpus": str(self.corpus.id)})
+ (
f"?worker_version_id={self.version.id}"
f"&worker_run_id={self.worker_run.id}"
f"&model_version_id={self.model_version.id}"
f"&configuration_id=false"
)
)
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
self.assertEqual(delay_mock.call_count, 1)
self.assertEqual(delay_mock.call_args, call(
corpus_id=self.corpus.id,
version_id=None,
element_id=None,
worker_run_id=self.worker_run.id,
model_version_id=None,
configuration_id=None,
user_id=self.user.id,
description=f"Deletion of worker results produced by {self.worker_run.summary}",
))
@patch("arkindex.project.triggers.documents_tasks.worker_results_delete.delay")
def test_filter_worker_run(self, delay_mock):
self.client.force_login(self.user)
with self.assertNumQueries(7):
response = self.client.delete(
reverse("api:worker-delete-results", kwargs={"corpus": str(self.corpus.id)})
+ f"?worker_run_id={self.worker_run.id}",
)
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
self.assertEqual(delay_mock.call_count, 1)
self.assertEqual(delay_mock.call_args, call(
corpus_id=self.corpus.id,
version_id=None,
element_id=None,
worker_run_id=self.worker_run.id,
model_version_id=None,
configuration_id=None,
user_id=self.user.id,
description=f"Deletion of worker results produced by {self.worker_run.summary}",
))
@patch("arkindex.project.triggers.documents_tasks.worker_results_delete.delay")
def test_filter_element(self, delay_mock):
self.client.force_login(self.user)
......@@ -115,6 +171,7 @@ class TestDestroyWorkerResults(FixtureAPITestCase):
self.assertEqual(delay_mock.call_args, call(
corpus_id=self.corpus.id,
element_id=self.page.id,
worker_run_id=None,
version_id=None,
model_version_id=None,
configuration_id=None,
......@@ -122,6 +179,31 @@ class TestDestroyWorkerResults(FixtureAPITestCase):
description="Deletion of worker results",
))
@patch("arkindex.project.triggers.documents_tasks.worker_results_delete.delay")
def test_filter_element_worker_run(self, delay_mock):
self.client.force_login(self.user)
with self.assertNumQueries(8):
response = self.client.delete(
reverse("api:worker-delete-results", kwargs={"corpus": str(self.corpus.id)})
+ (
f"?element_id={self.page.id}"
f"&worker_run_id={self.worker_run.id}"
)
)
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
self.assertEqual(delay_mock.call_count, 1)
self.assertEqual(delay_mock.call_args, call(
corpus_id=self.corpus.id,
element_id=self.page.id,
worker_run_id=self.worker_run.id,
version_id=None,
model_version_id=None,
configuration_id=None,
user_id=self.user.id,
description=f"Deletion of worker results produced by {self.worker_run.summary}",
))
@patch("arkindex.project.triggers.documents_tasks.worker_results_delete.delay")
def test_filter_unset_configuration(self, delay_mock):
self.client.force_login(self.user)
......@@ -136,6 +218,7 @@ class TestDestroyWorkerResults(FixtureAPITestCase):
self.assertEqual(delay_mock.call_args, call(
corpus_id=self.corpus.id,
element_id=None,
worker_run_id=None,
version_id=None,
model_version_id=None,
configuration_id=False,
......@@ -157,6 +240,7 @@ class TestDestroyWorkerResults(FixtureAPITestCase):
self.assertEqual(delay_mock.call_args, call(
corpus_id=self.corpus.id,
element_id=None,
worker_run_id=None,
version_id=None,
model_version_id=self.model_version.id,
configuration_id=None,
......@@ -183,6 +267,7 @@ class TestDestroyWorkerResults(FixtureAPITestCase):
self.assertEqual(delay_mock.call_args, call(
corpus_id=self.corpus.id,
element_id=self.page.id,
worker_run_id=None,
version_id=self.version.id,
model_version_id=self.model_version.id,
configuration_id=self.configuration.id,
......@@ -221,6 +306,32 @@ class TestDestroyWorkerResults(FixtureAPITestCase):
{"worker_version_id": ["This worker version does not exist."]}
)
def test_invalid_worker_run_id(self):
self.client.force_login(self.user)
with self.assertNumQueries(6):
response = self.client.delete(
reverse("api:worker-delete-results", kwargs={"corpus": str(self.corpus.id)})
+ "?worker_run_id=lol"
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertDictEqual(
response.json(),
{"worker_run_id": ["Invalid UUID."]},
)
def test_wrong_worker_run_id(self):
self.client.force_login(self.user)
with self.assertNumQueries(7):
response = self.client.delete(
reverse("api:worker-delete-results", kwargs={"corpus": str(self.corpus.id)})
+ "?worker_run_id=12341234-1234-1234-1234-123412341234"
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertDictEqual(
response.json(),
{"worker_run_id": ["This worker run does not exist."]}
)
def test_invalid_element_id(self):
self.client.force_login(self.user)
with self.assertNumQueries(6):
......@@ -377,6 +488,7 @@ class TestDestroyWorkerResults(FixtureAPITestCase):
self.assertEqual(delay_mock.call_count, 1)
self.assertEqual(delay_mock.call_args, call(
corpus_id=self.corpus.id,
worker_run_id=None,
version_id=None,
user_id=self.user.id,
model_version_id=None,
......@@ -400,6 +512,7 @@ class TestDestroyWorkerResults(FixtureAPITestCase):
self.assertEqual(delay_mock.call_count, 1)
self.assertEqual(delay_mock.call_args, call(
corpus_id=self.corpus.id,
worker_run_id=None,
version_id=self.version.id,
user_id=self.user.id,
model_version_id=None,
......@@ -407,3 +520,28 @@ class TestDestroyWorkerResults(FixtureAPITestCase):
description=f"Deletion of worker results on selected elements in {self.corpus.name} "
f"produced by {self.version}"
))
@override_settings(ARKINDEX_FEATURES={"selection": True})
@patch("arkindex.project.triggers.documents_tasks.selection_worker_results_delete.delay")
def test_selection_worker_run_filter(self, delay_mock):
self.user.selected_elements.add(self.page)
self.client.force_login(self.user)
with self.assertNumQueries(8):
response = self.client.delete(
reverse("api:worker-delete-results", kwargs={"corpus": str(self.corpus.id)})
+ f"?use_selection=true&worker_run_id={self.worker_run.id}"
)
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
self.assertEqual(delay_mock.call_count, 1)
self.assertEqual(delay_mock.call_args, call(
corpus_id=self.corpus.id,
worker_run_id=self.worker_run.id,
version_id=None,
user_id=self.user.id,
model_version_id=None,
configuration_id=None,
description=f"Deletion of worker results on selected elements in {self.corpus.name} "
f"produced by {self.worker_run.summary}"
))
arkindex/documents/tests/test_element.py
View file @ c7f1c938
......@@ -38,24 +38,22 @@ class TestElement(FixtureTestCase):
def test_iiif_thumbnail_url(self):
cases = [
(0, False, "http://server/img1/10,20,30,40/,400/0/default.jpg"),
(0, True, "http://server/img1/10,20,30,40/,400/!0/default.jpg"),
(180, False, "http://server/img1/10,20,30,40/,400/180/default.jpg"),
(180, True, "http://server/img1/10,20,30,40/,400/!180/default.jpg"),
(0, False, [[500, 100], [650, 100], [650, 800], [500, 800], [500, 100]], "http://server/img1/500,100,150,700/,400/0/default.jpg"),
(0, True, [[500, 100], [650, 100], [650, 800], [500, 800], [500, 100]], "http://server/img1/500,100,150,700/,400/!0/default.jpg"),
(180, False, [[500, 580], [650, 580], [650, 700], [500, 700], [500, 580]], "http://server/img1/500,580,150,120/,120/180/default.jpg"),
(180, True, [[500, 580], [650, 580], [650, 700], [500, 700], [500, 580]], "http://server/img1/500,580,150,120/,120/!180/default.jpg"),
(0, False, [[10, 20], [40, 20], [40, 60], [10, 60], [10, 20]], "http://server/img1/10,20,30,40/,40/0/default.jpg"),
(0, True, [[10, 20], [40, 20], [40, 60], [10, 60], [10, 20]], "http://server/img1/10,20,30,40/,40/!0/default.jpg"),
(180, False, [[10, 20], [40, 20], [40, 60], [10, 60], [10, 20]], "http://server/img1/10,20,30,40/,40/180/default.jpg"),
(180, True, [[10, 20], [40, 20], [40, 60], [10, 60], [10, 20]], "http://server/img1/10,20,30,40/,40/!180/default.jpg"),
]
for rotation_angle, mirrored, expected_url in cases:
for rotation_angle, mirrored, polygon, expected_url in cases:
with self.subTest(rotation_angle=rotation_angle, mirrored=mirrored):
element = Element(
name="Something",
type=self.element_type,
image=self.image,
polygon=[
[10, 20],
[40, 20],
[40, 60],
[10, 60],
[10, 20],
],
polygon=polygon,
rotation_angle=rotation_angle,
mirrored=mirrored,
)
......
arkindex/documents/tests/test_export.py
View file @ c7f1c938
......@@ -5,7 +5,7 @@ from django.test import override_settings
from django.urls import reverse
from rest_framework import status
from arkindex.documents.models import CorpusExportState
from arkindex.documents.models import Corpus, CorpusExportState
from arkindex.project.tests import FixtureAPITestCase
from arkindex.users.models import Role
......@@ -160,7 +160,7 @@ class TestExport(FixtureAPITestCase):
self.client.force_login(self.superuser)
export = self.corpus.exports.create(user=self.user, state=CorpusExportState.Done)
with self.assertNumQueries(3):
response = self.client.get(reverse("api:download-export", kwargs={"pk": export.id}))
response = self.client.get(reverse("api:manage-export", kwargs={"pk": export.id}))
self.assertEqual(response.status_code, status.HTTP_302_FOUND)
self.assertEqual(response.headers["Location"], "http://somewhere")
self.assertEqual(presigned_url_mock.call_args_list, [
......@@ -183,13 +183,13 @@ class TestExport(FixtureAPITestCase):
export = self.corpus.exports.create(user=self.superuser, state=CorpusExportState.Done)
with self.assertNumQueries(4):
response = self.client.get(reverse("api:download-export", kwargs={"pk": export.id}))
response = self.client.get(reverse("api:manage-export", kwargs={"pk": export.id}))
self.assertEqual(response.status_code, status.HTTP_302_FOUND)
self.assertEqual(response.headers["Location"], "http://somewhere")
def test_download_export_requires_login(self):
export = self.corpus.exports.create(user=self.user, state=CorpusExportState.Done)
response = self.client.get(reverse("api:download-export", kwargs={"pk": export.id}))
response = self.client.get(reverse("api:manage-export", kwargs={"pk": export.id}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_download_export_requires_verified(self):
......@@ -198,7 +198,7 @@ class TestExport(FixtureAPITestCase):
self.client.force_login(self.user)
export = self.corpus.exports.create(user=self.user, state=CorpusExportState.Done)
with self.assertNumQueries(2):
response = self.client.get(reverse("api:download-export", kwargs={"pk": export.id}))
response = self.client.get(reverse("api:manage-export", kwargs={"pk": export.id}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_download_export_requires_guest(self):
......@@ -208,7 +208,7 @@ class TestExport(FixtureAPITestCase):
self.client.force_login(self.user)
export = self.corpus.exports.create(user=self.user, state=CorpusExportState.Done)
with self.assertNumQueries(5):
response = self.client.get(reverse("api:download-export", kwargs={"pk": export.id}))
response = self.client.get(reverse("api:manage-export", kwargs={"pk": export.id}))
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
def test_download_export_not_done(self):
......@@ -217,5 +217,88 @@ class TestExport(FixtureAPITestCase):
with self.subTest(state=state):
export = self.corpus.exports.create(user=self.user, state=state)
with self.assertNumQueries(3):
response = self.client.get(reverse("api:download-export", kwargs={"pk": export.id}))
response = self.client.get(reverse("api:manage-export", kwargs={"pk": export.id}))
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
# DestroyExport
def test_delete_export_requires_login(self):
export = self.corpus.exports.create(user=self.user, state=CorpusExportState.Done)
response = self.client.delete(reverse("api:manage-export", kwargs={"pk": export.id}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_delete_export_requires_verified(self):
self.user.verified_email = False
self.user.save()
self.client.force_login(self.user)
export = self.corpus.exports.create(user=self.user, state=CorpusExportState.Done)
with self.assertNumQueries(2):
response = self.client.delete(reverse("api:manage-export", kwargs={"pk": export.id}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_delete_export_private_corpus(self):
private_corpus = Corpus.objects.create(name="private")
self.client.force_login(self.user)
export = private_corpus.exports.create(user=self.superuser, state=CorpusExportState.Done)
with self.assertNumQueries(5):
response = self.client.delete(reverse("api:manage-export", kwargs={"pk": export.id}))
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
def test_delete_export_wrong_state(self):
self.client.force_login(self.superuser)
for state in (CorpusExportState.Created, CorpusExportState.Failed):
with self.subTest(state=state):
export = self.corpus.exports.create(user=self.user, state=state)
with self.assertNumQueries(3):
response = self.client.delete(reverse("api:manage-export", kwargs={"pk": export.id}))
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
@override_settings(EXPORT_TTL_SECONDS=420)
def test_delete_export_running(self):
self.client.force_login(self.superuser)
with patch("django.utils.timezone.now") as mock_now:
mock_now.return_value = datetime.now(timezone.utc) - timedelta(weeks=3)
export = self.corpus.exports.create(user=self.user, state=CorpusExportState.Running)
self.corpus.exports.filter(id=export.id).update(updated=datetime.now(timezone.utc) - timedelta(minutes=2))
with self.assertNumQueries(3):
response = self.client.delete(reverse("api:manage-export", kwargs={"pk": export.id}))
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(response.json(), ["You cannot delete an export that is still running."])
@override_settings(EXPORT_TTL_SECONDS=420)
def test_delete_export_running_expired(self):
self.client.force_login(self.superuser)
with patch("django.utils.timezone.now") as mock_now:
mock_now.return_value = datetime.now(timezone.utc) - timedelta(weeks=3)
export = self.corpus.exports.create(user=self.user, state=CorpusExportState.Running)
self.corpus.exports.filter(id=export.id).update(updated=datetime.now(timezone.utc) - timedelta(minutes=10))
with self.assertNumQueries(4):
response = self.client.delete(reverse("api:manage-export", kwargs={"pk": export.id}))
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
assert not self.corpus.exports.exists()
def test_delete_export_requires_rights(self):
self.corpus.memberships.filter(user=self.user).update(level=Role.Contributor.value)
self.client.force_login(self.user)
export = self.corpus.exports.create(user=self.superuser, state=CorpusExportState.Done)
with self.assertNumQueries(5):
response = self.client.delete(reverse("api:manage-export", kwargs={"pk": export.id}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertDictEqual(response.json(), {"detail": "You do not have sufficient rights to delete this export."})
def test_delete_export_creator_contributor(self):
self.corpus.memberships.filter(user=self.user).update(level=Role.Contributor.value)
self.client.force_login(self.user)
export = self.corpus.exports.create(user=self.user, state=CorpusExportState.Done)
with self.assertNumQueries(6):
response = self.client.delete(reverse("api:manage-export", kwargs={"pk": export.id}))
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
assert not self.corpus.exports.exists()
def test_delete_export_corpus_admin(self):
self.client.force_login(self.user)
export = self.corpus.exports.create(user=self.superuser, state=CorpusExportState.Done)
with self.assertNumQueries(7):
response = self.client.delete(reverse("api:manage-export", kwargs={"pk": export.id}))
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
assert not self.corpus.exports.exists()
arkindex/ponos/serializers.py
View file @ c7f1c938
......@@ -106,6 +106,7 @@ class TaskLightSerializer(serializers.ModelSerializer):
└⟶ Error ├⟶ Failed
└⟶ Error
Stopping ⟶ Stopped
└⟶ Error
""").strip(),
)
......@@ -133,8 +134,8 @@ class TaskLightSerializer(serializers.ModelSerializer):
allowed_transitions = {
State.Unscheduled: [State.Pending],
State.Pending: [State.Running, State.Error],
State.Running: [State.Completed, State.Failed, State.Stopping, State.Error],
State.Stopping: [State.Stopped],
State.Running: [State.Completed, State.Failed, State.Error],
State.Stopping: [State.Stopped, State.Error],
}
if self.instance and state not in allowed_transitions.get(self.instance.state, []):
raise ValidationError(f"Transition from state {self.instance.state} to state {state} is forbidden.")
......
arkindex/ponos/tests/test_api.py
View file @ c7f1c938
......@@ -1275,25 +1275,93 @@ class TestAPI(FixtureAPITestCase):
self.assertEqual(self.task1.state, state)
@patch("arkindex.ponos.models.TaskLogs.latest", new_callable=PropertyMock)
def test_partial_update_task_from_agent(self, short_logs_mock):
@patch("arkindex.ponos.tasks.notify_process_completion.delay")
def test_partial_update_task_from_agent_allowed_states(self, notify_mock, short_logs_mock):
short_logs_mock.return_value = ""
self.task1.state = State.Pending
self.task1.agent = self.agent
self.task1.save()
with self.assertNumQueries(5):
resp = self.client.patch(
reverse("api:task-details", args=[self.task1.id]),
data={"state": State.Running.value},
HTTP_AUTHORIZATION=f"Bearer {self.agent.token.access_token}",
)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
cases = [
(State.Unscheduled, State.Pending, 5),
(State.Pending, State.Running, 5),
(State.Pending, State.Error, 11),
(State.Running, State.Completed, 9),
(State.Running, State.Failed, 9),
(State.Running, State.Error, 9),
(State.Stopping, State.Stopped, 9),
(State.Stopping, State.Error, 9),
]
for from_state, to_state, query_count in cases:
with self.subTest(from_state=from_state, to_state=to_state):
self.task1.state = from_state
self.task1.save()
data = resp.json()
self.assertEqual(data["id"], str(self.task1.id))
self.assertEqual(data["state"], State.Running.value)
self.task1.refresh_from_db()
self.assertEqual(self.task1.state, State.Running)
with self.assertNumQueries(query_count):
resp = self.client.patch(
reverse("api:task-details", args=[self.task1.id]),
data={"state": to_state.value},
HTTP_AUTHORIZATION=f"Bearer {self.agent.token.access_token}",
)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
data = resp.json()
self.assertEqual(data["id"], str(self.task1.id))
self.assertEqual(data["state"], to_state.value)
self.task1.refresh_from_db()
self.assertEqual(self.task1.state, to_state)
def test_partial_update_task_from_agent_forbidden_states(self):
self.task1.agent = self.agent
self.task1.save()
cases = [
(State.Unscheduled, State.Running),
(State.Unscheduled, State.Completed),
(State.Unscheduled, State.Failed),
(State.Unscheduled, State.Error),
(State.Unscheduled, State.Stopping),
(State.Unscheduled, State.Stopped),
(State.Pending, State.Unscheduled),
(State.Pending, State.Completed),
(State.Pending, State.Failed),
(State.Pending, State.Stopping),
(State.Pending, State.Stopped),
(State.Running, State.Unscheduled),
(State.Running, State.Pending),
(State.Running, State.Stopping),
(State.Running, State.Stopped),
(State.Stopping, State.Unscheduled),
(State.Stopping, State.Pending),
(State.Stopping, State.Running),
(State.Stopping, State.Completed),
(State.Stopping, State.Failed),
# Cannot go from one state to the same state
*((state, state) for state in State),
# Cannot go from a final state to anywhere
*((final_state, state) for final_state in FINAL_STATES for state in State),
]
for from_state, to_state in cases:
with self.subTest(from_state=from_state, to_state=to_state):
self.task1.state = from_state
self.task1.save()
with self.assertNumQueries(2):
resp = self.client.put(
reverse("api:task-details", args=[self.task1.id]),
data={"state": to_state.value},
HTTP_AUTHORIZATION=f"Bearer {self.agent.token.access_token}",
)
self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)
self.assertDictEqual(
resp.json(),
{"state": [f"Transition from state {from_state} to state {to_state} is forbidden."]},
)
self.task1.refresh_from_db()
self.assertEqual(self.task1.state, from_state)
def test_partial_update_task_from_agent_requires_login(self):
with self.assertNumQueries(0):
......