Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • arkindex/backend
1 result
Show changes
Commits on Source (7)
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 1259 additions and 1253 deletions
Makefile
View file @ acf2b82c
......@@ -13,13 +13,10 @@ clean:
find . -name '*.pyc' -exec rm {} \;
build:
CI_PROJECT_DIR=$(ROOT_DIR) CI_REGISTRY_IMAGE=$(IMAGE_TAG) $(ROOT_DIR)/ci/build.sh Dockerfile
binary:
CI_PROJECT_DIR=$(ROOT_DIR) CI_REGISTRY_IMAGE=$(IMAGE_TAG) $(ROOT_DIR)/ci/build.sh Dockerfile.binary -binary
CI_PROJECT_DIR=$(ROOT_DIR) CI_REGISTRY_IMAGE=$(IMAGE_TAG) $(ROOT_DIR)/ci/build.sh
worker:
arkindex/manage.py rqworker -v 2 default high
arkindex/manage.py rqworker -v 2 default high tasks
test-fixtures:
$(eval export PGPASSWORD=devdata)
......
README.md
View file @ acf2b82c
......@@ -160,6 +160,11 @@ We use [rq](https://python-rq.org/), integrated via [django-rq](https://pypi.org
To run them, use `make worker` to start a RQ worker. You will need to have Redis running; `make slim` or `make` in the architecture will provide it. `make` in the architecture also provides a RQ worker running in Docker from a binary build.
Process tasks are run in RQ by default (Community Edition). Two RQ workers must be running at the same time to actually run a process with worker activities, so the initialisation task can wait for the worker activity task to finish:
```sh
$ manage.py rqworker -v 3 default high & manage.py rqworker -v 3 tasks
```
## Metrics
The application serves metrics for Prometheus under the `/metrics` prefix.
A specific port can be used by setting the `PROMETHEUS_METRICS_PORT` environment variable, thus separating the application from the metrics API.
VERSION
View file @ acf2b82c
1.5.4-rc3
1.5.4
arkindex/documents/api/elements.py
View file @ acf2b82c
......@@ -1303,17 +1303,16 @@ class ElementNeighbors(ACLMixin, ListAPIView):
queryset = Element.objects.none()
def get_queryset(self):
element = get_object_or_404(
# Include the attributes required for ACL checks and the API response
Element.objects.select_related("corpus", "type").only("id", "name", "type__slug", "corpus__public"),
Element
.objects
.filter(corpus__in=Corpus.objects.readable(self.request.user))
.select_related("corpus", "type")
.only("id", "name", "type__slug", "corpus__public"),
id=self.kwargs["pk"]
)
# Check access permission
if not self.has_access(element.corpus, Role.Guest.value):
raise PermissionDenied(detail="You do not have a read access to this element.")
return Element.objects.get_neighbors(element)
......
arkindex/documents/fixtures/data.json
View file @ acf2b82c
This diff is collapsed.
arkindex/documents/tasks.py
View file @ acf2b82c
......@@ -185,6 +185,7 @@ def worker_results_delete(
# a worker version but without a worker run.)
# We use worker_version_id != None and not worker_version_id__isnull=False,
# because isnull would cause an unnecessary LEFT JOIN query.
# No extra filtering is needed on worker activities, since worker versions cannot be null there.
else:
elements = elements.exclude(worker_version_id=None)
classifications = classifications.exclude(worker_version_id=None)
......@@ -192,7 +193,6 @@ def worker_results_delete(
transcription_entities = transcription_entities.exclude(transcription__worker_version_id=None)
worker_transcription_entities = worker_transcription_entities.exclude(worker_version_id=None)
metadata = metadata.exclude(worker_version_id=None)
worker_activities = worker_activities.exclude(worker_version_id=None)
if element_id:
# include_children causes a deletion *only* on the element's descendants.
......@@ -227,8 +227,7 @@ def worker_results_delete(
transcription_entities = transcription_entities.filter(transcription__worker_run__model_version_id=model_version_id)
worker_transcription_entities = worker_transcription_entities.filter(worker_run__model_version_id=model_version_id)
metadata = metadata.filter(worker_run__model_version_id=model_version_id)
# Activities are not linked to a worker run and cannot be filtered by model version
worker_activities = worker_activities.none()
worker_activities = worker_activities.filter(model_version_id=model_version_id)
if not worker_run_id and configuration_id is not None:
if configuration_id is False:
......
arkindex/documents/tests/test_allowed_metadata.py
View file @ acf2b82c
from unittest.mock import patch
from django.urls import reverse
from rest_framework import status
......@@ -51,7 +53,8 @@ class TestAllowedMetaData(FixtureAPITestCase):
list(expected_meta[3:])
)
def test_list_private_corpus(self):
@patch("arkindex.project.mixins.has_access", return_value=False)
def test_list_private_corpus(self, has_access_mock):
with self.assertNumQueries(1):
response = self.client.get(
reverse("api:corpus-allowed-metadata", kwargs={"pk": str(self.private_corpus.id)}),
......@@ -77,7 +80,7 @@ class TestAllowedMetaData(FixtureAPITestCase):
def test_create(self):
self.client.force_login(self.user)
with self.assertNumQueries(8):
with self.assertNumQueries(5):
response = self.client.post(
reverse("api:corpus-allowed-metadata", kwargs={"pk": str(self.corpus.id)}),
data={"name": "flan", "type": "text"}
......@@ -93,7 +96,7 @@ class TestAllowedMetaData(FixtureAPITestCase):
def test_create_unique(self):
self.client.force_login(self.user)
self.corpus.allowed_metadatas.create(name="flan", type=MetaType.Text)
with self.assertNumQueries(7):
with self.assertNumQueries(4):
response = self.client.post(
reverse("api:corpus-allowed-metadata", kwargs={"pk": str(self.corpus.id)}),
data={"name": "flan", "type": "text"}
......@@ -122,21 +125,21 @@ class TestAllowedMetaData(FixtureAPITestCase):
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_create_requires_corpus_admin(self):
@patch("arkindex.project.mixins.has_access", return_value=False)
def test_create_requires_corpus_admin(self, has_access_mock):
self.client.force_login(self.user)
for role in [Role.Guest, Role.Contributor]:
with self.subTest(role=role):
self.corpus.memberships.filter(user=self.user).update(level=role.value)
with self.assertNumQueries(5):
response = self.client.post(
reverse("api:corpus-allowed-metadata", kwargs={"pk": str(self.corpus.id)}),
data={"name": "flan", "type": "text"}
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_create_private_corpus(self):
self.corpus.memberships.filter(user=self.user).update(level=Role.Contributor.value)
with self.assertNumQueries(3):
response = self.client.post(
reverse("api:corpus-allowed-metadata", kwargs={"pk": str(self.corpus.id)}),
data={"name": "flan", "type": "text"}
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
@patch("arkindex.project.mixins.has_access", return_value=False)
def test_create_private_corpus(self, has_access_mock):
self.client.force_login(self.user)
with self.assertNumQueries(6):
with self.assertNumQueries(3):
response = self.client.post(
reverse("api:corpus-allowed-metadata", kwargs={"pk": str(self.private_corpus.id)}),
data={"name": "flan", "type": "text"}
......@@ -145,7 +148,7 @@ class TestAllowedMetaData(FixtureAPITestCase):
def test_create_invalid(self):
self.client.force_login(self.user)
with self.assertNumQueries(6):
with self.assertNumQueries(3):
response = self.client.post(
reverse("api:corpus-allowed-metadata", kwargs={"pk": str(self.corpus.id)}),
data={"name": "flan", "type": "pouet"}
......@@ -175,9 +178,10 @@ class TestAllowedMetaData(FixtureAPITestCase):
response = self.client.get(reverse("api:allowed-metadata-edit", kwargs={"corpus": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", "pk": self.test_meta.id}))
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
def test_retrieve_private_corpus(self):
@patch("arkindex.project.mixins.has_access", return_value=False)
def test_retrieve_private_corpus(self, has_access_mock):
self.client.force_login(self.user)
with self.assertNumQueries(6):
with self.assertNumQueries(3):
response = self.client.get(reverse("api:allowed-metadata-edit", kwargs={"corpus": self.private_corpus.id, "pk": self.test_meta.id}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
......@@ -228,9 +232,10 @@ class TestAllowedMetaData(FixtureAPITestCase):
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_update_private_corpus(self):
@patch("arkindex.project.mixins.has_access", return_value=False)
def test_update_private_corpus(self, has_access_mock):
self.client.force_login(self.user)
with self.assertNumQueries(6):
with self.assertNumQueries(3):
response = self.client.put(
reverse("api:allowed-metadata-edit", kwargs={"corpus": self.private_corpus.id, "pk": self.test_meta.id}),
{"type": "url", "name": "newName"},
......@@ -239,7 +244,7 @@ class TestAllowedMetaData(FixtureAPITestCase):
def test_update_invalid_type(self):
self.client.force_login(self.user)
with self.assertNumQueries(7):
with self.assertNumQueries(4):
response = self.client.put(
reverse("api:allowed-metadata-edit", kwargs={"corpus": self.corpus.id, "pk": self.test_meta.id}),
{"type": "invalidType", "name": "newName"},
......@@ -251,7 +256,7 @@ class TestAllowedMetaData(FixtureAPITestCase):
def test_update_missing_argument(self):
self.client.force_login(self.user)
with self.assertNumQueries(7):
with self.assertNumQueries(4):
response = self.client.put(
reverse("api:allowed-metadata-edit", kwargs={"corpus": self.corpus.id, "pk": self.test_meta.id}),
{"name": "newName"},
......@@ -263,7 +268,7 @@ class TestAllowedMetaData(FixtureAPITestCase):
def test_update_duplicate(self):
self.client.force_login(self.user)
with self.assertNumQueries(8):
with self.assertNumQueries(5):
response = self.client.put(
reverse("api:allowed-metadata-edit", kwargs={"corpus": self.corpus.id, "pk": self.test_meta.id}),
{"type": self.allowed_meta[1].type.value, "name": self.allowed_meta[1].name},
......@@ -273,21 +278,20 @@ class TestAllowedMetaData(FixtureAPITestCase):
"detail": ["An AllowedMetaData with this type and name already exists in this corpus."]
})
def test_update_requires_corpus_admin(self):
@patch("arkindex.project.mixins.has_access", return_value=False)
def test_update_requires_corpus_admin(self, has_access_mock):
self.client.force_login(self.user)
for role in [Role.Guest, Role.Contributor]:
with self.subTest(role=role):
self.corpus.memberships.filter(user=self.user).update(level=role.value)
with self.assertNumQueries(5):
response = self.client.put(
reverse("api:allowed-metadata-edit", kwargs={"corpus": self.corpus.id, "pk": self.test_meta.id}),
{"type": self.allowed_meta[2].type.value, "name": self.allowed_meta[2].name},
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.corpus.memberships.filter(user=self.user).update(level=Role.Contributor.value)
with self.assertNumQueries(3):
response = self.client.put(
reverse("api:allowed-metadata-edit", kwargs={"corpus": self.corpus.id, "pk": self.test_meta.id}),
{"type": self.allowed_meta[2].type.value, "name": self.allowed_meta[2].name},
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_partial_update(self):
self.client.force_login(self.user)
with self.assertNumQueries(9):
with self.assertNumQueries(6):
response = self.client.patch(
reverse("api:allowed-metadata-edit", kwargs={"corpus": self.corpus.id, "pk": self.test_meta.id}),
{"name": "newName"},
......@@ -317,9 +321,10 @@ class TestAllowedMetaData(FixtureAPITestCase):
{"type": "reference"})
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_partial_update_private_corpus(self):
@patch("arkindex.project.mixins.has_access", return_value=False)
def test_partial_update_private_corpus(self, has_access_mock):
self.client.force_login(self.user)
with self.assertNumQueries(6):
with self.assertNumQueries(3):
response = self.client.patch(
reverse("api:allowed-metadata-edit", kwargs={"corpus": self.private_corpus.id, "pk": self.test_meta.id}),
{"type": "reference"})
......@@ -327,7 +332,7 @@ class TestAllowedMetaData(FixtureAPITestCase):
def test_partial_update_invalid_type(self):
self.client.force_login(self.user)
with self.assertNumQueries(7):
with self.assertNumQueries(4):
response = self.client.patch(
reverse("api:allowed-metadata-edit", kwargs={"corpus": self.corpus.id, "pk": self.test_meta.id}),
{"type": "invalidType"},
......@@ -339,7 +344,7 @@ class TestAllowedMetaData(FixtureAPITestCase):
def test_partial_update_duplicate(self):
self.client.force_login(self.user)
with self.assertNumQueries(8):
with self.assertNumQueries(5):
response = self.client.patch(
reverse("api:allowed-metadata-edit", kwargs={"corpus": self.corpus.id, "pk": self.test_meta.id}),
{"type": self.allowed_meta[2].type.value},
......@@ -349,21 +354,20 @@ class TestAllowedMetaData(FixtureAPITestCase):
"detail": ["An AllowedMetaData with this type and name already exists in this corpus."]
})
def test_partial_update_requires_corpus_admin(self):
@patch("arkindex.project.mixins.has_access", return_value=False)
def test_partial_update_requires_corpus_admin(self, has_access_mock):
self.client.force_login(self.user)
for role in [Role.Guest, Role.Contributor]:
with self.subTest(role=role):
self.corpus.memberships.filter(user=self.user).update(level=role.value)
with self.assertNumQueries(5):
response = self.client.patch(
reverse("api:allowed-metadata-edit", kwargs={"corpus": self.corpus.id, "pk": self.test_meta.id}),
{"type": self.allowed_meta[2].type.value},
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.corpus.memberships.filter(user=self.user).update(level=Role.Contributor.value)
with self.assertNumQueries(3):
response = self.client.patch(
reverse("api:allowed-metadata-edit", kwargs={"corpus": self.corpus.id, "pk": self.test_meta.id}),
{"type": self.allowed_meta[2].type.value},
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_destroy(self):
self.client.force_login(self.user)
with self.assertNumQueries(8):
with self.assertNumQueries(5):
response = self.client.delete(reverse("api:allowed-metadata-edit", kwargs={"corpus": self.corpus.id, "pk": self.test_meta.id}))
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
......@@ -372,7 +376,7 @@ class TestAllowedMetaData(FixtureAPITestCase):
def test_destroy_not_found(self):
self.client.force_login(self.user)
with self.assertNumQueries(7):
with self.assertNumQueries(4):
response = self.client.delete(reverse("api:allowed-metadata-edit", kwargs={"corpus": self.corpus.id, "pk": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}))
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
......@@ -395,17 +399,17 @@ class TestAllowedMetaData(FixtureAPITestCase):
response = self.client.delete(reverse("api:allowed-metadata-edit", kwargs={"corpus": self.corpus.id, "pk": self.test_meta.id}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_destroy_private_corpus(self):
@patch("arkindex.project.mixins.has_access", return_value=False)
def test_destroy_private_corpus(self, has_access_mock):
self.client.force_login(self.user)
with self.assertNumQueries(6):
with self.assertNumQueries(3):
response = self.client.delete(reverse("api:allowed-metadata-edit", kwargs={"corpus": self.private_corpus.id, "pk": self.test_meta.id}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_destroy_requires_corpus_admin(self):
@patch("arkindex.project.mixins.has_access", return_value=False)
def test_destroy_requires_corpus_admin(self, has_access_mock):
self.client.force_login(self.user)
for role in [Role.Guest, Role.Contributor]:
with self.subTest(role=role):
self.corpus.memberships.filter(user=self.user).update(level=role.value)
with self.assertNumQueries(5):
response = self.client.delete(reverse("api:allowed-metadata-edit", kwargs={"corpus": self.corpus.id, "pk": self.test_meta.id}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.corpus.memberships.filter(user=self.user).update(level=Role.Contributor.value)
with self.assertNumQueries(3):
response = self.client.delete(reverse("api:allowed-metadata-edit", kwargs={"corpus": self.corpus.id, "pk": self.test_meta.id}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
arkindex/documents/tests/test_bulk_transcriptions.py
View file @ acf2b82c
from unittest.mock import patch
from django.urls import reverse
from rest_framework import status
from arkindex.documents.models import TextOrientation
from arkindex.documents.models import Corpus, TextOrientation
from arkindex.process.models import ProcessMode, WorkerRun, WorkerVersion
from arkindex.project.tests import FixtureAPITestCase
......@@ -30,12 +32,13 @@ class TestBulkTranscriptions(FixtureAPITestCase):
response = self.client.post(reverse("api:transcription-bulk"))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_bulk_transcriptions_not_found(self):
@patch("arkindex.users.managers.BaseACLManager.filter_rights", return_value=Corpus.objects.none())
def test_bulk_transcriptions_not_found(self, filter_rights_mock):
self.client.force_login(self.user)
self.user.rights.all().delete()
forbidden_element = self.corpus.elements.get(name="Volume 1, page 1r")
with self.assertNumQueries(6):
with self.assertNumQueries(3):
response = self.client.post(reverse("api:transcription-bulk"), {
"worker_run_id": str(self.local_worker_run.id),
"transcriptions": [
......@@ -121,7 +124,7 @@ class TestBulkTranscriptions(FixtureAPITestCase):
test_element = self.corpus.elements.get(name="Volume 2, page 1r")
self.assertFalse(test_element.transcriptions.exists())
with self.assertNumQueries(7):
with self.assertNumQueries(5):
response = self.client.post(reverse("api:transcription-bulk"), {
"worker_run_id": str(self.local_worker_run.id),
"transcriptions": [
......@@ -323,7 +326,7 @@ class TestBulkTranscriptions(FixtureAPITestCase):
self.assertFalse(element1.transcriptions.exists())
self.assertFalse(element2.transcriptions.exists())
with self.assertNumQueries(7):
with self.assertNumQueries(5):
response = self.client.post(reverse("api:transcription-bulk"), {
"worker_run_id": str(self.local_worker_run.id),
"transcriptions": [
......@@ -413,7 +416,7 @@ class TestBulkTranscriptions(FixtureAPITestCase):
element = self.corpus.elements.get(name="Volume 2, page 1r")
self.assertFalse(element.transcriptions.exists())
with self.assertNumQueries(6):
with self.assertNumQueries(4):
response = self.client.post(
reverse("api:transcription-bulk"),
format="json",
......@@ -465,7 +468,7 @@ class TestBulkTranscriptions(FixtureAPITestCase):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
with self.assertNumQueries(6):
with self.assertNumQueries(4):
response = self.client.post(
reverse("api:transcription-bulk"),
format="json",
......
arkindex/documents/tests/test_classes.py
View file @ acf2b82c
This diff is collapsed.
arkindex/documents/tests/test_corpus.py
View file @ acf2b82c
This diff is collapsed.