Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • arkindex/backend
1 result
Show changes
Commits on Source (8)
Showing
with 362 additions and 14 deletions
......@@ -2,7 +2,15 @@ from django.contrib import admin
from django.db.models import Max
from enumfields.admin import EnumFieldListFilter
from arkindex.dataimport.models import DataFile, DataImport, Repository, Revision, Worker, WorkerVersion
from arkindex.dataimport.models import (
DataFile,
DataImport,
Repository,
Revision,
Worker,
WorkerConfiguration,
WorkerVersion,
)
from arkindex.users.admin import GroupMembershipInline, UserMembershipInline
......@@ -85,11 +93,16 @@ class WorkerVersionInline(admin.StackedInline):
return super().get_queryset(*args, **kwargs).prefetch_related('worker', 'revision')
class WorkerConfigurationInline(admin.StackedInline):
model = WorkerConfiguration
readonly_fields = ('configuration_hash', )
class WorkerAdmin(admin.ModelAdmin):
list_display = ('id', 'name', 'slug', 'type', 'repository')
field = ('id', 'name', 'slug', 'type', 'repository')
readonly_fields = ('id', )
inlines = [WorkerVersionInline, UserMembershipInline, GroupMembershipInline]
inlines = [WorkerVersionInline, UserMembershipInline, GroupMembershipInline, WorkerConfigurationInline]
class WorkerVersionAdmin(admin.ModelAdmin):
......@@ -99,9 +112,16 @@ class WorkerVersionAdmin(admin.ModelAdmin):
readonly_fields = ('id', )
class WorkerConfigurationAdmin(admin.ModelAdmin):
list_display = ('id', 'name', 'worker')
list_filter = ('worker', )
readonly_fields = ('id', 'configuration_hash')
admin.site.register(DataImport, DataImportAdmin)
admin.site.register(DataFile, DataFileAdmin)
admin.site.register(Revision, RevisionAdmin)
admin.site.register(Repository, RepositoryAdmin)
admin.site.register(Worker, WorkerAdmin)
admin.site.register(WorkerVersion, WorkerVersionAdmin)
admin.site.register(WorkerConfiguration, WorkerConfigurationAdmin)
......@@ -44,6 +44,7 @@ from arkindex.dataimport.models import (
Worker,
WorkerActivity,
WorkerActivityState,
WorkerConfiguration,
WorkerRun,
WorkerVersion,
)
......@@ -65,6 +66,7 @@ from arkindex.dataimport.serializers.imports import (
from arkindex.dataimport.serializers.workers import (
RepositorySerializer,
WorkerActivitySerializer,
WorkerConfigurationSerializer,
WorkerSerializer,
WorkerStatisticsSerializer,
WorkerVersionEditSerializer,
......@@ -985,6 +987,30 @@ class WorkerRetrieve(WorkerACLMixin, RetrieveAPIView):
return worker
@extend_schema(
tags=['repos'],
description=(
'List configurations for a given worker ID.\n\n'
'Requires a **read** access to the worker or its repository.'
)
)
class WorkerConfigurationList(WorkerACLMixin, ListAPIView):
permission_classes = (IsVerified, )
serializer_class = WorkerConfigurationSerializer
queryset = WorkerConfiguration.objects.none()
def get_queryset(self):
worker = get_object_or_404(
Worker.objects.select_related('repository'),
pk=self.kwargs['pk']
)
if not self.has_read_access(worker):
raise PermissionDenied(detail='You do not have a guest access to this worker.')
return worker.configurations.order_by('name')
@extend_schema(tags=['imports'])
@extend_schema_view(
get=extend_schema(description=(
......
# Generated by Django 3.2.5 on 2021-11-04 09:42
import uuid
import django.db.models.deletion
from django.db import migrations, models
import arkindex.project.fields
class Migration(migrations.Migration):
dependencies = [
('dataimport', '0038_dataimport_use_gpu'),
]
operations = [
migrations.CreateModel(
name='WorkerConfiguration',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
('name', models.CharField(max_length=250)),
('configuration', models.JSONField(blank=True, default=dict)),
('configuration_hash', arkindex.project.fields.MD5HashField(max_length=32)),
],
),
migrations.RemoveConstraint(
model_name='workerrun',
name='worker_run_configuration_objects',
),
migrations.RenameField(
model_name='workerrun',
old_name='configuration',
new_name='old_configuration',
),
migrations.AddConstraint(
model_name='workerrun',
constraint=models.CheckConstraint(check=models.Q(('old_configuration__typeof', 'object')), name='worker_run_old_configuration_objects'),
),
migrations.AddField(
model_name='workerconfiguration',
name='worker',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='configurations', to='dataimport.worker'),
),
migrations.AddField(
model_name='workeractivity',
name='configuration',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='worker_activities', to='dataimport.workerconfiguration'),
),
migrations.AddField(
model_name='workerrun',
name='configuration',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='worker_runs', to='dataimport.workerconfiguration'),
),
migrations.AddConstraint(
model_name='workerconfiguration',
constraint=models.CheckConstraint(check=models.Q(('configuration__typeof', 'object')), name='worker_configuration_configuration_objects'),
),
migrations.AlterUniqueTogether(
name='workerconfiguration',
unique_together={('worker', 'name'), ('worker', 'configuration_hash')},
),
]
......@@ -18,7 +18,7 @@ from arkindex.dataimport.providers import get_provider, git_providers
from arkindex.dataimport.utils import get_default_farm_id
from arkindex.documents.models import ClassificationState, Element
from arkindex.project.aws import S3FileMixin, S3FileStatus
from arkindex.project.fields import ArrayField
from arkindex.project.fields import ArrayField, MD5HashField
from arkindex.project.models import IndexableModel
from ponos.models import Artifact, State, Workflow
......@@ -601,19 +601,49 @@ class WorkerVersion(models.Model):
return self.configuration['docker'].get('command')
class WorkerConfiguration(IndexableModel):
name = models.CharField(max_length=250)
configuration = models.JSONField(default=dict, blank=True)
configuration_hash = MD5HashField()
worker = models.ForeignKey(
Worker,
on_delete=models.CASCADE,
related_name='configurations',
)
class Meta:
unique_together = (
('worker', 'configuration_hash'),
('worker', 'name')
)
constraints = [
models.CheckConstraint(
check=models.Q(configuration__typeof='object'),
name='worker_configuration_configuration_objects',
)
]
class WorkerRun(models.Model):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
dataimport = models.ForeignKey('dataimport.DataImport', on_delete=models.CASCADE, related_name='worker_runs')
version = models.ForeignKey('dataimport.WorkerVersion', on_delete=models.CASCADE, related_name='worker_runs')
parents = ArrayField(models.UUIDField())
configuration = models.JSONField(default=dict)
old_configuration = models.JSONField(default=dict)
configuration = models.ForeignKey(
WorkerConfiguration,
related_name='worker_runs',
on_delete=models.SET_NULL,
null=True,
blank=True,
)
class Meta:
unique_together = (('version', 'dataimport'),)
constraints = [
models.CheckConstraint(
check=models.Q(configuration__typeof='object'),
name='worker_run_configuration_objects',
check=models.Q(old_configuration__typeof='object'),
name='worker_run_old_configuration_objects',
)
]
......@@ -692,6 +722,13 @@ class WorkerActivity(IndexableModel):
null=True,
blank=True,
)
configuration = models.ForeignKey(
WorkerConfiguration,
related_name='worker_activities',
on_delete=models.SET_NULL,
null=True,
blank=True,
)
# Specific WorkerActivity manager
objects = ActivityManager()
......
......@@ -323,7 +323,7 @@ class WorkerRunSerializer(serializers.ModelSerializer):
# Serialize worker with its basic informations
worker = WorkerLightSerializer(source='version.worker', read_only=True)
# A DictField will require valid dicts, but without a child= argument, it will allow any value
configuration = serializers.DictField(allow_empty=True, default={})
configuration = serializers.DictField(source='old_configuration', allow_empty=True, default={})
class Meta:
model = WorkerRun
......
......@@ -12,6 +12,7 @@ from arkindex.dataimport.models import (
Worker,
WorkerActivity,
WorkerActivityState,
WorkerConfiguration,
WorkerVersion,
WorkerVersionGPUUsage,
WorkerVersionState,
......@@ -186,3 +187,13 @@ class WorkerStatisticsSerializer(serializers.Serializer):
started = serializers.IntegerField(read_only=True)
processed = serializers.IntegerField(read_only=True)
error = serializers.IntegerField(read_only=True)
class WorkerConfigurationSerializer(serializers.ModelSerializer):
class Meta:
model = WorkerConfiguration
fields = (
'id',
'name',
'configuration',
)
import json
from hashlib import md5
from django.db.models.signals import pre_save
from django.dispatch import receiver
from rest_framework.exceptions import ValidationError
from arkindex.dataimport.models import WorkerRun
from arkindex.dataimport.models import WorkerConfiguration, WorkerRun
def _list_ancestors(graph, parents):
......@@ -39,3 +42,9 @@ def check_parents(sender, instance, **kwargs):
ancestors = _list_ancestors(graph, instance.parents)
if instance.id in ancestors:
raise ValidationError(f"Can't add or update WorkerRun {instance.id} because parents field isn't properly defined. It would create a cycle.")
@receiver(pre_save, sender=WorkerConfiguration)
def update_configuration_hash(sender, instance, **kwargs):
configuration_json = json.dumps(instance.configuration, sort_keys=True).encode('utf-8')
instance.configuration_hash = md5(configuration_json).hexdigest()
......@@ -437,7 +437,7 @@ class TestWorkerRuns(FixtureAPITestCase):
def test_update_run_configuration(self):
self.client.force_login(self.user)
self.assertDictEqual(self.run_1.configuration, {})
self.assertDictEqual(self.run_1.old_configuration, {})
with self.assertNumQueries(8):
response = self.client.patch(
reverse('api:worker-run-details', kwargs={'pk': self.run_1.id}),
......@@ -453,7 +453,7 @@ class TestWorkerRuns(FixtureAPITestCase):
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.run_1.refresh_from_db()
self.assertDictEqual(self.run_1.configuration, {
self.assertDictEqual(self.run_1.old_configuration, {
'a': 'b',
'c': {
'd': 42
......
......@@ -959,3 +959,52 @@ class TestWorkersWorkerVersions(FixtureAPITestCase):
data={'revision': str(self.rev2.id), 'configuration': {"test": "test2"}, 'gpu_usage': 'not_supported'}, format='json'
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_configurations_list_requires_login(self):
response = self.client.get(reverse('api:worker-configurations', kwargs={'pk': str(self.worker_1.id)}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_configurations_list_requires_verified(self):
self.user.verified_email = False
self.user.save()
self.client.force_login(self.user)
response = self.client.get(reverse('api:worker-configurations', kwargs={'pk': str(self.worker_1.id)}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_configurations_list_no_rights(self):
self.worker_1.repository.memberships.filter(user=self.user).delete()
self.client.force_login(self.user)
with self.assertNumQueries(7):
response = self.client.get(reverse('api:worker-configurations', kwargs={'pk': str(self.worker_1.id)}))
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertEqual(response.json(), {'detail': 'You do not have a guest access to this worker.'})
def test_configurations_list(self):
"""
A user is able to retrieve a worker configuation if he has a guest access on it or its repository
"""
repo2 = Repository.objects.create(
url='http://gitlab/repo2',
type=RepositoryType.Worker,
hook_token='hook-token2',
credentials=self.creds,
provider_name='GitLabProvider'
)
worker_2 = repo2.workers.create(name='Worker 2', slug='worker_2', type='classifier')
config_1 = worker_2.configurations.create(name='config_1', configuration={'key': 'value'})
config_2 = worker_2.configurations.create(name='config_2')
repo2.memberships.create(user=self.user, level=Role.Guest.value)
self.client.force_login(self.user)
with self.assertNumQueries(9):
response = self.client.get(reverse('api:worker-configurations', kwargs={'pk': str(worker_2.id)}))
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertListEqual(response.json()['results'], [{
'id': str(config_1.id),
'name': 'config_1',
'configuration': {'key': 'value'}
}, {
'id': str(config_2.id),
'name': 'config_2',
'configuration': {}
}])
......@@ -14,6 +14,7 @@ from arkindex.documents.models import (
Element,
ElementType,
MLClass,
TextOrientation,
Transcription,
)
from arkindex.documents.serializers.light import ElementZoneSerializer
......@@ -218,6 +219,7 @@ class TranscriptionSerializer(serializers.ModelSerializer):
read_only=True,
help_text='This field is deprecated; please use the `confidence` field instead.',
)
orientation = EnumField(TextOrientation)
class Meta:
model = Transcription
......@@ -227,6 +229,7 @@ class TranscriptionSerializer(serializers.ModelSerializer):
'text',
'score',
'confidence',
'orientation',
'worker_version_id',
)
......@@ -268,10 +271,11 @@ class TranscriptionCreateSerializer(serializers.ModelSerializer):
max_value=1,
required=False,
)
orientation = EnumField(TextOrientation, default=TextOrientation.HorizontalLeftToRight, required=False)
class Meta:
model = Transcription
fields = ('text', 'worker_version', 'score', 'confidence')
fields = ('text', 'worker_version', 'score', 'confidence', 'orientation')
def validate(self, data):
data = super().validate(data)
......
......@@ -5,7 +5,7 @@ from django.urls import reverse
from rest_framework import status
from arkindex.dataimport.models import WorkerVersion
from arkindex.documents.models import Corpus, Transcription
from arkindex.documents.models import Corpus, TextOrientation, Transcription
from arkindex.project.tests import FixtureAPITestCase
from arkindex.users.models import Role, User
......@@ -73,6 +73,7 @@ class TestTranscriptionCreate(FixtureAPITestCase):
'confidence': None,
'score': None,
'text': 'A perfect day in a perfect place',
'orientation': TextOrientation.HorizontalLeftToRight.value,
'worker_version_id': None,
})
......@@ -101,6 +102,41 @@ class TestTranscriptionCreate(FixtureAPITestCase):
2
)
def test_create_transcription_with_orientation(self):
"""
Check that a transcription is created with the specified orientation
"""
self.client.force_login(self.user)
response = self.client.post(
reverse('api:transcription-create', kwargs={'pk': self.line.id}),
format='json',
data={'text': 'A perfect day in a perfect place', 'orientation': 'vertical-lr'}
)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
tr = Transcription.objects.get(text='A perfect day in a perfect place')
self.assertDictEqual(response.json(), {
'id': str(tr.id),
'confidence': None,
'score': None,
'text': 'A perfect day in a perfect place',
'orientation': 'vertical-lr',
'worker_version_id': None,
})
new_ts = Transcription.objects.get(text='A perfect day in a perfect place')
self.assertEqual(new_ts.orientation, TextOrientation.VerticalLeftToRight)
def test_create_transcription_invalid_orientation(self):
"""
Specifying an invalid text-orientation causes an error
"""
self.client.force_login(self.user)
response = self.client.post(
reverse('api:transcription-create', kwargs={'pk': self.line.id}),
format='json',
data={'text': 'A perfect day in a perfect place', 'orientation': 'wiggly'}
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
@override_settings(ARKINDEX_FEATURES={'search': False})
def test_create_transcription_no_search(self):
self.client.force_login(self.user)
......@@ -151,6 +187,7 @@ class TestTranscriptionCreate(FixtureAPITestCase):
'confidence': .42,
'score': .42,
'text': 'NEKUDOTAYIM',
'orientation': TextOrientation.HorizontalLeftToRight.value,
'worker_version_id': str(self.worker_version.id),
})
......@@ -176,6 +213,7 @@ class TestTranscriptionCreate(FixtureAPITestCase):
'confidence': .42,
'score': .42,
'text': 'NEKUDOTAYIM',
'orientation': TextOrientation.HorizontalLeftToRight.value,
'worker_version_id': str(self.worker_version.id),
})
......
......@@ -3,7 +3,7 @@ from uuid import uuid4
from django.urls import reverse
from rest_framework import status
from arkindex.documents.models import Corpus, Element, EntityType, Transcription
from arkindex.documents.models import Corpus, Element, EntityType, TextOrientation, Transcription
from arkindex.project.tests import FixtureAPITestCase
from arkindex.users.models import Role, User
......@@ -53,6 +53,7 @@ class TestEditTranscription(FixtureAPITestCase):
'confidence': None,
'score': None,
'text': 'A manual transcription',
'orientation': TextOrientation.HorizontalLeftToRight.value,
'worker_version_id': None,
})
......@@ -144,9 +145,92 @@ class TestEditTranscription(FixtureAPITestCase):
'confidence': None,
'score': None,
'text': 'a knight was living lonely',
'orientation': TextOrientation.HorizontalLeftToRight.value,
'worker_version_id': None,
})
def test_transcription_patch_orientation(self):
"""
Assert it is possible to edit only the text orientation
"""
self.client.force_login(self.user)
manual_tr_id = self.manual_transcription.id
response = self.client.patch(
reverse('api:transcription-edit', kwargs={'pk': manual_tr_id}),
format='json',
data={
'orientation': 'vertical-rl'
}
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.manual_transcription.refresh_from_db()
self.assertDictEqual(response.json(), {
'id': str(manual_tr_id),
'confidence': None,
'score': None,
'text': 'A manual transcription',
'orientation': 'vertical-rl',
'worker_version_id': None,
})
def test_transcription_patch_invalid_orientation(self):
"""
An invalid text orientation value causes an error
"""
self.client.force_login(self.user)
manual_tr_id = self.manual_transcription.id
response = self.client.patch(
reverse('api:transcription-edit', kwargs={'pk': manual_tr_id}),
format='json',
data={
'orientation': 'wobbly'
}
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(response.json(), {'orientation': ['Value is not of type TextOrientation']})
def test_transcription_edit_orientation(self):
"""
Assert it is possible to edit the text orientation with UpdateTranscription
"""
self.client.force_login(self.user)
manual_tr_id = self.manual_transcription.id
response = self.client.put(
reverse('api:transcription-edit', kwargs={'pk': manual_tr_id}),
format='json',
data={
'text': 'a knight was living lonely',
'orientation': 'vertical-rl',
}
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.manual_transcription.refresh_from_db()
self.assertDictEqual(response.json(), {
'id': str(manual_tr_id),
'confidence': None,
'score': None,
'text': 'a knight was living lonely',
'orientation': 'vertical-rl',
'worker_version_id': None,
})
def test_transcription_edit_invalid_orientation(self):
"""
An invalid text orientation value causes an error
"""
self.client.force_login(self.user)
manual_tr_id = self.manual_transcription.id
response = self.client.put(
reverse('api:transcription-edit', kwargs={'pk': manual_tr_id}),
format='json',
data={
'text': 'a knight was living lonely',
'orientation': 'wobbly',
}
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(response.json(), {'orientation': ['Value is not of type TextOrientation']})
def test_transcription_patch_write_right(self):
"""
A write right is required to patch a manual transcription
......
......@@ -2,7 +2,7 @@ from django.urls import reverse
from rest_framework import status
from arkindex.dataimport.models import WorkerVersion
from arkindex.documents.models import Corpus
from arkindex.documents.models import Corpus, TextOrientation
from arkindex.project.tests import FixtureAPITestCase
from arkindex.users.models import User
......@@ -52,6 +52,7 @@ class TestTranscriptions(FixtureAPITestCase):
'id': str(tr1.id),
'text': 'Lorem ipsum dolor sit amet',
'confidence': 1.0,
'orientation': TextOrientation.HorizontalLeftToRight.value,
'score': 1.0,
'worker_version_id': str(self.worker_version_1.id),
'element': None,
......@@ -60,6 +61,7 @@ class TestTranscriptions(FixtureAPITestCase):
'id': str(tr2.id),
'text': 'something',
'confidence': 0.369,
'orientation': TextOrientation.HorizontalLeftToRight.value,
'score': 0.369,
'worker_version_id': str(self.worker_version_2.id),
'element': None,
......@@ -129,6 +131,7 @@ class TestTranscriptions(FixtureAPITestCase):
'text': 'something',
'score': 0.369,
'confidence': 0.369,
'orientation': TextOrientation.HorizontalLeftToRight.value,
'worker_version_id': str(self.worker_version_2.id),
'element': {
'id': str(self.page.id),
......
......@@ -22,6 +22,7 @@ from arkindex.dataimport.api import (
RevisionRetrieve,
StartProcess,
UpdateWorkerActivity,
WorkerConfigurationList,
WorkerList,
WorkerRetrieve,
WorkerRunDetails,
......@@ -205,6 +206,7 @@ api = [
# Workers
path('workers/', WorkerList.as_view(), name='workers-list'),
path('workers/<uuid:pk>/', WorkerRetrieve.as_view(), name='worker-retrieve'),
path('workers/<uuid:pk>/configurations/', WorkerConfigurationList.as_view(), name='worker-configurations'),
path('workers/<uuid:pk>/versions/', WorkerVersionList.as_view(), name='worker-versions'),
path('workers/versions/<uuid:pk>/', WorkerVersionRetrieve.as_view(), name='version-retrieve'),
path('workers/versions/<uuid:pk>/activity/', UpdateWorkerActivity.as_view(), name='update-worker-activity'),
......