diff --git a/Dockerfile b/Dockerfile
index 888f8e778d2b3e87ff07359a72a9d10030e51232..366bebc2564c2ed5ae6741e1287dd03872b14958 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,10 +1,10 @@
-FROM registry.gitlab.com/arkindex/backend/base:python-3.7 as build
+FROM registry.gitlab.com/arkindex/backend/base:django-3.1 as build
RUN mkdir build
ADD . build
RUN cd build && python3 setup.py sdist
-FROM registry.gitlab.com/arkindex/backend/base:latest
+FROM registry.gitlab.com/arkindex/backend/base:django-3.1
ARG COMMON_BRANCH=master
ARG COMMON_ID=9855787
diff --git a/Dockerfile.binary b/Dockerfile.binary
index 3f38d8ee5a57c9b4ce4a4d59be44d22a9abc18f8..e072406b890fd96d28f2cc8ad6b437c1d7c1194b 100644
--- a/Dockerfile.binary
+++ b/Dockerfile.binary
@@ -57,7 +57,7 @@ RUN python -m nuitka \
arkindex/manage.py
# Start over from a clean setup
-FROM registry.gitlab.com/arkindex/backend/base:python-3.7 as build
+FROM registry.gitlab.com/arkindex/backend/base:django-3.1 as build
# Import files from compilation
RUN mkdir /usr/share/arkindex
diff --git a/arkindex/dataimport/migrations/0016_new_jsonfield.py b/arkindex/dataimport/migrations/0016_new_jsonfield.py
new file mode 100644
index 0000000000000000000000000000000000000000..d5a2ac741ae5275476071a8dfcb4411868a7ee8a
--- /dev/null
+++ b/arkindex/dataimport/migrations/0016_new_jsonfield.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.1 on 2020-08-10 14:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('dataimport', '0015_clear_payload'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='workerversion',
+ name='configuration',
+ field=models.JSONField(),
+ ),
+ ]
diff --git a/arkindex/dataimport/models.py b/arkindex/dataimport/models.py
index 7509378e1f6851362f5e1a549afe7c29c45de842..7111c0a6cb7cd40a20a4f536218c9265369bf40d 100644
--- a/arkindex/dataimport/models.py
+++ b/arkindex/dataimport/models.py
@@ -1,5 +1,4 @@
from django.db import models
-from django.contrib.postgres.fields import JSONField
from django.conf import settings
from django.utils.functional import cached_property
from rest_framework.exceptions import ValidationError
@@ -392,7 +391,7 @@ class WorkerVersion(models.Model):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
worker = models.ForeignKey('dataimport.Worker', on_delete=models.CASCADE)
revision = models.ForeignKey('dataimport.Revision', on_delete=models.CASCADE, related_name='versions')
- configuration = JSONField()
+ configuration = models.JSONField()
docker_image = models.ForeignKey(Artifact, on_delete=models.CASCADE, null=True)
state = EnumField(WorkerVersionState, default=WorkerVersionState.Created)
diff --git a/arkindex/dataimport/serializers/workers.py b/arkindex/dataimport/serializers/workers.py
index 32035d80b2833b37633ab6c3154ee178978ecd9e..cc5eb30dd5cc7f1f34aba8d5249f80de31cd9e73 100644
--- a/arkindex/dataimport/serializers/workers.py
+++ b/arkindex/dataimport/serializers/workers.py
@@ -27,6 +27,9 @@ class WorkerVersionSerializer(serializers.ModelSerializer):
"""
state = EnumField(WorkerVersionState, required=False)
worker = WorkerSerializer(read_only=True)
+ # ModelSerializer does not yet support Django 3.1's JSONField
+ # https://github.com/encode/django-rest-framework/pull/7467
+ configuration = serializers.JSONField()
class Meta:
model = WorkerVersion
diff --git a/arkindex/documents/api/entities.py b/arkindex/documents/api/entities.py
index 89377c11df86083c43254fe3b0db05390836c24e..508d451bdb05adf7f8c739b5a0e512723d921c36 100644
--- a/arkindex/documents/api/entities.py
+++ b/arkindex/documents/api/entities.py
@@ -107,15 +107,17 @@ class EntityElements(ListAPIView):
.filter(
corpus__in=Corpus.objects.readable(self.request.user),
metadatas__entity_id=pk
- ).select_related('type')
+ ) \
+ .select_related('type') \
+ .prefetch_related('metadatas__entity', 'metadatas__revision', 'corpus')
transcription_elements = Element.objects \
.filter(
corpus__in=Corpus.objects.readable(self.request.user),
transcriptions__transcription_entities__entity_id=pk
- ).select_related('type')
- return metadata_elements.union(transcription_elements) \
- .order_by('name', 'type') \
+ ).select_related('type') \
.prefetch_related('metadatas__entity', 'metadatas__revision', 'corpus')
+ return metadata_elements.union(transcription_elements) \
+ .order_by('name', 'type')
class EntityCreate(CreateAPIView):
diff --git a/arkindex/documents/api/ml.py b/arkindex/documents/api/ml.py
index 9879819b3bf98e31c700cf17b2a0a3589aad7a98..dad743131f84445b85659cbb209ad52b5b43eb58 100644
--- a/arkindex/documents/api/ml.py
+++ b/arkindex/documents/api/ml.py
@@ -381,7 +381,8 @@ class CorpusMLClassList(CorpusACLMixin, ListAPIView):
.annotate(nb_best=Count(
'classifications',
filter=best_classification_filter,
- ))
+ )) \
+ .order_by('name')
class MLClassList(ListAPIView):
diff --git a/arkindex/documents/tests/consumers/test_ml_results_consumer.py b/arkindex/documents/tests/consumers/test_ml_results_consumer.py
index 25feab0a4a254746c3a8a6dd622590a90ac3deb3..35a0844c1beb6953069e8ce4f97d30ca18f86dff 100644
--- a/arkindex/documents/tests/consumers/test_ml_results_consumer.py
+++ b/arkindex/documents/tests/consumers/test_ml_results_consumer.py
@@ -94,7 +94,7 @@ class TestMLResultsConsumer(FixtureTestCase):
self.assertEqual(self.page1.metadatas.count(), 2)
self.assertEqual(self.page2.metadatas.count(), 2)
- with self.assertNumQueries(14):
+ with self.assertNumQueries(13):
MLResultsConsumer({}).ml_results_delete({'corpus_id': str(self.corpus.id)})
for queryset in querysets:
@@ -144,7 +144,7 @@ class TestMLResultsConsumer(FixtureTestCase):
for queryset in folder2_querysets:
self.assertTrue(queryset.exists())
- with self.assertNumQueries(15):
+ with self.assertNumQueries(14):
MLResultsConsumer({}).ml_results_delete({'element_id': str(self.folder1.id)})
for queryset in folder1_querysets:
diff --git a/arkindex/project/config.py b/arkindex/project/config.py
index 39f05889d9814dfa036d440b66108071954e0b4f..d09d1fe2e2a6c0317e0d49945f4d2c74e588d9b5 100644
--- a/arkindex/project/config.py
+++ b/arkindex/project/config.py
@@ -12,11 +12,19 @@ class CacheType(Enum):
class CookieSameSiteOption(Enum):
+ """
+ Options for the SameSite flag on a cookie. Django accepts Lax, Strict, None and False.
+ None is a 'None' string, not Python's None, and disables the SameSite protection.
+ This can cause warnings when the Secure flag is active.
+ False removes the flag entirely from the cookie, leaving the decision up to the browser, which can cause warnings.
+
+ https://docs.djangoproject.com/en/3.1/ref/settings/#std:setting-SESSION_COOKIE_SAMESITE
+ """
Lax = 'lax'
Strict = 'strict'
# Cannot redefine Python's None!
- # Django needs a real None here to disable the check
- None_ = None
+ None_ = 'none'
+ Disabled = False
def get_settings_parser(base_dir):
@@ -86,9 +94,8 @@ def get_settings_parser(base_dir):
cors_parser = parser.add_subparser('cors', default={})
cors_parser.add_option('origin_whitelist', type=str, many=True, default=[
- 'universalviewer.io', # TODO: Remove this one?
- 'localhost:8080',
- '127.0.0.1:8080',
+ 'http://localhost:8080',
+ 'http://127.0.0.1:8080',
])
cors_parser.add_option('suffixes', type=str, many=True, default=[])
diff --git a/arkindex/project/polygon.py b/arkindex/project/polygon.py
index 4e13efb422be3caa2409291f8a9202f734b1c4cc..89b6a332b29a8453eba62c3fc25465b06af83775 100644
--- a/arkindex/project/polygon.py
+++ b/arkindex/project/polygon.py
@@ -14,7 +14,10 @@ class Point(namedtuple('Point', ['x', 'y'])):
"""
__slots__ = ()
- def __new__(cls, x, y):
+ def __new__(cls, x, y=None):
+ # Allow both Point(1, 2) and Point(tuple(1, 2)) for Django 3 compatibility
+ if isinstance(x, Iterable):
+ x, y = x
return super().__new__(cls, int(x), int(y))
def __str__(self):
diff --git a/arkindex/project/tests/config_samples/defaults.yaml b/arkindex/project/tests/config_samples/defaults.yaml
index 1c698d6d91d2f7c623557487bcfbacbe1c4b69a5..31bb76bfe6624259011e0a26652af00b5d1f9433 100644
--- a/arkindex/project/tests/config_samples/defaults.yaml
+++ b/arkindex/project/tests/config_samples/defaults.yaml
@@ -9,9 +9,8 @@ cache:
url: null
cors:
origin_whitelist:
- - universalviewer.io
- - localhost:8080
- - 127.0.0.1:8080
+ - http://localhost:8080
+ - http://127.0.0.1:8080
suffixes: []
csrf:
cookie_domain: null
diff --git a/arkindex/project/tests/config_samples/override.yaml b/arkindex/project/tests/config_samples/override.yaml
index de8575efe8d362fe67b8f2e6f4c9994505fa4a79..96d3482c2b54de763121ae1d54f99495f2417448 100644
--- a/arkindex/project/tests/config_samples/override.yaml
+++ b/arkindex/project/tests/config_samples/override.yaml
@@ -81,7 +81,7 @@ sentry:
session:
cookie_domain: cookie-dolmen
cookie_name: stonehenge
- cookie_samesite: null
+ cookie_samesite: false
static:
cdn_assets_url: http://cdn.teklia.horse/
frontend_version: 1.2.3-alpha4
diff --git a/arkindex/project/tests/test_config.py b/arkindex/project/tests/test_config.py
index b6f816c9d8d37af1cfce41903c53a659a9115f62..355f507446c50c7916b3e00ab8c10fec4d511013 100644
--- a/arkindex/project/tests/test_config.py
+++ b/arkindex/project/tests/test_config.py
@@ -22,11 +22,11 @@ class TestConfig(TestCase):
def str_representer(self, data):
if isinstance(data, Enum):
data = data.value
- else:
- data = str(data)
if data is None:
return self.represent_none(data)
- return self.represent_str(data)
+ elif isinstance(data, (bool, int, float, bytes, str)):
+ return self.represent_data(data)
+ return self.represent_str(str(data))
dumper.add_representer(None, str_representer)
dumper.ignore_aliases = lambda *args: True
diff --git a/base/requirements.txt b/base/requirements.txt
index 8f79c4baeebbf30252a8184d52ba660f8632160b..fbbd84f820f8e112e7fe5f061f4cc14bd343f103 100644
--- a/base/requirements.txt
+++ b/base/requirements.txt
@@ -1,6 +1,6 @@
boto3==1.9
cryptography>=2.8
-Django==2.2.13
+Django==3.1
elasticsearch==6.2.0
hiredis==1.0.0
ijson==2.3
diff --git a/requirements.txt b/requirements.txt
index b2807536cb3a5926cae4fc961abd801e1b0054a8..0dd9cae66ef4e8959bceae20f0ffcd200629cef3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,12 +8,12 @@ certifi==2017.7.27.1
channels==2.3.1
channels-redis==2.4.1
chardet==3.0.4
-django-admin-hstore-widget==1.0.1
+django-admin-hstore-widget==1.1.0
django-cachalot==2.2.2
-django-cors-headers==2.4.0
-django-enumfields==1.0.0
+django-cors-headers==3.4.0
+django-enumfields==2.0.0
django-redis==4.12.1
-djangorestframework==3.11.0
+djangorestframework==3.11.1
elasticsearch-dsl>=6.0.0,<7.0.0
gitpython==3.0.8
idna==2.6