Bump to Django 3.1

Dockerfile

-FROM registry.gitlab.com/arkindex/backend/base:python-3.7 as build
+FROM registry.gitlab.com/arkindex/backend/base:django-3.1 as build
 
 RUN mkdir build
 ADD . build
 RUN cd build && python3 setup.py sdist
 
-FROM registry.gitlab.com/arkindex/backend/base:latest
+FROM registry.gitlab.com/arkindex/backend/base:django-3.1
 
 ARG COMMON_BRANCH=master
 ARG COMMON_ID=9855787

Dockerfile.binary

@@ -57,7 +57,7 @@ RUN python -m nuitka \
       arkindex/manage.py
 
 # Start over from a clean setup
-FROM registry.gitlab.com/arkindex/backend/base:python-3.7 as build
+FROM registry.gitlab.com/arkindex/backend/base:django-3.1 as build
 
 # Import files from compilation
 RUN mkdir /usr/share/arkindex

arkindex/dataimport/migrations/0016_new_jsonfield.py

+# Generated by Django 3.1 on 2020-08-10 14:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dataimport', '0015_clear_payload'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='workerversion',
+            name='configuration',
+            field=models.JSONField(),
+        ),
+    ]

arkindex/dataimport/models.py

 from django.db import models
-from django.contrib.postgres.fields import JSONField
 from django.conf import settings
 from django.utils.functional import cached_property
 from rest_framework.exceptions import ValidationError
@@ -392,7 +391,7 @@ class WorkerVersion(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     worker = models.ForeignKey('dataimport.Worker', on_delete=models.CASCADE)
     revision = models.ForeignKey('dataimport.Revision', on_delete=models.CASCADE, related_name='versions')
-    configuration = JSONField()
+    configuration = models.JSONField()
     docker_image = models.ForeignKey(Artifact, on_delete=models.CASCADE, null=True)
     state = EnumField(WorkerVersionState, default=WorkerVersionState.Created)
 

arkindex/dataimport/serializers/workers.py

@@ -27,6 +27,9 @@ class WorkerVersionSerializer(serializers.ModelSerializer):
     """
     state = EnumField(WorkerVersionState, required=False)
     worker = WorkerSerializer(read_only=True)
+    # ModelSerializer does not yet support Django 3.1's JSONField
+    # https://github.com/encode/django-rest-framework/pull/7467
+    configuration = serializers.JSONField()
 
     class Meta:
         model = WorkerVersion

arkindex/documents/api/entities.py

@@ -107,15 +107,17 @@ class EntityElements(ListAPIView):
             .filter(
                 corpus__in=Corpus.objects.readable(self.request.user),
                 metadatas__entity_id=pk
-            ).select_related('type')
+            ) \
+            .select_related('type') \
+            .prefetch_related('metadatas__entity', 'metadatas__revision', 'corpus')
         transcription_elements = Element.objects \
             .filter(
                 corpus__in=Corpus.objects.readable(self.request.user),
                 transcriptions__transcription_entities__entity_id=pk
-            ).select_related('type')
-        return metadata_elements.union(transcription_elements) \
-            .order_by('name', 'type') \
+            ).select_related('type') \
             .prefetch_related('metadatas__entity', 'metadatas__revision', 'corpus')
+        return metadata_elements.union(transcription_elements) \
+            .order_by('name', 'type')
 
 
 class EntityCreate(CreateAPIView):

arkindex/documents/api/ml.py

@@ -381,7 +381,8 @@ class CorpusMLClassList(CorpusACLMixin, ListAPIView):
             .annotate(nb_best=Count(
                 'classifications',
                 filter=best_classification_filter,
-            ))
+            )) \
+            .order_by('name')
 
 
 class MLClassList(ListAPIView):

arkindex/documents/tests/consumers/test_ml_results_consumer.py

@@ -94,7 +94,7 @@ class TestMLResultsConsumer(FixtureTestCase):
         self.assertEqual(self.page1.metadatas.count(), 2)
         self.assertEqual(self.page2.metadatas.count(), 2)
 
-        with self.assertNumQueries(14):
+        with self.assertNumQueries(13):
             MLResultsConsumer({}).ml_results_delete({'corpus_id': str(self.corpus.id)})
 
         for queryset in querysets:
@@ -144,7 +144,7 @@ class TestMLResultsConsumer(FixtureTestCase):
         for queryset in folder2_querysets:
             self.assertTrue(queryset.exists())
 
-        with self.assertNumQueries(15):
+        with self.assertNumQueries(14):
             MLResultsConsumer({}).ml_results_delete({'element_id': str(self.folder1.id)})
 
         for queryset in folder1_querysets:

arkindex/project/config.py

@@ -12,11 +12,19 @@ class CacheType(Enum):
 
 
 class CookieSameSiteOption(Enum):
+    """
+    Options for the SameSite flag on a cookie. Django accepts Lax, Strict, None and False.
+    None is a 'None' string, not Python's None, and disables the SameSite protection.
+    This can cause warnings when the Secure flag is active.
+    False removes the flag entirely from the cookie, leaving the decision up to the browser, which can cause warnings.
+
+    https://docs.djangoproject.com/en/3.1/ref/settings/#std:setting-SESSION_COOKIE_SAMESITE
+    """
     Lax = 'lax'
     Strict = 'strict'
     # Cannot redefine Python's None!
-    # Django needs a real None here to disable the check
-    None_ = None
+    None_ = 'none'
+    Disabled = False
 
 
 def get_settings_parser(base_dir):
@@ -86,9 +94,8 @@ def get_settings_parser(base_dir):
 
     cors_parser = parser.add_subparser('cors', default={})
     cors_parser.add_option('origin_whitelist', type=str, many=True, default=[
-        'universalviewer.io',  # TODO: Remove this one?
-        'localhost:8080',
-        '127.0.0.1:8080',
+        'http://localhost:8080',
+        'http://127.0.0.1:8080',
     ])
     cors_parser.add_option('suffixes', type=str, many=True, default=[])
 

arkindex/project/polygon.py

@@ -14,7 +14,10 @@ class Point(namedtuple('Point', ['x', 'y'])):
     """
     __slots__ = ()
 
-    def __new__(cls, x, y):
+    def __new__(cls, x, y=None):
+        # Allow both Point(1, 2) and Point(tuple(1, 2)) for Django 3 compatibility
+        if isinstance(x, Iterable):
+            x, y = x
         return super().__new__(cls, int(x), int(y))
 
     def __str__(self):

arkindex/project/tests/config_samples/defaults.yaml

@@ -9,9 +9,8 @@ cache:
   url: null
 cors:
   origin_whitelist:
-  - universalviewer.io
-  - localhost:8080
-  - 127.0.0.1:8080
+  - http://localhost:8080
+  - http://127.0.0.1:8080
   suffixes: []
 csrf:
   cookie_domain: null

arkindex/project/tests/config_samples/override.yaml

@@ -81,7 +81,7 @@ sentry:
 session:
   cookie_domain: cookie-dolmen
   cookie_name: stonehenge
-  cookie_samesite: null
+  cookie_samesite: false
 static:
   cdn_assets_url: http://cdn.teklia.horse/
   frontend_version: 1.2.3-alpha4

arkindex/project/tests/test_config.py

@@ -22,11 +22,11 @@ class TestConfig(TestCase):
         def str_representer(self, data):
             if isinstance(data, Enum):
                 data = data.value
-            else:
-                data = str(data)
             if data is None:
                 return self.represent_none(data)
-            return self.represent_str(data)
+            elif isinstance(data, (bool, int, float, bytes, str)):
+                return self.represent_data(data)
+            return self.represent_str(str(data))
 
         dumper.add_representer(None, str_representer)
         dumper.ignore_aliases = lambda *args: True

base/requirements.txt

 boto3==1.9
 cryptography>=2.8
-Django==2.2.13
+Django==3.1
 elasticsearch==6.2.0
 hiredis==1.0.0
 ijson==2.3

requirements.txt

@@ -8,12 +8,12 @@ certifi==2017.7.27.1
 channels==2.3.1
 channels-redis==2.4.1
 chardet==3.0.4
-django-admin-hstore-widget==1.0.1
+django-admin-hstore-widget==1.1.0
 django-cachalot==2.2.2
-django-cors-headers==2.4.0
-django-enumfields==1.0.0
+django-cors-headers==3.4.0
+django-enumfields==2.0.0
 django-redis==4.12.1
-djangorestframework==3.11.0
+djangorestframework==3.11.1
 elasticsearch-dsl>=6.0.0,<7.0.0
 gitpython==3.0.8
 idna==2.6