From 5b937e0aba09ffebe538fac816c8506407c0b7bf Mon Sep 17 00:00:00 2001
From: Erwan Rouchet <rouchet@teklia.com>
Date: Thu, 24 Aug 2023 07:26:15 +0000
Subject: [PATCH] Add export TTL setting
---
arkindex/documents/api/export.py | 25 +++++++++++--------
arkindex/documents/tests/test_export.py | 7 ++++--
arkindex/project/config.py | 3 +++
arkindex/project/settings.py | 3 +++
.../tests/config_samples/defaults.yaml | 2 ++
.../project/tests/config_samples/errors.yaml | 2 ++
.../tests/config_samples/expected_errors.yaml | 2 ++
.../tests/config_samples/override.yaml | 2 ++
8 files changed, 34 insertions(+), 12 deletions(-)
diff --git a/arkindex/documents/api/export.py b/arkindex/documents/api/export.py
index 220a8efe98..f5145b8bad 100644
--- a/arkindex/documents/api/export.py
+++ b/arkindex/documents/api/export.py
@@ -1,5 +1,8 @@
-from datetime import datetime, timedelta, timezone
+from datetime import timedelta
+from textwrap import dedent
+from django.conf import settings
+from django.utils import timezone
from drf_spectacular.utils import extend_schema, extend_schema_view
from rest_framework import serializers, status
from rest_framework.exceptions import ValidationError
@@ -12,9 +15,6 @@ from arkindex.project.mixins import CorpusACLMixin
from arkindex.project.permissions import IsVerified
from arkindex.users.models import Role
-# Delay to generate a new export from a specific user
-EXPORT_DELAY_HOURS = 6
-
@extend_schema(tags=['exports'])
@extend_schema_view(
@@ -28,10 +28,15 @@ EXPORT_DELAY_HOURS = 6
post=extend_schema(
operation_id='StartExport',
request=None,
- description=(
- 'Start a corpus export job.\n'
- f'A user must wait {EXPORT_DELAY_HOURS} hours before being able to generate a new export of the same corpus.\n\n'
- 'Contributor access is required.'
+ description=dedent(
+ f"""
+ Start a corpus export job.
+
+ A user must wait for {settings.EXPORT_TTL_SECONDS} seconds after the last successful import
+ before being able to generate a new export of the same corpus.
+
+ Contributor access is required.
+ """
),
)
)
@@ -55,10 +60,10 @@ class CorpusExportAPIView(CorpusACLMixin, ListCreateAPIView):
available_exports = corpus.exports.filter(
state=CorpusExportState.Done,
- created__gte=datetime.now(timezone.utc) - timedelta(hours=EXPORT_DELAY_HOURS)
+ created__gte=timezone.now() - timedelta(seconds=settings.EXPORT_TTL_SECONDS)
)
if available_exports.exists():
- raise ValidationError(f'An export has already been made for this corpus in the last {EXPORT_DELAY_HOURS} hours.')
+ raise ValidationError(f'An export has already been made for this corpus in the last {settings.EXPORT_TTL_SECONDS} seconds.')
export = corpus.exports.create(user=self.request.user)
export.start()
diff --git a/arkindex/documents/tests/test_export.py b/arkindex/documents/tests/test_export.py
index 539cea687b..80611182a6 100644
--- a/arkindex/documents/tests/test_export.py
+++ b/arkindex/documents/tests/test_export.py
@@ -1,6 +1,7 @@
from datetime import datetime, timedelta, timezone
from unittest.mock import call, patch
+from django.test import override_settings
from django.urls import reverse
from rest_framework import status
@@ -12,6 +13,7 @@ from arkindex.users.models import Role
class TestExport(FixtureAPITestCase):
@patch('arkindex.project.triggers.export.export_corpus.delay')
+ @override_settings(EXPORT_TTL_SECONDS=420)
def test_start(self, delay_mock):
self.client.force_login(self.superuser)
response = self.client.post(reverse('api:corpus-export', kwargs={'pk': self.corpus.id}))
@@ -81,11 +83,12 @@ class TestExport(FixtureAPITestCase):
self.assertEqual(self.corpus.exports.count(), 1)
self.assertFalse(delay_mock.called)
+ @override_settings(EXPORT_TTL_SECONDS=420)
@patch('arkindex.project.triggers.export.export_corpus.delay')
def test_start_recent_export(self, delay_mock):
self.client.force_login(self.superuser)
with patch('django.utils.timezone.now') as mock_now:
- mock_now.return_value = datetime.now(timezone.utc) - timedelta(hours=2)
+ mock_now.return_value = datetime.now(timezone.utc) - timedelta(minutes=2)
self.corpus.exports.create(
user=self.user,
state=CorpusExportState.Done,
@@ -93,7 +96,7 @@ class TestExport(FixtureAPITestCase):
response = self.client.post(reverse('api:corpus-export', kwargs={'pk': self.corpus.id}))
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
- self.assertListEqual(response.json(), ['An export has already been made for this corpus in the last 6 hours.'])
+ self.assertListEqual(response.json(), ['An export has already been made for this corpus in the last 420 seconds.'])
self.assertEqual(self.corpus.exports.count(), 1)
self.assertFalse(delay_mock.called)
diff --git a/arkindex/project/config.py b/arkindex/project/config.py
index b00c0ae3e0..c98d3211eb 100644
--- a/arkindex/project/config.py
+++ b/arkindex/project/config.py
@@ -114,6 +114,9 @@ def get_settings_parser(base_dir):
email_parser.add_option('password', type=str)
email_parser.add_option('error_report_recipients', type=str, many=True, default=[])
+ export_parser = parser.add_subparser('export', default={})
+ export_parser.add_option('ttl', type=int, default=21600)
+
static_parser = parser.add_subparser('static', default={})
static_parser.add_option('root_path', type=dir_path, default=None)
static_parser.add_option('cdn_assets_url', type=str, default=None)
diff --git a/arkindex/project/settings.py b/arkindex/project/settings.py
index bae672ddc6..695ee75296 100644
--- a/arkindex/project/settings.py
+++ b/arkindex/project/settings.py
@@ -373,6 +373,9 @@ RQ = {
# How many keys to delete at once inside a sorted set in Redis using a single ZREM command
REDIS_ZREM_CHUNK_SIZE = 10000
+# How long before a corpus export can be run again after a successful one
+EXPORT_TTL_SECONDS = conf['export']['ttl']
+
LOGGING = {
'version': 1,
'filters': {
diff --git a/arkindex/project/tests/config_samples/defaults.yaml b/arkindex/project/tests/config_samples/defaults.yaml
index 2abdc4ffd7..1f2531b69c 100644
--- a/arkindex/project/tests/config_samples/defaults.yaml
+++ b/arkindex/project/tests/config_samples/defaults.yaml
@@ -33,6 +33,8 @@ doorbell:
appkey: null
id: null
email: null
+export:
+ ttl: 21600
features:
search: false
selection: true
diff --git a/arkindex/project/tests/config_samples/errors.yaml b/arkindex/project/tests/config_samples/errors.yaml
index e9228fc431..bee83a7c02 100644
--- a/arkindex/project/tests/config_samples/errors.yaml
+++ b/arkindex/project/tests/config_samples/errors.yaml
@@ -22,6 +22,8 @@ docker:
here: have a dict
email:
host: 123
+export:
+ ttl: forever
features:
sv_cheats: 1
gitlab:
diff --git a/arkindex/project/tests/config_samples/expected_errors.yaml b/arkindex/project/tests/config_samples/expected_errors.yaml
index 648227fc7f..892ad86043 100644
--- a/arkindex/project/tests/config_samples/expected_errors.yaml
+++ b/arkindex/project/tests/config_samples/expected_errors.yaml
@@ -13,6 +13,8 @@ email:
password: This option is required
port: This option is required
user: This option is required
+export:
+ ttl: "invalid literal for int() with base 10: 'forever'"
features:
sv_cheats: This option does not exist
ingest:
diff --git a/arkindex/project/tests/config_samples/override.yaml b/arkindex/project/tests/config_samples/override.yaml
index 3993e1e1f5..11d8487bd7 100644
--- a/arkindex/project/tests/config_samples/override.yaml
+++ b/arkindex/project/tests/config_samples/override.yaml
@@ -45,6 +45,8 @@ email:
password: hunter2
port: 25
user: teklia@wanadoo.fr
+export:
+ ttl: 123456
features:
search: true
selection: false
--
GitLab