diff --git a/arkindex/documents/migrations/0013_corpus_maximum_task_ttl.py b/arkindex/documents/migrations/0013_corpus_maximum_task_ttl.py
new file mode 100644
index 0000000000000000000000000000000000000000..d5fc5fba74b43de52467de4d3bf38d71d4df3de5
--- /dev/null
+++ b/arkindex/documents/migrations/0013_corpus_maximum_task_ttl.py
@@ -0,0 +1,22 @@
+# Generated by Django 5.0.8 on 2024-11-06 11:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("documents", "0012_alter_transcriptionentity_id"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="corpus",
+ name="maximum_task_ttl",
+ field=models.PositiveIntegerField(
+ blank=True,
+ help_text="Maximum time-to-live (expressed in seconds) for any WorkerRun created in this corpus. 0 means infinite. When not set, the instance-wide maximum time-to-live will be used instead.",
+ null=True,
+ ),
+ ),
+ ]
diff --git a/arkindex/documents/models.py b/arkindex/documents/models.py
index e6eb2bc4a66f5a14d6ae9a94885850cdc2d6521f..0353cd106e08c2921124f3034439d45d9844ac95 100644
--- a/arkindex/documents/models.py
+++ b/arkindex/documents/models.py
@@ -52,12 +52,23 @@ class Corpus(IndexableModel):
# Is this corpus indexable ?
indexable = models.BooleanField(default=False)
+ maximum_task_ttl = models.PositiveIntegerField(
+ blank=True,
+ null=True,
+ help_text="Maximum time-to-live (expressed in seconds) for any WorkerRun created in this corpus. 0 means infinite. "
+ "When not set, the instance-wide maximum time-to-live will be used instead.",
+ )
+
# Specific manager for ACL
objects = CorpusManager()
class Meta:
verbose_name_plural = "corpora"
+ @property
+ def applied_maximum_task_ttl(self) -> int:
+ return settings.PONOS_MAXIMUM_TASK_TTL if self.maximum_task_ttl is None else self.maximum_task_ttl
+
def __str__(self):
return self.name
diff --git a/arkindex/documents/serializers/elements.py b/arkindex/documents/serializers/elements.py
index 2125a6b08961227d1428e0f273acd6e92331d15b..ee9ec29171e743ad7b17eaa618f0ea80d0c54ba1 100644
--- a/arkindex/documents/serializers/elements.py
+++ b/arkindex/documents/serializers/elements.py
@@ -263,6 +263,13 @@ class CorpusSerializer(serializers.ModelSerializer):
read_only=True,
help_text="Number of users or groups that have been granted access rights on this corpus.",
)
+ maximum_task_ttl = serializers.IntegerField(
+ min_value=0,
+ read_only=True,
+ # Use the actually applied TTL, which defaults to the instance-wide setting and is never null
+ source="applied_maximum_task_ttl",
+ help_text="Maximum time-to-live for any WorkerRun created in this corpus, expressed in seconds. `0` means infinite.",
+ )
class Meta:
model = Corpus
@@ -277,6 +284,7 @@ class CorpusSerializer(serializers.ModelSerializer):
"created",
"authorized_users",
"indexable",
+ "maximum_task_ttl",
)
extra_kwargs = {
"public": {
diff --git a/arkindex/documents/tests/test_corpus.py b/arkindex/documents/tests/test_corpus.py
index b0953e78a74168a1a37788c8626703002bd8489a..805447a33f06bf23f7ac7a47f022b69bac8db688 100644
--- a/arkindex/documents/tests/test_corpus.py
+++ b/arkindex/documents/tests/test_corpus.py
@@ -4,6 +4,7 @@ from unittest.mock import call, patch
from uuid import uuid4
from django.contrib.auth.models import AnonymousUser
+from django.test import override_settings
from django.urls import reverse
from rest_framework import status
@@ -56,6 +57,7 @@ EXPECTED_CORPUS_TYPES = [
]
+@override_settings(PONOS_MAXIMUM_TASK_TTL=3600)
class TestCorpus(FixtureAPITestCase):
@classmethod
@@ -101,6 +103,7 @@ class TestCorpus(FixtureAPITestCase):
"created": DB_CREATED,
"authorized_users": 1,
"top_level_type": None,
+ "maximum_task_ttl": 3600,
}
]
)
@@ -134,6 +137,7 @@ class TestCorpus(FixtureAPITestCase):
"created": DB_CREATED,
"authorized_users": 2,
"top_level_type": None,
+ "maximum_task_ttl": 3600,
},
{
"id": str(self.corpus_public.id),
@@ -145,6 +149,7 @@ class TestCorpus(FixtureAPITestCase):
"created": DB_CREATED,
"authorized_users": 1,
"top_level_type": None,
+ "maximum_task_ttl": 3600,
}
]
)
@@ -178,6 +183,7 @@ class TestCorpus(FixtureAPITestCase):
"created": DB_CREATED,
"authorized_users": 2,
"top_level_type": None,
+ "maximum_task_ttl": 3600,
},
{
"id": str(self.corpus_hidden.id),
@@ -189,6 +195,7 @@ class TestCorpus(FixtureAPITestCase):
"created": DB_CREATED,
"authorized_users": 0,
"top_level_type": None,
+ "maximum_task_ttl": 3600,
},
{
"id": str(self.corpus_public.id),
@@ -200,6 +207,7 @@ class TestCorpus(FixtureAPITestCase):
"created": DB_CREATED,
"authorized_users": 1,
"top_level_type": None,
+ "maximum_task_ttl": 3600,
}
]
)
@@ -347,6 +355,7 @@ class TestCorpus(FixtureAPITestCase):
"created": DB_CREATED,
"authorized_users": 1,
"top_level_type": None,
+ "maximum_task_ttl": 3600,
})
def test_retrieve(self):
@@ -365,6 +374,7 @@ class TestCorpus(FixtureAPITestCase):
"created": DB_CREATED,
"authorized_users": 2,
"top_level_type": None,
+ "maximum_task_ttl": 3600,
})
def test_retrieve_not_found(self):
@@ -394,6 +404,7 @@ class TestCorpus(FixtureAPITestCase):
"created": DB_CREATED,
"authorized_users": 1,
"top_level_type": None,
+ "maximum_task_ttl": 3600,
})
@expectedFailure
@@ -422,18 +433,38 @@ class TestCorpus(FixtureAPITestCase):
"created": DB_CREATED,
"authorized_users": 2,
"top_level_type": None,
+ "maximum_task_ttl": 3600,
})
+ def test_retrieve_maximum_task_ttl(self):
+ """
+ Setting Corpus.maximum_task_ttl to any value should return it instead of the setting
+ """
+ self.corpus.maximum_task_ttl = 0
+ self.corpus.save()
+
+ with self.assertNumQueries(3):
+ response = self.client.get(reverse("api:corpus-retrieve", kwargs={"pk": self.corpus.id}))
+ self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+ self.assertEqual(response.json()["maximum_task_ttl"], 0)
+
def test_partial_update(self):
self.client.force_login(self.corpus_admin)
- response = self.client.patch(reverse("api:corpus-retrieve", kwargs={"pk": self.corpus_private.id}), {
- "name": "new name",
- "description": "new description",
- })
- self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+ with self.assertNumQueries(7):
+ response = self.client.patch(reverse("api:corpus-retrieve", kwargs={"pk": self.corpus_private.id}), {
+ "name": "new name",
+ "description": "new description",
+ # This field should be ignored
+ "maximum_task_ttl": 9999,
+ })
+ self.assertEqual(response.status_code, status.HTTP_200_OK)
+
self.corpus_private.refresh_from_db()
self.assertEqual(self.corpus_private.name, "new name")
self.assertEqual(self.corpus_private.description, "new description")
+ self.assertIsNone(self.corpus_private.maximum_task_ttl)
def test_partial_update_private_to_public_normal_user(self):
"""
@@ -526,14 +557,20 @@ class TestCorpus(FixtureAPITestCase):
def test_update(self):
self.client.force_login(self.corpus_admin)
- response = self.client.put(reverse("api:corpus-retrieve", kwargs={"pk": self.corpus_private.id}), {
- "name": "new name",
- "description": "new description",
- })
- self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+ with self.assertNumQueries(7):
+ response = self.client.put(reverse("api:corpus-retrieve", kwargs={"pk": self.corpus_private.id}), {
+ "name": "new name",
+ "description": "new description",
+ # This field should be ignored
+ "maximum_task_ttl": 9999,
+ })
+ self.assertEqual(response.status_code, status.HTTP_200_OK)
+
self.corpus_private.refresh_from_db()
self.assertEqual(self.corpus_private.name, "new name")
self.assertEqual(self.corpus_private.description, "new description")
+ self.assertIsNone(self.corpus_private.maximum_task_ttl)
def test_update_required_fields(self):
self.client.force_login(self.corpus_admin)
diff --git a/arkindex/project/config.py b/arkindex/project/config.py
index 84e76f14ff35d28581b7001cb88bd0c4a4f3a5c8..47a70f380101a86397a550a4312a98e49eb46260 100644
--- a/arkindex/project/config.py
+++ b/arkindex/project/config.py
@@ -175,6 +175,7 @@ def get_settings_parser(base_dir):
ponos_parser.add_option("artifact_max_size", type=int, default=5 * 1024**3)
# Default task expiry delay in days
ponos_parser.add_option("task_expiry", type=int, default=30)
+ ponos_parser.add_option("maximum_task_ttl", type=int, default=3600)
ponos_parser.add_option("auto_remove_container", type=bool, default=False)
sentry_parser = parser.add_subparser("sentry", default={})
diff --git a/arkindex/project/settings.py b/arkindex/project/settings.py
index 67b66f202f7e5e492560f80a64e7b5bfe7d1976d..1b2d950cbca60e711c9e2d0427f38fe3c00e1075 100644
--- a/arkindex/project/settings.py
+++ b/arkindex/project/settings.py
@@ -561,6 +561,7 @@ PONOS_DOCKER_AUTO_REMOVE_CONTAINER = conf["ponos"]["auto_remove_container"]
# Base data directory for RQ tasks execution (in the docker container)
PONOS_DATA_DIR = "/data"
PONOS_TASK_EXPIRY = conf["ponos"]["task_expiry"]
+PONOS_MAXIMUM_TASK_TTL = conf["ponos"]["maximum_task_ttl"]
# Robots.txt options
ROBOTS_TXT_DISALLOW = conf["robots_txt_disallow"]
diff --git a/arkindex/project/tests/config_samples/defaults.yaml b/arkindex/project/tests/config_samples/defaults.yaml
index 5a3e67754c310b19f27de75049a00da18d5e59d9..d2e108d675a9157f0a4040e4126485c1fd72e4d6 100644
--- a/arkindex/project/tests/config_samples/defaults.yaml
+++ b/arkindex/project/tests/config_samples/defaults.yaml
@@ -67,6 +67,7 @@ ponos:
artifact_max_size: 5368709120
auto_remove_container: false
default_env: {}
+ maximum_task_ttl: 3600
task_expiry: 30
public_hostname: https://default.config.arkindex.localhost
redis:
diff --git a/arkindex/project/tests/config_samples/errors.yaml b/arkindex/project/tests/config_samples/errors.yaml
index 3b5f91fa0b50aba2b5631b17f850a9776c560d43..5925b919da4f933f8c94df033bc368dc81a6e325 100644
--- a/arkindex/project/tests/config_samples/errors.yaml
+++ b/arkindex/project/tests/config_samples/errors.yaml
@@ -50,6 +50,7 @@ ponos:
artifact_max_size: .nan
auto_remove_container: please
default_env: {}
+ maximum_task_ttl: .inf
private_key: /dev/zero
task_expiry: zero
public_hostname: darkindex.lol
diff --git a/arkindex/project/tests/config_samples/expected_errors.yaml b/arkindex/project/tests/config_samples/expected_errors.yaml
index f0eaf9603de0d85f02c7774f60202ad63b80a3ce..b9ec8b030b4281343607231044d324cfa23d5dc3 100644
--- a/arkindex/project/tests/config_samples/expected_errors.yaml
+++ b/arkindex/project/tests/config_samples/expected_errors.yaml
@@ -33,6 +33,7 @@ job_timeouts:
send_verification_email: "invalid literal for int() with base 10: 'lol'"
ponos:
artifact_max_size: cannot convert float NaN to integer
+ maximum_task_ttl: cannot convert float infinity to integer
task_expiry: "invalid literal for int() with base 10: 'zero'"
public_hostname: The hostname must include an HTTP or HTTPS scheme.
redis:
diff --git a/arkindex/project/tests/config_samples/override.yaml b/arkindex/project/tests/config_samples/override.yaml
index 099d11b5ee20d04034c3f688539ca508890445d6..7a807a1fef29b69b3b84ce3a76a4bddb4d682e34 100644
--- a/arkindex/project/tests/config_samples/override.yaml
+++ b/arkindex/project/tests/config_samples/override.yaml
@@ -82,6 +82,7 @@ ponos:
auto_remove_container: true
default_env:
A: B
+ maximum_task_ttl: 3600
task_expiry: 42
public_hostname: https://darkindex.lol
redis:
diff --git a/arkindex/sql_validation/corpus_delete.sql b/arkindex/sql_validation/corpus_delete.sql
index 9fbed056b33fc5d67379cc54aa5bdb4f6b37d8f6..bf5f22713085f427c02cea00062664ba4268beab 100644
--- a/arkindex/sql_validation/corpus_delete.sql
+++ b/arkindex/sql_validation/corpus_delete.sql
@@ -5,7 +5,8 @@ SELECT "documents_corpus"."created",
"documents_corpus"."description",
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
- "documents_corpus"."indexable"
+ "documents_corpus"."indexable",
+ "documents_corpus"."maximum_task_ttl"
FROM "documents_corpus"
WHERE "documents_corpus"."id" = '{corpus_id}'::uuid
LIMIT 21;
diff --git a/arkindex/sql_validation/corpus_delete_top_level_type.sql b/arkindex/sql_validation/corpus_delete_top_level_type.sql
index 712cefb7df5f2b2dcdbe962c6d3569f3be95b0d5..dbbe418e7389f4b1d9b79c37984632511f432571 100644
--- a/arkindex/sql_validation/corpus_delete_top_level_type.sql
+++ b/arkindex/sql_validation/corpus_delete_top_level_type.sql
@@ -5,7 +5,8 @@ SELECT "documents_corpus"."created",
"documents_corpus"."description",
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
- "documents_corpus"."indexable"
+ "documents_corpus"."indexable",
+ "documents_corpus"."maximum_task_ttl"
FROM "documents_corpus"
WHERE "documents_corpus"."id" = '{corpus_id}'::uuid
LIMIT 21;
diff --git a/arkindex/sql_validation/corpus_rights_filter.sql b/arkindex/sql_validation/corpus_rights_filter.sql
index 9122e1515ed7c904ccee5b86cdc30dd3cf2013a1..6456c7da350966b8be2f82ef0215810931153463 100644
--- a/arkindex/sql_validation/corpus_rights_filter.sql
+++ b/arkindex/sql_validation/corpus_rights_filter.sql
@@ -22,6 +22,7 @@ SELECT "documents_corpus"."created",
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
"documents_corpus"."indexable",
+ "documents_corpus"."maximum_task_ttl",
LEAST("users_right"."level", T5."level") AS "max_level"
FROM "documents_corpus"
INNER JOIN "users_right" ON ("documents_corpus"."id" = "users_right"."content_id"
diff --git a/arkindex/sql_validation/corpus_rights_filter_public.sql b/arkindex/sql_validation/corpus_rights_filter_public.sql
index 03bc0851e01179c602279af11a40745878a860a3..60427780297b6d7fb4ab6a5d61865cdbc63bc03e 100644
--- a/arkindex/sql_validation/corpus_rights_filter_public.sql
+++ b/arkindex/sql_validation/corpus_rights_filter_public.sql
@@ -23,6 +23,7 @@ LIMIT 21;
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
"documents_corpus"."indexable",
+ "documents_corpus"."maximum_task_ttl",
LEAST("users_right"."level", T5."level") AS "max_level"
FROM "documents_corpus"
INNER JOIN "users_right" ON ("documents_corpus"."id" = "users_right"."content_id"
@@ -42,6 +43,7 @@ UNION
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
"documents_corpus"."indexable",
+ "documents_corpus"."maximum_task_ttl",
10 AS "max_level"
FROM "documents_corpus"
WHERE "documents_corpus"."public")
diff --git a/arkindex/sql_validation/list_elements.sql b/arkindex/sql_validation/list_elements.sql
index 8dede3c39d1e061caf3308060ff6310bd0a339fd..b04bd57d0d0afa1b16618f66a96be59d5d5c37b5 100644
--- a/arkindex/sql_validation/list_elements.sql
+++ b/arkindex/sql_validation/list_elements.sql
@@ -5,7 +5,8 @@ SELECT "documents_corpus"."created",
"documents_corpus"."description",
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
- "documents_corpus"."indexable"
+ "documents_corpus"."indexable",
+ "documents_corpus"."maximum_task_ttl"
FROM "documents_corpus"
WHERE "documents_corpus"."id" = '{corpus_id}'::uuid
LIMIT 21;
diff --git a/arkindex/sql_validation/process_elements_filter_ml_class.sql b/arkindex/sql_validation/process_elements_filter_ml_class.sql
index fcd76a4123e0a100972db96420456235299ba71e..fb1e9d4389cea1361a85639375aad08f83c50b18 100644
--- a/arkindex/sql_validation/process_elements_filter_ml_class.sql
+++ b/arkindex/sql_validation/process_elements_filter_ml_class.sql
@@ -51,7 +51,8 @@ SELECT "documents_corpus"."created",
"documents_corpus"."description",
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
- "documents_corpus"."indexable"
+ "documents_corpus"."indexable",
+ "documents_corpus"."maximum_task_ttl"
FROM "documents_corpus"
WHERE "documents_corpus"."id" = '{corpus_id}'::uuid
LIMIT 21;
diff --git a/arkindex/sql_validation/process_elements_filter_type.sql b/arkindex/sql_validation/process_elements_filter_type.sql
index 73e943b2542a10f131861e826b3c859f38aca966..a566bc4a782d58d2fe29839372458bf19dfb37d8 100644
--- a/arkindex/sql_validation/process_elements_filter_type.sql
+++ b/arkindex/sql_validation/process_elements_filter_type.sql
@@ -51,7 +51,8 @@ SELECT "documents_corpus"."created",
"documents_corpus"."description",
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
- "documents_corpus"."indexable"
+ "documents_corpus"."indexable",
+ "documents_corpus"."maximum_task_ttl"
FROM "documents_corpus"
WHERE "documents_corpus"."id" = '{corpus_id}'::uuid
LIMIT 21;
diff --git a/arkindex/sql_validation/process_elements_top_level.sql b/arkindex/sql_validation/process_elements_top_level.sql
index 7fe800743d1e5c2b9fa2f6611c04d125967d212b..77423e4582a0ca0620335a39ec096af91e4ee5b0 100644
--- a/arkindex/sql_validation/process_elements_top_level.sql
+++ b/arkindex/sql_validation/process_elements_top_level.sql
@@ -51,7 +51,8 @@ SELECT "documents_corpus"."created",
"documents_corpus"."description",
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
- "documents_corpus"."indexable"
+ "documents_corpus"."indexable",
+ "documents_corpus"."maximum_task_ttl"
FROM "documents_corpus"
WHERE "documents_corpus"."id" = '{corpus_id}'::uuid
LIMIT 21;
diff --git a/arkindex/sql_validation/process_elements_with_image.sql b/arkindex/sql_validation/process_elements_with_image.sql
index c8f0cd707ca209cb3132e8bc31af3aeb877e6971..2b6d311781ba790d1e63af7d571895f2c1beba55 100644
--- a/arkindex/sql_validation/process_elements_with_image.sql
+++ b/arkindex/sql_validation/process_elements_with_image.sql
@@ -51,7 +51,8 @@ SELECT "documents_corpus"."created",
"documents_corpus"."description",
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
- "documents_corpus"."indexable"
+ "documents_corpus"."indexable",
+ "documents_corpus"."maximum_task_ttl"
FROM "documents_corpus"
WHERE "documents_corpus"."id" = '{corpus_id}'::uuid
LIMIT 21;