From a179d773bb8b21604c163c20419e99737da67e2e Mon Sep 17 00:00:00 2001
From: Erwan Rouchet <rouchet@teklia.com>
Date: Thu, 13 Feb 2025 15:46:29 +0100
Subject: [PATCH] Remove WorkerRun TTL
---
arkindex/documents/fixtures/data.json | 15 +-
.../management/commands/build_fixtures.py | 13 +-
.../management/commands/load_export.py | 1 -
.../documents/tests/commands/test_cleanup.py | 4 +-
.../tests/commands/test_load_export.py | 2 +-
.../tests/tasks/test_corpus_delete.py | 2 +-
arkindex/documents/tests/tasks/test_export.py | 1 -
.../tests/tasks/test_worker_results_delete.py | 1 -
.../tests/test_bulk_classification.py | 2 +-
.../tests/test_bulk_element_transcriptions.py | 2 +-
.../documents/tests/test_bulk_elements.py | 2 +-
.../tests/test_bulk_transcription_entities.py | 2 +-
.../tests/test_bulk_transcriptions.py | 2 +-
.../documents/tests/test_classification.py | 2 +-
.../documents/tests/test_create_elements.py | 2 +-
.../tests/test_create_transcriptions.py | 2 +-
arkindex/documents/tests/test_entities_api.py | 4 +-
arkindex/documents/tests/test_metadata.py | 6 +-
arkindex/ponos/tests/rq/test_trigger.py | 4 +-
arkindex/ponos/tests/tasks/test_update.py | 13 +-
arkindex/process/builder.py | 13 +-
.../management/commands/fake_worker_run.py | 5 +-
.../migrations/0049_remove_workerrun_ttl.py | 17 ++
arkindex/process/models.py | 19 +-
arkindex/process/serializers/imports.py | 1 -
arkindex/process/serializers/worker_runs.py | 51 ----
.../tests/commands/test_fake_worker_run.py | 2 +-
arkindex/process/tests/process/test_clear.py | 2 -
arkindex/process/tests/process/test_create.py | 7 +-
.../process/test_default_process_name.py | 20 +-
.../process/tests/process/test_destroy.py | 2 +-
.../process/test_process_artifact_download.py | 2 +-
arkindex/process/tests/process/test_retry.py | 8 +-
arkindex/process/tests/process/test_run.py | 4 +-
arkindex/process/tests/process/test_start.py | 48 ++--
.../process/tests/templates/test_apply.py | 53 +---
.../process/tests/templates/test_create.py | 20 +-
.../process/tests/test_corpus_worker_runs.py | 5 -
.../tests/test_elements_initialisation.py | 6 +-
arkindex/process/tests/test_managers.py | 4 -
arkindex/process/tests/test_signals.py | 21 +-
.../process/tests/test_user_workerruns.py | 9 -
.../tests/worker_activity/test_bulk_insert.py | 1 -
.../tests/worker_activity/test_initialize.py | 4 +-
.../tests/worker_activity/test_list.py | 1 -
.../tests/worker_activity/test_update.py | 5 -
.../tests/worker_runs/test_build_task.py | 11 +-
.../process/tests/worker_runs/test_create.py | 8 +-
.../process/tests/worker_runs/test_delete.py | 6 +-
.../process/tests/worker_runs/test_list.py | 9 +-
.../tests/worker_runs/test_partial_update.py | 58 +---
.../tests/worker_runs/test_retrieve.py | 9 +-
.../process/tests/worker_runs/test_ttl.py | 272 ------------------
.../process/tests/worker_runs/test_update.py | 29 +-
arkindex/sql_validation/indexer_prefetch.sql | 6 +-
arkindex/sql_validation/list_elements.sql | 3 +-
56 files changed, 132 insertions(+), 691 deletions(-)
create mode 100644 arkindex/process/migrations/0049_remove_workerrun_ttl.py
delete mode 100644 arkindex/process/tests/worker_runs/test_ttl.py
diff --git a/arkindex/documents/fixtures/data.json b/arkindex/documents/fixtures/data.json
index 3b1b3abcdb..48662e5a99 100644
--- a/arkindex/documents/fixtures/data.json
+++ b/arkindex/documents/fixtures/data.json
@@ -355,8 +355,7 @@
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"has_results": false,
- "use_gpu": false,
- "ttl": 3600
+ "use_gpu": false
}
},
{
@@ -393,8 +392,7 @@
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"has_results": false,
- "use_gpu": false,
- "ttl": 3600
+ "use_gpu": false
}
},
{
@@ -410,8 +408,7 @@
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"has_results": false,
- "use_gpu": false,
- "ttl": 3600
+ "use_gpu": false
}
},
{
@@ -427,8 +424,7 @@
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"has_results": false,
- "use_gpu": false,
- "ttl": 0
+ "use_gpu": false
}
},
{
@@ -444,8 +440,7 @@
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"has_results": false,
- "use_gpu": false,
- "ttl": 0
+ "use_gpu": false
}
},
{
diff --git a/arkindex/documents/management/commands/build_fixtures.py b/arkindex/documents/management/commands/build_fixtures.py
index 87a12ff637..c691f669f8 100644
--- a/arkindex/documents/management/commands/build_fixtures.py
+++ b/arkindex/documents/management/commands/build_fixtures.py
@@ -206,14 +206,8 @@ class Command(BaseCommand):
mode=ProcessMode.Local,
creator=superuser,
)
- user_local_process.worker_runs.create(
- version=custom_version,
- ttl=0,
- )
- superuser_local_process.worker_runs.create(
- version=custom_version,
- ttl=0,
- )
+ user_local_process.worker_runs.create(version=custom_version)
+ superuser_local_process.worker_runs.create(version=custom_version)
# Create a corpus
corpus = Corpus.objects.create(
@@ -234,16 +228,13 @@ class Command(BaseCommand):
)
init_worker_run = process.worker_runs.create(
version=init_worker,
- ttl=3600,
)
dla_worker_run = process.worker_runs.create(
version=dla_worker,
- ttl=3600,
parents=[init_worker_run.id],
)
reco_run = process.worker_runs.create(
version=recognizer_worker,
- ttl=3600,
parents=[dla_worker_run.id],
)
diff --git a/arkindex/documents/management/commands/load_export.py b/arkindex/documents/management/commands/load_export.py
index 9b2fad0ee5..9096a4303a 100644
--- a/arkindex/documents/management/commands/load_export.py
+++ b/arkindex/documents/management/commands/load_export.py
@@ -444,7 +444,6 @@ class Command(BaseCommand):
version_id=worker_version_id,
model_version=model_version,
configuration=configuration,
- defaults={"ttl": 0},
)
def create_image_server(self, row):
diff --git a/arkindex/documents/tests/commands/test_cleanup.py b/arkindex/documents/tests/commands/test_cleanup.py
index 16fe3290c6..51489faf48 100644
--- a/arkindex/documents/tests/commands/test_cleanup.py
+++ b/arkindex/documents/tests/commands/test_cleanup.py
@@ -1163,14 +1163,13 @@ class TestCleanupCommand(FixtureTestCase):
# from the WorkerRuns, so there would be 2 runs with the same version and no configuration when they should be unique
process = self.corpus.processes.create(mode=ProcessMode.Workers, creator=self.superuser)
version = removable_worker.versions.first()
- process.worker_runs.create(version=version, ttl=0)
+ process.worker_runs.create(version=version)
process.worker_runs.create(
version=version,
configuration=removable_worker.configurations.create(
name="Some configuration",
configuration={},
),
- ttl=0,
)
# This worker cannot be cleaned up because it is used in ML results
@@ -1249,7 +1248,6 @@ class TestCleanupCommand(FixtureTestCase):
worker_run = process.worker_runs.create(
version=worker_version,
model_version=used_model.versions.create(),
- ttl=0,
)
self.corpus.elements.create(
type=self.corpus.types.first(),
diff --git a/arkindex/documents/tests/commands/test_load_export.py b/arkindex/documents/tests/commands/test_load_export.py
index 0787a57f66..383c1bc7ef 100644
--- a/arkindex/documents/tests/commands/test_load_export.py
+++ b/arkindex/documents/tests/commands/test_load_export.py
@@ -39,7 +39,7 @@ class TestLoadExport(FixtureTestCase):
"process.workerversion": ["created", "updated", "configuration", "state", "docker_image_iid"],
# The WorkerRuns lose their parents, use different worker versions that just got recreated,
# are assigned to the user's local process and not the original one
- "process.workerrun": ["parents", "version", "process", "summary", "created", "updated", "ttl"],
+ "process.workerrun": ["parents", "version", "process", "summary", "created", "updated"],
"process.workertype": [],
"images.imageserver": ["s3_bucket", "s3_region", "created", "updated", "read_only"],
"images.image": ["created", "updated", "hash", "status"],
diff --git a/arkindex/documents/tests/tasks/test_corpus_delete.py b/arkindex/documents/tests/tasks/test_corpus_delete.py
index 071af6a3af..cd5ad64153 100644
--- a/arkindex/documents/tests/tasks/test_corpus_delete.py
+++ b/arkindex/documents/tests/tasks/test_corpus_delete.py
@@ -45,7 +45,7 @@ class TestDeleteCorpus(FixtureTestCase):
ml_class=cls.corpus.ml_classes.create(name="a class"),
)
element_process.elements.add(element)
- worker_run = element_process.worker_runs.create(version=cls.worker_version, ttl=0)
+ worker_run = element_process.worker_runs.create(version=cls.worker_version)
task_1, task_2, task_3, task_4 = Task.objects.bulk_create(
[
Task(
diff --git a/arkindex/documents/tests/tasks/test_export.py b/arkindex/documents/tests/tasks/test_export.py
index 8b21ea830c..9831b8eed4 100644
--- a/arkindex/documents/tests/tasks/test_export.py
+++ b/arkindex/documents/tests/tasks/test_export.py
@@ -82,7 +82,6 @@ class TestExport(FixtureTestCase):
model=Model.objects.create(name="Some model"),
),
configuration=metadata_version.worker.configurations.create(name="Some configuration"),
- ttl=0,
)
element.metadatas.create(
diff --git a/arkindex/documents/tests/tasks/test_worker_results_delete.py b/arkindex/documents/tests/tasks/test_worker_results_delete.py
index 0d3c429ecf..cca0e0dcb0 100644
--- a/arkindex/documents/tests/tasks/test_worker_results_delete.py
+++ b/arkindex/documents/tests/tasks/test_worker_results_delete.py
@@ -40,7 +40,6 @@ class TestDeleteWorkerResults(FixtureTestCase):
version=cls.version_1,
model_version=cls.model_version,
configuration=cls.configuration,
- ttl=0,
)
cls.vol = cls.corpus.elements.get(name="Volume 1")
diff --git a/arkindex/documents/tests/test_bulk_classification.py b/arkindex/documents/tests/test_bulk_classification.py
index ddcc0d5c71..492209723d 100644
--- a/arkindex/documents/tests/test_bulk_classification.py
+++ b/arkindex/documents/tests/test_bulk_classification.py
@@ -287,7 +287,7 @@ class TestBulkClassification(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/documents/tests/test_bulk_element_transcriptions.py b/arkindex/documents/tests/test_bulk_element_transcriptions.py
index f4578bdce0..042eb16df2 100644
--- a/arkindex/documents/tests/test_bulk_element_transcriptions.py
+++ b/arkindex/documents/tests/test_bulk_element_transcriptions.py
@@ -748,7 +748,7 @@ class TestBulkElementTranscriptions(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/documents/tests/test_bulk_elements.py b/arkindex/documents/tests/test_bulk_elements.py
index 05d3516d2d..23964877c0 100644
--- a/arkindex/documents/tests/test_bulk_elements.py
+++ b/arkindex/documents/tests/test_bulk_elements.py
@@ -427,7 +427,7 @@ class TestBulkElements(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/documents/tests/test_bulk_transcription_entities.py b/arkindex/documents/tests/test_bulk_transcription_entities.py
index 8c60037671..e0f159919c 100644
--- a/arkindex/documents/tests/test_bulk_transcription_entities.py
+++ b/arkindex/documents/tests/test_bulk_transcription_entities.py
@@ -233,7 +233,7 @@ class TestBulkTranscriptionEntities(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
diff --git a/arkindex/documents/tests/test_bulk_transcriptions.py b/arkindex/documents/tests/test_bulk_transcriptions.py
index 8fbbcdac41..57ffaffdf3 100644
--- a/arkindex/documents/tests/test_bulk_transcriptions.py
+++ b/arkindex/documents/tests/test_bulk_transcriptions.py
@@ -263,7 +263,7 @@ class TestBulkTranscriptions(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/documents/tests/test_classification.py b/arkindex/documents/tests/test_classification.py
index 84ccefcfec..935d36958c 100644
--- a/arkindex/documents/tests/test_classification.py
+++ b/arkindex/documents/tests/test_classification.py
@@ -368,7 +368,7 @@ class TestClassifications(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/documents/tests/test_create_elements.py b/arkindex/documents/tests/test_create_elements.py
index 0cd08a757f..5c3a14342b 100644
--- a/arkindex/documents/tests/test_create_elements.py
+++ b/arkindex/documents/tests/test_create_elements.py
@@ -720,7 +720,7 @@ class TestCreateElements(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/documents/tests/test_create_transcriptions.py b/arkindex/documents/tests/test_create_transcriptions.py
index 18c03d0f16..819e1d18b8 100644
--- a/arkindex/documents/tests/test_create_transcriptions.py
+++ b/arkindex/documents/tests/test_create_transcriptions.py
@@ -337,7 +337,7 @@ class TestTranscriptionCreate(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/documents/tests/test_entities_api.py b/arkindex/documents/tests/test_entities_api.py
index 32b4dffa23..90675af67f 100644
--- a/arkindex/documents/tests/test_entities_api.py
+++ b/arkindex/documents/tests/test_entities_api.py
@@ -388,7 +388,7 @@ class TestEntitiesAPI(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_version_1, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_version_1)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run_1.process.run()
task = self.worker_run_1.process.tasks.first()
@@ -863,7 +863,7 @@ class TestEntitiesAPI(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_version_1, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_version_1)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run_1.process.run()
task = self.worker_run_1.process.tasks.first()
diff --git a/arkindex/documents/tests/test_metadata.py b/arkindex/documents/tests/test_metadata.py
index 55b026370d..ae0fc7a3c8 100644
--- a/arkindex/documents/tests/test_metadata.py
+++ b/arkindex/documents/tests/test_metadata.py
@@ -44,7 +44,7 @@ class TestMetaData(FixtureAPITestCase):
creator=cls.user,
farm=Farm.objects.first(),
)
- cls.process.worker_runs.create(version=cls.worker_version, ttl=0)
+ cls.process.worker_runs.create(version=cls.worker_version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
cls.process.run()
cls.task = cls.process.tasks.first()
@@ -463,7 +463,7 @@ class TestMetaData(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
@@ -1458,7 +1458,7 @@ class TestMetaData(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/ponos/tests/rq/test_trigger.py b/arkindex/ponos/tests/rq/test_trigger.py
index 406dd77a89..e65f9093e8 100644
--- a/arkindex/ponos/tests/rq/test_trigger.py
+++ b/arkindex/ponos/tests/rq/test_trigger.py
@@ -21,8 +21,8 @@ class TestTrigger(FixtureTestCase):
)
cls.worker_version1 = WorkerVersion.objects.get(worker__slug="reco")
cls.worker_version2 = WorkerVersion.objects.get(worker__slug="dla")
- cls.run1 = cls.process.worker_runs.create(version=cls.worker_version1, ttl=0)
- cls.run2 = cls.process.worker_runs.create(version=cls.worker_version2, parents=[cls.run1.id], ttl=0)
+ cls.run1 = cls.process.worker_runs.create(version=cls.worker_version1)
+ cls.run2 = cls.process.worker_runs.create(version=cls.worker_version2, parents=[cls.run1.id])
@override_settings(PONOS_RQ_EXECUTION=True)
@patch("arkindex.ponos.tasks.run_task_rq.delay")
diff --git a/arkindex/ponos/tests/tasks/test_update.py b/arkindex/ponos/tests/tasks/test_update.py
index 762a7fd31c..619c8bbfec 100644
--- a/arkindex/ponos/tests/tasks/test_update.py
+++ b/arkindex/ponos/tests/tasks/test_update.py
@@ -131,8 +131,8 @@ class TestTaskUpdate(FixtureAPITestCase):
corpus=self.corpus,
activity_state=ActivityState.Ready
)
- init_run = test_process.worker_runs.create(version=WorkerVersion.objects.get(worker__slug="initialisation"), ttl=0)
- test_run = test_process.worker_runs.create(version=self.recognizer, parents=[init_run.id], ttl=0)
+ init_run = test_process.worker_runs.create(version=WorkerVersion.objects.get(worker__slug="initialisation"))
+ test_run = test_process.worker_runs.create(version=self.recognizer, parents=[init_run.id])
test_process.run()
@@ -227,8 +227,8 @@ class TestTaskUpdate(FixtureAPITestCase):
activity_state=ActivityState.Ready
)
init_version = WorkerVersion.objects.get(worker__slug="initialisation")
- init_run = test_process.worker_runs.create(version=init_version, ttl=0)
- test_run = test_process.worker_runs.create(version=self.recognizer, parents=[init_run.id], ttl=0)
+ init_run = test_process.worker_runs.create(version=init_version)
+ test_run = test_process.worker_runs.create(version=self.recognizer, parents=[init_run.id])
test_process.run()
@@ -300,7 +300,7 @@ class TestTaskUpdate(FixtureAPITestCase):
chunks=2,
activity_state=ActivityState.Ready
)
- test_run = test_process.worker_runs.create(version=self.recognizer, ttl=0)
+ test_run = test_process.worker_runs.create(version=self.recognizer)
test_process.run()
@@ -402,18 +402,15 @@ class TestTaskUpdate(FixtureAPITestCase):
)
test_run_1 = test_process.worker_runs.create(
version=self.recognizer,
- ttl=0,
)
test_run = test_process.worker_runs.create(
version=self.recognizer,
model_version_id=test_model_version.id,
- ttl=0,
)
test_run_2 = test_process.worker_runs.create(
version=self.recognizer,
model_version_id=test_model_version.id,
configuration_id=test_configuration.id,
- ttl=0,
)
test_process.run()
diff --git a/arkindex/process/builder.py b/arkindex/process/builder.py
index 5a1c347a32..6db8b7afc9 100644
--- a/arkindex/process/builder.py
+++ b/arkindex/process/builder.py
@@ -102,7 +102,7 @@ class ProcessBuilder:
shm_size=shm_size,
extra_files=extra_files,
worker_run=worker_run,
- ttl=worker_run.ttl,
+ ttl=self.process.corpus.applied_maximum_task_ttl,
)
)
@@ -218,10 +218,7 @@ class ProcessBuilder:
from arkindex.process.models import ArkindexFeature, WorkerVersion
import_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.FileImport)
- worker_run, _ = self.process.worker_runs.get_or_create(
- version=import_version,
- defaults={"ttl": self.process.corpus.applied_maximum_task_ttl},
- )
+ worker_run, _ = self.process.worker_runs.get_or_create(version=import_version)
self._build_task(
slug="import_files",
@@ -250,7 +247,6 @@ class ProcessBuilder:
version=ingest_version,
model_version=None,
configuration=worker_configuration,
- defaults={"ttl": self.process.corpus.applied_maximum_task_ttl},
)
env = {
@@ -294,10 +290,7 @@ class ProcessBuilder:
worker_runs.remove(initialisation_worker_run)
# If there is no elements initialisation worker run in the process, create one
else:
- initialisation_worker_run = self.process.worker_runs.create(
- version=init_elements_version,
- ttl=self.process.corpus.applied_maximum_task_ttl,
- )
+ initialisation_worker_run = self.process.worker_runs.create(version=init_elements_version)
# Link all parentless worker runs to the initialisation worker run
no_parents = [run for run in worker_runs if not len(run.parents)]
for run in no_parents:
diff --git a/arkindex/process/management/commands/fake_worker_run.py b/arkindex/process/management/commands/fake_worker_run.py
index e1e9e1d72f..7533141bf5 100644
--- a/arkindex/process/management/commands/fake_worker_run.py
+++ b/arkindex/process/management/commands/fake_worker_run.py
@@ -33,10 +33,7 @@ class Command(BaseCommand):
else:
self.stdout.write(f"Using existing local process {process.id}")
- worker_run, created = process.worker_runs.get_or_create(
- version=worker_version,
- defaults={"ttl": 0},
- )
+ worker_run, created = process.worker_runs.get_or_create(version=worker_version)
if created:
self.stdout.write(self.style.SUCCESS(f"Created WorkerRun {worker_run.id}"))
diff --git a/arkindex/process/migrations/0049_remove_workerrun_ttl.py b/arkindex/process/migrations/0049_remove_workerrun_ttl.py
new file mode 100644
index 0000000000..fad0275417
--- /dev/null
+++ b/arkindex/process/migrations/0049_remove_workerrun_ttl.py
@@ -0,0 +1,17 @@
+# Generated by Django 5.0.8 on 2025-02-13 14:07
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("process", "0048_worker_cost_fields"),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name="workerrun",
+ name="ttl",
+ ),
+ ]
diff --git a/arkindex/process/models.py b/arkindex/process/models.py
index 3d5f00821c..920823bb69 100644
--- a/arkindex/process/models.py
+++ b/arkindex/process/models.py
@@ -342,18 +342,6 @@ class Process(IndexableModel):
if run.version.is_init_elements():
continue
- if new_process.mode == ProcessMode.Template or new_process.corpus.applied_maximum_task_ttl == 0:
- # When the destination process is a template, we do not apply any limits and copy the original TTL.
- # The limits will be applied only when applying a template.
- # With other modes, when the corpus has no limits, we also just use the original TTL.
- ttl = run.ttl
- elif run.ttl == 0:
- # The original TTL was infinite and there is a limit, so we use the corpus' limit
- ttl = new_process.corpus.applied_maximum_task_ttl
- else:
- # Apply the limit normally when no infinity is involved
- ttl = min(run.ttl, new_process.corpus.applied_maximum_task_ttl)
-
# Create a new WorkerRun with same version, configuration and parents.
new_run = WorkerRun(
process=new_process,
@@ -369,7 +357,6 @@ class Process(IndexableModel):
else run.version.gpu_usage == FeatureUsage.Required
),
summary=run.summary,
- ttl=ttl,
)
# Save the correspondence between this process' worker_run and the new one
new_runs[run.id] = new_run
@@ -938,10 +925,6 @@ class WorkerRun(models.Model):
updated = models.DateTimeField(auto_now=True)
has_results = models.BooleanField(default=False)
use_gpu = models.BooleanField(default=False)
- ttl = models.PositiveIntegerField(
- verbose_name="TTL",
- help_text="Maximum time-to-live for tasks created from this WorkerRun, in seconds. 0 means infinite.",
- )
objects = WorkerRunManager()
@@ -1057,7 +1040,7 @@ class WorkerRun(models.Model):
worker_run=self,
extra_files=extra_files,
requires_gpu=requires_gpu,
- ttl=self.ttl,
+ ttl=process.corpus.applied_maximum_task_ttl,
)
return task, parents
diff --git a/arkindex/process/serializers/imports.py b/arkindex/process/serializers/imports.py
index 71bdbf729f..9a345d520c 100644
--- a/arkindex/process/serializers/imports.py
+++ b/arkindex/process/serializers/imports.py
@@ -612,7 +612,6 @@ class ExportProcessSerializer(ProcessDetailsSerializer):
export_process.worker_runs.create(
version=worker_version,
configuration=worker_configuration,
- ttl=corpus.applied_maximum_task_ttl,
)
# Start the export process
export_process.run()
diff --git a/arkindex/process/serializers/worker_runs.py b/arkindex/process/serializers/worker_runs.py
index eb6ab16138..9f6146db04 100644
--- a/arkindex/process/serializers/worker_runs.py
+++ b/arkindex/process/serializers/worker_runs.py
@@ -1,7 +1,6 @@
from collections import defaultdict
from textwrap import dedent
-from django.core.validators import MaxValueValidator, MinValueValidator
from rest_framework import serializers
from rest_framework.exceptions import ValidationError
@@ -20,25 +19,6 @@ from arkindex.process.serializers.workers import WorkerConfigurationSerializer,
from arkindex.training.models import ModelVersion, ModelVersionState
from arkindex.training.serializers import ModelVersionLightSerializer
-# To prevent each element worker to retrieve contextual information
-# (process, worker version, model version…) with extra GET requests, we
-# do serialize all the related information on WorkerRun serializers.
-
-def _ttl_from_corpus(serializer_field) -> int:
- if isinstance(serializer_field.parent.instance, WorkerRun):
- process = serializer_field.parent.instance.process
- else:
- process = serializer_field.context["process"]
-
- # This function may be called on a local process, which does not have a corpus, even if the API blocks them later on
- if process.mode == ProcessMode.Local:
- return 0
-
- return process.corpus.applied_maximum_task_ttl
-
-
-_ttl_from_corpus.requires_context = True
-
class WorkerRunSerializer(serializers.ModelSerializer):
@@ -96,15 +76,6 @@ class WorkerRunSerializer(serializers.ModelSerializer):
"Only a configuration of the WorkerVersion's worker may be set.",
)
- ttl = serializers.IntegerField(
- default=_ttl_from_corpus,
- help_text=dedent("""
- Maximum time-to-live for tasks created from this WorkerRun, in seconds. `0` means infinite.
-
- Defaults to, and cannot exceed, the `maximum_task_ttl` on the corpus of the process.
- """),
- )
-
process = ProcessLightSerializer(read_only=True)
class Meta:
@@ -121,7 +92,6 @@ class WorkerRunSerializer(serializers.ModelSerializer):
"model_version",
"summary",
"use_gpu",
- "ttl",
)
read_only_fields = (
"id",
@@ -145,24 +115,6 @@ class WorkerRunSerializer(serializers.ModelSerializer):
return self.instance.process
return self.context["process"]
- def validate_ttl(self, value) -> int:
- if self._process.mode == ProcessMode.Local:
- # Don't validate anything, the endpoint will not work on local processes anyway
- return value
-
- corpus_ttl = self._process.corpus.applied_maximum_task_ttl
- if corpus_ttl == 0:
- # Allow infinity, and limit to the maximum value of an integer field
- min_ttl, max_ttl = 0, 2147483647
- else:
- # Restrict the maximum TTL further using the limit
- min_ttl, max_ttl = 1, corpus_ttl
-
- MinValueValidator(min_ttl)(value)
- MaxValueValidator(max_ttl)(value)
-
- return value
-
def validate(self, data):
data = super().validate(data)
errors = defaultdict(list)
@@ -296,8 +248,6 @@ class UserWorkerRunSerializer(serializers.ModelSerializer):
queryset=WorkerConfiguration.objects.all(),
style={"base_template": "input.html"},
)
- # Default value for the TTL, as the local process does not have a corpus and the run will never actually run
- ttl = serializers.HiddenField(default=0)
def validate_worker_version_id(self, worker_version_id):
# Check that the worker version exists
@@ -369,7 +319,6 @@ class UserWorkerRunSerializer(serializers.ModelSerializer):
"worker_version_id",
"model_version_id",
"configuration_id",
- "ttl",
)
diff --git a/arkindex/process/tests/commands/test_fake_worker_run.py b/arkindex/process/tests/commands/test_fake_worker_run.py
index 97055247ea..68020a16c9 100644
--- a/arkindex/process/tests/commands/test_fake_worker_run.py
+++ b/arkindex/process/tests/commands/test_fake_worker_run.py
@@ -64,7 +64,7 @@ class TestFakeWorkerRun(FixtureTestCase):
def test_existing_worker_run(self):
process = Process.objects.get(mode=ProcessMode.Local, creator=self.user)
- worker_run = process.worker_runs.create(version=self.worker_version, ttl=0)
+ worker_run = process.worker_runs.create(version=self.worker_version)
self.assertEqual(process.worker_runs.count(), 2)
output = self.fake_worker_run(["--user", str(self.user.id), "--worker-version", str(self.worker_version.id)])
diff --git a/arkindex/process/tests/process/test_clear.py b/arkindex/process/tests/process/test_clear.py
index 0593a54ce4..83ad163bc0 100644
--- a/arkindex/process/tests/process/test_clear.py
+++ b/arkindex/process/tests/process/test_clear.py
@@ -22,11 +22,9 @@ class TestProcessClear(FixtureAPITestCase):
)
cls.process.worker_runs.create(
version=WorkerVersion.objects.get(worker__slug="reco"),
- ttl=0,
)
cls.process.worker_runs.create(
version=WorkerVersion.objects.get(worker__slug="dla"),
- ttl=0,
)
def test_clear(self):
diff --git a/arkindex/process/tests/process/test_create.py b/arkindex/process/tests/process/test_create.py
index 3f1d6aaec3..b11f6da848 100644
--- a/arkindex/process/tests/process/test_create.py
+++ b/arkindex/process/tests/process/test_create.py
@@ -488,17 +488,14 @@ class TestCreateProcess(FixtureAPITestCase):
)
init_run = process_2.worker_runs.create(
version=self.init_elements_version,
- ttl=0,
)
run_1 = process_2.worker_runs.create(
version=self.version_1,
parents=[init_run.id],
- ttl=0,
)
run_2 = process_2.worker_runs.create(
version=self.version_2,
parents=[run_1.id],
- ttl=0,
)
self.assertFalse(process_2.tasks.exists())
@@ -589,7 +586,7 @@ class TestCreateProcess(FixtureAPITestCase):
dataset = self.corpus.datasets.first()
test_set = dataset.sets.get(name="test")
ProcessDatasetSet.objects.create(process=process, set=test_set)
- process.worker_runs.create(version=self.version_1, ttl=0)
+ process.worker_runs.create(version=self.version_1)
with self.assertNumQueries(9):
response = self.client.post(reverse("api:process-start", kwargs={"pk": str(process.id)}))
@@ -605,7 +602,7 @@ class TestCreateProcess(FixtureAPITestCase):
self.worker_1.save()
process = self.corpus.processes.create(creator=self.user, mode=ProcessMode.Workers)
- process.worker_runs.create(version=self.version_1, ttl=0)
+ process.worker_runs.create(version=self.version_1)
with self.assertNumQueries(9):
response = self.client.post(reverse("api:process-start", kwargs={"pk": str(process.id)}))
diff --git a/arkindex/process/tests/process/test_default_process_name.py b/arkindex/process/tests/process/test_default_process_name.py
index e496e7ff00..00223e3e59 100644
--- a/arkindex/process/tests/process/test_default_process_name.py
+++ b/arkindex/process/tests/process/test_default_process_name.py
@@ -50,22 +50,18 @@ class TestProcessName(FixtureAPITestCase):
init_elements_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.InitElements)
init_run = self.workers_process.worker_runs.create(
version=init_elements_version,
- ttl=0,
)
self.workers_process.worker_runs.create(
version=self.recognizer,
parents=[init_run.id],
- ttl=0,
)
dla_run = self.workers_process.worker_runs.create(
version=self.dla,
parents=[init_run.id],
- ttl=0,
)
self.workers_process.worker_runs.create(
version=self.version_gpu,
parents=[dla_run.id],
- ttl=0,
)
self.workers_process.save()
@@ -95,41 +91,34 @@ class TestProcessName(FixtureAPITestCase):
init_elements_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.InitElements)
init_run = self.workers_process.worker_runs.create(
version=init_elements_version,
- ttl=0,
)
reco_run_1 = self.workers_process.worker_runs.create(
version=self.recognizer,
parents=[init_run.id],
- ttl=0,
)
reco_run_2 = self.workers_process.worker_runs.create(
version=self.recognizer,
configuration=self.reco_config_1,
parents=[reco_run_1.id],
- ttl=0,
)
self.workers_process.worker_runs.create(
version=self.recognizer,
configuration=self.reco_config_2,
parents=[reco_run_2.id],
- ttl=0,
)
dla_run_1 = self.workers_process.worker_runs.create(
version=self.dla,
parents=[init_run.id],
- ttl=0,
)
reco_run_4 = self.workers_process.worker_runs.create(
version=self.recognizer,
configuration=self.reco_config_3,
parents=[dla_run_1.id],
- ttl=0,
)
self.workers_process.worker_runs.create(
version=self.dla,
configuration=self.dla_config,
parents=[reco_run_4.id],
- ttl=0,
)
self.workers_process.save()
@@ -150,26 +139,23 @@ class TestProcessName(FixtureAPITestCase):
If the default process worker name is too long (len() > 250) it gets truncated
"""
init_elements_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.InitElements)
- init_run = self.workers_process.worker_runs.create(version=init_elements_version, ttl=0)
+ init_run = self.workers_process.worker_runs.create(version=init_elements_version)
# Update the recognizer worker's name so that it is long
self.recognizer.worker.name = "animula vagula blandula hospes comesque corporis quae nunc abibis in loca pallidula rigida nudula ne"
self.recognizer.worker.save()
reco_run_1 = self.workers_process.worker_runs.create(
version=self.recognizer,
parents=[init_run.id],
- ttl=0,
)
reco_run_2 = self.workers_process.worker_runs.create(
version=self.recognizer,
configuration=self.reco_config_1,
parents=[reco_run_1.id],
- ttl=0,
)
self.workers_process.worker_runs.create(
version=self.recognizer,
configuration=self.reco_config_2,
parents=[reco_run_2.id],
- ttl=0,
)
self.workers_process.save()
@@ -191,8 +177,8 @@ class TestProcessName(FixtureAPITestCase):
self.workers_process.name = "My process"
self.workers_process.save()
init_elements_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.InitElements)
- init_run = self.workers_process.worker_runs.create(version=init_elements_version, ttl=0)
- self.workers_process.worker_runs.create(version=self.recognizer, parents=[init_run.id], ttl=0)
+ init_run = self.workers_process.worker_runs.create(version=init_elements_version)
+ self.workers_process.worker_runs.create(version=self.recognizer, parents=[init_run.id])
self.assertEqual(self.workers_process.name, "My process")
builder = ProcessBuilder(process=self.workers_process)
diff --git a/arkindex/process/tests/process/test_destroy.py b/arkindex/process/tests/process/test_destroy.py
index ffa37334ae..4bfe295929 100644
--- a/arkindex/process/tests/process/test_destroy.py
+++ b/arkindex/process/tests/process/test_destroy.py
@@ -111,7 +111,7 @@ class TestProcessDestroy(FixtureAPITestCase):
A process with worker runs linked to data cannot be deleted
"""
self.client.force_login(self.user)
- run = self.process.worker_runs.create(version=WorkerVersion.objects.first(), ttl=0)
+ run = self.process.worker_runs.create(version=WorkerVersion.objects.first())
page = self.corpus.elements.get(name="Volume 1, page 1r")
metadata = page.metadatas.get()
diff --git a/arkindex/process/tests/process/test_process_artifact_download.py b/arkindex/process/tests/process/test_process_artifact_download.py
index 004e375f67..394000cdb1 100644
--- a/arkindex/process/tests/process/test_process_artifact_download.py
+++ b/arkindex/process/tests/process/test_process_artifact_download.py
@@ -21,7 +21,7 @@ class TestProcessArtifactDownload(FixtureAPITestCase):
super().setUpTestData()
cls.pdf_export_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.ExportPDF)
cls.process = Process.objects.create(mode=ProcessMode.Export, creator=cls.user, corpus=cls.corpus)
- cls.worker_run = cls.process.worker_runs.create(version=cls.pdf_export_version, ttl=0)
+ cls.worker_run = cls.process.worker_runs.create(version=cls.pdf_export_version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
cls.process.run()
cls.task = cls.process.tasks.get()
diff --git a/arkindex/process/tests/process/test_retry.py b/arkindex/process/tests/process/test_retry.py
index 9407e2e4c5..547868fcba 100644
--- a/arkindex/process/tests/process/test_retry.py
+++ b/arkindex/process/tests/process/test_retry.py
@@ -156,7 +156,6 @@ class TestProcessRetry(FixtureAPITestCase):
process.worker_runs.create(
version=self.recognizer,
model_version=self.model_version,
- ttl=0,
)
with patch("arkindex.process.tasks.initialize_activity.delay"):
process.run()
@@ -209,7 +208,7 @@ class TestProcessRetry(FixtureAPITestCase):
worker_type, _ = WorkerType.objects.get_or_create(slug=f"type_{slug}", display_name=slug.capitalize())
worker, _ = Worker.objects.get_or_create(slug=slug, defaults={"type": worker_type, "repository_url": "fake"})
version, _ = worker.versions.get_or_create(version=1, defaults={"state": WorkerVersionState.Available, "docker_image_iid": "test"})
- return version.worker_runs.create(process=process, ttl=0)
+ return version.worker_runs.create(process=process)
init_version = WorkerVersion.objects.get_by_feature(feature=ArkindexFeature.InitElements)
@@ -263,7 +262,7 @@ class TestProcessRetry(FixtureAPITestCase):
mode=ProcessMode.Files,
creator=self.user,
)
- process.worker_runs.create(version=self.recognizer, ttl=0)
+ process.worker_runs.create(version=self.recognizer)
process.tasks.create(state=State.Error, run=0, depth=0, ttl=0)
self.assertEqual(process.state, State.Error)
process.finished = timezone.now()
@@ -305,7 +304,6 @@ class TestProcessRetry(FixtureAPITestCase):
"iiif_base_url": self.imgsrv.url,
},
),
- ttl=0,
)
process.tasks.create(state=State.Error, run=0, depth=0, ttl=0)
self.assertEqual(process.state, State.Error)
@@ -350,7 +348,7 @@ class TestProcessRetry(FixtureAPITestCase):
pdf_export_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.ExportPDF)
self.client.force_login(self.user)
process = self.corpus.processes.create(mode=ProcessMode.Export, creator=self.user)
- process.worker_runs.create(version=pdf_export_version, ttl=0)
+ process.worker_runs.create(version=pdf_export_version)
process.tasks.create(state=State.Error, run=0, depth=0, ttl=0)
self.assertEqual(process.state, State.Error)
process.finished = timezone.now()
diff --git a/arkindex/process/tests/process/test_run.py b/arkindex/process/tests/process/test_run.py
index ef0ba31c9f..8ca2347b3e 100644
--- a/arkindex/process/tests/process/test_run.py
+++ b/arkindex/process/tests/process/test_run.py
@@ -65,8 +65,8 @@ class TestProcessRun(FixtureTestCase):
mode=ProcessMode.Workers,
)
token_mock.side_effect = [b"12345", b"78945"]
- init_run = process.worker_runs.create(version=self.init_worker_version, ttl=0)
- run = process.worker_runs.create(version=self.version_with_model, parents=[init_run.id], ttl=0)
+ init_run = process.worker_runs.create(version=self.init_worker_version)
+ run = process.worker_runs.create(version=self.version_with_model, parents=[init_run.id])
run.model_version = self.model_version
run.save()
with patch("arkindex.process.tasks.initialize_activity.delay"):
diff --git a/arkindex/process/tests/process/test_start.py b/arkindex/process/tests/process/test_start.py
index 4d96cadcbb..32d9e5b2dd 100644
--- a/arkindex/process/tests/process/test_start.py
+++ b/arkindex/process/tests/process/test_start.py
@@ -117,7 +117,7 @@ class TestProcessStart(FixtureAPITestCase):
@override_settings(PUBLIC_HOSTNAME="https://darkindex.lol")
def test_without_required_model(self):
- self.workers_process.worker_runs.create(version=self.version_with_model, ttl=0)
+ self.workers_process.worker_runs.create(version=self.version_with_model)
self.client.force_login(self.user)
@@ -135,7 +135,7 @@ class TestProcessStart(FixtureAPITestCase):
@override_settings(PUBLIC_HOSTNAME="https://arkindex.localhost")
@patch("arkindex.project.triggers.process_tasks.initialize_activity.delay")
def test_with_required_model(self, activities_delay_mock):
- self.workers_process.worker_runs.create(version=self.version_with_model, model_version=self.model_version, ttl=0)
+ self.workers_process.worker_runs.create(version=self.version_with_model, model_version=self.model_version)
self.assertFalse(self.workers_process.tasks.exists())
self.client.force_login(self.user)
@@ -168,7 +168,7 @@ class TestProcessStart(FixtureAPITestCase):
)
def test_unavailable_worker_version(self):
- self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer)
self.recognizer.state = WorkerVersionState.Error
self.recognizer.save()
self.assertFalse(self.workers_process.tasks.exists())
@@ -185,7 +185,7 @@ class TestProcessStart(FixtureAPITestCase):
)
def test_unavailable_model_version(self):
- self.workers_process.worker_runs.create(version=self.recognizer, model_version=self.model_version, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer, model_version=self.model_version)
self.model_version.state = ModelVersionState.Error
self.model_version.save()
self.assertFalse(self.workers_process.tasks.exists())
@@ -202,7 +202,7 @@ class TestProcessStart(FixtureAPITestCase):
)
def test_archived_models(self):
- self.workers_process.worker_runs.create(version=self.recognizer, model_version=self.model_version, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer, model_version=self.model_version)
self.model.archived = timezone.now()
self.model.save()
self.assertFalse(self.workers_process.tasks.exists())
@@ -242,7 +242,6 @@ class TestProcessStart(FixtureAPITestCase):
version=self.recognizer,
configuration=None,
model_version=None,
- ttl=0,
)
# The other version is used with a configuration missing the required field
self.workers_process.worker_runs.create(
@@ -254,7 +253,6 @@ class TestProcessStart(FixtureAPITestCase):
},
),
model_version=None,
- ttl=0,
)
self.client.force_login(self.user)
@@ -278,8 +276,8 @@ class TestProcessStart(FixtureAPITestCase):
Default chunks, thumbnails and farm are used. Cache is disabled, and worker activities are enabled.
"""
init_elements_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.InitElements)
- init_run = self.workers_process.worker_runs.create(version=init_elements_version, ttl=0)
- run = self.workers_process.worker_runs.create(version=self.recognizer, parents=[init_run.id], ttl=0)
+ init_run = self.workers_process.worker_runs.create(version=init_elements_version)
+ run = self.workers_process.worker_runs.create(version=self.recognizer, parents=[init_run.id])
self.assertFalse(self.workers_process.tasks.exists())
self.client.force_login(self.user)
@@ -311,13 +309,13 @@ class TestProcessStart(FixtureAPITestCase):
def test_inconsistent_gpu_usages(self):
# The version's gpu_usage is Disabled, so the run's use_gpu is set to False
- self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer)
self.recognizer.gpu_usage = FeatureUsage.Required
self.recognizer.save()
self.dla.gpu_usage = FeatureUsage.Required
self.dla.save()
# The version's gpu_usage is Required, so the run's use_gpu is set to True
- self.workers_process.worker_runs.create(version=self.dla, ttl=0)
+ self.workers_process.worker_runs.create(version=self.dla)
self.dla.gpu_usage = FeatureUsage.Disabled
self.dla.save()
self.assertFalse(self.workers_process.tasks.exists())
@@ -338,7 +336,7 @@ class TestProcessStart(FixtureAPITestCase):
)
def test_dataset_requires_datasets(self):
- self.dataset_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.dataset_process.worker_runs.create(version=self.recognizer)
self.assertFalse(self.dataset_process.tasks.exists())
self.client.force_login(self.user)
@@ -355,7 +353,7 @@ class TestProcessStart(FixtureAPITestCase):
def test_dataset_requires_dataset_in_same_corpus(self):
test_set = self.other_dataset.sets.get(name="test")
ProcessDatasetSet.objects.create(process=self.dataset_process, set=test_set)
- self.dataset_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.dataset_process.worker_runs.create(version=self.recognizer)
self.assertFalse(self.dataset_process.tasks.exists())
self.client.force_login(self.user)
@@ -375,7 +373,7 @@ class TestProcessStart(FixtureAPITestCase):
test_set_2 = self.dataset2.sets.get(name="test")
ProcessDatasetSet.objects.create(process=self.dataset_process, set=test_set_1)
ProcessDatasetSet.objects.create(process=self.dataset_process, set=test_set_2)
- self.dataset_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.dataset_process.worker_runs.create(version=self.recognizer)
self.client.force_login(self.user)
@@ -400,7 +398,7 @@ class TestProcessStart(FixtureAPITestCase):
test_set_2 = self.other_dataset.sets.get(name="test")
ProcessDatasetSet.objects.create(process=self.dataset_process, set=test_set_1)
ProcessDatasetSet.objects.create(process=self.dataset_process, set=test_set_2)
- run = self.dataset_process.worker_runs.create(version=self.recognizer, ttl=0)
+ run = self.dataset_process.worker_runs.create(version=self.recognizer)
self.assertFalse(self.dataset_process.tasks.exists())
self.client.force_login(self.user)
@@ -434,7 +432,7 @@ class TestProcessStart(FixtureAPITestCase):
self.recognizer.save()
self.assertEqual(self.recognizer.state, WorkerVersionState.Available)
- run = self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ run = self.workers_process.worker_runs.create(version=self.recognizer)
self.assertFalse(self.workers_process.tasks.exists())
self.client.force_login(self.user)
@@ -463,7 +461,7 @@ class TestProcessStart(FixtureAPITestCase):
"""
A user can specify a ponos farm to use for a process
"""
- self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer)
farm = Farm.objects.get(name="Wheat farm")
self.client.force_login(self.user)
@@ -491,7 +489,7 @@ class TestProcessStart(FixtureAPITestCase):
farm = Farm.objects.get(name="Wheat farm")
get_default_farm_mock.return_value = farm
- self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer)
self.assertFalse(self.workers_process.tasks.exists())
self.client.force_login(self.user)
@@ -518,7 +516,7 @@ class TestProcessStart(FixtureAPITestCase):
@patch("arkindex.process.serializers.imports.get_default_farm")
def test_default_farm_guest(self, get_default_farm_mock, is_available_mock):
get_default_farm_mock.return_value = Farm.objects.first()
- self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer)
self.client.force_login(self.user)
with self.assertNumQueries(5):
@@ -539,7 +537,7 @@ class TestProcessStart(FixtureAPITestCase):
@patch("arkindex.ponos.models.Farm.is_available", return_value=False)
def test_farm_guest(self, is_available_mock):
- self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer)
self.client.force_login(self.user)
with self.assertNumQueries(7):
@@ -592,7 +590,7 @@ class TestProcessStart(FixtureAPITestCase):
"""
StartProcess should restrict the chunks to `settings.MAX_CHUNKS`
"""
- self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer)
self.client.force_login(self.user)
with self.assertNumQueries(5):
@@ -611,7 +609,7 @@ class TestProcessStart(FixtureAPITestCase):
"""
It should be possible to pass chunks parameters when starting a workers process
"""
- run = self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ run = self.workers_process.worker_runs.create(version=self.recognizer)
self.client.force_login(self.user)
with self.assertNumQueries(18):
@@ -644,7 +642,7 @@ class TestProcessStart(FixtureAPITestCase):
test_set_2 = self.dataset2.sets.get(name="test")
ProcessDatasetSet.objects.create(process=self.dataset_process, set=test_set_1)
ProcessDatasetSet.objects.create(process=self.dataset_process, set=test_set_2)
- run = self.dataset_process.worker_runs.create(version=self.recognizer, ttl=0)
+ run = self.dataset_process.worker_runs.create(version=self.recognizer)
self.client.force_login(self.user)
with self.assertNumQueries(12):
@@ -690,7 +688,7 @@ class TestProcessStart(FixtureAPITestCase):
"""
self.assertFalse(self.workers_process.use_cache)
self.assertEqual(self.workers_process.activity_state, ActivityState.Disabled)
- self.workers_process.worker_runs.create(version=self.version_gpu, ttl=0)
+ self.workers_process.worker_runs.create(version=self.version_gpu)
self.client.force_login(self.user)
@@ -723,12 +721,10 @@ class TestProcessStart(FixtureAPITestCase):
name="some_config",
configuration={"a": "b"},
),
- ttl=0,
)
run_2 = self.workers_process.worker_runs.create(
version=self.recognizer,
parents=[run_1.id],
- ttl=0,
)
self.assertNotEqual(run_1.task_slug, run_2.task_slug)
diff --git a/arkindex/process/tests/templates/test_apply.py b/arkindex/process/tests/templates/test_apply.py
index 0f59330309..43a4a98937 100644
--- a/arkindex/process/tests/templates/test_apply.py
+++ b/arkindex/process/tests/templates/test_apply.py
@@ -1,7 +1,6 @@
from datetime import datetime, timezone
from unittest.mock import call, patch
-from django.test import override_settings
from rest_framework import status
from rest_framework.reverse import reverse
@@ -19,7 +18,6 @@ from arkindex.training.models import Model, ModelVersionState
from arkindex.users.models import Role
-@override_settings(PONOS_MAXIMUM_TASK_TTL=3600)
class TestApplyTemplate(FixtureAPITestCase):
@classmethod
@@ -59,19 +57,16 @@ class TestApplyTemplate(FixtureAPITestCase):
cls.template_run_1 = cls.template.worker_runs.create(
version=cls.version_1,
configuration=cls.worker_configuration,
- ttl=7200,
)
cls.template_run_2 = cls.template.worker_runs.create(
version=cls.version_2,
parents=[cls.template_run_1.id],
model_version=cls.model_version,
- ttl=0,
)
cls.private_template.worker_runs.create(
version=cls.version_1,
configuration=cls.worker_configuration,
- ttl=0,
)
@patch("arkindex.project.mixins.get_max_level", return_value=Role.Guest.value)
@@ -170,7 +165,6 @@ class TestApplyTemplate(FixtureAPITestCase):
def test_apply(self):
self.assertIsNotNone(self.version_2.docker_image_iid)
- self.assertIsNone(self.corpus.maximum_task_ttl)
self.client.force_login(self.user)
with self.assertNumQueries(10):
response = self.client.post(
@@ -190,52 +184,16 @@ class TestApplyTemplate(FixtureAPITestCase):
self.assertIsNone(parent_run.model_version_id)
self.assertEqual(parent_run.configuration_id, self.worker_configuration.id)
self.assertListEqual(parent_run.parents, [])
- # This had a 7200 seconds TTL, but is limited by the instance limit
- self.assertEqual(parent_run.ttl, 3600)
self.assertEqual(child_run.process_id, self.process.id)
self.assertEqual(child_run.version_id, self.version_2.id)
self.assertEqual(child_run.model_version_id, self.model_version.id)
self.assertIsNone(child_run.configuration_id)
self.assertListEqual(child_run.parents, [parent_run.id])
- # This had an infinite TTL, but is limited by the instance limit
- self.assertEqual(child_run.ttl, 3600)
-
- def test_unlimited_ttl(self):
- self.corpus.maximum_task_ttl = 0
- self.corpus.save()
- self.client.force_login(self.user)
-
- with self.assertNumQueries(10):
- response = self.client.post(
- reverse("api:apply-process-template", kwargs={"pk": str(self.template.id)}),
- data={"process_id": str(self.process.id)},
- )
- self.assertEqual(response.status_code, status.HTTP_200_OK)
-
- child_run, parent_run = self.process.worker_runs.order_by("version__worker__slug")
- self.assertEqual(parent_run.ttl, 7200)
- self.assertEqual(child_run.ttl, 0)
-
- def test_corpus_limited_ttl(self):
- self.corpus.maximum_task_ttl = 9000
- self.corpus.save()
- self.client.force_login(self.user)
-
- with self.assertNumQueries(10):
- response = self.client.post(
- reverse("api:apply-process-template", kwargs={"pk": str(self.template.id)}),
- data={"process_id": str(self.process.id)},
- )
- self.assertEqual(response.status_code, status.HTTP_200_OK)
-
- child_run, parent_run = self.process.worker_runs.order_by("version__worker__slug")
- self.assertEqual(parent_run.ttl, 7200)
- self.assertEqual(child_run.ttl, 9000)
def test_excludes_init_elements(self):
init_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.InitElements)
- init_run = self.template.worker_runs.create(version=init_version, ttl=0)
+ init_run = self.template.worker_runs.create(version=init_version)
self.template_run_1.parents = [init_run.id]
self.template_run_1.save()
@@ -281,10 +239,7 @@ class TestApplyTemplate(FixtureAPITestCase):
process = self.corpus.processes.create(
creator=self.user, mode=ProcessMode.Workers
)
- process.worker_runs.create(
- version=self.version_2,
- ttl=0,
- )
+ process.worker_runs.create(version=self.version_2)
# Apply a template that has two other worker runs
with self.assertNumQueries(12):
response = self.client.post(
@@ -318,13 +273,13 @@ class TestApplyTemplate(FixtureAPITestCase):
# Set invalid values: the version with disabled GPU usage gets a GPU
self.template.worker_runs.filter(version=self.version_1).update(use_gpu=True)
# A signal is trying to set use_gpu to the correct values, so we create then update to give no GPU to a version that requires a GPU
- self.template.worker_runs.create(version=self.version_3, ttl=0)
+ self.template.worker_runs.create(version=self.version_3)
self.template.worker_runs.filter(version=self.version_3).update(use_gpu=False)
# Have two runs with a version that supports GPU usage, to test that both True and False are copied
self.version_2.gpu_usage = FeatureUsage.Supported
self.version_2.save()
- self.template.worker_runs.create(version=self.version_2, configuration=self.worker_configuration, use_gpu=True, ttl=0)
+ self.template.worker_runs.create(version=self.version_2, configuration=self.worker_configuration, use_gpu=True)
self.assertQuerySetEqual((
self.template.worker_runs
diff --git a/arkindex/process/tests/templates/test_create.py b/arkindex/process/tests/templates/test_create.py
index 8f819de010..f0bc778750 100644
--- a/arkindex/process/tests/templates/test_create.py
+++ b/arkindex/process/tests/templates/test_create.py
@@ -1,6 +1,5 @@
from unittest.mock import call, patch
-from django.test import override_settings
from rest_framework import status
from rest_framework.reverse import reverse
@@ -19,7 +18,6 @@ from arkindex.training.models import Model, ModelVersionState
from arkindex.users.models import Role, User
-@override_settings(PONOS_MAXIMUM_TASK_TTL=3600)
class TestCreateTemplate(FixtureAPITestCase):
@classmethod
@@ -61,12 +59,10 @@ class TestCreateTemplate(FixtureAPITestCase):
cls.run_1 = cls.process_template.worker_runs.create(
version=cls.version_1,
configuration=cls.worker_configuration,
- ttl=7200,
)
cls.run_2 = cls.process_template.worker_runs.create(
version=cls.version_2,
parents=[cls.run_1.id],
- ttl=0,
)
cls.model = Model.objects.create(name="moo")
@@ -75,24 +71,20 @@ class TestCreateTemplate(FixtureAPITestCase):
cls.template_run_1 = cls.template.worker_runs.create(
version=cls.version_1,
configuration=cls.worker_configuration,
- ttl=0,
)
cls.template_run_2 = cls.template.worker_runs.create(
version=cls.version_2,
parents=[cls.template_run_1.id],
model_version=cls.model_version,
- ttl=0,
)
cls.private_process_template.worker_runs.create(
version=cls.version_1,
configuration=cls.worker_configuration,
- ttl=0,
)
cls.private_template.worker_runs.create(
version=cls.version_1,
configuration=cls.worker_configuration,
- ttl=0,
)
def test_create(self):
@@ -116,10 +108,10 @@ class TestCreateTemplate(FixtureAPITestCase):
self.assertQuerySetEqual((
template_process.worker_runs
.order_by("version__worker__slug")
- .values_list("version_id", "model_version_id", "configuration_id", "parents", "ttl")
+ .values_list("version_id", "model_version_id", "configuration_id", "parents")
), [
- (self.version_2.id, None, None, [parent_run.id], 0),
- (self.version_1.id, None, self.worker_configuration.id, [], 7200),
+ (self.version_2.id, None, None, [parent_run.id]),
+ (self.version_1.id, None, self.worker_configuration.id, []),
])
def test_use_gpu(self):
@@ -128,13 +120,13 @@ class TestCreateTemplate(FixtureAPITestCase):
# Set invalid values: the version with disabled GPU usage gets a GPU
self.process_template.worker_runs.filter(version=self.version_1).update(use_gpu=True)
# A signal is trying to set use_gpu to the correct values, so we create then update to give no GPU to a version that requires a GPU
- self.process_template.worker_runs.create(version=self.version_3, ttl=0)
+ self.process_template.worker_runs.create(version=self.version_3)
self.process_template.worker_runs.filter(version=self.version_3).update(use_gpu=False)
# Have two runs with a version that supports GPU usage, to test that both True and False are copied
self.version_2.gpu_usage = FeatureUsage.Supported
self.version_2.save()
- self.process_template.worker_runs.create(version=self.version_2, configuration=self.worker_configuration, use_gpu=True, ttl=0)
+ self.process_template.worker_runs.create(version=self.version_2, configuration=self.worker_configuration, use_gpu=True)
self.assertQuerySetEqual((
self.process_template.worker_runs
@@ -170,7 +162,7 @@ class TestCreateTemplate(FixtureAPITestCase):
def test_excludes_init_elements(self):
init_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.InitElements)
- init_run = self.process_template.worker_runs.create(version=init_version, ttl=0)
+ init_run = self.process_template.worker_runs.create(version=init_version)
self.run_1.parents = [init_run.id]
self.run_1.save()
diff --git a/arkindex/process/tests/test_corpus_worker_runs.py b/arkindex/process/tests/test_corpus_worker_runs.py
index 590156dcc7..8b9754004f 100644
--- a/arkindex/process/tests/test_corpus_worker_runs.py
+++ b/arkindex/process/tests/test_corpus_worker_runs.py
@@ -29,7 +29,6 @@ class TestCorpusWorkerRuns(FixtureAPITestCase):
cls.run_1 = WorkerRun.objects.create(
process=cls.process,
version=cls.dla_worker_version,
- ttl=0,
has_results=True
)
@@ -57,12 +56,10 @@ class TestCorpusWorkerRuns(FixtureAPITestCase):
cls.run_2 = WorkerRun.objects.create(
process=cls.private_process,
version=cls.reco_worker_version,
- ttl=0,
)
cls.run_3 = WorkerRun.objects.create(
process=cls.private_process,
version=cls.dla_worker_version,
- ttl=0,
has_results=True
)
@@ -144,7 +141,6 @@ class TestCorpusWorkerRuns(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 0,
},
{
"id": str(self.run_3.id),
@@ -190,6 +186,5 @@ class TestCorpusWorkerRuns(FixtureAPITestCase):
},
"use_gpu": False,
"summary": "Worker Document layout analyser @ version 1",
- "ttl": 0,
}
])
diff --git a/arkindex/process/tests/test_elements_initialisation.py b/arkindex/process/tests/test_elements_initialisation.py
index 0ec36078b5..5deb50f40c 100644
--- a/arkindex/process/tests/test_elements_initialisation.py
+++ b/arkindex/process/tests/test_elements_initialisation.py
@@ -44,12 +44,10 @@ class TestElementsInit(FixtureAPITestCase):
)
init_run = process.worker_runs.create(
version=self.init_elements_version,
- ttl=0,
)
worker_run = process.worker_runs.create(
version=self.reco_version,
parents=[init_run.id],
- ttl=0,
)
with self.assertNumQueries(16):
@@ -86,8 +84,8 @@ class TestElementsInit(FixtureAPITestCase):
then one is created when the process is started.
"""
self.client.force_login(self.user)
- reco_run = self.process.worker_runs.create(version=self.reco_version, ttl=0)
- dla_run = self.process.worker_runs.create(version=self.dla_version, parents=[reco_run.id], ttl=0)
+ reco_run = self.process.worker_runs.create(version=self.reco_version)
+ dla_run = self.process.worker_runs.create(version=self.dla_version, parents=[reco_run.id])
with self.assertNumQueries(18):
response = self.client.post(
diff --git a/arkindex/process/tests/test_managers.py b/arkindex/process/tests/test_managers.py
index 26f0feada5..dc6b4a4c4f 100644
--- a/arkindex/process/tests/test_managers.py
+++ b/arkindex/process/tests/test_managers.py
@@ -28,23 +28,19 @@ class TestManagers(FixtureTestCase):
cls.worker_run_1 = cls.worker_version.worker_runs.create(
process=cls.corpus.processes.create(mode=ProcessMode.Workers, creator=cls.user),
- ttl=0,
)
cls.worker_run_2 = cls.worker_version.worker_runs.create(
process=cls.corpus.processes.create(mode=ProcessMode.Workers, creator=cls.user),
configuration=cls.worker_configuration,
- ttl=0,
)
cls.worker_run_3 = cls.worker_version.worker_runs.create(
process=cls.corpus.processes.create(mode=ProcessMode.Workers, creator=cls.user),
model_version=cls.model_version,
- ttl=0,
)
cls.worker_run_4 = cls.worker_version.worker_runs.create(
process=cls.corpus.processes.create(mode=ProcessMode.Workers, creator=cls.user),
model_version=cls.model_version,
configuration=cls.worker_configuration,
- ttl=0,
)
def test_corpus_worker_version_rebuild(self):
diff --git a/arkindex/process/tests/test_signals.py b/arkindex/process/tests/test_signals.py
index 1c6323c2a1..f1dc954aa8 100644
--- a/arkindex/process/tests/test_signals.py
+++ b/arkindex/process/tests/test_signals.py
@@ -38,10 +38,7 @@ class TestSignals(FixtureAPITestCase):
mode=ProcessMode.Workers,
farm=cls.farm,
)
- cls.run_1 = cls.process_1.worker_runs.create(
- version=cls.version_1,
- ttl=0,
- )
+ cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1)
cls.process_2 = cls.corpus.processes.create(
creator=cls.user,
mode=ProcessMode.Workers,
@@ -49,10 +46,7 @@ class TestSignals(FixtureAPITestCase):
)
def test_worker_run_check_parents_recursive(self):
- run_2 = self.process_1.worker_runs.create(
- version=self.version_2,
- ttl=0,
- )
+ run_2 = self.process_1.worker_runs.create(version=self.version_2)
self.assertListEqual(run_2.parents, [])
run_2.parents = [str(run_2.id)]
@@ -68,7 +62,6 @@ class TestSignals(FixtureAPITestCase):
self.process_2.worker_runs.create(
version=self.version_2,
parents=[str(self.run_1.id)],
- ttl=0,
)
self.process_1.refresh_from_db()
@@ -80,7 +73,6 @@ class TestSignals(FixtureAPITestCase):
self.process_2.worker_runs.create(
version=self.version_2,
parents=["12341234-1234-1234-1234-123412341234"],
- ttl=0,
)
self.process_1.refresh_from_db()
@@ -133,7 +125,6 @@ class TestSignals(FixtureAPITestCase):
run_2 = self.process_1.worker_runs.create(
version=self.version_2,
parents=[self.run_1.id],
- ttl=0,
)
self.run_1.parents = [run_2.id]
@@ -175,22 +166,18 @@ class TestSignals(FixtureAPITestCase):
run_2 = self.process_1.worker_runs.create(
version=self.version_2,
parents=[self.run_1.id],
- ttl=0,
)
run_3 = self.process_1.worker_runs.create(
version=version_3,
parents=[run_2.id],
- ttl=0,
)
run_4 = self.process_1.worker_runs.create(
version=version_4,
parents=[run_3.id],
- ttl=0,
)
run_5 = self.process_1.worker_runs.create(
version=version_5,
parents=[run_4.id],
- ttl=0,
)
self.run_1.parents = [run_5.id]
@@ -211,12 +198,10 @@ class TestSignals(FixtureAPITestCase):
run_2 = self.process_1.worker_runs.create(
version=self.version_2,
parents=[self.run_1.id],
- ttl=0,
)
run_3 = self.process_1.worker_runs.create(
version=version_3,
parents=[run_2.id],
- ttl=0,
)
run_3.parents.append(self.run_1.id)
@@ -231,7 +216,6 @@ class TestSignals(FixtureAPITestCase):
run_2 = self.process_1.worker_runs.create(
version=self.version_2,
parents=[self.run_1.id],
- ttl=0,
)
self.assertEqual(len(self.process_1.worker_runs.all()), 2)
@@ -245,7 +229,6 @@ class TestSignals(FixtureAPITestCase):
run = self.process_1.worker_runs.create(
version=self.version_2,
parents=[self.run_1.id],
- ttl=0,
)
self.assertIsNotNone(run.summary)
diff --git a/arkindex/process/tests/test_user_workerruns.py b/arkindex/process/tests/test_user_workerruns.py
index d98a8d5a33..4a515ef8c3 100644
--- a/arkindex/process/tests/test_user_workerruns.py
+++ b/arkindex/process/tests/test_user_workerruns.py
@@ -1,7 +1,6 @@
from datetime import datetime, timezone
from unittest.mock import call, patch
-from django.test import override_settings
from django.urls import reverse
from rest_framework import status
@@ -19,7 +18,6 @@ from arkindex.training.models import Model, ModelVersion, ModelVersionState
from arkindex.users.models import Right, Role, User
-@override_settings(PONOS_MAXIMUM_TASK_TTL=3600)
class TestUserWorkerRuns(FixtureAPITestCase):
@classmethod
def setUpTestData(cls):
@@ -67,7 +65,6 @@ class TestUserWorkerRuns(FixtureAPITestCase):
test_local_run = WorkerRun.objects.create(
process=self.local_process,
version=self.version_1,
- ttl=0,
)
self.client.force_login(self.user)
with self.assertNumQueries(5):
@@ -118,7 +115,6 @@ class TestUserWorkerRuns(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 0,
}, {
"configuration": None,
"id": str(self.local_run.id),
@@ -163,7 +159,6 @@ class TestUserWorkerRuns(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 0,
}])
def test_list_user_runs_only_own_runs(self):
@@ -175,7 +170,6 @@ class TestUserWorkerRuns(FixtureAPITestCase):
test_local_run = WorkerRun.objects.create(
process=test_local_process,
version=self.version_1,
- ttl=0,
)
assert WorkerRun.objects.filter(process__mode=ProcessMode.Local, process__creator=write_user).count() == 1
self.client.force_login(self.user)
@@ -264,8 +258,6 @@ class TestUserWorkerRuns(FixtureAPITestCase):
"use_cache": False,
},
"use_gpu": False,
- # The TTL is always 0 for user worker runs
- "ttl": 0,
})
def test_create_user_run_no_local_process(self):
@@ -483,7 +475,6 @@ class TestUserWorkerRuns(FixtureAPITestCase):
"use_cache": False,
},
"use_gpu": False,
- "ttl": 0,
})
def test_create_user_run_duplicate(self):
diff --git a/arkindex/process/tests/worker_activity/test_bulk_insert.py b/arkindex/process/tests/worker_activity/test_bulk_insert.py
index 0e2ac8c808..e51dd68498 100644
--- a/arkindex/process/tests/worker_activity/test_bulk_insert.py
+++ b/arkindex/process/tests/worker_activity/test_bulk_insert.py
@@ -40,7 +40,6 @@ class TestWorkerActivityBulkInsert(FixtureAPITestCase):
version=cls.worker_version,
configuration=cls.configuration,
model_version=cls.model_version,
- ttl=0,
)
def test_worker_version(self):
diff --git a/arkindex/process/tests/worker_activity/test_initialize.py b/arkindex/process/tests/worker_activity/test_initialize.py
index 321f1855f7..caaed902d0 100644
--- a/arkindex/process/tests/worker_activity/test_initialize.py
+++ b/arkindex/process/tests/worker_activity/test_initialize.py
@@ -19,8 +19,8 @@ class TestInitializeActivity(FixtureTestCase):
element_type=cls.corpus.types.get(slug="volume"),
activity_state=ActivityState.Pending,
)
- cls.process.worker_runs.create(version=cls.worker_version_1, ttl=0)
- cls.process.worker_runs.create(version=cls.worker_version_2, ttl=0)
+ cls.process.worker_runs.create(version=cls.worker_version_1)
+ cls.process.worker_runs.create(version=cls.worker_version_2)
@patch("arkindex.process.tasks.get_current_job")
def test_rq_progress(self, job_mock):
diff --git a/arkindex/process/tests/worker_activity/test_list.py b/arkindex/process/tests/worker_activity/test_list.py
index bc980a1189..75af915f29 100644
--- a/arkindex/process/tests/worker_activity/test_list.py
+++ b/arkindex/process/tests/worker_activity/test_list.py
@@ -48,7 +48,6 @@ class TestListWorkerActivities(FixtureAPITestCase):
version=cls.worker_version,
configuration=cls.configuration,
model_version=cls.model_version,
- ttl=0,
)
# Run the process, but skip the real activity initialization so that we can control it ourselves
diff --git a/arkindex/process/tests/worker_activity/test_update.py b/arkindex/process/tests/worker_activity/test_update.py
index e275425a7e..4d1a5aa34e 100644
--- a/arkindex/process/tests/worker_activity/test_update.py
+++ b/arkindex/process/tests/worker_activity/test_update.py
@@ -49,7 +49,6 @@ class TestUpdateWorkerActivity(FixtureAPITestCase):
version=cls.worker_version,
configuration=cls.configuration,
model_version=cls.model_version,
- ttl=0,
)
# Run the process, but skip the real activity initialization so that we can control it ourselves
@@ -201,7 +200,6 @@ class TestUpdateWorkerActivity(FixtureAPITestCase):
# Different configuration
configuration=None,
model_version=self.model_version,
- ttl=0,
)
with self.assertNumQueries(4):
@@ -457,17 +455,14 @@ class TestUpdateWorkerActivity(FixtureAPITestCase):
run_2 = self.process.worker_runs.create(
version=worker_version_2,
parents=[run_1.id],
- ttl=0,
)
self.process.worker_runs.create(
version=worker_version_3,
parents=[run_2.id],
- ttl=0,
)
self.process.worker_runs.create(
version=worker_version_4,
- ttl=0,
)
# Create activities for run_2, run_3 and run_4
diff --git a/arkindex/process/tests/worker_runs/test_build_task.py b/arkindex/process/tests/worker_runs/test_build_task.py
index 65ce96c232..022348a660 100644
--- a/arkindex/process/tests/worker_runs/test_build_task.py
+++ b/arkindex/process/tests/worker_runs/test_build_task.py
@@ -25,7 +25,7 @@ class TestWorkerRunsBuildTask(FixtureAPITestCase):
)
cls.version = WorkerVersion.objects.get(worker__slug="reco")
cls.worker = cls.version.worker
- cls.worker_run = cls.process.worker_runs.create(version=cls.version, ttl=0)
+ cls.worker_run = cls.process.worker_runs.create(version=cls.version)
# Model and Model version setup
cls.model_1 = Model.objects.create(name="My model")
@@ -52,7 +52,7 @@ class TestWorkerRunsBuildTask(FixtureAPITestCase):
"TASK_ELEMENTS": "/data/import/elements.json",
"ARKINDEX_WORKER_RUN_ID": str(self.worker_run.id),
})
- self.assertEqual(task.ttl, 0)
+ self.assertEqual(task.ttl, 3600)
def test_build_task_with_chunk(self):
task, parent_slugs = self.worker_run.build_task(self.process, ENV.copy(), "import", "/data/import/elements.json", chunk=4)
@@ -81,7 +81,6 @@ class TestWorkerRunsBuildTask(FixtureAPITestCase):
run_2 = self.process.worker_runs.create(
version=version_2,
parents=[self.worker_run.id],
- ttl=42,
)
task, parent_slugs = run_2.build_task(self.process, ENV.copy(), "import", "/data/import/elements.json")
@@ -97,7 +96,7 @@ class TestWorkerRunsBuildTask(FixtureAPITestCase):
"TASK_ELEMENTS": "/data/import/elements.json",
"ARKINDEX_WORKER_RUN_ID": str(run_2.id),
})
- self.assertEqual(task.ttl, 42)
+ self.assertEqual(task.ttl, 3600)
def test_build_task_with_parent_and_chunk(self):
version_2 = WorkerVersion.objects.create(
@@ -110,7 +109,6 @@ class TestWorkerRunsBuildTask(FixtureAPITestCase):
run_2 = self.process.worker_runs.create(
version=version_2,
parents=[self.worker_run.id],
- ttl=1000,
)
task, parent_slugs = run_2.build_task(self.process, ENV.copy(), "import", "/data/import/elements.json", chunk=4)
@@ -127,7 +125,7 @@ class TestWorkerRunsBuildTask(FixtureAPITestCase):
"TASK_ELEMENTS": "/data/import/elements.json",
"ARKINDEX_WORKER_RUN_ID": str(run_2.id),
})
- self.assertEqual(task.ttl, 1000)
+ self.assertEqual(task.ttl, 3600)
def test_build_task_shm_size(self):
self.version.configuration = {
@@ -160,7 +158,6 @@ class TestWorkerRunsBuildTask(FixtureAPITestCase):
run_2 = self.process.worker_runs.create(
version=version_2,
parents=[self.worker_run.id],
- ttl=0,
)
with self.assertRaisesRegex(
diff --git a/arkindex/process/tests/worker_runs/test_create.py b/arkindex/process/tests/worker_runs/test_create.py
index 41ac10e527..1d066bfb42 100644
--- a/arkindex/process/tests/worker_runs/test_create.py
+++ b/arkindex/process/tests/worker_runs/test_create.py
@@ -3,7 +3,6 @@ from datetime import datetime, timezone
from unittest.mock import call, patch
from django.db import transaction
-from django.test import override_settings
from django.urls import reverse
from rest_framework import status
@@ -20,7 +19,6 @@ from arkindex.training.models import Model, ModelVersion, ModelVersionState
from arkindex.users.models import Role
-@override_settings(PONOS_MAXIMUM_TASK_TTL=3600)
class TestWorkerRunsCreate(FixtureAPITestCase):
"""
Test worker runs create endpoint
@@ -38,7 +36,7 @@ class TestWorkerRunsCreate(FixtureAPITestCase):
)
cls.version_1 = WorkerVersion.objects.get(worker__slug="reco")
cls.worker_1 = cls.version_1.worker
- cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1, ttl=0)
+ cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1)
cls.configuration_1 = cls.worker_1.configurations.create(name="My config", configuration={"key": "value"})
worker_version = WorkerVersion.objects.exclude(worker=cls.version_1.worker).first()
cls.configuration_2 = worker_version.worker.configurations.create(name="Config")
@@ -330,7 +328,6 @@ class TestWorkerRunsCreate(FixtureAPITestCase):
},
"use_gpu": False,
"summary": "Worker Recognizer @ version 1",
- "ttl": 3600,
})
run = WorkerRun.objects.get(pk=pk)
# Check generated summary
@@ -415,7 +412,6 @@ class TestWorkerRunsCreate(FixtureAPITestCase):
},
"use_gpu": False,
"summary": "Worker Recognizer @ version 1 using configuration 'My config'",
- "ttl": 3600,
})
run = WorkerRun.objects.get(pk=pk)
# Check generated summary
@@ -563,7 +559,6 @@ class TestWorkerRunsCreate(FixtureAPITestCase):
},
"summary": f"Worker Recognizer @ version {worker_version.version}",
"use_gpu": use_gpu,
- "ttl": 3600,
})
run = WorkerRun.objects.get(pk=pk)
self.assertEqual(run.use_gpu, use_gpu)
@@ -623,7 +618,6 @@ class TestWorkerRunsCreate(FixtureAPITestCase):
},
"summary": "Worker Recognizer @ version 2",
"use_gpu": True,
- "ttl": 3600,
})
run = WorkerRun.objects.get(pk=pk)
self.assertEqual(run.use_gpu, True)
diff --git a/arkindex/process/tests/worker_runs/test_delete.py b/arkindex/process/tests/worker_runs/test_delete.py
index 410e5a3e44..e01006daa0 100644
--- a/arkindex/process/tests/worker_runs/test_delete.py
+++ b/arkindex/process/tests/worker_runs/test_delete.py
@@ -28,7 +28,7 @@ class TestWorkerRunsDelete(FixtureAPITestCase):
cls.version_1 = WorkerVersion.objects.get(worker__slug="reco")
cls.worker_1 = cls.version_1.worker
cls.version_2 = WorkerVersion.objects.get(worker__slug="dla")
- cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1, ttl=0)
+ cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1)
cls.agent = Agent.objects.create(
farm=cls.farm,
@@ -77,7 +77,7 @@ class TestWorkerRunsDelete(FixtureAPITestCase):
"""
A user cannot delete a worker run on a local process
"""
- run = self.local_process.worker_runs.create(version=self.version_1, ttl=0)
+ run = self.local_process.worker_runs.create(version=self.version_1)
self.client.force_login(self.user)
with self.assertNumQueries(4):
@@ -114,12 +114,10 @@ class TestWorkerRunsDelete(FixtureAPITestCase):
run_2 = self.process_1.worker_runs.create(
version=version_2,
parents=[self.run_1.id],
- ttl=0,
)
run_3 = self.process_1.worker_runs.create(
version=version_3,
parents=[self.run_1.id, run_2.id],
- ttl=0,
)
self.assertTrue(self.run_1.id in run_2.parents)
diff --git a/arkindex/process/tests/worker_runs/test_list.py b/arkindex/process/tests/worker_runs/test_list.py
index 8b0698d4a4..10bf646f47 100644
--- a/arkindex/process/tests/worker_runs/test_list.py
+++ b/arkindex/process/tests/worker_runs/test_list.py
@@ -22,7 +22,7 @@ class TestWorkerRunsList(FixtureAPITestCase):
)
cls.version_1 = WorkerVersion.objects.get(worker__slug="reco")
cls.worker_1 = cls.version_1.worker
- cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1, ttl=0)
+ cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1)
cls.process_2 = cls.corpus.processes.create(creator=cls.user, mode=ProcessMode.Workers)
def test_list_requires_login(self):
@@ -94,14 +94,10 @@ class TestWorkerRunsList(FixtureAPITestCase):
},
"use_gpu": False,
"summary": "Worker Recognizer @ version 1",
- "ttl": 0,
}])
def test_list_filter_process(self):
- run_2 = self.process_2.worker_runs.create(
- version=self.version_1,
- ttl=0,
- )
+ run_2 = self.process_2.worker_runs.create(version=self.version_1)
self.client.force_login(self.user)
with self.assertNumQueries(6):
@@ -165,5 +161,4 @@ class TestWorkerRunsList(FixtureAPITestCase):
},
"use_gpu": False,
"summary": "Worker Recognizer @ version 1",
- "ttl": 0,
}])
diff --git a/arkindex/process/tests/worker_runs/test_partial_update.py b/arkindex/process/tests/worker_runs/test_partial_update.py
index cb7ebb4664..e6a4606b5a 100644
--- a/arkindex/process/tests/worker_runs/test_partial_update.py
+++ b/arkindex/process/tests/worker_runs/test_partial_update.py
@@ -30,7 +30,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
cls.version_1 = WorkerVersion.objects.get(worker__slug="reco")
cls.worker_1 = cls.version_1.worker
cls.version_2 = WorkerVersion.objects.get(worker__slug="dla")
- cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1, ttl=1000)
+ cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1)
cls.configuration_1 = cls.worker_1.configurations.create(name="My config", configuration={"key": "value"})
worker_version = WorkerVersion.objects.exclude(worker=cls.version_1.worker).first()
cls.configuration_2 = worker_version.worker.configurations.create(name="Config")
@@ -115,10 +115,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
version=2,
configuration={"test": "test2"}
)
- run_2 = self.process_1.worker_runs.create(
- version=version_2,
- ttl=0,
- )
+ run_2 = self.process_1.worker_runs.create(version=version_2)
with self.assertNumQueries(0):
response = self.client.patch(
@@ -154,10 +151,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
version=2,
configuration={"test": "test2"}
)
- run_2 = self.process_1.worker_runs.create(
- version=version_2,
- ttl=0,
- )
+ run_2 = self.process_1.worker_runs.create(version=version_2)
self.client.force_login(self.user)
with self.assertNumQueries(3):
@@ -173,7 +167,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
"""
A user cannot update a worker run on a local process
"""
- run = self.local_process.worker_runs.create(version=self.version_1, ttl=1000)
+ run = self.local_process.worker_runs.create(version=self.version_1)
self.client.force_login(self.user)
with self.assertNumQueries(5):
@@ -266,7 +260,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 1000,
"summary": "Worker Recognizer @ version 1",
})
self.run_1.refresh_from_db()
@@ -332,7 +325,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 1000,
"summary": "Worker Recognizer @ version 1",
})
self.run_1.refresh_from_db()
@@ -403,7 +395,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 1000,
"summary": "Worker Recognizer @ version 1 using configuration 'My config'",
})
self.assertEqual(self.run_1.configuration.id, self.configuration_1.id)
@@ -455,10 +446,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Disabled
)
- run_2 = self.process_1.worker_runs.create(
- version=version_no_model,
- ttl=1000,
- )
+ run_2 = self.process_1.worker_runs.create(version=version_no_model)
with self.assertNumQueries(5):
response = self.client.patch(
@@ -484,10 +472,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Required
)
- run_2 = self.process_1.worker_runs.create(
- version=version_no_model,
- ttl=1000,
- )
+ run_2 = self.process_1.worker_runs.create(version=version_no_model)
random_model_version_uuid = str(uuid.uuid4())
with self.assertNumQueries(4):
@@ -515,10 +500,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Required
)
- run_2 = self.process_1.worker_runs.create(
- version=version_no_model,
- ttl=1000,
- )
+ run_2 = self.process_1.worker_runs.create(version=version_no_model)
# Create a model version, the user has no access to
model_no_access = Model.objects.create(name="Secret model")
@@ -555,10 +537,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Required
)
- run_2 = self.process_1.worker_runs.create(
- version=version_no_model,
- ttl=1000,
- )
+ run_2 = self.process_1.worker_runs.create(version=version_no_model)
def filter_rights(user, model, level):
"""
@@ -619,10 +598,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Required
)
- run = self.process_1.worker_runs.create(
- version=version,
- ttl=1000,
- )
+ run = self.process_1.worker_runs.create(version=version)
self.model_version_1.state = ModelVersionState.Error
self.model_version_1.save()
@@ -648,7 +624,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Required
)
- run = self.process_1.worker_runs.create(version=version, ttl=1000)
+ run = self.process_1.worker_runs.create(version=version)
self.model_1.archived = datetime.now(timezone.utc)
self.model_1.save()
@@ -676,10 +652,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Required
)
- run = self.process_1.worker_runs.create(
- version=version_with_model,
- ttl=0,
- )
+ run = self.process_1.worker_runs.create(version=version_with_model)
self.assertIsNone(run.model_version_id)
self.assertEqual(run.summary, "Worker Recognizer @ version 2")
@@ -758,7 +731,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 0,
"summary": f"Worker Recognizer @ version 2 with model {model_version.model.name} @ {str(model_version.id)[:6]}",
})
self.assertEqual(run.model_version_id, model_version.id)
@@ -778,7 +750,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
)
run = self.process_1.worker_runs.create(
version=version_with_model,
- ttl=0,
configuration=self.configuration_1
)
self.assertEqual(run.model_version_id, None)
@@ -853,7 +824,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 0,
"summary": f"Worker Recognizer @ version 2 with model My model @ {str(self.model_version_1.id)[:6]} using configuration 'My config'",
})
self.assertEqual(run.model_version_id, self.model_version_1.id)
@@ -867,7 +837,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_2,
- ttl=0,
)
self.client.force_login(self.user)
@@ -926,7 +895,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 1000,
"summary": "Worker Recognizer @ version 1",
})
@@ -954,7 +922,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
version=self.version_1,
model_version=None if model_version else self.model_version_1,
configuration=None if configuration else self.configuration_1,
- ttl=0,
)
# Having a model version or a configuration adds one query for each
@@ -1005,7 +972,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
run = WorkerRun.objects.create(
process=self.process_1,
version=worker_version,
- ttl=0
)
self.assertEqual(run.use_gpu, True if worker_version.gpu_usage == FeatureUsage.Required else False)
with self.assertNumQueries(3):
@@ -1035,7 +1001,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
run = WorkerRun.objects.create(
process=self.process_1,
version=worker_version,
- ttl=0
)
self.assertEqual(run.use_gpu, True if worker_version.gpu_usage == FeatureUsage.Required else False)
@@ -1093,7 +1058,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
},
"summary": f"Worker Recognizer @ version {worker_version.version}",
"use_gpu": use_gpu,
- "ttl": 0,
})
run.refresh_from_db()
self.assertEqual(run.use_gpu, use_gpu)
diff --git a/arkindex/process/tests/worker_runs/test_retrieve.py b/arkindex/process/tests/worker_runs/test_retrieve.py
index e253e85f25..42e7146a4a 100644
--- a/arkindex/process/tests/worker_runs/test_retrieve.py
+++ b/arkindex/process/tests/worker_runs/test_retrieve.py
@@ -35,7 +35,7 @@ class TestWorkerRunsRetrieve(FixtureAPITestCase):
cls.worker_custom = Worker.objects.get(slug="custom")
cls.version_custom = cls.worker_custom.versions.get()
- cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1, ttl=1000)
+ cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1)
cls.run_custom = cls.local_process.worker_runs.get(version=cls.version_custom)
cls.process_2 = cls.corpus.processes.create(creator=cls.user, mode=ProcessMode.Workers)
@@ -114,7 +114,6 @@ class TestWorkerRunsRetrieve(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 1000,
"summary": "Worker Recognizer @ version 1",
})
@@ -179,7 +178,6 @@ class TestWorkerRunsRetrieve(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 1000,
"summary": "Worker Recognizer @ version 1",
})
@@ -274,14 +272,13 @@ class TestWorkerRunsRetrieve(FixtureAPITestCase):
},
"summary": "Worker Custom worker @ version 1",
"use_gpu": False,
- "ttl": 0,
})
def test_retrieve_local(self):
"""
A user can retrieve a run on their own local process
"""
- run = self.local_process.worker_runs.create(version=self.version_1, ttl=0)
+ run = self.local_process.worker_runs.create(version=self.version_1)
self.client.force_login(self.user)
with self.assertNumQueries(5):
@@ -335,7 +332,6 @@ class TestWorkerRunsRetrieve(FixtureAPITestCase):
"prefix": None,
},
"use_gpu": False,
- "ttl": 0,
"summary": "Worker Recognizer @ version 1",
})
@@ -411,7 +407,6 @@ class TestWorkerRunsRetrieve(FixtureAPITestCase):
"prefix": None,
},
"use_gpu": False,
- "ttl": 1000,
"summary": "Worker Recognizer @ version 1",
})
diff --git a/arkindex/process/tests/worker_runs/test_ttl.py b/arkindex/process/tests/worker_runs/test_ttl.py
deleted file mode 100644
index edd98b6a9b..0000000000
--- a/arkindex/process/tests/worker_runs/test_ttl.py
+++ /dev/null
@@ -1,272 +0,0 @@
-from django.test import override_settings
-from django.urls import reverse
-from rest_framework import status
-
-from arkindex.ponos.models import Farm
-from arkindex.process.models import (
- ProcessMode,
- WorkerVersion,
-)
-from arkindex.project.tests import FixtureAPITestCase
-
-
-@override_settings(PONOS_MAXIMUM_TASK_TTL=3600)
-class TestWorkerRunTTL(FixtureAPITestCase):
-
- @classmethod
- def setUpTestData(cls):
- super().setUpTestData()
- cls.process = cls.corpus.processes.create(
- creator=cls.user,
- mode=ProcessMode.Workers,
- farm=Farm.objects.first(),
- )
- cls.recognizer = WorkerVersion.objects.get(worker__slug="reco")
- cls.dla = WorkerVersion.objects.get(worker__slug="dla")
- cls.worker_run = cls.process.worker_runs.create(version=cls.dla, ttl=0)
-
- def test_create_default_ttl(self):
- self.client.force_login(self.superuser)
- # Corpus TTL / WorkerRun TTL
- cases = [
- (0, 0),
- (10000, 10000),
- # No corpus TTL means the instance wide value should be set
- (None, 3600),
- ]
-
- for corpus_ttl, expected_ttl in cases:
- with self.subTest(corpus_ttl=corpus_ttl):
- self.process.worker_runs.filter(version=self.recognizer).delete()
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(6):
- response = self.client.post(
- reverse("api:worker-run-list", kwargs={"pk": str(self.process.id)}),
- {"worker_version_id": str(self.recognizer.id)},
- )
- self.assertEqual(response.status_code, status.HTTP_201_CREATED)
-
- data = response.json()
- self.assertEqual(data["ttl"], expected_ttl)
- run = self.process.worker_runs.get(id=data["id"])
- self.assertEqual(run.ttl, expected_ttl)
-
- def test_create_set_ttl(self):
- self.client.force_login(self.superuser)
- # Corpus TTL / WorkerRun TTL / Expected WorkerRun TTL
- cases = [
- (0, 0, 0),
- (0, 1000, 1000),
- (1800, 1000, 1000),
- (1800, 1800, 1800),
- # No corpus TTL means the instance wide value is the limit
- (None, 600, 600),
- (None, 3600, 3600),
- ]
-
- for corpus_ttl, worker_run_ttl, expected_ttl in cases:
- with self.subTest(corpus_ttl=corpus_ttl, worker_run_ttl=worker_run_ttl):
- self.process.worker_runs.filter(version=self.recognizer).delete()
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(6):
- response = self.client.post(
- reverse("api:worker-run-list", kwargs={"pk": str(self.process.id)}),
- {
- "worker_version_id": str(self.recognizer.id),
- "ttl": worker_run_ttl,
- },
- )
- self.assertEqual(response.status_code, status.HTTP_201_CREATED)
-
- data = response.json()
- self.assertEqual(data["ttl"], expected_ttl)
- run = self.process.worker_runs.get(id=data["id"])
- self.assertEqual(run.ttl, expected_ttl)
-
- def test_create_invalid_ttl(self):
- self.client.force_login(self.superuser)
- self.process.worker_runs.filter(version=self.recognizer).delete()
-
- # Corpus TTL, WorkerRun TTL, error message
- cases = [
- (None, "one hour", ["A valid integer is required."]),
- (None, -1, ["Ensure this value is greater than or equal to 1."]),
- (None, 0, ["Ensure this value is greater than or equal to 1."]),
- (None, 1e12, ["Ensure this value is less than or equal to 3600."]),
- (0, -1, ["Ensure this value is greater than or equal to 0."]),
- (0, 1e12, ["Ensure this value is less than or equal to 2147483647."]),
- (1800, -1, ["Ensure this value is greater than or equal to 1."]),
- (1800, 0, ["Ensure this value is greater than or equal to 1."]),
- (1800, 1e12, ["Ensure this value is less than or equal to 1800."]),
- ]
- for corpus_ttl, worker_run_ttl, expected_error in cases:
- with self.subTest(corpus_ttl=corpus_ttl, worker_run_ttl=worker_run_ttl):
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(4):
- response = self.client.post(
- reverse("api:worker-run-list", kwargs={"pk": str(self.process.id)}),
- {
- "worker_version_id": str(self.recognizer.id),
- "ttl": worker_run_ttl,
- },
- )
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
- self.assertEqual(response.json(), {
- "ttl": expected_error,
- })
-
- def test_partial_update_set_ttl(self):
- self.client.force_login(self.superuser)
- # Corpus TTL / WorkerRun TTL / Expected WorkerRun TTL
- cases = [
- (0, 0, 0),
- (0, 1000, 1000),
- (1800, 1000, 1000),
- (1800, 1800, 1800),
- # No corpus TTL means the instance wide value is the limit
- (None, 600, 600),
- (None, 3600, 3600),
- ]
-
- for corpus_ttl, worker_run_ttl, expected_ttl in cases:
- with self.subTest(corpus_ttl=corpus_ttl, worker_run_ttl=worker_run_ttl):
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(5):
- response = self.client.patch(
- reverse("api:worker-run-details", kwargs={"pk": str(self.worker_run.id)}),
- {"ttl": worker_run_ttl},
- )
- self.assertEqual(response.status_code, status.HTTP_200_OK)
-
- data = response.json()
- self.assertEqual(data["ttl"], expected_ttl)
- run = self.process.worker_runs.get(id=data["id"])
- self.assertEqual(run.ttl, expected_ttl)
-
- def test_partial_update_invalid_ttl(self):
- self.client.force_login(self.superuser)
-
- # Corpus TTL, WorkerRun TTL, error message
- cases = [
- (None, "one hour", ["A valid integer is required."]),
- (None, -1, ["Ensure this value is greater than or equal to 1."]),
- (None, 0, ["Ensure this value is greater than or equal to 1."]),
- (None, 1e12, ["Ensure this value is less than or equal to 3600."]),
- (0, -1, ["Ensure this value is greater than or equal to 0."]),
- (0, 1e12, ["Ensure this value is less than or equal to 2147483647."]),
- (1800, -1, ["Ensure this value is greater than or equal to 1."]),
- (1800, 0, ["Ensure this value is greater than or equal to 1."]),
- (1800, 1e12, ["Ensure this value is less than or equal to 1800."]),
- ]
- for corpus_ttl, worker_run_ttl, expected_error in cases:
- with self.subTest(corpus_ttl=corpus_ttl, worker_run_ttl=worker_run_ttl):
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(3):
- response = self.client.patch(
- reverse("api:worker-run-details", kwargs={"pk": str(self.worker_run.id)}),
- {"ttl": worker_run_ttl},
- )
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
- self.assertEqual(response.json(), {
- "ttl": expected_error,
- })
-
-
- def test_update_default_ttl(self):
- self.client.force_login(self.superuser)
- # Corpus TTL / WorkerRun TTL
- cases = [
- (0, 0),
- (10000, 10000),
- # No corpus TTL means the instance wide value should be set
- (None, 3600),
- ]
-
- for corpus_ttl, expected_ttl in cases:
- with self.subTest(corpus_ttl=corpus_ttl):
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(5):
- response = self.client.put(
- reverse("api:worker-run-details", kwargs={"pk": str(self.worker_run.id)}),
- {},
- )
- self.assertEqual(response.status_code, status.HTTP_200_OK)
-
- self.assertEqual(response.json()["ttl"], expected_ttl)
- self.worker_run.refresh_from_db()
- self.assertEqual(self.worker_run.ttl, expected_ttl)
-
- def test_update_set_ttl(self):
- self.client.force_login(self.superuser)
- # Corpus TTL / WorkerRun TTL / Expected WorkerRun TTL
- cases = [
- (0, 0, 0),
- (0, 1000, 1000),
- (1800, 1000, 1000),
- (1800, 1800, 1800),
- # No corpus TTL means the instance wide value is the limit
- (None, 600, 600),
- (None, 3600, 3600),
- ]
-
- for corpus_ttl, worker_run_ttl, expected_ttl in cases:
- with self.subTest(corpus_ttl=corpus_ttl, worker_run_ttl=worker_run_ttl):
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(5):
- response = self.client.put(
- reverse("api:worker-run-details", kwargs={"pk": str(self.worker_run.id)}),
- {"ttl": worker_run_ttl},
- )
- self.assertEqual(response.status_code, status.HTTP_200_OK)
-
- data = response.json()
- self.assertEqual(data["ttl"], expected_ttl)
- run = self.process.worker_runs.get(id=data["id"])
- self.assertEqual(run.ttl, expected_ttl)
-
- def test_update_invalid_ttl(self):
- self.client.force_login(self.superuser)
-
- # Corpus TTL, WorkerRun TTL, error message
- cases = [
- (None, "one hour", ["A valid integer is required."]),
- (None, -1, ["Ensure this value is greater than or equal to 1."]),
- (None, 0, ["Ensure this value is greater than or equal to 1."]),
- (None, 1e12, ["Ensure this value is less than or equal to 3600."]),
- (0, -1, ["Ensure this value is greater than or equal to 0."]),
- (0, 1e12, ["Ensure this value is less than or equal to 2147483647."]),
- (1800, -1, ["Ensure this value is greater than or equal to 1."]),
- (1800, 0, ["Ensure this value is greater than or equal to 1."]),
- (1800, 1e12, ["Ensure this value is less than or equal to 1800."]),
- ]
- for corpus_ttl, worker_run_ttl, expected_error in cases:
- with self.subTest(corpus_ttl=corpus_ttl, worker_run_ttl=worker_run_ttl):
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(3):
- response = self.client.put(
- reverse("api:worker-run-details", kwargs={"pk": str(self.worker_run.id)}),
- {"ttl": worker_run_ttl},
- )
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
- self.assertEqual(response.json(), {
- "ttl": expected_error,
- })
diff --git a/arkindex/process/tests/worker_runs/test_update.py b/arkindex/process/tests/worker_runs/test_update.py
index b412d3b9da..54226950e5 100644
--- a/arkindex/process/tests/worker_runs/test_update.py
+++ b/arkindex/process/tests/worker_runs/test_update.py
@@ -2,7 +2,6 @@ import uuid
from datetime import datetime, timezone
from unittest.mock import call, patch
-from django.test import override_settings
from django.urls import reverse
from rest_framework import status
from rest_framework.exceptions import ValidationError
@@ -14,7 +13,6 @@ from arkindex.training.models import Model, ModelVersion, ModelVersionState
from arkindex.users.models import Role
-@override_settings(PONOS_MAXIMUM_TASK_TTL=3600)
class TestWorkerRunsUpdate(FixtureAPITestCase):
"""
Test worker runs update endpoint
@@ -33,7 +31,7 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
cls.version_1 = WorkerVersion.objects.get(worker__slug="reco")
cls.worker_1 = cls.version_1.worker
cls.version_2 = WorkerVersion.objects.get(worker__slug="dla")
- cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1, ttl=1000)
+ cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1)
cls.configuration_1 = cls.worker_1.configurations.create(name="My config", configuration={"key": "value"})
worker_version = WorkerVersion.objects.exclude(worker=cls.version_1.worker).first()
cls.configuration_2 = worker_version.worker.configurations.create(name="Config")
@@ -99,7 +97,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_2,
- ttl=0,
)
with self.assertNumQueries(0):
@@ -137,7 +134,7 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
"""
A user cannot update a worker run on a local process
"""
- run = self.local_process.worker_runs.create(version=self.version_1, ttl=1000)
+ run = self.local_process.worker_runs.create(version=self.version_1)
self.client.force_login(self.user)
with self.assertNumQueries(5):
@@ -161,7 +158,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_2,
- ttl=0,
)
self.client.force_login(self.user)
@@ -194,7 +190,7 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
def test_update_duplicate_parents(self):
self.client.force_login(self.user)
- run_2 = self.process_1.worker_runs.create(version=self.version_2, ttl=0)
+ run_2 = self.process_1.worker_runs.create(version=self.version_2)
with self.assertNumQueries(4):
response = self.client.put(
@@ -218,7 +214,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
"""
run_2 = self.process_1.worker_runs.create(
version=self.version_2,
- ttl=0,
)
run_2.parents = [self.run_1.id, self.run_1.id]
@@ -286,7 +281,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 3600,
"summary": "Worker Recognizer @ version 1",
})
self.run_1.refresh_from_db()
@@ -352,7 +346,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
},
},
"use_gpu": False,
- "ttl": 3600,
"summary": "Worker Recognizer @ version 1",
})
self.run_1.refresh_from_db()
@@ -426,7 +419,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 3600,
"summary": "Worker Recognizer @ version 1 using configuration 'My config'",
})
self.assertEqual(self.run_1.configuration.id, self.configuration_1.id)
@@ -481,7 +473,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_no_model,
- ttl=0,
)
with self.assertNumQueries(5):
@@ -511,7 +502,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_no_model,
- ttl=0,
)
random_model_version_uuid = str(uuid.uuid4())
@@ -543,7 +533,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_no_model,
- ttl=0,
)
# Create a model version, the user has no access to
@@ -590,7 +579,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_no_model,
- ttl=0,
)
def filter_rights(user, model, level):
@@ -655,7 +643,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run = self.process_1.worker_runs.create(
version=version,
- ttl=0,
)
self.model_version_1.state = ModelVersionState.Error
self.model_version_1.save()
@@ -682,7 +669,7 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Required
)
- run = self.process_1.worker_runs.create(version=version, ttl=0)
+ run = self.process_1.worker_runs.create(version=version)
self.model_1.archived = datetime.now(timezone.utc)
self.model_1.save()
@@ -712,7 +699,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run = self.process_1.worker_runs.create(
version=version_with_model,
- ttl=0,
)
self.assertEqual(run.model_version, None)
# Check generated summary, before updating, there should be only information about the worker version
@@ -794,7 +780,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 3600,
"summary": f"Worker Recognizer @ version 2 with model {model_version.model.name} @ {str(model_version.id)[:6]}",
})
self.assertEqual(run.model_version_id, model_version.id)
@@ -813,7 +798,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run = self.process_1.worker_runs.create(
version=version_with_model,
- ttl=0,
)
self.assertIsNone(run.model_version)
self.assertIsNone(run.configuration)
@@ -892,7 +876,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 3600,
"summary": f"Worker Recognizer @ version 2 with model My model @ {str(self.model_version_1.id)[:6]} using configuration 'My config'",
})
self.assertEqual(run.model_version_id, self.model_version_1.id)
@@ -907,7 +890,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_2,
- ttl=0,
)
self.client.force_login(self.user)
@@ -916,7 +898,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
reverse("api:worker-run-details", kwargs={"pk": str(self.run_1.id)}),
data={
"parents": [str(run_2.id)],
- "ttl": 500,
},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -967,7 +948,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 500,
"summary": "Worker Recognizer @ version 1",
})
@@ -995,7 +975,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
version=self.version_1,
model_version=None if model_version else self.model_version_1,
configuration=None if configuration else self.configuration_1,
- ttl=0,
)
# Having a model version or a configuration adds one query for each
diff --git a/arkindex/sql_validation/indexer_prefetch.sql b/arkindex/sql_validation/indexer_prefetch.sql
index aab9525166..31118a36fb 100644
--- a/arkindex/sql_validation/indexer_prefetch.sql
+++ b/arkindex/sql_validation/indexer_prefetch.sql
@@ -56,8 +56,7 @@ SELECT "process_workerrun"."id",
"process_workerrun"."created",
"process_workerrun"."updated",
"process_workerrun"."has_results",
- "process_workerrun"."use_gpu",
- "process_workerrun"."ttl"
+ "process_workerrun"."use_gpu"
FROM "process_workerrun"
WHERE "process_workerrun"."id" IN ('{worker_run_id}'::uuid);
@@ -152,8 +151,7 @@ SELECT "process_workerrun"."id",
"process_workerrun"."created",
"process_workerrun"."updated",
"process_workerrun"."has_results",
- "process_workerrun"."use_gpu",
- "process_workerrun"."ttl"
+ "process_workerrun"."use_gpu"
FROM "process_workerrun"
WHERE "process_workerrun"."id" IN ('{worker_run_id}'::uuid);
diff --git a/arkindex/sql_validation/list_elements.sql b/arkindex/sql_validation/list_elements.sql
index 546d2c19cb..1a542e371a 100644
--- a/arkindex/sql_validation/list_elements.sql
+++ b/arkindex/sql_validation/list_elements.sql
@@ -41,8 +41,7 @@ SELECT "documents_element"."id",
"process_workerrun"."created",
"process_workerrun"."updated",
"process_workerrun"."has_results",
- "process_workerrun"."use_gpu",
- "process_workerrun"."ttl"
+ "process_workerrun"."use_gpu"
FROM "documents_element"
LEFT OUTER JOIN "process_workerrun" ON ("documents_element"."worker_run_id" = "process_workerrun"."id")
WHERE ("documents_element"."corpus_id" = '{corpus_id}'::uuid
--
GitLab