diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b8a56570c6c22c5b5058cb24d4aa8e83b92f9646..571548691a6780990f728bcee684e93516f5d578 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -11,7 +11,7 @@ include:
# For jobs that run backend scripts directly
.backend-setup:
- image: registry.gitlab.teklia.com/arkindex/backend/base:django-5.0.8
+ image: registry.gitlab.teklia.com/arkindex/backend/base:python3.12
cache:
paths:
@@ -61,7 +61,7 @@ backend-tests:
- arkindex test
backend-lint:
- image: python:3.10
+ image: python:3.12
stage: test
except:
@@ -159,7 +159,7 @@ backend-build:
backend-build-binary:
stage: build
- image: python:3.10
+ image: python:3.12
before_script:
- pip install nuitka
diff --git a/.isort.cfg b/.isort.cfg
index 0b8bd7b946695db57c49d6f0d920c390b8b27af7..83a3f011bdd9b6b79cf86a0ce4ac6759a50800b4 100644
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -7,4 +7,4 @@ use_parentheses = True
line_length = 120
default_section=FIRSTPARTY
-known_third_party = SolrClient,bleach,boto3,botocore,cryptography,corsheaders,django,django_admin_hstore_widget,django_rq,drf_spectacular,enumfields,gitlab,psycopg2,requests,responses,rest_framework,rq,setuptools,sqlparse,teklia_toolbox,tenacity,tripoli,yaml
+known_third_party = SolrClient,bleach,boto3,botocore,cryptography,corsheaders,django,django_admin_hstore_widget,django_rq,drf_spectacular,enumfields,gitlab,psycopg,requests,responses,rest_framework,rq,setuptools,sqlparse,teklia_toolbox,tenacity,tripoli,yaml
diff --git a/Dockerfile b/Dockerfile
index 01ec3163d4249441f31c1cabada9f7bc9f1da6ab..9ebc012df162b7d108da020dadb05d84d7d0eace 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,11 +1,11 @@
# syntax=docker/dockerfile:1
-FROM registry.gitlab.teklia.com/arkindex/backend/base:django-5.0.8 as build
+FROM registry.gitlab.teklia.com/arkindex/backend/base:python3.12 AS build
RUN mkdir build
ADD . build
RUN cd build && python3 setup.py sdist
-FROM registry.gitlab.teklia.com/arkindex/backend/base:django-5.0.8
+FROM registry.gitlab.teklia.com/arkindex/backend/base:python3.12
# Install arkindex and its deps
# Uses a source archive instead of full local copy to speedup docker build
diff --git a/VERSION b/VERSION
index f8a696c8dc56436062b4d179d96c78cbbbb718d7..661e7aeadf36f8e35f09094781c1abf2afab5c6a 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.7.2
+1.7.3
diff --git a/arkindex/budget/__init__.py b/arkindex/budget/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/arkindex/budget/apps.py b/arkindex/budget/apps.py
new file mode 100644
index 0000000000000000000000000000000000000000..d24215cc34d6bd983d8525543a655aecbb90edff
--- /dev/null
+++ b/arkindex/budget/apps.py
@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class BudgetConfig(AppConfig):
+ name = "arkindex.budget"
diff --git a/arkindex/budget/migrations/0001_initial.py b/arkindex/budget/migrations/0001_initial.py
new file mode 100644
index 0000000000000000000000000000000000000000..67ebfb8b710210ec03d3f261aa2e0957c11d3459
--- /dev/null
+++ b/arkindex/budget/migrations/0001_initial.py
@@ -0,0 +1,44 @@
+# Generated by Django 5.0.8 on 2025-02-10 09:37
+
+import uuid
+
+import django.core.validators
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ("process", "0048_worker_cost_fields"),
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="Budget",
+ fields=[
+ ("id", models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ("name", models.CharField(max_length=100, validators=[django.core.validators.MinLengthValidator(1)])),
+ ("total", models.DecimalField(decimal_places=3, default=0, max_digits=12)),
+ ],
+ ),
+ migrations.CreateModel(
+ name="BudgetEntry",
+ fields=[
+ ("id", models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ("created", models.DateTimeField(auto_now_add=True)),
+ ("description", models.CharField(max_length=255, validators=[django.core.validators.MinLengthValidator(1)])),
+ ("value", models.DecimalField(decimal_places=3, max_digits=12)),
+ ("budget", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name="entries", to="budget.budget")),
+ ("process", models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="budget_entries", to="process.process")),
+ ("user", models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="budget_entries", to=settings.AUTH_USER_MODEL)),
+ ],
+ options={
+ "verbose_name_plural": "budget entries",
+ },
+ ),
+ ]
diff --git a/arkindex/budget/migrations/__init__.py b/arkindex/budget/migrations/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/arkindex/budget/models.py b/arkindex/budget/models.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d1bd9757e3062ea63655f6a4d82bb7f5b694e05
--- /dev/null
+++ b/arkindex/budget/models.py
@@ -0,0 +1,46 @@
+import uuid
+
+from django.conf import settings
+from django.contrib.contenttypes.fields import GenericRelation
+from django.core.validators import MinLengthValidator
+from django.db import models
+
+
+class Budget(models.Model):
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ name = models.CharField(max_length=100, validators=[MinLengthValidator(1)])
+ total = models.DecimalField(max_digits=12, decimal_places=3, default=0)
+ memberships = GenericRelation("users.Right", "content_id")
+
+ def __str__(self) -> str:
+ return self.name
+
+
+class BudgetEntry(models.Model):
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ created = models.DateTimeField(auto_now_add=True)
+
+ budget = models.ForeignKey(Budget, on_delete=models.CASCADE, related_name="entries")
+ description = models.CharField(max_length=255, validators=[MinLengthValidator(1)])
+ value = models.DecimalField(max_digits=12, decimal_places=3)
+
+ user = models.ForeignKey(
+ settings.AUTH_USER_MODEL,
+ blank=True,
+ null=True,
+ on_delete=models.SET_NULL,
+ related_name="budget_entries",
+ )
+ process = models.ForeignKey(
+ "process.Process",
+ blank=True,
+ null=True,
+ on_delete=models.SET_NULL,
+ related_name="budget_entries",
+ )
+
+ class Meta:
+ verbose_name_plural = "budget entries"
+
+ def __str__(self) -> str:
+ return self.description
diff --git a/arkindex/documents/admin.py b/arkindex/documents/admin.py
index 1375921632ed67df5c8e8e20cb7dcbc3d79557f4..f4d4f4a63ca864555077abc73afee704ae282130 100644
--- a/arkindex/documents/admin.py
+++ b/arkindex/documents/admin.py
@@ -31,6 +31,8 @@ class ElementTypeInline(admin.TabularInline):
class CorpusAdmin(admin.ModelAdmin):
+ fields = ("id", "name", "description", "top_level_type", "public", "indexable", "maximum_task_ttl", "created")
+ readonly_fields = ("id", "created")
list_display = ("id", "name", "public", "top_level_type", "created")
search_fields = ("name", )
inlines = (ElementTypeInline, )
diff --git a/arkindex/documents/api/elements.py b/arkindex/documents/api/elements.py
index 5a027a5179824fb0243f06f1669544ad731c847a..d9644c226b4876dfee9cdef4cbc19b065e603e00 100644
--- a/arkindex/documents/api/elements.py
+++ b/arkindex/documents/api/elements.py
@@ -32,7 +32,6 @@ from drf_spectacular.utils import (
extend_schema_view,
inline_serializer,
)
-from psycopg2.extras import execute_values
from rest_framework import permissions, serializers, status
from rest_framework.exceptions import NotFound, PermissionDenied, ValidationError
from rest_framework.generics import (
@@ -133,13 +132,17 @@ def _fetch_has_children(elements):
if not elements:
return elements
+ # psycopg 2 used to have `execute_values` as a clean way to manage a VALUES statement, but psycopg 3 doesn't,
+ # so we will add a variable amount of placeholders to put each element ID separately.
+ # This results in `VALUES (%s),(%s),(%s)`.
+ placeholders = ",".join("(%s)" for _ in range(len(elements)))
+
with connection.cursor() as cursor:
- execute_values(
- cursor,
+ cursor.execute(
"SELECT DISTINCT ON (e.id) e.id, p.id is not null as has_children "
- "FROM (VALUES %s) e (id) "
+ f"FROM (VALUES {placeholders}) e (id) "
"LEFT JOIN documents_elementpath p ON ARRAY[e.id] && p.path AND p.path[array_length(p.path, 1)] = e.id",
- tuple((element.id, ) for element in elements),
+ tuple(element.id for element in elements),
)
has_children = dict(cursor.fetchall())
diff --git a/arkindex/documents/api/entities.py b/arkindex/documents/api/entities.py
index 7c60ae8f439c61c71247fd45fe1dd0fa6a0276c8..32fd14373930d4db767ba3b2fd0761ebce5c91de 100644
--- a/arkindex/documents/api/entities.py
+++ b/arkindex/documents/api/entities.py
@@ -6,7 +6,7 @@ from django.core.exceptions import ValidationError as DjangoValidationError
from django.db.utils import OperationalError
from django.shortcuts import get_object_or_404
from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema, extend_schema_view
-from psycopg2.errors import ProgramLimitExceeded
+from psycopg.errors import ProgramLimitExceeded
from rest_framework import permissions, serializers, status
from rest_framework.exceptions import NotFound, PermissionDenied, ValidationError
from rest_framework.generics import CreateAPIView, ListAPIView, RetrieveUpdateDestroyAPIView
diff --git a/arkindex/documents/export/__init__.py b/arkindex/documents/export/__init__.py
index ac1ccd78142bd42c8fece5e1299e0f9ab9d8f830..cdf88ee5ec1d0d4e9518e4271138f93341e93f41 100644
--- a/arkindex/documents/export/__init__.py
+++ b/arkindex/documents/export/__init__.py
@@ -48,7 +48,7 @@ EXPORT_QUERIES = [
def run_pg_query(query, source_db):
"""
Run a single Postgresql query and split the results into chunks.
- When a name is given to a cursor, psycopg2 uses a server-side cursor; we just use a random string as a name.
+ When a name is given to a cursor, psycopg uses a server-side cursor; we just use a random string as a name.
"""
db = connections[source_db]
@@ -86,7 +86,7 @@ def save_sqlite(rows, table, cursor):
return float(value)
# Show very explicit error messages if we stumble upon an unexpected type
- # https://docs.python.org/3.10/library/sqlite3.html#sqlite-and-python-types
+ # https://docs.python.org/3.12/library/sqlite3.html#sqlite-and-python-types
assert value is None or isinstance(value, (int, float, str, bytes)), f"Type {type(value)} is not supported by sqlite3"
return value
diff --git a/arkindex/documents/fixtures/data.json b/arkindex/documents/fixtures/data.json
index 3b1b3abcdba0e240785cf9a5a8329182f87c3c2f..48662e5a99292b19d3284508726ecfe8de180efb 100644
--- a/arkindex/documents/fixtures/data.json
+++ b/arkindex/documents/fixtures/data.json
@@ -355,8 +355,7 @@
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"has_results": false,
- "use_gpu": false,
- "ttl": 3600
+ "use_gpu": false
}
},
{
@@ -393,8 +392,7 @@
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"has_results": false,
- "use_gpu": false,
- "ttl": 3600
+ "use_gpu": false
}
},
{
@@ -410,8 +408,7 @@
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"has_results": false,
- "use_gpu": false,
- "ttl": 3600
+ "use_gpu": false
}
},
{
@@ -427,8 +424,7 @@
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"has_results": false,
- "use_gpu": false,
- "ttl": 0
+ "use_gpu": false
}
},
{
@@ -444,8 +440,7 @@
"created": "2020-02-02T01:23:45.678Z",
"updated": "2020-02-02T01:23:45.678Z",
"has_results": false,
- "use_gpu": false,
- "ttl": 0
+ "use_gpu": false
}
},
{
diff --git a/arkindex/documents/indexer.py b/arkindex/documents/indexer.py
index bee4bd7fc983ac95e51ccec5c374cb2a965a3440..fa273647b4e6d37eb1b560da1a41e1a5e18720b6 100644
--- a/arkindex/documents/indexer.py
+++ b/arkindex/documents/indexer.py
@@ -30,7 +30,7 @@ SELECT
element.name AS name,
elementtype.display_name AS type_name,
element.image_id AS image_id,
- element.polygon::bytea AS polygon,
+ element.polygon AS polygon,
element.worker_run_id AS worker_run_id
FROM documents_element element
INNER JOIN documents_elementtype elementtype ON (elementtype.id = element.type_id)
@@ -50,7 +50,7 @@ SELECT
element.name as name,
elementtype.display_name as type_name,
element.image_id AS image_id,
- element.polygon::bytea AS polygon,
+ element.polygon AS polygon,
element.worker_run_id AS worker_run_id
FROM (SELECT * FROM parent LIMIT %(limit)s OFFSET %(offset)s) AS parent_chunk
INNER JOIN documents_elementpath as elementpath ON (elementpath.path @> ARRAY[parent_chunk.id])
diff --git a/arkindex/documents/management/commands/build_fixtures.py b/arkindex/documents/management/commands/build_fixtures.py
index 87a12ff637d1ba11203d01ebe7a9a1e379e2afde..c691f669f8db03c6f73e653d650c8d1a6f881b00 100644
--- a/arkindex/documents/management/commands/build_fixtures.py
+++ b/arkindex/documents/management/commands/build_fixtures.py
@@ -206,14 +206,8 @@ class Command(BaseCommand):
mode=ProcessMode.Local,
creator=superuser,
)
- user_local_process.worker_runs.create(
- version=custom_version,
- ttl=0,
- )
- superuser_local_process.worker_runs.create(
- version=custom_version,
- ttl=0,
- )
+ user_local_process.worker_runs.create(version=custom_version)
+ superuser_local_process.worker_runs.create(version=custom_version)
# Create a corpus
corpus = Corpus.objects.create(
@@ -234,16 +228,13 @@ class Command(BaseCommand):
)
init_worker_run = process.worker_runs.create(
version=init_worker,
- ttl=3600,
)
dla_worker_run = process.worker_runs.create(
version=dla_worker,
- ttl=3600,
parents=[init_worker_run.id],
)
reco_run = process.worker_runs.create(
version=recognizer_worker,
- ttl=3600,
parents=[dla_worker_run.id],
)
diff --git a/arkindex/documents/management/commands/load_export.py b/arkindex/documents/management/commands/load_export.py
index 9b2fad0ee55dea000fbd0a93ec471f18b63e0648..9096a4303a16684bfdf7a97f75d97e2dacf7ae25 100644
--- a/arkindex/documents/management/commands/load_export.py
+++ b/arkindex/documents/management/commands/load_export.py
@@ -444,7 +444,6 @@ class Command(BaseCommand):
version_id=worker_version_id,
model_version=model_version,
configuration=configuration,
- defaults={"ttl": 0},
)
def create_image_server(self, row):
diff --git a/arkindex/documents/migrations/0014_corpus_budget.py b/arkindex/documents/migrations/0014_corpus_budget.py
new file mode 100644
index 0000000000000000000000000000000000000000..026c25cfb45183301b48072927efb9514456d19d
--- /dev/null
+++ b/arkindex/documents/migrations/0014_corpus_budget.py
@@ -0,0 +1,20 @@
+# Generated by Django 5.0.8 on 2025-02-10 09:41
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("budget", "0001_initial"),
+ ("documents", "0013_corpus_maximum_task_ttl"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="corpus",
+ name="budget",
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="corpora", to="budget.budget"),
+ ),
+ ]
diff --git a/arkindex/documents/models.py b/arkindex/documents/models.py
index 6ea6ee7745c9cba8008808ddf6b669ae2ea007b9..e21ef63b10a23bd1f029ac5c12f1d17919a37567 100644
--- a/arkindex/documents/models.py
+++ b/arkindex/documents/models.py
@@ -59,6 +59,14 @@ class Corpus(IndexableModel):
"When not set, the instance-wide maximum time-to-live will be used instead.",
)
+ budget = models.ForeignKey(
+ "budget.Budget",
+ on_delete=models.SET_NULL,
+ related_name="corpora",
+ blank=True,
+ null=True,
+ )
+
# Specific manager for ACL
objects = CorpusManager()
diff --git a/arkindex/documents/serializers/elements.py b/arkindex/documents/serializers/elements.py
index ee9ec29171e743ad7b17eaa618f0ea80d0c54ba1..528cd7b0d238b20c7a059be15a191073cf9001d9 100644
--- a/arkindex/documents/serializers/elements.py
+++ b/arkindex/documents/serializers/elements.py
@@ -285,6 +285,7 @@ class CorpusSerializer(serializers.ModelSerializer):
"authorized_users",
"indexable",
"maximum_task_ttl",
+ "budget_id",
)
extra_kwargs = {
"public": {
@@ -293,6 +294,9 @@ class CorpusSerializer(serializers.ModelSerializer):
},
"indexable": {
"help_text": "Whether this corpus should be indexed in Solr and searchable using the `SearchCorpus` endpoint."
+ },
+ "budget_id": {
+ "read_only": True,
}
}
diff --git a/arkindex/documents/serializers/entities.py b/arkindex/documents/serializers/entities.py
index bcc27783682a8a19bbf85198d0e1168983d13ac5..bd9a291d8ef730f6b2412896b8359061dcca4754 100644
--- a/arkindex/documents/serializers/entities.py
+++ b/arkindex/documents/serializers/entities.py
@@ -3,7 +3,7 @@ from textwrap import dedent
from django.db import transaction
from django.db.utils import OperationalError
-from psycopg2.errors import ProgramLimitExceeded
+from psycopg.errors import ProgramLimitExceeded
from rest_framework import serializers
from rest_framework.exceptions import ValidationError
diff --git a/arkindex/documents/tests/commands/test_cleanup.py b/arkindex/documents/tests/commands/test_cleanup.py
index 16fe3290c65e199dafe8a8ca5adf51c3ce68d78c..51489faf4803fbad0294a5a6344df6b20f32cbfb 100644
--- a/arkindex/documents/tests/commands/test_cleanup.py
+++ b/arkindex/documents/tests/commands/test_cleanup.py
@@ -1163,14 +1163,13 @@ class TestCleanupCommand(FixtureTestCase):
# from the WorkerRuns, so there would be 2 runs with the same version and no configuration when they should be unique
process = self.corpus.processes.create(mode=ProcessMode.Workers, creator=self.superuser)
version = removable_worker.versions.first()
- process.worker_runs.create(version=version, ttl=0)
+ process.worker_runs.create(version=version)
process.worker_runs.create(
version=version,
configuration=removable_worker.configurations.create(
name="Some configuration",
configuration={},
),
- ttl=0,
)
# This worker cannot be cleaned up because it is used in ML results
@@ -1249,7 +1248,6 @@ class TestCleanupCommand(FixtureTestCase):
worker_run = process.worker_runs.create(
version=worker_version,
model_version=used_model.versions.create(),
- ttl=0,
)
self.corpus.elements.create(
type=self.corpus.types.first(),
diff --git a/arkindex/documents/tests/commands/test_load_export.py b/arkindex/documents/tests/commands/test_load_export.py
index 0787a57f666980bdb4eab0240c459c20b6bfbe2e..383c1bc7ef1eeb733a17a7f60febfefcde60d417 100644
--- a/arkindex/documents/tests/commands/test_load_export.py
+++ b/arkindex/documents/tests/commands/test_load_export.py
@@ -39,7 +39,7 @@ class TestLoadExport(FixtureTestCase):
"process.workerversion": ["created", "updated", "configuration", "state", "docker_image_iid"],
# The WorkerRuns lose their parents, use different worker versions that just got recreated,
# are assigned to the user's local process and not the original one
- "process.workerrun": ["parents", "version", "process", "summary", "created", "updated", "ttl"],
+ "process.workerrun": ["parents", "version", "process", "summary", "created", "updated"],
"process.workertype": [],
"images.imageserver": ["s3_bucket", "s3_region", "created", "updated", "read_only"],
"images.image": ["created", "updated", "hash", "status"],
diff --git a/arkindex/documents/tests/tasks/test_corpus_delete.py b/arkindex/documents/tests/tasks/test_corpus_delete.py
index 7f0ff00642610028dedbe728915196333818e7af..cd5ad64153f21586311a3964b62a396365d19088 100644
--- a/arkindex/documents/tests/tasks/test_corpus_delete.py
+++ b/arkindex/documents/tests/tasks/test_corpus_delete.py
@@ -45,7 +45,7 @@ class TestDeleteCorpus(FixtureTestCase):
ml_class=cls.corpus.ml_classes.create(name="a class"),
)
element_process.elements.add(element)
- worker_run = element_process.worker_runs.create(version=cls.worker_version, ttl=0)
+ worker_run = element_process.worker_runs.create(version=cls.worker_version)
task_1, task_2, task_3, task_4 = Task.objects.bulk_create(
[
Task(
@@ -174,7 +174,7 @@ class TestDeleteCorpus(FixtureTestCase):
def test_run(self):
receivers = pre_delete.receivers
- with force_constraints_immediate(), self.assertExactQueries("corpus_delete.sql", params={"corpus_id": self.corpus.id}):
+ with force_constraints_immediate(), self.assertExactQueries("corpus_delete.sql", params={"corpus_id": self.corpus.id.hex}):
corpus_delete(self.corpus.id)
# Ensure the task restores the signal receivers
@@ -221,7 +221,7 @@ class TestDeleteCorpus(FixtureTestCase):
self.corpus.top_level_type = self.corpus.types.first()
self.corpus.save()
- with force_constraints_immediate(), self.assertExactQueries("corpus_delete_top_level_type.sql", params={"corpus_id": self.corpus.id}):
+ with force_constraints_immediate(), self.assertExactQueries("corpus_delete_top_level_type.sql", params={"corpus_id": self.corpus.id.hex}):
corpus_delete(self.corpus.id)
# Ensure the task restores the signal receivers
diff --git a/arkindex/documents/tests/tasks/test_export.py b/arkindex/documents/tests/tasks/test_export.py
index 8b21ea830c002881fa53b6e8b9935f57c407e231..9831b8eed4f1f7e73a91d17e3bbb45d855255a97 100644
--- a/arkindex/documents/tests/tasks/test_export.py
+++ b/arkindex/documents/tests/tasks/test_export.py
@@ -82,7 +82,6 @@ class TestExport(FixtureTestCase):
model=Model.objects.create(name="Some model"),
),
configuration=metadata_version.worker.configurations.create(name="Some configuration"),
- ttl=0,
)
element.metadatas.create(
diff --git a/arkindex/documents/tests/tasks/test_move_element.py b/arkindex/documents/tests/tasks/test_move_element.py
index 32d5a9cc788758283bb68d3013ff697e985928f0..84272f9adeb0aa700beaeeb5b7b65e6f28293903 100644
--- a/arkindex/documents/tests/tasks/test_move_element.py
+++ b/arkindex/documents/tests/tasks/test_move_element.py
@@ -26,9 +26,9 @@ class TestMoveElement(FixtureTestCase):
self.assertEqual(list(source_paths.values("path")), [{"path": [self.parent.id]}])
with self.assertExactQueries("element_move_without_child.sql", params={
- "source_id": str(self.source_without_child.id),
- "parent_id": str(self.parent.id),
- "destination_id": str(self.destination.id),
+ "source_id": self.source_without_child.id.hex,
+ "parent_id": self.parent.id.hex,
+ "destination_id": self.destination.id.hex,
"savepoints": [f"s{_thread.get_ident()}_x{connections['default'].savepoint_state + 1}", f"s{_thread.get_ident()}_x{connections['default'].savepoint_state + 2}"]
}):
move_element(self.source_without_child, self.destination)
@@ -52,9 +52,9 @@ class TestMoveElement(FixtureTestCase):
self.assertEqual(list(source_paths.values("path")), [{"path": [self.parent.id]}])
with self.assertExactQueries("element_move_with_children.sql", params={
- "source_id": str(self.source_with_children.id),
- "parent_id": str(self.parent.id),
- "destination_id": str(self.destination.id),
+ "source_id": self.source_with_children.id.hex,
+ "parent_id": self.parent.id.hex,
+ "destination_id": self.destination.id.hex,
"savepoints": [f"s{_thread.get_ident()}_x{connections['default'].savepoint_state + 1}", f"s{_thread.get_ident()}_x{connections['default'].savepoint_state + 2}"]
}):
move_element(self.source_with_children, self.destination)
diff --git a/arkindex/documents/tests/tasks/test_worker_results_delete.py b/arkindex/documents/tests/tasks/test_worker_results_delete.py
index 264d9fa821ab457e0e8996d01f0bf71d9302d563..cca0e0dcb0959a574ce1fa2aa5075c581ab736bf 100644
--- a/arkindex/documents/tests/tasks/test_worker_results_delete.py
+++ b/arkindex/documents/tests/tasks/test_worker_results_delete.py
@@ -40,7 +40,6 @@ class TestDeleteWorkerResults(FixtureTestCase):
version=cls.version_1,
model_version=cls.model_version,
configuration=cls.configuration,
- ttl=0,
)
cls.vol = cls.corpus.elements.get(name="Volume 1")
@@ -133,8 +132,8 @@ class TestDeleteWorkerResults(FixtureTestCase):
def test_run_on_corpus(self):
with self.assertExactQueries("worker_results_delete_in_corpus.sql", params={
- "corpus_id": str(self.corpus.id),
- "version_id": str(self.version_1.id),
+ "corpus_id": self.corpus.id.hex,
+ "version_id": self.version_1.id.hex,
}):
worker_results_delete(
corpus_id=self.corpus.id,
@@ -153,9 +152,9 @@ class TestDeleteWorkerResults(FixtureTestCase):
def test_run_on_parent(self):
with self.assertExactQueries("worker_results_delete_under_parent.sql", params={
- "corpus_id": str(self.corpus.id),
- "version_id": str(self.version_1.id),
- "element_id": str(self.page1.id),
+ "corpus_id": self.corpus.id.hex,
+ "version_id": self.version_1.id.hex,
+ "element_id": self.page1.id.hex,
}):
worker_results_delete(corpus_id=self.corpus.id, version_id=self.version_1.id, element_id=self.page1.id)
self.check_deleted(
@@ -169,9 +168,9 @@ class TestDeleteWorkerResults(FixtureTestCase):
The element itself is deleted after its related results from the same version
"""
with self.assertExactQueries("worker_results_delete_under_parent_included.sql", params={
- "corpus_id": str(self.corpus.id),
- "version_id": str(self.version_1.id),
- "element_id": str(self.page2.id),
+ "corpus_id": self.corpus.id.hex,
+ "version_id": self.version_1.id.hex,
+ "element_id": self.page2.id.hex,
}):
worker_results_delete(corpus_id=self.corpus.id, version_id=self.version_1.id, element_id=self.page2.id)
self.check_deleted(
@@ -183,9 +182,9 @@ class TestDeleteWorkerResults(FixtureTestCase):
def test_run_model_version_filter_on_parent(self):
with self.assertExactQueries("worker_results_delete_model_version_under_parent.sql", params={
- "corpus_id": str(self.corpus.id),
- "element_id": str(self.page2.id),
- "model_version_id": str(self.model_version.id),
+ "corpus_id": self.corpus.id.hex,
+ "element_id": self.page2.id.hex,
+ "model_version_id": self.model_version.id.hex,
}):
worker_results_delete(
corpus_id=self.corpus.id,
@@ -199,8 +198,8 @@ class TestDeleteWorkerResults(FixtureTestCase):
def test_run_configuration_filter(self):
with self.assertExactQueries("worker_results_delete_configuration_filter.sql", params={
- "corpus_id": str(self.corpus.id),
- "configuration_id": str(self.configuration.id),
+ "corpus_id": self.corpus.id.hex,
+ "configuration_id": self.configuration.id.hex,
}):
worker_results_delete(
corpus_id=self.corpus.id,
@@ -219,8 +218,8 @@ class TestDeleteWorkerResults(FixtureTestCase):
self.page2.worker_run = self.worker_run_2
self.page2.save()
with self.assertExactQueries("worker_results_delete_unset_configuration.sql", params={
- "corpus_id": str(self.corpus.id),
- "element_id": str(self.page2.id),
+ "corpus_id": self.corpus.id.hex,
+ "element_id": self.page2.id.hex,
}):
worker_results_delete(
corpus_id=self.corpus.id,
@@ -250,7 +249,7 @@ class TestDeleteWorkerResults(FixtureTestCase):
def test_run_all_versions(self):
with self.assertExactQueries("worker_results_delete_all_versions.sql", params={
- "corpus_id": str(self.corpus.id),
+ "corpus_id": self.corpus.id.hex,
}):
worker_results_delete(corpus_id=self.corpus.id)
self.check_deleted(
@@ -291,8 +290,8 @@ class TestDeleteWorkerResults(FixtureTestCase):
def test_run_worker_run_on_corpus(self):
with self.assertExactQueries("worker_results_delete_in_corpus_worker_run.sql", params={
- "corpus_id": str(self.corpus.id),
- "worker_run_id": str(self.worker_run_1.id),
+ "corpus_id": self.corpus.id.hex,
+ "worker_run_id": self.worker_run_1.id.hex,
}):
worker_results_delete(
corpus_id=self.corpus.id,
@@ -306,9 +305,9 @@ class TestDeleteWorkerResults(FixtureTestCase):
def test_run_worker_run_on_parent(self):
with self.assertExactQueries("worker_results_delete_under_parent_worker_run.sql", params={
- "corpus_id": str(self.corpus.id),
- "worker_run_id": str(self.worker_run_2.id),
- "element_id": str(self.page1.id),
+ "corpus_id": self.corpus.id.hex,
+ "worker_run_id": self.worker_run_2.id.hex,
+ "element_id": self.page1.id.hex,
}):
worker_results_delete(corpus_id=self.corpus.id, worker_run_id=self.worker_run_2.id, element_id=self.page1.id)
self.check_deleted(
@@ -324,9 +323,9 @@ class TestDeleteWorkerResults(FixtureTestCase):
self.page1.worker_version = self.version_2
self.page1.save()
with self.assertExactQueries("worker_results_delete_under_parent_included_worker_run.sql", params={
- "corpus_id": str(self.corpus.id),
- "worker_run_id": str(self.worker_run_2.id),
- "element_id": str(self.page1.id),
+ "corpus_id": self.corpus.id.hex,
+ "worker_run_id": self.worker_run_2.id.hex,
+ "element_id": self.page1.id.hex,
}):
worker_results_delete(corpus_id=self.corpus.id, worker_run_id=self.worker_run_2.id, element_id=self.page1.id)
self.check_deleted(
@@ -339,8 +338,8 @@ class TestDeleteWorkerResults(FixtureTestCase):
def test_run_worker_run_ignore_filters(self):
with self.assertExactQueries("worker_results_delete_in_corpus_worker_run.sql", params={
- "corpus_id": str(self.corpus.id),
- "worker_run_id": str(self.worker_run_1.id)
+ "corpus_id": self.corpus.id.hex,
+ "worker_run_id": self.worker_run_1.id.hex
}):
worker_results_delete(
corpus_id=self.corpus.id,
diff --git a/arkindex/documents/tests/test_bulk_classification.py b/arkindex/documents/tests/test_bulk_classification.py
index ddcc0d5c71d11584af9fb9c6d98a09f13c9d3399..492209723d686ed5d9ed6ac9c30d6f023538688e 100644
--- a/arkindex/documents/tests/test_bulk_classification.py
+++ b/arkindex/documents/tests/test_bulk_classification.py
@@ -287,7 +287,7 @@ class TestBulkClassification(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/documents/tests/test_bulk_element_transcriptions.py b/arkindex/documents/tests/test_bulk_element_transcriptions.py
index f4578bdce00ec3c58f24c48039f54ba05d86bc95..042eb16df25b730cfbc93156977919dba2e66f48 100644
--- a/arkindex/documents/tests/test_bulk_element_transcriptions.py
+++ b/arkindex/documents/tests/test_bulk_element_transcriptions.py
@@ -748,7 +748,7 @@ class TestBulkElementTranscriptions(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/documents/tests/test_bulk_elements.py b/arkindex/documents/tests/test_bulk_elements.py
index 05d3516d2df3acc5b3950d81de00214711764881..23964877c0093329ab0066e80c772e12971aac62 100644
--- a/arkindex/documents/tests/test_bulk_elements.py
+++ b/arkindex/documents/tests/test_bulk_elements.py
@@ -427,7 +427,7 @@ class TestBulkElements(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/documents/tests/test_bulk_transcription_entities.py b/arkindex/documents/tests/test_bulk_transcription_entities.py
index 8c60037671d12e50cfb6b3d08610ff1243f3d991..e0f159919c4e4043a6a686e7204e207efebd4fb3 100644
--- a/arkindex/documents/tests/test_bulk_transcription_entities.py
+++ b/arkindex/documents/tests/test_bulk_transcription_entities.py
@@ -233,7 +233,7 @@ class TestBulkTranscriptionEntities(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
diff --git a/arkindex/documents/tests/test_bulk_transcriptions.py b/arkindex/documents/tests/test_bulk_transcriptions.py
index 8fbbcdac41d84b2cb57f2221c5b4ea25dea9d38a..57ffaffdf329aaa5262f956d673f70d462bb17e2 100644
--- a/arkindex/documents/tests/test_bulk_transcriptions.py
+++ b/arkindex/documents/tests/test_bulk_transcriptions.py
@@ -263,7 +263,7 @@ class TestBulkTranscriptions(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/documents/tests/test_classification.py b/arkindex/documents/tests/test_classification.py
index 84ccefcfec56ecd8d506049f77db6e3fc7e4b314..935d36958c3e349d5fd49416608dc9523ded39f8 100644
--- a/arkindex/documents/tests/test_classification.py
+++ b/arkindex/documents/tests/test_classification.py
@@ -368,7 +368,7 @@ class TestClassifications(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/documents/tests/test_corpus.py b/arkindex/documents/tests/test_corpus.py
index 805447a33f06bf23f7ac7a47f022b69bac8db688..3055b5e8e67b7fb6787c53db95bb3568a9af3a51 100644
--- a/arkindex/documents/tests/test_corpus.py
+++ b/arkindex/documents/tests/test_corpus.py
@@ -104,6 +104,7 @@ class TestCorpus(FixtureAPITestCase):
"authorized_users": 1,
"top_level_type": None,
"maximum_task_ttl": 3600,
+ "budget_id": None,
}
]
)
@@ -138,6 +139,7 @@ class TestCorpus(FixtureAPITestCase):
"authorized_users": 2,
"top_level_type": None,
"maximum_task_ttl": 3600,
+ "budget_id": None,
},
{
"id": str(self.corpus_public.id),
@@ -150,6 +152,7 @@ class TestCorpus(FixtureAPITestCase):
"authorized_users": 1,
"top_level_type": None,
"maximum_task_ttl": 3600,
+ "budget_id": None,
}
]
)
@@ -184,6 +187,7 @@ class TestCorpus(FixtureAPITestCase):
"authorized_users": 2,
"top_level_type": None,
"maximum_task_ttl": 3600,
+ "budget_id": None,
},
{
"id": str(self.corpus_hidden.id),
@@ -196,6 +200,7 @@ class TestCorpus(FixtureAPITestCase):
"authorized_users": 0,
"top_level_type": None,
"maximum_task_ttl": 3600,
+ "budget_id": None,
},
{
"id": str(self.corpus_public.id),
@@ -208,6 +213,7 @@ class TestCorpus(FixtureAPITestCase):
"authorized_users": 1,
"top_level_type": None,
"maximum_task_ttl": 3600,
+ "budget_id": None,
}
]
)
@@ -356,6 +362,7 @@ class TestCorpus(FixtureAPITestCase):
"authorized_users": 1,
"top_level_type": None,
"maximum_task_ttl": 3600,
+ "budget_id": None,
})
def test_retrieve(self):
@@ -375,6 +382,7 @@ class TestCorpus(FixtureAPITestCase):
"authorized_users": 2,
"top_level_type": None,
"maximum_task_ttl": 3600,
+ "budget_id": None,
})
def test_retrieve_not_found(self):
@@ -405,6 +413,7 @@ class TestCorpus(FixtureAPITestCase):
"authorized_users": 1,
"top_level_type": None,
"maximum_task_ttl": 3600,
+ "budget_id": None,
})
@expectedFailure
@@ -434,6 +443,7 @@ class TestCorpus(FixtureAPITestCase):
"authorized_users": 2,
"top_level_type": None,
"maximum_task_ttl": 3600,
+ "budget_id": None,
})
def test_retrieve_maximum_task_ttl(self):
diff --git a/arkindex/documents/tests/test_corpus_elements.py b/arkindex/documents/tests/test_corpus_elements.py
index 78c3929e91a392c430441dc05166347f676d26af..0a252144e826decc2660034db6e410f6506f75c5 100644
--- a/arkindex/documents/tests/test_corpus_elements.py
+++ b/arkindex/documents/tests/test_corpus_elements.py
@@ -88,7 +88,7 @@ class TestListElements(FixtureAPITestCase):
expected_image_ids = set(filter(None, expected_elements.values_list("image_id", flat=True)))
expected_type_ids = set(expected_elements.values_list("type_id", flat=True))
- with self.assertExactQueries("list_elements.sql", params={"corpus_id": self.corpus.id}) as ctx:
+ with self.assertExactQueries("list_elements.sql", params={"corpus_id": self.corpus.id.hex}) as ctx:
response = self.client.get(
reverse("api:corpus-elements", kwargs={"corpus": self.corpus.id}),
)
diff --git a/arkindex/documents/tests/test_create_elements.py b/arkindex/documents/tests/test_create_elements.py
index 0cd08a757f2d0f78935cc70555d891cf35e1c54c..5c3a14342bb2fe58bd9ba0d8291ae7d756070bb5 100644
--- a/arkindex/documents/tests/test_create_elements.py
+++ b/arkindex/documents/tests/test_create_elements.py
@@ -720,7 +720,7 @@ class TestCreateElements(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/documents/tests/test_create_transcriptions.py b/arkindex/documents/tests/test_create_transcriptions.py
index 18c03d0f16a08fd4c353a90d4b514d74ef2ad34e..819e1d18b85bb55d340a3b1ee38906117cb9f0be 100644
--- a/arkindex/documents/tests/test_create_transcriptions.py
+++ b/arkindex/documents/tests/test_create_transcriptions.py
@@ -337,7 +337,7 @@ class TestTranscriptionCreate(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/documents/tests/test_destroy_elements.py b/arkindex/documents/tests/test_destroy_elements.py
index d6f39024b8bfa567d0810fe7bef6701b330842eb..c875c06a2130d7952525033de8dbc3e104d33626 100644
--- a/arkindex/documents/tests/test_destroy_elements.py
+++ b/arkindex/documents/tests/test_destroy_elements.py
@@ -219,7 +219,7 @@ class TestDestroyElements(FixtureAPITestCase):
self.assertEqual(len(ids), 24)
self.assertEqual(len(Element.objects.exclude(id__in=ids)), 5)
- with self.assertExactQueries("element_trash_children.sql", params={"id": str(self.vol.id)}):
+ with self.assertExactQueries("element_trash_children.sql", params={"id": self.vol.id.hex}):
Element.objects.filter(id=self.vol.id).trash()
with self.assertRaises(Element.DoesNotExist):
@@ -234,7 +234,7 @@ class TestDestroyElements(FixtureAPITestCase):
ids = list(children.values_list("id", flat=True))
self.assertEqual(len(ids), 24)
- with self.assertExactQueries("element_trash_no_children.sql", params={"id": str(self.vol.id)}):
+ with self.assertExactQueries("element_trash_no_children.sql", params={"id": self.vol.id.hex}):
Element.objects.filter(id=self.vol.id).trash(delete_children=False)
with self.assertRaises(Element.DoesNotExist):
@@ -248,7 +248,7 @@ class TestDestroyElements(FixtureAPITestCase):
ids = list(children.values_list("id", flat=True))
self.assertEqual(len(ids), 24)
- with self.assertExactQueries("element_trash_children.sql", params={"id": str(self.vol.id)}):
+ with self.assertExactQueries("element_trash_children.sql", params={"id": self.vol.id.hex}):
Element.objects.filter(id=self.vol.id).order_by("name").trash()
with self.assertRaises(Element.DoesNotExist):
@@ -324,7 +324,7 @@ class TestDestroyElements(FixtureAPITestCase):
corpus=self.corpus,
)
- with self.assertExactQueries("element_trash_deep.sql", params={"id": str(elements["A"].id)}):
+ with self.assertExactQueries("element_trash_deep.sql", params={"id": elements["A"].id.hex}):
Element.objects.filter(id=elements["A"].id).trash()
self.assertFalse(Element.objects.filter(id__in=[e.id for e in elements.values()]).exists())
@@ -563,7 +563,7 @@ class TestDestroyElements(FixtureAPITestCase):
test Element.delete method
"""
self.client.force_login(self.user)
- with self.assertExactQueries("element_dot_delete.sql", params={"id": str(self.vol.id)}):
+ with self.assertExactQueries("element_dot_delete.sql", params={"id": self.vol.id.hex}):
self.vol.delete()
with self.assertRaises(Element.DoesNotExist):
self.vol.refresh_from_db()
diff --git a/arkindex/documents/tests/test_edit_elementpath.py b/arkindex/documents/tests/test_edit_elementpath.py
index 250072d979f3dc9905e5b675043f4da05bbfe8c5..44e20b7c8791c47be59927a944a4140e0005fb5a 100644
--- a/arkindex/documents/tests/test_edit_elementpath.py
+++ b/arkindex/documents/tests/test_edit_elementpath.py
@@ -96,14 +96,14 @@ class TestEditElementPath(FixtureTestCase):
# add_parent uses transaction.atomic(), and we are running in a unit test, which is already in a transaction.
# This will cause a savepoint to be created, with a name that is hard to mock.
"savepoint": f"s{_thread.get_ident()}_x{connections['default'].savepoint_state + 1}",
- "A": elements["A"].id,
- "B": elements["B"].id,
+ "A": elements["A"].id.hex,
+ "B": elements["B"].id.hex,
# Element A has two parents so it has two paths.
# add_parent will pick the first one to perform updates on the new child's paths,
# so it will be seen in the SQL queries. To avoid intermittent failures,
# add_parent sorts parent paths by `path`, so we apply the same sort here.
# The paths only contain one ID, X's or Y's.
- "first_parent": elements["A"].paths.order_by("path").first().path[0],
+ "first_parent": elements["A"].paths.order_by("path").first().path[0].hex,
}
):
elements["B"].add_parent(elements["A"])
@@ -171,9 +171,9 @@ class TestEditElementPath(FixtureTestCase):
# add_parent uses transaction.atomic(), and we are running in a unit test, which is already in a transaction.
# This will cause a savepoint to be created, with a name that is hard to mock.
"savepoint": f"s{_thread.get_ident()}_x{connections['default'].savepoint_state + 1}",
- "A": elements["A"].id,
- "B": elements["B"].id,
- "K": elements["K"].id,
+ "A": elements["A"].id.hex,
+ "B": elements["B"].id.hex,
+ "K": elements["K"].id.hex,
}
):
elements["B"].add_parent(elements["A"])
@@ -285,8 +285,8 @@ class TestEditElementPath(FixtureTestCase):
# remove_child uses transaction.atomic(), and we are running in a unit test, which is already in a transaction.
# This will cause a savepoint to be created, with a name that is hard to mock.
"savepoint": f"s{_thread.get_ident()}_x{connections['default'].savepoint_state + 1}",
- "A": elements["A"].id,
- "B": elements["B"].id,
+ "A": elements["A"].id.hex,
+ "B": elements["B"].id.hex,
}
):
elements["A"].remove_child(elements["B"])
@@ -362,9 +362,9 @@ class TestEditElementPath(FixtureTestCase):
def __str__(self):
path_id = elements["B"].paths.get().id
if path1.id == path_id:
- return str(path1.path[0])
+ return path1.path[0].hex
if path2.id == path_id:
- return str(path2.path[0])
+ return path2.path[0].hex
raise AssertionError("Unexpected top-level path ID")
with self.assertExactQueries(
@@ -372,8 +372,8 @@ class TestEditElementPath(FixtureTestCase):
# remove_child uses transaction.atomic(), and we are running in a unit test, which is already in a transaction.
# This will cause a savepoint to be created, with a name that is hard to mock.
"savepoint": f"s{_thread.get_ident()}_x{connections['default'].savepoint_state + 1}",
- "A": elements["A"].id,
- "B": elements["B"].id,
+ "A": elements["A"].id.hex,
+ "B": elements["B"].id.hex,
"first_parent": FirstParent(),
}
):
@@ -514,7 +514,7 @@ class TestEditElementPath(FixtureTestCase):
# remove_children uses transaction.atomic(), and we are running in a unit test, which is already in a transaction.
# This will cause a savepoint to be created, with a name that is hard to mock.
"savepoint": f"s{_thread.get_ident()}_x{connections['default'].savepoint_state + 1}",
- "A": elements["A"].id,
+ "A": elements["A"].id.hex,
}
):
elements["A"].remove_children()
@@ -575,8 +575,8 @@ class TestEditElementPath(FixtureTestCase):
# remove_children uses transaction.atomic(), and we are running in a unit test, which is already in a transaction.
# This will cause a savepoint to be created, with a name that is hard to mock.
"savepoint": f"s{_thread.get_ident()}_x{connections['default'].savepoint_state + 1}",
- "A": elements["A"].id,
- "X": elements["X"].id,
+ "A": elements["A"].id.hex,
+ "X": elements["X"].id.hex,
}
):
elements["A"].remove_children()
@@ -645,8 +645,8 @@ class TestEditElementPath(FixtureTestCase):
# remove_children uses transaction.atomic(), and we are running in a unit test, which is already in a transaction.
# This will cause a savepoint to be created, with a name that is hard to mock.
"savepoint": f"s{_thread.get_ident()}_x{connections['default'].savepoint_state + 1}",
- "A": elements["A"].id,
- "first_parent": elements["A"].paths.order_by("id").first().path[0],
+ "A": elements["A"].id.hex,
+ "first_parent": elements["A"].paths.order_by("id").first().path[0].hex,
}
):
elements["A"].remove_children()
diff --git a/arkindex/documents/tests/test_entities_api.py b/arkindex/documents/tests/test_entities_api.py
index 32b4dffa231488169b0f3f34ac80ad5c08e3524a..90675af67f1e8e4bc7b76924eb4883294f9576df 100644
--- a/arkindex/documents/tests/test_entities_api.py
+++ b/arkindex/documents/tests/test_entities_api.py
@@ -388,7 +388,7 @@ class TestEntitiesAPI(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_version_1, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_version_1)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run_1.process.run()
task = self.worker_run_1.process.tasks.first()
@@ -863,7 +863,7 @@ class TestEntitiesAPI(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_version_1, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_version_1)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run_1.process.run()
task = self.worker_run_1.process.tasks.first()
diff --git a/arkindex/documents/tests/test_indexer.py b/arkindex/documents/tests/test_indexer.py
index 8fd06cb9e49d9d95511ea12ed86d563e807b899c..2cd29e02e2a5ff3d4ce20f48f1c4584fbe478827 100644
--- a/arkindex/documents/tests/test_indexer.py
+++ b/arkindex/documents/tests/test_indexer.py
@@ -295,14 +295,13 @@ class TestIndexerCommand(FixtureTestCase):
indexer = Indexer(self.private_corpus.id)
with self.assertExactQueries("indexer_prefetch.sql", params={
- "corpus_id": self.private_corpus.id,
- "page_id": self.page.id,
- "image_id": self.page.image_id,
- "worker_run_id": self.worker_run.id,
- "worker_version_id": self.worker_version.id,
- "worker_id": self.worker.id,
- "transcription_id": tr.id,
- "type_id": location_type.id
+ "corpus_id": self.private_corpus.id.hex,
+ "page_id": self.page.id.hex,
+ "worker_run_id": self.worker_run.id.hex,
+ "worker_version_id": self.worker_version.id.hex,
+ "worker_id": self.worker.id.hex,
+ "transcription_id": tr.id.hex,
+ "type_id": location_type.id.hex,
}):
indexer.index()
self.assertEqual(mock_solr.index.call_count, 1)
diff --git a/arkindex/documents/tests/test_metadata.py b/arkindex/documents/tests/test_metadata.py
index 55b026370d0252154f8f913acaf995cc3dcf3f50..ae0fc7a3c80023828b3d1a3afc242961fe54f61a 100644
--- a/arkindex/documents/tests/test_metadata.py
+++ b/arkindex/documents/tests/test_metadata.py
@@ -44,7 +44,7 @@ class TestMetaData(FixtureAPITestCase):
creator=cls.user,
farm=Farm.objects.first(),
)
- cls.process.worker_runs.create(version=cls.worker_version, ttl=0)
+ cls.process.worker_runs.create(version=cls.worker_version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
cls.process.run()
cls.task = cls.process.tasks.first()
@@ -463,7 +463,7 @@ class TestMetaData(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
@@ -1458,7 +1458,7 @@ class TestMetaData(FixtureAPITestCase):
mode=ProcessMode.Workers,
corpus=self.corpus,
)
- other_worker_run = process2.worker_runs.create(version=self.worker_run.version, ttl=0)
+ other_worker_run = process2.worker_runs.create(version=self.worker_run.version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
self.worker_run.process.run()
task = self.worker_run.process.tasks.first()
diff --git a/arkindex/documents/tests/test_path_constraints.py b/arkindex/documents/tests/test_path_constraints.py
index 751270a2219e7182c7df25541593d257c7a2d566..0e627cdb87d42b03c370485fea75035f6a47a751 100644
--- a/arkindex/documents/tests/test_path_constraints.py
+++ b/arkindex/documents/tests/test_path_constraints.py
@@ -1,5 +1,4 @@
-from django.db import IntegrityError, connections, transaction
-from django.db.utils import InternalError
+from django.db import IntegrityError, ProgrammingError, connections, transaction
from arkindex.documents.models import ElementPath
from arkindex.project.tests import FixtureTestCase
@@ -55,7 +54,7 @@ class TestPathConstraints(FixtureTestCase):
ordering=11111,
)
- with self.assertRaisesMessage(InternalError, "Each element may only have one ordering within the same parent"):
+ with self.assertRaisesMessage(ProgrammingError, "Each element may only have one ordering within the same parent"):
# Committing the savepoint would not execute the deferred trigger, and committing the transaction
# would mess with the test class and any subsequent unit tests, so the next best thing is to act
# like we are about to commit by forcing all constraint checks to run.
@@ -91,7 +90,7 @@ class TestPathConstraints(FixtureTestCase):
# We save using update_fields to really ensure only the element gets updated.
path.save(update_fields=["element"])
- with self.assertRaisesMessage(InternalError, "Each element may only have one ordering within the same parent"):
+ with self.assertRaisesMessage(ProgrammingError, "Each element may only have one ordering within the same parent"):
# Committing the savepoint would not execute the deferred trigger, and committing the transaction
# would mess with the test class and any subsequent unit tests, so the next best thing is to act
# like we are about to commit by forcing all constraint checks to run.
@@ -127,7 +126,7 @@ class TestPathConstraints(FixtureTestCase):
# We save using update_fields to really ensure only the path gets updated.
path.save(update_fields=["path"])
- with self.assertRaisesMessage(InternalError, "Each element may only have one ordering within the same parent"):
+ with self.assertRaisesMessage(ProgrammingError, "Each element may only have one ordering within the same parent"):
# Committing the savepoint would not execute the deferred trigger, and committing the transaction
# would mess with the test class and any subsequent unit tests, so the next best thing is to act
# like we are about to commit by forcing all constraint checks to run.
@@ -153,7 +152,7 @@ class TestPathConstraints(FixtureTestCase):
# We save using update_fields to really ensure only the ordering gets updated.
path.save(update_fields=["ordering"])
- with self.assertRaisesMessage(InternalError, "Each element may only have one ordering within the same parent"):
+ with self.assertRaisesMessage(ProgrammingError, "Each element may only have one ordering within the same parent"):
# Committing the savepoint would not execute the deferred trigger, and committing the transaction
# would mess with the test class and any subsequent unit tests, so the next best thing is to act
# like we are about to commit by forcing all constraint checks to run.
@@ -176,7 +175,7 @@ class TestPathConstraints(FixtureTestCase):
ordering=3,
)
- with self.assertRaisesMessage(InternalError, "Each element within a parent must have a distinct ordering"):
+ with self.assertRaisesMessage(ProgrammingError, "Each element within a parent must have a distinct ordering"):
# Committing the savepoint would not execute the deferred trigger, and committing the transaction
# would mess with the test class and any subsequent unit tests, so the next best thing is to act
# like we are about to commit by forcing all constraint checks to run.
@@ -201,7 +200,7 @@ class TestPathConstraints(FixtureTestCase):
# We save using update_fields to really ensure only the element gets updated.
path.save(update_fields=["element"])
- with self.assertRaisesMessage(InternalError, "Each element within a parent must have a distinct ordering"):
+ with self.assertRaisesMessage(ProgrammingError, "Each element within a parent must have a distinct ordering"):
# Committing the savepoint would not execute the deferred trigger, and committing the transaction
# would mess with the test class and any subsequent unit tests, so the next best thing is to act
# like we are about to commit by forcing all constraint checks to run.
@@ -234,7 +233,7 @@ class TestPathConstraints(FixtureTestCase):
# We save using update_fields to really ensure only the path gets updated.
path.save(update_fields=["path"])
- with self.assertRaisesMessage(InternalError, "Each element within a parent must have a distinct ordering"):
+ with self.assertRaisesMessage(ProgrammingError, "Each element within a parent must have a distinct ordering"):
# Committing the savepoint would not execute the deferred trigger, and committing the transaction
# would mess with the test class and any subsequent unit tests, so the next best thing is to act
# like we are about to commit by forcing all constraint checks to run.
@@ -261,7 +260,7 @@ class TestPathConstraints(FixtureTestCase):
# We save using update_fields to really ensure only the ordering gets updated.
path.save()
- with self.assertRaisesMessage(InternalError, "Each element within a parent must have a distinct ordering"):
+ with self.assertRaisesMessage(ProgrammingError, "Each element within a parent must have a distinct ordering"):
# Committing the savepoint would not execute the deferred trigger, and committing the transaction
# would mess with the test class and any subsequent unit tests, so the next best thing is to act
# like we are about to commit by forcing all constraint checks to run.
diff --git a/arkindex/metrics/tests/test_metrics_api.py b/arkindex/metrics/tests/test_metrics_api.py
index 917979e5978d8907d7568c7c66dbf53bdcf34be7..4a0719a5f53ba2c95eb39a99527059f61ef42d1b 100644
--- a/arkindex/metrics/tests/test_metrics_api.py
+++ b/arkindex/metrics/tests/test_metrics_api.py
@@ -35,6 +35,7 @@ class TestMetricsAPI(FixtureAPITestCase):
mode=AgentMode.Docker,
hostname="Demo Agent",
farm=farm,
+ fingerprint="demo" * 16,
last_ping=datetime.now(),
cpu_cores=42,
cpu_frequency=42e8,
diff --git a/arkindex/ponos/api.py b/arkindex/ponos/api.py
index 91f6d607dac16a1e63429ccd1baee27a6c4a2317..7616e944decdb0f56e25ef1a5e5f537a005d23dd 100644
--- a/arkindex/ponos/api.py
+++ b/arkindex/ponos/api.py
@@ -12,7 +12,7 @@ from rest_framework.generics import CreateAPIView, ListCreateAPIView, RetrieveUp
from rest_framework.response import Response
from rest_framework.views import APIView
-from arkindex.ponos.models import FINAL_STATES, Artifact, State, Task, task_token_default
+from arkindex.ponos.models import FINAL_STATES, Artifact, State, Task, token_default
from arkindex.ponos.permissions import (
IsAgentOrArtifactGuest,
IsAgentOrTaskGuest,
@@ -234,7 +234,7 @@ class TaskRestart(ProcessACLMixin, CreateAPIView):
copy.id = uuid.uuid4()
copy.slug = basename
copy.state = State.Pending
- copy.token = task_token_default()
+ copy.token = token_default()
copy.agent_id = None
copy.gpu_id = None
copy.started = None
diff --git a/arkindex/ponos/migrations/0001_initial.py b/arkindex/ponos/migrations/0001_initial.py
index 7a72645956ffc490e65c3d2f7d6cfc2a68733cd6..0c9d9a237838fc0c46cff5eca9b76cf9f9c70231 100644
--- a/arkindex/ponos/migrations/0001_initial.py
+++ b/arkindex/ponos/migrations/0001_initial.py
@@ -114,7 +114,7 @@ class Migration(migrations.Migration):
("updated", models.DateTimeField(auto_now=True)),
("expiry", models.DateTimeField(default=arkindex.ponos.models.expiry_default)),
("extra_files", django.contrib.postgres.fields.hstore.HStoreField(default=dict, blank=True)),
- ("token", models.CharField(default=arkindex.ponos.models.task_token_default, max_length=52, unique=True)),
+ ("token", models.CharField(default=arkindex.ponos.models.token_default, max_length=52, unique=True)),
("agent", models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="tasks", to="ponos.agent")),
("gpu", models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="tasks", to="ponos.gpu")),
("image_artifact", models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="tasks_using_image", to="ponos.artifact")),
diff --git a/arkindex/ponos/migrations/0004_index_cleanup.py b/arkindex/ponos/migrations/0004_index_cleanup.py
index 6e25ec7c743706c0bf9aa8b530e3dfc9834fbf6c..939e2531189a281d2db82afd1cb4468d3c226bef 100644
--- a/arkindex/ponos/migrations/0004_index_cleanup.py
+++ b/arkindex/ponos/migrations/0004_index_cleanup.py
@@ -3,7 +3,7 @@
from django.core.validators import RegexValidator
from django.db import migrations, models
-from arkindex.ponos.models import generate_seed, task_token_default
+from arkindex.ponos.models import generate_seed, token_default
class Migration(migrations.Migration):
@@ -89,7 +89,7 @@ class Migration(migrations.Migration):
model_name="task",
name="token",
field=models.CharField(
- default=task_token_default,
+ default=token_default,
max_length=52,
),
),
diff --git a/arkindex/ponos/migrations/0015_agent_token.py b/arkindex/ponos/migrations/0015_agent_token.py
new file mode 100644
index 0000000000000000000000000000000000000000..b3f118b8f0fdb2ab6ab3ed8bfa197768dab6541a
--- /dev/null
+++ b/arkindex/ponos/migrations/0015_agent_token.py
@@ -0,0 +1,38 @@
+# Generated by Django 5.0.8 on 2025-02-17 13:50
+
+from django.db import migrations, models
+
+from arkindex.ponos.models import token_default
+
+
+def add_agent_tokens(apps, schema_editor):
+ Agent = apps.get_model("ponos", "Agent")
+ to_update = []
+ for agent in Agent.objects.filter(token=None).only("id").iterator():
+ agent.token = token_default()
+ to_update.append(agent)
+ Agent.objects.bulk_update(to_update, ["token"], batch_size=100)
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("ponos", "0014_task_task_finished_requires_final_state"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="agent",
+ name="token",
+ field=models.CharField(
+ max_length=52,
+ # Make the field temporarily nullable and not unique, so that we can
+ # fill the tokens on existing agents before adding the constraints.
+ null=True,
+ ),
+ ),
+ migrations.RunPython(
+ add_agent_tokens,
+ reverse_code=migrations.RunPython.noop,
+ ),
+ ]
diff --git a/arkindex/ponos/migrations/0016_agent_token_constraints.py b/arkindex/ponos/migrations/0016_agent_token_constraints.py
new file mode 100644
index 0000000000000000000000000000000000000000..c0ae4fd40253090576dfbb165f886627a411531b
--- /dev/null
+++ b/arkindex/ponos/migrations/0016_agent_token_constraints.py
@@ -0,0 +1,24 @@
+# Generated by Django 5.0.8 on 2025-02-17 14:26
+
+from django.db import migrations, models
+
+import arkindex.ponos.models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("ponos", "0015_agent_token"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="agent",
+ name="token",
+ field=models.CharField(default=arkindex.ponos.models.token_default, max_length=52),
+ ),
+ migrations.AddConstraint(
+ model_name="agent",
+ constraint=models.UniqueConstraint(models.F("token"), name="unique_agent_token"),
+ ),
+ ]
diff --git a/arkindex/ponos/migrations/0017_remove_agents.py b/arkindex/ponos/migrations/0017_remove_agents.py
new file mode 100644
index 0000000000000000000000000000000000000000..87be71102c71d202b8518e70c84b4bcf7a8e91c1
--- /dev/null
+++ b/arkindex/ponos/migrations/0017_remove_agents.py
@@ -0,0 +1,38 @@
+# Generated by Django 5.0.8 on 2025-02-18 11:48
+
+from django.core.management.base import CommandError
+from django.db import migrations
+
+from arkindex.ponos.models import State
+
+
+def remove_agents(apps, schema_editor):
+ Agent = apps.get_model("ponos", "Agent")
+ GPU = apps.get_model("ponos", "GPU")
+ Task = apps.get_model("ponos", "Task")
+
+ if Task.objects.exclude(agent=None, gpu=None).filter(state=State.Running).exists():
+ raise CommandError(
+ "All existing Ponos agents and GPUs are about to be deleted, but some are currently assigned to running tasks.\n"
+ "Wait for the tasks to finish or stop them before running this migration."
+ )
+
+ Task.objects.exclude(gpu=None).update(gpu=None)
+ Task.objects.exclude(agent=None).update(agent=None)
+ GPU.objects.all().delete()
+ Agent.objects.all().delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("ponos", "0016_agent_token_constraints"),
+ ]
+
+ operations = [
+ migrations.RunPython(
+ remove_agents,
+ reverse_code=migrations.RunPython.noop,
+ elidable=True,
+ ),
+ ]
diff --git a/arkindex/ponos/migrations/0018_agent_fingerprint.py b/arkindex/ponos/migrations/0018_agent_fingerprint.py
new file mode 100644
index 0000000000000000000000000000000000000000..75c7a285dcac1bda2553032fd80affd2bdba4374
--- /dev/null
+++ b/arkindex/ponos/migrations/0018_agent_fingerprint.py
@@ -0,0 +1,32 @@
+# Generated by Django 5.0.8 on 2025-02-18 11:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("ponos", "0017_remove_agents"),
+ ]
+
+ operations = [
+ # Set a default value for the public key so that the migration is reversible
+ migrations.AlterField(
+ model_name="agent",
+ name="public_key",
+ field=models.TextField(default=""),
+ ),
+ migrations.RemoveField(
+ model_name="agent",
+ name="public_key",
+ ),
+ migrations.AddField(
+ model_name="agent",
+ name="fingerprint",
+ field=models.CharField(max_length=64),
+ ),
+ migrations.AddConstraint(
+ model_name="agent",
+ constraint=models.UniqueConstraint(models.F("fingerprint"), name="unique_agent_fingerprint"),
+ ),
+ ]
diff --git a/arkindex/ponos/models.py b/arkindex/ponos/models.py
index e77e10df10c44acead36174613d930f84b16ae54..2211c3e89d32619418d62835a62a333ccabc65e8 100644
--- a/arkindex/ponos/models.py
+++ b/arkindex/ponos/models.py
@@ -34,6 +34,15 @@ def gen_nonce(size=16):
return urandom(size)
+def token_default():
+ """
+ Default value for task and agent tokens.
+
+ :rtype: str
+ """
+ return base64.encodebytes(uuid.uuid4().bytes + uuid.uuid4().bytes).strip().decode("utf-8")
+
+
class Farm(models.Model):
"""
A group of agents, whose ID and seed can be used to register new agents
@@ -80,7 +89,7 @@ class Agent(models.Model):
created = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
farm = models.ForeignKey(Farm, on_delete=models.PROTECT)
- public_key = models.TextField()
+ fingerprint = models.CharField(max_length=64)
mode = EnumField(AgentMode, default=AgentMode.Docker, max_length=20)
accept_tasks = models.BooleanField(default=True)
@@ -95,12 +104,26 @@ class Agent(models.Model):
ram_load = models.FloatField(null=True, blank=True)
last_ping = models.DateTimeField(editable=False)
+ token = models.CharField(
+ default=token_default,
+ # The token generation always returns 52 characters
+ max_length=52,
+ )
+
class Meta:
constraints = [
models.CheckConstraint(
check=Q(mode=AgentMode.Slurm) | Q(cpu_cores__isnull=False, cpu_frequency__isnull=False, ram_total__isnull=False),
name="slurm_or_hardware_requirements",
),
+ models.UniqueConstraint(
+ "token",
+ name="unique_agent_token",
+ ),
+ models.UniqueConstraint(
+ "fingerprint",
+ name="unique_agent_fingerprint",
+ )
]
def __str__(self) -> str:
@@ -224,15 +247,6 @@ def expiry_default():
return timezone.now() + timedelta(days=settings.PONOS_TASK_EXPIRY)
-def task_token_default():
- """
- Default value for Task.token.
-
- :rtype: str
- """
- return base64.encodebytes(uuid.uuid4().bytes + uuid.uuid4().bytes).strip().decode("utf-8")
-
-
class TaskLogs(S3FileMixin):
s3_bucket = settings.PONOS_S3_LOGS_BUCKET
@@ -357,7 +371,7 @@ class Task(models.Model):
extra_files = HStoreField(default=dict, blank=True)
token = models.CharField(
- default=task_token_default,
+ default=token_default,
# The token generation always returns 52 characters
max_length=52,
)
diff --git a/arkindex/ponos/serializer_fields.py b/arkindex/ponos/serializer_fields.py
index 1bd83378c89a4b1c2b71abdfe3d8a48ebb40988a..80eef856a802795d60cf3aa85a77d2e3276579fa 100644
--- a/arkindex/ponos/serializer_fields.py
+++ b/arkindex/ponos/serializer_fields.py
@@ -1,61 +1,8 @@
-import base64
-from cryptography.exceptions import UnsupportedAlgorithm
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives.asymmetric import ec
-from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat, load_pem_public_key
-from rest_framework import serializers
from arkindex.ponos.utils import get_process_from_task_auth
-class PublicKeyField(serializers.CharField):
- """
- An EC public key, serialized in PEM format
- """
-
- default_error_messages = {
- "invalid_pem": "Incorrect PEM data",
- "unsupported_algorithm": "Key algorithm is not supported",
- "not_ec": "Key is not an EC public key",
- }
-
- def to_internal_value(self, data) -> ec.EllipticCurvePublicKey:
- data = super().to_internal_value(data)
- try:
- key = load_pem_public_key(
- data.encode("utf-8"),
- backend=default_backend(),
- )
- except ValueError:
- self.fail("invalid_pem")
- except UnsupportedAlgorithm:
- self.fail("unsupported_algorithm")
-
- if not isinstance(key, ec.EllipticCurvePublicKey):
- self.fail("not_ec")
-
- return key
-
- def to_representation(self, key: ec.EllipticCurvePublicKey) -> str:
- return key.public_bytes(
- Encoding.PEM,
- PublicFormat.SubjectPublicKeyInfo,
- ).decode("utf-8")
-
-
-class Base64Field(serializers.CharField):
- """
- A base64-encoded bytestring.
- """
-
- def to_internal_value(self, data) -> bytes:
- return base64.b64decode(super().to_internal_value(data))
-
- def to_representation(self, obj: bytes) -> str:
- return base64.b64encode(obj)
-
-
class CurrentProcessDefault:
"""
Use the process of the currently authenticated task as a default value.
diff --git a/arkindex/ponos/tests/rq/test_trigger.py b/arkindex/ponos/tests/rq/test_trigger.py
index 406dd77a8987f02ae48552a5d71bd11afbf4360e..e65f9093e8b4d60e7e1042300825f932941b44f2 100644
--- a/arkindex/ponos/tests/rq/test_trigger.py
+++ b/arkindex/ponos/tests/rq/test_trigger.py
@@ -21,8 +21,8 @@ class TestTrigger(FixtureTestCase):
)
cls.worker_version1 = WorkerVersion.objects.get(worker__slug="reco")
cls.worker_version2 = WorkerVersion.objects.get(worker__slug="dla")
- cls.run1 = cls.process.worker_runs.create(version=cls.worker_version1, ttl=0)
- cls.run2 = cls.process.worker_runs.create(version=cls.worker_version2, parents=[cls.run1.id], ttl=0)
+ cls.run1 = cls.process.worker_runs.create(version=cls.worker_version1)
+ cls.run2 = cls.process.worker_runs.create(version=cls.worker_version2, parents=[cls.run1.id])
@override_settings(PONOS_RQ_EXECUTION=True)
@patch("arkindex.ponos.tasks.run_task_rq.delay")
diff --git a/arkindex/ponos/tests/tasks/test_partial_update.py b/arkindex/ponos/tests/tasks/test_partial_update.py
index f4c8adbe30a6260adb86d1c18ff89f9943af60e6..45646f7edda745d8bff9306cabef7a97d8bc626a 100644
--- a/arkindex/ponos/tests/tasks/test_partial_update.py
+++ b/arkindex/ponos/tests/tasks/test_partial_update.py
@@ -29,6 +29,7 @@ class TestTaskPartialUpdate(FixtureAPITestCase):
cls.docker_agent = Agent.objects.create(
mode=AgentMode.Docker,
farm=cls.farm,
+ fingerprint="a" * 64,
last_ping=datetime.now(timezone.utc),
cpu_cores=42,
cpu_frequency=42e8,
@@ -37,6 +38,7 @@ class TestTaskPartialUpdate(FixtureAPITestCase):
cls.slurm_agent = Agent.objects.create(
mode=AgentMode.Slurm,
farm=cls.farm,
+ fingerprint="b" * 64,
last_ping=datetime.now(timezone.utc),
)
diff --git a/arkindex/ponos/tests/tasks/test_retrieve.py b/arkindex/ponos/tests/tasks/test_retrieve.py
index 5fe9581a411349d3f0a58d075a9416235a9058a1..7643f6d8531aee6db295bb0eb019ca994420135b 100644
--- a/arkindex/ponos/tests/tasks/test_retrieve.py
+++ b/arkindex/ponos/tests/tasks/test_retrieve.py
@@ -51,6 +51,7 @@ class TestTaskRetrieve(FixtureAPITestCase):
cls.docker_agent = Agent.objects.create(
mode=AgentMode.Docker,
farm=cls.farm,
+ fingerprint="a" * 64,
last_ping=datetime.now(timezone.utc),
cpu_cores=42,
cpu_frequency=42e8,
@@ -59,6 +60,7 @@ class TestTaskRetrieve(FixtureAPITestCase):
cls.slurm_agent = Agent.objects.create(
mode=AgentMode.Slurm,
farm=cls.farm,
+ fingerprint="b" * 64,
last_ping=datetime.now(timezone.utc),
)
diff --git a/arkindex/ponos/tests/tasks/test_update.py b/arkindex/ponos/tests/tasks/test_update.py
index 762a7fd31cac76e51dfad964cc3aee684902708c..bd93e8766eb12cc249492014a5b600d61f0c0082 100644
--- a/arkindex/ponos/tests/tasks/test_update.py
+++ b/arkindex/ponos/tests/tasks/test_update.py
@@ -39,14 +39,16 @@ class TestTaskUpdate(FixtureAPITestCase):
cls.docker_agent = Agent.objects.create(
mode=AgentMode.Docker,
farm=cls.farm,
+ fingerprint="a" * 64,
last_ping=datetime.now(timezone.utc),
cpu_cores=42,
cpu_frequency=42e8,
- ram_total=42e3
+ ram_total=42e3,
)
cls.slurm_agent = Agent.objects.create(
mode=AgentMode.Slurm,
farm=cls.farm,
+ fingerprint="b" * 64,
last_ping=datetime.now(timezone.utc),
)
@@ -131,8 +133,8 @@ class TestTaskUpdate(FixtureAPITestCase):
corpus=self.corpus,
activity_state=ActivityState.Ready
)
- init_run = test_process.worker_runs.create(version=WorkerVersion.objects.get(worker__slug="initialisation"), ttl=0)
- test_run = test_process.worker_runs.create(version=self.recognizer, parents=[init_run.id], ttl=0)
+ init_run = test_process.worker_runs.create(version=WorkerVersion.objects.get(worker__slug="initialisation"))
+ test_run = test_process.worker_runs.create(version=self.recognizer, parents=[init_run.id])
test_process.run()
@@ -227,8 +229,8 @@ class TestTaskUpdate(FixtureAPITestCase):
activity_state=ActivityState.Ready
)
init_version = WorkerVersion.objects.get(worker__slug="initialisation")
- init_run = test_process.worker_runs.create(version=init_version, ttl=0)
- test_run = test_process.worker_runs.create(version=self.recognizer, parents=[init_run.id], ttl=0)
+ init_run = test_process.worker_runs.create(version=init_version)
+ test_run = test_process.worker_runs.create(version=self.recognizer, parents=[init_run.id])
test_process.run()
@@ -300,7 +302,7 @@ class TestTaskUpdate(FixtureAPITestCase):
chunks=2,
activity_state=ActivityState.Ready
)
- test_run = test_process.worker_runs.create(version=self.recognizer, ttl=0)
+ test_run = test_process.worker_runs.create(version=self.recognizer)
test_process.run()
@@ -402,18 +404,15 @@ class TestTaskUpdate(FixtureAPITestCase):
)
test_run_1 = test_process.worker_runs.create(
version=self.recognizer,
- ttl=0,
)
test_run = test_process.worker_runs.create(
version=self.recognizer,
model_version_id=test_model_version.id,
- ttl=0,
)
test_run_2 = test_process.worker_runs.create(
version=self.recognizer,
model_version_id=test_model_version.id,
configuration_id=test_configuration.id,
- ttl=0,
)
test_process.run()
diff --git a/arkindex/ponos/tests/test_models.py b/arkindex/ponos/tests/test_models.py
index a94dc7373408de5e8e8fe786756c352c3bc9dc3c..ce83600bac8aa71190e5eddd065a08c9636b7b4c 100644
--- a/arkindex/ponos/tests/test_models.py
+++ b/arkindex/ponos/tests/test_models.py
@@ -145,7 +145,7 @@ class TestModels(FixtureAPITestCase):
hostname="agent_smith",
cpu_cores=2,
cpu_frequency=4.2e9,
- public_key="",
+ fingerprint="a" * 64,
farm=self.farm,
ram_total=2e9,
last_ping=timezone.now(),
@@ -159,7 +159,7 @@ class TestModels(FixtureAPITestCase):
def test_agent_slurm_mode(self):
Agent.objects.create(
hostname="agent_smith",
- public_key="",
+ fingerprint="a" * 64,
farm=self.farm,
last_ping=timezone.now(),
mode=AgentMode.Slurm.value
@@ -174,7 +174,7 @@ class TestModels(FixtureAPITestCase):
hostname="agent_smith",
cpu_cores=2,
cpu_frequency=4.2e9,
- public_key="",
+ fingerprint="a" * 64,
farm=self.farm,
ram_total=2e9,
last_ping=timezone.now(),
@@ -195,7 +195,7 @@ class TestModels(FixtureAPITestCase):
"params": {
"hostname": "agent_smith",
"cpu_frequency": 4.2e9,
- "public_key": "",
+ "fingerprint": "a" * 64,
"farm": self.farm,
"ram_total": 2e9,
"last_ping": timezone.now(),
@@ -210,7 +210,7 @@ class TestModels(FixtureAPITestCase):
"params": {
"hostname": "agent_smith",
"cpu_cores": 2,
- "public_key": "",
+ "fingerprint": "a" * 64,
"farm": self.farm,
"ram_total": 2e9,
"last_ping": timezone.now(),
@@ -226,7 +226,7 @@ class TestModels(FixtureAPITestCase):
"hostname": "agent_smith",
"cpu_cores": 2,
"cpu_frequency": 4.2e9,
- "public_key": "",
+ "fingerprint": "a" * 64,
"farm": self.farm,
"last_ping": timezone.now(),
"ram_load": 0.49,
@@ -241,7 +241,7 @@ class TestModels(FixtureAPITestCase):
"hostname": "agent_smith",
"cpu_cores": None,
"cpu_frequency": None,
- "public_key": "",
+ "fingerprint": "a" * 64,
"farm": self.farm,
"ram_total": None,
"last_ping": timezone.now(),
@@ -257,7 +257,7 @@ class TestModels(FixtureAPITestCase):
"hostname": "agent_smith",
"cpu_cores": 2,
"cpu_frequency": 4.2e9,
- "public_key": "",
+ "fingerprint": "a" * 64,
"farm": self.farm,
"ram_total": 2e9,
"last_ping": timezone.now(),
@@ -271,7 +271,7 @@ class TestModels(FixtureAPITestCase):
"mode": AgentMode.Slurm,
"params": {
"hostname": "agent_smith",
- "public_key": "",
+ "fingerprint": "b" * 64,
"farm": self.farm,
"last_ping": timezone.now(),
"ram_load": 0.49,
diff --git a/arkindex/process/builder.py b/arkindex/process/builder.py
index 5a1c347a32e95b8bc3fc0234e71338388c860c27..9756ba6944a7b36fe0598d6d96626dd810890499 100644
--- a/arkindex/process/builder.py
+++ b/arkindex/process/builder.py
@@ -13,7 +13,7 @@ from django.utils.functional import cached_property
from rest_framework.exceptions import ValidationError
from arkindex.images.models import ImageServer
-from arkindex.ponos.models import GPU, Task, task_token_default
+from arkindex.ponos.models import GPU, Task, token_default
class ProcessBuilder:
@@ -79,7 +79,7 @@ class ProcessBuilder:
Build a Task with default attributes and add it to the current stack.
Depth is not set while building individual Task instances.
"""
- token = task_token_default()
+ token = token_default()
env = {
**self.base_env,
@@ -102,7 +102,7 @@ class ProcessBuilder:
shm_size=shm_size,
extra_files=extra_files,
worker_run=worker_run,
- ttl=worker_run.ttl,
+ ttl=self.process.corpus.applied_maximum_task_ttl,
)
)
@@ -218,10 +218,7 @@ class ProcessBuilder:
from arkindex.process.models import ArkindexFeature, WorkerVersion
import_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.FileImport)
- worker_run, _ = self.process.worker_runs.get_or_create(
- version=import_version,
- defaults={"ttl": self.process.corpus.applied_maximum_task_ttl},
- )
+ worker_run, _ = self.process.worker_runs.get_or_create(version=import_version)
self._build_task(
slug="import_files",
@@ -250,7 +247,6 @@ class ProcessBuilder:
version=ingest_version,
model_version=None,
configuration=worker_configuration,
- defaults={"ttl": self.process.corpus.applied_maximum_task_ttl},
)
env = {
@@ -294,10 +290,7 @@ class ProcessBuilder:
worker_runs.remove(initialisation_worker_run)
# If there is no elements initialisation worker run in the process, create one
else:
- initialisation_worker_run = self.process.worker_runs.create(
- version=init_elements_version,
- ttl=self.process.corpus.applied_maximum_task_ttl,
- )
+ initialisation_worker_run = self.process.worker_runs.create(version=init_elements_version)
# Link all parentless worker runs to the initialisation worker run
no_parents = [run for run in worker_runs if not len(run.parents)]
for run in no_parents:
diff --git a/arkindex/process/management/commands/fake_worker_run.py b/arkindex/process/management/commands/fake_worker_run.py
index e1e9e1d72fc2ef648f2e4d4c3967507e6401b8d6..7533141bf5c4f67b4875de194986f1cbda085dcb 100644
--- a/arkindex/process/management/commands/fake_worker_run.py
+++ b/arkindex/process/management/commands/fake_worker_run.py
@@ -33,10 +33,7 @@ class Command(BaseCommand):
else:
self.stdout.write(f"Using existing local process {process.id}")
- worker_run, created = process.worker_runs.get_or_create(
- version=worker_version,
- defaults={"ttl": 0},
- )
+ worker_run, created = process.worker_runs.get_or_create(version=worker_version)
if created:
self.stdout.write(self.style.SUCCESS(f"Created WorkerRun {worker_run.id}"))
diff --git a/arkindex/process/migrations/0049_remove_workerrun_ttl.py b/arkindex/process/migrations/0049_remove_workerrun_ttl.py
new file mode 100644
index 0000000000000000000000000000000000000000..fad0275417963d229bec4151d6d5c7ac95ebf00c
--- /dev/null
+++ b/arkindex/process/migrations/0049_remove_workerrun_ttl.py
@@ -0,0 +1,17 @@
+# Generated by Django 5.0.8 on 2025-02-13 14:07
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("process", "0048_worker_cost_fields"),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name="workerrun",
+ name="ttl",
+ ),
+ ]
diff --git a/arkindex/process/models.py b/arkindex/process/models.py
index 3d5f00821cddb6d5eb9a65ab0173c4b0870499a6..16e89ce50961e7f1c0e1b2c7b42d5ba1ff0e23b3 100644
--- a/arkindex/process/models.py
+++ b/arkindex/process/models.py
@@ -15,7 +15,7 @@ from enumfields import Enum, EnumField
import pgtrigger
from arkindex.documents.models import Classification, Element
-from arkindex.ponos.models import FINAL_STATES, STATES_ORDERING, State, Task, task_token_default
+from arkindex.ponos.models import FINAL_STATES, STATES_ORDERING, State, Task, token_default
from arkindex.process.builder import ProcessBuilder
from arkindex.process.managers import (
ActivityManager,
@@ -342,18 +342,6 @@ class Process(IndexableModel):
if run.version.is_init_elements():
continue
- if new_process.mode == ProcessMode.Template or new_process.corpus.applied_maximum_task_ttl == 0:
- # When the destination process is a template, we do not apply any limits and copy the original TTL.
- # The limits will be applied only when applying a template.
- # With other modes, when the corpus has no limits, we also just use the original TTL.
- ttl = run.ttl
- elif run.ttl == 0:
- # The original TTL was infinite and there is a limit, so we use the corpus' limit
- ttl = new_process.corpus.applied_maximum_task_ttl
- else:
- # Apply the limit normally when no infinity is involved
- ttl = min(run.ttl, new_process.corpus.applied_maximum_task_ttl)
-
# Create a new WorkerRun with same version, configuration and parents.
new_run = WorkerRun(
process=new_process,
@@ -369,7 +357,6 @@ class Process(IndexableModel):
else run.version.gpu_usage == FeatureUsage.Required
),
summary=run.summary,
- ttl=ttl,
)
# Save the correspondence between this process' worker_run and the new one
new_runs[run.id] = new_run
@@ -736,6 +723,7 @@ class ArkindexFeature(Enum):
InitElements = "init_elements"
FileImport = "file_import"
S3Ingest = "s3_ingest"
+ DatasetExtractor = "dataset_extractor"
# When adding a new export worker, don't forget to also update the ExportFormat enum and the
# FEATURE_FORMAT_MAP dictionary which maps export formats to arkindex features
ExportPDF = "pdf_export"
@@ -938,10 +926,6 @@ class WorkerRun(models.Model):
updated = models.DateTimeField(auto_now=True)
has_results = models.BooleanField(default=False)
use_gpu = models.BooleanField(default=False)
- ttl = models.PositiveIntegerField(
- verbose_name="TTL",
- help_text="Maximum time-to-live for tasks created from this WorkerRun, in seconds. 0 means infinite.",
- )
objects = WorkerRunManager()
@@ -1028,7 +1012,7 @@ class WorkerRun(models.Model):
)
task_env = env.copy()
- token = task_token_default()
+ token = token_default()
task_env["ARKINDEX_TASK_TOKEN"] = token
task_env["TASK_ELEMENTS"] = elements_path
task_env["ARKINDEX_WORKER_RUN_ID"] = str(self.id)
@@ -1057,7 +1041,7 @@ class WorkerRun(models.Model):
worker_run=self,
extra_files=extra_files,
requires_gpu=requires_gpu,
- ttl=self.ttl,
+ ttl=process.corpus.applied_maximum_task_ttl,
)
return task, parents
diff --git a/arkindex/process/serializers/imports.py b/arkindex/process/serializers/imports.py
index 71bdbf729fdeb74b28596134658d7a946fcdce3c..9a345d520c403ca3be27bf5a43a04c8f591242cb 100644
--- a/arkindex/process/serializers/imports.py
+++ b/arkindex/process/serializers/imports.py
@@ -612,7 +612,6 @@ class ExportProcessSerializer(ProcessDetailsSerializer):
export_process.worker_runs.create(
version=worker_version,
configuration=worker_configuration,
- ttl=corpus.applied_maximum_task_ttl,
)
# Start the export process
export_process.run()
diff --git a/arkindex/process/serializers/worker_runs.py b/arkindex/process/serializers/worker_runs.py
index eb6ab16138af0062854135348292a6422ea330b9..9f6146db0405ff4537e4e34da6f0ee2bb9a8fac0 100644
--- a/arkindex/process/serializers/worker_runs.py
+++ b/arkindex/process/serializers/worker_runs.py
@@ -1,7 +1,6 @@
from collections import defaultdict
from textwrap import dedent
-from django.core.validators import MaxValueValidator, MinValueValidator
from rest_framework import serializers
from rest_framework.exceptions import ValidationError
@@ -20,25 +19,6 @@ from arkindex.process.serializers.workers import WorkerConfigurationSerializer,
from arkindex.training.models import ModelVersion, ModelVersionState
from arkindex.training.serializers import ModelVersionLightSerializer
-# To prevent each element worker to retrieve contextual information
-# (process, worker version, model version…) with extra GET requests, we
-# do serialize all the related information on WorkerRun serializers.
-
-def _ttl_from_corpus(serializer_field) -> int:
- if isinstance(serializer_field.parent.instance, WorkerRun):
- process = serializer_field.parent.instance.process
- else:
- process = serializer_field.context["process"]
-
- # This function may be called on a local process, which does not have a corpus, even if the API blocks them later on
- if process.mode == ProcessMode.Local:
- return 0
-
- return process.corpus.applied_maximum_task_ttl
-
-
-_ttl_from_corpus.requires_context = True
-
class WorkerRunSerializer(serializers.ModelSerializer):
@@ -96,15 +76,6 @@ class WorkerRunSerializer(serializers.ModelSerializer):
"Only a configuration of the WorkerVersion's worker may be set.",
)
- ttl = serializers.IntegerField(
- default=_ttl_from_corpus,
- help_text=dedent("""
- Maximum time-to-live for tasks created from this WorkerRun, in seconds. `0` means infinite.
-
- Defaults to, and cannot exceed, the `maximum_task_ttl` on the corpus of the process.
- """),
- )
-
process = ProcessLightSerializer(read_only=True)
class Meta:
@@ -121,7 +92,6 @@ class WorkerRunSerializer(serializers.ModelSerializer):
"model_version",
"summary",
"use_gpu",
- "ttl",
)
read_only_fields = (
"id",
@@ -145,24 +115,6 @@ class WorkerRunSerializer(serializers.ModelSerializer):
return self.instance.process
return self.context["process"]
- def validate_ttl(self, value) -> int:
- if self._process.mode == ProcessMode.Local:
- # Don't validate anything, the endpoint will not work on local processes anyway
- return value
-
- corpus_ttl = self._process.corpus.applied_maximum_task_ttl
- if corpus_ttl == 0:
- # Allow infinity, and limit to the maximum value of an integer field
- min_ttl, max_ttl = 0, 2147483647
- else:
- # Restrict the maximum TTL further using the limit
- min_ttl, max_ttl = 1, corpus_ttl
-
- MinValueValidator(min_ttl)(value)
- MaxValueValidator(max_ttl)(value)
-
- return value
-
def validate(self, data):
data = super().validate(data)
errors = defaultdict(list)
@@ -296,8 +248,6 @@ class UserWorkerRunSerializer(serializers.ModelSerializer):
queryset=WorkerConfiguration.objects.all(),
style={"base_template": "input.html"},
)
- # Default value for the TTL, as the local process does not have a corpus and the run will never actually run
- ttl = serializers.HiddenField(default=0)
def validate_worker_version_id(self, worker_version_id):
# Check that the worker version exists
@@ -369,7 +319,6 @@ class UserWorkerRunSerializer(serializers.ModelSerializer):
"worker_version_id",
"model_version_id",
"configuration_id",
- "ttl",
)
diff --git a/arkindex/process/tests/commands/test_fake_worker_run.py b/arkindex/process/tests/commands/test_fake_worker_run.py
index 97055247eafd7d759f77c860d379d89ed153e55e..68020a16c97ce9b24f4972cfc5a5d97520a9a0a8 100644
--- a/arkindex/process/tests/commands/test_fake_worker_run.py
+++ b/arkindex/process/tests/commands/test_fake_worker_run.py
@@ -64,7 +64,7 @@ class TestFakeWorkerRun(FixtureTestCase):
def test_existing_worker_run(self):
process = Process.objects.get(mode=ProcessMode.Local, creator=self.user)
- worker_run = process.worker_runs.create(version=self.worker_version, ttl=0)
+ worker_run = process.worker_runs.create(version=self.worker_version)
self.assertEqual(process.worker_runs.count(), 2)
output = self.fake_worker_run(["--user", str(self.user.id), "--worker-version", str(self.worker_version.id)])
diff --git a/arkindex/process/tests/process/test_clear.py b/arkindex/process/tests/process/test_clear.py
index 0593a54ce403a38d4fda7f0e1e42e70d112e9907..83ad163bc0ab0752b9c2591a4364038c29189442 100644
--- a/arkindex/process/tests/process/test_clear.py
+++ b/arkindex/process/tests/process/test_clear.py
@@ -22,11 +22,9 @@ class TestProcessClear(FixtureAPITestCase):
)
cls.process.worker_runs.create(
version=WorkerVersion.objects.get(worker__slug="reco"),
- ttl=0,
)
cls.process.worker_runs.create(
version=WorkerVersion.objects.get(worker__slug="dla"),
- ttl=0,
)
def test_clear(self):
diff --git a/arkindex/process/tests/process/test_create.py b/arkindex/process/tests/process/test_create.py
index 3f1d6aaec3e07f6f23e8d5c263d69f6340a96984..17278e26e37618d8e2523b8fd4b242ec853fbbd8 100644
--- a/arkindex/process/tests/process/test_create.py
+++ b/arkindex/process/tests/process/test_create.py
@@ -33,6 +33,7 @@ class TestCreateProcess(FixtureAPITestCase):
super().setUpTestData()
cls.agent = Agent.objects.create(
farm=Farm.objects.first(),
+ fingerprint="a" * 64,
hostname="claude",
cpu_cores=42,
cpu_frequency=1e15,
@@ -488,17 +489,14 @@ class TestCreateProcess(FixtureAPITestCase):
)
init_run = process_2.worker_runs.create(
version=self.init_elements_version,
- ttl=0,
)
run_1 = process_2.worker_runs.create(
version=self.version_1,
parents=[init_run.id],
- ttl=0,
)
run_2 = process_2.worker_runs.create(
version=self.version_2,
parents=[run_1.id],
- ttl=0,
)
self.assertFalse(process_2.tasks.exists())
@@ -589,7 +587,7 @@ class TestCreateProcess(FixtureAPITestCase):
dataset = self.corpus.datasets.first()
test_set = dataset.sets.get(name="test")
ProcessDatasetSet.objects.create(process=process, set=test_set)
- process.worker_runs.create(version=self.version_1, ttl=0)
+ process.worker_runs.create(version=self.version_1)
with self.assertNumQueries(9):
response = self.client.post(reverse("api:process-start", kwargs={"pk": str(process.id)}))
@@ -605,7 +603,7 @@ class TestCreateProcess(FixtureAPITestCase):
self.worker_1.save()
process = self.corpus.processes.create(creator=self.user, mode=ProcessMode.Workers)
- process.worker_runs.create(version=self.version_1, ttl=0)
+ process.worker_runs.create(version=self.version_1)
with self.assertNumQueries(9):
response = self.client.post(reverse("api:process-start", kwargs={"pk": str(process.id)}))
diff --git a/arkindex/process/tests/process/test_default_process_name.py b/arkindex/process/tests/process/test_default_process_name.py
index e496e7ff00af52728120e7d4e9876b8081ce70ca..00223e3e596084c99fc80c5aef295c316aa60b00 100644
--- a/arkindex/process/tests/process/test_default_process_name.py
+++ b/arkindex/process/tests/process/test_default_process_name.py
@@ -50,22 +50,18 @@ class TestProcessName(FixtureAPITestCase):
init_elements_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.InitElements)
init_run = self.workers_process.worker_runs.create(
version=init_elements_version,
- ttl=0,
)
self.workers_process.worker_runs.create(
version=self.recognizer,
parents=[init_run.id],
- ttl=0,
)
dla_run = self.workers_process.worker_runs.create(
version=self.dla,
parents=[init_run.id],
- ttl=0,
)
self.workers_process.worker_runs.create(
version=self.version_gpu,
parents=[dla_run.id],
- ttl=0,
)
self.workers_process.save()
@@ -95,41 +91,34 @@ class TestProcessName(FixtureAPITestCase):
init_elements_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.InitElements)
init_run = self.workers_process.worker_runs.create(
version=init_elements_version,
- ttl=0,
)
reco_run_1 = self.workers_process.worker_runs.create(
version=self.recognizer,
parents=[init_run.id],
- ttl=0,
)
reco_run_2 = self.workers_process.worker_runs.create(
version=self.recognizer,
configuration=self.reco_config_1,
parents=[reco_run_1.id],
- ttl=0,
)
self.workers_process.worker_runs.create(
version=self.recognizer,
configuration=self.reco_config_2,
parents=[reco_run_2.id],
- ttl=0,
)
dla_run_1 = self.workers_process.worker_runs.create(
version=self.dla,
parents=[init_run.id],
- ttl=0,
)
reco_run_4 = self.workers_process.worker_runs.create(
version=self.recognizer,
configuration=self.reco_config_3,
parents=[dla_run_1.id],
- ttl=0,
)
self.workers_process.worker_runs.create(
version=self.dla,
configuration=self.dla_config,
parents=[reco_run_4.id],
- ttl=0,
)
self.workers_process.save()
@@ -150,26 +139,23 @@ class TestProcessName(FixtureAPITestCase):
If the default process worker name is too long (len() > 250) it gets truncated
"""
init_elements_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.InitElements)
- init_run = self.workers_process.worker_runs.create(version=init_elements_version, ttl=0)
+ init_run = self.workers_process.worker_runs.create(version=init_elements_version)
# Update the recognizer worker's name so that it is long
self.recognizer.worker.name = "animula vagula blandula hospes comesque corporis quae nunc abibis in loca pallidula rigida nudula ne"
self.recognizer.worker.save()
reco_run_1 = self.workers_process.worker_runs.create(
version=self.recognizer,
parents=[init_run.id],
- ttl=0,
)
reco_run_2 = self.workers_process.worker_runs.create(
version=self.recognizer,
configuration=self.reco_config_1,
parents=[reco_run_1.id],
- ttl=0,
)
self.workers_process.worker_runs.create(
version=self.recognizer,
configuration=self.reco_config_2,
parents=[reco_run_2.id],
- ttl=0,
)
self.workers_process.save()
@@ -191,8 +177,8 @@ class TestProcessName(FixtureAPITestCase):
self.workers_process.name = "My process"
self.workers_process.save()
init_elements_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.InitElements)
- init_run = self.workers_process.worker_runs.create(version=init_elements_version, ttl=0)
- self.workers_process.worker_runs.create(version=self.recognizer, parents=[init_run.id], ttl=0)
+ init_run = self.workers_process.worker_runs.create(version=init_elements_version)
+ self.workers_process.worker_runs.create(version=self.recognizer, parents=[init_run.id])
self.assertEqual(self.workers_process.name, "My process")
builder = ProcessBuilder(process=self.workers_process)
diff --git a/arkindex/process/tests/process/test_destroy.py b/arkindex/process/tests/process/test_destroy.py
index b87856e50cc290f43f37fc638862e8f1f1c2c4d8..4bfe2959296185ddf5e47a2a39138523551ea2fb 100644
--- a/arkindex/process/tests/process/test_destroy.py
+++ b/arkindex/process/tests/process/test_destroy.py
@@ -111,7 +111,7 @@ class TestProcessDestroy(FixtureAPITestCase):
A process with worker runs linked to data cannot be deleted
"""
self.client.force_login(self.user)
- run = self.process.worker_runs.create(version=WorkerVersion.objects.first(), ttl=0)
+ run = self.process.worker_runs.create(version=WorkerVersion.objects.first())
page = self.corpus.elements.get(name="Volume 1, page 1r")
metadata = page.metadatas.get()
@@ -148,7 +148,7 @@ class TestProcessDestroy(FixtureAPITestCase):
"""
self.client.force_login(self.user)
- with self.assertNumQueries(19):
+ with self.assertNumQueries(20):
response = self.client.delete(reverse("api:process-details", kwargs={"pk": self.process.id}))
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
@@ -161,7 +161,7 @@ class TestProcessDestroy(FixtureAPITestCase):
"""
self.client.force_login(self.superuser)
- with self.assertNumQueries(19):
+ with self.assertNumQueries(20):
response = self.client.delete(reverse("api:process-details", kwargs={"pk": self.process.id}))
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
@@ -211,7 +211,7 @@ class TestProcessDestroy(FixtureAPITestCase):
state=WorkerActivityState.Processed,
)
- with self.assertNumQueries(20):
+ with self.assertNumQueries(21):
response = self.client.delete(reverse("api:process-details", kwargs={"pk": self.process.id}))
self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
diff --git a/arkindex/process/tests/process/test_process_artifact_download.py b/arkindex/process/tests/process/test_process_artifact_download.py
index 004e375f67c2128ae705e54e769c31ac1c7bedf6..394000cdb1449ef247a1270149824d02524cfeee 100644
--- a/arkindex/process/tests/process/test_process_artifact_download.py
+++ b/arkindex/process/tests/process/test_process_artifact_download.py
@@ -21,7 +21,7 @@ class TestProcessArtifactDownload(FixtureAPITestCase):
super().setUpTestData()
cls.pdf_export_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.ExportPDF)
cls.process = Process.objects.create(mode=ProcessMode.Export, creator=cls.user, corpus=cls.corpus)
- cls.worker_run = cls.process.worker_runs.create(version=cls.pdf_export_version, ttl=0)
+ cls.worker_run = cls.process.worker_runs.create(version=cls.pdf_export_version)
with patch("arkindex.process.tasks.initialize_activity.delay"):
cls.process.run()
cls.task = cls.process.tasks.get()
diff --git a/arkindex/process/tests/process/test_retry.py b/arkindex/process/tests/process/test_retry.py
index 9407e2e4c5b3dc82c5a697c9799d7742a0eb01c8..547868fcba410dd103368ab5df70f58f9ad9cd68 100644
--- a/arkindex/process/tests/process/test_retry.py
+++ b/arkindex/process/tests/process/test_retry.py
@@ -156,7 +156,6 @@ class TestProcessRetry(FixtureAPITestCase):
process.worker_runs.create(
version=self.recognizer,
model_version=self.model_version,
- ttl=0,
)
with patch("arkindex.process.tasks.initialize_activity.delay"):
process.run()
@@ -209,7 +208,7 @@ class TestProcessRetry(FixtureAPITestCase):
worker_type, _ = WorkerType.objects.get_or_create(slug=f"type_{slug}", display_name=slug.capitalize())
worker, _ = Worker.objects.get_or_create(slug=slug, defaults={"type": worker_type, "repository_url": "fake"})
version, _ = worker.versions.get_or_create(version=1, defaults={"state": WorkerVersionState.Available, "docker_image_iid": "test"})
- return version.worker_runs.create(process=process, ttl=0)
+ return version.worker_runs.create(process=process)
init_version = WorkerVersion.objects.get_by_feature(feature=ArkindexFeature.InitElements)
@@ -263,7 +262,7 @@ class TestProcessRetry(FixtureAPITestCase):
mode=ProcessMode.Files,
creator=self.user,
)
- process.worker_runs.create(version=self.recognizer, ttl=0)
+ process.worker_runs.create(version=self.recognizer)
process.tasks.create(state=State.Error, run=0, depth=0, ttl=0)
self.assertEqual(process.state, State.Error)
process.finished = timezone.now()
@@ -305,7 +304,6 @@ class TestProcessRetry(FixtureAPITestCase):
"iiif_base_url": self.imgsrv.url,
},
),
- ttl=0,
)
process.tasks.create(state=State.Error, run=0, depth=0, ttl=0)
self.assertEqual(process.state, State.Error)
@@ -350,7 +348,7 @@ class TestProcessRetry(FixtureAPITestCase):
pdf_export_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.ExportPDF)
self.client.force_login(self.user)
process = self.corpus.processes.create(mode=ProcessMode.Export, creator=self.user)
- process.worker_runs.create(version=pdf_export_version, ttl=0)
+ process.worker_runs.create(version=pdf_export_version)
process.tasks.create(state=State.Error, run=0, depth=0, ttl=0)
self.assertEqual(process.state, State.Error)
process.finished = timezone.now()
diff --git a/arkindex/process/tests/process/test_run.py b/arkindex/process/tests/process/test_run.py
index ef0ba31c9f3e15ecff7b2b368348f0056edbcc3b..99d606d8642dce04d3a71372ced67ce9c8341828 100644
--- a/arkindex/process/tests/process/test_run.py
+++ b/arkindex/process/tests/process/test_run.py
@@ -25,7 +25,7 @@ class TestProcessRun(FixtureTestCase):
)
@override_settings(PONOS_DEFAULT_ENV={"ARKINDEX_API_TOKEN": "testToken"})
- @patch("arkindex.process.builder.task_token_default")
+ @patch("arkindex.process.builder.token_default")
def test_pdf_import_run(self, token_mock):
process = self.corpus.processes.create(
creator=self.user,
@@ -65,8 +65,8 @@ class TestProcessRun(FixtureTestCase):
mode=ProcessMode.Workers,
)
token_mock.side_effect = [b"12345", b"78945"]
- init_run = process.worker_runs.create(version=self.init_worker_version, ttl=0)
- run = process.worker_runs.create(version=self.version_with_model, parents=[init_run.id], ttl=0)
+ init_run = process.worker_runs.create(version=self.init_worker_version)
+ run = process.worker_runs.create(version=self.version_with_model, parents=[init_run.id])
run.model_version = self.model_version
run.save()
with patch("arkindex.process.tasks.initialize_activity.delay"):
diff --git a/arkindex/process/tests/process/test_start.py b/arkindex/process/tests/process/test_start.py
index 4d96cadcbbe3cd43171bbd93a8d6809c70409f50..32d9e5b2dd33ef773caffffd24d6342102122027 100644
--- a/arkindex/process/tests/process/test_start.py
+++ b/arkindex/process/tests/process/test_start.py
@@ -117,7 +117,7 @@ class TestProcessStart(FixtureAPITestCase):
@override_settings(PUBLIC_HOSTNAME="https://darkindex.lol")
def test_without_required_model(self):
- self.workers_process.worker_runs.create(version=self.version_with_model, ttl=0)
+ self.workers_process.worker_runs.create(version=self.version_with_model)
self.client.force_login(self.user)
@@ -135,7 +135,7 @@ class TestProcessStart(FixtureAPITestCase):
@override_settings(PUBLIC_HOSTNAME="https://arkindex.localhost")
@patch("arkindex.project.triggers.process_tasks.initialize_activity.delay")
def test_with_required_model(self, activities_delay_mock):
- self.workers_process.worker_runs.create(version=self.version_with_model, model_version=self.model_version, ttl=0)
+ self.workers_process.worker_runs.create(version=self.version_with_model, model_version=self.model_version)
self.assertFalse(self.workers_process.tasks.exists())
self.client.force_login(self.user)
@@ -168,7 +168,7 @@ class TestProcessStart(FixtureAPITestCase):
)
def test_unavailable_worker_version(self):
- self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer)
self.recognizer.state = WorkerVersionState.Error
self.recognizer.save()
self.assertFalse(self.workers_process.tasks.exists())
@@ -185,7 +185,7 @@ class TestProcessStart(FixtureAPITestCase):
)
def test_unavailable_model_version(self):
- self.workers_process.worker_runs.create(version=self.recognizer, model_version=self.model_version, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer, model_version=self.model_version)
self.model_version.state = ModelVersionState.Error
self.model_version.save()
self.assertFalse(self.workers_process.tasks.exists())
@@ -202,7 +202,7 @@ class TestProcessStart(FixtureAPITestCase):
)
def test_archived_models(self):
- self.workers_process.worker_runs.create(version=self.recognizer, model_version=self.model_version, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer, model_version=self.model_version)
self.model.archived = timezone.now()
self.model.save()
self.assertFalse(self.workers_process.tasks.exists())
@@ -242,7 +242,6 @@ class TestProcessStart(FixtureAPITestCase):
version=self.recognizer,
configuration=None,
model_version=None,
- ttl=0,
)
# The other version is used with a configuration missing the required field
self.workers_process.worker_runs.create(
@@ -254,7 +253,6 @@ class TestProcessStart(FixtureAPITestCase):
},
),
model_version=None,
- ttl=0,
)
self.client.force_login(self.user)
@@ -278,8 +276,8 @@ class TestProcessStart(FixtureAPITestCase):
Default chunks, thumbnails and farm are used. Cache is disabled, and worker activities are enabled.
"""
init_elements_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.InitElements)
- init_run = self.workers_process.worker_runs.create(version=init_elements_version, ttl=0)
- run = self.workers_process.worker_runs.create(version=self.recognizer, parents=[init_run.id], ttl=0)
+ init_run = self.workers_process.worker_runs.create(version=init_elements_version)
+ run = self.workers_process.worker_runs.create(version=self.recognizer, parents=[init_run.id])
self.assertFalse(self.workers_process.tasks.exists())
self.client.force_login(self.user)
@@ -311,13 +309,13 @@ class TestProcessStart(FixtureAPITestCase):
def test_inconsistent_gpu_usages(self):
# The version's gpu_usage is Disabled, so the run's use_gpu is set to False
- self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer)
self.recognizer.gpu_usage = FeatureUsage.Required
self.recognizer.save()
self.dla.gpu_usage = FeatureUsage.Required
self.dla.save()
# The version's gpu_usage is Required, so the run's use_gpu is set to True
- self.workers_process.worker_runs.create(version=self.dla, ttl=0)
+ self.workers_process.worker_runs.create(version=self.dla)
self.dla.gpu_usage = FeatureUsage.Disabled
self.dla.save()
self.assertFalse(self.workers_process.tasks.exists())
@@ -338,7 +336,7 @@ class TestProcessStart(FixtureAPITestCase):
)
def test_dataset_requires_datasets(self):
- self.dataset_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.dataset_process.worker_runs.create(version=self.recognizer)
self.assertFalse(self.dataset_process.tasks.exists())
self.client.force_login(self.user)
@@ -355,7 +353,7 @@ class TestProcessStart(FixtureAPITestCase):
def test_dataset_requires_dataset_in_same_corpus(self):
test_set = self.other_dataset.sets.get(name="test")
ProcessDatasetSet.objects.create(process=self.dataset_process, set=test_set)
- self.dataset_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.dataset_process.worker_runs.create(version=self.recognizer)
self.assertFalse(self.dataset_process.tasks.exists())
self.client.force_login(self.user)
@@ -375,7 +373,7 @@ class TestProcessStart(FixtureAPITestCase):
test_set_2 = self.dataset2.sets.get(name="test")
ProcessDatasetSet.objects.create(process=self.dataset_process, set=test_set_1)
ProcessDatasetSet.objects.create(process=self.dataset_process, set=test_set_2)
- self.dataset_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.dataset_process.worker_runs.create(version=self.recognizer)
self.client.force_login(self.user)
@@ -400,7 +398,7 @@ class TestProcessStart(FixtureAPITestCase):
test_set_2 = self.other_dataset.sets.get(name="test")
ProcessDatasetSet.objects.create(process=self.dataset_process, set=test_set_1)
ProcessDatasetSet.objects.create(process=self.dataset_process, set=test_set_2)
- run = self.dataset_process.worker_runs.create(version=self.recognizer, ttl=0)
+ run = self.dataset_process.worker_runs.create(version=self.recognizer)
self.assertFalse(self.dataset_process.tasks.exists())
self.client.force_login(self.user)
@@ -434,7 +432,7 @@ class TestProcessStart(FixtureAPITestCase):
self.recognizer.save()
self.assertEqual(self.recognizer.state, WorkerVersionState.Available)
- run = self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ run = self.workers_process.worker_runs.create(version=self.recognizer)
self.assertFalse(self.workers_process.tasks.exists())
self.client.force_login(self.user)
@@ -463,7 +461,7 @@ class TestProcessStart(FixtureAPITestCase):
"""
A user can specify a ponos farm to use for a process
"""
- self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer)
farm = Farm.objects.get(name="Wheat farm")
self.client.force_login(self.user)
@@ -491,7 +489,7 @@ class TestProcessStart(FixtureAPITestCase):
farm = Farm.objects.get(name="Wheat farm")
get_default_farm_mock.return_value = farm
- self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer)
self.assertFalse(self.workers_process.tasks.exists())
self.client.force_login(self.user)
@@ -518,7 +516,7 @@ class TestProcessStart(FixtureAPITestCase):
@patch("arkindex.process.serializers.imports.get_default_farm")
def test_default_farm_guest(self, get_default_farm_mock, is_available_mock):
get_default_farm_mock.return_value = Farm.objects.first()
- self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer)
self.client.force_login(self.user)
with self.assertNumQueries(5):
@@ -539,7 +537,7 @@ class TestProcessStart(FixtureAPITestCase):
@patch("arkindex.ponos.models.Farm.is_available", return_value=False)
def test_farm_guest(self, is_available_mock):
- self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer)
self.client.force_login(self.user)
with self.assertNumQueries(7):
@@ -592,7 +590,7 @@ class TestProcessStart(FixtureAPITestCase):
"""
StartProcess should restrict the chunks to `settings.MAX_CHUNKS`
"""
- self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ self.workers_process.worker_runs.create(version=self.recognizer)
self.client.force_login(self.user)
with self.assertNumQueries(5):
@@ -611,7 +609,7 @@ class TestProcessStart(FixtureAPITestCase):
"""
It should be possible to pass chunks parameters when starting a workers process
"""
- run = self.workers_process.worker_runs.create(version=self.recognizer, ttl=0)
+ run = self.workers_process.worker_runs.create(version=self.recognizer)
self.client.force_login(self.user)
with self.assertNumQueries(18):
@@ -644,7 +642,7 @@ class TestProcessStart(FixtureAPITestCase):
test_set_2 = self.dataset2.sets.get(name="test")
ProcessDatasetSet.objects.create(process=self.dataset_process, set=test_set_1)
ProcessDatasetSet.objects.create(process=self.dataset_process, set=test_set_2)
- run = self.dataset_process.worker_runs.create(version=self.recognizer, ttl=0)
+ run = self.dataset_process.worker_runs.create(version=self.recognizer)
self.client.force_login(self.user)
with self.assertNumQueries(12):
@@ -690,7 +688,7 @@ class TestProcessStart(FixtureAPITestCase):
"""
self.assertFalse(self.workers_process.use_cache)
self.assertEqual(self.workers_process.activity_state, ActivityState.Disabled)
- self.workers_process.worker_runs.create(version=self.version_gpu, ttl=0)
+ self.workers_process.worker_runs.create(version=self.version_gpu)
self.client.force_login(self.user)
@@ -723,12 +721,10 @@ class TestProcessStart(FixtureAPITestCase):
name="some_config",
configuration={"a": "b"},
),
- ttl=0,
)
run_2 = self.workers_process.worker_runs.create(
version=self.recognizer,
parents=[run_1.id],
- ttl=0,
)
self.assertNotEqual(run_1.task_slug, run_2.task_slug)
diff --git a/arkindex/process/tests/templates/test_apply.py b/arkindex/process/tests/templates/test_apply.py
index 0f59330309ab6bb8b803884d30bfbd1e85225b3a..43a4a9893777d8b2e1b19f769f4403a8b88f0969 100644
--- a/arkindex/process/tests/templates/test_apply.py
+++ b/arkindex/process/tests/templates/test_apply.py
@@ -1,7 +1,6 @@
from datetime import datetime, timezone
from unittest.mock import call, patch
-from django.test import override_settings
from rest_framework import status
from rest_framework.reverse import reverse
@@ -19,7 +18,6 @@ from arkindex.training.models import Model, ModelVersionState
from arkindex.users.models import Role
-@override_settings(PONOS_MAXIMUM_TASK_TTL=3600)
class TestApplyTemplate(FixtureAPITestCase):
@classmethod
@@ -59,19 +57,16 @@ class TestApplyTemplate(FixtureAPITestCase):
cls.template_run_1 = cls.template.worker_runs.create(
version=cls.version_1,
configuration=cls.worker_configuration,
- ttl=7200,
)
cls.template_run_2 = cls.template.worker_runs.create(
version=cls.version_2,
parents=[cls.template_run_1.id],
model_version=cls.model_version,
- ttl=0,
)
cls.private_template.worker_runs.create(
version=cls.version_1,
configuration=cls.worker_configuration,
- ttl=0,
)
@patch("arkindex.project.mixins.get_max_level", return_value=Role.Guest.value)
@@ -170,7 +165,6 @@ class TestApplyTemplate(FixtureAPITestCase):
def test_apply(self):
self.assertIsNotNone(self.version_2.docker_image_iid)
- self.assertIsNone(self.corpus.maximum_task_ttl)
self.client.force_login(self.user)
with self.assertNumQueries(10):
response = self.client.post(
@@ -190,52 +184,16 @@ class TestApplyTemplate(FixtureAPITestCase):
self.assertIsNone(parent_run.model_version_id)
self.assertEqual(parent_run.configuration_id, self.worker_configuration.id)
self.assertListEqual(parent_run.parents, [])
- # This had a 7200 seconds TTL, but is limited by the instance limit
- self.assertEqual(parent_run.ttl, 3600)
self.assertEqual(child_run.process_id, self.process.id)
self.assertEqual(child_run.version_id, self.version_2.id)
self.assertEqual(child_run.model_version_id, self.model_version.id)
self.assertIsNone(child_run.configuration_id)
self.assertListEqual(child_run.parents, [parent_run.id])
- # This had an infinite TTL, but is limited by the instance limit
- self.assertEqual(child_run.ttl, 3600)
-
- def test_unlimited_ttl(self):
- self.corpus.maximum_task_ttl = 0
- self.corpus.save()
- self.client.force_login(self.user)
-
- with self.assertNumQueries(10):
- response = self.client.post(
- reverse("api:apply-process-template", kwargs={"pk": str(self.template.id)}),
- data={"process_id": str(self.process.id)},
- )
- self.assertEqual(response.status_code, status.HTTP_200_OK)
-
- child_run, parent_run = self.process.worker_runs.order_by("version__worker__slug")
- self.assertEqual(parent_run.ttl, 7200)
- self.assertEqual(child_run.ttl, 0)
-
- def test_corpus_limited_ttl(self):
- self.corpus.maximum_task_ttl = 9000
- self.corpus.save()
- self.client.force_login(self.user)
-
- with self.assertNumQueries(10):
- response = self.client.post(
- reverse("api:apply-process-template", kwargs={"pk": str(self.template.id)}),
- data={"process_id": str(self.process.id)},
- )
- self.assertEqual(response.status_code, status.HTTP_200_OK)
-
- child_run, parent_run = self.process.worker_runs.order_by("version__worker__slug")
- self.assertEqual(parent_run.ttl, 7200)
- self.assertEqual(child_run.ttl, 9000)
def test_excludes_init_elements(self):
init_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.InitElements)
- init_run = self.template.worker_runs.create(version=init_version, ttl=0)
+ init_run = self.template.worker_runs.create(version=init_version)
self.template_run_1.parents = [init_run.id]
self.template_run_1.save()
@@ -281,10 +239,7 @@ class TestApplyTemplate(FixtureAPITestCase):
process = self.corpus.processes.create(
creator=self.user, mode=ProcessMode.Workers
)
- process.worker_runs.create(
- version=self.version_2,
- ttl=0,
- )
+ process.worker_runs.create(version=self.version_2)
# Apply a template that has two other worker runs
with self.assertNumQueries(12):
response = self.client.post(
@@ -318,13 +273,13 @@ class TestApplyTemplate(FixtureAPITestCase):
# Set invalid values: the version with disabled GPU usage gets a GPU
self.template.worker_runs.filter(version=self.version_1).update(use_gpu=True)
# A signal is trying to set use_gpu to the correct values, so we create then update to give no GPU to a version that requires a GPU
- self.template.worker_runs.create(version=self.version_3, ttl=0)
+ self.template.worker_runs.create(version=self.version_3)
self.template.worker_runs.filter(version=self.version_3).update(use_gpu=False)
# Have two runs with a version that supports GPU usage, to test that both True and False are copied
self.version_2.gpu_usage = FeatureUsage.Supported
self.version_2.save()
- self.template.worker_runs.create(version=self.version_2, configuration=self.worker_configuration, use_gpu=True, ttl=0)
+ self.template.worker_runs.create(version=self.version_2, configuration=self.worker_configuration, use_gpu=True)
self.assertQuerySetEqual((
self.template.worker_runs
diff --git a/arkindex/process/tests/templates/test_create.py b/arkindex/process/tests/templates/test_create.py
index 8f819de010092c3714c3e15cb3715a7b22fd9ca8..f0bc7787500b00e258f1b87c671220d859f886a8 100644
--- a/arkindex/process/tests/templates/test_create.py
+++ b/arkindex/process/tests/templates/test_create.py
@@ -1,6 +1,5 @@
from unittest.mock import call, patch
-from django.test import override_settings
from rest_framework import status
from rest_framework.reverse import reverse
@@ -19,7 +18,6 @@ from arkindex.training.models import Model, ModelVersionState
from arkindex.users.models import Role, User
-@override_settings(PONOS_MAXIMUM_TASK_TTL=3600)
class TestCreateTemplate(FixtureAPITestCase):
@classmethod
@@ -61,12 +59,10 @@ class TestCreateTemplate(FixtureAPITestCase):
cls.run_1 = cls.process_template.worker_runs.create(
version=cls.version_1,
configuration=cls.worker_configuration,
- ttl=7200,
)
cls.run_2 = cls.process_template.worker_runs.create(
version=cls.version_2,
parents=[cls.run_1.id],
- ttl=0,
)
cls.model = Model.objects.create(name="moo")
@@ -75,24 +71,20 @@ class TestCreateTemplate(FixtureAPITestCase):
cls.template_run_1 = cls.template.worker_runs.create(
version=cls.version_1,
configuration=cls.worker_configuration,
- ttl=0,
)
cls.template_run_2 = cls.template.worker_runs.create(
version=cls.version_2,
parents=[cls.template_run_1.id],
model_version=cls.model_version,
- ttl=0,
)
cls.private_process_template.worker_runs.create(
version=cls.version_1,
configuration=cls.worker_configuration,
- ttl=0,
)
cls.private_template.worker_runs.create(
version=cls.version_1,
configuration=cls.worker_configuration,
- ttl=0,
)
def test_create(self):
@@ -116,10 +108,10 @@ class TestCreateTemplate(FixtureAPITestCase):
self.assertQuerySetEqual((
template_process.worker_runs
.order_by("version__worker__slug")
- .values_list("version_id", "model_version_id", "configuration_id", "parents", "ttl")
+ .values_list("version_id", "model_version_id", "configuration_id", "parents")
), [
- (self.version_2.id, None, None, [parent_run.id], 0),
- (self.version_1.id, None, self.worker_configuration.id, [], 7200),
+ (self.version_2.id, None, None, [parent_run.id]),
+ (self.version_1.id, None, self.worker_configuration.id, []),
])
def test_use_gpu(self):
@@ -128,13 +120,13 @@ class TestCreateTemplate(FixtureAPITestCase):
# Set invalid values: the version with disabled GPU usage gets a GPU
self.process_template.worker_runs.filter(version=self.version_1).update(use_gpu=True)
# A signal is trying to set use_gpu to the correct values, so we create then update to give no GPU to a version that requires a GPU
- self.process_template.worker_runs.create(version=self.version_3, ttl=0)
+ self.process_template.worker_runs.create(version=self.version_3)
self.process_template.worker_runs.filter(version=self.version_3).update(use_gpu=False)
# Have two runs with a version that supports GPU usage, to test that both True and False are copied
self.version_2.gpu_usage = FeatureUsage.Supported
self.version_2.save()
- self.process_template.worker_runs.create(version=self.version_2, configuration=self.worker_configuration, use_gpu=True, ttl=0)
+ self.process_template.worker_runs.create(version=self.version_2, configuration=self.worker_configuration, use_gpu=True)
self.assertQuerySetEqual((
self.process_template.worker_runs
@@ -170,7 +162,7 @@ class TestCreateTemplate(FixtureAPITestCase):
def test_excludes_init_elements(self):
init_version = WorkerVersion.objects.get_by_feature(ArkindexFeature.InitElements)
- init_run = self.process_template.worker_runs.create(version=init_version, ttl=0)
+ init_run = self.process_template.worker_runs.create(version=init_version)
self.run_1.parents = [init_run.id]
self.run_1.save()
diff --git a/arkindex/process/tests/test_corpus_worker_runs.py b/arkindex/process/tests/test_corpus_worker_runs.py
index 590156dcc7708d2e8580e5777c0590bd2733800b..8b9754004f5c970925041eb17baaf7b6521485b9 100644
--- a/arkindex/process/tests/test_corpus_worker_runs.py
+++ b/arkindex/process/tests/test_corpus_worker_runs.py
@@ -29,7 +29,6 @@ class TestCorpusWorkerRuns(FixtureAPITestCase):
cls.run_1 = WorkerRun.objects.create(
process=cls.process,
version=cls.dla_worker_version,
- ttl=0,
has_results=True
)
@@ -57,12 +56,10 @@ class TestCorpusWorkerRuns(FixtureAPITestCase):
cls.run_2 = WorkerRun.objects.create(
process=cls.private_process,
version=cls.reco_worker_version,
- ttl=0,
)
cls.run_3 = WorkerRun.objects.create(
process=cls.private_process,
version=cls.dla_worker_version,
- ttl=0,
has_results=True
)
@@ -144,7 +141,6 @@ class TestCorpusWorkerRuns(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 0,
},
{
"id": str(self.run_3.id),
@@ -190,6 +186,5 @@ class TestCorpusWorkerRuns(FixtureAPITestCase):
},
"use_gpu": False,
"summary": "Worker Document layout analyser @ version 1",
- "ttl": 0,
}
])
diff --git a/arkindex/process/tests/test_elements_initialisation.py b/arkindex/process/tests/test_elements_initialisation.py
index 0ec36078b5955b4561276a2b9a24efe37c4693bb..5deb50f40c13f3edd2bd2ae794d473d32869757c 100644
--- a/arkindex/process/tests/test_elements_initialisation.py
+++ b/arkindex/process/tests/test_elements_initialisation.py
@@ -44,12 +44,10 @@ class TestElementsInit(FixtureAPITestCase):
)
init_run = process.worker_runs.create(
version=self.init_elements_version,
- ttl=0,
)
worker_run = process.worker_runs.create(
version=self.reco_version,
parents=[init_run.id],
- ttl=0,
)
with self.assertNumQueries(16):
@@ -86,8 +84,8 @@ class TestElementsInit(FixtureAPITestCase):
then one is created when the process is started.
"""
self.client.force_login(self.user)
- reco_run = self.process.worker_runs.create(version=self.reco_version, ttl=0)
- dla_run = self.process.worker_runs.create(version=self.dla_version, parents=[reco_run.id], ttl=0)
+ reco_run = self.process.worker_runs.create(version=self.reco_version)
+ dla_run = self.process.worker_runs.create(version=self.dla_version, parents=[reco_run.id])
with self.assertNumQueries(18):
response = self.client.post(
diff --git a/arkindex/process/tests/test_managers.py b/arkindex/process/tests/test_managers.py
index aa5c3c12f99c5bd64cfacfb6116494758403e7be..dc6b4a4c4fda61260f39d1dffd9fbfbe0fce6ec8 100644
--- a/arkindex/process/tests/test_managers.py
+++ b/arkindex/process/tests/test_managers.py
@@ -28,23 +28,19 @@ class TestManagers(FixtureTestCase):
cls.worker_run_1 = cls.worker_version.worker_runs.create(
process=cls.corpus.processes.create(mode=ProcessMode.Workers, creator=cls.user),
- ttl=0,
)
cls.worker_run_2 = cls.worker_version.worker_runs.create(
process=cls.corpus.processes.create(mode=ProcessMode.Workers, creator=cls.user),
configuration=cls.worker_configuration,
- ttl=0,
)
cls.worker_run_3 = cls.worker_version.worker_runs.create(
process=cls.corpus.processes.create(mode=ProcessMode.Workers, creator=cls.user),
model_version=cls.model_version,
- ttl=0,
)
cls.worker_run_4 = cls.worker_version.worker_runs.create(
process=cls.corpus.processes.create(mode=ProcessMode.Workers, creator=cls.user),
model_version=cls.model_version,
configuration=cls.worker_configuration,
- ttl=0,
)
def test_corpus_worker_version_rebuild(self):
@@ -105,5 +101,5 @@ class TestManagers(FixtureTestCase):
Ensure the in_use method iterates over all related items
"""
worker_run_id = uuid4()
- with self.assertExactQueries("worker_run_in_use.sql", params={"id": worker_run_id}):
+ with self.assertExactQueries("worker_run_in_use.sql", params={"id": worker_run_id.hex}):
self.assertFalse(WorkerRun.objects.filter(id=worker_run_id).in_use())
diff --git a/arkindex/process/tests/test_process_elements.py b/arkindex/process/tests/test_process_elements.py
index 0881b4fe1cf77cc96e507d7831c58e916329164d..e2c60dd54113cb978d8046fb450ca0587f855bb8 100644
--- a/arkindex/process/tests/test_process_elements.py
+++ b/arkindex/process/tests/test_process_elements.py
@@ -304,9 +304,9 @@ class TestProcessElements(FixtureAPITestCase):
self.client.force_login(self.superuser)
with self.assertExactQueries("process_elements_filter_type.sql", skip=1, params={
"user_id": self.superuser.id,
- "process_id": str(self.process.id),
- "corpus_id": str(self.private_corpus.id),
- "type_id": str(self.folder_type.id),
+ "process_id": self.process.id.hex,
+ "corpus_id": self.private_corpus.id.hex,
+ "type_id": self.folder_type.id.hex,
}):
response = self.client.get(reverse("api:process-elements-list", kwargs={"pk": self.process.id}))
@@ -333,9 +333,9 @@ class TestProcessElements(FixtureAPITestCase):
self.client.force_login(self.superuser)
with self.assertExactQueries("process_elements_filter_ml_class.sql", skip=1, params={
"user_id": self.superuser.id,
- "process_id": str(self.process.id),
- "corpus_id": str(self.private_corpus.id),
- "ml_class_id": str(self.ml_class.id),
+ "process_id": self.process.id.hex,
+ "corpus_id": self.private_corpus.id.hex,
+ "ml_class_id": self.ml_class.id.hex,
}):
response = self.client.get(reverse("api:process-elements-list", kwargs={"pk": self.process.id}))
self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -719,8 +719,8 @@ class TestProcessElements(FixtureAPITestCase):
self.process.save()
with self.assertExactQueries("process_elements_with_image.sql", skip=1, params={
"user_id": self.superuser.id,
- "process_id": str(self.process.id),
- "corpus_id": str(self.private_corpus.id),
+ "process_id": self.process.id.hex,
+ "corpus_id": self.private_corpus.id.hex,
}):
response = self.client.get(
reverse("api:process-elements-list", kwargs={"pk": self.process.id}),
@@ -749,8 +749,8 @@ class TestProcessElements(FixtureAPITestCase):
self.client.force_login(self.superuser)
with self.assertExactQueries("process_elements_top_level.sql", skip=1, params={
"user_id": self.superuser.id,
- "process_id": str(self.process.id),
- "corpus_id": str(self.private_corpus.id),
+ "process_id": self.process.id.hex,
+ "corpus_id": self.private_corpus.id.hex,
}):
response = self.client.get(
reverse("api:process-elements-list", kwargs={"pk": self.process.id}),
diff --git a/arkindex/process/tests/test_signals.py b/arkindex/process/tests/test_signals.py
index 1c6323c2a153924f0e80eb473fe9543e5b36b8b2..f1dc954aa854bb8c45e90ec5d8643bd2993d0b96 100644
--- a/arkindex/process/tests/test_signals.py
+++ b/arkindex/process/tests/test_signals.py
@@ -38,10 +38,7 @@ class TestSignals(FixtureAPITestCase):
mode=ProcessMode.Workers,
farm=cls.farm,
)
- cls.run_1 = cls.process_1.worker_runs.create(
- version=cls.version_1,
- ttl=0,
- )
+ cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1)
cls.process_2 = cls.corpus.processes.create(
creator=cls.user,
mode=ProcessMode.Workers,
@@ -49,10 +46,7 @@ class TestSignals(FixtureAPITestCase):
)
def test_worker_run_check_parents_recursive(self):
- run_2 = self.process_1.worker_runs.create(
- version=self.version_2,
- ttl=0,
- )
+ run_2 = self.process_1.worker_runs.create(version=self.version_2)
self.assertListEqual(run_2.parents, [])
run_2.parents = [str(run_2.id)]
@@ -68,7 +62,6 @@ class TestSignals(FixtureAPITestCase):
self.process_2.worker_runs.create(
version=self.version_2,
parents=[str(self.run_1.id)],
- ttl=0,
)
self.process_1.refresh_from_db()
@@ -80,7 +73,6 @@ class TestSignals(FixtureAPITestCase):
self.process_2.worker_runs.create(
version=self.version_2,
parents=["12341234-1234-1234-1234-123412341234"],
- ttl=0,
)
self.process_1.refresh_from_db()
@@ -133,7 +125,6 @@ class TestSignals(FixtureAPITestCase):
run_2 = self.process_1.worker_runs.create(
version=self.version_2,
parents=[self.run_1.id],
- ttl=0,
)
self.run_1.parents = [run_2.id]
@@ -175,22 +166,18 @@ class TestSignals(FixtureAPITestCase):
run_2 = self.process_1.worker_runs.create(
version=self.version_2,
parents=[self.run_1.id],
- ttl=0,
)
run_3 = self.process_1.worker_runs.create(
version=version_3,
parents=[run_2.id],
- ttl=0,
)
run_4 = self.process_1.worker_runs.create(
version=version_4,
parents=[run_3.id],
- ttl=0,
)
run_5 = self.process_1.worker_runs.create(
version=version_5,
parents=[run_4.id],
- ttl=0,
)
self.run_1.parents = [run_5.id]
@@ -211,12 +198,10 @@ class TestSignals(FixtureAPITestCase):
run_2 = self.process_1.worker_runs.create(
version=self.version_2,
parents=[self.run_1.id],
- ttl=0,
)
run_3 = self.process_1.worker_runs.create(
version=version_3,
parents=[run_2.id],
- ttl=0,
)
run_3.parents.append(self.run_1.id)
@@ -231,7 +216,6 @@ class TestSignals(FixtureAPITestCase):
run_2 = self.process_1.worker_runs.create(
version=self.version_2,
parents=[self.run_1.id],
- ttl=0,
)
self.assertEqual(len(self.process_1.worker_runs.all()), 2)
@@ -245,7 +229,6 @@ class TestSignals(FixtureAPITestCase):
run = self.process_1.worker_runs.create(
version=self.version_2,
parents=[self.run_1.id],
- ttl=0,
)
self.assertIsNotNone(run.summary)
diff --git a/arkindex/process/tests/test_user_workerruns.py b/arkindex/process/tests/test_user_workerruns.py
index d98a8d5a33d32c9f80f557ec477730165006fbfc..4a515ef8c31e2ec8deb96d595c4df9bd7de69a27 100644
--- a/arkindex/process/tests/test_user_workerruns.py
+++ b/arkindex/process/tests/test_user_workerruns.py
@@ -1,7 +1,6 @@
from datetime import datetime, timezone
from unittest.mock import call, patch
-from django.test import override_settings
from django.urls import reverse
from rest_framework import status
@@ -19,7 +18,6 @@ from arkindex.training.models import Model, ModelVersion, ModelVersionState
from arkindex.users.models import Right, Role, User
-@override_settings(PONOS_MAXIMUM_TASK_TTL=3600)
class TestUserWorkerRuns(FixtureAPITestCase):
@classmethod
def setUpTestData(cls):
@@ -67,7 +65,6 @@ class TestUserWorkerRuns(FixtureAPITestCase):
test_local_run = WorkerRun.objects.create(
process=self.local_process,
version=self.version_1,
- ttl=0,
)
self.client.force_login(self.user)
with self.assertNumQueries(5):
@@ -118,7 +115,6 @@ class TestUserWorkerRuns(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 0,
}, {
"configuration": None,
"id": str(self.local_run.id),
@@ -163,7 +159,6 @@ class TestUserWorkerRuns(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 0,
}])
def test_list_user_runs_only_own_runs(self):
@@ -175,7 +170,6 @@ class TestUserWorkerRuns(FixtureAPITestCase):
test_local_run = WorkerRun.objects.create(
process=test_local_process,
version=self.version_1,
- ttl=0,
)
assert WorkerRun.objects.filter(process__mode=ProcessMode.Local, process__creator=write_user).count() == 1
self.client.force_login(self.user)
@@ -264,8 +258,6 @@ class TestUserWorkerRuns(FixtureAPITestCase):
"use_cache": False,
},
"use_gpu": False,
- # The TTL is always 0 for user worker runs
- "ttl": 0,
})
def test_create_user_run_no_local_process(self):
@@ -483,7 +475,6 @@ class TestUserWorkerRuns(FixtureAPITestCase):
"use_cache": False,
},
"use_gpu": False,
- "ttl": 0,
})
def test_create_user_run_duplicate(self):
diff --git a/arkindex/process/tests/worker_activity/test_bulk_insert.py b/arkindex/process/tests/worker_activity/test_bulk_insert.py
index 026292cd5476b6ed60b138c13dbf4dd8de848013..e51dd68498424bad9c79a695673659475a205840 100644
--- a/arkindex/process/tests/worker_activity/test_bulk_insert.py
+++ b/arkindex/process/tests/worker_activity/test_bulk_insert.py
@@ -40,7 +40,6 @@ class TestWorkerActivityBulkInsert(FixtureAPITestCase):
version=cls.worker_version,
configuration=cls.configuration,
model_version=cls.model_version,
- ttl=0,
)
def test_worker_version(self):
@@ -57,9 +56,9 @@ class TestWorkerActivityBulkInsert(FixtureAPITestCase):
elements_qs = Element.objects.filter(type__slug="act", type__corpus_id=self.corpus.id)
params = {
- "worker_version_id": self.worker_version.id,
- "corpus_id": self.corpus.id,
- "process_id": self.process.id,
+ "worker_version_id": self.worker_version.id.hex,
+ "corpus_id": self.corpus.id.hex,
+ "process_id": self.process.id.hex,
}
with self.assertExactQueries("workeractivity_bulk_insert_worker_version_only.sql", params=params):
WorkerActivity.objects.bulk_insert(
@@ -87,10 +86,10 @@ class TestWorkerActivityBulkInsert(FixtureAPITestCase):
elements_qs = Element.objects.filter(type__slug="act", type__corpus_id=self.corpus.id)
params = {
- "worker_version_id": self.worker_version.id,
- "corpus_id": self.corpus.id,
- "process_id": self.process.id,
- "configuration_id": self.configuration.id,
+ "worker_version_id": self.worker_version.id.hex,
+ "corpus_id": self.corpus.id.hex,
+ "process_id": self.process.id.hex,
+ "configuration_id": self.configuration.id.hex,
}
with self.assertExactQueries("workeractivity_bulk_insert_no_model.sql", params=params):
WorkerActivity.objects.bulk_insert(
@@ -118,10 +117,10 @@ class TestWorkerActivityBulkInsert(FixtureAPITestCase):
elements_qs = Element.objects.filter(type__slug="act", type__corpus_id=self.corpus.id)
params = {
- "worker_version_id": self.worker_version.id,
- "model_version_id": self.model_version.id,
- "corpus_id": self.corpus.id,
- "process_id": self.process.id,
+ "worker_version_id": self.worker_version.id.hex,
+ "model_version_id": self.model_version.id.hex,
+ "corpus_id": self.corpus.id.hex,
+ "process_id": self.process.id.hex,
}
with self.assertExactQueries("workeractivity_bulk_insert_no_configuration.sql", params=params):
WorkerActivity.objects.bulk_insert(
@@ -184,11 +183,11 @@ class TestWorkerActivityBulkInsert(FixtureAPITestCase):
elements_qs = Element.objects.filter(type__slug="act", type__corpus_id=self.corpus.id)
params = {
- "worker_version_id": self.worker_version.id,
- "configuration_id": self.configuration.id,
- "model_version_id": self.model_version.id,
- "corpus_id": self.corpus.id,
- "process_id": self.process.id,
+ "worker_version_id": self.worker_version.id.hex,
+ "configuration_id": self.configuration.id.hex,
+ "model_version_id": self.model_version.id.hex,
+ "corpus_id": self.corpus.id.hex,
+ "process_id": self.process.id.hex,
}
with self.assertExactQueries("workeractivity_bulk_insert.sql", params=params):
WorkerActivity.objects.bulk_insert(
diff --git a/arkindex/process/tests/worker_activity/test_initialize.py b/arkindex/process/tests/worker_activity/test_initialize.py
index 321f1855f7c8d2e9062538552abb57aceddea987..caaed902d0c140d5051d4d7685dcbb4cd00f75c9 100644
--- a/arkindex/process/tests/worker_activity/test_initialize.py
+++ b/arkindex/process/tests/worker_activity/test_initialize.py
@@ -19,8 +19,8 @@ class TestInitializeActivity(FixtureTestCase):
element_type=cls.corpus.types.get(slug="volume"),
activity_state=ActivityState.Pending,
)
- cls.process.worker_runs.create(version=cls.worker_version_1, ttl=0)
- cls.process.worker_runs.create(version=cls.worker_version_2, ttl=0)
+ cls.process.worker_runs.create(version=cls.worker_version_1)
+ cls.process.worker_runs.create(version=cls.worker_version_2)
@patch("arkindex.process.tasks.get_current_job")
def test_rq_progress(self, job_mock):
diff --git a/arkindex/process/tests/worker_activity/test_list.py b/arkindex/process/tests/worker_activity/test_list.py
index bc980a1189a5738caac9943f165c41a3b160948c..75af915f2948d86625833312b0bc4f34a17221b4 100644
--- a/arkindex/process/tests/worker_activity/test_list.py
+++ b/arkindex/process/tests/worker_activity/test_list.py
@@ -48,7 +48,6 @@ class TestListWorkerActivities(FixtureAPITestCase):
version=cls.worker_version,
configuration=cls.configuration,
model_version=cls.model_version,
- ttl=0,
)
# Run the process, but skip the real activity initialization so that we can control it ourselves
diff --git a/arkindex/process/tests/worker_activity/test_update.py b/arkindex/process/tests/worker_activity/test_update.py
index e275425a7e5f933cb2378e9775192038ac2078cd..4d1a5aa34e6f8cf775a31f9b74990d1ec290111e 100644
--- a/arkindex/process/tests/worker_activity/test_update.py
+++ b/arkindex/process/tests/worker_activity/test_update.py
@@ -49,7 +49,6 @@ class TestUpdateWorkerActivity(FixtureAPITestCase):
version=cls.worker_version,
configuration=cls.configuration,
model_version=cls.model_version,
- ttl=0,
)
# Run the process, but skip the real activity initialization so that we can control it ourselves
@@ -201,7 +200,6 @@ class TestUpdateWorkerActivity(FixtureAPITestCase):
# Different configuration
configuration=None,
model_version=self.model_version,
- ttl=0,
)
with self.assertNumQueries(4):
@@ -457,17 +455,14 @@ class TestUpdateWorkerActivity(FixtureAPITestCase):
run_2 = self.process.worker_runs.create(
version=worker_version_2,
parents=[run_1.id],
- ttl=0,
)
self.process.worker_runs.create(
version=worker_version_3,
parents=[run_2.id],
- ttl=0,
)
self.process.worker_runs.create(
version=worker_version_4,
- ttl=0,
)
# Create activities for run_2, run_3 and run_4
diff --git a/arkindex/process/tests/worker_runs/test_build_task.py b/arkindex/process/tests/worker_runs/test_build_task.py
index 65ce96c232f643b396f70afcfbc4d21ad0f36400..022348a66055f610eb5344ee248efefede735c24 100644
--- a/arkindex/process/tests/worker_runs/test_build_task.py
+++ b/arkindex/process/tests/worker_runs/test_build_task.py
@@ -25,7 +25,7 @@ class TestWorkerRunsBuildTask(FixtureAPITestCase):
)
cls.version = WorkerVersion.objects.get(worker__slug="reco")
cls.worker = cls.version.worker
- cls.worker_run = cls.process.worker_runs.create(version=cls.version, ttl=0)
+ cls.worker_run = cls.process.worker_runs.create(version=cls.version)
# Model and Model version setup
cls.model_1 = Model.objects.create(name="My model")
@@ -52,7 +52,7 @@ class TestWorkerRunsBuildTask(FixtureAPITestCase):
"TASK_ELEMENTS": "/data/import/elements.json",
"ARKINDEX_WORKER_RUN_ID": str(self.worker_run.id),
})
- self.assertEqual(task.ttl, 0)
+ self.assertEqual(task.ttl, 3600)
def test_build_task_with_chunk(self):
task, parent_slugs = self.worker_run.build_task(self.process, ENV.copy(), "import", "/data/import/elements.json", chunk=4)
@@ -81,7 +81,6 @@ class TestWorkerRunsBuildTask(FixtureAPITestCase):
run_2 = self.process.worker_runs.create(
version=version_2,
parents=[self.worker_run.id],
- ttl=42,
)
task, parent_slugs = run_2.build_task(self.process, ENV.copy(), "import", "/data/import/elements.json")
@@ -97,7 +96,7 @@ class TestWorkerRunsBuildTask(FixtureAPITestCase):
"TASK_ELEMENTS": "/data/import/elements.json",
"ARKINDEX_WORKER_RUN_ID": str(run_2.id),
})
- self.assertEqual(task.ttl, 42)
+ self.assertEqual(task.ttl, 3600)
def test_build_task_with_parent_and_chunk(self):
version_2 = WorkerVersion.objects.create(
@@ -110,7 +109,6 @@ class TestWorkerRunsBuildTask(FixtureAPITestCase):
run_2 = self.process.worker_runs.create(
version=version_2,
parents=[self.worker_run.id],
- ttl=1000,
)
task, parent_slugs = run_2.build_task(self.process, ENV.copy(), "import", "/data/import/elements.json", chunk=4)
@@ -127,7 +125,7 @@ class TestWorkerRunsBuildTask(FixtureAPITestCase):
"TASK_ELEMENTS": "/data/import/elements.json",
"ARKINDEX_WORKER_RUN_ID": str(run_2.id),
})
- self.assertEqual(task.ttl, 1000)
+ self.assertEqual(task.ttl, 3600)
def test_build_task_shm_size(self):
self.version.configuration = {
@@ -160,7 +158,6 @@ class TestWorkerRunsBuildTask(FixtureAPITestCase):
run_2 = self.process.worker_runs.create(
version=version_2,
parents=[self.worker_run.id],
- ttl=0,
)
with self.assertRaisesRegex(
diff --git a/arkindex/process/tests/worker_runs/test_create.py b/arkindex/process/tests/worker_runs/test_create.py
index 41ac10e52738e197473f8d2e434c033ada84a3df..1d066bfb420854071d510929527f3ee18213362b 100644
--- a/arkindex/process/tests/worker_runs/test_create.py
+++ b/arkindex/process/tests/worker_runs/test_create.py
@@ -3,7 +3,6 @@ from datetime import datetime, timezone
from unittest.mock import call, patch
from django.db import transaction
-from django.test import override_settings
from django.urls import reverse
from rest_framework import status
@@ -20,7 +19,6 @@ from arkindex.training.models import Model, ModelVersion, ModelVersionState
from arkindex.users.models import Role
-@override_settings(PONOS_MAXIMUM_TASK_TTL=3600)
class TestWorkerRunsCreate(FixtureAPITestCase):
"""
Test worker runs create endpoint
@@ -38,7 +36,7 @@ class TestWorkerRunsCreate(FixtureAPITestCase):
)
cls.version_1 = WorkerVersion.objects.get(worker__slug="reco")
cls.worker_1 = cls.version_1.worker
- cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1, ttl=0)
+ cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1)
cls.configuration_1 = cls.worker_1.configurations.create(name="My config", configuration={"key": "value"})
worker_version = WorkerVersion.objects.exclude(worker=cls.version_1.worker).first()
cls.configuration_2 = worker_version.worker.configurations.create(name="Config")
@@ -330,7 +328,6 @@ class TestWorkerRunsCreate(FixtureAPITestCase):
},
"use_gpu": False,
"summary": "Worker Recognizer @ version 1",
- "ttl": 3600,
})
run = WorkerRun.objects.get(pk=pk)
# Check generated summary
@@ -415,7 +412,6 @@ class TestWorkerRunsCreate(FixtureAPITestCase):
},
"use_gpu": False,
"summary": "Worker Recognizer @ version 1 using configuration 'My config'",
- "ttl": 3600,
})
run = WorkerRun.objects.get(pk=pk)
# Check generated summary
@@ -563,7 +559,6 @@ class TestWorkerRunsCreate(FixtureAPITestCase):
},
"summary": f"Worker Recognizer @ version {worker_version.version}",
"use_gpu": use_gpu,
- "ttl": 3600,
})
run = WorkerRun.objects.get(pk=pk)
self.assertEqual(run.use_gpu, use_gpu)
@@ -623,7 +618,6 @@ class TestWorkerRunsCreate(FixtureAPITestCase):
},
"summary": "Worker Recognizer @ version 2",
"use_gpu": True,
- "ttl": 3600,
})
run = WorkerRun.objects.get(pk=pk)
self.assertEqual(run.use_gpu, True)
diff --git a/arkindex/process/tests/worker_runs/test_delete.py b/arkindex/process/tests/worker_runs/test_delete.py
index 410e5a3e4422a1d81dbce6fb42ea2d679d94ab34..364cda2564db59a7b0932daacc88036b8b4cbc4c 100644
--- a/arkindex/process/tests/worker_runs/test_delete.py
+++ b/arkindex/process/tests/worker_runs/test_delete.py
@@ -28,10 +28,11 @@ class TestWorkerRunsDelete(FixtureAPITestCase):
cls.version_1 = WorkerVersion.objects.get(worker__slug="reco")
cls.worker_1 = cls.version_1.worker
cls.version_2 = WorkerVersion.objects.get(worker__slug="dla")
- cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1, ttl=0)
+ cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1)
cls.agent = Agent.objects.create(
farm=cls.farm,
+ fingerprint="a" * 64,
hostname="claude",
cpu_cores=42,
cpu_frequency=1e15,
@@ -77,7 +78,7 @@ class TestWorkerRunsDelete(FixtureAPITestCase):
"""
A user cannot delete a worker run on a local process
"""
- run = self.local_process.worker_runs.create(version=self.version_1, ttl=0)
+ run = self.local_process.worker_runs.create(version=self.version_1)
self.client.force_login(self.user)
with self.assertNumQueries(4):
@@ -114,12 +115,10 @@ class TestWorkerRunsDelete(FixtureAPITestCase):
run_2 = self.process_1.worker_runs.create(
version=version_2,
parents=[self.run_1.id],
- ttl=0,
)
run_3 = self.process_1.worker_runs.create(
version=version_3,
parents=[self.run_1.id, run_2.id],
- ttl=0,
)
self.assertTrue(self.run_1.id in run_2.parents)
diff --git a/arkindex/process/tests/worker_runs/test_list.py b/arkindex/process/tests/worker_runs/test_list.py
index 8b0698d4a482ba9d56369be6d0f8b0e9458a2cf0..10bf646f478e2d251ed5b8983cef61fab9e602a6 100644
--- a/arkindex/process/tests/worker_runs/test_list.py
+++ b/arkindex/process/tests/worker_runs/test_list.py
@@ -22,7 +22,7 @@ class TestWorkerRunsList(FixtureAPITestCase):
)
cls.version_1 = WorkerVersion.objects.get(worker__slug="reco")
cls.worker_1 = cls.version_1.worker
- cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1, ttl=0)
+ cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1)
cls.process_2 = cls.corpus.processes.create(creator=cls.user, mode=ProcessMode.Workers)
def test_list_requires_login(self):
@@ -94,14 +94,10 @@ class TestWorkerRunsList(FixtureAPITestCase):
},
"use_gpu": False,
"summary": "Worker Recognizer @ version 1",
- "ttl": 0,
}])
def test_list_filter_process(self):
- run_2 = self.process_2.worker_runs.create(
- version=self.version_1,
- ttl=0,
- )
+ run_2 = self.process_2.worker_runs.create(version=self.version_1)
self.client.force_login(self.user)
with self.assertNumQueries(6):
@@ -165,5 +161,4 @@ class TestWorkerRunsList(FixtureAPITestCase):
},
"use_gpu": False,
"summary": "Worker Recognizer @ version 1",
- "ttl": 0,
}])
diff --git a/arkindex/process/tests/worker_runs/test_partial_update.py b/arkindex/process/tests/worker_runs/test_partial_update.py
index cb7ebb4664e613a2b70e5e93637d876c793e527c..91c635b57c0d9709763bd2d623848b4191e53685 100644
--- a/arkindex/process/tests/worker_runs/test_partial_update.py
+++ b/arkindex/process/tests/worker_runs/test_partial_update.py
@@ -30,7 +30,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
cls.version_1 = WorkerVersion.objects.get(worker__slug="reco")
cls.worker_1 = cls.version_1.worker
cls.version_2 = WorkerVersion.objects.get(worker__slug="dla")
- cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1, ttl=1000)
+ cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1)
cls.configuration_1 = cls.worker_1.configurations.create(name="My config", configuration={"key": "value"})
worker_version = WorkerVersion.objects.exclude(worker=cls.version_1.worker).first()
cls.configuration_2 = worker_version.worker.configurations.create(name="Config")
@@ -95,6 +95,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
cls.agent = Agent.objects.create(
farm=cls.farm,
+ fingerprint="a" * 64,
hostname="claude",
cpu_cores=42,
cpu_frequency=1e15,
@@ -115,10 +116,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
version=2,
configuration={"test": "test2"}
)
- run_2 = self.process_1.worker_runs.create(
- version=version_2,
- ttl=0,
- )
+ run_2 = self.process_1.worker_runs.create(version=version_2)
with self.assertNumQueries(0):
response = self.client.patch(
@@ -154,10 +152,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
version=2,
configuration={"test": "test2"}
)
- run_2 = self.process_1.worker_runs.create(
- version=version_2,
- ttl=0,
- )
+ run_2 = self.process_1.worker_runs.create(version=version_2)
self.client.force_login(self.user)
with self.assertNumQueries(3):
@@ -173,7 +168,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
"""
A user cannot update a worker run on a local process
"""
- run = self.local_process.worker_runs.create(version=self.version_1, ttl=1000)
+ run = self.local_process.worker_runs.create(version=self.version_1)
self.client.force_login(self.user)
with self.assertNumQueries(5):
@@ -266,7 +261,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 1000,
"summary": "Worker Recognizer @ version 1",
})
self.run_1.refresh_from_db()
@@ -332,7 +326,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 1000,
"summary": "Worker Recognizer @ version 1",
})
self.run_1.refresh_from_db()
@@ -403,7 +396,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 1000,
"summary": "Worker Recognizer @ version 1 using configuration 'My config'",
})
self.assertEqual(self.run_1.configuration.id, self.configuration_1.id)
@@ -455,10 +447,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Disabled
)
- run_2 = self.process_1.worker_runs.create(
- version=version_no_model,
- ttl=1000,
- )
+ run_2 = self.process_1.worker_runs.create(version=version_no_model)
with self.assertNumQueries(5):
response = self.client.patch(
@@ -484,10 +473,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Required
)
- run_2 = self.process_1.worker_runs.create(
- version=version_no_model,
- ttl=1000,
- )
+ run_2 = self.process_1.worker_runs.create(version=version_no_model)
random_model_version_uuid = str(uuid.uuid4())
with self.assertNumQueries(4):
@@ -515,10 +501,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Required
)
- run_2 = self.process_1.worker_runs.create(
- version=version_no_model,
- ttl=1000,
- )
+ run_2 = self.process_1.worker_runs.create(version=version_no_model)
# Create a model version, the user has no access to
model_no_access = Model.objects.create(name="Secret model")
@@ -555,10 +538,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Required
)
- run_2 = self.process_1.worker_runs.create(
- version=version_no_model,
- ttl=1000,
- )
+ run_2 = self.process_1.worker_runs.create(version=version_no_model)
def filter_rights(user, model, level):
"""
@@ -619,10 +599,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Required
)
- run = self.process_1.worker_runs.create(
- version=version,
- ttl=1000,
- )
+ run = self.process_1.worker_runs.create(version=version)
self.model_version_1.state = ModelVersionState.Error
self.model_version_1.save()
@@ -648,7 +625,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Required
)
- run = self.process_1.worker_runs.create(version=version, ttl=1000)
+ run = self.process_1.worker_runs.create(version=version)
self.model_1.archived = datetime.now(timezone.utc)
self.model_1.save()
@@ -676,10 +653,7 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Required
)
- run = self.process_1.worker_runs.create(
- version=version_with_model,
- ttl=0,
- )
+ run = self.process_1.worker_runs.create(version=version_with_model)
self.assertIsNone(run.model_version_id)
self.assertEqual(run.summary, "Worker Recognizer @ version 2")
@@ -758,7 +732,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 0,
"summary": f"Worker Recognizer @ version 2 with model {model_version.model.name} @ {str(model_version.id)[:6]}",
})
self.assertEqual(run.model_version_id, model_version.id)
@@ -778,7 +751,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
)
run = self.process_1.worker_runs.create(
version=version_with_model,
- ttl=0,
configuration=self.configuration_1
)
self.assertEqual(run.model_version_id, None)
@@ -853,7 +825,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 0,
"summary": f"Worker Recognizer @ version 2 with model My model @ {str(self.model_version_1.id)[:6]} using configuration 'My config'",
})
self.assertEqual(run.model_version_id, self.model_version_1.id)
@@ -867,7 +838,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_2,
- ttl=0,
)
self.client.force_login(self.user)
@@ -926,7 +896,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 1000,
"summary": "Worker Recognizer @ version 1",
})
@@ -954,7 +923,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
version=self.version_1,
model_version=None if model_version else self.model_version_1,
configuration=None if configuration else self.configuration_1,
- ttl=0,
)
# Having a model version or a configuration adds one query for each
@@ -1005,7 +973,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
run = WorkerRun.objects.create(
process=self.process_1,
version=worker_version,
- ttl=0
)
self.assertEqual(run.use_gpu, True if worker_version.gpu_usage == FeatureUsage.Required else False)
with self.assertNumQueries(3):
@@ -1035,7 +1002,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
run = WorkerRun.objects.create(
process=self.process_1,
version=worker_version,
- ttl=0
)
self.assertEqual(run.use_gpu, True if worker_version.gpu_usage == FeatureUsage.Required else False)
@@ -1093,7 +1059,6 @@ class TestWorkerRunsPartialUpdate(FixtureAPITestCase):
},
"summary": f"Worker Recognizer @ version {worker_version.version}",
"use_gpu": use_gpu,
- "ttl": 0,
})
run.refresh_from_db()
self.assertEqual(run.use_gpu, use_gpu)
diff --git a/arkindex/process/tests/worker_runs/test_retrieve.py b/arkindex/process/tests/worker_runs/test_retrieve.py
index e253e85f25da5e605625ac07faa49e1906654593..0ffe02b790455b509122f2ae0416dd4f3d463845 100644
--- a/arkindex/process/tests/worker_runs/test_retrieve.py
+++ b/arkindex/process/tests/worker_runs/test_retrieve.py
@@ -35,12 +35,13 @@ class TestWorkerRunsRetrieve(FixtureAPITestCase):
cls.worker_custom = Worker.objects.get(slug="custom")
cls.version_custom = cls.worker_custom.versions.get()
- cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1, ttl=1000)
+ cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1)
cls.run_custom = cls.local_process.worker_runs.get(version=cls.version_custom)
cls.process_2 = cls.corpus.processes.create(creator=cls.user, mode=ProcessMode.Workers)
cls.agent = Agent.objects.create(
farm=cls.farm,
+ fingerprint="a" * 64,
hostname="claude",
cpu_cores=42,
cpu_frequency=1e15,
@@ -114,7 +115,6 @@ class TestWorkerRunsRetrieve(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 1000,
"summary": "Worker Recognizer @ version 1",
})
@@ -179,7 +179,6 @@ class TestWorkerRunsRetrieve(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 1000,
"summary": "Worker Recognizer @ version 1",
})
@@ -274,14 +273,13 @@ class TestWorkerRunsRetrieve(FixtureAPITestCase):
},
"summary": "Worker Custom worker @ version 1",
"use_gpu": False,
- "ttl": 0,
})
def test_retrieve_local(self):
"""
A user can retrieve a run on their own local process
"""
- run = self.local_process.worker_runs.create(version=self.version_1, ttl=0)
+ run = self.local_process.worker_runs.create(version=self.version_1)
self.client.force_login(self.user)
with self.assertNumQueries(5):
@@ -335,7 +333,6 @@ class TestWorkerRunsRetrieve(FixtureAPITestCase):
"prefix": None,
},
"use_gpu": False,
- "ttl": 0,
"summary": "Worker Recognizer @ version 1",
})
@@ -411,7 +408,6 @@ class TestWorkerRunsRetrieve(FixtureAPITestCase):
"prefix": None,
},
"use_gpu": False,
- "ttl": 1000,
"summary": "Worker Recognizer @ version 1",
})
diff --git a/arkindex/process/tests/worker_runs/test_ttl.py b/arkindex/process/tests/worker_runs/test_ttl.py
deleted file mode 100644
index edd98b6a9b2b821370a857a6a9022561487e2fa5..0000000000000000000000000000000000000000
--- a/arkindex/process/tests/worker_runs/test_ttl.py
+++ /dev/null
@@ -1,272 +0,0 @@
-from django.test import override_settings
-from django.urls import reverse
-from rest_framework import status
-
-from arkindex.ponos.models import Farm
-from arkindex.process.models import (
- ProcessMode,
- WorkerVersion,
-)
-from arkindex.project.tests import FixtureAPITestCase
-
-
-@override_settings(PONOS_MAXIMUM_TASK_TTL=3600)
-class TestWorkerRunTTL(FixtureAPITestCase):
-
- @classmethod
- def setUpTestData(cls):
- super().setUpTestData()
- cls.process = cls.corpus.processes.create(
- creator=cls.user,
- mode=ProcessMode.Workers,
- farm=Farm.objects.first(),
- )
- cls.recognizer = WorkerVersion.objects.get(worker__slug="reco")
- cls.dla = WorkerVersion.objects.get(worker__slug="dla")
- cls.worker_run = cls.process.worker_runs.create(version=cls.dla, ttl=0)
-
- def test_create_default_ttl(self):
- self.client.force_login(self.superuser)
- # Corpus TTL / WorkerRun TTL
- cases = [
- (0, 0),
- (10000, 10000),
- # No corpus TTL means the instance wide value should be set
- (None, 3600),
- ]
-
- for corpus_ttl, expected_ttl in cases:
- with self.subTest(corpus_ttl=corpus_ttl):
- self.process.worker_runs.filter(version=self.recognizer).delete()
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(6):
- response = self.client.post(
- reverse("api:worker-run-list", kwargs={"pk": str(self.process.id)}),
- {"worker_version_id": str(self.recognizer.id)},
- )
- self.assertEqual(response.status_code, status.HTTP_201_CREATED)
-
- data = response.json()
- self.assertEqual(data["ttl"], expected_ttl)
- run = self.process.worker_runs.get(id=data["id"])
- self.assertEqual(run.ttl, expected_ttl)
-
- def test_create_set_ttl(self):
- self.client.force_login(self.superuser)
- # Corpus TTL / WorkerRun TTL / Expected WorkerRun TTL
- cases = [
- (0, 0, 0),
- (0, 1000, 1000),
- (1800, 1000, 1000),
- (1800, 1800, 1800),
- # No corpus TTL means the instance wide value is the limit
- (None, 600, 600),
- (None, 3600, 3600),
- ]
-
- for corpus_ttl, worker_run_ttl, expected_ttl in cases:
- with self.subTest(corpus_ttl=corpus_ttl, worker_run_ttl=worker_run_ttl):
- self.process.worker_runs.filter(version=self.recognizer).delete()
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(6):
- response = self.client.post(
- reverse("api:worker-run-list", kwargs={"pk": str(self.process.id)}),
- {
- "worker_version_id": str(self.recognizer.id),
- "ttl": worker_run_ttl,
- },
- )
- self.assertEqual(response.status_code, status.HTTP_201_CREATED)
-
- data = response.json()
- self.assertEqual(data["ttl"], expected_ttl)
- run = self.process.worker_runs.get(id=data["id"])
- self.assertEqual(run.ttl, expected_ttl)
-
- def test_create_invalid_ttl(self):
- self.client.force_login(self.superuser)
- self.process.worker_runs.filter(version=self.recognizer).delete()
-
- # Corpus TTL, WorkerRun TTL, error message
- cases = [
- (None, "one hour", ["A valid integer is required."]),
- (None, -1, ["Ensure this value is greater than or equal to 1."]),
- (None, 0, ["Ensure this value is greater than or equal to 1."]),
- (None, 1e12, ["Ensure this value is less than or equal to 3600."]),
- (0, -1, ["Ensure this value is greater than or equal to 0."]),
- (0, 1e12, ["Ensure this value is less than or equal to 2147483647."]),
- (1800, -1, ["Ensure this value is greater than or equal to 1."]),
- (1800, 0, ["Ensure this value is greater than or equal to 1."]),
- (1800, 1e12, ["Ensure this value is less than or equal to 1800."]),
- ]
- for corpus_ttl, worker_run_ttl, expected_error in cases:
- with self.subTest(corpus_ttl=corpus_ttl, worker_run_ttl=worker_run_ttl):
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(4):
- response = self.client.post(
- reverse("api:worker-run-list", kwargs={"pk": str(self.process.id)}),
- {
- "worker_version_id": str(self.recognizer.id),
- "ttl": worker_run_ttl,
- },
- )
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
- self.assertEqual(response.json(), {
- "ttl": expected_error,
- })
-
- def test_partial_update_set_ttl(self):
- self.client.force_login(self.superuser)
- # Corpus TTL / WorkerRun TTL / Expected WorkerRun TTL
- cases = [
- (0, 0, 0),
- (0, 1000, 1000),
- (1800, 1000, 1000),
- (1800, 1800, 1800),
- # No corpus TTL means the instance wide value is the limit
- (None, 600, 600),
- (None, 3600, 3600),
- ]
-
- for corpus_ttl, worker_run_ttl, expected_ttl in cases:
- with self.subTest(corpus_ttl=corpus_ttl, worker_run_ttl=worker_run_ttl):
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(5):
- response = self.client.patch(
- reverse("api:worker-run-details", kwargs={"pk": str(self.worker_run.id)}),
- {"ttl": worker_run_ttl},
- )
- self.assertEqual(response.status_code, status.HTTP_200_OK)
-
- data = response.json()
- self.assertEqual(data["ttl"], expected_ttl)
- run = self.process.worker_runs.get(id=data["id"])
- self.assertEqual(run.ttl, expected_ttl)
-
- def test_partial_update_invalid_ttl(self):
- self.client.force_login(self.superuser)
-
- # Corpus TTL, WorkerRun TTL, error message
- cases = [
- (None, "one hour", ["A valid integer is required."]),
- (None, -1, ["Ensure this value is greater than or equal to 1."]),
- (None, 0, ["Ensure this value is greater than or equal to 1."]),
- (None, 1e12, ["Ensure this value is less than or equal to 3600."]),
- (0, -1, ["Ensure this value is greater than or equal to 0."]),
- (0, 1e12, ["Ensure this value is less than or equal to 2147483647."]),
- (1800, -1, ["Ensure this value is greater than or equal to 1."]),
- (1800, 0, ["Ensure this value is greater than or equal to 1."]),
- (1800, 1e12, ["Ensure this value is less than or equal to 1800."]),
- ]
- for corpus_ttl, worker_run_ttl, expected_error in cases:
- with self.subTest(corpus_ttl=corpus_ttl, worker_run_ttl=worker_run_ttl):
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(3):
- response = self.client.patch(
- reverse("api:worker-run-details", kwargs={"pk": str(self.worker_run.id)}),
- {"ttl": worker_run_ttl},
- )
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
- self.assertEqual(response.json(), {
- "ttl": expected_error,
- })
-
-
- def test_update_default_ttl(self):
- self.client.force_login(self.superuser)
- # Corpus TTL / WorkerRun TTL
- cases = [
- (0, 0),
- (10000, 10000),
- # No corpus TTL means the instance wide value should be set
- (None, 3600),
- ]
-
- for corpus_ttl, expected_ttl in cases:
- with self.subTest(corpus_ttl=corpus_ttl):
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(5):
- response = self.client.put(
- reverse("api:worker-run-details", kwargs={"pk": str(self.worker_run.id)}),
- {},
- )
- self.assertEqual(response.status_code, status.HTTP_200_OK)
-
- self.assertEqual(response.json()["ttl"], expected_ttl)
- self.worker_run.refresh_from_db()
- self.assertEqual(self.worker_run.ttl, expected_ttl)
-
- def test_update_set_ttl(self):
- self.client.force_login(self.superuser)
- # Corpus TTL / WorkerRun TTL / Expected WorkerRun TTL
- cases = [
- (0, 0, 0),
- (0, 1000, 1000),
- (1800, 1000, 1000),
- (1800, 1800, 1800),
- # No corpus TTL means the instance wide value is the limit
- (None, 600, 600),
- (None, 3600, 3600),
- ]
-
- for corpus_ttl, worker_run_ttl, expected_ttl in cases:
- with self.subTest(corpus_ttl=corpus_ttl, worker_run_ttl=worker_run_ttl):
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(5):
- response = self.client.put(
- reverse("api:worker-run-details", kwargs={"pk": str(self.worker_run.id)}),
- {"ttl": worker_run_ttl},
- )
- self.assertEqual(response.status_code, status.HTTP_200_OK)
-
- data = response.json()
- self.assertEqual(data["ttl"], expected_ttl)
- run = self.process.worker_runs.get(id=data["id"])
- self.assertEqual(run.ttl, expected_ttl)
-
- def test_update_invalid_ttl(self):
- self.client.force_login(self.superuser)
-
- # Corpus TTL, WorkerRun TTL, error message
- cases = [
- (None, "one hour", ["A valid integer is required."]),
- (None, -1, ["Ensure this value is greater than or equal to 1."]),
- (None, 0, ["Ensure this value is greater than or equal to 1."]),
- (None, 1e12, ["Ensure this value is less than or equal to 3600."]),
- (0, -1, ["Ensure this value is greater than or equal to 0."]),
- (0, 1e12, ["Ensure this value is less than or equal to 2147483647."]),
- (1800, -1, ["Ensure this value is greater than or equal to 1."]),
- (1800, 0, ["Ensure this value is greater than or equal to 1."]),
- (1800, 1e12, ["Ensure this value is less than or equal to 1800."]),
- ]
- for corpus_ttl, worker_run_ttl, expected_error in cases:
- with self.subTest(corpus_ttl=corpus_ttl, worker_run_ttl=worker_run_ttl):
- self.corpus.maximum_task_ttl = corpus_ttl
- self.corpus.save()
-
- with self.assertNumQueries(3):
- response = self.client.put(
- reverse("api:worker-run-details", kwargs={"pk": str(self.worker_run.id)}),
- {"ttl": worker_run_ttl},
- )
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
- self.assertEqual(response.json(), {
- "ttl": expected_error,
- })
diff --git a/arkindex/process/tests/worker_runs/test_update.py b/arkindex/process/tests/worker_runs/test_update.py
index b412d3b9da98f1ef6af1ef23d32e7bbbd519c701..a79cc7059ac668b4f611d3e284f1ce0c1b69f448 100644
--- a/arkindex/process/tests/worker_runs/test_update.py
+++ b/arkindex/process/tests/worker_runs/test_update.py
@@ -2,7 +2,6 @@ import uuid
from datetime import datetime, timezone
from unittest.mock import call, patch
-from django.test import override_settings
from django.urls import reverse
from rest_framework import status
from rest_framework.exceptions import ValidationError
@@ -14,7 +13,6 @@ from arkindex.training.models import Model, ModelVersion, ModelVersionState
from arkindex.users.models import Role
-@override_settings(PONOS_MAXIMUM_TASK_TTL=3600)
class TestWorkerRunsUpdate(FixtureAPITestCase):
"""
Test worker runs update endpoint
@@ -33,7 +31,7 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
cls.version_1 = WorkerVersion.objects.get(worker__slug="reco")
cls.worker_1 = cls.version_1.worker
cls.version_2 = WorkerVersion.objects.get(worker__slug="dla")
- cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1, ttl=1000)
+ cls.run_1 = cls.process_1.worker_runs.create(version=cls.version_1)
cls.configuration_1 = cls.worker_1.configurations.create(name="My config", configuration={"key": "value"})
worker_version = WorkerVersion.objects.exclude(worker=cls.version_1.worker).first()
cls.configuration_2 = worker_version.worker.configurations.create(name="Config")
@@ -71,6 +69,7 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
cls.agent = Agent.objects.create(
farm=cls.farm,
+ fingerprint="a" * 64,
hostname="claude",
cpu_cores=42,
cpu_frequency=1e15,
@@ -99,7 +98,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_2,
- ttl=0,
)
with self.assertNumQueries(0):
@@ -137,7 +135,7 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
"""
A user cannot update a worker run on a local process
"""
- run = self.local_process.worker_runs.create(version=self.version_1, ttl=1000)
+ run = self.local_process.worker_runs.create(version=self.version_1)
self.client.force_login(self.user)
with self.assertNumQueries(5):
@@ -161,7 +159,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_2,
- ttl=0,
)
self.client.force_login(self.user)
@@ -194,7 +191,7 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
def test_update_duplicate_parents(self):
self.client.force_login(self.user)
- run_2 = self.process_1.worker_runs.create(version=self.version_2, ttl=0)
+ run_2 = self.process_1.worker_runs.create(version=self.version_2)
with self.assertNumQueries(4):
response = self.client.put(
@@ -218,7 +215,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
"""
run_2 = self.process_1.worker_runs.create(
version=self.version_2,
- ttl=0,
)
run_2.parents = [self.run_1.id, self.run_1.id]
@@ -286,7 +282,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 3600,
"summary": "Worker Recognizer @ version 1",
})
self.run_1.refresh_from_db()
@@ -352,7 +347,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
},
},
"use_gpu": False,
- "ttl": 3600,
"summary": "Worker Recognizer @ version 1",
})
self.run_1.refresh_from_db()
@@ -426,7 +420,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 3600,
"summary": "Worker Recognizer @ version 1 using configuration 'My config'",
})
self.assertEqual(self.run_1.configuration.id, self.configuration_1.id)
@@ -481,7 +474,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_no_model,
- ttl=0,
)
with self.assertNumQueries(5):
@@ -511,7 +503,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_no_model,
- ttl=0,
)
random_model_version_uuid = str(uuid.uuid4())
@@ -543,7 +534,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_no_model,
- ttl=0,
)
# Create a model version, the user has no access to
@@ -590,7 +580,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_no_model,
- ttl=0,
)
def filter_rights(user, model, level):
@@ -655,7 +644,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run = self.process_1.worker_runs.create(
version=version,
- ttl=0,
)
self.model_version_1.state = ModelVersionState.Error
self.model_version_1.save()
@@ -682,7 +670,7 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
configuration={"test": "test2"},
model_usage=FeatureUsage.Required
)
- run = self.process_1.worker_runs.create(version=version, ttl=0)
+ run = self.process_1.worker_runs.create(version=version)
self.model_1.archived = datetime.now(timezone.utc)
self.model_1.save()
@@ -712,7 +700,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run = self.process_1.worker_runs.create(
version=version_with_model,
- ttl=0,
)
self.assertEqual(run.model_version, None)
# Check generated summary, before updating, there should be only information about the worker version
@@ -794,7 +781,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 3600,
"summary": f"Worker Recognizer @ version 2 with model {model_version.model.name} @ {str(model_version.id)[:6]}",
})
self.assertEqual(run.model_version_id, model_version.id)
@@ -813,7 +799,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run = self.process_1.worker_runs.create(
version=version_with_model,
- ttl=0,
)
self.assertIsNone(run.model_version)
self.assertIsNone(run.configuration)
@@ -892,7 +877,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 3600,
"summary": f"Worker Recognizer @ version 2 with model My model @ {str(self.model_version_1.id)[:6]} using configuration 'My config'",
})
self.assertEqual(run.model_version_id, self.model_version_1.id)
@@ -907,7 +891,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
)
run_2 = self.process_1.worker_runs.create(
version=version_2,
- ttl=0,
)
self.client.force_login(self.user)
@@ -916,7 +899,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
reverse("api:worker-run-details", kwargs={"pk": str(self.run_1.id)}),
data={
"parents": [str(run_2.id)],
- "ttl": 500,
},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -967,7 +949,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
}
},
"use_gpu": False,
- "ttl": 500,
"summary": "Worker Recognizer @ version 1",
})
@@ -995,7 +976,6 @@ class TestWorkerRunsUpdate(FixtureAPITestCase):
version=self.version_1,
model_version=None if model_version else self.model_version_1,
configuration=None if configuration else self.configuration_1,
- ttl=0,
)
# Having a model version or a configuration adds one query for each
diff --git a/arkindex/project/config.py b/arkindex/project/config.py
index b97f561fc4b63e234fb82432f7d4847c544aa4e8..71a74d8282f86f52f85d19884a9feffd139e3b46 100644
--- a/arkindex/project/config.py
+++ b/arkindex/project/config.py
@@ -91,7 +91,6 @@ def get_settings_parser(base_dir):
# SECURITY WARNING: keep the secret key used in production secret!
parser.add_option("secret_key", type=str, default="jf0w^y&ml(caax8f&a1mub)(js9(l5mhbbhosz3gi+m01ex+lo")
- parser.add_option("jwt_signing_key", type=str, default=None)
database_parser = parser.add_subparser("database", default={})
database_parser.add_option("name", type=str, default="arkindex_dev")
diff --git a/arkindex/project/settings.py b/arkindex/project/settings.py
index b9fbcad8c7af6fb75c9cf77979ea5ca34f330e6c..eaed15d931496e9e27069cdc2689654255972dff 100644
--- a/arkindex/project/settings.py
+++ b/arkindex/project/settings.py
@@ -14,7 +14,6 @@ import decimal
import os
import sys
import warnings
-from datetime import timedelta
from pathlib import Path
from textwrap import dedent
@@ -127,6 +126,7 @@ INSTALLED_APPS = [
"arkindex.users",
"arkindex.process",
"arkindex.training",
+ "arkindex.budget",
]
MIDDLEWARE = [
@@ -218,13 +218,6 @@ REST_FRAMEWORK = {
"TEST_REQUEST_DEFAULT_FORMAT": "json",
}
-SIMPLE_JWT = {
- "USER_ID_CLAIM": "agent_id",
- "ROTATE_REFRESH_TOKENS": True,
- "ACCESS_TOKEN_LIFETIME": timedelta(hours=6),
- "SIGNING_KEY": conf["jwt_signing_key"] or SECRET_KEY,
-}
-
SPECTACULAR_SETTINGS = {
"CAMELIZE_NAMES": True,
# Remove the automatically generated `description` that lists the members of all enums,
diff --git a/arkindex/project/tests/__init__.py b/arkindex/project/tests/__init__.py
index f1bc9e90bae7adc959beb8f116aec90cfdfaad3d..fc326f20e0a4770d3b40de1649baeb3ab428ef58 100644
--- a/arkindex/project/tests/__init__.py
+++ b/arkindex/project/tests/__init__.py
@@ -77,7 +77,7 @@ class _AssertExactQueriesContext(CaptureQueriesContext):
items = enumerate(self.params)
for name, value in items:
- actual_sql = actual_sql.replace(value, f"{{{name}}}")
+ actual_sql = actual_sql.replace(str(value), f"{{{name}}}")
self.path.write_text(actual_sql)
except OSError as e:
diff --git a/arkindex/project/tests/config_samples/defaults.yaml b/arkindex/project/tests/config_samples/defaults.yaml
index 24aa0d51d4b66cab220df1ad131648ab26039348..d45fe54f5d5bd1450e367f7615e82c46675df887 100644
--- a/arkindex/project/tests/config_samples/defaults.yaml
+++ b/arkindex/project/tests/config_samples/defaults.yaml
@@ -60,7 +60,6 @@ job_timeouts:
send_verification_email: 120
task: 36000
worker_results_delete: 3600
-jwt_signing_key: null
local_imageserver_id: 1
metrics_port: 3000
ponos:
diff --git a/arkindex/project/tests/config_samples/errors.yaml b/arkindex/project/tests/config_samples/errors.yaml
index fd795eded5eb97f408d21b5263afabbb62e8f64e..455ae2d4ca25f153dff5577f2db6c455564b9f9e 100644
--- a/arkindex/project/tests/config_samples/errors.yaml
+++ b/arkindex/project/tests/config_samples/errors.yaml
@@ -43,7 +43,6 @@ job_timeouts:
task: ''
worker_results_delete: null
send_verification_email: lol
-jwt_signing_key: null
local_imageserver_id: 1
metrics_port: 12
ponos:
diff --git a/arkindex/project/tests/config_samples/override.yaml b/arkindex/project/tests/config_samples/override.yaml
index fed278c1c26d9b17837037b68b9b6922cffe7745..52b0d9869fc5863f513130342cc049b868d7910c 100644
--- a/arkindex/project/tests/config_samples/override.yaml
+++ b/arkindex/project/tests/config_samples/override.yaml
@@ -75,7 +75,6 @@ job_timeouts:
send_verification_email: 10
task: 11
worker_results_delete: 12
-jwt_signing_key: deadbeef
local_imageserver_id: 45
metrics_port: 4242
ponos:
diff --git a/arkindex/project/tests/test_gis.py b/arkindex/project/tests/test_gis.py
index 1e0dfde3157242c3f5c6345c0116a5622b35a29d..9d3c7647aa6098b127517059988bacbb40d25aee 100644
--- a/arkindex/project/tests/test_gis.py
+++ b/arkindex/project/tests/test_gis.py
@@ -1,5 +1,5 @@
from django.contrib.gis.geos import LinearRing, LineString, Point
-from psycopg2 import Binary
+from psycopg.sql import quote
from arkindex.documents.models import Element
from arkindex.project.gis import ensure_linear_ring
@@ -69,8 +69,8 @@ class TestGis(FixtureTestCase):
"""
polygon = LinearRing((0, 0), (0, 10), (10, 10), (10, 0), (0, 0))
- # psycopg2.Binary provides the encoding from a geometry's Extended Well Known Binary to a PostgreSQL bytea
- encoded_polygon = str(Binary(bytes(polygon.ewkb)))
+ # psycopg.Binary provides the encoding from a geometry's Extended Well Known Binary to a PostgreSQL bytea
+ encoded_polygon = quote(bytes(polygon.ewkb))
self.assertEqual(
str(Element.objects.filter(polygon=polygon).only("id").query),
diff --git a/arkindex/sql_validation/add_first_parent.sql b/arkindex/sql_validation/add_first_parent.sql
index 296ee48210be34bf8a23848b1b3586df69cbf9cf..31f0ed72bcf776a20f4d58f49ff83d91553a819a 100644
--- a/arkindex/sql_validation/add_first_parent.sql
+++ b/arkindex/sql_validation/add_first_parent.sql
@@ -30,7 +30,7 @@ SELECT EXISTS
(SELECT COALESCE(MAX(ordering) + 1, 0)
FROM documents_elementpath
WHERE path @> ARRAY['{A}'::uuid]
- AND path[array_length(path, 1)] = '{A}'::uuid ) ;
+ AND path[array_length(path, 1)] = '{A}'::uuid ) ;
INSERT INTO documents_elementpath (id, element_id, path, ordering)
SELECT gen_random_uuid(),
@@ -39,16 +39,15 @@ SELECT gen_random_uuid(),
1
FROM documents_elementpath
WHERE element_id = '{A}'::uuid
- AND path <> ARRAY['{first_parent}'::uuid];
+ AND path <> '{{{first_parent}}}'::uuid[];
UPDATE "documents_elementpath"
-SET "path" = ARRAY['{first_parent}'::uuid,
- '{A}'::uuid]::uuid[], "ordering" = 1
+SET "path" = '{{{first_parent},{A}}}'::uuid[]::uuid[], "ordering" = 1
WHERE ("documents_elementpath"."element_id" = '{B}'::uuid
AND "documents_elementpath"."path" = '{{}}'::uuid[]);
UPDATE "documents_elementpath"
-SET "path" = array_cat(ARRAY['{first_parent}'::uuid, '{A}'::uuid], "documents_elementpath"."path")::uuid[]
+SET "path" = array_cat('{{{first_parent},{A}}}'::uuid[], "documents_elementpath"."path")::uuid[]
WHERE "documents_elementpath"."path" && (ARRAY['{B}'::uuid])::uuid[];
INSERT INTO documents_elementpath (id, element_id, path, ordering)
@@ -58,10 +57,8 @@ SELECT gen_random_uuid(),
child_paths.ordering
FROM documents_elementpath child_paths,
documents_elementpath new_parent_paths
-WHERE child_paths.path @> ARRAY['{first_parent}'::uuid,
- '{A}'::uuid,
- '{B}'::uuid]
+WHERE child_paths.path @> '{{{first_parent},{A},{B}}}'::uuid[]
AND new_parent_paths.element_id = '{A}'::uuid
- AND new_parent_paths.path <> ARRAY['{first_parent}'::uuid];
+ AND new_parent_paths.path <> '{{{first_parent}}}'::uuid[];
RELEASE SAVEPOINT "{savepoint}"
diff --git a/arkindex/sql_validation/add_second_parent.sql b/arkindex/sql_validation/add_second_parent.sql
index 27375cfedebc965bf8443bed799a01847d6a589c..ad0fb7c23ec502d4152d0519416c5bd24bc1ff5e 100644
--- a/arkindex/sql_validation/add_second_parent.sql
+++ b/arkindex/sql_validation/add_second_parent.sql
@@ -30,7 +30,7 @@ SELECT EXISTS
(SELECT COALESCE(MAX(ordering) + 1, 0)
FROM documents_elementpath
WHERE path @> ARRAY['{A}'::uuid]
- AND path[array_length(path, 1)] = '{A}'::uuid ) ;
+ AND path[array_length(path, 1)] = '{A}'::uuid ) ;
INSERT INTO documents_elementpath (id, element_id, path, ordering)
SELECT gen_random_uuid(),
@@ -47,8 +47,7 @@ SELECT gen_random_uuid(),
child_paths.ordering
FROM documents_elementpath child_paths,
documents_elementpath new_parent_paths
-WHERE child_paths.path @> ARRAY['{K}'::uuid,
- '{B}'::uuid]
+WHERE child_paths.path @> '{{{K},{B}}}'::uuid[]
AND new_parent_paths.element_id = '{A}'::uuid ;
RELEASE SAVEPOINT "{savepoint}"
diff --git a/arkindex/sql_validation/corpus_delete.sql b/arkindex/sql_validation/corpus_delete.sql
index bf5f22713085f427c02cea00062664ba4268beab..f789e6f8c7cd5ca1f78a0ef925f011695f106e65 100644
--- a/arkindex/sql_validation/corpus_delete.sql
+++ b/arkindex/sql_validation/corpus_delete.sql
@@ -6,7 +6,8 @@ SELECT "documents_corpus"."created",
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
"documents_corpus"."indexable",
- "documents_corpus"."maximum_task_ttl"
+ "documents_corpus"."maximum_task_ttl",
+ "documents_corpus"."budget_id"
FROM "documents_corpus"
WHERE "documents_corpus"."id" = '{corpus_id}'::uuid
LIMIT 21;
diff --git a/arkindex/sql_validation/corpus_delete_top_level_type.sql b/arkindex/sql_validation/corpus_delete_top_level_type.sql
index dbbe418e7389f4b1d9b79c37984632511f432571..d3ccdb0d2d4ceec1105a22f3a6e0dfbd97be161f 100644
--- a/arkindex/sql_validation/corpus_delete_top_level_type.sql
+++ b/arkindex/sql_validation/corpus_delete_top_level_type.sql
@@ -6,7 +6,8 @@ SELECT "documents_corpus"."created",
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
"documents_corpus"."indexable",
- "documents_corpus"."maximum_task_ttl"
+ "documents_corpus"."maximum_task_ttl",
+ "documents_corpus"."budget_id"
FROM "documents_corpus"
WHERE "documents_corpus"."id" = '{corpus_id}'::uuid
LIMIT 21;
diff --git a/arkindex/sql_validation/corpus_rights_filter.sql b/arkindex/sql_validation/corpus_rights_filter.sql
index 6456c7da350966b8be2f82ef0215810931153463..f914e4f909827dbddd3783b6a5e44ac19f9bb342 100644
--- a/arkindex/sql_validation/corpus_rights_filter.sql
+++ b/arkindex/sql_validation/corpus_rights_filter.sql
@@ -23,6 +23,7 @@ SELECT "documents_corpus"."created",
"documents_corpus"."public",
"documents_corpus"."indexable",
"documents_corpus"."maximum_task_ttl",
+ "documents_corpus"."budget_id",
LEAST("users_right"."level", T5."level") AS "max_level"
FROM "documents_corpus"
INNER JOIN "users_right" ON ("documents_corpus"."id" = "users_right"."content_id"
diff --git a/arkindex/sql_validation/corpus_rights_filter_public.sql b/arkindex/sql_validation/corpus_rights_filter_public.sql
index 60427780297b6d7fb4ab6a5d61865cdbc63bc03e..d46840b4d4d4ed8862c234a1e7a708f7d139b113 100644
--- a/arkindex/sql_validation/corpus_rights_filter_public.sql
+++ b/arkindex/sql_validation/corpus_rights_filter_public.sql
@@ -24,6 +24,7 @@ LIMIT 21;
"documents_corpus"."public",
"documents_corpus"."indexable",
"documents_corpus"."maximum_task_ttl",
+ "documents_corpus"."budget_id",
LEAST("users_right"."level", T5."level") AS "max_level"
FROM "documents_corpus"
INNER JOIN "users_right" ON ("documents_corpus"."id" = "users_right"."content_id"
@@ -44,6 +45,7 @@ UNION
"documents_corpus"."public",
"documents_corpus"."indexable",
"documents_corpus"."maximum_task_ttl",
+ "documents_corpus"."budget_id",
10 AS "max_level"
FROM "documents_corpus"
WHERE "documents_corpus"."public")
diff --git a/arkindex/sql_validation/element_dot_delete.sql b/arkindex/sql_validation/element_dot_delete.sql
index 31722f6a59efd51b8306722e36224174ad02f723..c28d23ce6ee67814379395dd95fb1a9e9831aa5b 100644
--- a/arkindex/sql_validation/element_dot_delete.sql
+++ b/arkindex/sql_validation/element_dot_delete.sql
@@ -1,59 +1,75 @@
-DELETE FROM documents_transcriptionentity WHERE transcription_id IN (
- SELECT t.id FROM documents_transcription t
- LEFT JOIN documents_elementpath elementpath USING (element_id)
- WHERE t.element_id = '{id}'::uuid OR elementpath.path && ARRAY['{id}'::uuid]
-) ;
+DELETE
+FROM documents_transcriptionentity
+WHERE transcription_id IN
+ (SELECT t.id
+ FROM documents_transcription t
+ LEFT JOIN documents_elementpath elementpath USING (element_id)
+ WHERE t.element_id = '{id}'::uuid
+ OR elementpath.path && ARRAY['{id}'::uuid] ) ;
-DELETE FROM documents_transcription
+DELETE
+FROM documents_transcription
WHERE element_id = '{id}'::uuid
-OR element_id IN (
- SELECT element_id FROM documents_elementpath WHERE path && ARRAY['{id}'::uuid]
-) ;
+ OR element_id IN
+ (SELECT element_id
+ FROM documents_elementpath
+ WHERE path && ARRAY['{id}'::uuid] ) ;
-DELETE FROM documents_classification
+DELETE
+FROM documents_classification
WHERE element_id = '{id}'::uuid
-OR element_id IN (
- SELECT element_id FROM documents_elementpath WHERE path && ARRAY['{id}'::uuid]
-) ;
+ OR element_id IN
+ (SELECT element_id
+ FROM documents_elementpath
+ WHERE path && ARRAY['{id}'::uuid] ) ;
-DELETE FROM documents_metadata
+DELETE
+FROM documents_metadata
WHERE element_id = '{id}'::uuid
-OR element_id IN (
- SELECT element_id FROM documents_elementpath WHERE path && ARRAY['{id}'::uuid]
- ) ;
+ OR element_id IN
+ (SELECT element_id
+ FROM documents_elementpath
+ WHERE path && ARRAY['{id}'::uuid] ) ;
-DELETE FROM process_processelement
+DELETE
+FROM process_processelement
WHERE element_id = '{id}'::uuid
-OR element_id IN (
- SELECT element_id FROM documents_elementpath WHERE path && ARRAY['{id}'::uuid]
-) ;
+ OR element_id IN
+ (SELECT element_id
+ FROM documents_elementpath
+ WHERE path && ARRAY['{id}'::uuid] ) ;
UPDATE process_process
SET element_id = NULL
WHERE element_id = '{id}'::uuid
-OR element_id IN (
- SELECT element_id FROM documents_elementpath WHERE path && ARRAY['{id}'::uuid]
-) ;
+ OR element_id IN
+ (SELECT element_id
+ FROM documents_elementpath
+ WHERE path && ARRAY['{id}'::uuid] ) ;
-DELETE FROM documents_selection selection
+DELETE
+FROM documents_selection selection
WHERE element_id = '{id}'::uuid
-OR element_id IN (
- SELECT element_id FROM documents_elementpath WHERE path && ARRAY['{id}'::uuid]
-) ;
+ OR element_id IN
+ (SELECT element_id
+ FROM documents_elementpath
+ WHERE path && ARRAY['{id}'::uuid] ) ;
-DELETE FROM process_workeractivity
+DELETE
+FROM process_workeractivity
WHERE element_id = '{id}'::uuid
-OR element_id IN (
- SELECT element_id FROM documents_elementpath WHERE path && ARRAY['{id}'::uuid]
-) ;
-
-WITH children_ids (id) AS (
- DELETE FROM documents_elementpath
- WHERE element_id = '{id}'::uuid OR path && ARRAY['{id}'::uuid]
- RETURNING element_id
-)
-DELETE FROM documents_element element
-USING children_ids
+ OR element_id IN
+ (SELECT element_id
+ FROM documents_elementpath
+ WHERE path && ARRAY['{id}'::uuid] ) ;
+
+WITH children_ids (id) AS
+ (DELETE
+ FROM documents_elementpath
+ WHERE element_id = '{id}'::uuid
+ OR path && ARRAY['{id}'::uuid] RETURNING element_id)
+DELETE
+FROM documents_element element USING children_ids
WHERE element.id = children_ids.id ;
DELETE
diff --git a/arkindex/sql_validation/element_move_with_children.sql b/arkindex/sql_validation/element_move_with_children.sql
index 354e208d6070f14d16b1290e5228ee07924927c4..9466ccaeb2ec438e36fb2b593ec58a3ab1d83d89 100644
--- a/arkindex/sql_validation/element_move_with_children.sql
+++ b/arkindex/sql_validation/element_move_with_children.sql
@@ -21,10 +21,8 @@ WHERE element_id = '{source_id}'::uuid ;
UPDATE documents_elementpath
SET path = path[2:]
-WHERE path @> ARRAY['{parent_id}'::uuid,
- '{source_id}'::uuid]
- AND path[0:2] = ARRAY['{parent_id}'::uuid,
- '{source_id}'::uuid] ;
+WHERE path @> '{{{parent_id},{source_id}}}'::uuid[]
+ AND path[0:2] = '{{{parent_id},{source_id}}}'::uuid[] ;
UPDATE "documents_elementpath"
SET "path" = '{{}}'::uuid[]
@@ -65,15 +63,15 @@ SELECT EXISTS
(SELECT COALESCE(MAX(ordering) + 1, 0)
FROM documents_elementpath
WHERE path @> ARRAY['{destination_id}'::uuid]
- AND path[array_length(path, 1)] = '{destination_id}'::uuid ) ;
+ AND path[array_length(path, 1)] = '{destination_id}'::uuid ) ;
UPDATE "documents_elementpath"
-SET "path" = ARRAY['{destination_id}'::uuid]::uuid[], "ordering" = 3
+SET "path" = '{{{destination_id}}}'::uuid[]::uuid[], "ordering" = 3
WHERE ("documents_elementpath"."element_id" = '{source_id}'::uuid
AND "documents_elementpath"."path" = '{{}}'::uuid[]);
UPDATE "documents_elementpath"
-SET "path" = array_cat(ARRAY['{destination_id}'::uuid], "documents_elementpath"."path")::uuid[]
+SET "path" = array_cat('{{{destination_id}}}'::uuid[], "documents_elementpath"."path")::uuid[]
WHERE "documents_elementpath"."path" && (ARRAY['{source_id}'::uuid])::uuid[];
RELEASE SAVEPOINT "{savepoints[1]}"
diff --git a/arkindex/sql_validation/element_move_without_child.sql b/arkindex/sql_validation/element_move_without_child.sql
index 354e208d6070f14d16b1290e5228ee07924927c4..9466ccaeb2ec438e36fb2b593ec58a3ab1d83d89 100644
--- a/arkindex/sql_validation/element_move_without_child.sql
+++ b/arkindex/sql_validation/element_move_without_child.sql
@@ -21,10 +21,8 @@ WHERE element_id = '{source_id}'::uuid ;
UPDATE documents_elementpath
SET path = path[2:]
-WHERE path @> ARRAY['{parent_id}'::uuid,
- '{source_id}'::uuid]
- AND path[0:2] = ARRAY['{parent_id}'::uuid,
- '{source_id}'::uuid] ;
+WHERE path @> '{{{parent_id},{source_id}}}'::uuid[]
+ AND path[0:2] = '{{{parent_id},{source_id}}}'::uuid[] ;
UPDATE "documents_elementpath"
SET "path" = '{{}}'::uuid[]
@@ -65,15 +63,15 @@ SELECT EXISTS
(SELECT COALESCE(MAX(ordering) + 1, 0)
FROM documents_elementpath
WHERE path @> ARRAY['{destination_id}'::uuid]
- AND path[array_length(path, 1)] = '{destination_id}'::uuid ) ;
+ AND path[array_length(path, 1)] = '{destination_id}'::uuid ) ;
UPDATE "documents_elementpath"
-SET "path" = ARRAY['{destination_id}'::uuid]::uuid[], "ordering" = 3
+SET "path" = '{{{destination_id}}}'::uuid[]::uuid[], "ordering" = 3
WHERE ("documents_elementpath"."element_id" = '{source_id}'::uuid
AND "documents_elementpath"."path" = '{{}}'::uuid[]);
UPDATE "documents_elementpath"
-SET "path" = array_cat(ARRAY['{destination_id}'::uuid], "documents_elementpath"."path")::uuid[]
+SET "path" = array_cat('{{{destination_id}}}'::uuid[], "documents_elementpath"."path")::uuid[]
WHERE "documents_elementpath"."path" && (ARRAY['{source_id}'::uuid])::uuid[];
RELEASE SAVEPOINT "{savepoints[1]}"
diff --git a/arkindex/sql_validation/indexer_prefetch.sql b/arkindex/sql_validation/indexer_prefetch.sql
index 74c6e2bc86bd08c3d908b82bbe0186e9c9f945b8..31118a36fb017e3469755d35d223db5b4a8e2bef 100644
--- a/arkindex/sql_validation/indexer_prefetch.sql
+++ b/arkindex/sql_validation/indexer_prefetch.sql
@@ -5,7 +5,7 @@ SELECT element.id AS parent_id,
element.name AS name,
elementtype.display_name AS type_name,
element.image_id AS image_id,
- element.polygon::bytea AS polygon,
+ element.polygon AS polygon,
element.worker_run_id AS worker_run_id
FROM documents_element element
INNER JOIN documents_elementtype elementtype ON (elementtype.id = element.type_id)
@@ -21,7 +21,7 @@ WITH parent AS
element.name AS name,
elementtype.display_name AS type_name,
element.image_id AS image_id,
- element.polygon::bytea AS polygon,
+ element.polygon AS polygon,
element.worker_run_id AS worker_run_id
FROM documents_element element
INNER JOIN documents_elementtype elementtype ON (elementtype.id = element.type_id)
@@ -35,7 +35,7 @@ SELECT parent_id,
element.name as name,
elementtype.display_name as type_name,
element.image_id AS image_id,
- element.polygon::bytea AS polygon,
+ element.polygon AS polygon,
element.worker_run_id AS worker_run_id
FROM
(SELECT *
@@ -56,8 +56,7 @@ SELECT "process_workerrun"."id",
"process_workerrun"."created",
"process_workerrun"."updated",
"process_workerrun"."has_results",
- "process_workerrun"."use_gpu",
- "process_workerrun"."ttl"
+ "process_workerrun"."use_gpu"
FROM "process_workerrun"
WHERE "process_workerrun"."id" IN ('{worker_run_id}'::uuid);
@@ -152,8 +151,7 @@ SELECT "process_workerrun"."id",
"process_workerrun"."created",
"process_workerrun"."updated",
"process_workerrun"."has_results",
- "process_workerrun"."use_gpu",
- "process_workerrun"."ttl"
+ "process_workerrun"."use_gpu"
FROM "process_workerrun"
WHERE "process_workerrun"."id" IN ('{worker_run_id}'::uuid);
diff --git a/arkindex/sql_validation/list_elements.sql b/arkindex/sql_validation/list_elements.sql
index 98e3284cfa465154415b194f505440a0ae88d3b5..1a542e371a3e03cecff3059fe69f92ab010b7ee5 100644
--- a/arkindex/sql_validation/list_elements.sql
+++ b/arkindex/sql_validation/list_elements.sql
@@ -6,7 +6,8 @@ SELECT "documents_corpus"."created",
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
"documents_corpus"."indexable",
- "documents_corpus"."maximum_task_ttl"
+ "documents_corpus"."maximum_task_ttl",
+ "documents_corpus"."budget_id"
FROM "documents_corpus"
WHERE "documents_corpus"."id" = '{corpus_id}'::uuid
LIMIT 21;
@@ -26,7 +27,7 @@ SELECT "documents_element"."id",
"documents_element"."worker_version_id",
"documents_element"."worker_run_id",
"documents_element"."image_id",
- "documents_element"."polygon"::bytea,
+ "documents_element"."polygon",
"documents_element"."rotation_angle",
"documents_element"."mirrored",
"documents_element"."confidence",
@@ -40,8 +41,7 @@ SELECT "documents_element"."id",
"process_workerrun"."created",
"process_workerrun"."updated",
"process_workerrun"."has_results",
- "process_workerrun"."use_gpu",
- "process_workerrun"."ttl"
+ "process_workerrun"."use_gpu"
FROM "documents_element"
LEFT OUTER JOIN "process_workerrun" ON ("documents_element"."worker_run_id" = "process_workerrun"."id")
WHERE ("documents_element"."corpus_id" = '{corpus_id}'::uuid
diff --git a/arkindex/sql_validation/process_elements_filter_ml_class.sql b/arkindex/sql_validation/process_elements_filter_ml_class.sql
index fb1e9d4389cea1361a85639375aad08f83c50b18..bc15a46955079de959819a7ad85860be0e2e951f 100644
--- a/arkindex/sql_validation/process_elements_filter_ml_class.sql
+++ b/arkindex/sql_validation/process_elements_filter_ml_class.sql
@@ -52,7 +52,8 @@ SELECT "documents_corpus"."created",
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
"documents_corpus"."indexable",
- "documents_corpus"."maximum_task_ttl"
+ "documents_corpus"."maximum_task_ttl",
+ "documents_corpus"."budget_id"
FROM "documents_corpus"
WHERE "documents_corpus"."id" = '{corpus_id}'::uuid
LIMIT 21;
diff --git a/arkindex/sql_validation/process_elements_filter_type.sql b/arkindex/sql_validation/process_elements_filter_type.sql
index a566bc4a782d58d2fe29839372458bf19dfb37d8..931808c1ed99f5d98d030add763a5103a9bbee20 100644
--- a/arkindex/sql_validation/process_elements_filter_type.sql
+++ b/arkindex/sql_validation/process_elements_filter_type.sql
@@ -52,7 +52,8 @@ SELECT "documents_corpus"."created",
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
"documents_corpus"."indexable",
- "documents_corpus"."maximum_task_ttl"
+ "documents_corpus"."maximum_task_ttl",
+ "documents_corpus"."budget_id"
FROM "documents_corpus"
WHERE "documents_corpus"."id" = '{corpus_id}'::uuid
LIMIT 21;
diff --git a/arkindex/sql_validation/process_elements_top_level.sql b/arkindex/sql_validation/process_elements_top_level.sql
index 77423e4582a0ca0620335a39ec096af91e4ee5b0..d10220c4fe27cfe33da7fe6b9eaf4aaf67543ff2 100644
--- a/arkindex/sql_validation/process_elements_top_level.sql
+++ b/arkindex/sql_validation/process_elements_top_level.sql
@@ -52,7 +52,8 @@ SELECT "documents_corpus"."created",
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
"documents_corpus"."indexable",
- "documents_corpus"."maximum_task_ttl"
+ "documents_corpus"."maximum_task_ttl",
+ "documents_corpus"."budget_id"
FROM "documents_corpus"
WHERE "documents_corpus"."id" = '{corpus_id}'::uuid
LIMIT 21;
diff --git a/arkindex/sql_validation/process_elements_with_image.sql b/arkindex/sql_validation/process_elements_with_image.sql
index 2b6d311781ba790d1e63af7d571895f2c1beba55..75a14883916f3a452c2c789c71309237b4fe83e5 100644
--- a/arkindex/sql_validation/process_elements_with_image.sql
+++ b/arkindex/sql_validation/process_elements_with_image.sql
@@ -52,7 +52,8 @@ SELECT "documents_corpus"."created",
"documents_corpus"."top_level_type_id",
"documents_corpus"."public",
"documents_corpus"."indexable",
- "documents_corpus"."maximum_task_ttl"
+ "documents_corpus"."maximum_task_ttl",
+ "documents_corpus"."budget_id"
FROM "documents_corpus"
WHERE "documents_corpus"."id" = '{corpus_id}'::uuid
LIMIT 21;
@@ -70,7 +71,7 @@ SELECT "documents_element"."id",
"documents_element"."image_id",
"images_image"."width",
"images_image"."height",
- "documents_element"."polygon"::bytea,
+ "documents_element"."polygon",
"documents_element"."rotation_angle",
"documents_element"."mirrored",
NULLIF(CONCAT((RTRIM("images_imageserver"."url", '/'))::text, (CONCAT(('/')::text, ("images_image"."path")::text))::text), '/') AS "image_url"
diff --git a/arkindex/sql_validation/remove_child_last_parent.sql b/arkindex/sql_validation/remove_child_last_parent.sql
index fa81733d12fb29f028d5348cb35ed6d96794c7df..c44943c5c5d5fecb42cfe93376550a70bf69b7b2 100644
--- a/arkindex/sql_validation/remove_child_last_parent.sql
+++ b/arkindex/sql_validation/remove_child_last_parent.sql
@@ -14,12 +14,8 @@ WHERE element_id = '{B}'::uuid ;
UPDATE documents_elementpath
SET path = path[3:]
-WHERE path @> ARRAY['{first_parent}'::uuid,
- '{A}'::uuid,
- '{B}'::uuid]
- AND path[0:3] = ARRAY['{first_parent}'::uuid,
- '{A}'::uuid,
- '{B}'::uuid] ;
+WHERE path @> '{{{first_parent},{A},{B}}}'::uuid[]
+ AND path[0:3] = '{{{first_parent},{A},{B}}}'::uuid[] ;
UPDATE "documents_elementpath"
SET "path" = '{{}}'::uuid[]
diff --git a/arkindex/sql_validation/remove_children_multiple_parents.sql b/arkindex/sql_validation/remove_children_multiple_parents.sql
index c1565c51de3235b33068628fec68dd730438e3c6..03cfbf7a930623288d84db198474463dc58fcde5 100644
--- a/arkindex/sql_validation/remove_children_multiple_parents.sql
+++ b/arkindex/sql_validation/remove_children_multiple_parents.sql
@@ -10,7 +10,7 @@ LIMIT 1;
DELETE
FROM documents_elementpath child_paths USING documents_elementpath parent_paths
WHERE parent_paths.element_id = '{A}'::uuid
- AND parent_paths.path <> ARRAY['{first_parent}'::uuid]
+ AND parent_paths.path <> '{{{first_parent}}}'::uuid[]
AND child_paths.path @> (parent_paths.path || '{A}'::uuid) ;
DELETE
@@ -21,9 +21,7 @@ WHERE parent_paths.path && ARRAY['{A}'::uuid]
UPDATE documents_elementpath
SET path = path[2 + 1:]
-WHERE path @> ARRAY['{first_parent}'::uuid,
- '{A}'::uuid]
- AND path[:2] = ARRAY['{first_parent}'::uuid,
- '{A}'::uuid] ;
+WHERE path @> '{{{first_parent},{A}}}'::uuid[]
+ AND path[:2] = '{{{first_parent},{A}}}'::uuid[] ;
RELEASE SAVEPOINT "{savepoint}"
diff --git a/arkindex/sql_validation/remove_children_no_parents.sql b/arkindex/sql_validation/remove_children_no_parents.sql
index dba5386430e325efe67307007ec0b1af839f9177..1c939384a7fb430f580ee82865888dd7de999ab8 100644
--- a/arkindex/sql_validation/remove_children_no_parents.sql
+++ b/arkindex/sql_validation/remove_children_no_parents.sql
@@ -15,7 +15,7 @@ WHERE parent_paths.path && ARRAY['{A}'::uuid]
UPDATE documents_elementpath
SET path = path[1 + 1:]
-WHERE path @> ARRAY['{A}'::uuid]
- AND path[:1] = ARRAY['{A}'::uuid] ;
+WHERE path @> '{{{A}}}'::uuid[]
+ AND path[:1] = '{{{A}}}'::uuid[] ;
RELEASE SAVEPOINT "{savepoint}"
diff --git a/arkindex/sql_validation/remove_children_single_parent.sql b/arkindex/sql_validation/remove_children_single_parent.sql
index 7d86e0d01945344eb36a0f4c88df5e20f5eaca33..e884f1d34f65b6c66dee27eb800eca041034546c 100644
--- a/arkindex/sql_validation/remove_children_single_parent.sql
+++ b/arkindex/sql_validation/remove_children_single_parent.sql
@@ -15,9 +15,7 @@ WHERE parent_paths.path && ARRAY['{A}'::uuid]
UPDATE documents_elementpath
SET path = path[2 + 1:]
-WHERE path @> ARRAY['{X}'::uuid,
- '{A}'::uuid]
- AND path[:2] = ARRAY['{X}'::uuid,
- '{A}'::uuid] ;
+WHERE path @> '{{{X},{A}}}'::uuid[]
+ AND path[:2] = '{{{X},{A}}}'::uuid[] ;
RELEASE SAVEPOINT "{savepoint}"
diff --git a/arkindex/sql_validation/workeractivity_bulk_insert_no_model.sql b/arkindex/sql_validation/workeractivity_bulk_insert_no_model.sql
index cd539dd953c2b4c3a74fd7ccc0009cc4da7b1ea5..461456a39e64b4b0a49370fbcd3314ee6df90cfb 100644
--- a/arkindex/sql_validation/workeractivity_bulk_insert_no_model.sql
+++ b/arkindex/sql_validation/workeractivity_bulk_insert_no_model.sql
@@ -15,7 +15,7 @@ FROM
WHERE ("documents_elementtype"."corpus_id" = '{corpus_id}'::uuid
AND "documents_elementtype"."slug" = 'act')) AS elt ON CONFLICT (worker_version_id,
element_id,
- configuration_id)
+ configuration_id)
WHERE configuration_id IS NOT NULL
AND model_version_id IS NULL DO
UPDATE
diff --git a/arkindex/system_workers.yml b/arkindex/system_workers.yml
index 03efc879b2a34400fa2164c4c93b761867682b87..7bf51563bf61461ac0e58537e2ed25a87035096c 100644
--- a/arkindex/system_workers.yml
+++ b/arkindex/system_workers.yml
@@ -1,7 +1,7 @@
# When releasing Arkindex, check that the Docker images set here are up to date,
# then update the `version` to the current Arkindex version as set in the `VERSION` file
# to confirm that the images have been manually checked.
-version: 1.7.2
+version: 1.7.3
features:
file_import:
@@ -10,7 +10,7 @@ features:
image: registry.gitlab.teklia.com/arkindex/workers/init-elements:0.1.1
command: worker-init-elements
s3_ingest:
- image: registry.gitlab.teklia.com/arkindex/workers/import/s3:0.2.0rc3
+ image: registry.gitlab.teklia.com/arkindex/workers/import/s3:0.2.0
pdf_export:
teklia_worker:
name: arkindex/workers/export
@@ -31,3 +31,8 @@ features:
name: arkindex/workers/export
version: 0.2.1
slug: csv-export
+ dataset_extractor:
+ teklia_worker:
+ name: arkindex/workers/generic-training-dataset
+ version: 0.3.0
+ slug: generic-training-dataset
diff --git a/arkindex/training/api.py b/arkindex/training/api.py
index b23502c88fa40f70063e5386a1d748f6cedffe78..fd56c44f0ad256b92d1c1bd4a74ac3ecb1f627dc 100644
--- a/arkindex/training/api.py
+++ b/arkindex/training/api.py
@@ -69,9 +69,16 @@ def _fetch_datasetelement_neighbors(datasetelements):
"""
if not datasetelements:
return datasetelements
+
+ # psycopg 3 does not support `IN %s`, so we use a variable amount of placeholders for each ID instead.
+ # https://www.psycopg.org/psycopg3/docs/basic/from_pg2.html#you-cannot-use-in-s-with-a-tuple
+ # We use named placeholders with list indices so that we can refer to the same ID twice,
+ # because we are filtering on DatasetElement IDs twice
+ placeholders = ",".join(f"%({i})s" for i in range(len(datasetelements)))
+
with connection.cursor() as cursor:
cursor.execute(
- """
+ f"""
WITH neighbors AS (
SELECT
n.id,
@@ -89,7 +96,7 @@ def _fetch_datasetelement_neighbors(datasetelements):
WHERE set_id IN (
SELECT set_id
FROM training_datasetelement
- WHERE id IN %(ids)s
+ WHERE id IN ({placeholders})
)
ORDER BY n.element_id
)
@@ -97,9 +104,9 @@ def _fetch_datasetelement_neighbors(datasetelements):
neighbors.id, neighbors.previous, neighbors.next
FROM
neighbors
- WHERE neighbors.id in %(ids)s
+ WHERE neighbors.id in ({placeholders})
""",
- {"ids": tuple(datasetelement.id for datasetelement in datasetelements)}
+ {str(i): datasetelement.id for i, datasetelement in enumerate(datasetelements)},
)
neighbors = {
diff --git a/base/Dockerfile b/base/Dockerfile
index 5bf809476cd2e090d7ec41a60b3a4184a4bad7db..1efe60a29f7c9116f288922d513c323341980f59 100644
--- a/base/Dockerfile
+++ b/base/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.10-slim-bookworm
+FROM python:3.12-slim-bookworm
# Install some runtime deps and python dependencies that are slow to install or require specific build deps
ADD requirements.txt bootstrap.sh /
diff --git a/base/requirements.txt b/base/requirements.txt
index 3d48b30e397a495b3470c6e592663d6ce13c060f..691940d4e599534cbf466336b306b711a50d9f12 100644
--- a/base/requirements.txt
+++ b/base/requirements.txt
@@ -1,6 +1,3 @@
-boto3==1.18.13
-cryptography==3.4.7
+boto3==1.36.16
+cryptography==44.0.1
Django==5.0.8
-ed25519==1.5
-lxml==4.9.2
-psycopg2-binary==2.9.1
diff --git a/requirements.txt b/requirements.txt
index fdfda8d85c4c1f1858e6c14a93588fde9d5aa9b9..29cf9f325bc4239d08d8d281d51889df9cd77162 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,5 @@
# -r ./base/requirements.txt
-
bleach==6.0.0
django-admin-hstore-widget==1.2.1
django-cors-headers==3.14.0
@@ -8,12 +7,11 @@ django-enumfields2==3.0.2
django-pgtrigger==4.7.0
django-rq==2.10.1
djangorestframework==3.15.2
-djangorestframework-simplejwt==5.2.2
docker==7.0.0
drf-spectacular==0.27.2
+psycopg[binary]==3.2.4
python-magic==0.4.27
python-memcached==1.59
-PyYAML==6.0
requests==2.28.2
rq==1.16.0
sentry-sdk==2.7.1
@@ -22,4 +20,4 @@ SolrClient==0.3.1
teklia-toolbox==0.1.3
tenacity==8.2.2
uritemplate==4.1.1
-zstandard==0.20.0
+zstandard==0.23.0
diff --git a/ruff.toml b/ruff.toml
index 8a02c65695d77b3c18cae483a6ff6db75abbd0ba..453bb5b5c9bb7afaa8413f7c29e6968ee5404854 100644
--- a/ruff.toml
+++ b/ruff.toml
@@ -45,7 +45,7 @@ known-third-party = [
"drf_spectacular",
"enumfields",
"gitlab",
- "psycopg2",
+ "psycopg",
"requests",
"responses",
"rest_framework",
diff --git a/setup.py b/setup.py
index fdd40bb80d4261c25a3fc17db605abceb2aa2cba..6dce684b6c90c29fe11f43d69c11405276647787 100755
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@ setup(
license_files=("LICENSE",),
description="Manuscripts indexation framework",
author="Teklia",
- author_email="abadie@teklia.com",
+ author_email="contact@teklia.com",
url="https://arkindex.teklia.com",
python_requires=">=3.10",
install_requires=install_requires,