Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • arkindex/backend
1 result
Show changes
Commits on Source (5)
Hide whitespace changes
Inline Side-by-side
Showing
with 68 additions and 8 deletions
VERSION
View file @ 925bca67
1.6.1-beta2
1.6.1-beta3
arkindex/documents/serializers/entities.py
View file @ 925bca67
......@@ -86,7 +86,7 @@ class BaseEntitySerializer(serializers.ModelSerializer):
@extend_schema_serializer(
deprecate_fields=("worker_version_id")
deprecate_fields=("worker_version_id", )
)
class EntitySerializer(BaseEntitySerializer):
"""
......
arkindex/ponos/models.py
View file @ 925bca67
......@@ -212,7 +212,7 @@ def expiry_default():
:rtype: datetime
"""
return timezone.now() + timedelta(days=30)
return timezone.now() + timedelta(days=settings.PONOS_TASK_EXPIRY)
def task_token_default():
......
arkindex/process/serializers/imports.py
View file @ 925bca67
......@@ -52,8 +52,9 @@ class ProcessLightSerializer(serializers.ModelSerializer):
"creator",
"activity_state",
"use_cache",
"chunks",
)
read_only_fields = ("id", "state", "mode", "corpus", "creator", "activity_state", "use_cache")
read_only_fields = ("id", "state", "mode", "corpus", "creator", "activity_state", "use_cache", "chunks",)
class CorpusWorkerRunProcessSerializer(ProcessLightSerializer):
......
arkindex/process/tests/test_corpus_worker_runs.py
View file @ 925bca67
......@@ -105,6 +105,7 @@ class TestCorpusWorkerRuns(FixtureAPITestCase):
"parents": [],
"process": {
"activity_state": "disabled",
"chunks": 1,
"corpus": None,
"element_id": str(self.local_process.element.id),
"element_count": 0,
......@@ -173,6 +174,7 @@ class TestCorpusWorkerRuns(FixtureAPITestCase):
"id": str(self.private_process.id),
"activity_state": "disabled",
"corpus": str(self.private_corpus.id),
"chunks": 1,
"element_id": None,
"element_count": 10,
"mode": "workers",
......
arkindex/process/tests/test_create_process.py
View file @ 925bca67
......@@ -101,6 +101,7 @@ class TestCreateProcess(FixtureAPITestCase):
"name": None,
"id": str(process.id),
"state": "unscheduled",
"chunks": 1,
"corpus": str(self.volume.corpus.id),
"files": [],
"mode": "workers",
......@@ -158,6 +159,7 @@ class TestCreateProcess(FixtureAPITestCase):
"id": str(process.id),
"state": "unscheduled",
"mode": "workers",
"chunks": 1,
"corpus": str(self.corpus.id),
"files": [],
"folder_type": None,
......@@ -380,6 +382,7 @@ class TestCreateProcess(FixtureAPITestCase):
"name": None,
"id": str(process.id),
"state": "unscheduled",
"chunks": 1,
"corpus": str(self.corpus.id),
"files": [],
"mode": "workers",
......@@ -417,6 +420,7 @@ class TestCreateProcess(FixtureAPITestCase):
"name": None,
"id": str(process.id),
"state": "unscheduled",
"chunks": 1,
"corpus": str(self.volume.corpus.id),
"files": [],
"mode": "workers",
......@@ -947,6 +951,7 @@ class TestCreateProcess(FixtureAPITestCase):
"name": None,
"id": str(process.id),
"state": "unscheduled",
"chunks": 1,
"corpus": str(self.corpus.id),
"files": [],
"mode": "workers",
......@@ -989,6 +994,7 @@ class TestCreateProcess(FixtureAPITestCase):
"name": "Test Process",
"id": str(process.id),
"state": "unscheduled",
"chunks": 1,
"corpus": str(self.corpus.id),
"files": [],
"mode": mode if mode else "workers",
......
arkindex/process/tests/test_processes.py
View file @ 925bca67
......@@ -145,6 +145,7 @@ class TestProcesses(FixtureAPITestCase):
"id": str(self.workers_process2.id),
"name": None,
"mode": "workers",
"chunks": 1,
"corpus": str(self.other_corpus.id),
"state": "unscheduled",
"activity_state": "disabled",
......@@ -159,6 +160,7 @@ class TestProcesses(FixtureAPITestCase):
"id": str(self.user_img_process.id),
"name": None,
"mode": "files",
"chunks": 1,
"corpus": str(self.private_corpus.id),
"state": "unscheduled",
"activity_state": "disabled",
......@@ -173,6 +175,7 @@ class TestProcesses(FixtureAPITestCase):
"id": str(self.elts_process.id),
"name": None,
"mode": "workers",
"chunks": 1,
"corpus": str(self.corpus.id),
"state": "unscheduled",
"activity_state": "disabled",
......@@ -187,6 +190,7 @@ class TestProcesses(FixtureAPITestCase):
"id": str(self.workers_process.id),
"name": "Process fixture",
"mode": "workers",
"chunks": 1,
"corpus": str(self.corpus.id),
"state": "unscheduled",
"activity_state": "disabled",
......@@ -226,6 +230,7 @@ class TestProcesses(FixtureAPITestCase):
"id": str(self.user_img_process.id),
"name": None,
"mode": "files",
"chunks": 1,
"corpus": str(self.private_corpus.id),
"state": "unscheduled",
"activity_state": "ready",
......@@ -261,6 +266,7 @@ class TestProcesses(FixtureAPITestCase):
"id": str(self.user_img_process.id),
"name": None,
"mode": "files",
"chunks": 1,
"corpus": str(self.private_corpus.id),
"state": "unscheduled",
"activity_state": "disabled",
......@@ -275,6 +281,7 @@ class TestProcesses(FixtureAPITestCase):
"id": str(self.workers_process.id),
"name": "Process fixture",
"mode": "workers",
"chunks": 1,
"corpus": str(self.corpus.id),
"state": "unscheduled",
"activity_state": "disabled",
......@@ -342,6 +349,7 @@ class TestProcesses(FixtureAPITestCase):
"id": str(self.local_process.id),
"name": None,
"mode": "local",
"chunks": 1,
"corpus": None,
"state": "unscheduled",
"activity_state": "disabled",
......@@ -384,6 +392,7 @@ class TestProcesses(FixtureAPITestCase):
"id": str(self.user_img_process.id),
"name": None,
"mode": "files",
"chunks": 1,
"corpus": str(self.private_corpus.id),
"state": "unscheduled",
"activity_state": "disabled",
......@@ -398,6 +407,7 @@ class TestProcesses(FixtureAPITestCase):
"id": str(self.workers_process.id),
"name": "Process fixture",
"mode": "workers",
"chunks": 1,
"corpus": str(self.corpus.id),
"state": "unscheduled",
"activity_state": "disabled",
......@@ -578,6 +588,7 @@ class TestProcesses(FixtureAPITestCase):
"id": str(unscheduled_with_tasks.id),
"name": None,
"mode": "workers",
"chunks": 1,
"corpus": str(self.corpus.id),
"state": "unscheduled",
"activity_state": "disabled",
......@@ -605,6 +616,7 @@ class TestProcesses(FixtureAPITestCase):
"id": str(self.elts_process.id),
"name": None,
"mode": "workers",
"chunks": 1,
"corpus": str(self.corpus.id),
"state": "unscheduled",
"activity_state": "disabled",
......@@ -619,6 +631,7 @@ class TestProcesses(FixtureAPITestCase):
"id": str(self.workers_process2.id),
"name": None,
"mode": "workers",
"chunks": 1,
"corpus": str(self.other_corpus.id),
"state": "unscheduled",
"activity_state": "disabled",
......@@ -633,6 +646,7 @@ class TestProcesses(FixtureAPITestCase):
"id": str(self.user_img_process.id),
"name": None,
"mode": "files",
"chunks": 1,
"corpus": str(self.private_corpus.id),
"state": "unscheduled",
"activity_state": "disabled",
......@@ -647,6 +661,7 @@ class TestProcesses(FixtureAPITestCase):
"id": str(self.workers_process.id),
"name": "Process fixture",
"mode": "workers",
"chunks": 1,
"corpus": str(self.corpus.id),
"state": "unscheduled",
"activity_state": "disabled",
......@@ -681,6 +696,7 @@ class TestProcesses(FixtureAPITestCase):
self.assertEqual(response.json(), {
"activity_state": ActivityState.Disabled.value,
"chunks": 1,
"corpus": str(self.private_corpus.id),
"element": None,
"element_name_contains": None,
......@@ -1290,6 +1306,7 @@ class TestProcesses(FixtureAPITestCase):
"creator": str(uuid.uuid4()),
"files": [1, 2, 3],
"mode": "iiif",
"chunks": 42,
"folder_type": "register",
"name": "newName",
"element_name_contains": "AAA",
......@@ -1319,6 +1336,7 @@ class TestProcesses(FixtureAPITestCase):
"load_children": True,
"use_cache": False,
"use_gpu": False,
"chunks": 1,
"activity_state": ActivityState.Disabled.value,
"id": str(process.id),
"corpus": str(self.corpus.id),
......@@ -1525,6 +1543,7 @@ class TestProcesses(FixtureAPITestCase):
"creator": str(uuid.uuid4()),
"files": [1, 2, 3],
"mode": "iiif",
"chunks": 42,
"folder_type": "register",
"name": "newName",
"element_name_contains": "AAA",
......@@ -1554,6 +1573,7 @@ class TestProcesses(FixtureAPITestCase):
"activity_state": ActivityState.Disabled.value,
"id": str(process.id),
"corpus": str(self.corpus.id),
"chunks": 1,
"element": None,
"files": [],
"mode": "workers",
......@@ -3188,6 +3208,7 @@ class TestProcesses(FixtureAPITestCase):
new_id = data.pop("id")
self.assertDictEqual(data, {
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": ProcessMode.Workers.value,
"name": f"Failures from {self.workers_process.id}",
"state": State.Unscheduled.value,
......
arkindex/process/tests/test_user_workerruns.py
View file @ 925bca67
......@@ -69,7 +69,6 @@ class TestUserWorkerRuns(FixtureAPITestCase):
version=self.version_1,
parents=[],
)
self.client.force_login(self.user)
with self.assertNumQueries(6):
response = self.client.get(reverse("api:user-worker-run-list"))
......@@ -83,6 +82,7 @@ class TestUserWorkerRuns(FixtureAPITestCase):
"process": {
"activity_state": "disabled",
"corpus": None,
"chunks": 1,
"id": str(self.local_process.id),
"mode": "local",
"name": None,
......@@ -127,6 +127,7 @@ class TestUserWorkerRuns(FixtureAPITestCase):
"process": {
"activity_state": "disabled",
"corpus": None,
"chunks": 1,
"id": str(self.local_process.id),
"mode": "local",
"name": None,
......@@ -241,6 +242,7 @@ class TestUserWorkerRuns(FixtureAPITestCase):
"process": {
"activity_state": "disabled",
"corpus": None,
"chunks": 1,
"id": str(self.local_process.id),
"mode": "local",
"name": None,
......@@ -447,6 +449,7 @@ class TestUserWorkerRuns(FixtureAPITestCase):
"process": {
"activity_state": "disabled",
"corpus": None,
"chunks": 1,
"id": str(self.local_process.id),
"mode": "local",
"name": None,
......
arkindex/process/tests/test_workerruns.py
View file @ 925bca67
......@@ -163,6 +163,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_1.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -440,6 +441,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_2.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": mode.value,
"name": None,
"state": "unscheduled",
......@@ -538,6 +540,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_2.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -638,6 +641,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_2.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -676,6 +680,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_1.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -736,6 +741,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_1.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -786,6 +792,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.local_process.id),
"activity_state": "disabled",
"corpus": None,
"chunks": 1,
"mode": "local",
"name": None,
"state": "unscheduled",
......@@ -939,6 +946,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.local_process.id),
"activity_state": "disabled",
"corpus": None,
"chunks": 1,
"mode": "local",
"name": None,
"state": "unscheduled",
......@@ -1011,6 +1019,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"mode": "workers",
"chunks": 1,
"name": None,
"state": "unscheduled",
"use_cache": False,
......@@ -1217,6 +1226,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_1.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -1279,6 +1289,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_1.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -1349,6 +1360,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_1.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -1755,6 +1767,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_1.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -1854,6 +1867,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_1.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -1929,6 +1943,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_1.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -2161,6 +2176,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_1.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -2223,6 +2239,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_1.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -2290,6 +2307,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_1.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -2683,6 +2701,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_1.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -2779,6 +2798,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_1.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......@@ -2853,6 +2873,7 @@ class TestWorkerRuns(FixtureAPITestCase):
"id": str(self.process_1.id),
"activity_state": "disabled",
"corpus": str(self.corpus.id),
"chunks": 1,
"mode": "workers",
"name": None,
"state": "unscheduled",
......
arkindex/project/config.py
View file @ 925bca67
......@@ -175,6 +175,8 @@ def get_settings_parser(base_dir):
# Do not use file_path here to allow the backend to start without a Ponos key
ponos_parser.add_option("default_env", type=dict, default={})
ponos_parser.add_option("artifact_max_size", type=int, default=5 * 1024**3)
# Default task expiry delay in days
ponos_parser.add_option("task_expiry", type=int, default=30)
docker_parser = parser.add_subparser("docker", default={})
docker_parser.add_option("tasks_image", type=str, default="registry.gitlab.teklia.com/arkindex/tasks")
......
arkindex/project/settings.py
View file @ 925bca67
......@@ -500,6 +500,7 @@ PONOS_ARTIFACT_MAX_SIZE = conf["ponos"]["artifact_max_size"]
PONOS_RQ_EXECUTION = True
# Base data directory for RQ tasks execution (in the docker container)
PONOS_DATA_DIR = "/data"
PONOS_TASK_EXPIRY = conf["ponos"]["task_expiry"]
# Docker images used by our ponos workflow
ARKINDEX_TASKS_IMAGE = conf["docker"]["tasks_image"]
......
arkindex/project/tests/config_samples/defaults.yaml
View file @ 925bca67
......@@ -68,6 +68,7 @@ metrics_port: 3000
ponos:
artifact_max_size: 5368709120
default_env: {}
task_expiry: 30
public_hostname: https://default.config.arkindex.localhost
redis:
db: 0
......
arkindex/project/tests/config_samples/errors.yaml
View file @ 925bca67
......@@ -51,6 +51,7 @@ ponos:
artifact_max_size: .nan
default_env: {}
private_key: /dev/zero
task_expiry: zero
public_hostname: darkindex.lol
redis:
host: radish
......
arkindex/project/tests/config_samples/expected_errors.yaml
View file @ 925bca67
......@@ -31,6 +31,7 @@ job_timeouts:
worker_results_delete: "int() argument must be a string, a bytes-like object or a real number, not 'NoneType'"
ponos:
artifact_max_size: cannot convert float NaN to integer
task_expiry: "invalid literal for int() with base 10: 'zero'"
public_hostname: The hostname must include an HTTP or HTTPS scheme.
redis:
port: "invalid literal for int() with base 10: 'over nine thousand'"
......
arkindex/project/tests/config_samples/override.yaml
View file @ 925bca67
......@@ -83,6 +83,7 @@ ponos:
artifact_max_size: 12345678901234567890
default_env:
A: B
task_expiry: 42
public_hostname: https://darkindex.lol
redis:
db: 42
......
arkindex/project/tests/openapi/test_schema.py
View file @ 925bca67
......@@ -41,7 +41,6 @@ class TestAutoSchema(TestCase):
{
"operationId": "listThings",
"deprecated": True,
"description": "",
"responses": {
"410": {
"content": {
......
arkindex/training/serializers.py
View file @ 925bca67
......@@ -717,7 +717,7 @@ class DatasetElementSerializer(serializers.ModelSerializer):
super().__init__(*args, **kwargs)
if dataset := self.context.get("dataset"):
self.fields["element_id"].queryset = Element.objects.filter(corpus=dataset.corpus)
self.fields["set"].queryset = dataset.sets.all()
self.fields["set"].queryset = dataset.sets.using("default")
def validate_element_id(self, element):
dataset = self.context.get("dataset")
......
requirements.txt
View file @ 925bca67
......@@ -10,7 +10,7 @@ django-rq==2.10.1
djangorestframework==3.13.1
djangorestframework-simplejwt==5.2.2
docker==7.0.0
drf-spectacular==0.18.2
drf-spectacular==0.21.2
python-magic==0.4.27
python-memcached==1.59
pytz==2023.3
......