diff --git a/arkindex/dataimport/tasks.py b/arkindex/dataimport/tasks.py index e8bb8231cbd451fe81e3d0fc73f919a0ec6ca9f4..eb159744074a235e44e2db339f5dc4a2eb7468eb 100644 --- a/arkindex/dataimport/tasks.py +++ b/arkindex/dataimport/tasks.py @@ -1,12 +1,13 @@ from typing import Optional +from django.conf import settings from django.db import transaction from django_rq import job from arkindex.dataimport.models import ActivityState, DataImport, WorkerActivity, WorkerActivityState -@job('default', timeout=3600) +@job('default', timeout=settings.RQ_TIMEOUTS['initialize_activity']) def initialize_activity(process: DataImport): """ List all worker versions used in a process and initialize their activity on processed elements. @@ -28,7 +29,7 @@ def initialize_activity(process: DataImport): process.save() -@job('default', timeout=3600) +@job('default', timeout=settings.RQ_TIMEOUTS['process_delete']) def process_delete(process: DataImport, delete_activity: Optional[WorkerActivityState] = None): """ Delete a process and removes its reference from related worker activities. diff --git a/arkindex/documents/export/__init__.py b/arkindex/documents/export/__init__.py index 7b174f5f0b1acbe2884a10f52dfd4bd8bd59fee8..d9b94f00ba2505404d985f7f2c92385243638fe2 100644 --- a/arkindex/documents/export/__init__.py +++ b/arkindex/documents/export/__init__.py @@ -99,7 +99,7 @@ def send_email(subject, template_name, corpus_export, **context): logger.error(f'Failed to send email to {corpus_export.user.email}') -@job('high', timeout=7200) +@job('high', timeout=settings.RQ_TIMEOUTS['export_corpus']) def export_corpus(corpus_export: CorpusExport) -> None: _, db_path = tempfile.mkstemp(suffix='db') try: diff --git a/arkindex/documents/tasks.py b/arkindex/documents/tasks.py index 4b6f0545c41bb560add441acc7298c90643d884e..d58b268f3baa25bee37cd16ed4edcc02aa679a38 100644 --- a/arkindex/documents/tasks.py +++ b/arkindex/documents/tasks.py @@ -1,5 +1,6 @@ import logging +from django.conf import settings from django.db.models import Q from django_rq import job from rq import get_current_job @@ -21,7 +22,7 @@ from arkindex.documents.models import ( logger = logging.getLogger(__name__) -@job('high') +@job('high', timeout=settings.RQ_TIMEOUTS['corpus_delete']) def corpus_delete(corpus_id: str) -> None: # Note that this can be None when the task is run outside of a RQ worker (e.g. unit test) rq_job = get_current_job() @@ -70,12 +71,12 @@ def corpus_delete(corpus_id: str) -> None: logger.info(f'Deleted corpus {corpus_id}') -@job('high') +@job('high', timeout=settings.RQ_TIMEOUTS['element_trash']) def element_trash(queryset: ElementQuerySet, delete_children: bool) -> None: queryset.trash(delete_children=delete_children) -@job('high') +@job('high', timeout=settings.RQ_TIMEOUTS['worker_results_delete']) def worker_results_delete(corpus_id: str, version_id: str, parent_id: str) -> None: """ Recursively delete all Worker Results produced by a specific WorkerVersion on a whole corpus or under @@ -112,7 +113,7 @@ def worker_results_delete(corpus_id: str, version_id: str, parent_id: str) -> No transcriptions._raw_delete(using='default') -@job('high') +@job('high', timeout=settings.RQ_TIMEOUTS['move_element']) def move_element(source: Element, destination: Element) -> None: parents = Element.objects.filter(id__in=source.paths.values('path__last')).only('id') for parent in parents: diff --git a/arkindex/project/config.py b/arkindex/project/config.py index b50ae1346846b1e5ffc38c5edec1871621da6eb3..71613fbcd0814c8448286e82487c41af5676b255 100644 --- a/arkindex/project/config.py +++ b/arkindex/project/config.py @@ -114,6 +114,15 @@ def get_settings_parser(base_dir): redis_parser.add_option('password', type=str, default=None) redis_parser.add_option('timeout', type=int, default=1800) + job_timeouts_parser = parser.add_subparser('job_timeouts', default={}) + job_timeouts_parser.add_option('export_corpus', type=int, default=7200) + job_timeouts_parser.add_option('corpus_delete', type=int, default=7200) + job_timeouts_parser.add_option('worker_results_delete', type=int, default=3600) + job_timeouts_parser.add_option('element_trash', type=int, default=3600) + job_timeouts_parser.add_option('move_element', type=int, default=3600) + job_timeouts_parser.add_option('initialize_activity', type=int, default=3600) + job_timeouts_parser.add_option('process_delete', type=int, default=3600) + csrf_parser = parser.add_subparser('csrf', default={}) csrf_parser.add_option('cookie_name', type=str, default='arkindex.csrf') csrf_parser.add_option('cookie_domain', type=str, default=None) diff --git a/arkindex/project/settings.py b/arkindex/project/settings.py index 842c432110ee4c0109fe208e9e4a6be3b0f37068..f3f6612bdcbc2f98ba4896b5658bc5cdf11f8e61 100644 --- a/arkindex/project/settings.py +++ b/arkindex/project/settings.py @@ -383,6 +383,8 @@ RQ_QUEUES = { } } +RQ_TIMEOUTS = conf['job_timeouts'] + RQ = { 'JOB_CLASS': 'arkindex.project.rq_overrides.Job', 'QUEUE_CLASS': 'arkindex.project.rq_overrides.Queue' diff --git a/arkindex/project/tests/config_samples/defaults.yaml b/arkindex/project/tests/config_samples/defaults.yaml index f58e8c607e011e8f3ae9ffae5596e5b30a1b0eca..ea5f88fc55e3e89c00bcc5bbf3b8772fef00e8e7 100644 --- a/arkindex/project/tests/config_samples/defaults.yaml +++ b/arkindex/project/tests/config_samples/defaults.yaml @@ -45,6 +45,14 @@ imports_worker_version: null influxdb: api_url: http://localhost:8086/ internal_group_id: 2 +job_timeouts: + corpus_delete: 7200 + element_trash: 3600 + export_corpus: 7200 + initialize_activity: 3600 + move_element: 3600 + process_delete: 3600 + worker_results_delete: 3600 jwt_signing_key: null local_imageserver_id: 1 ponos: diff --git a/arkindex/project/tests/config_samples/errors.yaml b/arkindex/project/tests/config_samples/errors.yaml index a2db06b62a82d6e75b8edf0f784c498943e3e6f3..d8877b7914c8b41eeeeb094fd5ae4bd67dea3b98 100644 --- a/arkindex/project/tests/config_samples/errors.yaml +++ b/arkindex/project/tests/config_samples/errors.yaml @@ -30,6 +30,13 @@ gitlab: influxdb: api_url: no internal_group_id: 2 +job_timeouts: + corpus_delete: lol + element_trash: no + export_corpus: [] + move_element: + a: b + worker_results_delete: null jwt_signing_key: null local_imageserver_id: 1 ponos: diff --git a/arkindex/project/tests/config_samples/expected_errors.yaml b/arkindex/project/tests/config_samples/expected_errors.yaml index 930ec1a76e127ebdb7a44b8907159f31887fcb5d..0bb4c8530e7fcda4b657c776d5fb9e1fddf26c9e 100644 --- a/arkindex/project/tests/config_samples/expected_errors.yaml +++ b/arkindex/project/tests/config_samples/expected_errors.yaml @@ -13,6 +13,11 @@ email: user: This option is required features: sv_cheats: This option does not exist +job_timeouts: + corpus_delete: "invalid literal for int() with base 10: 'lol'" + export_corpus: "int() argument must be a string, a bytes-like object or a number, not 'list'" + move_element: "int() argument must be a string, a bytes-like object or a number, not 'dict'" + worker_results_delete: "int() argument must be a string, a bytes-like object or a number, not 'NoneType'" public_hostname: The hostname must include an HTTP or HTTPS scheme. redis: port: "invalid literal for int() with base 10: 'over nine thousand'" diff --git a/arkindex/project/tests/config_samples/override.yaml b/arkindex/project/tests/config_samples/override.yaml index 95aeb01fd910a624a8aca7cc2b1d77bb5b87257d..be46d7c9bf4b5980c735934994f629e6c1ff9730 100644 --- a/arkindex/project/tests/config_samples/override.yaml +++ b/arkindex/project/tests/config_samples/override.yaml @@ -59,6 +59,14 @@ imports_worker_version: aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa influxdb: api_url: http://graph/ internal_group_id: 4 +job_timeouts: + corpus_delete: 1 + element_trash: 2 + export_corpus: 3 + initialize_activity: 4 + move_element: 5 + process_delete: 6 + worker_results_delete: 7 jwt_signing_key: deadbeef local_imageserver_id: 45 ponos: