Skip to content
Snippets Groups Projects
Commit 6688e756 authored by Bastien Abadie's avatar Bastien Abadie Committed by Erwan Rouchet
Browse files

Add a dedicated RQ queue for exports.

parent 5d3624a1
No related branches found
No related tags found
1 merge request!2306Add a dedicated RQ queue for exports.
......@@ -16,7 +16,7 @@ build:
CI_PROJECT_DIR=$(ROOT_DIR) CI_REGISTRY_IMAGE=$(IMAGE_TAG) $(ROOT_DIR)/ci/build.sh
worker:
arkindex rqworker -v 2 default high tasks
arkindex rqworker -v 2 default high tasks export
test-fixtures:
$(eval export PGPASSWORD=devdata)
......
......@@ -118,7 +118,7 @@ def send_email(subject, template_name, corpus_export, **context):
logger.error(f"Failed to send email to {corpus_export.user.email}")
@job("high", timeout=settings.RQ_TIMEOUTS["export_corpus"])
@job("export", timeout=settings.RQ_TIMEOUTS["export_corpus"])
def export_corpus(corpus_export: CorpusExport) -> None:
_, db_path = tempfile.mkstemp(suffix=".db")
try:
......
......@@ -348,6 +348,7 @@ RQ_QUEUES = {
"default": _rq_queue_conf,
"high": _rq_queue_conf,
"tasks": _rq_queue_conf,
"export": _rq_queue_conf,
}
RQ_TIMEOUTS = conf["job_timeouts"]
......
......@@ -222,10 +222,12 @@ class TestJobs(FixtureAPITestCase):
self.assertEqual(get_queue_mock.call_args_list, [
call("default"),
call("high"),
call("export"),
])
# Called once per queue
self.assertListEqual(get_queue_mock().fetch_job.call_args_list, [
call("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"),
call("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"),
call("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
])
......@@ -291,9 +293,11 @@ class TestJobs(FixtureAPITestCase):
self.assertEqual(get_queue_mock.call_args_list, [
call("default"),
call("high"),
call("export"),
])
self.assertListEqual(get_queue_mock().fetch_job.call_args_list, [
call("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"),
call("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"),
call("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
])
......
......@@ -32,7 +32,7 @@ services:
worker:
container_name: ark-worker
build: .
command: arkindex rqworker-pool --num-workers 2 -v 1 default high tasks
command: arkindex rqworker-pool --num-workers 2 -v 1 default high tasks export
depends_on:
- db
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment