Add a dedicated RQ queue for exports.

Makefile

@@ -16,7 +16,7 @@ build:
 	CI_PROJECT_DIR=$(ROOT_DIR) CI_REGISTRY_IMAGE=$(IMAGE_TAG) $(ROOT_DIR)/ci/build.sh
 
 worker:
-	arkindex rqworker -v 2 default high tasks
+	arkindex rqworker -v 2 default high tasks export
 
 test-fixtures:
 	$(eval export PGPASSWORD=devdata)

arkindex/documents/export/__init__.py

@@ -118,7 +118,7 @@ def send_email(subject, template_name, corpus_export, **context):
         logger.error(f"Failed to send email to {corpus_export.user.email}")
 
 
-@job("high", timeout=settings.RQ_TIMEOUTS["export_corpus"])
+@job("export", timeout=settings.RQ_TIMEOUTS["export_corpus"])
 def export_corpus(corpus_export: CorpusExport) -> None:
     _, db_path = tempfile.mkstemp(suffix=".db")
     try:

arkindex/project/settings.py

@@ -348,6 +348,7 @@ RQ_QUEUES = {
     "default": _rq_queue_conf,
     "high": _rq_queue_conf,
     "tasks": _rq_queue_conf,
+    "export": _rq_queue_conf,
 }
 
 RQ_TIMEOUTS = conf["job_timeouts"]

arkindex/users/tests/test_jobs.py

@@ -222,10 +222,12 @@ class TestJobs(FixtureAPITestCase):
         self.assertEqual(get_queue_mock.call_args_list, [
             call("default"),
             call("high"),
+            call("export"),
         ])
 
         # Called once per queue
         self.assertListEqual(get_queue_mock().fetch_job.call_args_list, [
+            call("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"),
             call("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"),
             call("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
         ])
@@ -291,9 +293,11 @@ class TestJobs(FixtureAPITestCase):
         self.assertEqual(get_queue_mock.call_args_list, [
             call("default"),
             call("high"),
+            call("export"),
         ])
 
         self.assertListEqual(get_queue_mock().fetch_job.call_args_list, [
+            call("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"),
             call("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"),
             call("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
         ])

docker-compose.yml

@@ -32,7 +32,7 @@ services:
   worker:
     container_name: ark-worker
     build: .
-    command: arkindex rqworker-pool --num-workers 2 -v 1 default high tasks
+    command: arkindex rqworker-pool --num-workers 2 -v 1 default high tasks export
 
     depends_on:
       - db