Split project exports between two queues

arkindex/documents/export/__init__.py

@@ -135,7 +135,18 @@ def update_state(corpus_export: CorpusExport, state: CorpusExportState):
         corpus_export.save()
 
 
+@job("high", timeout=settings.RQ_TIMEOUTS["export_corpus"])
+def local_export(corpus_export: CorpusExport) -> None:
+    assert corpus_export.source == "default"
+    export_corpus(corpus_export)
+
+
 @job("export", timeout=settings.RQ_TIMEOUTS["export_corpus"])
+def remote_export(corpus_export: CorpusExport) -> None:
+    assert corpus_export.source != "default"
+    export_corpus(corpus_export)
+
+
 def export_corpus(corpus_export: CorpusExport) -> None:
     _, db_path = tempfile.mkstemp(suffix=".db")
     try:

arkindex/documents/tests/test_export.py

@@ -5,6 +5,7 @@ from django.test import override_settings
 from django.urls import reverse
 from rest_framework import status
 
+from arkindex.documents.export import local_export, remote_export
 from arkindex.documents.models import Corpus, CorpusExportState
 from arkindex.project.tests import FixtureAPITestCase
 from arkindex.users.models import Role
@@ -12,7 +13,7 @@ from arkindex.users.models import Role
 
 class TestExport(FixtureAPITestCase):
 
-    @patch("arkindex.project.triggers.export.export_corpus.delay")
+    @patch("arkindex.project.triggers.export.local_export.delay")
     @override_settings(EXPORT_TTL_SECONDS=420)
     def test_start(self, delay_mock):
         self.client.force_login(self.superuser)
@@ -41,7 +42,7 @@ class TestExport(FixtureAPITestCase):
             description="Export of corpus Unit Tests"
         ))
 
-    @patch("arkindex.project.triggers.export.export_corpus.delay")
+    @patch("arkindex.project.triggers.export.local_export.delay")
     def test_start_requires_login(self, delay_mock):
         response = self.client.post(reverse("api:corpus-export", kwargs={"pk": self.corpus.id}))
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
@@ -49,7 +50,7 @@ class TestExport(FixtureAPITestCase):
         self.assertFalse(self.corpus.exports.exists())
         self.assertFalse(delay_mock.called)
 
-    @patch("arkindex.project.triggers.export.export_corpus.delay")
+    @patch("arkindex.project.triggers.export.local_export.delay")
     def test_start_requires_verified(self, delay_mock):
         self.user.verified_email = False
         self.user.save()
@@ -61,7 +62,7 @@ class TestExport(FixtureAPITestCase):
         self.assertFalse(self.corpus.exports.exists())
         self.assertFalse(delay_mock.called)
 
-    @patch("arkindex.project.triggers.export.export_corpus.delay")
+    @patch("arkindex.project.triggers.export.local_export.delay")
     @patch("arkindex.users.utils.get_max_level", return_value=Role.Guest.value)
     def test_start_requires_contributor(self, max_level_mock, delay_mock):
         self.user.rights.update(level=Role.Guest.value)
@@ -75,7 +76,7 @@ class TestExport(FixtureAPITestCase):
         self.assertFalse(self.corpus.exports.exists())
         self.assertFalse(delay_mock.called)
 
-    @patch("arkindex.project.triggers.export.export_corpus.delay")
+    @patch("arkindex.project.triggers.export.remote_export.delay")
     def test_start_bad_source(self, delay_mock):
         self.client.force_login(self.superuser)
 
@@ -87,7 +88,7 @@ class TestExport(FixtureAPITestCase):
         self.assertFalse(delay_mock.called)
 
     @patch("arkindex.documents.models.CorpusExport.source")
-    @patch("arkindex.project.triggers.export.export_corpus.delay")
+    @patch("arkindex.project.triggers.export.remote_export.delay")
     @override_settings(EXPORT_TTL_SECONDS=420)
     def test_start_with_source(self, delay_mock, source_field_mock):
         source_field_mock.field.choices.return_value = [("default", "default"), ("jouvence", "jouvence")]
@@ -118,7 +119,7 @@ class TestExport(FixtureAPITestCase):
             description="Export of corpus Unit Tests from source jouvence"
         ))
 
-    @patch("arkindex.project.triggers.export.export_corpus.delay")
+    @patch("arkindex.project.triggers.export.local_export.delay")
     def test_start_running(self, delay_mock):
         self.client.force_login(self.superuser)
         self.corpus.exports.create(user=self.user, state=CorpusExportState.Running)
@@ -133,7 +134,7 @@ class TestExport(FixtureAPITestCase):
         self.assertFalse(delay_mock.called)
 
     @override_settings(EXPORT_TTL_SECONDS=420)
-    @patch("arkindex.project.triggers.export.export_corpus.delay")
+    @patch("arkindex.project.triggers.export.local_export.delay")
     def test_start_recent_export(self, delay_mock):
         self.client.force_login(self.superuser)
         with patch("django.utils.timezone.now") as mock_now:
@@ -153,7 +154,7 @@ class TestExport(FixtureAPITestCase):
         self.assertFalse(delay_mock.called)
 
     @override_settings(EXPORT_TTL_SECONDS=420)
-    @patch("arkindex.project.triggers.export.export_corpus.delay")
+    @patch("arkindex.project.triggers.export.local_export.delay")
     def test_start_recent_export_different_source(self, delay_mock):
         from arkindex.documents.models import CorpusExport
         CorpusExport.source.field.choices = [("default", "default"), ("jouvence", "jouvence")]
@@ -433,3 +434,23 @@ class TestExport(FixtureAPITestCase):
             response = self.client.delete(reverse("api:manage-export", kwargs={"pk": export.id}))
             self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
         assert not self.corpus.exports.exists()
+
+    @patch("arkindex.documents.models.CorpusExport.source")
+    def test_local_export_default_db_only(self, source_field_mock):
+        source_field_mock.field.choices.return_value = [("default", "default"), ("jouvence", "jouvence")]
+        export = self.corpus.exports.create(user=self.superuser, state=CorpusExportState.Created, source="jouvence")
+        with self.assertRaises(AssertionError):
+            local_export(export)
+
+        export.refresh_from_db()
+        self.assertEqual(export.state, CorpusExportState.Created)
+
+    @patch("arkindex.documents.models.CorpusExport.source")
+    def test_remote_export_not_default_db(self, source_field_mock):
+        source_field_mock.field.choices.return_value = [("default", "default"), ("jouvence", "jouvence")]
+        export = self.corpus.exports.create(user=self.superuser, state=CorpusExportState.Created, source="default")
+        with self.assertRaises(AssertionError):
+            remote_export(export)
+
+        export.refresh_from_db()
+        self.assertEqual(export.state, CorpusExportState.Created)

arkindex/project/triggers.py

@@ -198,10 +198,12 @@ def export_corpus(corpus_export: CorpusExport) -> None:
     """
     Export a corpus to a SQLite database
     """
+    export_function = export.local_export
     description = f"Export of corpus {corpus_export.corpus.name}"
     if corpus_export.source != "default":
         description += f" from source {corpus_export.source}"
-    export.export_corpus.delay(
+        export_function = export.remote_export
+    export_function.delay(
         corpus_export=corpus_export,
         user_id=corpus_export.user_id,
         description=description