Configurable SQL chunk size in reindex command

Closes #1890 (closed)

arkindex/documents/management/commands/reindex.py

@@ -29,13 +29,22 @@ class Command(BaseCommand):
             help="Only setup a collection. Create a collection and fields if they do not exist or update the fields",
             action="store_true",
         )
+        parser.add_argument(
+            "--sql-chunk-size",
+            help="Size of the chunks of parent elements used when retrieving all children of indexable elements from the database.",
+            type=int,
+            default=10000,
+        )
 
-    def handle(self, corpus_id, **options):
+    def handle(self, corpus_id, sql_chunk_size, **options):
         if not settings.ARKINDEX_FEATURES["search"]:
             raise CommandError("Reindexation is not possible if the search feature flag is disabled. "
                                "Consider setting `features.search` to `on` or `true` or `yes` in the YAML "
                                "configuration file, and configuring Solr properly.")
 
+        if sql_chunk_size <= 0:
+            raise CommandError("--sql-chunk-size must be set to a strictly positive integer.")
+
         if options["all"]:
             corpora = Corpus.objects.filter(indexable=True)
         elif corpus_id:
@@ -47,7 +56,7 @@ class Command(BaseCommand):
 
         for corpus in corpora:
             self.stdout.write(f"Indexing {corpus.name}")
-            indexer = Indexer(corpus.id)
+            indexer = Indexer(corpus.id, sql_chunk_size=sql_chunk_size)
             if options.get("drop"):
                 indexer.drop_index()
             indexer.setup()