Fix multiprocessing pool

arkindex/documents/management/commands/from_csv.py

@@ -19,38 +19,40 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 
-class Command(BaseCommand):
-    help = "Import Himanis volumes from CSV file. May require the Arkindex backend in PATH."
+def import_manifest(url, name):
+    "Import a manifest from a given URL with a given volume name."
+
+    logger.info('Importing volume {} from {}'.format(name, url))
+    URLManifestsImporter([], url, offline=True, volume_name=name).run()
 
-    def import_manifest(url, name):
-        "Import a manifest from a given URL with a given volume name."
 
-        logger.info('Importing volume {} from {}'.format(name, url))
-        URLManifestsImporter([], url, offline=True, volume_name=name).run()
+def import_annotations(source, raw_path, name, index_root):
+    """
+    Import annotations.
+    source: 'bvmm' or 'gallica'
+    raw_path: Raw path to index file
+    name: Volume name
+    index_root: Root folder for index files
+    """
 
-    def import_annotations(source, raw_path, name, index_root):
-        """
-        Import annotations.
-        source: 'bvmm' or 'gallica'
-        raw_path: Raw path to index file
-        name: Volume name
-        index_root: Root folder for index files
-        """
+    if raw_path.startswith('/home/data/indexes'):
+        raw_path = raw_path[18:]
+    raw_path = raw_path.lstrip('/')
+    index_path = os.path.join(index_root, raw_path)
+    assert os.path.exists(index_path)
 
-        if raw_path.startswith('/home/data/indexes'):
-            raw_path = raw_path[18:]
-        raw_path = raw_path.lstrip('/')
-        index_path = os.path.join(index_root, raw_path)
-        assert os.path.exists(index_path)
+    volume = Element.objects.get(name=name, type=ElementType.Volume)
 
-        volume = Element.objects.get(name=name, type=ElementType.Volume)
+    if source == 'bvmm':
+        IndexImporter(index_path, volume, mask=r'(?:.*/)?([^/]+)_[A-Z]\.idx\.gz').run()
+    elif source == 'gallica':
+        GallicaIndexImporter(index_path, volume).run()
+    else:  # Try anyway
+        IndexImporter(index_path, volume).run()
 
-        if source == 'bvmm':
-            IndexImporter(index_path, volume, mask=r'(?:.*/)?([^/]+)_[A-Z]\.idx\.gz').run()
-        elif source == 'gallica':
-            GallicaIndexImporter(index_path, volume).run()
-        else:  # Try anyway
-            IndexImporter(index_path, volume).run()
+
+class Command(BaseCommand):
+    help = "Import Himanis volumes from CSV file. May require the Arkindex backend in PATH."
 
     def add_arguments(self, parser):
         parser.add_argument('csv')
@@ -84,6 +86,6 @@ class Command(BaseCommand):
 
         with Pool(options['processes']) as pool:
             logger.info('Importing all manifests')
-            pool.starmap(self.import_manifest, [(row[1], row[0]) for row in data])
+            pool.starmap(import_manifest, [(row[1], row[0]) for row in data])
             logger.info('Importing all annotations')
-            pool.starmap(self.import_annotations, [(row[3], row[2], row[0], args['index_root']) for row in data])
+            pool.starmap(import_annotations, [(row[3], row[2], row[0], args['index_root']) for row in data])