use corpus id

README.md

@@ -24,6 +24,11 @@ Simple example:
 python kaldi_data_generator.py --dataset_name my_balsac --out_dir /tmp/balsac/ --volumes 8f4005e9-1921-47b0-be7b-e27c7fd29486  d2f7c563-1622-4721-bd51-96fab97189f7 
 ```
 
+With corpus ids
+```bash
+python kaldi_data_generator.py --dataset_name cz --out_dir /tmp/home_cz/ --corpora 1ed45e94-9108-4029-a529-9abe37f55ba0  
+```
+
 Polygon example:
 ```bash
 python kaldi_data_generator.py --dataset_name my_balsac2 --extraction_mode polygon --out_dir /tmp/balsac/ --pages 50e1c3c0-2fe9-4216-805e-1a2fd2e7e9f4

kaldi_data_generator.py

@@ -96,6 +96,10 @@ class KaldiDataGenerator:
             print("ListTranscriptions failed", e.status_code, e.title, e.content, page_id)
             raise e
         print("Num of lines", count)
+        if count == 0:
+            print(f"Page {page_id} skipped, because it has no lines")
+            return
+
         full_image_url = res['zone']['image']['s3_url']
 
         img = self.get_image(full_image_url, page_id=page_id)
@@ -141,6 +145,12 @@ class KaldiDataGenerator:
             page_ids = [page['id'] for page in api_client.paginate('ListElementChildren', id=volume_id)]
             self.run_pages(page_ids)
 
+    def run_corpora(self, corpus_ids: list):
+        for corpus_id in corpus_ids:
+            print("Cor", corpus_id)
+            vol_ids = [vol['id'] for vol in api_client.paginate('ListElements', corpus=corpus_id, type='volume')]
+            self.run_volumes(vol_ids)
+
 
 class Split(Enum):
     Train: int = 0
@@ -219,6 +229,8 @@ def create_parser():
                        help='Use color images')
     parser.set_defaults(grayscale=True)
 
+    parser.add_argument('--corpora', nargs='*',
+                        help='List of corpus ids to be used, separated by spaces')
     parser.add_argument('--volumes', nargs='*',
                         help='List of volume ids to be used, separated by spaces')
     parser.add_argument('--pages', nargs='*',
@@ -245,6 +257,8 @@ def main():
         kaldi_data_generator.run_pages(args.pages)
     if args.volumes:
         kaldi_data_generator.run_volumes(args.volumes)
+    if args.corpora:
+        kaldi_data_generator.run_corpora(args.corpora)
 
     print()
     # create partitions from all the extracted data