Catch runtimeError when formatting LM files

Closes #221 (closed)

dan/datasets/extract/arkindex.py

@@ -384,6 +384,9 @@ class ArkindexExtractor:
             subword_vocab_size=self.subword_vocab_size,
         )
 
+        if not tokenizer.sentencepiece_model:
+            return
+
         for level, tokenize in (
             ("characters", tokenizer.char_tokenize),
             ("words", tokenizer.word_tokenize),
@@ -454,7 +457,7 @@ class ArkindexExtractor:
                 )
 
         if failed_downloads:
-            logger.error(f"Failed to download {len(failed_downloads)} image(s).")
+            logger.warning(f"Failed to download {len(failed_downloads)} image(s).")
             print(*list(map(": ".join, failed_downloads)), sep="\n")
 
     def run(self):