Catch runtimeError when formatting LM files

dan/datasets/extract/arkindex.py

@@ -384,6 +384,9 @@ class ArkindexExtractor:
             subword_vocab_size=self.subword_vocab_size,
         )
 
+        if not tokenizer.sentencepiece_model:
+            return
+
         for level, tokenize in (
             ("characters", tokenizer.char_tokenize),
             ("words", tokenizer.word_tokenize),
@@ -454,7 +457,7 @@ class ArkindexExtractor:
                 )
 
         if failed_downloads:
-            logger.error(f"Failed to download {len(failed_downloads)} image(s).")
+            logger.warning(f"Failed to download {len(failed_downloads)} image(s).")
             print(*list(map(": ".join, failed_downloads)), sep="\n")
 
     def run(self):

dan/datasets/extract/utils.py

@@ -186,12 +186,19 @@ class Tokenizer:
         with NamedTemporaryFile(dir=self.outdir, suffix=".txt", mode="w") as tmp:
             tmp.write("\n".join(self.training_corpus))
             tmp.flush()
-            spm.SentencePieceTrainer.train(
-                input=tmp.name,
-                vocab_size=self.subword_vocab_size,
-                model_prefix=self.prefix,
-                user_defined_symbols=self.special_tokens,
-            )
+
+            try:
+                spm.SentencePieceTrainer.train(
+                    input=tmp.name,
+                    vocab_size=self.subword_vocab_size,
+                    model_prefix=self.prefix,
+                    user_defined_symbols=self.special_tokens,
+                    minloglevel=1,
+                )
+            except Exception as e:
+                logger.warning(f"Failed to train a sentencepiece model for subword tokenization: {e}")
+                self.sentencepiece_model = None
+                return
 
         # Load the model
         self.sentencepiece_model = spm.SentencePieceProcessor(