Support subword and word language models

Closes #199 (closed)

Ref https://redmine.teklia.com/issues/4941

dan/datasets/extract/extract.py

@@ -370,7 +370,7 @@ class ArkindexExtractor:
         ), "Tokens should be single characters."
 
         # Build LM corpus
-        train_corpus = [text for text in self.data["train"].values()]
+        train_corpus = [text.replace("\n", " ") for text in self.data["train"].values()]
         tokenizer = Tokenizer(
             train_corpus,
             outdir=self.output / "language_model",