Support subword and word language models

Closes #199 (closed)

Ref https://redmine.teklia.com/issues/4941

dan/datasets/extract/utils.py

@@ -226,9 +226,10 @@ class Tokenizer:
         :param text: Text to be tokenized.
         """
         return " ".join(
-            self.encode(
-                [char if char in self.charset else self.unknown_token for char in text]
-            )
+            [
+                char if char in self.charset else self.unknown_token
+                for char in self.encode(text)
+            ]
         )
 
     def encode(self, text: List[str]) -> List[str]: