Support subword and word language models

Closes #199 (closed)

Ref https://redmine.teklia.com/issues/4941

dan/datasets/extract/arkindex.py

@@ -367,7 +367,6 @@ class ArkindexExtractor:
                 self.mapping.encode[token]
             ) if token in self.mapping.encode else self.language_tokens.append(token)
         self.language_tokens.append(self.mapping.ctc.encoded)
-        self.language_tokens.append(self.unknown_token)
 
         # Build LM corpus
         train_corpus = [