Add the unknown character to the list of tokens

dan/datasets/extract/extract.py

@@ -365,6 +365,7 @@ class ArkindexExtractor:
                 self.mapping.encode[token]
             ) if token in self.mapping.encode else self.language_tokens.append(token)
         self.language_tokens.append(self.mapping.ctc.encoded)
+        self.language_tokens.append(self.unknown_token)
 
         # Build LM corpus
         train_corpus = [

tests/test_extract.py

@@ -654,6 +654,7 @@ def test_extract(
         "▁" if t.isspace() else t for t in sorted(list(expected_charset))
     ]
     expected_language_tokens.append("◌")
+    expected_language_tokens.append("⁇")
     assert (output / "language_model" / "tokens.txt").read_text() == "\n".join(
         expected_language_tokens
     )