Support subword and word language models

Closes #199 (closed)

Ref https://redmine.teklia.com/issues/4941

tests/test_extract.py

@@ -654,7 +654,6 @@ def test_extract(
         "▁" if t.isspace() else t for t in sorted(list(expected_charset))
     ]
     expected_language_tokens.append("◌")
-    expected_language_tokens.append("⁇")
     assert (output / "language_model" / "tokens.txt").read_text() == "\n".join(
         expected_language_tokens
     )