Support subword and word language models

Closes #199 (closed)

Ref https://redmine.teklia.com/issues/4941

dan/datasets/extract/utils.py

@@ -87,16 +87,18 @@ def insert_token(text: str, entity_type: EntityType, offset: int, length: int) -
 
 def normalize_linebreaks(text: str) -> str:
     """
-    Remove begin/ending linebreaks
-    Replace \r with regular linebreak and consecutive linebreaks
+    Remove begin/ending linebreaks.
+    Replace \r with regular linebreak and consecutive linebreaks.
+    :param text: Text to normalize.
     """
     return TRIM_RETURN_REGEX.sub("\n", text.strip())
 
 
 def normalize_spaces(text: str) -> str:
     """
-    Remove begin/ending spaces
-    Replace \t with regular space and consecutive spaces
+    Remove begin/ending spaces.
+    Replace \t with regular space and consecutive spaces.
+    :param text: Text to normalize.
     """
     return TRIM_SPACE_REGEX.sub(" ", text.strip())