Skip to content
Snippets Groups Projects
Commit 9d66f700 authored by Manon Blanco's avatar Manon Blanco
Browse files

Keep NER text as is

parent c0eb5bb8
No related branches found
No related tags found
1 merge request!336Keep NER text as is
......@@ -93,7 +93,7 @@ def compute_prob_by_ner(
return zip(
*[
(
f"{characters[current: next_token]}".replace("\n", " "),
characters[current:next_token],
np.mean(probabilities[current:next_token]),
)
for current, next_token in indices
......@@ -154,10 +154,7 @@ def split_text(
return [], []
indices = build_ner_indices(text, tokens)
text_split = [
f"{text[current: next_token]}".replace("\n", " ")
for current, next_token in indices
]
text_split = [text[current:next_token] for current, next_token in indices]
case _:
logger.error(f"Level should be either {list(map(str, Level))}")
return [], []
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment