Skip to content
Snippets Groups Projects
Commit 2f61d377 authored by Yoann Schneider's avatar Yoann Schneider :tennis:
Browse files

Merge branch 'do-not-edit-ner-text' into 'main'

Keep NER text as is

Closes #246

See merge request !336
parents c0eb5bb8 9d66f700
No related branches found
No related tags found
1 merge request!336Keep NER text as is
...@@ -93,7 +93,7 @@ def compute_prob_by_ner( ...@@ -93,7 +93,7 @@ def compute_prob_by_ner(
return zip( return zip(
*[ *[
( (
f"{characters[current: next_token]}".replace("\n", " "), characters[current:next_token],
np.mean(probabilities[current:next_token]), np.mean(probabilities[current:next_token]),
) )
for current, next_token in indices for current, next_token in indices
...@@ -154,10 +154,7 @@ def split_text( ...@@ -154,10 +154,7 @@ def split_text(
return [], [] return [], []
indices = build_ner_indices(text, tokens) indices = build_ner_indices(text, tokens)
text_split = [ text_split = [text[current:next_token] for current, next_token in indices]
f"{text[current: next_token]}".replace("\n", " ")
for current, next_token in indices
]
case _: case _:
logger.error(f"Level should be either {list(map(str, Level))}") logger.error(f"Level should be either {list(map(str, Level))}")
return [], [] return [], []
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment