Compute confidence scores by char, word or line

Closes #33 (closed)

dan/predict/prediction.py

@@ -218,11 +218,9 @@ def compute_prob_by_separator(characters, probabilities, separator):
     # match anything except separators, get start and end index
     pattern = re.compile(f"[^{separator.pattern}]+")
     matches = [(m.start(), m.end()) for m in re.finditer(pattern, characters)]
-    return [
-        np.mean(probabilities[start:end])
-        for (start, end) in matches
-    ]
 
+    # Iterate over text pieces and compute mean confidence
+    return [np.mean(probabilities[start:end]) for (start, end) in matches]
 
 
 def run(