Skip to content
Snippets Groups Projects
Commit 3df7ae44 authored by Nolan's avatar Nolan Committed by Yoann Schneider
Browse files

Remove new_tokens from OCR code

parent c3d2f8fe
No related branches found
No related tags found
1 merge request!126Remove new_tokens from OCR code
...@@ -42,11 +42,6 @@ class OCRDatasetManager(DatasetManager): ...@@ -42,11 +42,6 @@ class OCRDatasetManager(DatasetManager):
): ):
self.synthetic_data = self.params["config"]["synthetic_data"] self.synthetic_data = self.params["config"]["synthetic_data"]
if "new_tokens" in params:
self.charset = sorted(
list(set(self.charset).union(set(params["new_tokens"])))
)
self.tokens = { self.tokens = {
"pad": params["config"]["padding_token"], "pad": params["config"]["padding_token"],
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment