From 2ed4bde20f648d8cc8ff302b2d16ae7a70984f8b Mon Sep 17 00:00:00 2001
From: NolanB <nboukachab@teklia.com>
Date: Mon, 15 May 2023 16:58:53 +0200
Subject: [PATCH] remove new_tokens
---
dan/manager/ocr.py | 5 -----
1 file changed, 5 deletions(-)
diff --git a/dan/manager/ocr.py b/dan/manager/ocr.py
index 50389b87..bd464073 100644
--- a/dan/manager/ocr.py
+++ b/dan/manager/ocr.py
@@ -42,11 +42,6 @@ class OCRDatasetManager(DatasetManager):
):
self.synthetic_data = self.params["config"]["synthetic_data"]
- if "new_tokens" in params:
- self.charset = sorted(
- list(set(self.charset).union(set(params["new_tokens"])))
- )
-
self.tokens = {
"pad": params["config"]["padding_token"],
}
--
GitLab