Skip to content
Snippets Groups Projects

No more DPI adjusting

Merged Yoann Schneider requested to merge replace-dpi-adjust into main
All threads resolved!
Files
2
@@ -384,6 +384,9 @@ class ArkindexExtractor:
subword_vocab_size=self.subword_vocab_size,
)
if not tokenizer.sentencepiece_model:
return
for level, tokenize in (
("characters", tokenizer.char_tokenize),
("words", tokenizer.word_tokenize),
@@ -478,6 +481,11 @@ class ArkindexExtractor:
pbar.update()
pbar.refresh()
if not self.data:
raise Exception(
"No data was extracted using the provided export database and parameters."
)
self.download_images()
self.format_lm_files()
self.export()
Loading