diff --git a/Datasets/dataset_formatters/generic_dataset_formatter.py b/Datasets/dataset_formatters/generic_dataset_formatter.py
index e474ed45a27e14919f69e78923dc807cc1815ac4..ae55af026c841af4a9ab8ee08e3437bf9d16c034 100644
--- a/Datasets/dataset_formatters/generic_dataset_formatter.py
+++ b/Datasets/dataset_formatters/generic_dataset_formatter.py
@@ -1,38 +1,3 @@
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in Python whose purpose is to
-# provide public implementation of deep learning works, in pytorch.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
-
import os
import shutil
import tarfile
diff --git a/Datasets/dataset_formatters/read2016_formatter.py b/Datasets/dataset_formatters/read2016_formatter.py
deleted file mode 100644
index 5a0b0452f7d797b0b37c3a391b1e59783bdbf5e0..0000000000000000000000000000000000000000
--- a/Datasets/dataset_formatters/read2016_formatter.py
+++ /dev/null
@@ -1,509 +0,0 @@
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in Python whose purpose is to
-# provide public implementation of deep learning works, in pytorch.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
-
-from Datasets.dataset_formatters.generic_dataset_formatter import OCRDatasetFormatter
-import os
-import numpy as np
-from PIL import Image
-import xml.etree.ElementTree as ET
-
-
-# Layout begin-token to end-token
-SEM_MATCHING_TOKENS = {
- "â“‘": "â’·", # paragraph (body)
- "â“": "â’¶", # annotation
- "â“Ÿ": "â“…", # page
- "â“": "Ⓝ", # page number
- "ⓢ": "Ⓢ", # section (=linked annotation + body)
- }
-
-
-class READ2016DatasetFormatter(OCRDatasetFormatter):
- def __init__(self, level, set_names=["train", "valid", "test"], dpi=150, end_token=True, sem_token=True):
- super(READ2016DatasetFormatter, self).__init__("READ_2016", level, "_sem" if sem_token else "", set_names)
-
- self.map_datasets_files.update({
- "READ_2016": {
- # (350 for train, 50 for validation and 50 for test)
- "page": {
- "arx_files": ["Test-ICFHR-2016.tgz", "Train-And-Val-ICFHR-2016.tgz"],
- "needed_files": [],
- "format_function": self.format_read2016_page,
- },
- # (169 for train, 24 for validation and 24 for test)
- "double_page": {
- "arx_files": ["Test-ICFHR-2016.tgz", "Train-And-Val-ICFHR-2016.tgz"],
- "needed_files": [],
- "format_function": self.format_read2016_double_page,
- }
- }
- })
- self.dpi = dpi
- self.end_token = end_token
- self.sem_token = sem_token
- self.matching_token = SEM_MATCHING_TOKENS
-
- def init_format(self):
- super().init_format()
- os.rename(os.path.join(self.temp_fold, "PublicData", "Training"), os.path.join(self.temp_fold, "train"))
- os.rename(os.path.join(self.temp_fold, "PublicData", "Validation"), os.path.join(self.temp_fold, "valid"))
- os.rename(os.path.join(self.temp_fold, "Test-ICFHR-2016"), os.path.join(self.temp_fold, "test"))
- os.rmdir(os.path.join(self.temp_fold, "PublicData"))
- for set_name in ["train", "valid", ]:
- for filename in os.listdir(os.path.join(self.temp_fold, set_name, "Images")):
- filepath = os.path.join(self.temp_fold, set_name, "Images", filename)
- if os.path.isfile(filepath):
- os.rename(filepath, os.path.join(self.temp_fold, set_name, filename))
- os.rmdir(os.path.join(self.temp_fold, set_name, "Images"))
-
- def preformat_read2016(self):
- """
- Extract all information from READ 2016 dataset and correct some mistakes
- """
- def coord_str_to_points(coord_str):
- """
- Extract bounding box from coord string
- """
- points = coord_str.split(" ")
- x_points, y_points = list(), list()
- for p in points:
- y_points.append(int(p.split(",")[1]))
- x_points.append(int(p.split(",")[0]))
- top, bottom, left, right = np.min(y_points), np.max(y_points), np.min(x_points), np.max(x_points)
- return {
- "left": left,
- "bottom": bottom,
- "top": top,
- "right": right
- }
-
- def baseline_str_to_points(coord_str):
- """
- Extract bounding box from baseline string
- """
- points = coord_str.split(" ")
- x_points, y_points = list(), list()
- for p in points:
- y_points.append(int(p.split(",")[1]))
- x_points.append(int(p.split(",")[0]))
- top, bottom, left, right = np.min(y_points), np.max(y_points), np.min(x_points), np.max(x_points)
- return {
- "left": left,
- "bottom": bottom,
- "top": top,
- "right": right
- }
-
- dataset = {
- "train": list(),
- "valid": list(),
- "test": list(),
- }
- for set_name in ["train", "valid", "test"]:
- img_fold_path = os.path.join(self.temp_fold, set_name)
- xml_fold_path = os.path.join(self.temp_fold, set_name, "page")
- for xml_file_name in sorted(os.listdir(xml_fold_path)):
- if xml_file_name.split(".")[-1] != "xml":
- continue
- filename = xml_file_name.split(".")[0]
- img_path = os.path.join(img_fold_path, filename + ".JPG")
- xml_file_path = os.path.join(xml_fold_path, xml_file_name)
- xml_root = ET.parse(xml_file_path).getroot()
- pages = xml_root.findall("{http://schema.primaresearch.org/PAGE/gts/pagecontent/2013-07-15}Page")
- for page in pages:
- page_dict = {
- "label": list(),
- "text_regions": list(),
- "img_path": img_path,
- "width": int(page.attrib["imageWidth"]),
- "height": int(page.attrib["imageHeight"])
- }
- text_regions = page.findall("{http://schema.primaresearch.org/PAGE/gts/pagecontent/2013-07-15}TextRegion")
- for text_region in text_regions:
- text_region_dict = {
- "label": list(),
- "lines": list(),
- "coords": coord_str_to_points(text_region.find("{http://schema.primaresearch.org/PAGE/gts/pagecontent/2013-07-15}Coords").attrib["points"])
- }
- text_lines = text_region.findall("{http://schema.primaresearch.org/PAGE/gts/pagecontent/2013-07-15}TextLine")
- for text_line in text_lines:
- text_line_label = text_line.find("{http://schema.primaresearch.org/PAGE/gts/pagecontent/2013-07-15}TextEquiv")\
- .find("{http://schema.primaresearch.org/PAGE/gts/pagecontent/2013-07-15}Unicode")\
- .text
- if text_line_label is None and \
- text_line.attrib["id"] not in ["line_a5f4ab4e-2ea0-4c65-840c-4a89b04bd477",
- "line_e1288df8-8a0d-40df-be91-4b4a332027ec",
- "line_455330f3-9e27-4340-ae86-9d6c448dc091",
- "line_ecbbccee-e8c2-495d-ac47-0aff93f3d9ac",
- "line_e918616d-64f8-43d2-869c-f687726212be",
- "line_ebd8f850-1da5-45b1-b59c-9349497ecc8e",
- "line_816fb2ce-06b0-4e00-bb28-10c8b9c367f2"]:
- print("ignored null line{}".format(page_dict["img_path"]))
- continue
- if text_line.attrib["id"] == "line_816fb2ce-06b0-4e00-bb28-10c8b9c367f2":
- label = "16"
- elif text_line.attrib["id"] == "line_a5f4ab4e-2ea0-4c65-840c-4a89b04bd477":
- label = "108"
- elif text_line.attrib["id"] == "line_e1288df8-8a0d-40df-be91-4b4a332027ec":
- label = "196"
- elif text_line.attrib["id"] == "line_455330f3-9e27-4340-ae86-9d6c448dc091":
- label = "199"
- elif text_line.attrib["id"] == "line_ecbbccee-e8c2-495d-ac47-0aff93f3d9ac":
- label = "202"
- elif text_line.attrib["id"] == "line_e918616d-64f8-43d2-869c-f687726212be":
- label = "214"
- elif text_line.attrib["id"] == "line_ebd8f850-1da5-45b1-b59c-9349497ecc8e":
- label = "216"
- else:
- label = self.format_text_label(text_line_label)
- line_baseline = text_line.find("{http://schema.primaresearch.org/PAGE/gts/pagecontent/2013-07-15}Baseline")
- line_coord = coord_str_to_points(text_line.find("{http://schema.primaresearch.org/PAGE/gts/pagecontent/2013-07-15}Coords").attrib["points"])
- text_line_dict = {
- "label": label,
- "coords": line_coord,
- "baseline_coords": baseline_str_to_points(line_baseline.attrib["points"]) if line_baseline is not None else line_coord
- }
- text_line_dict["label"] = text_line_dict["label"]
- text_region_dict["label"].append(text_line_dict["label"])
- text_region_dict["lines"].append(text_line_dict)
- if text_region_dict["label"] == list():
- print("ignored null region {}".format(page_dict["img_path"]))
- continue
- text_region_dict["label"] = self.format_text_label("\n".join(text_region_dict["label"]))
- text_region_dict["baseline_coords"] = {
- "left": min([line["baseline_coords"]["left"] for line in text_region_dict["lines"]]),
- "right": max([line["baseline_coords"]["right"] for line in text_region_dict["lines"]]),
- "bottom": max([line["baseline_coords"]["bottom"] for line in text_region_dict["lines"]]),
- "top": min([line["baseline_coords"]["top"] for line in text_region_dict["lines"]]),
- }
- page_dict["label"].append(text_region_dict["label"])
- page_dict["text_regions"].append(text_region_dict)
- page_dict["label"] = self.format_text_label("\n".join(page_dict["label"]))
- dataset[set_name].append(page_dict)
-
- return dataset
-
- def format_read2016_page(self):
- """
- Format the READ 2016 dataset at single-page level
- """
- dataset = self.preformat_read2016()
- for set_name in ["train", "valid", "test"]:
- for i, page in enumerate(dataset[set_name]):
- new_img_name = "{}_{}.jpeg".format(set_name, i)
- new_img_path = os.path.join(self.target_fold_path, set_name, new_img_name)
- self.load_resize_save(page["img_path"], new_img_path, 300, self.dpi)
- new_label, sorted_text_regions, nb_cols, side = self.sort_text_regions(page["text_regions"], page["width"])
- paragraphs = list()
- for paragraph in page["text_regions"]:
- paragraph_label = {
- "label": paragraph["label"],
- "lines": list(),
- "mode": paragraph["mode"]
- }
- for line in paragraph["lines"]:
- paragraph_label["lines"].append({
- "text": line["label"],
- "top": line["coords"]["top"],
- "bottom": line["coords"]["bottom"],
- "left": line["coords"]["left"],
- "right": line["coords"]["right"],
- })
- paragraph_label["lines"][-1] = self.adjust_coord_ratio(paragraph_label["lines"][-1], self.dpi / 300)
- paragraph_label["top"] = min([line["top"] for line in paragraph_label["lines"]])
- paragraph_label["bottom"] = max([line["bottom"] for line in paragraph_label["lines"]])
- paragraph_label["left"] = min([line["left"] for line in paragraph_label["lines"]])
- paragraph_label["right"] = max([line["right"] for line in paragraph_label["lines"]])
- paragraphs.append(paragraph_label)
-
- if self.sem_token:
- if self.end_token:
- new_label = "â“Ÿ" + new_label + "â“…"
- else:
- new_label = "â“Ÿ" + new_label
-
- page_label = {
- "text": new_label,
- "paragraphs": paragraphs,
- "nb_cols": nb_cols,
- "side": side,
- "top": min([pg["top"] for pg in paragraphs]),
- "bottom": max([pg["bottom"] for pg in paragraphs]),
- "left": min([pg["left"] for pg in paragraphs]),
- "right": max([pg["right"] for pg in paragraphs]),
- "page_width": int(np.array(Image.open(page["img_path"])).shape[1] * self.dpi / 300)
- }
-
- self.gt[set_name][new_img_name] = {
- "text": new_label,
- "nb_cols": nb_cols,
- "pages": [page_label, ],
- }
- self.charset = self.charset.union(set(page["label"]))
- self.add_tokens_in_charset()
-
- def format_read2016_double_page(self):
- """
- Format the READ 2016 dataset at double-page level
- """
- dataset = self.preformat_read2016()
- for set_name in ["train", "valid", "test"]:
- for i, page in enumerate(dataset[set_name]):
- dataset[set_name][i]["label"], dataset[set_name][i]["text_regions"], dataset[set_name][i]["nb_cols"], dataset[set_name][i]["side"] = \
- self.sort_text_regions(dataset[set_name][i]["text_regions"], dataset[set_name][i]["width"])
- dataset = self.group_by_page_number(dataset)
- for set_name in ["train", "valid", "test"]:
- i = 0
- for document in dataset[set_name]:
- if len(document["pages"]) != 2:
- continue
- new_img_name = "{}_{}.jpeg".format(set_name, i)
- new_img_path = os.path.join(self.target_fold_path, set_name, new_img_name)
- img_left = np.array(Image.open(document["pages"][0]["img_path"]))
- img_right = np.array(Image.open(document["pages"][1]["img_path"]))
- left_page_width = img_left.shape[1]
- right_page_width = img_right.shape[1]
- img = np.concatenate([img_left, img_right], axis=1)
- img = self.resize(img, 300, self.dpi)
- img = Image.fromarray(img)
- img.save(new_img_path)
- pages = list()
- for page_id, page in enumerate(document["pages"]):
- page_label = {
- "text": page["label"],
- "paragraphs": list(),
- "nb_cols": page["nb_cols"]
- }
- for paragraph in page["text_regions"]:
- paragraph_label = {
- "label": paragraph["label"],
- "lines": list(),
- "mode": paragraph["mode"]
- }
- for line in paragraph["lines"]:
- paragraph_label["lines"].append({
- "text": line["label"],
- "top": line["coords"]["top"],
- "bottom": line["coords"]["bottom"],
- "left": line["coords"]["left"] if page_id == 0 else line["coords"]["left"] + left_page_width,
- "right": line["coords"]["right"]if page_id == 0 else line["coords"]["right"] + left_page_width,
- })
- paragraph_label["lines"][-1] = self.adjust_coord_ratio(paragraph_label["lines"][-1], self.dpi / 300)
- paragraph_label["top"] = min([line["top"] for line in paragraph_label["lines"]])
- paragraph_label["bottom"] = max([line["bottom"] for line in paragraph_label["lines"]])
- paragraph_label["left"] = min([line["left"] for line in paragraph_label["lines"]])
- paragraph_label["right"] = max([line["right"] for line in paragraph_label["lines"]])
- page_label["paragraphs"].append(paragraph_label)
- page_label["top"] = min([pg["top"] for pg in page_label["paragraphs"]])
- page_label["bottom"] = max([pg["bottom"] for pg in page_label["paragraphs"]])
- page_label["left"] = min([pg["left"] for pg in page_label["paragraphs"]])
- page_label["right"] = max([pg["right"] for pg in page_label["paragraphs"]])
- page_label["page_width"] = int(left_page_width * self.dpi / 300) if page_id == 0 else int(right_page_width * self.dpi / 300)
- page_label["side"] = page["side"]
- pages.append(page_label)
-
- label_left = document["pages"][0]["label"]
- label_right = document["pages"][1]["label"]
- if self.sem_token:
- if self.end_token:
- document_label = "â“Ÿ" + label_left + "â“…" + "â“Ÿ" + label_right + "â“…"
- else:
- document_label = "â“Ÿ" + label_left + "â“Ÿ" + label_right
- else:
- document_label = label_left + "\n" + label_right
- self.gt[set_name][new_img_name] = {
- "text": document_label,
- "nb_cols": document["pages"][0]["nb_cols"] + document["pages"][1]["nb_cols"],
- "pages": pages,
- }
- self.charset = self.charset.union(set(document_label))
- i += 1
- self.add_tokens_in_charset()
-
- def add_tokens_in_charset(self):
- """
- Add layout tokens to the charset
- """
- if self.sem_token:
- if self.end_token:
- self.charset = self.charset.union(set("ⓢⓑâ“â“ⓈⒷⒶⓃⓟⓅ"))
- else:
- self.charset = self.charset.union(set("ⓢⓑâ“â“â“Ÿ"))
-
- def group_by_page_number(self, dataset):
- """
- Group page data by pairs of successive pages
- """
- new_dataset = {
- "train": dict(),
- "valid": dict(),
- "test": dict()
- }
- for set_name in ["train", "valid", "test"]:
- for page in dataset[set_name]:
- page_num = int(page["text_regions"][0]["label"].replace(".", "").replace(" ", "").replace("â“", "").replace("Ⓝ", ""))
- page["page_num"] = page_num
- if page_num in new_dataset[set_name]:
- new_dataset[set_name][page_num].append(page)
- else:
- new_dataset[set_name][page_num] = [page, ]
- new_dataset[set_name] = [{"pages": new_dataset[set_name][key],
- "page_num": new_dataset[set_name][key][0]["page_num"]
- } for key in new_dataset[set_name]]
- return new_dataset
-
- def update_label(self, label, start_token):
- """
- Add layout token to text region transcription
- """
- if self.sem_token:
- if self.end_token:
- return start_token + label + self.matching_token[start_token]
- else:
- return start_token + label
- return label
-
- def merge_group_tr(self, group, text_region):
- group["text_regions"].append(text_region)
- group["coords"]["top"] = min([tr["coords"]["top"] for tr in group["text_regions"]])
- group["coords"]["bottom"] = max([tr["coords"]["bottom"] for tr in group["text_regions"]])
- group["coords"]["left"] = min([tr["coords"]["left"] for tr in group["text_regions"]])
- group["coords"]["right"] = max([tr["coords"]["right"] for tr in group["text_regions"]])
- group["baseline_coords"]["top"] = min([tr["baseline_coords"]["top"] for tr in group["text_regions"]])
- group["baseline_coords"]["bottom"] = max([tr["baseline_coords"]["bottom"] for tr in group["text_regions"]])
- group["baseline_coords"]["left"] = min([tr["baseline_coords"]["left"] for tr in group["text_regions"]])
- group["baseline_coords"]["right"] = max([tr["baseline_coords"]["right"] for tr in group["text_regions"]])
-
- def is_annotation_alone(self, groups, page_width):
- for group in groups:
- if all([tr["coords"]["right"] < page_width / 2 and not tr["label"].replace(".", "").replace(" ", "").isdigit() for tr in group["text_regions"]]):
- return True
- return False
-
- def sort_text_regions(self, text_regions, page_width):
- """
- Establish reading order based on paragraph pixel position:
- page number then section by section: first all annotations, then associated body
- """
- nb_cols = 1
- groups = list()
- for text_region in text_regions:
- added_in_group = False
- temp_label = text_region["label"].replace(".", "").replace(" ", "")
- if len(temp_label) <= 4 and temp_label.isdigit():
- groups.append({
- "coords": text_region["coords"].copy(),
- "baseline_coords": text_region["baseline_coords"].copy(),
- "text_regions": [text_region, ]
- })
- groups[-1]["coords"]["top"] = 0
- groups[-1]["coords"]["bottom"] = 0
- groups[-1]["baseline_coords"]["top"] = 0
- groups[-1]["baseline_coords"]["bottom"] = 0
- continue
- for group in groups:
- if not (group["baseline_coords"]["bottom"] <= text_region["baseline_coords"]["top"] or
- group["baseline_coords"]["top"] >= text_region["baseline_coords"]["bottom"] or
- (text_region["coords"]["right"]-text_region["coords"]["left"] > 0.4*page_width and
- group["coords"]["right"]-group["coords"]["left"] > 0.4*page_width)):
- self.merge_group_tr(group, text_region)
- added_in_group = True
- break
-
- if not added_in_group:
- groups.append({
- "coords": text_region["coords"].copy(),
- "baseline_coords": text_region["baseline_coords"].copy(),
- "text_regions": [text_region, ]
- })
- while self.is_annotation_alone(groups, page_width):
- new_groups = list()
- annotations_groups = list()
- body_groups = list()
- for i in range(len(groups)):
- group = groups[i]
- if all([tr["label"].replace(".", "").replace(" ","").isdigit() for tr in group["text_regions"]]):
- new_groups.append(group)
- continue
- if all([tr["coords"]["right"] < page_width / 2 for tr in group["text_regions"]]):
- annotations_groups.append(group)
- continue
- body_groups.append(group)
- for ag in annotations_groups:
- dist = [min([abs(ag["coords"]["bottom"]-g["coords"]["top"]), abs(ag["coords"]["top"]-g["coords"]["bottom"])]) for g in body_groups]
- index = np.argmin(dist)
- for tr in ag["text_regions"]:
- self.merge_group_tr(body_groups[index], tr)
- new_groups.extend(body_groups)
- groups = new_groups
- ordered_groups = sorted(groups, key=lambda g: g["coords"]["top"])
- sorted_text_regions = list()
- for group in ordered_groups:
- text_regions = group["text_regions"]
- if len(text_regions) == 1 and text_regions[0]["label"].replace(".", "").replace(" ", "").isdigit():
- side = "right" if text_regions[0]["coords"]["left"] > page_width / 2 else "left"
- sorted_text_regions.append(text_regions[0])
- sorted_text_regions[-1]["mode"] = "page_number"
- sorted_text_regions[-1]["label"] = self.update_label(sorted_text_regions[-1]["label"], "â“")
- else:
- left = [tr for tr in group["text_regions"] if tr["coords"]["right"] < page_width / 2]
- right = [tr for tr in group["text_regions"] if tr["coords"]["right"] >= page_width / 2]
- nb_cols = max(2 if len(left) > 0 else 1, nb_cols)
- for i, text_region in enumerate(sorted(left, key=lambda tr: tr["coords"]["top"])):
- sorted_text_regions.append(text_region)
- sorted_text_regions[-1]["mode"] = "annotation"
- sorted_text_regions[-1]["label"] = self.update_label(sorted_text_regions[-1]["label"], "â“")
- if i == 0 and self.sem_token:
- sorted_text_regions[-1]["label"] = "â“¢" + sorted_text_regions[-1]["label"]
- for i, text_region in enumerate(sorted(right, key=lambda tr: tr["coords"]["top"])):
- sorted_text_regions.append(text_region)
- sorted_text_regions[-1]["mode"] = "body"
- sorted_text_regions[-1]["label"] = self.update_label(sorted_text_regions[-1]["label"], "â“‘")
- if i == 0 and self.sem_token and len(left) == 0:
- sorted_text_regions[-1]["label"] = "â“¢" + sorted_text_regions[-1]["label"]
- if i == len(right)-1 and self.sem_token and self.end_token:
- sorted_text_regions[-1]["label"] = sorted_text_regions[-1]["label"] + self.matching_token["â“¢"]
-
- sep = "" if self.sem_token else "\n"
- new_label = sep.join(t["label"] for t in sorted_text_regions)
- return new_label, sorted_text_regions, nb_cols, side
-
-
-if __name__ == "__main__":
-
- READ2016DatasetFormatter("page", sem_token=True).format()
- READ2016DatasetFormatter("page", sem_token=False).format()
- READ2016DatasetFormatter("double_page", sem_token=True).format()
- READ2016DatasetFormatter("double_page", sem_token=False).format()
\ No newline at end of file
diff --git a/Datasets/dataset_formatters/rimes_formatter.py b/Datasets/dataset_formatters/rimes_formatter.py
deleted file mode 100644
index 929a42a44bafccc2bb88e1138af1042b109d2cd2..0000000000000000000000000000000000000000
--- a/Datasets/dataset_formatters/rimes_formatter.py
+++ /dev/null
@@ -1,257 +0,0 @@
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in Python whose purpose is to
-# provide public implementation of deep learning works, in pytorch.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
-
-from Datasets.dataset_formatters.generic_dataset_formatter import OCRDatasetFormatter
-import os
-import numpy as np
-from Datasets.dataset_formatters.utils_dataset import natural_sort
-from PIL import Image
-import xml.etree.ElementTree as ET
-import re
-
-# Layout string to token
-SEM_MATCHING_TOKENS_STR = {
- 'Ouverture': "â“ž", # opening
- 'Corps de texte': "â“‘", # body
- 'PS/PJ': "â“Ÿ", # post scriptum
- 'Coordonnées Expéditeur': "ⓢ", # sender
- 'Reference': "â“¢", # also counted as sender information
- 'Objet': "ⓨ", # why
- 'Date, Lieu': "ⓦ", # where, when
- 'Coordonnées Destinataire': "ⓡ", # recipient
- }
-
-# Layout begin-token to end-token
-SEM_MATCHING_TOKENS = {
- "â“‘": "â’·",
- "â“ž": "â“„",
- "ⓡ": "Ⓡ",
- "ⓢ": "Ⓢ",
- "ⓦ": "Ⓦ",
- "ⓨ": "Ⓨ",
- "â“Ÿ": "â“…"
- }
-
-
-class RIMESDatasetFormatter(OCRDatasetFormatter):
- def __init__(self, level, set_names=["train", "valid", "test"], dpi=150, sem_token=True):
- super(RIMESDatasetFormatter, self).__init__("RIMES", level, "_sem" if sem_token else "", set_names)
-
- self.source_fold_path = os.path.join("../raw", "RIMES")
- self.dpi = dpi
- self.sem_token = sem_token
- self.map_datasets_files.update({
- "RIMES": {
- # (1,050 for train, 100 for validation and 100 for test)
- "page": {
- "arx_files": ["RIMES_page.tar.gz", ],
- "needed_files": [],
- "format_function": self.format_rimes_page,
- },
- }
- })
-
- self.matching_tokens_str = SEM_MATCHING_TOKENS_STR
- self.matching_tokens = SEM_MATCHING_TOKENS
- self.ordering_function = order_text_regions
-
- def preformat_rimes_page(self):
- """
- Extract all information from dataset and correct some annotations
- """
- dataset = {
- "train": list(),
- "valid": list(),
- "test": list()
- }
- img_folder_path = os.path.join(self.temp_fold, "RIMES page", "Images")
- xml_folder_path = os.path.join(self.temp_fold, "RIMES page", "XML")
- xml_files = natural_sort([os.path.join(xml_folder_path, name) for name in os.listdir(xml_folder_path)])
- train_xml = xml_files[:1050]
- valid_xml = xml_files[1050:1150]
- test_xml = xml_files[1150:]
- for set_name, xml_files in zip(self.set_names, [train_xml, valid_xml, test_xml]):
- for i, xml_path in enumerate(xml_files):
- text_regions = list()
- root = ET.parse(xml_path).getroot()
- img_name = root.find("source").text
- if img_name == "01160_L.png":
- text_regions.append({
- "label": "LETTRE RECOMMANDEE\nAVEC ACCUSE DE RECEPTION",
- "type": "",
- "coords": {
- "left": 88,
- "right": 1364,
- "top": 1224,
- "bottom": 1448,
- }
- })
- for text_region in root.findall("box"):
- type = text_region.find("type").text
- label = text_region.find("text").text
- if label is None or len(label.strip()) <= 0:
- continue
- if label == "Ref : QVLCP¨65":
- label = label.replace("¨", "")
- if img_name == "01094_L.png" and type == "Corps de texte":
- label = "Suite à la tempête du 19.11.06, un\narbre est tombé sur mon toît et l'a endommagé.\nJe d'eplore une cinquantaine de tuiles à changer,\nune poutre à réparer et une gouttière à \nremplacer. Veuillez trouver ci-joint le devis\nde réparation. Merci de m'envoyer votre\nexpert le plus rapidement possible.\nEn esperant une réponse rapide de votre\npart, veuillez accepter, madame, monsieur,\nmes salutations distinguées."
- elif img_name == "01111_L.png" and type == "Corps de texte":
- label = "Je vous ai envoyé un courrier le 20 octobre 2006\nvous signalant un sinistre survenu dans ma\nmaison, un dégât des eaux consécutif aux\nfortes pluis.\nVous deviez envoyer un expert pour constater\nles dégâts. Personne n'est venu à ce jour\nJe vous prie donc de faire le nécessaire\nafin que les réparations nécessaires puissent\nêtre commencés.\nDans l'attente, veuillez agréer, Monsieur,\nmes sincères salutations"
-
- label = self.convert_label_accent(label)
- label = self.convert_label(label)
- label = self.format_text_label(label)
- coords = {
- "left": int(text_region.attrib["top_left_x"]),
- "right": int(text_region.attrib["bottom_right_x"]),
- "top": int(text_region.attrib["top_left_y"]),
- "bottom": int(text_region.attrib["bottom_right_y"]),
- }
- text_regions.append({
- "label": label,
- "type": type,
- "coords": coords
- })
- text_regions = self.ordering_function(text_regions)
- dataset[set_name].append({
- "text_regions": text_regions,
- "img_path": os.path.join(img_folder_path, img_name),
- "label": "\n".join([tr["label"] for tr in text_regions]),
- "sem_label": "".join([self.sem_label(tr["label"], tr["type"]) for tr in text_regions]),
- })
- return dataset
-
- def convert_label_accent(self, label):
- """
- Solve encoding issues
- """
- return label.replace("\\n", "\n").replace("<euro>", "€").replace(">euro>", "€").replace(">fligne>", " ")\
- .replace("¤", "¤").replace("û", "û").replace("�", "").replace("ï¿©", "é").replace("ç", "ç")\
- .replace("é", "é").replace("ô", "ô").replace(u'\xa0', " ").replace("è", "è").replace("°", "°")\
- .replace("À", "À").replace("ì", "À").replace("ê", "ê").replace("î", "î").replace("â", "â")\
- .replace("²", "²").replace("ù", "ù").replace("Ã", "à ").replace("¬", "€")
-
- def format_rimes_page(self):
- """
- Format RIMES page dataset
- """
- dataset = self.preformat_rimes_page()
- for set_name in self.set_names:
- fold = os.path.join(self.target_fold_path, set_name)
- for sample in dataset[set_name]:
- new_name = "{}_{}.png".format(set_name, len(os.listdir(fold)))
- new_img_path = os.path.join(fold, new_name)
- self.load_resize_save(sample["img_path"], new_img_path, 300, self.dpi)
- for tr in sample["text_regions"]:
- tr["coords"] = self.adjust_coord_ratio(tr["coords"], self.dpi / 300)
- page = {
- "text": sample["label"] if not self.sem_token else sample["sem_label"],
- "paragraphs": sample["text_regions"],
- "nb_cols": 1,
- }
- self.charset = self.charset.union(set(page["text"]))
- self.gt[set_name][new_name] = page
-
- def convert_label(self, label):
- """
- Some annotations presents many options for a given text part, always keep the first one only
- """
- if "¤" in label:
- label = re.sub('¤{([^¤]*)[/|]([^¤]*)}¤', r'\1', label, flags=re.DOTALL)
- label = re.sub('¤{([^¤]*)[/|]([^¤]*)[/|]([^¤]*)>', r'\1', label, flags=re.DOTALL)
- label = re.sub('¤([^¤]*)[/|]([^¤]*)¤', r'\1', label, flags=re.DOTALL)
- label = re.sub('¤{}¤([^¤]*)[/|]([^ ]*)', r'\1', label, flags=re.DOTALL)
- label = re.sub('¤{/([^¤]*)/([^ ]*)', r'\1', label, flags=re.DOTALL)
- label = re.sub('¤{([^¤]*)[/|]([^ ]*)', r'\1', label, flags=re.DOTALL)
- label = re.sub('([^¤]*)/(.*)[¤}{]+', r'\1', label, flags=re.DOTALL)
- label = re.sub('[¤}{]+([^¤}{]*)[¤}{]+', r'\1', label, flags=re.DOTALL)
- label = re.sub('¤([^¤]*)¤', r'\1', label, flags=re.DOTALL)
- label = re.sub('[ ]+', " ", label, flags=re.DOTALL)
- label = label.strip()
- return label
-
- def sem_label(self, label, type):
- """
- Add layout tokens
- """
- if type == "":
- return label
- begin_token = self.matching_tokens_str[type]
- end_token = self.matching_tokens[begin_token]
- return begin_token + label + end_token
-
-
-def order_text_regions(text_regions):
- """
- Establish reading order based on text region pixel positions
- """
- sorted_text_regions = list()
- for tr in text_regions:
- added = False
- if len(sorted_text_regions) == 0:
- sorted_text_regions.append(tr)
- added = True
- else:
- for i, sorted_tr in enumerate(sorted_text_regions):
- tr_height = tr["coords"]["bottom"] - tr["coords"]["top"]
- sorted_tr_height = sorted_tr["coords"]["bottom"] - sorted_tr["coords"]["top"]
- tr_is_totally_above = tr["coords"]["bottom"] < sorted_tr["coords"]["top"]
- tr_is_top_above = tr["coords"]["top"] < sorted_tr["coords"]["top"]
- is_same_level = sorted_tr["coords"]["top"] <= tr["coords"]["bottom"] <= sorted_tr["coords"]["bottom"] or\
- sorted_tr["coords"]["top"] <= tr["coords"]["top"] <= sorted_tr["coords"]["bottom"] or\
- tr["coords"]["top"] <= sorted_tr["coords"]["bottom"] <= tr["coords"]["bottom"] or\
- tr["coords"]["top"] <= sorted_tr["coords"]["top"] <= tr["coords"]["bottom"]
- vertical_shared_space = tr["coords"]["bottom"]-sorted_tr["coords"]["top"] if tr_is_top_above else sorted_tr["coords"]["bottom"]-tr["coords"]["top"]
- reach_same_level_limit = vertical_shared_space > 0.3*min(tr_height, sorted_tr_height)
- is_more_at_left = tr["coords"]["left"] < sorted_tr["coords"]["left"]
- equivalent_height = abs(tr_height-sorted_tr_height) < 0.3*min(tr_height, sorted_tr_height)
- is_middle_above_top = np.mean([tr["coords"]["top"], tr["coords"]["bottom"]]) < sorted_tr["coords"]["top"]
- if tr_is_totally_above or\
- (is_same_level and equivalent_height and is_more_at_left and reach_same_level_limit) or\
- (is_same_level and equivalent_height and tr_is_top_above and not reach_same_level_limit) or\
- (is_same_level and not equivalent_height and is_middle_above_top):
- sorted_text_regions.insert(i, tr)
- added = True
- break
- if not added:
- sorted_text_regions.append(tr)
-
- return sorted_text_regions
-
-
-if __name__ == "__main__":
-
- RIMESDatasetFormatter("page", sem_token=True).format()
- RIMESDatasetFormatter("page", sem_token=False).format()
diff --git a/Datasets/dataset_formatters/simara_formatter.py b/Datasets/dataset_formatters/simara_formatter.py
index c8479ed0c7bcd671b438f87f62e1f8f03090c0f2..c60eb30f94cf569d3b2d9d9197a16756ee994c01 100644
--- a/Datasets/dataset_formatters/simara_formatter.py
+++ b/Datasets/dataset_formatters/simara_formatter.py
@@ -1,38 +1,3 @@
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in Python whose purpose is to
-# provide public implementation of deep learning works, in pytorch.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
-
from Datasets.dataset_formatters.generic_dataset_formatter import OCRDatasetFormatter
import os
import numpy as np
diff --git a/Datasets/dataset_formatters/utils_dataset.py b/Datasets/dataset_formatters/utils_dataset.py
index 2495873e580322d46340d1e8d2a5c909a1bc640c..f4d980dadb926ceee5f4d13e28620e21a11a1177 100644
--- a/Datasets/dataset_formatters/utils_dataset.py
+++ b/Datasets/dataset_formatters/utils_dataset.py
@@ -1,39 +1,3 @@
-
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in Python whose purpose is to
-# provide public implementation of deep learning works, in pytorch.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
-
import re
import random
import cv2
diff --git a/Fonts/list_fonts_read_2016.txt b/Fonts/list_fonts_read_2016.txt
deleted file mode 100644
index ab092fa8a0826a122644bbf295d0378659528f9c..0000000000000000000000000000000000000000
--- a/Fonts/list_fonts_read_2016.txt
+++ /dev/null
@@ -1,41 +0,0 @@
-"../../../Fonts/lato/Lato-HairlineItalic.ttf",
-"../../../Fonts/lato/Lato-HeavyItalic.ttf",
-"../../../Fonts/lato/Lato-BoldItalic.ttf",
-"../../../Fonts/lato/Lato-Black.ttf",
-"../../../Fonts/lato/Lato-Heavy.ttf",
-"../../../Fonts/lato/Lato-Regular.ttf",
-"../../../Fonts/lato/Lato-LightItalic.ttf",
-"../../../Fonts/lato/Lato-Italic.ttf",
-"../../../Fonts/lato/Lato-ThinItalic.ttf",
-"../../../Fonts/lato/Lato-Bold.ttf",
-"../../../Fonts/lato/Lato-Hairline.ttf",
-"../../../Fonts/lato/Lato-Medium.ttf",
-"../../../Fonts/lato/Lato-SemiboldItalic.ttf",
-"../../../Fonts/lato/Lato-BlackItalic.ttf",
-"../../../Fonts/lato/Lato-MediumItalic.ttf",
-"../../../Fonts/lato/Lato-Semibold.ttf",
-"../../../Fonts/lato/Lato-Thin.ttf",
-"../../../Fonts/lato/Lato-Light.ttf",
-"../../../Fonts/gentiumplus/GentiumPlus-I.ttf",
-"../../../Fonts/gentiumplus/GentiumPlus-R.ttf",
-"../../../Fonts/dejavu/DejaVuSansMono-BoldOblique.ttf",
-"../../../Fonts/dejavu/DejaVuSerifCondensed.ttf",
-"../../../Fonts/dejavu/DejaVuSans-BoldOblique.ttf",
-"../../../Fonts/dejavu/DejaVuSans-ExtraLight.ttf",
-"../../../Fonts/dejavu/DejaVuSansCondensed-Oblique.ttf",
-"../../../Fonts/dejavu/DejaVuSerifCondensed-BoldItalic.ttf",
-"../../../Fonts/dejavu/DejaVuSansCondensed-Bold.ttf",
-"../../../Fonts/dejavu/DejaVuSerif-Italic.ttf",
-"../../../Fonts/dejavu/DejaVuSansCondensed.ttf",
-"../../../Fonts/dejavu/DejaVuSerifCondensed-Italic.ttf",
-"../../../Fonts/dejavu/DejaVuSerifCondensed-Bold.ttf",
-"../../../Fonts/dejavu/DejaVuSansMono.ttf",
-"../../../Fonts/dejavu/DejaVuSerif-Bold.ttf",
-"../../../Fonts/dejavu/DejaVuSans-Bold.ttf",
-"../../../Fonts/dejavu/DejaVuSerif.ttf",
-"../../../Fonts/dejavu/DejaVuSansMono-Bold.ttf",
-"../../../Fonts/dejavu/DejaVuSerif-BoldItalic.ttf",
-"../../../Fonts/dejavu/DejaVuSansMono-Oblique.ttf",
-"../../../Fonts/dejavu/DejaVuSans.ttf",
-"../../../Fonts/dejavu/DejaVuSans-Oblique.ttf",
-"../../../Fonts/dejavu/DejaVuSansCondensed-BoldOblique.ttf"
\ No newline at end of file
diff --git a/Fonts/list_fonts_rimes.txt b/Fonts/list_fonts_rimes.txt
deleted file mode 100644
index 4628dbd54f2d3275f43ebabae96b605ff179319f..0000000000000000000000000000000000000000
--- a/Fonts/list_fonts_rimes.txt
+++ /dev/null
@@ -1,95 +0,0 @@
-"../../../Fonts/handwritten-mix/Parisienne-Regular.ttf",
-"../../../Fonts/handwritten-mix/A little sunshine.ttf",
-"../../../Fonts/handwritten-mix/Massillo.ttf",
-"../../../Fonts/handwritten-mix/Cursive standard Bold.ttf",
-"../../../Fonts/handwritten-mix/Merveille-mj8j.ttf",
-"../../../Fonts/handwritten-mix/Cursive standard.ttf",
-"../../../Fonts/handwritten-mix/Roustel.ttf",
-"../../../Fonts/handwritten-mix/Baby Doll.ttf",
-"../../../Fonts/handwritten-mix/flashback Demo.ttf",
-"../../../Fonts/handwritten-mix/CreamShoes.ttf",
-"../../../Fonts/handwritten-mix/Gentle Remind.ttf",
-"../../../Fonts/handwritten-mix/Alexandria Rose.ttf",
-"../../../Fonts/lato/Lato-HairlineItalic.ttf",
-"../../../Fonts/lato/Lato-HeavyItalic.ttf",
-"../../../Fonts/lato/Lato-BoldItalic.ttf",
-"../../../Fonts/lato/Lato-Black.ttf",
-"../../../Fonts/lato/Lato-Heavy.ttf",
-"../../../Fonts/lato/Lato-Regular.ttf",
-"../../../Fonts/lato/Lato-LightItalic.ttf",
-"../../../Fonts/lato/Lato-Italic.ttf",
-"../../../Fonts/lato/Lato-ThinItalic.ttf",
-"../../../Fonts/lato/Lato-Bold.ttf",
-"../../../Fonts/lato/Lato-Hairline.ttf",
-"../../../Fonts/lato/Lato-Medium.ttf",
-"../../../Fonts/lato/Lato-SemiboldItalic.ttf",
-"../../../Fonts/lato/Lato-BlackItalic.ttf",
-"../../../Fonts/lato/Lato-MediumItalic.ttf",
-"../../../Fonts/lato/Lato-Semibold.ttf",
-"../../../Fonts/lato/Lato-Thin.ttf",
-"../../../Fonts/lato/Lato-Light.ttf",
-"../../../Fonts/gentiumplus/GentiumPlus-I.ttf",
-"../../../Fonts/gentiumplus/GentiumPlus-R.ttf",
-"../../../Fonts/dejavu/DejaVuSansMono-BoldOblique.ttf",
-"../../../Fonts/dejavu/DejaVuSerifCondensed.ttf",
-"../../../Fonts/dejavu/DejaVuSans-BoldOblique.ttf",
-"../../../Fonts/dejavu/DejaVuSans-ExtraLight.ttf",
-"../../../Fonts/dejavu/DejaVuSansCondensed-Oblique.ttf",
-"../../../Fonts/dejavu/DejaVuSerifCondensed-BoldItalic.ttf",
-"../../../Fonts/dejavu/DejaVuSansCondensed-Bold.ttf",
-"../../../Fonts/dejavu/DejaVuSerif-Italic.ttf",
-"../../../Fonts/dejavu/DejaVuSansCondensed.ttf",
-"../../../Fonts/dejavu/DejaVuSerifCondensed-Italic.ttf",
-"../../../Fonts/dejavu/DejaVuSerifCondensed-Bold.ttf",
-"../../../Fonts/dejavu/DejaVuSansMono.ttf",
-"../../../Fonts/dejavu/DejaVuSerif-Bold.ttf",
-"../../../Fonts/dejavu/DejaVuSans-Bold.ttf",
-"../../../Fonts/dejavu/DejaVuSerif.ttf",
-"../../../Fonts/dejavu/DejaVuSansMono-Bold.ttf",
-"../../../Fonts/dejavu/DejaVuSerif-BoldItalic.ttf",
-"../../../Fonts/dejavu/DejaVuSansMono-Oblique.ttf",
-"../../../Fonts/dejavu/DejaVuSans.ttf",
-"../../../Fonts/dejavu/DejaVuSans-Oblique.ttf",
-"../../../Fonts/dejavu/DejaVuSansCondensed-BoldOblique.ttf",
-"../../../Fonts/open-sans/OpenSans-SemiboldItalic.ttf",
-"../../../Fonts/open-sans/OpenSans-CondLight.ttf",
-"../../../Fonts/open-sans/OpenSans-Light.ttf",
-"../../../Fonts/open-sans/OpenSans-Italic.ttf",
-"../../../Fonts/open-sans/OpenSans-CondBold.ttf",
-"../../../Fonts/open-sans/OpenSans-Bold.ttf",
-"../../../Fonts/open-sans/OpenSans-CondLightItalic.ttf",
-"../../../Fonts/open-sans/OpenSans-ExtraBold.ttf",
-"../../../Fonts/open-sans/OpenSans-Semibold.ttf",
-"../../../Fonts/open-sans/OpenSans-Regular.ttf",
-"../../../Fonts/open-sans/OpenSans-BoldItalic.ttf",
-"../../../Fonts/open-sans/OpenSans-LightItalic.ttf",
-"../../../Fonts/open-sans/OpenSans-ExtraBoldItalic.ttf",
-"../../../Fonts/msttcorefonts/Arial.ttf",
-"../../../Fonts/msttcorefonts/Verdana_Italic.ttf",
-"../../../Fonts/msttcorefonts/Georgia_Bold_Italic.ttf",
-"../../../Fonts/msttcorefonts/Andale_Mono.ttf",
-"../../../Fonts/msttcorefonts/Courier_New_Italic.ttf",
-"../../../Fonts/msttcorefonts/Georgia_Italic.ttf",
-"../../../Fonts/msttcorefonts/Arial_Black.ttf",
-"../../../Fonts/msttcorefonts/Trebuchet_MS_Italic.ttf",
-"../../../Fonts/msttcorefonts/Verdana.ttf",
-"../../../Fonts/msttcorefonts/Courier_New.ttf",
-"../../../Fonts/msttcorefonts/Verdana_Bold.ttf",
-"../../../Fonts/msttcorefonts/Arial_Bold_Italic.ttf",
-"../../../Fonts/msttcorefonts/Georgia.ttf",
-"../../../Fonts/msttcorefonts/Trebuchet_MS_Bold_Italic.ttf",
-"../../../Fonts/msttcorefonts/Impact.ttf",
-"../../../Fonts/msttcorefonts/Courier_New_Bold.ttf",
-"../../../Fonts/msttcorefonts/Times_New_Roman_Italic.ttf",
-"../../../Fonts/msttcorefonts/Georgia_Bold.ttf",
-"../../../Fonts/msttcorefonts/Times_New_Roman_Bold.ttf",
-"../../../Fonts/msttcorefonts/Times_New_Roman.ttf",
-"../../../Fonts/msttcorefonts/Comic_Sans_MS.ttf",
-"../../../Fonts/msttcorefonts/Trebuchet_MS_Bold.ttf",
-"../../../Fonts/msttcorefonts/Trebuchet_MS.ttf",
-"../../../Fonts/msttcorefonts/Arial_Italic.ttf",
-"../../../Fonts/msttcorefonts/Courier_New_Bold_Italic.ttf",
-"../../../Fonts/msttcorefonts/Verdana_Bold_Italic.ttf",
-"../../../Fonts/msttcorefonts/Arial_Bold.ttf",
-"../../../Fonts/msttcorefonts/Times_New_Roman_Bold_Italic.ttf",
-"../../../Fonts/msttcorefonts/Comic_Sans_MS_Bold.ttf"
\ No newline at end of file
diff --git a/LICENSE_CECILL-C.md b/LICENSE_CECILL-C.md
deleted file mode 100644
index 1f0442b5c61ddf4e915ae4f7f0bacccc944bc440..0000000000000000000000000000000000000000
--- a/LICENSE_CECILL-C.md
+++ /dev/null
@@ -1,517 +0,0 @@
-
-CeCILL-C FREE SOFTWARE LICENSE AGREEMENT
-
-
- Notice
-
-This Agreement is a Free Software license agreement that is the result
-of discussions between its authors in order to ensure compliance with
-the two main principles guiding its drafting:
-
- * firstly, compliance with the principles governing the distribution
- of Free Software: access to source code, broad rights granted to
- users,
- * secondly, the election of a governing law, French law, with which
- it is conformant, both as regards the law of torts and
- intellectual property law, and the protection that it offers to
- both authors and holders of the economic rights over software.
-
-The authors of the CeCILL-C (for Ce[a] C[nrs] I[nria] L[ogiciel] L[ibre])
-license are: Université de Rouen Normandie (1), INSA Rouen (2), tutelles du laboratoire LITIS (1 et 2)
-
-Commissariat à l'Energie Atomique - CEA, a public scientific, technical
-and industrial research establishment, having its principal place of
-business at 25 rue Leblanc, immeuble Le Ponant D, 75015 Paris, France.
-
-Centre National de la Recherche Scientifique - CNRS, a public scientific
-and technological establishment, having its principal place of business
-at 3 rue Michel-Ange, 75794 Paris cedex 16, France.
-
-Institut National de Recherche en Informatique et en Automatique -
-INRIA, a public scientific and technological establishment, having its
-principal place of business at Domaine de Voluceau, Rocquencourt, BP
-105, 78153 Le Chesnay cedex, France.
-
-
- Preamble
-
-The purpose of this Free Software license agreement is to grant users
-the right to modify and re-use the software governed by this license.
-
-The exercising of this right is conditional upon the obligation to make
-available to the community the modifications made to the source code of
-the software so as to contribute to its evolution.
-
-In consideration of access to the source code and the rights to copy,
-modify and redistribute granted by the license, users are provided only
-with a limited warranty and the software's author, the holder of the
-economic rights, and the successive licensors only have limited liability.
-
-In this respect, the risks associated with loading, using, modifying
-and/or developing or reproducing the software by the user are brought to
-the user's attention, given its Free Software status, which may make it
-complicated to use, with the result that its use is reserved for
-developers and experienced professionals having in-depth computer
-knowledge. Users are therefore encouraged to load and test the
-suitability of the software as regards their requirements in conditions
-enabling the security of their systems and/or data to be ensured and,
-more generally, to use and operate it in the same conditions of
-security. This Agreement may be freely reproduced and published,
-provided it is not altered, and that no provisions are either added or
-removed herefrom.
-
-This Agreement may apply to any or all software for which the holder of
-the economic rights decides to submit the use thereof to its provisions.
-
-
- Article 1 - DEFINITIONS
-
-For the purpose of this Agreement, when the following expressions
-commence with a capital letter, they shall have the following meaning:
-
-Agreement: means this license agreement, and its possible subsequent
-versions and annexes.
-
-Software: means the software in its Object Code and/or Source Code form
-and, where applicable, its documentation, "as is" when the Licensee
-accepts the Agreement.
-
-Initial Software: means the Software in its Source Code and possibly its
-Object Code form and, where applicable, its documentation, "as is" when
-it is first distributed under the terms and conditions of the Agreement.
-
-Modified Software: means the Software modified by at least one
-Integrated Contribution.
-
-Source Code: means all the Software's instructions and program lines to
-which access is required so as to modify the Software.
-
-Object Code: means the binary files originating from the compilation of
-the Source Code.
-
-Holder: means the holder(s) of the economic rights over the Initial
-Software.
-
-Licensee: means the Software user(s) having accepted the Agreement.
-
-Contributor: means a Licensee having made at least one Integrated
-Contribution.
-
-Licensor: means the Holder, or any other individual or legal entity, who
-distributes the Software under the Agreement.
-
-Integrated Contribution: means any or all modifications, corrections,
-translations, adaptations and/or new functions integrated into the
-Source Code by any or all Contributors.
-
-Related Module: means a set of sources files including their
-documentation that, without modification to the Source Code, enables
-supplementary functions or services in addition to those offered by the
-Software.
-
-Derivative Software: means any combination of the Software, modified or
-not, and of a Related Module.
-
-Parties: mean both the Licensee and the Licensor.
-
-These expressions may be used both in singular and plural form.
-
-
- Article 2 - PURPOSE
-
-The purpose of the Agreement is the grant by the Licensor to the
-Licensee of a non-exclusive, transferable and worldwide license for the
-Software as set forth in Article 5 hereinafter for the whole term of the
-protection granted by the rights over said Software.
-
-
- Article 3 - ACCEPTANCE
-
-3.1 The Licensee shall be deemed as having accepted the terms and
-conditions of this Agreement upon the occurrence of the first of the
-following events:
-
- * (i) loading the Software by any or all means, notably, by
- downloading from a remote server, or by loading from a physical
- medium;
- * (ii) the first time the Licensee exercises any of the rights
- granted hereunder.
-
-3.2 One copy of the Agreement, containing a notice relating to the
-characteristics of the Software, to the limited warranty, and to the
-fact that its use is restricted to experienced users has been provided
-to the Licensee prior to its acceptance as set forth in Article 3.1
-hereinabove, and the Licensee hereby acknowledges that it has read and
-understood it.
-
-
- Article 4 - EFFECTIVE DATE AND TERM
-
-
- 4.1 EFFECTIVE DATE
-
-The Agreement shall become effective on the date when it is accepted by
-the Licensee as set forth in Article 3.1.
-
-
- 4.2 TERM
-
-The Agreement shall remain in force for the entire legal term of
-protection of the economic rights over the Software.
-
-
- Article 5 - SCOPE OF RIGHTS GRANTED
-
-The Licensor hereby grants to the Licensee, who accepts, the following
-rights over the Software for any or all use, and for the term of the
-Agreement, on the basis of the terms and conditions set forth hereinafter.
-
-Besides, if the Licensor owns or comes to own one or more patents
-protecting all or part of the functions of the Software or of its
-components, the Licensor undertakes not to enforce the rights granted by
-these patents against successive Licensees using, exploiting or
-modifying the Software. If these patents are transferred, the Licensor
-undertakes to have the transferees subscribe to the obligations set
-forth in this paragraph.
-
-
- 5.1 RIGHT OF USE
-
-The Licensee is authorized to use the Software, without any limitation
-as to its fields of application, with it being hereinafter specified
-that this comprises:
-
- 1. permanent or temporary reproduction of all or part of the Software
- by any or all means and in any or all form.
-
- 2. loading, displaying, running, or storing the Software on any or
- all medium.
-
- 3. entitlement to observe, study or test its operation so as to
- determine the ideas and principles behind any or all constituent
- elements of said Software. This shall apply when the Licensee
- carries out any or all loading, displaying, running, transmission
- or storage operation as regards the Software, that it is entitled
- to carry out hereunder.
-
-
- 5.2 RIGHT OF MODIFICATION
-
-The right of modification includes the right to translate, adapt,
-arrange, or make any or all modifications to the Software, and the right
-to reproduce the resulting software. It includes, in particular, the
-right to create a Derivative Software.
-
-The Licensee is authorized to make any or all modification to the
-Software provided that it includes an explicit notice that it is the
-author of said modification and indicates the date of the creation thereof.
-
-
- 5.3 RIGHT OF DISTRIBUTION
-
-In particular, the right of distribution includes the right to publish,
-transmit and communicate the Software to the general public on any or
-all medium, and by any or all means, and the right to market, either in
-consideration of a fee, or free of charge, one or more copies of the
-Software by any means.
-
-The Licensee is further authorized to distribute copies of the modified
-or unmodified Software to third parties according to the terms and
-conditions set forth hereinafter.
-
-
- 5.3.1 DISTRIBUTION OF SOFTWARE WITHOUT MODIFICATION
-
-The Licensee is authorized to distribute true copies of the Software in
-Source Code or Object Code form, provided that said distribution
-complies with all the provisions of the Agreement and is accompanied by:
-
- 1. a copy of the Agreement,
-
- 2. a notice relating to the limitation of both the Licensor's
- warranty and liability as set forth in Articles 8 and 9,
-
-and that, in the event that only the Object Code of the Software is
-redistributed, the Licensee allows effective access to the full Source
-Code of the Software at a minimum during the entire period of its
-distribution of the Software, it being understood that the additional
-cost of acquiring the Source Code shall not exceed the cost of
-transferring the data.
-
-
- 5.3.2 DISTRIBUTION OF MODIFIED SOFTWARE
-
-When the Licensee makes an Integrated Contribution to the Software, the
-terms and conditions for the distribution of the resulting Modified
-Software become subject to all the provisions of this Agreement.
-
-The Licensee is authorized to distribute the Modified Software, in
-source code or object code form, provided that said distribution
-complies with all the provisions of the Agreement and is accompanied by:
-
- 1. a copy of the Agreement,
-
- 2. a notice relating to the limitation of both the Licensor's
- warranty and liability as set forth in Articles 8 and 9,
-
-and that, in the event that only the object code of the Modified
-Software is redistributed, the Licensee allows effective access to the
-full source code of the Modified Software at a minimum during the entire
-period of its distribution of the Modified Software, it being understood
-that the additional cost of acquiring the source code shall not exceed
-the cost of transferring the data.
-
-
- 5.3.3 DISTRIBUTION OF DERIVATIVE SOFTWARE
-
-When the Licensee creates Derivative Software, this Derivative Software
-may be distributed under a license agreement other than this Agreement,
-subject to compliance with the requirement to include a notice
-concerning the rights over the Software as defined in Article 6.4.
-In the event the creation of the Derivative Software required modification
-of the Source Code, the Licensee undertakes that:
-
- 1. the resulting Modified Software will be governed by this Agreement,
- 2. the Integrated Contributions in the resulting Modified Software
- will be clearly identified and documented,
- 3. the Licensee will allow effective access to the source code of the
- Modified Software, at a minimum during the entire period of
- distribution of the Derivative Software, such that such
- modifications may be carried over in a subsequent version of the
- Software; it being understood that the additional cost of
- purchasing the source code of the Modified Software shall not
- exceed the cost of transferring the data.
-
-
- 5.3.4 COMPATIBILITY WITH THE CeCILL LICENSE
-
-When a Modified Software contains an Integrated Contribution subject to
-the CeCILL license agreement, or when a Derivative Software contains a
-Related Module subject to the CeCILL license agreement, the provisions
-set forth in the third item of Article 6.4 are optional.
-
-
- Article 6 - INTELLECTUAL PROPERTY
-
-
- 6.1 OVER THE INITIAL SOFTWARE
-
-The Holder owns the economic rights over the Initial Software. Any or
-all use of the Initial Software is subject to compliance with the terms
-and conditions under which the Holder has elected to distribute its work
-and no one shall be entitled to modify the terms and conditions for the
-distribution of said Initial Software.
-
-The Holder undertakes that the Initial Software will remain ruled at
-least by this Agreement, for the duration set forth in Article 4.2.
-
-
- 6.2 OVER THE INTEGRATED CONTRIBUTIONS
-
-The Licensee who develops an Integrated Contribution is the owner of the
-intellectual property rights over this Contribution as defined by
-applicable law.
-
-
- 6.3 OVER THE RELATED MODULES
-
-The Licensee who develops a Related Module is the owner of the
-intellectual property rights over this Related Module as defined by
-applicable law and is free to choose the type of agreement that shall
-govern its distribution under the conditions defined in Article 5.3.3.
-
-
- 6.4 NOTICE OF RIGHTS
-
-The Licensee expressly undertakes:
-
- 1. not to remove, or modify, in any manner, the intellectual property
- notices attached to the Software;
-
- 2. to reproduce said notices, in an identical manner, in the copies
- of the Software modified or not;
-
- 3. to ensure that use of the Software, its intellectual property
- notices and the fact that it is governed by the Agreement is
- indicated in a text that is easily accessible, specifically from
- the interface of any Derivative Software.
-
-The Licensee undertakes not to directly or indirectly infringe the
-intellectual property rights of the Holder and/or Contributors on the
-Software and to take, where applicable, vis-�-vis its staff, any and all
-measures required to ensure respect of said intellectual property rights
-of the Holder and/or Contributors.
-
-
- Article 7 - RELATED SERVICES
-
-7.1 Under no circumstances shall the Agreement oblige the Licensor to
-provide technical assistance or maintenance services for the Software.
-
-However, the Licensor is entitled to offer this type of services. The
-terms and conditions of such technical assistance, and/or such
-maintenance, shall be set forth in a separate instrument. Only the
-Licensor offering said maintenance and/or technical assistance services
-shall incur liability therefor.
-
-7.2 Similarly, any Licensor is entitled to offer to its licensees, under
-its sole responsibility, a warranty, that shall only be binding upon
-itself, for the redistribution of the Software and/or the Modified
-Software, under terms and conditions that it is free to decide. Said
-warranty, and the financial terms and conditions of its application,
-shall be subject of a separate instrument executed between the Licensor
-and the Licensee.
-
-
- Article 8 - LIABILITY
-
-8.1 Subject to the provisions of Article 8.2, the Licensee shall be
-entitled to claim compensation for any direct loss it may have suffered
-from the Software as a result of a fault on the part of the relevant
-Licensor, subject to providing evidence thereof.
-
-8.2 The Licensor's liability is limited to the commitments made under
-this Agreement and shall not be incurred as a result of in particular:
-(i) loss due the Licensee's total or partial failure to fulfill its
-obligations, (ii) direct or consequential loss that is suffered by the
-Licensee due to the use or performance of the Software, and (iii) more
-generally, any consequential loss. In particular the Parties expressly
-agree that any or all pecuniary or business loss (i.e. loss of data,
-loss of profits, operating loss, loss of customers or orders,
-opportunity cost, any disturbance to business activities) or any or all
-legal proceedings instituted against the Licensee by a third party,
-shall constitute consequential loss and shall not provide entitlement to
-any or all compensation from the Licensor.
-
-
- Article 9 - WARRANTY
-
-9.1 The Licensee acknowledges that the scientific and technical
-state-of-the-art when the Software was distributed did not enable all
-possible uses to be tested and verified, nor for the presence of
-possible defects to be detected. In this respect, the Licensee's
-attention has been drawn to the risks associated with loading, using,
-modifying and/or developing and reproducing the Software which are
-reserved for experienced users.
-
-The Licensee shall be responsible for verifying, by any or all means,
-the suitability of the product for its requirements, its good working
-order, and for ensuring that it shall not cause damage to either persons
-or properties.
-
-9.2 The Licensor hereby represents, in good faith, that it is entitled
-to grant all the rights over the Software (including in particular the
-rights set forth in Article 5).
-
-9.3 The Licensee acknowledges that the Software is supplied "as is" by
-the Licensor without any other express or tacit warranty, other than
-that provided for in Article 9.2 and, in particular, without any warranty
-as to its commercial value, its secured, safe, innovative or relevant
-nature.
-
-Specifically, the Licensor does not warrant that the Software is free
-from any error, that it will operate without interruption, that it will
-be compatible with the Licensee's own equipment and software
-configuration, nor that it will meet the Licensee's requirements.
-
-9.4 The Licensor does not either expressly or tacitly warrant that the
-Software does not infringe any third party intellectual property right
-relating to a patent, software or any other property right. Therefore,
-the Licensor disclaims any and all liability towards the Licensee
-arising out of any or all proceedings for infringement that may be
-instituted in respect of the use, modification and redistribution of the
-Software. Nevertheless, should such proceedings be instituted against
-the Licensee, the Licensor shall provide it with technical and legal
-assistance for its defense. Such technical and legal assistance shall be
-decided on a case-by-case basis between the relevant Licensor and the
-Licensee pursuant to a memorandum of understanding. The Licensor
-disclaims any and all liability as regards the Licensee's use of the
-name of the Software. No warranty is given as regards the existence of
-prior rights over the name of the Software or as regards the existence
-of a trademark.
-
-
- Article 10 - TERMINATION
-
-10.1 In the event of a breach by the Licensee of its obligations
-hereunder, the Licensor may automatically terminate this Agreement
-thirty (30) days after notice has been sent to the Licensee and has
-remained ineffective.
-
-10.2 A Licensee whose Agreement is terminated shall no longer be
-authorized to use, modify or distribute the Software. However, any
-licenses that it may have granted prior to termination of the Agreement
-shall remain valid subject to their having been granted in compliance
-with the terms and conditions hereof.
-
-
- Article 11 - MISCELLANEOUS
-
-
- 11.1 EXCUSABLE EVENTS
-
-Neither Party shall be liable for any or all delay, or failure to
-perform the Agreement, that may be attributable to an event of force
-majeure, an act of God or an outside cause, such as defective
-functioning or interruptions of the electricity or telecommunications
-networks, network paralysis following a virus attack, intervention by
-government authorities, natural disasters, water damage, earthquakes,
-fire, explosions, strikes and labor unrest, war, etc.
-
-11.2 Any failure by either Party, on one or more occasions, to invoke
-one or more of the provisions hereof, shall under no circumstances be
-interpreted as being a waiver by the interested Party of its right to
-invoke said provision(s) subsequently.
-
-11.3 The Agreement cancels and replaces any or all previous agreements,
-whether written or oral, between the Parties and having the same
-purpose, and constitutes the entirety of the agreement between said
-Parties concerning said purpose. No supplement or modification to the
-terms and conditions hereof shall be effective as between the Parties
-unless it is made in writing and signed by their duly authorized
-representatives.
-
-11.4 In the event that one or more of the provisions hereof were to
-conflict with a current or future applicable act or legislative text,
-said act or legislative text shall prevail, and the Parties shall make
-the necessary amendments so as to comply with said act or legislative
-text. All other provisions shall remain effective. Similarly, invalidity
-of a provision of the Agreement, for any reason whatsoever, shall not
-cause the Agreement as a whole to be invalid.
-
-
- 11.5 LANGUAGE
-
-The Agreement is drafted in both French and English and both versions
-are deemed authentic.
-
-
- Article 12 - NEW VERSIONS OF THE AGREEMENT
-
-12.1 Any person is authorized to duplicate and distribute copies of this
-Agreement.
-
-12.2 So as to ensure coherence, the wording of this Agreement is
-protected and may only be modified by the authors of the License, who
-reserve the right to periodically publish updates or new versions of the
-Agreement, each with a separate number. These subsequent versions may
-address new issues encountered by Free Software.
-
-12.3 Any Software distributed under a given version of the Agreement may
-only be subsequently distributed under the same version of the Agreement
-or a subsequent version.
-
-
- Article 13 - GOVERNING LAW AND JURISDICTION
-
-13.1 The Agreement is governed by French law. The Parties agree to
-endeavor to seek an amicable solution to any disagreements or disputes
-that may arise during the performance of the Agreement.
-
-13.2 Failing an amicable solution within two (2) months as from their
-occurrence, and unless emergency proceedings are necessary, the
-disagreements or disputes shall be referred to the Paris Courts having
-jurisdiction, by the more diligent Party.
-
-
-Version 1.0 dated 2006-09-05.
diff --git a/OCR/line_OCR/ctc/main_line_ctc.py b/OCR/line_OCR/ctc/main_line_ctc.py
index ee517031fa5bcd87d4621c842ed455b0be4b8652..bb8fa1d6c277b8978bb251eb0259e17603e9e28e 100644
--- a/OCR/line_OCR/ctc/main_line_ctc.py
+++ b/OCR/line_OCR/ctc/main_line_ctc.py
@@ -1,36 +1,3 @@
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in XXX whose purpose is XXX.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
import os
import sys
diff --git a/OCR/line_OCR/ctc/main_syn_line.py b/OCR/line_OCR/ctc/main_syn_line.py
index de2974646d638c874cc1a5e56ac827e20ac70b14..c16f9de5ab1fb38ce18486146f997ba60baedc61 100644
--- a/OCR/line_OCR/ctc/main_syn_line.py
+++ b/OCR/line_OCR/ctc/main_syn_line.py
@@ -1,36 +1,3 @@
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in XXX whose purpose is XXX.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
import os
import sys
diff --git a/OCR/line_OCR/ctc/models_line_ctc.py b/OCR/line_OCR/ctc/models_line_ctc.py
index cda34ebc0ef22e33cff03ca61686a23784ae33d4..c13c072ebd10e810d41b8aab1e318c2170637ea0 100644
--- a/OCR/line_OCR/ctc/models_line_ctc.py
+++ b/OCR/line_OCR/ctc/models_line_ctc.py
@@ -1,36 +1,3 @@
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in XXX whose purpose is XXX.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
from torch.nn.functional import log_softmax
from torch.nn import AdaptiveMaxPool2d, Conv1d
diff --git a/OCR/line_OCR/ctc/trainer_line_ctc.py b/OCR/line_OCR/ctc/trainer_line_ctc.py
index cd229206e90ff6096d44a4cac736f80501aa6cad..eba12d81ecb4a417115121822d640be374cf185a 100644
--- a/OCR/line_OCR/ctc/trainer_line_ctc.py
+++ b/OCR/line_OCR/ctc/trainer_line_ctc.py
@@ -1,36 +1,3 @@
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in XXX whose purpose is XXX.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
from basic.metric_manager import MetricManager
from OCR.ocr_manager import OCRManager
diff --git a/OCR/ocr_manager.py b/OCR/ocr_manager.py
index dd920890baf773c735dc81abfce6036b0520f56e..e34b36c0a0793868c5a8f590b20510db72bc0681 100644
--- a/OCR/ocr_manager.py
+++ b/OCR/ocr_manager.py
@@ -1,38 +1,3 @@
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in Python whose purpose is to
-# provide public implementation of deep learning works, in pytorch.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
-
from basic.generic_training_manager import GenericTrainingManager
import os
from PIL import Image
diff --git a/README.md b/README.md
index 45bb7eeabe6acc37a8522ef8fcd00a8dde40f793..28d0dc9201d28151f4b494df0fc6685b991d0479 100644
--- a/README.md
+++ b/README.md
@@ -49,68 +49,6 @@ Install the dependencies:
pip install -r requirements.txt
```
-
-## Datasets
-This section is dedicated to the datasets used in the paper: download and formatting instructions are provided
-for experiment replication purposes.
-
-RIMES dataset at page level was distributed during the [evaluation compaign of 2009](https://ieeexplore.ieee.org/document/5277557).
-
-READ 2016 dataset corresponds to the one used in the [ICFHR 2016 competition on handwritten text recognition](https://ieeexplore.ieee.org/document/7814136).
-It can be found [here](https://zenodo.org/record/1164045#.YiINkBvjKEA)
-
-Raw dataset files must be placed in Datasets/raw/{dataset_name} \
-where dataset name is "READ 2016" or "RIMES"
-
-## Training And Evaluation
-### Step 1: Download the dataset
-
-### Step 2: Format the dataset
-```
-python3 Datasets/dataset_formatters/read2016_formatter.py
-python3 Datasets/dataset_formatters/rimes_formatter.py
-```
-
-### Step 3: Add any font you want as .ttf file in the folder Fonts
-
-### Step 4 : Generate synthetic line dataset for pre-training
-```
-python3 OCR/line_OCR/ctc/main_syn_line.py
-```
-There are two lines in this script to adapt to the used dataset:
-```
-model.generate_syn_line_dataset("READ_2016_syn_line")
-dataset_name = "READ_2016"
-```
-
-### Step 5 : Pre-training on synthetic lines
-```
-python3 OCR/line_OCR/ctc/main_line_ctc.py
-```
-There are two lines in this script to adapt to the used dataset:
-```
-dataset_name = "READ_2016"
-"output_folder": "FCN_read_line_syn"
-```
-Weights and evaluation results are stored in OCR/line_OCR/ctc/outputs
-
-### Step 6 : Training the DAN
-```
-python3 OCR/document_OCR/dan/main_dan.py
-```
-The following lines must be adapted to the dataset used and pre-training folder names:
-```
-dataset_name = "READ_2016"
-"transfer_learning": {
- # model_name: [state_dict_name, checkpoint_path, learnable, strict]
- "encoder": ["encoder", "../../line_OCR/ctc/outputs/FCN_read_2016_line_syn/checkpoints/best.pt", True, True],
- "decoder": ["decoder", "../../line_OCR/ctc/outputs/FCN_read_2016_line_syn/best.pt", True, False],
-},
-```
-
-Weights and evaluation results are stored in OCR/document_OCR/dan/outputs
-
-
### Remarks (for pre-training and training)
All hyperparameters are specified and editable in the training scripts (meaning are in comments).\
Evaluation is performed just after training ending (training is stopped when the maximum elapsed time is reached or after a maximum number of epoch as specified in the training script).\
@@ -154,20 +92,3 @@ To run the inference on a GPU, one can replace `cpu` by the name of the GPU. In
```python
text, confidence_scores = model.predict(image, confidences=True)
```
-
-## Citation
-
-```bibtex
-@misc{Coquenet2022b,
- author = {Coquenet, Denis and Chatelain, Clément and Paquet, Thierry},
- title = {DAN: a Segmentation-free Document Attention Network for Handwritten Document Recognition},
- doi = {10.48550/ARXIV.2203.12273},
- url = {https://arxiv.org/abs/2203.12273},
- publisher = {arXiv},
- year = {2022},
-}
-```
-
-## License
-
-This whole project is under Cecill-C license.
diff --git a/basic/generic_dataset_manager.py b/basic/generic_dataset_manager.py
index 7e9c7c7f32302ce9ae2978474672d625d55b7862..dd272d152884b4fc2d27f6467ad915f099f77acc 100644
--- a/basic/generic_dataset_manager.py
+++ b/basic/generic_dataset_manager.py
@@ -1,37 +1,3 @@
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in XXX whose purpose is XXX.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
-
import torch
import random
from torch.utils.data import Dataset, DataLoader
diff --git a/basic/generic_training_manager.py b/basic/generic_training_manager.py
index 2e2a7c95a4c6e546227803d67eeba2af70ca81a6..a8f41b28274678b23083123e33d2343bb6b942b9 100644
--- a/basic/generic_training_manager.py
+++ b/basic/generic_training_manager.py
@@ -1,37 +1,3 @@
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in XXX whose purpose is XXX.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
-
import torch
import os
import sys
diff --git a/basic/metric_manager.py b/basic/metric_manager.py
index f537c9118710f75f2ee4b632131a3a78b8a88998..6c8576863a2d26a85c2d5dbc4321323dedf870a0 100644
--- a/basic/metric_manager.py
+++ b/basic/metric_manager.py
@@ -1,38 +1,3 @@
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in Python whose purpose is to
-# provide public implementation of deep learning works, in pytorch.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
-
from Datasets.dataset_formatters.rimes_formatter import SEM_MATCHING_TOKENS as RIMES_MATCHING_TOKENS
from Datasets.dataset_formatters.read2016_formatter import SEM_MATCHING_TOKENS as READ_MATCHING_TOKENS
diff --git a/basic/scheduler.py b/basic/scheduler.py
index 2781e49aca7a75fb386aaaad2bae08388cef3baa..6c875c1da2f57ab0306635cbc77309c2b83af116 100644
--- a/basic/scheduler.py
+++ b/basic/scheduler.py
@@ -1,37 +1,3 @@
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in Python whose purpose is to
-# provide public implementation of deep learning works, in pytorch.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
from torch.nn import Dropout, Dropout2d
import numpy as np
diff --git a/basic/transforms.py b/basic/transforms.py
index 3c73ada329786a7bb1591882430ad74ff42de96d..18c8084dafe668025ea7ae58ce99384b58a672a1 100644
--- a/basic/transforms.py
+++ b/basic/transforms.py
@@ -1,36 +1,3 @@
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in XXX whose purpose is XXX.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
import numpy as np
from numpy import random
diff --git a/basic/utils.py b/basic/utils.py
index 8578279a9a2a1143a37d7583e32d44737ed02dd0..ac50e57ab1136a2f2e1d209c39271406316d0479 100644
--- a/basic/utils.py
+++ b/basic/utils.py
@@ -1,36 +1,3 @@
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in XXX whose purpose is XXX.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
import numpy as np
diff --git a/dan/cli.py b/dan/cli.py
new file mode 100644
index 0000000000000000000000000000000000000000..b0ed21995d9c4e0f70460062dd0e080c1f42de35
--- /dev/null
+++ b/dan/cli.py
@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+import argparse
+import errno
+from dan.datasets.extract.extract_from_arkindex import add_extract_parser
+from dan.ocr.line.generate_synthetic import add_generate_parser
+
+from dan.ocr.train import add_train_parser
+
+
+
+def get_parser():
+ parser = argparse.ArgumentParser(prog="TEKLIA DAN training")
+ subcommands = parser.add_subparsers(metavar="subcommand")
+
+ add_train_parser(subcommands)
+ add_extract_parser(subcommands)
+ add_generate_parser(subcommands)
+ return parser
+
+def main():
+ parser = get_parser()
+ args = vars(parser.parse_args())
+ if "func" in args:
+ # Run the subcommand's function
+ try:
+ status = args.pop("func")(**args)
+ parser.exit(status=status)
+ except KeyboardInterrupt:
+ # Just quit silently on ^C instead of displaying a long traceback
+ parser.exit(status=errno.EOWNERDEAD)
+ else:
+ parser.error("A subcommand is required.")
diff --git a/dan/datasets/__init__.py b/dan/datasets/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/dan/datasets/extract/__init__.py b/dan/datasets/extract/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/dan/datasets/extract/arkindex_utils.py b/dan/datasets/extract/arkindex_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..5216a7e0dd286cdfb53c73802324cd40cd07f7bd
--- /dev/null
+++ b/dan/datasets/extract/arkindex_utils.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+ The arkindex_utils module
+ ======================
+"""
+
+import errno
+import logging
+import sys
+
+from apistar.exceptions import ErrorResponse
+
+
+def retrieve_corpus(client, corpus_name: str) -> str:
+ """
+ Retrieve the corpus id from the corpus name.
+ :param client: The arkindex client.
+ :param corpus_name: The name of the corpus to retrieve.
+ :return target_corpus: The id of the retrieved corpus.
+ """
+ for corpus in client.request("ListCorpus"):
+ if corpus["name"] == corpus_name:
+ target_corpus = corpus["id"]
+ try:
+ logging.info(f"Corpus id retrieved: {target_corpus}")
+ except NameError:
+ logging.error(f"Corpus {corpus_name} not found")
+ sys.exit(errno.EINVAL)
+
+ return target_corpus
+
+
+def retrieve_subsets(
+ client, corpus: str, parents_types: list, parents_names: list
+) -> list:
+ """
+ Retrieve the requested subsets.
+ :param client: The arkindex client.
+ :param corpus: The id of the retrieved corpus.
+ :param parents_types: The types of parents of the elements to retrieve.
+ :param parents_names: The names of parents of the elements to retrieve.
+ :return subsets: The retrieved subsets.
+ """
+ subsets = []
+ for parent_type in parents_types:
+ try:
+ subsets.extend(
+ client.request("ListElements", corpus=corpus, type=parent_type)[
+ "results"
+ ]
+ )
+ except ErrorResponse as e:
+ logging.error(f"{e.content}: {parent_type}")
+ sys.exit(errno.EINVAL)
+ # Retrieve subsets with name in parents-names. If no parents-names given, keep all subsets.
+ if parents_names is not None:
+ logging.info(f"Retrieving {parents_names} subset(s)")
+ subsets = [subset for subset in subsets if subset["name"] in parents_names]
+ else:
+ logging.info("Retrieving all subsets")
+
+ if len(subsets) == 0:
+ logging.info("No subset found")
+
+ return subsets
diff --git a/dan/datasets/extract/extract_from_arkindex.py b/dan/datasets/extract/extract_from_arkindex.py
new file mode 100644
index 0000000000000000000000000000000000000000..1a6e207dac7fee11f45926377cf72b4401b3294f
--- /dev/null
+++ b/dan/datasets/extract/extract_from_arkindex.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Example of minimal usage:
+# python extract_from_arkindex.py
+# --corpus "AN-Simara annotations E (2022-06-20)"
+# --parents-types folder
+# --parents-names FRAN_IR_032031_4538.pdf
+# --output-dir ../
+
+"""
+ The extraction module
+ ======================
+"""
+
+import logging
+import os
+
+import cv2
+import imageio.v2 as iio
+from arkindex import ArkindexClient, options_from_env
+from tqdm import tqdm
+
+from dan.datasets.extract.arkindex_utils import retrieve_corpus, retrieve_subsets
+from dan.datasets.extract.utils import get_cli_args
+
+logging.basicConfig(
+ level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
+
+
+IMAGES_DIR = "./images/" # Path to the images directory.
+LABELS_DIR = "./labels/" # Path to the labels directory.
+
+# Layout string to token
+SEM_MATCHING_TOKENS_STR = {
+ "INTITULE": "ⓘ",
+ "DATE": "â““",
+ "COTE_SERIE": "â“¢",
+ "ANALYSE_COMPL.": "â“’",
+ "PRECISIONS_SUR_COTE": "â“Ÿ",
+ "COTE_ARTICLE": "â“",
+}
+
+# Layout begin-token to end-token
+SEM_MATCHING_TOKENS = {"ⓘ": "â’¾", "â““": "â’¹", "â“¢": "Ⓢ", "â“’": "â’¸", "â“Ÿ": "â“…", "â“": "â’¶"}
+
+
+def add_extract_parser(subcommands) -> None:
+ parser = subcommands.add_parser(
+ "extract",
+ description=__doc__,
+ )
+ parser.set_defaults(func=run)
+
+
+def run():
+ args = get_cli_args()
+
+ # Get and initialize the parameters.
+ os.makedirs(IMAGES_DIR, exist_ok=True)
+ os.makedirs(LABELS_DIR, exist_ok=True)
+
+ # Login to arkindex.
+ client = ArkindexClient(**options_from_env())
+
+ corpus = retrieve_corpus(client, args.corpus)
+ subsets = retrieve_subsets(client, corpus, args.parents_types, args.parents_names)
+
+ # Iterate over the subsets to find the page images and labels.
+ for subset in subsets:
+
+ os.makedirs(
+ os.path.join(args.output_dir, IMAGES_DIR, subset["name"]), exist_ok=True
+ )
+ os.makedirs(
+ os.path.join(args.output_dir, LABELS_DIR, subset["name"]), exist_ok=True
+ )
+
+ for page in tqdm(
+ client.paginate(
+ "ListElementChildren", id=subset["id"], type="page", recursive=True
+ ),
+ desc="Set " + subset["name"],
+ ):
+
+ image = iio.imread(page["zone"]["url"])
+ cv2.imwrite(
+ os.path.join(
+ args.output_dir, IMAGES_DIR, subset["name"], f"{page['id']}.jpg"
+ ),
+ cv2.cvtColor(image, cv2.COLOR_BGR2RGB),
+ )
+
+ tr = client.request(
+ "ListTranscriptions", id=page["id"], worker_version=None
+ )["results"]
+ tr = [one for one in tr if one["worker_version_id"] is None]
+ assert len(tr) == 1, page["id"]
+
+ for one_tr in tr:
+ ent = client.request("ListTranscriptionEntities", id=one_tr["id"])[
+ "results"
+ ]
+ ent = [one for one in ent if one["worker_version_id"] is None]
+ if len(ent) == 0:
+ continue
+ else:
+ text = one_tr["text"]
+
+ new_text = text
+ count = 0
+ for e in ent:
+ start_token = SEM_MATCHING_TOKENS_STR[e["entity"]["metas"]["subtype"]]
+ end_token = SEM_MATCHING_TOKENS[start_token]
+ new_text = (
+ new_text[: count + e["offset"]]
+ + start_token
+ + new_text[count + e["offset"] :]
+ )
+ count += 1
+ new_text = (
+ new_text[: count + e["offset"] + e["length"]]
+ + end_token
+ + new_text[count + e["offset"] + e["length"] :]
+ )
+ count += 1
+
+ with open(
+ os.path.join(
+ args.output_dir, LABELS_DIR, subset["name"], f"{page['id']}.txt"
+ ),
+ "w",
+ ) as f:
+ f.write(new_text)
diff --git a/dan/datasets/extract/utils.py b/dan/datasets/extract/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..72ab461b8214a14b144cc6c2edc725cad38cf6bc
--- /dev/null
+++ b/dan/datasets/extract/utils.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+ The utils module
+ ======================
+"""
+
+import argparse
+
+
+def get_cli_args():
+ """
+ Get the command-line arguments.
+ :return: The command-line arguments.
+ """
+ parser = argparse.ArgumentParser(
+ description="Arkindex DAN Training Label Generation"
+ )
+
+ # Required arguments.
+ parser.add_argument(
+ "--corpus",
+ type=str,
+ help="Name of the corpus from which the data will be retrieved.",
+ required=True,
+ )
+ parser.add_argument(
+ "--parents-types",
+ nargs="+",
+ type=str,
+ help="Type of parents of the elements.",
+ required=True,
+ )
+ parser.add_argument(
+ "--output-dir",
+ type=str,
+ help="Path to the output directory.",
+ required=True,
+ )
+
+ # Optional arguments.
+ parser.add_argument(
+ "--parents-names",
+ nargs="+",
+ type=str,
+ help="Names of parents of the elements.",
+ default=None,
+ )
+ return parser.parse_args()
diff --git a/dan/datasets/format/__init__.py b/dan/datasets/format/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/dan/datasets/format/generic.py b/dan/datasets/format/generic.py
new file mode 100644
index 0000000000000000000000000000000000000000..8bdc795157d37890039230108fc3611c865867d2
--- /dev/null
+++ b/dan/datasets/format/generic.py
@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+import os
+import pickle
+import shutil
+
+import numpy as np
+from PIL import Image
+
+
+class DatasetFormatter:
+ """
+ Global pipeline/functions for dataset formatting
+ """
+
+ def __init__(
+ self, dataset_name, level, extra_name="", set_names=["train", "valid", "test"]
+ ):
+ self.dataset_name = dataset_name
+ self.level = level
+ self.set_names = set_names
+ self.target_fold_path = os.path.join(
+ "Datasets", "formatted", "{}_{}{}".format(dataset_name, level, extra_name)
+ )
+ self.map_datasets_files = dict()
+
+ def format(self):
+ self.init_format()
+ self.map_datasets_files[self.dataset_name][self.level]["format_function"]()
+ self.end_format()
+
+ def init_format(self):
+ """
+ Load and extracts needed files
+ """
+ os.makedirs(self.target_fold_path, exist_ok=True)
+
+ for set_name in self.set_names:
+ os.makedirs(os.path.join(self.target_fold_path, set_name), exist_ok=True)
+
+
+class OCRDatasetFormatter(DatasetFormatter):
+ """
+ Specific pipeline/functions for OCR/HTR dataset formatting
+ """
+
+ def __init__(
+ self, source_dataset, level, extra_name="", set_names=["train", "valid", "test"]
+ ):
+ super(OCRDatasetFormatter, self).__init__(
+ source_dataset, level, extra_name, set_names
+ )
+ self.charset = set()
+ self.gt = dict()
+ for set_name in set_names:
+ self.gt[set_name] = dict()
+
+ def load_resize_save(self, source_path, target_path):
+ """
+ Load image, apply resolution modification and save it
+ """
+ shutil.copyfile(source_path, target_path)
+
+ def resize(self, img, source_dpi, target_dpi):
+ """
+ Apply resolution modification to image
+ """
+ if source_dpi == target_dpi:
+ return img
+ if isinstance(img, np.ndarray):
+ h, w = img.shape[:2]
+ img = Image.fromarray(img)
+ else:
+ w, h = img.size
+ ratio = target_dpi / source_dpi
+ img = img.resize((int(w * ratio), int(h * ratio)), Image.BILINEAR)
+ return np.array(img)
+
+ def end_format(self):
+ """
+ Save label and charset files
+ """
+ with open(os.path.join(self.target_fold_path, "labels.pkl"), "wb") as f:
+ pickle.dump(
+ {
+ "ground_truth": self.gt,
+ "charset": sorted(list(self.charset)),
+ },
+ f,
+ )
+ with open(os.path.join(self.target_fold_path, "charset.pkl"), "wb") as f:
+ pickle.dump(sorted(list(self.charset)), f)
diff --git a/dan/datasets/format/simara.py b/dan/datasets/format/simara.py
new file mode 100644
index 0000000000000000000000000000000000000000..2d81c31abfd70e6632628a0bfc651bbb70b88f88
--- /dev/null
+++ b/dan/datasets/format/simara.py
@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+import os
+from collections import defaultdict
+
+from tqdm import tqdm
+
+from dan.datasets.format.generic import OCRDatasetFormatter
+
+# Layout string to token
+SEM_MATCHING_TOKENS_STR = {
+ "INTITULE": "ⓘ",
+ "DATE": "â““",
+ "COTE_SERIE": "â“¢",
+ "ANALYSE_COMPL": "â“’",
+ "PRECISIONS_SUR_COTE": "â“Ÿ",
+ "COTE_ARTICLE": "â“",
+}
+
+# Layout begin-token to end-token
+SEM_MATCHING_TOKENS = {"ⓘ": "â’¾", "â““": "â’¹", "â“¢": "Ⓢ", "â“’": "â’¸", "â“Ÿ": "â“…", "â“": "â’¶"}
+
+
+class SimaraDatasetFormatter(OCRDatasetFormatter):
+ def __init__(
+ self, level, set_names=["train", "valid", "test"], dpi=150, sem_token=True
+ ):
+ super(SimaraDatasetFormatter, self).__init__(
+ "simara", level, "_sem" if sem_token else "", set_names
+ )
+
+ self.dpi = dpi
+ self.sem_token = sem_token
+ self.map_datasets_files.update(
+ {
+ "simara": {
+ # (1,050 for train, 100 for validation and 100 for test)
+ "page": {
+ "format_function": self.format_simara_page,
+ },
+ }
+ }
+ )
+ self.matching_tokens_str = SEM_MATCHING_TOKENS_STR
+ self.matching_tokens = SEM_MATCHING_TOKENS
+
+ def preformat_simara_page(self):
+ """
+ Extract all information from dataset and correct some annotations
+ """
+ dataset = defaultdict(list)
+ img_folder_path = os.path.join("Datasets", "raw", "simara", "images")
+ labels_folder_path = os.path.join("Datasets", "raw", "simara", "labels")
+ sem_labels_folder_path = os.path.join("Datasets", "raw", "simara", "labels_sem")
+ train_files = [
+ os.path.join(labels_folder_path, "train", name)
+ for name in os.listdir(os.path.join(sem_labels_folder_path, "train"))
+ ]
+ valid_files = [
+ os.path.join(labels_folder_path, "valid", name)
+ for name in os.listdir(os.path.join(sem_labels_folder_path, "valid"))
+ ]
+ test_files = [
+ os.path.join(labels_folder_path, "test", name)
+ for name in os.listdir(os.path.join(sem_labels_folder_path, "test"))
+ ]
+ for set_name, files in zip(
+ self.set_names, [train_files, valid_files, test_files]
+ ):
+ for i, label_file in enumerate(
+ tqdm(files, desc="Pre-formatting " + set_name)
+ ):
+ with open(label_file, "r") as f:
+ text = f.read()
+ with open(label_file.replace("labels", "labels_sem"), "r") as f:
+ sem_text = f.read()
+ dataset[set_name].append(
+ {
+ "img_path": os.path.join(
+ img_folder_path,
+ set_name,
+ label_file.split("/")[-1].replace("txt", "jpg"),
+ ),
+ "label": text,
+ "sem_label": sem_text,
+ }
+ )
+ return dataset
+
+ def format_simara_page(self):
+ """
+ Format simara page dataset
+ """
+ dataset = self.preformat_simara_page()
+ for set_name in self.set_names:
+ fold = os.path.join(self.target_fold_path, set_name)
+ for sample in tqdm(dataset[set_name], desc="Formatting " + set_name):
+ new_name = sample["img_path"].split("/")[-1]
+ new_img_path = os.path.join(fold, new_name)
+ self.load_resize_save(
+ sample["img_path"], new_img_path
+ ) # , 300, self.dpi)
+ page = {
+ "text": sample["label"]
+ if not self.sem_token
+ else sample["sem_label"],
+ }
+ self.charset = self.charset.union(set(page["text"]))
+ self.gt[set_name][new_name] = page
diff --git a/dan/datasets/utils.py b/dan/datasets/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..c911cc9b06c4d6c258acc668312b54a0ea2b79b3
--- /dev/null
+++ b/dan/datasets/utils.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+import re
+
+
+def convert(text):
+ return int(text) if text.isdigit() else text.lower()
+
+
+def natural_sort(data):
+ return sorted(data, key=lambda key: [convert(c) for c in re.split("([0-9]+)", key)])
diff --git a/dan/decoder.py b/dan/decoder.py
index aa811235d5f9c6d0615c088910ba54ede7e2908c..d2070fc6110854a3c2cb567ad1e61f1efb11c7ad 100644
--- a/dan/decoder.py
+++ b/dan/decoder.py
@@ -79,9 +79,6 @@ class PositionalEncoding2D(Module):
"""
return x + self.pe[:, :, : x.size(2), : x.size(3)]
- def get_pe_by_size(self, h, w, device):
- return self.pe[:, :, :h, :w].to(device)
-
class CustomMultiHeadAttention(Module):
"""
diff --git a/dan/manager/__init__.py b/dan/manager/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/dan/manager/dataset.py b/dan/manager/dataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..64633ebe53445b4b1768e773ba76c9195e079f77
--- /dev/null
+++ b/dan/manager/dataset.py
@@ -0,0 +1,441 @@
+# -*- coding: utf-8 -*-
+import os
+import pickle
+import random
+
+import cv2
+import numpy as np
+import torch
+from PIL import Image
+from torch.utils.data import DataLoader, Dataset
+from torch.utils.data.distributed import DistributedSampler
+
+from dan.datasets.utils import natural_sort
+from dan.transforms import apply_data_augmentation
+
+
+class DatasetManager:
+ def __init__(self, params):
+ self.params = params
+ self.dataset_class = params["dataset_class"]
+ self.img_padding_value = params["config"]["padding_value"]
+
+ self.my_collate_function = None
+
+ self.train_dataset = None
+ self.valid_datasets = dict()
+ self.test_datasets = dict()
+
+ self.train_loader = None
+ self.valid_loaders = dict()
+ self.test_loaders = dict()
+
+ self.train_sampler = None
+ self.valid_samplers = dict()
+ self.test_samplers = dict()
+
+ self.generator = torch.Generator()
+ self.generator.manual_seed(0)
+
+ self.batch_size = {
+ "train": self.params["batch_size"],
+ "valid": self.params["valid_batch_size"]
+ if "valid_batch_size" in self.params
+ else self.params["batch_size"],
+ "test": self.params["test_batch_size"]
+ if "test_batch_size" in self.params
+ else 1,
+ }
+
+ def apply_specific_treatment_after_dataset_loading(self, dataset):
+ raise NotImplementedError
+
+ def load_datasets(self):
+ """
+ Load training and validation datasets
+ """
+ self.train_dataset = self.dataset_class(
+ self.params,
+ "train",
+ self.params["train"]["name"],
+ self.get_paths_and_sets(self.params["train"]["datasets"]),
+ )
+ (
+ self.params["config"]["mean"],
+ self.params["config"]["std"],
+ ) = self.train_dataset.compute_std_mean()
+
+ self.my_collate_function = self.train_dataset.collate_function(
+ self.params["config"]
+ )
+ self.apply_specific_treatment_after_dataset_loading(self.train_dataset)
+
+ for custom_name in self.params["valid"].keys():
+ self.valid_datasets[custom_name] = self.dataset_class(
+ self.params,
+ "valid",
+ custom_name,
+ self.get_paths_and_sets(self.params["valid"][custom_name]),
+ )
+ self.apply_specific_treatment_after_dataset_loading(
+ self.valid_datasets[custom_name]
+ )
+
+ def load_ddp_samplers(self):
+ """
+ Load training and validation data samplers
+ """
+ if self.params["use_ddp"]:
+ self.train_sampler = DistributedSampler(
+ self.train_dataset,
+ num_replicas=self.params["num_gpu"],
+ rank=self.params["ddp_rank"],
+ shuffle=True,
+ )
+ for custom_name in self.valid_datasets.keys():
+ self.valid_samplers[custom_name] = DistributedSampler(
+ self.valid_datasets[custom_name],
+ num_replicas=self.params["num_gpu"],
+ rank=self.params["ddp_rank"],
+ shuffle=False,
+ )
+ else:
+ for custom_name in self.valid_datasets.keys():
+ self.valid_samplers[custom_name] = None
+
+ def load_dataloaders(self):
+ """
+ Load training and validation data loaders
+ """
+ self.train_loader = DataLoader(
+ self.train_dataset,
+ batch_size=self.batch_size["train"],
+ shuffle=True if self.train_sampler is None else False,
+ drop_last=False,
+ batch_sampler=self.train_sampler,
+ sampler=self.train_sampler,
+ num_workers=self.params["num_gpu"] * self.params["worker_per_gpu"],
+ pin_memory=True,
+ collate_fn=self.my_collate_function,
+ worker_init_fn=self.seed_worker,
+ generator=self.generator,
+ )
+
+ for key in self.valid_datasets.keys():
+ self.valid_loaders[key] = DataLoader(
+ self.valid_datasets[key],
+ batch_size=self.batch_size["valid"],
+ sampler=self.valid_samplers[key],
+ batch_sampler=self.valid_samplers[key],
+ shuffle=False,
+ num_workers=self.params["num_gpu"] * self.params["worker_per_gpu"],
+ pin_memory=True,
+ drop_last=False,
+ collate_fn=self.my_collate_function,
+ worker_init_fn=self.seed_worker,
+ generator=self.generator,
+ )
+
+ @staticmethod
+ def seed_worker(worker_id):
+ worker_seed = torch.initial_seed() % 2**32
+ np.random.seed(worker_seed)
+ random.seed(worker_seed)
+
+ def generate_test_loader(self, custom_name, sets_list):
+ """
+ Load test dataset, data sampler and data loader
+ """
+ if custom_name in self.test_loaders.keys():
+ return
+ paths_and_sets = list()
+ for set_info in sets_list:
+ paths_and_sets.append(
+ {"path": self.params["datasets"][set_info[0]], "set_name": set_info[1]}
+ )
+ self.test_datasets[custom_name] = self.dataset_class(
+ self.params, "test", custom_name, paths_and_sets
+ )
+ self.apply_specific_treatment_after_dataset_loading(
+ self.test_datasets[custom_name]
+ )
+ if self.params["use_ddp"]:
+ self.test_samplers[custom_name] = DistributedSampler(
+ self.test_datasets[custom_name],
+ num_replicas=self.params["num_gpu"],
+ rank=self.params["ddp_rank"],
+ shuffle=False,
+ )
+ else:
+ self.test_samplers[custom_name] = None
+ self.test_loaders[custom_name] = DataLoader(
+ self.test_datasets[custom_name],
+ batch_size=self.batch_size["test"],
+ sampler=self.test_samplers[custom_name],
+ shuffle=False,
+ num_workers=self.params["num_gpu"] * self.params["worker_per_gpu"],
+ pin_memory=True,
+ drop_last=False,
+ collate_fn=self.my_collate_function,
+ worker_init_fn=self.seed_worker,
+ generator=self.generator,
+ )
+
+ def get_paths_and_sets(self, dataset_names_folds):
+ paths_and_sets = list()
+ for dataset_name, fold in dataset_names_folds:
+ path = self.params["datasets"][dataset_name]
+ paths_and_sets.append({"path": path, "set_name": fold})
+ return paths_and_sets
+
+
+class GenericDataset(Dataset):
+ """
+ Main class to handle dataset loading
+ """
+
+ def __init__(self, params, set_name, custom_name, paths_and_sets):
+ self.params = params
+ self.name = custom_name
+ self.set_name = set_name
+ self.mean = (
+ np.array(params["config"]["mean"])
+ if "mean" in params["config"].keys()
+ else None
+ )
+ self.std = (
+ np.array(params["config"]["std"])
+ if "std" in params["config"].keys()
+ else None
+ )
+
+ self.load_in_memory = (
+ self.params["config"]["load_in_memory"]
+ if "load_in_memory" in self.params["config"]
+ else True
+ )
+
+ self.samples = self.load_samples(
+ paths_and_sets, load_in_memory=self.load_in_memory
+ )
+
+ if self.load_in_memory:
+ self.apply_preprocessing(params["config"]["preprocessings"])
+
+ self.padding_value = params["config"]["padding_value"]
+ if self.padding_value == "mean":
+ if self.mean is None:
+ _, _ = self.compute_std_mean()
+ self.padding_value = self.mean
+ self.params["config"]["padding_value"] = self.padding_value
+
+ self.curriculum_config = None
+ self.training_info = None
+
+ def __len__(self):
+ return len(self.samples)
+
+ @staticmethod
+ def load_image(path):
+ with Image.open(path) as pil_img:
+ img = np.array(pil_img)
+ # grayscale images
+ if len(img.shape) == 2:
+ img = np.expand_dims(img, axis=2)
+ return img
+
+ @staticmethod
+ def load_samples(paths_and_sets, load_in_memory=True):
+ """
+ Load images and labels
+ """
+ samples = list()
+ for path_and_set in paths_and_sets:
+ path = path_and_set["path"]
+ set_name = path_and_set["set_name"]
+ with open(os.path.join(path, "labels.pkl"), "rb") as f:
+ info = pickle.load(f)
+ gt = info["ground_truth"][set_name]
+ for filename in natural_sort(gt.keys()):
+ name = os.path.join(os.path.basename(path), set_name, filename)
+ full_path = os.path.join(path, set_name, filename)
+ if isinstance(gt[filename], dict) and "text" in gt[filename]:
+ label = gt[filename]["text"]
+ else:
+ label = gt[filename]
+ samples.append(
+ {
+ "name": name,
+ "label": label,
+ "unchanged_label": label,
+ "path": full_path,
+ "nb_cols": 1
+ if "nb_cols" not in gt[filename]
+ else gt[filename]["nb_cols"],
+ }
+ )
+ if load_in_memory:
+ samples[-1]["img"] = GenericDataset.load_image(full_path)
+ if type(gt[filename]) is dict:
+ if "lines" in gt[filename].keys():
+ samples[-1]["raw_line_seg_label"] = gt[filename]["lines"]
+ if "paragraphs" in gt[filename].keys():
+ samples[-1]["paragraphs_label"] = gt[filename]["paragraphs"]
+ if "pages" in gt[filename].keys():
+ samples[-1]["pages_label"] = gt[filename]["pages"]
+ return samples
+
+ def apply_preprocessing(self, preprocessings):
+ for i in range(len(self.samples)):
+ self.samples[i] = apply_preprocessing(self.samples[i], preprocessings)
+
+ def compute_std_mean(self):
+ """
+ Compute cumulated variance and mean of whole dataset
+ """
+ if self.mean is not None and self.std is not None:
+ return self.mean, self.std
+ if not self.load_in_memory:
+ sample = self.samples[0].copy()
+ sample["img"] = self.get_sample_img(0)
+ img = apply_preprocessing(sample, self.params["config"]["preprocessings"])[
+ "img"
+ ]
+ else:
+ img = self.get_sample_img(0)
+ _, _, c = img.shape
+ sum = np.zeros((c,))
+ nb_pixels = 0
+
+ for i in range(len(self.samples)):
+ if not self.load_in_memory:
+ sample = self.samples[i].copy()
+ sample["img"] = self.get_sample_img(i)
+ img = apply_preprocessing(
+ sample, self.params["config"]["preprocessings"]
+ )["img"]
+ else:
+ img = self.get_sample_img(i)
+ sum += np.sum(img, axis=(0, 1))
+ nb_pixels += np.prod(img.shape[:2])
+ mean = sum / nb_pixels
+ diff = np.zeros((c,))
+ for i in range(len(self.samples)):
+ if not self.load_in_memory:
+ sample = self.samples[i].copy()
+ sample["img"] = self.get_sample_img(i)
+ img = apply_preprocessing(
+ sample, self.params["config"]["preprocessings"]
+ )["img"]
+ else:
+ img = self.get_sample_img(i)
+ diff += [np.sum((img[:, :, k] - mean[k]) ** 2) for k in range(c)]
+ std = np.sqrt(diff / nb_pixels)
+
+ self.mean = mean
+ self.std = std
+ return mean, std
+
+ def apply_data_augmentation(self, img):
+ """
+ Apply data augmentation strategy on the input image
+ """
+ augs = [
+ self.params["config"][key] if key in self.params["config"].keys() else None
+ for key in ["augmentation", "valid_augmentation", "test_augmentation"]
+ ]
+ for aug, set_name in zip(augs, ["train", "valid", "test"]):
+ if aug and self.set_name == set_name:
+ return apply_data_augmentation(img, aug)
+ return img, list()
+
+ def get_sample_img(self, i):
+ """
+ Get image by index
+ """
+ if self.load_in_memory:
+ return self.samples[i]["img"]
+ else:
+ return GenericDataset.load_image(self.samples[i]["path"])
+
+
+def apply_preprocessing(sample, preprocessings):
+ """
+ Apply preprocessings on each sample
+ """
+ resize_ratio = [1, 1]
+ img = sample["img"]
+ for preprocessing in preprocessings:
+
+ if preprocessing["type"] == "dpi":
+ ratio = preprocessing["target"] / preprocessing["source"]
+ temp_img = img
+ h, w, c = temp_img.shape
+ temp_img = cv2.resize(
+ temp_img, (int(np.ceil(w * ratio)), int(np.ceil(h * ratio)))
+ )
+ if len(temp_img.shape) == 2:
+ temp_img = np.expand_dims(temp_img, axis=2)
+ img = temp_img
+
+ resize_ratio = [ratio, ratio]
+
+ if preprocessing["type"] == "to_grayscaled":
+ temp_img = img
+ h, w, c = temp_img.shape
+ if c == 3:
+ img = np.expand_dims(
+ 0.2125 * temp_img[:, :, 0]
+ + 0.7154 * temp_img[:, :, 1]
+ + 0.0721 * temp_img[:, :, 2],
+ axis=2,
+ ).astype(np.uint8)
+
+ if preprocessing["type"] == "to_RGB":
+ temp_img = img
+ h, w, c = temp_img.shape
+ if c == 1:
+ img = np.concatenate([temp_img, temp_img, temp_img], axis=2)
+
+ if preprocessing["type"] == "resize":
+ keep_ratio = preprocessing["keep_ratio"]
+ max_h, max_w = preprocessing["max_height"], preprocessing["max_width"]
+ temp_img = img
+ h, w, c = temp_img.shape
+
+ ratio_h = max_h / h if max_h else 1
+ ratio_w = max_w / w if max_w else 1
+ if keep_ratio:
+ ratio_h = ratio_w = min(ratio_w, ratio_h)
+ new_h = min(max_h, int(h * ratio_h))
+ new_w = min(max_w, int(w * ratio_w))
+ temp_img = cv2.resize(temp_img, (new_w, new_h))
+ if len(temp_img.shape) == 2:
+ temp_img = np.expand_dims(temp_img, axis=2)
+
+ img = temp_img
+ resize_ratio = [ratio_h, ratio_w]
+
+ if preprocessing["type"] == "fixed_height":
+ new_h = preprocessing["height"]
+ temp_img = img
+ h, w, c = temp_img.shape
+ ratio = new_h / h
+ temp_img = cv2.resize(temp_img, (int(w * ratio), new_h))
+ if len(temp_img.shape) == 2:
+ temp_img = np.expand_dims(temp_img, axis=2)
+ img = temp_img
+ resize_ratio = [ratio, ratio]
+ if resize_ratio != [1, 1] and "raw_line_seg_label" in sample:
+ for li in range(len(sample["raw_line_seg_label"])):
+ for side, ratio in zip(
+ (["bottom", "top"], ["right", "left"]), resize_ratio
+ ):
+ for s in side:
+ sample["raw_line_seg_label"][li][s] = (
+ sample["raw_line_seg_label"][li][s] * ratio
+ )
+
+ sample["img"] = img
+ sample["resize_ratio"] = resize_ratio
+ return sample
diff --git a/dan/manager/metrics.py b/dan/manager/metrics.py
new file mode 100644
index 0000000000000000000000000000000000000000..3bb572b25054455aadf14fdb72f9273c8c57bd03
--- /dev/null
+++ b/dan/manager/metrics.py
@@ -0,0 +1,522 @@
+# -*- coding: utf-8 -*-
+import re
+
+import editdistance
+import networkx as nx
+import numpy as np
+
+from dan.datasets.format.simara import SEM_MATCHING_TOKENS as SIMARA_MATCHING_TOKENS
+from dan.post_processing import PostProcessingModuleSIMARA
+
+
+class MetricManager:
+ def __init__(self, metric_names, dataset_name):
+ self.dataset_name = dataset_name
+
+ if "simara" in dataset_name and "page" in dataset_name:
+ self.post_processing_module = PostProcessingModuleSIMARA
+ self.matching_tokens = SIMARA_MATCHING_TOKENS
+ self.edit_and_num_edge_nodes = edit_and_num_items_for_ged_from_str_simara
+ else:
+ self.matching_tokens = dict()
+
+ self.layout_tokens = "".join(
+ list(self.matching_tokens.keys()) + list(self.matching_tokens.values())
+ )
+ if len(self.layout_tokens) == 0:
+ self.layout_tokens = None
+ self.metric_names = metric_names
+ self.epoch_metrics = None
+
+ self.linked_metrics = {
+ "cer": ["edit_chars", "nb_chars"],
+ "wer": ["edit_words", "nb_words"],
+ "loer": [
+ "edit_graph",
+ "nb_nodes_and_edges",
+ "nb_pp_op_layout",
+ "nb_gt_layout_token",
+ ],
+ "precision": ["precision", "weights"],
+ "map_cer_per_class": [
+ "map_cer",
+ ],
+ "layout_precision_per_class_per_threshold": [
+ "map_cer",
+ ],
+ }
+
+ self.init_metrics()
+
+ def init_metrics(self):
+ """
+ Initialization of the metrics specified in metrics_name
+ """
+ self.epoch_metrics = {
+ "nb_samples": list(),
+ "names": list(),
+ "ids": list(),
+ }
+
+ for metric_name in self.metric_names:
+ if metric_name in self.linked_metrics:
+ for linked_metric_name in self.linked_metrics[metric_name]:
+ if linked_metric_name not in self.epoch_metrics.keys():
+ self.epoch_metrics[linked_metric_name] = list()
+ else:
+ self.epoch_metrics[metric_name] = list()
+
+ def update_metrics(self, batch_metrics):
+ """
+ Add batch metrics to the metrics
+ """
+ for key in batch_metrics.keys():
+ if key in self.epoch_metrics:
+ self.epoch_metrics[key] += batch_metrics[key]
+
+ def get_display_values(self, output=False):
+ """
+ format metrics values for shell display purposes
+ """
+ metric_names = self.metric_names.copy()
+ if output:
+ metric_names.extend(["nb_samples"])
+ display_values = dict()
+ for metric_name in metric_names:
+ value = None
+ if output:
+ if metric_name in ["nb_samples", "weights"]:
+ value = np.sum(self.epoch_metrics[metric_name])
+ elif metric_name in [
+ "time",
+ ]:
+ total_time = np.sum(self.epoch_metrics[metric_name])
+ sample_time = total_time / np.sum(self.epoch_metrics["nb_samples"])
+ display_values["sample_time"] = round(sample_time, 4)
+ value = total_time
+ elif metric_name == "loer":
+ display_values["pper"] = round(
+ np.sum(self.epoch_metrics["nb_pp_op_layout"])
+ / np.sum(self.epoch_metrics["nb_gt_layout_token"]),
+ 4,
+ )
+ elif metric_name == "map_cer_per_class":
+ value = compute_global_mAP_per_class(self.epoch_metrics["map_cer"])
+ for key in value.keys():
+ display_values["map_cer_" + key] = round(value[key], 4)
+ continue
+ elif metric_name == "layout_precision_per_class_per_threshold":
+ value = compute_global_precision_per_class_per_threshold(
+ self.epoch_metrics["map_cer"]
+ )
+ for key_class in value.keys():
+ for threshold in value[key_class].keys():
+ display_values[
+ "map_cer_{}_{}".format(key_class, threshold)
+ ] = round(value[key_class][threshold], 4)
+ continue
+ if metric_name == "cer":
+ value = np.sum(self.epoch_metrics["edit_chars"]) / np.sum(
+ self.epoch_metrics["nb_chars"]
+ )
+ if output:
+ display_values["nb_chars"] = np.sum(self.epoch_metrics["nb_chars"])
+ elif metric_name == "wer":
+ value = np.sum(self.epoch_metrics["edit_words"]) / np.sum(
+ self.epoch_metrics["nb_words"]
+ )
+ if output:
+ display_values["nb_words"] = np.sum(self.epoch_metrics["nb_words"])
+ elif metric_name in ["loss", "loss_ctc", "loss_ce", "syn_max_lines"]:
+ value = np.average(
+ self.epoch_metrics[metric_name],
+ weights=np.array(self.epoch_metrics["nb_samples"]),
+ )
+ elif metric_name == "map_cer":
+ value = compute_global_mAP(self.epoch_metrics[metric_name])
+ elif metric_name == "loer":
+ value = np.sum(self.epoch_metrics["edit_graph"]) / np.sum(
+ self.epoch_metrics["nb_nodes_and_edges"]
+ )
+ elif value is None:
+ continue
+
+ display_values[metric_name] = round(value, 4)
+ return display_values
+
+ def compute_metrics(self, values, metric_names):
+ metrics = {
+ "nb_samples": [
+ values["nb_samples"],
+ ],
+ }
+ for v in ["weights", "time"]:
+ if v in values:
+ metrics[v] = [values[v]]
+ for metric_name in metric_names:
+ if metric_name == "cer":
+ metrics["edit_chars"] = [
+ edit_cer_from_string(u, v, self.layout_tokens)
+ for u, v in zip(values["str_y"], values["str_x"])
+ ]
+ metrics["nb_chars"] = [
+ nb_chars_cer_from_string(gt, self.layout_tokens)
+ for gt in values["str_y"]
+ ]
+ elif metric_name == "wer":
+ split_gt = [
+ format_string_for_wer(gt, self.layout_tokens)
+ for gt in values["str_y"]
+ ]
+ split_pred = [
+ format_string_for_wer(pred, self.layout_tokens)
+ for pred in values["str_x"]
+ ]
+ metrics["edit_words"] = [
+ edit_wer_from_formatted_split_text(gt, pred)
+ for (gt, pred) in zip(split_gt, split_pred)
+ ]
+ metrics["nb_words"] = [len(gt) for gt in split_gt]
+ elif metric_name in [
+ "loss_ctc",
+ "loss_ce",
+ "loss",
+ "syn_max_lines",
+ ]:
+ metrics[metric_name] = [
+ values[metric_name],
+ ]
+ elif metric_name == "map_cer":
+ pp_pred = list()
+ pp_score = list()
+ for pred, score in zip(values["str_x"], values["confidence_score"]):
+ pred_score = self.post_processing_module().post_process(pred, score)
+ pp_pred.append(pred_score[0])
+ pp_score.append(pred_score[1])
+ metrics[metric_name] = [
+ compute_layout_mAP_per_class(y, x, conf, self.matching_tokens)
+ for x, conf, y in zip(pp_pred, pp_score, values["str_y"])
+ ]
+ elif metric_name == "loer":
+ pp_pred = list()
+ metrics["nb_pp_op_layout"] = list()
+ for pred in values["str_x"]:
+ pp_module = self.post_processing_module()
+ pp_pred.append(pp_module.post_process(pred))
+ metrics["nb_pp_op_layout"].append(pp_module.num_op)
+ metrics["nb_gt_layout_token"] = [
+ len(keep_only_tokens(str_x, self.layout_tokens))
+ for str_x in values["str_x"]
+ ]
+ edit_and_num_items = [
+ self.edit_and_num_edge_nodes(y, x)
+ for x, y in zip(pp_pred, values["str_y"])
+ ]
+ metrics["edit_graph"], metrics["nb_nodes_and_edges"] = [
+ ei[0] for ei in edit_and_num_items
+ ], [ei[1] for ei in edit_and_num_items]
+ return metrics
+
+ def get(self, name):
+ return self.epoch_metrics[name]
+
+
+def keep_only_tokens(str, tokens):
+ """
+ Remove all but layout tokens from string
+ """
+ return re.sub("([^" + tokens + "])", "", str)
+
+
+def keep_all_but_tokens(str, tokens):
+ """
+ Remove all layout tokens from string
+ """
+ return re.sub("([" + tokens + "])", "", str)
+
+
+def edit_cer_from_string(gt, pred, layout_tokens=None):
+ """
+ Format and compute edit distance between two strings at character level
+ """
+ gt = format_string_for_cer(gt, layout_tokens)
+ pred = format_string_for_cer(pred, layout_tokens)
+ return editdistance.eval(gt, pred)
+
+
+def nb_chars_cer_from_string(gt, layout_tokens=None):
+ """
+ Compute length after formatting of ground truth string
+ """
+ return len(format_string_for_cer(gt, layout_tokens))
+
+
+def edit_wer_from_string(gt, pred, layout_tokens=None):
+ """
+ Format and compute edit distance between two strings at word level
+ """
+ split_gt = format_string_for_wer(gt, layout_tokens)
+ split_pred = format_string_for_wer(pred, layout_tokens)
+ return edit_wer_from_formatted_split_text(split_gt, split_pred)
+
+
+def format_string_for_wer(str, layout_tokens):
+ """
+ Format string for WER computation: remove layout tokens, treat punctuation as word, replace line break by space
+ """
+ str = re.sub(
+ r"([\[\]{}/\\()\"'&+*=<>?.;:,!\-—_€#%°])", r" \1 ", str
+ ) # punctuation processed as word
+ if layout_tokens is not None:
+ str = keep_all_but_tokens(
+ str, layout_tokens
+ ) # remove layout tokens from metric
+ str = re.sub("([ \n])+", " ", str).strip() # keep only one space character
+ return str.split(" ")
+
+
+def format_string_for_cer(str, layout_tokens):
+ """
+ Format string for CER computation: remove layout tokens and extra spaces
+ """
+ if layout_tokens is not None:
+ str = keep_all_but_tokens(
+ str, layout_tokens
+ ) # remove layout tokens from metric
+ str = re.sub("([\n])+", "\n", str) # remove consecutive line breaks
+ str = re.sub("([ ])+", " ", str).strip() # remove consecutive spaces
+ return str
+
+
+def edit_wer_from_formatted_split_text(gt, pred):
+ """
+ Compute edit distance at word level from formatted string as list
+ """
+ return editdistance.eval(gt, pred)
+
+
+def extract_by_tokens(
+ input_str, begin_token, end_token, associated_score=None, order_by_score=False
+):
+ """
+ Extract list of text regions by begin and end tokens
+ Order the list by confidence score
+ """
+ if order_by_score:
+ assert associated_score is not None
+ res = list()
+ for match in re.finditer(
+ "{}[^{}]*{}".format(begin_token, end_token, end_token), input_str
+ ):
+ begin, end = match.regs[0]
+ if order_by_score:
+ res.append(
+ {
+ "confidence": np.mean(
+ [associated_score[begin], associated_score[end - 1]]
+ ),
+ "content": input_str[begin + 1 : end - 1],
+ }
+ )
+ else:
+ res.append(input_str[begin + 1 : end - 1])
+ if order_by_score:
+ res = sorted(res, key=lambda x: x["confidence"], reverse=True)
+ res = [r["content"] for r in res]
+ return res
+
+
+def compute_layout_precision_per_threshold(
+ gt, pred, score, begin_token, end_token, layout_tokens, return_weight=True
+):
+ """
+ Compute average precision of a given class for CER threshold from 5% to 50% with a step of 5%
+ """
+ pred_list = extract_by_tokens(
+ pred, begin_token, end_token, associated_score=score, order_by_score=True
+ )
+ gt_list = extract_by_tokens(gt, begin_token, end_token)
+ pred_list = [keep_all_but_tokens(p, layout_tokens) for p in pred_list]
+ gt_list = [keep_all_but_tokens(gt, layout_tokens) for gt in gt_list]
+ precision_per_threshold = [
+ compute_layout_AP_for_given_threshold(gt_list, pred_list, threshold / 100)
+ for threshold in range(5, 51, 5)
+ ]
+ if return_weight:
+ return precision_per_threshold, len(gt_list)
+ return precision_per_threshold
+
+
+def compute_layout_AP_for_given_threshold(gt_list, pred_list, threshold):
+ """
+ Compute average precision of a given class for a given CER threshold
+ """
+ remaining_gt_list = gt_list.copy()
+ num_true = len(gt_list)
+ correct = np.zeros((len(pred_list)), dtype=np.bool)
+ for i, pred in enumerate(pred_list):
+ if len(remaining_gt_list) == 0:
+ break
+ cer_with_gt = [
+ edit_cer_from_string(gt, pred) / nb_chars_cer_from_string(gt)
+ for gt in remaining_gt_list
+ ]
+ cer, ind = np.min(cer_with_gt), np.argmin(cer_with_gt)
+ if cer <= threshold:
+ correct[i] = True
+ del remaining_gt_list[ind]
+ precision = np.cumsum(correct, dtype=np.int) / np.arange(1, len(pred_list) + 1)
+ recall = np.cumsum(correct, dtype=np.int) / num_true
+ max_precision_from_recall = np.maximum.accumulate(precision[::-1])[::-1]
+ recall_diff = recall - np.concatenate(
+ [
+ np.array(
+ [
+ 0,
+ ]
+ ),
+ recall[:-1],
+ ]
+ )
+ P = np.sum(recall_diff * max_precision_from_recall)
+ return P
+
+
+def compute_layout_mAP_per_class(gt, pred, score, tokens):
+ """
+ Compute the mAP_cer for each class for a given sample
+ """
+ layout_tokens = "".join(list(tokens.keys()))
+ AP_per_class = dict()
+ for token in tokens.keys():
+ if token in gt:
+ AP_per_class[token] = compute_layout_precision_per_threshold(
+ gt, pred, score, token, tokens[token], layout_tokens=layout_tokens
+ )
+ return AP_per_class
+
+
+def compute_global_mAP(list_AP_per_class):
+ """
+ Compute the global mAP_cer for several samples
+ """
+ weights_per_doc = list()
+ mAP_per_doc = list()
+ for doc_AP_per_class in list_AP_per_class:
+ APs = np.array(
+ [np.mean(doc_AP_per_class[key][0]) for key in doc_AP_per_class.keys()]
+ )
+ weights = np.array(
+ [doc_AP_per_class[key][1] for key in doc_AP_per_class.keys()]
+ )
+ if np.sum(weights) == 0:
+ mAP_per_doc.append(0)
+ else:
+ mAP_per_doc.append(np.average(APs, weights=weights))
+ weights_per_doc.append(np.sum(weights))
+ if np.sum(weights_per_doc) == 0:
+ return 0
+ return np.average(mAP_per_doc, weights=weights_per_doc)
+
+
+def compute_global_mAP_per_class(list_AP_per_class):
+ """
+ Compute the mAP_cer per class for several samples
+ """
+ mAP_per_class = dict()
+ for doc_AP_per_class in list_AP_per_class:
+ for key in doc_AP_per_class.keys():
+ if key not in mAP_per_class:
+ mAP_per_class[key] = {"AP": list(), "weights": list()}
+ mAP_per_class[key]["AP"].append(np.mean(doc_AP_per_class[key][0]))
+ mAP_per_class[key]["weights"].append(doc_AP_per_class[key][1])
+ for key in mAP_per_class.keys():
+ mAP_per_class[key] = np.average(
+ mAP_per_class[key]["AP"], weights=mAP_per_class[key]["weights"]
+ )
+ return mAP_per_class
+
+
+def compute_global_precision_per_class_per_threshold(list_AP_per_class):
+ """
+ Compute the mAP_cer per class and per threshold for several samples
+ """
+ mAP_per_class = dict()
+ for doc_AP_per_class in list_AP_per_class:
+ for key in doc_AP_per_class.keys():
+ if key not in mAP_per_class:
+ mAP_per_class[key] = dict()
+ for threshold in range(5, 51, 5):
+ mAP_per_class[key][threshold] = {
+ "precision": list(),
+ "weights": list(),
+ }
+ for i, threshold in enumerate(range(5, 51, 5)):
+ mAP_per_class[key][threshold]["precision"].append(
+ np.mean(doc_AP_per_class[key][0][i])
+ )
+ mAP_per_class[key][threshold]["weights"].append(
+ doc_AP_per_class[key][1]
+ )
+ for key_class in mAP_per_class.keys():
+ for threshold in mAP_per_class[key_class]:
+ mAP_per_class[key_class][threshold] = np.average(
+ mAP_per_class[key_class][threshold]["precision"],
+ weights=mAP_per_class[key_class][threshold]["weights"],
+ )
+ return mAP_per_class
+
+
+def str_to_graph_simara(str):
+ """
+ Compute graph from string of layout tokens for the SIMARA dataset at page level
+ """
+ begin_layout_tokens = "".join(list(SIMARA_MATCHING_TOKENS.keys()))
+ layout_token_sequence = keep_only_tokens(str, begin_layout_tokens)
+ g = nx.DiGraph()
+ g.add_node("D", type="document", level=2, page=0)
+ token_name_dict = {"ⓘ": "I", "â““": "D", "â“¢": "S", "â“’": "C", "â“Ÿ": "P", "â“": "A"}
+ num = dict()
+ previous_node = None
+ for token in begin_layout_tokens:
+ num[token] = 0
+ for ind, c in enumerate(layout_token_sequence):
+ num[c] += 1
+ node_name = "{}_{}".format(token_name_dict[c], num[c])
+ g.add_node(node_name, type=token_name_dict[c], level=1, page=0)
+ g.add_edge("D", node_name)
+ if previous_node:
+ g.add_edge(previous_node, node_name)
+ previous_node = node_name
+ return g
+
+
+def graph_edit_distance(g1, g2):
+ """
+ Compute graph edit distance between two graphs
+ """
+ for v in nx.optimize_graph_edit_distance(
+ g1,
+ g2,
+ node_ins_cost=lambda node: 1,
+ node_del_cost=lambda node: 1,
+ node_subst_cost=lambda node1, node2: 0 if node1["type"] == node2["type"] else 1,
+ edge_ins_cost=lambda edge: 1,
+ edge_del_cost=lambda edge: 1,
+ edge_subst_cost=lambda edge1, edge2: 0 if edge1 == edge2 else 1,
+ ):
+ new_edit = v
+ return new_edit
+
+
+def edit_and_num_items_for_ged_from_str_simara(str_gt, str_pred):
+ """
+ Compute graph edit distance and num nodes/edges for normalized graph edit distance
+ For the SIMARA dataset
+ """
+ g_gt = str_to_graph_simara(str_gt)
+ g_pred = str_to_graph_simara(str_pred)
+ return (
+ graph_edit_distance(g_gt, g_pred),
+ g_gt.number_of_nodes() + g_gt.number_of_edges(),
+ )
diff --git a/dan/manager/ocr.py b/dan/manager/ocr.py
new file mode 100644
index 0000000000000000000000000000000000000000..71463042e14e381161385ba443433e69ca9b404a
--- /dev/null
+++ b/dan/manager/ocr.py
@@ -0,0 +1,640 @@
+# -*- coding: utf-8 -*-
+import copy
+import os
+import pickle
+
+import cv2
+import numpy as np
+import torch
+from fontTools.ttLib import TTFont
+from PIL import Image, ImageDraw, ImageFont
+
+from dan.manager.dataset import DatasetManager, GenericDataset, apply_preprocessing
+from dan.ocr.utils import LM_str_to_ind
+from dan.utils import (
+ pad_image,
+ pad_image_width_right,
+ pad_images,
+ pad_sequences_1D,
+ rand,
+ rand_uniform,
+ randint,
+)
+
+
+class OCRDatasetManager(DatasetManager):
+ """
+ Specific class to handle OCR/HTR tasks
+ """
+
+ def __init__(self, params):
+ super(OCRDatasetManager, self).__init__(params)
+
+ self.charset = (
+ params["charset"] if "charset" in params else self.get_merged_charsets()
+ )
+
+ if (
+ "synthetic_data" in self.params["config"]
+ and self.params["config"]["synthetic_data"]
+ and "config" in self.params["config"]["synthetic_data"]
+ ):
+ self.char_only_set = self.charset.copy()
+ for token in [
+ "\n",
+ ]:
+ if token in self.char_only_set:
+ self.char_only_set.remove(token)
+ self.params["config"]["synthetic_data"]["config"][
+ "valid_fonts"
+ ] = get_valid_fonts(self.char_only_set)
+
+ if "new_tokens" in params:
+ self.charset = sorted(
+ list(set(self.charset).union(set(params["new_tokens"])))
+ )
+
+ self.tokens = {
+ "pad": params["config"]["padding_token"],
+ }
+ if self.params["config"]["charset_mode"].lower() == "ctc":
+ self.tokens["blank"] = len(self.charset)
+ self.tokens["pad"] = (
+ self.tokens["pad"] if self.tokens["pad"] else len(self.charset) + 1
+ )
+ self.params["config"]["padding_token"] = self.tokens["pad"]
+ elif self.params["config"]["charset_mode"] == "seq2seq":
+ self.tokens["end"] = len(self.charset)
+ self.tokens["start"] = len(self.charset) + 1
+ self.tokens["pad"] = (
+ self.tokens["pad"] if self.tokens["pad"] else len(self.charset) + 2
+ )
+ self.params["config"]["padding_token"] = self.tokens["pad"]
+
+ def get_merged_charsets(self):
+ """
+ Merge the charset of the different datasets used
+ """
+ datasets = self.params["datasets"]
+ charset = set()
+ for key in datasets.keys():
+ with open(os.path.join(datasets[key], "labels.pkl"), "rb") as f:
+ info = pickle.load(f)
+ charset = charset.union(set(info["charset"]))
+ if (
+ "\n" in charset
+ and "remove_linebreaks" in self.params["config"]["constraints"]
+ ):
+ charset.remove("\n")
+ if "" in charset:
+ charset.remove("")
+ return sorted(list(charset))
+
+ def apply_specific_treatment_after_dataset_loading(self, dataset):
+ dataset.charset = self.charset
+ dataset.tokens = self.tokens
+ dataset.convert_labels()
+ if (
+ "padding" in dataset.params["config"]
+ and dataset.params["config"]["padding"]["min_height"] == "max"
+ ):
+ dataset.params["config"]["padding"]["min_height"] = max(
+ [s["img"].shape[0] for s in self.train_dataset.samples]
+ )
+ if (
+ "padding" in dataset.params["config"]
+ and dataset.params["config"]["padding"]["min_width"] == "max"
+ ):
+ dataset.params["config"]["padding"]["min_width"] = max(
+ [s["img"].shape[1] for s in self.train_dataset.samples]
+ )
+
+
+class OCRDataset(GenericDataset):
+ """
+ Specific class to handle OCR/HTR datasets
+ """
+
+ def __init__(self, params, set_name, custom_name, paths_and_sets):
+ super(OCRDataset, self).__init__(params, set_name, custom_name, paths_and_sets)
+ self.charset = None
+ self.tokens = None
+ self.reduce_dims_factor = np.array(
+ [params["config"]["height_divisor"], params["config"]["width_divisor"], 1]
+ )
+ self.collate_function = OCRCollateFunction
+ self.synthetic_id = 0
+
+ def __getitem__(self, idx):
+ sample = copy.deepcopy(self.samples[idx])
+
+ if not self.load_in_memory:
+ sample["img"] = self.get_sample_img(idx)
+ sample = apply_preprocessing(
+ sample, self.params["config"]["preprocessings"]
+ )
+
+ if (
+ "synthetic_data" in self.params["config"]
+ and self.params["config"]["synthetic_data"]
+ and self.set_name == "train"
+ ):
+ sample = self.generate_synthetic_data(sample)
+
+ # Data augmentation
+ sample["img"], sample["applied_da"] = self.apply_data_augmentation(
+ sample["img"]
+ )
+
+ if "max_size" in self.params["config"] and self.params["config"]["max_size"]:
+ max_ratio = max(
+ sample["img"].shape[0]
+ / self.params["config"]["max_size"]["max_height"],
+ sample["img"].shape[1] / self.params["config"]["max_size"]["max_width"],
+ )
+ if max_ratio > 1:
+ new_h, new_w = int(np.ceil(sample["img"].shape[0] / max_ratio)), int(
+ np.ceil(sample["img"].shape[1] / max_ratio)
+ )
+ sample["img"] = cv2.resize(sample["img"], (new_w, new_h))
+
+ # Normalization if requested
+ if "normalize" in self.params["config"] and self.params["config"]["normalize"]:
+ sample["img"] = (sample["img"] - self.mean) / self.std
+
+ sample["img_shape"] = sample["img"].shape
+ sample["img_reduced_shape"] = np.ceil(
+ sample["img_shape"] / self.reduce_dims_factor
+ ).astype(int)
+
+ # Padding to handle CTC requirements
+ if self.set_name == "train":
+ max_label_len = 0
+ height = 1
+ ctc_padding = False
+ if "CTC_line" in self.params["config"]["constraints"]:
+ max_label_len = sample["label_len"]
+ ctc_padding = True
+ if "CTC_va" in self.params["config"]["constraints"]:
+ max_label_len = max(sample["line_label_len"])
+ ctc_padding = True
+ if "CTC_pg" in self.params["config"]["constraints"]:
+ max_label_len = sample["label_len"]
+ height = max(sample["img_reduced_shape"][0], 1)
+ ctc_padding = True
+ if (
+ ctc_padding
+ and 2 * max_label_len + 1 > sample["img_reduced_shape"][1] * height
+ ):
+ sample["img"] = pad_image_width_right(
+ sample["img"],
+ int(
+ np.ceil((2 * max_label_len + 1) / height)
+ * self.reduce_dims_factor[1]
+ ),
+ self.padding_value,
+ )
+ sample["img_shape"] = sample["img"].shape
+ sample["img_reduced_shape"] = np.ceil(
+ sample["img_shape"] / self.reduce_dims_factor
+ ).astype(int)
+ sample["img_reduced_shape"] = [
+ max(1, t) for t in sample["img_reduced_shape"]
+ ]
+
+ sample["img_position"] = [
+ [0, sample["img_shape"][0]],
+ [0, sample["img_shape"][1]],
+ ]
+ # Padding constraints to handle model needs
+ if "padding" in self.params["config"] and self.params["config"]["padding"]:
+ if (
+ self.set_name == "train"
+ or not self.params["config"]["padding"]["train_only"]
+ ):
+ min_pad = self.params["config"]["padding"]["min_pad"]
+ max_pad = self.params["config"]["padding"]["max_pad"]
+ pad_width = (
+ randint(min_pad, max_pad)
+ if min_pad is not None and max_pad is not None
+ else None
+ )
+ pad_height = (
+ randint(min_pad, max_pad)
+ if min_pad is not None and max_pad is not None
+ else None
+ )
+
+ sample["img"], sample["img_position"] = pad_image(
+ sample["img"],
+ padding_value=self.padding_value,
+ new_width=self.params["config"]["padding"]["min_width"],
+ new_height=self.params["config"]["padding"]["min_height"],
+ pad_width=pad_width,
+ pad_height=pad_height,
+ padding_mode=self.params["config"]["padding"]["mode"],
+ return_position=True,
+ )
+ sample["img_reduced_position"] = [
+ np.ceil(p / factor).astype(int)
+ for p, factor in zip(sample["img_position"], self.reduce_dims_factor[:2])
+ ]
+ return sample
+
+ def convert_labels(self):
+ """
+ Label str to token at character level
+ """
+ for i in range(len(self.samples)):
+ self.samples[i] = self.convert_sample_labels(self.samples[i])
+
+ def convert_sample_labels(self, sample):
+ label = sample["label"]
+ line_labels = label.split("\n")
+ if "remove_linebreaks" in self.params["config"]["constraints"]:
+ full_label = label.replace("\n", " ").replace(" ", " ")
+ word_labels = full_label.split(" ")
+ else:
+ full_label = label
+ word_labels = label.replace("\n", " ").replace(" ", " ").split(" ")
+
+ sample["label"] = full_label
+ sample["token_label"] = LM_str_to_ind(self.charset, full_label)
+ if "add_eot" in self.params["config"]["constraints"]:
+ sample["token_label"].append(self.tokens["end"])
+ sample["label_len"] = len(sample["token_label"])
+ if "add_sot" in self.params["config"]["constraints"]:
+ sample["token_label"].insert(0, self.tokens["start"])
+
+ sample["line_label"] = line_labels
+ sample["token_line_label"] = [
+ LM_str_to_ind(self.charset, label) for label in line_labels
+ ]
+ sample["line_label_len"] = [len(label) for label in line_labels]
+ sample["nb_lines"] = len(line_labels)
+
+ sample["word_label"] = word_labels
+ sample["token_word_label"] = [
+ LM_str_to_ind(self.charset, label) for label in word_labels
+ ]
+ sample["word_label_len"] = [len(label) for label in word_labels]
+ sample["nb_words"] = len(word_labels)
+ return sample
+
+ def generate_synthetic_data(self, sample):
+ config = self.params["config"]["synthetic_data"]
+
+ if not (config["init_proba"] == config["end_proba"] == 1):
+ nb_samples = self.training_info["step"] * self.params["batch_size"]
+ if config["start_scheduler_at_max_line"]:
+ max_step = config["num_steps_proba"]
+ current_step = max(
+ 0,
+ min(
+ nb_samples
+ - config["curr_step"]
+ * (config["max_nb_lines"] - config["min_nb_lines"]),
+ max_step,
+ ),
+ )
+ proba = (
+ config["init_proba"]
+ if self.get_syn_max_lines() < config["max_nb_lines"]
+ else config["proba_scheduler_function"](
+ config["init_proba"],
+ config["end_proba"],
+ current_step,
+ max_step,
+ )
+ )
+ else:
+ proba = config["proba_scheduler_function"](
+ config["init_proba"],
+ config["end_proba"],
+ min(nb_samples, config["num_steps_proba"]),
+ config["num_steps_proba"],
+ )
+ if rand() > proba:
+ return sample
+
+ if "mode" in config and config["mode"] == "line_hw_to_printed":
+ sample["img"] = self.generate_typed_text_line_image(sample["label"])
+ return sample
+
+ return self.generate_synthetic_page_sample()
+
+ def get_syn_max_lines(self):
+ config = self.params["config"]["synthetic_data"]
+ if config["curriculum"]:
+ nb_samples = self.training_info["step"] * self.params["batch_size"]
+ max_nb_lines = min(
+ config["max_nb_lines"],
+ (nb_samples - config["curr_start"]) // config["curr_step"] + 1,
+ )
+ return max(config["min_nb_lines"], max_nb_lines)
+ return config["max_nb_lines"]
+
+ def generate_synthetic_page_sample(self):
+ config = self.params["config"]["synthetic_data"]
+ max_nb_lines_per_page = self.get_syn_max_lines()
+ crop = (
+ config["crop_curriculum"] and max_nb_lines_per_page < config["max_nb_lines"]
+ )
+ sample = {"name": "synthetic_data_{}".format(self.synthetic_id), "path": None}
+ self.synthetic_id += 1
+ nb_pages = 2 if "double" in config["dataset_level"] else 1
+ background_sample = copy.deepcopy(self.samples[randint(0, len(self))])
+ pages = list()
+ backgrounds = list()
+
+ h, w, c = background_sample["img"].shape
+ page_width = w // 2 if nb_pages == 2 else w
+ for i in range(nb_pages):
+ nb_lines_per_page = randint(
+ config["min_nb_lines"], max_nb_lines_per_page + 1
+ )
+ background = (
+ np.ones((h, page_width, c), dtype=background_sample["img"].dtype) * 255
+ )
+ if i == 0 and nb_pages == 2:
+ background[:, -2:, :] = 0
+ backgrounds.append(background)
+ if "READ_2016" in self.params["datasets"].keys():
+ side = background_sample["pages_label"][i]["side"]
+ coords = {
+ "left": int(0.15 * page_width)
+ if side == "left"
+ else int(0.05 * page_width),
+ "right": int(0.95 * page_width)
+ if side == "left"
+ else int(0.85 * page_width),
+ "top": int(0.05 * h),
+ "bottom": int(0.85 * h),
+ }
+ pages.append(
+ self.generate_synthetic_read2016_page(
+ background,
+ coords,
+ side=side,
+ crop=crop,
+ nb_lines=nb_lines_per_page,
+ )
+ )
+ elif "RIMES" in self.params["datasets"].keys():
+ pages.append(
+ self.generate_synthetic_rimes_page(
+ background, nb_lines=nb_lines_per_page, crop=crop
+ )
+ )
+ else:
+ raise NotImplementedError
+
+ if nb_pages == 1:
+ sample["img"] = pages[0][0]
+ sample["label_raw"] = pages[0][1]["raw"]
+ sample["label_begin"] = pages[0][1]["begin"]
+ sample["label_sem"] = pages[0][1]["sem"]
+ sample["label"] = pages[0][1]
+ sample["nb_cols"] = pages[0][2]
+ else:
+ if pages[0][0].shape[0] != pages[1][0].shape[0]:
+ max_height = max(pages[0][0].shape[0], pages[1][0].shape[0])
+ backgrounds[0] = backgrounds[0][:max_height]
+ backgrounds[0][: pages[0][0].shape[0]] = pages[0][0]
+ backgrounds[1] = backgrounds[1][:max_height]
+ backgrounds[1][: pages[1][0].shape[0]] = pages[1][0]
+ pages[0][0] = backgrounds[0]
+ pages[1][0] = backgrounds[1]
+ sample["label_raw"] = pages[0][1]["raw"] + "\n" + pages[1][1]["raw"]
+ sample["label_begin"] = pages[0][1]["begin"] + pages[1][1]["begin"]
+ sample["label_sem"] = pages[0][1]["sem"] + pages[1][1]["sem"]
+ sample["img"] = np.concatenate([pages[0][0], pages[1][0]], axis=1)
+ sample["nb_cols"] = pages[0][2] + pages[1][2]
+ sample["label"] = sample["label_raw"]
+ if "â“‘" in self.charset:
+ sample["label"] = sample["label_begin"]
+ if "â’·" in self.charset:
+ sample["label"] = sample["label_sem"]
+ sample["unchanged_label"] = sample["label"]
+ sample = self.convert_sample_labels(sample)
+ return sample
+
+ def generate_typed_text_line_image(self, text):
+ return generate_typed_text_line_image(
+ text, self.params["config"]["synthetic_data"]["config"]
+ )
+
+
+class OCRCollateFunction:
+ """
+ Merge samples data to mini-batch data for OCR task
+ """
+
+ def __init__(self, config):
+ self.img_padding_value = float(config["padding_value"])
+ self.label_padding_value = config["padding_token"]
+ self.config = config
+
+ def __call__(self, batch_data):
+ names = [batch_data[i]["name"] for i in range(len(batch_data))]
+ ids = [
+ batch_data[i]["name"].split("/")[-1].split(".")[0]
+ for i in range(len(batch_data))
+ ]
+ applied_da = [batch_data[i]["applied_da"] for i in range(len(batch_data))]
+
+ labels = [batch_data[i]["token_label"] for i in range(len(batch_data))]
+ labels = pad_sequences_1D(labels, padding_value=self.label_padding_value)
+ labels = torch.tensor(labels).long()
+ reverse_labels = [
+ [
+ batch_data[i]["token_label"][0],
+ ]
+ + batch_data[i]["token_label"][-2:0:-1]
+ + [
+ batch_data[i]["token_label"][-1],
+ ]
+ for i in range(len(batch_data))
+ ]
+ reverse_labels = pad_sequences_1D(
+ reverse_labels, padding_value=self.label_padding_value
+ )
+ reverse_labels = torch.tensor(reverse_labels).long()
+ labels_len = [batch_data[i]["label_len"] for i in range(len(batch_data))]
+
+ raw_labels = [batch_data[i]["label"] for i in range(len(batch_data))]
+ unchanged_labels = [
+ batch_data[i]["unchanged_label"] for i in range(len(batch_data))
+ ]
+
+ nb_cols = [batch_data[i]["nb_cols"] for i in range(len(batch_data))]
+ nb_lines = [batch_data[i]["nb_lines"] for i in range(len(batch_data))]
+ line_raw = [batch_data[i]["line_label"] for i in range(len(batch_data))]
+ line_token = [batch_data[i]["token_line_label"] for i in range(len(batch_data))]
+ pad_line_token = list()
+ line_len = [batch_data[i]["line_label_len"] for i in range(len(batch_data))]
+ for i in range(max(nb_lines)):
+ current_lines = [
+ line_token[j][i] if i < nb_lines[j] else [self.label_padding_value]
+ for j in range(len(batch_data))
+ ]
+ pad_line_token.append(
+ torch.tensor(
+ pad_sequences_1D(
+ current_lines, padding_value=self.label_padding_value
+ )
+ ).long()
+ )
+ for j in range(len(batch_data)):
+ if i >= nb_lines[j]:
+ line_len[j].append(0)
+ line_len = [i for i in zip(*line_len)]
+
+ nb_words = [batch_data[i]["nb_words"] for i in range(len(batch_data))]
+ word_raw = [batch_data[i]["word_label"] for i in range(len(batch_data))]
+ word_token = [batch_data[i]["token_word_label"] for i in range(len(batch_data))]
+ pad_word_token = list()
+ word_len = [batch_data[i]["word_label_len"] for i in range(len(batch_data))]
+ for i in range(max(nb_words)):
+ current_words = [
+ word_token[j][i] if i < nb_words[j] else [self.label_padding_value]
+ for j in range(len(batch_data))
+ ]
+ pad_word_token.append(
+ torch.tensor(
+ pad_sequences_1D(
+ current_words, padding_value=self.label_padding_value
+ )
+ ).long()
+ )
+ for j in range(len(batch_data)):
+ if i >= nb_words[j]:
+ word_len[j].append(0)
+ word_len = [i for i in zip(*word_len)]
+
+ padding_mode = (
+ self.config["padding_mode"] if "padding_mode" in self.config else "br"
+ )
+ imgs = [batch_data[i]["img"] for i in range(len(batch_data))]
+ imgs_shape = [batch_data[i]["img_shape"] for i in range(len(batch_data))]
+ imgs_reduced_shape = [
+ batch_data[i]["img_reduced_shape"] for i in range(len(batch_data))
+ ]
+ imgs_position = [batch_data[i]["img_position"] for i in range(len(batch_data))]
+ imgs_reduced_position = [
+ batch_data[i]["img_reduced_position"] for i in range(len(batch_data))
+ ]
+ imgs = pad_images(
+ imgs, padding_value=self.img_padding_value, padding_mode=padding_mode
+ )
+ imgs = torch.tensor(imgs).float().permute(0, 3, 1, 2)
+ formatted_batch_data = {
+ "names": names,
+ "ids": ids,
+ "nb_lines": nb_lines,
+ "nb_cols": nb_cols,
+ "labels": labels,
+ "reverse_labels": reverse_labels,
+ "raw_labels": raw_labels,
+ "unchanged_labels": unchanged_labels,
+ "labels_len": labels_len,
+ "imgs": imgs,
+ "imgs_shape": imgs_shape,
+ "imgs_reduced_shape": imgs_reduced_shape,
+ "imgs_position": imgs_position,
+ "imgs_reduced_position": imgs_reduced_position,
+ "line_raw": line_raw,
+ "line_labels": pad_line_token,
+ "line_labels_len": line_len,
+ "nb_words": nb_words,
+ "word_raw": word_raw,
+ "word_labels": pad_word_token,
+ "word_labels_len": word_len,
+ "applied_da": applied_da,
+ }
+
+ return formatted_batch_data
+
+
+def generate_typed_text_line_image(
+ text, config, bg_color=(255, 255, 255), txt_color=(0, 0, 0)
+):
+ if text == "":
+ text = " "
+ if "text_color_default" in config:
+ txt_color = config["text_color_default"]
+ if "background_color_default" in config:
+ bg_color = config["background_color_default"]
+
+ font_path = config["valid_fonts"][randint(0, len(config["valid_fonts"]))]
+ font_size = randint(config["font_size_min"], config["font_size_max"] + 1)
+ fnt = ImageFont.truetype(font_path, font_size)
+
+ text_width, text_height = fnt.getsize(text)
+ padding_top = int(
+ rand_uniform(config["padding_top_ratio_min"], config["padding_top_ratio_max"])
+ * text_height
+ )
+ padding_bottom = int(
+ rand_uniform(
+ config["padding_bottom_ratio_min"], config["padding_bottom_ratio_max"]
+ )
+ * text_height
+ )
+ padding_left = int(
+ rand_uniform(config["padding_left_ratio_min"], config["padding_left_ratio_max"])
+ * text_width
+ )
+ padding_right = int(
+ rand_uniform(
+ config["padding_right_ratio_min"], config["padding_right_ratio_max"]
+ )
+ * text_width
+ )
+ padding = [padding_top, padding_bottom, padding_left, padding_right]
+ return generate_typed_text_line_image_from_params(
+ text, fnt, bg_color, txt_color, config["color_mode"], padding
+ )
+
+
+def generate_typed_text_line_image_from_params(
+ text, font, bg_color, txt_color, color_mode, padding
+):
+ padding_top, padding_bottom, padding_left, padding_right = padding
+ text_width, text_height = font.getsize(text)
+ img_height = padding_top + padding_bottom + text_height
+ img_width = padding_left + padding_right + text_width
+ img = Image.new(color_mode, (img_width, img_height), color=bg_color)
+ d = ImageDraw.Draw(img)
+ d.text((padding_left, padding_bottom), text, font=font, fill=txt_color, spacing=0)
+ return np.array(img)
+
+
+def char_in_font(unicode_char, font_path):
+ with TTFont(font_path) as font:
+ for cmap in font["cmap"].tables:
+ if cmap.isUnicode():
+ if ord(unicode_char) in cmap.cmap:
+ return True
+ return False
+
+
+def get_valid_fonts(alphabet=None):
+ valid_fonts = list()
+ for fold_detail in os.walk("../../../Fonts"):
+ if fold_detail[2]:
+ for font_name in fold_detail[2]:
+ if ".ttf" not in font_name:
+ continue
+ font_path = os.path.join(fold_detail[0], font_name)
+ to_add = True
+ if alphabet is not None:
+ for char in alphabet:
+ if not char_in_font(char, font_path):
+ to_add = False
+ break
+ if to_add:
+ valid_fonts.append(font_path)
+ else:
+ valid_fonts.append(font_path)
+ return valid_fonts
diff --git a/dan/manager/training.py b/dan/manager/training.py
new file mode 100644
index 0000000000000000000000000000000000000000..90f1362648ed96fc8b130d5039ccae98556a15e9
--- /dev/null
+++ b/dan/manager/training.py
@@ -0,0 +1,1291 @@
+# -*- coding: utf-8 -*-
+import copy
+import json
+import os
+import pickle
+import random
+import sys
+from datetime import date
+from time import time
+
+import numpy as np
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+from PIL import Image
+from torch.cuda.amp import GradScaler, autocast
+from torch.nn import CrossEntropyLoss
+from torch.nn.init import kaiming_uniform_
+from torch.nn.parallel import DistributedDataParallel as DDP
+from torch.utils.tensorboard import SummaryWriter
+from tqdm import tqdm
+
+from dan.manager.metrics import MetricManager
+from dan.ocr.utils import LM_ind_to_str
+from dan.schedulers import DropoutScheduler
+
+
+class GenericTrainingManager:
+ def __init__(self, params):
+ self.type = None
+ self.is_master = False
+ self.params = params
+ self.dropout_scheduler = None
+ self.models = {}
+ self.begin_time = None
+ self.dataset = None
+ self.dataset_name = list(self.params["dataset_params"]["datasets"].values())[0]
+ self.paths = None
+ self.latest_step = 0
+ self.latest_epoch = -1
+ self.latest_batch = 0
+ self.total_batch = 0
+ self.grad_acc_step = 0
+ self.latest_train_metrics = dict()
+ self.latest_valid_metrics = dict()
+ self.curriculum_info = dict()
+ self.curriculum_info["latest_valid_metrics"] = dict()
+ self.phase = None
+ self.max_mem_usage_by_epoch = list()
+ self.losses = list()
+ self.lr_values = list()
+
+ self.scaler = None
+
+ self.optimizers = dict()
+ self.optimizers_named_params_by_group = dict()
+ self.lr_schedulers = dict()
+ self.best = None
+ self.writer = None
+ self.metric_manager = dict()
+
+ self.init_hardware_config()
+ self.init_paths()
+ self.load_dataset()
+ self.params["model_params"]["use_amp"] = self.params["training_params"][
+ "use_amp"
+ ]
+
+ def init_paths(self):
+ """
+ Create output folders for results and checkpoints
+ """
+ output_path = os.path.join(
+ "outputs", self.params["training_params"]["output_folder"]
+ )
+ os.makedirs(output_path, exist_ok=True)
+ checkpoints_path = os.path.join(output_path, "checkpoints")
+ os.makedirs(checkpoints_path, exist_ok=True)
+ results_path = os.path.join(output_path, "results")
+ os.makedirs(results_path, exist_ok=True)
+
+ self.paths = {
+ "results": results_path,
+ "checkpoints": checkpoints_path,
+ "output_folder": output_path,
+ }
+
+ def load_dataset(self):
+ """
+ Load datasets, data samplers and data loaders
+ """
+ self.params["dataset_params"]["use_ddp"] = self.params["training_params"][
+ "use_ddp"
+ ]
+ self.params["dataset_params"]["batch_size"] = self.params["training_params"][
+ "batch_size"
+ ]
+ if "valid_batch_size" in self.params["training_params"]:
+ self.params["dataset_params"]["valid_batch_size"] = self.params[
+ "training_params"
+ ]["valid_batch_size"]
+ if "test_batch_size" in self.params["training_params"]:
+ self.params["dataset_params"]["test_batch_size"] = self.params[
+ "training_params"
+ ]["test_batch_size"]
+ self.params["dataset_params"]["num_gpu"] = self.params["training_params"][
+ "nb_gpu"
+ ]
+ self.params["dataset_params"]["worker_per_gpu"] = (
+ 4
+ if "worker_per_gpu" not in self.params["dataset_params"]
+ else self.params["dataset_params"]["worker_per_gpu"]
+ )
+ self.dataset = self.params["dataset_params"]["dataset_manager"](
+ self.params["dataset_params"]
+ )
+ self.dataset.load_datasets()
+ self.dataset.load_ddp_samplers()
+ self.dataset.load_dataloaders()
+
+ def init_hardware_config(self):
+ # Debug mode
+ if self.params["training_params"]["force_cpu"]:
+ self.params["training_params"]["use_ddp"] = False
+ self.params["training_params"]["use_amp"] = False
+ # Manage Distributed Data Parallel & GPU usage
+ self.manual_seed = (
+ 1111
+ if "manual_seed" not in self.params["training_params"].keys()
+ else self.params["training_params"]["manual_seed"]
+ )
+ self.ddp_config = {
+ "master": self.params["training_params"]["use_ddp"]
+ and self.params["training_params"]["ddp_rank"] == 0,
+ "address": "localhost"
+ if "ddp_addr" not in self.params["training_params"].keys()
+ else self.params["training_params"]["ddp_addr"],
+ "port": "11111"
+ if "ddp_port" not in self.params["training_params"].keys()
+ else self.params["training_params"]["ddp_port"],
+ "backend": "nccl"
+ if "ddp_backend" not in self.params["training_params"].keys()
+ else self.params["training_params"]["ddp_backend"],
+ "rank": self.params["training_params"]["ddp_rank"],
+ }
+ self.is_master = (
+ self.ddp_config["master"] or not self.params["training_params"]["use_ddp"]
+ )
+ if self.params["training_params"]["force_cpu"]:
+ self.device = "cpu"
+ else:
+ if self.params["training_params"]["use_ddp"]:
+ self.device = torch.device(self.ddp_config["rank"])
+ self.params["dataset_params"]["ddp_rank"] = self.ddp_config["rank"]
+ self.launch_ddp()
+ else:
+ self.device = torch.device(
+ "cuda:0" if torch.cuda.is_available() else "cpu"
+ )
+ self.params["model_params"]["device"] = self.device.type
+ # Print GPU info
+ # global
+ if (
+ self.params["training_params"]["use_ddp"] and self.ddp_config["master"]
+ ) or not self.params["training_params"]["use_ddp"]:
+ print("##################")
+ print("Available GPUS: {}".format(self.params["training_params"]["nb_gpu"]))
+ for i in range(self.params["training_params"]["nb_gpu"]):
+ print(
+ "Rank {}: {} {}".format(
+ i,
+ torch.cuda.get_device_name(i),
+ torch.cuda.get_device_properties(i),
+ )
+ )
+ print("##################")
+ # local
+ print("Local GPU:")
+ if self.device != "cpu":
+ print(
+ "Rank {}: {} {}".format(
+ self.params["training_params"]["ddp_rank"],
+ torch.cuda.get_device_name(),
+ torch.cuda.get_device_properties(self.device),
+ )
+ )
+ else:
+ print("WORKING ON CPU !\n")
+ print("##################")
+
+ def load_model(self, reset_optimizer=False, strict=True):
+ """
+ Load model weights from scratch or from checkpoints
+ """
+ # Instantiate Model
+ for model_name in self.params["model_params"]["models"].keys():
+ self.models[model_name] = self.params["model_params"]["models"][model_name](
+ self.params["model_params"]
+ )
+ self.models[model_name].to(self.device) # To GPU or CPU
+ # make the model compatible with Distributed Data Parallel if used
+ if self.params["training_params"]["use_ddp"]:
+ self.models[model_name] = DDP(
+ self.models[model_name], [self.ddp_config["rank"]]
+ )
+
+ # Handle curriculum dropout
+ if "dropout_scheduler" in self.params["model_params"]:
+ func = self.params["model_params"]["dropout_scheduler"]["function"]
+ T = self.params["model_params"]["dropout_scheduler"]["T"]
+ self.dropout_scheduler = DropoutScheduler(self.models, func, T)
+
+ self.scaler = GradScaler(enabled=self.params["training_params"]["use_amp"])
+
+ # Check if checkpoint exists
+ checkpoint = self.get_checkpoint()
+ if checkpoint is not None:
+ self.load_existing_model(checkpoint, strict=strict)
+ else:
+ self.init_new_model()
+
+ self.load_optimizers(checkpoint, reset_optimizer=reset_optimizer)
+
+ if self.is_master:
+ print("LOADED EPOCH: {}\n".format(self.latest_epoch), flush=True)
+
+ def get_checkpoint(self):
+ """
+ Seek if checkpoint exist, return None otherwise
+ """
+ if self.params["training_params"]["load_epoch"] in ("best", "last"):
+ for filename in os.listdir(self.paths["checkpoints"]):
+ if self.params["training_params"]["load_epoch"] in filename:
+ return torch.load(os.path.join(self.paths["checkpoints"], filename))
+ return None
+
+ def load_existing_model(self, checkpoint, strict=True):
+ """
+ Load information and weights from previous training
+ """
+ self.load_save_info(checkpoint)
+ self.latest_epoch = checkpoint["epoch"]
+ if "step" in checkpoint:
+ self.latest_step = checkpoint["step"]
+ self.best = checkpoint["best"]
+ if "scaler_state_dict" in checkpoint:
+ self.scaler.load_state_dict(checkpoint["scaler_state_dict"])
+ # Load model weights from past training
+ for model_name in self.models.keys():
+ self.models[model_name].load_state_dict(
+ checkpoint["{}_state_dict".format(model_name)], strict=strict
+ )
+
+ def init_new_model(self):
+ """
+ Initialize model
+ """
+ # Specific weights initialization if exists
+ for model_name in self.models.keys():
+ try:
+ self.models[model_name].init_weights()
+ except Exception:
+ pass
+
+ # Handle transfer learning instructions
+ if self.params["model_params"]["transfer_learning"]:
+ # Iterates over models
+ for model_name in self.params["model_params"]["transfer_learning"].keys():
+ state_dict_name, path, learnable, strict = self.params["model_params"][
+ "transfer_learning"
+ ][model_name]
+ # Loading pretrained weights file
+ checkpoint = torch.load(path)
+ try:
+ # Load pretrained weights for model
+ self.models[model_name].load_state_dict(
+ checkpoint["{}_state_dict".format(state_dict_name)],
+ strict=strict,
+ )
+ print(
+ "transferred weights for {}".format(state_dict_name), flush=True
+ )
+ except RuntimeError as e:
+ print(e, flush=True)
+ # if error, try to load each parts of the model (useful if only few layers are different)
+ for key in checkpoint[
+ "{}_state_dict".format(state_dict_name)
+ ].keys():
+ try:
+ # for pre-training of decision layer
+ if (
+ "end_conv" in key
+ and "transfered_charset" in self.params["model_params"]
+ ):
+ self.adapt_decision_layer_to_old_charset(
+ model_name, key, checkpoint, state_dict_name
+ )
+ else:
+ self.models[model_name].load_state_dict(
+ {
+ key: checkpoint[
+ "{}_state_dict".format(state_dict_name)
+ ][key]
+ },
+ strict=False,
+ )
+ except RuntimeError as e:
+ # exception when adding linebreak token from pretraining
+ print(e, flush=True)
+ # Set parameters no trainable
+ if not learnable:
+ self.set_model_learnable(self.models[model_name], False)
+
+ def adapt_decision_layer_to_old_charset(
+ self, model_name, key, checkpoint, state_dict_name
+ ):
+ """
+ Transfer learning of the decision learning in case of close charsets between pre-training and training
+ """
+ pretrained_chars = list()
+ weights = checkpoint["{}_state_dict".format(state_dict_name)][key]
+ new_size = list(weights.size())
+ new_size[0] = (
+ len(self.dataset.charset) + self.params["model_params"]["additional_tokens"]
+ )
+ new_weights = torch.zeros(new_size, device=weights.device, dtype=weights.dtype)
+ old_charset = (
+ checkpoint["charset"]
+ if "charset" in checkpoint
+ else self.params["model_params"]["old_charset"]
+ )
+ if "bias" not in key:
+ kaiming_uniform_(new_weights, nonlinearity="relu")
+ for i, c in enumerate(self.dataset.charset):
+ if c in old_charset:
+ new_weights[i] = weights[old_charset.index(c)]
+ pretrained_chars.append(c)
+ if (
+ "transfered_charset_last_is_ctc_blank" in self.params["model_params"]
+ and self.params["model_params"]["transfered_charset_last_is_ctc_blank"]
+ ):
+ new_weights[-1] = weights[-1]
+ pretrained_chars.append("<blank>")
+ checkpoint["{}_state_dict".format(state_dict_name)][key] = new_weights
+ self.models[model_name].load_state_dict(
+ {key: checkpoint["{}_state_dict".format(state_dict_name)][key]},
+ strict=False,
+ )
+ print(
+ "Pretrained chars for {} ({}): {}".format(
+ key, len(pretrained_chars), pretrained_chars
+ )
+ )
+
+ def load_optimizers(self, checkpoint, reset_optimizer=False):
+ """
+ Load the optimizer of each model
+ """
+ for model_name in self.models.keys():
+ new_params = dict()
+ if (
+ checkpoint
+ and "optimizer_named_params_{}".format(model_name) in checkpoint
+ ):
+ self.optimizers_named_params_by_group[model_name] = checkpoint[
+ "optimizer_named_params_{}".format(model_name)
+ ]
+ # for progressively growing models
+ for name, param in self.models[model_name].named_parameters():
+ existing = False
+ for gr in self.optimizers_named_params_by_group[model_name]:
+ if name in gr:
+ gr[name] = param
+ existing = True
+ break
+ if not existing:
+ new_params.update({name: param})
+ else:
+ self.optimizers_named_params_by_group[model_name] = [
+ dict(),
+ ]
+ self.optimizers_named_params_by_group[model_name][0].update(
+ self.models[model_name].named_parameters()
+ )
+
+ # Instantiate optimizer
+ self.reset_optimizer(model_name)
+
+ # Handle learning rate schedulers
+ if (
+ "lr_schedulers" in self.params["training_params"]
+ and self.params["training_params"]["lr_schedulers"]
+ ):
+ key = (
+ "all"
+ if "all" in self.params["training_params"]["lr_schedulers"]
+ else model_name
+ )
+ if key in self.params["training_params"]["lr_schedulers"]:
+ self.lr_schedulers[model_name] = self.params["training_params"][
+ "lr_schedulers"
+ ][key]["class"](
+ self.optimizers[model_name],
+ **self.params["training_params"]["lr_schedulers"][key]["args"]
+ )
+
+ # Load optimizer state from past training
+ if checkpoint and not reset_optimizer:
+ self.optimizers[model_name].load_state_dict(
+ checkpoint["optimizer_{}_state_dict".format(model_name)]
+ )
+ # Load optimizer scheduler config from past training if used
+ if (
+ "lr_schedulers" in self.params["training_params"]
+ and self.params["training_params"]["lr_schedulers"]
+ and "lr_scheduler_{}_state_dict".format(model_name)
+ in checkpoint.keys()
+ ):
+ self.lr_schedulers[model_name].load_state_dict(
+ checkpoint["lr_scheduler_{}_state_dict".format(model_name)]
+ )
+
+ # for progressively growing models, keeping learning rate
+ if checkpoint and new_params:
+ self.optimizers_named_params_by_group[model_name].append(new_params)
+ self.optimizers[model_name].add_param_group(
+ {"params": list(new_params.values())}
+ )
+
+ @staticmethod
+ def set_model_learnable(model, learnable=True):
+ for p in list(model.parameters()):
+ p.requires_grad = learnable
+
+ def save_model(self, epoch, name, keep_weights=False):
+ """
+ Save model weights and training info for curriculum learning or learning rate for instance
+ """
+ if not self.is_master:
+ return
+ to_del = []
+ for filename in os.listdir(self.paths["checkpoints"]):
+ if name in filename:
+ to_del.append(os.path.join(self.paths["checkpoints"], filename))
+ path = os.path.join(self.paths["checkpoints"], "{}_{}.pt".format(name, epoch))
+ content = {
+ "optimizers_named_params": self.optimizers_named_params_by_group,
+ "epoch": epoch,
+ "step": self.latest_step,
+ "scaler_state_dict": self.scaler.state_dict(),
+ "best": self.best,
+ "charset": self.dataset.charset,
+ }
+ for model_name in self.optimizers:
+ content["optimizer_{}_state_dict".format(model_name)] = self.optimizers[
+ model_name
+ ].state_dict()
+ for model_name in self.lr_schedulers:
+ content[
+ "lr_scheduler_{}_state_dict".format(model_name)
+ ] = self.lr_schedulers[model_name].state_dict()
+ content = self.add_save_info(content)
+ for model_name in self.models.keys():
+ content["{}_state_dict".format(model_name)] = self.models[
+ model_name
+ ].state_dict()
+ torch.save(content, path)
+ if not keep_weights:
+ for path_to_del in to_del:
+ if path_to_del != path:
+ os.remove(path_to_del)
+
+ def reset_optimizer(self, model_name):
+ """
+ Reset optimizer learning rate for given model
+ """
+ params = list(self.optimizers_named_params_by_group[model_name][0].values())
+ key = (
+ "all"
+ if "all" in self.params["training_params"]["optimizers"]
+ else model_name
+ )
+ self.optimizers[model_name] = self.params["training_params"]["optimizers"][key][
+ "class"
+ ](params, **self.params["training_params"]["optimizers"][key]["args"])
+ for i in range(1, len(self.optimizers_named_params_by_group[model_name])):
+ self.optimizers[model_name].add_param_group(
+ {
+ "params": list(
+ self.optimizers_named_params_by_group[model_name][i].values()
+ )
+ }
+ )
+
+ def save_params(self):
+ """
+ Output text file containing a summary of all hyperparameters chosen for the training
+ """
+
+ def compute_nb_params(module):
+ return sum([np.prod(p.size()) for p in list(module.parameters())])
+
+ def class_to_str_dict(my_dict):
+ for key in my_dict.keys():
+ if callable(my_dict[key]):
+ my_dict[key] = my_dict[key].__name__
+ elif isinstance(my_dict[key], np.ndarray):
+ my_dict[key] = my_dict[key].tolist()
+ elif isinstance(my_dict[key], dict):
+ my_dict[key] = class_to_str_dict(my_dict[key])
+ return my_dict
+
+ path = os.path.join(self.paths["results"], "params")
+ if os.path.isfile(path):
+ return
+ params = copy.deepcopy(self.params)
+ params = class_to_str_dict(params)
+ params["date"] = date.today().strftime("%d/%m/%Y")
+ total_params = 0
+ for model_name in self.models.keys():
+ current_params = compute_nb_params(self.models[model_name])
+ params["model_params"]["models"][model_name] = [
+ params["model_params"]["models"][model_name],
+ "{:,}".format(current_params),
+ ]
+ total_params += current_params
+ params["model_params"]["total_params"] = "{:,}".format(total_params)
+
+ params["hardware"] = dict()
+ if self.device != "cpu":
+ for i in range(self.params["training_params"]["nb_gpu"]):
+ params["hardware"][str(i)] = "{} {}".format(
+ torch.cuda.get_device_name(i), torch.cuda.get_device_properties(i)
+ )
+ else:
+ params["hardware"]["0"] = "CPU"
+ params["software"] = {
+ "python_version": sys.version,
+ "pytorch_version": torch.__version__,
+ "cuda_version": torch.version.cuda,
+ "cudnn_version": torch.backends.cudnn.version(),
+ }
+ with open(path, "w") as f:
+ json.dump(params, f, indent=4)
+
+ def backward_loss(self, loss, retain_graph=False):
+ self.scaler.scale(loss).backward(retain_graph=retain_graph)
+
+ def step_optimizers(self, increment_step=True, names=None):
+ for model_name in self.optimizers:
+ if names and model_name not in names:
+ continue
+ if (
+ "gradient_clipping" in self.params["training_params"]
+ and model_name
+ in self.params["training_params"]["gradient_clipping"]["models"]
+ ):
+ self.scaler.unscale_(self.optimizers[model_name])
+ torch.nn.utils.clip_grad_norm_(
+ self.models[model_name].parameters(),
+ self.params["training_params"]["gradient_clipping"]["max"],
+ )
+ self.scaler.step(self.optimizers[model_name])
+ self.scaler.update()
+ self.latest_step += 1
+
+ def zero_optimizers(self, set_to_none=True):
+ for model_name in self.optimizers:
+ self.zero_optimizer(model_name, set_to_none)
+
+ def zero_optimizer(self, model_name, set_to_none=True):
+ self.optimizers[model_name].zero_grad(set_to_none=set_to_none)
+
+ def train(self):
+ """
+ Main training loop
+ """
+ # init tensorboard file and output param summary file
+ if self.is_master:
+ self.writer = SummaryWriter(self.paths["results"])
+ self.save_params()
+ # init variables
+ self.begin_time = time()
+ focus_metric_name = self.params["training_params"]["focus_metric"]
+ nb_epochs = self.params["training_params"]["max_nb_epochs"]
+ interval_save_weights = self.params["training_params"]["interval_save_weights"]
+ metric_names = self.params["training_params"]["train_metrics"]
+
+ display_values = None
+ # init curriculum learning
+ if (
+ "curriculum_learning" in self.params["training_params"].keys()
+ and self.params["training_params"]["curriculum_learning"]
+ ):
+ self.init_curriculum()
+ # perform epochs
+ for num_epoch in range(self.latest_epoch + 1, nb_epochs):
+ self.dataset.train_dataset.training_info = {
+ "epoch": self.latest_epoch,
+ "step": self.latest_step,
+ }
+ self.phase = "train"
+ # Check maximum training time stop condition
+ if (
+ self.params["training_params"]["max_training_time"]
+ and time() - self.begin_time
+ > self.params["training_params"]["max_training_time"]
+ ):
+ break
+ # set models trainable
+ for model_name in self.models.keys():
+ self.models[model_name].train()
+ self.latest_epoch = num_epoch
+ if self.dataset.train_dataset.curriculum_config:
+ self.dataset.train_dataset.curriculum_config[
+ "epoch"
+ ] = self.latest_epoch
+ # init epoch metrics values
+ self.metric_manager["train"] = MetricManager(
+ metric_names=metric_names, dataset_name=self.dataset_name
+ )
+
+ with tqdm(total=len(self.dataset.train_loader.dataset)) as pbar:
+ pbar.set_description("EPOCH {}/{}".format(num_epoch, nb_epochs))
+ # iterates over mini-batch data
+ for ind_batch, batch_data in enumerate(self.dataset.train_loader):
+ self.latest_batch = ind_batch + 1
+ self.total_batch += 1
+ # train on batch data and compute metrics
+ batch_values = self.train_batch(batch_data, metric_names)
+ batch_metrics = self.metric_manager["train"].compute_metrics(
+ batch_values, metric_names
+ )
+ batch_metrics["names"] = batch_data["names"]
+ batch_metrics["ids"] = batch_data["ids"]
+ # Merge metrics if Distributed Data Parallel is used
+ if self.params["training_params"]["use_ddp"]:
+ batch_metrics = self.merge_ddp_metrics(batch_metrics)
+ # Update learning rate via scheduler if one is used
+ if self.params["training_params"]["lr_schedulers"]:
+ for model_name in self.models:
+ key = (
+ "all"
+ if "all"
+ in self.params["training_params"]["lr_schedulers"]
+ else model_name
+ )
+ if (
+ model_name in self.lr_schedulers
+ and ind_batch
+ % self.params["training_params"]["lr_schedulers"][key][
+ "step_interval"
+ ]
+ == 0
+ ):
+ self.lr_schedulers[model_name].step(
+ len(batch_metrics["names"])
+ )
+ if "lr" in metric_names:
+ self.writer.add_scalar(
+ "lr_{}".format(model_name),
+ self.lr_schedulers[model_name].lr,
+ self.lr_schedulers[model_name].step_num,
+ )
+ # Update dropout scheduler if used
+ if self.dropout_scheduler:
+ self.dropout_scheduler.step(len(batch_metrics["names"]))
+ self.dropout_scheduler.update_dropout_rate()
+
+ # Add batch metrics values to epoch metrics values
+ self.metric_manager["train"].update_metrics(batch_metrics)
+ display_values = self.metric_manager["train"].get_display_values()
+ pbar.set_postfix(values=str(display_values))
+ pbar.update(len(batch_data["names"]))
+
+ # log metrics in tensorboard file
+ if self.is_master:
+ for key in display_values.keys():
+ self.writer.add_scalar(
+ "{}_{}".format(
+ self.params["dataset_params"]["train"]["name"], key
+ ),
+ display_values[key],
+ num_epoch,
+ )
+ self.latest_train_metrics = display_values
+
+ # evaluate and compute metrics for valid sets
+ if (
+ self.params["training_params"]["eval_on_valid"]
+ and num_epoch % self.params["training_params"]["eval_on_valid_interval"]
+ == 0
+ ):
+ for valid_set_name in self.dataset.valid_loaders.keys():
+ # evaluate set and compute metrics
+ eval_values = self.evaluate(valid_set_name)
+ self.latest_valid_metrics = eval_values
+ # log valid metrics in tensorboard file
+ if self.is_master:
+ for key in eval_values.keys():
+ self.writer.add_scalar(
+ "{}_{}".format(valid_set_name, key),
+ eval_values[key],
+ num_epoch,
+ )
+ if valid_set_name == self.params["training_params"][
+ "set_name_focus_metric"
+ ] and (
+ self.best is None
+ or (
+ eval_values[focus_metric_name] <= self.best
+ and self.params["training_params"][
+ "expected_metric_value"
+ ]
+ == "low"
+ )
+ or (
+ eval_values[focus_metric_name] >= self.best
+ and self.params["training_params"][
+ "expected_metric_value"
+ ]
+ == "high"
+ )
+ ):
+ self.save_model(epoch=num_epoch, name="best")
+ self.best = eval_values[focus_metric_name]
+
+ # Handle curriculum learning update
+ if self.dataset.train_dataset.curriculum_config:
+ self.check_and_update_curriculum()
+
+ if (
+ "curriculum_model" in self.params["model_params"]
+ and self.params["model_params"]["curriculum_model"]
+ ):
+ self.update_curriculum_model()
+
+ # save model weights
+ if self.is_master:
+ self.save_model(epoch=num_epoch, name="last")
+ if interval_save_weights and num_epoch % interval_save_weights == 0:
+ self.save_model(epoch=num_epoch, name="weights", keep_weights=True)
+ self.writer.flush()
+
+ def evaluate(self, set_name, **kwargs):
+ """
+ Main loop for validation
+ """
+ self.phase = "eval"
+ loader = self.dataset.valid_loaders[set_name]
+ # Set models in eval mode
+ for model_name in self.models.keys():
+ self.models[model_name].eval()
+ metric_names = self.params["training_params"]["eval_metrics"]
+ display_values = None
+
+ # initialize epoch metrics
+ self.metric_manager[set_name] = MetricManager(
+ metric_names, dataset_name=self.dataset_name
+ )
+ with tqdm(total=len(loader.dataset)) as pbar:
+ pbar.set_description("Evaluation E{}".format(self.latest_epoch))
+ with torch.no_grad():
+ # iterate over batch data
+ for ind_batch, batch_data in enumerate(loader):
+ self.latest_batch = ind_batch + 1
+ # eval batch data and compute metrics
+ batch_values = self.evaluate_batch(batch_data, metric_names)
+ batch_metrics = self.metric_manager[set_name].compute_metrics(
+ batch_values, metric_names
+ )
+ batch_metrics["names"] = batch_data["names"]
+ batch_metrics["ids"] = batch_data["ids"]
+ # merge metrics values if Distributed Data Parallel is used
+ if self.params["training_params"]["use_ddp"]:
+ batch_metrics = self.merge_ddp_metrics(batch_metrics)
+
+ # add batch metrics to epoch metrics
+ self.metric_manager[set_name].update_metrics(batch_metrics)
+ display_values = self.metric_manager[set_name].get_display_values()
+
+ pbar.set_postfix(values=str(display_values))
+ pbar.update(len(batch_data["names"]))
+ if "cer_by_nb_cols" in metric_names:
+ self.log_cer_by_nb_cols(set_name)
+ return display_values
+
+ def predict(self, custom_name, sets_list, metric_names, output=False):
+ """
+ Main loop for evaluation
+ """
+ self.phase = "predict"
+ metric_names = metric_names.copy()
+ self.dataset.generate_test_loader(custom_name, sets_list)
+ loader = self.dataset.test_loaders[custom_name]
+ # Set models in eval mode
+ for model_name in self.models.keys():
+ self.models[model_name].eval()
+
+ # initialize epoch metrics
+ self.metric_manager[custom_name] = MetricManager(
+ metric_names, self.dataset_name
+ )
+
+ with tqdm(total=len(loader.dataset)) as pbar:
+ pbar.set_description("Prediction")
+ with torch.no_grad():
+ for ind_batch, batch_data in enumerate(loader):
+ # iterates over batch data
+ self.latest_batch = ind_batch + 1
+ # eval batch data and compute metrics
+ batch_values = self.evaluate_batch(batch_data, metric_names)
+ batch_metrics = self.metric_manager[custom_name].compute_metrics(
+ batch_values, metric_names
+ )
+ batch_metrics["names"] = batch_data["names"]
+ batch_metrics["ids"] = batch_data["ids"]
+ # merge batch metrics if Distributed Data Parallel is used
+ if self.params["training_params"]["use_ddp"]:
+ batch_metrics = self.merge_ddp_metrics(batch_metrics)
+
+ # add batch metrics to epoch metrics
+ self.metric_manager[custom_name].update_metrics(batch_metrics)
+ display_values = self.metric_manager[
+ custom_name
+ ].get_display_values()
+
+ pbar.set_postfix(values=str(display_values))
+ pbar.update(len(batch_data["names"]))
+
+ self.dataset.remove_test_dataset(custom_name)
+ # output metrics values if requested
+ if output:
+ if "pred" in metric_names:
+ self.output_pred(custom_name)
+ metrics = self.metric_manager[custom_name].get_display_values(output=True)
+ path = os.path.join(
+ self.paths["results"],
+ "predict_{}_{}.txt".format(custom_name, self.latest_epoch),
+ )
+ with open(path, "w") as f:
+ for metric_name in metrics.keys():
+ f.write("{}: {}\n".format(metric_name, metrics[metric_name]))
+
+ def output_pred(self, name):
+ path = os.path.join(
+ self.paths["results"], "pred_{}_{}.txt".format(name, self.latest_epoch)
+ )
+ pred = "\n".join(self.metric_manager[name].get("pred"))
+ with open(path, "w") as f:
+ f.write(pred)
+
+ def launch_ddp(self):
+ """
+ Initialize Distributed Data Parallel system
+ """
+ mp.set_start_method("fork", force=True)
+ os.environ["MASTER_ADDR"] = self.ddp_config["address"]
+ os.environ["MASTER_PORT"] = str(self.ddp_config["port"])
+ dist.init_process_group(
+ self.ddp_config["backend"],
+ rank=self.ddp_config["rank"],
+ world_size=self.params["training_params"]["nb_gpu"],
+ )
+ torch.cuda.set_device(self.ddp_config["rank"])
+ random.seed(self.manual_seed)
+ np.random.seed(self.manual_seed)
+ torch.manual_seed(self.manual_seed)
+ torch.cuda.manual_seed(self.manual_seed)
+
+ def merge_ddp_metrics(self, metrics):
+ """
+ Merge metrics when Distributed Data Parallel is used
+ """
+ for metric_name in metrics.keys():
+ if metric_name in [
+ "edit_words",
+ "nb_words",
+ "edit_chars",
+ "nb_chars",
+ "edit_chars_force_len",
+ "edit_chars_curr",
+ "nb_chars_curr",
+ "ids",
+ ]:
+ metrics[metric_name] = self.cat_ddp_metric(metrics[metric_name])
+ elif metric_name in [
+ "nb_samples",
+ "loss",
+ "loss_ce",
+ "loss_ctc",
+ "loss_ce_end",
+ ]:
+ metrics[metric_name] = self.sum_ddp_metric(
+ metrics[metric_name], average=False
+ )
+ return metrics
+
+ def sum_ddp_metric(self, metric, average=False):
+ """
+ Sum metrics for Distributed Data Parallel
+ """
+ sum = torch.tensor(metric[0]).to(self.device)
+ dist.all_reduce(sum, op=dist.ReduceOp.SUM)
+ if average:
+ sum.true_divide(dist.get_world_size())
+ return [
+ sum.item(),
+ ]
+
+ def cat_ddp_metric(self, metric):
+ """
+ Concatenate metrics for Distributed Data Parallel
+ """
+ tensor = torch.tensor(metric).unsqueeze(0).to(self.device)
+ res = [
+ torch.zeros(tensor.size()).long().to(self.device)
+ for _ in range(dist.get_world_size())
+ ]
+ dist.all_gather(res, tensor)
+ return list(torch.cat(res, dim=0).flatten().cpu().numpy())
+
+ @staticmethod
+ def cleanup():
+ dist.destroy_process_group()
+
+ def train_batch(self, batch_data, metric_names):
+ raise NotImplementedError
+
+ def evaluate_batch(self, batch_data, metric_names):
+ raise NotImplementedError
+
+ def init_curriculum(self):
+ raise NotImplementedError
+
+ def update_curriculum(self):
+ raise NotImplementedError
+
+ def add_checkpoint_info(self, load_mode="last", **kwargs):
+ for filename in os.listdir(self.paths["checkpoints"]):
+ if load_mode in filename:
+ checkpoint_path = os.path.join(self.paths["checkpoints"], filename)
+ checkpoint = torch.load(checkpoint_path)
+ for key in kwargs.keys():
+ checkpoint[key] = kwargs[key]
+ torch.save(checkpoint, checkpoint_path)
+ return
+ self.save_model(self.latest_epoch, "last")
+
+ def load_save_info(self, info_dict):
+ """
+ Load curriculum info from saved model info
+ """
+ if "curriculum_config" in info_dict.keys():
+ self.dataset.train_dataset.curriculum_config = info_dict[
+ "curriculum_config"
+ ]
+
+ def add_save_info(self, info_dict):
+ """
+ Add curriculum info to model info to be saved
+ """
+ info_dict["curriculum_config"] = self.dataset.train_dataset.curriculum_config
+ return info_dict
+
+
+class OCRManager(GenericTrainingManager):
+ def __init__(self, params):
+ super(OCRManager, self).__init__(params)
+ self.params["model_params"]["vocab_size"] = len(self.dataset.charset)
+
+ def generate_syn_line_dataset(self, name):
+ """
+ Generate synthetic line dataset from currently loaded dataset
+ """
+ dataset_name = list(self.params["dataset_params"]["datasets"].keys())[0]
+ path = os.path.join(
+ os.path.dirname(self.params["dataset_params"]["datasets"][dataset_name]),
+ name,
+ )
+ os.makedirs(path, exist_ok=True)
+ charset = set()
+ dataset = None
+ gt = {"train": dict(), "valid": dict(), "test": dict()}
+ for set_name in ["train", "valid", "test"]:
+ set_path = os.path.join(path, set_name)
+ os.makedirs(set_path, exist_ok=True)
+ if set_name == "train":
+ dataset = self.dataset.train_dataset
+ elif set_name == "valid":
+ dataset = self.dataset.valid_datasets["{}-valid".format(dataset_name)]
+ elif set_name == "test":
+ self.dataset.generate_test_loader(
+ "{}-test".format(dataset_name),
+ [
+ (dataset_name, "test"),
+ ],
+ )
+ dataset = self.dataset.test_datasets["{}-test".format(dataset_name)]
+
+ samples = list()
+ for sample in dataset.samples:
+ for line_label in sample["label"].split("\n"):
+ for chunk in [
+ line_label[i : i + 100] for i in range(0, len(line_label), 100)
+ ]:
+ charset = charset.union(set(chunk))
+ if len(chunk) > 0:
+ samples.append(
+ {
+ "path": sample["path"],
+ "label": chunk,
+ "nb_cols": 1,
+ }
+ )
+
+ for i, sample in enumerate(samples):
+ ext = sample["path"].split(".")[-1]
+ img_name = "{}_{}.{}".format(set_name, i, ext)
+ img_path = os.path.join(set_path, img_name)
+
+ img = dataset.generate_typed_text_line_image(sample["label"])
+ Image.fromarray(img).save(img_path)
+ gt[set_name][img_name] = {
+ "text": sample["label"],
+ "nb_cols": sample["nb_cols"] if "nb_cols" in sample else 1,
+ }
+ if "line_label" in sample:
+ gt[set_name][img_name]["lines"] = sample["line_label"]
+
+ with open(os.path.join(path, "labels.pkl"), "wb") as f:
+ pickle.dump(
+ {
+ "ground_truth": gt,
+ "charset": sorted(list(charset)),
+ },
+ f,
+ )
+
+
+class Manager(OCRManager):
+ def __init__(self, params):
+ super(Manager, self).__init__(params)
+
+ def load_save_info(self, info_dict):
+ if "curriculum_config" in info_dict.keys():
+ if self.dataset.train_dataset is not None:
+ self.dataset.train_dataset.curriculum_config = info_dict[
+ "curriculum_config"
+ ]
+
+ def add_save_info(self, info_dict):
+ info_dict["curriculum_config"] = self.dataset.train_dataset.curriculum_config
+ return info_dict
+
+ def apply_teacher_forcing(self, y, y_len, error_rate):
+ y_error = y.clone()
+ for b in range(len(y_len)):
+ for i in range(1, y_len[b]):
+ if (
+ np.random.rand() < error_rate
+ and y[b][i] != self.dataset.tokens["pad"]
+ ):
+ y_error[b][i] = np.random.randint(0, len(self.dataset.charset) + 2)
+ return y_error, y_len
+
+ def train_batch(self, batch_data, metric_names):
+ loss_func = CrossEntropyLoss(ignore_index=self.dataset.tokens["pad"])
+
+ sum_loss = 0
+ x = batch_data["imgs"].to(self.device)
+ y = batch_data["labels"].to(self.device)
+ reduced_size = [s[:2] for s in batch_data["imgs_reduced_shape"]]
+ y_len = batch_data["labels_len"]
+
+ # add errors in teacher forcing
+ if (
+ "teacher_forcing_error_rate" in self.params["training_params"]
+ and self.params["training_params"]["teacher_forcing_error_rate"] is not None
+ ):
+ error_rate = self.params["training_params"]["teacher_forcing_error_rate"]
+ simulated_y_pred, y_len = self.apply_teacher_forcing(y, y_len, error_rate)
+ elif "teacher_forcing_scheduler" in self.params["training_params"]:
+ error_rate = (
+ self.params["training_params"]["teacher_forcing_scheduler"][
+ "min_error_rate"
+ ]
+ + min(
+ self.latest_step,
+ self.params["training_params"]["teacher_forcing_scheduler"][
+ "total_num_steps"
+ ],
+ )
+ * (
+ self.params["training_params"]["teacher_forcing_scheduler"][
+ "max_error_rate"
+ ]
+ - self.params["training_params"]["teacher_forcing_scheduler"][
+ "min_error_rate"
+ ]
+ )
+ / self.params["training_params"]["teacher_forcing_scheduler"][
+ "total_num_steps"
+ ]
+ )
+ simulated_y_pred, y_len = self.apply_teacher_forcing(y, y_len, error_rate)
+ else:
+ simulated_y_pred = y
+
+ with autocast(enabled=self.params["training_params"]["use_amp"]):
+ hidden_predict = None
+ cache = None
+
+ raw_features = self.models["encoder"](x)
+ features_size = raw_features.size()
+ b, c, h, w = features_size
+
+ pos_features = self.models["decoder"].features_updater.get_pos_features(
+ raw_features
+ )
+ features = torch.flatten(pos_features, start_dim=2, end_dim=3).permute(
+ 2, 0, 1
+ )
+ enhanced_features = pos_features
+ enhanced_features = torch.flatten(
+ enhanced_features, start_dim=2, end_dim=3
+ ).permute(2, 0, 1)
+ output, pred, hidden_predict, cache, weights = self.models["decoder"](
+ features,
+ enhanced_features,
+ simulated_y_pred[:, :-1],
+ reduced_size,
+ [max(y_len) for _ in range(b)],
+ features_size,
+ start=0,
+ hidden_predict=hidden_predict,
+ cache=cache,
+ keep_all_weights=True,
+ )
+
+ loss_ce = loss_func(pred, y[:, 1:])
+ sum_loss += loss_ce
+ with autocast(enabled=False):
+ self.backward_loss(sum_loss)
+ self.step_optimizers()
+ self.zero_optimizers()
+ predicted_tokens = torch.argmax(pred, dim=1).detach().cpu().numpy()
+ predicted_tokens = [predicted_tokens[i, : y_len[i]] for i in range(b)]
+ str_x = [
+ LM_ind_to_str(self.dataset.charset, t, oov_symbol="")
+ for t in predicted_tokens
+ ]
+
+ values = {
+ "nb_samples": b,
+ "str_y": batch_data["raw_labels"],
+ "str_x": str_x,
+ "loss": sum_loss.item(),
+ "loss_ce": loss_ce.item(),
+ "syn_max_lines": self.dataset.train_dataset.get_syn_max_lines()
+ if self.params["dataset_params"]["config"]["synthetic_data"]
+ else 0,
+ }
+
+ return values
+
+ def evaluate_batch(self, batch_data, metric_names):
+ x = batch_data["imgs"].to(self.device)
+ reduced_size = [s[:2] for s in batch_data["imgs_reduced_shape"]]
+
+ max_chars = self.params["training_params"]["max_char_prediction"]
+
+ start_time = time()
+ with autocast(enabled=self.params["training_params"]["use_amp"]):
+ b = x.size(0)
+ reached_end = torch.zeros((b,), dtype=torch.bool, device=self.device)
+ prediction_len = torch.zeros((b,), dtype=torch.int, device=self.device)
+ predicted_tokens = (
+ torch.ones((b, 1), dtype=torch.long, device=self.device)
+ * self.dataset.tokens["start"]
+ )
+ predicted_tokens_len = torch.ones((b,), dtype=torch.int, device=self.device)
+
+ whole_output = list()
+ confidence_scores = list()
+ cache = None
+ hidden_predict = None
+ if b > 1:
+ features_list = list()
+ for i in range(b):
+ pos = batch_data["imgs_position"]
+ features_list.append(
+ self.models["encoder"](
+ x[
+ i : i + 1,
+ :,
+ pos[i][0][0] : pos[i][0][1],
+ pos[i][1][0] : pos[i][1][1],
+ ]
+ )
+ )
+ max_height = max([f.size(2) for f in features_list])
+ max_width = max([f.size(3) for f in features_list])
+ features = torch.zeros(
+ (b, features_list[0].size(1), max_height, max_width),
+ device=self.device,
+ dtype=features_list[0].dtype,
+ )
+ for i in range(b):
+ features[
+ i, :, : features_list[i].size(2), : features_list[i].size(3)
+ ] = features_list[i]
+ else:
+ features = self.models["encoder"](x)
+ features_size = features.size()
+ coverage_vector = torch.zeros(
+ (features.size(0), 1, features.size(2), features.size(3)),
+ device=self.device,
+ )
+ pos_features = self.models["decoder"].features_updater.get_pos_features(
+ features
+ )
+ features = torch.flatten(pos_features, start_dim=2, end_dim=3).permute(
+ 2, 0, 1
+ )
+ enhanced_features = pos_features
+ enhanced_features = torch.flatten(
+ enhanced_features, start_dim=2, end_dim=3
+ ).permute(2, 0, 1)
+
+ for i in range(0, max_chars):
+ output, pred, hidden_predict, cache, weights = self.models["decoder"](
+ features,
+ enhanced_features,
+ predicted_tokens,
+ reduced_size,
+ predicted_tokens_len,
+ features_size,
+ start=0,
+ hidden_predict=hidden_predict,
+ cache=cache,
+ num_pred=1,
+ )
+ whole_output.append(output)
+ confidence_scores.append(
+ torch.max(torch.softmax(pred[:, :], dim=1), dim=1).values
+ )
+ coverage_vector = torch.clamp(coverage_vector + weights, 0, 1)
+ predicted_tokens = torch.cat(
+ [
+ predicted_tokens,
+ torch.argmax(pred[:, :, -1], dim=1, keepdim=True),
+ ],
+ dim=1,
+ )
+ reached_end = torch.logical_or(
+ reached_end,
+ torch.eq(predicted_tokens[:, -1], self.dataset.tokens["end"]),
+ )
+ predicted_tokens_len += 1
+
+ prediction_len[reached_end is False] = i + 1
+ if torch.all(reached_end):
+ break
+
+ confidence_scores = (
+ torch.cat(confidence_scores, dim=1).cpu().detach().numpy()
+ )
+ predicted_tokens = predicted_tokens[:, 1:]
+ prediction_len[torch.eq(reached_end, False)] = max_chars - 1
+ predicted_tokens = [
+ predicted_tokens[i, : prediction_len[i]] for i in range(b)
+ ]
+ confidence_scores = [
+ confidence_scores[i, : prediction_len[i]].tolist() for i in range(b)
+ ]
+ str_x = [
+ LM_ind_to_str(self.dataset.charset, t, oov_symbol="")
+ for t in predicted_tokens
+ ]
+
+ process_time = time() - start_time
+
+ values = {
+ "nb_samples": b,
+ "str_y": batch_data["raw_labels"],
+ "str_x": str_x,
+ "confidence_score": confidence_scores,
+ "time": process_time,
+ }
+ return values
diff --git a/dan/manager/utils.py b/dan/manager/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..c95ca7ca81a65f10e030f76286d3c71128d1137c
--- /dev/null
+++ b/dan/manager/utils.py
@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+import os
+import pickle
+
+from PIL import Image
+
+from dan.manager.training import GenericTrainingManager
+
+
+class OCRManager(GenericTrainingManager):
+ def __init__(self, params):
+ super(OCRManager, self).__init__(params)
+ self.params["model_params"]["vocab_size"] = len(self.dataset.charset)
+
+ def generate_syn_line_dataset(self, name):
+ """
+ Generate synthetic line dataset from currently loaded dataset
+ """
+ dataset_name = list(self.params["dataset_params"]["datasets"].keys())[0]
+ path = os.path.join(
+ os.path.dirname(self.params["dataset_params"]["datasets"][dataset_name]),
+ name,
+ )
+ os.makedirs(path, exist_ok=True)
+ charset = set()
+ dataset = None
+ gt = {"train": dict(), "valid": dict(), "test": dict()}
+ for set_name in ["train", "valid", "test"]:
+ set_path = os.path.join(path, set_name)
+ os.makedirs(set_path, exist_ok=True)
+ if set_name == "train":
+ dataset = self.dataset.train_dataset
+ elif set_name == "valid":
+ dataset = self.dataset.valid_datasets["{}-valid".format(dataset_name)]
+ elif set_name == "test":
+ self.dataset.generate_test_loader(
+ "{}-test".format(dataset_name),
+ [
+ (dataset_name, "test"),
+ ],
+ )
+ dataset = self.dataset.test_datasets["{}-test".format(dataset_name)]
+
+ samples = list()
+ for sample in dataset.samples:
+ for line_label in sample["label"].split("\n"):
+ for chunk in [
+ line_label[i : i + 100] for i in range(0, len(line_label), 100)
+ ]:
+ charset = charset.union(set(chunk))
+ if len(chunk) > 0:
+ samples.append(
+ {
+ "path": sample["path"],
+ "label": chunk,
+ "nb_cols": 1,
+ }
+ )
+
+ for i, sample in enumerate(samples):
+ ext = sample["path"].split(".")[-1]
+ img_name = "{}_{}.{}".format(set_name, i, ext)
+ img_path = os.path.join(set_path, img_name)
+
+ img = dataset.generate_typed_text_line_image(sample["label"])
+ Image.fromarray(img).save(img_path)
+ gt[set_name][img_name] = {
+ "text": sample["label"],
+ "nb_cols": sample["nb_cols"] if "nb_cols" in sample else 1,
+ }
+ if "line_label" in sample:
+ gt[set_name][img_name]["lines"] = sample["line_label"]
+
+ with open(os.path.join(path, "labels.pkl"), "wb") as f:
+ pickle.dump(
+ {
+ "ground_truth": gt,
+ "charset": sorted(list(charset)),
+ },
+ f,
+ )
diff --git a/dan/models.py b/dan/models.py
index 225d004ceb9c342eb0b74805601969a8108df9bf..6057caddb1e96bcb6a6d0c1e2e38cb637e2757a0 100644
--- a/dan/models.py
+++ b/dan/models.py
@@ -1,39 +1,4 @@
# -*- coding: utf-8 -*-
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in XXX whose purpose is XXX.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
-
-
import random
from torch.nn import (
diff --git a/dan/ocr/__init__.py b/dan/ocr/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/dan/ocr/document/__init__.py b/dan/ocr/document/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/dan/ocr/document/train.py b/dan/ocr/document/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..1f8216a98af40e16b935e3dd81c9c7cc9a706d7e
--- /dev/null
+++ b/dan/ocr/document/train.py
@@ -0,0 +1,234 @@
+# -*- coding: utf-8 -*-
+import random
+
+import numpy as np
+import torch
+import torch.multiprocessing as mp
+from torch.optim import Adam
+
+from dan.decoder import GlobalHTADecoder
+from dan.manager.ocr import OCRDataset, OCRDatasetManager
+from dan.manager.training import Manager
+from dan.models import FCN_Encoder
+from dan.schedulers import exponential_dropout_scheduler
+from dan.transforms import aug_config
+
+
+def add_document_parser(subcommands) -> None:
+ parser = subcommands.add_parser(
+ "document",
+ description=__doc__,
+ )
+ parser.set_defaults(func=run)
+
+
+def train_and_test(rank, params):
+ torch.manual_seed(0)
+ torch.cuda.manual_seed(0)
+ np.random.seed(0)
+ random.seed(0)
+ torch.backends.cudnn.benchmark = False
+ torch.backends.cudnn.deterministic = True
+
+ params["training_params"]["ddp_rank"] = rank
+ model = Manager(params)
+ model.load_model()
+
+ model.train()
+
+ # load weights giving best CER on valid set
+ model.params["training_params"]["load_epoch"] = "best"
+ model.load_model()
+
+ metrics = ["cer", "wer", "time", "map_cer", "loer"]
+ for dataset_name in params["dataset_params"]["datasets"].keys():
+ for set_name in ["test", "valid", "train"]:
+ model.predict(
+ "{}-{}".format(dataset_name, set_name),
+ [
+ (dataset_name, set_name),
+ ],
+ metrics,
+ output=True,
+ )
+
+
+def run():
+ dataset_name = "simara"
+ dataset_level = "page"
+ dataset_variant = "_sem"
+
+ params = {
+ "dataset_params": {
+ "dataset_manager": OCRDatasetManager,
+ "dataset_class": OCRDataset,
+ "datasets": {
+ dataset_name: "../../../Datasets/formatted/{}_{}{}".format(
+ dataset_name, dataset_level, dataset_variant
+ ),
+ },
+ "train": {
+ "name": "{}-train".format(dataset_name),
+ "datasets": [
+ (dataset_name, "train"),
+ ],
+ },
+ "valid": {
+ "{}-valid".format(dataset_name): [
+ (dataset_name, "valid"),
+ ],
+ },
+ "config": {
+ "load_in_memory": True, # Load all images in CPU memory
+ "worker_per_gpu": 4, # Num of parallel processes per gpu for data loading
+ "width_divisor": 8, # Image width will be divided by 8
+ "height_divisor": 32, # Image height will be divided by 32
+ "padding_value": 0, # Image padding value
+ "padding_token": None, # Label padding value
+ "charset_mode": "seq2seq", # add end-of-transcription and start-of-transcription tokens to charset
+ "constraints": [
+ "add_eot",
+ "add_sot",
+ ], # add end-of-transcription and start-of-transcription tokens in labels
+ "normalize": True, # Normalize with mean and variance of training dataset
+ "preprocessings": [
+ {
+ "type": "to_RGB",
+ # if grayscaled image, produce RGB one (3 channels with same value) otherwise do nothing
+ },
+ ],
+ "augmentation": aug_config(0.9, 0.1),
+ "synthetic_data": None,
+ # "synthetic_data": {
+ # "init_proba": 0.9, # begin proba to generate synthetic document
+ # "end_proba": 0.2, # end proba to generate synthetic document
+ # "num_steps_proba": 200000, # linearly decrease the percent of synthetic document from 90% to 20% through 200000 samples
+ # "proba_scheduler_function": linear_scheduler, # decrease proba rate linearly
+ # "start_scheduler_at_max_line": True, # start decreasing proba only after curriculum reach max number of lines
+ # "dataset_level": dataset_level,
+ # "curriculum": True, # use curriculum learning (slowly increase number of lines per synthetic samples)
+ # "crop_curriculum": True, # during curriculum learning, crop images under the last text line
+ # "curr_start": 0, # start curriculum at iteration
+ # "curr_step": 10000, # interval to increase the number of lines for curriculum learning
+ # "min_nb_lines": 1, # initial number of lines for curriculum learning
+ # "max_nb_lines": max_nb_lines[dataset_name], # maximum number of lines for curriculum learning
+ # "padding_value": 255,
+ # # config for synthetic line generation
+ # "config": {
+ # "background_color_default": (255, 255, 255),
+ # "background_color_eps": 15,
+ # "text_color_default": (0, 0, 0),
+ # "text_color_eps": 15,
+ # "font_size_min": 35,
+ # "font_size_max": 45,
+ # "color_mode": "RGB",
+ # "padding_left_ratio_min": 0.00,
+ # "padding_left_ratio_max": 0.05,
+ # "padding_right_ratio_min": 0.02,
+ # "padding_right_ratio_max": 0.2,
+ # "padding_top_ratio_min": 0.02,
+ # "padding_top_ratio_max": 0.1,
+ # "padding_bottom_ratio_min": 0.02,
+ # "padding_bottom_ratio_max": 0.1,
+ # },
+ # }
+ },
+ },
+ "model_params": {
+ "models": {
+ "encoder": FCN_Encoder,
+ "decoder": GlobalHTADecoder,
+ },
+ # "transfer_learning": None,
+ "transfer_learning": {
+ # model_name: [state_dict_name, checkpoint_path, learnable, strict]
+ "encoder": ["encoder", "dan_rimes_page.pt", True, True],
+ "decoder": ["decoder", "dan_rimes_page.pt", True, False],
+ },
+ "transfered_charset": True, # Transfer learning of the decision layer based on charset of the line HTR model
+ "additional_tokens": 1, # for decision layer = [<eot>, ], only for transferred charset
+ "input_channels": 3, # number of channels of input image
+ "dropout": 0.5, # dropout rate for encoder
+ "enc_dim": 256, # dimension of extracted features
+ "nb_layers": 5, # encoder
+ "h_max": 500, # maximum height for encoder output (for 2D positional embedding)
+ "w_max": 1000, # maximum width for encoder output (for 2D positional embedding)
+ "l_max": 15000, # max predicted sequence (for 1D positional embedding)
+ "dec_num_layers": 8, # number of transformer decoder layers
+ "dec_num_heads": 4, # number of heads in transformer decoder layers
+ "dec_res_dropout": 0.1, # dropout in transformer decoder layers
+ "dec_pred_dropout": 0.1, # dropout rate before decision layer
+ "dec_att_dropout": 0.1, # dropout rate in multi head attention
+ "dec_dim_feedforward": 256, # number of dimension for feedforward layer in transformer decoder layers
+ "use_2d_pe": True, # use 2D positional embedding
+ "use_1d_pe": True, # use 1D positional embedding
+ "use_lstm": False,
+ "attention_win": 100, # length of attention window
+ # Curriculum dropout
+ "dropout_scheduler": {
+ "function": exponential_dropout_scheduler,
+ "T": 5e4,
+ },
+ },
+ "training_params": {
+ "output_folder": "dan_simara_page", # folder name for checkpoint and results
+ "max_nb_epochs": 50000, # maximum number of epochs before to stop
+ "max_training_time": 3600
+ * 24
+ * 1.9, # maximum time before to stop (in seconds)
+ "load_epoch": "last", # ["best", "last"]: last to continue training, best to evaluate
+ "interval_save_weights": None, # None: keep best and last only
+ "batch_size": 2, # mini-batch size for training
+ "valid_batch_size": 4, # mini-batch size for valdiation
+ "use_ddp": False, # Use DistributedDataParallel
+ "ddp_port": "20027",
+ "use_amp": True, # Enable automatic mix-precision
+ "nb_gpu": torch.cuda.device_count(),
+ "optimizers": {
+ "all": {
+ "class": Adam,
+ "args": {
+ "lr": 0.0001,
+ "amsgrad": False,
+ },
+ },
+ },
+ "lr_schedulers": None, # Learning rate schedulers
+ "eval_on_valid": True, # Whether to eval and logs metrics on validation set during training or not
+ "eval_on_valid_interval": 5, # Interval (in epochs) to evaluate during training
+ "focus_metric": "cer", # Metrics to focus on to determine best epoch
+ "expected_metric_value": "low", # ["high", "low"] What is best for the focus metric value
+ "set_name_focus_metric": "{}-valid".format(
+ dataset_name
+ ), # Which dataset to focus on to select best weights
+ "train_metrics": [
+ "loss_ce",
+ "cer",
+ "wer",
+ "syn_max_lines",
+ ], # Metrics name for training
+ "eval_metrics": [
+ "cer",
+ "wer",
+ "map_cer",
+ ], # Metrics name for evaluation on validation set during training
+ "force_cpu": False, # True for debug purposes
+ "max_char_prediction": 1000, # max number of token prediction
+ # Keep teacher forcing rate to 20% during whole training
+ "teacher_forcing_scheduler": {
+ "min_error_rate": 0.2,
+ "max_error_rate": 0.2,
+ "total_num_steps": 5e4,
+ },
+ },
+ }
+
+ if (
+ params["training_params"]["use_ddp"]
+ and not params["training_params"]["force_cpu"]
+ ):
+ mp.spawn(
+ train_and_test, args=(params,), nprocs=params["training_params"]["nb_gpu"]
+ )
+ else:
+ train_and_test(0, params)
diff --git a/dan/ocr/line/__init__.py b/dan/ocr/line/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/dan/ocr/line/generate_synthetic.py b/dan/ocr/line/generate_synthetic.py
new file mode 100644
index 0000000000000000000000000000000000000000..435e19eddfe2e2f5f9e7bb01cc2a8150c1fca425
--- /dev/null
+++ b/dan/ocr/line/generate_synthetic.py
@@ -0,0 +1,165 @@
+# -*- coding: utf-8 -*-
+import random
+
+import numpy as np
+import torch
+import torch.multiprocessing as mp
+from torch.optim import Adam
+
+from dan.manager.ocr import OCRDataset, OCRDatasetManager
+from dan.models import FCN_Encoder
+from dan.ocr.line.model_utils import Decoder
+from dan.ocr.line.utils import TrainerLineCTC
+from dan.schedulers import exponential_dropout_scheduler, exponential_scheduler
+from dan.transforms import line_aug_config
+
+
+def add_generate_parser(subcommands) -> None:
+ parser = subcommands.add_parser(
+ "generate",
+ description=__doc__,
+ )
+ parser.set_defaults(func=run)
+
+
+def train_and_test(rank, params):
+ torch.manual_seed(0)
+ torch.cuda.manual_seed(0)
+ np.random.seed(0)
+ random.seed(0)
+ torch.backends.cudnn.benchmark = False
+ torch.backends.cudnn.deterministic = True
+
+ params["training_params"]["ddp_rank"] = rank
+ model = TrainerLineCTC(params)
+
+ model.generate_syn_line_dataset(
+ "READ_2016_syn_line"
+ ) # ["RIMES_syn_line", "READ_2016_syn_line"]
+
+
+def run():
+ dataset_name = "READ_2016"
+ dataset_level = "page"
+ params = {
+ "dataset_params": {
+ "dataset_manager": OCRDatasetManager,
+ "dataset_class": OCRDataset,
+ "datasets": {
+ dataset_name: "../../../Datasets/formatted/{}_{}".format(
+ dataset_name, dataset_level
+ ),
+ },
+ "train": {
+ "name": "{}-train".format(dataset_name),
+ "datasets": [
+ (dataset_name, "train"),
+ ],
+ },
+ "valid": {
+ "{}-valid".format(dataset_name): [
+ (dataset_name, "valid"),
+ ],
+ },
+ "config": {
+ "load_in_memory": False, # Load all images in CPU memory
+ "worker_per_gpu": 4,
+ "width_divisor": 8, # Image width will be divided by 8
+ "height_divisor": 32, # Image height will be divided by 32
+ "padding_value": 0, # Image padding value
+ "padding_token": 1000, # Label padding value (None: default value is chosen)
+ "padding_mode": "br", # Padding at bottom and right
+ "charset_mode": "CTC", # add blank token
+ "constraints": [], # Padding for CTC requirements if necessary
+ "normalize": True, # Normalize with mean and variance of training dataset
+ "preprocessings": [],
+ # Augmentation techniques to use at training time
+ "augmentation": line_aug_config(0.9, 0.1),
+ #
+ "synthetic_data": {
+ "mode": "line_hw_to_printed",
+ "init_proba": 1,
+ "end_proba": 1,
+ "num_steps_proba": 1e5,
+ "proba_scheduler_function": exponential_scheduler,
+ "config": {
+ "background_color_default": (255, 255, 255),
+ "background_color_eps": 15,
+ "text_color_default": (0, 0, 0),
+ "text_color_eps": 15,
+ "font_size_min": 30,
+ "font_size_max": 50,
+ "color_mode": "RGB",
+ "padding_left_ratio_min": 0.02,
+ "padding_left_ratio_max": 0.1,
+ "padding_right_ratio_min": 0.02,
+ "padding_right_ratio_max": 0.1,
+ "padding_top_ratio_min": 0.02,
+ "padding_top_ratio_max": 0.2,
+ "padding_bottom_ratio_min": 0.02,
+ "padding_bottom_ratio_max": 0.2,
+ },
+ },
+ },
+ },
+ "model_params": {
+ # Model classes to use for each module
+ "models": {
+ "encoder": FCN_Encoder,
+ "decoder": Decoder,
+ },
+ "transfer_learning": None,
+ "input_channels": 3, # 1 for grayscale images, 3 for RGB ones (or grayscale as RGB)
+ "enc_size": 256,
+ "dropout_scheduler": {
+ "function": exponential_dropout_scheduler,
+ "T": 5e4,
+ },
+ "dropout": 0.5,
+ },
+ "training_params": {
+ "output_folder": "FCN_Encoder_read_syn_line_all_pad_max_cursive", # folder names for logs and weights
+ "max_nb_epochs": 10000, # max number of epochs for the training
+ "max_training_time": 3600
+ * 24
+ * 1.9, # max training time limit (in seconds)
+ "load_epoch": "last", # ["best", "last"], to load weights from best epoch or last trained epoch
+ "interval_save_weights": None, # None: keep best and last only
+ "use_ddp": False, # Use DistributedDataParallel
+ "use_amp": True, # Enable automatic mix-precision
+ "nb_gpu": torch.cuda.device_count(),
+ "batch_size": 1, # mini-batch size per GPU
+ "optimizers": {
+ "all": {
+ "class": Adam,
+ "args": {
+ "lr": 0.0001,
+ "amsgrad": False,
+ },
+ }
+ },
+ "lr_schedulers": None,
+ "eval_on_valid": True, # Whether to eval and logs metrics on validation set during training or not
+ "eval_on_valid_interval": 2, # Interval (in epochs) to evaluate during training
+ "focus_metric": "cer", # Metrics to focus on to determine best epoch
+ "expected_metric_value": "low", # ["high", "low"] What is best for the focus metric value
+ "set_name_focus_metric": "{}-valid".format(dataset_name),
+ "train_metrics": ["loss_ctc", "cer", "wer"], # Metrics name for training
+ "eval_metrics": [
+ "loss_ctc",
+ "cer",
+ "wer",
+ ], # Metrics name for evaluation on validation set during training
+ "force_cpu": False, # True for debug purposes to run on cpu only
+ },
+ }
+
+ if (
+ params["training_params"]["use_ddp"]
+ and not params["training_params"]["force_cpu"]
+ ):
+ mp.spawn(
+ train_and_test, args=(params,), nprocs=params["training_params"]["nb_gpu"]
+ )
+ else:
+ train_and_test(0, params)
diff --git a/dan/ocr/line/model_utils.py b/dan/ocr/line/model_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..9417e8003a104dd0fd047453011805c877169ddc
--- /dev/null
+++ b/dan/ocr/line/model_utils.py
@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+from torch.nn import AdaptiveMaxPool2d, Conv1d, Module
+from torch.nn.functional import log_softmax
+
+
+class Decoder(Module):
+ def __init__(self, params):
+ super(Decoder, self).__init__()
+
+ self.vocab_size = params["vocab_size"]
+
+ self.ada_pool = AdaptiveMaxPool2d((1, None))
+ self.end_conv = Conv1d(
+ in_channels=params["enc_size"],
+ out_channels=self.vocab_size + 1,
+ kernel_size=1,
+ )
+
+ def forward(self, x):
+ x = self.ada_pool(x).squeeze(2)
+ x = self.end_conv(x)
+ return log_softmax(x, dim=1)
diff --git a/dan/ocr/line/train.py b/dan/ocr/line/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..9e092fd966a958e3244545f6f7488776c6701a0c
--- /dev/null
+++ b/dan/ocr/line/train.py
@@ -0,0 +1,207 @@
+# -*- coding: utf-8 -*-
+import random
+
+import numpy as np
+import torch
+import torch.multiprocessing as mp
+from torch.optim import Adam
+
+from dan.manager.ocr import OCRDataset, OCRDatasetManager
+from dan.models import FCN_Encoder
+from dan.ocr.line.model_utils import Decoder
+from dan.ocr.line.utils import TrainerLineCTC
+from dan.schedulers import exponential_dropout_scheduler, exponential_scheduler
+from dan.transforms import line_aug_config
+
+
+def add_line_parser(subcommands) -> None:
+ parser = subcommands.add_parser(
+ "line",
+ description=__doc__,
+ )
+ parser.set_defaults(func=run)
+
+
+def train_and_test(rank, params):
+ torch.manual_seed(0)
+ torch.cuda.manual_seed(0)
+ np.random.seed(0)
+ random.seed(0)
+ torch.backends.cudnn.benchmark = False
+ torch.backends.cudnn.deterministic = True
+
+ params["training_params"]["ddp_rank"] = rank
+ model = TrainerLineCTC(params)
+ model.load_model()
+
+ # Model trains until max_time_training or max_nb_epochs is reached
+ model.train()
+
+ # load weights giving best CER on valid set
+ model.params["training_params"]["load_epoch"] = "best"
+ model.load_model()
+
+ # compute metrics on train, valid and test sets (in eval conditions)
+ metrics = [
+ "cer",
+ "wer",
+ "time",
+ ]
+ for dataset_name in params["dataset_params"]["datasets"].keys():
+ for set_name in [
+ "test",
+ "valid",
+ "train",
+ ]:
+ model.predict(
+ "{}-{}".format(dataset_name, set_name),
+ [
+ (dataset_name, set_name),
+ ],
+ metrics,
+ output=True,
+ )
+
+
+def run():
+ dataset_name = "READ_2016" # ["RIMES", "READ_2016"]
+ dataset_level = "syn_line"
+ params = {
+ "dataset_params": {
+ "dataset_manager": OCRDatasetManager,
+ "dataset_class": OCRDataset,
+ "datasets": {
+ dataset_name: "../../../Datasets/formatted/{}_{}".format(
+ dataset_name, dataset_level
+ ),
+ },
+ "train": {
+ "name": "{}-train".format(dataset_name),
+ "datasets": [
+ (dataset_name, "train"),
+ ],
+ },
+ "valid": {
+ "{}-valid".format(dataset_name): [
+ (dataset_name, "valid"),
+ ],
+ },
+ "config": {
+ "load_in_memory": True, # Load all images in CPU memory
+ "worker_per_gpu": 8, # Num of parallel processes per gpu for data loading
+ "width_divisor": 8, # Image width will be divided by 8
+ "height_divisor": 32, # Image height will be divided by 32
+ "padding_value": 0, # Image padding value
+ "padding_token": 1000, # Label padding value (None: default value is chosen)
+ "padding_mode": "br", # Padding at bottom and right
+ "charset_mode": "CTC", # add blank token
+ "constraints": [
+ "CTC_line",
+ ], # Padding for CTC requirements if necessary
+ "normalize": True, # Normalize with mean and variance of training dataset
+ "padding": {
+ "min_height": "max", # Pad to reach max height of training samples
+ "min_width": "max", # Pad to reach max width of training samples
+ "min_pad": None,
+ "max_pad": None,
+ "mode": "br", # Padding at bottom and right
+ "train_only": False, # Add padding at training time and evaluation time
+ },
+ "preprocessings": [
+ {
+ "type": "to_RGB",
+ # if grayscale image, produce RGB one (3 channels with same value) otherwise do nothing
+ },
+ ],
+ # Augmentation techniques to use at training time
+ "augmentation": line_aug_config(0.9, 0.1),
+ #
+ "synthetic_data": {
+ "mode": "line_hw_to_printed",
+ "init_proba": 1,
+ "end_proba": 1,
+ "num_steps_proba": 1e5,
+ "probadocument_scheduler_function": exponential_scheduler,
+ "config": {
+ "background_color_default": (255, 255, 255),
+ "background_color_eps": 15,
+ "text_color_default": (0, 0, 0),
+ "text_color_eps": 15,
+ "font_size_min": 30,
+ "font_size_max": 50,
+ "color_mode": "RGB",
+ "padding_left_ratio_min": 0.02,
+ "padding_left_ratio_max": 0.1,
+ "padding_right_ratio_min": 0.02,
+ "padding_right_ratio_max": 0.1,
+ "padding_top_ratio_min": 0.02,
+ "padding_top_ratio_max": 0.2,
+ "padding_bottom_ratio_min": 0.02,
+ "padding_bottom_ratio_max": 0.2,
+ },
+ },
+ },
+ },
+ "model_params": {
+ # Model classes to use for each module
+ "models": {
+ "encoder": FCN_Encoder,
+ "decoder": Decoder,
+ },
+ "transfer_learning": None,
+ "input_channels": 3, # 1 for grayscale images, 3 for RGB ones (or grayscale as RGB)
+ "enc_size": 256,
+ "dropout_scheduler": {
+ "function": exponential_dropout_scheduler,
+ "T": 5e4,
+ },
+ "dropout": 0.5,
+ },
+ "training_params": {
+ "output_folder": "FCN_read_2016_line_syn", # folder names for logs and weights
+ "max_nb_epochs": 10000, # max number of epochs for the training
+ "max_training_time": 3600
+ * 24
+ * 1.9, # max training time limit (in seconds)
+ "load_epoch": "last", # ["best", "last"], to load weights from best epoch or last trained epoch
+ "interval_save_weights": None, # None: keep best and last only
+ "use_ddp": False, # Use DistributedDataParallel
+ "use_amp": True, # Enable automatic mix-precision
+ "nb_gpu": torch.cuda.device_count(),
+ "batch_size": 16, # mini-batch size per GPU
+ "optimizers": {
+ "all": {
+ "class": Adam,
+ "args": {
+ "lr": 0.0001,
+ "amsgrad": False,
+ },
+ }
+ },
+ "lr_schedulers": None, # Learning rate schedulers
+ "eval_on_valid": True, # Whether to eval and logs metrics on validation set during training or not
+ "eval_on_valid_interval": 2, # Interval (in epochs) to evaluate during training
+ "focus_metric": "cer", # Metrics to focus on to determine best epoch
+ "expected_metric_value": "low", # ["high", "low"] What is best for the focus metric value
+ "set_name_focus_metric": "{}-valid".format(
+ dataset_name
+ ), # Which dataset to focus on to select best weights
+ "train_metrics": ["loss_ctc", "cer", "wer"], # Metrics name for training
+ "eval_metrics": [
+ "loss_ctc",
+ "cer",
+ "wer",
+ ], # Metrics name for evaluation on validation set during training
+ "force_cpu": False, # True for debug purposes to run on cpu only
+ },
+ }
+
+ if (
+ params["training_params"]["use_ddp"]
+ and not params["training_params"]["force_cpu"]
+ ):
+ mp.spawn(
+ train_and_test, args=(params,), nprocs=params["training_params"]["nb_gpu"]
+ )
+ else:
+ train_and_test(0, params)
diff --git a/dan/ocr/line/utils.py b/dan/ocr/line/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..01ed68be78629637eb19562b5dbca26e848971bf
--- /dev/null
+++ b/dan/ocr/line/utils.py
@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+import re
+import time
+
+import torch
+from torch.cuda.amp import autocast
+from torch.nn import CTCLoss
+
+from dan.manager.training import OCRManager
+from dan.ocr.utils import LM_ind_to_str
+
+
+class TrainerLineCTC(OCRManager):
+ def __init__(self, params):
+ super(TrainerLineCTC, self).__init__(params)
+
+ def train_batch(self, batch_data, metric_names):
+ """
+ Forward and backward pass for training
+ """
+ x = batch_data["imgs"].to(self.device)
+ y = batch_data["labels"].to(self.device)
+ x_reduced_len = [s[1] for s in batch_data["imgs_reduced_shape"]]
+ y_len = batch_data["labels_len"]
+
+ loss_ctc = CTCLoss(blank=self.dataset.tokens["blank"])
+ self.zero_optimizers()
+
+ with autocast(enabled=self.params["training_params"]["use_amp"]):
+ x = self.models["encoder"](x)
+ global_pred = self.models["decoder"](x)
+
+ loss = loss_ctc(global_pred.permute(2, 0, 1), y, x_reduced_len, y_len)
+
+ self.backward_loss(loss)
+
+ self.step_optimizers()
+ pred = torch.argmax(global_pred, dim=1).cpu().numpy()
+
+ values = {
+ "nb_samples": len(batch_data["raw_labels"]),
+ "loss_ctc": loss.item(),
+ "str_x": self.pred_to_str(pred, x_reduced_len),
+ "str_y": batch_data["raw_labels"],
+ }
+
+ return values
+
+ def evaluate_batch(self, batch_data, metric_names):
+ """
+ Forward pass only for validation and test
+ """
+ x = batch_data["imgs"].to(self.device)
+ y = batch_data["labels"].to(self.device)
+ x_reduced_len = [s[1] for s in batch_data["imgs_reduced_shape"]]
+ y_len = batch_data["labels_len"]
+
+ loss_ctc = CTCLoss(blank=self.dataset.tokens["blank"])
+
+ start_time = time.time()
+ with autocast(enabled=self.params["training_params"]["use_amp"]):
+ x = self.models["encoder"](x)
+ global_pred = self.models["decoder"](x)
+
+ loss = loss_ctc(global_pred.permute(2, 0, 1), y, x_reduced_len, y_len)
+ pred = torch.argmax(global_pred, dim=1).cpu().numpy()
+ str_x = self.pred_to_str(pred, x_reduced_len)
+
+ process_time = time.time() - start_time
+
+ values = {
+ "nb_samples": len(batch_data["raw_labels"]),
+ "loss_ctc": loss.item(),
+ "str_x": str_x,
+ "str_y": batch_data["raw_labels"],
+ "time": process_time,
+ }
+ return values
+
+ def ctc_remove_successives_identical_ind(self, ind):
+ res = []
+ for i in ind:
+ if res and res[-1] == i:
+ continue
+ res.append(i)
+ return res
+
+ def pred_to_str(self, pred, pred_len):
+ """
+ convert prediction tokens to string
+ """
+ ind_x = [pred[i][: pred_len[i]] for i in range(pred.shape[0])]
+ ind_x = [self.ctc_remove_successives_identical_ind(t) for t in ind_x]
+ str_x = [LM_ind_to_str(self.dataset.charset, t, oov_symbol="") for t in ind_x]
+ str_x = [re.sub("( )+", " ", t).strip(" ") for t in str_x]
+ return str_x
diff --git a/dan/ocr/train.py b/dan/ocr/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..1bcbe1c75965c371cf71dc903a707482c5c0c1c8
--- /dev/null
+++ b/dan/ocr/train.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+from dan.ocr.line.train import add_line_parser
+from dan.ocr.document.train import add_document_parser
+
+def add_train_parser(subcommands) -> None:
+ parser = subcommands.add_parser(
+ "train",
+ description=__doc__,
+ )
+ subcommands = parser.add_subparsers(metavar="subcommand")
+
+ add_line_parser(subcommands)
+ add_document_parser(subcommands)
diff --git a/dan/ocr/utils.py b/dan/ocr/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..8038f6f9cab2bc21b80fda7b36b0fea66b445c96
--- /dev/null
+++ b/dan/ocr/utils.py
@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+# Charset / labels conversion
+def LM_str_to_ind(labels, str):
+ return [labels.index(c) for c in str]
+
+
+def LM_ind_to_str(labels, ind, oov_symbol=None):
+ if oov_symbol is not None:
+ res = []
+ for i in ind:
+ if i < len(labels):
+ res.append(labels[i])
+ else:
+ res.append(oov_symbol)
+ else:
+ res = [labels[i] for i in ind]
+ return "".join(res)
diff --git a/dan/ocr_utils.py b/dan/ocr_utils.py
deleted file mode 100644
index 4685f66f15176446daaccdb51043ca42c71ba7ce..0000000000000000000000000000000000000000
--- a/dan/ocr_utils.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in Python whose purpose is to
-# provide public implementation of deep learning works, in pytorch.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
-
-
-# Charset / labels conversion
-def LM_str_to_ind(labels, str):
- return [labels.index(c) for c in str]
-
-
-def LM_ind_to_str(labels, ind, oov_symbol=None):
- if oov_symbol is not None:
- res = []
- for i in ind:
- if i < len(labels):
- res.append(labels[i])
- else:
- res.append(oov_symbol)
- else:
- res = [labels[i] for i in ind]
- return "".join(res)
diff --git a/dan/post_processing.py b/dan/post_processing.py
index 3b456ba5a8d315555e43198a563c6dab692dba83..1242959c8c315aa643c624d686e366d8e0a35b9a 100644
--- a/dan/post_processing.py
+++ b/dan/post_processing.py
@@ -1,50 +1,7 @@
# -*- coding: utf-8 -*-
-# Copyright Université de Rouen Normandie (1), INSA Rouen (2),
-# tutelles du laboratoire LITIS (1 et 2)
-# contributors :
-# - Denis Coquenet
-#
-#
-# This software is a computer program written in Python whose purpose is to
-# provide public implementation of deep learning works, in pytorch.
-#
-# This software is governed by the CeCILL-C license under French law and
-# abiding by the rules of distribution of free software. You can use,
-# modify and/ or redistribute the software under the terms of the CeCILL-C
-# license as circulated by CEA, CNRS and INRIA at the following URL
-# "http://www.cecill.info".
-#
-# As a counterpart to the access to the source code and rights to copy,
-# modify and redistribute granted by the license, users are provided only
-# with a limited warranty and the software's author, the holder of the
-# economic rights, and the successive licensors have only limited
-# liability.
-#
-# In this respect, the user's attention is drawn to the risks associated
-# with loading, using, modifying and/or developing or reproducing the
-# software by the user in light of its specific status of free software,
-# that may mean that it is complicated to manipulate, and that also
-# therefore means that it is reserved for developers and experienced
-# professionals having in-depth computer knowledge. Users are therefore
-# encouraged to load and test the software's suitability as regards their
-# requirements in conditions enabling the security of their systems and/or
-# data to be ensured and, more generally, to use and operate it in the
-# same conditions as regards security.
-#
-# The fact that you are presently reading this means that you have had
-# knowledge of the CeCILL-C license and that you accept its terms.
-
import numpy as np
-from Datasets.dataset_formatters.read2016_formatter import (
- SEM_MATCHING_TOKENS as READ_MATCHING_TOKENS,
-)
-from Datasets.dataset_formatters.rimes_formatter import (
- SEM_MATCHING_TOKENS as RIMES_MATCHING_TOKENS,
-)
-from Datasets.dataset_formatters.simara_formatter import (
- SEM_MATCHING_TOKENS as SIMARA_MATCHING_TOKENS,
-)
+from dan.datasets.format.simara import SEM_MATCHING_TOKENS as SIMARA_MATCHING_TOKENS
class PostProcessingModule:
@@ -94,193 +51,6 @@ class PostProcessingModule:
self.num_op += 1
-class PostProcessingModuleREAD(PostProcessingModule):
- """
- Specific post-processing for the READ 2016 dataset at single-page and double-page levels
- """
-
- def __init__(self):
- super(PostProcessingModuleREAD, self).__init__()
-
- self.matching_tokens = READ_MATCHING_TOKENS
- self.reverse_matching_tokens = dict()
- for key in self.matching_tokens:
- self.reverse_matching_tokens[self.matching_tokens[key]] = key
-
- def post_processing_page_labels(self):
- """
- Correct tokens of page detection.
- """
- ind = 0
- while ind != len(self.prediction):
- # Label must start with a begin-page token
- if ind == 0 and self.prediction[ind] != "â“Ÿ":
- self.insert_label(0, "â“Ÿ")
- continue
- # There cannot be tokens out of begin-page end-page scope: begin-page must be preceded by end-page
- if (
- self.prediction[ind] == "â“Ÿ"
- and ind != 0
- and self.prediction[ind - 1] != "â“…"
- ):
- self.insert_label(ind, "â“…")
- continue
- # There cannot be tokens out of begin-page end-page scope: end-page must be followed by begin-page
- if (
- self.prediction[ind] == "â“…"
- and ind < len(self.prediction) - 1
- and self.prediction[ind + 1] != "â“Ÿ"
- ):
- self.insert_label(ind + 1, "â“Ÿ")
- ind += 1
- # Label must start with a begin-page token even for empty prediction
- if len(self.prediction) == 0:
- self.insert_label(0, "â“Ÿ")
- ind += 1
- # Label must end with a end-page token
- if self.prediction[-1] != "â“…":
- self.insert_label(ind, "â“…")
-
- def post_processing(self):
- """
- Correct tokens of page number, section, body and annotations.
- """
- self.post_processing_page_labels()
- ind = 0
- begin_token = None
- in_section = False
- while ind != len(self.prediction):
- # each tags must be closed while changing page
- if self.prediction[ind] == "â“…":
- if begin_token is not None:
- self.insert_label(ind, self.matching_tokens[begin_token])
- begin_token = None
- ind += 1
- elif in_section:
- self.insert_label(ind, self.matching_tokens["â“¢"])
- in_section = False
- ind += 1
- else:
- ind += 1
- continue
- # End token is removed if the previous begin token does not match with it
- if self.prediction[ind] in "ⓃⒶⒷ":
- if begin_token == self.reverse_matching_tokens[self.prediction[ind]]:
- begin_token = None
- ind += 1
- else:
- self.del_label(ind)
- continue
- if self.prediction[ind] == "Ⓢ":
- # each sub-tags must be closed while closing section
- if in_section:
- if begin_token is None:
- in_section = False
- ind += 1
- else:
- self.insert_label(ind, self.matching_tokens[begin_token])
- begin_token = None
- ind += 2
- else:
- self.del_label(ind)
- continue
- if self.prediction[ind] == "â“¢":
- # A sub-tag must be closed before opening a section
- if begin_token is not None:
- self.insert_label(ind, self.matching_tokens[begin_token])
- begin_token = None
- ind += 1
- # A section must be closed before opening a new one
- elif in_section:
- self.insert_label(ind, "Ⓢ")
- in_section = False
- ind += 1
- else:
- in_section = True
- ind += 1
- continue
- if self.prediction[ind] == "â“":
- # Page number cannot be in section: a started section must be closed
- if begin_token is None:
- if in_section:
- in_section = False
- self.insert_label(ind, "Ⓢ")
- ind += 1
- begin_token = self.prediction[ind]
- ind += 1
- else:
- self.insert_label(ind, self.matching_tokens[begin_token])
- begin_token = None
- ind += 1
- continue
- if self.prediction[ind] in "â“â“‘":
- # Annotation and body must be in section
- if begin_token is None:
- if in_section:
- begin_token = self.prediction[ind]
- ind += 1
- else:
- in_section = True
- self.insert_label(ind, "â“¢")
- ind += 1
- # Previous sub-tag must be closed
- else:
- self.insert_label(ind, self.matching_tokens[begin_token])
- begin_token = None
- ind += 1
- continue
- ind += 1
- res = "".join(self.prediction)
- if self.confidence is not None:
- return res, np.array(self.confidence)
- return res
-
-
-class PostProcessingModuleRIMES(PostProcessingModule):
- """
- Specific post-processing for the RIMES dataset at page level
- """
-
- def __init__(self):
- super(PostProcessingModuleRIMES, self).__init__()
- self.matching_tokens = RIMES_MATCHING_TOKENS
- self.reverse_matching_tokens = dict()
- for key in self.matching_tokens:
- self.reverse_matching_tokens[self.matching_tokens[key]] = key
-
- def post_processing(self):
- ind = 0
- begin_token = None
- while ind != len(self.prediction):
- char = self.prediction[ind]
- # a tag must be closed before starting a new one
- if char in self.matching_tokens.keys():
- if begin_token is None:
- ind += 1
- else:
- self.insert_label(ind, self.matching_tokens[begin_token])
- ind += 2
- begin_token = char
- continue
- # an end token without prior corresponding begin token is removed
- elif char in self.matching_tokens.values():
- if begin_token == self.reverse_matching_tokens[char]:
- ind += 1
- begin_token = None
- else:
- self.del_label(ind)
- continue
- else:
- ind += 1
- # a tag must be closed
- if begin_token is not None:
- self.insert_label(ind + 1, self.matching_tokens[begin_token])
- res = "".join(self.prediction)
- if self.confidence is not None:
- return res, np.array(self.confidence)
- return res
-
-
class PostProcessingModuleSIMARA(PostProcessingModule):
"""
Specific post-processing for the SIMARA dataset at page level
diff --git a/dan/predict.py b/dan/predict.py
index 4473af173efa2f3a488d65c9a458d939a196720f..753e584d91416b47c81c07580e8f15eebb7dfb53 100644
--- a/dan/predict.py
+++ b/dan/predict.py
@@ -10,7 +10,7 @@ import yaml
from dan.decoder import GlobalHTADecoder
from dan.models import FCN_Encoder
-from dan.ocr_utils import LM_ind_to_str
+from dan.ocr.utils import LM_ind_to_str
class DAN:
diff --git a/dan/schedulers.py b/dan/schedulers.py
new file mode 100644
index 0000000000000000000000000000000000000000..f801804b6216251dc6251cf26813322b4785bcbe
--- /dev/null
+++ b/dan/schedulers.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+import numpy as np
+from torch.nn import Dropout, Dropout2d
+
+
+class DropoutScheduler:
+ def __init__(self, models, function, T=1e5):
+ """
+ T: number of gradient updates to converge
+ """
+
+ self.teta_list = list()
+ self.init_teta_list(models)
+ self.function = function
+ self.T = T
+ self.step_num = 0
+
+ def step(self, num):
+ self.step_num += num
+
+ def init_teta_list(self, models):
+ for model_name in models.keys():
+ self.init_teta_list_module(models[model_name])
+
+ def init_teta_list_module(self, module):
+ for child in module.children():
+ if isinstance(child, Dropout) or isinstance(child, Dropout2d):
+ self.teta_list.append([child, child.p])
+ else:
+ self.init_teta_list_module(child)
+
+ def update_dropout_rate(self):
+ for (module, p) in self.teta_list:
+ module.p = self.function(p, self.step_num, self.T)
+
+
+def exponential_dropout_scheduler(dropout_rate, step, max_step):
+ return dropout_rate * (1 - np.exp(-10 * step / max_step))
+
+
+def exponential_scheduler(init_value, end_value, step, max_step):
+ step = min(step, max_step - 1)
+ return init_value - (init_value - end_value) * (1 - np.exp(-10 * step / max_step))
+
+
+def linear_scheduler(init_value, end_value, step, max_step):
+ return init_value + step * (end_value - init_value) / max_step
diff --git a/dan/transforms.py b/dan/transforms.py
new file mode 100644
index 0000000000000000000000000000000000000000..b33489c63cc88e42f43077286f550af587b987aa
--- /dev/null
+++ b/dan/transforms.py
@@ -0,0 +1,510 @@
+# -*- coding: utf-8 -*-
+"""
+Each transform class defined here takes as input a PIL Image and returns the modified PIL Image
+"""
+import math
+
+import cv2
+import numpy as np
+from cv2 import dilate, erode, normalize
+from numpy import random
+from PIL import Image, ImageOps
+from torchvision.transforms import (
+ ColorJitter,
+ GaussianBlur,
+ RandomCrop,
+ RandomPerspective,
+)
+from torchvision.transforms.functional import InterpolationMode
+
+from dan.utils import rand, rand_uniform, randint
+
+
+class SignFlipping:
+ """
+ Color inversion
+ """
+
+ def __init__(self):
+ pass
+
+ def __call__(self, x):
+ return ImageOps.invert(x)
+
+
+class DPIAdjusting:
+ """
+ Resolution modification
+ """
+
+ def __init__(self, factor, preserve_ratio):
+ self.factor = factor
+
+ def __call__(self, x):
+ w, h = x.size
+ return x.resize(
+ (int(np.ceil(w * self.factor)), int(np.ceil(h * self.factor))),
+ Image.BILINEAR,
+ )
+
+
+class Dilation:
+ """
+ OCR: stroke width increasing
+ """
+
+ def __init__(self, kernel, iterations):
+ self.kernel = np.ones(kernel, np.uint8)
+ self.iterations = iterations
+
+ def __call__(self, x):
+ return Image.fromarray(
+ dilate(np.array(x), self.kernel, iterations=self.iterations)
+ )
+
+
+class Erosion:
+ """
+ OCR: stroke width decreasing
+ """
+
+ def __init__(self, kernel, iterations):
+ self.kernel = np.ones(kernel, np.uint8)
+ self.iterations = iterations
+
+ def __call__(self, x):
+ return Image.fromarray(
+ erode(np.array(x), self.kernel, iterations=self.iterations)
+ )
+
+
+class GaussianNoise:
+ """
+ Add Gaussian Noise
+ """
+
+ def __init__(self, std):
+ self.std = std
+
+ def __call__(self, x):
+ x_np = np.array(x)
+ mean, std = np.mean(x_np), np.std(x_np)
+ std = math.copysign(max(abs(std), 0.000001), std)
+ min_, max_ = np.min(
+ x_np,
+ ), np.max(x_np)
+ normal_noise = np.random.randn(*x_np.shape)
+ if (
+ len(x_np.shape) == 3
+ and x_np.shape[2] == 3
+ and np.all(x_np[:, :, 0] == x_np[:, :, 1])
+ and np.all(x_np[:, :, 0] == x_np[:, :, 2])
+ ):
+ normal_noise[:, :, 1] = normal_noise[:, :, 2] = normal_noise[:, :, 0]
+ x_np = ((x_np - mean) / std + normal_noise * self.std) * std + mean
+ x_np = normalize(x_np, x_np, max_, min_, cv2.NORM_MINMAX)
+
+ return Image.fromarray(x_np.astype(np.uint8))
+
+
+class Sharpen:
+ """
+ Add Gaussian Noise
+ """
+
+ def __init__(self, alpha, strength):
+ self.alpha = alpha
+ self.strength = strength
+
+ def __call__(self, x):
+ x_np = np.array(x)
+ id_matrix = np.array([[0, 0, 0], [0, 1, 0], [0, 0, 0]])
+ effect_matrix = np.array([[1, 1, 1], [1, -(8 + self.strength), 1], [1, 1, 1]])
+ kernel = (1 - self.alpha) * id_matrix - self.alpha * effect_matrix
+ kernel = np.expand_dims(kernel, axis=2)
+ kernel = np.concatenate([kernel, kernel, kernel], axis=2)
+ sharpened = cv2.filter2D(x_np, -1, kernel=kernel[:, :, 0])
+ return Image.fromarray(sharpened.astype(np.uint8))
+
+
+class ZoomRatio:
+ """
+ Crop by ratio
+ Preserve dimensions if keep_dim = True (= zoom)
+ """
+
+ def __init__(self, ratio_h, ratio_w, keep_dim=True):
+ self.ratio_w = ratio_w
+ self.ratio_h = ratio_h
+ self.keep_dim = keep_dim
+
+ def __call__(self, x):
+ w, h = x.size
+ x = RandomCrop((int(h * self.ratio_h), int(w * self.ratio_w)))(x)
+ if self.keep_dim:
+ x = x.resize((w, h), Image.BILINEAR)
+ return x
+
+
+class ElasticDistortion:
+ def __init__(self, kernel_size=(7, 7), sigma=5, alpha=1):
+
+ self.kernel_size = kernel_size
+ self.sigma = sigma
+ self.alpha = alpha
+
+ def __call__(self, x):
+ x_np = np.array(x)
+
+ h, w = x_np.shape[:2]
+
+ dx = np.random.uniform(-1, 1, (h, w))
+ dy = np.random.uniform(-1, 1, (h, w))
+
+ x_gauss = cv2.GaussianBlur(dx, self.kernel_size, self.sigma)
+ y_gauss = cv2.GaussianBlur(dy, self.kernel_size, self.sigma)
+
+ n = np.sqrt(x_gauss**2 + y_gauss**2)
+
+ nd_x = self.alpha * x_gauss / n
+ nd_y = self.alpha * y_gauss / n
+
+ ind_y, ind_x = np.indices((h, w), dtype=np.float32)
+
+ map_x = nd_x + ind_x
+ map_x = map_x.reshape(h, w).astype(np.float32)
+ map_y = nd_y + ind_y
+ map_y = map_y.reshape(h, w).astype(np.float32)
+
+ dst = cv2.remap(x_np, map_x, map_y, cv2.INTER_LINEAR)
+ return Image.fromarray(dst.astype(np.uint8))
+
+
+class Tightening:
+ """
+ Reduce interline spacing
+ """
+
+ def __init__(self, color=255, remove_proba=0.75):
+ self.color = color
+ self.remove_proba = remove_proba
+
+ def __call__(self, x):
+ x_np = np.array(x)
+ interline_indices = [np.all(line == 255) for line in x_np]
+ indices_to_removed = np.logical_and(
+ np.random.choice(
+ [True, False],
+ size=len(x_np),
+ replace=True,
+ p=[self.remove_proba, 1 - self.remove_proba],
+ ),
+ interline_indices,
+ )
+ new_x = x_np[np.logical_not(indices_to_removed)]
+ return Image.fromarray(new_x.astype(np.uint8))
+
+
+def get_list_augmenters(img, aug_configs, fill_value):
+ """
+ Randomly select a list of data augmentation techniques to used based on aug_configs
+ """
+ augmenters = list()
+ for aug_config in aug_configs:
+ if rand() > aug_config["proba"]:
+ continue
+ if aug_config["type"] == "dpi":
+ valid_factor = False
+ while not valid_factor:
+ factor = rand_uniform(
+ aug_config["min_factor"], aug_config["max_factor"]
+ )
+ valid_factor = not (
+ (
+ "max_width" in aug_config
+ and factor * img.size[0] > aug_config["max_width"]
+ )
+ or (
+ "max_height" in aug_config
+ and factor * img.size[1] > aug_config["max_height"]
+ )
+ or (
+ "min_width" in aug_config
+ and factor * img.size[0] < aug_config["min_width"]
+ )
+ or (
+ "min_height" in aug_config
+ and factor * img.size[1] < aug_config["min_height"]
+ )
+ )
+ augmenters.append(
+ DPIAdjusting(factor, preserve_ratio=aug_config["preserve_ratio"])
+ )
+
+ elif aug_config["type"] == "zoom_ratio":
+ ratio_h = rand_uniform(aug_config["min_ratio_h"], aug_config["max_ratio_h"])
+ ratio_w = rand_uniform(aug_config["min_ratio_w"], aug_config["max_ratio_w"])
+ augmenters.append(
+ ZoomRatio(
+ ratio_h=ratio_h, ratio_w=ratio_w, keep_dim=aug_config["keep_dim"]
+ )
+ )
+
+ elif aug_config["type"] == "perspective":
+ scale = rand_uniform(aug_config["min_factor"], aug_config["max_factor"])
+ augmenters.append(
+ RandomPerspective(
+ distortion_scale=scale,
+ p=1,
+ interpolation=InterpolationMode.BILINEAR,
+ fill=fill_value,
+ )
+ )
+
+ elif aug_config["type"] == "elastic_distortion":
+ kernel_size = (
+ randint(aug_config["min_kernel_size"], aug_config["max_kernel_size"])
+ // 2
+ * 2
+ + 1
+ )
+ sigma = rand_uniform(aug_config["min_sigma"], aug_config["max_sigma"])
+ alpha = rand_uniform(aug_config["min_alpha"], aug_config["max_alpha"])
+ augmenters.append(
+ ElasticDistortion(
+ kernel_size=(kernel_size, kernel_size), sigma=sigma, alpha=alpha
+ )
+ )
+
+ elif aug_config["type"] == "dilation_erosion":
+ kernel_h = randint(aug_config["min_kernel"], aug_config["max_kernel"] + 1)
+ kernel_w = randint(aug_config["min_kernel"], aug_config["max_kernel"] + 1)
+ if randint(0, 2) == 0:
+ augmenters.append(
+ Erosion((kernel_w, kernel_h), aug_config["iterations"])
+ )
+ else:
+ augmenters.append(
+ Dilation((kernel_w, kernel_h), aug_config["iterations"])
+ )
+
+ elif aug_config["type"] == "color_jittering":
+ augmenters.append(
+ ColorJitter(
+ contrast=aug_config["factor_contrast"],
+ brightness=aug_config["factor_brightness"],
+ saturation=aug_config["factor_saturation"],
+ hue=aug_config["factor_hue"],
+ )
+ )
+
+ elif aug_config["type"] == "gaussian_blur":
+ max_kernel_h = min(aug_config["max_kernel"], img.size[1])
+ max_kernel_w = min(aug_config["max_kernel"], img.size[0])
+ kernel_h = randint(aug_config["min_kernel"], max_kernel_h + 1) // 2 * 2 + 1
+ kernel_w = randint(aug_config["min_kernel"], max_kernel_w + 1) // 2 * 2 + 1
+ sigma = rand_uniform(aug_config["min_sigma"], aug_config["max_sigma"])
+ augmenters.append(
+ GaussianBlur(kernel_size=(kernel_w, kernel_h), sigma=sigma)
+ )
+
+ elif aug_config["type"] == "gaussian_noise":
+ augmenters.append(GaussianNoise(std=aug_config["std"]))
+
+ elif aug_config["type"] == "sharpen":
+ alpha = rand_uniform(aug_config["min_alpha"], aug_config["max_alpha"])
+ strength = rand_uniform(
+ aug_config["min_strength"], aug_config["max_strength"]
+ )
+ augmenters.append(Sharpen(alpha=alpha, strength=strength))
+
+ else:
+ print("Error - unknown augmentor: {}".format(aug_config["type"]))
+ exit(-1)
+
+ return augmenters
+
+
+def apply_data_augmentation(img, da_config):
+ """
+ Apply data augmentation strategy on input image
+ """
+ applied_da = list()
+ if da_config["proba"] != 1 and rand() > da_config["proba"]:
+ return img, applied_da
+
+ # Convert to PIL Image
+ img = img[:, :, 0] if img.shape[2] == 1 else img
+ img = Image.fromarray(img)
+
+ fill_value = da_config["fill_value"] if "fill_value" in da_config else 255
+ augmenters = get_list_augmenters(
+ img, da_config["augmentations"], fill_value=fill_value
+ )
+ if da_config["order"] == "random":
+ random.shuffle(augmenters)
+
+ for augmenter in augmenters:
+ img = augmenter(img)
+ applied_da.append(type(augmenter).__name__)
+
+ # convert to numpy array
+ img = np.array(img)
+ img = np.expand_dims(img, axis=2) if len(img.shape) == 2 else img
+ return img, applied_da
+
+
+def apply_transform(img, transform):
+ """
+ Apply data augmentation technique on input image
+ """
+ img = img[:, :, 0] if img.shape[2] == 1 else img
+ img = Image.fromarray(img)
+ img = transform(img)
+ img = np.array(img)
+ return np.expand_dims(img, axis=2) if len(img.shape) == 2 else img
+
+
+def line_aug_config(proba_use_da, p):
+ return {
+ "order": "random",
+ "proba": proba_use_da,
+ "augmentations": [
+ {
+ "type": "dpi",
+ "proba": p,
+ "min_factor": 0.5,
+ "max_factor": 1.5,
+ "preserve_ratio": True,
+ },
+ {
+ "type": "perspective",
+ "proba": p,
+ "min_factor": 0,
+ "max_factor": 0.4,
+ },
+ {
+ "type": "elastic_distortion",
+ "proba": p,
+ "min_alpha": 0.5,
+ "max_alpha": 1,
+ "min_sigma": 1,
+ "max_sigma": 10,
+ "min_kernel_size": 3,
+ "max_kernel_size": 9,
+ },
+ {
+ "type": "dilation_erosion",
+ "proba": p,
+ "min_kernel": 1,
+ "max_kernel": 3,
+ "iterations": 1,
+ },
+ {
+ "type": "color_jittering",
+ "proba": p,
+ "factor_hue": 0.2,
+ "factor_brightness": 0.4,
+ "factor_contrast": 0.4,
+ "factor_saturation": 0.4,
+ },
+ {
+ "type": "gaussian_blur",
+ "proba": p,
+ "min_kernel": 3,
+ "max_kernel": 5,
+ "min_sigma": 3,
+ "max_sigma": 5,
+ },
+ {
+ "type": "gaussian_noise",
+ "proba": p,
+ "std": 0.5,
+ },
+ {
+ "type": "sharpen",
+ "proba": p,
+ "min_alpha": 0,
+ "max_alpha": 1,
+ "min_strength": 0,
+ "max_strength": 1,
+ },
+ {
+ "type": "zoom_ratio",
+ "proba": p,
+ "min_ratio_h": 0.8,
+ "max_ratio_h": 1,
+ "min_ratio_w": 0.99,
+ "max_ratio_w": 1,
+ "keep_dim": True,
+ },
+ ],
+ }
+
+
+def aug_config(proba_use_da, p):
+ return {
+ "order": "random",
+ "proba": proba_use_da,
+ "augmentations": [
+ {
+ "type": "dpi",
+ "proba": p,
+ "min_factor": 0.75,
+ "max_factor": 1,
+ "preserve_ratio": True,
+ },
+ {
+ "type": "perspective",
+ "proba": p,
+ "min_factor": 0,
+ "max_factor": 0.4,
+ },
+ {
+ "type": "elastic_distortion",
+ "proba": p,
+ "min_alpha": 0.5,
+ "max_alpha": 1,
+ "min_sigma": 1,
+ "max_sigma": 10,
+ "min_kernel_size": 3,
+ "max_kernel_size": 9,
+ },
+ {
+ "type": "dilation_erosion",
+ "proba": p,
+ "min_kernel": 1,
+ "max_kernel": 3,
+ "iterations": 1,
+ },
+ {
+ "type": "color_jittering",
+ "proba": p,
+ "factor_hue": 0.2,
+ "factor_brightness": 0.4,
+ "factor_contrast": 0.4,
+ "factor_saturation": 0.4,
+ },
+ {
+ "type": "gaussian_blur",
+ "proba": p,
+ "min_kernel": 3,
+ "max_kernel": 5,
+ "min_sigma": 3,
+ "max_sigma": 5,
+ },
+ {
+ "type": "gaussian_noise",
+ "proba": p,
+ "std": 0.5,
+ },
+ {
+ "type": "sharpen",
+ "proba": p,
+ "min_alpha": 0,
+ "max_alpha": 1,
+ "min_strength": 0,
+ "max_strength": 1,
+ },
+ ],
+ }
diff --git a/dan/utils.py b/dan/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..3c8ea8f7118122a2cf1c5563a94367ef6117945a
--- /dev/null
+++ b/dan/utils.py
@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+import cv2
+import numpy as np
+import torch
+from torch.distributions.uniform import Uniform
+
+
+def randint(low, high):
+ """
+ call torch.randint to preserve random among dataloader workers
+ """
+ return int(torch.randint(low, high, (1,)))
+
+
+def rand():
+ """
+ call torch.rand to preserve random among dataloader workers
+ """
+ return float(torch.rand((1,)))
+
+
+def rand_uniform(low, high):
+ """
+ call torch uniform to preserve random among dataloader workers
+ """
+ return float(Uniform(low, high).sample())
+
+
+def pad_sequences_1D(data, padding_value):
+ """
+ Pad data with padding_value to get same length
+ """
+ x_lengths = [len(x) for x in data]
+ longest_x = max(x_lengths)
+ padded_data = np.ones((len(data), longest_x)).astype(np.int32) * padding_value
+ for i, x_len in enumerate(x_lengths):
+ padded_data[i, :x_len] = data[i][:x_len]
+ return padded_data
+
+
+def resize_max(img, max_width=None, max_height=None):
+ if max_width is not None and img.shape[1] > max_width:
+ ratio = max_width / img.shape[1]
+ new_h = int(np.floor(ratio * img.shape[0]))
+ new_w = int(np.floor(ratio * img.shape[1]))
+ img = cv2.resize(img, (new_w, new_h), interpolation=cv2.INTER_LINEAR)
+ if max_height is not None and img.shape[0] > max_height:
+ ratio = max_height / img.shape[0]
+ new_h = int(np.floor(ratio * img.shape[0]))
+ new_w = int(np.floor(ratio * img.shape[1]))
+ img = cv2.resize(img, (new_w, new_h), interpolation=cv2.INTER_LINEAR)
+ return img
+
+
+def pad_images(data, padding_value, padding_mode="br"):
+ """
+ data: list of numpy array
+ mode: "br"/"tl"/"random" (bottom-right, top-left, random)
+ """
+ x_lengths = [x.shape[0] for x in data]
+ y_lengths = [x.shape[1] for x in data]
+ longest_x = max(x_lengths)
+ longest_y = max(y_lengths)
+ padded_data = (
+ np.ones((len(data), longest_x, longest_y, data[0].shape[2])) * padding_value
+ )
+ for i, xy_len in enumerate(zip(x_lengths, y_lengths)):
+ x_len, y_len = xy_len
+ if padding_mode == "br":
+ padded_data[i, :x_len, :y_len, ...] = data[i]
+ elif padding_mode == "tl":
+ padded_data[i, -x_len:, -y_len:, ...] = data[i]
+ elif padding_mode == "random":
+ xmax = longest_x - x_len
+ ymax = longest_y - y_len
+ xi = randint(0, xmax) if xmax >= 1 else 0
+ yi = randint(0, ymax) if ymax >= 1 else 0
+ padded_data[i, xi : xi + x_len, yi : yi + y_len, ...] = data[i]
+ else:
+ raise NotImplementedError("Undefined padding mode: {}".format(padding_mode))
+ return padded_data
+
+
+def pad_image(
+ image,
+ padding_value,
+ new_height=None,
+ new_width=None,
+ pad_width=None,
+ pad_height=None,
+ padding_mode="br",
+ return_position=False,
+):
+ """
+ data: list of numpy array
+ mode: "br"/"tl"/"random" (bottom-right, top-left, random)
+ """
+ if pad_width is not None and new_width is not None:
+ raise NotImplementedError("pad_with and new_width are not compatible")
+ if pad_height is not None and new_height is not None:
+ raise NotImplementedError("pad_height and new_height are not compatible")
+
+ h, w, c = image.shape
+ pad_width = (
+ pad_width
+ if pad_width is not None
+ else max(0, new_width - w)
+ if new_width is not None
+ else 0
+ )
+ pad_height = (
+ pad_height
+ if pad_height is not None
+ else max(0, new_height - h)
+ if new_height is not None
+ else 0
+ )
+
+ if not (pad_width == 0 and pad_height == 0):
+ padded_image = np.ones((h + pad_height, w + pad_width, c)) * padding_value
+ if padding_mode == "br":
+ hi, wi = 0, 0
+ elif padding_mode == "tl":
+ hi, wi = pad_height, pad_width
+ elif padding_mode == "random":
+ hi = randint(0, pad_height) if pad_height >= 1 else 0
+ wi = randint(0, pad_width) if pad_width >= 1 else 0
+ else:
+ raise NotImplementedError("Undefined padding mode: {}".format(padding_mode))
+ padded_image[hi : hi + h, wi : wi + w, ...] = image
+ output = padded_image
+ else:
+ hi, wi = 0, 0
+ output = image
+
+ if return_position:
+ return output, [[hi, hi + h], [wi, wi + w]]
+ return output
+
+
+def pad_image_width_right(img, new_width, padding_value):
+ """
+ Pad img to right side with padding value to reach new_width as width
+ """
+ h, w, c = img.shape
+ pad_width = max((new_width - w), 0)
+ pad_right = np.ones((h, pad_width, c), dtype=img.dtype) * padding_value
+ img = np.concatenate([img, pad_right], axis=1)
+ return img
diff --git a/images/visual.png b/images/visual.png
new file mode 100644
index 0000000000000000000000000000000000000000..e228e7924cb2760dc61636651b525ef5359408c5
Binary files /dev/null and b/images/visual.png differ
diff --git a/images/visual_slanted_lines.png b/images/visual_slanted_lines.png
new file mode 100644
index 0000000000000000000000000000000000000000..478af3ac1eb40316bbb1f0e7400f592b0d31ec70
Binary files /dev/null and b/images/visual_slanted_lines.png differ