From 86163b14886235c5d4d598626e4f42e631f07ff0 Mon Sep 17 00:00:00 2001
From: Manon blanco <blanco@teklia.com>
Date: Wed, 14 Jun 2023 11:32:38 +0000
Subject: [PATCH] Remove DPIAdjusting transform
---
dan/manager/dataset.py | 13 --------
dan/transforms.py | 49 +---------------------------
docs/usage/train/parameters.md | 16 ---------
tests/data/training/models/best_0.pt | 2 +-
tests/data/training/models/last_3.pt | 4 +--
tests/test_training.py | 6 ++--
6 files changed, 7 insertions(+), 83 deletions(-)
diff --git a/dan/manager/dataset.py b/dan/manager/dataset.py
index 525e637c..547ada96 100644
--- a/dan/manager/dataset.py
+++ b/dan/manager/dataset.py
@@ -347,19 +347,6 @@ def apply_preprocessing(sample, preprocessings):
resize_ratio = [1, 1]
img = sample["img"]
for preprocessing in preprocessings:
- if preprocessing["type"] == "dpi":
- ratio = preprocessing["target"] / preprocessing["source"]
- temp_img = img
- h, w, c = temp_img.shape
- temp_img = cv2.resize(
- temp_img, (int(np.ceil(w * ratio)), int(np.ceil(h * ratio)))
- )
- if len(temp_img.shape) == 2:
- temp_img = np.expand_dims(temp_img, axis=2)
- img = temp_img
-
- resize_ratio = [ratio, ratio]
-
if preprocessing["type"] == "to_grayscaled":
temp_img = img
h, w, c = temp_img.shape
diff --git a/dan/transforms.py b/dan/transforms.py
index 2c596e3c..00bd6385 100644
--- a/dan/transforms.py
+++ b/dan/transforms.py
@@ -20,22 +20,6 @@ from torchvision.transforms import (
from torchvision.transforms.functional import InterpolationMode
-class DPIAdjusting:
- """
- Resolution modification
- """
-
- def __init__(self, factor):
- self.factor = factor
-
- def __call__(self, x):
- w, h = x.size
- return x.resize(
- (int(np.ceil(w * self.factor)), int(np.ceil(h * self.factor))),
- Image.BILINEAR,
- )
-
-
class Dilation:
"""
OCR: stroke width increasing
@@ -175,33 +159,8 @@ def get_list_augmenters(img, aug_configs, fill_value):
for aug_config in aug_configs:
if rand((1,)) > aug_config["proba"]:
continue
- if aug_config["type"] == "dpi":
- valid_factor = False
- while not valid_factor:
- factor = Uniform(
- aug_config["min_factor"], aug_config["max_factor"]
- ).sample()
- valid_factor = not (
- (
- "max_width" in aug_config
- and factor * img.size[0] > aug_config["max_width"]
- )
- or (
- "max_height" in aug_config
- and factor * img.size[1] > aug_config["max_height"]
- )
- or (
- "min_width" in aug_config
- and factor * img.size[0] < aug_config["min_width"]
- )
- or (
- "min_height" in aug_config
- and factor * img.size[1] < aug_config["min_height"]
- )
- )
- augmenters.append(DPIAdjusting(factor))
- elif aug_config["type"] == "zoom_ratio":
+ if aug_config["type"] == "zoom_ratio":
ratio_h = Uniform(
aug_config["min_ratio_h"], aug_config["max_ratio_h"]
).sample()
@@ -340,12 +299,6 @@ def aug_config(proba_use_da, p):
"order": "random",
"proba": proba_use_da,
"augmentations": [
- {
- "type": "dpi",
- "proba": p,
- "min_factor": 0.75,
- "max_factor": 1,
- },
{
"type": "perspective",
"proba": p,
diff --git a/docs/usage/train/parameters.md b/docs/usage/train/parameters.md
index bc2b55ad..fbcb0ab1 100644
--- a/docs/usage/train/parameters.md
+++ b/docs/usage/train/parameters.md
@@ -23,14 +23,6 @@ All hyperparameters are specified and editable in the training scripts (meaning
Preprocessing is applied before training the network (see `dan/manager/dataset.py`).
The following transformations are implemented:
-* DPI adjustment
-```py
- {
- "type": "dpi",
- "source": 300,
- "target": 150,
- }
-```
* Convert to grayscale
```py
{
@@ -80,7 +72,6 @@ Augmentation transformations are applied on-the-fly during training to artificia
The following transformations are implemented in `dan/transforms.py`:
* Color inversion
-* DPI adjusting
* Dilation and erosion
* Elastic distortion
* Reducing interline spacing
@@ -100,13 +91,6 @@ The following configuration is used by default when using the `teklia-dan train
"order": "random",
"proba": 0.9,
"augmentations": [
- {
- "type": "dpi",
- "proba": 0.1,
- "min_factor": 0.75,
- "max_factor": 1,
- "preserve_ratio": True,
- },
{
"type": "perspective",
"proba": 0.1,
diff --git a/tests/data/training/models/best_0.pt b/tests/data/training/models/best_0.pt
index 1bf2702b..e7ef8402 100644
--- a/tests/data/training/models/best_0.pt
+++ b/tests/data/training/models/best_0.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:3199e188056836ee2b907319c72c24abfb3b83d850dad8951033a63effa89e72
+oid sha256:48871b9082cf3fc61822dda39a2dc73142c9a9e73947333b17911b0283c133c7
size 84773087
diff --git a/tests/data/training/models/last_3.pt b/tests/data/training/models/last_3.pt
index 27e32d82..519615bd 100644
--- a/tests/data/training/models/last_3.pt
+++ b/tests/data/training/models/last_3.pt
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:c62f5090b1ae30e55a758a4d3ce9814c754f336b241438e106a98c630e0c31e6
-size 84773279
+oid sha256:577051cb2b3ef1e3f666c0b831369c19a02f123514baec43b1a1ec0f31e3ad16
+size 84773087
diff --git a/tests/test_training.py b/tests/test_training.py
index b2c4ebbb..f146afba 100644
--- a/tests/test_training.py
+++ b/tests/test_training.py
@@ -16,7 +16,7 @@ from tests.conftest import FIXTURES
"last_3.pt",
{
"nb_chars": 43,
- "cer": 1.2791,
+ "cer": 1.2558,
"nb_words": 9,
"wer": 1.0,
"nb_words_no_punct": 9,
@@ -34,7 +34,7 @@ from tests.conftest import FIXTURES
},
{
"nb_chars": 49,
- "cer": 1.1429,
+ "cer": 1.1224,
"nb_words": 9,
"wer": 1.0,
"nb_words_no_punct": 9,
@@ -99,7 +99,7 @@ def test_train_and_test(
assert torch.allclose(
trained_tensor,
expected_tensor,
- atol=1e-04,
+ atol=1e-03,
)
assert (
trained_model[optimizer_part]["param_groups"]
--
GitLab