diff --git a/dan/datasets/__init__.py b/dan/datasets/__init__.py
index 8c3464c2951253aac6e51060e9ccbce3010fa3b4..12aff639f570eba38d1a8d4c74794007336cc481 100644
--- a/dan/datasets/__init__.py
+++ b/dan/datasets/__init__.py
@@ -4,6 +4,7 @@ Preprocess datasets for training.
"""
from dan.datasets.analyze import add_analyze_parser
+from dan.datasets.download import add_download_parser
from dan.datasets.entities import add_entities_parser
from dan.datasets.extract import add_extract_parser
from dan.datasets.tokens import add_tokens_parser
@@ -18,6 +19,7 @@ def add_dataset_parser(subcommands) -> None:
subcommands = parser.add_subparsers(metavar="subcommand")
add_extract_parser(subcommands)
+ add_download_parser(subcommands)
add_analyze_parser(subcommands)
add_entities_parser(subcommands)
add_tokens_parser(subcommands)
diff --git a/dan/datasets/download/__init__.py b/dan/datasets/download/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4de26f2a5c1c77e1421f601a9b16c385f291a9dc
--- /dev/null
+++ b/dan/datasets/download/__init__.py
@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+"""
+Download images of a dataset from a split extracted by DAN
+"""
+
+import pathlib
+
+from dan.datasets.download.images import run
+
+
+def _valid_image_format(value: str):
+ im_format = value
+ if not im_format.startswith("."):
+ im_format = "." + im_format
+ return im_format
+
+
+def add_download_parser(subcommands) -> None:
+ parser = subcommands.add_parser(
+ "download",
+ description=__doc__,
+ help=__doc__,
+ )
+
+ # Required arguments.
+ parser.add_argument(
+ "--output",
+ type=pathlib.Path,
+ help="Path where the `split.json` file is stored and where the data will be generated.",
+ required=True,
+ )
+
+ parser.add_argument(
+ "--max-width",
+ type=int,
+ help="Images larger than this width will be resized to this width.",
+ )
+
+ parser.add_argument(
+ "--max-height",
+ type=int,
+ help="Images larger than this height will be resized to this height.",
+ )
+
+ # Formatting arguments
+ parser.add_argument(
+ "--image-format",
+ type=_valid_image_format,
+ default=".jpg",
+ help="Images will be saved under this format.",
+ )
+
+ parser.set_defaults(func=run)
diff --git a/dan/datasets/download/exceptions.py b/dan/datasets/download/exceptions.py
new file mode 100644
index 0000000000000000000000000000000000000000..435d30a7c4acf6e01223c5f374e5723852534dfb
--- /dev/null
+++ b/dan/datasets/download/exceptions.py
@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+from pathlib import Path
+
+
+class ImageDownloadError(Exception):
+ """
+ Raised when an element's image could not be downloaded
+ """
+
+ def __init__(
+ self, split: str, path: Path, url: str, exc: Exception, *args: object
+ ) -> None:
+ super().__init__(*args)
+ self.split: str = split
+ self.path: str = str(path)
+ self.url: str = url
+ self.message = f"{str(exc)} for element {path.stem}"
diff --git a/dan/datasets/download/images.py b/dan/datasets/download/images.py
new file mode 100644
index 0000000000000000000000000000000000000000..feaf10fd69aa4233dc25d0184838583dc6360700
--- /dev/null
+++ b/dan/datasets/download/images.py
@@ -0,0 +1,267 @@
+# -*- coding: utf-8 -*-
+
+import json
+import logging
+from collections import defaultdict
+from concurrent.futures import Future, ThreadPoolExecutor
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+import cv2
+import numpy as np
+from PIL import Image
+from tqdm import tqdm
+
+from dan.datasets.download.exceptions import ImageDownloadError
+from dan.datasets.download.utils import download_image, get_bbox
+from line_image_extractor.extractor import extract
+from line_image_extractor.image_utils import (
+ BoundingBox,
+ Extraction,
+ polygon_to_bbox,
+)
+
+IMAGES_DIR = "images" # Subpath to the images directory.
+
+IIIF_URL = "{image_url}/{bbox}/{size}/0/default.jpg"
+# IIIF 2.0 uses `full`
+IIIF_FULL_SIZE = "full"
+
+logger = logging.getLogger(__name__)
+
+
+class ImageDownloader:
+ """
+ Download images from extracted data
+ """
+
+ def __init__(
+ self,
+ output: Path = None,
+ max_width: Optional[int] = None,
+ max_height: Optional[int] = None,
+ image_extension: str = "",
+ ) -> None:
+ self.output = output
+
+ self.max_width = max_width
+ self.max_height = max_height
+ self.image_extension = image_extension
+
+ # Load split file
+ split_file = output / "split.json" if output else None
+ self.split: Dict = (
+ json.loads(split_file.read_text())
+ if split_file and split_file.is_file()
+ else {}
+ )
+ # Create directories
+ for split_name in self.split:
+ Path(output, IMAGES_DIR, split_name).mkdir(parents=True, exist_ok=True)
+
+ self.data: Dict = defaultdict(dict)
+
+ def check_extraction(self, values: dict) -> Optional[str]:
+ # Check image parameters
+ if not (image := values.get("image")):
+ return "Image information not found"
+
+ # Only support `iiif_url` with `polygon` for now
+ if not image.get("iiif_url"):
+ return "Image IIIF URL not found"
+ if not image.get("polygon"):
+ return "Image polygon not found"
+
+ # Check text parameter
+ if values.get("text") is None:
+ return "Text not found"
+
+ def get_iiif_size_arg(self, width: int, height: int) -> str:
+ if (self.max_width is None or width <= self.max_width) and (
+ self.max_height is None or height <= self.max_height
+ ):
+ return IIIF_FULL_SIZE
+
+ bigger_width = self.max_width and width >= self.max_width
+ bigger_height = self.max_height and height >= self.max_height
+
+ if bigger_width and bigger_height:
+ # Resize to the biggest dim to keep aspect ratio
+ # Only resize width is bigger than max size
+ # This ratio tells which dim needs the biggest shrinking
+ ratio = width * self.max_height / (height * self.max_width)
+ return f"{self.max_width}," if ratio > 1 else f",{self.max_height}"
+ elif bigger_width:
+ return f"{self.max_width},"
+ # Only resize height is bigger than max size
+ elif bigger_height:
+ return f",{self.max_height}"
+
+ def build_iiif_url(
+ self, polygon: List[List[int]], image_url: str
+ ) -> Tuple[BoundingBox, str]:
+ bbox = polygon_to_bbox(polygon)
+ size = self.get_iiif_size_arg(width=bbox.width, height=bbox.height)
+ # Rotations are done using the lib
+ return IIIF_URL.format(image_url=image_url, bbox=get_bbox(polygon), size=size)
+
+ def build_tasks(self) -> List[Dict[str, str]]:
+ tasks = []
+ for split, items in self.split.items():
+ # Create directories
+ destination = self.output / IMAGES_DIR / split
+ destination.mkdir(parents=True, exist_ok=True)
+
+ for element_id, values in items.items():
+ image_path = (destination / element_id).with_suffix(
+ self.image_extension
+ )
+
+ error = self.check_extraction(values)
+ if error:
+ logger.warning(f"{image_path}: {error}")
+ continue
+
+ self.data[split][str(image_path)] = values["text"]
+
+ # Create task for multithreading pool if image does not exist yet
+ if image_path.exists():
+ continue
+
+ polygon = values["image"]["polygon"]
+ iiif_url = values["image"]["iiif_url"]
+ tasks.append(
+ {
+ "split": split,
+ "polygon": polygon,
+ "image_url": self.build_iiif_url(polygon, iiif_url),
+ "destination": image_path,
+ }
+ )
+ return tasks
+
+ def get_image(
+ self,
+ split: str,
+ polygon: List[List[int]],
+ image_url: str,
+ destination: Path,
+ ) -> None:
+ """Save the element's image to the given path and applies any image operations needed.
+
+ :param split: Dataset split this image belongs to.
+ :param polygon: Polygon of the processed element.
+ :param image_url: Base IIIF URL of the image.
+ :param destination: Where the image should be saved.
+ """
+ bbox = polygon_to_bbox(polygon)
+ try:
+ img: Image.Image = download_image(image_url)
+
+ # The polygon's coordinate are in the referential of the full image
+ # We need to remove the offset of the bounding rectangle
+ polygon = [(x - bbox.x, y - bbox.y) for x, y in polygon]
+
+ # Normalize bbox
+ bbox = BoundingBox(x=0, y=0, width=bbox.width, height=bbox.height)
+
+ image = extract(
+ img=cv2.cvtColor(np.asarray(img), cv2.COLOR_RGB2BGR),
+ polygon=np.asarray(polygon).clip(0),
+ bbox=bbox,
+ extraction_mode=Extraction.boundingRect,
+ max_deskew_angle=45,
+ )
+
+ cv2.imwrite(str(destination), image)
+
+ except Exception as e:
+ raise ImageDownloadError(
+ split=split, path=destination, url=image_url, exc=e
+ )
+
+ def download_images(self, tasks: List[Dict[str, str]]) -> None:
+ """
+ Execute each image download task in parallel
+
+ :param tasks: List of tasks to execute.
+ """
+ failed_downloads = []
+ with tqdm(
+ desc="Downloading images", total=len(tasks)
+ ) as pbar, ThreadPoolExecutor() as executor:
+
+ def process_future(future: Future):
+ """
+ Callback function called at the end of the thread
+ """
+ # Update the progress bar count
+ pbar.update(1)
+
+ exc = future.exception()
+ if exc is None:
+ # No error
+ return
+ # If failed, tag for removal
+ assert isinstance(exc, ImageDownloadError)
+ # Remove transcription from labels dict
+ del self.data[exc.split][exc.path]
+ # Save tried URL
+ failed_downloads.append((exc.url, exc.message))
+
+ # Submit all tasks
+ for task in tasks:
+ executor.submit(self.get_image, **task).add_done_callback(
+ process_future
+ )
+
+ if failed_downloads:
+ logger.error(f"Failed to download {len(failed_downloads)} image(s).")
+ print(*list(map(": ".join, failed_downloads)), sep="\n")
+
+ def export(self) -> None:
+ """
+ Writes a `labels.json` file containing a mapping of the images that have been correctly uploaded (identified by its path)
+ to the ground-truth transcription (with NER tokens if needed).
+ """
+ (self.output / "labels.json").write_text(
+ json.dumps(
+ self.data,
+ sort_keys=True,
+ indent=4,
+ )
+ )
+
+ def run(self) -> None:
+ """
+ Download the missing images from a `split.json` file and build a `labels.json` file containing
+ a mapping of the images that have been correctly uploaded (identified by its path)
+ to the ground-truth transcription (with NER tokens if needed).
+ """
+ tasks: List[Dict[str, str]] = self.build_tasks()
+ self.download_images(tasks)
+ self.export()
+
+
+def run(
+ output: Path,
+ max_width: Optional[int],
+ max_height: Optional[int],
+ image_format: str,
+):
+ """
+ Download the missing images from a `split.json` file and build a `labels.json` file containing
+ a mapping of the images that have been correctly uploaded (identified by its path)
+ to the ground-truth transcription (with NER tokens if needed).
+
+ :param output: Path where the `split.json` file is stored and where the data will be generated
+ :param max_width: Images larger than this width will be resized to this width
+ :param max_height: Images larger than this height will be resized to this height
+ :param image_format: Images will be saved under this format
+ """
+ ImageDownloader(
+ output=output,
+ max_width=max_width,
+ max_height=max_height,
+ image_extension=image_format,
+ ).run()
diff --git a/dan/datasets/download/utils.py b/dan/datasets/download/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..a332536d5f15d26db592016f68734c580493f934
--- /dev/null
+++ b/dan/datasets/download/utils.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+import logging
+from io import BytesIO
+from typing import List
+
+import requests
+from PIL import Image, ImageOps
+from tenacity import (
+ retry,
+ retry_if_exception_type,
+ stop_after_attempt,
+ wait_exponential,
+)
+
+logger = logging.getLogger(__name__)
+
+# See http://docs.python-requests.org/en/master/user/advanced/#timeouts
+DOWNLOAD_TIMEOUT = (30, 60)
+
+
+def _retry_log(retry_state, *args, **kwargs):
+ logger.warning(
+ f"Request to {retry_state.args[0]} failed ({repr(retry_state.outcome.exception())}), "
+ f"retrying in {retry_state.idle_for} seconds"
+ )
+
+
+@retry(
+ stop=stop_after_attempt(3),
+ wait=wait_exponential(multiplier=2),
+ retry=retry_if_exception_type(requests.RequestException),
+ before_sleep=_retry_log,
+ reraise=True,
+)
+def _retried_request(url: str) -> requests.Response:
+ resp = requests.get(url, timeout=DOWNLOAD_TIMEOUT)
+ resp.raise_for_status()
+ return resp
+
+
+def download_image(url: str) -> Image.Image:
+ """
+ Download an image and open it with Pillow
+ """
+ assert url.startswith("http"), "Image URL must be HTTP(S)"
+
+ # Download the image
+ # Cannot use stream=True as urllib's responses do not support the seek(int) method,
+ # which is explicitly required by Image.open on file-like objects
+ try:
+ resp = _retried_request(url)
+ except requests.HTTPError as e:
+ if "/full/" in url and 400 <= e.response.status_code < 500:
+ # Retry with max instead of full as IIIF size
+ resp = _retried_request(url.replace("/full/", "/max/"))
+ else:
+ raise e
+
+ # Preprocess the image and prepare it for classification
+ image = Image.open(BytesIO(resp.content)).convert("RGB")
+
+ # Do not rotate JPEG images (see https://github.com/python-pillow/Pillow/issues/4703)
+ image = ImageOps.exif_transpose(image)
+
+ logger.debug(
+ "Downloaded image {} - size={}x{}".format(url, image.size[0], image.size[1])
+ )
+
+ return image
+
+
+def get_bbox(polygon: List[List[int]]) -> str:
+ """
+ Returns a comma-separated string of upper left-most pixel, width + height of the image
+ """
+ all_x, all_y = zip(*polygon)
+ x, y = min(all_x), min(all_y)
+ width, height = max(all_x) - x, max(all_y) - y
+ return ",".join(list(map(str, [int(x), int(y), int(width), int(height)])))
diff --git a/dan/datasets/extract/__init__.py b/dan/datasets/extract/__init__.py
index c2634de59741cc05d6dfc998938a8528768266c0..6dbf84bd7c987413206282291e4fe6f1a7212dc3 100644
--- a/dan/datasets/extract/__init__.py
+++ b/dan/datasets/extract/__init__.py
@@ -34,13 +34,6 @@ def validate_char(char):
return char
-def _valid_image_format(value: str):
- im_format = value
- if not im_format.startswith("."):
- im_format = "." + im_format
- return im_format
-
-
def add_extract_parser(subcommands) -> None:
parser = subcommands.add_parser(
"extract",
@@ -131,18 +124,6 @@ def add_extract_parser(subcommands) -> None:
required=False,
)
- parser.add_argument(
- "--max-width",
- type=int,
- help="Images larger than this width will be resized to this width.",
- )
-
- parser.add_argument(
- "--max-height",
- type=int,
- help="Images larger than this height will be resized to this height.",
- )
-
parser.add_argument(
"--subword-vocab-size",
type=int,
@@ -151,13 +132,6 @@ def add_extract_parser(subcommands) -> None:
)
# Formatting arguments
- parser.add_argument(
- "--image-format",
- type=_valid_image_format,
- default=".jpg",
- help="Images will be saved under this format.",
- )
-
parser.add_argument(
"--keep-spaces",
action="store_true",
diff --git a/dan/datasets/extract/arkindex.py b/dan/datasets/extract/arkindex.py
index 31af3e0a25565689786da8c3f236f0a172d6bf98..28fd03c0c3e44045b46991a947378ba8663db3a2 100644
--- a/dan/datasets/extract/arkindex.py
+++ b/dan/datasets/extract/arkindex.py
@@ -5,14 +5,10 @@ import logging
import pickle
import random
from collections import defaultdict
-from concurrent.futures import Future, ThreadPoolExecutor
from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Union
from uuid import UUID
-import cv2
-import numpy as np
-from PIL import Image
from tqdm import tqdm
from arkindex_export import open_database
@@ -23,37 +19,24 @@ from dan.datasets.extract.db import (
get_transcriptions,
)
from dan.datasets.extract.exceptions import (
- ImageDownloadError,
NoTranscriptionError,
ProcessingError,
UnknownTokenInText,
)
from dan.datasets.extract.utils import (
Tokenizer,
- download_image,
entities_to_xml,
- get_bbox,
get_translation_map,
get_vocabulary,
normalize_linebreaks,
normalize_spaces,
)
from dan.utils import LMTokenMapping, parse_tokens
-from line_image_extractor.extractor import extract
-from line_image_extractor.image_utils import (
- BoundingBox,
- Extraction,
- polygon_to_bbox,
-)
-IMAGES_DIR = "images" # Subpath to the images directory.
LANGUAGE_DIR = "language_model" # Subpath to the language model directory.
TRAIN_NAME = "train"
SPLIT_NAMES = [TRAIN_NAME, "val", "test"]
-IIIF_URL = "{image_url}/{bbox}/{size}/0/default.jpg"
-# IIIF 2.0 uses `full`
-IIIF_FULL_SIZE = "full"
logger = logging.getLogger(__name__)
@@ -74,10 +57,7 @@ class ArkindexExtractor:
tokens: Path = None,
transcription_worker_version: Optional[Union[str, bool]] = None,
entity_worker_version: Optional[Union[str, bool]] = None,
- max_width: Optional[int] = None,
- max_height: Optional[int] = None,
keep_spaces: bool = False,
- image_extension: str = "",
allow_empty: bool = False,
subword_vocab_size: int = 1000,
) -> None:
@@ -90,9 +70,6 @@ class ArkindexExtractor:
self.tokens = parse_tokens(tokens) if tokens else {}
self.transcription_worker_version = transcription_worker_version
self.entity_worker_version = entity_worker_version
- self.max_width = max_width
- self.max_height = max_height
- self.image_extension = image_extension
self.allow_empty = allow_empty
self.mapping = LMTokenMapping()
self.keep_spaces = keep_spaces
@@ -104,41 +81,9 @@ class ArkindexExtractor:
self.language_tokens = []
self.language_lexicon = defaultdict(list)
- # Image download tasks to process
- self.tasks: List[Dict[str, str]] = []
-
# NER extraction
self.translation_map: Dict[str, str] | None = get_translation_map(self.tokens)
- def get_iiif_size_arg(self, width: int, height: int) -> str:
- if (self.max_width is None or width <= self.max_width) and (
- self.max_height is None or height <= self.max_height
- ):
- return IIIF_FULL_SIZE
-
- bigger_width = self.max_width and width >= self.max_width
- bigger_height = self.max_height and height >= self.max_height
-
- if bigger_width and bigger_height:
- # Resize to the biggest dim to keep aspect ratio
- # Only resize width is bigger than max size
- # This ratio tells which dim needs the biggest shrinking
- ratio = width * self.max_height / (height * self.max_width)
- return f"{self.max_width}," if ratio > 1 else f",{self.max_height}"
- elif bigger_width:
- return f"{self.max_width},"
- # Only resize height is bigger than max size
- elif bigger_height:
- return f",{self.max_height}"
-
- def build_iiif_url(self, polygon, image_url) -> Tuple[BoundingBox, str]:
- bbox = polygon_to_bbox(json.loads(str(polygon)))
- size = self.get_iiif_size_arg(width=bbox.width, height=bbox.height)
- # Rotations are done using the lib
- return bbox, IIIF_URL.format(
- image_url=image_url, bbox=get_bbox(polygon), size=size
- )
-
def translate(self, text: str):
"""
Use translation map to replace XML tags to actual tokens
@@ -177,47 +122,6 @@ class ArkindexExtractor:
)
)
- def get_image(
- self,
- split: str,
- polygon: List[List[int]],
- image_url: str,
- destination: Path,
- ) -> None:
- """Save the element's image to the given path and applies any image operations needed.
-
- :param split: Dataset split this image belongs to.
- :param polygon: Polygon of the processed element.
- :param image_url: Base IIIF URL of the image.
- :param destination: Where the image should be saved.
- """
- bbox, download_url = self.build_iiif_url(polygon=polygon, image_url=image_url)
- try:
- img: Image.Image = download_image(download_url)
-
- # The polygon's coordinate are in the referential of the full image
- # We need to remove the offset of the bounding rectangle
- polygon = [(x - bbox.x, y - bbox.y) for x, y in polygon]
-
- # Normalize bbox
- bbox = BoundingBox(x=0, y=0, width=bbox.width, height=bbox.height)
-
- image = extract(
- img=cv2.cvtColor(np.asarray(img), cv2.COLOR_RGB2BGR),
- polygon=np.asarray(polygon).clip(0),
- bbox=bbox,
- extraction_mode=Extraction.boundingRect,
- max_deskew_angle=45,
- )
-
- destination.parent.mkdir(parents=True, exist_ok=True)
- cv2.imwrite(str(destination), image)
-
- except Exception as e:
- raise ImageDownloadError(
- split=split, path=destination, url=download_url, exc=e
- )
-
def format_text(self, text: str, charset: Optional[set] = None):
if not self.keep_spaces:
text = normalize_spaces(text)
@@ -234,11 +138,7 @@ class ArkindexExtractor:
)
return text.strip()
- def process_element(
- self,
- element: Element,
- split: str,
- ):
+ def process_element(self, element: Element, split: str):
"""
Extract an element's data and save it to disk.
The output path is directly related to the split of the element.
@@ -248,36 +148,23 @@ class ArkindexExtractor:
if self.unknown_token in text:
raise UnknownTokenInText(element_id=element.id)
- image_path = Path(self.output, IMAGES_DIR, split, element.id).with_suffix(
- self.image_extension
- )
-
- # Create task for multithreading pool if image does not exist yet
- if not image_path.exists():
- self.tasks.append(
- {
- "split": split,
- "polygon": json.loads(str(element.polygon)),
- "image_url": element.image.url,
- "destination": image_path,
- }
- )
-
text = self.format_text(
text,
# Do not replace unknown characters in train split
charset=self.charset if split != TRAIN_NAME else None,
)
- self.data[split][str(image_path)] = text
+ self.data[split][element.id] = {
+ "text": text,
+ "image": {
+ "iiif_url": element.image.url,
+ "polygon": json.loads(element.polygon),
+ },
+ }
+
self.charset = self.charset.union(set(text))
- def process_parent(
- self,
- pbar,
- parent: Element,
- split: str,
- ):
+ def process_parent(self, pbar, parent: Element, split: str):
"""
Extract data from a parent element.
"""
@@ -326,8 +213,10 @@ class ArkindexExtractor:
# Build LM corpus
train_corpus = [
- text.replace(self.mapping.linebreak.display, self.mapping.space.display)
- for text in self.data["train"].values()
+ values["text"].replace(
+ self.mapping.linebreak.display, self.mapping.space.display
+ )
+ for values in self.data[TRAIN_NAME].values()
]
tokenizer = Tokenizer(
@@ -361,7 +250,7 @@ class ArkindexExtractor:
]
def export(self):
- (self.output / "labels.json").write_text(
+ (self.output / "split.json").write_text(
json.dumps(
self.data,
sort_keys=True,
@@ -382,40 +271,6 @@ class ArkindexExtractor:
pickle.dumps(sorted(list(self.charset)))
)
- def download_images(self):
- failed_downloads = []
- with tqdm(
- desc="Downloading images", total=len(self.tasks)
- ) as pbar, ThreadPoolExecutor() as executor:
-
- def process_future(future: Future):
- """
- Callback function called at the end of the thread
- """
- # Update the progress bar count
- pbar.update(1)
-
- exc = future.exception()
- if exc is None:
- # No error
- return
- # If failed, tag for removal
- assert isinstance(exc, ImageDownloadError)
- # Remove transcription from labels dict
- del self.data[exc.split][exc.path]
- # Save tried URL
- failed_downloads.append((exc.url, exc.message))
-
- # Submit all tasks
- for task in self.tasks:
- executor.submit(self.get_image, **task).add_done_callback(
- process_future
- )
-
- if failed_downloads:
- logger.error(f"Failed to download {len(failed_downloads)} image(s).")
- print(*list(map(": ".join, failed_downloads)), sep="\n")
-
def run(self):
# Iterate over the subsets to find the page images and labels.
for folder_id, split in zip(self.folders, SPLIT_NAMES):
@@ -442,7 +297,6 @@ class ArkindexExtractor:
"No data was extracted using the provided export database and parameters."
)
- self.download_images()
self.format_lm_files()
self.export()
@@ -460,9 +314,6 @@ def run(
test_folder: UUID,
transcription_worker_version: Optional[Union[str, bool]],
entity_worker_version: Optional[Union[str, bool]],
- max_width: Optional[int],
- max_height: Optional[int],
- image_format: str,
keep_spaces: bool,
allow_empty: bool,
subword_vocab_size: int,
@@ -473,8 +324,6 @@ def run(
folders = [str(train_folder), str(val_folder), str(test_folder)]
# Create directories
- for split in SPLIT_NAMES:
- Path(output, IMAGES_DIR, split).mkdir(parents=True, exist_ok=True)
Path(output, LANGUAGE_DIR).mkdir(parents=True, exist_ok=True)
ArkindexExtractor(
@@ -487,10 +336,7 @@ def run(
tokens=tokens,
transcription_worker_version=transcription_worker_version,
entity_worker_version=entity_worker_version,
- max_width=max_width,
- max_height=max_height,
keep_spaces=keep_spaces,
- image_extension=image_format,
allow_empty=allow_empty,
subword_vocab_size=subword_vocab_size,
).run()
diff --git a/dan/datasets/extract/exceptions.py b/dan/datasets/extract/exceptions.py
index ef7ba5b9b36809ac013ff2b6d6195d96681fcf08..ad4de1f653960e2132270a86a1486c08a112d100 100644
--- a/dan/datasets/extract/exceptions.py
+++ b/dan/datasets/extract/exceptions.py
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
-from pathlib import Path
class ProcessingError(Exception):
@@ -21,21 +20,6 @@ class ElementProcessingError(ProcessingError):
self.element_id = element_id
-class ImageDownloadError(Exception):
- """
- Raised when an element's image could not be downloaded
- """
-
- def __init__(
- self, split: str, path: Path, url: str, exc: Exception, *args: object
- ) -> None:
- super().__init__(*args)
- self.split: str = split
- self.path: str = str(path)
- self.url: str = url
- self.message = f"{str(exc)} for element {path.stem}"
-
-
class NoTranscriptionError(ElementProcessingError):
"""
Raised when there are no transcriptions on an element
diff --git a/dan/datasets/extract/utils.py b/dan/datasets/extract/utils.py
index ef43c7bd81ca8673299d1ef92c31083525038ba5..7a39107f1678bd5c5c8306490f4a8e2750d89e8d 100644
--- a/dan/datasets/extract/utils.py
+++ b/dan/datasets/extract/utils.py
@@ -4,31 +4,19 @@ import logging
import operator
import re
from dataclasses import dataclass, field
-from io import BytesIO
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Dict, Iterator, List, Optional, Union
-import requests
import sentencepiece as spm
from lxml.etree import Element, SubElement, tostring
from nltk import wordpunct_tokenize
-from PIL import Image, ImageOps
-from tenacity import (
- retry,
- retry_if_exception_type,
- stop_after_attempt,
- wait_exponential,
-)
from arkindex_export import TranscriptionEntity
from dan.utils import EntityType, LMTokenMapping
logger = logging.getLogger(__name__)
-# See http://docs.python-requests.org/en/master/user/advanced/#timeouts
-DOWNLOAD_TIMEOUT = (30, 60)
-
# replace \t with regular space and consecutive spaces
TRIM_SPACE_REGEX = re.compile(r"[\t ]+")
TRIM_RETURN_REGEX = re.compile(r"[\r\n]+")
@@ -42,57 +30,6 @@ ENCODING_MAP = {
}
-def _retry_log(retry_state, *args, **kwargs):
- logger.warning(
- f"Request to {retry_state.args[0]} failed ({repr(retry_state.outcome.exception())}), "
- f"retrying in {retry_state.idle_for} seconds"
- )
-
-
-@retry(
- stop=stop_after_attempt(3),
- wait=wait_exponential(multiplier=2),
- retry=retry_if_exception_type(requests.RequestException),
- before_sleep=_retry_log,
- reraise=True,
-)
-def _retried_request(url):
- resp = requests.get(url, timeout=DOWNLOAD_TIMEOUT)
- resp.raise_for_status()
- return resp
-
-
-def download_image(url):
- """
- Download an image and open it with Pillow
- """
- assert url.startswith("http"), "Image URL must be HTTP(S)"
-
- # Download the image
- # Cannot use stream=True as urllib's responses do not support the seek(int) method,
- # which is explicitly required by Image.open on file-like objects
- try:
- resp = _retried_request(url)
- except requests.HTTPError as e:
- if "/full/" in url and 400 <= e.response.status_code < 500:
- # Retry with max instead of full as IIIF size
- resp = _retried_request(url.replace("/full/", "/max/"))
- else:
- raise e
-
- # Preprocess the image and prepare it for classification
- image = Image.open(BytesIO(resp.content)).convert("RGB")
-
- # Do not rotate JPEG images (see https://github.com/python-pillow/Pillow/issues/4703)
- image = ImageOps.exif_transpose(image)
-
- logger.debug(
- "Downloaded image {} - size={}x{}".format(url, image.size[0], image.size[1])
- )
-
- return image
-
-
def normalize_linebreaks(text: str) -> str:
"""
Remove begin/ending linebreaks.
@@ -111,17 +48,6 @@ def normalize_spaces(text: str) -> str:
return TRIM_SPACE_REGEX.sub(" ", text.strip())
-def get_bbox(polygon: List[List[int]]) -> str:
- """
- Arkindex polygon stored as string
- returns a comma-separated string of upper left-most pixel, width + height of the image
- """
- all_x, all_y = zip(*polygon)
- x, y = min(all_x), min(all_y)
- width, height = max(all_x) - x, max(all_y) - y
- return ",".join(list(map(str, [int(x), int(y), int(width), int(height)])))
-
-
def get_vocabulary(tokenized_text: List[str]) -> set[str]:
"""
Compute set of vocabulary from tokenzied text.
diff --git a/docs/get_started/training.md b/docs/get_started/training.md
index f3ae783e83c8218f9bc64a0b9419785011ade3ca..de06a3b3d897acd8ed20757c746e0f5830de5b71 100644
--- a/docs/get_started/training.md
+++ b/docs/get_started/training.md
@@ -9,8 +9,9 @@ To extract the data, DAN uses an Arkindex export database in SQLite format. You
1. Structure the data into folders (`train` / `val` / `test`) in [Arkindex](https://demo.arkindex.org/).
1. [Export the project](https://doc.arkindex.org/howto/export/) in SQLite format.
1. Extract the data with the [extract command](../usage/datasets/extract.md).
+1. Download images with the [download command](../usage/datasets/download.md).
-This command will extract and format the images and labels needed to train DAN. It will also tokenize the training corpus at character, subword, and word levels, allowing you to combine DAN with an explicit statistical language model to improve performance.
+These commands will extract and format the images and labels needed to train DAN. It will also tokenize the training corpus at character, subword, and word levels, allowing you to combine DAN with an explicit statistical language model to improve performance.
At the end, you should get the following tree structure:
diff --git a/docs/ref/datasets/download/exceptions.md b/docs/ref/datasets/download/exceptions.md
new file mode 100644
index 0000000000000000000000000000000000000000..f6921dcd57ef9846cf9813c1f05608f417446e2e
--- /dev/null
+++ b/docs/ref/datasets/download/exceptions.md
@@ -0,0 +1,3 @@
+# Exceptions
+
+::: dan.datasets.download.exceptions
diff --git a/docs/ref/datasets/download/images.md b/docs/ref/datasets/download/images.md
new file mode 100644
index 0000000000000000000000000000000000000000..d0f97bf5ab56503d7d23a8ee0ac27a560a134a35
--- /dev/null
+++ b/docs/ref/datasets/download/images.md
@@ -0,0 +1,3 @@
+# Image
+
+::: dan.datasets.download.images
diff --git a/docs/ref/datasets/download/index.md b/docs/ref/datasets/download/index.md
new file mode 100644
index 0000000000000000000000000000000000000000..0ab0c174ec87c0ee17cdf00549cb04a2f7feb7ee
--- /dev/null
+++ b/docs/ref/datasets/download/index.md
@@ -0,0 +1,3 @@
+# Download
+
+::: dan.datasets.download
diff --git a/docs/ref/datasets/download/utils.md b/docs/ref/datasets/download/utils.md
new file mode 100644
index 0000000000000000000000000000000000000000..9f431bb1f56e9cdd865ba26784fc906f96d419aa
--- /dev/null
+++ b/docs/ref/datasets/download/utils.md
@@ -0,0 +1,3 @@
+# Utils
+
+::: dan.datasets.download.utils
diff --git a/docs/ref/datasets/extract/exceptions.md b/docs/ref/datasets/extract/exceptions.md
index 7bb4b0d91e7818071b61df08f53bb27b07664b90..669216b0d2e2c46736489da9935c50fb8a0ba1f2 100644
--- a/docs/ref/datasets/extract/exceptions.md
+++ b/docs/ref/datasets/extract/exceptions.md
@@ -1,5 +1,3 @@
# Exceptions
::: dan.datasets.extract.exceptions
-options:
-show_source: false
diff --git a/docs/usage/datasets/download.md b/docs/usage/datasets/download.md
new file mode 100644
index 0000000000000000000000000000000000000000..77f3c7f9f24b73f2023f2292fab293c1f80f797c
--- /dev/null
+++ b/docs/usage/datasets/download.md
@@ -0,0 +1,79 @@
+# Dataset download
+
+## Description
+
+Use the `teklia-dan dataset download` command to download images of a dataset from a split extracted by DAN. This will:
+
+- Generate the images of each element (in the `images/` folder),
+- Create the mapping of the images that have been correctly uploaded (identified by its path) to the ground-truth transcription (with NER tokens if needed) (in the `labels.json` file).
+
+If an image download fails for whatever reason, it won't appear in the transcriptions file. The reason will be printed to stdout at the end of the process. Before trying to download the image, it checks that it wasn't downloaded previously. It is thus safe to run this command twice if a few images failed.
+
+| Parameter | Description | Type | Default |
+| ---------------- | -------------------------------------------------------------------------------- | -------------- | ------- |
+| `--output` | Path where the `split.json` file is stored and where the data will be generated. | `pathlib.Path` | |
+| `--max-width` | Images larger than this width will be resized to this width. | `int` | |
+| `--max-height` | Images larger than this height will be resized to this height. | `int` | |
+| `--image-format` | Images will be saved under this format. | `str` | `.jpg` |
+
+The `--output` directory should have a `split.json` JSON-formatted file with a specific format. A mapping of the elements (identified by its ID) to the image information and the ground-truth transcription (with NER tokens if needed). This file can be generated by the `teklia-dan dataset extract` command. More details in the [dedicated page](./extract.md).
+
+```json
+{
+ "train": {
+ "<element_id>": {
+ "image": {
+ "iiif_url": "https://<iiif_server>/iiif/2/<path>",
+ "polygon": [
+ [37, 191],
+ [37, 339],
+ [767, 339],
+ [767, 191],
+ [37, 191]
+ ]
+ },
+ "text": "â“¢Couâ‡e⇠ⓕBouis â“‘â‡.12.14"
+ },
+ },
+ "val": {},
+ "test": {}
+}
+```
+
+## Examples
+
+### Download full images
+
+To download images from an extracted split, please use the following:
+
+```shell
+teklia-dan dataset download \
+ --output data
+```
+
+### Download resized images
+
+To download cropped images from an extracted split and limit the width and/or the height of images, please use the following:
+
+```shell
+teklia-dan dataset download \
+ --output data \
+ --max-width 1800
+```
+
+or
+
+```shell
+teklia-dan dataset download \
+ --output data \
+ --max-height 2000
+```
+
+or
+
+```shell
+teklia-dan dataset download \
+ --output data \
+ --max-width 1800 \
+ --max-height 2000
+```
diff --git a/docs/usage/datasets/extract.md b/docs/usage/datasets/extract.md
index b4b7992a644ddc54813793519a598c7384f2263b..2ac15b6b90799675ede631ab5214e9e67fecde43 100644
--- a/docs/usage/datasets/extract.md
+++ b/docs/usage/datasets/extract.md
@@ -4,13 +4,10 @@
Use the `teklia-dan dataset extract` command to extract a dataset from an Arkindex export database (SQLite format). This will:
-- Generate the images of each element (in the `images/` folder),
-- Create the mapping of the images (identified by its path) to the ground-truth transcription (with NER tokens if needed) (in the `labels.json` file),
+- Create a mapping of the elements (identified by its ID) to the image information and the ground-truth transcription (with NER tokens if needed) (in the `split.json` file),
- Store the set of characters encountered in the dataset (in the `charset.pkl` file),
- Generate the resources needed to build a n-gram language model at character, subword or word-level with [kenlm](https://github.com/kpu/kenlm) (in the `language_model/` folder).
-If an image download fails for whatever reason, it won't appear in the transcriptions file. The reason will be printed to stdout at the end of the process. Before trying to download the image, it checks that it wasn't downloaded previously. It is thus safe to run this command twice if a few images failed.
-
| Parameter | Description | Type | Default |
| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | ------- |
| `database` | Path to an Arkindex export database in SQLite format. | `pathlib.Path` | |
@@ -25,10 +22,7 @@ If an image download fails for whatever reason, it won't appear in the transcrip
| `--test-folder` | ID of the training folder to extract from Arkindex. | `uuid` | |
| `--transcription-worker-version` | Filter transcriptions by worker_version. Use `manual` for manual filtering. | `str` or `uuid` | |
| `--entity-worker-version` | Filter transcriptions entities by worker_version. Use `manual` for manual filtering | `str` or `uuid` | |
-| `--max-width` | Images larger than this width will be resized to this width. | `int` | |
-| `--max-height` | Images larger than this height will be resized to this height. | `int` | |
| `--keep-spaces` | Transcriptions are trimmed by default. Use this flag to disable this behaviour. | `bool` | `False` |
-| `--image-format` | Images will be saved under this format. | `str` | `.jpg` |
| `--allow-empty` | Elements with no transcriptions are skipped by default. This flag disables this behaviour. | `bool` | `False` |
| `--subword-vocab-size` | Size of the vocabulary used to train the sentencepiece subword tokenizer used to train the optional language model. | `int` | `1000` |
diff --git a/docs/usage/datasets/index.md b/docs/usage/datasets/index.md
index 4fbca80ebda8a2316d3a77583da78fa949c0ea72..130951e9cf35b4ab7423e308bda65cbaf4237afa 100644
--- a/docs/usage/datasets/index.md
+++ b/docs/usage/datasets/index.md
@@ -13,3 +13,6 @@ Two operations are available through subcommands:
`teklia-dan dataset extract`
: To extract a dataset from an [Arkindex export](https://doc.arkindex.org/howto/export/). More details in the [dedicated page](./extract.md).
+
+`teklia-dan dataset download`
+: To download images of a dataset. More details in the [dedicated page](./download.md).
diff --git a/docs/usage/predict/index.md b/docs/usage/predict/index.md
index bc8340d8fb3c17aa8dac3187eb0a3be22d8a25bd..b96149dbafaba253ec694f2d16cf16629aa6fc67 100644
--- a/docs/usage/predict/index.md
+++ b/docs/usage/predict/index.md
@@ -26,14 +26,14 @@ Use the `teklia-dan predict` command to apply a trained DAN model on an image.
| `--start-token` | Use a specific starting token at the beginning of the prediction. Useful when making predictions on different single pages. | `str` | |
| `--use-language-model` | Whether to use an explicit language model to rescore text hypotheses. | `bool` | `False` |
-## Examples
-
-In the following examples the `models` directory should have:
+The `--model` argument expects a directory with the following files:
- a `model.pt` file,
- a `charset.pkl` file,
- a `parameters.yml` file corresponding to the `inference_parameters.yml` file generated during training.
+## Examples
+
### Predict with confidence scores
To run a prediction with confidence scores, run this command:
diff --git a/mkdocs.yml b/mkdocs.yml
index 1a9e00fae86e743ae59bb0ccbbe5398adea96882..611dcb92e7f373e6767ec285954d422028eb05e8 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -65,6 +65,7 @@ nav:
- Dataset entities: usage/datasets/entities.md
- Dataset tokens: usage/datasets/tokens.md
- Dataset extraction: usage/datasets/extract.md
+ - Dataset download: usage/datasets/download.md
- Training:
- usage/train/index.md
- Configuration: usage/train/config.md
@@ -81,6 +82,11 @@ nav:
- Analyze:
- ref/datasets/analyze/index.md
- Statistics: ref/datasets/analyze/statistics.md
+ - Download:
+ - ref/datasets/download/index.md
+ - Images: ref/datasets/download/images.md
+ - Utils: ref/datasets/download/utils.md
+ - Exceptions: ref/datasets/download/exceptions.md
- Entities:
- ref/datasets/entities/index.md
- Extract: ref/datasets/entities/extract.md
diff --git a/tests/conftest.py b/tests/conftest.py
index e629efb9c840f52c3273900b51b3665387b6ad8b..4bc38b0733cc8ea76ed012eefa9a953f2461058c 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -257,5 +257,12 @@ def evaluate_config():
@pytest.fixture
-def prediction_data_path():
- return FIXTURES / "prediction"
+def split_content():
+ splits = json.loads((FIXTURES / "extraction" / "split.json").read_text())
+ for split in splits:
+ for element_id in splits[split]:
+ splits[split][element_id]["image"]["iiif_url"] = splits[split][element_id][
+ "image"
+ ]["iiif_url"].replace("{FIXTURES}", str(FIXTURES))
+
+ return splits
diff --git a/tests/data/extraction/split.json b/tests/data/extraction/split.json
new file mode 100644
index 0000000000000000000000000000000000000000..36a6ce49eb9dc62bd61abe6b94d236651ac10755
--- /dev/null
+++ b/tests/data/extraction/split.json
@@ -0,0 +1,456 @@
+{
+ "test": {
+ "test-page_1-line_1": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/test-page_1-line_1.jpg",
+ "polygon": [
+ [
+ 37,
+ 191
+ ],
+ [
+ 37,
+ 339
+ ],
+ [
+ 767,
+ 339
+ ],
+ [
+ 767,
+ 191
+ ],
+ [
+ 37,
+ 191
+ ]
+ ]
+ },
+ "text": "â“¢Couâ‡e⇠ⓕBouis â“‘â‡.12.14"
+ },
+ "test-page_1-line_2": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/test-page_1-line_2.jpg",
+ "polygon": [
+ [
+ 28,
+ 339
+ ],
+ [
+ 28,
+ 464
+ ],
+ [
+ 767,
+ 464
+ ],
+ [
+ 767,
+ 339
+ ],
+ [
+ 28,
+ 339
+ ]
+ ]
+ },
+ "text": "â“¢â‡outrain â“•Aâ‡olâ‡â‡e â“‘9.4.13"
+ },
+ "test-page_1-line_3": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/test-page_1-line_3.jpg",
+ "polygon": [
+ [
+ 28,
+ 464
+ ],
+ [
+ 28,
+ 614
+ ],
+ [
+ 767,
+ 614
+ ],
+ [
+ 767,
+ 464
+ ],
+ [
+ 28,
+ 464
+ ]
+ ]
+ },
+ "text": "â“¢â‡abale â“•â‡ranâ‡ais â“‘26.3.11"
+ },
+ "test-page_2-line_1": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/test-page_2-line_1.jpg",
+ "polygon": [
+ [
+ 14,
+ 199
+ ],
+ [
+ 14,
+ 330
+ ],
+ [
+ 767,
+ 330
+ ],
+ [
+ 767,
+ 199
+ ],
+ [
+ 14,
+ 199
+ ]
+ ]
+ },
+ "text": "â“¢â‡urosoy â“•Bouis â“‘22â‡4â‡18"
+ },
+ "test-page_2-line_2": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/test-page_2-line_2.jpg",
+ "polygon": [
+ [
+ 16,
+ 330
+ ],
+ [
+ 16,
+ 471
+ ],
+ [
+ 765,
+ 471
+ ],
+ [
+ 765,
+ 330
+ ],
+ [
+ 16,
+ 330
+ ]
+ ]
+ },
+ "text": "â“¢Colaiani â“•Anâ‡els â“‘28.11.1â‡"
+ },
+ "test-page_2-line_3": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/test-page_2-line_3.jpg",
+ "polygon": [
+ [
+ 11,
+ 473
+ ],
+ [
+ 11,
+ 598
+ ],
+ [
+ 772,
+ 598
+ ],
+ [
+ 772,
+ 473
+ ],
+ [
+ 11,
+ 473
+ ]
+ ]
+ },
+ "text": "â“¢Renouar⇠ⓕMaurice â“‘2â‡.â‡.04"
+ }
+ },
+ "train": {
+ "train-page_1-line_1": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/train-page_1-line_1.jpg",
+ "polygon": [
+ [
+ 27,
+ 187
+ ],
+ [
+ 27,
+ 327
+ ],
+ [
+ 754,
+ 327
+ ],
+ [
+ 754,
+ 187
+ ],
+ [
+ 27,
+ 187
+ ]
+ ]
+ },
+ "text": "â“¢Caillet â“•Maurice â“‘28.9.06"
+ },
+ "train-page_1-line_2": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/train-page_1-line_2.jpg",
+ "polygon": [
+ [
+ 28,
+ 328
+ ],
+ [
+ 28,
+ 465
+ ],
+ [
+ 755,
+ 465
+ ],
+ [
+ 755,
+ 328
+ ],
+ [
+ 28,
+ 328
+ ]
+ ]
+ },
+ "text": "â“¢Reboul â“•Jean â“‘30.9.02"
+ },
+ "train-page_1-line_3": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/train-page_1-line_3.jpg",
+ "polygon": [
+ [
+ 23,
+ 463
+ ],
+ [
+ 23,
+ 604
+ ],
+ [
+ 803,
+ 604
+ ],
+ [
+ 803,
+ 463
+ ],
+ [
+ 23,
+ 463
+ ]
+ ]
+ },
+ "text": "â“¢Bareyre â“•Jean â“‘28.3.11"
+ },
+ "train-page_1-line_4": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/train-page_1-line_4.jpg",
+ "polygon": [
+ [
+ 21,
+ 604
+ ],
+ [
+ 21,
+ 743
+ ],
+ [
+ 812,
+ 743
+ ],
+ [
+ 812,
+ 604
+ ],
+ [
+ 21,
+ 604
+ ]
+ ]
+ },
+ "text": "â“¢Roussy â“•Jean â“‘4.11.14"
+ },
+ "train-page_2-line_1": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/train-page_2-line_1.jpg",
+ "polygon": [
+ [
+ 18,
+ 197
+ ],
+ [
+ 18,
+ 340
+ ],
+ [
+ 751,
+ 340
+ ],
+ [
+ 751,
+ 197
+ ],
+ [
+ 18,
+ 197
+ ]
+ ]
+ },
+ "text": "â“¢Marin â“•Marcel â“‘10.8.06"
+ },
+ "train-page_2-line_2": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/train-page_2-line_2.jpg",
+ "polygon": [
+ [
+ 18,
+ 340
+ ],
+ [
+ 18,
+ 476
+ ],
+ [
+ 751,
+ 476
+ ],
+ [
+ 751,
+ 340
+ ],
+ [
+ 18,
+ 340
+ ]
+ ]
+ },
+ "text": "â“¢Amical â“•Eloi â“‘11.10.04"
+ },
+ "train-page_2-line_3": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/train-page_2-line_3.jpg",
+ "polygon": [
+ [
+ 21,
+ 476
+ ],
+ [
+ 21,
+ 615
+ ],
+ [
+ 746,
+ 615
+ ],
+ [
+ 746,
+ 476
+ ],
+ [
+ 21,
+ 476
+ ]
+ ]
+ },
+ "text": "â“¢Biros â“•Mael â“‘30.10.10"
+ }
+ },
+ "val": {
+ "val-page_1-line_1": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/val-page_1-line_1.jpg",
+ "polygon": [
+ [
+ 14,
+ 211
+ ],
+ [
+ 14,
+ 347
+ ],
+ [
+ 755,
+ 347
+ ],
+ [
+ 755,
+ 211
+ ],
+ [
+ 14,
+ 211
+ ]
+ ]
+ },
+ "text": "â“¢Monar⇠ⓕBouis â“‘29â‡â‡â‡04"
+ },
+ "val-page_1-line_2": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/val-page_1-line_2.jpg",
+ "polygon": [
+ [
+ 14,
+ 350
+ ],
+ [
+ 14,
+ 484
+ ],
+ [
+ 748,
+ 484
+ ],
+ [
+ 748,
+ 350
+ ],
+ [
+ 14,
+ 350
+ ]
+ ]
+ },
+ "text": "â“¢Astier â“•Artâ‡ur â“‘11â‡2â‡13"
+ },
+ "val-page_1-line_3": {
+ "image": {
+ "iiif_url": "{FIXTURES}/extraction/images/text_line/val-page_1-line_3.jpg",
+ "polygon": [
+ [
+ 11,
+ 484
+ ],
+ [
+ 11,
+ 622
+ ],
+ [
+ 751,
+ 622
+ ],
+ [
+ 751,
+ 484
+ ],
+ [
+ 11,
+ 484
+ ]
+ ]
+ },
+ "text": "â“¢â‡e â‡lieâ‡er â“•Jules â“‘21â‡11â‡11"
+ }
+ }
+}
diff --git a/tests/test_download.py b/tests/test_download.py
new file mode 100644
index 0000000000000000000000000000000000000000..d5348ec3844ece8b5e827e6db8a8d8491ba8365e
--- /dev/null
+++ b/tests/test_download.py
@@ -0,0 +1,202 @@
+# -*- coding: utf-8 -*-
+
+import json
+import logging
+from operator import attrgetter, methodcaller
+from pathlib import Path
+
+import pytest
+from PIL import Image, ImageChops
+
+from dan.datasets.download.images import IIIF_FULL_SIZE, ImageDownloader
+from dan.datasets.download.utils import download_image
+from line_image_extractor.image_utils import BoundingBox
+from tests import FIXTURES
+
+EXTRACTION_DATA_PATH = FIXTURES / "extraction"
+
+
+@pytest.mark.parametrize(
+ "max_width, max_height, width, height, resize",
+ (
+ (1000, 2000, 900, 800, IIIF_FULL_SIZE),
+ (1000, 2000, 1100, 800, "1000,"),
+ (1000, 2000, 1100, 2800, ",2000"),
+ (1000, 2000, 2000, 3000, "1000,"),
+ ),
+)
+def test_get_iiif_size_arg(max_width, max_height, width, height, resize):
+ assert (
+ ImageDownloader(max_width=max_width, max_height=max_height).get_iiif_size_arg(
+ width=width, height=height
+ )
+ == resize
+ )
+
+
+def test_download(split_content, monkeypatch, tmp_path):
+ # Mock download_image so that it simply opens it with Pillow
+ monkeypatch.setattr(
+ "dan.datasets.download.images.download_image", lambda url: Image.open(url)
+ )
+
+ output = tmp_path / "download"
+ output.mkdir(parents=True, exist_ok=True)
+ (output / "split.json").write_text(json.dumps(split_content))
+
+ def mock_build_image_url(polygon, image_url, *args, **kwargs):
+ # During tests, the image URL is its local path
+ return image_url
+
+ extractor = ImageDownloader(
+ output=output,
+ image_extension=".jpg",
+ )
+ # Mock build_image_url to simply return the path to the image
+ extractor.build_iiif_url = mock_build_image_url
+ extractor.run()
+
+ # Check files
+ IMAGE_DIR = output / "images"
+ TEST_DIR = IMAGE_DIR / "test"
+ TRAIN_DIR = IMAGE_DIR / "train"
+ VAL_DIR = IMAGE_DIR / "val"
+
+ expected_paths = [
+ # Images of test folder
+ TEST_DIR / "test-page_1-line_1.jpg",
+ TEST_DIR / "test-page_1-line_2.jpg",
+ TEST_DIR / "test-page_1-line_3.jpg",
+ TEST_DIR / "test-page_2-line_1.jpg",
+ TEST_DIR / "test-page_2-line_2.jpg",
+ TEST_DIR / "test-page_2-line_3.jpg",
+ # Images of train folder
+ TRAIN_DIR / "train-page_1-line_1.jpg",
+ TRAIN_DIR / "train-page_1-line_2.jpg",
+ TRAIN_DIR / "train-page_1-line_3.jpg",
+ TRAIN_DIR / "train-page_1-line_4.jpg",
+ TRAIN_DIR / "train-page_2-line_1.jpg",
+ TRAIN_DIR / "train-page_2-line_2.jpg",
+ TRAIN_DIR / "train-page_2-line_3.jpg",
+ # Images of val folder
+ VAL_DIR / "val-page_1-line_1.jpg",
+ VAL_DIR / "val-page_1-line_2.jpg",
+ VAL_DIR / "val-page_1-line_3.jpg",
+ output / "labels.json",
+ output / "split.json",
+ ]
+ assert sorted(filter(methodcaller("is_file"), output.rglob("*"))) == expected_paths
+
+ # Check "labels.json"
+ expected_labels = {
+ "test": {
+ str(TEST_DIR / "test-page_1-line_1.jpg"): "â“¢Couâ‡e⇠ⓕBouis â“‘â‡.12.14",
+ str(TEST_DIR / "test-page_1-line_2.jpg"): "â“¢â‡outrain â“•Aâ‡olâ‡â‡e â“‘9.4.13",
+ str(TEST_DIR / "test-page_1-line_3.jpg"): "â“¢â‡abale â“•â‡ranâ‡ais â“‘26.3.11",
+ str(TEST_DIR / "test-page_2-line_1.jpg"): "â“¢â‡urosoy â“•Bouis â“‘22â‡4â‡18",
+ str(TEST_DIR / "test-page_2-line_2.jpg"): "â“¢Colaiani â“•Anâ‡els â“‘28.11.1â‡",
+ str(TEST_DIR / "test-page_2-line_3.jpg"): "â“¢Renouar⇠ⓕMaurice â“‘2â‡.â‡.04",
+ },
+ "train": {
+ str(TRAIN_DIR / "train-page_1-line_1.jpg"): "â“¢Caillet â“•Maurice â“‘28.9.06",
+ str(TRAIN_DIR / "train-page_1-line_2.jpg"): "â“¢Reboul â“•Jean â“‘30.9.02",
+ str(TRAIN_DIR / "train-page_1-line_3.jpg"): "â“¢Bareyre â“•Jean â“‘28.3.11",
+ str(TRAIN_DIR / "train-page_1-line_4.jpg"): "â“¢Roussy â“•Jean â“‘4.11.14",
+ str(TRAIN_DIR / "train-page_2-line_1.jpg"): "â“¢Marin â“•Marcel â“‘10.8.06",
+ str(TRAIN_DIR / "train-page_2-line_2.jpg"): "â“¢Amical â“•Eloi â“‘11.10.04",
+ str(TRAIN_DIR / "train-page_2-line_3.jpg"): "â“¢Biros â“•Mael â“‘30.10.10",
+ },
+ "val": {
+ str(VAL_DIR / "val-page_1-line_1.jpg"): "â“¢Monar⇠ⓕBouis â“‘29â‡â‡â‡04",
+ str(VAL_DIR / "val-page_1-line_2.jpg"): "â“¢Astier â“•Artâ‡ur â“‘11â‡2â‡13",
+ str(VAL_DIR / "val-page_1-line_3.jpg"): "â“¢â‡e â‡lieâ‡er â“•Jules â“‘21â‡11â‡11",
+ },
+ }
+
+ assert json.loads((output / "labels.json").read_text()) == expected_labels
+
+ # Check cropped images
+ for expected_path in expected_paths:
+ if expected_path.suffix != ".jpg":
+ continue
+
+ assert ImageChops.difference(
+ Image.open(
+ EXTRACTION_DATA_PATH / "images" / "text_line" / expected_path.name
+ ),
+ Image.open(expected_path),
+ )
+
+
+def test_download_image_error(monkeypatch, caplog, capsys):
+ task = {
+ "split": "train",
+ "polygon": [],
+ "image_url": "deadbeef",
+ "destination": Path("/dev/null"),
+ }
+ monkeypatch.setattr(
+ "dan.datasets.download.images.polygon_to_bbox",
+ lambda polygon: BoundingBox(0, 0, 0, 0),
+ )
+
+ extractor = ImageDownloader(image_extension=".jpg")
+
+ # Add the key in data
+ extractor.data[task["split"]][str(task["destination"])] = "deadbeefdata"
+
+ # Build a random task
+ extractor.download_images([task])
+
+ # Key should have been removed
+ assert str(task["destination"]) not in extractor.data[task["split"]]
+
+ # Check error log
+ assert len(caplog.record_tuples) == 1
+ _, level, msg = caplog.record_tuples[0]
+ assert level == logging.ERROR
+ assert msg == "Failed to download 1 image(s)."
+
+ # Check stdout
+ captured = capsys.readouterr()
+ assert captured.out == "deadbeef: Image URL must be HTTP(S) for element null\n"
+
+
+def test_download_image_error_try_max(responses, caplog):
+ # An image's URL
+ url = (
+ "https://blabla.com/iiif/2/image_path.jpg/231,699,2789,3659/full/0/default.jpg"
+ )
+ fixed_url = (
+ "https://blabla.com/iiif/2/image_path.jpg/231,699,2789,3659/max/0/default.jpg"
+ )
+
+ # Fake responses error
+ responses.add(
+ responses.GET,
+ url,
+ status=400,
+ )
+ # Correct response with max
+ responses.add(
+ responses.GET,
+ fixed_url,
+ status=200,
+ body=next((FIXTURES / "prediction" / "images").iterdir()).read_bytes(),
+ )
+
+ image = download_image(url)
+
+ assert image
+ # We try 3 times with the first URL
+ # Then the first try with the new URL is successful
+ assert len(responses.calls) == 4
+ assert list(map(attrgetter("request.url"), responses.calls)) == [url] * 3 + [
+ fixed_url
+ ]
+
+ # Check error log
+ assert len(caplog.record_tuples) == 2
+
+ # We should only have WARNING levels
+ assert set(level for _, level, _ in caplog.record_tuples) == {logging.WARNING}
diff --git a/tests/test_extract.py b/tests/test_extract.py
index 265649d043e758abf4176a500be9db9886d41458..5f857044ea1756743e687bb7aef5a07b7a8f97cd 100644
--- a/tests/test_extract.py
+++ b/tests/test_extract.py
@@ -1,19 +1,15 @@
# -*- coding: utf-8 -*-
import json
-import logging
import pickle
import re
-from operator import attrgetter, methodcaller
-from pathlib import Path
+from operator import methodcaller
from typing import NamedTuple
-from unittest.mock import patch
import pytest
-from PIL import Image, ImageChops
from arkindex_export import Element, Transcription, TranscriptionEntity
-from dan.datasets.extract.arkindex import IIIF_FULL_SIZE, ArkindexExtractor
+from dan.datasets.extract.arkindex import ArkindexExtractor
from dan.datasets.extract.db import get_transcription_entities
from dan.datasets.extract.exceptions import (
NoTranscriptionError,
@@ -21,13 +17,11 @@ from dan.datasets.extract.exceptions import (
)
from dan.datasets.extract.utils import (
EntityType,
- download_image,
entities_to_xml,
normalize_linebreaks,
normalize_spaces,
)
from dan.utils import parse_tokens
-from line_image_extractor.image_utils import BoundingBox, polygon_to_bbox
from tests import FIXTURES
EXTRACTION_DATA_PATH = FIXTURES / "extraction"
@@ -51,24 +45,6 @@ def filter_tokens(keys):
return {key: value for key, value in TOKENS.items() if key in keys}
-@pytest.mark.parametrize(
- "max_width, max_height, width, height, resize",
- (
- (1000, 2000, 900, 800, IIIF_FULL_SIZE),
- (1000, 2000, 1100, 800, "1000,"),
- (1000, 2000, 1100, 2800, ",2000"),
- (1000, 2000, 2000, 3000, "1000,"),
- ),
-)
-def test_get_iiif_size_arg(max_width, max_height, width, height, resize):
- assert (
- ArkindexExtractor(max_width=max_width, max_height=max_height).get_iiif_size_arg(
- width=width, height=height
- )
- == resize
- )
-
-
@pytest.mark.parametrize(
"text,trimmed",
(
@@ -255,12 +231,11 @@ def test_process_element_unknown_token_in_text_error(mock_database, tmp_path):
),
),
)
-@patch("dan.datasets.extract.arkindex.download_image")
def test_extract(
- mock_download_image,
load_entities,
keep_spaces,
transcription_entities_worker_version,
+ split_content,
mock_database,
expected_subword_language_corpus,
subword_vocab_size,
@@ -277,10 +252,6 @@ def test_extract(
if token
]
- def mock_build_image_url(image_url, polygon, *args, **kwargs):
- # During tests, the image URL is its local path
- return polygon_to_bbox(json.loads(str(polygon))), image_url
-
extractor = ArkindexExtractor(
folders=["train", "val", "test"],
element_type=["text_line"],
@@ -294,43 +265,12 @@ def test_extract(
if load_entities
else None,
keep_spaces=keep_spaces,
- image_extension=".jpg",
subword_vocab_size=subword_vocab_size,
)
- # Mock build_image_url to simply return the path to the image
- extractor.build_iiif_url = mock_build_image_url
- # Mock download_image so that it simply opens it with Pillow
- mock_download_image.side_effect = Image.open
extractor.run()
- # Check files
- IMAGE_DIR = output / "images"
- TEST_DIR = IMAGE_DIR / "test"
- TRAIN_DIR = IMAGE_DIR / "train"
- VAL_DIR = IMAGE_DIR / "val"
-
expected_paths = [
output / "charset.pkl",
- # Images of test folder
- TEST_DIR / "test-page_1-line_1.jpg",
- TEST_DIR / "test-page_1-line_2.jpg",
- TEST_DIR / "test-page_1-line_3.jpg",
- TEST_DIR / "test-page_2-line_1.jpg",
- TEST_DIR / "test-page_2-line_2.jpg",
- TEST_DIR / "test-page_2-line_3.jpg",
- # Images of train folder
- TRAIN_DIR / "train-page_1-line_1.jpg",
- TRAIN_DIR / "train-page_1-line_2.jpg",
- TRAIN_DIR / "train-page_1-line_3.jpg",
- TRAIN_DIR / "train-page_1-line_4.jpg",
- TRAIN_DIR / "train-page_2-line_1.jpg",
- TRAIN_DIR / "train-page_2-line_2.jpg",
- TRAIN_DIR / "train-page_2-line_3.jpg",
- # Images of val folder
- VAL_DIR / "val-page_1-line_1.jpg",
- VAL_DIR / "val-page_1-line_2.jpg",
- VAL_DIR / "val-page_1-line_3.jpg",
- output / "labels.json",
# Language resources
output / "language_model" / "corpus_characters.txt",
output / "language_model" / "corpus_subwords.txt",
@@ -341,64 +281,42 @@ def test_extract(
output / "language_model" / "subword_tokenizer.model",
output / "language_model" / "subword_tokenizer.vocab",
output / "language_model" / "tokens.txt",
+ output / "split.json",
]
assert sorted(filter(methodcaller("is_file"), output.rglob("*"))) == expected_paths
- # Check "labels.json"
- expected_labels = {
- "test": {
- str(TEST_DIR / "test-page_1-line_1.jpg"): "â“¢Couâ‡e⇠ⓕBouis â“‘â‡.12.14",
- str(TEST_DIR / "test-page_1-line_2.jpg"): "â“¢â‡outrain â“•Aâ‡olâ‡â‡e â“‘9.4.13",
- str(TEST_DIR / "test-page_1-line_3.jpg"): "â“¢â‡abale â“•â‡ranâ‡ais â“‘26.3.11",
- str(TEST_DIR / "test-page_2-line_1.jpg"): "â“¢â‡urosoy â“•Bouis â“‘22â‡4â‡18",
- str(TEST_DIR / "test-page_2-line_2.jpg"): "â“¢Colaiani â“•Anâ‡els â“‘28.11.1â‡",
- str(TEST_DIR / "test-page_2-line_3.jpg"): "â“¢Renouar⇠ⓕMaurice â“‘2â‡.â‡.04",
- },
- "train": {
- str(TRAIN_DIR / "train-page_1-line_1.jpg"): "â“¢Caillet â“•Maurice â“‘28.9.06",
- str(TRAIN_DIR / "train-page_1-line_2.jpg"): "â“¢Reboul â“•Jean â“‘30.9.02",
- str(TRAIN_DIR / "train-page_1-line_3.jpg"): "â“¢Bareyre â“•Jean â“‘28.3.11",
- str(TRAIN_DIR / "train-page_1-line_4.jpg"): "â“¢Roussy â“•Jean â“‘4.11.14",
- str(TRAIN_DIR / "train-page_2-line_1.jpg"): "â“¢Marin â“•Marcel â“‘10.8.06",
- str(TRAIN_DIR / "train-page_2-line_2.jpg"): "â“¢Amical â“•Eloi â“‘11.10.04",
- str(TRAIN_DIR / "train-page_2-line_3.jpg"): "â“¢Biros â“•Mael â“‘30.10.10",
- },
- "val": {
- str(VAL_DIR / "val-page_1-line_1.jpg"): "â“¢Monar⇠ⓕBouis â“‘29â‡â‡â‡04",
- str(VAL_DIR / "val-page_1-line_2.jpg"): "â“¢Astier â“•Artâ‡ur â“‘11â‡2â‡13",
- str(VAL_DIR / "val-page_1-line_3.jpg"): "â“¢â‡e â‡lieâ‡er â“•Jules â“‘21â‡11â‡11",
- },
- }
-
+ # Check "split.json"
# Transcriptions with worker version are in lowercase
if transcription_entities_worker_version:
- for split in expected_labels:
- for path in expected_labels[split]:
- expected_labels[split][path] = expected_labels[split][path].lower()
+ for split in split_content:
+ for element_id in split_content[split]:
+ split_content[split][element_id]["text"] = split_content[split][
+ element_id
+ ]["text"].lower()
# If we do not load entities, remove tokens
if not load_entities:
token_translations = {ord(token): None for token in tokens}
- for split in expected_labels:
- for path in expected_labels[split]:
- expected_labels[split][path] = expected_labels[split][path].translate(
- token_translations
- )
+ for split in split_content:
+ for element_id in split_content[split]:
+ split_content[split][element_id]["text"] = split_content[split][
+ element_id
+ ]["text"].translate(token_translations)
# Replace double spaces with regular space
if not keep_spaces:
- for split in expected_labels:
- for path in expected_labels[split]:
- expected_labels[split][path] = TWO_SPACES_REGEX.sub(
- " ", expected_labels[split][path]
+ for split in split_content:
+ for element_id in split_content[split]:
+ split_content[split][element_id]["text"] = TWO_SPACES_REGEX.sub(
+ " ", split_content[split][element_id]["text"]
)
- assert json.loads((output / "labels.json").read_text()) == expected_labels
+ assert json.loads((output / "split.json").read_text()) == split_content
# Check "charset.pkl"
expected_charset = set()
- for label in expected_labels["train"].values():
- expected_charset.update(set(label))
+ for values in split_content["train"].values():
+ expected_charset.update(set(values["text"]))
if load_entities:
expected_charset.update(tokens)
@@ -497,104 +415,6 @@ def test_extract(
output / "language_model" / "lexicon_subwords.txt"
).read_text() == "\n".join(expected_language_subword_lexicon)
- # Check cropped images
- for expected_path in expected_paths:
- if expected_path.suffix != ".jpg":
- continue
-
- assert ImageChops.difference(
- Image.open(
- EXTRACTION_DATA_PATH / "images" / "text_line" / expected_path.name
- ),
- Image.open(expected_path),
- )
-
-
-@patch("dan.datasets.extract.arkindex.ArkindexExtractor.build_iiif_url")
-def test_download_image_error(iiif_url, caplog, capsys):
- task = {
- "split": "train",
- "polygon": [],
- "image_url": "deadbeef",
- "destination": Path("/dev/null"),
- }
- # Make download_image crash
- iiif_url.return_value = BoundingBox(0, 0, 0, 0), task["image_url"]
-
- extractor = ArkindexExtractor(
- folders=["train", "val", "test"],
- element_type=["text_line"],
- parent_element_type="double_page",
- output=None,
- entity_separators=None,
- tokens=None,
- transcription_worker_version=None,
- entity_worker_version=None,
- keep_spaces=False,
- image_extension=".jpg",
- )
-
- # Build a random task
- extractor.tasks = [task]
-
- # Add the key in data
- extractor.data[task["split"]][str(task["destination"])] = "deadbeefdata"
-
- extractor.download_images()
-
- # Key should have been removed
- assert task["destination"] not in extractor.data[task["split"]]
-
- # Check error log
- assert len(caplog.record_tuples) == 1
- _, level, msg = caplog.record_tuples[0]
- assert level == logging.ERROR
- assert msg == "Failed to download 1 image(s)."
-
- # Check stdout
- captured = capsys.readouterr()
- assert captured.out == "deadbeef: Image URL must be HTTP(S) for element null\n"
-
-
-def test_download_image_error_try_max(responses, caplog):
- # An image's URL
- url = (
- "https://blabla.com/iiif/2/image_path.jpg/231,699,2789,3659/full/0/default.jpg"
- )
- fixed_url = (
- "https://blabla.com/iiif/2/image_path.jpg/231,699,2789,3659/max/0/default.jpg"
- )
-
- # Fake responses error
- responses.add(
- responses.GET,
- url,
- status=400,
- )
- # Correct response with max
- responses.add(
- responses.GET,
- fixed_url,
- status=200,
- body=next((FIXTURES / "prediction" / "images").iterdir()).read_bytes(),
- )
-
- image = download_image(url)
-
- assert image
- # We try 3 times with the first URL
- # Then the first try with the new URL is successful
- assert len(responses.calls) == 4
- assert list(map(attrgetter("request.url"), responses.calls)) == [url] * 3 + [
- fixed_url
- ]
-
- # Check error log
- assert len(caplog.record_tuples) == 2
-
- # We should only have WARNING levels
- assert set(level for _, level, _ in caplog.record_tuples) == {logging.WARNING}
-
@pytest.mark.parametrize("allow_empty", (True, False))
def test_empty_transcription(allow_empty, mock_database):
@@ -608,7 +428,6 @@ def test_empty_transcription(allow_empty, mock_database):
transcription_worker_version=None,
entity_worker_version=None,
keep_spaces=False,
- image_extension=".jpg",
allow_empty=allow_empty,
)
element_no_transcription = Element(id="unknown")