.gitlab-ci.yml

@@ -151,7 +151,7 @@ docs-stop-surge:
 
 bump-python-deps:
   stage: deploy
-  image: registry.gitlab.com/teklia/devops:latest
+  image: registry.gitlab.teklia.com/infra/devops:latest
 
   only:
     - schedules
@@ -161,7 +161,7 @@ bump-python-deps:
 
 release-notes:
   stage: deploy
-  image: registry.gitlab.com/teklia/devops:latest
+  image: registry.gitlab.teklia.com/infra/devops:latest
 
   rules:
     - if: '$CI_COMMIT_TAG'

dan/datasets/extract/arkindex.py

@@ -259,7 +259,7 @@ class ArkindexExtractor:
 
         except Exception as e:
             raise ImageDownloadError(
-                split=split, path=str(destination), url=download_url, exc=e
+                split=split, path=destination, url=download_url, exc=e
             )
 
     def format_text(self, text: str, charset: Optional[set] = None):

dan/datasets/extract/exceptions.py

 # -*- coding: utf-8 -*-
+from pathlib import Path
 
 
 class ProcessingError(Exception):
@@ -26,13 +27,13 @@ class ImageDownloadError(Exception):
     """
 
     def __init__(
-        self, split: str, path: str, url: str, exc: Exception, *args: object
+        self, split: str, path: Path, url: str, exc: Exception, *args: object
     ) -> None:
         super().__init__(*args)
         self.split: str = split
-        self.path: str = path
+        self.path: str = str(path)
         self.url: str = url
-        self.message = str(exc)
+        self.message = f"{str(exc)} for element {path.stem}"
 
 
 class NoTranscriptionError(ElementProcessingError):

dan/datasets/extract/utils.py

@@ -33,7 +33,7 @@ TRIM_RETURN_REGEX = re.compile(r"[\r\n]+")
 
 
 def _retry_log(retry_state, *args, **kwargs):
-    logger.debug(
+    logger.warning(
         f"Request to {retry_state.args[0]} failed ({repr(retry_state.outcome.exception())}), "
         f"retrying in {retry_state.idle_for} seconds"
     )

tests/test_extract.py

@@ -5,6 +5,7 @@ import logging
 import pickle
 import re
 from operator import attrgetter, methodcaller
+from pathlib import Path
 from typing import NamedTuple
 from unittest.mock import patch
 
@@ -701,7 +702,7 @@ def test_download_image_error(iiif_url, caplog, capsys):
         "split": "train",
         "polygon": [],
         "image_url": "deadbeef",
-        "destination": "/dev/null",
+        "destination": Path("/dev/null"),
     }
     # Make download_image crash
     iiif_url.return_value = BoundingBox(0, 0, 0, 0), task["image_url"]
@@ -723,7 +724,7 @@ def test_download_image_error(iiif_url, caplog, capsys):
     extractor.tasks = [task]
 
     # Add the key in data
-    extractor.data[task["split"]][task["destination"]] = "deadbeefdata"
+    extractor.data[task["split"]][str(task["destination"])] = "deadbeefdata"
 
     extractor.download_images()
 
@@ -738,10 +739,10 @@ def test_download_image_error(iiif_url, caplog, capsys):
 
     # Check stdout
     captured = capsys.readouterr()
-    assert captured.out == "deadbeef: Image URL must be HTTP(S)\n"
+    assert captured.out == "deadbeef: Image URL must be HTTP(S) for element null\n"
 
 
-def test_download_image_error_try_max(responses):
+def test_download_image_error_try_max(responses, caplog):
     # An image's URL
     url = (
         "https://blabla.com/iiif/2/image_path.jpg/231,699,2789,3659/full/0/default.jpg"
@@ -774,6 +775,12 @@ def test_download_image_error_try_max(responses):
         fixed_url
     ]
 
+    # Check error log
+    assert len(caplog.record_tuples) == 2
+
+    # We should only have WARNING levels
+    assert set(level for _, level, _ in caplog.record_tuples) == {logging.WARNING}
+
 
 @pytest.mark.parametrize("allow_empty", (True, False))
 def test_empty_transcription(allow_empty, mock_database):