Do not crash on missing elements
Refs https://redmine.teklia.com/issues/9806
If the S3 import code encounters a 404 on element creation, it suddenly crashes the whole flow. It should retry instead:
2025-01-13 13:39:39,069 INFO/worker_bucket_import.worker: [189463/374925] Handling node 'Ofpra-JPEG07/FROFPRA_FESP_060/FROFPRA_FESP_060_2280_L.jpg'
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/worker_bucket_import/worker.py", line 278, in create_image
return self.create_iiif_url(iiif_complete_url)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/arkindex_worker/worker/image.py", line 21, in create_iiif_url
return Image(self.api_client.request("CreateIIIFURL", body={"url": url}))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/tenacity/__init__.py", line 289, in wrapped_f
return self(f, *args, **kw)
^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/tenacity/__init__.py", line 379, in __call__
do = self.iter(retry_state=retry_state)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/tenacity/__init__.py", line 314, in iter
return fut.result()
^^^^^^^^^^^^
File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 449, in result
return self.__get_result()
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result
raise self._exception
File "/usr/local/lib/python3.11/site-packages/tenacity/__init__.py", line 382, in __call__
result = fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/arkindex/client.py", line 315, in request
return self.single_request(operation_id, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/arkindex/client.py", line 295, in single_request
return super().request(operation_id, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/apistar/client/client.py", line 108, in request
return self.transport.send(
^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/apistar/client/transports.py", line 77, in send
raise exceptions.ErrorResponse(
apistar.exceptions.ErrorResponse
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/bin/worker-bucket-import", line 8, in <module>
sys.exit(main())
^^^^^^
File "/usr/local/lib/python3.11/site-packages/worker_bucket_import/worker.py", line 515, in main
).run()
^^^^^
File "/usr/local/lib/python3.11/site-packages/worker_bucket_import/worker.py", line 501, in run
self.build_elements(child)
File "/usr/local/lib/python3.11/site-packages/worker_bucket_import/worker.py", line 455, in build_elements
self.build_elements(child_node)
File "/usr/local/lib/python3.11/site-packages/worker_bucket_import/worker.py", line 455, in build_elements
self.build_elements(child_node)
File "/usr/local/lib/python3.11/site-packages/worker_bucket_import/worker.py", line 429, in build_elements
image = self.create_image(node)
^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/worker_bucket_import/worker.py", line 282, in create_image
raise Exception(
Exception: Failed to create image from URL https://europe.iiif.teklia.com/iiif/2/arkhenum-ofpra%2FOfpra-JPEG07%2FFROFPRA_FESP_060%2FFROFPRA_FESP_060_2280_L.jpg: 404 - 404 page not found