From 528bc322b2aa5962e88264986d57afb1fc00c610 Mon Sep 17 00:00:00 2001
From: Valentin Rigal <rigal@teklia.com>
Date: Mon, 21 Jun 2021 13:14:18 +0200
Subject: [PATCH] Use the generic error catch and handle exceptions when
patching to the Error state
---
arkindex_worker/worker/__init__.py | 43 ++++++++++++------------------
1 file changed, 17 insertions(+), 26 deletions(-)
diff --git a/arkindex_worker/worker/__init__.py b/arkindex_worker/worker/__init__.py
index 96135192..cc32636d 100644
--- a/arkindex_worker/worker/__init__.py
+++ b/arkindex_worker/worker/__init__.py
@@ -27,11 +27,6 @@ class ActivityState(Enum):
Error = "error"
-class ConflictResponse(ErrorResponse):
- # Custom Exception type for HTTP 409 conflict
- pass
-
-
class ElementsWorker(
BaseWorker,
ClassificationMixin,
@@ -134,24 +129,23 @@ class ElementsWorker(
# Process the element and report its progress if activities are enabled
try:
self.update_activity(element.id, ActivityState.Started)
- except ConflictResponse as e:
- # Skip this element in case of conflict while initializing the activity
+ except ErrorResponse as e:
+ if e.status_code != 409:
+ raise e
+ # 409 conflict error when updating the state of an activity to "started" mean that we
+ # cannot process this element. We assume that the reason is that the state transition
+ # was forbidden i.e. that the activity was already in a started or processed state.
+ # This allow concurrent access to an element activity between multiple processes.
+ # Element should not be counted as failed as it is probably handled somewhere else.
logger.warning(
- f"Cannot start processing element {element.id} due to a conflict, "
- f"another process could have processed it with the same version already: {e.content}"
+ f"Cannot start processing element {element.id} due to a conflict. "
+ f"Another process could have processed it with the same version already."
)
continue
self.process_element(element)
- try:
- self.update_activity(element.id, ActivityState.Processed)
- except ConflictResponse as e:
- # Do not count this element as failed
- logger.warning(
- f"Element {element.id} was processed but its activity could not be updated: {e.content}"
- )
- continue
+ self.update_activity(element.id, ActivityState.Processed)
except Exception as e:
- # Handle errors occurring while retrieving, processing or patching the activity of the element
+ # Handle errors occurring while retrieving, processing or patching the activity for this element.
# Count the element as failed in case the activity update to "started" failed with no conflict.
# This prevent from processing the element
failed += 1
@@ -169,7 +163,11 @@ class ElementsWorker(
exc_info=e if self.args.verbose else None,
)
if element:
- self.update_activity(element.id, ActivityState.Error)
+ # Try to update the activity to error state regardless of the response
+ try:
+ self.update_activity(element.id, ActivityState.Error)
+ except Exception:
+ pass
self.report.error(element_id, e)
# Save report as local artifact
@@ -221,13 +219,6 @@ class ElementsWorker(
logger.warning(
f"Failed to update activity of element {element_id} to {state.value} due to an API error: {e.content}"
)
- # 409 conflict error when updating the state of an activity mean that we cannot
- # process this element. We assume that the reason is that the state transition
- # was forbidden i.e. that the activity was already in a started or processed state.
- # This allow concurrent access to an element activity between multiple processes.
- # Element should not be counted as failed as it is probably handled somewhere else.
- if e.status_code == 409:
- raise ConflictResponse(e)
raise e
logger.debug(f"Updated activity of element {element_id} to {state}")
--
GitLab