Catch `403` and `500` error responses when calling `ValidateModelVersion`

arkindex_worker/worker/training.py

@@ -280,8 +280,17 @@ class TrainingMixin:
                 },
             )
         except ErrorResponse as e:
+            # Temporary fix while waiting for `ValidateModelVersion` refactoring as it can
+            # return errors even when the model version is properly validated
+            if e.status_code in [403, 500]:
+                logger.warning(
+                    f'An error occurred while validating model version {self.model_version["id"]}, please check its status.'
+                )
+                return
+
             if e.status_code != 409:
                 raise e
+
             logger.warning(
                 f"An available model version exists with hash {hash}, using it instead of the pending version."
             )

tests/test_elements_worker/test_training.py

@@ -179,6 +179,41 @@ def test_validate_model_version_not_created(mock_training_worker):
         mock_training_worker.validate_model_version(hash="a", size=1, archive_hash="b")
 
 
+@pytest.mark.parametrize("status_code", [403, 500])
+def test_validate_model_version_catch_errors(
+    mocker, mock_training_worker, caplog, status_code
+):
+    mocker.patch(
+        "arkindex_worker.worker.base.BaseWorker.request.retry.retry", return_value=False
+    )
+
+    mock_training_worker.model_version = {"id": "model_version_id"}
+    args = {
+        "hash": "hash",
+        "archive_hash": "archive_hash",
+        "size": 30,
+    }
+    mock_training_worker.api_client.add_error_response(
+        "ValidateModelVersion",
+        id="model_version_id",
+        status_code=status_code,
+        body=args,
+    )
+
+    mock_training_worker.validate_model_version(**args)
+    assert mock_training_worker.model_version == {"id": "model_version_id"}
+    assert [
+        (level, message)
+        for module, level, message in caplog.record_tuples
+        if module == "arkindex_worker"
+    ] == [
+        (
+            logging.WARNING,
+            "An error occurred while validating model version model_version_id, please check its status.",
+        ),
+    ]
+
+
 @pytest.mark.parametrize("deletion_failed", [True, False])
 def test_validate_model_version_hash_conflict(
     mock_training_worker, default_model_version, caplog, deletion_failed