From 3aba05edfeb625249a966fa2e8ea33c4990587c0 Mon Sep 17 00:00:00 2001
From: Yoann Schneider <yschneider@teklia.com>
Date: Wed, 15 Feb 2023 16:32:03 +0100
Subject: [PATCH] retry check
---
dan/mlflow.py | 9 ++++++++-
dan/ocr/document/train.py | 5 +++++
2 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/dan/mlflow.py b/dan/mlflow.py
index 9fdd4a65..366a119a 100644
--- a/dan/mlflow.py
+++ b/dan/mlflow.py
@@ -3,6 +3,8 @@ import os
from contextlib import contextmanager
import mlflow
+from mlflow.environment_variables import MLFLOW_HTTP_REQUEST_MAX_RETRIES
+
import requests
from dan import logger
@@ -16,7 +18,7 @@ def make_mlflow_request(mlflow_method, *args, **kwargs):
mlflow_method(*args, **kwargs)
except requests.exceptions.ConnectionError as e:
logger.error(f"Call to `{str(mlflow_method)}` failed with error: {str(e)}")
-
+ raise e
def setup_environment(config: dict):
"""
@@ -34,6 +36,11 @@ def setup_environment(config: dict):
if config_key in config:
os.environ[variable_name] = config[config_key]
+ # Check max retry setting
+ max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get()
+ if max_retries and int(max_retries) <= 1:
+ logger.warning(f"The maximum number of retries for MLflow HTTP requests is set to {max_retries}, which is low. Consider using a higher value.")
+
def logging_metrics(
display_values: dict,
diff --git a/dan/ocr/document/train.py b/dan/ocr/document/train.py
index 72abb8b8..4b005ec4 100644
--- a/dan/ocr/document/train.py
+++ b/dan/ocr/document/train.py
@@ -103,6 +103,11 @@ def get_config():
(dataset_name, "val"),
],
},
+ "test": {
+ "{}-test".format(dataset_name): [
+ (dataset_name, "test"),
+ ],
+ },
"config": {
"load_in_memory": True, # Load all images in CPU memory
"worker_per_gpu": 4, # Num of parallel processes per gpu for data loading
--
GitLab