Do not limit training by time

Closes #133 (closed)

dan/manager/training.py

@@ -33,7 +33,6 @@ class GenericTrainingManager:
         self.params = params
         self.dropout_scheduler = None
         self.models = {}
-        self.begin_time = None
         self.dataset = None
         self.dataset_name = list(self.params["dataset_params"]["datasets"].values())[0]
         self.paths = None
@@ -534,7 +533,6 @@ class GenericTrainingManager:
             self.writer = SummaryWriter(self.paths["results"])
             self.save_params()
         # init variables
-        self.begin_time = time()
         nb_epochs = self.params["training_params"]["max_nb_epochs"]
         metric_names = self.params["training_params"]["train_metrics"]
 
@@ -547,9 +545,6 @@ class GenericTrainingManager:
             self.init_curriculum()
         # perform epochs
         for num_epoch in range(self.latest_epoch + 1, nb_epochs):
-            # Check maximum training time stop condition
-            if time() - self.begin_time > 3600 * 24 * 1.9:
-                break
             # set models trainable
             for model_name in self.models.keys():
                 self.models[model_name].train()