diff --git a/.arkindex.yml b/.arkindex.yml index 1f435a4bbdf7f6476795a1af19d462a5ac1b4331..276ff645ba654f67cb3c1f371d58768dc73f6152 100644 --- a/.arkindex.yml +++ b/.arkindex.yml @@ -9,3 +9,16 @@ workers: type: data-extract docker: build: Dockerfile + user_configuration: + train_folder_id: + type: str + title: ID of the training folder on Arkindex + required: true + validation_folder_id: + type: str + title: ID of the validation folder on Arkindex + required: true + test_folder_id: + type: str + title: ID of the testing folder on Arkindex + required: true diff --git a/worker_generic_training_dataset/worker.py b/worker_generic_training_dataset/worker.py index 4c475a4ae5331d271f286a6ab12da4e94002127b..78ba703307d038d7ab9e77e8094a62cc9a07bf79 100644 --- a/worker_generic_training_dataset/worker.py +++ b/worker_generic_training_dataset/worker.py @@ -44,14 +44,13 @@ class DatasetExtractor(BaseWorker): self.args = self.parser.parse_args() if self.is_read_only: super().configure_for_developers() - self.process_information = { - "train_folder_id": "47a0e07b-d07a-4969-aced-44450d132f0d", - "validation_folder_id": "8cbc4b53-9e07-4a72-b4e6-93f7f5b0cbed", - "test_folder_id": "659a37ea-3b26-42f0-8b65-78964f9e433e", - } else: super().configure() + if self.user_configuration: + logger.info("Overriding with user_configuration") + self.config.update(self.user_configuration) + # database arg is mandatory in dev mode assert ( not self.is_read_only or self.args.database is not None @@ -79,15 +78,15 @@ class DatasetExtractor(BaseWorker): """ logger.info("Retrieving information from process_information") - train_folder_id = self.process_information.get("train_folder_id") + train_folder_id = self.config.get("train_folder_id") assert train_folder_id, "A training folder id is necessary to use this worker" self.training_folder_id = UUID(train_folder_id) - val_folder_id = self.process_information.get("validation_folder_id") + val_folder_id = self.config.get("validation_folder_id") assert val_folder_id, "A validation folder id is necessary to use this worker" self.validation_folder_id = UUID(val_folder_id) - test_folder_id = self.process_information.get("test_folder_id") + test_folder_id = self.config.get("test_folder_id") self.testing_folder_id = UUID(test_folder_id) if test_folder_id else None def initialize_database(self):