Support extra_files in RQ tasks

closes #1695 (closed)

arkindex/ponos/tasks.py

@@ -91,7 +91,7 @@ def upload_logs(task, text):
         logger.warning(f"Failed uploading logs for task {task}: {e}")
 
 
-def download_extra_files(task) -> None:
+def download_extra_files(task, temp_dir) -> None:
     """
     Download the task's extra_files and store them in a dedicated `extra_files` folder.
     This folder is mounted as `PONOS_DATA_DIR/extra_files` in docker containers.
@@ -106,7 +106,7 @@ def download_extra_files(task) -> None:
             resp.raise_for_status()
 
             # Write file to a specific data directory
-            extra_files_dir = Path(settings.PONOS_DATA_DIR) / "extra_files"
+            extra_files_dir = temp_dir / "extra_files"
             extra_files_dir.mkdir(exist_ok=True)
             with open(extra_files_dir / path_name, "wb") as f:
                 for chunk in resp.iter_content(chunk_size=8192):
@@ -183,7 +183,7 @@ def run_docker_task(client, task, temp_dir):
     if task.extra_files:
         logger.info("Downloading extra_files for task {!s}".format(task))
         try:
-            download_extra_files(task)
+            download_extra_files(task, temp_dir)
         except Exception as e:
             logger.warning(
                 "Failed downloading extra_files for task {!s}: {!s}".format(