From c47f283ff7d394316baa829b85393738fa9c20af Mon Sep 17 00:00:00 2001
From: EvaBardou <bardou@teklia.com>
Date: Mon, 13 Nov 2023 12:07:11 +0100
Subject: [PATCH] Update dataset archive extension to .tar.zst
---
worker_generic_training_dataset/worker.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/worker_generic_training_dataset/worker.py b/worker_generic_training_dataset/worker.py
index 43109d8..caef383 100644
--- a/worker_generic_training_dataset/worker.py
+++ b/worker_generic_training_dataset/worker.py
@@ -355,11 +355,11 @@ class DatasetExtractor(DatasetWorker):
casted_elements = list(map(_format_element, elements))
self.process_split(split_name, casted_elements)
- # TAR + ZSTD the cache and the images folder, and store as task artifact
- zstd_archive_path: Path = self.work_dir / f"{dataset.id}.zstd"
- logger.info(f"Compressing the images to {zstd_archive_path}")
+ # TAR + ZST the cache and the images folder, and store as task artifact
+ zst_archive_path: Path = self.work_dir / f"{dataset.id}.tar.zst"
+ logger.info(f"Compressing the images to {zst_archive_path}")
create_tar_zst_archive(
- source=self.data_folder_path, destination=zstd_archive_path
+ source=self.data_folder_path, destination=zst_archive_path
)
self.data_folder.cleanup()
--
GitLab