Skip to content
Snippets Groups Projects

Implement worker

Merged Yoann Schneider requested to merge implem into main
2 files
+ 3
3
Compare changes
  • Side-by-side
  • Inline
Files
2
@@ -96,7 +96,7 @@ class DatasetExtractor(BaseWorker):
database_path = (
Path(self.args.database)
if self.is_read_only
else self.workdir / "db.sqlite"
else self.work_dir / "db.sqlite"
)
if database_path.exists():
database_path.unlink()
@@ -276,7 +276,7 @@ class DatasetExtractor(BaseWorker):
self.process_split(split_name, split_id, image_folder)
# TAR + ZSTD Image folder and store as task artifact
zstd_archive_path = Path(self.work_dir) / "arkindex_data.zstd"
zstd_archive_path = self.work_dir / "arkindex_data.zstd"
logger.info(f"Compressing the images to {zstd_archive_path}")
create_tar_zstd_archive(folder_path=image_folder, destination=zstd_archive_path)
Loading