Cleanup the downloaded dataset artifact after processing

arkindex_worker/worker/__init__.py

@@ -440,6 +440,8 @@ class DatasetWorker(BaseWorker, DatasetMixin, TaskMixin):
         failed = 0
         for i, item in enumerate(datasets, start=1):
             dataset = None
+            dataset_artifact = None
+
             try:
                 if not self.is_read_only:
                     # Just use the result of list_datasets as the dataset
@@ -465,7 +467,7 @@ class DatasetWorker(BaseWorker, DatasetMixin, TaskMixin):
                     self.update_dataset_state(dataset, DatasetState.Building)
                 else:
                     logger.info(f"Downloading data for {dataset} ({i}/{count})")
-                    self.download_dataset_artifact(dataset)
+                    dataset_artifact = self.download_dataset_artifact(dataset)
 
                 # Process the dataset
                 self.process_dataset(dataset)
@@ -496,6 +498,10 @@ class DatasetWorker(BaseWorker, DatasetMixin, TaskMixin):
                     # Try to update the state to Error regardless of the response
                     with contextlib.suppress(Exception):
                         self.update_dataset_state(dataset, DatasetState.Error)
+            finally:
+                # Cleanup the dataset artifact if it was downloaded, no matter what
+                if dataset_artifact:
+                    dataset_artifact.unlink(missing_ok=True)
 
         if failed:
             logger.error(