Skip to content
Snippets Groups Projects

Rework the worker due to `Dataset` API changes

Merged Eva Bardou requested to merge rework-worker into main
All threads resolved!
@@ -382,7 +382,8 @@ class DatasetExtractor(DatasetWorker):
dataset_sets: list[Set] = list(self.list_sets())
datasets_and_sets: list[tuple[Dataset, list[Set]]] = [
groupby(dataset_sets, attrgetter("dataset.id"))
(dataset, list(sets))
for dataset, sets in groupby(dataset_sets, attrgetter("dataset"))
]
if not datasets_and_sets:
logger.warning("No datasets to process, stopping.")
@@ -425,7 +426,7 @@ class DatasetExtractor(DatasetWorker):
with contextlib.suppress(Exception):
self.update_dataset_state(dataset, DatasetState.Error)
message = f'Ran on {count} dataset{"s"[:count>1]}: {count - failed} completed, {failed} failed'
message = f'Ran on {count} dataset{"s"[:count > 1]}: {count - failed} completed, {failed} failed'
if failed:
logger.error(message)
if failed >= count: # Everything failed!
Loading