Draft: Refactor and implement API version of the worker

Closes #17

worker_generic_training_dataset/__init__.py

@@ -306,10 +306,10 @@ class Extractor(DatasetWorker):
         failed = 0
         for i, (dataset, sets) in enumerate(grouped_sets, start=1):
             try:
-                assert dataset.state in [
-                    DatasetState.Open.value,
-                    DatasetState.Error.value,
-                ], "When generating a new dataset, its state should be Open or Error."
+                # assert dataset.state in [
+                #     DatasetState.Open.value,
+                #     DatasetState.Error.value,
+                # ], "When generating a new dataset, its state should be Open or Error."
 
                 # Update the dataset state to Building
                 logger.info(f"Building {dataset} ({i}/{count})")
@@ -324,7 +324,9 @@ class Extractor(DatasetWorker):
             except Exception as e:
                 # Handle errors occurring while processing or patching the state for this dataset
                 failed += 1
+                import traceback
 
+                traceback.print_exc()
                 if isinstance(e, ErrorResponse):
                     message = f"An API error occurred while processing {dataset}: {e.title} - {e.content}"
                 else: