push for debug

arkindex_worker/worker/__init__.py

@@ -67,7 +67,9 @@ class ElementsWorker(
     """
 
     def __init__(self, description="Arkindex Elements Worker", support_cache=False):
+        print("in __init__ in __init__.py before super")
         super().__init__(description, support_cache)
+        print("in __init__ in __init__.py after super")
 
         # Add mandatory argument to process elements
         self.parser.add_argument(
@@ -87,6 +89,9 @@ class ElementsWorker(
 
         self._worker_version_cache = {}
 
+        # CLI args are stored on the instance so that implementations can access them
+        self.args = self.parser.parse_args()
+
     def list_elements(self):
         """
         List the elements to be processed, either from the CLI arguments or
@@ -136,8 +141,11 @@ class ElementsWorker(
         return self.process_information.get("activity_state") == "ready"
 
     def configure(self):
-        super().configure()
-        super().configure_cache()
+        if self.is_read_only:
+            super().configure_for_developers()
+        else:
+            super().configure()
+            super().configure_cache()
 
         # Add report concerning elements
         self.report = Reporter(

arkindex_worker/worker/base.py

@@ -59,6 +59,39 @@ class BaseWorker(object):
         """
 
         self.parser = argparse.ArgumentParser(description=description)
+        self.parser.add_argument(
+            "-c",
+            "--config",
+            help="Alternative configuration file when running without a Worker Version ID",
+            type=open,
+        )
+        self.parser.add_argument(
+            "-d",
+            "--database",
+            help="Alternative SQLite database to use for worker caching",
+            type=str,
+            default=None,
+        )
+        self.parser.add_argument(
+            "-v",
+            "--verbose",
+            "--debug",
+            help="Display more information on events and errors",
+            action="store_true",
+            default=False,
+        )
+        self.parser.add_argument(
+            "--dev",
+            help=(
+                "Run worker in developer mode. "
+                "Worker will be in read-only state even if a worker_version is supplied. "
+                "ARKINDEX_PROCESS_ID environment variable is not required with this flag."
+            ),
+            action="store_true",
+        )
+
+        # Call potential extra arguments
+        self.add_arguments()
 
         # Setup workdir either in Ponos environment or on host's home
         if os.environ.get("PONOS_DATA"):
@@ -98,84 +131,14 @@ class BaseWorker(object):
         """
         return self.args.dev or self.worker_version_id is None
 
-    def configure(self):
-        """
-        Configure worker using CLI args and environment variables.
-        """
-        self.parser.add_argument(
-            "-c",
-            "--config",
-            help="Alternative configuration file when running without a Worker Version ID",
-            type=open,
-        )
-        self.parser.add_argument(
-            "-d",
-            "--database",
-            help="Alternative SQLite database to use for worker caching",
-            type=str,
-            default=None,
-        )
-        self.parser.add_argument(
-            "-v",
-            "--verbose",
-            "--debug",
-            help="Display more information on events and errors",
-            action="store_true",
-            default=False,
-        )
-        self.parser.add_argument(
-            "--dev",
-            help=(
-                "Run worker in developer mode. "
-                "Worker will be in read-only state even if a worker_version is supplied. "
-                "ARKINDEX_PROCESS_ID environment variable is not required with this flag."
-            ),
-            action="store_true",
-        )
-
-        # Call potential extra arguments
-        self.add_arguments()
-
-        # CLI args are stored on the instance so that implementations can access them
-        self.args = self.parser.parse_args()
-
+    def configure_for_developers(self):
+        print("in configure_for_developers in base.py")
         # Setup logging level if verbose or if ARKINDEX_DEBUG is set to true
         if self.args.verbose or os.environ.get("ARKINDEX_DEBUG"):
             logger.setLevel(logging.DEBUG)
             logger.debug("Debug output enabled")
 
-        # Build Arkindex API client from environment variables
-        self.api_client = ArkindexClient(**options_from_env())
-        logger.debug(f"Setup Arkindex API client on {self.api_client.document.url}")
-
-        # Load process information except in developer mode
-        if not self.args.dev:
-            assert os.environ.get(
-                "ARKINDEX_PROCESS_ID"
-            ), "ARKINDEX_PROCESS_ID environment variable is not defined"
-            self.process_information = self.request(
-                "RetrieveDataImport", id=os.environ["ARKINDEX_PROCESS_ID"]
-            )
-
-        if self.worker_version_id:
-            # Retrieve initial configuration from API
-            worker_version = self.request(
-                "RetrieveWorkerVersion", id=self.worker_version_id
-            )
-            logger.info(
-                f"Loaded worker {worker_version['worker']['name']} revision {worker_version['revision']['hash'][0:7]} from API"
-            )
-            self.config = worker_version["configuration"]["configuration"]
-            if "user_configuration" in worker_version["configuration"]:
-                # Add default values (if set) to user_configuration
-                for key, value in worker_version["configuration"][
-                    "user_configuration"
-                ].items():
-                    if "default" in value:
-                        self.user_configuration[key] = value["default"]
-            self.worker_details = worker_version["worker"]
-            required_secrets = worker_version["configuration"].get("secrets", [])
-        elif self.args.config:
+        if self.args.config:
             # Load config from YAML file
             self.config = yaml.safe_load(self.args.config)
             self.worker_details = {"name": "Local worker"}
@@ -192,6 +155,50 @@ class BaseWorker(object):
         # Load all required secrets
         self.secrets = {name: self.load_secret(name) for name in required_secrets}
 
+    def configure(self):
+        """
+        Configure worker using CLI args and environment variables.
+        """
+        print("in configure in base.py")
+        assert not self.is_read_only
+        # Setup logging level if verbose or if ARKINDEX_DEBUG is set to true
+        if self.args.verbose or os.environ.get("ARKINDEX_DEBUG"):
+            logger.setLevel(logging.DEBUG)
+            logger.debug("Debug output enabled")
+
+        # Build Arkindex API client from environment variables
+        self.api_client = ArkindexClient(**options_from_env())
+        logger.debug(f"Setup Arkindex API client on {self.api_client.document.url}")
+
+        # Load process information except in developer mode
+        assert os.environ.get(
+            "ARKINDEX_PROCESS_ID"
+        ), "ARKINDEX_PROCESS_ID environment variable is not defined"
+        self.process_information = self.request(
+            "RetrieveDataImport", id=os.environ["ARKINDEX_PROCESS_ID"]
+        )
+
+        # Retrieve initial configuration from API
+        worker_version = self.request(
+            "RetrieveWorkerVersion", id=self.worker_version_id
+        )
+        logger.info(
+            f"Loaded worker {worker_version['worker']['name']} revision {worker_version['revision']['hash'][0:7]} from API"
+        )
+        self.config = worker_version["configuration"]["configuration"]
+        if "user_configuration" in worker_version["configuration"]:
+            # Add default values (if set) to user_configuration
+            for key, value in worker_version["configuration"][
+                "user_configuration"
+            ].items():
+                if "default" in value:
+                    self.user_configuration[key] = value["default"]
+        self.worker_details = worker_version["worker"]
+        required_secrets = worker_version["configuration"].get("secrets", [])
+
+        # Load all required secrets
+        self.secrets = {name: self.load_secret(name) for name in required_secrets}
+
         # Load worker run configuration when available and not in dev mode
         if os.environ.get("ARKINDEX_WORKER_RUN_ID") and not self.args.dev:
             worker_run = self.request(

tests/test_base_worker.py

@@ -45,17 +45,17 @@ def test_init_var_ponos_data_given(monkeypatch):
     assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
 
 
-def test_init_var_worker_version_id_missing(monkeypatch, mock_process_api):
+def test_init_var_worker_version_id_missing(monkeypatch):
     monkeypatch.setattr(sys, "argv", ["worker"])
     monkeypatch.delenv("WORKER_VERSION_ID")
     worker = BaseWorker()
-    worker.configure()
+    worker.configure_for_developers()
     assert worker.worker_version_id is None
     assert worker.is_read_only is True
     assert worker.config == {}  # default empty case
 
 
-def test_init_var_worker_local_file(monkeypatch, tmp_path, mock_process_api):
+def test_init_var_worker_local_file(monkeypatch, tmp_path):
     # Build a dummy yaml config file
     config = tmp_path / "config.yml"
     config.write_text("---\nlocalKey: abcdef123")
@@ -63,7 +63,7 @@ def test_init_var_worker_local_file(monkeypatch, tmp_path, mock_process_api):
     monkeypatch.setattr(sys, "argv", ["worker", "-c", str(config)])
     monkeypatch.delenv("WORKER_VERSION_ID")
     worker = BaseWorker()
-    worker.configure()
+    worker.configure_for_developers()
     assert worker.worker_version_id is None
     assert worker.is_read_only is True
     assert worker.config == {"localKey": "abcdef123"}  # Use a local file for devs
@@ -71,18 +71,13 @@ def test_init_var_worker_local_file(monkeypatch, tmp_path, mock_process_api):
     config.unlink()
 
 
-def test_cli_default(mocker, mock_config_api):
+def test_cli_default(mocker):
     worker = BaseWorker()
-    spy = mocker.spy(worker, "add_arguments")
-    assert not spy.called
     assert logger.level == logging.NOTSET
     assert not hasattr(worker, "api_client")
 
     mocker.patch.object(sys, "argv", ["worker"])
-    worker.configure()
-
-    assert spy.called
-    assert spy.call_count == 1
+    worker.configure_for_developers()
     assert not worker.args.verbose
     assert logger.level == logging.NOTSET
     assert worker.api_client
@@ -93,7 +88,7 @@ def test_cli_default(mocker, mock_config_api):
     logger.setLevel(logging.NOTSET)
 
 
-def test_cli_arg_verbose_given(mocker, mock_config_api):
+def test_cli_arg_verbose_given(mocker):
     worker = BaseWorker()
     spy = mocker.spy(worker, "add_arguments")
     assert not spy.called
@@ -115,7 +110,7 @@ def test_cli_arg_verbose_given(mocker, mock_config_api):
     logger.setLevel(logging.NOTSET)
 
 
-def test_cli_envvar_debug_given(mocker, monkeypatch, mock_config_api):
+def test_cli_envvar_debug_given(mocker, monkeypatch):
     worker = BaseWorker()
 
     assert logger.level == logging.NOTSET
@@ -133,7 +128,7 @@ def test_cli_envvar_debug_given(mocker, monkeypatch, mock_config_api):
     logger.setLevel(logging.NOTSET)
 
 
-def test_configure_dev_mode(mocker, monkeypatch, mock_worker_version_api):
+def test_configure_dev_mode(mocker, monkeypatch):
     """
     Configuring a worker in developer mode avoid retrieving process information
     """