Load secrets from backend & local storage

.isort.cfg

@@ -8,4 +8,4 @@ line_length = 88
 
 default_section=FIRSTPARTY
 known_first_party = arkindex,arkindex_common
-known_third_party =PIL,apistar,pytest,requests,setuptools,tenacity,yaml
+known_third_party =PIL,apistar,gnupg,pytest,requests,setuptools,tenacity,yaml

arkindex_worker/worker.py

@@ -6,7 +6,9 @@ import os
 import sys
 import uuid
 from enum import Enum
+from pathlib import Path
 
+import gnupg
 import yaml
 from apistar.exceptions import ErrorResponse
 
@@ -87,16 +89,68 @@ class BaseWorker(object):
                 f"Loaded worker {worker_version['worker']['name']} revision {worker_version['revision']['hash'][0:7]} from API"
             )
             self.config = worker_version["configuration"]["configuration"]
+            required_secrets = worker_version["configuration"].get("secrets", [])
         elif self.args.config:
             # Load config from YAML file
             self.config = yaml.safe_load(self.args.config)
+            required_secrets = self.config.get("secrets", [])
             logger.info(
                 f"Running with local configuration from {self.args.config.name}"
             )
         else:
             self.config = {}
+            required_secrets = []
             logger.warning("Running without any extra configuration")
 
+        # Load all required secrets
+        self.secrets = {name: self.load_secret(name) for name in required_secrets}
+
+    def load_secret(self, name):
+        """Load all secrets described in the worker configuration"""
+        secret = None
+
+        # Load from the backend
+        try:
+            resp = self.api_client.request("RetrieveSecret", name=name)
+            secret = resp["content"]
+            logging.info(f"Loaded API secret {name}")
+        except ErrorResponse as e:
+            logger.warning(f"Secret {name} not available: {e.content}")
+
+        # Load from local developer storage
+        base_dir = Path(os.environ.get("XDG_CONFIG_HOME") or "~/.config").expanduser()
+        path = base_dir / "arkindex" / "secrets" / name
+        if path.exists():
+            logging.debug(f"Loading local secret from {path}")
+
+            try:
+                gpg = gnupg.GPG()
+                decrypted = gpg.decrypt_file(open(path, "rb"))
+                assert (
+                    decrypted.ok
+                ), f"GPG error: {decrypted.status} - {decrypted.stderr}"
+                secret = decrypted.data.decode("utf-8")
+                logging.info(f"Loaded local secret {name}")
+            except Exception as e:
+                logger.error(f"Local secret {name} is not available as {path}: {e}")
+
+        if secret is None:
+            raise Exception(f"Secret {name} is not available on the API nor locally")
+
+        # Parse secret payload, according to its extension
+        _, ext = os.path.splitext(os.path.basename(name))
+        try:
+            ext = ext.lower()
+            if ext == ".json":
+                return json.loads(secret)
+            elif ext in (".yaml", ".yml"):
+                return yaml.safe_load(secret)
+        except Exception as e:
+            logger.error(f"Failed to parse secret {name}: {e}")
+
+        # By default give raw secret payload
+        return secret
+
     def add_arguments(self):
         """Override this method to add argparse argument to this worker"""
 

requirements.txt

 arkindex-client==1.0.2
 Pillow==7.2.0
+python-gnupg==0.4.6
 tenacity==6.2.0

tests/conftest.py

@@ -54,6 +54,7 @@ def mock_worker_version_api(responses):
         "configuration": {
             "docker": {"image": "python:3"},
             "configuration": {"someKey": "someValue"},
+            "secrets": [],
         },
         "revision": {
             "hash": "deadbeef1234",

tests/test_base_worker.py

@@ -4,6 +4,10 @@ import os
 import sys
 from pathlib import Path
 
+import gnupg
+import pytest
+
+from arkindex.mock import MockApiClient
 from arkindex_worker import logger
 from arkindex_worker.worker import BaseWorker
 
@@ -101,3 +105,107 @@ def test_cli_arg_verbose_given(mocker, mock_worker_version_api):
     assert worker.config == {"someKey": "someValue"}  # from API
 
     logger.setLevel(logging.NOTSET)
+
+
+def test_load_missing_secret():
+    worker = BaseWorker()
+    worker.api_client = MockApiClient()
+
+    with pytest.raises(
+        Exception, match="Secret missing/secret is not available on the API nor locally"
+    ):
+        worker.load_secret("missing/secret")
+
+
+def test_load_remote_secret():
+    worker = BaseWorker()
+    worker.api_client = MockApiClient()
+    worker.api_client.add_response(
+        "RetrieveSecret",
+        name="testRemote",
+        response={"content": "this is a secret value !"},
+    )
+
+    assert worker.load_secret("testRemote") == "this is a secret value !"
+
+    # The one mocked call has been used
+    assert len(worker.api_client.history) == 1
+    assert len(worker.api_client.responses) == 0
+
+
+def test_load_json_secret():
+    worker = BaseWorker()
+    worker.api_client = MockApiClient()
+    worker.api_client.add_response(
+        "RetrieveSecret",
+        name="path/to/file.json",
+        response={"content": '{"key": "value", "number": 42}'},
+    )
+
+    assert worker.load_secret("path/to/file.json") == {
+        "key": "value",
+        "number": 42,
+    }
+
+    # The one mocked call has been used
+    assert len(worker.api_client.history) == 1
+    assert len(worker.api_client.responses) == 0
+
+
+def test_load_yaml_secret():
+    worker = BaseWorker()
+    worker.api_client = MockApiClient()
+    worker.api_client.add_response(
+        "RetrieveSecret",
+        name="path/to/file.yaml",
+        response={
+            "content": """---
+somekey: value
+aList:
+  - A
+  - B
+  - C
+struct:
+ level:
+   X
+"""
+        },
+    )
+
+    assert worker.load_secret("path/to/file.yaml") == {
+        "aList": ["A", "B", "C"],
+        "somekey": "value",
+        "struct": {"level": "X"},
+    }
+
+    # The one mocked call has been used
+    assert len(worker.api_client.history) == 1
+    assert len(worker.api_client.responses) == 0
+
+
+def test_load_local_secret(monkeypatch, tmpdir):
+    # Setup arkindex config dir in a temp directory
+    monkeypatch.setenv("XDG_CONFIG_HOME", str(tmpdir))
+
+    # Write a dummy secret
+    secrets_dir = tmpdir / "arkindex" / "secrets"
+    os.makedirs(secrets_dir)
+    secret = secrets_dir / "testLocal"
+    secret.write_text("this is a local secret value", encoding="utf-8")
+
+    # Mock GPG decryption
+    class GpgDecrypt(object):
+        def __init__(self, fd):
+            self.ok = True
+            self.data = fd.read()
+
+    monkeypatch.setattr(gnupg.GPG, "decrypt_file", lambda gpg, f: GpgDecrypt(f))
+
+    worker = BaseWorker()
+    worker.api_client = MockApiClient()
+
+    assert worker.load_secret("testLocal") == "this is a local secret value"
+
+    # The remote api is checked first
+    assert len(worker.api_client.history) == 1
+    assert worker.api_client.history[0].operation == "RetrieveSecret"