diff --git a/.isort.cfg b/.isort.cfg
index 7fa1b117d09b99cf7ce2a459b8f071e9af81dd89..5bc2f237603d937d42e4db424ed6e105ca2d0c45 100644
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -8,4 +8,4 @@ line_length = 88
default_section=FIRSTPARTY
known_first_party = arkindex,arkindex_common
-known_third_party =PIL,apistar,pytest,requests,setuptools,tenacity,yaml
+known_third_party =PIL,apistar,gnupg,pytest,requests,setuptools,tenacity,yaml
diff --git a/arkindex_worker/worker.py b/arkindex_worker/worker.py
index 664de9465a97f1157dd44b41626d63b8b540961e..be46a870e6f6d22ba89fd60fea79dce978ade54c 100644
--- a/arkindex_worker/worker.py
+++ b/arkindex_worker/worker.py
@@ -6,7 +6,9 @@ import os
import sys
import uuid
from enum import Enum
+from pathlib import Path
+import gnupg
import yaml
from apistar.exceptions import ErrorResponse
@@ -87,16 +89,68 @@ class BaseWorker(object):
f"Loaded worker {worker_version['worker']['name']} revision {worker_version['revision']['hash'][0:7]} from API"
)
self.config = worker_version["configuration"]["configuration"]
+ required_secrets = worker_version["configuration"].get("secrets", [])
elif self.args.config:
# Load config from YAML file
self.config = yaml.safe_load(self.args.config)
+ required_secrets = self.config.get("secrets", [])
logger.info(
f"Running with local configuration from {self.args.config.name}"
)
else:
self.config = {}
+ required_secrets = []
logger.warning("Running without any extra configuration")
+ # Load all required secrets
+ self.secrets = {name: self.load_secret(name) for name in required_secrets}
+
+ def load_secret(self, name):
+ """Load all secrets described in the worker configuration"""
+ secret = None
+
+ # Load from the backend
+ try:
+ resp = self.api_client.request("RetrieveSecret", name=name)
+ secret = resp["content"]
+ logging.info(f"Loaded API secret {name}")
+ except ErrorResponse as e:
+ logger.warning(f"Secret {name} not available: {e.content}")
+
+ # Load from local developer storage
+ base_dir = Path(os.environ.get("XDG_CONFIG_HOME") or "~/.config").expanduser()
+ path = base_dir / "arkindex" / "secrets" / name
+ if path.exists():
+ logging.debug(f"Loading local secret from {path}")
+
+ try:
+ gpg = gnupg.GPG()
+ decrypted = gpg.decrypt_file(open(path, "rb"))
+ assert (
+ decrypted.ok
+ ), f"GPG error: {decrypted.status} - {decrypted.stderr}"
+ secret = decrypted.data.decode("utf-8")
+ logging.info(f"Loaded local secret {name}")
+ except Exception as e:
+ logger.error(f"Local secret {name} is not available as {path}: {e}")
+
+ if secret is None:
+ raise Exception(f"Secret {name} is not available on the API nor locally")
+
+ # Parse secret payload, according to its extension
+ _, ext = os.path.splitext(os.path.basename(name))
+ try:
+ ext = ext.lower()
+ if ext == ".json":
+ return json.loads(secret)
+ elif ext in (".yaml", ".yml"):
+ return yaml.safe_load(secret)
+ except Exception as e:
+ logger.error(f"Failed to parse secret {name}: {e}")
+
+ # By default give raw secret payload
+ return secret
+
def add_arguments(self):
"""Override this method to add argparse argument to this worker"""
diff --git a/requirements.txt b/requirements.txt
index 827a074e475ff0195689aa3e001f0c2cb55a67cb..5ae91b593903359e9d9cd01b9063fab17b7cf10b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
arkindex-client==1.0.2
Pillow==7.2.0
+python-gnupg==0.4.6
tenacity==6.2.0
diff --git a/tests/conftest.py b/tests/conftest.py
index 672f51c36a9061ebb1432762c103cb38a82670fd..b293eee996d13cfe1398a47627e8c080c224aa51 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -54,6 +54,7 @@ def mock_worker_version_api(responses):
"configuration": {
"docker": {"image": "python:3"},
"configuration": {"someKey": "someValue"},
+ "secrets": [],
},
"revision": {
"hash": "deadbeef1234",
diff --git a/tests/test_base_worker.py b/tests/test_base_worker.py
index bb329321a9aaed5eab224c987bc301172edaed20..dc39b4c0fdb6cc1db7d264bed5ecb820291215f3 100644
--- a/tests/test_base_worker.py
+++ b/tests/test_base_worker.py
@@ -4,6 +4,10 @@ import os
import sys
from pathlib import Path
+import gnupg
+import pytest
+
+from arkindex.mock import MockApiClient
from arkindex_worker import logger
from arkindex_worker.worker import BaseWorker
@@ -101,3 +105,107 @@ def test_cli_arg_verbose_given(mocker, mock_worker_version_api):
assert worker.config == {"someKey": "someValue"} # from API
logger.setLevel(logging.NOTSET)
+
+
+def test_load_missing_secret():
+ worker = BaseWorker()
+ worker.api_client = MockApiClient()
+
+ with pytest.raises(
+ Exception, match="Secret missing/secret is not available on the API nor locally"
+ ):
+ worker.load_secret("missing/secret")
+
+
+def test_load_remote_secret():
+ worker = BaseWorker()
+ worker.api_client = MockApiClient()
+ worker.api_client.add_response(
+ "RetrieveSecret",
+ name="testRemote",
+ response={"content": "this is a secret value !"},
+ )
+
+ assert worker.load_secret("testRemote") == "this is a secret value !"
+
+ # The one mocked call has been used
+ assert len(worker.api_client.history) == 1
+ assert len(worker.api_client.responses) == 0
+
+
+def test_load_json_secret():
+ worker = BaseWorker()
+ worker.api_client = MockApiClient()
+ worker.api_client.add_response(
+ "RetrieveSecret",
+ name="path/to/file.json",
+ response={"content": '{"key": "value", "number": 42}'},
+ )
+
+ assert worker.load_secret("path/to/file.json") == {
+ "key": "value",
+ "number": 42,
+ }
+
+ # The one mocked call has been used
+ assert len(worker.api_client.history) == 1
+ assert len(worker.api_client.responses) == 0
+
+
+def test_load_yaml_secret():
+ worker = BaseWorker()
+ worker.api_client = MockApiClient()
+ worker.api_client.add_response(
+ "RetrieveSecret",
+ name="path/to/file.yaml",
+ response={
+ "content": """---
+somekey: value
+aList:
+ - A
+ - B
+ - C
+struct:
+ level:
+ X
+"""
+ },
+ )
+
+ assert worker.load_secret("path/to/file.yaml") == {
+ "aList": ["A", "B", "C"],
+ "somekey": "value",
+ "struct": {"level": "X"},
+ }
+
+ # The one mocked call has been used
+ assert len(worker.api_client.history) == 1
+ assert len(worker.api_client.responses) == 0
+
+
+def test_load_local_secret(monkeypatch, tmpdir):
+ # Setup arkindex config dir in a temp directory
+ monkeypatch.setenv("XDG_CONFIG_HOME", str(tmpdir))
+
+ # Write a dummy secret
+ secrets_dir = tmpdir / "arkindex" / "secrets"
+ os.makedirs(secrets_dir)
+ secret = secrets_dir / "testLocal"
+ secret.write_text("this is a local secret value", encoding="utf-8")
+
+ # Mock GPG decryption
+ class GpgDecrypt(object):
+ def __init__(self, fd):
+ self.ok = True
+ self.data = fd.read()
+
+ monkeypatch.setattr(gnupg.GPG, "decrypt_file", lambda gpg, f: GpgDecrypt(f))
+
+ worker = BaseWorker()
+ worker.api_client = MockApiClient()
+
+ assert worker.load_secret("testLocal") == "this is a local secret value"
+
+ # The remote api is checked first
+ assert len(worker.api_client.history) == 1
+ assert worker.api_client.history[0].operation == "RetrieveSecret"