From 4b4e2a5525cc991b0f0a39ace4690c3c94fddd7e Mon Sep 17 00:00:00 2001
From: Bastien Abadie <bastien@nextcairn.com>
Date: Wed, 30 Sep 2020 13:20:51 +0000
Subject: [PATCH] Load secrets from backend & local storage
---
.isort.cfg | 2 +-
arkindex_worker/worker.py | 54 +++++++++++++++++++
requirements.txt | 1 +
tests/conftest.py | 1 +
tests/test_base_worker.py | 108 ++++++++++++++++++++++++++++++++++++++
5 files changed, 165 insertions(+), 1 deletion(-)
diff --git a/.isort.cfg b/.isort.cfg
index 7fa1b117..5bc2f237 100644
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -8,4 +8,4 @@ line_length = 88
default_section=FIRSTPARTY
known_first_party = arkindex,arkindex_common
-known_third_party =PIL,apistar,pytest,requests,setuptools,tenacity,yaml
+known_third_party =PIL,apistar,gnupg,pytest,requests,setuptools,tenacity,yaml
diff --git a/arkindex_worker/worker.py b/arkindex_worker/worker.py
index 664de946..be46a870 100644
--- a/arkindex_worker/worker.py
+++ b/arkindex_worker/worker.py
@@ -6,7 +6,9 @@ import os
import sys
import uuid
from enum import Enum
+from pathlib import Path
+import gnupg
import yaml
from apistar.exceptions import ErrorResponse
@@ -87,16 +89,68 @@ class BaseWorker(object):
f"Loaded worker {worker_version['worker']['name']} revision {worker_version['revision']['hash'][0:7]} from API"
)
self.config = worker_version["configuration"]["configuration"]
+ required_secrets = worker_version["configuration"].get("secrets", [])
elif self.args.config:
# Load config from YAML file
self.config = yaml.safe_load(self.args.config)
+ required_secrets = self.config.get("secrets", [])
logger.info(
f"Running with local configuration from {self.args.config.name}"
)
else:
self.config = {}
+ required_secrets = []
logger.warning("Running without any extra configuration")
+ # Load all required secrets
+ self.secrets = {name: self.load_secret(name) for name in required_secrets}
+
+ def load_secret(self, name):
+ """Load all secrets described in the worker configuration"""
+ secret = None
+
+ # Load from the backend
+ try:
+ resp = self.api_client.request("RetrieveSecret", name=name)
+ secret = resp["content"]
+ logging.info(f"Loaded API secret {name}")
+ except ErrorResponse as e:
+ logger.warning(f"Secret {name} not available: {e.content}")
+
+ # Load from local developer storage
+ base_dir = Path(os.environ.get("XDG_CONFIG_HOME") or "~/.config").expanduser()
+ path = base_dir / "arkindex" / "secrets" / name
+ if path.exists():
+ logging.debug(f"Loading local secret from {path}")
+
+ try:
+ gpg = gnupg.GPG()
+ decrypted = gpg.decrypt_file(open(path, "rb"))
+ assert (
+ decrypted.ok
+ ), f"GPG error: {decrypted.status} - {decrypted.stderr}"
+ secret = decrypted.data.decode("utf-8")
+ logging.info(f"Loaded local secret {name}")
+ except Exception as e:
+ logger.error(f"Local secret {name} is not available as {path}: {e}")
+
+ if secret is None:
+ raise Exception(f"Secret {name} is not available on the API nor locally")
+
+ # Parse secret payload, according to its extension
+ _, ext = os.path.splitext(os.path.basename(name))
+ try:
+ ext = ext.lower()
+ if ext == ".json":
+ return json.loads(secret)
+ elif ext in (".yaml", ".yml"):
+ return yaml.safe_load(secret)
+ except Exception as e:
+ logger.error(f"Failed to parse secret {name}: {e}")
+
+ # By default give raw secret payload
+ return secret
+
def add_arguments(self):
"""Override this method to add argparse argument to this worker"""
diff --git a/requirements.txt b/requirements.txt
index 827a074e..5ae91b59 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
arkindex-client==1.0.2
Pillow==7.2.0
+python-gnupg==0.4.6
tenacity==6.2.0
diff --git a/tests/conftest.py b/tests/conftest.py
index 672f51c3..b293eee9 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -54,6 +54,7 @@ def mock_worker_version_api(responses):
"configuration": {
"docker": {"image": "python:3"},
"configuration": {"someKey": "someValue"},
+ "secrets": [],
},
"revision": {
"hash": "deadbeef1234",
diff --git a/tests/test_base_worker.py b/tests/test_base_worker.py
index bb329321..dc39b4c0 100644
--- a/tests/test_base_worker.py
+++ b/tests/test_base_worker.py
@@ -4,6 +4,10 @@ import os
import sys
from pathlib import Path
+import gnupg
+import pytest
+
+from arkindex.mock import MockApiClient
from arkindex_worker import logger
from arkindex_worker.worker import BaseWorker
@@ -101,3 +105,107 @@ def test_cli_arg_verbose_given(mocker, mock_worker_version_api):
assert worker.config == {"someKey": "someValue"} # from API
logger.setLevel(logging.NOTSET)
+
+
+def test_load_missing_secret():
+ worker = BaseWorker()
+ worker.api_client = MockApiClient()
+
+ with pytest.raises(
+ Exception, match="Secret missing/secret is not available on the API nor locally"
+ ):
+ worker.load_secret("missing/secret")
+
+
+def test_load_remote_secret():
+ worker = BaseWorker()
+ worker.api_client = MockApiClient()
+ worker.api_client.add_response(
+ "RetrieveSecret",
+ name="testRemote",
+ response={"content": "this is a secret value !"},
+ )
+
+ assert worker.load_secret("testRemote") == "this is a secret value !"
+
+ # The one mocked call has been used
+ assert len(worker.api_client.history) == 1
+ assert len(worker.api_client.responses) == 0
+
+
+def test_load_json_secret():
+ worker = BaseWorker()
+ worker.api_client = MockApiClient()
+ worker.api_client.add_response(
+ "RetrieveSecret",
+ name="path/to/file.json",
+ response={"content": '{"key": "value", "number": 42}'},
+ )
+
+ assert worker.load_secret("path/to/file.json") == {
+ "key": "value",
+ "number": 42,
+ }
+
+ # The one mocked call has been used
+ assert len(worker.api_client.history) == 1
+ assert len(worker.api_client.responses) == 0
+
+
+def test_load_yaml_secret():
+ worker = BaseWorker()
+ worker.api_client = MockApiClient()
+ worker.api_client.add_response(
+ "RetrieveSecret",
+ name="path/to/file.yaml",
+ response={
+ "content": """---
+somekey: value
+aList:
+ - A
+ - B
+ - C
+struct:
+ level:
+ X
+"""
+ },
+ )
+
+ assert worker.load_secret("path/to/file.yaml") == {
+ "aList": ["A", "B", "C"],
+ "somekey": "value",
+ "struct": {"level": "X"},
+ }
+
+ # The one mocked call has been used
+ assert len(worker.api_client.history) == 1
+ assert len(worker.api_client.responses) == 0
+
+
+def test_load_local_secret(monkeypatch, tmpdir):
+ # Setup arkindex config dir in a temp directory
+ monkeypatch.setenv("XDG_CONFIG_HOME", str(tmpdir))
+
+ # Write a dummy secret
+ secrets_dir = tmpdir / "arkindex" / "secrets"
+ os.makedirs(secrets_dir)
+ secret = secrets_dir / "testLocal"
+ secret.write_text("this is a local secret value", encoding="utf-8")
+
+ # Mock GPG decryption
+ class GpgDecrypt(object):
+ def __init__(self, fd):
+ self.ok = True
+ self.data = fd.read()
+
+ monkeypatch.setattr(gnupg.GPG, "decrypt_file", lambda gpg, f: GpgDecrypt(f))
+
+ worker = BaseWorker()
+ worker.api_client = MockApiClient()
+
+ assert worker.load_secret("testLocal") == "this is a local secret value"
+
+ # The remote api is checked first
+ assert len(worker.api_client.history) == 1
+ assert worker.api_client.history[0].operation == "RetrieveSecret"
--
GitLab