diff --git a/arkindex_worker/git.py b/arkindex_worker/git.py
index cc904010ba5af8ad0e26f4b01755dfcfd3229ef6..92a38125eb39a1556f2a8f89a3da22ce35e00ae8 100644
--- a/arkindex_worker/git.py
+++ b/arkindex_worker/git.py
@@ -207,7 +207,6 @@ class GitHelper:
in worker.configure() configure the git helper and start the cloning:
```
gitlab = GitlabHelper(...)
- workflow_id = os.environ["ARKINDEX_PROCESS_ID"]
prepare_git_key(...)
self.git_helper = GitHelper(workflow_id=workflow_id, gitlab_helper=gitlab, ...)
self.git_helper.run_clone_in_background()
diff --git a/arkindex_worker/worker/base.py b/arkindex_worker/worker/base.py
index 1e63819edfd7314012a968517db94de6d41bd943..cc78ce07c10e10d9cca460f1b05895274b67a28e 100644
--- a/arkindex_worker/worker/base.py
+++ b/arkindex_worker/worker/base.py
@@ -85,7 +85,6 @@ class BaseWorker(object):
help=(
"Run worker in developer mode. "
"Worker will be in read-only state even if a worker_version is supplied. "
- "ARKINDEX_PROCESS_ID environment variable is not required with this flag."
),
action="store_true",
default=False,
@@ -111,6 +110,11 @@ class BaseWorker(object):
logger.warning(
"Missing WORKER_VERSION_ID environment variable, worker is in read-only mode"
)
+ self.worker_run_id = os.environ.get("ARKINDEX_WORKER_RUN_ID")
+ if not self.worker_run_id:
+ logger.warning(
+ "Missing ARKINDEX_WORKER_RUN_ID environment variable, worker is in read-only mode"
+ )
logger.info(f"Worker will use {self.work_dir} as working directory")
@@ -130,7 +134,11 @@ class BaseWorker(object):
or when no worker version ID is provided.
:rtype: bool
"""
- return self.args.dev or self.worker_version_id is None
+ return (
+ self.args.dev
+ or self.worker_version_id is None
+ or self.worker_run_id is None
+ )
def configure_for_developers(self):
assert self.is_read_only
@@ -170,21 +178,20 @@ class BaseWorker(object):
self.api_client = ArkindexClient(**options_from_env())
logger.debug(f"Setup Arkindex API client on {self.api_client.document.url}")
+ # Load worker run information
+ worker_run = self.request("RetrieveWorkerRun", id=self.worker_run_id)
+
# Load process information
- assert os.environ.get(
- "ARKINDEX_PROCESS_ID"
- ), "ARKINDEX_PROCESS_ID environment variable is not defined"
- self.process_information = self.request(
- "RetrieveDataImport", id=os.environ["ARKINDEX_PROCESS_ID"]
- )
+ self.process_information = worker_run["process"]
- # Retrieve initial configuration from API
- worker_version = self.request(
- "RetrieveWorkerVersion", id=self.worker_version_id
- )
+ # Load worker version information
+ worker_version = worker_run["worker_version"]
+ self.worker_details = worker_version["worker"]
logger.info(
- f"Loaded worker {worker_version['worker']['name']} revision {worker_version['revision']['hash'][0:7]} from API"
+ f"Loaded worker {self.worker_details['name']} revision {worker_version['revision']['hash'][0:7]} from API"
)
+
+ # Retrieve initial configuration from API
self.config = worker_version["configuration"]["configuration"]
if "user_configuration" in worker_version["configuration"]:
# Add default values (if set) to user_configuration
@@ -193,30 +200,20 @@ class BaseWorker(object):
].items():
if "default" in value:
self.user_configuration[key] = value["default"]
- self.worker_details = worker_version["worker"]
- required_secrets = worker_version["configuration"].get("secrets", [])
# Load all required secrets
+ required_secrets = worker_version["configuration"].get("secrets", [])
self.secrets = {name: self.load_secret(name) for name in required_secrets}
# Load worker run configuration when available
- if os.environ.get("ARKINDEX_WORKER_RUN_ID"):
- worker_run = self.request(
- "RetrieveWorkerRun", id=os.environ["ARKINDEX_WORKER_RUN_ID"]
- )
- configuration_id = worker_run.get("configuration_id")
- if configuration_id:
- worker_configuration = self.request(
- "RetrieveWorkerConfiguration", id=configuration_id
- )
- self.user_configuration = worker_configuration.get("configuration")
- if self.user_configuration:
- logger.info("Loaded user configuration from WorkerRun")
-
- # if debug mode is set to true activate debug mode in logger
- if self.user_configuration and self.user_configuration.get("debug"):
- logger.setLevel(logging.DEBUG)
- logger.debug("Debug output enabled")
+ worker_configuration = worker_run.get("configuration")
+ self.user_configuration = worker_configuration.get("configuration")
+ if self.user_configuration:
+ logger.info("Loaded user configuration from WorkerRun")
+ # if debug mode is set to true activate debug mode in logger
+ if self.user_configuration.get("debug"):
+ logger.setLevel(logging.DEBUG)
+ logger.debug("Debug output enabled")
def configure_cache(self):
task_id = os.environ.get("PONOS_TASK")
diff --git a/tests/conftest.py b/tests/conftest.py
index 64af8eb13a10e4299e3b4dcf71841ab821da3cbb..ea16385f33c0e1a0d5dec64c59d273208d58bd07 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -104,119 +104,86 @@ def setup_api(responses, monkeypatch, cache_yaml):
def give_env_variable(request, monkeypatch):
"""Defines required environment variables"""
monkeypatch.setenv("WORKER_VERSION_ID", "12341234-1234-1234-1234-123412341234")
- monkeypatch.setenv("ARKINDEX_PROCESS_ID", "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff")
+ monkeypatch.setenv("ARKINDEX_WORKER_RUN_ID", "56785678-5678-5678-5678-567856785678")
monkeypatch.setenv("ARKINDEX_CORPUS_ID", "11111111-1111-1111-1111-111111111111")
@pytest.fixture
-def mock_config_api(mock_worker_version_api, mock_process_api):
- """Mock all API endpoints required to configure a worker"""
- pass
-
-
-@pytest.fixture
-def mock_worker_version_api(responses):
- """Provide a mock API response to get worker configuration"""
+def mock_worker_run_api(responses):
+ """Provide a mock API response to get worker run information"""
payload = {
- "id": "12341234-1234-1234-1234-123412341234",
- "configuration": {
- "docker": {"image": "python:3"},
- "configuration": {"someKey": "someValue"},
- "secrets": [],
- },
- "revision": {
- "hash": "deadbeef1234",
- "name": "some git revision",
- },
- "docker_image": "python:3",
- "docker_image_name": "python:3",
- "state": "created",
+ "id": "56785678-5678-5678-5678-567856785678",
+ "parents": [],
+ "worker_version_id": "12341234-1234-1234-1234-123412341234",
+ "model_version_id": None,
+ "dataimport_id": "0e6053d5-0d50-41dd-88b6-90907493c433",
"worker": {
"id": "deadbeef-1234-5678-1234-worker",
"name": "Fake worker",
"slug": "fake_worker",
"type": "classifier",
},
- }
- responses.add(
- responses.GET,
- "http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
- status=200,
- body=json.dumps(payload),
- content_type="application/json",
- )
-
-
-@pytest.fixture
-def mock_worker_version_user_configuration_api(responses):
- """
- Provide a mock API response to get a worker configuration
- that includes a `user_configuration`
- """
- payload = {
- "worker": {"id": "1234", "name": "Workerino", "slug": "workerino"},
- "revision": {"hash": "1234lala-lalalalala-lala"},
+ "configuration_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
+ "worker_version": {
+ "id": "12341234-1234-1234-1234-123412341234",
+ "configuration": {
+ "docker": {"image": "python:3"},
+ "configuration": {"someKey": "someValue"},
+ "secrets": [],
+ },
+ "revision": {
+ "hash": "deadbeef1234",
+ "name": "some git revision",
+ },
+ "docker_image": "python:3",
+ "docker_image_name": "python:3",
+ "state": "created",
+ "worker": {
+ "id": "deadbeef-1234-5678-1234-worker",
+ "name": "Fake worker",
+ "slug": "fake_worker",
+ "type": "classifier",
+ },
+ },
"configuration": {
- "configuration": {"param_1": "/some/path/file.pth", "param_2": 12},
- "user_configuration": {
- "param_3": {
- "title": "A Third Parameter",
- "type": "string",
- "default": "Animula vagula blandula",
- },
- "param_4": {"title": "Parameter The Fourth", "type": "int"},
- "param_5": {
- "title": "Parameter 5 (Five)",
- "type": "bool",
- "default": True,
+ "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
+ "name": "string",
+ "configuration": {},
+ },
+ "process": {
+ "name": None,
+ "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
+ "state": "running",
+ "mode": "workers",
+ "corpus": "11111111-1111-1111-1111-111111111111",
+ "workflow": "http://testserver/ponos/v1/workflow/12341234-1234-1234-1234-123412341234/",
+ "files": [],
+ "revision": None,
+ "element": {
+ "id": "1234-deadbeef",
+ "type": "folder",
+ "name": "Test folder",
+ "corpus": {
+ "id": "11111111-1111-1111-1111-111111111111",
+ "name": "John Doe project",
+ "public": False,
},
+ "thumbnail_url": "http://testserver/thumbnail.png",
+ "zone": None,
+ "thumbnail_put_url": "http://testserver/thumbnail.png",
},
+ "folder_type": None,
+ "element_type": "page",
+ "element_name_contains": None,
+ "load_children": True,
+ "use_cache": False,
+ "activity_state": "ready",
},
}
- responses.add(
- responses.GET,
- "http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
- status=200,
- body=json.dumps(payload),
- content_type="application/json",
- )
-
-@pytest.fixture
-def mock_process_api(responses):
- """Provide a mock of the API response to get information on a process. Workers activity is enabled"""
- payload = {
- "name": None,
- "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
- "state": "running",
- "mode": "workers",
- "corpus": "11111111-1111-1111-1111-111111111111",
- "workflow": "http://testserver/ponos/v1/workflow/12341234-1234-1234-1234-123412341234/",
- "files": [],
- "revision": None,
- "element": {
- "id": "12341234-1234-1234-1234-123412341234",
- "type": "folder",
- "name": "Test folder",
- "corpus": {
- "id": "11111111-1111-1111-1111-111111111111",
- "name": "John Doe project",
- "public": False,
- },
- "thumbnail_url": "http://testserver/thumbnail.png",
- "zone": None,
- "thumbnail_put_url": "http://testserver/thumbnail.png",
- },
- "folder_type": None,
- "element_type": "page",
- "element_name_contains": None,
- "load_children": True,
- "use_cache": False,
- "activity_state": "ready",
- }
responses.add(
responses.GET,
- "http://testserver/api/v1/imports/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/",
+ "http://testserver/api/v1/imports/workers/56785678-5678-5678-5678-567856785678/",
status=200,
body=json.dumps(payload),
content_type="application/json",
@@ -236,7 +203,7 @@ def mock_activity_calls(responses):
@pytest.fixture
-def mock_elements_worker(monkeypatch, mock_config_api):
+def mock_elements_worker(monkeypatch, mock_worker_run_api):
"""Build and configure an ElementsWorker with fixed CLI parameters to avoid issues with pytest"""
monkeypatch.setattr(sys, "argv", ["worker"])
@@ -267,7 +234,7 @@ def mock_elements_worker_with_list(monkeypatch, responses, mock_elements_worker)
@pytest.fixture
-def mock_base_worker_with_cache(mocker, monkeypatch, mock_config_api):
+def mock_base_worker_with_cache(mocker, monkeypatch, mock_worker_run_api):
"""Build a BaseWorker using SQLite cache, also mocking a PONOS_TASK"""
monkeypatch.setattr(sys, "argv", ["worker"])
@@ -277,7 +244,7 @@ def mock_base_worker_with_cache(mocker, monkeypatch, mock_config_api):
@pytest.fixture
-def mock_elements_worker_with_cache(monkeypatch, mock_config_api, tmp_path):
+def mock_elements_worker_with_cache(monkeypatch, mock_worker_run_api, tmp_path):
"""Build and configure an ElementsWorker using SQLite cache with fixed CLI parameters to avoid issues with pytest"""
cache_path = tmp_path / "db.sqlite"
init_cache_db(cache_path)
diff --git a/tests/test_base_worker.py b/tests/test_base_worker.py
index 376aee002dac330d95937fe96aa19cf0cd63eaa2..0a6cf496b9774f429e3dfc27a4d05f19af79a973 100644
--- a/tests/test_base_worker.py
+++ b/tests/test_base_worker.py
@@ -1,4 +1,5 @@
# -*- coding: utf-8 -*-
+import json
import logging
import os
import sys
@@ -48,6 +49,7 @@ def test_init_var_ponos_data_given(monkeypatch):
def test_init_var_worker_version_id_missing(monkeypatch):
monkeypatch.setattr(sys, "argv", ["worker"])
monkeypatch.delenv("WORKER_VERSION_ID")
+ monkeypatch.delenv("ARKINDEX_WORKER_RUN_ID")
worker = BaseWorker()
worker.args = worker.parser.parse_args()
worker.configure_for_developers()
@@ -56,6 +58,17 @@ def test_init_var_worker_version_id_missing(monkeypatch):
assert worker.config == {} # default empty case
+def test_init_var_worker_run_id_missing(monkeypatch):
+ monkeypatch.setattr(sys, "argv", ["worker"])
+ monkeypatch.delenv("ARKINDEX_WORKER_RUN_ID")
+ worker = BaseWorker()
+ worker.args = worker.parser.parse_args()
+ worker.configure_for_developers()
+ assert worker.worker_run_id is None
+ assert worker.is_read_only is True
+ assert worker.config == {} # default empty case
+
+
def test_init_var_worker_local_file(monkeypatch, tmp_path):
# Build a dummy yaml config file
config = tmp_path / "config.yml"
@@ -63,6 +76,7 @@ def test_init_var_worker_local_file(monkeypatch, tmp_path):
monkeypatch.setattr(sys, "argv", ["worker", "-c", str(config)])
monkeypatch.delenv("WORKER_VERSION_ID")
+ monkeypatch.delenv("ARKINDEX_WORKER_RUN_ID")
worker = BaseWorker()
worker.args = worker.parser.parse_args()
worker.configure_for_developers()
@@ -73,43 +87,47 @@ def test_init_var_worker_local_file(monkeypatch, tmp_path):
config.unlink()
-def test_cli_default(mocker, mock_config_api):
+def test_cli_default(mocker, mock_worker_run_api):
worker = BaseWorker()
assert logger.level == logging.NOTSET
assert not hasattr(worker, "api_client")
mocker.patch.object(sys, "argv", ["worker"])
worker.args = worker.parser.parse_args()
+ assert worker.is_read_only is False
+ assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
+ assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
+
worker.configure()
assert not worker.args.verbose
assert logger.level == logging.NOTSET
assert worker.api_client
- assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
- assert worker.is_read_only is False
assert worker.config == {"someKey": "someValue"} # from API
logger.setLevel(logging.NOTSET)
-def test_cli_arg_verbose_given(mocker, mock_config_api):
+def test_cli_arg_verbose_given(mocker, mock_worker_run_api):
worker = BaseWorker()
assert logger.level == logging.NOTSET
assert not hasattr(worker, "api_client")
mocker.patch.object(sys, "argv", ["worker", "-v"])
worker.args = worker.parser.parse_args()
+ assert worker.is_read_only is False
+ assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
+ assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
+
worker.configure()
assert worker.args.verbose
assert logger.level == logging.DEBUG
assert worker.api_client
- assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
- assert worker.is_read_only is False
assert worker.config == {"someKey": "someValue"} # from API
logger.setLevel(logging.NOTSET)
-def test_cli_envvar_debug_given(mocker, monkeypatch, mock_config_api):
+def test_cli_envvar_debug_given(mocker, monkeypatch, mock_worker_run_api):
worker = BaseWorker()
assert logger.level == logging.NOTSET
@@ -117,11 +135,13 @@ def test_cli_envvar_debug_given(mocker, monkeypatch, mock_config_api):
mocker.patch.object(sys, "argv", ["worker"])
monkeypatch.setenv("ARKINDEX_DEBUG", True)
worker.args = worker.parser.parse_args()
+ assert worker.is_read_only is False
+ assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
+ assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
+
worker.configure()
assert logger.level == logging.DEBUG
assert worker.api_client
- assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
- assert worker.is_read_only is False
assert worker.config == {"someKey": "someValue"} # from API
logger.setLevel(logging.NOTSET)
@@ -133,36 +153,65 @@ def test_configure_dev_mode(mocker, monkeypatch):
"""
worker = BaseWorker()
mocker.patch.object(sys, "argv", ["worker", "--dev"])
- monkeypatch.setenv(
- "ARKINDEX_WORKER_RUN_ID", "aaaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
- )
worker.args = worker.parser.parse_args()
worker.configure_for_developers()
assert worker.args.dev is True
assert worker.process_information is None
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
+ assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
assert worker.is_read_only is True
assert worker.user_configuration == {}
-def test_configure_worker_run(mocker, monkeypatch, responses, mock_config_api):
+def test_configure_worker_run(mocker, monkeypatch, responses):
worker = BaseWorker()
mocker.patch.object(sys, "argv", ["worker"])
- run_id = "aaaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
- configuration_id = "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"
- monkeypatch.setenv("ARKINDEX_WORKER_RUN_ID", run_id)
- responses.add(
- responses.GET,
- f"http://testserver/api/v1/imports/workers/{run_id}/",
- json={"id": run_id, "configuration_id": configuration_id},
- )
+ user_configuration = {
+ "id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
+ "name": "BBB",
+ "configuration": {"a": "b"},
+ }
+ payload = {
+ "id": "56785678-5678-5678-5678-567856785678",
+ "parents": [],
+ "worker_version_id": "12341234-1234-1234-1234-123412341234",
+ "model_version_id": None,
+ "dataimport_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
+ "worker": {
+ "id": "deadbeef-1234-5678-1234-worker",
+ "name": "Fake worker",
+ "slug": "fake_worker",
+ "type": "classifier",
+ },
+ "configuration_id": user_configuration["id"],
+ "worker_version": {
+ "id": "12341234-1234-1234-1234-123412341234",
+ "worker": {
+ "id": "deadbeef-1234-5678-1234-worker",
+ "name": "Fake worker",
+ "slug": "fake_worker",
+ "type": "classifier",
+ },
+ "revision": {"hash": "deadbeef1234"},
+ "configuration": {"configuration": {}},
+ },
+ "configuration": user_configuration,
+ "process": {"id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"},
+ }
+
responses.add(
responses.GET,
- f"http://testserver/api/v1/workers/configurations/{configuration_id}/",
- json={"id": configuration_id, "name": "BBB", "configuration": {"a": "b"}},
+ "http://testserver/api/v1/imports/workers/56785678-5678-5678-5678-567856785678/",
+ status=200,
+ body=json.dumps(payload),
+ content_type="application/json",
)
worker.args = worker.parser.parse_args()
+ assert worker.is_read_only is False
+ assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
+ assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
+
worker.configure()
assert worker.user_configuration == {"a": "b"}
@@ -171,20 +220,56 @@ def test_configure_worker_run(mocker, monkeypatch, responses, mock_config_api):
def test_configure_user_configuration_defaults(
mocker,
monkeypatch,
- mock_worker_version_user_configuration_api,
- mock_process_api,
responses,
):
worker = BaseWorker()
mocker.patch.object(sys, "argv")
- run_id = "aaaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
- monkeypatch.setenv("ARKINDEX_WORKER_RUN_ID", run_id)
+ worker.args = worker.parser.parse_args()
+
+ payload = {
+ "id": "56785678-5678-5678-5678-567856785678",
+ "parents": [],
+ "worker_version_id": "12341234-1234-1234-1234-123412341234",
+ "model_version_id": None,
+ "dataimport_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
+ "worker": {
+ "id": "deadbeef-1234-5678-1234-worker",
+ "name": "Fake worker",
+ "slug": "fake_worker",
+ "type": "classifier",
+ },
+ "configuration_id": "af0daaf4-983e-4703-a7ed-a10f146d6684",
+ "worker_version": {
+ "id": "12341234-1234-1234-1234-123412341234",
+ "worker": {
+ "id": "deadbeef-1234-5678-1234-worker",
+ "name": "Fake worker",
+ "slug": "fake_worker",
+ "type": "classifier",
+ },
+ "revision": {"hash": "deadbeef1234"},
+ "configuration": {
+ "configuration": {"param_1": "/some/path/file.pth", "param_2": 12}
+ },
+ },
+ "configuration": {
+ "id": "af0daaf4-983e-4703-a7ed-a10f146d6684",
+ "name": "my-userconfig",
+ "configuration": {
+ "param_3": "Animula vagula blandula",
+ "param_5": True,
+ },
+ },
+ "process": {"id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"},
+ }
responses.add(
responses.GET,
- f"http://testserver/api/v1/imports/workers/{run_id}/",
- json={"id": run_id},
+ "http://testserver/api/v1/imports/workers/56785678-5678-5678-5678-567856785678/",
+ status=200,
+ body=json.dumps(payload),
+ content_type="application/json",
)
- worker.args = worker.parser.parse_args()
+
worker.configure()
assert worker.config == {"param_1": "/some/path/file.pth", "param_2": 12}
@@ -194,52 +279,95 @@ def test_configure_user_configuration_defaults(
}
-@pytest.mark.parametrize("debug_dict", [{"debug": True}, {"debug": False}])
-def test_configure_user_config_debug(
- mocker, monkeypatch, responses, debug_dict, mock_config_api
-):
+@pytest.mark.parametrize("debug", (True, False))
+def test_configure_user_config_debug(mocker, monkeypatch, responses, debug):
worker = BaseWorker()
mocker.patch.object(sys, "argv", ["worker"])
- run_id = "aaaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
- configuration_id = "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"
- monkeypatch.setenv("ARKINDEX_WORKER_RUN_ID", run_id)
assert logger.level == logging.NOTSET
+ payload = {
+ "id": "56785678-5678-5678-5678-567856785678",
+ "parents": [],
+ "worker_version_id": "12341234-1234-1234-1234-123412341234",
+ "model_version_id": None,
+ "dataimport_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
+ "worker": {
+ "id": "deadbeef-1234-5678-1234-worker",
+ "name": "Fake worker",
+ "slug": "fake_worker",
+ "type": "classifier",
+ },
+ "configuration_id": "af0daaf4-983e-4703-a7ed-a10f146d6684",
+ "worker_version": {
+ "id": "12341234-1234-1234-1234-123412341234",
+ "worker": {
+ "id": "deadbeef-1234-5678-1234-worker",
+ "name": "Fake worker",
+ "slug": "fake_worker",
+ "type": "classifier",
+ },
+ "revision": {"hash": "deadbeef1234"},
+ "configuration": {"configuration": {}},
+ },
+ "configuration": {
+ "id": "af0daaf4-983e-4703-a7ed-a10f146d6684",
+ "name": "BBB",
+ "configuration": {"debug": debug},
+ },
+ "process": {"id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"},
+ }
responses.add(
responses.GET,
- f"http://testserver/api/v1/imports/workers/{run_id}/",
- json={"id": run_id, "configuration_id": configuration_id},
- )
- responses.add(
- responses.GET,
- f"http://testserver/api/v1/workers/configurations/{configuration_id}/",
- json={"id": configuration_id, "name": "BBB", "configuration": debug_dict},
+ "http://testserver/api/v1/imports/workers/56785678-5678-5678-5678-567856785678/",
+ status=200,
+ body=json.dumps(payload),
+ content_type="application/json",
)
worker.args = worker.parser.parse_args()
worker.configure()
- assert worker.user_configuration == debug_dict
- expected_log_level = logging.DEBUG if debug_dict["debug"] else logging.NOTSET
+ assert worker.user_configuration == {"debug": debug}
+ expected_log_level = logging.DEBUG if debug else logging.NOTSET
assert logger.level == expected_log_level
logger.setLevel(logging.NOTSET)
-def test_configure_worker_run_missing_conf(
- mocker, monkeypatch, responses, mock_config_api
-):
+def test_configure_worker_run_missing_conf(mocker, monkeypatch, responses):
worker = BaseWorker()
mocker.patch.object(sys, "argv", ["worker"])
- run_id = "aaaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
- configuration_id = "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"
- monkeypatch.setenv("ARKINDEX_WORKER_RUN_ID", run_id)
- responses.add(
- responses.GET,
- f"http://testserver/api/v1/imports/workers/{run_id}/",
- json={"id": run_id, "configuration_id": configuration_id},
- )
+
+ payload = {
+ "id": "56785678-5678-5678-5678-567856785678",
+ "parents": [],
+ "worker_version_id": "12341234-1234-1234-1234-123412341234",
+ "model_version_id": None,
+ "dataimport_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
+ "worker": {
+ "id": "deadbeef-1234-5678-1234-worker",
+ "name": "Fake worker",
+ "slug": "fake_worker",
+ "type": "classifier",
+ },
+ "configuration_id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
+ "worker_version": {
+ "id": "12341234-1234-1234-1234-123412341234",
+ "worker": {
+ "id": "deadbeef-1234-5678-1234-worker",
+ "name": "Fake worker",
+ "slug": "fake_worker",
+ "type": "classifier",
+ },
+ "revision": {"hash": "deadbeef1234"},
+ "configuration": {"configuration": {}},
+ },
+ "configuration": {"id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", "name": "BBB"},
+ "process": {"id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"},
+ }
responses.add(
responses.GET,
- f"http://testserver/api/v1/workers/configurations/{configuration_id}/",
- json={"id": configuration_id, "name": "BBB"},
+ "http://testserver/api/v1/imports/workers/56785678-5678-5678-5678-567856785678/",
+ status=200,
+ body=json.dumps(payload),
+ content_type="application/json",
)
worker.args = worker.parser.parse_args()
worker.configure()
diff --git a/tests/test_elements_worker/__init__.py b/tests/test_elements_worker/__init__.py
index 3748d0a2618a3ce191413f8efd9ea329cc96cd3c..6b4429bcdf145a8a526e0fd20cca08953049a00c 100644
--- a/tests/test_elements_worker/__init__.py
+++ b/tests/test_elements_worker/__init__.py
@@ -1,9 +1,8 @@
# -*- coding: utf-8 -*-
# API calls during worker configuration
BASE_API_CALLS = [
- ("GET", "http://testserver/api/v1/imports/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/"),
(
"GET",
- "http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
+ "http://testserver/api/v1/imports/workers/56785678-5678-5678-5678-567856785678/",
),
]
diff --git a/tests/test_elements_worker/test_cli.py b/tests/test_elements_worker/test_cli.py
index 948ddb6321100f82dc566e87129a625bb7e6022b..ed2dfbe1ab7c6e146436c4918283a6cd203c1ebf 100644
--- a/tests/test_elements_worker/test_cli.py
+++ b/tests/test_elements_worker/test_cli.py
@@ -10,7 +10,7 @@ import pytest
from arkindex_worker.worker import ElementsWorker
-def test_cli_default(monkeypatch, mock_config_api):
+def test_cli_default(monkeypatch, mock_worker_run_api):
_, path = tempfile.mkstemp()
with open(path, "w") as f:
json.dump(
@@ -33,7 +33,7 @@ def test_cli_default(monkeypatch, mock_config_api):
os.unlink(path)
-def test_cli_arg_elements_list_given(mocker, mock_config_api):
+def test_cli_arg_elements_list_given(mocker, mock_worker_run_api):
_, path = tempfile.mkstemp()
with open(path, "w") as f:
json.dump(
@@ -62,7 +62,7 @@ def test_cli_arg_element_one_given_not_uuid(mocker):
worker.configure()
-def test_cli_arg_element_one_given(mocker, mock_config_api):
+def test_cli_arg_element_one_given(mocker, mock_worker_run_api):
mocker.patch.object(
sys, "argv", ["worker", "--element", "12341234-1234-1234-1234-123412341234"]
)
@@ -74,7 +74,7 @@ def test_cli_arg_element_one_given(mocker, mock_config_api):
assert not worker.args.elements_list
-def test_cli_arg_element_many_given(mocker, mock_config_api):
+def test_cli_arg_element_many_given(mocker, mock_worker_run_api):
mocker.patch.object(
sys,
"argv",
diff --git a/tests/test_elements_worker/test_worker.py b/tests/test_elements_worker/test_worker.py
index e0bedf3122b1c3e41d08d93ba0cdd6777731ea60..1eeb21d13a6caad4ee97aece6088f188464f5f07 100644
--- a/tests/test_elements_worker/test_worker.py
+++ b/tests/test_elements_worker/test_worker.py
@@ -79,15 +79,8 @@ def test_readonly(responses, mock_elements_worker):
] == BASE_API_CALLS
-def test_activities_disabled(responses, monkeypatch, mock_worker_version_api):
+def test_activities_disabled(responses, monkeypatch, mock_worker_run_api):
"""Test worker process elements without updating activities when they are disabled for the process"""
- responses.add(
- responses.GET,
- "http://testserver/api/v1/imports/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/",
- status=200,
- body=json.dumps({"activity_state": "disabled"}),
- content_type="application/json",
- )
monkeypatch.setattr(sys, "argv", ["worker"])
worker = ElementsWorker()
worker.configure()
@@ -113,7 +106,7 @@ def test_activities_dev_mode(mocker):
assert worker.store_activity is False
-def test_update_call(responses, mock_elements_worker, mock_process_api):
+def test_update_call(responses, mock_elements_worker, mock_worker_run_api):
"""Test an update call with feature enabled triggers an API call"""
responses.add(
responses.PUT,