Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • workers/base-worker
1 result
Show changes
Commits on Source (2)
Showing
with 42 additions and 183 deletions
0.3.1-rc1
0.3.1-rc2
......@@ -94,6 +94,7 @@ class CachedElement(Model):
rotation_angle = IntegerField(default=0)
mirrored = BooleanField(default=False)
initial = BooleanField(default=False)
# Needed to filter elements with cache
worker_version_id = UUIDField(null=True)
worker_run_id = UUIDField(null=True)
confidence = FloatField(null=True)
......@@ -173,6 +174,7 @@ class CachedTranscription(Model):
text = TextField()
confidence = FloatField()
orientation = CharField(max_length=50)
# Needed to filter transcriptions with cache
worker_version_id = UUIDField(null=True)
worker_run_id = UUIDField(null=True)
......@@ -191,7 +193,6 @@ class CachedClassification(Model):
class_name = TextField()
confidence = FloatField()
state = CharField(max_length=10)
worker_version_id = UUIDField(null=True)
worker_run_id = UUIDField(null=True)
class Meta:
......@@ -209,7 +210,6 @@ class CachedEntity(Model):
name = TextField()
validated = BooleanField(default=False)
metas = JSONField(null=True)
worker_version_id = UUIDField(null=True)
worker_run_id = UUIDField(null=True)
class Meta:
......@@ -228,7 +228,6 @@ class CachedTranscriptionEntity(Model):
entity = ForeignKeyField(CachedEntity, backref="transcription_entities")
offset = IntegerField(constraints=[Check("offset >= 0")])
length = IntegerField(constraints=[Check("length > 0")])
worker_version_id = UUIDField(null=True)
worker_run_id = UUIDField(null=True)
confidence = FloatField(null=True)
......
......@@ -126,11 +126,6 @@ class BaseWorker(object):
# through a ponos agent
self.task_id = os.environ.get("PONOS_TASK")
self.worker_version_id = os.environ.get("WORKER_VERSION_ID")
if not self.worker_version_id:
logger.warning(
"Missing WORKER_VERSION_ID environment variable, worker is in read-only mode"
)
self.worker_run_id = os.environ.get("ARKINDEX_WORKER_RUN_ID")
if not self.worker_run_id:
logger.warning(
......@@ -159,13 +154,9 @@ class BaseWorker(object):
Whether or not the worker can publish data.
:returns: False when dev mode is enabled with the ``--dev`` CLI argument,
when no worker version ID is provided or when no worker run ID is provided
when no worker run ID is provided
"""
return (
self.args.dev
or self.worker_version_id is None
or self.worker_run_id is None
)
return self.args.dev or self.worker_run_id is None
def setup_api_client(self):
"""
......@@ -228,6 +219,10 @@ class BaseWorker(object):
# Load worker version information
worker_version = worker_run["worker_version"]
# Store worker version id
self.worker_version_id = worker_version["id"]
self.worker_details = worker_version["worker"]
logger.info(
f"Loaded worker {self.worker_details['name']} revision {worker_version['revision']['hash'][0:7]} from API"
......
......@@ -104,7 +104,6 @@ class ClassificationMixin(object):
body={
"element": str(element.id),
"ml_class": self.get_ml_class_id(ml_class),
"worker_version": self.worker_version_id,
"worker_run_id": self.worker_run_id,
"confidence": confidence,
"high_confidence": high_confidence,
......@@ -121,7 +120,6 @@ class ClassificationMixin(object):
"class_name": ml_class,
"confidence": created["confidence"],
"state": created["state"],
"worker_version_id": self.worker_version_id,
"worker_run_id": self.worker_run_id,
}
]
......@@ -211,7 +209,6 @@ class ClassificationMixin(object):
"CreateClassifications",
body={
"parent": str(element.id),
"worker_version": self.worker_version_id,
"worker_run_id": self.worker_run_id,
"classifications": classifications,
},
......@@ -230,7 +227,6 @@ class ClassificationMixin(object):
"class_name": created_cl["class_name"],
"confidence": created_cl["confidence"],
"state": created_cl["state"],
"worker_version_id": self.worker_version_id,
"worker_run_id": self.worker_run_id,
}
for created_cl in created_cls
......
......@@ -138,7 +138,6 @@ class ElementMixin(object):
"corpus": element.corpus.id,
"polygon": polygon,
"parent": element.id,
"worker_version": self.worker_version_id,
"worker_run_id": self.worker_run_id,
"confidence": confidence,
},
......@@ -234,7 +233,6 @@ class ElementMixin(object):
"CreateElements",
id=parent.id,
body={
"worker_version": self.worker_version_id,
"worker_run_id": self.worker_run_id,
"elements": elements,
},
......@@ -267,7 +265,6 @@ class ElementMixin(object):
"type": element["type"],
"image_id": image_id,
"polygon": element["polygon"],
"worker_version_id": self.worker_version_id,
"worker_run_id": self.worker_run_id,
"confidence": element.get("confidence"),
}
......
......@@ -70,7 +70,6 @@ class EntityMixin(object):
"metas": metas,
"validated": validated,
"corpus": self.corpus_id,
"worker_version": self.worker_version_id,
"worker_run_id": self.worker_run_id,
},
)
......@@ -86,7 +85,7 @@ class EntityMixin(object):
"name": name,
"validated": validated if validated is not None else False,
"metas": metas,
"worker_version_id": self.worker_version_id,
"worker_run_id": self.worker_run_id,
}
]
CachedEntity.insert_many(to_insert).execute()
......@@ -141,7 +140,6 @@ class EntityMixin(object):
"entity": entity,
"length": length,
"offset": offset,
"worker_version_id": self.worker_version_id,
"worker_run_id": self.worker_run_id,
}
if confidence is not None:
......@@ -162,7 +160,6 @@ class EntityMixin(object):
entity=entity,
offset=offset,
length=length,
worker_version_id=self.worker_version_id,
worker_run_id=self.worker_run_id,
confidence=confidence,
)
......
......@@ -101,7 +101,6 @@ class MetaDataMixin(object):
"name": name,
"value": value,
"entity_id": entity,
"worker_version": self.worker_version_id,
"worker_run_id": self.worker_run_id,
},
)
......@@ -177,7 +176,6 @@ class MetaDataMixin(object):
"CreateMetaDataBulk",
id=element.id,
body={
"worker_version": self.worker_version_id,
"worker_run_id": self.worker_run_id,
"metadata_list": metas,
},
......
......@@ -82,7 +82,6 @@ class TranscriptionMixin(object):
id=element.id,
body={
"text": text,
"worker_version": self.worker_version_id,
"worker_run_id": self.worker_run_id,
"confidence": confidence,
"orientation": orientation.value,
......@@ -101,7 +100,6 @@ class TranscriptionMixin(object):
"text": created["text"],
"confidence": created["confidence"],
"orientation": created["orientation"],
"worker_version_id": self.worker_version_id,
"worker_run_id": self.worker_run_id,
}
]
......@@ -172,7 +170,6 @@ class TranscriptionMixin(object):
created_trs = self.request(
"CreateTranscriptions",
body={
"worker_version": self.worker_version_id,
"worker_run_id": self.worker_run_id,
"transcriptions": transcriptions_payload,
},
......@@ -191,7 +188,6 @@ class TranscriptionMixin(object):
"text": created_tr["text"],
"confidence": created_tr["confidence"],
"orientation": created_tr["orientation"],
"worker_version_id": self.worker_version_id,
"worker_run_id": self.worker_run_id,
}
for created_tr in created_trs
......@@ -287,7 +283,6 @@ class TranscriptionMixin(object):
id=element.id,
body={
"element_type": sub_element_type,
"worker_version": self.worker_version_id,
"worker_run_id": self.worker_run_id,
"transcriptions": transcriptions_payload,
"return_elements": True,
......@@ -325,7 +320,6 @@ class TranscriptionMixin(object):
"type": sub_element_type,
"image_id": element.image_id,
"polygon": transcription["polygon"],
"worker_version_id": self.worker_version_id,
"worker_run_id": self.worker_run_id,
}
)
......@@ -341,7 +335,6 @@ class TranscriptionMixin(object):
"orientation": transcription.get(
"orientation", TextOrientation.HorizontalLeftToRight
).value,
"worker_version_id": self.worker_version_id,
"worker_run_id": self.worker_run_id,
}
)
......
......@@ -3,6 +3,10 @@
Once you have implemented a worker, you can run it on some Arkindex elements
on your own machine to test it.
!!! warning
This section has been deprecated as of the latest version of base-worker.
## Retrieving credentials
For a worker to run properly, you will need two types of credentials:
......@@ -66,7 +70,9 @@ linked to this revision. You should get the following output:
Created a worker version: 392bd299-bc8f-4ec6-aa3c-e6503ecc7730
```
> **Warning:** This feature should only be used when a normal worker cannot be created using the Git workflow.
!!! warning
This feature should only be used when a normal worker cannot be created using the Git workflow.
## Setting credentials
......@@ -94,9 +100,10 @@ export ARKINDEX_API_TOKEN="YOUR_TOKEN_HERE"
export WORKER_VERSION_ID="xxxxx"
```
> **Warning:** Do not add these instructions to a script such as `.bashrc`;
> this would mean storing credentials in plaintext and can lead to security
> breaches.
!!! warning
Do not add these instructions to a script such as `.bashrc`;
this would mean storing credentials in plaintext and can lead to security breaches
## Running your worker
......
......@@ -97,6 +97,7 @@ markdown_extensions:
- def_list # enable definition lists
- admonition # syntax coloration in code blocks
- codehilite
- pymdownx.details
- pymdownx.superfences
copyright: Copyright © Teklia
......
......@@ -104,7 +104,6 @@ def setup_api(responses, monkeypatch, cache_yaml):
@pytest.fixture(autouse=True)
def give_env_variable(request, monkeypatch):
"""Defines required environment variables"""
monkeypatch.setenv("WORKER_VERSION_ID", "12341234-1234-1234-1234-123412341234")
monkeypatch.setenv("ARKINDEX_WORKER_RUN_ID", "56785678-5678-5678-5678-567856785678")
......
......@@ -18,7 +18,6 @@ def test_init_default_local_share(monkeypatch):
worker = BaseWorker()
assert worker.work_dir == os.path.expanduser("~/.local/share/arkindex")
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
def test_init_default_xdg_data_home(monkeypatch):
......@@ -27,14 +26,12 @@ def test_init_default_xdg_data_home(monkeypatch):
worker = BaseWorker()
assert worker.work_dir == f"{path}/arkindex"
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
def test_init_with_local_cache(monkeypatch):
worker = BaseWorker(support_cache=True)
assert worker.work_dir == os.path.expanduser("~/.local/share/arkindex")
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
assert worker.support_cache is True
......@@ -44,19 +41,6 @@ def test_init_var_ponos_data_given(monkeypatch):
worker = BaseWorker()
assert worker.work_dir == f"{path}/current"
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
def test_init_var_worker_version_id_missing(monkeypatch):
monkeypatch.setattr(sys, "argv", ["worker"])
monkeypatch.delenv("WORKER_VERSION_ID")
monkeypatch.delenv("ARKINDEX_WORKER_RUN_ID")
worker = BaseWorker()
worker.args = worker.parser.parse_args()
worker.configure_for_developers()
assert worker.worker_version_id is None
assert worker.is_read_only is True
assert worker.config == {} # default empty case
def test_init_var_worker_run_id_missing(monkeypatch):
......@@ -76,12 +60,11 @@ def test_init_var_worker_local_file(monkeypatch, tmp_path):
config.write_text("---\nlocalKey: abcdef123")
monkeypatch.setattr(sys, "argv", ["worker", "-c", str(config)])
monkeypatch.delenv("WORKER_VERSION_ID")
monkeypatch.delenv("ARKINDEX_WORKER_RUN_ID")
worker = BaseWorker()
worker.args = worker.parser.parse_args()
worker.configure_for_developers()
assert worker.worker_version_id is None
assert worker.worker_run_id is None
assert worker.is_read_only is True
assert worker.config == {"localKey": "abcdef123"} # Use a local file for devs
......@@ -95,7 +78,6 @@ def test_cli_default(mocker, mock_worker_run_api):
mocker.patch.object(sys, "argv", ["worker"])
worker.args = worker.parser.parse_args()
assert worker.is_read_only is False
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
worker.configure()
......@@ -103,6 +85,7 @@ def test_cli_default(mocker, mock_worker_run_api):
assert logger.level == logging.NOTSET
assert worker.api_client
assert worker.config == {"someKey": "someValue"} # from API
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
logger.setLevel(logging.NOTSET)
......@@ -114,7 +97,6 @@ def test_cli_arg_verbose_given(mocker, mock_worker_run_api):
mocker.patch.object(sys, "argv", ["worker", "-v"])
worker.args = worker.parser.parse_args()
assert worker.is_read_only is False
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
worker.configure()
......@@ -122,6 +104,7 @@ def test_cli_arg_verbose_given(mocker, mock_worker_run_api):
assert logger.level == logging.DEBUG
assert worker.api_client
assert worker.config == {"someKey": "someValue"} # from API
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
logger.setLevel(logging.NOTSET)
......@@ -134,13 +117,13 @@ def test_cli_envvar_debug_given(mocker, monkeypatch, mock_worker_run_api):
monkeypatch.setenv("ARKINDEX_DEBUG", "True")
worker.args = worker.parser.parse_args()
assert worker.is_read_only is False
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
worker.configure()
assert logger.level == logging.DEBUG
assert worker.api_client
assert worker.config == {"someKey": "someValue"} # from API
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
logger.setLevel(logging.NOTSET)
......@@ -156,7 +139,6 @@ def test_configure_dev_mode(mocker, monkeypatch):
assert worker.args.dev is True
assert worker.process_information is None
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
assert worker.is_read_only is True
assert worker.user_configuration == {}
......@@ -211,11 +193,11 @@ def test_configure_worker_run(mocker, monkeypatch, responses):
)
worker.args = worker.parser.parse_args()
assert worker.is_read_only is False
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
worker.configure()
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
assert worker.user_configuration == {"a": "b"}
......@@ -501,12 +483,12 @@ def test_configure_load_model_configuration(mocker, monkeypatch, responses):
)
worker.args = worker.parser.parse_args()
assert worker.is_read_only is False
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
assert worker.model_configuration == {}
worker.configure()
assert worker.worker_version_id == "12341234-1234-1234-1234-123412341234"
assert worker.model_configuration == {
"param1": "value1",
"param2": 2,
......
......@@ -58,11 +58,11 @@ def test_create_tables(tmp_path):
init_cache_db(db_path)
create_tables()
expected_schema = """CREATE TABLE "classifications" ("id" TEXT NOT NULL PRIMARY KEY, "element_id" TEXT NOT NULL, "class_name" TEXT NOT NULL, "confidence" REAL NOT NULL, "state" VARCHAR(10) NOT NULL, "worker_version_id" TEXT, "worker_run_id" TEXT, FOREIGN KEY ("element_id") REFERENCES "elements" ("id"))
expected_schema = """CREATE TABLE "classifications" ("id" TEXT NOT NULL PRIMARY KEY, "element_id" TEXT NOT NULL, "class_name" TEXT NOT NULL, "confidence" REAL NOT NULL, "state" VARCHAR(10) NOT NULL, "worker_run_id" TEXT, FOREIGN KEY ("element_id") REFERENCES "elements" ("id"))
CREATE TABLE "elements" ("id" TEXT NOT NULL PRIMARY KEY, "parent_id" TEXT, "type" VARCHAR(50) NOT NULL, "image_id" TEXT, "polygon" text, "rotation_angle" INTEGER NOT NULL, "mirrored" INTEGER NOT NULL, "initial" INTEGER NOT NULL, "worker_version_id" TEXT, "worker_run_id" TEXT, "confidence" REAL, FOREIGN KEY ("image_id") REFERENCES "images" ("id"))
CREATE TABLE "entities" ("id" TEXT NOT NULL PRIMARY KEY, "type" VARCHAR(50) NOT NULL, "name" TEXT NOT NULL, "validated" INTEGER NOT NULL, "metas" text, "worker_version_id" TEXT, "worker_run_id" TEXT)
CREATE TABLE "entities" ("id" TEXT NOT NULL PRIMARY KEY, "type" VARCHAR(50) NOT NULL, "name" TEXT NOT NULL, "validated" INTEGER NOT NULL, "metas" text, "worker_run_id" TEXT)
CREATE TABLE "images" ("id" TEXT NOT NULL PRIMARY KEY, "width" INTEGER NOT NULL, "height" INTEGER NOT NULL, "url" TEXT NOT NULL)
CREATE TABLE "transcription_entities" ("transcription_id" TEXT NOT NULL, "entity_id" TEXT NOT NULL, "offset" INTEGER NOT NULL CHECK (offset >= 0), "length" INTEGER NOT NULL CHECK (length > 0), "worker_version_id" TEXT, "worker_run_id" TEXT, "confidence" REAL, PRIMARY KEY ("transcription_id", "entity_id"), FOREIGN KEY ("transcription_id") REFERENCES "transcriptions" ("id"), FOREIGN KEY ("entity_id") REFERENCES "entities" ("id"))
CREATE TABLE "transcription_entities" ("transcription_id" TEXT NOT NULL, "entity_id" TEXT NOT NULL, "offset" INTEGER NOT NULL CHECK (offset >= 0), "length" INTEGER NOT NULL CHECK (length > 0), "worker_run_id" TEXT, "confidence" REAL, PRIMARY KEY ("transcription_id", "entity_id"), FOREIGN KEY ("transcription_id") REFERENCES "transcriptions" ("id"), FOREIGN KEY ("entity_id") REFERENCES "entities" ("id"))
CREATE TABLE "transcriptions" ("id" TEXT NOT NULL PRIMARY KEY, "element_id" TEXT NOT NULL, "text" TEXT NOT NULL, "confidence" REAL NOT NULL, "orientation" VARCHAR(50) NOT NULL, "worker_version_id" TEXT, "worker_run_id" TEXT, FOREIGN KEY ("element_id") REFERENCES "elements" ("id"))"""
actual_schema = "\n".join(
......
......@@ -249,7 +249,6 @@ def test_create_classification_wrong_ml_class(mock_elements_worker, responses):
{
"element": "12341234-1234-1234-1234-123412341234",
"ml_class": "new-ml-class-1234",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"confidence": 0.42,
"high_confidence": True,
......@@ -401,7 +400,6 @@ def test_create_classification(responses, mock_elements_worker):
assert json.loads(responses.calls[-1].request.body) == {
"element": "12341234-1234-1234-1234-123412341234",
"ml_class": "0000",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"confidence": 0.42,
"high_confidence": True,
......@@ -427,7 +425,6 @@ def test_create_classification_with_cache(responses, mock_elements_worker_with_c
"id": "56785678-5678-5678-5678-567856785678",
"element": "12341234-1234-1234-1234-123412341234",
"ml_class": "0000",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"confidence": 0.42,
"high_confidence": True,
......@@ -452,7 +449,6 @@ def test_create_classification_with_cache(responses, mock_elements_worker_with_c
assert json.loads(responses.calls[-1].request.body) == {
"element": "12341234-1234-1234-1234-123412341234",
"ml_class": "0000",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"confidence": 0.42,
"high_confidence": True,
......@@ -471,57 +467,11 @@ def test_create_classification_with_cache(responses, mock_elements_worker_with_c
class_name="a_class",
confidence=0.42,
state="pending",
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
)
]
def test_create_classification_duplicate_worker_version(
responses, mock_elements_worker
):
mock_elements_worker.classes = {
"11111111-1111-1111-1111-111111111111": {"a_class": "0000"}
}
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
responses.add(
responses.POST,
"http://testserver/api/v1/classifications/",
status=400,
json={
"non_field_errors": [
"The fields element, worker_version, ml_class must make a unique set."
]
},
)
mock_elements_worker.create_classification(
element=elt,
ml_class="a_class",
confidence=0.42,
high_confidence=True,
)
assert len(responses.calls) == len(BASE_API_CALLS) + 1
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS + [
("POST", "http://testserver/api/v1/classifications/"),
]
assert json.loads(responses.calls[-1].request.body) == {
"element": "12341234-1234-1234-1234-123412341234",
"ml_class": "0000",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"confidence": 0.42,
"high_confidence": True,
}
# Classification has NOT been created
assert mock_elements_worker.report.report_data["elements"] == {}
def test_create_classification_duplicate_worker_run(responses, mock_elements_worker):
mock_elements_worker.classes = {
"11111111-1111-1111-1111-111111111111": {"a_class": "0000"}
......@@ -555,7 +505,6 @@ def test_create_classification_duplicate_worker_run(responses, mock_elements_wor
assert json.loads(responses.calls[-1].request.body) == {
"element": "12341234-1234-1234-1234-123412341234",
"ml_class": "0000",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"confidence": 0.42,
"high_confidence": True,
......@@ -877,7 +826,6 @@ def test_create_classifications(responses, mock_elements_worker_with_cache):
status=200,
json={
"parent": str(elt.id),
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"classifications": [
{
......@@ -911,7 +859,6 @@ def test_create_classifications(responses, mock_elements_worker_with_cache):
assert json.loads(responses.calls[-1].request.body) == {
"parent": str(elt.id),
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"classifications": classes,
}
......@@ -924,7 +871,6 @@ def test_create_classifications(responses, mock_elements_worker_with_cache):
class_name="portrait",
confidence=0.75,
state="pending",
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
),
CachedClassification(
......@@ -933,7 +879,6 @@ def test_create_classifications(responses, mock_elements_worker_with_cache):
class_name="landscape",
confidence=0.25,
state="pending",
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
),
]
......@@ -573,7 +573,6 @@ def test_create_sub_element(responses, mock_elements_worker, slim_output):
"corpus": "11111111-1111-1111-1111-111111111111",
"polygon": [[1, 1], [2, 2], [2, 1], [1, 2]],
"parent": "12341234-1234-1234-1234-123412341234",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"confidence": None,
}
......@@ -619,7 +618,6 @@ def test_create_sub_element_confidence(responses, mock_elements_worker):
"corpus": "11111111-1111-1111-1111-111111111111",
"polygon": [[1, 1], [2, 2], [2, 1], [1, 2]],
"parent": "12341234-1234-1234-1234-123412341234",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"confidence": 0.42,
}
......@@ -995,7 +993,6 @@ def test_create_elements_cached_element(responses, mock_elements_worker_with_cac
"polygon": [[1, 1], [2, 2], [2, 1], [1, 2]],
}
],
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
}
assert created_ids == [{"id": "497f6eca-6276-4993-bfeb-53cbbbba6f08"}]
......@@ -1009,7 +1006,6 @@ def test_create_elements_cached_element(responses, mock_elements_worker_with_cac
type="something",
image_id="c0fec0fe-c0fe-c0fe-c0fe-c0fec0fec0fe",
polygon=[[1, 1], [2, 2], [2, 1], [1, 2]],
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
),
]
......@@ -1064,7 +1060,6 @@ def test_create_elements(responses, mock_elements_worker_with_cache, tmp_path):
"polygon": [[1, 1], [2, 2], [2, 1], [1, 2]],
}
],
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
}
assert created_ids == [{"id": "497f6eca-6276-4993-bfeb-53cbbbba6f08"}]
......@@ -1079,7 +1074,6 @@ def test_create_elements(responses, mock_elements_worker_with_cache, tmp_path):
type="something",
image_id="c0fec0fe-c0fe-c0fe-c0fe-c0fec0fec0fe",
polygon=[[1, 1], [2, 2], [2, 1], [1, 2]],
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
confidence=None,
)
......@@ -1139,7 +1133,6 @@ def test_create_elements_confidence(
"confidence": 0.42,
}
],
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
}
assert created_ids == [{"id": "497f6eca-6276-4993-bfeb-53cbbbba6f08"}]
......@@ -1154,7 +1147,6 @@ def test_create_elements_confidence(
type="something",
image_id="c0fec0fe-c0fe-c0fe-c0fe-c0fec0fec0fe",
polygon=[[1, 1], [2, 2], [2, 1], [1, 2]],
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
confidence=0.42,
)
......
......@@ -186,7 +186,6 @@ def test_create_entity(responses, mock_elements_worker):
"metas": {},
"validated": None,
"corpus": "11111111-1111-1111-1111-111111111111",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
}
assert entity_id == "12345678-1234-1234-1234-123456789123"
......@@ -220,7 +219,6 @@ def test_create_entity_with_cache(responses, mock_elements_worker_with_cache):
"metas": {},
"validated": None,
"corpus": "11111111-1111-1111-1111-111111111111",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
}
assert entity_id == "12345678-1234-1234-1234-123456789123"
......@@ -233,7 +231,6 @@ def test_create_entity_with_cache(responses, mock_elements_worker_with_cache):
name="Bob Bob",
validated=False,
metas={},
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
)
]
......@@ -421,7 +418,6 @@ def test_create_transcription_entity_no_confidence(responses, mock_elements_work
"entity": "11111111-1111-1111-1111-111111111111",
"offset": 5,
"length": 10,
"worker_version_id": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
}
......@@ -460,7 +456,6 @@ def test_create_transcription_entity_with_confidence(responses, mock_elements_wo
"entity": "11111111-1111-1111-1111-111111111111",
"offset": 5,
"length": 10,
"worker_version_id": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"confidence": 0.33,
}
......@@ -500,7 +495,6 @@ def test_create_transcription_entity_confidence_none(responses, mock_elements_wo
"entity": "11111111-1111-1111-1111-111111111111",
"offset": 5,
"length": 10,
"worker_version_id": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
}
......@@ -518,14 +512,12 @@ def test_create_transcription_entity_with_cache(
text="Hello, it's me.",
confidence=0.42,
orientation=TextOrientation.HorizontalLeftToRight,
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
)
CachedEntity.create(
id=UUID("11111111-1111-1111-1111-111111111111"),
type="person",
name="Bob Bob",
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
)
......@@ -560,7 +552,6 @@ def test_create_transcription_entity_with_cache(
"entity": "11111111-1111-1111-1111-111111111111",
"offset": 5,
"length": 10,
"worker_version_id": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
}
......@@ -571,7 +562,6 @@ def test_create_transcription_entity_with_cache(
entity=UUID("11111111-1111-1111-1111-111111111111"),
offset=5,
length=10,
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
)
]
......@@ -590,14 +580,12 @@ def test_create_transcription_entity_with_confidence_with_cache(
text="Hello, it's me.",
confidence=0.42,
orientation=TextOrientation.HorizontalLeftToRight,
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
)
CachedEntity.create(
id=UUID("11111111-1111-1111-1111-111111111111"),
type="person",
name="Bob Bob",
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
)
......@@ -634,7 +622,6 @@ def test_create_transcription_entity_with_confidence_with_cache(
"entity": "11111111-1111-1111-1111-111111111111",
"offset": 5,
"length": 10,
"worker_version_id": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"confidence": 0.77,
}
......@@ -646,7 +633,6 @@ def test_create_transcription_entity_with_confidence_with_cache(
entity=UUID("11111111-1111-1111-1111-111111111111"),
offset=5,
length=10,
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
confidence=0.77,
)
......
......@@ -193,7 +193,6 @@ def test_create_metadata(responses, mock_elements_worker):
"name": "Teklia",
"value": "La Turbine, Grenoble 38000",
"entity_id": None,
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
}
assert metadata_id == "12345678-1234-1234-1234-123456789123"
......@@ -222,7 +221,6 @@ def test_create_metadatas(responses, mock_elements_worker, metadatas):
"http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
status=201,
json={
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"metadata_list": [
{
......
......@@ -133,7 +133,6 @@ def test_create_transcription_default_orientation(responses, mock_elements_worke
"id": "56785678-5678-5678-5678-567856785678",
"text": "Animula vagula blandula",
"confidence": 0.42,
"worker_version_id": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
},
)
......@@ -144,7 +143,6 @@ def test_create_transcription_default_orientation(responses, mock_elements_worke
)
assert json.loads(responses.calls[-1].request.body) == {
"text": "Animula vagula blandula",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"confidence": 0.42,
"orientation": "horizontal-lr",
......@@ -161,7 +159,6 @@ def test_create_transcription_orientation(responses, mock_elements_worker):
"id": "56785678-5678-5678-5678-567856785678",
"text": "Animula vagula blandula",
"confidence": 0.42,
"worker_version_id": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
},
)
......@@ -173,7 +170,6 @@ def test_create_transcription_orientation(responses, mock_elements_worker):
)
assert json.loads(responses.calls[-1].request.body) == {
"text": "Animula vagula blandula",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"confidence": 0.42,
"orientation": "vertical-lr",
......@@ -233,7 +229,6 @@ def test_create_transcription(responses, mock_elements_worker):
"id": "56785678-5678-5678-5678-567856785678",
"text": "i am a line",
"confidence": 0.42,
"worker_version_id": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
},
)
......@@ -253,7 +248,6 @@ def test_create_transcription(responses, mock_elements_worker):
assert json.loads(responses.calls[-1].request.body) == {
"text": "i am a line",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"confidence": 0.42,
"orientation": "horizontal-lr",
......@@ -272,7 +266,6 @@ def test_create_transcription_with_cache(responses, mock_elements_worker_with_ca
"text": "i am a line",
"confidence": 0.42,
"orientation": "horizontal-lr",
"worker_version_id": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
},
)
......@@ -292,7 +285,6 @@ def test_create_transcription_with_cache(responses, mock_elements_worker_with_ca
assert json.loads(responses.calls[-1].request.body) == {
"text": "i am a line",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"orientation": "horizontal-lr",
"confidence": 0.42,
......@@ -306,7 +298,7 @@ def test_create_transcription_with_cache(responses, mock_elements_worker_with_ca
text="i am a line",
confidence=0.42,
orientation=TextOrientation.HorizontalLeftToRight,
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_version_id=None,
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
)
]
......@@ -325,7 +317,6 @@ def test_create_transcription_orientation_with_cache(
"text": "Animula vagula blandula",
"confidence": 0.42,
"orientation": "vertical-lr",
"worker_version_id": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
},
)
......@@ -337,7 +328,6 @@ def test_create_transcription_orientation_with_cache(
)
assert json.loads(responses.calls[-1].request.body) == {
"text": "Animula vagula blandula",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"orientation": "vertical-lr",
"confidence": 0.42,
......@@ -362,7 +352,7 @@ def test_create_transcription_orientation_with_cache(
"text": "Animula vagula blandula",
"confidence": 0.42,
"orientation": TextOrientation.VerticalLeftToRight.value,
"worker_version_id": UUID("12341234-1234-1234-1234-123412341234"),
"worker_version_id": None,
"worker_run_id": UUID("56785678-5678-5678-5678-567856785678"),
}
]
......@@ -675,7 +665,6 @@ def test_create_transcriptions(responses, mock_elements_worker_with_cache):
"http://testserver/api/v1/transcription/bulk/",
status=200,
json={
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"transcriptions": [
{
......@@ -708,7 +697,6 @@ def test_create_transcriptions(responses, mock_elements_worker_with_cache):
]
assert json.loads(responses.calls[-1].request.body) == {
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"transcriptions": [
{
......@@ -734,7 +722,6 @@ def test_create_transcriptions(responses, mock_elements_worker_with_cache):
text="The",
confidence=0.75,
orientation=TextOrientation.HorizontalLeftToRight,
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
),
CachedTranscription(
......@@ -743,7 +730,6 @@ def test_create_transcriptions(responses, mock_elements_worker_with_cache):
text="word",
confidence=0.42,
orientation=TextOrientation.HorizontalLeftToRight,
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
),
]
......@@ -771,7 +757,6 @@ def test_create_transcriptions_orientation(responses, mock_elements_worker_with_
"http://testserver/api/v1/transcription/bulk/",
status=200,
json={
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"transcriptions": [
{
......@@ -797,7 +782,6 @@ def test_create_transcriptions_orientation(responses, mock_elements_worker_with_
)
assert json.loads(responses.calls[-1].request.body) == {
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"transcriptions": [
{
......@@ -835,7 +819,7 @@ def test_create_transcriptions_orientation(responses, mock_elements_worker_with_
"text": "Animula vagula blandula",
"confidence": 0.12,
"orientation": TextOrientation.HorizontalRightToLeft.value,
"worker_version_id": UUID("12341234-1234-1234-1234-123412341234"),
"worker_version_id": None,
"worker_run_id": UUID("56785678-5678-5678-5678-567856785678"),
},
{
......@@ -856,7 +840,7 @@ def test_create_transcriptions_orientation(responses, mock_elements_worker_with_
"text": "Hospes comesque corporis",
"confidence": 0.21,
"orientation": TextOrientation.VerticalLeftToRight.value,
"worker_version_id": UUID("12341234-1234-1234-1234-123412341234"),
"worker_version_id": None,
"worker_run_id": UUID("56785678-5678-5678-5678-567856785678"),
},
]
......@@ -1329,7 +1313,6 @@ def test_create_element_transcriptions(responses, mock_elements_worker):
assert json.loads(responses.calls[-1].request.body) == {
"element_type": "page",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"transcriptions": [
{
......@@ -1415,7 +1398,6 @@ def test_create_element_transcriptions_with_cache(
assert json.loads(responses.calls[-1].request.body) == {
"element_type": "page",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"transcriptions": [
{
......@@ -1464,7 +1446,6 @@ def test_create_element_transcriptions_with_cache(
parent_id=UUID("12341234-1234-1234-1234-123412341234"),
type="page",
polygon="[[100, 150], [700, 150], [700, 200], [100, 200]]",
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
),
CachedElement(
......@@ -1472,7 +1453,6 @@ def test_create_element_transcriptions_with_cache(
parent_id=UUID("12341234-1234-1234-1234-123412341234"),
type="page",
polygon="[[0, 0], [2000, 0], [2000, 3000], [0, 3000]]",
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
),
]
......@@ -1483,7 +1463,6 @@ def test_create_element_transcriptions_with_cache(
text="The",
confidence=0.5,
orientation=TextOrientation.HorizontalLeftToRight.value,
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
),
CachedTranscription(
......@@ -1492,7 +1471,6 @@ def test_create_element_transcriptions_with_cache(
text="first",
confidence=0.75,
orientation=TextOrientation.HorizontalLeftToRight.value,
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
),
CachedTranscription(
......@@ -1501,7 +1479,6 @@ def test_create_element_transcriptions_with_cache(
text="line",
confidence=0.9,
orientation=TextOrientation.HorizontalLeftToRight.value,
worker_version_id=UUID("12341234-1234-1234-1234-123412341234"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
),
]
......@@ -1563,7 +1540,6 @@ def test_create_transcriptions_orientation_with_cache(
assert json.loads(responses.calls[-1].request.body) == {
"element_type": "page",
"worker_version": "12341234-1234-1234-1234-123412341234",
"worker_run_id": "56785678-5678-5678-5678-567856785678",
"transcriptions": [
{
......@@ -1618,14 +1594,14 @@ def test_create_transcriptions_orientation_with_cache(
"rotation_angle": 0,
"mirrored": False,
"initial": False,
"worker_version_id": UUID("12341234-1234-1234-1234-123412341234"),
"worker_version_id": None,
"worker_run_id": UUID("56785678-5678-5678-5678-567856785678"),
"confidence": None,
},
"text": "Animula vagula blandula",
"confidence": 0.5,
"orientation": TextOrientation.HorizontalLeftToRight.value,
"worker_version_id": UUID("12341234-1234-1234-1234-123412341234"),
"worker_version_id": None,
"worker_run_id": UUID("56785678-5678-5678-5678-567856785678"),
},
{
......@@ -1639,14 +1615,14 @@ def test_create_transcriptions_orientation_with_cache(
"rotation_angle": 0,
"mirrored": False,
"initial": False,
"worker_version_id": UUID("12341234-1234-1234-1234-123412341234"),
"worker_version_id": None,
"worker_run_id": UUID("56785678-5678-5678-5678-567856785678"),
"confidence": None,
},
"text": "Hospes comesque corporis",
"confidence": 0.75,
"orientation": TextOrientation.VerticalLeftToRight.value,
"worker_version_id": UUID("12341234-1234-1234-1234-123412341234"),
"worker_version_id": None,
"worker_run_id": UUID("56785678-5678-5678-5678-567856785678"),
},
{
......@@ -1660,14 +1636,14 @@ def test_create_transcriptions_orientation_with_cache(
"rotation_angle": 0,
"mirrored": False,
"initial": False,
"worker_version_id": UUID("12341234-1234-1234-1234-123412341234"),
"worker_version_id": None,
"worker_run_id": UUID("56785678-5678-5678-5678-567856785678"),
"confidence": None,
},
"text": "Quae nunc abibis in loca",
"confidence": 0.9,
"orientation": TextOrientation.HorizontalRightToLeft.value,
"worker_version_id": UUID("12341234-1234-1234-1234-123412341234"),
"worker_version_id": None,
"worker_run_id": UUID("56785678-5678-5678-5678-567856785678"),
},
]
......
......@@ -66,7 +66,7 @@ def test_readonly(responses, mock_elements_worker):
"""Test readonly worker does not trigger any API calls"""
# Setup the worker as read-only
mock_elements_worker.worker_version_id = None
mock_elements_worker.worker_run_id = None
assert mock_elements_worker.is_read_only is True
out = mock_elements_worker.update_activity("1234-deadbeef", ActivityState.Processed)
......
......@@ -18,7 +18,5 @@ def setup_environment(responses):
# Set schema url in environment
os.environ["ARKINDEX_API_SCHEMA_URL"] = schema_url
# Setup a fake worker version ID
os.environ["WORKER_VERSION_ID"] = "1234-{{ cookiecutter.slug }}"
# Setup a fake worker run ID
os.environ["ARKINDEX_WORKER_RUN_ID"] = "1234-{{ cookiecutter.slug }}"