Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • workers/base-worker
1 result
Show changes
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 902 additions and 522 deletions
tests/conftest.py
View file @ e76a273a
......@@ -16,6 +16,7 @@ from arkindex_worker.cache import (
MODELS,
SQL_VERSION,
CachedElement,
CachedImage,
CachedTranscription,
Version,
create_version_table,
......@@ -210,6 +211,15 @@ def mock_elements_worker(monkeypatch, mock_worker_run_api):
return worker
@pytest.fixture
def mock_elements_worker_read_only(monkeypatch):
"""Build and configure an ElementsWorker with fixed CLI parameters to avoid issues with pytest"""
monkeypatch.setattr(sys, "argv", ["worker", "--dev"])
worker = ElementsWorker()
worker.configure()
return worker
@pytest.fixture
def mock_elements_worker_with_list(monkeypatch, responses, mock_elements_worker):
"""
......@@ -321,9 +331,25 @@ def fake_gitlab_helper_factory():
@pytest.fixture
def mock_cached_elements():
"""Insert few elements in local cache"""
CachedElement.create(
id=UUID("99999999-9999-9999-9999-999999999999"),
parent_id=None,
type="something",
polygon="[[1, 1], [2, 2], [2, 1], [1, 2]]",
worker_version_id=None,
worker_run_id=None,
)
CachedElement.create(
id=UUID("12341234-1234-1234-1234-123412341234"),
parent_id=UUID("99999999-9999-9999-9999-999999999999"),
type="double_page",
polygon="[[1, 1], [2, 2], [2, 1], [1, 2]]",
worker_version_id=UUID("56785678-5678-5678-5678-567856785678"),
worker_run_id=UUID("56785678-5678-5678-5678-567856785678"),
)
CachedElement.create(
id=UUID("11111111-1111-1111-1111-111111111111"),
parent_id="12341234-1234-1234-1234-123412341234",
parent_id=UUID("12341234-1234-1234-1234-123412341234"),
type="something",
polygon="[[1, 1], [2, 2], [2, 1], [1, 2]]",
worker_version_id=UUID("56785678-5678-5678-5678-567856785678"),
......@@ -344,7 +370,19 @@ def mock_cached_elements():
worker_version_id=None,
worker_run_id=None,
)
assert CachedElement.select().count() == 3
assert CachedElement.select().count() == 5
@pytest.fixture
def mock_cached_images():
"""Insert few elements in local cache"""
CachedImage.create(
id=UUID("99999999-9999-9999-9999-999999999999"),
width=1250,
height=2500,
url="http://testserver/iiif/3/image",
)
assert CachedImage.select().count() == 1
@pytest.fixture
......
tests/test_base_worker.py
View file @ e76a273a
......@@ -10,7 +10,7 @@ import pytest
from arkindex.mock import MockApiClient
from arkindex_worker import logger
from arkindex_worker.worker import BaseWorker
from arkindex_worker.worker import BaseWorker, ElementsWorker
from arkindex_worker.worker.base import ModelNotFoundError
from tests.conftest import FIXTURES_DIR
......@@ -739,3 +739,25 @@ def test_extract_parent_archives(tmp_path):
)
mode = "rb" if extracted_file.suffix == ".png" else "r"
assert extracted_file.open(mode).read() == expected_file.open(mode).read()
def test_corpus_id_not_set_read_only_mode(
mock_elements_worker_read_only: ElementsWorker,
):
mock_elements_worker_read_only.configure()
with pytest.raises(
Exception, match="Missing ARKINDEX_CORPUS_ID environment variable"
):
mock_elements_worker_read_only.corpus_id
def test_corpus_id_set_read_only_mode(
monkeypatch, mock_elements_worker_read_only: ElementsWorker
):
corpus_id = str(uuid.uuid4())
monkeypatch.setenv("ARKINDEX_CORPUS_ID", corpus_id)
mock_elements_worker_read_only.configure()
assert mock_elements_worker_read_only.corpus_id == corpus_id
tests/test_cache.py
View file @ e76a273a
......@@ -148,10 +148,20 @@ def test_check_version_same_version(tmp_path):
@pytest.mark.parametrize(
"image_width,image_height,polygon_x,polygon_y,polygon_width,polygon_height,max_size,expected_url",
"image_width,image_height,polygon_x,polygon_y,polygon_width,polygon_height,max_width,max_height,expected_url",
[
# No max_size: no resize
(400, 600, 0, 0, 400, 600, None, "http://something/full/full/0/default.jpg"),
(
400,
600,
0,
0,
400,
600,
None,
None,
"http://something/full/full/0/default.jpg",
),
# No max_size: resize on bbox
(
400,
......@@ -161,6 +171,7 @@ def test_check_version_same_version(tmp_path):
200,
100,
None,
None,
"http://something/0,0,200,100/full/0/default.jpg",
),
(
......@@ -171,12 +182,43 @@ def test_check_version_same_version(tmp_path):
200,
100,
None,
None,
"http://something/50,50,200,100/full/0/default.jpg",
),
# max_size equal to the image size, no resize
(400, 600, 0, 0, 400, 600, 600, "http://something/full/full/0/default.jpg"),
(600, 400, 0, 0, 600, 400, 600, "http://something/full/full/0/default.jpg"),
(400, 400, 0, 0, 400, 400, 400, "http://something/full/full/0/default.jpg"),
(
400,
600,
0,
0,
400,
600,
400,
None,
"http://something/full/full/0/default.jpg",
),
(
600,
400,
0,
0,
600,
400,
None,
400,
"http://something/full/full/0/default.jpg",
),
(
400,
400,
0,
0,
400,
400,
400,
400,
"http://something/full/full/0/default.jpg",
),
(
400,
400,
......@@ -185,11 +227,32 @@ def test_check_version_same_version(tmp_path):
200,
100,
200,
100,
"http://something/50,50,200,100/full/0/default.jpg",
),
# max_size is smaller than the image, resize
(400, 600, 0, 0, 400, 600, 400, "http://something/full/266,400/0/default.jpg"),
(600, 400, 0, 0, 600, 400, 400, "http://something/full/400,266/0/default.jpg"),
(
400,
600,
0,
0,
400,
600,
None,
400,
"http://something/full/,400/0/default.jpg",
),
(
600,
400,
0,
0,
600,
400,
400,
None,
"http://something/full/400,/0/default.jpg",
),
(
400,
600,
......@@ -198,6 +261,7 @@ def test_check_version_same_version(tmp_path):
200,
600,
400,
600,
"http://something/0,0,200,600/full/0/default.jpg",
),
(
......@@ -208,13 +272,54 @@ def test_check_version_same_version(tmp_path):
200,
600,
400,
600,
"http://something/50,50,200,600/full/0/default.jpg",
),
(400, 400, 0, 0, 400, 400, 200, "http://something/full/200,200/0/default.jpg"),
(
400,
400,
0,
0,
400,
400,
200,
200,
"http://something/full/200,200/0/default.jpg",
),
# max_size above the image size, no resize
(400, 600, 0, 0, 400, 600, 800, "http://something/full/full/0/default.jpg"),
(600, 400, 0, 0, 600, 400, 800, "http://something/full/full/0/default.jpg"),
(400, 400, 0, 0, 400, 400, 800, "http://something/full/full/0/default.jpg"),
(
400,
600,
0,
0,
400,
600,
800,
None,
"http://something/full/full/0/default.jpg",
),
(
600,
400,
0,
0,
600,
400,
None,
800,
"http://something/full/full/0/default.jpg",
),
(
400,
400,
0,
0,
400,
400,
800,
800,
"http://something/full/full/0/default.jpg",
),
(
400,
400,
......@@ -223,6 +328,7 @@ def test_check_version_same_version(tmp_path):
200,
100,
800,
800,
"http://something/50,50,200,100/full/0/default.jpg",
),
],
......@@ -235,7 +341,8 @@ def test_element_open_image(
polygon_y,
polygon_width,
polygon_height,
max_size,
max_width,
max_height,
expected_url,
):
open_mock = mocker.patch(
......@@ -261,7 +368,7 @@ def test_element_open_image(
],
)
assert elt.open_image(max_size=max_size) == "an image!"
assert elt.open_image(max_width=max_width, max_height=max_height) == "an image!"
assert open_mock.call_count == 1
assert open_mock.call_args == mocker.call(
expected_url, mirrored=False, rotation_angle=0
......
tests/test_element.py
View file @ e76a273a
......@@ -94,7 +94,7 @@ def test_open_image_resize_portrait(mocker):
}
)
# Resize = original size
assert elt.open_image(max_size=600, use_full_image=True) == "an image!"
assert elt.open_image(max_height=600, use_full_image=True) == "an image!"
assert open_mock.call_count == 1
assert open_mock.call_args == mocker.call(
"http://something/full/full/0/default.jpg",
......@@ -102,15 +102,15 @@ def test_open_image_resize_portrait(mocker):
mirrored=False,
)
# Resize = smaller height
assert elt.open_image(max_size=400, use_full_image=True) == "an image!"
assert elt.open_image(max_height=400, use_full_image=True) == "an image!"
assert open_mock.call_count == 2
assert open_mock.call_args == mocker.call(
"http://something/full/266,400/0/default.jpg",
"http://something/full/,400/0/default.jpg",
rotation_angle=0,
mirrored=False,
)
# Resize = bigger height
assert elt.open_image(max_size=800, use_full_image=True) == "an image!"
assert elt.open_image(max_height=800, use_full_image=True) == "an image!"
assert open_mock.call_count == 3
assert open_mock.call_args == mocker.call(
"http://something/full/full/0/default.jpg",
......@@ -138,10 +138,10 @@ def test_open_image_resize_partial_element(mocker):
"mirrored": False,
}
)
assert elt.open_image(max_size=400, use_full_image=True) == "an image!"
assert elt.open_image(max_height=400, use_full_image=True) == "an image!"
assert open_mock.call_count == 1
assert open_mock.call_args == mocker.call(
"http://something/full/full/0/default.jpg",
"http://something/full/,400/0/default.jpg",
rotation_angle=0,
mirrored=False,
)
......@@ -167,7 +167,7 @@ def test_open_image_resize_landscape(mocker):
}
)
# Resize = original size
assert elt.open_image(max_size=600, use_full_image=True) == "an image!"
assert elt.open_image(max_width=600, use_full_image=True) == "an image!"
assert open_mock.call_count == 1
assert open_mock.call_args == mocker.call(
"http://something/full/full/0/default.jpg",
......@@ -175,15 +175,15 @@ def test_open_image_resize_landscape(mocker):
mirrored=False,
)
# Resize = smaller width
assert elt.open_image(max_size=400, use_full_image=True) == "an image!"
assert elt.open_image(max_width=400, use_full_image=True) == "an image!"
assert open_mock.call_count == 2
assert open_mock.call_args == mocker.call(
"http://something/full/400,266/0/default.jpg",
"http://something/full/400,/0/default.jpg",
rotation_angle=0,
mirrored=False,
)
# Resize = bigger width
assert elt.open_image(max_size=800, use_full_image=True) == "an image!"
assert elt.open_image(max_width=800, use_full_image=True) == "an image!"
assert open_mock.call_count == 3
assert open_mock.call_args == mocker.call(
"http://something/full/full/0/default.jpg",
......@@ -212,7 +212,14 @@ def test_open_image_resize_square(mocker):
}
)
# Resize = original size
assert elt.open_image(max_size=400, use_full_image=True) == "an image!"
assert (
elt.open_image(
max_width=400,
max_height=400,
use_full_image=True,
)
== "an image!"
)
assert open_mock.call_count == 1
assert open_mock.call_args == mocker.call(
"http://something/full/full/0/default.jpg",
......@@ -220,7 +227,14 @@ def test_open_image_resize_square(mocker):
mirrored=False,
)
# Resize = smaller
assert elt.open_image(max_size=200, use_full_image=True) == "an image!"
assert (
elt.open_image(
max_width=200,
max_height=200,
use_full_image=True,
)
== "an image!"
)
assert open_mock.call_count == 2
assert open_mock.call_args == mocker.call(
"http://something/full/200,200/0/default.jpg",
......@@ -228,7 +242,14 @@ def test_open_image_resize_square(mocker):
mirrored=False,
)
# Resize = bigger
assert elt.open_image(max_size=800, use_full_image=True) == "an image!"
assert (
elt.open_image(
max_width=800,
max_height=800,
use_full_image=True,
)
== "an image!"
)
assert open_mock.call_count == 3
assert open_mock.call_args == mocker.call(
"http://something/full/full/0/default.jpg",
......@@ -251,7 +272,7 @@ def test_open_image_resize_tiles(mocker):
}
)
with pytest.raises(NotImplementedError):
elt.open_image(max_size=400)
elt.open_image(max_width=400)
def test_open_image_requires_zone():
......@@ -364,10 +385,10 @@ def test_open_image_resize_use_full_image_false(mocker):
}
)
# Resize = smaller
assert elt.open_image(max_size=200, use_full_image=False) == "an image!"
assert elt.open_image(max_height=200, use_full_image=False) == "an image!"
assert open_mock.call_count == 1
assert open_mock.call_args == mocker.call(
"http://zoneurl/0,0,400,600/133,200/0/default.jpg",
"http://zoneurl/0,0,400,600/,200/0/default.jpg",
rotation_angle=0,
mirrored=False,
)
......
tests/test_elements_worker/test_classifications.py
View file @ e76a273a
......@@ -423,11 +423,6 @@ def test_create_classification(responses, mock_elements_worker):
"high_confidence": True,
}
# Classification has been created and reported
assert mock_elements_worker.report.report_data["elements"][elt.id][
"classifications"
] == {"a_class": 1}
def test_create_classification_with_cache(responses, mock_elements_worker_with_cache):
mock_elements_worker_with_cache.classes = {"a_class": "0000"}
......@@ -470,11 +465,6 @@ def test_create_classification_with_cache(responses, mock_elements_worker_with_c
"high_confidence": True,
}
# Classification has been created and reported
assert mock_elements_worker_with_cache.report.report_data["elements"][elt.id][
"classifications"
] == {"a_class": 1}
# Check that created classification was properly stored in SQLite cache
assert list(CachedClassification.select()) == [
CachedClassification(
......@@ -524,9 +514,6 @@ def test_create_classification_duplicate_worker_run(responses, mock_elements_wor
"high_confidence": True,
}
# Classification has NOT been created
assert mock_elements_worker.report.report_data["elements"] == {}
def test_create_classifications_wrong_element(mock_elements_worker):
with pytest.raises(AssertionError) as e:
......
tests/test_elements_worker/test_entities.py
View file @ e76a273a
......@@ -12,43 +12,16 @@ from arkindex_worker.cache import (
CachedTranscription,
CachedTranscriptionEntity,
)
from arkindex_worker.models import Element, Transcription
from arkindex_worker.models import Transcription
from arkindex_worker.worker.entity import MissingEntityType
from arkindex_worker.worker.transcription import TextOrientation
from . import BASE_API_CALLS
def test_create_entity_wrong_element(mock_elements_worker):
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=None,
name="Bob Bob",
type="person",
)
assert (
str(e.value)
== "element shouldn't be null and should be an Element or CachedElement"
)
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element="not element type",
name="Bob Bob",
type="person",
)
assert (
str(e.value)
== "element shouldn't be null and should be an Element or CachedElement"
)
def test_create_entity_wrong_name(mock_elements_worker):
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=elt,
name=None,
type="person",
)
......@@ -56,7 +29,6 @@ def test_create_entity_wrong_name(mock_elements_worker):
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=elt,
name=1234,
type="person",
)
......@@ -64,11 +36,8 @@ def test_create_entity_wrong_name(mock_elements_worker):
def test_create_entity_wrong_type(mock_elements_worker):
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type=None,
)
......@@ -76,7 +45,6 @@ def test_create_entity_wrong_type(mock_elements_worker):
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type=1234,
)
......@@ -84,13 +52,10 @@ def test_create_entity_wrong_type(mock_elements_worker):
def test_create_entity_wrong_corpus(monkeypatch, mock_elements_worker):
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
# Triggering an error on metas param, not giving corpus should work since
# ARKINDEX_CORPUS_ID environment variable is set on mock_elements_worker
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type="person",
metas="wrong metas",
......@@ -99,11 +64,8 @@ def test_create_entity_wrong_corpus(monkeypatch, mock_elements_worker):
def test_create_entity_wrong_metas(mock_elements_worker):
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type="person",
metas="wrong metas",
......@@ -112,11 +74,8 @@ def test_create_entity_wrong_metas(mock_elements_worker):
def test_create_entity_wrong_validated(mock_elements_worker):
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type="person",
validated="wrong validated",
......@@ -127,7 +86,6 @@ def test_create_entity_wrong_validated(mock_elements_worker):
def test_create_entity_api_error(responses, mock_elements_worker):
# Set one entity type
mock_elements_worker.entity_types = {"person": "person-entity-type-id"}
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
responses.add(
responses.POST,
"http://testserver/api/v1/entity/",
......@@ -136,7 +94,6 @@ def test_create_entity_api_error(responses, mock_elements_worker):
with pytest.raises(ErrorResponse):
mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type="person",
)
......@@ -158,7 +115,6 @@ def test_create_entity(responses, mock_elements_worker):
# Set one entity type
mock_elements_worker.entity_types = {"person": "person-entity-type-id"}
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
responses.add(
responses.POST,
"http://testserver/api/v1/entity/",
......@@ -167,7 +123,6 @@ def test_create_entity(responses, mock_elements_worker):
)
entity_id = mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type="person",
)
......@@ -193,8 +148,6 @@ def test_create_entity_missing_type(responses, mock_elements_worker):
"""
Create entity with an unknown type will fail.
"""
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
# Call to list entity types
responses.add(
responses.GET,
......@@ -213,7 +166,6 @@ def test_create_entity_missing_type(responses, mock_elements_worker):
AssertionError, match="Entity type `new-entity` not found in the corpus."
):
mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type="new-entity",
)
......@@ -232,7 +184,6 @@ def test_create_entity_missing_type(responses, mock_elements_worker):
def test_create_entity_with_cache(responses, mock_elements_worker_with_cache):
# Set one entity type
mock_elements_worker_with_cache.entity_types = {"person": "person-entity-type-id"}
elt = CachedElement.create(id="12341234-1234-1234-1234-123412341234", type="thing")
responses.add(
responses.POST,
"http://testserver/api/v1/entity/",
......@@ -241,7 +192,6 @@ def test_create_entity_with_cache(responses, mock_elements_worker_with_cache):
)
entity_id = mock_elements_worker_with_cache.create_entity(
element=elt,
name="Bob Bob",
type="person",
)
......@@ -910,22 +860,11 @@ def test_create_transcription_entities_wrong_transcription(
)
def test_create_transcription_entities_no_transcription_element(mock_elements_worker):
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_transcription_entities(
transcription=Transcription(id="transcription_id"),
entities=[],
)
assert str(e.value) == "No element linked to Transcription (transcription_id)"
@pytest.mark.parametrize("entities", (None, "not a list of entities", 1))
def test_create_transcription_entities_wrong_entities(mock_elements_worker, entities):
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_transcription_entities(
transcription=Transcription(
id="transcription_id", element={"id": "element_id"}
),
transcription=Transcription(id="transcription_id"),
entities=entities,
)
assert str(e.value) == "entities shouldn't be null and should be of type list"
......@@ -934,9 +873,7 @@ def test_create_transcription_entities_wrong_entities(mock_elements_worker, enti
def test_create_transcription_entities_wrong_entities_subtype(mock_elements_worker):
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_transcription_entities(
transcription=Transcription(
id="transcription_id", element={"id": "element_id"}
),
transcription=Transcription(id="transcription_id"),
entities=["not a dict"],
)
assert str(e.value) == "Entity at index 0 in entities: Should be of type dict"
......@@ -1030,17 +967,14 @@ def test_create_transcription_entities_wrong_entity(
):
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_transcription_entities(
transcription=Transcription(
id="transcription_id", element={"id": "element_id"}
),
transcription=Transcription(id="transcription_id"),
entities=[entity],
)
assert str(e.value) == error
def test_create_transcription_entities(responses, mock_elements_worker):
element_id = "element_id"
transcription = Transcription(id="transcription-id", element={"id": element_id})
transcription = Transcription(id="transcription-id")
# Call to Transcription entities creation in bulk
responses.add(
responses.POST,
......@@ -1091,35 +1025,6 @@ def test_create_transcription_entities(responses, mock_elements_worker):
assert len(created_objects) == 1
assert element_id in mock_elements_worker.report.report_data["elements"]
ml_report = mock_elements_worker.report.report_data["elements"][element_id]
assert "started" in ml_report
del ml_report["started"]
# Check reporting
assert ml_report == {
"elements": {},
"transcriptions": 0,
"classifications": {},
"entities": [
{
"id": "entity-id",
"type": "22222222-2222-2222-2222-222222222222",
"name": "Teklia",
}
],
"transcription_entities": [
{
"transcription_id": "transcription-id",
"entity_id": "entity-id",
"transcription_entity_id": "transc-entity-id",
}
],
"metadata": [],
"errors": [],
}
assert len(responses.calls) == len(BASE_API_CALLS) + 1
assert [
(call.request.method, call.request.url) for call in responses.calls
......
tests/test_reporting.py deleted 100644 → 0
View file @ aabda26b
# -*- coding: utf-8 -*-
import json
import uuid
from datetime import datetime
from tempfile import NamedTemporaryFile
import pytest
from apistar.exceptions import ErrorResponse
from arkindex_worker.models import Transcription
from arkindex_worker.reporting import Reporter
def test_init():
version_id = str(uuid.uuid4())
reporter = Reporter(name="Worker", slug="worker-slug", version=version_id)
assert "started" in reporter.report_data
del reporter.report_data["started"]
assert reporter.report_data == {
"name": "Worker",
"slug": "worker-slug",
"version": version_id,
"elements": {},
}
def test_process():
reporter = Reporter("worker")
reporter.process("myelement")
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
assert "started" in element_data
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 0,
"classifications": {},
"entities": [],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_element():
reporter = Reporter("worker")
reporter.add_element("myelement", type="text_line")
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {"text_line": 1},
"transcriptions": 0,
"classifications": {},
"entities": [],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_element_count():
"""
Report multiple elements with the same parent and type
"""
reporter = Reporter("worker")
reporter.add_element("myelement", type="text_line", type_count=42)
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {"text_line": 42},
"transcriptions": 0,
"classifications": {},
"entities": [],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_classification():
reporter = Reporter("worker")
reporter.add_classification("myelement", "three")
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 0,
"classifications": {"three": 1},
"entities": [],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_classifications():
reporter = Reporter("worker")
with pytest.raises(AssertionError):
reporter.add_classifications("myelement", {"not": "a list"})
reporter.add_classifications(
"myelement", [{"class_name": "three"}, {"class_name": "two"}]
)
reporter.add_classifications(
"myelement",
[
{"class_name": "three"},
{"class_name": "two", "high_confidence": True},
{"class_name": "three", "confidence": 0.42},
],
)
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 0,
"classifications": {"three": 3, "two": 2},
"entities": [],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_transcription():
reporter = Reporter("worker")
reporter.add_transcription("myelement")
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 1,
"classifications": {},
"entities": [],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_transcription_count():
"""
Report multiple transcriptions with the same element and type
"""
reporter = Reporter("worker")
reporter.add_transcription("myelement", 1337)
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 1337,
"classifications": {},
"entities": [],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_entity():
reporter = Reporter("worker")
reporter.add_entity(
"myelement",
"12341234-1234-1234-1234-123412341234",
"person-entity-type-id",
"Bob Bob",
)
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 0,
"classifications": {},
"entities": [
{
"id": "12341234-1234-1234-1234-123412341234",
"type": "person-entity-type-id",
"name": "Bob Bob",
}
],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_transcription_entity():
reporter = Reporter("worker")
reporter.add_transcription_entity(
"5678",
Transcription({"id": "1234-5678", "element": {"id": "myelement"}}),
"1234",
)
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 0,
"classifications": {},
"entities": [],
"transcription_entities": [
{
"transcription_id": "1234-5678",
"entity_id": "5678",
"transcription_entity_id": "1234",
}
],
"metadata": [],
"errors": [],
}
def test_add_metadata():
reporter = Reporter("worker")
reporter.add_metadata(
"myelement", "12341234-1234-1234-1234-123412341234", "location", "Teklia"
)
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 0,
"classifications": {},
"entities": [],
"transcription_entities": [],
"metadata": [
{
"id": "12341234-1234-1234-1234-123412341234",
"type": "location",
"name": "Teklia",
}
],
"errors": [],
}
def test_error():
reporter = Reporter("worker")
reporter.error("myelement", ZeroDivisionError("What have you done"))
reporter.error(
"myelement",
ErrorResponse(
title="I'm a teapot",
status_code=418,
content='{"coffee": "Can\'t touch this"}',
),
)
assert reporter.report_data["elements"]["myelement"]["errors"] == [
{"class": "ZeroDivisionError", "message": "What have you done"},
{
"class": "ErrorResponse",
"message": "I'm a teapot",
"status_code": 418,
"content": '{"coffee": "Can\'t touch this"}',
},
]
def test_reporter_save(mocker):
datetime_mock = mocker.MagicMock()
datetime_mock.utcnow.return_value = datetime(2000, 1, 1)
mocker.patch("arkindex_worker.reporting.datetime", datetime_mock)
version_id = str(uuid.uuid4())
reporter = Reporter(name="Worker", slug="worker-slug", version=version_id)
reporter.add_element("myelement", type="text_line", type_count=4)
with NamedTemporaryFile() as f:
reporter.save(f.name)
exported_data = json.load(f)
assert exported_data == {
"name": "Worker",
"slug": "worker-slug",
"started": "2000-01-01T00:00:00",
"version": version_id,
"elements": {
"myelement": {
"classifications": {},
"elements": {"text_line": 4},
"entities": [],
"transcription_entities": [],
"errors": [],
"metadata": [],
"started": "2000-01-01T00:00:00",
"transcriptions": 0,
}
},
}
worker-{{cookiecutter.slug}}/.gitlab-ci.yml
View file @ e76a273a
......@@ -50,7 +50,7 @@ lint:
docker-build:
stage: build
image: docker:19.03.1
image: docker:24.0.6
services:
- docker:dind
variables:
......
worker-{{cookiecutter.slug}}/Dockerfile
View file @ e76a273a
FROM python:3
FROM python:3.11
WORKDIR /src
......
worker-{{cookiecutter.slug}}/requirements.txt
View file @ e76a273a
arkindex-base-worker==0.3.3
arkindex-base-worker==0.3.4