Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • workers/base-worker
1 result
Show changes
Hide whitespace changes
Inline Side-by-side
Showing
with 412 additions and 518 deletions
tests/test_base_worker.py
View file @ 426b05af
......@@ -10,7 +10,7 @@ import pytest
from arkindex.mock import MockApiClient
from arkindex_worker import logger
from arkindex_worker.worker import BaseWorker
from arkindex_worker.worker import BaseWorker, ElementsWorker
from arkindex_worker.worker.base import ModelNotFoundError
from tests.conftest import FIXTURES_DIR
......@@ -739,3 +739,25 @@ def test_extract_parent_archives(tmp_path):
)
mode = "rb" if extracted_file.suffix == ".png" else "r"
assert extracted_file.open(mode).read() == expected_file.open(mode).read()
def test_corpus_id_not_set_read_only_mode(
mock_elements_worker_read_only: ElementsWorker,
):
mock_elements_worker_read_only.configure()
with pytest.raises(
Exception, match="Missing ARKINDEX_CORPUS_ID environment variable"
):
mock_elements_worker_read_only.corpus_id
def test_corpus_id_set_read_only_mode(
monkeypatch, mock_elements_worker_read_only: ElementsWorker
):
corpus_id = str(uuid.uuid4())
monkeypatch.setenv("ARKINDEX_CORPUS_ID", corpus_id)
mock_elements_worker_read_only.configure()
assert mock_elements_worker_read_only.corpus_id == corpus_id
tests/test_cache.py
View file @ 426b05af
......@@ -148,10 +148,20 @@ def test_check_version_same_version(tmp_path):
@pytest.mark.parametrize(
"image_width,image_height,polygon_x,polygon_y,polygon_width,polygon_height,max_size,expected_url",
"image_width,image_height,polygon_x,polygon_y,polygon_width,polygon_height,max_width,max_height,expected_url",
[
# No max_size: no resize
(400, 600, 0, 0, 400, 600, None, "http://something/full/full/0/default.jpg"),
(
400,
600,
0,
0,
400,
600,
None,
None,
"http://something/full/full/0/default.jpg",
),
# No max_size: resize on bbox
(
400,
......@@ -161,6 +171,7 @@ def test_check_version_same_version(tmp_path):
200,
100,
None,
None,
"http://something/0,0,200,100/full/0/default.jpg",
),
(
......@@ -171,12 +182,43 @@ def test_check_version_same_version(tmp_path):
200,
100,
None,
None,
"http://something/50,50,200,100/full/0/default.jpg",
),
# max_size equal to the image size, no resize
(400, 600, 0, 0, 400, 600, 600, "http://something/full/full/0/default.jpg"),
(600, 400, 0, 0, 600, 400, 600, "http://something/full/full/0/default.jpg"),
(400, 400, 0, 0, 400, 400, 400, "http://something/full/full/0/default.jpg"),
(
400,
600,
0,
0,
400,
600,
400,
None,
"http://something/full/full/0/default.jpg",
),
(
600,
400,
0,
0,
600,
400,
None,
400,
"http://something/full/full/0/default.jpg",
),
(
400,
400,
0,
0,
400,
400,
400,
400,
"http://something/full/full/0/default.jpg",
),
(
400,
400,
......@@ -185,11 +227,32 @@ def test_check_version_same_version(tmp_path):
200,
100,
200,
100,
"http://something/50,50,200,100/full/0/default.jpg",
),
# max_size is smaller than the image, resize
(400, 600, 0, 0, 400, 600, 400, "http://something/full/266,400/0/default.jpg"),
(600, 400, 0, 0, 600, 400, 400, "http://something/full/400,266/0/default.jpg"),
(
400,
600,
0,
0,
400,
600,
None,
400,
"http://something/full/,400/0/default.jpg",
),
(
600,
400,
0,
0,
600,
400,
400,
None,
"http://something/full/400,/0/default.jpg",
),
(
400,
600,
......@@ -198,6 +261,7 @@ def test_check_version_same_version(tmp_path):
200,
600,
400,
600,
"http://something/0,0,200,600/full/0/default.jpg",
),
(
......@@ -208,13 +272,54 @@ def test_check_version_same_version(tmp_path):
200,
600,
400,
600,
"http://something/50,50,200,600/full/0/default.jpg",
),
(400, 400, 0, 0, 400, 400, 200, "http://something/full/200,200/0/default.jpg"),
(
400,
400,
0,
0,
400,
400,
200,
200,
"http://something/full/200,200/0/default.jpg",
),
# max_size above the image size, no resize
(400, 600, 0, 0, 400, 600, 800, "http://something/full/full/0/default.jpg"),
(600, 400, 0, 0, 600, 400, 800, "http://something/full/full/0/default.jpg"),
(400, 400, 0, 0, 400, 400, 800, "http://something/full/full/0/default.jpg"),
(
400,
600,
0,
0,
400,
600,
800,
None,
"http://something/full/full/0/default.jpg",
),
(
600,
400,
0,
0,
600,
400,
None,
800,
"http://something/full/full/0/default.jpg",
),
(
400,
400,
0,
0,
400,
400,
800,
800,
"http://something/full/full/0/default.jpg",
),
(
400,
400,
......@@ -223,6 +328,7 @@ def test_check_version_same_version(tmp_path):
200,
100,
800,
800,
"http://something/50,50,200,100/full/0/default.jpg",
),
],
......@@ -235,7 +341,8 @@ def test_element_open_image(
polygon_y,
polygon_width,
polygon_height,
max_size,
max_width,
max_height,
expected_url,
):
open_mock = mocker.patch(
......@@ -261,7 +368,7 @@ def test_element_open_image(
],
)
assert elt.open_image(max_size=max_size) == "an image!"
assert elt.open_image(max_width=max_width, max_height=max_height) == "an image!"
assert open_mock.call_count == 1
assert open_mock.call_args == mocker.call(
expected_url, mirrored=False, rotation_angle=0
......
tests/test_element.py
View file @ 426b05af
......@@ -94,7 +94,7 @@ def test_open_image_resize_portrait(mocker):
}
)
# Resize = original size
assert elt.open_image(max_size=600, use_full_image=True) == "an image!"
assert elt.open_image(max_height=600, use_full_image=True) == "an image!"
assert open_mock.call_count == 1
assert open_mock.call_args == mocker.call(
"http://something/full/full/0/default.jpg",
......@@ -102,15 +102,15 @@ def test_open_image_resize_portrait(mocker):
mirrored=False,
)
# Resize = smaller height
assert elt.open_image(max_size=400, use_full_image=True) == "an image!"
assert elt.open_image(max_height=400, use_full_image=True) == "an image!"
assert open_mock.call_count == 2
assert open_mock.call_args == mocker.call(
"http://something/full/266,400/0/default.jpg",
"http://something/full/,400/0/default.jpg",
rotation_angle=0,
mirrored=False,
)
# Resize = bigger height
assert elt.open_image(max_size=800, use_full_image=True) == "an image!"
assert elt.open_image(max_height=800, use_full_image=True) == "an image!"
assert open_mock.call_count == 3
assert open_mock.call_args == mocker.call(
"http://something/full/full/0/default.jpg",
......@@ -138,10 +138,10 @@ def test_open_image_resize_partial_element(mocker):
"mirrored": False,
}
)
assert elt.open_image(max_size=400, use_full_image=True) == "an image!"
assert elt.open_image(max_height=400, use_full_image=True) == "an image!"
assert open_mock.call_count == 1
assert open_mock.call_args == mocker.call(
"http://something/full/full/0/default.jpg",
"http://something/full/,400/0/default.jpg",
rotation_angle=0,
mirrored=False,
)
......@@ -167,7 +167,7 @@ def test_open_image_resize_landscape(mocker):
}
)
# Resize = original size
assert elt.open_image(max_size=600, use_full_image=True) == "an image!"
assert elt.open_image(max_width=600, use_full_image=True) == "an image!"
assert open_mock.call_count == 1
assert open_mock.call_args == mocker.call(
"http://something/full/full/0/default.jpg",
......@@ -175,15 +175,15 @@ def test_open_image_resize_landscape(mocker):
mirrored=False,
)
# Resize = smaller width
assert elt.open_image(max_size=400, use_full_image=True) == "an image!"
assert elt.open_image(max_width=400, use_full_image=True) == "an image!"
assert open_mock.call_count == 2
assert open_mock.call_args == mocker.call(
"http://something/full/400,266/0/default.jpg",
"http://something/full/400,/0/default.jpg",
rotation_angle=0,
mirrored=False,
)
# Resize = bigger width
assert elt.open_image(max_size=800, use_full_image=True) == "an image!"
assert elt.open_image(max_width=800, use_full_image=True) == "an image!"
assert open_mock.call_count == 3
assert open_mock.call_args == mocker.call(
"http://something/full/full/0/default.jpg",
......@@ -212,7 +212,14 @@ def test_open_image_resize_square(mocker):
}
)
# Resize = original size
assert elt.open_image(max_size=400, use_full_image=True) == "an image!"
assert (
elt.open_image(
max_width=400,
max_height=400,
use_full_image=True,
)
== "an image!"
)
assert open_mock.call_count == 1
assert open_mock.call_args == mocker.call(
"http://something/full/full/0/default.jpg",
......@@ -220,7 +227,14 @@ def test_open_image_resize_square(mocker):
mirrored=False,
)
# Resize = smaller
assert elt.open_image(max_size=200, use_full_image=True) == "an image!"
assert (
elt.open_image(
max_width=200,
max_height=200,
use_full_image=True,
)
== "an image!"
)
assert open_mock.call_count == 2
assert open_mock.call_args == mocker.call(
"http://something/full/200,200/0/default.jpg",
......@@ -228,7 +242,14 @@ def test_open_image_resize_square(mocker):
mirrored=False,
)
# Resize = bigger
assert elt.open_image(max_size=800, use_full_image=True) == "an image!"
assert (
elt.open_image(
max_width=800,
max_height=800,
use_full_image=True,
)
== "an image!"
)
assert open_mock.call_count == 3
assert open_mock.call_args == mocker.call(
"http://something/full/full/0/default.jpg",
......@@ -251,7 +272,7 @@ def test_open_image_resize_tiles(mocker):
}
)
with pytest.raises(NotImplementedError):
elt.open_image(max_size=400)
elt.open_image(max_width=400)
def test_open_image_requires_zone():
......@@ -364,10 +385,10 @@ def test_open_image_resize_use_full_image_false(mocker):
}
)
# Resize = smaller
assert elt.open_image(max_size=200, use_full_image=False) == "an image!"
assert elt.open_image(max_height=200, use_full_image=False) == "an image!"
assert open_mock.call_count == 1
assert open_mock.call_args == mocker.call(
"http://zoneurl/0,0,400,600/133,200/0/default.jpg",
"http://zoneurl/0,0,400,600/,200/0/default.jpg",
rotation_angle=0,
mirrored=False,
)
......
tests/test_elements_worker/test_classifications.py
View file @ 426b05af
......@@ -423,11 +423,6 @@ def test_create_classification(responses, mock_elements_worker):
"high_confidence": True,
}
# Classification has been created and reported
assert mock_elements_worker.report.report_data["elements"][elt.id][
"classifications"
] == {"a_class": 1}
def test_create_classification_with_cache(responses, mock_elements_worker_with_cache):
mock_elements_worker_with_cache.classes = {"a_class": "0000"}
......@@ -470,11 +465,6 @@ def test_create_classification_with_cache(responses, mock_elements_worker_with_c
"high_confidence": True,
}
# Classification has been created and reported
assert mock_elements_worker_with_cache.report.report_data["elements"][elt.id][
"classifications"
] == {"a_class": 1}
# Check that created classification was properly stored in SQLite cache
assert list(CachedClassification.select()) == [
CachedClassification(
......@@ -524,9 +514,6 @@ def test_create_classification_duplicate_worker_run(responses, mock_elements_wor
"high_confidence": True,
}
# Classification has NOT been created
assert mock_elements_worker.report.report_data["elements"] == {}
def test_create_classifications_wrong_element(mock_elements_worker):
with pytest.raises(AssertionError) as e:
......
tests/test_elements_worker/test_elements.py
View file @ 426b05af
......@@ -1210,94 +1210,208 @@ def test_create_elements_integrity_error(
assert list(CachedElement.select()) == []
def test_update_element_wrong_element(mock_elements_worker):
with pytest.raises(AssertionError) as e:
mock_elements_worker.update_element(
element=None,
)
assert (
str(e.value)
== "element shouldn't be null and should be an Element or CachedElement"
)
@pytest.mark.parametrize(
"payload, error",
(
# Element
(
{"element": None},
"element shouldn't be null and should be an Element or CachedElement",
),
(
{"element": "not element type"},
"element shouldn't be null and should be an Element or CachedElement",
),
),
)
def test_partial_update_element_wrong_param_element(
mock_elements_worker, payload, error
):
api_payload = {
"element": Element({"zone": None}),
**payload,
}
with pytest.raises(AssertionError) as e:
mock_elements_worker.update_element(
element="not element type",
mock_elements_worker.partial_update_element(
**api_payload,
)
assert (
str(e.value)
== "element shouldn't be null and should be an Element or CachedElement"
)
assert str(e.value) == error
def test_update_element_wrong_type(mock_elements_worker):
@pytest.mark.parametrize(
"payload, error",
(
# Type
({"type": 1234}, "type should be a str"),
({"type": None}, "type should be a str"),
),
)
def test_partial_update_element_wrong_param_type(mock_elements_worker, payload, error):
api_payload = {
"element": Element({"zone": None}),
**payload,
}
with pytest.raises(AssertionError) as e:
mock_elements_worker.update_element(
element=Element({"zone": None}),
type=1234,
mock_elements_worker.partial_update_element(
**api_payload,
)
assert str(e.value) == "type should be None or a str"
assert str(e.value) == error
def test_update_element_wrong_name(mock_elements_worker):
@pytest.mark.parametrize(
"payload, error",
(
# Name
({"name": 1234}, "name should be a str"),
({"name": None}, "name should be a str"),
),
)
def test_partial_update_element_wrong_param_name(mock_elements_worker, payload, error):
api_payload = {
"element": Element({"zone": None}),
**payload,
}
with pytest.raises(AssertionError) as e:
mock_elements_worker.update_element(
element=Element({"zone": None}),
name=1234,
mock_elements_worker.partial_update_element(
**api_payload,
)
assert str(e.value) == "name should be None or a str"
assert str(e.value) == error
def test_update_element_wrong_polygon(mock_elements_worker):
elt = Element({"zone": None})
@pytest.mark.parametrize(
"payload, error",
(
# Polygon
({"polygon": "not a polygon"}, "polygon should be a list"),
({"polygon": None}, "polygon should be a list"),
({"polygon": [[1, 1], [2, 2]]}, "polygon should have at least three points"),
(
{"polygon": [[1, 1, 1], [2, 2, 1], [2, 1, 1], [1, 2, 1]]},
"polygon points should be lists of two items",
),
(
{"polygon": [[1], [2], [2], [1]]},
"polygon points should be lists of two items",
),
(
{"polygon": [["not a coord", 1], [2, 2], [2, 1], [1, 2]]},
"polygon points should be lists of two numbers",
),
),
)
def test_partial_update_element_wrong_param_polygon(
mock_elements_worker, payload, error
):
api_payload = {
"element": Element({"zone": None}),
**payload,
}
with pytest.raises(AssertionError) as e:
mock_elements_worker.update_element(
element=elt,
polygon="not a polygon",
mock_elements_worker.partial_update_element(
**api_payload,
)
assert str(e.value) == "polygon should be None or a list"
assert str(e.value) == error
with pytest.raises(AssertionError) as e:
mock_elements_worker.update_element(
element=elt,
polygon=[[1, 1], [2, 2]],
)
assert str(e.value) == "polygon should have at least three points"
@pytest.mark.parametrize(
"payload, error",
(
# Confidence
({"confidence": "lol"}, "confidence should be None or a float in [0..1] range"),
({"confidence": "0.2"}, "confidence should be None or a float in [0..1] range"),
({"confidence": -1.0}, "confidence should be None or a float in [0..1] range"),
({"confidence": 1.42}, "confidence should be None or a float in [0..1] range"),
(
{"confidence": float("inf")},
"confidence should be None or a float in [0..1] range",
),
),
)
def test_partial_update_element_wrong_param_conf(mock_elements_worker, payload, error):
api_payload = {
"element": Element({"zone": None}),
**payload,
}
with pytest.raises(AssertionError) as e:
mock_elements_worker.update_element(
element=elt,
polygon=[[1, 1, 1], [2, 2, 1], [2, 1, 1], [1, 2, 1]],
mock_elements_worker.partial_update_element(
**api_payload,
)
assert str(e.value) == "polygon points should be lists of two items"
assert str(e.value) == error
@pytest.mark.parametrize(
"payload, error",
(
# Rotation angle
({"rotation_angle": "lol"}, "rotation_angle should be a positive integer"),
({"rotation_angle": -1}, "rotation_angle should be a positive integer"),
({"rotation_angle": 0.5}, "rotation_angle should be a positive integer"),
({"rotation_angle": None}, "rotation_angle should be a positive integer"),
),
)
def test_partial_update_element_wrong_param_rota(mock_elements_worker, payload, error):
api_payload = {
"element": Element({"zone": None}),
**payload,
}
with pytest.raises(AssertionError) as e:
mock_elements_worker.update_element(
element=elt,
polygon=[[1], [2], [2], [1]],
mock_elements_worker.partial_update_element(
**api_payload,
)
assert str(e.value) == "polygon points should be lists of two items"
assert str(e.value) == error
@pytest.mark.parametrize(
"payload, error",
(
# Mirrored
({"mirrored": "lol"}, "mirrored should be a boolean"),
({"mirrored": 1234}, "mirrored should be a boolean"),
({"mirrored": None}, "mirrored should be a boolean"),
),
)
def test_partial_update_element_wrong_param_mir(mock_elements_worker, payload, error):
api_payload = {
"element": Element({"zone": None}),
**payload,
}
with pytest.raises(AssertionError) as e:
mock_elements_worker.update_element(
element=elt,
polygon=[["not a coord", 1], [2, 2], [2, 1], [1, 2]],
mock_elements_worker.partial_update_element(
**api_payload,
)
assert str(e.value) == "polygon points should be lists of two numbers"
assert str(e.value) == error
@pytest.mark.parametrize("confidence", ["lol", "0.2", -1.0, 1.42, float("inf")])
def test_update_element_wrong_confidence(mock_elements_worker, confidence):
@pytest.mark.parametrize(
"payload, error",
(
# Image
({"image": "lol"}, "image should be a UUID"),
({"image": 1234}, "image should be a UUID"),
({"image": None}, "image should be a UUID"),
),
)
def test_partial_update_element_wrong_param_image(mock_elements_worker, payload, error):
api_payload = {
"element": Element({"zone": None}),
**payload,
}
with pytest.raises(AssertionError) as e:
mock_elements_worker.update_element(
element=Element({"zone": None}),
confidence=confidence,
mock_elements_worker.partial_update_element(
**api_payload,
)
assert str(e.value) == "confidence should be None or a float in [0..1] range"
assert str(e.value) == error
def test_update_element_api_error(responses, mock_elements_worker):
def test_partial_update_element_api_error(responses, mock_elements_worker):
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
responses.add(
responses.PATCH,
......@@ -1306,7 +1420,7 @@ def test_update_element_api_error(responses, mock_elements_worker):
)
with pytest.raises(ErrorResponse):
mock_elements_worker.update_element(
mock_elements_worker.partial_update_element(
element=elt,
type="something",
name="0",
......@@ -1326,26 +1440,56 @@ def test_update_element_api_error(responses, mock_elements_worker):
]
def test_update_element(
responses, mock_elements_worker_with_cache, mock_cached_elements
@pytest.mark.parametrize(
"payload",
(
(
{
"polygon": [[10, 10], [20, 20], [20, 10], [10, 20]],
"confidence": None,
}
),
(
{
"rotation_angle": 45,
"mirrored": False,
}
),
(
{
"polygon": [[10, 10], [20, 20], [20, 10], [10, 20]],
"confidence": None,
"rotation_angle": 45,
"mirrored": False,
}
),
),
)
def test_partial_update_element(
responses,
mock_elements_worker_with_cache,
mock_cached_elements,
mock_cached_images,
payload,
):
elt = CachedElement.select().first()
new_image = CachedImage.select().first()
elt_response = {
"type": "new type",
"name": "new name",
"polygon": [[10, 10], [20, 20], [20, 10], [10, 20]],
"confidence": None,
"image": str(new_image.id),
**payload,
}
responses.add(
responses.PATCH,
f"http://testserver/api/v1/element/{elt.id}/",
status=200,
# UUID not allowed in JSON
json=elt_response,
)
element_update_response = mock_elements_worker_with_cache.update_element(
element_update_response = mock_elements_worker_with_cache.partial_update_element(
element=elt,
**elt_response,
**{**elt_response, "image": new_image.id},
)
assert len(responses.calls) == len(BASE_API_CALLS) + 1
......@@ -1361,20 +1505,25 @@ def test_update_element(
assert element_update_response == elt_response
cached_element = CachedElement.get(CachedElement.id == elt.id)
assert cached_element.type == elt_response["type"]
assert cached_element.polygon == str(elt_response["polygon"])
assert cached_element.confidence == elt_response["confidence"]
# Always present in payload
assert str(cached_element.image_id) == elt_response["image"]
# Optional params
if "polygon" in payload:
# Cast to string as this is the only difference compared to model
elt_response["polygon"] = str(elt_response["polygon"])
for param in payload:
assert getattr(cached_element, param) == elt_response[param]
def test_update_element_confidence(
responses, mock_elements_worker_with_cache, mock_cached_elements
@pytest.mark.parametrize("confidence", (None, 0.42))
def test_partial_update_element_confidence(
responses, mock_elements_worker_with_cache, mock_cached_elements, confidence
):
elt = CachedElement.select().first()
elt_response = {
"type": "new type",
"name": "new name",
"polygon": [[10, 10], [20, 20], [20, 10], [10, 20]],
"confidence": 0.42,
"confidence": confidence,
}
responses.add(
responses.PATCH,
......@@ -1383,7 +1532,7 @@ def test_update_element_confidence(
json=elt_response,
)
element_update_response = mock_elements_worker_with_cache.update_element(
element_update_response = mock_elements_worker_with_cache.partial_update_element(
element=elt,
**elt_response,
)
......@@ -1401,9 +1550,8 @@ def test_update_element_confidence(
assert element_update_response == elt_response
cached_element = CachedElement.get(CachedElement.id == elt.id)
assert cached_element.type == elt_response["type"]
assert cached_element.polygon == str(elt_response["polygon"])
assert cached_element.confidence == elt_response["confidence"]
assert cached_element.confidence == confidence
def test_list_element_children_wrong_element(mock_elements_worker):
......
tests/test_elements_worker/test_entities.py
View file @ 426b05af
......@@ -12,43 +12,16 @@ from arkindex_worker.cache import (
CachedTranscription,
CachedTranscriptionEntity,
)
from arkindex_worker.models import Element, Transcription
from arkindex_worker.models import Transcription
from arkindex_worker.worker.entity import MissingEntityType
from arkindex_worker.worker.transcription import TextOrientation
from . import BASE_API_CALLS
def test_create_entity_wrong_element(mock_elements_worker):
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=None,
name="Bob Bob",
type="person",
)
assert (
str(e.value)
== "element shouldn't be null and should be an Element or CachedElement"
)
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element="not element type",
name="Bob Bob",
type="person",
)
assert (
str(e.value)
== "element shouldn't be null and should be an Element or CachedElement"
)
def test_create_entity_wrong_name(mock_elements_worker):
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=elt,
name=None,
type="person",
)
......@@ -56,7 +29,6 @@ def test_create_entity_wrong_name(mock_elements_worker):
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=elt,
name=1234,
type="person",
)
......@@ -64,11 +36,8 @@ def test_create_entity_wrong_name(mock_elements_worker):
def test_create_entity_wrong_type(mock_elements_worker):
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type=None,
)
......@@ -76,7 +45,6 @@ def test_create_entity_wrong_type(mock_elements_worker):
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type=1234,
)
......@@ -84,13 +52,10 @@ def test_create_entity_wrong_type(mock_elements_worker):
def test_create_entity_wrong_corpus(monkeypatch, mock_elements_worker):
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
# Triggering an error on metas param, not giving corpus should work since
# ARKINDEX_CORPUS_ID environment variable is set on mock_elements_worker
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type="person",
metas="wrong metas",
......@@ -99,11 +64,8 @@ def test_create_entity_wrong_corpus(monkeypatch, mock_elements_worker):
def test_create_entity_wrong_metas(mock_elements_worker):
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type="person",
metas="wrong metas",
......@@ -112,11 +74,8 @@ def test_create_entity_wrong_metas(mock_elements_worker):
def test_create_entity_wrong_validated(mock_elements_worker):
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type="person",
validated="wrong validated",
......@@ -127,7 +86,6 @@ def test_create_entity_wrong_validated(mock_elements_worker):
def test_create_entity_api_error(responses, mock_elements_worker):
# Set one entity type
mock_elements_worker.entity_types = {"person": "person-entity-type-id"}
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
responses.add(
responses.POST,
"http://testserver/api/v1/entity/",
......@@ -136,7 +94,6 @@ def test_create_entity_api_error(responses, mock_elements_worker):
with pytest.raises(ErrorResponse):
mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type="person",
)
......@@ -158,7 +115,6 @@ def test_create_entity(responses, mock_elements_worker):
# Set one entity type
mock_elements_worker.entity_types = {"person": "person-entity-type-id"}
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
responses.add(
responses.POST,
"http://testserver/api/v1/entity/",
......@@ -167,7 +123,6 @@ def test_create_entity(responses, mock_elements_worker):
)
entity_id = mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type="person",
)
......@@ -193,8 +148,6 @@ def test_create_entity_missing_type(responses, mock_elements_worker):
"""
Create entity with an unknown type will fail.
"""
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
# Call to list entity types
responses.add(
responses.GET,
......@@ -213,7 +166,6 @@ def test_create_entity_missing_type(responses, mock_elements_worker):
AssertionError, match="Entity type `new-entity` not found in the corpus."
):
mock_elements_worker.create_entity(
element=elt,
name="Bob Bob",
type="new-entity",
)
......@@ -232,7 +184,6 @@ def test_create_entity_missing_type(responses, mock_elements_worker):
def test_create_entity_with_cache(responses, mock_elements_worker_with_cache):
# Set one entity type
mock_elements_worker_with_cache.entity_types = {"person": "person-entity-type-id"}
elt = CachedElement.create(id="12341234-1234-1234-1234-123412341234", type="thing")
responses.add(
responses.POST,
"http://testserver/api/v1/entity/",
......@@ -241,7 +192,6 @@ def test_create_entity_with_cache(responses, mock_elements_worker_with_cache):
)
entity_id = mock_elements_worker_with_cache.create_entity(
element=elt,
name="Bob Bob",
type="person",
)
......@@ -910,22 +860,11 @@ def test_create_transcription_entities_wrong_transcription(
)
def test_create_transcription_entities_no_transcription_element(mock_elements_worker):
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_transcription_entities(
transcription=Transcription(id="transcription_id"),
entities=[],
)
assert str(e.value) == "No element linked to Transcription (transcription_id)"
@pytest.mark.parametrize("entities", (None, "not a list of entities", 1))
def test_create_transcription_entities_wrong_entities(mock_elements_worker, entities):
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_transcription_entities(
transcription=Transcription(
id="transcription_id", element={"id": "element_id"}
),
transcription=Transcription(id="transcription_id"),
entities=entities,
)
assert str(e.value) == "entities shouldn't be null and should be of type list"
......@@ -934,9 +873,7 @@ def test_create_transcription_entities_wrong_entities(mock_elements_worker, enti
def test_create_transcription_entities_wrong_entities_subtype(mock_elements_worker):
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_transcription_entities(
transcription=Transcription(
id="transcription_id", element={"id": "element_id"}
),
transcription=Transcription(id="transcription_id"),
entities=["not a dict"],
)
assert str(e.value) == "Entity at index 0 in entities: Should be of type dict"
......@@ -1030,17 +967,14 @@ def test_create_transcription_entities_wrong_entity(
):
with pytest.raises(AssertionError) as e:
mock_elements_worker.create_transcription_entities(
transcription=Transcription(
id="transcription_id", element={"id": "element_id"}
),
transcription=Transcription(id="transcription_id"),
entities=[entity],
)
assert str(e.value) == error
def test_create_transcription_entities(responses, mock_elements_worker):
element_id = "element_id"
transcription = Transcription(id="transcription-id", element={"id": element_id})
transcription = Transcription(id="transcription-id")
# Call to Transcription entities creation in bulk
responses.add(
responses.POST,
......@@ -1091,35 +1025,6 @@ def test_create_transcription_entities(responses, mock_elements_worker):
assert len(created_objects) == 1
assert element_id in mock_elements_worker.report.report_data["elements"]
ml_report = mock_elements_worker.report.report_data["elements"][element_id]
assert "started" in ml_report
del ml_report["started"]
# Check reporting
assert ml_report == {
"elements": {},
"transcriptions": 0,
"classifications": {},
"entities": [
{
"id": "entity-id",
"type": "22222222-2222-2222-2222-222222222222",
"name": "Teklia",
}
],
"transcription_entities": [
{
"transcription_id": "transcription-id",
"entity_id": "entity-id",
"transcription_entity_id": "transc-entity-id",
}
],
"metadata": [],
"errors": [],
}
assert len(responses.calls) == len(BASE_API_CALLS) + 1
assert [
(call.request.method, call.request.url) for call in responses.calls
......
tests/test_reporting.py deleted 100644 → 0
View file @ 37eddfc3
# -*- coding: utf-8 -*-
import json
import uuid
from datetime import datetime
from tempfile import NamedTemporaryFile
import pytest
from apistar.exceptions import ErrorResponse
from arkindex_worker.models import Transcription
from arkindex_worker.reporting import Reporter
def test_init():
version_id = str(uuid.uuid4())
reporter = Reporter(name="Worker", slug="worker-slug", version=version_id)
assert "started" in reporter.report_data
del reporter.report_data["started"]
assert reporter.report_data == {
"name": "Worker",
"slug": "worker-slug",
"version": version_id,
"elements": {},
}
def test_process():
reporter = Reporter("worker")
reporter.process("myelement")
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
assert "started" in element_data
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 0,
"classifications": {},
"entities": [],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_element():
reporter = Reporter("worker")
reporter.add_element("myelement", type="text_line")
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {"text_line": 1},
"transcriptions": 0,
"classifications": {},
"entities": [],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_element_count():
"""
Report multiple elements with the same parent and type
"""
reporter = Reporter("worker")
reporter.add_element("myelement", type="text_line", type_count=42)
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {"text_line": 42},
"transcriptions": 0,
"classifications": {},
"entities": [],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_classification():
reporter = Reporter("worker")
reporter.add_classification("myelement", "three")
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 0,
"classifications": {"three": 1},
"entities": [],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_classifications():
reporter = Reporter("worker")
with pytest.raises(AssertionError):
reporter.add_classifications("myelement", {"not": "a list"})
reporter.add_classifications(
"myelement", [{"class_name": "three"}, {"class_name": "two"}]
)
reporter.add_classifications(
"myelement",
[
{"class_name": "three"},
{"class_name": "two", "high_confidence": True},
{"class_name": "three", "confidence": 0.42},
],
)
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 0,
"classifications": {"three": 3, "two": 2},
"entities": [],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_transcription():
reporter = Reporter("worker")
reporter.add_transcription("myelement")
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 1,
"classifications": {},
"entities": [],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_transcription_count():
"""
Report multiple transcriptions with the same element and type
"""
reporter = Reporter("worker")
reporter.add_transcription("myelement", 1337)
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 1337,
"classifications": {},
"entities": [],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_entity():
reporter = Reporter("worker")
reporter.add_entity(
"myelement",
"12341234-1234-1234-1234-123412341234",
"person-entity-type-id",
"Bob Bob",
)
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 0,
"classifications": {},
"entities": [
{
"id": "12341234-1234-1234-1234-123412341234",
"type": "person-entity-type-id",
"name": "Bob Bob",
}
],
"transcription_entities": [],
"metadata": [],
"errors": [],
}
def test_add_transcription_entity():
reporter = Reporter("worker")
reporter.add_transcription_entity(
"5678",
Transcription({"id": "1234-5678", "element": {"id": "myelement"}}),
"1234",
)
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 0,
"classifications": {},
"entities": [],
"transcription_entities": [
{
"transcription_id": "1234-5678",
"entity_id": "5678",
"transcription_entity_id": "1234",
}
],
"metadata": [],
"errors": [],
}
def test_add_metadata():
reporter = Reporter("worker")
reporter.add_metadata(
"myelement", "12341234-1234-1234-1234-123412341234", "location", "Teklia"
)
assert "myelement" in reporter.report_data["elements"]
element_data = reporter.report_data["elements"]["myelement"]
del element_data["started"]
assert element_data == {
"elements": {},
"transcriptions": 0,
"classifications": {},
"entities": [],
"transcription_entities": [],
"metadata": [
{
"id": "12341234-1234-1234-1234-123412341234",
"type": "location",
"name": "Teklia",
}
],
"errors": [],
}
def test_error():
reporter = Reporter("worker")
reporter.error("myelement", ZeroDivisionError("What have you done"))
reporter.error(
"myelement",
ErrorResponse(
title="I'm a teapot",
status_code=418,
content='{"coffee": "Can\'t touch this"}',
),
)
assert reporter.report_data["elements"]["myelement"]["errors"] == [
{"class": "ZeroDivisionError", "message": "What have you done"},
{
"class": "ErrorResponse",
"message": "I'm a teapot",
"status_code": 418,
"content": '{"coffee": "Can\'t touch this"}',
},
]
def test_reporter_save(mocker):
datetime_mock = mocker.MagicMock()
datetime_mock.utcnow.return_value = datetime(2000, 1, 1)
mocker.patch("arkindex_worker.reporting.datetime", datetime_mock)
version_id = str(uuid.uuid4())
reporter = Reporter(name="Worker", slug="worker-slug", version=version_id)
reporter.add_element("myelement", type="text_line", type_count=4)
with NamedTemporaryFile() as f:
reporter.save(f.name)
exported_data = json.load(f)
assert exported_data == {
"name": "Worker",
"slug": "worker-slug",
"started": "2000-01-01T00:00:00",
"version": version_id,
"elements": {
"myelement": {
"classifications": {},
"elements": {"text_line": 4},
"entities": [],
"transcription_entities": [],
"errors": [],
"metadata": [],
"started": "2000-01-01T00:00:00",
"transcriptions": 0,
}
},
}
worker-{{cookiecutter.slug}}/.gitlab-ci.yml
View file @ 426b05af
......@@ -50,7 +50,7 @@ lint:
docker-build:
stage: build
image: docker:19.03.1
image: docker:24.0.6
services:
- docker:dind
variables:
......
worker-{{cookiecutter.slug}}/Dockerfile
View file @ 426b05af
FROM python:3
FROM python:3.11
WORKDIR /src
......
worker-{{cookiecutter.slug}}/requirements.txt
View file @ 426b05af
arkindex-base-worker==0.3.3
arkindex-base-worker==0.3.4