Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • workers/base-worker
1 result
Show changes
Hide whitespace changes
Inline Side-by-side
tests/test_elements_worker/test_transcriptions.py
View file @ b6f9d725
......@@ -8,6 +8,8 @@ from apistar.exceptions import ErrorResponse
from arkindex_worker.cache import CachedElement, CachedTranscription
from arkindex_worker.models import Element
from . import BASE_API_CALLS
TRANSCRIPTIONS_SAMPLE = [
{
"polygon": [[100, 150], [700, 150], [700, 200], [100, 200]],
......@@ -130,16 +132,16 @@ def test_create_transcription_api_error(responses, mock_elements_worker):
score=0.42,
)
assert len(responses.calls) == 7
assert [call.request.url for call in responses.calls] == [
"http://testserver/api/v1/user/",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
assert len(responses.calls) == len(BASE_API_CALLS) + 5
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS + [
# We retry 5 times the API call
f"http://testserver/api/v1/element/{elt.id}/transcription/",
f"http://testserver/api/v1/element/{elt.id}/transcription/",
f"http://testserver/api/v1/element/{elt.id}/transcription/",
f"http://testserver/api/v1/element/{elt.id}/transcription/",
f"http://testserver/api/v1/element/{elt.id}/transcription/",
("POST", f"http://testserver/api/v1/element/{elt.id}/transcription/"),
("POST", f"http://testserver/api/v1/element/{elt.id}/transcription/"),
("POST", f"http://testserver/api/v1/element/{elt.id}/transcription/"),
("POST", f"http://testserver/api/v1/element/{elt.id}/transcription/"),
("POST", f"http://testserver/api/v1/element/{elt.id}/transcription/"),
]
......@@ -164,14 +166,14 @@ def test_create_transcription(responses, mock_elements_worker):
score=0.42,
)
assert len(responses.calls) == 3
assert [call.request.url for call in responses.calls] == [
"http://testserver/api/v1/user/",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
f"http://testserver/api/v1/element/{elt.id}/transcription/",
assert len(responses.calls) == len(BASE_API_CALLS) + 1
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS + [
("POST", f"http://testserver/api/v1/element/{elt.id}/transcription/"),
]
assert json.loads(responses.calls[2].request.body) == {
assert json.loads(responses.calls[-1].request.body) == {
"text": "i am a line",
"worker_version": "12341234-1234-1234-1234-123412341234",
"score": 0.42,
......@@ -200,14 +202,14 @@ def test_create_transcription_with_cache(responses, mock_elements_worker_with_ca
score=0.42,
)
assert len(responses.calls) == 3
assert [call.request.url for call in responses.calls] == [
"http://testserver/api/v1/user/",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
f"http://testserver/api/v1/element/{elt.id}/transcription/",
assert len(responses.calls) == len(BASE_API_CALLS) + 1
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS + [
("POST", f"http://testserver/api/v1/element/{elt.id}/transcription/"),
]
assert json.loads(responses.calls[2].request.body) == {
assert json.loads(responses.calls[-1].request.body) == {
"text": "i am a line",
"worker_version": "12341234-1234-1234-1234-123412341234",
"score": 0.42,
......@@ -478,16 +480,16 @@ def test_create_transcriptions_api_error(responses, mock_elements_worker):
with pytest.raises(ErrorResponse):
mock_elements_worker.create_transcriptions(transcriptions=trans)
assert len(responses.calls) == 7
assert [call.request.url for call in responses.calls] == [
"http://testserver/api/v1/user/",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
assert len(responses.calls) == len(BASE_API_CALLS) + 5
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS + [
# We retry 5 times the API call
"http://testserver/api/v1/transcription/bulk/",
"http://testserver/api/v1/transcription/bulk/",
"http://testserver/api/v1/transcription/bulk/",
"http://testserver/api/v1/transcription/bulk/",
"http://testserver/api/v1/transcription/bulk/",
("POST", "http://testserver/api/v1/transcription/bulk/"),
("POST", "http://testserver/api/v1/transcription/bulk/"),
("POST", "http://testserver/api/v1/transcription/bulk/"),
("POST", "http://testserver/api/v1/transcription/bulk/"),
("POST", "http://testserver/api/v1/transcription/bulk/"),
]
......@@ -533,14 +535,14 @@ def test_create_transcriptions(responses, mock_elements_worker_with_cache):
transcriptions=trans,
)
assert len(responses.calls) == 3
assert [call.request.url for call in responses.calls] == [
"http://testserver/api/v1/user/",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
"http://testserver/api/v1/transcription/bulk/",
assert len(responses.calls) == len(BASE_API_CALLS) + 1
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS + [
("POST", "http://testserver/api/v1/transcription/bulk/"),
]
assert json.loads(responses.calls[2].request.body) == {
assert json.loads(responses.calls[-1].request.body) == {
"worker_version": "12341234-1234-1234-1234-123412341234",
"transcriptions": trans,
}
......@@ -955,16 +957,16 @@ def test_create_element_transcriptions_api_error(responses, mock_elements_worker
transcriptions=TRANSCRIPTIONS_SAMPLE,
)
assert len(responses.calls) == 7
assert [call.request.url for call in responses.calls] == [
"http://testserver/api/v1/user/",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
assert len(responses.calls) == len(BASE_API_CALLS) + 5
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS + [
# We retry 5 times the API call
f"http://testserver/api/v1/element/{elt.id}/transcriptions/bulk/",
f"http://testserver/api/v1/element/{elt.id}/transcriptions/bulk/",
f"http://testserver/api/v1/element/{elt.id}/transcriptions/bulk/",
f"http://testserver/api/v1/element/{elt.id}/transcriptions/bulk/",
f"http://testserver/api/v1/element/{elt.id}/transcriptions/bulk/",
("POST", f"http://testserver/api/v1/element/{elt.id}/transcriptions/bulk/"),
("POST", f"http://testserver/api/v1/element/{elt.id}/transcriptions/bulk/"),
("POST", f"http://testserver/api/v1/element/{elt.id}/transcriptions/bulk/"),
("POST", f"http://testserver/api/v1/element/{elt.id}/transcriptions/bulk/"),
("POST", f"http://testserver/api/v1/element/{elt.id}/transcriptions/bulk/"),
]
......@@ -999,14 +1001,14 @@ def test_create_element_transcriptions(responses, mock_elements_worker):
transcriptions=TRANSCRIPTIONS_SAMPLE,
)
assert len(responses.calls) == 3
assert [call.request.url for call in responses.calls] == [
"http://testserver/api/v1/user/",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
f"http://testserver/api/v1/element/{elt.id}/transcriptions/bulk/",
assert len(responses.calls) == len(BASE_API_CALLS) + 1
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS + [
("POST", f"http://testserver/api/v1/element/{elt.id}/transcriptions/bulk/"),
]
assert json.loads(responses.calls[2].request.body) == {
assert json.loads(responses.calls[-1].request.body) == {
"element_type": "page",
"worker_version": "12341234-1234-1234-1234-123412341234",
"transcriptions": TRANSCRIPTIONS_SAMPLE,
......@@ -1065,14 +1067,14 @@ def test_create_element_transcriptions_with_cache(
transcriptions=TRANSCRIPTIONS_SAMPLE,
)
assert len(responses.calls) == 3
assert [call.request.url for call in responses.calls] == [
"http://testserver/api/v1/user/",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
f"http://testserver/api/v1/element/{elt.id}/transcriptions/bulk/",
assert len(responses.calls) == len(BASE_API_CALLS) + 1
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS + [
("POST", f"http://testserver/api/v1/element/{elt.id}/transcriptions/bulk/"),
]
assert json.loads(responses.calls[2].request.body) == {
assert json.loads(responses.calls[-1].request.body) == {
"element_type": "page",
"worker_version": "12341234-1234-1234-1234-123412341234",
"transcriptions": TRANSCRIPTIONS_SAMPLE,
......@@ -1200,16 +1202,31 @@ def test_list_transcriptions_api_error(responses, mock_elements_worker):
):
next(mock_elements_worker.list_transcriptions(element=elt))
assert len(responses.calls) == 7
assert [call.request.url for call in responses.calls] == [
"http://testserver/api/v1/user/",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
assert len(responses.calls) == len(BASE_API_CALLS) + 5
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS + [
# We do 5 retries
"http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/transcriptions/",
"http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/transcriptions/",
"http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/transcriptions/",
"http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/transcriptions/",
"http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/transcriptions/",
(
"GET",
"http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/transcriptions/",
),
(
"GET",
"http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/transcriptions/",
),
(
"GET",
"http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/transcriptions/",
),
(
"GET",
"http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/transcriptions/",
),
(
"GET",
"http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/transcriptions/",
),
]
......@@ -1254,87 +1271,95 @@ def test_list_transcriptions(responses, mock_elements_worker):
):
assert transcription == trans[idx]
assert len(responses.calls) == 3
assert [call.request.url for call in responses.calls] == [
"http://testserver/api/v1/user/",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
"http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/transcriptions/",
]
def test_list_transcriptions_with_cache_unhandled_param(
responses, mock_elements_worker_with_cache
):
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
with pytest.raises(AssertionError) as e:
mock_elements_worker_with_cache.list_transcriptions(
element=elt, element_type="page"
)
assert (
str(e.value)
== "When using the local cache, you can only filter by 'worker_version'"
)
def test_list_transcriptions_with_cache_skip_recursive(
responses, mock_elements_worker_with_cache
):
# When the local cache is activated and the user defines the recursive filter, we should fallback to the API
elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
trans = [
{
"id": "0000",
"text": "hey",
"confidence": 0.42,
"worker_version_id": "56785678-5678-5678-5678-567856785678",
"element": None,
},
]
responses.add(
responses.GET,
"http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/transcriptions/?recursive=True",
status=200,
json={
"count": 3,
"next": None,
"results": trans,
},
)
for idx, transcription in enumerate(
mock_elements_worker_with_cache.list_transcriptions(element=elt, recursive=True)
):
assert transcription == trans[idx]
assert len(responses.calls) == 3
assert [call.request.url for call in responses.calls] == [
"http://testserver/api/v1/user/",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
"http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/transcriptions/?recursive=True",
assert len(responses.calls) == len(BASE_API_CALLS) + 1
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS + [
(
"GET",
"http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/transcriptions/",
),
]
@pytest.mark.parametrize(
"filters, expected_ids",
(
# Filter on element should give all elements inserted
# Filter on element should give first transcription
(
{
"element": CachedElement(
id="11111111-1111-1111-1111-111111111111", type="page"
),
},
("11111111-1111-1111-1111-111111111111",),
),
# Filter on element and element_type should give first transcription
(
{
"element": CachedElement(
id="11111111-1111-1111-1111-111111111111", type="page"
),
"element_type": "page",
},
("11111111-1111-1111-1111-111111111111",),
),
# Filter on element and worker_version should give first transcription
(
{
"element": CachedElement(
id="11111111-1111-1111-1111-111111111111", type="page"
),
"worker_version": "56785678-5678-5678-5678-567856785678",
},
("11111111-1111-1111-1111-111111111111",),
),
# Filter recursively on element should give all transcriptions inserted
(
{
"element": CachedElement(id="12341234-1234-1234-1234-123412341234"),
"element": CachedElement(
id="11111111-1111-1111-1111-111111111111", type="page"
),
"recursive": True,
},
(
"11111111-1111-1111-1111-111111111111",
"22222222-2222-2222-2222-222222222222",
"33333333-3333-3333-3333-333333333333",
"44444444-4444-4444-4444-444444444444",
"55555555-5555-5555-5555-555555555555",
),
),
# Filter on element and worker version should give first element
# Filter recursively on element and worker_version should give four transcriptions
(
{
"element": CachedElement(id="12341234-1234-1234-1234-123412341234"),
"worker_version": "56785678-5678-5678-5678-567856785678",
"element": CachedElement(
id="11111111-1111-1111-1111-111111111111", type="page"
),
"worker_version": "90129012-9012-9012-9012-901290129012",
"recursive": True,
},
("11111111-1111-1111-1111-111111111111",),
(
"22222222-2222-2222-2222-222222222222",
"33333333-3333-3333-3333-333333333333",
"44444444-4444-4444-4444-444444444444",
"55555555-5555-5555-5555-555555555555",
),
),
# Filter recursively on element and element_type should give three transcriptions
(
{
"element": CachedElement(
id="11111111-1111-1111-1111-111111111111", type="page"
),
"element_type": "something_else",
"recursive": True,
},
(
"22222222-2222-2222-2222-222222222222",
"44444444-4444-4444-4444-444444444444",
"55555555-5555-5555-5555-555555555555",
),
),
),
)
......@@ -1345,8 +1370,8 @@ def test_list_transcriptions_with_cache(
filters,
expected_ids,
):
# Check we have 2 elements already present in database
assert CachedTranscription.select().count() == 2
# Check we have 5 elements already present in database
assert CachedTranscription.select().count() == 5
# Query database through cache
transcriptions = mock_elements_worker_with_cache.list_transcriptions(**filters)
......@@ -1355,8 +1380,7 @@ def test_list_transcriptions_with_cache(
assert transcription.id == UUID(expected_id)
# Check the worker never hits the API for elements
assert len(responses.calls) == 2
assert [call.request.url for call in responses.calls] == [
"http://testserver/api/v1/user/",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
]
assert len(responses.calls) == len(BASE_API_CALLS)
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS
tests/test_elements_worker/test_worker.py
View file @ b6f9d725
# -*- coding: utf-8 -*-
# . -*- coding: utf-8 -*-
import json
import sys
import pytest
from apistar.exceptions import ErrorResponse
from arkindex_worker.cache import CachedElement
from arkindex_worker.worker import ActivityState
from arkindex_worker.worker import ActivityState, ElementsWorker
# Common API calls for all workers
BASE_API_CALLS = [
"http://testserver/api/v1/user/",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/",
]
from . import BASE_API_CALLS
TEST_VERSION_ID = "test_123"
TEST_SLUG = "some_slug"
......@@ -65,45 +62,45 @@ def test_get_worker_version_slug_none(fake_dummy_worker):
assert str(excinfo.value) == "No worker version ID"
def test_defaults(responses, mock_elements_worker):
"""Test the default values from mocked calls"""
assert not mock_elements_worker.is_read_only
assert mock_elements_worker.features == {
"workers_activity": False,
"signup": False,
}
assert len(responses.calls) == 2
assert [call.request.url for call in responses.calls] == BASE_API_CALLS
def test_feature_disabled(responses, mock_elements_worker):
"""Test disabled calls do not trigger any API calls"""
assert not mock_elements_worker.is_read_only
out = mock_elements_worker.update_activity("1234-deadbeef", ActivityState.Processed)
assert out is None
assert len(responses.calls) == 2
assert [call.request.url for call in responses.calls] == BASE_API_CALLS
def test_readonly(responses, mock_elements_worker):
"""Test readonly worker does not trigger any API calls"""
# Setup the worker as read-only, but with workers_activity enabled
# Setup the worker as read-only
mock_elements_worker.worker_version_id = None
assert mock_elements_worker.is_read_only is True
mock_elements_worker.features["workers_activity"] = True
out = mock_elements_worker.update_activity("1234-deadbeef", ActivityState.Processed)
assert out is None
assert len(responses.calls) == 2
assert [call.request.url for call in responses.calls] == BASE_API_CALLS
assert len(responses.calls) == len(BASE_API_CALLS)
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS
def test_update_call(responses, mock_elements_worker):
def test_activities_disabled(
responses, monkeypatch, mock_worker_version_api, mock_user_api
):
"""Test worker process elements without updating activities when they are disabled for the process"""
responses.add(
responses.GET,
"http://testserver/api/v1/imports/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/",
status=200,
body=json.dumps({"activity_state": "disabled"}),
content_type="application/json",
)
monkeypatch.setattr(sys, "argv", ["worker"])
worker = ElementsWorker()
worker.configure()
assert not worker.is_read_only
assert len(responses.calls) == len(BASE_API_CALLS)
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS
def test_update_call(responses, mock_elements_worker, mock_process_api):
"""Test an update call with feature enabled triggers an API call"""
responses.add(
responses.PUT,
......@@ -111,29 +108,34 @@ def test_update_call(responses, mock_elements_worker):
status=200,
json={
"element_id": "1234-deadbeef",
"process_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
"state": "processed",
},
)
# Enable worker activity
mock_elements_worker.features["workers_activity"] = True
out = mock_elements_worker.update_activity("1234-deadbeef", ActivityState.Processed)
# Check the response received by worker
assert out == {
"element_id": "1234-deadbeef",
"process_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
"state": "processed",
}
assert len(responses.calls) == 3
assert [call.request.url for call in responses.calls] == BASE_API_CALLS + [
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/activity/",
assert len(responses.calls) == len(BASE_API_CALLS) + 1
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS + [
(
"PUT",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/activity/",
),
]
# Check the request sent by worker
assert json.loads(responses.calls[2].request.body) == {
assert json.loads(responses.calls[-1].request.body) == {
"element_id": "1234-deadbeef",
"process_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
"state": "processed",
}
......@@ -185,6 +187,7 @@ def test_run(
status=200,
json={
"element_id": "1234-deadbeef",
"process_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
"state": "started",
},
)
......@@ -194,13 +197,12 @@ def test_run(
status=200,
json={
"element_id": "1234-deadbeef",
"process_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
"state": final_state,
},
)
# Enable worker activity
assert mock_elements_worker.is_read_only is False
mock_elements_worker.features["workers_activity"] = True
# Mock exception in process_element
if process_exception:
......@@ -217,20 +219,30 @@ def test_run(
# Simply run the process
mock_elements_worker.run()
assert len(responses.calls) == 5
assert [call.request.url for call in responses.calls] == BASE_API_CALLS + [
"http://testserver/api/v1/element/1234-deadbeef/",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/activity/",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/activity/",
assert len(responses.calls) == len(BASE_API_CALLS) + 3
assert [
(call.request.method, call.request.url) for call in responses.calls
] == BASE_API_CALLS + [
("GET", "http://testserver/api/v1/element/1234-deadbeef/"),
(
"PUT",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/activity/",
),
(
"PUT",
"http://testserver/api/v1/workers/versions/12341234-1234-1234-1234-123412341234/activity/",
),
]
# Check the requests sent by worker
assert json.loads(responses.calls[3].request.body) == {
assert json.loads(responses.calls[-2].request.body) == {
"element_id": "1234-deadbeef",
"process_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
"state": "started",
}
assert json.loads(responses.calls[4].request.body) == {
assert json.loads(responses.calls[-1].request.body) == {
"element_id": "1234-deadbeef",
"process_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
"state": final_state,
}
......
tests/test_merge.py
View file @ b6f9d725
......@@ -5,9 +5,14 @@ import pytest
from arkindex_worker.cache import (
MODELS,
CachedClassification,
CachedElement,
CachedEntity,
CachedImage,
CachedTranscription,
CachedTranscriptionEntity,
merge_parents_cache,
retrieve_parents_cache_path,
)
......@@ -76,20 +81,27 @@ def test_merge_databases(
# We always start with an empty database
with mock_databases["target"]["db"].bind_ctx(MODELS):
assert CachedImage.select().count() == 0
assert CachedElement.select().count() == 0
assert CachedTranscription.select().count() == 0
assert CachedClassification.select().count() == 0
assert CachedEntity.select().count() == 0
assert CachedTranscriptionEntity.select().count() == 0
# Retrieve parents databases paths
paths = retrieve_parents_cache_path(parents, data_dir=tmpdir)
# Merge all requested parents databases into our target
merge_parents_cache(
parents,
mock_databases["target"]["path"],
data_dir=tmpdir,
)
merge_parents_cache(paths, mock_databases["target"]["path"])
# The target now should have the expected elements and transcriptions
with mock_databases["target"]["db"].bind_ctx(MODELS):
assert CachedImage.select().count() == 0
assert CachedElement.select().count() == len(expected_elements)
assert CachedTranscription.select().count() == len(expected_transcriptions)
assert CachedClassification.select().count() == 0
assert CachedEntity.select().count() == 0
assert CachedTranscriptionEntity.select().count() == 0
assert [
e.id for e in CachedElement.select().order_by("id")
] == expected_elements
......@@ -105,27 +117,35 @@ def test_merge_chunk(mock_databases, tmpdir, monkeypatch):
"""
# At first we have nothing in target
with mock_databases["target"]["db"].bind_ctx(MODELS):
assert CachedImage.select().count() == 0
assert CachedElement.select().count() == 0
assert CachedTranscription.select().count() == 0
assert CachedClassification.select().count() == 0
assert CachedEntity.select().count() == 0
assert CachedTranscriptionEntity.select().count() == 0
# Check filenames
assert mock_databases["chunk_42"]["path"].basename == "db_42.sqlite"
assert mock_databases["second"]["path"].basename == "db.sqlite"
merge_parents_cache(
paths = retrieve_parents_cache_path(
[
"chunk_42",
"first",
],
mock_databases["target"]["path"],
data_dir=tmpdir,
chunk="42",
)
merge_parents_cache(paths, mock_databases["target"]["path"])
# The target should now have 3 elements and 0 transcription
with mock_databases["target"]["db"].bind_ctx(MODELS):
assert CachedImage.select().count() == 0
assert CachedElement.select().count() == 3
assert CachedTranscription.select().count() == 0
assert CachedClassification.select().count() == 0
assert CachedEntity.select().count() == 0
assert CachedTranscriptionEntity.select().count() == 0
assert [e.id for e in CachedElement.select().order_by("id")] == [
UUID("42424242-4242-4242-4242-424242424242"),
UUID("12341234-1234-1234-1234-123412341234"),
......@@ -146,9 +166,14 @@ def test_merge_from_worker(
json={"parents": ["first", "second"]},
)
# At first we have no data in our main database
assert CachedElement.select().count() == 0
assert CachedTranscription.select().count() == 0
# At first we have nothing in target
with mock_databases["target"]["db"].bind_ctx(MODELS):
assert CachedImage.select().count() == 0
assert CachedElement.select().count() == 0
assert CachedTranscription.select().count() == 0
assert CachedClassification.select().count() == 0
assert CachedEntity.select().count() == 0
assert CachedTranscriptionEntity.select().count() == 0
# Configure worker with a specific data directory
monkeypatch.setenv("PONOS_DATA", str(tmpdir))
......@@ -157,8 +182,12 @@ def test_merge_from_worker(
mock_base_worker_with_cache.configure()
# Then we have 2 elements and a transcription
assert CachedImage.select().count() == 0
assert CachedElement.select().count() == 3
assert CachedTranscription.select().count() == 1
assert CachedClassification.select().count() == 0
assert CachedEntity.select().count() == 0
assert CachedTranscriptionEntity.select().count() == 0
assert [e.id for e in CachedElement.select().order_by("id")] == [
UUID("12341234-1234-1234-1234-123412341234"),
UUID("56785678-5678-5678-5678-567856785678"),
......
worker-{{cookiecutter.slug}}/requirements.txt
View file @ b6f9d725
arkindex-base-worker==0.1.14
arkindex-base-worker==0.2.0