From d725c62abd32272c6075941be6a89410bcb38e5d Mon Sep 17 00:00:00 2001
From: Eva Bardou <ebardou@teklia.com>
Date: Tue, 23 Mar 2021 17:40:28 +0100
Subject: [PATCH] Fix existing tests
---
arkindex_worker/worker.py | 86 ++++++++++++++++++---------------
tests/data/cache/lines.sqlite | Bin 12288 -> 20480 bytes
tests/data/cache/tables.sqlite | Bin 12288 -> 20480 bytes
tests/test_cache.py | 39 ++++++++++++++-
4 files changed, 85 insertions(+), 40 deletions(-)
diff --git a/arkindex_worker/worker.py b/arkindex_worker/worker.py
index d25badd5..ad802495 100644
--- a/arkindex_worker/worker.py
+++ b/arkindex_worker/worker.py
@@ -536,20 +536,25 @@ class ElementsWorker(BaseWorker):
self.report.add_transcription(element.id)
- # Store transcription in local cache
- try:
- to_insert = [
- CachedTranscription(
- id=convert_str_uuid_to_hex(created["id"]),
- element_id=convert_str_uuid_to_hex(element.id),
- text=created["text"],
- confidence=created["confidence"],
- worker_version_id=convert_str_uuid_to_hex(self.worker_version_id),
+ if self.use_cache:
+ # Store transcription in local cache
+ try:
+ to_insert = [
+ CachedTranscription(
+ id=convert_str_uuid_to_hex(created["id"]),
+ element_id=convert_str_uuid_to_hex(element.id),
+ text=created["text"],
+ confidence=created["confidence"],
+ worker_version_id=convert_str_uuid_to_hex(
+ self.worker_version_id
+ ),
+ )
+ ]
+ self.cache.insert("transcriptions", to_insert)
+ except sqlite3.IntegrityError as e:
+ logger.warning(
+ f"Couldn't save created transcription in local cache: {e}"
)
- ]
- self.cache.insert("transcriptions", to_insert)
- except sqlite3.IntegrityError as e:
- logger.warning(f"Couldn't save created transcription in local cache: {e}")
def create_classification(
self, element, ml_class, confidence, high_confidence=False
@@ -720,21 +725,25 @@ class ElementsWorker(BaseWorker):
},
)
- created_ids = []
- elements_to_insert = []
- transcriptions_to_insert = []
- parent_id_hex = convert_str_uuid_to_hex(element.id)
- worker_version_id_hex = convert_str_uuid_to_hex(self.worker_version_id)
- for index, annotation in enumerate(annotations):
- transcription = transcriptions[index]
- element_id_hex = convert_str_uuid_to_hex(annotation["id"])
+ for annotation in annotations:
if annotation["created"]:
logger.debug(
f"A sub_element of {element.id} with type {sub_element_type} was created during transcriptions bulk creation"
)
self.report.add_element(element.id, sub_element_type)
+ self.report.add_transcription(annotation["id"])
- if annotation["id"] not in created_ids:
+ if self.use_cache:
+ # Store transcriptions and their associated element (if created) in local cache
+ created_ids = []
+ elements_to_insert = []
+ transcriptions_to_insert = []
+ parent_id_hex = convert_str_uuid_to_hex(element.id)
+ worker_version_id_hex = convert_str_uuid_to_hex(self.worker_version_id)
+ for index, annotation in enumerate(annotations):
+ transcription = transcriptions[index]
+ element_id_hex = convert_str_uuid_to_hex(annotation["id"])
+ if annotation["created"] and annotation["id"] not in created_ids:
# TODO: Retrieve real element_name through API
elements_to_insert.append(
CachedElement(
@@ -748,25 +757,24 @@ class ElementsWorker(BaseWorker):
)
created_ids.append(annotation["id"])
- self.report.add_transcription(annotation["id"])
-
- transcriptions_to_insert.append(
- CachedTranscription(
- # TODO: Retrieve real transcription_id through API
- id=convert_str_uuid_to_hex(uuid.uuid4()),
- element_id=element_id_hex,
- text=transcription["text"],
- confidence=transcription["score"],
- worker_version_id=worker_version_id_hex,
+ transcriptions_to_insert.append(
+ CachedTranscription(
+ # TODO: Retrieve real transcription_id through API
+ id=convert_str_uuid_to_hex(uuid.uuid4()),
+ element_id=element_id_hex,
+ text=transcription["text"],
+ confidence=transcription["score"],
+ worker_version_id=worker_version_id_hex,
+ )
)
- )
- # Store transcriptions and their associated element (if created) in local cache
- try:
- self.cache.insert("elements", elements_to_insert)
- self.cache.insert("transcriptions", transcriptions_to_insert)
- except sqlite3.IntegrityError as e:
- logger.warning(f"Couldn't save created transcriptions in local cache: {e}")
+ try:
+ self.cache.insert("elements", elements_to_insert)
+ self.cache.insert("transcriptions", transcriptions_to_insert)
+ except sqlite3.IntegrityError as e:
+ logger.warning(
+ f"Couldn't save created transcriptions in local cache: {e}"
+ )
return annotations
diff --git a/tests/data/cache/lines.sqlite b/tests/data/cache/lines.sqlite
index 4294b9b0f3789629fb466debb0d8931f589f9c59..e7024d74adf29e30a46addfcc0b4c55a3203a3a1 100644
GIT binary patch
delta 388
zcmZojXjs5FL0XW7fq{V)h+%+fqK>gB3xl5hO<w*V3@p6S41D|ekMl<JHEt|?&g0gk
z%*-yXsma*FSdy5OlUh=gm{*)!lvz-cnV(mT%42a3a&-)GRS0o(@^Mu_k(>O9PlF{j
zCpCAnEWe_9NoqxjLWpZbh=QL#5QqBs=x`|jL2`axT4qXWUUI4e&?KM9Gx>EU>+|!Q
zyZHyXdb;~5c)LbwfDB2^D~Zoc(F7`Ta}9F!b9N0@fC?9D0L8g9ty$Q_{WT|_<(H60
zaYk{xAu6AB^FBT&MIK&YU@-CXG4OZs^KBMXVB%-5Vq)|4Rb`+2Ku&S;2RT6?<`?nE
zFIPw`N>!-LFSWP7_0WBtNY%s159Ad%N*IBPCE0x^Kh&3>{6Sv;g!v6UQgd?h6=4cD
I0~HDY0CJsp{{R30
delta 56
zcmZozz}S#5L0XWBfq{V;h+%+nqK+|8P|yA*FaHk)Ccb?PeEay1Zx$5r<J-KC&q)yg
D7q$zd
diff --git a/tests/data/cache/tables.sqlite b/tests/data/cache/tables.sqlite
index be29e25631f18d386ad8c644ead09e91792a3d43..b4347205395dd22dba0ce3446f3260d2cd7cb0bb 100644
GIT binary patch
delta 325
zcmZojXjs5FL0XWBfq{V)h+%+fqK>gB6N8@pO<w*V3@p4U41CA=kMpMRb!{yCz~k1W
z%*-yXsma*FSdy5OlUh=gm{*)!lvz-cnV(mT%42a3a&-)GRS0o(@^Mu_k(>ORPs1lQ
zCp9-UuOvP*MIp>F$l1d&NW<7jQ^C(aM8PlA$47@t0SHP`D@qhXTqA&DDB{Wad1;v`
zsd>q%3P4+YCeP&8aYIw$<{#wh>F%fC?HZ|puuBuD-pw`0)z8^ASOF?rtdW_b$)#z{
f!Y1yoIr%KVgglCCi{lMZ`K+6d^I0k`5>NmDC30c*
delta 56
zcmZozz}S#5L0XWJfq{V;h+%+nqK+|8P|yA*FaHk)Ccfhge8>5ZZx$2?=i7Xo&r%Tp
D7{Lp?
diff --git a/tests/test_cache.py b/tests/test_cache.py
index fde2c681..9564655f 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -6,7 +6,7 @@ from pathlib import Path
import pytest
-from arkindex_worker.cache import CachedElement, LocalDB
+from arkindex_worker.cache import CachedElement, CachedTranscription, LocalDB
from arkindex_worker.utils import convert_str_uuid_to_hex
FIXTURES = Path(__file__).absolute().parent / "data/cache"
@@ -32,6 +32,26 @@ ELEMENTS_TO_INSERT = [
),
),
]
+TRANSCRIPTIONS_TO_INSERT = [
+ CachedTranscription(
+ id=convert_str_uuid_to_hex("11111111-1111-1111-1111-111111111111"),
+ element_id=convert_str_uuid_to_hex("11111111-1111-1111-1111-111111111111"),
+ text="Hello!",
+ confidence=0.42,
+ worker_version_id=convert_str_uuid_to_hex(
+ "56785678-5678-5678-5678-567856785678"
+ ),
+ ),
+ CachedTranscription(
+ id=convert_str_uuid_to_hex("22222222-2222-2222-2222-222222222222"),
+ element_id=convert_str_uuid_to_hex("22222222-2222-2222-2222-222222222222"),
+ text="How are you?",
+ confidence=0.42,
+ worker_version_id=convert_str_uuid_to_hex(
+ "56785678-5678-5678-5678-567856785678"
+ ),
+ ),
+]
def test_init_non_existent_path():
@@ -110,6 +130,10 @@ def test_insert_existing_lines():
cache.insert("elements", ELEMENTS_TO_INSERT)
assert str(e.value) == "UNIQUE constraint failed: elements.id"
+ with pytest.raises(sqlite3.IntegrityError) as e:
+ cache.insert("transcriptions", TRANSCRIPTIONS_TO_INSERT)
+ assert str(e.value) == "UNIQUE constraint failed: transcriptions.id"
+
with open(db_path, "rb") as after_file:
after = after_file.read()
@@ -130,3 +154,16 @@ def test_insert():
)
assert [CachedElement(**dict(row)) for row in generated_rows] == ELEMENTS_TO_INSERT
+
+ cache.insert("transcriptions", TRANSCRIPTIONS_TO_INSERT)
+ generated_rows = cache.cursor.execute("SELECT * FROM transcriptions").fetchall()
+
+ expected_cache = LocalDB(f"{FIXTURES}/lines.sqlite")
+ assert (
+ generated_rows
+ == expected_cache.cursor.execute("SELECT * FROM transcriptions").fetchall()
+ )
+
+ assert [
+ CachedTranscription(**dict(row)) for row in generated_rows
+ ] == TRANSCRIPTIONS_TO_INSERT
--
GitLab