From b38754b50227f119a47e902e888edd4d9cf2155a Mon Sep 17 00:00:00 2001
From: Eva Bardou <ebardou@teklia.com>
Date: Wed, 17 Mar 2021 13:00:19 +0100
Subject: [PATCH] Store created elements in a local SQLite database
---
arkindex_worker/cache.py | 31 +++++++++++++++++++++++++++++++
arkindex_worker/worker.py | 17 +++++++++++++++++
2 files changed, 48 insertions(+)
create mode 100644 arkindex_worker/cache.py
diff --git a/arkindex_worker/cache.py b/arkindex_worker/cache.py
new file mode 100644
index 00000000..d6c18c9d
--- /dev/null
+++ b/arkindex_worker/cache.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+import os
+import sqlite3
+
+
+class LocalDB(object):
+ def __init__(self, path):
+ if not os.path.exists(path):
+ open(path, "x").close()
+
+ self.db = sqlite3.connect(path)
+ self.cursor = self.db.cursor()
+
+ def create_elements_table(self):
+ try:
+ self.cursor.execute(
+ """CREATE TABLE elements (
+ id TEXT PRIMARY KEY,
+ parent_id TEXT,
+ name TEXT NOT NULL,
+ type TEXT NOT NULL,
+ polygon TEXT,
+ worker_version_id TEXT
+ )"""
+ )
+ except sqlite3.OperationalError:
+ print("Table 'elements' already exists")
+
+ def insert(self, table, lines):
+ self.cursor.executemany(f"INSERT INTO {table} VALUES (?,?,?,?,?,?)", lines)
+ self.db.commit()
diff --git a/arkindex_worker/worker.py b/arkindex_worker/worker.py
index 3bfc1be0..830f7b51 100644
--- a/arkindex_worker/worker.py
+++ b/arkindex_worker/worker.py
@@ -16,6 +16,7 @@ from apistar.exceptions import ErrorResponse
from arkindex import ArkindexClient, options_from_env
from arkindex_worker import logger
+from arkindex_worker.cache import LocalDB
from arkindex_worker.models import Element
from arkindex_worker.reporting import Reporter
@@ -451,6 +452,22 @@ class ElementsWorker(BaseWorker):
for element in elements:
self.report.add_element(parent.id, element["type"])
+ # Store elements in local cache
+ cache = LocalDB(f"/data/{os.environ.get('TASK_ID')}/db.sqlite")
+ cache.create_elements_table()
+ to_insert = [
+ (
+ created_ids[idx],
+ parent.id,
+ element["name"],
+ element["type"],
+ json.dumps(element["polygon"]),
+ self.worker_version_id,
+ )
+ for idx, element in enumerate(elements)
+ ]
+ cache.insert("elements", to_insert)
+
return created_ids
def create_transcription(self, element, text, type=None, score=None):
--
GitLab