Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • arkindex/workers/thumbnails
1 result
Show changes
Commits on Source (11)
...@@ -4,9 +4,9 @@ version: 2 ...@@ -4,9 +4,9 @@ version: 2
workers: workers:
- slug: thumbnails-generator - slug: thumbnails-generator
name: Thumbnails Generator name: Thumbnails Generator
type: generator type: Utils
docker: docker:
build: Dockerfile command: worker-thumbnails-generator
configuration: configuration:
width: 900 width: 900
height: 400 height: 400
......
...@@ -14,7 +14,7 @@ variables: ...@@ -14,7 +14,7 @@ variables:
DEBIAN_FRONTEND: non-interactive DEBIAN_FRONTEND: non-interactive
test: test:
image: python:slim image: python:3.12-slim
stage: test stage: test
cache: cache:
...@@ -41,7 +41,7 @@ test: ...@@ -41,7 +41,7 @@ test:
- tox -- --junitxml=test-report.xml --durations=50 - tox -- --junitxml=test-report.xml --durations=50
lint: lint:
image: python:slim image: python:3.12-slim
cache: cache:
paths: paths:
...@@ -134,3 +134,4 @@ publish-worker: ...@@ -134,3 +134,4 @@ publish-worker:
- demo - demo
# Publish worker on https://arkindex.teklia.com # Publish worker on https://arkindex.teklia.com
- prod - prod
- elephant
repos: repos:
- repo: https://github.com/astral-sh/ruff-pre-commit - repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version. # Ruff version.
rev: v0.3.1 rev: v0.8.2
hooks: hooks:
# Run the linter. # Run the linter.
- id: ruff - id: ruff
...@@ -9,7 +9,7 @@ repos: ...@@ -9,7 +9,7 @@ repos:
# Run the formatter. # Run the formatter.
- id: ruff-format - id: ruff-format
- repo: https://github.com/pre-commit/pre-commit-hooks - repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0 rev: v5.0.0
hooks: hooks:
- id: check-ast - id: check-ast
- id: check-docstring-first - id: check-docstring-first
...@@ -27,10 +27,16 @@ repos: ...@@ -27,10 +27,16 @@ repos:
- id: check-toml - id: check-toml
- id: requirements-txt-fixer - id: requirements-txt-fixer
- repo: https://github.com/codespell-project/codespell - repo: https://github.com/codespell-project/codespell
rev: v2.2.6 rev: v2.3.0
hooks: hooks:
- id: codespell - id: codespell
args: ['--write-changes'] args: ['--write-changes']
- repo: meta - repo: meta
hooks: hooks:
- id: check-useless-excludes - id: check-useless-excludes
- repo: https://gitlab.teklia.com/tools/pre-commit-hooks
rev: 0.1.0
hooks:
- id: long-test-files
args: ['1000']
files: '^tests\/(.*\/)?test_[^\/]*\.py$'
\ No newline at end of file
...@@ -11,8 +11,4 @@ COPY worker_thumbnails_generator worker_thumbnails_generator ...@@ -11,8 +11,4 @@ COPY worker_thumbnails_generator worker_thumbnails_generator
COPY requirements.txt setup.py pyproject.toml ./ COPY requirements.txt setup.py pyproject.toml ./
RUN pip install . --no-cache-dir RUN pip install . --no-cache-dir
# Add archi local CA
RUN curl https://assets.teklia.com/teklia_dev_ca.pem > /usr/local/share/ca-certificates/arkindex-dev.crt && update-ca-certificates
ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt
CMD ["worker-thumbnails-generator"] CMD ["worker-thumbnails-generator"]
This diff is collapsed.
...@@ -4,6 +4,7 @@ build-backend = "setuptools.build_meta" ...@@ -4,6 +4,7 @@ build-backend = "setuptools.build_meta"
[project] [project]
name = "worker_thumbnails_generator" name = "worker_thumbnails_generator"
license = { file = "LICENSE" }
version = "0.1.0" version = "0.1.0"
description = "Worker to generate thumbnails for Arkindex elements" description = "Worker to generate thumbnails for Arkindex elements"
dynamic = ["dependencies"] dynamic = ["dependencies"]
......
...@@ -45,7 +45,7 @@ def _setup_environment(responses, monkeypatch) -> None: ...@@ -45,7 +45,7 @@ def _setup_environment(responses, monkeypatch) -> None:
monkeypatch.setattr(BaseWorker, "setup_api_client", mock_setup_api_client) monkeypatch.setattr(BaseWorker, "setup_api_client", mock_setup_api_client)
@pytest.fixture() @pytest.fixture
def mock_worker() -> ThumbnailsGenerator: def mock_worker() -> ThumbnailsGenerator:
worker = ThumbnailsGenerator() worker = ThumbnailsGenerator()
worker.args = Namespace(dev=False) worker.args = Namespace(dev=False)
...@@ -63,7 +63,7 @@ def mock_worker() -> ThumbnailsGenerator: ...@@ -63,7 +63,7 @@ def mock_worker() -> ThumbnailsGenerator:
return worker return worker
@pytest.fixture() @pytest.fixture
def page_1_payload() -> dict: def page_1_payload() -> dict:
return { return {
"id": "page_1", "id": "page_1",
...@@ -89,7 +89,7 @@ def page_1_payload() -> dict: ...@@ -89,7 +89,7 @@ def page_1_payload() -> dict:
} }
@pytest.fixture() @pytest.fixture
def page_2_payload() -> dict: def page_2_payload() -> dict:
return { return {
"id": "page_2", "id": "page_2",
...@@ -115,7 +115,7 @@ def page_2_payload() -> dict: ...@@ -115,7 +115,7 @@ def page_2_payload() -> dict:
} }
@pytest.fixture() @pytest.fixture
def page_3_payload() -> dict: def page_3_payload() -> dict:
return { return {
"id": "page_3", "id": "page_3",
...@@ -141,7 +141,7 @@ def page_3_payload() -> dict: ...@@ -141,7 +141,7 @@ def page_3_payload() -> dict:
} }
@pytest.fixture() @pytest.fixture
def pages_payload( def pages_payload(
page_1_payload: dict, page_2_payload: dict, page_3_payload: dict page_1_payload: dict, page_2_payload: dict, page_3_payload: dict
) -> list[dict]: ) -> list[dict]:
......
...@@ -175,6 +175,13 @@ def test_process_element_no_images_found(caplog, mock_worker): ...@@ -175,6 +175,13 @@ def test_process_element_no_images_found(caplog, mock_worker):
folder=False, folder=False,
response=[], response=[],
) )
mock_worker.api_client.add_response(
"ListElementChildren",
id=folder.id,
folder=False,
recursive=True,
response=[],
)
mock_worker.process_element(folder) mock_worker.process_element(folder)
......
...@@ -14,32 +14,23 @@ class ThumbnailsGenerator(ElementsWorker): ...@@ -14,32 +14,23 @@ class ThumbnailsGenerator(ElementsWorker):
def configure(self) -> None: def configure(self) -> None:
super().configure() super().configure()
if self.user_configuration:
logger.info("Loading user_configuration")
self.config.update(self.user_configuration)
self.thumbnail_width: int = self.config["width"] self.thumbnail_width: int = self.config["width"]
self.thumbnail_height: int = self.config["height"] self.thumbnail_height: int = self.config["height"]
self.first_n: int = self.config.get("first_n") self.first_n: int = self.config.get("first_n")
corpus = self.request("RetrieveCorpus", id=self.corpus_id) self.list_corpus_types()
self.corpus_types = {
element_type["slug"]: element_type for element_type in corpus["types"]
}
def _list_folder_elements(self, folder): def _list_folder_elements(self, folder):
""" """
List all elements in a folder that could have images and could be used as thumbnails. List all elements in a folder that could have images and could be used as thumbnails.
""" """
# First try with the first page of results returned by a non-recursive list, much faster than recursive # First try with a non-recursive list, much faster than recursive
results = self.list_element_children(folder, folder=False) yield from self.list_element_children(folder, folder=False)
yield from results
# Fallback to going through all of the non-folder elements recursively. # Fallback to going through all of the non-folder elements recursively.
# If there were no results at all, we do not even try to make the other query, # This request may return some elements even though the previous one did not,
# since we already know there will be no results on this one too. # since this folder might only contain folder elements that have non-folder elements.
if results: yield from self.list_element_children(folder, folder=False, recursive=True)
yield from self.list_element_children(folder, folder=False, recursive=True)
def get_first_images(self, folder): def get_first_images(self, folder):
""" """
...@@ -129,7 +120,7 @@ class ThumbnailsGenerator(ElementsWorker): ...@@ -129,7 +120,7 @@ class ThumbnailsGenerator(ElementsWorker):
def main() -> None: def main() -> None:
ThumbnailsGenerator( ThumbnailsGenerator(
description="Worker to generate thumbnails for Arkindex elements" description="Worker to generate thumbnails for Arkindex elements: Utility process"
).run() ).run()
......