.flake8

-[flake8]
-max-line-length = 88
-exclude = .git,__pycache__
-ignore = E203,E501,W503

.isort.cfg

-[settings]
-# Compatible with black
-profile = black
-
-default_section=FIRSTPARTY
-known_first_party = arkindex,arkindex_common,arkindex_worker
-known_third_party =PIL,apistar,gitlab,gnupg,peewee,playhouse,pytest,requests,responses,setuptools,sh,shapely,tenacity,yaml,zstandard

.pre-commit-config.yaml

 repos:
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.0.278
     hooks:
-      - id: isort
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+        exclude: "^worker-{{cookiecutter.slug}}/"
   - repo: https://github.com/ambv/black
     rev: 23.1.0
     hooks:
     - id: black
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
-        additional_dependencies:
-          - 'flake8-coding==1.3.2'
-          - 'flake8-debugger==4.1.2'
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:

docs-requirements.txt

-black==23.3.0
+black==23.7.0
 doc8==1.1.1
 mkdocs==1.4.3
 mkdocs-material==9.1.14

docs/releases.md

 # Releases
 
+
+
+## 0.3.3
+
+Released on **26 May 2023** • View on [Gitlab](https://gitlab.com/teklia/workers/base-worker/-/releases/0.3.3)
+
+- The `Timer` class previously defined in `arkindex_worker.utils` was removed as it was already defined [Teklia's python toolbox](https://pypi.org/project/teklia-toolbox/).
+```python
+# Old usage
+from arkindex_worker.utils import Timer
+# New usage
+from teklia_toolbox.time import Timer
+```
+- The [create_element_transcriptions](../ref/api/transcription/#arkindex_worker.worker.transcription.TranscriptionMixin.create_element_transcriptions) API helper now accepts an `element_confidence` float field in the dictionaries provided through the `transcriptions` field. This confidence will be set on the created element.
+- More query filters are available on the [list_element_children](../ref/api/element/#arkindex_worker.worker.element.ElementMixin.list_element_children) API helper. More details about their usage is available in the documentation:
+    - `transcription_worker_version`
+    - `transcription_worker_run`
+    - `with_metadata`
+    - `worker_run`
+- `Arkindex Base-Worker` now fully uses [pathlib](https://docs.python.org/3/library/pathlib.html) to handle filesystem paths as suggested by [PEP 428](https://peps.python.org/pep-0428/).
+- Many helpers were added to handle [ZSTD](https://en.wikipedia.org/wiki/Zstd) and [TAR](https://en.wikipedia.org/wiki/Tar_(computing)) archives as well as delete files cleanly. More details about that in the documentation of the [arkindex_worker.utils](../ref/utils/) module.
+- A bug affecting the parsing of the configuration of workers that use a Machine learning model stored on an Arkindex instance was fixed.
+
+
 ## 0.3.2
 
 Released on **8 March 2023** • View on [Gitlab](https://gitlab.com/teklia/workers/base-worker/-/releases/0.3.2)
@@ -10,13 +34,13 @@ Released on **8 March 2023** • View on [Gitlab](https://gitlab.com/teklia/w
 - The model version API endpoints were updated in the [latest Arkindex release](https://teklia.com/solutions/arkindex/releases/1-4-0/) and a new helper was introduced subsequently. However, there are no breaking changes and the main helper, `publish_model_version`, still has the same signature and behaviour.
 - The latest Arkindex release changed the way NER entities are stored and published.
     - The `EntityType` enum was removed as type slug are no longer restrcited to a small options,
-    - [create_entity](https://workers.arkindex.org/ref/api/entity/#arkindex_worker.worker.entity.EntityMixin.create_entity) now expects a type slug as a String,
-    - a new helper [list_corpus_entity_types](https://workers.arkindex.org/ref/api/entity/#arkindex_worker.worker.entity.EntityMixin.list_corpus_entity_types) was added to load the Entity types in the corpus,
-    - a new helper [check_required_entity_types](https://workers.arkindex.org/ref/api/entity/#arkindex_worker.worker.entity.EntityMixin.check_required_entity_types) to make sure that needed entity types are available in the corpus was added. Missing ones are created by default (this can be disabled).
-- The [create_classifications](https://workers.arkindex.org/ref/api/classification/#arkindex_worker.worker.classification.ClassificationMixin.create_classifications) helper now expects the UUID of each MLClass instead of their name.
+    - [create_entity](../ref/api/entity/#arkindex_worker.worker.entity.EntityMixin.create_entity) now expects a type slug as a String,
+    - a new helper [list_corpus_entity_types](../ref/api/entity/#arkindex_worker.worker.entity.EntityMixin.list_corpus_entity_types) was added to load the Entity types in the corpus,
+    - a new helper [check_required_entity_types](../ref/api/entity/#arkindex_worker.worker.entity.EntityMixin.check_required_entity_types) to make sure that needed entity types are available in the corpus was added. Missing ones are created by default (this can be disabled).
+- The [create_classifications](../ref/api/classification/#arkindex_worker.worker.classification.ClassificationMixin.create_classifications) helper now expects the UUID of each MLClass instead of their name.
 - In developer mode, the only way to set the `corpus_id` attribute is to use the `ARKINDEX_CORPUS_ID` environment variable. When it's not set, all API requests using the `corpus_id` as path parameter will fail with `500` status code. A warning log was added to help developers troubleshoot this error by advising them to set this variable.
-- The [create_transcriptions](https://workers.arkindex.org/ref/api/transcription/#arkindex_worker.worker.transcription.TranscriptionMixin.create_transcriptions) helper no longer makes the API call in developer mode. This behaviour aligns with all other publication helpers.
-- Fixes hash computation when publishing a model using [publish_model_version](https://workers.arkindex.org/ref/api/training/#arkindex_worker.worker.training.TrainingMixin.publish_model_version).
+- The [create_transcriptions](../ref/api/transcription/#arkindex_worker.worker.transcription.TranscriptionMixin.create_transcriptions) helper no longer makes the API call in developer mode. This behaviour aligns with all other publication helpers.
+- Fixes hash computation when publishing a model using [publish_model_version](../ref/api/training/#arkindex_worker.worker.training.TrainingMixin.publish_model_version).
 - If a process is linked to a model version, its id will be available to the worker through its `model_version_id` attribute.
 - The URLs of the API endpoint related to Ponos were changed in the latest Arkindex release. Some changes were needed in the test suite.
 - The `classes` attribute no directly contains the classes of the corpus of the processed element.

pyproject.toml

+[tool.ruff]
+exclude = [".git", "__pycache__"]
+ignore = ["E501"]
+select = ["E", "F", "T1", "W", "I"]
+
+[tool.ruff.isort]
+known-first-party = ["arkindex", "arkindex_common", "arkindex_worker"]
+known-third-party = [
+    "PIL",
+    "apistar",
+    "gitlab",
+    "gnupg",
+    "peewee",
+    "playhouse",
+    "pytest",
+    "requests",
+    "responses",
+    "setuptools",
+    "sh",
+    "shapely",
+    "tenacity",
+    "yaml",
+    "zstandard",
+]

tests/test_elements_worker/test_transcriptions.py

@@ -1978,7 +1978,9 @@ def test_list_transcriptions_with_cache(
     # Query database through cache
     transcriptions = mock_elements_worker_with_cache.list_transcriptions(**filters)
     assert transcriptions.count() == len(expected_ids)
-    for transcription, expected_id in zip(transcriptions.order_by("id"), expected_ids):
+    for transcription, expected_id in zip(
+        transcriptions.order_by(CachedTranscription.id), expected_ids
+    ):
         assert transcription.id == UUID(expected_id)
 
     # Check the worker never hits the API for elements

worker-{{cookiecutter.slug}}/.flake8

-[flake8]
-max-line-length = 150
-exclude = .git,__pycache__
-ignore = E203,E501,W503

worker-{{cookiecutter.slug}}/.isort.cfg

-[settings]
-# Compatible with black
-profile = black
-
-default_section=FIRSTPARTY
-known_first_party = arkindex,arkindex_worker
-known_third_party = pytest,setuptools

worker-{{cookiecutter.slug}}/.pre-commit-config.yaml

 repos:
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.0.278
     hooks:
-      - id: isort
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
   - repo: https://github.com/ambv/black
     rev: 23.1.0
     hooks:
     - id: black
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
-        additional_dependencies:
-          - 'flake8-coding==1.3.2'
-          - 'flake8-debugger==4.1.2'
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:

worker-{{cookiecutter.slug}}/pyproject.toml

+[tool.ruff]
+exclude = [".git", "__pycache__"]
+ignore = ["E501"]
+select = ["E", "F", "T1", "W", "I"]
+
+[tool.ruff.isort]
+known-first-party = ["arkindex", "arkindex_worker"]
+known-third-party = ["pytest", "setuptools"]

worker-{{cookiecutter.slug}}/requirements.txt

-arkindex-base-worker==0.3.2
+arkindex-base-worker==0.3.3

worker-{{cookiecutter.slug}}/tests/conftest.py

@@ -25,4 +25,7 @@ def setup_environment(responses, monkeypatch) -> None:
     os.environ["ARKINDEX_WORKER_RUN_ID"] = "1234-{{ cookiecutter.slug }}"
 
     # Setup a mock api client instead of using a real one
-    monkeypatch.setattr(BaseWorker, "setup_api_client", lambda _: MockApiClient())
+    def mock_setup_api_client(self):
+        self.api_client = MockApiClient()
+
+    monkeypatch.setattr(BaseWorker, "setup_api_client", mock_setup_api_client)