add ruff and add all ignore to make the CI pass for now

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     rev: 'v0.0.257'
     hooks:
       - id: ruff
-        args: [--exit-non-zero-on-fix]
+        args: [--exit-non-zero-on-fix, "--fix"]
         exclude: "^worker-{{cookiecutter.slug}}/"
   - repo: https://github.com/ambv/black
     rev: 23.1.0

.ruff.toml

+# See https://github.com/charliermarsh/ruff#rules for error code definitions.
 select = [
     # Pyflakes
     "F",
@@ -8,23 +9,41 @@ select = [
     "I001",
     # flake8-debugger
     "T10",
+    # flake8-use-pathlib
+    "PTH",
+    # flake8-comprehensions
+    "C4",
+    # logging format
+    "G",
+    # Pep8-naming
+    "N",
+    # flake8-pie
+    "PIE",
+    # Pylint
+    "PL",
+    # Ruff-specific
+    "RUF",
+    # flake8-bandit
+    "S",
+    # flake8-simplify
+    "SIM",
 ]
 
-# Never enforce `E501` (line length violations).
-ignore = ["E501"]
-
-# Do not lint __pycache__ folders
-exclude = ["__pycache__"]
-
-# Autofix issues
-fix = true
+# Never enforce
+ignore = [
+    "E501", # line-too-long
+    "RUF001", # unicode
+    "RUF002", # unicode
+    "PLR0915", # Too many statements (functions too big)
+    "PLR0912", # Too many branches (functions too complex)
+    "PLR0913", # Toom any arguments to function call
+    "S101", # assert checks forbidden
+    "PLR2004", # Magic value used in comparison, consider replacing 3 with a constant variable
+]
 
 # By default, always enumerate fixed violations.
 show-fixes = true
 
-# Group violations by containing file.
-format = "gitlab"
-
 # By default, always show source code snippets.
 show-source = true
 

arkindex_worker/.ruff.toml

+# Extend the `.ruff.toml` file in the parent directory.
+extend = "../.ruff.toml"
+
+ignore = [
+    "G004", # Logging statement uses f-string
+]
+
+[per-file-ignores]
+"worker/training.py" = [
+    "S113", # Probable use of requests call without timeout
+]
+"worker/entity.py" = [
+    "N818", # Exception name `MissingEntityType` should be named with an Error suffix
+]
+"cache.py" = [
+    "SIM117", # Use a single `with` statement with multiple contexts instead of nested `with` statements
+    "PLR5501", # Consider using `elif` instead of `else` then `if` to remove one indentation level
+    "SIM114", # Combine `if` branches using logical `or` operator
+    "S608", # Possible SQL injection vector through string-based query construction
+]
+"worker/base.py" = [
+    "SIM115", # Use context handler for opening files
+    "PTH123", # `open()` should be replaced by `Path.open()`
+]
+"utils.py" = [
+    "S324", # Probable use of insecure hash functions in `hashlib`: `md5`
+    "PTH123", # `open()` should be replaced by `Path.open()`
+]
+"reporting.py" = [
+    "PTH123", # `open()` should be replaced by `Path.open()`
+]
+"image.py" = [
+    "G001", # Logging statement uses `string.format()`
+]
+"models.py" = [
+    "G003", # Logging statement uses `+`
+]
+"__init__.py" = [
+    "G001", # Logging statement uses `string.format()`
+]

arkindex_worker/cache.py

@@ -7,7 +7,6 @@ and writes will go both to the Arkindex API and the database,
 reducing network usage.
 """
 import json
-import sqlite3
 from pathlib import Path
 import sqlite3
 from typing import Optional, Union
@@ -323,9 +322,8 @@ def merge_parents_cache(paths: list, current_database: Path):
         # Check that the parent cache uses a compatible version
         check_version(path)
 
-        with SqliteDatabase(path) as source:
-            with source.bind_ctx(MODELS):
-                source.create_tables(MODELS)
+        with SqliteDatabase(path) as source, source.bind_ctx(MODELS):
+            source.create_tables(MODELS)
 
         logger.info(f"Merging parent db {path} into {current_database}")
         statements = [

arkindex_worker/git.py

@@ -193,13 +193,11 @@ def prepare_git_key(
     private_key_path = Path(private_key_path).expanduser()
     known_hosts_path = Path(known_hosts_path).expanduser()
 
-    if private_key_path.exists():
-        if private_key_path.read_text() != private_key:
-            make_backup(private_key_path)
+    if private_key_path.exists() and private_key_path.read_text() != private_key:
+        make_backup(private_key_path)
 
-    if known_hosts_path.exists():
-        if known_hosts_path.read_text() != known_hosts:
-            make_backup(known_hosts_path)
+    if known_hosts_path.exists() and known_hosts_path.read_text() != known_hosts:
+        make_backup(known_hosts_path)
 
     private_key_path.write_text(private_key)
     # private key must be private, otherwise git will fail

arkindex_worker/image.py

@@ -5,7 +5,6 @@ Helper methods to download and open IIIF images, and manage polygons.
 from collections import namedtuple
 from io import BytesIO
 from math import ceil
-import os
 from pathlib import Path
 from typing import TYPE_CHECKING, List, Optional, Union
 
@@ -55,11 +54,7 @@ def open_image(
     :param mirrored: Whether or not to mirror the image horizontally.
     :returns: A Pillow image.
     """
-    if (
-        path.startswith("http://")
-        or path.startswith("https://")
-        or not Path(path).exists()
-    ):
+    if path.startswith(("http://", "https://")) or not Path(path).exists():
         image = download_image(path)
     else:
         try:

arkindex_worker/models.py

@@ -201,10 +201,7 @@ class Element(MagicDict):
         else:
             resize = "full"
 
-        if use_full_image:
-            url = self.image_url(resize)
-        else:
-            url = self.resize_zone_url(resize)
+        url = self.image_url(resize) if use_full_image else self.resize_zone_url(resize)
 
         try:
             return open_image(

arkindex_worker/utils.py

@@ -2,9 +2,9 @@
 import hashlib
 import logging
 import os
+from pathlib import Path
 import tarfile
 import tempfile
-from pathlib import Path
 from typing import Optional, Tuple, Union
 
 import zstandard

arkindex_worker/worker/__init__.py

@@ -3,6 +3,7 @@
 Base classes to implement Arkindex workers.
 """
 
+import contextlib
 from enum import Enum
 import json
 import os
@@ -228,10 +229,9 @@ class ElementsWorker(
                 )
                 if element:
                     # Try to update the activity to error state regardless of the response
-                    try:
+                    with contextlib.suppress(Exception):
                         self.update_activity(element.id, ActivityState.Error)
-                    except Exception:
-                        pass
+
                 self.report.error(element_id, e)
 
         # Save report as local artifact

arkindex_worker/worker/entity.py

@@ -71,7 +71,7 @@ class EntityMixin(object):
         element: Union[Element, CachedElement],
         name: str,
         type: str,
-        metas=dict(),
+        metas={},
         validated=None,
     ):
         """

hooks/pre-commit

@@ -3,6 +3,7 @@
 # File generated by pre-commit: https://pre-commit.com
 # ID: 138fd403232d2ddd5efb44317e38bf03
 import os
+from pathlib import Path
 import sys
 
 # we try our best, but the shebang of this script is difficult to determine:
@@ -26,7 +27,7 @@ ARGS = [
     "--skip-on-missing-config",
 ]
 # end templated
-ARGS.extend(("--hook-dir", os.path.realpath(os.path.dirname(__file__))))
+ARGS.extend(("--hook-dir", os.path.realpath(Path(__file__).parent)))
 ARGS.append("--")
 ARGS.extend(sys.argv[1:])
 

setup.py

@@ -11,7 +11,7 @@ def requirements(path: Path):
         return list(map(str.strip, f.read().splitlines()))
 
 
-with open("VERSION") as f:
+with Path("VERSION").open() as f:
     VERSION = f.read()
 
 setup(

tests/.ruff.toml

+# Extend the `.ruff.toml` file in the parent directory.
+extend = "../.ruff.toml"
+
+ignore = [
+    "RUF005",
+]
+
+[per-file-ignores]
+"test_image.py" = [
+    "S108", # Probable insecure usage of temporary file or directory
+]
+"conftest.py" = [
+    "S324", # use of insecure hash functions in hashlib
+    "PLW2901", # `for` loop variable `path` overwritten by assignment target
+    "S108", # Probable insecure usage of temporary file or directory
+    "PTH123", # `open()` should be replaced by `Path.open()`
+]
+"test_utils.py" = [
+    "C414", # Unnecessary `list` call within `sorted()`
+]
+"test_elements_worker/test_cli.py" = [
+    "PTH108", # `os.unlink()` should be replaced by `Path.unlink()`
+    "PTH123", # `open()` should be replaced by `Path.open()`
+]
+"test_cache.py" = [
+    "PLR0913", # Too many arguments to function call (6 > 5)
+    "PTH123", # `open()` should be replaced by `Path.open()`
+]
+"test_elements_worker/test_worker.py" = [
+    "PLR0913", # Too many arguments to function call (6 > 5)
+]
+"test_git.py" = [
+    "S108", # Probable insecure usage of temporary file or directory
+]

tests/test_base_worker.py

 # -*- coding: utf-8 -*-
 import json
 import logging
-import sys
 from pathlib import Path
 import sys
 

worker-{{cookiecutter.slug}}/.isort.cfg

-[settings]
-# Compatible with black
-profile = black
-
-default_section=FIRSTPARTY
-known_first_party = arkindex,arkindex_worker
-known_third_party = pytest,setuptools

worker-{{cookiecutter.slug}}/.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     rev: 'v0.0.257'
     hooks:
       - id: ruff
-        args: [--fix, --exit-non-zero-on-fix]
+        args: [--exit-non-zero-on-fix, "--fix"]
   - repo: https://github.com/ambv/black
     rev: 23.1.0
     hooks:

worker-{{cookiecutter.slug}}/.ruff.toml

+# See https://github.com/charliermarsh/ruff#rules for error code definitions.
 select = [
     # Pyflakes
     "F",
@@ -8,27 +9,44 @@ select = [
     "I001",
     # flake8-debugger
     "T10",
+    # flake8-use-pathlib
+    "PTH",
+    # flake8-comprehensions
+    "C4",
+    # logging format
+    "G",
+    # Pep8-naming
+    "N",
+    # flake8-pie
+    "PIE",
+    # Pylint
+    "PL",
+    # Ruff-specific
+    "RUF",
+    # flake8-bandit
+    "S",
+    # flake8-simplify
+    "SIM",
 ]
 
-# Never enforce `E501` (line length violations).
-ignore = ["E501"]
-
-# Do not lint __pycache__ folders
-exclude = ["__pycache__"]
-
-# Autofix issues
-fix = true
+# Never enforce
+ignore = [
+    "E501", # line-too-long
+    "PLR0915", # Too many statements (functions too big)
+    "PLR0912", # Too many branches (functions too complex)
+    "PLR0913", # Toom any arguments to function call
+    "S101", # assert checks forbidden
+    "PLR2004", # Magic value used in comparison, consider replacing ... with a constant variable
+    "N999", # Invalid module name
+]
 
 # By default, always enumerate fixed violations.
 show-fixes = true
 
-# Group violations by containing file.
-format = "gitlab"
-
 # By default, always show source code snippets.
 show-source = true
 
 [isort]
-known-first-party = ["arkindex","arkindex_common","arkindex_worker"]
-known-third-party = ["PIL","apistar","gitlab","gnupg","peewee","playhouse","pytest","requests","responses","setuptools","sh","shapely","tenacity","yaml","zstandard"]
+known-first-party = ["arkindex","arkindex_worker"]
+known-third-party = ["pytest","setuptools"]
 force-sort-within-sections = true

worker-{{cookiecutter.slug}}/setup.py

@@ -30,9 +30,12 @@ def parse_requirements():
     )
 
 
+with Path("VERSION").open() as file:
+    VERSION = file.read()
+
 setup(
     name=MODULE,
-    version=open("VERSION").read(),
+    version=VERSION,
     description="{{ cookiecutter.description }}",
     author="{{ cookiecutter.author }}",
     author_email="{{ cookiecutter.email }}",

worker-{{cookiecutter.slug}}/worker_{{cookiecutter.slug}}/worker.py

@@ -8,7 +8,9 @@ logger = logging.getLogger(__name__)
 
 class Demo(ElementsWorker):
     def process_element(self, element):
-        logger.info(f"Demo processing element ({element.id})")
+        logger.info(
+            "Demo processing element ({element_id})", extra={"element_id": element.id}
+        )
 
 
 def main():