diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 788e58561644176c02e75eadf66d728bba92aa9b..0000000000000000000000000000000000000000 --- a/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -max-line-length = 88 -exclude = .git,__pycache__ -ignore = E203,E501,W503 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6582747024585df8b13de884b8b3f4f723dd1bfe..de7fac962db0272729f3f7f66ec55c7b18f294b5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,19 +1,14 @@ repos: - - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: 'v0.0.257' hooks: - - id: isort + - id: ruff + args: [--exit-non-zero-on-fix] + exclude: "^worker-{{cookiecutter.slug}}/" - repo: https://github.com/ambv/black rev: 23.1.0 hooks: - id: black - - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - additional_dependencies: - - 'flake8-coding==1.3.2' - - 'flake8-debugger==4.1.2' - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 0000000000000000000000000000000000000000..fa6e473e40a03f71528de271c37af6b41f2c9503 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,34 @@ +select = [ + # Pyflakes + "F", + # Pycodestyle + "E", + "W", + # isort + "I001", + # flake8-debugger + "T10", +] + +# Never enforce `E501` (line length violations). +ignore = ["E501"] + +# Do not lint __pycache__ folders +exclude = ["__pycache__"] + +# Autofix issues +fix = true + +# By default, always enumerate fixed violations. +show-fixes = true + +# Group violations by containing file. +format = "gitlab" + +# By default, always show source code snippets. +show-source = true + +[isort] +known-first-party = ["arkindex","arkindex_common","arkindex_worker"] +known-third-party = ["PIL","apistar","gitlab","gnupg","peewee","playhouse","pytest","requests","responses","setuptools","sh","shapely","tenacity","yaml","zstandard"] +force-sort-within-sections = true diff --git a/arkindex_worker/cache.py b/arkindex_worker/cache.py index 428713500fae0cfea1c548c3895cc5d8edec9659..c748ea6de55417e3bad8c11fa5d77237a5b7b0df 100644 --- a/arkindex_worker/cache.py +++ b/arkindex_worker/cache.py @@ -6,10 +6,10 @@ On methods that support caching, the database will be used for all reads, and writes will go both to the Arkindex API and the database, reducing network usage. """ - import json import sqlite3 from pathlib import Path +import sqlite3 from typing import Optional, Union from peewee import ( diff --git a/arkindex_worker/git.py b/arkindex_worker/git.py index f30b781898cd06b360ca9544126fd17056aae8c3..140627af3311eedf6d4f5311644aa91ee1e88687 100644 --- a/arkindex_worker/git.py +++ b/arkindex_worker/git.py @@ -2,16 +2,16 @@ """ Helper classes for workers that interact with Git repositories and the GitLab API. """ -import shutil -import time from datetime import datetime from pathlib import Path +import shutil +import time from typing import Optional, Union import gitlab +from gitlab.v4.objects import MergeRequest, ProjectMergeRequest import requests import sh -from gitlab.v4.objects import MergeRequest, ProjectMergeRequest from arkindex_worker import logger diff --git a/arkindex_worker/image.py b/arkindex_worker/image.py index c01f17fd469276e1b38c4354a9edc03dd39a4f15..f1db64470b820f4993583267def2ab77e18dfb42 100644 --- a/arkindex_worker/image.py +++ b/arkindex_worker/image.py @@ -5,11 +5,12 @@ Helper methods to download and open IIIF images, and manage polygons. from collections import namedtuple from io import BytesIO from math import ceil +import os from pathlib import Path from typing import TYPE_CHECKING, List, Optional, Union -import requests from PIL import Image +import requests from shapely.affinity import rotate, scale, translate from shapely.geometry import LinearRing from tenacity import ( diff --git a/arkindex_worker/models.py b/arkindex_worker/models.py index 924610b693e33c4dfe9abc24437247e67801175c..4a353f4be952d4be3f119909334c2bdcab2d90ce 100644 --- a/arkindex_worker/models.py +++ b/arkindex_worker/models.py @@ -3,8 +3,8 @@ Wrappers around API results to provide more convenient attribute access and IIIF helpers. """ -import tempfile from contextlib import contextmanager +import tempfile from typing import Generator, List, Optional from PIL import Image diff --git a/arkindex_worker/reporting.py b/arkindex_worker/reporting.py index b9ee03ef465795cefc377fdaf9f300f3f3d252b1..4c34a340d89a70a0c08bab8de439fab94a423e99 100644 --- a/arkindex_worker/reporting.py +++ b/arkindex_worker/reporting.py @@ -3,11 +3,11 @@ Generator for the ``ml_report.json`` file, to report created worker results and exceptions. """ -import json -import traceback from collections import Counter from datetime import datetime +import json from pathlib import Path +import traceback from typing import Dict, List, Optional, Union from uuid import UUID diff --git a/arkindex_worker/worker/__init__.py b/arkindex_worker/worker/__init__.py index 2ea54f18fe3529a9972aefefa9ac0b16d30f8282..4b349f5b0ab6c6e24ed66c8599b2e2e815a4ab18 100644 --- a/arkindex_worker/worker/__init__.py +++ b/arkindex_worker/worker/__init__.py @@ -3,12 +3,12 @@ Base classes to implement Arkindex workers. """ +from enum import Enum import json import os import sys -import uuid -from enum import Enum from typing import Iterable, List, Union +import uuid from apistar.exceptions import ErrorResponse @@ -19,10 +19,10 @@ from arkindex_worker.reporting import Reporter from arkindex_worker.worker.base import BaseWorker from arkindex_worker.worker.classification import ClassificationMixin from arkindex_worker.worker.element import ElementMixin -from arkindex_worker.worker.entity import EntityMixin # noqa: F401 -from arkindex_worker.worker.metadata import MetaDataMixin, MetaType # noqa: F401 +from arkindex_worker.worker.entity import EntityMixin +from arkindex_worker.worker.metadata import MetaDataMixin from arkindex_worker.worker.transcription import TranscriptionMixin -from arkindex_worker.worker.version import WorkerVersionMixin # noqa: F401 +from arkindex_worker.worker.version import WorkerVersionMixin class ActivityState(Enum): diff --git a/arkindex_worker/worker/base.py b/arkindex_worker/worker/base.py index 8d2e454b9ebba34dc9f9b7ad1aa6df7668b4f0ad..f1400e4f656192afff6fe5f0757a7517f31b17a4 100644 --- a/arkindex_worker/worker/base.py +++ b/arkindex_worker/worker/base.py @@ -10,9 +10,8 @@ import os from pathlib import Path from typing import List, Optional -import gnupg -import yaml from apistar.exceptions import ErrorResponse +import gnupg from tenacity import ( before_sleep_log, retry, @@ -20,6 +19,7 @@ from tenacity import ( stop_after_attempt, wait_exponential, ) +import yaml from arkindex import ArkindexClient, options_from_env from arkindex_worker import logger diff --git a/arkindex_worker/worker/training.py b/arkindex_worker/worker/training.py index fc51760772ce9d13fa0f89856155febf069aa4a7..c4b8dea8f46dcae4229d9d262cfc870c6840c8af 100644 --- a/arkindex_worker/worker/training.py +++ b/arkindex_worker/worker/training.py @@ -3,14 +3,14 @@ BaseWorker methods for training. """ -import functools from contextlib import contextmanager +import functools from pathlib import Path from typing import NewType, Optional, Tuple, Union from uuid import UUID -import requests from apistar.exceptions import ErrorResponse +import requests from arkindex_worker import logger from arkindex_worker.utils import close_delete_file, create_tar_zst_archive diff --git a/tests/conftest.py b/tests/conftest.py index c11d0bdbf58e25eb28041e2e05de00cff79a6221..ac4494c5e85cbd81fedc7b994c58fbdb3c230715 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,14 +2,14 @@ import hashlib import json import os +from pathlib import Path import sys import time -from pathlib import Path from uuid import UUID +from peewee import SqliteDatabase import pytest import yaml -from peewee import SqliteDatabase from arkindex.mock import MockApiClient from arkindex_worker.cache import ( diff --git a/tests/test_base_worker.py b/tests/test_base_worker.py index b975313ec498dbc60b76e305cf3fe1279617f569..ce8db4cfa33f279a0fe916e3b3067b92aeb16e64 100644 --- a/tests/test_base_worker.py +++ b/tests/test_base_worker.py @@ -3,6 +3,7 @@ import json import logging import sys from pathlib import Path +import sys import gnupg import pytest diff --git a/tests/test_cache.py b/tests/test_cache.py index 77c9ab4c2a782fc4730c6c3353abf604d3ec419d..f85c36296bbeb886da12d3c846206b2f5ba69d96 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -2,8 +2,8 @@ from pathlib import Path from uuid import UUID -import pytest from peewee import OperationalError +import pytest from arkindex_worker.cache import ( SQL_VERSION, diff --git a/tests/test_elements_worker/test_classifications.py b/tests/test_elements_worker/test_classifications.py index 23db74264b268accc1d120ef0e33673ef59f3642..ca2d5ffadbad34298ecce9ed9b53a3461b17e244 100644 --- a/tests/test_elements_worker/test_classifications.py +++ b/tests/test_elements_worker/test_classifications.py @@ -2,8 +2,8 @@ import json from uuid import UUID, uuid4 -import pytest from apistar.exceptions import ErrorResponse +import pytest from arkindex_worker.cache import CachedClassification, CachedElement from arkindex_worker.models import Element diff --git a/tests/test_elements_worker/test_elements.py b/tests/test_elements_worker/test_elements.py index fda0673753ba35e80432185b71699acd12ddf421..4b7e3c02bba46aedd4c3ab234757114e5a52122e 100644 --- a/tests/test_elements_worker/test_elements.py +++ b/tests/test_elements_worker/test_elements.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- -import json from argparse import Namespace +import json from uuid import UUID -import pytest from apistar.exceptions import ErrorResponse +import pytest from responses import matchers from arkindex_worker.cache import ( diff --git a/tests/test_elements_worker/test_entities.py b/tests/test_elements_worker/test_entities.py index b989f3b3eede5912db0523b01f911a7fbfe82d08..28bff7b3624e1ee618171aa8219a767f77a71056 100644 --- a/tests/test_elements_worker/test_entities.py +++ b/tests/test_elements_worker/test_entities.py @@ -2,8 +2,8 @@ import json from uuid import UUID -import pytest from apistar.exceptions import ErrorResponse +import pytest from responses import matchers from arkindex_worker.cache import ( diff --git a/tests/test_elements_worker/test_metadata.py b/tests/test_elements_worker/test_metadata.py index c18e3f60103255df3aee50f3ec21267af2b6c014..5507b30ddd55c98f0495323abb823c808460e5d4 100644 --- a/tests/test_elements_worker/test_metadata.py +++ b/tests/test_elements_worker/test_metadata.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- import json -import pytest from apistar.exceptions import ErrorResponse +import pytest from arkindex.mock import MockApiClient from arkindex_worker.cache import CachedElement diff --git a/tests/test_elements_worker/test_transcriptions.py b/tests/test_elements_worker/test_transcriptions.py index 70bebc9084220a29ad7cf13352654b2a7694a8da..d54beb9bec73176b489051e3defde8f189e50b5c 100644 --- a/tests/test_elements_worker/test_transcriptions.py +++ b/tests/test_elements_worker/test_transcriptions.py @@ -2,9 +2,9 @@ import json from uuid import UUID -import pytest from apistar.exceptions import ErrorResponse from playhouse.shortcuts import model_to_dict +import pytest from arkindex_worker.cache import CachedElement, CachedTranscription from arkindex_worker.models import Element diff --git a/tests/test_elements_worker/test_worker.py b/tests/test_elements_worker/test_worker.py index 8be5a573a53a33cc046b5ec6f8c105a4242e11b5..cac9c1617060ca94d4bb2793b4f9011cb53f07d5 100644 --- a/tests/test_elements_worker/test_worker.py +++ b/tests/test_elements_worker/test_worker.py @@ -2,8 +2,8 @@ import json import sys -import pytest from apistar.exceptions import ErrorResponse +import pytest from arkindex_worker.cache import CachedElement from arkindex_worker.worker import ActivityState, ElementsWorker diff --git a/tests/test_git.py b/tests/test_git.py index 0c78a456ab8a7f5367c182f89cbd8d8f3c763cdf..61716d93ec39a1f933cc71f607531f3f590afb03 100644 --- a/tests/test_git.py +++ b/tests/test_git.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- from pathlib import Path -import pytest from gitlab import GitlabCreateError, GitlabError +import pytest from requests import ConnectionError from responses import matchers diff --git a/tests/test_image.py b/tests/test_image.py index 4f1002cad53593ea28e4174f6f6eb0589543635b..a0c2ac9126285707fe8236e1949f57b37ebfb2dd 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- +from io import BytesIO import math +from pathlib import Path import unittest import uuid -from io import BytesIO -from pathlib import Path -import pytest from PIL import Image, ImageChops, ImageOps +import pytest from arkindex_worker.cache import CachedElement, create_tables, init_cache_db from arkindex_worker.image import ( diff --git a/tests/test_reporting.py b/tests/test_reporting.py index af214f011978d1df5c0a07b142ace8ab75493acf..352a98987c23f594deb7c7b4702b9afc461e2d18 100644 --- a/tests/test_reporting.py +++ b/tests/test_reporting.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- -import json -import uuid from datetime import datetime +import json from tempfile import NamedTemporaryFile +import uuid -import pytest from apistar.exceptions import ErrorResponse +import pytest from arkindex_worker.models import Transcription from arkindex_worker.reporting import Reporter diff --git a/worker-{{cookiecutter.slug}}/.flake8 b/worker-{{cookiecutter.slug}}/.flake8 deleted file mode 100644 index 7a3797fc6b71677df500d8386cf70e3173a7f79f..0000000000000000000000000000000000000000 --- a/worker-{{cookiecutter.slug}}/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -max-line-length = 150 -exclude = .git,__pycache__ -ignore = E203,E501,W503 diff --git a/worker-{{cookiecutter.slug}}/.pre-commit-config.yaml b/worker-{{cookiecutter.slug}}/.pre-commit-config.yaml index 93d07e6993e07300b856e6c8457e9074805baa49..d75ecd758b9e44070d0ef46c3b37d9919e3afbf3 100644 --- a/worker-{{cookiecutter.slug}}/.pre-commit-config.yaml +++ b/worker-{{cookiecutter.slug}}/.pre-commit-config.yaml @@ -1,19 +1,13 @@ repos: - - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: 'v0.0.257' hooks: - - id: isort + - id: ruff + args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/ambv/black rev: 23.1.0 hooks: - id: black - - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - additional_dependencies: - - 'flake8-coding==1.3.2' - - 'flake8-debugger==4.1.2' - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: diff --git a/worker-{{cookiecutter.slug}}/.ruff.toml b/worker-{{cookiecutter.slug}}/.ruff.toml new file mode 100644 index 0000000000000000000000000000000000000000..fa6e473e40a03f71528de271c37af6b41f2c9503 --- /dev/null +++ b/worker-{{cookiecutter.slug}}/.ruff.toml @@ -0,0 +1,34 @@ +select = [ + # Pyflakes + "F", + # Pycodestyle + "E", + "W", + # isort + "I001", + # flake8-debugger + "T10", +] + +# Never enforce `E501` (line length violations). +ignore = ["E501"] + +# Do not lint __pycache__ folders +exclude = ["__pycache__"] + +# Autofix issues +fix = true + +# By default, always enumerate fixed violations. +show-fixes = true + +# Group violations by containing file. +format = "gitlab" + +# By default, always show source code snippets. +show-source = true + +[isort] +known-first-party = ["arkindex","arkindex_common","arkindex_worker"] +known-third-party = ["PIL","apistar","gitlab","gnupg","peewee","playhouse","pytest","requests","responses","setuptools","sh","shapely","tenacity","yaml","zstandard"] +force-sort-within-sections = true