Skip to content
Snippets Groups Projects
Verified Commit c85ccbae authored by Yoann Schneider's avatar Yoann Schneider :tennis:
Browse files

add ruff and add all ignore to make the CI pass for now

parent 13a5f8bc
No related branches found
No related tags found
1 merge request!322Replace isort and flake8 by ruff for linting
Showing
with 160 additions and 63 deletions
......@@ -3,7 +3,7 @@ repos:
rev: 'v0.0.257'
hooks:
- id: ruff
args: [--exit-non-zero-on-fix]
args: [--exit-non-zero-on-fix, "--fix"]
exclude: "^worker-{{cookiecutter.slug}}/"
- repo: https://github.com/ambv/black
rev: 23.1.0
......
# See https://github.com/charliermarsh/ruff#rules for error code definitions.
select = [
# Pyflakes
"F",
......@@ -8,23 +9,41 @@ select = [
"I001",
# flake8-debugger
"T10",
# flake8-use-pathlib
"PTH",
# flake8-comprehensions
"C4",
# logging format
"G",
# Pep8-naming
"N",
# flake8-pie
"PIE",
# Pylint
"PL",
# Ruff-specific
"RUF",
# flake8-bandit
"S",
# flake8-simplify
"SIM",
]
# Never enforce `E501` (line length violations).
ignore = ["E501"]
# Do not lint __pycache__ folders
exclude = ["__pycache__"]
# Autofix issues
fix = true
# Never enforce
ignore = [
"E501", # line-too-long
"RUF001", # unicode
"RUF002", # unicode
"PLR0915", # Too many statements (functions too big)
"PLR0912", # Too many branches (functions too complex)
"PLR0913", # Toom any arguments to function call
"S101", # assert checks forbidden
"PLR2004", # Magic value used in comparison, consider replacing 3 with a constant variable
]
# By default, always enumerate fixed violations.
show-fixes = true
# Group violations by containing file.
format = "gitlab"
# By default, always show source code snippets.
show-source = true
......
# Extend the `.ruff.toml` file in the parent directory.
extend = "../.ruff.toml"
ignore = [
"G004", # Logging statement uses f-string
]
[per-file-ignores]
"worker/training.py" = [
"S113", # Probable use of requests call without timeout
]
"worker/entity.py" = [
"N818", # Exception name `MissingEntityType` should be named with an Error suffix
]
"cache.py" = [
"SIM117", # Use a single `with` statement with multiple contexts instead of nested `with` statements
"PLR5501", # Consider using `elif` instead of `else` then `if` to remove one indentation level
"SIM114", # Combine `if` branches using logical `or` operator
"S608", # Possible SQL injection vector through string-based query construction
]
"worker/base.py" = [
"SIM115", # Use context handler for opening files
"PTH123", # `open()` should be replaced by `Path.open()`
]
"utils.py" = [
"S324", # Probable use of insecure hash functions in `hashlib`: `md5`
"PTH123", # `open()` should be replaced by `Path.open()`
]
"reporting.py" = [
"PTH123", # `open()` should be replaced by `Path.open()`
]
"image.py" = [
"G001", # Logging statement uses `string.format()`
]
"models.py" = [
"G003", # Logging statement uses `+`
]
"__init__.py" = [
"G001", # Logging statement uses `string.format()`
]
......@@ -7,7 +7,6 @@ and writes will go both to the Arkindex API and the database,
reducing network usage.
"""
import json
import sqlite3
from pathlib import Path
import sqlite3
from typing import Optional, Union
......@@ -323,9 +322,8 @@ def merge_parents_cache(paths: list, current_database: Path):
# Check that the parent cache uses a compatible version
check_version(path)
with SqliteDatabase(path) as source:
with source.bind_ctx(MODELS):
source.create_tables(MODELS)
with SqliteDatabase(path) as source, source.bind_ctx(MODELS):
source.create_tables(MODELS)
logger.info(f"Merging parent db {path} into {current_database}")
statements = [
......
......@@ -193,13 +193,11 @@ def prepare_git_key(
private_key_path = Path(private_key_path).expanduser()
known_hosts_path = Path(known_hosts_path).expanduser()
if private_key_path.exists():
if private_key_path.read_text() != private_key:
make_backup(private_key_path)
if private_key_path.exists() and private_key_path.read_text() != private_key:
make_backup(private_key_path)
if known_hosts_path.exists():
if known_hosts_path.read_text() != known_hosts:
make_backup(known_hosts_path)
if known_hosts_path.exists() and known_hosts_path.read_text() != known_hosts:
make_backup(known_hosts_path)
private_key_path.write_text(private_key)
# private key must be private, otherwise git will fail
......
......@@ -5,7 +5,6 @@ Helper methods to download and open IIIF images, and manage polygons.
from collections import namedtuple
from io import BytesIO
from math import ceil
import os
from pathlib import Path
from typing import TYPE_CHECKING, List, Optional, Union
......@@ -55,11 +54,7 @@ def open_image(
:param mirrored: Whether or not to mirror the image horizontally.
:returns: A Pillow image.
"""
if (
path.startswith("http://")
or path.startswith("https://")
or not Path(path).exists()
):
if path.startswith(("http://", "https://")) or not Path(path).exists():
image = download_image(path)
else:
try:
......
......@@ -201,10 +201,7 @@ class Element(MagicDict):
else:
resize = "full"
if use_full_image:
url = self.image_url(resize)
else:
url = self.resize_zone_url(resize)
url = self.image_url(resize) if use_full_image else self.resize_zone_url(resize)
try:
return open_image(
......
......@@ -2,9 +2,9 @@
import hashlib
import logging
import os
from pathlib import Path
import tarfile
import tempfile
from pathlib import Path
from typing import Optional, Tuple, Union
import zstandard
......
......@@ -3,6 +3,7 @@
Base classes to implement Arkindex workers.
"""
import contextlib
from enum import Enum
import json
import os
......@@ -228,10 +229,9 @@ class ElementsWorker(
)
if element:
# Try to update the activity to error state regardless of the response
try:
with contextlib.suppress(Exception):
self.update_activity(element.id, ActivityState.Error)
except Exception:
pass
self.report.error(element_id, e)
# Save report as local artifact
......
......@@ -71,7 +71,7 @@ class EntityMixin(object):
element: Union[Element, CachedElement],
name: str,
type: str,
metas=dict(),
metas={},
validated=None,
):
"""
......
......@@ -3,6 +3,7 @@
# File generated by pre-commit: https://pre-commit.com
# ID: 138fd403232d2ddd5efb44317e38bf03
import os
from pathlib import Path
import sys
# we try our best, but the shebang of this script is difficult to determine:
......@@ -26,7 +27,7 @@ ARGS = [
"--skip-on-missing-config",
]
# end templated
ARGS.extend(("--hook-dir", os.path.realpath(os.path.dirname(__file__))))
ARGS.extend(("--hook-dir", os.path.realpath(Path(__file__).parent)))
ARGS.append("--")
ARGS.extend(sys.argv[1:])
......
......@@ -11,7 +11,7 @@ def requirements(path: Path):
return list(map(str.strip, f.read().splitlines()))
with open("VERSION") as f:
with Path("VERSION").open() as f:
VERSION = f.read()
setup(
......
# Extend the `.ruff.toml` file in the parent directory.
extend = "../.ruff.toml"
ignore = [
"RUF005",
]
[per-file-ignores]
"test_image.py" = [
"S108", # Probable insecure usage of temporary file or directory
]
"conftest.py" = [
"S324", # use of insecure hash functions in hashlib
"PLW2901", # `for` loop variable `path` overwritten by assignment target
"S108", # Probable insecure usage of temporary file or directory
"PTH123", # `open()` should be replaced by `Path.open()`
]
"test_utils.py" = [
"C414", # Unnecessary `list` call within `sorted()`
]
"test_elements_worker/test_cli.py" = [
"PTH108", # `os.unlink()` should be replaced by `Path.unlink()`
"PTH123", # `open()` should be replaced by `Path.open()`
]
"test_cache.py" = [
"PLR0913", # Too many arguments to function call (6 > 5)
"PTH123", # `open()` should be replaced by `Path.open()`
]
"test_elements_worker/test_worker.py" = [
"PLR0913", # Too many arguments to function call (6 > 5)
]
"test_git.py" = [
"S108", # Probable insecure usage of temporary file or directory
]
# -*- coding: utf-8 -*-
import json
import logging
import sys
from pathlib import Path
import sys
......
[settings]
# Compatible with black
profile = black
default_section=FIRSTPARTY
known_first_party = arkindex,arkindex_worker
known_third_party = pytest,setuptools
......@@ -3,7 +3,7 @@ repos:
rev: 'v0.0.257'
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix]
args: [--exit-non-zero-on-fix, "--fix"]
- repo: https://github.com/ambv/black
rev: 23.1.0
hooks:
......
# See https://github.com/charliermarsh/ruff#rules for error code definitions.
select = [
# Pyflakes
"F",
......@@ -8,27 +9,44 @@ select = [
"I001",
# flake8-debugger
"T10",
# flake8-use-pathlib
"PTH",
# flake8-comprehensions
"C4",
# logging format
"G",
# Pep8-naming
"N",
# flake8-pie
"PIE",
# Pylint
"PL",
# Ruff-specific
"RUF",
# flake8-bandit
"S",
# flake8-simplify
"SIM",
]
# Never enforce `E501` (line length violations).
ignore = ["E501"]
# Do not lint __pycache__ folders
exclude = ["__pycache__"]
# Autofix issues
fix = true
# Never enforce
ignore = [
"E501", # line-too-long
"PLR0915", # Too many statements (functions too big)
"PLR0912", # Too many branches (functions too complex)
"PLR0913", # Toom any arguments to function call
"S101", # assert checks forbidden
"PLR2004", # Magic value used in comparison, consider replacing ... with a constant variable
"N999", # Invalid module name
]
# By default, always enumerate fixed violations.
show-fixes = true
# Group violations by containing file.
format = "gitlab"
# By default, always show source code snippets.
show-source = true
[isort]
known-first-party = ["arkindex","arkindex_common","arkindex_worker"]
known-third-party = ["PIL","apistar","gitlab","gnupg","peewee","playhouse","pytest","requests","responses","setuptools","sh","shapely","tenacity","yaml","zstandard"]
known-first-party = ["arkindex","arkindex_worker"]
known-third-party = ["pytest","setuptools"]
force-sort-within-sections = true
......@@ -30,9 +30,12 @@ def parse_requirements():
)
with Path("VERSION").open() as file:
VERSION = file.read()
setup(
name=MODULE,
version=open("VERSION").read(),
version=VERSION,
description="{{ cookiecutter.description }}",
author="{{ cookiecutter.author }}",
author_email="{{ cookiecutter.email }}",
......
......@@ -8,7 +8,9 @@ logger = logging.getLogger(__name__)
class Demo(ElementsWorker):
def process_element(self, element):
logger.info(f"Demo processing element ({element.id})")
logger.info(
"Demo processing element ({element_id})", extra={"element_id": element.id}
)
def main():
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment