bootstrap repo

.arkindex.yml

+---
+version: 2
+
+type: worker
+
+workers:
+  - slug: generic-training-dataset
+    name: Generic Training Dataset Extractor
+    type: data-extract
+    docker:
+      build: Dockerfile

.cookiecutter.json

+{
+  "slug": "generic-training-dataset",
+  "name": "Generic Training Dataset Extractor",
+  "description": "Fill base-worker cache with information about dataset and extract images",
+  "worker_type": "data-extract",
+  "author": "Teklia",
+  "email": "contact@teklia.com"
+}

.dockerignore

+.tox
+.git

.flake8

+[flake8]
+max-line-length = 150
+exclude = .git,__pycache__
+ignore = E203,E501,W503

.gitignore

+*.pyc
+*.egg-info/
+.tox/

.gitlab-ci.yml

+stages:
+  - test
+  - build
+  - release
+
+test:
+  image: python:3
+
+  stage: test
+  cache:
+    paths:
+      - .cache/pip
+
+  variables:
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+    ARKINDEX_API_SCHEMA_URL: schema.yml
+
+  before_script:
+    - pip install tox
+
+    # Download OpenAPI schema from last backend build
+    - curl https://assets.teklia.com/arkindex/openapi.yml > schema.yml
+
+  except:
+    - schedules
+
+  script:
+    - tox -- --junitxml=test-report.xml --durations=50
+
+lint:
+  image: python:3
+
+  cache:
+    paths:
+      - .cache/pip
+      - .cache/pre-commit
+
+  variables:
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+    PRE_COMMIT_HOME: "$CI_PROJECT_DIR/.cache/pre-commit"
+
+  before_script:
+    - pip install pre-commit
+
+  except:
+    - schedules
+
+  script:
+    - pre-commit run -a
+
+docker-build:
+  stage: build
+  image: docker:19.03.1
+  services:
+    - docker:dind
+  variables:
+    DOCKER_DRIVER: overlay2
+    DOCKER_HOST: tcp://docker:2375/
+
+  except:
+    - schedules
+
+  script:
+    - ci/build.sh
+
+release-notes:
+  stage: release
+  image: registry.gitlab.com/teklia/devops:latest
+
+  only:
+    - tags
+
+  script:
+    - devops release-notes
+
+bump-python-deps:
+  stage: release
+  image: registry.gitlab.com/teklia/devops:latest
+
+  only:
+    - schedules
+
+  script:
+    - devops python-deps requirements.txt

.isort.cfg

+[settings]
+# Compatible with black
+profile = black
+
+default_section=FIRSTPARTY
+known_first_party = arkindex,arkindex_worker
+known_third_party = pytest,setuptools

.pre-commit-config.yaml

+repos:
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+  - repo: https://github.com/ambv/black
+    rev: 23.1.0
+    hooks:
+    - id: black
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8
+        additional_dependencies:
+          - 'flake8-coding==1.3.2'
+          - 'flake8-debugger==4.1.2'
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-ast
+      - id: check-docstring-first
+      - id: check-executables-have-shebangs
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: debug-statements
+      - id: trailing-whitespace
+      - id: check-yaml
+        args: [--allow-multiple-documents]
+      - id: mixed-line-ending
+      - id: name-tests-test
+        args: ['--django']
+      - id: check-json
+      - id: requirements-txt-fixer
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.2
+    hooks:
+      - id: codespell
+        args: ['--write-changes']
+  - repo: meta
+    hooks:
+      - id: check-useless-excludes

Dockerfile

+FROM python:3
+
+WORKDIR /src
+
+# Install worker as a package
+COPY worker_generic-training-dataset worker_generic-training-dataset
+COPY requirements.txt setup.py VERSION ./
+RUN pip install .
+
+# Add archi local CA
+RUN curl https://assets.teklia.com/teklia_dev_ca.pem > /usr/local/share/ca-certificates/arkindex-dev.crt && update-ca-certificates
+ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt
+
+CMD ["worker-generic-training-dataset"]

MANIFEST.in

+include requirements.txt
+include VERSION

README.md

+# generic-training-dataset
+
+Fill base-worker cache with information about dataset and extract images
+
+### Development
+
+For development and tests purpose it may be useful to install the worker as a editable package with pip.
+
+```shell
+pip3 install -e .
+```
+
+### Linter
+
+Code syntax is analyzed before submitting the code.\
+To run the linter tools suite you may use pre-commit.
+
+```shell
+pip install pre-commit
+pre-commit run -a
+```
+
+### Run tests
+
+Tests are executed with tox using [pytest](https://pytest.org).
+
+```shell
+pip install tox
+tox
+```
+
+To recreate tox virtual environment (e.g. a dependencies update), you may run `tox -r`

VERSION

+0.1.0

ci/build.sh

+#!/bin/sh -e
+# Build the tasks Docker image.
+# Requires CI_PROJECT_DIR and CI_REGISTRY_IMAGE to be set.
+# VERSION defaults to latest.
+# Will automatically login to a registry if CI_REGISTRY, CI_REGISTRY_USER and CI_REGISTRY_PASSWORD are set.
+# Will only push an image if $CI_REGISTRY is set.
+
+if [ -z "$VERSION" ]; then
+	VERSION=${CI_COMMIT_TAG:-latest}
+fi
+
+if [ -z "$VERSION" -o -z "$CI_PROJECT_DIR" -o -z "$CI_REGISTRY_IMAGE" ]; then
+	echo Missing environment variables
+	exit 1
+fi
+
+IMAGE_TAG="$CI_REGISTRY_IMAGE:$VERSION"
+
+cd $CI_PROJECT_DIR
+docker build -f Dockerfile . -t "$IMAGE_TAG"
+
+# Publish the image on the main branch or on a tag
+if [ "$CI_COMMIT_REF_NAME" = "main" -o -n "$CI_COMMIT_TAG" ]; then
+  if [ -n "$CI_REGISTRY" -a -n "$CI_REGISTRY_USER" -a -n "$CI_REGISTRY_PASSWORD" ]; then
+    echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+    docker push $IMAGE_TAG
+  else
+    echo "Missing environment variables to log in to the container registry…"
+  fi
+else
+  echo "The build was not published to the repository registry (only for main branch or tags)…"
+fi

requirements.txt

+arkindex-base-worker==0.3.2

setup.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pathlib import Path
+
+from setuptools import find_packages, setup
+
+MODULE = "worker_generic_training_dataset"
+COMMAND = "worker-generic-training-dataset"
+
+
+def parse_requirements_line(line):
+    """Special case for git requirements"""
+    if line.startswith("git+http"):
+        assert "@" in line, "Branch should be specified with suffix (ex: @master)"
+        assert (
+            "#egg=" in line
+        ), "Package name should be specified with suffix (ex: #egg=kraken)"
+        package_name = line.split("#egg=")[-1]
+        return f"{package_name} @ {line}"
+    else:
+        return line
+
+
+def parse_requirements():
+    path = Path(__file__).parent.resolve() / "requirements.txt"
+    assert path.exists(), f"Missing requirements: {path}"
+    return list(
+        map(parse_requirements_line, map(str.strip, path.read_text().splitlines()))
+    )
+
+
+setup(
+    name=MODULE,
+    version=open("VERSION").read(),
+    description="Fill base-worker cache with information about dataset and extract images",
+    author="Teklia",
+    author_email="contact@teklia.com",
+    install_requires=parse_requirements(),
+    entry_points={"console_scripts": [f"{COMMAND}={MODULE}.worker:main"]},
+    packages=find_packages(),
+)

tests/conftest.py

+# -*- coding: utf-8 -*-
+import os
+
+import pytest
+
+from arkindex.mock import MockApiClient
+from arkindex_worker.worker.base import BaseWorker
+
+
+@pytest.fixture(autouse=True)
+def setup_environment(responses, monkeypatch):
+    """Setup needed environment variables"""
+
+    # Allow accessing remote API schemas
+    # defaulting to the prod environment
+    schema_url = os.environ.get(
+        "ARKINDEX_API_SCHEMA_URL",
+        "https://arkindex.teklia.com/api/v1/openapi/?format=openapi-json",
+    )
+    responses.add_passthru(schema_url)
+
+    # Set schema url in environment
+    os.environ["ARKINDEX_API_SCHEMA_URL"] = schema_url
+    # Setup a fake worker run ID
+    os.environ["ARKINDEX_WORKER_RUN_ID"] = "1234-generic-training-dataset"
+
+    # Setup a mock api client instead of using a real one
+    monkeypatch.setattr(BaseWorker, "setup_api_client", lambda _: MockApiClient())

tests/test_worker.py

+# -*- coding: utf-8 -*-
+import importlib
+
+
+def test_dummy():
+    assert True
+
+
+def test_import():
+    """Import our newly created module, through importlib to avoid parsing issues"""
+    worker = importlib.import_module("worker_generic_training_dataset.worker")
+    assert hasattr(worker, "Demo")
+    assert hasattr(worker.Demo, "process_element")

tox.ini

+[tox]
+envlist = worker-generic-training-dataset
+
+[testenv]
+passenv = ARKINDEX_API_SCHEMA_URL
+commands =
+  pytest {posargs}
+
+deps =
+  pytest
+  pytest-responses
+  -rrequirements.txt

worker_generic_training_dataset/worker.py

+# -*- coding: utf-8 -*-
+from arkindex_worker.worker import ElementsWorker
+
+
+class Demo(ElementsWorker):
+    def process_element(self, element):
+        print("Demo processing element", element)
+
+
+def main():
+    Demo(description="Fill base-worker cache with information about dataset and extract images").run()
+
+
+if __name__ == "__main__":
+    main()