Skip to content
Snippets Groups Projects
Commit 5a19dde7 authored by Nishanth artham's avatar Nishanth artham
Browse files

bootstrap with worker template

parent 0fcf9b3f
No related branches found
No related tags found
1 merge request!1Bootstrap the repo with the base-worker tempalte
Pipeline #130007 failed
.DS_Store 0 → 100644
File added
---
version: 2
type: worker
workers:
- slug: palmira
name: plamira-worker
type: dla
docker:
build: Dockerfile
{
"slug": "palmira",
"name": "plamira-worker",
"description": "Palm Leaf Manuscript Region Annotator",
"worker_type": "dla",
"author": "Nishanth artham",
"email": "arthamnishanth123@gmail.com"
}
.tox
.git
[flake8]
max-line-length = 150
exclude = .git,__pycache__
ignore = E203,E501,W503
*.pyc
*.egg-info/
.tox/
stages:
- test
- build
- release
test:
image: python:3
stage: test
cache:
paths:
- .cache/pip
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
ARKINDEX_API_SCHEMA_URL: schema.yml
before_script:
- pip install tox
# Download OpenAPI schema from last backend build
- curl https://assets.teklia.com/arkindex/openapi.yml > schema.yml
except:
- schedules
script:
- tox
lint:
image: python:3
cache:
paths:
- .cache/pip
- .cache/pre-commit
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
PRE_COMMIT_HOME: "$CI_PROJECT_DIR/.cache/pre-commit"
before_script:
- pip install pre-commit
except:
- schedules
script:
- pre-commit run -a
docker-build:
stage: build
image: docker:19.03.1
services:
- docker:dind
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375/
except:
- schedules
script:
- ci/build.sh
release-notes:
stage: release
image: registry.gitlab.com/teklia/devops:latest
only:
- tags
script:
- devops release-notes
bump-python-deps:
stage: release
image: registry.gitlab.com/teklia/devops:latest
only:
- schedules
script:
- devops python-deps requirements.txt
[settings]
# Compatible with black
multi_line_output = 3
include_trailing_comma = True
force_grid_wrap = 0
use_parentheses = True
line_length = 88
default_section=FIRSTPARTY
known_first_party =
known_third_party = arkindex_worker,pytest,setuptools
repos:
- repo: https://github.com/pre-commit/mirrors-isort
rev: v5.10.1
hooks:
- id: isort
- repo: https://github.com/ambv/black
rev: 22.6.0
hooks:
- id: black
- repo: https://github.com/pycqa/flake8
rev: 5.0.4
hooks:
- id: flake8
additional_dependencies:
- flake8-coding==1.3.1
- flake8-copyright==0.2.3
- flake8-debugger==3.1.0
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.3.0
hooks:
- id: check-ast
- id: check-docstring-first
- id: check-executables-have-shebangs
- id: check-merge-conflict
- id: check-symlinks
- id: debug-statements
- id: trailing-whitespace
- id: check-yaml
args: [--allow-multiple-documents]
- id: mixed-line-ending
- id: name-tests-test
args: ['--django']
- id: check-json
- id: requirements-txt-fixer
- repo: https://github.com/codespell-project/codespell
rev: v2.2.1
hooks:
- id: codespell
args: ['--write-changes']
- repo: meta
hooks:
- id: check-useless-excludes
FROM python:3
WORKDIR /src
# Install worker as a package
COPY worker_{{cookiecutter.slug}} worker_{{cookiecutter.slug}}
COPY requirements.txt setup.py VERSION ./
RUN pip install .
# Add archi local CA
RUN curl https://assets.teklia.com/teklia_dev_ca.pem > /usr/local/share/ca-certificates/arkindex-dev.crt && update-ca-certificates
ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt
CMD ["worker-{{ cookiecutter.slug }}"]
include requirements.txt
include VERSION
# palmira
Palm Leaf Manuscript Region Annotator
### Development
For development and tests purpose it may be useful to install the worker as a editable package with pip.
```shell
pip3 install -e .
```
### Linter
Code syntax is analyzed before submitting the code.\
To run the linter tools suite you may use pre-commit.
```shell
pip install pre-commit
pre-commit run -a
```
### Run tests
Tests are executed with tox using [pytest](https://pytest.org).
```shell
pip install tox
tox
```
To recreate tox virtual environment (e.g. a dependencies update), you may run `tox -r`
0.1.0
#!/bin/sh -e
# Build the tasks Docker image.
# Requires CI_PROJECT_DIR and CI_REGISTRY_IMAGE to be set.
# VERSION defaults to latest.
# Will automatically login to a registry if CI_REGISTRY, CI_REGISTRY_USER and CI_REGISTRY_PASSWORD are set.
# Will only push an image if $CI_REGISTRY is set.
if [ -z "$VERSION" ]; then
VERSION=${CI_COMMIT_TAG:-latest}
fi
if [ -z "$VERSION" -o -z "$CI_PROJECT_DIR" -o -z "$CI_REGISTRY_IMAGE" ]; then
echo Missing environment variables
exit 1
fi
IMAGE_TAG="$CI_REGISTRY_IMAGE:$VERSION"
cd $CI_PROJECT_DIR
docker build -f Dockerfile . -t "$IMAGE_TAG"
# Publish the image on the main branch or on a tag
if [ "$CI_COMMIT_REF_NAME" = "main" -o -n "$CI_COMMIT_TAG" ]; then
if [ -n "$CI_REGISTRY" -a -n "$CI_REGISTRY_USER" -a -n "$CI_REGISTRY_PASSWORD" ]; then
echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
docker push $IMAGE_TAG
else
echo "Missing environment variables to log in to the container registry…"
fi
else
echo "The build was not published to the repository registry (only for main branch or tags)…"
fi
setup.py 0 → 100755
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from pathlib import Path
from setuptools import find_packages, setup
MODULE = "worker_{{cookiecutter.slug}}"
COMMAND = "worker-{{cookiecutter.slug}}"
def parse_requirements_line(line):
"""Special case for git requirements"""
if line.startswith("git+http"):
assert "@" in line, "Branch should be specified with suffix (ex: @master)"
assert (
"#egg=" in line
), "Package name should be specified with suffix (ex: #egg=kraken)"
package_name = line.split("#egg=")[-1]
return f"{package_name} @ {line}"
else:
return line
def parse_requirements():
path = Path(__file__).parent.resolve() / "requirements.txt"
assert path.exists(), f"Missing requirements: {path}"
return list(
map(parse_requirements_line, map(str.strip, path.read_text().splitlines()))
)
setup(
name=MODULE,
version=open("VERSION").read(),
description="{{ cookiecutter.description }}",
author="{{ cookiecutter.author }}",
author_email="{{ cookiecutter.email }}",
install_requires=parse_requirements(),
entry_points={"console_scripts": [f"{COMMAND}={MODULE}.worker:main"]},
packages=find_packages(),
)
# -*- coding: utf-8 -*-
import os
import pytest
from arkindex_worker.worker.base import BaseWorker
from arkindex.mock import MockApiClient
@pytest.fixture(autouse=True)
def setup_environment(responses, monkeypatch):
"""Setup needed environment variables"""
# Allow accessing remote API schemas
# defaulting to the prod environment
schema_url = os.environ.get(
"ARKINDEX_API_SCHEMA_URL",
"https://arkindex.teklia.com/api/v1/openapi/?format=openapi-json",
)
responses.add_passthru(schema_url)
# Set schema url in environment
os.environ["ARKINDEX_API_SCHEMA_URL"] = schema_url
# Setup a fake worker run ID
os.environ["ARKINDEX_WORKER_RUN_ID"] = "1234-{{ cookiecutter.slug }}"
# Setup a mock api client instead of using a real one
monkeypatch.setattr(BaseWorker, "setup_api_client", lambda _: MockApiClient())
# -*- coding: utf-8 -*-
import importlib
def test_dummy():
assert True
def test_import():
"""Import our newly created module, through importlib to avoid parsing issues"""
worker = importlib.import_module("worker_{{ cookiecutter.slug }}.worker")
assert hasattr(worker, "Demo")
assert hasattr(worker.Demo, "process_element")
tox.ini 0 → 100644
[tox]
envlist = worker-{{ cookiecutter.slug }}
[testenv]
passenv = ARKINDEX_API_SCHEMA_URL
commands =
pytest {posargs}
deps =
pytest
pytest-responses
-rrequirements.txt
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment