diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05829fb0c6bd2475f5fa4e7ec3835e62a4cb3993..ff0bf0eb82b69b2901e441df7519184c4afe5ae2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ stages: backend-tests: stage: test - image: python:3.5-alpine + image: registry.gitlab.com/arkindex/backend:base-0.8.0 services: - postgres:latest @@ -20,11 +20,10 @@ backend-tests: DB_PORT: 5432 before_script: - - apk --update add postgresql-dev libjpeg-turbo-dev gcc musl-dev zlib-dev libmagic libxml2-dev libxslt-dev git - - pip install codecov + - apk --update add build-base + - pip install -r tests-requirements.txt codecov script: - python setup.py test - - pip install .[test] - flake8 - codecov diff --git a/Dockerfile b/Dockerfile index 6ae1d9b459a980fca9b425af30dea375e3484d02..45c91a0ab9e04070c53585ca64dca5159de28808 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,12 @@ -FROM python:3.5-alpine as base +FROM registry.gitlab.com/arkindex/backend:base-0.8.0 -FROM base as staging - -# Add system dependencies, use cache by copying requirements -RUN apk add --update --no-cache postgresql-dev jpeg-dev build-base gzip zlib-dev libmagic libxml2-dev libxslt-dev +ARG FRONTEND_BRANCH=master +ARG GITLAB_TOKEN="gTPA5UQYesSuKMCRM2r_" # Install arkindex and its deps # Uses a source archive instead of full local copy to speedup docker build COPY dist/arkindex-*.tar.gz /tmp/arkindex.tar.gz -RUN pip install --install-option="--prefix=/build" /tmp/arkindex.tar.gz gunicorn - -# Restart from base, using staging build -FROM base -ARG FRONTEND_BRANCH=master -ARG GITLAB_TOKEN="gTPA5UQYesSuKMCRM2r_" - -# Import from staging the python build, so the final image is slimmer -# and has no build tool -COPY --from=staging /build /usr -ENV PYTHONPATH=/usr/lib/python3.5/site-packages - -# Add runtime system deps -RUN apk add --update --no-cache wget gzip libmagic git unzip libpq libxslt libjpeg - -# Add unprivilegied user -RUN addgroup -g 1000 teklia && adduser -D -u 1000 -G teklia ark +RUN pip install /tmp/arkindex.tar.gz gunicorn # Allow access to medias and logs RUN mkdir -p /medias/staging /medias/iiif /logs /workers diff --git a/MANIFEST.in b/MANIFEST.in index bef1560b93334be6581597b9ccc12244ad866b7e..e5d4319faf6f97d547fcde6f9a4565b0119ae425 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include VERSION include requirements.txt +include base/requirements.txt include tests-requirements.txt include arkindex/documents/*.xsl recursive-include arkindex/templates *.html diff --git a/Makefile b/Makefile index 807703470f6a9a79f102bf0b798fae834a0c2b32..5c334250e330b48d9501fb432ee5d234cbc9bd5c 100644 --- a/Makefile +++ b/Makefile @@ -3,30 +3,36 @@ TUNNEL_HOST:=panetios-dev TUNNEL_PORT:=8000 VERSION=$(shell git rev-parse --short HEAD) -TAG=arkindex-app -.PHONY: build +TAG_APP=arkindex-app +TAG_BASE=arkindex-base +.PHONY: build base all: clean build +base: require-version + echo $(version) > $(ROOT_DIR)/VERSION + docker build $(ROOT_DIR)/base -t $(TAG_BASE):$(version) -t $(TAG_BASE):latest -t registry.gitlab.com/arkindex/backend:base-$(version) + docker push registry.gitlab.com/arkindex/backend:base-$(version) + git commit $(ROOT_DIR)/VERSION -m 'Bump to $(version)' + clean: rm -rf *.egg-info build dist .eggs find . -name '*.pyc' -exec rm {} \; build: python setup.py sdist - docker build $(ROOT_DIR) -t $(TAG):$(VERSION) -t $(TAG):latest + docker build $(ROOT_DIR) -t $(TAG_APP):$(VERSION) -t $(TAG_APP):latest publish-version: require-docker-auth - $(MAKE) build TAG=registry.gitlab.com/arkindex/backend + $(MAKE) build TAG_APP=registry.gitlab.com/arkindex/backend docker push registry.gitlab.com/arkindex/backend:$(VERSION) latest: $(MAKE) publish-version VERSION=latest -release: require-version - echo $(version) > $(ROOT_DIR)/VERSION +release: + version:=$(shell cat VERSION) $(MAKE) publish-version VERSION=$(version) - git commit $(ROOT_DIR)/VERSION -m 'Release $(version)' git tag $(version) worker: diff --git a/base/Dockerfile b/base/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..210e72bb2398d51aa29b401358bcc338301aa314 --- /dev/null +++ b/base/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.5-alpine as base + +FROM base as staging + +# Add system dependencies, use cache by copying requirements +RUN apk add --update --no-cache postgresql-dev jpeg-dev build-base gzip zlib-dev libmagic libxml2-dev libxslt-dev + +# Install long buildtime deps +# Uses a source archive instead of full local copy to speedup docker build +ADD requirements.txt /requirements.txt +RUN pip install --install-option="--prefix=/build" -r requirements.txt + +# Restart from base, using staging build +FROM base + +# Import from staging the python build, so the final image is slimmer +# and has no build tool +COPY --from=staging /build /usr +ENV PYTHONPATH=/usr/lib/python3.5/site-packages + +# Add runtime system deps +RUN apk add --update --no-cache wget gzip libmagic git unzip libpq libxslt libjpeg + +# Add unprivilegied user +RUN addgroup -g 1000 teklia && adduser -D -u 1000 -G teklia ark diff --git a/base/requirements.txt b/base/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3ab286dabcf5b0bc496f51c344161b2a9cee327e --- /dev/null +++ b/base/requirements.txt @@ -0,0 +1,9 @@ +Django==2.1 +elasticsearch==6.2.0 +ijson==2.3 +lxml==4.2.3 +openpyxl==2.4.9 +Pillow==4.3.0 +psycopg2==2.7.3.2 +python-Levenshtein==0.12.0 +redis==2.10.6 diff --git a/requirements.txt b/requirements.txt index 530cabe3c80d4b71d712f4cb9746141cdc50d6c4..2be728a555ff55b0dbec62e6ef64fc84cb963555 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,27 +1,20 @@ +# -r ./base/requirements.txt + celery==4.2.0 celery_once==2.0.0 certifi==2017.7.27.1 chardet==3.0.4 -Django==2.1 django-enumfields==0.10.0 djangorestframework==3.7.1 django-webpack-loader==0.5.0 -elasticsearch==6.2.0 et-xmlfile==1.0.1 gitpython==2.1.11 idna==2.6 -ijson==2.3 jdcal==1.3 -lxml==4.2.3 olefile==0.44 -openpyxl==2.4.9 -Pillow==4.3.0 -psycopg2==2.7.3.2 -python-Levenshtein==0.12.0 python-magic==0.4.15 python-memcached==1.59 pytz==2017.2 -redis==2.10.6 requests==2.18.4 roman==2.0.0 urllib3==1.22 diff --git a/setup.py b/setup.py index 7e120f314fa570af072d71ee489361e03082f458..2b45d2f053afb51b5eccd2c4a2e0a471f57422d0 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,19 @@ #!/usr/bin/env python +import os.path from setuptools import setup, find_packages + +def requirements(path): + assert os.path.exists(path), 'Missing requirements {}'.format(path) + with open(path) as f: + return f.read().splitlines() + + with open('VERSION') as f: VERSION = f.read() -with open('requirements.txt') as f: - requirements = f.read().splitlines() - -with open('tests-requirements.txt') as f: - tests_requirements = f.read().splitlines() +install_requires = requirements('requirements.txt') + requirements('base/requirements.txt') +tests_requires = requirements('tests-requirements.txt') setup( name='arkindex', @@ -17,11 +22,11 @@ setup( author='Teklia', author_email='abadie@teklia.com', url='https://arkindex.teklia.com', - install_requires=requirements, + install_requires=install_requires, test_suite='arkindex.project.runtests.run', - tests_require=tests_requirements, + tests_require=tests_requires, extras_require={ - 'test': tests_requirements, + 'test': tests_requires, }, packages=find_packages(), include_package_data=True,