Docker base image

.gitlab-ci.yml

@@ -3,7 +3,7 @@ stages:
 
 backend-tests:
   stage: test
-  image: python:3.5-alpine
+  image: registry.gitlab.com/arkindex/backend:base-0.8.0
 
   services:
     - postgres:latest
@@ -20,11 +20,10 @@ backend-tests:
     DB_PORT: 5432
 
   before_script:
-    - apk --update add postgresql-dev libjpeg-turbo-dev gcc musl-dev zlib-dev libmagic libxml2-dev libxslt-dev git
-    - pip install codecov
+    - apk --update add build-base
+    - pip install -r tests-requirements.txt codecov
 
   script:
     - python setup.py test
-    - pip install .[test]
     - flake8
     - codecov

Dockerfile

-FROM python:3.5-alpine as base
+FROM registry.gitlab.com/arkindex/backend:base-0.8.0
 
-FROM base as staging
-
-# Add system dependencies, use cache by copying requirements
-RUN apk add --update --no-cache postgresql-dev jpeg-dev build-base gzip zlib-dev libmagic libxml2-dev libxslt-dev
+ARG FRONTEND_BRANCH=master
+ARG GITLAB_TOKEN="gTPA5UQYesSuKMCRM2r_"
 
 # Install arkindex and its deps
 # Uses a source archive instead of full local copy to speedup docker build
 COPY dist/arkindex-*.tar.gz /tmp/arkindex.tar.gz
-RUN pip install --install-option="--prefix=/build" /tmp/arkindex.tar.gz gunicorn
-
-# Restart from base, using staging build
-FROM base
-ARG FRONTEND_BRANCH=master
-ARG GITLAB_TOKEN="gTPA5UQYesSuKMCRM2r_"
-
-# Import from staging the python build, so the final image is slimmer
-# and has no build tool
-COPY --from=staging /build /usr
-ENV PYTHONPATH=/usr/lib/python3.5/site-packages
-
-# Add runtime system deps
-RUN apk add --update --no-cache wget gzip libmagic git unzip libpq libxslt libjpeg
-
-# Add unprivilegied user
-RUN addgroup -g 1000 teklia && adduser -D -u 1000 -G teklia ark
+RUN pip install /tmp/arkindex.tar.gz gunicorn
 
 # Allow access to medias and logs
 RUN mkdir -p /medias/staging /medias/iiif /logs /workers

MANIFEST.in

 include VERSION
 include requirements.txt
+include base/requirements.txt
 include tests-requirements.txt
 include arkindex/documents/*.xsl
 recursive-include arkindex/templates *.html

Makefile

@@ -3,30 +3,36 @@ TUNNEL_HOST:=panetios-dev
 TUNNEL_PORT:=8000
 
 VERSION=$(shell git rev-parse --short HEAD)
-TAG=arkindex-app
-.PHONY: build
+TAG_APP=arkindex-app
+TAG_BASE=arkindex-base
+.PHONY: build base
 
 all: clean build
 
+base: require-version
+	echo $(version) > $(ROOT_DIR)/VERSION
+	docker build $(ROOT_DIR)/base -t $(TAG_BASE):$(version) -t $(TAG_BASE):latest -t registry.gitlab.com/arkindex/backend:base-$(version)
+	docker push registry.gitlab.com/arkindex/backend:base-$(version)
+	git commit $(ROOT_DIR)/VERSION -m 'Bump to $(version)'
+
 clean:
 	rm -rf *.egg-info build dist .eggs
 	find . -name '*.pyc' -exec rm {} \;
 
 build:
 	python setup.py sdist
-	docker build $(ROOT_DIR) -t $(TAG):$(VERSION) -t $(TAG):latest
+	docker build $(ROOT_DIR) -t $(TAG_APP):$(VERSION) -t $(TAG_APP):latest
 
 publish-version: require-docker-auth
-	$(MAKE) build TAG=registry.gitlab.com/arkindex/backend
+	$(MAKE) build TAG_APP=registry.gitlab.com/arkindex/backend
 	docker push registry.gitlab.com/arkindex/backend:$(VERSION)
 
 latest:
 	$(MAKE) publish-version VERSION=latest
 
-release: require-version
-	echo $(version) > $(ROOT_DIR)/VERSION
+release:
+	version:=$(shell cat VERSION)
 	$(MAKE) publish-version VERSION=$(version)
-	git commit $(ROOT_DIR)/VERSION -m 'Release $(version)'
 	git tag $(version)
 
 worker:

base/Dockerfile

+FROM python:3.5-alpine as base
+
+FROM base as staging
+
+# Add system dependencies, use cache by copying requirements
+RUN apk add --update --no-cache postgresql-dev jpeg-dev build-base gzip zlib-dev libmagic libxml2-dev libxslt-dev
+
+# Install long buildtime deps
+# Uses a source archive instead of full local copy to speedup docker build
+ADD requirements.txt /requirements.txt
+RUN pip install --install-option="--prefix=/build" -r requirements.txt
+
+# Restart from base, using staging build
+FROM base
+
+# Import from staging the python build, so the final image is slimmer
+# and has no build tool
+COPY --from=staging /build /usr
+ENV PYTHONPATH=/usr/lib/python3.5/site-packages
+
+# Add runtime system deps
+RUN apk add --update --no-cache wget gzip libmagic git unzip libpq libxslt libjpeg
+
+# Add unprivilegied user
+RUN addgroup -g 1000 teklia && adduser -D -u 1000 -G teklia ark

base/requirements.txt

+Django==2.1
+elasticsearch==6.2.0
+ijson==2.3
+lxml==4.2.3
+openpyxl==2.4.9
+Pillow==4.3.0
+psycopg2==2.7.3.2
+python-Levenshtein==0.12.0
+redis==2.10.6

requirements.txt

+# -r ./base/requirements.txt
+
 celery==4.2.0
 celery_once==2.0.0
 certifi==2017.7.27.1
 chardet==3.0.4
-Django==2.1
 django-enumfields==0.10.0
 djangorestframework==3.7.1
 django-webpack-loader==0.5.0
-elasticsearch==6.2.0
 et-xmlfile==1.0.1
 gitpython==2.1.11
 idna==2.6
-ijson==2.3
 jdcal==1.3
-lxml==4.2.3
 olefile==0.44
-openpyxl==2.4.9
-Pillow==4.3.0
-psycopg2==2.7.3.2
-python-Levenshtein==0.12.0
 python-magic==0.4.15
 python-memcached==1.59
 pytz==2017.2
-redis==2.10.6
 requests==2.18.4
 roman==2.0.0
 urllib3==1.22

setup.py

 #!/usr/bin/env python
+import os.path
 from setuptools import setup, find_packages
 
+
+def requirements(path):
+    assert os.path.exists(path), 'Missing requirements {}'.format(path)
+    with open(path) as f:
+        return f.read().splitlines()
+
+
 with open('VERSION') as f:
     VERSION = f.read()
 
-with open('requirements.txt') as f:
-    requirements = f.read().splitlines()
-
-with open('tests-requirements.txt') as f:
-    tests_requirements = f.read().splitlines()
+install_requires = requirements('requirements.txt') + requirements('base/requirements.txt')
+tests_requires = requirements('tests-requirements.txt')
 
 setup(
     name='arkindex',
@@ -17,11 +22,11 @@ setup(
     author='Teklia',
     author_email='abadie@teklia.com',
     url='https://arkindex.teklia.com',
-    install_requires=requirements,
+    install_requires=install_requires,
     test_suite='arkindex.project.runtests.run',
-    tests_require=tests_requirements,
+    tests_require=tests_requires,
     extras_require={
-        'test': tests_requirements,
+        'test': tests_requires,
     },
     packages=find_packages(),
     include_package_data=True,