diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3b73d0730aee37c14cb56ecf29172077eaca7606..28e6272d550a417244798ff3c53b517e76fab087 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,33 +1,37 @@
-image: registry.gitlab.com/arkindex/backend:base-0.11.3
+image: registry.gitlab.com/arkindex/backend/base:latest
stages:
- test
- build
+ - deploy
-cache:
- paths:
- - .cache/pip
+# For jobs that run backend scripts directly
+.backend-setup:
+ cache:
+ paths:
+ - .cache/pip
-before_script:
- - apk --update add build-base
- # Custom line to install our own deps from Git using GitLab CI credentials
- - "pip install -e git+https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/arkindex/common#egg=arkindex-common"
- - "pip install -e git+https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/arkindex/ponos#egg=ponos-server"
- - pip install -r tests-requirements.txt codecov
+ before_script:
+ - apk --update add build-base
+ # Custom line to install our own deps from Git using GitLab CI credentials
+ - "pip install -e git+https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/arkindex/common#egg=arkindex-common"
+ - "pip install -e git+https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/arkindex/ponos#egg=ponos-server"
+ - pip install -r tests-requirements.txt codecov
-variables:
- # For the postgres image
- POSTGRES_DB: arkindex_dev
- POSTGRES_USER: devuser
- POSTGRES_PASSWORD: devdata
+ variables:
+ # For the postgres image
+ POSTGRES_DB: arkindex_dev
+ POSTGRES_USER: devuser
+ POSTGRES_PASSWORD: devdata
- # For the backend
- DB_HOST: postgres
- DB_PORT: 5432
+ # For the backend
+ DB_HOST: postgres
+ DB_PORT: 5432
- # Pip cache
- PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+ # Pip cache
+ PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
backend-tests:
+ extends: .backend-setup
stage: test
services:
@@ -38,12 +42,14 @@ backend-tests:
- codecov
backend-lint:
+ extends: .backend-setup
stage: test
script:
- flake8
backend-migrations:
+ extends: .backend-setup
stage: test
services:
@@ -54,33 +60,81 @@ backend-migrations:
- arkindex/manage.py makemigrations --check --noinput --dry-run -v 3
backend-openapi:
+ extends: .backend-setup
stage: build
script:
- - mkdir -p output
- - pip install -e .
- - pip install uritemplate==3 apistar>=0.7.2
- - arkindex/manage.py generateschema --generator_class arkindex.project.openapi.SchemaGenerator > output/original.yml
- - openapi/patch.py openapi/paths.yml output/original.yml > output/schema.yml
-
- variables:
- PONOS_DATA_DIR: /tmp
+ - ci/openapi.sh
artifacts:
paths:
- output/
backend-static:
+ extends: .backend-setup
stage: build
script:
- - mkdir -p static
- - pip install -e .
- - STATIC_ROOT=$(pwd)/static arkindex/manage.py collectstatic
-
- variables:
- PONOS_DATA_DIR: /tmp
+ - ci/static-collect.sh
artifacts:
paths:
- static
+
+backend-build-base:
+ stage: build
+ image: docker:19.03.1
+ services:
+ - docker:dind
+ variables:
+ DOCKER_DRIVER: overlay2
+ DOCKER_HOST: tcp://docker:2375/
+
+ # Run this only on base tags
+ rules:
+ - if: '$CI_COMMIT_TAG =~ /^base-.*/'
+ when: on_success
+ - when: never
+
+ script:
+ - ci/build-base.sh
+
+backend-build:
+ stage: build
+ image: docker:19.03.1
+ services:
+ - docker:dind
+ variables:
+ DOCKER_DRIVER: overlay2
+ DOCKER_HOST: tcp://docker:2375/
+
+ # Run this on master and tags except base tags
+ rules:
+ - if: '$CI_COMMIT_BRANCH == "master"'
+ when: on_success
+ - if: '$CI_COMMIT_TAG && $CI_COMMIT_TAG !~ /^base-.*/'
+ when: on_success
+ - when: never
+
+ script:
+ - ci/build.sh
+
+backend-static-deploy:
+ stage: deploy
+
+ # Run this on any version tag except base images
+ rules:
+ - if: '$CI_COMMIT_TAG && $CI_COMMIT_TAG !~ /^base-.*/'
+ when: on_success
+ - when: never
+
+ # Run immediately once backend-static ends without waiting for others
+ needs:
+ - backend-static
+
+ # Ensure artifacts are available
+ dependencies:
+ - backend-static
+
+ script:
+ - ci/static-deploy.sh
diff --git a/Dockerfile b/Dockerfile
index e19ca8231939012d16a9f42281cd4eb065f80b05..3eb327cfba3e65931c3574dd76ef58195894d766 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,10 @@
-FROM registry.gitlab.com/arkindex/backend:base-0.11.3
+FROM registry.gitlab.com/arkindex/backend/base:latest as build
+
+RUN mkdir build
+ADD . build
+RUN cd build && python3 setup.py sdist
+
+FROM registry.gitlab.com/arkindex/backend/base:latest
ARG COMMON_BRANCH=master
ARG COMMON_ID=9855787
@@ -25,7 +31,7 @@ RUN \
# Install arkindex and its deps
# Uses a source archive instead of full local copy to speedup docker build
-COPY dist/arkindex-*.tar.gz /tmp/arkindex.tar.gz
+COPY --from=build /build/dist/arkindex-*.tar.gz /tmp/arkindex.tar.gz
RUN pip install /tmp/arkindex.tar.gz && rm /tmp/arkindex.tar.gz
# Allow access to static files directory
diff --git a/Makefile b/Makefile
index 4e30a94a7c895c2689136f47a3b91ae5fca8c35e..fc55a4af5631094da03d83014e68e9e903507e93 100644
--- a/Makefile
+++ b/Makefile
@@ -1,47 +1,21 @@
ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-TUNNEL_HOST:=arkindex-dev
-TUNNEL_PORT:=8000
PONOS_BRANCH=master
COMMON_BRANCH=master
-VERSION=$(shell git rev-parse --short HEAD)
-TAG_APP=arkindex-app
-.PHONY: build base
+IMAGE_TAG=registry.gitlab.com/arkindex/backend
+.PHONY: all
all: clean build
base: require-version
- docker build $(ROOT_DIR)/base -t registry.gitlab.com/arkindex/backend:base-$(version)
- docker push registry.gitlab.com/arkindex/backend:base-$(version)
- sed -i '/image:/s/base-.*$$/base-$(version)/' $(ROOT_DIR)/.gitlab-ci.yml
- sed -i '/FROM/s/base-.*$$/base-$(version)/' $(ROOT_DIR)/Dockerfile
+ VERSION=$(version) CI_PROJECT_DIR=$(ROOT_DIR) CI_REGISTRY_IMAGE=$(IMAGE_TAG) $(ROOT_DIR)/ci/build-base.sh
clean:
rm -rf *.egg-info build dist .eggs
find . -name '*.pyc' -exec rm {} \;
build:
- rm -f dist/arkindex-*.tar.gz
- python setup.py sdist
- docker build --no-cache $(ROOT_DIR) -t $(TAG_APP):$(VERSION) -t $(TAG_APP):latest --build-arg PONOS_BRANCH=$(PONOS_BRANCH) --build-arg COMMON_BRANCH=$(COMMON_BRANCH)
-
-publish-version: require-docker-auth
- [ -f $(ROOT_DIR)/arkindex/project/local_settings.py ] && mv $(ROOT_DIR)/arkindex/project/local_settings.py $(ROOT_DIR)/arkindex/project/local_settings.py.bak || true
- $(MAKE) build TAG_APP=registry.gitlab.com/arkindex/backend
- docker push registry.gitlab.com/arkindex/backend:$(VERSION)
- [ -f $(ROOT_DIR)/arkindex/project/local_settings.py.bak ] && mv $(ROOT_DIR)/arkindex/project/local_settings.py.bak $(ROOT_DIR)/arkindex/project/local_settings.py || true
-
-latest:
- $(MAKE) publish-version VERSION=latest
-
-release:
- $(eval version:=$(shell cat VERSION))
- $(MAKE) publish-version VERSION=$(version)
- docker push registry.gitlab.com/arkindex/backend:latest
- git tag $(version)
-
-tunnel:
- ssh $(TUNNEL_HOST) -NR *:$(TUNNEL_PORT):localhost:$(TUNNEL_PORT)
+ CI_PROJECT_DIR=$(ROOT_DIR) CI_REGISTRY_IMAGE=$(IMAGE_TAG) COMMON_BRANCH=$(COMMON_BRANCH) PONOS_BRANCH=$(PONOS_BRANCH) $(ROOT_DIR)/ci/build.sh
test-fixtures:
$(eval export PGPASSWORD=devdata)
@@ -63,9 +37,6 @@ test-fixtures-restore:
psql -h 127.0.0.1 -p 9100 -U devuser -c 'DROP DATABASE arkindex_dev' template1
psql -h 127.0.0.1 -p 9100 -U devuser -c 'ALTER DATABASE arkindex_dev_replace RENAME TO arkindex_dev' template1
-require-docker-auth:
- @grep registry.gitlab.com ~/.docker/config.json > /dev/null || (echo "Docker Login on registry.gitlab.com"; docker login registry.gitlab.com)
-
require-version:
@if [ ! "$(version)" ]; then echo "Missing version to publish"; exit 1; fi
@git rev-parse $(version) >/dev/null 2>&1 && (echo "Version $(version) already exists on local git repo !" && exit 1) || true
diff --git a/README.md b/README.md
index 487f752689b105328b5c5d32873bce10133841ff..a031a6d27a6009a31bf7c30049f6b3132f589d92 100644
--- a/README.md
+++ b/README.md
@@ -97,10 +97,6 @@ At the root of the repository is a Makefile that provides commands for common op
* `make base`: Create and push the `arkindex-base` Docker image that is used to build the `arkindex-app` image;
* `make clean`: Cleanup the Python package build and cache files;
* `make build`: Build the arkindex Python package and recreate the `arkindex-app:latest` without pushing to the GitLab container registry;
-* `make latest`: Build and push the `latest` Docker image to the GitLab container registry;
-* `make release`: Build and push a release Docker image to the GitLab container registry (use the `VERSION` file to update the version number);
-* `make worker`: Start a local (non-Docker) Celery worker;
-* `make tunnel`: Open a SSH tunnel via the preproduction server, making your dev server available on `arkindex.dev.teklia.com:8000` — useful for webhook related development;
* `make test-fixtures`: Create the unit tests fixtures on a temporary PostgreSQL database and save them to the `data.json` file used by most Django unit tests.
### Django commands
diff --git a/ci/build-base.sh b/ci/build-base.sh
new file mode 100755
index 0000000000000000000000000000000000000000..83fb856ee1b58041de4848120cf908d430bf5a08
--- /dev/null
+++ b/ci/build-base.sh
@@ -0,0 +1,33 @@
+#!/bin/sh -e
+# Build the backend base image.
+# Requires CI_PROJECT_DIR and CI_REGISTRY_IMAGE as well as either VERSION or CI_COMMIT_TAG.
+# If VERSION is not set, CI_COMMIT_TAG must start with "base-" and will be used as "base-$VERSION".
+# If CI_REGISTRY is set, will push the image to the specified registry.
+# If CI_REGISTRY, CI_REGISTRY_USER and CI_REGISTRY_PASSWORD are set, will run `docker login` before pushing.
+
+if [ -z "$VERSION" ]; then
+ # Ensure this is a base tag, then tell sh to remove the base- prefix.
+ case $CI_COMMIT_TAG in
+ base-*)
+ VERSION=${CI_COMMIT_TAG#base-};;
+ *)
+ echo build-base can only be used with 'base-*' tags.
+ exit 1;;
+ esac
+fi
+
+if [ -z "$VERSION" -o -z "$CI_PROJECT_DIR" -o -z "$CI_REGISTRY_IMAGE" ]; then
+ echo Missing environment variables
+ exit 1
+fi
+
+if [ -n "$CI_REGISTRY" -a -n "$CI_REGISTRY_USER" -a -n "$CI_REGISTRY_PASSWORD" ]; then
+ echo Logging in to container registry…
+ echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+fi
+
+IMAGE_TAG="$CI_REGISTRY_IMAGE/base:$VERSION"
+IMAGE_LATEST="$CI_REGISTRY_IMAGE/base:latest"
+docker build --no-cache "$CI_PROJECT_DIR/base" -t "$IMAGE_TAG" -t "$IMAGE_LATEST"
+docker push "$IMAGE_TAG"
+docker push "$IMAGE_LATEST"
diff --git a/ci/build.sh b/ci/build.sh
new file mode 100755
index 0000000000000000000000000000000000000000..e84a3c97b8cf32126dd58b1f7750659d09c19341
--- /dev/null
+++ b/ci/build.sh
@@ -0,0 +1,37 @@
+#!/bin/sh -e
+# Build the backend Docker image.
+# Requires CI_PROJECT_DIR and CI_REGISTRY_IMAGE to be set.
+# VERSION defaults to latest.
+# Will automatically login to a registry if CI_REGISTRY, CI_REGISTRY_USER and CI_REGISTRY_PASSWORD are set.
+# Will only push an image if $CI_REGISTRY is set.
+
+if [ -z "$VERSION" ]; then
+ #Ensure this is not a base tag
+ case $CI_COMMIT_TAG in
+ base-*)
+ echo build can only be used with non-base tags.
+ exit 1;;
+ esac
+ VERSION=${CI_COMMIT_TAG:-latest}
+fi
+
+if [ -z "$VERSION" -o -z "$CI_PROJECT_DIR" -o -z "$CI_REGISTRY_IMAGE" ]; then
+ echo Missing environment variables
+ exit 1
+fi
+
+if [ -n "$CI_REGISTRY" -a -n "$CI_REGISTRY_USER" -a -n "$CI_REGISTRY_PASSWORD" ]; then
+ echo Logging in to container registry…
+ echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+fi
+
+PONOS_BRANCH=${PONOS_BRANCH:-master}
+COMMON_BRANCH=${COMMON_BRANCH:-master}
+IMAGE_TAG="$CI_REGISTRY_IMAGE:$VERSION"
+
+cd $CI_PROJECT_DIR
+docker pull "$CI_REGISTRY_IMAGE/base:latest"
+docker build . -t "$IMAGE_TAG" --build-arg "PONOS_BRANCH=$PONOS_BRANCH" --build-arg "COMMON_BRANCH=$COMMON_BRANCH"
+if [ -n "$CI_REGISTRY" ]; then
+ docker push "$IMAGE_TAG"
+fi
diff --git a/ci/openapi.sh b/ci/openapi.sh
new file mode 100755
index 0000000000000000000000000000000000000000..6433f8c6f91e8a5e3a56a6d152809dccf3f30fec
--- /dev/null
+++ b/ci/openapi.sh
@@ -0,0 +1,6 @@
+#!/bin/sh -e
+mkdir -p output
+pip install -e .
+pip install uritemplate==3 apistar>=0.7.2
+PONOS_DATA_DIR=/tmp arkindex/manage.py generateschema --generator_class arkindex.project.openapi.SchemaGenerator > output/original.yml
+openapi/patch.py openapi/paths.yml output/original.yml > output/schema.yml
diff --git a/ci/static-collect.sh b/ci/static-collect.sh
new file mode 100755
index 0000000000000000000000000000000000000000..aa1e28588a7ad2234fa68e61a60b3686ceabc658
--- /dev/null
+++ b/ci/static-collect.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+mkdir -p static
+pip install -e .
+PONOS_DATA_DIR=/tmp STATIC_ROOT=$(pwd)/static arkindex/manage.py collectstatic
diff --git a/ci/static-deploy.sh b/ci/static-deploy.sh
new file mode 100755
index 0000000000000000000000000000000000000000..fb837a450f1d77d9be13be8cd2eaa22ad3c45b29
--- /dev/null
+++ b/ci/static-deploy.sh
@@ -0,0 +1,5 @@
+VERSION=${VERSION:-${CI_COMMIT_TAG}}
+[ -z "$VERSION" ] && echo "No version specified" && exit 1
+
+pip install awscli
+aws s3 cp --recursive static s3://teklia-assets-release/arkindex/$VERSION/