Merge branch 'super-ci' into 'master'

Build in CI See merge request !691

Merge branch 'super-ci' into 'master'
51d572d2 · Bastien Abadie · 9265e972 · d2246843 · 51d572d2 · 51d572d2
Commit 51d572d2 authored 5 years ago by Bastien Abadie
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
-image: registry.gitlab.com/arkindex/backend:base-0.11.3
+image: registry.gitlab.com/arkindex/backend/base:latest
 stages:
  - test
  - build
+  - deploy

-cache:
-  paths:
-    - .cache/pip
+# For jobs that run backend scripts directly
+.backend-setup:
+  cache:
+    paths:
+      - .cache/pip

-before_script:
-  - apk --update add build-base
-  # Custom line to install our own deps from Git using GitLab CI credentials
-  - "pip install -e git+https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/arkindex/common#egg=arkindex-common"
-  - "pip install -e git+https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/arkindex/ponos#egg=ponos-server"
-  - pip install -r tests-requirements.txt codecov
+  before_script:
+    - apk --update add build-base
+    # Custom line to install our own deps from Git using GitLab CI credentials
+    - "pip install -e git+https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/arkindex/common#egg=arkindex-common"
+    - "pip install -e git+https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/arkindex/ponos#egg=ponos-server"
+    - pip install -r tests-requirements.txt codecov

-variables:
-  # For the postgres image
-  POSTGRES_DB: arkindex_dev
-  POSTGRES_USER: devuser
-  POSTGRES_PASSWORD: devdata
+  variables:
+    # For the postgres image
+    POSTGRES_DB: arkindex_dev
+    POSTGRES_USER: devuser
+    POSTGRES_PASSWORD: devdata

-  # For the backend
-  DB_HOST: postgres
-  DB_PORT: 5432
+    # For the backend
+    DB_HOST: postgres
+    DB_PORT: 5432

-  # Pip cache
-  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+    # Pip cache
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"

 backend-tests:
+  extends: .backend-setup
  stage: test

  services:
@@ -38,12 +42,14 @@ backend-tests:
    - codecov

 backend-lint:
+  extends: .backend-setup
  stage: test

  script:
    - flake8

 backend-migrations:
+  extends: .backend-setup
  stage: test

  services:
@@ -54,33 +60,81 @@ backend-migrations:
    - arkindex/manage.py makemigrations --check --noinput --dry-run -v 3

 backend-openapi:
+  extends: .backend-setup
  stage: build

  script:
-    - mkdir -p output
-    - pip install -e .
-    - pip install uritemplate==3 apistar>=0.7.2
-    - arkindex/manage.py generateschema --generator_class arkindex.project.openapi.SchemaGenerator > output/original.yml
-    - openapi/patch.py openapi/paths.yml output/original.yml > output/schema.yml
-
-  variables:
-    PONOS_DATA_DIR: /tmp
+    - ci/openapi.sh

  artifacts:
    paths:
      - output/

 backend-static:
+  extends: .backend-setup
  stage: build

  script:
-    - mkdir -p static
-    - pip install -e .
-    - STATIC_ROOT=$(pwd)/static arkindex/manage.py collectstatic
-
-  variables:
-    PONOS_DATA_DIR: /tmp
+    - ci/static-collect.sh

  artifacts:
    paths:
      - static
+
+backend-build-base:
+  stage: build
+  image: docker:19.03.1
+  services:
+    - docker:dind
+  variables:
+    DOCKER_DRIVER: overlay2
+    DOCKER_HOST: tcp://docker:2375/
+
+  # Run this only on base tags
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /^base-.*/'
+      when: on_success
+    - when: never
+
+  script:
+    - ci/build-base.sh
+
+backend-build:
+  stage: build
+  image: docker:19.03.1
+  services:
+    - docker:dind
+  variables:
+    DOCKER_DRIVER: overlay2
+    DOCKER_HOST: tcp://docker:2375/
+
+  # Run this on master and tags except base tags
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      when: on_success
+    - if: '$CI_COMMIT_TAG && $CI_COMMIT_TAG !~ /^base-.*/'
+      when: on_success
+    - when: never
+
+  script:
+    - ci/build.sh
+
+backend-static-deploy:
+  stage: deploy
+
+  # Run this on any version tag except base images
+  rules:
+    - if: '$CI_COMMIT_TAG && $CI_COMMIT_TAG !~ /^base-.*/'
+      when: on_success
+    - when: never
+
+  # Run immediately once backend-static ends without waiting for others
+  needs:
+    - backend-static
+
+  # Ensure artifacts are available
+  dependencies:
+    - backend-static
+
+  script:
+    - ci/static-deploy.sh
--- a/Dockerfile
+++ b/Dockerfile
-FROM registry.gitlab.com/arkindex/backend:base-0.11.3
+FROM registry.gitlab.com/arkindex/backend/base:latest as build
+
+RUN mkdir build
+ADD . build
+RUN cd build && python3 setup.py sdist
+
+FROM registry.gitlab.com/arkindex/backend/base:latest

 ARG COMMON_BRANCH=master
 ARG COMMON_ID=9855787
@@ -25,7 +31,7 @@ RUN \

 # Install arkindex and its deps
 # Uses a source archive instead of full local copy to speedup docker build
-COPY dist/arkindex-*.tar.gz /tmp/arkindex.tar.gz
+COPY --from=build /build/dist/arkindex-*.tar.gz /tmp/arkindex.tar.gz
 RUN pip install /tmp/arkindex.tar.gz && rm /tmp/arkindex.tar.gz

 # Allow access to static files directory

--- a/Makefile
+++ b/Makefile
 ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-TUNNEL_HOST:=arkindex-dev
-TUNNEL_PORT:=8000
 PONOS_BRANCH=master
 COMMON_BRANCH=master

-VERSION=$(shell git rev-parse --short HEAD)
-TAG_APP=arkindex-app
-.PHONY: build base
+IMAGE_TAG=registry.gitlab.com/arkindex/backend
+.PHONY: all

 all: clean build

 base: require-version
-	docker build $(ROOT_DIR)/base -t registry.gitlab.com/arkindex/backend:base-$(version)
-	docker push registry.gitlab.com/arkindex/backend:base-$(version)
-	sed -i '/image:/s/base-.*$$/base-$(version)/' $(ROOT_DIR)/.gitlab-ci.yml
-	sed -i '/FROM/s/base-.*$$/base-$(version)/' $(ROOT_DIR)/Dockerfile
+	VERSION=$(version) CI_PROJECT_DIR=$(ROOT_DIR) CI_REGISTRY_IMAGE=$(IMAGE_TAG) $(ROOT_DIR)/ci/build-base.sh

 clean:
 	rm -rf *.egg-info build dist .eggs
 	find . -name '*.pyc' -exec rm {} \;

 build:
-	rm -f dist/arkindex-*.tar.gz
-	python setup.py sdist
-	docker build --no-cache $(ROOT_DIR) -t $(TAG_APP):$(VERSION) -t $(TAG_APP):latest --build-arg PONOS_BRANCH=$(PONOS_BRANCH) --build-arg COMMON_BRANCH=$(COMMON_BRANCH)
-
-publish-version: require-docker-auth
-	[ -f $(ROOT_DIR)/arkindex/project/local_settings.py ] && mv $(ROOT_DIR)/arkindex/project/local_settings.py $(ROOT_DIR)/arkindex/project/local_settings.py.bak || true
-	$(MAKE) build TAG_APP=registry.gitlab.com/arkindex/backend
-	docker push registry.gitlab.com/arkindex/backend:$(VERSION)
-	[ -f $(ROOT_DIR)/arkindex/project/local_settings.py.bak ] && mv $(ROOT_DIR)/arkindex/project/local_settings.py.bak $(ROOT_DIR)/arkindex/project/local_settings.py || true
-
-latest:
-	$(MAKE) publish-version VERSION=latest
-
-release:
-	$(eval version:=$(shell cat VERSION))
-	$(MAKE) publish-version VERSION=$(version)
-	docker push registry.gitlab.com/arkindex/backend:latest
-	git tag $(version)
-
-tunnel:
-	ssh $(TUNNEL_HOST) -NR *:$(TUNNEL_PORT):localhost:$(TUNNEL_PORT)
+	CI_PROJECT_DIR=$(ROOT_DIR) CI_REGISTRY_IMAGE=$(IMAGE_TAG) COMMON_BRANCH=$(COMMON_BRANCH) PONOS_BRANCH=$(PONOS_BRANCH) $(ROOT_DIR)/ci/build.sh

 test-fixtures:
 	$(eval export PGPASSWORD=devdata)
@@ -63,9 +37,6 @@ test-fixtures-restore:
 	psql -h 127.0.0.1 -p 9100 -U devuser -c 'DROP DATABASE arkindex_dev' template1
 	psql -h 127.0.0.1 -p 9100 -U devuser -c 'ALTER DATABASE arkindex_dev_replace RENAME TO arkindex_dev' template1

-require-docker-auth:
-	@grep registry.gitlab.com ~/.docker/config.json > /dev/null || (echo "Docker Login on registry.gitlab.com"; docker login registry.gitlab.com)
-
 require-version:
 	@if [ ! "$(version)" ]; then echo "Missing version to publish"; exit 1; fi
 	@git rev-parse $(version) >/dev/null 2>&1 && (echo "Version $(version) already exists on local git repo !" && exit 1) || true
--- a/README.md
+++ b/README.md
@@ -97,10 +97,6 @@ At the root of the repository is a Makefile that provides commands for common op
 * `make base`: Create and push the `arkindex-base` Docker image that is used to build the `arkindex-app` image;
 * `make clean`: Cleanup the Python package build and cache files;
 * `make build`: Build the arkindex Python package and recreate the `arkindex-app:latest` without pushing to the GitLab container registry;
-* `make latest`: Build and push the `latest` Docker image to the GitLab container registry;
-* `make release`: Build and push a release Docker image to the GitLab container registry (use the `VERSION` file to update the version number);
-* `make worker`: Start a local (non-Docker) Celery worker;
-* `make tunnel`: Open a SSH tunnel via the preproduction server, making your dev server available on `arkindex.dev.teklia.com:8000` &mdash; useful for webhook related development;
 * `make test-fixtures`: Create the unit tests fixtures on a temporary PostgreSQL database and save them to the `data.json` file used by most Django unit tests.

 ### Django commands

--- a/ci/build-base.sh
+++ b/ci/build-base.sh
+#!/bin/sh -e
+# Build the backend base image.
+# Requires CI_PROJECT_DIR and CI_REGISTRY_IMAGE as well as either VERSION or CI_COMMIT_TAG.
+# If VERSION is not set, CI_COMMIT_TAG must start with "base-" and will be used as "base-$VERSION".
+# If CI_REGISTRY is set, will push the image to the specified registry.
+# If CI_REGISTRY, CI_REGISTRY_USER and CI_REGISTRY_PASSWORD are set, will run `docker login` before pushing.
+
+if [ -z "$VERSION" ]; then
+	# Ensure this is a base tag, then tell sh to remove the base- prefix.
+	case $CI_COMMIT_TAG in
+		base-*)
+			VERSION=${CI_COMMIT_TAG#base-};;
+		*)
+			echo build-base can only be used with 'base-*' tags.
+			exit 1;;
+	esac
+fi
+
+if [ -z "$VERSION" -o -z "$CI_PROJECT_DIR" -o -z "$CI_REGISTRY_IMAGE" ]; then
+	echo Missing environment variables
+	exit 1
+fi
+
+if [ -n "$CI_REGISTRY" -a -n "$CI_REGISTRY_USER" -a -n "$CI_REGISTRY_PASSWORD" ]; then
+	echo Logging in to container registry…
+	echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+fi
+
+IMAGE_TAG="$CI_REGISTRY_IMAGE/base:$VERSION"
+IMAGE_LATEST="$CI_REGISTRY_IMAGE/base:latest"
+docker build --no-cache "$CI_PROJECT_DIR/base" -t "$IMAGE_TAG" -t "$IMAGE_LATEST"
+docker push "$IMAGE_TAG"
+docker push "$IMAGE_LATEST"
--- a/ci/build.sh
+++ b/ci/build.sh
+#!/bin/sh -e
+# Build the backend Docker image.
+# Requires CI_PROJECT_DIR and CI_REGISTRY_IMAGE to be set.
+# VERSION defaults to latest.
+# Will automatically login to a registry if CI_REGISTRY, CI_REGISTRY_USER and CI_REGISTRY_PASSWORD are set.
+# Will only push an image if $CI_REGISTRY is set.
+
+if [ -z "$VERSION" ]; then
+	#Ensure this is not a base tag
+	case $CI_COMMIT_TAG in
+		base-*)
+			echo build can only be used with non-base tags.
+			exit 1;;
+	esac
+	VERSION=${CI_COMMIT_TAG:-latest}
+fi
+
+if [ -z "$VERSION" -o -z "$CI_PROJECT_DIR" -o -z "$CI_REGISTRY_IMAGE" ]; then
+	echo Missing environment variables
+	exit 1
+fi
+
+if [ -n "$CI_REGISTRY" -a -n "$CI_REGISTRY_USER" -a -n "$CI_REGISTRY_PASSWORD" ]; then
+	echo Logging in to container registry…
+	echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+fi
+
+PONOS_BRANCH=${PONOS_BRANCH:-master}
+COMMON_BRANCH=${COMMON_BRANCH:-master}
+IMAGE_TAG="$CI_REGISTRY_IMAGE:$VERSION"
+
+cd $CI_PROJECT_DIR
+docker pull "$CI_REGISTRY_IMAGE/base:latest"
+docker build . -t "$IMAGE_TAG" --build-arg "PONOS_BRANCH=$PONOS_BRANCH" --build-arg "COMMON_BRANCH=$COMMON_BRANCH"
+if [ -n "$CI_REGISTRY" ]; then
+	docker push "$IMAGE_TAG"
+fi
--- a/ci/openapi.sh
+++ b/ci/openapi.sh
+#!/bin/sh -e
+mkdir -p output
+pip install -e .
+pip install uritemplate==3 apistar>=0.7.2
+PONOS_DATA_DIR=/tmp arkindex/manage.py generateschema --generator_class arkindex.project.openapi.SchemaGenerator > output/original.yml
+openapi/patch.py openapi/paths.yml output/original.yml > output/schema.yml
--- a/ci/static-collect.sh
+++ b/ci/static-collect.sh
+#!/bin/sh
+mkdir -p static
+pip install -e .
+PONOS_DATA_DIR=/tmp STATIC_ROOT=$(pwd)/static arkindex/manage.py collectstatic
--- a/ci/static-deploy.sh
+++ b/ci/static-deploy.sh
+VERSION=${VERSION:-${CI_COMMIT_TAG}}
+[ -z "$VERSION" ] && echo "No version specified" && exit 1
+
+pip install awscli
+aws s3 cp --recursive static s3://teklia-assets-release/arkindex/$VERSION/