chore: bring staging changes to production

.dockerignore

@@ -6,3 +6,4 @@
 !requirements/
 !gunicorn.py
 !setup.cfg
+!pyproject.toml

.travis.yml

@@ -1,5 +1,5 @@
-sudo: required
-language: minimal
+os: linux
+language: shell
 
 git:
   depth: 2
@@ -10,33 +10,37 @@ branches:
   - devel
 
 services:
-  - docker
+- docker
 
 env:
   global:
-    - IMAGE_REPO=gcr.io/dd-decaf-cfbf6/warehouse
-    - IMAGE_TAG=${TRAVIS_BRANCH}
+  - IMAGE=gcr.io/dd-decaf-cfbf6/warehouse
+  - BRANCH=${TRAVIS_BRANCH}
+  - BUILD_COMMIT=${TRAVIS_COMMIT}
+  - SHORT_COMMIT=${TRAVIS_COMMIT:0:7}
+  - BUILD_DATE=$(date -u +%Y-%m-%d)
+  - BUILD_TAG=${BRANCH}_${BUILD_DATE}_${SHORT_COMMIT}
+
+before_install:
+- make setup
 
 install:
-  - docker build -t ${IMAGE_REPO}:${TRAVIS_COMMIT::12} -t ${IMAGE_REPO}:${TRAVIS_BRANCH} .
-  - make network
-  - make databases
+- make build
+- make build-travis
+- make post-build
+- make start
 
 script:
-  - docker-compose run --rm web flake8 src tests
-  - docker-compose run --rm web black --check src tests
-  - docker-compose run --rm web isort --check-only --recursive src tests
-  - # Run the tests and report coverage (see https://docs.codecov.io/docs/testing-with-docker).
-  - mkdir --parents /tmp/coverage
-  - docker-compose run --rm -e ENVIRONMENT=testing -v "/tmp/coverage:/tmp/coverage" web pytest --cov-report "xml:/tmp/coverage/coverage.xml" --cov-report term --cov=src/warehouse
-  - bash <(curl -s https://codecov.io/bash) -f "/tmp/coverage/coverage.xml"
-  - ./scripts/verify_license_headers.sh src tests
+- make style
+- make safety
+# Run the tests and report coverage (see https://docs.codecov.io/docs/testing-with-docker).
+- docker-compose exec -e ENVIRONMENT=testing web pytest --cov=warehouse --cov-report=term --cov-report=xml
+- bash <(curl -s https://codecov.io/bash)
 
 before_deploy:
-  - ./scripts/install_gcloud.sh
-  - ./scripts/install_kubectl.sh
-  - docker push ${IMAGE_REPO}:${TRAVIS_COMMIT::12}
-  - docker push ${IMAGE_REPO}:${TRAVIS_BRANCH}
+- ./scripts/install_gcloud.sh
+- ./scripts/install_kubectl.sh
+- if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then make push; fi
 
 deploy:
   provider: script

Dockerfile

@@ -1,25 +1,49 @@
-FROM dddecaf/postgres-base:master
+# Copyright (c) 2018-2020 Novo Nordisk Foundation Center for Biosustainability,
+# Technical University of Denmark.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_TAG=alpine
+
+FROM dddecaf/postgres-base:${BASE_TAG}
+
+ARG BASE_TAG=alpine
+ARG BUILD_COMMIT
+
+LABEL dk.dtu.biosustain.warehouse.alpine.vendor="Novo Nordisk Foundation \
+Center for Biosustainability, Technical University of Denmark"
+LABEL maintainer="[email protected]"
+LABEL dk.dtu.biosustain.warehouse.alpine.build.base-tag="${BASE_TAG}"
+LABEL dk.dtu.biosustain.warehouse.alpine.build.commit="${BUILD_COMMIT}"
+
+ARG CWD="/app"
+
+ENV PYTHONPATH="${CWD}/src"
 
-ENV APP_USER=giraffe
-
-ARG UID=1000
-ARG GID=1000
-
-ARG CWD=/app
-
-ENV PYTHONPATH=${CWD}/src
+WORKDIR "${CWD}"
 
-RUN addgroup -g "${GID}" -S "${APP_USER}" && \
-    adduser -u "${UID}" -G "${APP_USER}" -S "${APP_USER}"
+COPY requirements ./requirements/
 
-WORKDIR "${CWD}"
+RUN set -eux \
+    && apk add --no-cache --virtual .build-deps build-base \
+    && pip install -r requirements/requirements.txt \
+    && rm -rf /root/.cache/pip \
+    && apk del .build-deps
 
-COPY requirements ./requirements
+COPY . ./
 
-RUN apk add --no-cache build-base && \
-    pip-sync requirements/requirements.txt && \
-    apk del build-base
+RUN chown -R "${APP_USER}:${APP_USER}" .
 
-COPY . "${CWD}/"
+EXPOSE 8000
 
-RUN chown -R "${APP_USER}:${APP_USER}" "${CWD}"
+CMD ["gunicorn", "-c", "gunicorn.py", "warehouse.wsgi:app"]

LATEST_BASE_TAG

@@ -0,0 +1 @@
+ alpine_2020-05-28_97a4608

Makefile

@@ -1,43 +1,71 @@
-.PHONY: setup network build start qa style test flake8 isort \
-		isort-save license stop clean logs
-SHELL:=/bin/bash
+.PHONY: setup lock own build post-build push start qa style safety test qc stop clean logs
 
-#################################################################################
-# COMMANDS                                                                      #
-#################################################################################
+################################################################################
+# Variables                                                                    #
+################################################################################
 
-## Run all initialization targets.
-setup: build
+IMAGE ?= gcr.io/dd-decaf-cfbf6/warehouse
+BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
+BUILD_COMMIT ?= $(shell git rev-parse HEAD)
+SHORT_COMMIT ?= $(shell git rev-parse --short HEAD)
+BUILD_DATE ?= $(shell date -u +%Y-%m-%d)
+BUILD_TAG ?= ${BRANCH}_${BUILD_DATE}_${SHORT_COMMIT}
+
+################################################################################
+# Commands                                                                     #
+################################################################################
 
 ## Create the docker bridge network if necessary.
 network:
 	docker network inspect DD-DeCaF >/dev/null 2>&1 || \
 		docker network create DD-DeCaF
 
-## Build local docker images.
-build: network volume
-	docker-compose build
-
-## Recompile requirements and store pinned dependencies with hashes.
-pip-compile:
-	docker run --rm -v `pwd`/requirements:/build dddecaf/postgres-base:compiler \
-		pip-compile --generate-hashes --upgrade --output-file /build/requirements.txt \
-		/build/requirements.in
-
-## Start all services in the background.
-start:
-	docker-compose up -d
-
-## Run all QA targets.
-qa: test style
-
-## Run all style related targets.
-style: black flake8 isort license
-
-## Run the tests.
-test:
-	-docker-compose run --rm -e ENVIRONMENT=testing web \
-		pytest --cov=src/warehouse tests
+## Run all initialization targets.
+setup: network
+
+## Generate the compiled requirements files.
+lock:
+	docker pull dddecaf/tag-spy:latest
+	$(eval LATEST_BASE_TAG := $(shell docker run --rm dddecaf/tag-spy:latest tag-spy dddecaf/postgres-base alpine))
+	$(file >LATEST_BASE_TAG, $(LATEST_BASE_TAG))
+	$(eval COMPILER_TAG := $(subst alpine,alpine-compiler,$(LATEST_BASE_TAG)))
+	$(info ************************************************************)
+	$(info * Compiling service dependencies on the basis of:)
+	$(info * dddecaf/postgres-base:$(COMPILER_TAG))
+	$(info ************************************************************)
+	docker pull dddecaf/postgres-base:$(COMPILER_TAG)
+	docker run --rm --mount \
+		"source=$(CURDIR)/requirements,target=/opt/requirements,type=bind" \
+		dddecaf/postgres-base:$(COMPILER_TAG) \
+		pip-compile --allow-unsafe --verbose --generate-hashes --upgrade \
+		/opt/requirements/requirements.in
+
+## Change file ownership from root to local user.
+own:
+	sudo chown "$(shell id --user --name):$(shell id --group --name)" .
+
+## Build the Docker image for deployment.
+build-travis:
+	$(eval LATEST_BASE_TAG := $(shell cat LATEST_BASE_TAG))
+	$(info ************************************************************)
+	$(info * Building the service on the basis of:)
+	$(info * dddecaf/postgres-base:$(LATEST_BASE_TAG))
+	$(info * Today is $(shell date -u +%Y-%m-%d).)
+	$(info * Please re-run `make lock` if you want to check for and)
+	$(info * depend on a later version.)
+	$(info ************************************************************)
+	docker pull dddecaf/postgres-base:$(LATEST_BASE_TAG)
+	docker build \
+		--build-arg BASE_TAG=$(LATEST_BASE_TAG) \
+		--build-arg BUILD_COMMIT=$(BUILD_COMMIT) \
+		--tag $(IMAGE):$(BRANCH) \
+		--tag $(IMAGE):$(BUILD_TAG) \
+		.
+
+## Build the local docker-compose image.
+build:
+	$(eval LATEST_BASE_TAG := $(shell cat LATEST_BASE_TAG))
+	BASE_TAG=$(LATEST_BASE_TAG) docker-compose build
 
 ## Create the testing database.
 databases:
@@ -47,28 +75,60 @@ databases:
 	docker-compose run --rm web flask db upgrade
 	docker-compose stop
 
-## Run black.
-black:
-	docker-compose run --rm web black src/warehouse tests
+## Only run once!
+post-build: databases
 
-## Run flake8.
-flake8:
-	-docker-compose run --rm web \
-		flake8 src/warehouse tests
+## Push local Docker images to their registries.
+push:
+	docker push $(IMAGE):$(BRANCH)
+	docker push $(IMAGE):$(BUILD_TAG)
+
+## Start all services in the background.
+start:
+	docker-compose up --force-recreate -d
+
+## Apply all quality assurance (QA) tools.
+qa:
+	docker-compose exec -e ENVIRONMENT=testing web \
+		isort --recursive src tests
+	docker-compose exec -e ENVIRONMENT=testing web \
+		black src tests
 
-## Check Python package import order.
 isort:
-	-docker-compose run --rm web \
-		isort --check-only --recursive src/warehouse tests
+	docker-compose exec -e ENVIRONMENT=testing web \
+		isort --check-only --diff --recursive src tests
 
-## Sort imports and write changes to files.
-isort-save:
-	docker-compose run --rm web \
-		isort --recursive src/warehouse tests
+black:
+	docker-compose exec -e ENVIRONMENT=testing web \
+		black --check --diff src tests
+
+flake8:
+	docker-compose exec -e ENVIRONMENT=testing web \
+		flake8 src tests
 
-## Verify source code license headers.
 license:
-	-./scripts/verify_license_headers.sh src/warehouse tests
+	docker-compose exec -e ENVIRONMENT=testing web \
+		./scripts/verify_license_headers.sh src tests
+
+## Run all style checks.
+style: isort black flake8 license
+
+## Check installed dependencies for vulnerabilities.
+safety:
+	docker-compose exec -e ENVIRONMENT=testing web \
+		safety check --full-report
+
+## Run the test suite.
+test:
+	docker-compose exec -e ENVIRONMENT=testing web \
+		pytest --cov=warehouse --cov-report=term
+
+## Run all quality control (QC) tools.
+qc: style safety test
+
+## Check the gunicorn configuration.
+gunicorn:
+	docker-compose run --rm web gunicorn --check-config -c gunicorn.py warehouse.wsgi:app
 
 ## Stop all services.
 stop:
@@ -77,19 +137,19 @@ stop:
 ## Stop all services and remove containers.
 clean:
 	docker-compose down
-	@echo "If you really want to remove all data, also run 'docker volume rm warehouse'."
 
 ## Follow the logs.
 logs:
 	docker-compose logs --tail="all" -f
 
-#################################################################################
-# Self Documenting Commands                                                     #
-#################################################################################
+################################################################################
+# Self Documenting Commands                                                    #
+################################################################################
 
 .DEFAULT_GOAL := show-help
 
-# Inspired by <http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html>
+# Inspired by
+# <http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html>
 # sed script explained:
 # /^##/:
 # 	* save line in hold space
@@ -142,4 +202,5 @@ show-help:
 		} \
 		printf "\n"; \
 	}' \
-	| more $(shell test $(shell uname) = Darwin && echo '--no-init --raw-control-chars')
+	| more $(shell test $(shell uname) = Darwin \
+	&& echo '--no-init --raw-control-chars')

deployment/production/deployment.yml

@@ -66,6 +66,11 @@ spec:
               name: warehouse-production
               key: BASIC_AUTH_PASSWORD
         command: ["flask", "db", "upgrade"]
+        resources:
+          requests:
+            cpu: "1m"
+          limits:
+            cpu: "2000m"
       containers:
       - name: web
         image: gcr.io/dd-decaf-cfbf6/warehouse:master
@@ -121,7 +126,6 @@ spec:
               key: BASIC_AUTH_PASSWORD
         resources:
           requests:
-            cpu: "10m"
+            cpu: "1m"
           limits:
             cpu: "2000m"
-        command: ["gunicorn", "-c", "gunicorn.py", "warehouse.wsgi:app"]

deployment/staging/deployment.yml

@@ -66,6 +66,11 @@ spec:
               name: warehouse-staging
               key: BASIC_AUTH_PASSWORD
         command: ["flask", "db", "upgrade"]
+        resources:
+          requests:
+            cpu: "1m"
+          limits:
+            cpu: "2000m"
       containers:
       - name: web
         image: gcr.io/dd-decaf-cfbf6/warehouse:devel
@@ -121,7 +126,6 @@ spec:
               key: BASIC_AUTH_PASSWORD
         resources:
           requests:
-            cpu: "10m"
+            cpu: "1m"
           limits:
             cpu: "2000m"
-        command: ["gunicorn", "-c", "gunicorn.py", "warehouse.wsgi:app"]