Refactor CI jobs and CI makefile (#1610)

* Refactor CI jobs and CI makefile

* Add Jenkinsfile example link

* Rename COMMAND env var to TARGET

* Rename environment file to .env

* Move upload-yaml target to build/ci makefile

* Remove more of 'operators/'

* Rename more

* Remove CLUSTER_NAME env var as it's no longer used

* Run dep ensure before ci targets

* Remove vendor/ from CI docker image as it's not used

* Fix fetching VAULT_TOKEN

* Export vault token for vault CLI to use

* Fix passing VAULT_TOKEN

* Add back creating license.key file

.gitignore

@@ -37,3 +37,7 @@ docs/html/*
 # ignore deployer files
 hack/deployer/deployer
 hack/deployer/config/run-config.yml
+
+# ignore CI config files
+run-config.yml
+.env

.golangci.yml

@@ -1,5 +1,5 @@
 run:
-  deadline: 90s
+  deadline: 300s
   skip-dirs:
     - config
     - hack

Makefile

@@ -6,6 +6,8 @@
 ##  --      Variables      --  ##
 #################################
 
+# reads file '.env', ignores if it doesn't exist
+-include .env
 
 # make sure sub-commands don't use eg. fish shell
 export SHELL := /bin/bash
@@ -25,11 +27,8 @@ LATEST_RELEASED_IMG ?= "docker.elastic.co/eck/$(NAME):0.8.0"
 # on GKE, use GCR and GCLOUD_PROJECT
 ifneq ($(findstring gke_,$(KUBECTL_CLUSTER)),)
 	REGISTRY ?= eu.gcr.io
-	REPOSITORY = ${GCLOUD_PROJECT}
-else ifneq ($(findstring azmk8s.io:443,$(shell kubectl config view --minify -o=jsonpath={.clusters[*].cluster.server} 2> /dev/null)),)
-	REGISTRY ?= cloudonk8s.azurecr.io
-	REPOSITORY ?= operators
-else ifeq ($(REGISTRY),)
+	REPOSITORY ?= ${GCLOUD_PROJECT}
+else
 	# default to local registry
 	REGISTRY ?= localhost:5000
 endif
@@ -38,11 +37,6 @@ endif
 IMG_SUFFIX ?= -$(subst _,,$(USER))
 IMG ?= $(REGISTRY)/$(REPOSITORY)/$(NAME)$(IMG_SUFFIX)
 TAG ?= $(shell git rev-parse --short --verify HEAD)
-
-ifeq ($(OPERATOR_IMAGE),)
-	# we never want this empty
-	OPERATOR_IMAGE := $(IMG):$(VERSION)-$(TAG)
-endif
 OPERATOR_IMAGE ?= $(IMG):$(VERSION)-$(TAG)
 
 
@@ -300,9 +294,7 @@ purge-gcr-images:
 # can be overriden to eg. TESTS_MATCH=TestMutationMoreNodes to match a single test
 TESTS_MATCH ?= "^Test"
 E2E_IMG ?= $(IMG)-e2e-tests:$(TAG)
-ifeq ($(STACK_VERSION),)
-	STACK_VERSION = 7.3.0
-endif
+STACK_VERSION ?= 7.3.0
 
 # Run e2e tests as a k8s batch job
 e2e: build-operator-image e2e-docker-build e2e-docker-push e2e-run
@@ -344,16 +336,12 @@ e2e-local:
 ci: dep-vendor-only check-fmt lint generate check-local-changes unit integration e2e-compile docker-build
 
 # Run e2e tests in a dedicated cluster.
-ci-e2e: run-deployer
-	$(MAKE) IMG_SUFFIX=-ci install-crds apply-psp e2e
+ci-e2e: dep-vendor-only run-deployer install-crds apply-psp e2e
 
 run-deployer: dep-vendor-only build-deployer
 	./hack/deployer/deployer execute --plans-file hack/deployer/config/plans.yml --run-config-file run-config.yml
 
-ci-release: export GO_TAGS = release
-ci-release: export LICENSE_PUBKEY = $(CURDIR)/build/ci/license.key
-ci-release: clean
-	@ $(MAKE) dep-vendor-only generate docker-build docker-push
+ci-release: clean dep-vendor-only generate build-operator-image
 	@ echo $(OPERATOR_IMAGE) was pushed!
 
 ##########################

build/ci/Dockerfile

@@ -54,7 +54,7 @@ RUN curl -sSL https://aka.ms/InstallAzureCLIDeb | bash
 WORKDIR /go/src/github.com/elastic/cloud-on-k8s
 COPY Gopkg.lock .
 COPY Gopkg.toml .
-RUN dep ensure --vendor-only -v
+RUN dep ensure --vendor-only -v && rm -rf vendor/
 
 # Cleanup
 RUN rm /go/src/github.com/elastic/cloud-on-k8s/Gopkg.lock && \

build/ci/Makefile

@@ -7,141 +7,54 @@
 ROOT_DIR = $(CURDIR)/../..
 GO_MOUNT_PATH ?= /go/src/github.com/elastic/cloud-on-k8s
 
-CI_IMAGE ?= docker.elastic.co/eck/eck-ci:$(shell md5sum $(ROOT_DIR)/Gopkg.lock $(ROOT_DIR)/build/ci/Dockerfile | awk '{print $$1}' | md5sum | awk '{print $$1}')
-
-VAULT_GKE_CREDS_SECRET ?= secret/cloud-team/cloud-ci/ci-gcp-k8s-operator
-GKE_CREDS_FILE ?= credentials.json
-VAULT_PUBLIC_KEY ?= secret/release/license
-PUBLIC_KEY_FILE ?= license.key
-VAULT_DOCKER_CREDENTIALS ?= secret/devops-ci/cloud-on-k8s/eckadmin
-DOCKER_LOGIN ?= eckadmin
-DOCKER_CREDENTIALS_FILE ?= docker_credentials.file
-VAULT_AWS_CREDS ?= secret/cloud-team/cloud-ci/eck-release
-VAULT_AWS_ACCESS_KEY_FILE ?= aws_access_key.file
-VAULT_AWS_SECRET_KEY_FILE ?= aws_secret_key.file
-
-VAULT_TOKEN ?= $(shell vault write -field=token auth/approle/login role_id=$(VAULT_ROLE_ID) secret_id=$(VAULT_SECRET_ID))
+# BUILD_ID is present during run on Jenkins machine, but not on dev box, hence using it here to distinguish between those cases
+ifdef BUILD_ID
+VAULT_TOKEN = $(shell vault write -address=$(VAULT_ADDR) -field=token auth/approle/login role_id=$(VAULT_ROLE_ID) secret_id=$(VAULT_SECRET_ID))
+else
+VAULT_TOKEN = $(shell vault write -address=$(VAULT_ADDR) -field=token auth/github/login token=$(GITHUB_TOKEN))
+# we use roleId as a string that has to be there for authn/z for CI, but it's empty and not needed for local execution
+NOT_USED = $(shell test -e ../../run-config.yml && sed -i -e "s;roleId:;token: $(GITHUB_TOKEN);g" ../../run-config.yml)
+endif
 
-check-license-header:
-	./../check-license-header.sh
+CI_IMAGE ?= docker.elastic.co/eck/eck-ci:$(shell md5sum $(ROOT_DIR)/Gopkg.lock $(ROOT_DIR)/build/ci/Dockerfile | awk '{print $$1}' | md5sum | awk '{print $$1}')
 
 show-image:
 	@ echo $(CI_IMAGE)
 
-# login to vault and retrieve gke creds into $GKE_CREDS_FILE
-vault-gke-creds:
-	@ VAULT_TOKEN=$(VAULT_TOKEN) \
-		vault read \
-		-address=$(VAULT_ADDR) \
-		-field=service-account \
-		$(VAULT_GKE_CREDS_SECRET) \
-		> $(GKE_CREDS_FILE)
-
-# reads Elastic public key from Vault into $PUBLIC_KEY_FILE
-vault-public-key:
-	@ VAULT_TOKEN=$(VAULT_TOKEN) \
-	 	vault read \
-		-address=$(VAULT_ADDR) \
-		-field=pubkey \
-		$(VAULT_PUBLIC_KEY) \
-		| base64 --decode \
-		> $(PUBLIC_KEY_FILE)
-
-# reads Docker password from Vault
-vault-docker-creds:
-	@ VAULT_TOKEN=$(VAULT_TOKEN) \
-	 	vault read \
-		-address=$(VAULT_ADDR) \
-		-field=value \
-		$(VAULT_DOCKER_CREDENTIALS) \
-		> $(DOCKER_CREDENTIALS_FILE)
-
-# reads AWS creds for yaml upload
-vault-aws-creds:
-	@ VAULT_TOKEN=$(VAULT_TOKEN) \
-		vault read \
-		-address=$(VAULT_ADDR) \
-		-field=access-key-id \
-		$(VAULT_AWS_CREDS) \
-		> $(VAULT_AWS_ACCESS_KEY_FILE)
-	@ VAULT_TOKEN=$(VAULT_TOKEN) \
-		vault read \
-		-address=$(VAULT_ADDR) \
-		-field=secret-access-key \
-		$(VAULT_AWS_CREDS) \
-		> $(VAULT_AWS_SECRET_KEY_FILE)
-
-## -- Job executed on all PRs
-
-ci-pr: check-license-header
-	@ docker run --rm -t \
-		-v /var/run/docker.sock:/var/run/docker.sock \
-		-v $(ROOT_DIR):$(GO_MOUNT_PATH) \
-		-w $(GO_MOUNT_PATH) \
-		-e "IMG_SUFFIX=-ci" \
-		--net=host \
-		$(CI_IMAGE) \
-		bash -c \
-			"make ci"
-
-## -- Release job
-
-ci-release: vault-public-key vault-docker-creds
-	@ docker run --rm -t \
-    	-v /var/run/docker.sock:/var/run/docker.sock \
-    	-v $(ROOT_DIR):$(GO_MOUNT_PATH) \
-    	-w $(GO_MOUNT_PATH) \
-    	-e "ELASTIC_DOCKER_LOGIN=$(DOCKER_LOGIN)" \
-    	-e "ELASTIC_DOCKER_PASSWORD=$(shell cat $(DOCKER_CREDENTIALS_FILE))" \
-    	-e "USE_ELASTIC_DOCKER_REGISTRY=true" \
-    	-e "OPERATOR_IMAGE=$(OPERATOR_IMAGE)" \
-    	-e "LATEST_RELEASED_IMG=$(LATEST_RELEASED_IMG)" \
-    	-e "VERSION=$(VERSION)" \
-    	-e "SNAPSHOT=$(SNAPSHOT)" \
-    	$(CI_IMAGE) \
-		bash -c "make ci-release"
+# runs $TARGET in context of CI container and dev makefile
+ci:
+	@ $(MAKE) DOCKER_CMD="make $(TARGET)" ci-internal
 
-# Will be uploaded to https://download.elastic.co/downloads/eck/$TAG_NAME/all-in-one.yaml
-yaml-upload: vault-aws-creds
-	@ docker run --rm -t \
-        -v $(ROOT_DIR):$(GO_MOUNT_PATH) \
-        -w $(GO_MOUNT_PATH) \
-        -e "AWS_ACCESS_KEY_ID=$(shell cat $(VAULT_AWS_ACCESS_KEY_FILE))" \
-        -e "AWS_SECRET_ACCESS_KEY=$(shell cat $(VAULT_AWS_SECRET_KEY_FILE))" \
-        $(CI_IMAGE) \
-        bash -c "aws s3 cp $(GO_MOUNT_PATH)/config/all-in-one.yaml \
-		s3://download.elasticsearch.org/downloads/eck/$(TAG_NAME)/all-in-one.yaml"
+ci-interactive:
+	@ $(MAKE) DOCKER_OPTS=-i DOCKER_CMD=bash ci-internal
 
-## -- End-to-end tests job
-
-ci-e2e:
-	@ docker run --rm -t \
+ci-internal: ci-build-image
+	@ docker run --rm -t $(DOCKER_OPTS) \
 		-v /var/run/docker.sock:/var/run/docker.sock \
 		-v $(ROOT_DIR):$(GO_MOUNT_PATH) \
 		-w $(GO_MOUNT_PATH) \
-		-e "IMG_SUFFIX=-ci" \
-		-e "GCLOUD_PROJECT=$(GCLOUD_PROJECT)" \
-		-e "REGISTRY=$(REGISTRY)" \
-		-e "REPOSITORY=$(GCLOUD_PROJECT)" \
-		-e "TESTS_MATCH=$(TESTS_MATCH)" \
-		-e "SKIP_DOCKER_COMMAND=$(SKIP_DOCKER_COMMAND)" \
-		-e "OPERATOR_IMAGE=$(OPERATOR_IMAGE)" \
-		-e "STACK_VERSION=$(STACK_VERSION)" \
 		$(CI_IMAGE) \
-		bash -c "make ci-e2e"
+		bash -c "$(DOCKER_CMD)"
 
-ci-run-deployer:
-	@ docker run --rm -t \
-		-v /var/run/docker.sock:/var/run/docker.sock \
-		-v $(ROOT_DIR):$(GO_MOUNT_PATH) \
-		-w $(GO_MOUNT_PATH) \
-		-e "GCLOUD_PROJECT=$(GCLOUD_PROJECT)" \
-		$(CI_IMAGE) \
-		bash -c "make run-deployer"
 
-# Check if Docker image exists by trying to pull it. If there is no image, then build and push it.
-ci-build-image: vault-docker-creds
+# reads Docker password from Vault,
+# checks if Docker image exists by trying to pull it. If there is no image, then build and push it.
+ci-build-image:
 	@ docker pull $(CI_IMAGE) || (docker build -f $(ROOT_DIR)/build/ci/Dockerfile -t push.$(CI_IMAGE) \
-	--label "commit.hash=$(shell git rev-parse --short --verify HEAD)" $(ROOT_DIR) &&\
-	docker login -u $(DOCKER_LOGIN) -p $(shell cat $(DOCKER_CREDENTIALS_FILE)) push.docker.elastic.co &&\
-	docker push push.$(CI_IMAGE))
+		--label "commit.hash=$(shell git rev-parse --short --verify HEAD)" $(ROOT_DIR) && docker login -u eckadmin \
+		-p $(shell VAULT_TOKEN=$(VAULT_TOKEN) vault read -address=$(VAULT_ADDR) -field=value secret/devops-ci/cloud-on-k8s/eckadmin) \
+		push.docker.elastic.co && docker push push.$(CI_IMAGE))
+
+VAULT_AWS_CREDS = secret/cloud-team/cloud-ci/eck-release
+AWS_ACCESS_KEY_ID = $(shell VAULT_TOKEN=$(VAULT_TOKEN) vault read -address=$(VAULT_ADDR) -field=access-key-id $(VAULT_AWS_CREDS))
+AWS_SECRET_ACCESS_KEY = $(shell VAULT_TOKEN=$(VAULT_TOKEN) vault read -address=$(VAULT_ADDR) -field=secret-access-key $(VAULT_AWS_CREDS))
+# reads AWS creds for yaml upload to https://download.elastic.co/downloads/eck/$TAG_NAME/all-in-one.yaml
+yaml-upload:
+	@ $(MAKE) \
+		DOCKER_OPTS="-e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) -e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY)" \
+		DOCKER_CMD="aws s3 cp $(GO_MOUNT_PATH)/config/all-in-one.yaml \
+		s3://download.elasticsearch.org/downloads/eck/$(TAG_NAME)/all-in-one.yaml" ci-internal
+
+# reads Elastic public key from Vault into license.key
+get-elastic-public-key:
+	@ VAULT_TOKEN=$(VAULT_TOKEN) vault read -address=$(VAULT_ADDR) -field=pubkey secret/release/license | base64 --decode > license.key

build/ci/README.md

@@ -0,0 +1,71 @@
+# Continuous integration
+
+### Structure
+
+We are using Jenkins as CI runner and keep its configuration as code in the repo. The address of the instance we use is https://devops-ci.elastic.co/view/cloud-on-k8s/.
+
+There are few layers in most of our jobs:
+
+1. [Job definition](../../.ci/jobs) - description of the job.
+2. Jenkinsfile (e.g.: [e2e/Jenkinsfile](e2e/Jenkinsfile)) - loads vault credentials, sets up configuration. 
+3. [CI makefile](Makefile) - creates container to run CI in, consolidates dev and CI setups.
+4. [dev makefile](../../Makefile) - contains logic, delegates to specific tools as needed.
+5. tools - e.g. for [e2e test running](../../test/e2e) and [cluster provisioning](../../hack/deployer).
+
+### Local repro
+
+For debugging and development purposes it's possible to run CI jobs from dev box. It requires minimal setup and it mirrors CI closely, starting at CI makefile layer.
+
+Once, run:
+```
+export BUILD_TAG=local-ci-$(USER//_)
+
+# fill out:
+export GCLOUD_PROJECT=YOUR_GCLOUD_PROJECT
+export VAULT_ADDR=YOUR_VAULT_INSTANCE_ADDRESS
+export GITHUB_TOKEN=YOUR_PERSONAL_ACCESS_TOKEN
+``` 
+
+Per repro, depending on the job, set up .env and run-config.yml files. E.g.: to repro e2e tests run, look at its [Jenkinsfile](e2e/Jenkinsfile) and rerun the script locally in repo root: 
+```
+cat >.env <<EOF
+GCLOUD_PROJECT = "$GCLOUD_PROJECT"
+REGISTRY = eu.gcr.io
+REPOSITORY = "$GCLOUD_PROJECT"
+SKIP_DOCKER_COMMAND = false
+IMG_SUFFIX = -ci
+EOF
+
+cat >run-config.yml <<EOF
+id: gke-ci
+overrides:
+  kubernetesVersion: "1.12"
+  clusterName: $BUILD_TAG
+  vaultInfo:
+    address: $VAULT_ADDR
+    roleId: $VAULT_ROLE_ID
+    secretId: $VAULT_SECRET_ID
+  gke:
+    gCloudProject: $GCLOUD_PROJECT
+EOF
+
+make -C build/ci COMMAND=ci-e2e ci
+```
+
+CI makefile will take care of setting up correct credentials in the .env and run-config.yml file.
+
+This will run e2e test using the same:
+1. container
+1. credentials
+1. settings
+1. call path
+
+as the CI job.
+
+### CI container
+
+You can build and run CI container interactively with:
+
+```
+make -C build/ci ci-interactive
+```