From 279f4f575449a6968cdb5979a25be35969b809f4 Mon Sep 17 00:00:00 2001 From: Samuel Lang Date: Fri, 14 Aug 2020 12:57:32 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=91=B7=20Add=20high-level=20E2E=20test=20?= =?UTF-8?q?steps=20for=20Skipper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add e2e-skipper* files for test setup It does the following things: * install Skipper ingress with Kustomize * load Flagger image onto the local cluster * install Flagger and Prometheus in the flagger-system namespace --- .circleci/config.yml | 2 +- kustomize/skipper/kustomization.yaml | 67 ++++++++++++++++++++++ kustomize/skipper/namespace.yaml | 4 ++ kustomize/skipper/patch.yaml | 13 +++++ test/e2e-skipper-canary.yaml | 66 ++++++++++++++++++++++ test/e2e-skipper-cleanup.sh | 14 +++++ test/e2e-skipper-tests.sh | 83 ++++++++++++++++++++++++++++ test/e2e-skipper.sh | 22 ++++++++ test/e2e-workload-ingress.yaml | 17 ++++++ 9 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 kustomize/skipper/kustomization.yaml create mode 100644 kustomize/skipper/namespace.yaml create mode 100644 kustomize/skipper/patch.yaml create mode 100644 test/e2e-skipper-canary.yaml create mode 100755 test/e2e-skipper-cleanup.sh create mode 100755 test/e2e-skipper-tests.sh create mode 100755 test/e2e-skipper.sh create mode 100644 test/e2e-workload-ingress.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml index c98be8f10..09c6a5982 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -153,7 +153,7 @@ jobs: - run: test/e2e-kind.sh - run: test/e2e-skipper.sh - run: test/e2e-skipper-tests.sh - - run: test/e2e-nginx-cleanup.sh + - run: test/e2e-skipper-cleanup.sh push-helm-charts: docker: diff --git a/kustomize/skipper/kustomization.yaml b/kustomize/skipper/kustomization.yaml new file mode 100644 index 000000000..cc609c449 --- /dev/null +++ b/kustomize/skipper/kustomization.yaml @@ -0,0 +1,67 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../base/prometheus/ + - ../base/flagger/ + - https://raw.githubusercontent.com/zalando/skipper/v0.11.140/docs/kubernetes/deploy/deployment/rbac.yaml + - https://raw.githubusercontent.com/zalando/skipper/v0.11.140/docs/kubernetes/deploy/deployment/service.yaml + - https://raw.githubusercontent.com/zalando/skipper/v0.11.140/docs/kubernetes/deploy/deployment/deployment.yaml + - namespace.yaml +patchesStrategicMerge: + - patch.yaml +patches: + - target: + kind: Deployment + name: skipper-ingress + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: skipper-ingress + spec: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "9911" + prometheus.io/scrape: "true" + spec: + nodeSelector: + $patch: delete + affinity: + $patch: delete + containers: + - name: skipper-ingress + image: registry.opensource.zalan.do/pathfinder/skipper:latest + ports: + - name: metrics-port + containerPort: 9911 + resources: + $patch: delete + readinessProbe: + initialDelaySeconds: 5 + args: + - skipper + - -default-filters-prepend=enableAccessLog(4,5) + - -address=:9999 + - -disable-metrics-compat + - -enable-connection-metrics + - -enable-profile + - -enable-ratelimits + - -experimental-upgrade + - -histogram-metric-buckets=.01,1,10,100 + - -kubernetes-in-cluster + - -kubernetes-path-mode=path-prefix + - -kubernetes + - -lb-healthcheck-interval=3s + - -max-audit-body=0 + - -max-idle-connection-backend=0 + - -metrics-exp-decay-sample + - -metrics-flavour=prometheus + - -proxy-preserve-host + - -route-backend-metrics + - -route-backend-error-counters + - -route-response-metrics + - -serve-host-metrics + - -serve-route-metrics + - -whitelisted-healthcheck-cidr=0.0.0.0/0 # kind uses other IP addresse diff --git a/kustomize/skipper/namespace.yaml b/kustomize/skipper/namespace.yaml new file mode 100644 index 000000000..1f7400c3b --- /dev/null +++ b/kustomize/skipper/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: flagger-system diff --git a/kustomize/skipper/patch.yaml b/kustomize/skipper/patch.yaml new file mode 100644 index 000000000..fa89e85a7 --- /dev/null +++ b/kustomize/skipper/patch.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flagger +spec: + template: + spec: + containers: + - name: flagger + args: + - -log-level=debug + - -mesh-provider=kubernetes + - -metrics-server=http://flagger-prometheus:9090 diff --git a/test/e2e-skipper-canary.yaml b/test/e2e-skipper-canary.yaml new file mode 100644 index 000000000..7e0965e45 --- /dev/null +++ b/test/e2e-skipper-canary.yaml @@ -0,0 +1,66 @@ +apiVersion: flagger.app/v1beta1 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + provider: skipper + progressDeadlineSeconds: 120 + revertOnDeletion: true + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + ingressRef: + apiVersion: networking.k8s.io/v1beta1 + kind: Ingress + name: podinfo-ingress + service: + # service name (defaults to targetRef.name) + name: podinfo-service + # ClusterIP port number + port: 80 + # container port name or number (optional) + targetPort: http + # port name can be http or grpc (default http) + # portName: http + # add all the other container ports + # to the ClusterIP services (default false) + # portDiscovery: false + analysis: + interval: 15s + threshold: 5 + maxWeight: 100 + stepWeight: 10 + metrics: + - name: request-success-rate + interval: 15s + # minimum req success rate (non 5xx responses) + # percentage (0-100) + thresholdRange: + min: 99 + - name: request-duration + interval: 15s + # maximum req duration P99 + # milliseconds + thresholdRange: + max: 500 + webhooks: + - name: gate + type: confirm-rollout + url: http://flagger-loadtester.test/gate/approve + - name: acceptance-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 10s + metadata: + type: bash + cmd: "curl -sd 'test' http://podinfo-service-canary/token | grep token" + - name: "load test" + type: rollout + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 10m -q 10 -c 2 -host app.example.com http://skipper-ingress.kube-system" + logCmdOutput: "true" diff --git a/test/e2e-skipper-cleanup.sh b/test/e2e-skipper-cleanup.sh new file mode 100755 index 000000000..c1678f70d --- /dev/null +++ b/test/e2e-skipper-cleanup.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo '>>> Deleting Skipper Ingress' +kustomize build test/skipper/ | kubectl delete -f - + +echo '>>> Deleting Flagger' +kubectl delete -k ${REPO_ROOT}/kustomize/kubernetes + +echo '>>> Cleanup test namespace' +kubectl delete namespace test --ignore-not-found=true diff --git a/test/e2e-skipper-tests.sh b/test/e2e-skipper-tests.sh new file mode 100755 index 000000000..2af94271f --- /dev/null +++ b/test/e2e-skipper-tests.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary initialization, analysis and promotion +# Prerequisites: Kubernetes Kind and Skipper ingress controller + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo '>>> Creating test namespace' +kubectl create namespace test || true + +echo '>>> Initialising workload' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml +kubectl apply -f ${REPO_ROOT}/test/e2e-workload-ingress.yaml + +echo '>>> Installing load tester' +kubectl apply -k ${REPO_ROOT}/kustomize/tester +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Create canary CRD' +kubectl apply -f ${REPO_ROOT}/test/e2e-skipper-canary.yaml +echo '>>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +failed=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Failed' && failed=true || failed=false + if ${failed}; then + kubectl -n flagger-system logs deployment/flagger + echo "Canary failed!" + exit 1 + fi + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n flagger-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' diff --git a/test/e2e-skipper.sh b/test/e2e-skipper.sh new file mode 100755 index 000000000..3bf88c317 --- /dev/null +++ b/test/e2e-skipper.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo '>>> Loading Flagger image' +kind load docker-image test/flagger:latest + +echo '>>> Installing Skipper Ingress, Flagger and Prometheus' +# use kustomize to avoid compatibility issues: +# https://github.com/kubernetes-sigs/kustomize/issues/2390 +# Skipper will throw an Prometheus warning which can be ignored: +# https://github.com/weaveworks/flagger/issues/664 +kustomize build ${REPO_ROOT}/kustomize/skipper | kubectl apply -f - + +kubectl rollout status deployment/skipper-ingress -n kube-system +kubectl rollout status deployment/flagger-prometheus -n flagger-system + +kubectl -n flagger-system set image deployment/flagger flagger=test/flagger:latest + +kubectl -n flagger-system rollout status deployment/flagger diff --git a/test/e2e-workload-ingress.yaml b/test/e2e-workload-ingress.yaml new file mode 100644 index 000000000..b7a2c619b --- /dev/null +++ b/test/e2e-workload-ingress.yaml @@ -0,0 +1,17 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: podinfo-ingress + namespace: test + labels: + app: podinfo + annotations: + kubernetes.io/ingress.class: skipper +spec: + rules: + - host: app.example.com + http: + paths: + - backend: + serviceName: podinfo-service + servicePort: 80