Implement helm and bash pre-rollout hooks

Dockerfile.loadtester

@@ -21,6 +21,10 @@ WORKDIR /home/app
 RUN curl -sSLo hey "https://storage.googleapis.com/jblabs/dist/hey_linux_v0.1.2" && \
 chmod +x hey && mv hey /usr/local/bin/hey
 
+RUN curl -sSL "https://get.helm.sh/helm-v2.12.3-linux-amd64.tar.gz" | tar xvz && \
+chmod +x linux-amd64/helm && mv linux-amd64/helm /usr/local/bin/helm && \
+rm -rf linux-amd64
+
 COPY --from=builder /go/src/github.com/weaveworks/flagger/loadtester .
 
 RUN chown -R app:app ./

Makefile

@@ -7,7 +7,7 @@ LT_VERSION?=$(shell grep 'VERSION' cmd/loadtester/main.go | awk '{ print $$4 }'
 TS=$(shell date +%Y-%m-%d_%H-%M-%S)
 
 run:
-	go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info \
+	go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=istio -namespace=test \
 	-metrics-server=https://prometheus.istio.weavedx.com \
 	-slack-url=https://hooks.slack.com/services/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 \
 	-slack-channel="devops-alerts"
@@ -106,6 +106,11 @@ reset-test:
 	kubectl apply -f ./artifacts/namespaces
 	kubectl apply -f ./artifacts/canaries
 
+loadtester-run:
+	docker build -t weaveworks/flagger-loadtester:$(LT_VERSION) . -f Dockerfile.loadtester
+	docker rm -f tester || true
+	docker run -dp 8888:9090 --name tester weaveworks/flagger-loadtester:$(LT_VERSION)
+
 loadtester-push:
 	docker build -t weaveworks/flagger-loadtester:$(LT_VERSION) . -f Dockerfile.loadtester
-	docker push weaveworks/flagger-loadtester:$(LT_VERSION)
+	docker push weaveworks/flagger-loadtester:$(LT_VERSION)

artifacts/canaries/canary.yaml

@@ -73,5 +73,5 @@ spec:
         timeout: 5s
         metadata:
           type: cmd
-          cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/"
+          cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898/"
           logCmdOutput: "true"

artifacts/helmtester/deployment.yaml

@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flagger-helmtester
+  namespace: kube-system
+  labels:
+    app: flagger-helmtester
+spec:
+  selector:
+    matchLabels:
+      app: flagger-helmtester
+  template:
+    metadata:
+      labels:
+        app: flagger-helmtester
+      annotations:
+        prometheus.io/scrape: "true"
+    spec:
+      serviceAccountName: tiller
+      containers:
+        - name: helmtester
+          image: weaveworks/flagger-loadtester:0.4.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 8080
+          command:
+            - ./loadtester
+            - -port=8080
+            - -log-level=info
+            - -timeout=1h
+          livenessProbe:
+            exec:
+              command:
+                - wget
+                - --quiet
+                - --tries=1
+                - --timeout=4
+                - --spider
+                - http://localhost:8080/healthz
+            timeoutSeconds: 5
+          readinessProbe:
+            exec:
+              command:
+                - wget
+                - --quiet
+                - --tries=1
+                - --timeout=4
+                - --spider
+                - http://localhost:8080/healthz
+            timeoutSeconds: 5
+          resources:
+            limits:
+              memory: "512Mi"
+              cpu: "1000m"
+            requests:
+              memory: "32Mi"
+              cpu: "10m"

artifacts/helmtester/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: flagger-helmtester
+  namespace: kube-system
+  labels:
+    app: flagger-helmtester
+spec:
+  type: ClusterIP
+  selector:
+    app: flagger-helmtester
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: http

charts/loadtester/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: loadtester
-version: 0.4.0
-appVersion: 0.3.0
+version: 0.4.1
+appVersion: 0.4.0
 kubeVersion: ">=1.11.0-0"
 engine: gotpl
 description: Flagger's load testing services based on rakyll/hey that generates traffic during canary analysis when configured as a webhook.

charts/loadtester/README.md

@@ -7,7 +7,6 @@ and can be used to generates traffic during canary analysis when configured as a
 ## Prerequisites
 
 * Kubernetes >= 1.11
-* Istio >= 1.0
 
 ## Installing the Chart
 
@@ -25,7 +24,7 @@ helm upgrade -i flagger-loadtester flagger/loadtester
 
 The command deploys Grafana on the Kubernetes cluster in the default namespace.
 
-> **Tip**: Note that the namespace where you deploy the load tester should have the Istio sidecar injection enabled
+> **Tip**: Note that the namespace where you deploy the load tester should have the Istio or App Mesh sidecar injection enabled
 
 The [configuration](#configuration) section lists the parameters that can be configured during installation.
 
@@ -48,13 +47,14 @@ Parameter | Description | Default
 `image.repository` | Image repository | `quay.io/stefanprodan/flagger-loadtester`
 `image.pullPolicy` | Image pull policy | `IfNotPresent`
 `image.tag` | Image tag | `<VERSION>`
-`replicaCount` | desired number of pods | `1`
+`replicaCount` | Desired number of pods | `1`
+`serviceAccountName` | Kubernetes service account name | `none` 
 `resources.requests.cpu` | CPU requests | `10m`
-`resources.requests.memory` | memory requests | `64Mi`
+`resources.requests.memory` | Memory requests | `64Mi`
 `tolerations` | List of node taints to tolerate | `[]`
 `affinity` | node/pod affinities | `node`
-`nodeSelector` | node labels for pod assignment | `{}`
-`service.type` | type of service | `ClusterIP`
+`nodeSelector` | Node labels for pod assignment | `{}`
+`service.type` | Type of service | `ClusterIP`
 `service.port` | ClusterIP port | `80`
 `cmd.timeout` | Command execution timeout | `1h`
 `logLevel` | Log level can be debug, info, warning, error or panic | `info`

charts/loadtester/templates/deployment.yaml

@@ -19,6 +19,9 @@ spec:
       annotations:
         appmesh.k8s.aws/ports: "444"
     spec:
+      {{- if .Values.serviceAccountName }}
+      serviceAccountName: {{ .Values.serviceAccountName }}
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"

charts/loadtester/values.yaml

@@ -2,7 +2,7 @@ replicaCount: 1
 
 image:
   repository: weaveworks/flagger-loadtester
-  tag: 0.3.0
+  tag: 0.4.0
   pullPolicy: IfNotPresent
 
 logLevel: info
@@ -27,6 +27,8 @@ tolerations: []
 
 affinity: {}
 
+serviceAccountName: ""
+
 # App Mesh virtual node settings
 meshName: ""
 #backends:

charts/podinfo/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
-version: 2.0.1
+version: 2.1.0
 appVersion: 1.4.0
 name: podinfo
 engine: gotpl

charts/podinfo/templates/canary.yaml

@@ -38,8 +38,17 @@ spec:
     - name: request-duration
       threshold: {{ .Values.canary.thresholds.latency }}
       interval: 1m
-    {{- if .Values.canary.loadtest.enabled }}
     webhooks:
+      {{- if .Values.canary.helmtest.enabled }}
+      - name: "helm test"
+        type: pre-rollout
+        url: {{ .Values.canary.helmtest.url }}
+        timeout: 3m
+        metadata:
+          type: "helm"
+          cmd: "test {{ .Release.Name }} --cleanup"
+      {{- end }}
+      {{- if .Values.canary.loadtest.enabled }}
       - name: load-test-get
         url: {{ .Values.canary.loadtest.url }}
         timeout: 5s
@@ -50,5 +59,5 @@ spec:
         timeout: 5s
         metadata:
           cmd: "hey -z 1m -q 5 -c 2 -m POST -d '{\"test\": true}' http://{{ template "podinfo.fullname" . }}.{{ .Release.Namespace }}:{{ .Values.service.port }}/echo"
-    {{- end }}
+      {{- end }}
 {{- end }}

charts/podinfo/values.yaml

@@ -45,6 +45,10 @@ canary:
     enabled: false
     # load tester address
     url: http://flagger-loadtester.test/
+  helmtest:
+    enabled: false
+    # helm tester address
+    url: http://flagger-helmtester.kube-system/
 
 resources:
   limits:

cmd/loadtester/main.go

@@ -10,7 +10,7 @@ import (
 	"time"
 )
 
-var VERSION = "0.3.0"
+var VERSION = "0.4.0"
 var (
 	logLevel          string
 	port              string

docs/gitbook/how-it-works.md

@@ -593,11 +593,11 @@ Spec:
     webhooks:
       - name: "smoke test"
         type: pre-rollout
-        url: http://migration-check.db/query
-        timeout: 30s
+        url: http://flagger-helmtester.kube-system/
+        timeout: 3m
         metadata:
-          key1: "val1"
-          key2: "val2"
+          type: "helm"
+          cmd: "test podinfo --cleanup"
       - name: "load test"
         type: rollout
         url: http://flagger-loadtester.test/
@@ -640,7 +640,7 @@ On a non-2xx response Flagger will include the response body (if any) in the fai
 For workloads that are not receiving constant traffic Flagger can be configured with a webhook, 
 that when called, will start a load test for the target workload.
 If the target workload doesn't receive any traffic during the canary analysis, 
-Flagger metric checks will fail with "no values found for metric istio_requests_total".
+Flagger metric checks will fail with "no values found for metric request-success-rate".
 
 Flagger comes with a load testing service based on [rakyll/hey](https://github.com/rakyll/hey) 
 that generates traffic during analysis when configured as a webhook.
@@ -677,18 +677,18 @@ webhooks:
     timeout: 5s
     metadata:
       type: cmd
-      cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/"
+      cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898/"
   - name: load-test-post
     url: http://flagger-loadtester.test/
     timeout: 5s
     metadata:
       type: cmd
-      cmd: "hey -z 1m -q 10 -c 2 -m POST -d '{test: 2}' http://podinfo.test:9898/echo"
+      cmd: "hey -z 1m -q 10 -c 2 -m POST -d '{test: 2}' http://podinfo-canary.test:9898/echo"
 ```
 
 When the canary analysis starts, Flagger will call the webhooks and the load tester will run the `hey` commands 
 in the background, if they are not already running. This will ensure that during the 
-analysis, the `podinfo.test` virtual service will receive a steady stream of GET and POST requests.
+analysis, the `podinfo-canary.test` service will receive a steady stream of GET and POST requests.
 
 If your workload is exposed outside the mesh with the Istio Gateway and TLS you can point `hey` to the 
 public URL and use HTTP2.
@@ -707,7 +707,7 @@ The load tester can run arbitrary commands as long as the binary is present in t
 For example if you you want to replace `hey` with another CLI, you can create your own Docker image:
 
 ```dockerfile
-FROM quay.io/stefanprodan/flagger-loadtester:<VER>
+FROM weaveworks/flagger-loadtester:<VER>
 
 RUN curl -Lo /usr/local/bin/my-cli https://github.com/user/repo/releases/download/ver/my-cli \
     && chmod +x /usr/local/bin/my-cli
@@ -741,3 +741,52 @@ webhooks:
 When the canary analysis starts, the load tester will initiate a [clone_and_start request](https://github.com/naver/ngrinder/wiki/REST-API-PerfTest)
 to the nGrinder server and start a new performance test. the load tester will periodically poll the nGrinder server
 for the status of the test, and prevent duplicate requests from being sent in subsequent analysis loops.
+
+### Integration Testing
+
+Flagger comes with a testing service that can run Helm tests or Bats tests when configured as a webhook.
+
+Deploy the Helm test runner in the `kube-system` namespace using the `tiller` service account:
+
+```bash
+helm repo add flagger https://flagger.app
+
+helm upgrade -i flagger-helmtester flagger/loadtester \
+--namespace=kube-system \
+--set serviceAccountName=tiller
+```
+
+When deployed the Helm tester API will be available at `http://flagger-helmtester.kube-system/`. 
+
+Now you can add pre-rollout webhooks to the canary analysis spec:
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "smoke test"
+        type: pre-rollout
+        url: http://flagger-helmtester.kube-system/
+        timeout: 3m
+        metadata:
+          type: "helm"
+          cmd: "test {{ .Release.Name }} --cleanup"
+```
+
+When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary.
+If the helm test fails, Flagger will retry until the analysis threshold is reached and the canary is rolled back.
+
+As an alternative to Helm you can use the [Bash Automated Testing System](https://github.com/bats-core/bats-core) to run your tests. 
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "acceptance tests"
+        type: pre-rollout
+        url: http://flagger-batstester.default/
+        timeout: 5m
+        metadata:
+          type: "bash"
+          cmd: "bats /tests/acceptance.bats"
+```
+
+Note that you should create a ConfigMap with your Bats tests and mount it inside the tester container.