From dfdcfed26eb3df6c1330faad1514d2006596f54e Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 12:00:04 +0300 Subject: [PATCH 01/15] Add `Waiting` canary status phase means the canary rollout is paused (waiting for confirmation to proceed) --- artifacts/flagger/crd.yaml | 1 + charts/flagger/templates/crd.yaml | 1 + kustomize/base/flagger/crd.yaml | 1 + pkg/apis/flagger/v1alpha3/status.go | 2 ++ pkg/canary/status.go | 3 +++ 5 files changed, 8 insertions(+) diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index 1b2355774..ed54fbfeb 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -174,6 +174,7 @@ spec: - "" - Initializing - Initialized + - Waiting - Progressing - Succeeded - Failed diff --git a/charts/flagger/templates/crd.yaml b/charts/flagger/templates/crd.yaml index fcf326f1b..e7cd7641b 100644 --- a/charts/flagger/templates/crd.yaml +++ b/charts/flagger/templates/crd.yaml @@ -175,6 +175,7 @@ spec: - "" - Initializing - Initialized + - Waiting - Progressing - Succeeded - Failed diff --git a/kustomize/base/flagger/crd.yaml b/kustomize/base/flagger/crd.yaml index 1b2355774..ed54fbfeb 100644 --- a/kustomize/base/flagger/crd.yaml +++ b/kustomize/base/flagger/crd.yaml @@ -174,6 +174,7 @@ spec: - "" - Initializing - Initialized + - Waiting - Progressing - Succeeded - Failed diff --git a/pkg/apis/flagger/v1alpha3/status.go b/pkg/apis/flagger/v1alpha3/status.go index 5c43b0fd8..b0ca1fd9d 100644 --- a/pkg/apis/flagger/v1alpha3/status.go +++ b/pkg/apis/flagger/v1alpha3/status.go @@ -43,6 +43,8 @@ const ( // CanaryPhaseInitialized means the primary deployment, hpa and ClusterIP services // have been created along with the service mesh or ingress objects CanaryPhaseInitialized CanaryPhase = "Initialized" + // CanaryPhaseWaiting means the canary rollout is paused (waiting for confirmation to proceed) + CanaryPhaseWaiting CanaryPhase = "Waiting" // CanaryPhaseProgressing means the canary analysis is underway CanaryPhaseProgressing CanaryPhase = "Progressing" // CanaryPhaseSucceeded means the canary analysis has been successful diff --git a/pkg/canary/status.go b/pkg/canary/status.go index 2001470c3..cfbacd9cc 100644 --- a/pkg/canary/status.go +++ b/pkg/canary/status.go @@ -205,6 +205,9 @@ func (c *Deployer) MakeStatusConditions(canaryStatus flaggerv1.CanaryStatus, case flaggerv1.CanaryPhaseInitialized: status = corev1.ConditionTrue message = "Deployment initialization completed." + case flaggerv1.CanaryPhaseWaiting: + status = corev1.ConditionUnknown + message = "Waiting for approval." case flaggerv1.CanaryPhaseProgressing: status = corev1.ConditionUnknown message = "New revision detected, starting canary analysis." From 919dafa5671f65b251828f49dd096747a1aae5ff Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 12:02:44 +0300 Subject: [PATCH 02/15] Add gate halt and approve endpoints --- artifacts/loadtester/deployment.yaml | 2 +- charts/loadtester/Chart.yaml | 4 ++-- charts/loadtester/values.yaml | 2 +- cmd/loadtester/main.go | 2 +- kustomize/tester/deployment.yaml | 2 +- pkg/loadtester/server.go | 8 ++++++++ 6 files changed, 14 insertions(+), 6 deletions(-) diff --git a/artifacts/loadtester/deployment.yaml b/artifacts/loadtester/deployment.yaml index c5f1004d1..336155155 100644 --- a/artifacts/loadtester/deployment.yaml +++ b/artifacts/loadtester/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: loadtester - image: weaveworks/flagger-loadtester:0.5.0 + image: weaveworks/flagger-loadtester:0.6.0 imagePullPolicy: IfNotPresent ports: - name: http diff --git a/charts/loadtester/Chart.yaml b/charts/loadtester/Chart.yaml index 53f5c2325..3e651a045 100644 --- a/charts/loadtester/Chart.yaml +++ b/charts/loadtester/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: loadtester -version: 0.5.0 -appVersion: 0.5.0 +version: 0.6.0 +appVersion: 0.6.0 kubeVersion: ">=1.11.0-0" engine: gotpl description: Flagger's load testing services based on rakyll/hey and bojand/ghz that generates traffic during canary analysis when configured as a webhook. diff --git a/charts/loadtester/values.yaml b/charts/loadtester/values.yaml index 486ab92bc..3a03a5cc3 100644 --- a/charts/loadtester/values.yaml +++ b/charts/loadtester/values.yaml @@ -2,7 +2,7 @@ replicaCount: 1 image: repository: weaveworks/flagger-loadtester - tag: 0.4.0 + tag: 0.6.0 pullPolicy: IfNotPresent logLevel: info diff --git a/cmd/loadtester/main.go b/cmd/loadtester/main.go index a042f3b76..4e29ffa4f 100644 --- a/cmd/loadtester/main.go +++ b/cmd/loadtester/main.go @@ -10,7 +10,7 @@ import ( "time" ) -var VERSION = "0.5.0" +var VERSION = "0.6.0" var ( logLevel string port string diff --git a/kustomize/tester/deployment.yaml b/kustomize/tester/deployment.yaml index c5f1004d1..336155155 100644 --- a/kustomize/tester/deployment.yaml +++ b/kustomize/tester/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: loadtester - image: weaveworks/flagger-loadtester:0.5.0 + image: weaveworks/flagger-loadtester:0.6.0 imagePullPolicy: IfNotPresent ports: - name: http diff --git a/pkg/loadtester/server.go b/pkg/loadtester/server.go index c8ce4c1da..ca907756b 100644 --- a/pkg/loadtester/server.go +++ b/pkg/loadtester/server.go @@ -21,6 +21,14 @@ func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogge w.WriteHeader(http.StatusOK) w.Write([]byte("OK")) }) + mux.HandleFunc("/gate/approve", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) + }) + mux.HandleFunc("/gate/halt", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("Forbidden")) + }) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { From c46c7b9e2145f76e9576d54cbe512cedad04a9a6 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 12:04:05 +0300 Subject: [PATCH 03/15] Add canary status conditions to docs --- docs/gitbook/how-it-works.md | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index 439821c74..72bfb0b10 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -105,6 +105,53 @@ convention you can specify your label with the `-selector-labels` flag. The target deployment should expose a TCP port that will be used by Flagger to create the ClusterIP Service and the Istio Virtual Service. The container port from the target deployment should match the `service.port` value. +### Canary status + +Get the current status of canary deployments cluster wide: + +```bash +kubectl get canaries --all-namespaces + +NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME +test podinfo Progressing 15 2019-06-30T14:05:07Z +prod frontend Succeeded 0 2019-06-30T16:15:07Z +prod backend Failed 0 2019-06-30T17:05:07Z +``` + +The status condition reflects the last know state of the canary analysis: + +```bash +kubectl -n test get canary/podinfo -oyaml | awk '/status/,0' +``` + +A successful rollout status: + +```yaml +status: + canaryWeight: 0 + failedChecks: 0 + iterations: 0 + lastAppliedSpec: "14788816656920327485" + lastPromotedSpec: "14788816656920327485" + conditions: + - lastTransitionTime: "2019-07-10T08:23:18Z" + lastUpdateTime: "2019-07-10T08:23:18Z" + message: Canary analysis completed successfully, promotion finished. + reason: Succeeded + status: "True" + type: Promoted +``` + +The `Promoted` status condition can have one of the following reasons: Initialized, Waiting, Progressing, Succeeded or Failed. +A failed canary will have the promoted status set to `false`, +the reason to `failed` and the last applied spec will be different to the last promoted one. + +Wait for a successful rollout: + +```bash +kubectl wait canary/podinfo --for=condition=promoted +``` + ### Istio routing Flagger creates an Istio Virtual Service and Destination Rules based on the Canary service spec. From 04cbacb6e038e726ee47767e34906891c5fbd584 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 12:09:39 +0300 Subject: [PATCH 04/15] Implement confirm rollout gate and hook The confirm-rollout hooks are executed before the pre-rollout hooks. Flagger will halt the canary rollout until the confirm webhook returns HTTP status 200. --- pkg/apis/flagger/v1alpha3/types.go | 2 ++ pkg/controller/scheduler.go | 38 +++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/pkg/apis/flagger/v1alpha3/types.go b/pkg/apis/flagger/v1alpha3/types.go index 1f995e028..9cd7cea43 100755 --- a/pkg/apis/flagger/v1alpha3/types.go +++ b/pkg/apis/flagger/v1alpha3/types.go @@ -137,6 +137,8 @@ const ( PreRolloutHook HookType = "pre-rollout" // PreRolloutHook execute webhook after the canary analysis PostRolloutHook HookType = "post-rollout" + // ConfirmRolloutHook halt canary analysis until webhook returns HTTP 200 + ConfirmRolloutHook HookType = "confirm-rollout" ) // CanaryWebhook holds the reference to external checks used for canary analysis diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index de02d079e..f08ef22e1 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -134,6 +134,12 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh return } + isApproved := c.runConfirmRolloutHooks(cd) + + if !isApproved { + return + } + // set max weight default value to 100% maxWeight := 100 if cd.Spec.CanaryAnalysis.MaxWeight > 0 { @@ -454,7 +460,10 @@ func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router. } func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) { - if cd.Status.LastAppliedSpec == "" || cd.Status.Phase == flaggerv1.CanaryPhaseInitializing || cd.Status.Phase == flaggerv1.CanaryPhaseProgressing { + if cd.Status.LastAppliedSpec == "" || + cd.Status.Phase == flaggerv1.CanaryPhaseInitializing || + cd.Status.Phase == flaggerv1.CanaryPhaseProgressing || + cd.Status.Phase == flaggerv1.CanaryPhaseWaiting { return true, nil } @@ -523,6 +532,33 @@ func (c *Controller) hasCanaryRevisionChanged(cd *flaggerv1.Canary) bool { return false } +func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary) bool { + for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks { + if webhook.Type == flaggerv1.ConfirmRolloutHook { + err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook) + if err != nil { + if canary.Status.Phase != flaggerv1.CanaryPhaseWaiting { + c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for approval %s", + canary.Name, canary.Namespace, webhook.Name) + if err := c.deployer.SetStatusPhase(canary, flaggerv1.CanaryPhaseWaiting); err != nil { + c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err) + } + } + return false + } else { + if canary.Status.Phase == flaggerv1.CanaryPhaseWaiting { + if err := c.deployer.SetStatusPhase(canary, flaggerv1.CanaryPhaseProgressing); err != nil { + c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err) + return false + } + c.recordEventInfof(canary, "Confirm-rollout check %s passed", webhook.Name) + } + } + } + } + return true +} + func (c *Controller) runPreRolloutHooks(canary *flaggerv1.Canary) bool { for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks { if webhook.Type == flaggerv1.PreRolloutHook { From 75d49304f3d58a0289d6d0c9577868b6a6dc0369 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 12:17:11 +0300 Subject: [PATCH 05/15] Add confirm-rollout hook to docs --- docs/gitbook/how-it-works.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index 72bfb0b10..afd2ffab8 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -655,6 +655,8 @@ The canary analysis can be extended with webhooks. Flagger will call each webhoo determine from the response status code (HTTP 2xx) if the canary is failing or not. There are three types of hooks: +* Confirm-rollout hooks are executed before scaling up the canary deployment and ca be used for manual approval. +The rollout is paused until the hook returns a successful HTTP status code. * Pre-rollout hooks are executed before routing traffic to canary. The canary advancement is paused if a pre-rollout hook fails and if the number of failures reach the threshold the canary will be rollback. @@ -668,6 +670,9 @@ Spec: ```yaml canaryAnalysis: webhooks: + - name: "start gate" + type: confirm-rollout + url: http://flagger-loadtester.test/gate/approve - name: "smoke test" type: pre-rollout url: http://flagger-helmtester.kube-system/ From 28e7e89047218d703d706130bc6f445f6baa40f8 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 16:09:13 +0300 Subject: [PATCH 06/15] Pause or resume analysis on confirmation gate toggle --- pkg/canary/status.go | 2 +- pkg/controller/scheduler.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/canary/status.go b/pkg/canary/status.go index cfbacd9cc..a744d437f 100644 --- a/pkg/canary/status.go +++ b/pkg/canary/status.go @@ -156,7 +156,7 @@ func (c *Deployer) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPh cdCopy.Status.Phase = phase cdCopy.Status.LastTransitionTime = metav1.Now() - if phase != flaggerv1.CanaryPhaseProgressing { + if phase != flaggerv1.CanaryPhaseProgressing && phase != flaggerv1.CanaryPhaseWaiting { cdCopy.Status.CanaryWeight = 0 cdCopy.Status.Iterations = 0 } diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index f08ef22e1..bca276007 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -134,9 +134,8 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh return } - isApproved := c.runConfirmRolloutHooks(cd) - - if !isApproved { + // check gates + if isApproved := c.runConfirmRolloutHooks(cd); !isApproved { return } @@ -552,6 +551,7 @@ func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary) bool { return false } c.recordEventInfof(canary, "Confirm-rollout check %s passed", webhook.Name) + return false } } } From f204fe53f464b612c1cbe14ce1759265e527c209 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 16:14:22 +0300 Subject: [PATCH 07/15] Implement canary gating API with in-memory storage POST /gate/[check|open|close] --- cmd/loadtester/main.go | 4 +- pkg/loadtester/gate.go | 31 +++++++++++++++ pkg/loadtester/server.go | 82 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 pkg/loadtester/gate.go diff --git a/cmd/loadtester/main.go b/cmd/loadtester/main.go index 4e29ffa4f..74c4d09b2 100644 --- a/cmd/loadtester/main.go +++ b/cmd/loadtester/main.go @@ -47,5 +47,7 @@ func main() { go taskRunner.Start(100*time.Millisecond, stopCh) logger.Infof("Starting load tester v%s API on port %s", VERSION, port) - loadtester.ListenAndServe(port, time.Minute, logger, taskRunner, stopCh) + + gateStorage := loadtester.NewGateStorage("in-memory") + loadtester.ListenAndServe(port, time.Minute, logger, taskRunner, gateStorage, stopCh) } diff --git a/pkg/loadtester/gate.go b/pkg/loadtester/gate.go new file mode 100644 index 000000000..fdacdd7ec --- /dev/null +++ b/pkg/loadtester/gate.go @@ -0,0 +1,31 @@ +package loadtester + +import "sync" + +type GateStorage struct { + backend string + data *sync.Map +} + +func NewGateStorage(backend string) *GateStorage { + return &GateStorage{ + backend: backend, + data: new(sync.Map), + } +} + +func (gs *GateStorage) open(key string) { + gs.data.Store(key, true) +} + +func (gs *GateStorage) close(key string) { + gs.data.Store(key, false) +} + +func (gs *GateStorage) isOpen(key string) (locked bool) { + val, ok := gs.data.LoadOrStore(key, false) + if ok { + return val.(bool) + } + return +} diff --git a/pkg/loadtester/server.go b/pkg/loadtester/server.go index ca907756b..3ddc7903f 100644 --- a/pkg/loadtester/server.go +++ b/pkg/loadtester/server.go @@ -14,7 +14,7 @@ import ( ) // ListenAndServe starts a web server and waits for SIGTERM -func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogger, taskRunner *TaskRunner, stopCh <-chan struct{}) { +func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogger, taskRunner *TaskRunner, gate *GateStorage, stopCh <-chan struct{}) { mux := http.DefaultServeMux mux.Handle("/metrics", promhttp.Handler()) mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { @@ -29,6 +29,86 @@ func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogge w.WriteHeader(http.StatusForbidden) w.Write([]byte("Forbidden")) }) + mux.HandleFunc("/gate/check", func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + logger.Error("reading the request body failed", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + defer r.Body.Close() + + canary := &flaggerv1.CanaryWebhookPayload{} + err = json.Unmarshal(body, canary) + if err != nil { + logger.Error("decoding the request body failed", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + + canaryName := fmt.Sprintf("%s.%s", canary.Name, canary.Namespace) + approved := gate.isOpen(canaryName) + if approved { + w.WriteHeader(http.StatusOK) + w.Write([]byte("Approved")) + } else { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("Forbidden")) + } + + logger.Infof("%s gate check: approved %v", canaryName, approved) + }) + + mux.HandleFunc("/gate/open", func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + logger.Error("reading the request body failed", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + defer r.Body.Close() + + canary := &flaggerv1.CanaryWebhookPayload{} + err = json.Unmarshal(body, canary) + if err != nil { + logger.Error("decoding the request body failed", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + + canaryName := fmt.Sprintf("%s.%s", canary.Name, canary.Namespace) + gate.open(canaryName) + + w.WriteHeader(http.StatusAccepted) + + logger.Infof("%s gate opened", canaryName) + }) + + mux.HandleFunc("/gate/close", func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + logger.Error("reading the request body failed", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + defer r.Body.Close() + + canary := &flaggerv1.CanaryWebhookPayload{} + err = json.Unmarshal(body, canary) + if err != nil { + logger.Error("decoding the request body failed", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + + canaryName := fmt.Sprintf("%s.%s", canary.Name, canary.Namespace) + gate.close(canaryName) + + w.WriteHeader(http.StatusAccepted) + + logger.Infof("%s gate closed", canaryName) + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { From be22ff9951df8940fa23b934fd1b905a9d02d497 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 16:27:30 +0300 Subject: [PATCH 08/15] Bump load tester version --- artifacts/loadtester/deployment.yaml | 2 +- charts/loadtester/Chart.yaml | 2 +- charts/loadtester/values.yaml | 2 +- cmd/loadtester/main.go | 2 +- kustomize/tester/deployment.yaml | 2 +- kustomize/tester/kustomization.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/artifacts/loadtester/deployment.yaml b/artifacts/loadtester/deployment.yaml index 336155155..8e7a117f9 100644 --- a/artifacts/loadtester/deployment.yaml +++ b/artifacts/loadtester/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: loadtester - image: weaveworks/flagger-loadtester:0.6.0 + image: weaveworks/flagger-loadtester:0.6.1 imagePullPolicy: IfNotPresent ports: - name: http diff --git a/charts/loadtester/Chart.yaml b/charts/loadtester/Chart.yaml index 3e651a045..745d4b133 100644 --- a/charts/loadtester/Chart.yaml +++ b/charts/loadtester/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: loadtester version: 0.6.0 -appVersion: 0.6.0 +appVersion: 0.6.1 kubeVersion: ">=1.11.0-0" engine: gotpl description: Flagger's load testing services based on rakyll/hey and bojand/ghz that generates traffic during canary analysis when configured as a webhook. diff --git a/charts/loadtester/values.yaml b/charts/loadtester/values.yaml index 3a03a5cc3..36ff9bf12 100644 --- a/charts/loadtester/values.yaml +++ b/charts/loadtester/values.yaml @@ -2,7 +2,7 @@ replicaCount: 1 image: repository: weaveworks/flagger-loadtester - tag: 0.6.0 + tag: 0.6.1 pullPolicy: IfNotPresent logLevel: info diff --git a/cmd/loadtester/main.go b/cmd/loadtester/main.go index 74c4d09b2..7f7907672 100644 --- a/cmd/loadtester/main.go +++ b/cmd/loadtester/main.go @@ -10,7 +10,7 @@ import ( "time" ) -var VERSION = "0.6.0" +var VERSION = "0.6.1" var ( logLevel string port string diff --git a/kustomize/tester/deployment.yaml b/kustomize/tester/deployment.yaml index 336155155..8e7a117f9 100644 --- a/kustomize/tester/deployment.yaml +++ b/kustomize/tester/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: loadtester - image: weaveworks/flagger-loadtester:0.6.0 + image: weaveworks/flagger-loadtester:0.6.1 imagePullPolicy: IfNotPresent ports: - name: http diff --git a/kustomize/tester/kustomization.yaml b/kustomize/tester/kustomization.yaml index 1c9287958..2e66439d3 100644 --- a/kustomize/tester/kustomization.yaml +++ b/kustomize/tester/kustomization.yaml @@ -4,4 +4,4 @@ resources: - deployment.yaml images: - name: weaveworks/flagger-loadtester - newTag: 0.5.0 + newTag: 0.6.1 From bc4bdcdc1c5f7c232e32002cf059349399af8f98 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 19:21:41 +0300 Subject: [PATCH 09/15] Upgrade Gloo e2e to v0.17.6 --- test/e2e-gloo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-gloo.sh b/test/e2e-gloo.sh index c3d172277..05d5d0432 100755 --- a/test/e2e-gloo.sh +++ b/test/e2e-gloo.sh @@ -2,7 +2,7 @@ set -o errexit -GLOO_VER="0.14.2" +GLOO_VER="0.17.6" REPO_ROOT=$(git rev-parse --show-toplevel) export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" From c4d28e14fc55e98b9d0addce8bc6ebfbbd911ec3 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 19:35:02 +0300 Subject: [PATCH 10/15] Upgrade Gloo e2e to v0.17.5 --- test/e2e-gloo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-gloo.sh b/test/e2e-gloo.sh index 05d5d0432..d821ef937 100755 --- a/test/e2e-gloo.sh +++ b/test/e2e-gloo.sh @@ -2,7 +2,7 @@ set -o errexit -GLOO_VER="0.17.6" +GLOO_VER="0.17.5" REPO_ROOT=$(git rev-parse --show-toplevel) export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" From aa6a180bcc4b403f4ab86b6a54e1e74604a76db8 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 19:44:06 +0300 Subject: [PATCH 11/15] Remove Gloo NodePort from e2e tests --- test/e2e-gloo.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/e2e-gloo.sh b/test/e2e-gloo.sh index d821ef937..3f86b2bf8 100755 --- a/test/e2e-gloo.sh +++ b/test/e2e-gloo.sh @@ -9,9 +9,7 @@ export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" echo '>>> Installing Gloo' helm repo add gloo https://storage.googleapis.com/solo-public-helm helm upgrade -i gloo gloo/gloo --version ${GLOO_VER} \ ---wait \ ---namespace gloo-system \ ---set gatewayProxies.gateway-proxy.service.type=NodePort +--namespace gloo-system kubectl -n gloo-system rollout status deployment/gloo kubectl -n gloo-system rollout status deployment/gateway-proxy From 046245a8b5bb8df4d638996f531a6f38b26fa17b Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 24 Jul 2019 19:54:33 +0300 Subject: [PATCH 12/15] Use Gloo 0.17.6 in e2e tests --- test/e2e-gloo-tests.sh | 2 +- test/e2e-gloo.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e-gloo-tests.sh b/test/e2e-gloo-tests.sh index 6f9483677..891d9c53a 100755 --- a/test/e2e-gloo-tests.sh +++ b/test/e2e-gloo-tests.sh @@ -55,7 +55,7 @@ spec: canaryAnalysis: interval: 15s threshold: 15 - maxWeight: 30 + maxWeight: 50 stepWeight: 10 metrics: - name: request-success-rate diff --git a/test/e2e-gloo.sh b/test/e2e-gloo.sh index 3f86b2bf8..19fcf9fc7 100755 --- a/test/e2e-gloo.sh +++ b/test/e2e-gloo.sh @@ -2,7 +2,7 @@ set -o errexit -GLOO_VER="0.17.5" +GLOO_VER="0.17.6" REPO_ROOT=$(git rev-parse --show-toplevel) export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" From e07a82d02476af4580b48564e30535ef9d5981de Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 25 Jul 2019 13:29:33 +0300 Subject: [PATCH 13/15] Add manual gating to docs --- docs/gitbook/how-it-works.md | 52 ++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index afd2ffab8..bec6f796e 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -884,3 +884,55 @@ As an alternative to Helm you can use the [Bash Automated Testing System](https: ``` Note that you should create a ConfigMap with your Bats tests and mount it inside the tester container. + +### Manual Gating + +For manual approval of a canary deployment you can use the `confirm-rollout` webhook. +The confirmation hooks are executed before the pre-rollout hooks. +Flagger will halt the canary traffic shifting and analysis until the confirm webhook returns HTTP status 200. + +Manual gating with Flagger's tester: + +```yaml + canaryAnalysis: + webhooks: + - name: "gate" + type: confirm-rollout + url: http://flagger-loadtester.test/gate/halt +``` + +The `/gate/halt` returns HTTP 403 thus blocking the rollout. + +Change the URL to `/gate/approve` to start the canary analysis: + +```yaml + canaryAnalysis: + webhooks: + - name: "start gate" + type: confirm-rollout + url: http://flagger-loadtester.test/gate/approve +``` + +Manual gating can be driven with Flagger's tester API. Set the confirmation URL to `/gate/check`: + +```yaml + canaryAnalysis: + webhooks: + - name: "ask confirmation" + type: confirm-rollout + url: http://flagger-loadtester.test/gate/check +``` + +By default the gate is closed, you can start or resume the canary rollout with: + +```bash +kubectl -n test exec -it flagger-loadtester-xxxx-xxxx sh + +curl -d '{"name": "podinfo","namespace":"test"}' http://localhost:8080/gate/open +``` + +You can pause the rollout at any time with: + +```bash +curl -d '{"name": "podinfo","namespace":"test"}' http://localhost:8080/gate/close +``` From 163f5292b0a44b860f888b1a075e776fb00fa404 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 25 Jul 2019 19:10:46 +0300 Subject: [PATCH 14/15] Push a notification when a canary is waiting for approval --- docs/gitbook/how-it-works.md | 3 ++- pkg/controller/scheduler.go | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index bec6f796e..483b6226a 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -901,7 +901,8 @@ Manual gating with Flagger's tester: url: http://flagger-loadtester.test/gate/halt ``` -The `/gate/halt` returns HTTP 403 thus blocking the rollout. +The `/gate/halt` returns HTTP 403 thus blocking the rollout. +If you have notifications enabled, Flagger will post a message to Slack or MS Teams if a canary rollout is waiting for approval. Change the URL to `/gate/approve` to start the canary analysis: diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index bca276007..cf9a595b8 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -537,11 +537,12 @@ func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary) bool { err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook) if err != nil { if canary.Status.Phase != flaggerv1.CanaryPhaseWaiting { - c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for approval %s", - canary.Name, canary.Namespace, webhook.Name) if err := c.deployer.SetStatusPhase(canary, flaggerv1.CanaryPhaseWaiting); err != nil { c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err) } + c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for approval %s", + canary.Name, canary.Namespace, webhook.Name) + c.sendNotification(canary, "Canary is waiting for approval.", false, false) } return false } else { From e3b03debdee109158e945b20b351de75174723bb Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 26 Jul 2019 01:19:15 +0300 Subject: [PATCH 15/15] Use podinfo v1.7 --- artifacts/ab-testing/deployment.yaml | 2 +- artifacts/appmesh/deployment.yaml | 2 +- artifacts/canaries/deployment.yaml | 2 +- artifacts/gloo/deployment.yaml | 2 +- artifacts/nginx/deployment.yaml | 2 +- artifacts/workloads/canary-deployment.yaml | 2 +- artifacts/workloads/primary-deployment.yaml | 2 +- charts/podinfo/Chart.yaml | 4 ++-- charts/podinfo/values.yaml | 2 +- docs/gitbook/how-it-works.md | 10 ++++++++++ docs/gitbook/tutorials/canary-helm-gitops.md | 2 +- docs/gitbook/tutorials/flagger-smi-istio.md | 2 +- docs/gitbook/usage/ab-testing.md | 2 +- docs/gitbook/usage/appmesh-progressive-delivery.md | 2 +- docs/gitbook/usage/blue-green.md | 8 ++++---- docs/gitbook/usage/gloo-progressive-delivery.md | 2 +- docs/gitbook/usage/linkerd-progressive-delivery.md | 2 +- docs/gitbook/usage/nginx-progressive-delivery.md | 2 +- docs/gitbook/usage/progressive-delivery.md | 2 +- 19 files changed, 32 insertions(+), 22 deletions(-) diff --git a/artifacts/ab-testing/deployment.yaml b/artifacts/ab-testing/deployment.yaml index 373eb3bfb..66c0174e9 100644 --- a/artifacts/ab-testing/deployment.yaml +++ b/artifacts/ab-testing/deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.4.0 + image: quay.io/stefanprodan/podinfo:1.7.0 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/artifacts/appmesh/deployment.yaml b/artifacts/appmesh/deployment.yaml index beeae3265..6166cfaf1 100644 --- a/artifacts/appmesh/deployment.yaml +++ b/artifacts/appmesh/deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.4.0 + image: quay.io/stefanprodan/podinfo:1.7.0 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/artifacts/canaries/deployment.yaml b/artifacts/canaries/deployment.yaml index 57ed8a419..dacb34be2 100644 --- a/artifacts/canaries/deployment.yaml +++ b/artifacts/canaries/deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.4.0 + image: quay.io/stefanprodan/podinfo:1.7.0 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/artifacts/gloo/deployment.yaml b/artifacts/gloo/deployment.yaml index 57ed8a419..dacb34be2 100644 --- a/artifacts/gloo/deployment.yaml +++ b/artifacts/gloo/deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.4.0 + image: quay.io/stefanprodan/podinfo:1.7.0 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/artifacts/nginx/deployment.yaml b/artifacts/nginx/deployment.yaml index 814dd9c2f..4baf94242 100644 --- a/artifacts/nginx/deployment.yaml +++ b/artifacts/nginx/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.4.0 + image: quay.io/stefanprodan/podinfo:1.7.0 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/artifacts/workloads/canary-deployment.yaml b/artifacts/workloads/canary-deployment.yaml index 814dd9c2f..4baf94242 100644 --- a/artifacts/workloads/canary-deployment.yaml +++ b/artifacts/workloads/canary-deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.4.0 + image: quay.io/stefanprodan/podinfo:1.7.0 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/artifacts/workloads/primary-deployment.yaml b/artifacts/workloads/primary-deployment.yaml index cc17b82fa..6bc393a7d 100644 --- a/artifacts/workloads/primary-deployment.yaml +++ b/artifacts/workloads/primary-deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.4.1 + image: quay.io/stefanprodan/podinfo:1.7.1 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/charts/podinfo/Chart.yaml b/charts/podinfo/Chart.yaml index 075936f39..0c8d63db2 100644 --- a/charts/podinfo/Chart.yaml +++ b/charts/podinfo/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 -version: 2.2.0 -appVersion: 1.4.0 +version: 2.3.0 +appVersion: 1.7.0 name: podinfo engine: gotpl description: Flagger canary deployment demo chart diff --git a/charts/podinfo/values.yaml b/charts/podinfo/values.yaml index deff5c3d9..95eb9f3ee 100644 --- a/charts/podinfo/values.yaml +++ b/charts/podinfo/values.yaml @@ -1,7 +1,7 @@ # Default values for podinfo. image: repository: quay.io/stefanprodan/podinfo - tag: 1.4.0 + tag: 1.7.0 pullPolicy: IfNotPresent service: diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index 483b6226a..829a2c41f 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -902,6 +902,7 @@ Manual gating with Flagger's tester: ``` The `/gate/halt` returns HTTP 403 thus blocking the rollout. + If you have notifications enabled, Flagger will post a message to Slack or MS Teams if a canary rollout is waiting for approval. Change the URL to `/gate/approve` to start the canary analysis: @@ -937,3 +938,12 @@ You can pause the rollout at any time with: ```bash curl -d '{"name": "podinfo","namespace":"test"}' http://localhost:8080/gate/close ``` + +If a canary analysis is paused the status will change to waiting: + +```bash +kubectl get canary/podinfo + +NAME STATUS WEIGHT +podinfo Waiting 0 +``` diff --git a/docs/gitbook/tutorials/canary-helm-gitops.md b/docs/gitbook/tutorials/canary-helm-gitops.md index ca7f7f938..9633c8bff 100644 --- a/docs/gitbook/tutorials/canary-helm-gitops.md +++ b/docs/gitbook/tutorials/canary-helm-gitops.md @@ -344,7 +344,7 @@ launch the `frontend` and `backend` apps. A CI/CD pipeline for the `frontend` release could look like this: * cut a release from the master branch of the podinfo code repo with the git tag `1.4.1` -* CI builds the image and pushes the `podinfo:1.4.1` image to the container registry +* CI builds the image and pushes the `podinfo:1.7.1` image to the container registry * Flux scans the registry and updates the Helm release `image.tag` to `1.4.1` * Flux commits and push the change to the cluster repo * Flux applies the updated Helm release on the cluster diff --git a/docs/gitbook/tutorials/flagger-smi-istio.md b/docs/gitbook/tutorials/flagger-smi-istio.md index 6b622e581..3fae5dda9 100644 --- a/docs/gitbook/tutorials/flagger-smi-istio.md +++ b/docs/gitbook/tutorials/flagger-smi-istio.md @@ -236,7 +236,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: diff --git a/docs/gitbook/usage/ab-testing.md b/docs/gitbook/usage/ab-testing.md index aa5a93722..1a99f94b9 100644 --- a/docs/gitbook/usage/ab-testing.md +++ b/docs/gitbook/usage/ab-testing.md @@ -131,7 +131,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/abtest \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: diff --git a/docs/gitbook/usage/appmesh-progressive-delivery.md b/docs/gitbook/usage/appmesh-progressive-delivery.md index 02fe3c29a..8a069f3b8 100644 --- a/docs/gitbook/usage/appmesh-progressive-delivery.md +++ b/docs/gitbook/usage/appmesh-progressive-delivery.md @@ -176,7 +176,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: diff --git a/docs/gitbook/usage/blue-green.md b/docs/gitbook/usage/blue-green.md index 61ab6152b..1f8b3ea1e 100644 --- a/docs/gitbook/usage/blue-green.md +++ b/docs/gitbook/usage/blue-green.md @@ -59,8 +59,8 @@ Create a deployment and a horizontal pod autoscaler: ```bash export REPO=https://raw.githubusercontent.com/weaveworks/flagger/master -kubectl apply -f ${REPO}/artifacts/canary/deployment.yaml -kubectl apply -f ${REPO}/artifacts/canary/hpa.yaml +kubectl apply -f ${REPO}/artifacts/canaries/deployment.yaml +kubectl apply -f ${REPO}/artifacts/canaries/hpa.yaml ``` Deploy the load testing service to generate traffic during the analysis: @@ -172,7 +172,7 @@ Trigger a deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: @@ -297,7 +297,7 @@ Trigger a deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.3 +podinfod=quay.io/stefanprodan/podinfo:1.7.3 ``` Generate 404s: diff --git a/docs/gitbook/usage/gloo-progressive-delivery.md b/docs/gitbook/usage/gloo-progressive-delivery.md index bff29cee7..81359b2fa 100644 --- a/docs/gitbook/usage/gloo-progressive-delivery.md +++ b/docs/gitbook/usage/gloo-progressive-delivery.md @@ -197,7 +197,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: diff --git a/docs/gitbook/usage/linkerd-progressive-delivery.md b/docs/gitbook/usage/linkerd-progressive-delivery.md index 40da36278..3b9f64c94 100644 --- a/docs/gitbook/usage/linkerd-progressive-delivery.md +++ b/docs/gitbook/usage/linkerd-progressive-delivery.md @@ -165,7 +165,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: diff --git a/docs/gitbook/usage/nginx-progressive-delivery.md b/docs/gitbook/usage/nginx-progressive-delivery.md index 4d660bc40..6261c3511 100644 --- a/docs/gitbook/usage/nginx-progressive-delivery.md +++ b/docs/gitbook/usage/nginx-progressive-delivery.md @@ -189,7 +189,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: diff --git a/docs/gitbook/usage/progressive-delivery.md b/docs/gitbook/usage/progressive-delivery.md index db2d925fe..939e0940f 100644 --- a/docs/gitbook/usage/progressive-delivery.md +++ b/docs/gitbook/usage/progressive-delivery.md @@ -119,7 +119,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.4.1 +podinfod=quay.io/stefanprodan/podinfo:1.7.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: