From 625eed084035fec830778e524638ef9b95fd913d Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sat, 5 Oct 2019 17:59:34 +0300 Subject: [PATCH 1/2] Enforce blue/green when using kubernetes networking Use blue/green with ten iterations and warn that progressive traffic shifting and HTTP headers routing are not compatible with Kubernetes L4 networking. --- pkg/controller/scheduler.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 7d9b909c8..8fec79c03 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -325,6 +325,19 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh } } + // use blue/green strategy for kubernetes provider + if provider == "kubernetes" { + if len(cd.Spec.CanaryAnalysis.Match) > 0 { + c.recordEventWarningf(cd, "A/B testing is not supported when using the kubernetes provider") + cd.Spec.CanaryAnalysis.Match = nil + } + if cd.Spec.CanaryAnalysis.Iterations < 1 { + c.recordEventWarningf(cd, "Progressive traffic is not supported when using the kubernetes provider") + c.recordEventWarningf(cd, "Setting canaryAnalysis.iterations: 10") + cd.Spec.CanaryAnalysis.Iterations = 10 + } + } + // strategy: A/B testing if len(cd.Spec.CanaryAnalysis.Match) > 0 && cd.Spec.CanaryAnalysis.Iterations > 0 { // route traffic to canary and increment iterations From b76d0001ed5bbe6955feb80a8140fd323a9c12e5 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sat, 5 Oct 2019 18:00:16 +0300 Subject: [PATCH 2/2] Move Istio routing docs to FAQ --- README.md | 36 ++-- docs/gitbook/faq.md | 199 ++++++++++++++++++- docs/gitbook/how-it-works.md | 214 +++------------------ docs/gitbook/usage/progressive-delivery.md | 12 +- 4 files changed, 234 insertions(+), 227 deletions(-) diff --git a/README.md b/README.md index 637c5737d..e98ec02a0 100644 --- a/README.md +++ b/README.md @@ -87,16 +87,8 @@ spec: service: # container port port: 9898 - # Istio gateways (optional) - gateways: - - public-gateway.istio-system.svc.cluster.local - # Istio virtual service host names (optional) - hosts: - - podinfo.example.com - # Istio traffic policy (optional) - trafficPolicy: - tls: - mode: ISTIO_MUTUAL + # port name can be http or grpc (default http) + portName: http # HTTP match conditions (optional) match: - uri: @@ -104,10 +96,6 @@ spec: # HTTP rewrite (optional) rewrite: uri: / - # cross-origin resource sharing policy (optional) - corsPolicy: - allowOrigin: - - example.com # request timeout (optional) timeout: 5s # promote the canary without analysing it (default false) @@ -160,15 +148,17 @@ For more details on how the canary analysis and promotion works please [read the ## Features -| Feature | Istio | Linkerd | App Mesh | NGINX | Gloo | -| -------------------------------------------- | ------------------ | ------------------ |------------------ |------------------ |------------------ | -| Canary deployments (weighted traffic) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| A/B testing (headers and cookies filters) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_minus_sign: | -| Webhooks (acceptance/load testing) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Request success rate check (L7 metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Request duration check (L7 metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Custom promql checks | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Traffic policy, CORS, retries and timeouts | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: | +| Feature | Istio | Linkerd | App Mesh | NGINX | Gloo | Kubernetes CNI | +| -------------------------------------------- | ------------------ | ------------------ |------------------ |------------------ |------------------ |------------------ | +| Canary deployments (weighted traffic) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: | +| A/B testing (headers and cookies routing) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | +| Blue/Green deployments (traffic switch) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Webhooks (acceptance/load testing) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Manual gating (approve/pause/resume) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Request success rate check (L7 metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: | +| Request duration check (L7 metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: | +| Custom promql checks | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Traffic policy, CORS, retries and timeouts | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: | ## Roadmap diff --git a/docs/gitbook/faq.md b/docs/gitbook/faq.md index 10a7a6ccd..1db364e18 100644 --- a/docs/gitbook/faq.md +++ b/docs/gitbook/faq.md @@ -7,10 +7,12 @@ Flagger can run automated application analysis, promotion and rollback for the following deployment strategies: * Canary (progressive traffic shifting) * Istio, Linkerd, App Mesh, NGINX, Gloo +* Canary (traffic mirroring) + * Istio * A/B Testing (HTTP headers and cookies traffic routing) * Istio, NGINX * Blue/Green (traffic switch) - * Kubernetes CNI + * Kubernetes CNI, Istio, Linkerd, App Mesh, NGINX, Gloo For Canary deployments and A/B testing you'll need a Layer 7 traffic management solution like a service mesh or an ingress controller. For Blue/Green deployments no service mesh or ingress controller is required. @@ -123,7 +125,7 @@ idempotent. Before using mirroring on requests that may be writes, you should consider what will happen if a write is duplicated and handled by the primary and canary. -To use mirroring, set `spec.canaryAnalysis.mirror` to `true`. Example for +To use mirroring, set `spec.canaryAnalysis.mirror` to `true`. Example for traffic shifting: ```yaml @@ -132,17 +134,10 @@ kind: Canary spec: provider: istio canaryAnalysis: - interval: 30s mirror: true + interval: 30s stepWeight: 20 maxWeight: 50 - metrics: - - interval: 29s - name: request-success-rate - threshold: 99 - - interval: 29s - name: request-duration - threshold: 500 ``` ### Kubernetes services @@ -334,6 +329,190 @@ spec: topologyKey: kubernetes.io/hostname ``` +### Istio routing + +**How does Flagger interact with Istio?** + +Flagger creates an Istio Virtual Service and Destination Rules based on the Canary service spec. +The service configuration lets you expose an app inside or outside the mesh. +You can also define traffic policies, HTTP match conditions, URI rewrite rules, CORS policies, timeout and retries. + +The following spec exposes the `frontend` workload inside the mesh on `frontend.test.svc.cluster.local:9898` +and outside the mesh on `frontend.example.com`. You'll have to specify an Istio ingress gateway for external hosts. + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: frontend + namespace: test +spec: + service: + # container port + port: 9898 + # service port name (optional, will default to "http") + portName: http-frontend + # Istio gateways (optional) + gateways: + - public-gateway.istio-system.svc.cluster.local + - mesh + # Istio virtual service host names (optional) + hosts: + - frontend.example.com + # Istio traffic policy + trafficPolicy: + tls: + # use ISTIO_MUTUAL when mTLS is enabled + mode: DISABLE + # HTTP match conditions (optional) + match: + - uri: + prefix: / + # HTTP rewrite (optional) + rewrite: + uri: / + # Envoy timeout and retry policy (optional) + headers: + request: + add: + x-envoy-upstream-rq-timeout-ms: "15000" + x-envoy-max-retries: "10" + x-envoy-retry-on: "gateway-error,connect-failure,refused-stream" + # cross-origin resource sharing policy (optional) + corsPolicy: + allowOrigin: + - example.com + allowMethods: + - GET + allowCredentials: false + allowHeaders: + - x-some-header + maxAge: 24h +``` + +For the above spec Flagger will generate the following virtual service: + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: frontend + namespace: test + ownerReferences: + - apiVersion: flagger.app/v1alpha3 + blockOwnerDeletion: true + controller: true + kind: Canary + name: podinfo + uid: 3a4a40dd-3875-11e9-8e1d-42010a9c0fd1 +spec: + gateways: + - public-gateway.istio-system.svc.cluster.local + - mesh + hosts: + - frontend.example.com + - frontend + http: + - appendHeaders: + x-envoy-max-retries: "10" + x-envoy-retry-on: gateway-error,connect-failure,refused-stream + x-envoy-upstream-rq-timeout-ms: "15000" + corsPolicy: + allowHeaders: + - x-some-header + allowMethods: + - GET + allowOrigin: + - example.com + maxAge: 24h + match: + - uri: + prefix: / + rewrite: + uri: / + route: + - destination: + host: podinfo-primary + weight: 100 + - destination: + host: podinfo-canary + weight: 0 +``` + +For each destination in the virtual service a rule is generated: + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: frontend-primary + namespace: test +spec: + host: frontend-primary + trafficPolicy: + tls: + mode: DISABLE +--- +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: frontend-canary + namespace: test +spec: + host: frontend-canary + trafficPolicy: + tls: + mode: DISABLE +``` + +Flagger keeps in sync the virtual service and destination rules with the canary service spec. +Any direct modification to the virtual service spec will be overwritten. + +To expose a workload inside the mesh on `http://backend.test.svc.cluster.local:9898`, +the service spec can contain only the container port and the traffic policy: + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: backend + namespace: test +spec: + service: + port: 9898 + trafficPolicy: + tls: + mode: DISABLE +``` + +Based on the above spec, Flagger will create several ClusterIP services like: + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: backend-primary + ownerReferences: + - apiVersion: flagger.app/v1alpha3 + blockOwnerDeletion: true + controller: true + kind: Canary + name: backend + uid: 2ca1a9c7-2ef6-11e9-bd01-42010a9c0145 +spec: + type: ClusterIP + ports: + - name: http + port: 9898 + protocol: TCP + targetPort: 9898 + selector: + app: backend-primary +``` + +Flagger works for user facing apps exposed outside the cluster via an ingress gateway +and for backend HTTP APIs that are accessible only from inside the mesh. + ### Istio Ingress Gateway **How can I expose multiple canaries on the same external domain?** diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index 6669cf4ad..afd5e2398 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -4,8 +4,6 @@ a horizontal pod autoscaler \(HPA\) and creates a series of objects \(Kubernetes deployments, ClusterIP services, virtual service, traffic split or ingress\) to drive the canary analysis and promotion. -![Flagger Canary Process](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-hpa.png) - ### Canary Custom Resource For a deployment named _podinfo_, a canary promotion can be defined using Flagger's custom resource: @@ -19,7 +17,7 @@ metadata: spec: # service mesh provider (optional) # can be: kubernetes, istio, linkerd, appmesh, nginx, gloo, supergloo - provider: istio + provider: linkerd # deployment reference targetRef: apiVersion: apps/v1 @@ -36,8 +34,11 @@ spec: service: # container port port: 9898 - # service port name (optional, will default to "http") + # port name can be http or grpc (default http) portName: http + # add all the other container ports + # to the ClusterIP services (default false) + portDiscovery: false # promote the canary without analysing it (default false) skipAnalysis: false # define the canary analysis timing and KPIs @@ -164,188 +165,6 @@ kubectl wait canary/podinfo --for=condition=promoted --timeout=5m kubectl get canary/podinfo | grep Succeeded ``` -### Istio routing - -Flagger creates an Istio Virtual Service and Destination Rules based on the Canary service spec. -The service configuration lets you expose an app inside or outside the mesh. -You can also define traffic policies, HTTP match conditions, URI rewrite rules, CORS policies, timeout and retries. - -The following spec exposes the `frontend` workload inside the mesh on `frontend.test.svc.cluster.local:9898` -and outside the mesh on `frontend.example.com`. You'll have to specify an Istio ingress gateway for external hosts. - -```yaml -apiVersion: flagger.app/v1alpha3 -kind: Canary -metadata: - name: frontend - namespace: test -spec: - service: - # container port - port: 9898 - # service port name (optional, will default to "http") - portName: http-frontend - # Istio gateways (optional) - gateways: - - public-gateway.istio-system.svc.cluster.local - - mesh - # Istio virtual service host names (optional) - hosts: - - frontend.example.com - # Istio traffic policy - trafficPolicy: - tls: - # use ISTIO_MUTUAL when mTLS is enabled - mode: DISABLE - # HTTP match conditions (optional) - match: - - uri: - prefix: / - # HTTP rewrite (optional) - rewrite: - uri: / - # Envoy timeout and retry policy (optional) - headers: - request: - add: - x-envoy-upstream-rq-timeout-ms: "15000" - x-envoy-max-retries: "10" - x-envoy-retry-on: "gateway-error,connect-failure,refused-stream" - # cross-origin resource sharing policy (optional) - corsPolicy: - allowOrigin: - - example.com - allowMethods: - - GET - allowCredentials: false - allowHeaders: - - x-some-header - maxAge: 24h -``` - -For the above spec Flagger will generate the following virtual service: - -```yaml -apiVersion: networking.istio.io/v1alpha3 -kind: VirtualService -metadata: - name: frontend - namespace: test - ownerReferences: - - apiVersion: flagger.app/v1alpha3 - blockOwnerDeletion: true - controller: true - kind: Canary - name: podinfo - uid: 3a4a40dd-3875-11e9-8e1d-42010a9c0fd1 -spec: - gateways: - - public-gateway.istio-system.svc.cluster.local - - mesh - hosts: - - frontend.example.com - - frontend - http: - - appendHeaders: - x-envoy-max-retries: "10" - x-envoy-retry-on: gateway-error,connect-failure,refused-stream - x-envoy-upstream-rq-timeout-ms: "15000" - corsPolicy: - allowHeaders: - - x-some-header - allowMethods: - - GET - allowOrigin: - - example.com - maxAge: 24h - match: - - uri: - prefix: / - rewrite: - uri: / - route: - - destination: - host: podinfo-primary - weight: 100 - - destination: - host: podinfo-canary - weight: 0 -``` - -For each destination in the virtual service a rule is generated: - -```yaml -apiVersion: networking.istio.io/v1alpha3 -kind: DestinationRule -metadata: - name: frontend-primary - namespace: test -spec: - host: frontend-primary - trafficPolicy: - tls: - mode: DISABLE ---- -apiVersion: networking.istio.io/v1alpha3 -kind: DestinationRule -metadata: - name: frontend-canary - namespace: test -spec: - host: frontend-canary - trafficPolicy: - tls: - mode: DISABLE -``` - -Flagger keeps in sync the virtual service and destination rules with the canary service spec. -Any direct modification to the virtual service spec will be overwritten. - -To expose a workload inside the mesh on `http://backend.test.svc.cluster.local:9898`, -the service spec can contain only the container port and the traffic policy: - -```yaml -apiVersion: flagger.app/v1alpha3 -kind: Canary -metadata: - name: backend - namespace: test -spec: - service: - port: 9898 - trafficPolicy: - tls: - mode: DISABLE -``` - -Based on the above spec, Flagger will create several ClusterIP services like: - -```yaml -apiVersion: v1 -kind: Service -metadata: - name: backend-primary - ownerReferences: - - apiVersion: flagger.app/v1alpha3 - blockOwnerDeletion: true - controller: true - kind: Canary - name: backend - uid: 2ca1a9c7-2ef6-11e9-bd01-42010a9c0145 -spec: - type: ClusterIP - ports: - - name: http - port: 9898 - protocol: TCP - targetPort: 9898 - selector: - app: backend-primary -``` - -Flagger works for user facing apps exposed outside the cluster via an ingress gateway -and for backend HTTP APIs that are accessible only from inside the mesh. - ### Canary Stages ![Flagger Canary Stages](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-steps.png) @@ -483,6 +302,29 @@ interval * threshold Make sure that the analysis threshold is lower than the number of iterations. +### Blue/Green deployments + +For applications that are not deployed on a service mesh, Flagger can orchestrate blue/green style deployments +with Kubernetes L4 networking. When using Istio you have the option to mirror traffic between blue and green. + +You can use the blue/green deployment strategy by replacing `stepWeight/maxWeight` with `iterations` in the `canaryAnalysis` spec: + +```yaml + canaryAnalysis: + # schedule interval (default 60s) + interval: 1m + # total number of iterations + iterations: 10 + # max number of failed iterations before rollback + threshold: 2 + # Traffic shadowing (compatible with Istio only) + mirror: true +``` + +With the above configuration Flagger will run conformance and load tests on the canary pods for ten minutes. +If the metrics analysis succeeds, live traffic will be switched from the old version to the new one when the +canary is promoted. + ### HTTP Metrics The canary analysis is using the following Prometheus queries: diff --git a/docs/gitbook/usage/progressive-delivery.md b/docs/gitbook/usage/progressive-delivery.md index 9d0bfcdce..17d3c4fec 100644 --- a/docs/gitbook/usage/progressive-delivery.md +++ b/docs/gitbook/usage/progressive-delivery.md @@ -115,6 +115,8 @@ kubectl apply -f ./podinfo-canary.yaml When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary. The canary analysis will run for five minutes while validating the HTTP metrics and rollout hooks every minute. +![Flagger Canary Process](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-hpa.png) + After a couple of seconds Flagger will create the canary objects: ```bash @@ -250,10 +252,10 @@ Events: ![Flagger Canary Traffic Shadowing](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-traffic-mirroring.png) -For applications that perform read operations, Flagger can be configured to drive a canary release with traffic mirroring. +For applications that perform read operations, Flagger can be configured to drive canary releases with traffic mirroring. Istio traffic mirroring will copy each incoming request, sending one request to the primary and one to the canary service. The response from the primary is sent back to the user and the response from the canary is discarded. -Metrics are collected on both requests so that the deployment will only proceed if the canary metrics are healthy. +Metrics are collected on both requests so that the deployment will only proceed if the canary metrics are within the threshold values. Note that mirroring should be used for requests that are **idempotent** or capable of being processed twice (once by the primary and once by the canary). @@ -324,9 +326,3 @@ The above procedure can be extended with [custom metrics](https://docs.flagger.a [webhooks](https://docs.flagger.app/how-it-works#webhooks), [manual promotion](https://docs.flagger.app/how-it-works#manual-gating) approval and [Slack or MS Teams](https://docs.flagger.app/usage/alerting) notifications. - - - - - -