diff --git a/CHANGELOG.md b/CHANGELOG.md index 34603f5e8..f9c6b93a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ All notable changes to this project are documented in this file. +## vNext (TBA) + +Adds support for multiple ports services + +#### Features + +- Implement port discovery [#207](https://github.com/weaveworks/flagger/pull/207) + +#### Improvements + +- Add [FAQ page](https://docs.flagger.app/faq) to docs website +- Use Istio v1.1.9 in [e2e testing]() + + ## 0.15.0 (2019-06-12) Adds support for customising the Istio [traffic policy](https://docs.flagger.app/how-it-works#istio-routing) in the canary service spec diff --git a/README.md b/README.md index f00a7baf4..8540500ba 100644 --- a/README.md +++ b/README.md @@ -35,12 +35,13 @@ Flagger documentation can be found at [docs.flagger.app](https://docs.flagger.ap * [Custom metrics](https://docs.flagger.app/how-it-works#custom-metrics) * [Webhooks](https://docs.flagger.app/how-it-works#webhooks) * [Load testing](https://docs.flagger.app/how-it-works#load-testing) + * [FAQ](https://docs.flagger.app/faq) * Usage * [Istio canary deployments](https://docs.flagger.app/usage/progressive-delivery) * [Istio A/B testing](https://docs.flagger.app/usage/ab-testing) * [App Mesh canary deployments](https://docs.flagger.app/usage/appmesh-progressive-delivery) * [NGINX ingress controller canary deployments](https://docs.flagger.app/usage/nginx-progressive-delivery) - * [Gloo Canary Deployments](https://docs.flagger.app/usage/gloo-progressive-delivery) + * [Gloo ingress controller canary deployments](https://docs.flagger.app/usage/gloo-progressive-delivery) * [Monitoring](https://docs.flagger.app/usage/monitoring) * [Alerting](https://docs.flagger.app/usage/alerting) * Tutorials @@ -149,27 +150,15 @@ For more details on how the canary analysis and promotion works please [read the ## Features -| Service Mesh Feature | Istio | App Mesh | SuperGloo | -| -------------------------------------------- | ------------------ | ------------------ |------------------ | -| Canary deployments (weighted traffic) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| A/B testing (headers and cookies filters) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | -| Load testing | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Webhooks (acceptance testing) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Request success rate check (L7 metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Request duration check (L7 metric) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | -| Custom promql checks | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Ingress gateway (CORS, retries and timeouts) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | - -| Ingress Controller Feature | NGINX | Gloo | -| -------------------------------------------- | ------------------ | ------------------ | -| Canary deployments (weighted traffic) | :heavy_check_mark: | :heavy_check_mark: | -| A/B testing (headers and cookies filters) | :heavy_check_mark: | :heavy_minus_sign: | -| Load testing | :heavy_check_mark: | :heavy_check_mark: | -| Webhooks (acceptance testing) | :heavy_check_mark: | :heavy_check_mark: | -| Request success rate check (L7 metric) | :heavy_minus_sign: | :heavy_check_mark: | -| Request duration check (L7 metric) | :heavy_minus_sign: | :heavy_check_mark: | -| Custom promql checks | :heavy_check_mark: | :heavy_check_mark: | - +| Feature | Istio | App Mesh | NGINX | Gloo | +| -------------------------------------------- | ------------------ | ------------------ |------------------ |------------------ | +| Canary deployments (weighted traffic) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| A/B testing (headers and cookies filters) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_minus_sign: | +| Webhooks (acceptance/load testing) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Request success rate check (L7 metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Request duration check (L7 metric) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_check_mark: | +| Custom promql checks | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Traffic policy, CORS, retries and timeouts | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: | ## Roadmap diff --git a/artifacts/canaries/canary.yaml b/artifacts/canaries/canary.yaml index 383aac329..1421a90d4 100644 --- a/artifacts/canaries/canary.yaml +++ b/artifacts/canaries/canary.yaml @@ -20,17 +20,25 @@ spec: service: # container port port: 9898 + # port name can be http or grpc (default http) + portName: http + # add all the other container ports + # when generating ClusterIP services (default false) + portDiscovery: false # Istio gateways (optional) gateways: - public-gateway.istio-system.svc.cluster.local + # remove the mesh gateway if the public host is + # shared across multiple virtual services - mesh # Istio virtual service host names (optional) hosts: - app.istio.weavedx.com # Istio traffic policy (optional) trafficPolicy: - loadBalancer: - simple: LEAST_CONN + tls: + # use ISTIO_MUTUAL when mTLS is enabled + mode: DISABLE # HTTP match conditions (optional) match: - uri: @@ -68,7 +76,7 @@ spec: # external checks (optional) webhooks: - name: load-test - url: http://flagger-loadtester.test/ + url: http://tester.istio.weavedx.com/ timeout: 5s metadata: type: cmd diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index 61eac4087..3dffd6213 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -89,6 +89,8 @@ spec: type: number portName: type: string + portDiscovery: + type: boolean meshName: type: string timeout: diff --git a/artifacts/gke/istio-prometheus.yaml b/artifacts/gke/istio-prometheus.yaml index ad9dbdcf4..07944d6e4 100644 --- a/artifacts/gke/istio-prometheus.yaml +++ b/artifacts/gke/istio-prometheus.yaml @@ -1,49 +1,4 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: prometheus - labels: - app: prometheus -rules: - - apiGroups: [""] - resources: - - nodes - - services - - endpoints - - pods - - nodes/proxy - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: - - configmaps - verbs: ["get"] - - nonResourceURLs: ["/metrics"] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: prometheus - labels: - app: prometheus -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: prometheus -subjects: - - kind: ServiceAccount - name: prometheus - namespace: istio-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: prometheus - namespace: istio-system - labels: - app: prometheus ---- +# Source: istio/charts/prometheus/templates/configmap.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -51,6 +6,9 @@ metadata: namespace: istio-system labels: app: prometheus + chart: prometheus-1.0.6 + heritage: Tiller + release: istio data: prometheus.yml: |- global: @@ -363,6 +321,70 @@ data: - source_labels: [__meta_kubernetes_pod_name] action: replace target_label: pod_name + +--- + +# Source: istio/charts/prometheus/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: prometheus-istio-system + labels: + app: prometheus + chart: prometheus-1.0.6 + heritage: Tiller + release: istio +rules: + - apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + - nodes/proxy + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: + - configmaps + verbs: ["get"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] + +--- + +# Source: istio/charts/prometheus/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: istio-system + labels: + app: prometheus + chart: prometheus-1.0.6 + heritage: Tiller + release: istio + +--- + +# Source: istio/charts/prometheus/templates/clusterrolebindings.yaml +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: prometheus-istio-system + labels: + app: prometheus + chart: prometheus-1.0.6 + heritage: Tiller + release: istio +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-istio-system +subjects: + - kind: ServiceAccount + name: prometheus + namespace: istio-system + --- # Source: istio/charts/prometheus/templates/service.yaml @@ -384,13 +406,18 @@ spec: port: 9090 --- -apiVersion: apps/v1 + +# Source: istio/charts/prometheus/templates/deployment.yaml +apiVersion: apps/v1beta1 kind: Deployment metadata: name: prometheus namespace: istio-system labels: app: prometheus + chart: prometheus-1.0.6 + heritage: Tiller + release: istio spec: replicas: 1 selector: @@ -407,7 +434,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: "docker.io/prom/prometheus:v2.7.1" + image: "docker.io/prom/prometheus:v2.3.1" imagePullPolicy: IfNotPresent args: - '--storage.tsdb.retention=6h' @@ -441,3 +468,367 @@ spec: defaultMode: 420 optional: true secretName: istio.default + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + - ppc64le + - s390x + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 2 + preference: + matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + - weight: 2 + preference: + matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - ppc64le + - weight: 2 + preference: + matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - s390x + +--- +apiVersion: "config.istio.io/v1alpha2" +kind: metric +metadata: + name: requestcount + namespace: istio-system +spec: + value: "1" + dimensions: + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + source_workload: source.workload.name | "unknown" + source_workload_namespace: source.workload.namespace | "unknown" + source_principal: source.principal | "unknown" + source_app: source.labels["app"] | "unknown" + source_version: source.labels["version"] | "unknown" + destination_workload: destination.workload.name | "unknown" + destination_workload_namespace: destination.workload.namespace | "unknown" + destination_principal: destination.principal | "unknown" + destination_app: destination.labels["app"] | "unknown" + destination_version: destination.labels["version"] | "unknown" + destination_service: destination.service.host | "unknown" + destination_service_name: destination.service.name | "unknown" + destination_service_namespace: destination.service.namespace | "unknown" + request_protocol: api.protocol | context.protocol | "unknown" + response_code: response.code | 200 + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: metric +metadata: + name: requestduration + namespace: istio-system +spec: + value: response.duration | "0ms" + dimensions: + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + source_workload: source.workload.name | "unknown" + source_workload_namespace: source.workload.namespace | "unknown" + source_principal: source.principal | "unknown" + source_app: source.labels["app"] | "unknown" + source_version: source.labels["version"] | "unknown" + destination_workload: destination.workload.name | "unknown" + destination_workload_namespace: destination.workload.namespace | "unknown" + destination_principal: destination.principal | "unknown" + destination_app: destination.labels["app"] | "unknown" + destination_version: destination.labels["version"] | "unknown" + destination_service: destination.service.host | "unknown" + destination_service_name: destination.service.name | "unknown" + destination_service_namespace: destination.service.namespace | "unknown" + request_protocol: api.protocol | context.protocol | "unknown" + response_code: response.code | 200 + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: metric +metadata: + name: requestsize + namespace: istio-system +spec: + value: request.size | 0 + dimensions: + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + source_workload: source.workload.name | "unknown" + source_workload_namespace: source.workload.namespace | "unknown" + source_principal: source.principal | "unknown" + source_app: source.labels["app"] | "unknown" + source_version: source.labels["version"] | "unknown" + destination_workload: destination.workload.name | "unknown" + destination_workload_namespace: destination.workload.namespace | "unknown" + destination_principal: destination.principal | "unknown" + destination_app: destination.labels["app"] | "unknown" + destination_version: destination.labels["version"] | "unknown" + destination_service: destination.service.host | "unknown" + destination_service_name: destination.service.name | "unknown" + destination_service_namespace: destination.service.namespace | "unknown" + request_protocol: api.protocol | context.protocol | "unknown" + response_code: response.code | 200 + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: metric +metadata: + name: responsesize + namespace: istio-system +spec: + value: response.size | 0 + dimensions: + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + source_workload: source.workload.name | "unknown" + source_workload_namespace: source.workload.namespace | "unknown" + source_principal: source.principal | "unknown" + source_app: source.labels["app"] | "unknown" + source_version: source.labels["version"] | "unknown" + destination_workload: destination.workload.name | "unknown" + destination_workload_namespace: destination.workload.namespace | "unknown" + destination_principal: destination.principal | "unknown" + destination_app: destination.labels["app"] | "unknown" + destination_version: destination.labels["version"] | "unknown" + destination_service: destination.service.host | "unknown" + destination_service_name: destination.service.name | "unknown" + destination_service_namespace: destination.service.namespace | "unknown" + request_protocol: api.protocol | context.protocol | "unknown" + response_code: response.code | 200 + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: metric +metadata: + name: tcpbytesent + namespace: istio-system +spec: + value: connection.sent.bytes | 0 + dimensions: + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + source_workload: source.workload.name | "unknown" + source_workload_namespace: source.workload.namespace | "unknown" + source_principal: source.principal | "unknown" + source_app: source.labels["app"] | "unknown" + source_version: source.labels["version"] | "unknown" + destination_workload: destination.workload.name | "unknown" + destination_workload_namespace: destination.workload.namespace | "unknown" + destination_principal: destination.principal | "unknown" + destination_app: destination.labels["app"] | "unknown" + destination_version: destination.labels["version"] | "unknown" + destination_service: destination.service.name | "unknown" + destination_service_name: destination.service.name | "unknown" + destination_service_namespace: destination.service.namespace | "unknown" + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: metric +metadata: + name: tcpbytereceived + namespace: istio-system +spec: + value: connection.received.bytes | 0 + dimensions: + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + source_workload: source.workload.name | "unknown" + source_workload_namespace: source.workload.namespace | "unknown" + source_principal: source.principal | "unknown" + source_app: source.labels["app"] | "unknown" + source_version: source.labels["version"] | "unknown" + destination_workload: destination.workload.name | "unknown" + destination_workload_namespace: destination.workload.namespace | "unknown" + destination_principal: destination.principal | "unknown" + destination_app: destination.labels["app"] | "unknown" + destination_version: destination.labels["version"] | "unknown" + destination_service: destination.service.name | "unknown" + destination_service_name: destination.service.name | "unknown" + destination_service_namespace: destination.service.namespace | "unknown" + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: prometheus +metadata: + name: handler + namespace: istio-system +spec: + metrics: + - name: requests_total + instance_name: requestcount.metric.istio-system + kind: COUNTER + label_names: + - reporter + - source_app + - source_principal + - source_workload + - source_workload_namespace + - source_version + - destination_app + - destination_principal + - destination_workload + - destination_workload_namespace + - destination_version + - destination_service + - destination_service_name + - destination_service_namespace + - request_protocol + - response_code + - connection_security_policy + - name: request_duration_seconds + instance_name: requestduration.metric.istio-system + kind: DISTRIBUTION + label_names: + - reporter + - source_app + - source_principal + - source_workload + - source_workload_namespace + - source_version + - destination_app + - destination_principal + - destination_workload + - destination_workload_namespace + - destination_version + - destination_service + - destination_service_name + - destination_service_namespace + - request_protocol + - response_code + - connection_security_policy + buckets: + explicit_buckets: + bounds: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10] + - name: request_bytes + instance_name: requestsize.metric.istio-system + kind: DISTRIBUTION + label_names: + - reporter + - source_app + - source_principal + - source_workload + - source_workload_namespace + - source_version + - destination_app + - destination_principal + - destination_workload + - destination_workload_namespace + - destination_version + - destination_service + - destination_service_name + - destination_service_namespace + - request_protocol + - response_code + - connection_security_policy + buckets: + exponentialBuckets: + numFiniteBuckets: 8 + scale: 1 + growthFactor: 10 + - name: response_bytes + instance_name: responsesize.metric.istio-system + kind: DISTRIBUTION + label_names: + - reporter + - source_app + - source_principal + - source_workload + - source_workload_namespace + - source_version + - destination_app + - destination_principal + - destination_workload + - destination_workload_namespace + - destination_version + - destination_service + - destination_service_name + - destination_service_namespace + - request_protocol + - response_code + - connection_security_policy + buckets: + exponentialBuckets: + numFiniteBuckets: 8 + scale: 1 + growthFactor: 10 + - name: tcp_sent_bytes_total + instance_name: tcpbytesent.metric.istio-system + kind: COUNTER + label_names: + - reporter + - source_app + - source_principal + - source_workload + - source_workload_namespace + - source_version + - destination_app + - destination_principal + - destination_workload + - destination_workload_namespace + - destination_version + - destination_service + - destination_service_name + - destination_service_namespace + - connection_security_policy + - name: tcp_received_bytes_total + instance_name: tcpbytereceived.metric.istio-system + kind: COUNTER + label_names: + - reporter + - source_app + - source_principal + - source_workload + - source_workload_namespace + - source_version + - destination_app + - destination_principal + - destination_workload + - destination_workload_namespace + - destination_version + - destination_service + - destination_service_name + - destination_service_namespace + - connection_security_policy +--- +apiVersion: "config.istio.io/v1alpha2" +kind: rule +metadata: + name: promhttp + namespace: istio-system +spec: + match: context.protocol == "http" || context.protocol == "grpc" + actions: + - handler: handler.prometheus + instances: + - requestcount.metric + - requestduration.metric + - requestsize.metric + - responsesize.metric +--- +apiVersion: "config.istio.io/v1alpha2" +kind: rule +metadata: + name: promtcp + namespace: istio-system +spec: + match: context.protocol == "tcp" + actions: + - handler: handler.prometheus + instances: + - tcpbytesent.metric + - tcpbytereceived.metric +--- diff --git a/charts/flagger/README.md b/charts/flagger/README.md index d47034a6d..9a2492d60 100644 --- a/charts/flagger/README.md +++ b/charts/flagger/README.md @@ -1,7 +1,7 @@ # Flagger [Flagger](https://github.com/weaveworks/flagger) is a Kubernetes operator that automates the promotion of -canary deployments using Istio routing for traffic shifting and Prometheus metrics for canary analysis. +canary deployments using Istio, App Mesh, NGINX or Gloo routing for traffic shifting and Prometheus metrics for canary analysis. Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance indicators like HTTP requests success rate, requests average duration and pods health. Based on the KPIs analysis a canary is promoted or aborted and the analysis result is published to Slack. @@ -9,7 +9,6 @@ Based on the KPIs analysis a canary is promoted or aborted and the analysis resu ## Prerequisites * Kubernetes >= 1.11 -* Istio >= 1.0 * Prometheus >= 2.6 ## Installing the Chart @@ -48,7 +47,8 @@ Parameter | Description | Default `image.repository` | image repository | `weaveworks/flagger` `image.tag` | image tag | `` `image.pullPolicy` | image pull policy | `IfNotPresent` -`metricsServer` | Prometheus URL | `http://prometheus.istio-system:9090` +`prometheus.install` | if `true`, installs Prometheus configured to scrape all pods in the custer including the App Mesh sidecar | `false` +`metricsServer` | Prometheus URL, used when `prometheus.install` is `false` | `http://prometheus.istio-system:9090` `slack.url` | Slack incoming webhook | None `slack.channel` | Slack channel | None `slack.user` | Slack username | `flagger` diff --git a/charts/flagger/templates/crd.yaml b/charts/flagger/templates/crd.yaml index f189a31a0..44b3a1153 100644 --- a/charts/flagger/templates/crd.yaml +++ b/charts/flagger/templates/crd.yaml @@ -90,6 +90,8 @@ spec: type: number portName: type: string + portDiscovery: + type: boolean meshName: type: string timeout: diff --git a/docs/gitbook/README.md b/docs/gitbook/README.md index 5f0271154..db6be6d66 100644 --- a/docs/gitbook/README.md +++ b/docs/gitbook/README.md @@ -5,9 +5,8 @@ description: Flagger is a progressive delivery Kubernetes operator # Introduction [Flagger](https://github.com/weaveworks/flagger) is a **Kubernetes** operator that automates the promotion of canary -deployments using **Istio**, **App Mesh** or **NGINX** routing for traffic shifting and **Prometheus** metrics for canary analysis. -The canary analysis can be extended with webhooks for running -system integration/acceptance tests, load tests, or any other custom validation. +deployments using **Istio**, **App Mesh**, **NGINX** or **Gloo** routing for traffic shifting and **Prometheus** metrics for canary analysis. +The canary analysis can be extended with webhooks for running system integration/acceptance tests, load tests, or any other custom validation. Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance indicators like HTTP requests success rate, requests average duration and pods health. diff --git a/docs/gitbook/SUMMARY.md b/docs/gitbook/SUMMARY.md index 346d7c4d8..4a674a2d9 100644 --- a/docs/gitbook/SUMMARY.md +++ b/docs/gitbook/SUMMARY.md @@ -2,6 +2,7 @@ * [Introduction](README.md) * [How it works](how-it-works.md) +* [FAQ](faq.md) ## Install diff --git a/docs/gitbook/faq.md b/docs/gitbook/faq.md index bc2e1b8a7..ebf5b284e 100644 --- a/docs/gitbook/faq.md +++ b/docs/gitbook/faq.md @@ -1,21 +1,318 @@ # Frequently asked questions -**Can Flagger be part of my integration tests?** -> Yes, Flagger supports webhooks to do integration testing. - -**What if I only want to target beta testers?** -> That's a feature in Flagger, not in App Mesh. It's on the App Mesh roadmap. - -**When do I use A/B testing when Canary?** -> One advantage of using A/B testing is that each version remains separated and routes aren't mixed. -> -> Using a Canary deployment can lead to behaviour like this one observed by a -> user: -> -> [..] during a canary deployment of our nodejs app, the version that is being served <50% traffic reports mime type mismatch errors in the browser (js as "text/html") -> When the deployment Passes/ Fails (doesn't really matter) the version that stays alive works as expected. If anyone has any tips or direction I would greatly appreciate it. Even if its as simple as I'm looking in the wrong place. Thanks in advance! -> -> The issue was that we were not maintaining session affinity while serving files for our frontend. Which resulted in any redirects or refreshes occasionally returning a mismatched app.*.js file (generated from vue) -> -> Read up on [A/B testing](https://docs.flagger.app/usage/ab-testing). +### A/B Testing +When should I use A/B testing instead of progressive traffic shifting? + +For frontend applications that require session affinity you should use HTTP headers or cookies match conditions +to ensure a set of users will stay on the same version for the whole duration of the canary analysis. +A/B testing is supported by Istio and NGINX only. + +Istio example: + +```yaml + canaryAnalysis: + # schedule interval (default 60s) + interval: 1m + # total number of iterations + iterations: 10 + # max number of failed iterations before rollback + threshold: 2 + # canary match condition + match: + - headers: + x-canary: + regex: ".*insider.*" + - headers: + cookie: + regex: "^(.*?;)?(canary=always)(;.*)?$" +``` + +NGINX example: + +```yaml + canaryAnalysis: + interval: 1m + threshold: 10 + iterations: 2 + match: + - headers: + x-canary: + exact: "insider" + - headers: + cookie: + exact: "canary" +``` + +Note that the NGINX ingress controller supports only exact matching for a single header and the cookie value is set to `always`. + +The above configurations will route users with the x-canary header or canary cookie to the canary instance during analysis: + +```bash +curl -H 'X-Canary: insider' http://app.example.com +curl -b 'canary=always' http://app.example.com +``` + +### Kubernetes services + +How is an application exposed inside the cluster? + +Assuming the app name is podinfo you can define a canary like: + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + service: + # container port (required) + port: 9898 + # port name can be http or grpc (default http) + portName: http +``` + +Based on the canary spec service, Flagger generates the following Kubernetes ClusterIP service: + +* `..vc.cluster.local` with selector `app=-primary` +* `-primary..vc.cluster.local` with selector `app=-primary` +* `-canary..vc.cluster.local` with selector `app=` + +This ensures that traffic coming from a namespace outside the mesh to `podinfo.test:9898` +will be routed to the latest stable release of your app. + + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: podinfo +spec: + type: ClusterIP + selector: + app: podinfo-primary + ports: + - name: http + port: 9898 + protocol: TCP + targetPort: http +``` + +The `podinfo-canary.test:9898` address is available only during the +canary analysis and can be used for conformance testing or load testing. + +### Multiple ports + +My application listens on multiple ports, how can I expose them inside the cluster? + +If port discovery is enabled, Flagger scans the deployment spec and extracts the containers +ports excluding the port specified in the canary service and Envoy sidecar ports. +`These ports will be used when generating the ClusterIP services. + +For a deployment that exposes two ports: + +```yaml +apiVersion: apps/v1 +kind: Deployment +spec: + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9899" + spec: + containers: + - name: app + ports: + - containerPort: 8080 + - containerPort: 9090 +``` + +You can enable port discovery so that Prometheus will be able to reach port `9090` over mTLS: + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +spec: + service: + # container port used for canary analysis + port: 8080 + # port name can be http or grpc (default http) + portName: http + # add all the other container ports + # to the ClusterIP services (default false) + portDiscovery: true + trafficPolicy: + tls: + mode: ISTIO_MUTUAL +``` + +Both port `8080` and `9090` will be added to the ClusterIP services but the virtual service +will point to the port specified in `spec.service.port`. + +### Label selectors + +What labels selectors are supported by Flagger? + +The target deployment must have a single label selector in the format `app: `: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo +spec: + selector: + matchLabels: + app: podinfo + template: + metadata: + labels: + app: podinfo +``` + +Besides `app` Flagger supports `name` and `app.kubernetes.io/name` selectors. If you use a different +convention you can specify your label with the `-selector-labels` flag. + +Is pod affinity and anti affinity supported? + +For pod affinity to work you need to use a different label than the `app`, `name` or `app.kubernetes.io/name`. + +Anti affinity example: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo +spec: + selector: + matchLabels: + app: podinfo + affinity: podinfo + template: + metadata: + labels: + app: podinfo + affinity: podinfo + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + affinity: podinfo + topologyKey: kubernetes.io/hostname +``` + +### Istio Ingress Gateway + +How can I expose multiple canaries on the same external domain? + +Assuming you have two apps, one that servers the main website and one that serves the REST API. +For each app you can define a canary object as: + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: website +spec: + service: + port: 8080 + gateways: + - public-gateway.istio-system.svc.cluster.local + hosts: + - my-site.com + match: + - uri: + prefix: / + rewrite: + uri: / +--- +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: webapi +spec: + service: + port: 8080 + gateways: + - public-gateway.istio-system.svc.cluster.local + hosts: + - my-site.com + match: + - uri: + prefix: /api + rewrite: + uri: / +``` + +Based on the above configuration, Flagger will create two virtual services bounded to the same ingress gateway and external host. +Istio Pilot will [merge](https://istio.io/help/ops/traffic-management/deploy-guidelines/#multiple-virtual-services-and-destination-rules-for-the-same-host) +the two services and the website rule will be moved to the end of the list in the merged configuration. + +Note that host merging only works if the canaries are bounded to a ingress gateway other than the `mesh` gateway. + +### Istio Mutual TLS + +How can I enable mTLS for a canary? + +When deploying Istio with global mTLS enabled, you have to set the TLS mode to `ISTIO_MUTUAL`: + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +spec: + service: + trafficPolicy: + tls: + mode: ISTIO_MUTUAL +``` + +If you run Istio in permissive mode you can disable TLS: + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +spec: + service: + trafficPolicy: + tls: + mode: DISABLE +``` + +If Flagger is outside of the mesh, how can it start the load test? + +In order for Flagger to be able to call the load tester service from outside the mesh, you need to disable mTLS on port 80: + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: flagger-namespace + namespace: flagger +spec: + host: "*.flagger.svc.cluster.local" + trafficPolicy: + tls: + mode: DISABLE +--- +apiVersion: authentication.istio.io/v1alpha1 +kind: Policy +metadata: + name: loadtester-mtls-disabled + namespace: flagger +spec: + targets: + - name: flagger-loadtester + ports: + - number: 80 +``` diff --git a/docs/gitbook/install/flagger-install-on-google-cloud.md b/docs/gitbook/install/flagger-install-on-google-cloud.md index 25dfa3730..9f5a00110 100644 --- a/docs/gitbook/install/flagger-install-on-google-cloud.md +++ b/docs/gitbook/install/flagger-install-on-google-cloud.md @@ -333,10 +333,17 @@ The GKE Istio add-on does not include a Prometheus instance that scrapes the Ist Because Flagger uses the Istio HTTP metrics to run the canary analysis you have to deploy the following Prometheus configuration that's similar to the one that comes with the official Istio Helm chart. +Find the GKE Istio version with: + ```bash -REPO=https://raw.githubusercontent.com/weaveworks/flagger/master +kubectl -n istio-system get deploy istio-pilot -oyaml | grep image: +``` -kubectl apply -f ${REPO}/artifacts/gke/istio-prometheus.yaml +Install Prometheus in istio-system namespace (replace `1.0.6-gke.3` with your version): + +```bash +kubectl -n istio-system apply -f \ +https://storage.googleapis.com/gke-release/istio/release/1.0.6-gke.3/patches/install-prometheus.yaml ``` ### Install Flagger and Grafana diff --git a/go.sum b/go.sum index 39c68efae..0500c0187 100644 --- a/go.sum +++ b/go.sum @@ -141,7 +141,6 @@ github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Z github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -165,6 +164,7 @@ github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1: github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f h1:ShTPMJQes6tubcjzGMODIVG5hlrCeImaBnZzKF2N8SM= github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= @@ -279,7 +279,6 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -306,6 +305,7 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -424,6 +424,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -451,6 +452,7 @@ gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmK google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/pkg/apis/flagger/v1alpha3/types.go b/pkg/apis/flagger/v1alpha3/types.go index f65e6577a..a07900aa1 100755 --- a/pkg/apis/flagger/v1alpha3/types.go +++ b/pkg/apis/flagger/v1alpha3/types.go @@ -115,9 +115,10 @@ type CanaryStatus struct { // CanaryService is used to create ClusterIP services // and Istio Virtual Service type CanaryService struct { - Port int32 `json:"port"` - PortName string `json:"portName,omitempty"` - Timeout string `json:"timeout,omitempty"` + Port int32 `json:"port"` + PortName string `json:"portName,omitempty"` + PortDiscovery bool `json:"portDiscovery"` + Timeout string `json:"timeout,omitempty"` // Istio Gateways []string `json:"gateways,omitempty"` Hosts []string `json:"hosts,omitempty"` diff --git a/pkg/canary/deployer.go b/pkg/canary/deployer.go index e6d86c35e..fdeb2fbc3 100644 --- a/pkg/canary/deployer.go +++ b/pkg/canary/deployer.go @@ -31,27 +31,27 @@ type Deployer struct { } // Initialize creates the primary deployment, hpa, -// scales to zero the canary deployment and returns the pod selector label -func (c *Deployer) Initialize(cd *flaggerv1.Canary) (string, error) { +// scales to zero the canary deployment and returns the pod selector label and container ports +func (c *Deployer) Initialize(cd *flaggerv1.Canary) (label string, ports *map[string]int32, err error) { primaryName := fmt.Sprintf("%s-primary", cd.Spec.TargetRef.Name) - label, err := c.createPrimaryDeployment(cd) + label, ports, err = c.createPrimaryDeployment(cd) if err != nil { - return "", fmt.Errorf("creating deployment %s.%s failed: %v", primaryName, cd.Namespace, err) + return "", ports, fmt.Errorf("creating deployment %s.%s failed: %v", primaryName, cd.Namespace, err) } if cd.Status.Phase == "" { c.Logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("Scaling down %s.%s", cd.Spec.TargetRef.Name, cd.Namespace) if err := c.Scale(cd, 0); err != nil { - return "", err + return "", ports, err } } if cd.Spec.AutoscalerRef != nil && cd.Spec.AutoscalerRef.Kind == "HorizontalPodAutoscaler" { if err := c.createPrimaryHpa(cd); err != nil { - return "", fmt.Errorf("creating hpa %s.%s failed: %v", primaryName, cd.Namespace, err) + return "", ports, fmt.Errorf("creating hpa %s.%s failed: %v", primaryName, cd.Namespace, err) } } - return label, nil + return label, ports, nil } // Promote copies the pod spec, secrets and config maps from canary to primary @@ -172,37 +172,46 @@ func (c *Deployer) Scale(cd *flaggerv1.Canary, replicas int32) error { return nil } -func (c *Deployer) createPrimaryDeployment(cd *flaggerv1.Canary) (string, error) { +func (c *Deployer) createPrimaryDeployment(cd *flaggerv1.Canary) (string, *map[string]int32, error) { targetName := cd.Spec.TargetRef.Name primaryName := fmt.Sprintf("%s-primary", cd.Spec.TargetRef.Name) canaryDep, err := c.KubeClient.AppsV1().Deployments(cd.Namespace).Get(targetName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { - return "", fmt.Errorf("deployment %s.%s not found, retrying", targetName, cd.Namespace) + return "", nil, fmt.Errorf("deployment %s.%s not found, retrying", targetName, cd.Namespace) } - return "", err + return "", nil, err } label, err := c.getSelectorLabel(canaryDep) if err != nil { - return "", fmt.Errorf("invalid label selector! Deployment %s.%s spec.selector.matchLabels must contain selector 'app: %s'", + return "", nil, fmt.Errorf("invalid label selector! Deployment %s.%s spec.selector.matchLabels must contain selector 'app: %s'", targetName, cd.Namespace, targetName) } + var ports *map[string]int32 + if cd.Spec.Service.PortDiscovery { + p, err := c.getPorts(canaryDep, cd.Spec.Service.Port) + if err != nil { + return "", nil, fmt.Errorf("port discovery failed with error: %v", err) + } + ports = &p + } + primaryDep, err := c.KubeClient.AppsV1().Deployments(cd.Namespace).Get(primaryName, metav1.GetOptions{}) if errors.IsNotFound(err) { // create primary secrets and config maps configRefs, err := c.ConfigTracker.GetTargetConfigs(cd) if err != nil { - return "", err + return "", nil, err } if err := c.ConfigTracker.CreatePrimaryConfigs(cd, configRefs); err != nil { - return "", err + return "", nil, err } annotations, err := c.makeAnnotations(canaryDep.Spec.Template.Annotations) if err != nil { - return "", err + return "", nil, err } replicas := int32(1) @@ -247,13 +256,13 @@ func (c *Deployer) createPrimaryDeployment(cd *flaggerv1.Canary) (string, error) _, err = c.KubeClient.AppsV1().Deployments(cd.Namespace).Create(primaryDep) if err != nil { - return "", err + return "", nil, err } c.Logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof("Deployment %s.%s created", primaryDep.GetName(), cd.Namespace) } - return label, nil + return label, ports, nil } func (c *Deployer) createPrimaryHpa(cd *flaggerv1.Canary) error { @@ -339,6 +348,37 @@ func (c *Deployer) getSelectorLabel(deployment *appsv1.Deployment) (string, erro return "", fmt.Errorf("selector not found") } +var sidecars = map[string]bool{ + "istio-proxy": true, + "envoy": true, +} + +// getPorts returns a list of all container ports +func (c *Deployer) getPorts(deployment *appsv1.Deployment, canaryPort int32) (map[string]int32, error) { + ports := make(map[string]int32) + + for _, container := range deployment.Spec.Template.Spec.Containers { + // exclude service mesh proxies based on container name + if _, ok := sidecars[container.Name]; ok { + continue + } + for i, p := range container.Ports { + // exclude canary.service.port + if p.ContainerPort == canaryPort { + continue + } + name := fmt.Sprintf("tcp-%s-%v", container.Name, i) + if p.Name != "" { + name = p.Name + } + + ports[name] = p.ContainerPort + } + } + + return ports, nil +} + func makePrimaryLabels(labels map[string]string, primaryName string, label string) map[string]string { res := make(map[string]string) for k, v := range labels { diff --git a/pkg/canary/deployer_test.go b/pkg/canary/deployer_test.go index d978c78c8..cba0bb5db 100644 --- a/pkg/canary/deployer_test.go +++ b/pkg/canary/deployer_test.go @@ -9,7 +9,7 @@ import ( func TestCanaryDeployer_Sync(t *testing.T) { mocks := SetupMocks() - _, err := mocks.deployer.Initialize(mocks.canary) + _, _, err := mocks.deployer.Initialize(mocks.canary) if err != nil { t.Fatal(err.Error()) } @@ -95,7 +95,7 @@ func TestCanaryDeployer_Sync(t *testing.T) { func TestCanaryDeployer_IsNewSpec(t *testing.T) { mocks := SetupMocks() - _, err := mocks.deployer.Initialize(mocks.canary) + _, _, err := mocks.deployer.Initialize(mocks.canary) if err != nil { t.Fatal(err.Error()) } @@ -118,7 +118,7 @@ func TestCanaryDeployer_IsNewSpec(t *testing.T) { func TestCanaryDeployer_Promote(t *testing.T) { mocks := SetupMocks() - _, err := mocks.deployer.Initialize(mocks.canary) + _, _, err := mocks.deployer.Initialize(mocks.canary) if err != nil { t.Fatal(err.Error()) } @@ -163,7 +163,7 @@ func TestCanaryDeployer_Promote(t *testing.T) { func TestCanaryDeployer_IsReady(t *testing.T) { mocks := SetupMocks() - _, err := mocks.deployer.Initialize(mocks.canary) + _, _, err := mocks.deployer.Initialize(mocks.canary) if err != nil { t.Error("Expected primary readiness check to fail") } @@ -181,7 +181,7 @@ func TestCanaryDeployer_IsReady(t *testing.T) { func TestCanaryDeployer_SetFailedChecks(t *testing.T) { mocks := SetupMocks() - _, err := mocks.deployer.Initialize(mocks.canary) + _, _, err := mocks.deployer.Initialize(mocks.canary) if err != nil { t.Fatal(err.Error()) } @@ -203,7 +203,7 @@ func TestCanaryDeployer_SetFailedChecks(t *testing.T) { func TestCanaryDeployer_SetState(t *testing.T) { mocks := SetupMocks() - _, err := mocks.deployer.Initialize(mocks.canary) + _, _, err := mocks.deployer.Initialize(mocks.canary) if err != nil { t.Fatal(err.Error()) } @@ -225,7 +225,7 @@ func TestCanaryDeployer_SetState(t *testing.T) { func TestCanaryDeployer_SyncStatus(t *testing.T) { mocks := SetupMocks() - _, err := mocks.deployer.Initialize(mocks.canary) + _, _, err := mocks.deployer.Initialize(mocks.canary) if err != nil { t.Fatal(err.Error()) } @@ -264,7 +264,7 @@ func TestCanaryDeployer_SyncStatus(t *testing.T) { func TestCanaryDeployer_Scale(t *testing.T) { mocks := SetupMocks() - _, err := mocks.deployer.Initialize(mocks.canary) + _, _, err := mocks.deployer.Initialize(mocks.canary) if err != nil { t.Fatal(err.Error()) } diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index 423fb27f4..61af34750 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -354,6 +354,14 @@ func newTestDeployment() *appsv1.Deployment { ContainerPort: 9898, Protocol: corev1.ProtocolTCP, }, + { + Name: "http-metrics", + ContainerPort: 8080, + Protocol: corev1.ProtocolTCP, + }, + { + ContainerPort: 8888, + }, }, Env: []corev1.EnvVar{ { diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 2a1e5c6f5..777f27ab6 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -91,7 +91,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh primaryName := fmt.Sprintf("%s-primary", cd.Spec.TargetRef.Name) // create primary deployment and hpa if needed - label, err := c.deployer.Initialize(cd) + label, ports, err := c.deployer.Initialize(cd) if err != nil { c.recordEventWarningf(cd, "%v", err) return @@ -101,7 +101,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh meshRouter := c.routerFactory.MeshRouter(c.meshProvider) // create or update ClusterIP services - if err := c.routerFactory.KubernetesRouter(label).Reconcile(cd); err != nil { + if err := c.routerFactory.KubernetesRouter(label, ports).Reconcile(cd); err != nil { c.recordEventWarningf(cd, "%v", err) return } diff --git a/pkg/controller/scheduler_test.go b/pkg/controller/scheduler_test.go index 5e9e26ec2..32ba819d2 100644 --- a/pkg/controller/scheduler_test.go +++ b/pkg/controller/scheduler_test.go @@ -1,6 +1,7 @@ package controller import ( + "fmt" "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "testing" @@ -329,3 +330,47 @@ func TestScheduler_ABTesting(t *testing.T) { t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded) } } + +func TestScheduler_PortDiscovery(t *testing.T) { + mocks := SetupMocks(false) + + // enable port discovery + cd, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + cd.Spec.Service.PortDiscovery = true + _, err = mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Update(cd) + if err != nil { + t.Fatal(err.Error()) + } + + mocks.ctrl.advanceCanary("podinfo", "default", true) + + canarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get("podinfo-canary", metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + if len(canarySvc.Spec.Ports) != 3 { + t.Fatalf("Got svc port count %v wanted %v", len(canarySvc.Spec.Ports), 3) + } + + matchPorts := func(lookup string) bool { + switch lookup { + case + "http 9898", + "http-metrics 8080", + "tcp-podinfo-2 8888": + return true + } + return false + } + + for _, port := range canarySvc.Spec.Ports { + if !matchPorts(fmt.Sprintf("%s %v", port.Name, port.Port)) { + t.Fatalf("Got wrong svc port %v", port.Name) + } + + } +} diff --git a/pkg/router/factory.go b/pkg/router/factory.go index 212b5460d..c330dbac1 100644 --- a/pkg/router/factory.go +++ b/pkg/router/factory.go @@ -32,12 +32,13 @@ func NewFactory(kubeConfig *restclient.Config, kubeClient kubernetes.Interface, } // KubernetesRouter returns a ClusterIP service router -func (factory *Factory) KubernetesRouter(label string) *KubernetesRouter { +func (factory *Factory) KubernetesRouter(label string, ports *map[string]int32) *KubernetesRouter { return &KubernetesRouter{ logger: factory.logger, flaggerClient: factory.flaggerClient, kubeClient: factory.kubeClient, label: label, + ports: ports, } } diff --git a/pkg/router/istio.go b/pkg/router/istio.go index 7c12878a4..e7f45122e 100644 --- a/pkg/router/istio.go +++ b/pkg/router/istio.go @@ -102,6 +102,7 @@ func (ir *IstioRouter) reconcileDestinationRule(canary *flaggerv1.Canary, name s func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error { targetName := canary.Spec.TargetRef.Name primaryName := fmt.Sprintf("%s-primary", targetName) + canaryName := fmt.Sprintf("%s-canary", targetName) // set hosts and add the ClusterIP service host if it doesn't exists hosts := canary.Spec.Service.Hosts @@ -133,18 +134,8 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error { // create destinations with primary weight 100% and canary weight 0% canaryRoute := []istiov1alpha3.DestinationWeight{ - { - Destination: istiov1alpha3.Destination{ - Host: primaryName, - }, - Weight: 100, - }, - { - Destination: istiov1alpha3.Destination{ - Host: fmt.Sprintf("%s-canary", targetName), - }, - Weight: 0, - }, + makeDestination(canary, primaryName, 100), + makeDestination(canary, canaryName, 0), } newSpec := istiov1alpha3.VirtualServiceSpec{ @@ -183,12 +174,7 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error { CorsPolicy: canary.Spec.Service.CorsPolicy, AppendHeaders: addHeaders(canary), Route: []istiov1alpha3.DestinationWeight{ - { - Destination: istiov1alpha3.Destination{ - Host: primaryName, - }, - Weight: 100, - }, + makeDestination(canary, primaryName, 100), }, }, } @@ -294,6 +280,9 @@ func (ir *IstioRouter) SetRoutes( canaryWeight int, ) error { targetName := canary.Spec.TargetRef.Name + primaryName := fmt.Sprintf("%s-primary", targetName) + canaryName := fmt.Sprintf("%s-canary", targetName) + vs, err := ir.istioClient.NetworkingV1alpha3().VirtualServices(canary.Namespace).Get(targetName, v1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { @@ -315,18 +304,8 @@ func (ir *IstioRouter) SetRoutes( CorsPolicy: canary.Spec.Service.CorsPolicy, AppendHeaders: addHeaders(canary), Route: []istiov1alpha3.DestinationWeight{ - { - Destination: istiov1alpha3.Destination{ - Host: fmt.Sprintf("%s-primary", targetName), - }, - Weight: primaryWeight, - }, - { - Destination: istiov1alpha3.Destination{ - Host: fmt.Sprintf("%s-canary", targetName), - }, - Weight: canaryWeight, - }, + makeDestination(canary, primaryName, primaryWeight), + makeDestination(canary, canaryName, canaryWeight), }, }, } @@ -344,18 +323,8 @@ func (ir *IstioRouter) SetRoutes( CorsPolicy: canary.Spec.Service.CorsPolicy, AppendHeaders: addHeaders(canary), Route: []istiov1alpha3.DestinationWeight{ - { - Destination: istiov1alpha3.Destination{ - Host: fmt.Sprintf("%s-primary", targetName), - }, - Weight: primaryWeight, - }, - { - Destination: istiov1alpha3.Destination{ - Host: fmt.Sprintf("%s-canary", targetName), - }, - Weight: canaryWeight, - }, + makeDestination(canary, primaryName, primaryWeight), + makeDestination(canary, canaryName, canaryWeight), }, }, { @@ -366,12 +335,7 @@ func (ir *IstioRouter) SetRoutes( CorsPolicy: canary.Spec.Service.CorsPolicy, AppendHeaders: addHeaders(canary), Route: []istiov1alpha3.DestinationWeight{ - { - Destination: istiov1alpha3.Destination{ - Host: fmt.Sprintf("%s-primary", targetName), - }, - Weight: primaryWeight, - }, + makeDestination(canary, primaryName, primaryWeight), }, }, } @@ -409,3 +373,22 @@ func mergeMatchConditions(canary, defaults []istiov1alpha3.HTTPMatchRequest) []i return canary } + +// makeDestination returns a an destination weight for the specified host +func makeDestination(canary *flaggerv1.Canary, host string, weight int) istiov1alpha3.DestinationWeight { + dest := istiov1alpha3.DestinationWeight{ + Destination: istiov1alpha3.Destination{ + Host: host, + }, + Weight: weight, + } + + // if port discovery is enabled then we need to explicitly set the destination port + if canary.Spec.Service.PortDiscovery { + dest.Destination.Port = &istiov1alpha3.PortSelector{ + Number: uint32(canary.Spec.Service.Port), + } + } + + return dest +} diff --git a/pkg/router/kubernetes.go b/pkg/router/kubernetes.go index 2d38dfe6f..8760a41ff 100644 --- a/pkg/router/kubernetes.go +++ b/pkg/router/kubernetes.go @@ -20,6 +20,7 @@ type KubernetesRouter struct { flaggerClient clientset.Interface logger *zap.SugaredLogger label string + ports *map[string]int32 } // Reconcile creates or updates the primary and canary services @@ -79,6 +80,22 @@ func (c *KubernetesRouter) reconcileService(canary *flaggerv1.Canary, name strin }, } + if c.ports != nil { + for n, p := range *c.ports { + cp := corev1.ServicePort{ + Name: n, + Protocol: corev1.ProtocolTCP, + Port: p, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: p, + }, + } + + svcSpec.Ports = append(svcSpec.Ports, cp) + } + } + svc, err := c.kubeClient.CoreV1().Services(canary.Namespace).Get(name, metav1.GetOptions{}) if errors.IsNotFound(err) { svc = &corev1.Service{ @@ -114,6 +131,7 @@ func (c *KubernetesRouter) reconcileService(canary *flaggerv1.Canary, name strin if diff := cmp.Diff(svcSpec.Ports, svc.Spec.Ports); diff != "" { svcClone := svc.DeepCopy() svcClone.Spec = svcSpec + svcClone.Spec.ClusterIP = svc.Spec.ClusterIP _, err = c.kubeClient.CoreV1().Services(canary.Namespace).Update(svcClone) if err != nil { return fmt.Errorf("service %s update error %v", name, err) diff --git a/test/e2e-istio.sh b/test/e2e-istio.sh index 933eaee29..b57db4640 100755 --- a/test/e2e-istio.sh +++ b/test/e2e-istio.sh @@ -2,7 +2,7 @@ set -o errexit -ISTIO_VER="1.1.8" +ISTIO_VER="1.1.9" REPO_ROOT=$(git rev-parse --show-toplevel) export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"