pkg/metrics/providers: add AWS CloudWatch provider

artifacts/flagger/crd.yaml

@@ -733,7 +733,6 @@ spec:
               type: object
               required:
                 - type
-                - address
               properties:
                 type:
                   description: Type of this provider
@@ -742,6 +741,7 @@ spec:
                     - prometheus
                     - influxdb
                     - datadog
+                    - cloudwatch
                 address:
                   description: API address of this provider
                   type: string
@@ -754,6 +754,9 @@ spec:
                     name:
                       description: Name of the Kubernetes secret
                       type: string
+                region:
+                  description: Region of the provider
+                  type: string
             query:
               description: Query of this metric template
               type: string

charts/flagger/crds/crd.yaml

@@ -733,7 +733,6 @@ spec:
               type: object
               required:
                 - type
-                - address
               properties:
                 type:
                   description: Type of this provider
@@ -742,6 +741,7 @@ spec:
                     - prometheus
                     - influxdb
                     - datadog
+                    - cloudwatch
                 address:
                   description: API address of this provider
                   type: string
@@ -754,6 +754,9 @@ spec:
                     name:
                       description: Name of the Kubernetes secret
                       type: string
+                region:
+                  description: Region of the provider
+                  type: string
             query:
               description: Query of this metric template
               type: string

docs/gitbook/usage/metrics.md

@@ -233,3 +233,86 @@ Reference the template in the canary analysis:
           max: 5
         interval: 1m
 ```
+
+
+### AWS CloudWatch metrics
+
+You can create custom metric checks using the AWS CloudWatch metrics provider.
+
+The template example:
+
+```yaml
+apiVersion: flagger.app/v1alpha1
+kind: MetricTemplate
+metadata:
+  name: cloudwatch-error-rate
+  namespace: istio-system
+spec:
+  provider:
+    type: cloudwatch
+    region: ap-northeast-1 # specify the region of your metrics
+  query: |
+    [
+        {
+            "Id": "e1",
+            "Expression": "m1 / m2",
+            "Label": "ErrorRate"
+        },
+        {
+            "Id": "m1",
+            "MetricStat": {
+                "Metric": {
+                    "Namespace": "MyKubernetesCluster",
+                    "MetricName": "ErrorCount",
+                    "Dimensions": [
+                        {
+                            "Name": "appName",
+                            "Value": "{{ name }}.{{ namespace }}"
+                        }
+                    ]
+                },
+                "Period": 60,
+                "Stat": "Sum",
+                "Unit": "Count"
+            },
+            "ReturnData": false
+        },
+        {
+            "Id": "m2",
+            "MetricStat": {
+                "Metric": {
+                    "Namespace": "MyKubernetesCluster",
+                    "MetricName": "RequestCount",
+                    "Dimensions": [
+                        {
+                            "Name": "appName",
+                            "Value": "{{ name }}.{{ namespace }}"
+                        }
+                    ]
+                },
+                "Period": 60,
+                "Stat": "Sum",
+                "Unit": "Count"
+            },
+            "ReturnData": false
+        }
+    ]
+```
+
+where the query is in the form as in [the AWS' official document](https://aws.amazon.com/premiumsupport/knowledge-center/cloudwatch-getmetricdata-api/).
+
+Reference the template in the canary analysis:
+
+```yaml
+  analysis:
+    metrics:
+      - name: "cw custom error rate"
+        templateRef:
+          name: cloudwatch-error-rate
+          namespace: istio-system
+        thresholdRange:
+          max: 0.1
+        interval: 1m
+```
+
+Please note that the flagger need AWS IAM permission to perform `cloudwatch:GetMetricData` to use this provider.

go.mod

@@ -4,12 +4,13 @@ go 1.13
 
 require (
 	github.com/Masterminds/semver/v3 v3.0.3
+	github.com/aws/aws-sdk-go v1.29.12
 	github.com/davecgh/go-spew v1.1.1
 	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
 	github.com/google/go-cmp v0.3.0
 	github.com/googleapis/gnostic v0.2.0 // indirect
 	github.com/imdario/mergo v0.3.7 // indirect
-	github.com/pkg/errors v0.8.1
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.0.0
 	go.uber.org/atomic v1.3.2 // indirect
 	go.uber.org/multierr v1.1.0 // indirect

go.sum

@@ -20,6 +20,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/aws/aws-sdk-go v1.29.12 h1:V2Z0/kQZsogK/KqkGkgcbcFh1pH7fhXWY+6aTty2G3k=
+github.com/aws/aws-sdk-go v1.29.12/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -38,7 +40,9 @@ github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
@@ -51,6 +55,7 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
@@ -96,6 +101,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
 github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -141,8 +148,8 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -210,6 +217,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

kustomize/base/flagger/crd.yaml

@@ -733,7 +733,6 @@ spec:
               type: object
               required:
                 - type
-                - address
               properties:
                 type:
                   description: Type of this provider
@@ -742,6 +741,7 @@ spec:
                     - prometheus
                     - influxdb
                     - datadog
+                    - cloudwatch
                 address:
                   description: API address of this provider
                   type: string
@@ -754,6 +754,9 @@ spec:
                     name:
                       description: Name of the Kubernetes secret
                       type: string
+                region:
+                  description: Region of the provider
+                  type: string
             query:
               description: Query of this metric template
               type: string

pkg/apis/flagger/v1beta1/metric.go

@@ -64,11 +64,16 @@ type MetricTemplateProvider struct {
 	Type string `json:"type,omitempty"`
 
 	// HTTP(S) address of this provider
+	// +optional
 	Address string `json:"address,omitempty"`
 
 	// Secret reference containing the provider credentials
 	// +optional
 	SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"`
+
+	// Region of the provider
+	// +optional
+	Region string `json:"region,omitempty"`
 }
 
 // MetricTemplateModel is the query template model

pkg/metrics/providers/cloudwatch.go

@@ -0,0 +1,113 @@
+package providers
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/cloudwatch"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
+)
+
+const (
+	cloudWatchMaxRetries                           = 3
+	cloudWatchStartDeltaMultiplierOnMetricInterval = 10
+)
+
+type CloudWatchProvider struct {
+	client     cloudWatchClient
+	startDelta time.Duration
+}
+
+// for the testing purpose
+type cloudWatchClient interface {
+	GetMetricData(input *cloudwatch.GetMetricDataInput) (*cloudwatch.GetMetricDataOutput, error)
+}
+
+// NewCloudWatchProvider takes a metricInterval, a provider spec and the credentials map, and
+// returns a cloudWatchProvider ready to execute queries against the AWS CloudWatch metrics
+func NewCloudWatchProvider(metricInterval string, provider flaggerv1.MetricTemplateProvider) (*CloudWatchProvider, error) {
+	if provider.Region == "" {
+		return nil, fmt.Errorf("region not specified")
+	}
+
+	sess, err := session.NewSession(aws.NewConfig().
+		WithRegion(provider.Region).WithMaxRetries(cloudWatchMaxRetries))
+
+	if err != nil {
+		return nil, fmt.Errorf("error creating aws session: %s", err.Error())
+	}
+
+	md, err := time.ParseDuration(metricInterval)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing metric interval: %s", err.Error())
+	}
+
+	return &CloudWatchProvider{
+		client:     cloudwatch.New(sess),
+		startDelta: cloudWatchStartDeltaMultiplierOnMetricInterval * md,
+	}, err
+}
+
+// RunQuery executes the aws cloud watch metrics query against GetMetricData endpoint
+// and returns the the first result as float64
+func (p *CloudWatchProvider) RunQuery(query string) (float64, error) {
+	var cq []*cloudwatch.MetricDataQuery
+	if err := json.Unmarshal([]byte(query), &cq); err != nil {
+		return 0, fmt.Errorf("error unmarshaling query: %s", err.Error())
+	}
+
+	end := time.Now()
+	start := end.Add(-p.startDelta)
+	res, err := p.client.GetMetricData(&cloudwatch.GetMetricDataInput{
+		EndTime:           aws.Time(end),
+		MaxDatapoints:     aws.Int64(20),
+		StartTime:         aws.Time(start),
+		MetricDataQueries: cq,
+	})
+
+	if err != nil {
+		return 0, fmt.Errorf("error requesting cloudwatch: %s", err.Error())
+	}
+
+	mr := res.MetricDataResults
+	if len(mr) < 1 {
+		return 0, fmt.Errorf("no values found in response: %s", res.String())
+	}
+
+	vs := res.MetricDataResults[0].Values
+	if len(vs) < 1 {
+		return 0, fmt.Errorf("no values found in response: %s", res.String())
+	}
+
+	return aws.Float64Value(vs[0]), nil
+}
+
+// IsOnline calls GetMetricData endpoint with the empty query
+// and returns an error if the returned status code is NOT http.StatusBadRequests.
+// For example, if the flagger does not have permission to perform `cloudwatch:GetMetricData`,
+// the returned status code would be http.StatusForbidden
+func (p *CloudWatchProvider) IsOnline() (bool, error) {
+	_, err := p.client.GetMetricData(&cloudwatch.GetMetricDataInput{
+		EndTime:           aws.Time(time.Time{}),
+		MetricDataQueries: []*cloudwatch.MetricDataQuery{},
+		StartTime:         aws.Time(time.Time{}),
+	})
+
+	if err == nil {
+		return true, nil
+	}
+
+	ae, ok := err.(awserr.RequestFailure)
+	if !ok {
+		return false, fmt.Errorf("unexpected error: %v", err)
+	} else if ae.StatusCode() != http.StatusBadRequest {
+		return false, fmt.Errorf("unexpected status code: %v", ae)
+	}
+	return true, nil
+}