Skip to content

Commit

Permalink
Merge pull request #464 from weaveworks/cloud-watch-metrics
Browse files Browse the repository at this point in the history
pkg/metrics/providers: add AWS CloudWatch provider
  • Loading branch information
stefanprodan authored Mar 3, 2020
2 parents 62f2851 + ecb8207 commit b47cfb6
Show file tree
Hide file tree
Showing 11 changed files with 419 additions and 11 deletions.
5 changes: 4 additions & 1 deletion artifacts/flagger/crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -733,7 +733,6 @@ spec:
type: object
required:
- type
- address
properties:
type:
description: Type of this provider
Expand All @@ -742,6 +741,7 @@ spec:
- prometheus
- influxdb
- datadog
- cloudwatch
address:
description: API address of this provider
type: string
Expand All @@ -754,6 +754,9 @@ spec:
name:
description: Name of the Kubernetes secret
type: string
region:
description: Region of the provider
type: string
query:
description: Query of this metric template
type: string
Expand Down
5 changes: 4 additions & 1 deletion charts/flagger/crds/crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -733,7 +733,6 @@ spec:
type: object
required:
- type
- address
properties:
type:
description: Type of this provider
Expand All @@ -742,6 +741,7 @@ spec:
- prometheus
- influxdb
- datadog
- cloudwatch
address:
description: API address of this provider
type: string
Expand All @@ -754,6 +754,9 @@ spec:
name:
description: Name of the Kubernetes secret
type: string
region:
description: Region of the provider
type: string
query:
description: Query of this metric template
type: string
Expand Down
83 changes: 83 additions & 0 deletions docs/gitbook/usage/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -233,3 +233,86 @@ Reference the template in the canary analysis:
max: 5
interval: 1m
```


### AWS CloudWatch metrics

You can create custom metric checks using the AWS CloudWatch metrics provider.

The template example:

```yaml
apiVersion: flagger.app/v1alpha1
kind: MetricTemplate
metadata:
name: cloudwatch-error-rate
namespace: istio-system
spec:
provider:
type: cloudwatch
region: ap-northeast-1 # specify the region of your metrics
query: |
[
{
"Id": "e1",
"Expression": "m1 / m2",
"Label": "ErrorRate"
},
{
"Id": "m1",
"MetricStat": {
"Metric": {
"Namespace": "MyKubernetesCluster",
"MetricName": "ErrorCount",
"Dimensions": [
{
"Name": "appName",
"Value": "{{ name }}.{{ namespace }}"
}
]
},
"Period": 60,
"Stat": "Sum",
"Unit": "Count"
},
"ReturnData": false
},
{
"Id": "m2",
"MetricStat": {
"Metric": {
"Namespace": "MyKubernetesCluster",
"MetricName": "RequestCount",
"Dimensions": [
{
"Name": "appName",
"Value": "{{ name }}.{{ namespace }}"
}
]
},
"Period": 60,
"Stat": "Sum",
"Unit": "Count"
},
"ReturnData": false
}
]
```

where the query is in the form as in [the AWS' official document](https://aws.amazon.com/premiumsupport/knowledge-center/cloudwatch-getmetricdata-api/).

Reference the template in the canary analysis:

```yaml
analysis:
metrics:
- name: "cw custom error rate"
templateRef:
name: cloudwatch-error-rate
namespace: istio-system
thresholdRange:
max: 0.1
interval: 1m
```

Please note that the flagger need AWS IAM permission to perform `cloudwatch:GetMetricData` to use this provider.
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@ go 1.13

require (
github.com/Masterminds/semver/v3 v3.0.3
github.com/aws/aws-sdk-go v1.29.12
github.com/davecgh/go-spew v1.1.1
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
github.com/google/go-cmp v0.3.0
github.com/googleapis/gnostic v0.2.0 // indirect
github.com/imdario/mergo v0.3.7 // indirect
github.com/pkg/errors v0.8.1
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.0.0
go.uber.org/atomic v1.3.2 // indirect
go.uber.org/multierr v1.1.0 // indirect
Expand Down
13 changes: 11 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/aws/aws-sdk-go v1.29.12 h1:V2Z0/kQZsogK/KqkGkgcbcFh1pH7fhXWY+6aTty2G3k=
github.com/aws/aws-sdk-go v1.29.12/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
Expand All @@ -38,7 +40,9 @@ github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
Expand All @@ -51,6 +55,7 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8
github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
Expand Down Expand Up @@ -96,6 +101,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
Expand Down Expand Up @@ -141,8 +148,8 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
Expand Down Expand Up @@ -210,6 +217,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down
5 changes: 4 additions & 1 deletion kustomize/base/flagger/crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -733,7 +733,6 @@ spec:
type: object
required:
- type
- address
properties:
type:
description: Type of this provider
Expand All @@ -742,6 +741,7 @@ spec:
- prometheus
- influxdb
- datadog
- cloudwatch
address:
description: API address of this provider
type: string
Expand All @@ -754,6 +754,9 @@ spec:
name:
description: Name of the Kubernetes secret
type: string
region:
description: Region of the provider
type: string
query:
description: Query of this metric template
type: string
Expand Down
5 changes: 5 additions & 0 deletions pkg/apis/flagger/v1beta1/metric.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,16 @@ type MetricTemplateProvider struct {
Type string `json:"type,omitempty"`

// HTTP(S) address of this provider
// +optional
Address string `json:"address,omitempty"`

// Secret reference containing the provider credentials
// +optional
SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"`

// Region of the provider
// +optional
Region string `json:"region,omitempty"`
}

// MetricTemplateModel is the query template model
Expand Down
113 changes: 113 additions & 0 deletions pkg/metrics/providers/cloudwatch.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
package providers

import (
"encoding/json"
"fmt"
"net/http"
"time"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatch"

flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
)

const (
cloudWatchMaxRetries = 3
cloudWatchStartDeltaMultiplierOnMetricInterval = 10
)

type CloudWatchProvider struct {
client cloudWatchClient
startDelta time.Duration
}

// for the testing purpose
type cloudWatchClient interface {
GetMetricData(input *cloudwatch.GetMetricDataInput) (*cloudwatch.GetMetricDataOutput, error)
}

// NewCloudWatchProvider takes a metricInterval, a provider spec and the credentials map, and
// returns a cloudWatchProvider ready to execute queries against the AWS CloudWatch metrics
func NewCloudWatchProvider(metricInterval string, provider flaggerv1.MetricTemplateProvider) (*CloudWatchProvider, error) {
if provider.Region == "" {
return nil, fmt.Errorf("region not specified")
}

sess, err := session.NewSession(aws.NewConfig().
WithRegion(provider.Region).WithMaxRetries(cloudWatchMaxRetries))

if err != nil {
return nil, fmt.Errorf("error creating aws session: %s", err.Error())
}

md, err := time.ParseDuration(metricInterval)
if err != nil {
return nil, fmt.Errorf("error parsing metric interval: %s", err.Error())
}

return &CloudWatchProvider{
client: cloudwatch.New(sess),
startDelta: cloudWatchStartDeltaMultiplierOnMetricInterval * md,
}, err
}

// RunQuery executes the aws cloud watch metrics query against GetMetricData endpoint
// and returns the the first result as float64
func (p *CloudWatchProvider) RunQuery(query string) (float64, error) {
var cq []*cloudwatch.MetricDataQuery
if err := json.Unmarshal([]byte(query), &cq); err != nil {
return 0, fmt.Errorf("error unmarshaling query: %s", err.Error())
}

end := time.Now()
start := end.Add(-p.startDelta)
res, err := p.client.GetMetricData(&cloudwatch.GetMetricDataInput{
EndTime: aws.Time(end),
MaxDatapoints: aws.Int64(20),
StartTime: aws.Time(start),
MetricDataQueries: cq,
})

if err != nil {
return 0, fmt.Errorf("error requesting cloudwatch: %s", err.Error())
}

mr := res.MetricDataResults
if len(mr) < 1 {
return 0, fmt.Errorf("no values found in response: %s", res.String())
}

vs := res.MetricDataResults[0].Values
if len(vs) < 1 {
return 0, fmt.Errorf("no values found in response: %s", res.String())
}

return aws.Float64Value(vs[0]), nil
}

// IsOnline calls GetMetricData endpoint with the empty query
// and returns an error if the returned status code is NOT http.StatusBadRequests.
// For example, if the flagger does not have permission to perform `cloudwatch:GetMetricData`,
// the returned status code would be http.StatusForbidden
func (p *CloudWatchProvider) IsOnline() (bool, error) {
_, err := p.client.GetMetricData(&cloudwatch.GetMetricDataInput{
EndTime: aws.Time(time.Time{}),
MetricDataQueries: []*cloudwatch.MetricDataQuery{},
StartTime: aws.Time(time.Time{}),
})

if err == nil {
return true, nil
}

ae, ok := err.(awserr.RequestFailure)
if !ok {
return false, fmt.Errorf("unexpected error: %v", err)
} else if ae.StatusCode() != http.StatusBadRequest {
return false, fmt.Errorf("unexpected status code: %v", ae)
}
return true, nil
}
Loading

0 comments on commit b47cfb6

Please sign in to comment.