Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement MS Teams notifications #235

Merged
merged 6 commits into from
Jul 7, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions charts/flagger/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,24 @@ Add Flagger Helm repository:
helm repo add flagger https://flagger.app
```

To install the chart with the release name `flagger`:
To install the chart with the release name `flagger` for Istio:

```console
$ helm install --name flagger --namespace istio-system flagger/flagger
$ helm upgrade -i flagger flagger/flagger \
--namespace=istio-system \
--set meshProvider=istio \
--set metricsServer=http://prometheus:9090
```

To install the chart with the release name `flagger` for Linkerd:

```console
$ helm upgrade -i flagger flagger/flagger \
--namespace=linkerd \
--set meshProvider=linkerd \
--set metricsServer=http://linkerd-prometheus:9090
```

The command deploys Flagger on the Kubernetes cluster in the istio-system namespace.
The [configuration](#configuration) section lists the parameters that can be configured during installation.

## Uninstalling the Chart
Expand All @@ -52,6 +63,7 @@ Parameter | Description | Default
`slack.url` | Slack incoming webhook | None
`slack.channel` | Slack channel | None
`slack.user` | Slack username | `flagger`
`msteams.url` | Microsoft Teams incoming webhook | None
`rbac.create` | if `true`, create and use RBAC resources | `true`
`rbac.pspEnabled` | If `true`, create and use a restricted pod security policy | `false`
`crd.create` | if `true`, create Flagger's CRDs | `true`
Expand Down
3 changes: 3 additions & 0 deletions charts/flagger/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ spec:
- -slack-user={{ .Values.slack.user }}
- -slack-channel={{ .Values.slack.channel }}
{{- end }}
{{- if .Values.msteams.url }}
- -msteams-url={{ .Values.msteams.url }}
{{- end }}
livenessProbe:
exec:
command:
Expand Down
4 changes: 4 additions & 0 deletions charts/flagger/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ slack:
# incoming webhook https://api.slack.com/incoming-webhooks
url:

msteams:
# MS Teams incoming webhook URL
url:

serviceAccount:
# serviceAccount.create: Whether to create a service account or not
create: true
Expand Down
37 changes: 26 additions & 11 deletions cmd/flagger/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ var (
controlLoopInterval time.Duration
logLevel string
port string
msteamsURL string
slackURL string
slackUser string
slackChannel string
Expand All @@ -56,6 +57,7 @@ func init() {
flag.StringVar(&slackURL, "slack-url", "", "Slack hook URL.")
flag.StringVar(&slackUser, "slack-user", "flagger", "Slack user name.")
flag.StringVar(&slackChannel, "slack-channel", "", "Slack channel.")
flag.StringVar(&msteamsURL, "msteams-url", "", "MS Teams incoming webhook URL.")
flag.IntVar(&threadiness, "threadiness", 2, "Worker concurrency.")
flag.BoolVar(&zapReplaceGlobals, "zap-replace-globals", false, "Whether to change the logging level of the global zap logger.")
flag.StringVar(&zapEncoding, "zap-encoding", "json", "Zap logger encoding.")
Expand Down Expand Up @@ -158,15 +160,8 @@ func main() {
logger.Errorf("Metrics server %s unreachable %v", metricsServer, err)
}

var slack *notifier.Slack
if slackURL != "" {
slack, err = notifier.NewSlack(slackURL, slackUser, slackChannel)
if err != nil {
logger.Errorf("Notifier %v", err)
} else {
logger.Infof("Slack notifications enabled for channel %s", slack.Channel)
}
}
// setup Slack or MS Teams notifications
notifierClient := initNotifier(logger)

// start HTTP server
go server.ListenAndServe(port, 3*time.Second, logger, stopCh)
Expand All @@ -179,9 +174,8 @@ func main() {
flaggerClient,
canaryInformer,
controlLoopInterval,
metricsServer,
logger,
slack,
notifierClient,
routerFactory,
observerFactory,
meshProvider,
Expand Down Expand Up @@ -209,3 +203,24 @@ func main() {

<-stopCh
}

func initNotifier(logger *zap.SugaredLogger) (client notifier.Interface) {
provider := "slack"
notifierURL := slackURL
if msteamsURL != "" {
provider = "msteams"
notifierURL = msteamsURL
}
notifierFactory := notifier.NewFactory(notifierURL, slackUser, slackChannel)

if notifierURL != "" {
var err error
client, err = notifierFactory.Notifier(provider)
if err != nil {
logger.Errorf("Notifier %v", err)
} else {
logger.Infof("Notifications enabled for %s", notifierURL[0:30])
}
}
return
}
4 changes: 2 additions & 2 deletions docs/gitbook/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ description: Flagger is a progressive delivery Kubernetes operator
# Introduction

[Flagger](https://github.com/weaveworks/flagger) is a **Kubernetes** operator that automates the promotion of canary
deployments using **Istio**, **App Mesh**, **NGINX** or **Gloo** routing for traffic shifting and **Prometheus** metrics for canary analysis.
deployments using **Istio**, **Linkerd**, **App Mesh**, **NGINX** or **Gloo** routing for traffic shifting and **Prometheus** metrics for canary analysis.
The canary analysis can be extended with webhooks for running system integration/acceptance tests, load tests, or any other custom validation.

Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance
indicators like HTTP requests success rate, requests average duration and pods health.
Based on analysis of the **KPIs** a canary is promoted or aborted, and the analysis result is published to **Slack**.
Based on analysis of the **KPIs** a canary is promoted or aborted, and the analysis result is published to **Slack** or **MS Teams**.

![Flagger overview diagram](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-overview.png)

Expand Down
8 changes: 8 additions & 0 deletions docs/gitbook/install/flagger-install-on-kubernetes.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,14 @@ helm upgrade -i flagger flagger/flagger \
--set slack.user=flagger
```

Enable **Microsoft Teams** notifications:

```bash
helm upgrade -i flagger flagger/flagger \
--namespace=istio-system \
--set msteams.url=https://outlook.office.com/webhook/YOUR/TEAMS/WEBHOOK
```

If you don't have Tiller you can use the helm template command and apply the generated yaml with kubectl:

```bash
Expand Down
18 changes: 17 additions & 1 deletion docs/gitbook/usage/alerting.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ Flagger can be configured to send Slack notifications:

```bash
helm upgrade -i flagger flagger/flagger \
--namespace=istio-system \
--set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \
--set slack.channel=general \
--set slack.user=flagger
Expand All @@ -22,6 +21,23 @@ maximum number of failed checks:

![Slack Notifications](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/screens/slack-canary-failed.png)

### Microsoft Teams

Flagger can be configured to send notifications to Microsoft Teams:

```bash
helm upgrade -i flagger flagger/flagger \
--set msteams.url=https://outlook.office.com/webhook/YOUR/TEAMS/WEBHOOK
```

Flagger will post a message card to MS Teams when a new revision has been detected and if the canary analysis failed or succeeded:

![MS Teams Notifications](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/screens/flagger-ms-teams-notifications.png)

And you'll get a notification on rollback:

![MS Teams Notifications](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/screens/flagger-ms-teams-failed.png)

### Prometheus Alert Manager

Besides Slack, you can use Alertmanager to trigger alerts when a canary deployment failed:
Expand Down
Binary file added docs/screens/flagger-ms-teams-failed.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added docs/screens/flagger-ms-teams-notifications.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
44 changes: 28 additions & 16 deletions pkg/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ type Controller struct {
jobs map[string]CanaryJob
deployer canary.Deployer
recorder metrics.Recorder
notifier *notifier.Slack
notifier notifier.Interface
routerFactory *router.Factory
observerFactory *metrics.Factory
meshProvider string
Expand All @@ -58,9 +58,8 @@ func NewController(
flaggerClient clientset.Interface,
flaggerInformer flaggerinformers.CanaryInformer,
flaggerWindow time.Duration,
metricServer string,
logger *zap.SugaredLogger,
notifier *notifier.Slack,
notifier notifier.Interface,
routerFactory *router.Factory,
observerFactory *metrics.Factory,
meshProvider string,
Expand Down Expand Up @@ -271,29 +270,42 @@ func (c *Controller) sendNotification(cd *flaggerv1.Canary, message string, meta
return
}

var fields []notifier.SlackField
var fields []notifier.Field

if metadata {
fields = append(fields,
notifier.SlackField{
Title: "Target",
notifier.Field{
Name: "Target",
Value: fmt.Sprintf("%s/%s.%s", cd.Spec.TargetRef.Kind, cd.Spec.TargetRef.Name, cd.Namespace),
},
notifier.SlackField{
Title: "Traffic routing",
Value: fmt.Sprintf("Weight step: %v max: %v",
cd.Spec.CanaryAnalysis.StepWeight,
cd.Spec.CanaryAnalysis.MaxWeight),
},
notifier.SlackField{
Title: "Failed checks threshold",
notifier.Field{
Name: "Failed checks threshold",
Value: fmt.Sprintf("%v", cd.Spec.CanaryAnalysis.Threshold),
},
notifier.SlackField{
Title: "Progress deadline",
notifier.Field{
Name: "Progress deadline",
Value: fmt.Sprintf("%vs", cd.GetProgressDeadlineSeconds()),
},
)

if cd.Spec.CanaryAnalysis.StepWeight > 0 {
fields = append(fields, notifier.Field{
Name: "Traffic routing",
Value: fmt.Sprintf("Weight step: %v max: %v",
cd.Spec.CanaryAnalysis.StepWeight,
cd.Spec.CanaryAnalysis.MaxWeight),
})
} else if len(cd.Spec.CanaryAnalysis.Match) > 0 {
fields = append(fields, notifier.Field{
Name: "Traffic routing",
Value: "A/B Testing",
})
} else if cd.Spec.CanaryAnalysis.Iterations > 0 {
fields = append(fields, notifier.Field{
Name: "Traffic routing",
Value: "Blue/Green",
})
}
}
err := c.notifier.Post(cd.Name, cd.Namespace, message, fields, warn)
if err != nil {
Expand Down
43 changes: 43 additions & 0 deletions pkg/notifier/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package notifier

import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"time"
)

func postMessage(address string, payload interface{}) error {
data, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("marshalling notification payload failed %v", err)
}

b := bytes.NewBuffer(data)

req, err := http.NewRequest("POST", address, b)
if err != nil {
return err
}
req.Header.Set("Content-type", "application/json")

ctx, cancel := context.WithTimeout(req.Context(), 5*time.Second)
defer cancel()

res, err := http.DefaultClient.Do(req.WithContext(ctx))
if err != nil {
return fmt.Errorf("sending notification failed %v", err)
}

defer res.Body.Close()
statusCode := res.StatusCode
if statusCode != 200 {
body, _ := ioutil.ReadAll(res.Body)
return fmt.Errorf("sending notification failed %v", string(body))
}

return nil
}
26 changes: 26 additions & 0 deletions pkg/notifier/factory.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package notifier

type Factory struct {
URL string
Username string
Channel string
}

func NewFactory(URL string, username string, channel string) *Factory {
return &Factory{
URL: URL,
Channel: channel,
Username: username,
}
}

func (f Factory) Notifier(provider string) (Interface, error) {
switch {
case provider == "slack":
return NewSlack(f.URL, f.Username, f.Channel)
case provider == "msteams":
return NewMSTeams(f.URL)
}

return nil, nil
}
10 changes: 10 additions & 0 deletions pkg/notifier/notifier.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package notifier

type Interface interface {
Post(workload string, namespace string, message string, fields []Field, warn bool) error
}

type Field struct {
Name string
Value string
}
Loading