Add the option to update message and description when sending alerts to opsgenie

config/notifiers.go

@@ -455,16 +455,17 @@ type OpsGenieConfig struct {
 
 	HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"`
 
-	APIKey      Secret                    `yaml:"api_key,omitempty" json:"api_key,omitempty"`
-	APIURL      *URL                      `yaml:"api_url,omitempty" json:"api_url,omitempty"`
-	Message     string                    `yaml:"message,omitempty" json:"message,omitempty"`
-	Description string                    `yaml:"description,omitempty" json:"description,omitempty"`
-	Source      string                    `yaml:"source,omitempty" json:"source,omitempty"`
-	Details     map[string]string         `yaml:"details,omitempty" json:"details,omitempty"`
-	Responders  []OpsGenieConfigResponder `yaml:"responders,omitempty" json:"responders,omitempty"`
-	Tags        string                    `yaml:"tags,omitempty" json:"tags,omitempty"`
-	Note        string                    `yaml:"note,omitempty" json:"note,omitempty"`
-	Priority    string                    `yaml:"priority,omitempty" json:"priority,omitempty"`
+	APIKey       Secret                    `yaml:"api_key,omitempty" json:"api_key,omitempty"`
+	APIURL       *URL                      `yaml:"api_url,omitempty" json:"api_url,omitempty"`
+	Message      string                    `yaml:"message,omitempty" json:"message,omitempty"`
+	Description  string                    `yaml:"description,omitempty" json:"description,omitempty"`
+	Source       string                    `yaml:"source,omitempty" json:"source,omitempty"`
+	Details      map[string]string         `yaml:"details,omitempty" json:"details,omitempty"`
+	Responders   []OpsGenieConfigResponder `yaml:"responders,omitempty" json:"responders,omitempty"`
+	Tags         string                    `yaml:"tags,omitempty" json:"tags,omitempty"`
+	Note         string                    `yaml:"note,omitempty" json:"note,omitempty"`
+	Priority     string                    `yaml:"priority,omitempty" json:"priority,omitempty"`
+	UpdateAlerts bool                      `yaml:"update_alerts,omitempty" json:"update_alerts,omitempty"`
 }
 
 const opsgenieValidTypesRe = `^(team|user|escalation|schedule)$`

docs/configuration.md

@@ -843,6 +843,10 @@ responders:
 # Priority level of alert. Possible values are P1, P2, P3, P4, and P5.
 [ priority: <tmpl_string> ]
 
+# Whether or not to update message and description of the alert in OpsGenie if it already exists
+# By default, the alert is never updated in OpsGenie, the new message only appears in activity log.
+[ update_alerts: <boolean> | default = false ]
+
 # The HTTP client's configuration.
 [ http_config: <http_config> | default = global.http_config ]
 ```

notify/opsgenie/opsgenie.go

@@ -80,20 +80,33 @@ type opsGenieCloseMessage struct {
 	Source string `json:"source"`
 }
 
+type opsGenieUpdateMessageMessage struct {
+	Message string `json:"message,omitempty"`
+}
+
+type opsGenieUpdateDescriptionMessage struct {
+	Description string `json:"description,omitempty"`
+}
+
 // Notify implements the Notifier interface.
 func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
-	req, retry, err := n.createRequest(ctx, as...)
+	requests, retry, err := n.createRequests(ctx, as...)
 	if err != nil {
 		return retry, err
 	}
 
-	resp, err := n.client.Do(req)
-	if err != nil {
-		return true, err
+	for _, req := range requests {
+		resp, err := n.client.Do(req)
+		if err != nil {
+			return true, err
+		}
+		shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+		notify.Drain(resp)
+		if err != nil {
+			return shouldRetry, err
+		}
 	}
-	defer notify.Drain(resp)
-
-	return n.retrier.Check(resp.StatusCode, resp.Body)
+	return true, nil
 }
 
 // Like Split but filter out empty strings.
@@ -109,7 +122,7 @@ func safeSplit(s string, sep string) []string {
 }
 
 // Create requests for a list of alerts.
-func (n *Notifier) createRequest(ctx context.Context, as ...*types.Alert) (*http.Request, bool, error) {
+func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*http.Request, bool, error) {
 	key, err := notify.ExtractGroupKey(ctx)
 	if err != nil {
 		return nil, false, err
@@ -130,26 +143,37 @@ func (n *Notifier) createRequest(ctx context.Context, as ...*types.Alert) (*http
 		details[k] = tmpl(v)
 	}
 
+	requests := []*http.Request{}
+
 	var (
-		msg    interface{}
-		apiURL = n.conf.APIURL.Copy()
 		alias  = key.Hash()
 		alerts = types.Alerts(as...)
 	)
 	switch alerts.Status() {
 	case model.AlertResolved:
-		apiURL.Path += fmt.Sprintf("v2/alerts/%s/close", alias)
-		q := apiURL.Query()
+		resolvedEndpointURL := n.conf.APIURL.Copy()
+		resolvedEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/close", alias)
+		q := resolvedEndpointURL.Query()
 		q.Set("identifierType", "alias")
-		apiURL.RawQuery = q.Encode()
-		msg = &opsGenieCloseMessage{Source: tmpl(n.conf.Source)}
+		resolvedEndpointURL.RawQuery = q.Encode()
+		var msg = &opsGenieCloseMessage{Source: tmpl(n.conf.Source)}
+		var buf bytes.Buffer
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return nil, false, err
+		}
+		req, err := http.NewRequest("POST", resolvedEndpointURL.String(), &buf)
+		if err != nil {
+			return nil, true, err
+		}
+		requests = append(requests, req.WithContext(ctx))
 	default:
 		message, truncated := notify.Truncate(tmpl(n.conf.Message), 130)
 		if truncated {
 			level.Debug(n.logger).Log("msg", "truncated message", "truncated_message", message, "alert", key)
 		}
 
-		apiURL.Path += "v2/alerts"
+		createEndpointURL := n.conf.APIURL.Copy()
+		createEndpointURL.Path += "v2/alerts"
 
 		var responders []opsGenieCreateMessageResponder
 		for _, r := range n.conf.Responders {
@@ -169,7 +193,7 @@ func (n *Notifier) createRequest(ctx context.Context, as ...*types.Alert) (*http
 			responders = append(responders, responder)
 		}
 
-		msg = &opsGenieCreateMessage{
+		var msg = &opsGenieCreateMessage{
 			Alias:       alias,
 			Message:     message,
 			Description: tmpl(n.conf.Description),
@@ -180,6 +204,54 @@ func (n *Notifier) createRequest(ctx context.Context, as ...*types.Alert) (*http
 			Note:        tmpl(n.conf.Note),
 			Priority:    tmpl(n.conf.Priority),
 		}
+		var buf bytes.Buffer
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return nil, false, err
+		}
+		req, err := http.NewRequest("POST", createEndpointURL.String(), &buf)
+		if err != nil {
+			return nil, true, err
+		}
+		requests = append(requests, req.WithContext(ctx))
+
+		if n.conf.UpdateAlerts {
+			updateMessageEndpointUrl := n.conf.APIURL.Copy()
+			updateMessageEndpointUrl.Path += fmt.Sprintf("v2/alerts/%s/message", alias)
+			q := updateMessageEndpointUrl.Query()
+			q.Set("identifierType", "alias")
+			updateMessageEndpointUrl.RawQuery = q.Encode()
+			updateMsgMsg := &opsGenieUpdateMessageMessage{
+				Message: msg.Message,
+			}
+			var updateMessageBuf bytes.Buffer
+			if err := json.NewEncoder(&updateMessageBuf).Encode(updateMsgMsg); err != nil {
+				return nil, false, err
+			}
+			req, err := http.NewRequest("PUT", updateMessageEndpointUrl.String(), &updateMessageBuf)
+			if err != nil {
+				return nil, true, err
+			}
+			requests = append(requests, req)
+
+			updateDescriptionEndpointURL := n.conf.APIURL.Copy()
+			updateDescriptionEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/description", alias)
+			q = updateDescriptionEndpointURL.Query()
+			q.Set("identifierType", "alias")
+			updateDescriptionEndpointURL.RawQuery = q.Encode()
+			updateDescMsg := &opsGenieUpdateDescriptionMessage{
+				Description: msg.Description,
+			}
+
+			var updateDescriptionBuf bytes.Buffer
+			if err := json.NewEncoder(&updateDescriptionBuf).Encode(updateDescMsg); err != nil {
+				return nil, false, err
+			}
+			req, err = http.NewRequest("PUT", updateDescriptionEndpointURL.String(), &updateDescriptionBuf)
+			if err != nil {
+				return nil, true, err
+			}
+			requests = append(requests, req.WithContext(ctx))
+		}
 	}
 
 	apiKey := tmpl(string(n.conf.APIKey))
@@ -188,16 +260,10 @@ func (n *Notifier) createRequest(ctx context.Context, as ...*types.Alert) (*http
 		return nil, false, errors.Wrap(err, "templating error")
 	}
 
-	var buf bytes.Buffer
-	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
-		return nil, false, err
+	for _, req := range requests {
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Authorization", fmt.Sprintf("GenieKey %s", apiKey))
 	}
 
-	req, err := http.NewRequest("POST", apiURL.String(), &buf)
-	if err != nil {
-		return nil, true, err
-	}
-	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("Authorization", fmt.Sprintf("GenieKey %s", apiKey))
-	return req.WithContext(ctx), true, nil
+	return requests, true, nil
 }

notify/opsgenie/opsgenie_test.go

@@ -167,12 +167,13 @@ func TestOpsGenie(t *testing.T) {
 				},
 			}
 
-			req, retry, err := notifier.createRequest(ctx, alert1)
+			req, retry, err := notifier.createRequests(ctx, alert1)
 			require.NoError(t, err)
+			require.Len(t, req, 1)
 			require.Equal(t, true, retry)
-			require.Equal(t, expectedURL, req.URL)
-			require.Equal(t, "GenieKey http://am", req.Header.Get("Authorization"))
-			require.Equal(t, tc.expectedEmptyAlertBody, readBody(t, req))
+			require.Equal(t, expectedURL, req[0].URL)
+			require.Equal(t, "GenieKey http://am", req[0].Header.Get("Authorization"))
+			require.Equal(t, tc.expectedEmptyAlertBody, readBody(t, req[0]))
 
 			// Fully defined alert.
 			alert2 := &types.Alert{
@@ -193,20 +194,69 @@ func TestOpsGenie(t *testing.T) {
 					EndsAt:   time.Now().Add(time.Hour),
 				},
 			}
-			req, retry, err = notifier.createRequest(ctx, alert2)
+			req, retry, err = notifier.createRequests(ctx, alert2)
 			require.NoError(t, err)
 			require.Equal(t, true, retry)
-			require.Equal(t, tc.expectedBody, readBody(t, req))
+			require.Len(t, req, 1)
+			require.Equal(t, tc.expectedBody, readBody(t, req[0]))
 
 			// Broken API Key Template.
 			tc.cfg.APIKey = "{{ kaput "
-			_, _, err = notifier.createRequest(ctx, alert2)
+			_, _, err = notifier.createRequests(ctx, alert2)
 			require.Error(t, err)
 			require.Equal(t, err.Error(), "templating error: template: :1: function \"kaput\" not defined")
 		})
 	}
 }
 
+func TestOpsGenieWithUpdate(t *testing.T) {
+	u, err := url.Parse("https://test-opsgenie-url")
+	require.NoError(t, err)
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "1")
+	opsGenieConfigWithUpdate := config.OpsGenieConfig{
+		Message:      `{{ .CommonLabels.Message }}`,
+		Description:  `{{ .CommonLabels.Description }}`,
+		UpdateAlerts: true,
+		APIKey:       "test-api-key",
+		APIURL:       &config.URL{URL: u},
+		HTTPConfig:   &commoncfg.HTTPClientConfig{},
+	}
+	notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, log.NewNopLogger())
+	alert := &types.Alert{
+		Alert: model.Alert{
+			StartsAt: time.Now(),
+			EndsAt:   time.Now().Add(time.Hour),
+			Labels: model.LabelSet{
+				"Message":     "new message",
+				"Description": "new description",
+			},
+		},
+	}
+	require.NoError(t, err)
+	requests, retry, err := notifierWithUpdate.createRequests(ctx, alert)
+	require.NoError(t, err)
+	require.True(t, retry)
+	require.Len(t, requests, 3)
+
+	body0 := readBody(t, requests[0])
+	body1 := readBody(t, requests[1])
+	body2 := readBody(t, requests[2])
+	key, _ := notify.ExtractGroupKey(ctx)
+	alias := key.Hash()
+
+	require.Equal(t, requests[0].URL.String(), "https://test-opsgenie-url/v2/alerts")
+	require.NotEmpty(t, body0)
+
+	require.Equal(t, requests[1].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/message?identifierType=alias", alias))
+	require.Equal(t, body1, `{"message":"new message"}
+`)
+	require.Equal(t, requests[2].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/description?identifierType=alias", alias))
+	require.Equal(t, body2, `{"description":"new description"}
+`)
+}
+
 func readBody(t *testing.T, r *http.Request) string {
 	t.Helper()
 	body, err := ioutil.ReadAll(r.Body)