feat: allow to return rules with matching active alerts

This change adds a `-rules-with-active-alerts` CLI argument to return rules with active alerts matching the enforced label. The use case is for an admin to write general-purpose alerting rules which individual users can still retrieve from the API. To avoid surprises ("why do I see this alert now?"), the feature needs to be explicitly enabled. Signed-off-by: Simon Pasquier <[email protected]>
prometheus-community · Jul 31, 2024 · 4ece18c · 4ece18c
1 parent a177f41
commit 4ece18c
Show file tree

Hide file tree

Showing 6 changed files with 253 additions and 23 deletions.
diff --git a/README.md b/README.md
@@ -206,6 +206,17 @@ NOTE: When the `/api/v1/labels` and `/api/v1/label/<name>/values` endpoints were
 
 The proxy requests the `/api/v1/rules` Prometheus endpoint, discards the rules that don't contain an exact match of the label(s) and returns the modified response to the client.
 
+To return alerting rules which have active alerts matching the label(s), you can use the `-rules-with-active-alerts` option. For example:
+
+```
+prom-label-proxy \
+   -header-name X-Namespace \
+   -label namespace \
+   -upstream http://demo.do.prometheus.io:9090 \
+   -insecure-listen-address 127.0.0.1:8080 \
+   -rules-with-active-alerts
+```
+
 ### Alerts endpoint
 
 The proxy requests the `/api/v1/alerts` Prometheus endpoint, discards the rules that don't contain an exact match of the label(s) and returns the modified response to the client.

diff --git a/injectproxy/routes.go b/injectproxy/routes.go
@@ -45,20 +45,22 @@ type routes struct {
 	label    string
 	el       ExtractLabeler
 
-	mux            http.Handler
-	modifiers      map[string]func(*http.Response) error
-	errorOnReplace bool
-	regexMatch     bool
+	mux                   http.Handler
+	modifiers             map[string]func(*http.Response) error
+	errorOnReplace        bool
+	regexMatch            bool
+	rulesWithActiveAlerts bool
 
 	logger *log.Logger
 }
 
 type options struct {
-	enableLabelAPIs  bool
-	passthroughPaths []string
-	errorOnReplace   bool
-	registerer       prometheus.Registerer
-	regexMatch       bool
+	enableLabelAPIs       bool
+	passthroughPaths      []string
+	errorOnReplace        bool
+	registerer            prometheus.Registerer
+	regexMatch            bool
+	rulesWithActiveAlerts bool
 }
 
 type Option interface {
@@ -102,6 +104,13 @@ func WithErrorOnReplace() Option {
 	})
 }
 
+// WithActiveAlerts causes the proxy to return rules with active alerts.
+func WithActiveAlerts() Option {
+	return optionFunc(func(o *options) {
+		o.rulesWithActiveAlerts = true
+	})
+}
+
 // WithRegexMatch causes the proxy to handle tenant name as regexp
 func WithRegexMatch() Option {
 	return optionFunc(func(o *options) {
@@ -294,13 +303,14 @@ func NewRoutes(upstream *url.URL, label string, extractLabeler ExtractLabeler, o
 	proxy := httputil.NewSingleHostReverseProxy(upstream)
 
 	r := &routes{
-		upstream:       upstream,
-		handler:        proxy,
-		label:          label,
-		el:             extractLabeler,
-		errorOnReplace: opt.errorOnReplace,
-		regexMatch:     opt.regexMatch,
-		logger:         log.Default(),
+		upstream:              upstream,
+		handler:               proxy,
+		label:                 label,
+		el:                    extractLabeler,
+		errorOnReplace:        opt.errorOnReplace,
+		regexMatch:            opt.regexMatch,
+		rulesWithActiveAlerts: opt.rulesWithActiveAlerts,
+		logger:                log.Default(),
 	}
 	mux := newStrictMux(newInstrumentedMux(http.NewServeMux(), opt.registerer))
 

diff --git a/injectproxy/rules.go b/injectproxy/rules.go
@@ -174,7 +174,7 @@ var errModifyResponseFailed = errors.New("failed to process the API response")
 // modifyAPIResponse unwraps the Prometheus API response, passes the enforced
 // label value and the response to the given function and finally replaces the
 // result in the response.
-func modifyAPIResponse(f func([]string, *apiResponse) (interface{}, error)) func(*http.Response) error {
+func modifyAPIResponse(f func([]string, *http.Request, *apiResponse) (interface{}, error)) func(*http.Response) error {
 	return func(resp *http.Response) error {
 		if resp.StatusCode != http.StatusOK {
 			// Pass non-200 responses as-is.
@@ -186,7 +186,7 @@ func modifyAPIResponse(f func([]string, *apiResponse) (interface{}, error)) func
 			return fmt.Errorf("can't decode the response: %w", err)
 		}
 
-		v, err := f(MustLabelValues(resp.Request.Context()), apir)
+		v, err := f(MustLabelValues(resp.Request.Context()), resp.Request, apir)
 		if err != nil {
 			return fmt.Errorf("%w: %w", errModifyResponseFailed, err)
 		}
@@ -209,7 +209,7 @@ func modifyAPIResponse(f func([]string, *apiResponse) (interface{}, error)) func
 	}
 }
 
-func (r *routes) filterRules(lvalues []string, resp *apiResponse) (interface{}, error) {
+func (r *routes) filterRules(lvalues []string, req *http.Request, resp *apiResponse) (interface{}, error) {
 	var rgs rulesData
 	if err := json.Unmarshal(resp.Data, &rgs); err != nil {
 		return nil, fmt.Errorf("can't decode rules data: %w", err)
@@ -223,9 +223,51 @@ func (r *routes) filterRules(lvalues []string, resp *apiResponse) (interface{},
 	filtered := []*ruleGroup{}
 	for _, rg := range rgs.RuleGroups {
 		var rules []rule
-		for _, rule := range rg.Rules {
-			if lval := rule.Labels().Get(r.label); lval != "" && m.Matches(lval) {
-				rules = append(rules, rule)
+		for _, rgr := range rg.Rules {
+			if lval := rgr.Labels().Get(r.label); lval != "" && m.Matches(lval) {
+				rules = append(rules, rgr)
+				continue
+			}
+
+			if !r.rulesWithActiveAlerts || rgr.alertingRule == nil {
+				continue
+			}
+
+			var ar *alertingRule
+			for i := range rgr.alertingRule.Alerts {
+				if lval := rgr.alertingRule.Alerts[i].Labels.Get(r.label); lval == "" || !m.Matches(lval) {
+					continue
+				}
+
+				if ar == nil {
+					ar = &alertingRule{
+						Name:           rgr.alertingRule.Name,
+						Query:          rgr.alertingRule.Query,
+						Duration:       rgr.alertingRule.Duration,
+						KeepFiringFor:  rgr.alertingRule.KeepFiringFor,
+						Labels:         rgr.alertingRule.Labels.Copy(),
+						Annotations:    rgr.alertingRule.Annotations.Copy(),
+						Health:         rgr.alertingRule.Health,
+						LastError:      rgr.alertingRule.LastError,
+						EvaluationTime: rgr.alertingRule.EvaluationTime,
+						LastEvaluation: rgr.alertingRule.LastEvaluation,
+						Type:           rgr.alertingRule.Type,
+					}
+				}
+
+				ar.Alerts = append(ar.Alerts, rgr.alertingRule.Alerts[i])
+				switch ar.State {
+				case "pending":
+					if rgr.alertingRule.Alerts[i].State == "firing" {
+						ar.State = rgr.alertingRule.Alerts[i].State
+					}
+				case "":
+					ar.State = rgr.alertingRule.Alerts[i].State
+				}
+			}
+
+			if ar != nil {
+				rules = append(rules, rule{alertingRule: ar})
 			}
 		}
 
@@ -238,7 +280,7 @@ func (r *routes) filterRules(lvalues []string, resp *apiResponse) (interface{},
 	return &rulesData{RuleGroups: filtered}, nil
 }
 
-func (r *routes) filterAlerts(lvalues []string, resp *apiResponse) (interface{}, error) {
+func (r *routes) filterAlerts(lvalues []string, _ *http.Request, resp *apiResponse) (interface{}, error) {
 	var data alertsData
 	if err := json.Unmarshal(resp.Data, &data); err != nil {
 		return nil, fmt.Errorf("can't decode alerts data: %w", err)

diff --git a/injectproxy/rules_test.go b/injectproxy/rules_test.go
@@ -298,6 +298,83 @@ func validRules() http.Handler {
           }
         ],
         "interval": 10
+      },
+      {
+        "name": "group3",
+        "file": "testdata/rules3.yml",
+        "rules": [
+          {
+            "state": "firing",
+            "name": "Alert3",
+            "query": "metric4{ns!=\"default\"} == 0",
+            "duration": 300,
+            "labels": {},
+            "annotations": {},
+            "alerts": [
+              {
+                "labels": {
+                  "alertname": "Alert3",
+                  "namespace": "ns1"
+                },
+                "annotations": {},
+                "state": "firing",
+                "activeAt": "2019-12-18T13:14:39.972915521+01:00",
+                "value": "0e+00"
+              },
+              {
+                "labels": {
+                  "alertname": "Alert3",
+                  "namespace": "ns3"
+                },
+                "annotations": {},
+                "state": "pending",
+                "activeAt": "2019-12-18T13:20:39.972915521+01:00",
+                "value": "0e+00"
+              }
+            ],
+            "health": "ok",
+            "type": "alerting",
+            "evaluationTime": 0.000214,
+            "lastEvaluation": "2024-04-29T14:23:52.903557247+02:00"
+          },
+          {
+            "state": "firing",
+            "name": "Alert4",
+            "query": "metric5 == 0",
+            "duration": 300,
+            "labels": {},
+            "annotations": {},
+            "alerts": [
+              {
+                "labels": {
+                  "alertname": "Alert4",
+                  "namespace": "ns3",
+                  "state": "foo"
+                },
+                "annotations": {},
+                "state": "pending",
+                "activeAt": "2019-12-18T13:20:39.972915521+01:00",
+                "value": "0e+00"
+              },
+              {
+                "labels": {
+                  "alertname": "Alert1",
+                  "namespace": "ns3",
+                  "state": "bar"
+                },
+                "annotations": {},
+                "state": "firing",
+                "activeAt": "2019-12-18T13:14:39.972915521+01:00",
+                "value": "0e+00"
+              }
+            ],
+            "health": "ok",
+            "type": "alerting",
+            "evaluationTime": 0.000214,
+            "lastEvaluation": "2024-04-29T14:23:52.903557247+02:00"
+          }
+        ],
+        "interval": 10
       }
     ]
   }
@@ -472,6 +549,14 @@ func TestRules(t *testing.T) {
 			expCode: http.StatusBadRequest,
 			golden:  "rules_invalid_upstream_response.golden",
 		},
+		{
+			labelv:   []string{"ns3"},
+			upstream: validRules(),
+			opts:     []Option{WithActiveAlerts()},
+
+			expCode: http.StatusOK,
+			golden:  "rules_with_active_alerts.golden",
+		},
 	} {
 		t.Run(fmt.Sprintf("%s=%s", proxyLabel, tc.labelv), func(t *testing.T) {
 			m := newMockUpstream(tc.upstream)

diff --git a/injectproxy/testdata/rules_with_active_alerts.golden b/injectproxy/testdata/rules_with_active_alerts.golden
@@ -0,0 +1,76 @@
+{
+  "status": "success",
+  "data": {
+    "groups": [
+      {
+        "name": "group3",
+        "file": "testdata/rules3.yml",
+        "rules": [
+          {
+            "state": "pending",
+            "name": "Alert3",
+            "query": "metric4{ns!=\"default\"} == 0",
+            "duration": 300,
+            "keepFiringFor": 0,
+            "labels": {},
+            "annotations": {},
+            "alerts": [
+              {
+                "labels": {
+                  "alertname": "Alert3",
+                  "namespace": "ns3"
+                },
+                "annotations": {},
+                "state": "pending",
+                "activeAt": "2019-12-18T13:20:39.972915521+01:00",
+                "value": "0e+00"
+              }
+            ],
+            "health": "ok",
+            "evaluationTime": 0.000214,
+            "lastEvaluation": "2024-04-29T14:23:52.903557247+02:00",
+            "type": "alerting"
+          },
+          {
+            "state": "firing",
+            "name": "Alert4",
+            "query": "metric5 == 0",
+            "duration": 300,
+            "keepFiringFor": 0,
+            "labels": {},
+            "annotations": {},
+            "alerts": [
+              {
+                "labels": {
+                  "alertname": "Alert4",
+                  "namespace": "ns3",
+                  "state": "foo"
+                },
+                "annotations": {},
+                "state": "pending",
+                "activeAt": "2019-12-18T13:20:39.972915521+01:00",
+                "value": "0e+00"
+              },
+              {
+                "labels": {
+                  "alertname": "Alert1",
+                  "namespace": "ns3",
+                  "state": "bar"
+                },
+                "annotations": {},
+                "state": "firing",
+                "activeAt": "2019-12-18T13:14:39.972915521+01:00",
+                "value": "0e+00"
+              }
+            ],
+            "health": "ok",
+            "evaluationTime": 0.000214,
+            "lastEvaluation": "2024-04-29T14:23:52.903557247+02:00",
+            "type": "alerting"
+          }
+        ],
+        "interval": 10
+      }
+    ]
+  }
+}
diff --git a/main.go b/main.go
@@ -67,6 +67,7 @@ func main() {
 		errorOnReplace         bool
 		regexMatch             bool
 		headerUsesListSyntax   bool
+		rulesWithActiveAlerts  bool
 	)
 
 	flagset := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
@@ -86,6 +87,7 @@ func main() {
 	flagset.BoolVar(&errorOnReplace, "error-on-replace", false, "When specified, the proxy will return HTTP status code 400 if the query already contains a label matcher that differs from the one the proxy would inject.")
 	flagset.BoolVar(&regexMatch, "regex-match", false, "When specified, the tenant name is treated as a regular expression. In this case, only one tenant name should be provided.")
 	flagset.BoolVar(&headerUsesListSyntax, "header-uses-list-syntax", false, "When specified, the header line value will be parsed as a comma-separated list. This allows a single tenant header line to specify multiple tenant names.")
+	flagset.BoolVar(&rulesWithActiveAlerts, "rules-with-active-alerts", false, "When true, the proxy will return alerting rules with active alerts matching the tenant label even when the tenant label isn't present in the rule's labels.")
 
 	//nolint: errcheck // Parse() will exit on error.
 	flagset.Parse(os.Args[1:])
@@ -133,6 +135,10 @@ func main() {
 		opts = append(opts, injectproxy.WithErrorOnReplace())
 	}
 
+	if rulesWithActiveAlerts {
+		opts = append(opts, injectproxy.WithActiveAlerts())
+	}
+
 	if regexMatch {
 		if len(labelValues) > 0 {
 			if len(labelValues) > 1 {