prometheus.operator.*: Allow Customization of Default `scrape_inter…

…val` and `scrape_timeout` (#5139)

* feat(prometheus.operator): add scrape block

Add `scrape` block to customize the default behavior of
`prometheus.operator.podmonitors`, `prometheus.operator.probes`,
and `prometheus.operator.servicemonitors`.

Closes #4806

Signed-off-by: Simon Berz <[email protected]>

* fix(prometheus.operator.probes): pass RelabelConfigs to ConfigGenerator

Fix `prometheus.operator.probes` no longer ignores relabeling `rule`
blocks.

Signed-off-by: Simon Berz <[email protected]>

---------

Signed-off-by: Simon Berz <[email protected]>
Co-authored-by: Craig Peterson <[email protected]>

CHANGELOG.md

@@ -104,6 +104,8 @@ Main (unreleased)
 
 - Agent Management: Honor 503 ServiceUnavailable `Retry-After` header. (@jcreixell)
 
+- Added `scrape` block to customize the default behavior of `prometheus.operator.podmonitors`, `prometheus.operator.probes`, and `prometheus.operator.servicemonitors`. (@sberz)
+=======
 ### Bugfixes
 
 - Fixed `otelcol.exporter.prometheus` label names for the `otel_scope_info`
@@ -135,6 +137,8 @@ Main (unreleased)
 
 - Switch to `IBM/sarama` module. (@hainenber)
 
+- `prometheus.operator.probes` no longer ignores relabeling `rule` blocks. (@sberz)
+
 v0.36.2 (2023-09-22)
 --------------------
 

component/prometheus/operator/common/crdmanager.go

@@ -366,6 +366,7 @@ func (c *crdManager) addPodMonitor(pm *promopv1.PodMonitor) {
 		Secrets:                  configgen.NewSecretManager(c.client),
 		Client:                   &c.args.Client,
 		AdditionalRelabelConfigs: c.args.RelabelConfigs,
+		ScrapeOptions:            c.args.Scrape,
 	}
 	for i, ep := range pm.Spec.PodMetricsEndpoints {
 		var scrapeConfig *config.ScrapeConfig
@@ -415,6 +416,7 @@ func (c *crdManager) addServiceMonitor(sm *promopv1.ServiceMonitor) {
 		Secrets:                  configgen.NewSecretManager(c.client),
 		Client:                   &c.args.Client,
 		AdditionalRelabelConfigs: c.args.RelabelConfigs,
+		ScrapeOptions:            c.args.Scrape,
 	}
 	for i, ep := range sm.Spec.Endpoints {
 		var scrapeConfig *config.ScrapeConfig
@@ -461,8 +463,10 @@ func (c *crdManager) onDeleteServiceMonitor(obj interface{}) {
 func (c *crdManager) addProbe(p *promopv1.Probe) {
 	var err error
 	gen := configgen.ConfigGenerator{
-		Secrets: configgen.NewSecretManager(c.client),
-		Client:  &c.args.Client,
+		Secrets:                  configgen.NewSecretManager(c.client),
+		Client:                   &c.args.Client,
+		AdditionalRelabelConfigs: c.args.RelabelConfigs,
+		ScrapeOptions:            c.args.Scrape,
 	}
 	var pmc *config.ScrapeConfig
 	pmc, err = gen.GenerateProbeConfig(p)

component/prometheus/operator/configgen/config_gen.go

@@ -7,9 +7,11 @@ import (
 
 	k8sConfig "github.com/grafana/agent/component/common/kubernetes"
 	flow_relabel "github.com/grafana/agent/component/common/relabel"
+	"github.com/grafana/agent/component/prometheus/operator"
 	promopv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	commonConfig "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/config"
 	promk8s "github.com/prometheus/prometheus/discovery/kubernetes"
 	"github.com/prometheus/prometheus/model/relabel"
 )
@@ -18,6 +20,7 @@ type ConfigGenerator struct {
 	Client                   *k8sConfig.ClientArguments
 	Secrets                  SecretFetcher
 	AdditionalRelabelConfigs []*flow_relabel.Config
+	ScrapeOptions            operator.ScrapeOptions
 }
 
 var (
@@ -164,6 +167,24 @@ func (cg *ConfigGenerator) generateAuthorization(a promopv1.SafeAuthorization, n
 	return auth, nil
 }
 
+func (cg *ConfigGenerator) generateDefaultScrapeConfig() *config.ScrapeConfig {
+	opt := cg.ScrapeOptions
+
+	c := config.DefaultScrapeConfig
+	c.ScrapeInterval = config.DefaultGlobalConfig.ScrapeInterval
+	c.ScrapeTimeout = config.DefaultGlobalConfig.ScrapeTimeout
+
+	if opt.DefaultScrapeInterval != 0 {
+		c.ScrapeInterval = model.Duration(opt.DefaultScrapeInterval)
+	}
+
+	if opt.DefaultScrapeTimeout != 0 {
+		c.ScrapeTimeout = model.Duration(opt.DefaultScrapeTimeout)
+	}
+
+	return &c
+}
+
 type relabeler struct {
 	configs []*relabel.Config
 }

component/prometheus/operator/configgen/config_gen_podmonitor.go

@@ -27,10 +27,8 @@ var (
 )
 
 func (cg *ConfigGenerator) GeneratePodMonitorConfig(m *promopv1.PodMonitor, ep promopv1.PodMetricsEndpoint, i int) (cfg *config.ScrapeConfig, err error) {
-	c := config.DefaultScrapeConfig
-	cfg = &c
-	cfg.ScrapeInterval = config.DefaultGlobalConfig.ScrapeInterval
-	cfg.ScrapeTimeout = config.DefaultGlobalConfig.ScrapeTimeout
+	cfg = cg.generateDefaultScrapeConfig()
+
 	cfg.JobName = fmt.Sprintf("podMonitor/%s/%s/%d", m.Namespace, m.Name, i)
 	cfg.HonorLabels = ep.HonorLabels
 	if ep.HonorTimestamps != nil {

component/prometheus/operator/configgen/config_gen_probe.go

@@ -21,10 +21,8 @@ import (
 // See https://github.com/prometheus-operator/prometheus-operator/blob/aa8222d7e9b66e9293ed11c9291ea70173021029/pkg/prometheus/promcfg.go#L835
 
 func (cg *ConfigGenerator) GenerateProbeConfig(m *promopv1.Probe) (cfg *config.ScrapeConfig, err error) {
-	c := config.DefaultScrapeConfig
-	cfg = &c
-	cfg.ScrapeInterval = config.DefaultGlobalConfig.ScrapeInterval
-	cfg.ScrapeTimeout = config.DefaultGlobalConfig.ScrapeTimeout
+	cfg = cg.generateDefaultScrapeConfig()
+
 	cfg.JobName = fmt.Sprintf("probe/%s/%s", m.Namespace, m.Name)
 	cfg.HonorTimestamps = true
 	cfg.MetricsPath = m.Spec.ProberSpec.Path

component/prometheus/operator/configgen/config_gen_probe_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/grafana/agent/component/common/kubernetes"
+	flow_relabel "github.com/grafana/agent/component/common/relabel"
 	"github.com/grafana/agent/pkg/util"
 	promopv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	commonConfig "github.com/prometheus/common/config"
@@ -53,6 +54,8 @@ func TestGenerateProbeConfig(t *testing.T) {
 				},
 			},
 			expectedRelabels: util.Untab(`
+- target_label: __meta_foo
+  replacement: bar
 - source_labels: [job]
   target_label: __tmp_prometheus_job_name
 - source_labels: [__meta_kubernetes_ingress_label_foo, __meta_kubernetes_ingress_labelpresent_foo]
@@ -135,6 +138,8 @@ func TestGenerateProbeConfig(t *testing.T) {
 				},
 			},
 			expectedRelabels: util.Untab(`
+- target_label: __meta_foo
+  replacement: bar
 - source_labels:
   - job
   target_label: __tmp_prometheus_job_name
@@ -181,7 +186,12 @@ func TestGenerateProbeConfig(t *testing.T) {
 	}
 	for _, tc := range suite {
 		t.Run(tc.name, func(t *testing.T) {
-			cg := &ConfigGenerator{Client: &kubernetes.ClientArguments{}}
+			cg := &ConfigGenerator{
+				Client: &kubernetes.ClientArguments{},
+				AdditionalRelabelConfigs: []*flow_relabel.Config{
+					{TargetLabel: "__meta_foo", Replacement: "bar"},
+				},
+			}
 			cfg, err := cg.GenerateProbeConfig(tc.m)
 			require.NoError(t, err)
 			// check relabel configs separately

component/prometheus/operator/configgen/config_gen_servicemonitor.go

@@ -17,11 +17,8 @@ import (
 )
 
 func (cg *ConfigGenerator) GenerateServiceMonitorConfig(m *promopv1.ServiceMonitor, ep promopv1.Endpoint, i int) (cfg *config.ScrapeConfig, err error) {
-	c := config.DefaultScrapeConfig
-	cfg = &c
+	cfg = cg.generateDefaultScrapeConfig()
 
-	cfg.ScrapeInterval = config.DefaultGlobalConfig.ScrapeInterval
-	cfg.ScrapeTimeout = config.DefaultGlobalConfig.ScrapeTimeout
 	cfg.JobName = fmt.Sprintf("serviceMonitor/%s/%s/%d", m.Namespace, m.Name, i)
 	cfg.HonorLabels = ep.HonorLabels
 	if ep.HonorTimestamps != nil {

component/prometheus/operator/configgen/config_gen_test.go

@@ -4,10 +4,12 @@ import (
 	"fmt"
 	"net/url"
 	"testing"
+	"time"
 
 	"github.com/grafana/agent/component/common/config"
 	"github.com/grafana/agent/component/common/kubernetes"
 	flow_relabel "github.com/grafana/agent/component/common/relabel"
+	"github.com/grafana/agent/component/prometheus/operator"
 	promopv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	promConfig "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
@@ -418,6 +420,42 @@ func TestGenerateAuthorization(t *testing.T) {
 	}
 }
 
+func TestGenerateDefaultScrapeConfig(t *testing.T) {
+	tests := []struct {
+		name             string
+		scrapeOptions    operator.ScrapeOptions
+		expectedInterval time.Duration
+		expectedTimeout  time.Duration
+	}{
+		{
+			name:             "empty",
+			scrapeOptions:    operator.ScrapeOptions{},
+			expectedInterval: 1 * time.Minute,
+			expectedTimeout:  10 * time.Second,
+		},
+		{
+			name: "defaults set",
+			scrapeOptions: operator.ScrapeOptions{
+				DefaultScrapeInterval: 30 * time.Second,
+				DefaultScrapeTimeout:  5 * time.Second,
+			},
+			expectedInterval: 30 * time.Second,
+			expectedTimeout:  5 * time.Second,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cg := &ConfigGenerator{
+				ScrapeOptions: tt.scrapeOptions,
+			}
+			got := cg.generateDefaultScrapeConfig()
+
+			assert.Equal(t, model.Duration(tt.expectedInterval), got.ScrapeInterval)
+			assert.Equal(t, model.Duration(tt.expectedTimeout), got.ScrapeTimeout)
+		})
+	}
+}
+
 func TestRelabelerAdd(t *testing.T) {
 	relabeler := &relabeler{}
 

component/prometheus/operator/types.go

@@ -27,6 +27,8 @@ type Arguments struct {
 	Clustering Clustering `river:"clustering,block,optional"`
 
 	RelabelConfigs []*flow_relabel.Config `river:"rule,block,optional"`
+
+	Scrape ScrapeOptions `river:"scrape,block,optional"`
 }
 
 // Clustering holds values that configure clustering-specific behavior.
@@ -35,6 +37,15 @@ type Clustering struct {
 	Enabled bool `river:"enabled,attr"`
 }
 
+// ScrapeOptions holds values that configure scraping behavior.
+type ScrapeOptions struct {
+	// DefaultScrapeInterval is the default interval to scrape targets.
+	DefaultScrapeInterval time.Duration `river:"default_scrape_interval,attr,optional"`
+
+	// DefaultScrapeTimeout is the default timeout to scrape targets.
+	DefaultScrapeTimeout time.Duration `river:"default_scrape_timeout,attr,optional"`
+}
+
 var DefaultArguments = Arguments{
 	Client: kubernetes.ClientArguments{
 		HTTPClientConfig: config.DefaultHTTPClientConfig,

docs/sources/flow/reference/components/prometheus.operator.podmonitors.md

@@ -53,6 +53,7 @@ client > oauth2 | [oauth2][] | Configure OAuth2 for authenticating to the Kubern
 client > oauth2 > tls_config | [tls_config][] | Configure TLS settings for connecting to the Kubernetes API. | no
 client > tls_config | [tls_config][] | Configure TLS settings for connecting to the Kubernetes API. | no
 rule | [rule][] | Relabeling rules to apply to discovered targets. | no
+scrape | [scrape][] | Default scrape configuration to apply to discovered targets. | no
 selector | [selector][] | Label selector for which PodMonitors to discover. | no
 selector > match_expression | [match_expression][] | Label selector expression for which PodMonitors to discover. | no
 clustering | [clustering][] | Configure the component for when the Agent is running in clustered mode. | no
@@ -69,6 +70,7 @@ inside a `client` block.
 [selector]: #selector-block
 [match_expression]: #match_expression-block
 [rule]: #rule-block
+[scrape]: #scrape-block
 [clustering]: #clustering-beta
 
 ### client block
@@ -115,6 +117,10 @@ Name | Type | Description | Default | Required
 
 {{< docs/shared lookup="flow/reference/components/rule-block.md" source="agent" version="<AGENT VERSION>" >}}
 
+### scrape block
+
+{{< docs/shared lookup="flow/reference/components/prom-operator-scrape.md" source="agent" version="<AGENT VERSION>" >}}
+
 ### selector block
 
 The `selector` block describes a Kubernetes label selector for PodMonitors.
@@ -147,7 +153,7 @@ The `operator` argument must be one of the following strings:
 * `"Exists"`
 * `"DoesNotExist"`
 
-If there are multiple `match_expressions` blocks inside of a `selector` block, they are combined together with AND clauses. 
+If there are multiple `match_expressions` blocks inside of a `selector` block, they are combined together with AND clauses.
 
 ### clustering (beta)
 

docs/sources/flow/reference/components/prometheus.operator.probes.md

@@ -53,6 +53,7 @@ client > oauth2 | [oauth2][] | Configure OAuth2 for authenticating to the Kubern
 client > oauth2 > tls_config | [tls_config][] | Configure TLS settings for connecting to the Kubernetes API. | no
 client > tls_config | [tls_config][] | Configure TLS settings for connecting to the Kubernetes API. | no
 rule | [rule][] | Relabeling rules to apply to discovered targets. | no
+scrape | [scrape][] | Default scrape configuration to apply to discovered targets. | no
 selector | [selector][] | Label selector for which Probes to discover. | no
 selector > match_expression | [match_expression][] | Label selector expression for which Probes to discover. | no
 clustering | [clustering][] | Configure the component for when the Agent is running in clustered mode. | no
@@ -69,6 +70,7 @@ inside a `client` block.
 [selector]: #selector-block
 [match_expression]: #match_expression-block
 [rule]: #rule-block
+[scrape]: #scrape-block
 [clustering]: #clustering-experimental
 
 ### client block
@@ -115,6 +117,10 @@ Name | Type | Description | Default | Required
 
 {{< docs/shared lookup="flow/reference/components/rule-block.md" source="agent" version="<AGENT VERSION>" >}}
 
+### scrape block
+
+{{< docs/shared lookup="flow/reference/components/prom-operator-scrape.md" source="agent" version="<AGENT VERSION>" >}}
+
 ### selector block
 
 The `selector` block describes a Kubernetes label selector for Probes.
@@ -147,7 +153,7 @@ The `operator` argument must be one of the following strings:
 * `"Exists"`
 * `"DoesNotExist"`
 
-If there are multiple `match_expressions` blocks inside of a `selector` block, they are combined together with AND clauses. 
+If there are multiple `match_expressions` blocks inside of a `selector` block, they are combined together with AND clauses.
 
 ### clustering (experimental)
 

docs/sources/flow/reference/components/prometheus.operator.servicemonitors.md

@@ -53,6 +53,7 @@ client > oauth2 | [oauth2][] | Configure OAuth2 for authenticating to the Kubern
 client > oauth2 > tls_config | [tls_config][] | Configure TLS settings for connecting to the Kubernetes API. | no
 client > tls_config | [tls_config][] | Configure TLS settings for connecting to the Kubernetes API. | no
 rule | [rule][] | Relabeling rules to apply to discovered targets. | no
+scrape | [scrape][] | Default scrape configuration to apply to discovered targets. | no
 selector | [selector][] | Label selector for which ServiceMonitors to discover. | no
 selector > match_expression | [match_expression][] | Label selector expression for which ServiceMonitors to discover. | no
 clustering | [clustering][] | Configure the component for when the Agent is running in clustered mode. | no
@@ -69,6 +70,7 @@ inside a `client` block.
 [selector]: #selector-block
 [match_expression]: #match_expression-block
 [rule]: #rule-block
+[scrape]: #scrape-block
 [clustering]: #clustering-beta
 
 ### client block
@@ -115,6 +117,10 @@ Name | Type | Description | Default | Required
 
 {{< docs/shared lookup="flow/reference/components/rule-block.md" source="agent" version="<AGENT VERSION>" >}}
 
+### scrape block
+
+{{< docs/shared lookup="flow/reference/components/prom-operator-scrape.md" source="agent" version="<AGENT VERSION>" >}}
+
 ### selector block
 
 The `selector` block describes a Kubernetes label selector for ServiceMonitors.
@@ -147,7 +153,7 @@ The `operator` argument must be one of the following strings:
 * `"Exists"`
 * `"DoesNotExist"`
 
-If there are multiple `match_expressions` blocks inside of a `selector` block, they are combined together with AND clauses. 
+If there are multiple `match_expressions` blocks inside of a `selector` block, they are combined together with AND clauses.
 
 ### clustering (beta)
 

docs/sources/shared/flow/reference/components/prom-operator-scrape.md

@@ -0,0 +1,11 @@
+---
+aliases:
+- /docs/agent/shared/flow/reference/components/prom-operator-scrape/
+canonical: https://grafana.com/docs/agent/latest/shared/flow/reference/components/prom-operator-scrape/
+headless: true
+---
+
+Name | Type | Description | Default | Required
+---- | ---- | ----------- | ------- | --------
+`default_scrape_interval` | `duration` | The default interval between scraping targets. Used as the default if the target resource does not provide a scrape interval. | `1m` | no
+`default_scrape_timeout` | `duration` | The default timeout for scrape requests. Used as the default if the target resource does not provide a scrape timeout. | `10s` | no