From d2d5237c2dd36d44627bb9f5a8a9d75b67b37e30 Mon Sep 17 00:00:00 2001 From: kaaaaaaang Date: Thu, 7 Nov 2024 14:59:44 +0800 Subject: [PATCH] add metricsexclude (#474) --- cmd/diag/command/collect.go | 1 + cmd/diag/command/collectdm.go | 1 + cmd/diag/command/collectk.go | 1 + cmd/diag/command/util_metricdump.go | 1 + collector/collect.go | 4 +++- collector/prometheus.go | 16 +++++++-------- collector/prometheus_test.go | 31 +++++++++++++++++++++++++++++ pkg/utils/utils.go | 9 +++++++++ 8 files changed, 54 insertions(+), 10 deletions(-) create mode 100644 collector/prometheus_test.go diff --git a/cmd/diag/command/collect.go b/cmd/diag/command/collect.go index a4ef27d9..a5cbffd6 100644 --- a/cmd/diag/command/collect.go +++ b/cmd/diag/command/collect.go @@ -156,6 +156,7 @@ func newCollectCmd() *cobra.Command { cmd.Flags().StringSliceVar(&inc, "include", []string{"system", "config", "monitor", "log.std", "log.slow"}, "types of data to collect") cmd.Flags().StringSliceVar(&ext, "exclude", nil, "types of data not to collect") cmd.Flags().StringSliceVar(&cOpt.MetricsFilter, "metricsfilter", nil, "prefix of metrics to collect") + cmd.Flags().StringSliceVar(&cOpt.MetricsExclude, "metricsexclude", []string{"node_interrupts_total"}, "prefix of metrics to exclude") cmd.Flags().IntVar(&cOpt.MetricsLimit, "metricslimit", 10000, "metric size limit of single request, specified in series*hour per request") cmd.Flags().StringVar(&metricsConf, "metricsconfig", "", "config file of metricsfilter") cmd.Flags().StringSliceVar(&labels, "metricslabel", nil, "only collect metrics that match labels") diff --git a/cmd/diag/command/collectdm.go b/cmd/diag/command/collectdm.go index 300e753d..1a14e230 100644 --- a/cmd/diag/command/collectdm.go +++ b/cmd/diag/command/collectdm.go @@ -156,6 +156,7 @@ func newCollectDMCmd() *cobra.Command { cmd.Flags().BoolVar(&collectAll, "all", false, "Collect all data") cmd.Flags().StringSliceVar(&inc, "include", []string{"system", "config", "monitor", "log.std"}, "types of data not to collect") cmd.Flags().StringSliceVar(&cOpt.MetricsFilter, "metricsfilter", nil, "prefix of metrics to collect") + cmd.Flags().StringSliceVar(&cOpt.MetricsExclude, "metricsexclude", []string{"node_interrupts_total"}, "prefix of metrics to exclude") cmd.Flags().IntVar(&cOpt.MetricsLimit, "metricslimit", 10000, "metric size limit of single request, specified in series*hour per request") cmd.Flags().StringVar(&metricsConf, "metricsconfig", "", "config file of metricsfilter") cmd.Flags().StringSliceVar(&labels, "metricslabel", nil, "only collect metrics that match labels") diff --git a/cmd/diag/command/collectk.go b/cmd/diag/command/collectk.go index b7d296d2..7ebc2d27 100644 --- a/cmd/diag/command/collectk.go +++ b/cmd/diag/command/collectk.go @@ -137,6 +137,7 @@ func newCollectkCmd() *cobra.Command { cmd.Flags().StringSliceVar(&inc, "include", []string{"monitor.metric", "log.std", "log.slow"}, "types of data to collect") cmd.Flags().StringSliceVar(&ext, "exclude", nil, "types of data not to collect") cmd.Flags().StringSliceVar(&cOpt.MetricsFilter, "metricsfilter", nil, "prefix of metrics to collect") + cmd.Flags().StringSliceVar(&cOpt.MetricsExclude, "metricsexclude", []string{"node_interrupts_total"}, "prefix of metrics to exclude") cmd.Flags().IntVar(&cOpt.MetricsLimit, "metricslimit", 10000, "metric size limit of single request, specified in series*hour per request") cmd.Flags().StringVar(&metricsConf, "metricsconfig", "", "config file of metricsfilter") cmd.Flags().StringSliceVar(&labels, "metricslabel", nil, "only collect metrics that match labels") diff --git a/cmd/diag/command/util_metricdump.go b/cmd/diag/command/util_metricdump.go index ab136329..f2c72101 100644 --- a/cmd/diag/command/util_metricdump.go +++ b/cmd/diag/command/util_metricdump.go @@ -132,6 +132,7 @@ func newMetricDumpCmd() *cobra.Command { cmd.Flags().StringVarP(&opt.ScrapeBegin, "from", "f", time.Now().Add(time.Hour*-2).Format(time.RFC3339), "start timepoint when collecting timeseries data") cmd.Flags().StringVarP(&opt.ScrapeEnd, "to", "t", time.Now().Format(time.RFC3339), "stop timepoint when collecting timeseries data") cmd.Flags().StringSliceVar(&cOpt.MetricsFilter, "metricsfilter", nil, "prefix of metrics to collect") + cmd.Flags().StringSliceVar(&cOpt.MetricsExclude, "metricsexclude", []string{"node_interrupts_total"}, "prefix of metrics to exclude") cmd.Flags().StringSliceVar(&labels, "metricslabel", nil, "only collect metrics that match labels") cmd.Flags().IntVar(&cOpt.MetricsLimit, "metricslimit", 10000, "metric size limit of single request, specified in series*hour per request") cmd.Flags().StringSliceVarP(&cOpt.Header, "header", "H", nil, "custom headers of http request when collect metrics") diff --git a/collector/collect.go b/collector/collect.go index 4bfbcdb2..fc834150 100644 --- a/collector/collect.go +++ b/collector/collect.go @@ -113,7 +113,8 @@ type CollectOptions struct { DiagMode string // run diag collect at command line mode or server mode ProfileName string // the name of a pre-defined collecting profile Collectors CollectTree // struct to show which collector is enabled - MetricsFilter []string // prefix of metrics to collect" + MetricsFilter []string // prefix of metrics to collect + MetricsExclude []string //prefix of metrics to exclude MetricsLabel map[string]string // label to filte metrics Dir string // target directory to store collected data Limit int // rate limit of SCP @@ -299,6 +300,7 @@ func (m *Manager) CollectClusterInfo( resultDir: resultDir, label: cOpt.MetricsLabel, filter: cOpt.MetricsFilter, + exclude: cOpt.MetricsExclude, limit: cOpt.MetricsLimit, compress: cOpt.CompressMetrics, customHeader: cOpt.Header, diff --git a/collector/prometheus.go b/collector/prometheus.go index e2370616..08cf20fd 100644 --- a/collector/prometheus.go +++ b/collector/prometheus.go @@ -175,6 +175,7 @@ type MetricCollectOptions struct { label map[string]string metrics []string // metric list filter []string + exclude []string limit int // series*min per query compress bool customHeader []string @@ -259,7 +260,7 @@ func (c *MetricCollectOptions) Prepare(m *Manager, topo *models.TiDBCluster) (ma return nil, fmt.Errorf("failed to get metric list from %s: %s", c.endpoint, err) } - c.metrics = filterMetrics(c.metrics, c.filter) + c.metrics = filterMetrics(c.metrics, c.filter, c.exclude) result := make(map[string][]CollectStat) insCnt := len(topo.Components()) @@ -570,18 +571,15 @@ func ensureMonitorDir(base string, sub ...string) error { return os.MkdirAll(dir, 0755) } -func filterMetrics(src, filter []string) []string { - if filter == nil { - return src - } +func filterMetrics(src, filter, exclude []string) []string { var res []string for _, metric := range src { - for _, prefix := range filter { - if strings.HasPrefix(metric, prefix) { - res = append(res, metric) - } + if (len(filter) < 1 || utils.MatchPrefixs(metric, filter)) && + !utils.MatchPrefixs(metric, exclude) { + res = append(res, metric) } } + return res } diff --git a/collector/prometheus_test.go b/collector/prometheus_test.go new file mode 100644 index 00000000..e67549e0 --- /dev/null +++ b/collector/prometheus_test.go @@ -0,0 +1,31 @@ +package collector + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMetricFilter(t *testing.T) { + assert := require.New(t) + + list := []string{ + "tikv_xxx", + "tidb_xxx", + "ticdc_xxx", + "node_xxx", + } + filter := []string{ + "tikv", + "tidb", + "node", + } + exclude := []string{ + "node", + } + + assert.Equal(filterMetrics(list, filter, exclude), []string{"tikv_xxx", "tidb_xxx"}) + assert.Equal(filterMetrics(list, nil, exclude), []string{"tikv_xxx", "tidb_xxx", "ticdc_xxx"}) + assert.Equal(filterMetrics(list, filter, nil), []string{"tikv_xxx", "tidb_xxx", "node_xxx"}) + assert.Equal(filterMetrics(list, nil, nil), []string{"tikv_xxx", "tidb_xxx", "ticdc_xxx", "node_xxx"}) +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 0e8c9e79..afdc6220 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -49,3 +49,12 @@ func AddHeaders(exist http.Header, addons []string) { exist.Add(line[:index], TrimLeftSpace(line[index+1:])) } } + +func MatchPrefixs(str string, prefixs []string) bool { + for _, prefix := range prefixs { + if strings.HasPrefix(str, prefix) { + return true + } + } + return false +}