add metricsexclude (#474)

cmd/diag/command/collect.go

@@ -156,6 +156,7 @@ func newCollectCmd() *cobra.Command {
 	cmd.Flags().StringSliceVar(&inc, "include", []string{"system", "config", "monitor", "log.std", "log.slow"}, "types of data to collect")
 	cmd.Flags().StringSliceVar(&ext, "exclude", nil, "types of data not to collect")
 	cmd.Flags().StringSliceVar(&cOpt.MetricsFilter, "metricsfilter", nil, "prefix of metrics to collect")
+	cmd.Flags().StringSliceVar(&cOpt.MetricsExclude, "metricsexclude", []string{"node_interrupts_total"}, "prefix of metrics to exclude")
 	cmd.Flags().IntVar(&cOpt.MetricsLimit, "metricslimit", 10000, "metric size limit of single request, specified in series*hour per request")
 	cmd.Flags().StringVar(&metricsConf, "metricsconfig", "", "config file of metricsfilter")
 	cmd.Flags().StringSliceVar(&labels, "metricslabel", nil, "only collect metrics that match labels")

cmd/diag/command/collectdm.go

@@ -156,6 +156,7 @@ func newCollectDMCmd() *cobra.Command {
 	cmd.Flags().BoolVar(&collectAll, "all", false, "Collect all data")
 	cmd.Flags().StringSliceVar(&inc, "include", []string{"system", "config", "monitor", "log.std"}, "types of data not to collect")
 	cmd.Flags().StringSliceVar(&cOpt.MetricsFilter, "metricsfilter", nil, "prefix of metrics to collect")
+	cmd.Flags().StringSliceVar(&cOpt.MetricsExclude, "metricsexclude", []string{"node_interrupts_total"}, "prefix of metrics to exclude")
 	cmd.Flags().IntVar(&cOpt.MetricsLimit, "metricslimit", 10000, "metric size limit of single request, specified in series*hour per request")
 	cmd.Flags().StringVar(&metricsConf, "metricsconfig", "", "config file of metricsfilter")
 	cmd.Flags().StringSliceVar(&labels, "metricslabel", nil, "only collect metrics that match labels")

cmd/diag/command/collectk.go

@@ -137,6 +137,7 @@ func newCollectkCmd() *cobra.Command {
 	cmd.Flags().StringSliceVar(&inc, "include", []string{"monitor.metric", "log.std", "log.slow"}, "types of data to collect")
 	cmd.Flags().StringSliceVar(&ext, "exclude", nil, "types of data not to collect")
 	cmd.Flags().StringSliceVar(&cOpt.MetricsFilter, "metricsfilter", nil, "prefix of metrics to collect")
+	cmd.Flags().StringSliceVar(&cOpt.MetricsExclude, "metricsexclude", []string{"node_interrupts_total"}, "prefix of metrics to exclude")
 	cmd.Flags().IntVar(&cOpt.MetricsLimit, "metricslimit", 10000, "metric size limit of single request, specified in series*hour per request")
 	cmd.Flags().StringVar(&metricsConf, "metricsconfig", "", "config file of metricsfilter")
 	cmd.Flags().StringSliceVar(&labels, "metricslabel", nil, "only collect metrics that match labels")

cmd/diag/command/util_metricdump.go

@@ -132,6 +132,7 @@ func newMetricDumpCmd() *cobra.Command {
 	cmd.Flags().StringVarP(&opt.ScrapeBegin, "from", "f", time.Now().Add(time.Hour*-2).Format(time.RFC3339), "start timepoint when collecting timeseries data")
 	cmd.Flags().StringVarP(&opt.ScrapeEnd, "to", "t", time.Now().Format(time.RFC3339), "stop timepoint when collecting timeseries data")
 	cmd.Flags().StringSliceVar(&cOpt.MetricsFilter, "metricsfilter", nil, "prefix of metrics to collect")
+	cmd.Flags().StringSliceVar(&cOpt.MetricsExclude, "metricsexclude", []string{"node_interrupts_total"}, "prefix of metrics to exclude")
 	cmd.Flags().StringSliceVar(&labels, "metricslabel", nil, "only collect metrics that match labels")
 	cmd.Flags().IntVar(&cOpt.MetricsLimit, "metricslimit", 10000, "metric size limit of single request, specified in series*hour per request")
 	cmd.Flags().StringSliceVarP(&cOpt.Header, "header", "H", nil, "custom headers of http request when collect metrics")

collector/collect.go

@@ -113,7 +113,8 @@ type CollectOptions struct {
 	DiagMode        string            // run diag collect at command line mode or server mode
 	ProfileName     string            // the name of a pre-defined collecting profile
 	Collectors      CollectTree       // struct to show which collector is enabled
-	MetricsFilter   []string          // prefix of metrics to collect"
+	MetricsFilter   []string          // prefix of metrics to collect
+	MetricsExclude  []string          //prefix of metrics to exclude
 	MetricsLabel    map[string]string // label to filte metrics
 	Dir             string            // target directory to store collected data
 	Limit           int               // rate limit of SCP
@@ -299,6 +300,7 @@ func (m *Manager) CollectClusterInfo(
 				resultDir:    resultDir,
 				label:        cOpt.MetricsLabel,
 				filter:       cOpt.MetricsFilter,
+				exclude:      cOpt.MetricsExclude,
 				limit:        cOpt.MetricsLimit,
 				compress:     cOpt.CompressMetrics,
 				customHeader: cOpt.Header,

collector/prometheus.go

@@ -175,6 +175,7 @@ type MetricCollectOptions struct {
 	label        map[string]string
 	metrics      []string // metric list
 	filter       []string
+	exclude      []string
 	limit        int // series*min per query
 	compress     bool
 	customHeader []string
@@ -259,7 +260,7 @@ func (c *MetricCollectOptions) Prepare(m *Manager, topo *models.TiDBCluster) (ma
 		return nil, fmt.Errorf("failed to get metric list from %s: %s", c.endpoint, err)
 	}
 
-	c.metrics = filterMetrics(c.metrics, c.filter)
+	c.metrics = filterMetrics(c.metrics, c.filter, c.exclude)
 
 	result := make(map[string][]CollectStat)
 	insCnt := len(topo.Components())
@@ -570,18 +571,15 @@ func ensureMonitorDir(base string, sub ...string) error {
 	return os.MkdirAll(dir, 0755)
 }
 
-func filterMetrics(src, filter []string) []string {
-	if filter == nil {
-		return src
-	}
+func filterMetrics(src, filter, exclude []string) []string {
 	var res []string
 	for _, metric := range src {
-		for _, prefix := range filter {
-			if strings.HasPrefix(metric, prefix) {
-				res = append(res, metric)
-			}
+		if (len(filter) < 1 || utils.MatchPrefixs(metric, filter)) &&
+			!utils.MatchPrefixs(metric, exclude) {
+			res = append(res, metric)
 		}
 	}
+
 	return res
 }
 

collector/prometheus_test.go

@@ -0,0 +1,31 @@
+package collector
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMetricFilter(t *testing.T) {
+	assert := require.New(t)
+
+	list := []string{
+		"tikv_xxx",
+		"tidb_xxx",
+		"ticdc_xxx",
+		"node_xxx",
+	}
+	filter := []string{
+		"tikv",
+		"tidb",
+		"node",
+	}
+	exclude := []string{
+		"node",
+	}
+
+	assert.Equal(filterMetrics(list, filter, exclude), []string{"tikv_xxx", "tidb_xxx"})
+	assert.Equal(filterMetrics(list, nil, exclude), []string{"tikv_xxx", "tidb_xxx", "ticdc_xxx"})
+	assert.Equal(filterMetrics(list, filter, nil), []string{"tikv_xxx", "tidb_xxx", "node_xxx"})
+	assert.Equal(filterMetrics(list, nil, nil), []string{"tikv_xxx", "tidb_xxx", "ticdc_xxx", "node_xxx"})
+}

pkg/utils/utils.go

@@ -49,3 +49,12 @@ func AddHeaders(exist http.Header, addons []string) {
 		exist.Add(line[:index], TrimLeftSpace(line[index+1:]))
 	}
 }
+
+func MatchPrefixs(str string, prefixs []string) bool {
+	for _, prefix := range prefixs {
+		if strings.HasPrefix(str, prefix) {
+			return true
+		}
+	}
+	return false
+}