From 1834250d16556666564d84eeef8887dc19814ef0 Mon Sep 17 00:00:00 2001 From: buzhimingyonghu <1512049108@qq.com> Date: Wed, 13 Nov 2024 06:31:22 +0000 Subject: [PATCH 1/5] fix pika_exporter warnf not found valuename --- .../pika_exporter/exporter/metrics/parser.go | 97 +++++++++++++++++-- tools/pika_exporter/exporter/pika.go | 23 ++++- 2 files changed, 108 insertions(+), 12 deletions(-) diff --git a/tools/pika_exporter/exporter/metrics/parser.go b/tools/pika_exporter/exporter/metrics/parser.go index 90dfd9ce72..683145965c 100644 --- a/tools/pika_exporter/exporter/metrics/parser.go +++ b/tools/pika_exporter/exporter/metrics/parser.go @@ -17,10 +17,86 @@ const ( ) type ParseOption struct { - Version *semver.Version - Extracts map[string]string - ExtractsProxy map[string][]int64 - Info string + Version *semver.Version + Extracts map[string]string + ExtractsProxy map[string][]int64 + Info string + CurrentVersion VersionChecker +} +type VersionChecker interface { + CheckContainsEmptyValue(key string) bool +} +type VersionChecker336 struct { + EmptyValueName []string +} + +func (v *VersionChecker336) CheckContainsEmptyValue(key string) bool { + if v.EmptyValueName == nil { + v.EmptyValueName = []string{ + "instantaneous_output_repl_kbps", + "total_net_output_bytes", + "cache_db_num", + "hits_per_sec", + "cache_status", + "total_net_input_bytes", + "instantaneous_output_kbps", + "instantaneous_input_kbps", + "total_net_repl_input_bytes", + "instantaneous_input_repl_kbps", + "slow_logs_count", + "total_net_repl_output_bytes", + "cache_memory", + } + } + for _, str := range v.EmptyValueName { + if str == key { + return true + } + } + return false +} + +type VersionChecker355 struct { + EmptyValueName []string +} + +func (v *VersionChecker355) CheckContainsEmptyValue(key string) bool { + if v.EmptyValueName == nil { + v.EmptyValueName = []string{ + "cache_db_num", + "cache_status", + "cache_memory", + "hits_per_sec", + } + } + for _, str := range v.EmptyValueName { + if str == key { + return true + } + } + return false +} + +type VersionChecker350 struct { + EmptyValueName []string +} + +func (v *VersionChecker350) CheckContainsEmptyValue(key string) bool { + if v.EmptyValueName == nil { + v.EmptyValueName = []string{ + "cache_db_num", + "cache_status", + "cache_memory", + "hits_per_sec", + "slow_logs_count", + } + } + for _, str := range v.EmptyValueName { + if str == key { + return true + } + } + return false } type Parser interface { @@ -112,9 +188,10 @@ func (p *regexParser) Parse(m MetricMeta, c Collector, opt ParseOption) { matchMaps := p.regMatchesToMap(s) if len(matchMaps) == 0 { - log.Warnf("regexParser::Parse reg find sub match nil. name:%s", p.name) + if opt.CurrentVersion == nil || !opt.CurrentVersion.CheckContainsEmptyValue(p.name) { + log.Warnf("regexParser::Parse reg find sub match nil. name:%s", p.name) + } } - extracts := make(map[string]string) for k, v := range opt.Extracts { extracts[k] = v @@ -172,8 +249,11 @@ func (p *normalParser) Parse(m MetricMeta, c Collector, opt ParseOption) { if m.ValueName != "" { if v, ok := findInMap(m.ValueName, opt.Extracts); !ok { - log.Warnf("normalParser::Parse not found value. metricName:%s valueName:%s", m.Name, m.ValueName) + if opt.CurrentVersion == nil || !opt.CurrentVersion.CheckContainsEmptyValue(m.ValueName) { + log.Warnf("normalParser::Parse not found value. metricName:%s valueName:%s", m.Name, m.ValueName) + } return + } else { metric.Value = convertToFloat64(v) } @@ -208,7 +288,6 @@ func (p *timeParser) Parse(m MetricMeta, c Collector, opt ParseOption) { if m.ValueName != "" { if v, ok := findInMap(m.ValueName, opt.Extracts); !ok { - log.Warnf("timeParser::Parse not found value. metricName:%s valueName:%s", m.Name, m.ValueName) return } else { t, err := convertTimeToUnix(v) @@ -227,6 +306,7 @@ func (p *timeParser) Parse(m MetricMeta, c Collector, opt ParseOption) { } func findInMap(key string, ms ...map[string]string) (string, bool) { + for _, m := range ms { if v, ok := m[key]; ok { return v, true @@ -234,7 +314,6 @@ func findInMap(key string, ms ...map[string]string) (string, bool) { } return "", false } - func trimSpace(s string) string { return strings.TrimRight(strings.TrimLeft(s, " "), " ") } diff --git a/tools/pika_exporter/exporter/pika.go b/tools/pika_exporter/exporter/pika.go index 2e5c27e2dd..77eb7f0311 100644 --- a/tools/pika_exporter/exporter/pika.go +++ b/tools/pika_exporter/exporter/pika.go @@ -252,9 +252,10 @@ func (e *exporter) collectInfo(c *client, ch chan<- prometheus.Metric) error { return nil }) parseOpt := metrics.ParseOption{ - Version: version, - Extracts: extracts, - Info: info, + Version: version, + Extracts: extracts, + Info: info, + CurrentVersion: selectversion(version.Original()), } for _, m := range metrics.MetricConfigs { m.Parse(m, collector, parseOpt) @@ -263,6 +264,22 @@ func (e *exporter) collectInfo(c *client, ch chan<- prometheus.Metric) error { return nil } +func selectversion(version string) metrics.VersionChecker { + var v metrics.VersionChecker + + switch version { + case "3.3.6": + v = &metrics.VersionChecker336{} + case "3.5.5": + v = &metrics.VersionChecker355{} + case "3.5.0": + v = &metrics.VersionChecker350{} + default: + return nil + } + return v +} + func (e *exporter) collectKeys(c *client) error { allKeys := append([]dbKeyPair{}, e.keys...) keys, err := getKeysFromPatterns(c, e.keyPatterns, e.scanCount) From d681cf637b8889721a936c15fcbc6048355bbfc6 Mon Sep 17 00:00:00 2001 From: buzhimingyonghu <1512049108@qq.com> Date: Wed, 13 Nov 2024 07:58:14 +0000 Subject: [PATCH 2/5] Remove duplicate logs and add version management analysis --- .../pika_exporter/exporter/metrics/parser.go | 67 ++++++++++++++++--- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/tools/pika_exporter/exporter/metrics/parser.go b/tools/pika_exporter/exporter/metrics/parser.go index 683145965c..5d36968c03 100644 --- a/tools/pika_exporter/exporter/metrics/parser.go +++ b/tools/pika_exporter/exporter/metrics/parser.go @@ -24,13 +24,15 @@ type ParseOption struct { CurrentVersion VersionChecker } type VersionChecker interface { - CheckContainsEmptyValue(key string) bool + CheckContainsEmptyValueName(key string) bool + CheckContainsEmptyRegexName(key string) bool } type VersionChecker336 struct { EmptyValueName []string + EmptyRegexName []string } -func (v *VersionChecker336) CheckContainsEmptyValue(key string) bool { +func (v *VersionChecker336) CheckContainsEmptyValueName(key string) bool { if v.EmptyValueName == nil { v.EmptyValueName = []string{ "instantaneous_output_repl_kbps", @@ -55,18 +57,33 @@ func (v *VersionChecker336) CheckContainsEmptyValue(key string) bool { } return false } +func (v *VersionChecker336) CheckContainsEmptyRegexName(key string) bool { + if v.EmptyRegexName == nil { + v.EmptyRegexName = []string{ + "hitratio_per_sec", + } + } + for _, str := range v.EmptyRegexName { + if str == key { + return true + } + } + return false +} -type VersionChecker355 struct { +type VersionChecker350 struct { EmptyValueName []string + EmptyRegexName []string } -func (v *VersionChecker355) CheckContainsEmptyValue(key string) bool { +func (v *VersionChecker350) CheckContainsEmptyValueName(key string) bool { if v.EmptyValueName == nil { v.EmptyValueName = []string{ "cache_db_num", "cache_status", "cache_memory", "hits_per_sec", + "slow_logs_count", } } for _, str := range v.EmptyValueName { @@ -76,19 +93,32 @@ func (v *VersionChecker355) CheckContainsEmptyValue(key string) bool { } return false } +func (v *VersionChecker350) CheckContainsEmptyRegexName(key string) bool { + if v.EmptyRegexName == nil { + v.EmptyRegexName = []string{ + "hitratio_per_sec", + } + } + for _, str := range v.EmptyRegexName { + if str == key { + return true + } + } + return false +} -type VersionChecker350 struct { +type VersionChecker355 struct { EmptyValueName []string + EmptyRegexName []string } -func (v *VersionChecker350) CheckContainsEmptyValue(key string) bool { +func (v *VersionChecker355) CheckContainsEmptyValueName(key string) bool { if v.EmptyValueName == nil { v.EmptyValueName = []string{ "cache_db_num", "cache_status", "cache_memory", "hits_per_sec", - "slow_logs_count", } } for _, str := range v.EmptyValueName { @@ -98,6 +128,24 @@ func (v *VersionChecker350) CheckContainsEmptyValue(key string) bool { } return false } +func (v *VersionChecker355) CheckContainsEmptyRegexName(key string) bool { + if v.EmptyRegexName == nil { + v.EmptyRegexName = []string{ + "hitratio_per_sec", + "keyspace_info_>=3.1.0", + "keyspace_info_all_>=3.3.3", + "binlog_>=3.2.0", + "keyspace_last_start_time", + "is_scaning_keyspace", + } + } + for _, str := range v.EmptyRegexName { + if str == key { + return true + } + } + return false +} type Parser interface { Parse(m MetricMeta, c Collector, opt ParseOption) @@ -188,7 +236,7 @@ func (p *regexParser) Parse(m MetricMeta, c Collector, opt ParseOption) { matchMaps := p.regMatchesToMap(s) if len(matchMaps) == 0 { - if opt.CurrentVersion == nil || !opt.CurrentVersion.CheckContainsEmptyValue(p.name) { + if opt.CurrentVersion == nil || !opt.CurrentVersion.CheckContainsEmptyRegexName(p.name) { log.Warnf("regexParser::Parse reg find sub match nil. name:%s", p.name) } } @@ -213,7 +261,6 @@ func (p *regexParser) regMatchesToMap(s string) []map[string]string { multiMatches := p.reg.FindAllStringSubmatch(s, -1) if len(multiMatches) == 0 { - log.Errorf("regexParser::regMatchesToMap reg find sub match nil. name:%s", p.name) return nil } @@ -249,7 +296,7 @@ func (p *normalParser) Parse(m MetricMeta, c Collector, opt ParseOption) { if m.ValueName != "" { if v, ok := findInMap(m.ValueName, opt.Extracts); !ok { - if opt.CurrentVersion == nil || !opt.CurrentVersion.CheckContainsEmptyValue(m.ValueName) { + if opt.CurrentVersion == nil || !opt.CurrentVersion.CheckContainsEmptyValueName(m.ValueName) { log.Warnf("normalParser::Parse not found value. metricName:%s valueName:%s", m.Name, m.ValueName) } return From 67e92bce1101675787981bc6634d9fa612154a9f Mon Sep 17 00:00:00 2001 From: buzhimingyonghu <1512049108@qq.com> Date: Fri, 15 Nov 2024 02:13:01 +0000 Subject: [PATCH 3/5] fix --- .../pika_exporter/exporter/metrics/parser.go | 49 +++++++++++-------- tools/pika_exporter/exporter/pika.go | 23 +++++++-- 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/tools/pika_exporter/exporter/metrics/parser.go b/tools/pika_exporter/exporter/metrics/parser.go index 5d36968c03..b39ef486a8 100644 --- a/tools/pika_exporter/exporter/metrics/parser.go +++ b/tools/pika_exporter/exporter/metrics/parser.go @@ -26,13 +26,14 @@ type ParseOption struct { type VersionChecker interface { CheckContainsEmptyValueName(key string) bool CheckContainsEmptyRegexName(key string) bool + InitVersionChecker() } type VersionChecker336 struct { EmptyValueName []string EmptyRegexName []string } -func (v *VersionChecker336) CheckContainsEmptyValueName(key string) bool { +func (v *VersionChecker336) InitVersionChecker() { if v.EmptyValueName == nil { v.EmptyValueName = []string{ "instantaneous_output_repl_kbps", @@ -50,6 +51,13 @@ func (v *VersionChecker336) CheckContainsEmptyValueName(key string) bool { "cache_memory", } } + if v.EmptyRegexName == nil { + v.EmptyRegexName = []string{ + "hitratio_per_sec", + } + } +} +func (v *VersionChecker336) CheckContainsEmptyValueName(key string) bool { for _, str := range v.EmptyValueName { if str == key { return true @@ -58,11 +66,6 @@ func (v *VersionChecker336) CheckContainsEmptyValueName(key string) bool { return false } func (v *VersionChecker336) CheckContainsEmptyRegexName(key string) bool { - if v.EmptyRegexName == nil { - v.EmptyRegexName = []string{ - "hitratio_per_sec", - } - } for _, str := range v.EmptyRegexName { if str == key { return true @@ -76,7 +79,7 @@ type VersionChecker350 struct { EmptyRegexName []string } -func (v *VersionChecker350) CheckContainsEmptyValueName(key string) bool { +func (v *VersionChecker350) InitVersionChecker() { if v.EmptyValueName == nil { v.EmptyValueName = []string{ "cache_db_num", @@ -86,6 +89,13 @@ func (v *VersionChecker350) CheckContainsEmptyValueName(key string) bool { "slow_logs_count", } } + if v.EmptyRegexName == nil { + v.EmptyRegexName = []string{ + "hitratio_per_sec", + } + } +} +func (v *VersionChecker350) CheckContainsEmptyValueName(key string) bool { for _, str := range v.EmptyValueName { if str == key { return true @@ -94,11 +104,6 @@ func (v *VersionChecker350) CheckContainsEmptyValueName(key string) bool { return false } func (v *VersionChecker350) CheckContainsEmptyRegexName(key string) bool { - if v.EmptyRegexName == nil { - v.EmptyRegexName = []string{ - "hitratio_per_sec", - } - } for _, str := range v.EmptyRegexName { if str == key { return true @@ -112,7 +117,7 @@ type VersionChecker355 struct { EmptyRegexName []string } -func (v *VersionChecker355) CheckContainsEmptyValueName(key string) bool { +func (v *VersionChecker355) InitVersionChecker() { if v.EmptyValueName == nil { v.EmptyValueName = []string{ "cache_db_num", @@ -121,14 +126,6 @@ func (v *VersionChecker355) CheckContainsEmptyValueName(key string) bool { "hits_per_sec", } } - for _, str := range v.EmptyValueName { - if str == key { - return true - } - } - return false -} -func (v *VersionChecker355) CheckContainsEmptyRegexName(key string) bool { if v.EmptyRegexName == nil { v.EmptyRegexName = []string{ "hitratio_per_sec", @@ -139,6 +136,16 @@ func (v *VersionChecker355) CheckContainsEmptyRegexName(key string) bool { "is_scaning_keyspace", } } +} +func (v *VersionChecker355) CheckContainsEmptyValueName(key string) bool { + for _, str := range v.EmptyValueName { + if str == key { + return true + } + } + return false +} +func (v *VersionChecker355) CheckContainsEmptyRegexName(key string) bool { for _, str := range v.EmptyRegexName { if str == key { return true diff --git a/tools/pika_exporter/exporter/pika.go b/tools/pika_exporter/exporter/pika.go index 77eb7f0311..e3fe2631c9 100644 --- a/tools/pika_exporter/exporter/pika.go +++ b/tools/pika_exporter/exporter/pika.go @@ -264,22 +264,37 @@ func (e *exporter) collectInfo(c *client, ch chan<- prometheus.Metric) error { return nil } +const ( + VERSION_336 = "3.3.6" + VERSION_350 = "3.5.0" + VERSION_355 = "3.5.5" +) + func selectversion(version string) metrics.VersionChecker { + if !isValidVersion(version) { + log.Warnf("Invalid version format: %s", version) + return nil + } var v metrics.VersionChecker - switch version { - case "3.3.6": + case VERSION_336: v = &metrics.VersionChecker336{} - case "3.5.5": + case VERSION_355: v = &metrics.VersionChecker355{} - case "3.5.0": + case VERSION_350: v = &metrics.VersionChecker350{} default: return nil } + v.InitVersionChecker() return v } +// isValidVersion validates the version string format (e.g., x.y.z) +func isValidVersion(version string) bool { + matched, _ := regexp.MatchString(`^\d+\.\d+\.\d+$`, version) + return matched +} func (e *exporter) collectKeys(c *client) error { allKeys := append([]dbKeyPair{}, e.keys...) keys, err := getKeysFromPatterns(c, e.keyPatterns, e.scanCount) From f5c0cdbc92e1644aab275d6f499c3aa0482eae0f Mon Sep 17 00:00:00 2001 From: buzhimingyonghu <1512049108@qq.com> Date: Mon, 9 Dec 2024 03:04:16 +0000 Subject: [PATCH 4/5] add 336 ignore --- .../pika_exporter/exporter/metrics/parser.go | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tools/pika_exporter/exporter/metrics/parser.go b/tools/pika_exporter/exporter/metrics/parser.go index b39ef486a8..c5771af734 100644 --- a/tools/pika_exporter/exporter/metrics/parser.go +++ b/tools/pika_exporter/exporter/metrics/parser.go @@ -52,9 +52,50 @@ func (v *VersionChecker336) InitVersionChecker() { } } if v.EmptyRegexName == nil { + v.EmptyRegexName = []string{ "hitratio_per_sec", + "total_blob_file_size", + "block_cache_capacity", + "background_errors", + "num_running_flushes", + "mem_table_flush_pending", + "estimate_pending_compaction_bytes", + "block_cache_pinned_usage", + "pending_compaction_bytes_stops", + "estimate_live_data_size", + "pending_compaction_bytes_delays", + "num_running_compactions", + "live_blob_file_size", + "cur_size_active_mem_table", + "block_cache_usage", + "cf_l0_file_count_limit_stops_with_ongoing_compaction", + "cur_size_all_mem_tables", + "num_immutable_mem_table", + "compaction_pending", + "live_sst_files_size", + "memtable_limit_stops", + "total_delays", + "l0_file_count_limit_delays", + "estimate_table_readers_mem", + "num_immutable_mem_table_flushed", + "compaction_Sum", + "size_all_mem_tables", + "total_sst_files_size", + "commandstats_info", + "num_snapshots", + "current_super_version_number", + "memtable_limit_delays", + "estimate_num_keys", + "num_blob_files", + "total_stops", + "cf_l0_file_count_limit_delays_with_ongoing_compaction", + "num_live_versions", + "l0_file_count_limit_stops", + "compaction", + "blob_stats", } + } } func (v *VersionChecker336) CheckContainsEmptyValueName(key string) bool { From ffc275290608da7878b8f1c48e037d08e957522b Mon Sep 17 00:00:00 2001 From: buzhimingyonghu <1512049108@qq.com> Date: Mon, 9 Dec 2024 03:13:28 +0000 Subject: [PATCH 5/5] delete keyspace-stats-clock log --- tools/pika_exporter/exporter/pika.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/pika_exporter/exporter/pika.go b/tools/pika_exporter/exporter/pika.go index e3fe2631c9..0e74da6356 100644 --- a/tools/pika_exporter/exporter/pika.go +++ b/tools/pika_exporter/exporter/pika.go @@ -390,7 +390,6 @@ func (e *exporter) statsKeySpace(hour int) { defer e.wg.Done() if hour < 0 { - log.Infoln("stats KeySpace not open") return }