From 87f20253b562dc09d2094f4c292a1329a34573fa Mon Sep 17 00:00:00 2001 From: tehbooom Date: Fri, 17 May 2024 11:10:48 -0400 Subject: [PATCH 01/13] added support for graphite tags --- .../metricbeat/module/statsd/server/data.go | 33 +++++++++++++++---- .../module/statsd/server/data_test.go | 14 ++++++++ 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/x-pack/metricbeat/module/statsd/server/data.go b/x-pack/metricbeat/module/statsd/server/data.go index 27024e26284..289d200d18f 100644 --- a/x-pack/metricbeat/module/statsd/server/data.go +++ b/x-pack/metricbeat/module/statsd/server/data.go @@ -33,13 +33,24 @@ type statsdMetric struct { func splitTags(rawTags, kvSep []byte) map[string]string { tags := map[string]string{} - for _, kv := range bytes.Split(rawTags, []byte(",")) { - kvSplit := bytes.SplitN(kv, kvSep, 2) - if len(kvSplit) != 2 { - logger.Warn("could not parse tags") - continue + if bytes.Contains(rawTags, []byte(",")) { + for _, kv := range bytes.Split(rawTags, []byte(",")) { + kvSplit := bytes.SplitN(kv, kvSep, 2) + if len(kvSplit) != 2 { + logger.Warn("could not parse tags") + continue + } + tags[string(kvSplit[0])] = string(kvSplit[1]) + } + } else { + for _, kv := range bytes.Split(rawTags, []byte(";")) { + kvSplit := bytes.SplitN(kv, kvSep, 2) + if len(kvSplit) != 2 { + logger.Warn("could not parse tags") + continue + } + tags[string(kvSplit[0])] = string(kvSplit[1]) } - tags[string(kvSplit[0])] = string(kvSplit[1]) } return tags } @@ -47,6 +58,7 @@ func splitTags(rawTags, kvSep []byte) map[string]string { func parseSingle(b []byte) (statsdMetric, error) { // format: :|[|@samplerate][|#:,:] // alternative: [,=,=]:|[|@samplerate] + // alternative: [;=;=]:|[|@samplerate] s := statsdMetric{} parts := bytes.SplitN(b, []byte("|"), 4) @@ -73,7 +85,14 @@ func parseSingle(b []byte) (statsdMetric, error) { return s, errInvalidPacket } - nameTagsSplit := bytes.SplitN(nameSplit[0], []byte(","), 2) + var nameTagsSplit [][]byte + + if bytes.Contains(nameSplit[0], []byte(",")) { + nameTagsSplit = bytes.SplitN(nameSplit[0], []byte(","), 2) + } else { + nameTagsSplit = bytes.SplitN(nameSplit[0], []byte(";"), 2) + } + s.name = string(nameTagsSplit[0]) if len(nameTagsSplit) > 1 { s.tags = splitTags(nameTagsSplit[1], []byte("=")) diff --git a/x-pack/metricbeat/module/statsd/server/data_test.go b/x-pack/metricbeat/module/statsd/server/data_test.go index 2bdc97ab5c2..a905ec79b01 100644 --- a/x-pack/metricbeat/module/statsd/server/data_test.go +++ b/x-pack/metricbeat/module/statsd/server/data_test.go @@ -1022,6 +1022,20 @@ func TestParseMetrics(t *testing.T) { }, }, }, + { // Graphite 1.1.x Tags + input: "tags3;k1=v1;k2=v2:1|c", + expected: []statsdMetric{ + { + name: "tags3", + metricType: "c", + value: "1", + tags: map[string]string{ + "k1": "v1", + "k2": "v2", + }, + }, + }, + }, /// errors { input: "meter1-1.4|m", From 6998046a1159a5f4f5122751b8889d6a68fb2e33 Mon Sep 17 00:00:00 2001 From: tehbooom Date: Fri, 17 May 2024 12:28:09 -0400 Subject: [PATCH 02/13] added more unit tests --- .../module/statsd/server/data_test.go | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/x-pack/metricbeat/module/statsd/server/data_test.go b/x-pack/metricbeat/module/statsd/server/data_test.go index a905ec79b01..b3e11db61f0 100644 --- a/x-pack/metricbeat/module/statsd/server/data_test.go +++ b/x-pack/metricbeat/module/statsd/server/data_test.go @@ -1078,6 +1078,17 @@ func TestParseSingle(t *testing.T) { tags: map[string]string{"k1": "v1", "k2": "v2"}, }, }, + "valid packet: counter with Graphite tags": { + input: "tags2;k1=v1;k2=v2:1|c", + err: nil, + want: statsdMetric{ + name: "tags2", + metricType: "c", + sampleRate: "", + value: "1", + tags: map[string]string{"k1": "v1", "k2": "v2"}, + }, + }, "valid packet: gauge": { input: "gauge1:1.0|g", err: nil, @@ -1138,13 +1149,14 @@ func TestTagsGrouping(t *testing.T) { "metric3:3|c|@0.1|#k1:v2,k2:v3", "metric4:4|ms|#k1:v2,k2:v3", + "metric5;k1=v3;k2=v4:5|c", } err := process(testData, ms) require.NoError(t, err) events := ms.getEvents() - assert.Len(t, events, 4) + assert.Len(t, events, 5) actualTags := []mapstr.M{} for _, e := range events { @@ -1176,6 +1188,12 @@ func TestTagsGrouping(t *testing.T) { "k2": "v3", }, }, + { + "labels": mapstr.M{ + "k1": "v3", + "k2": "v4", + }, + }, } assert.ElementsMatch(t, expectedTags, actualTags) @@ -1187,6 +1205,7 @@ func TestTagsCleanup(t *testing.T) { "metric1:1|g|#k1:v1,k2:v2", "metric2:3|ms|#k1:v2,k2:v3", + "metric3;k1=v3;k2=v4:5|c", } err := process(testData, ms) require.NoError(t, err) @@ -1194,7 +1213,7 @@ func TestTagsCleanup(t *testing.T) { time.Sleep(1000 * time.Millisecond) // they will be reported at least once - assert.Len(t, ms.getEvents(), 2) + assert.Len(t, ms.getEvents(), 3) testData = []string{ "metric1:+2|g|#k1:v1,k2:v2", @@ -1243,12 +1262,13 @@ func TestData(t *testing.T) { "metric08:seven|s|#k1:v1,k2:v2", "metric09,k1=v1,k2=v2:8|h", "metric10.with.dots,k1=v1,k2=v2:9|h", + "metric11;k1=v1;k2=v2:10|c", } err := process(testData, ms) require.NoError(t, err) events := ms.getEvents() - assert.Len(t, events, 10) + assert.Len(t, events, 11) mbevent := mbtest.StandardizeEvent(ms, *events[0]) mbtest.WriteEventToDataJSON(t, mbevent, "") @@ -1393,6 +1413,7 @@ func BenchmarkIngest(b *testing.B) { "metric08:seven|s|#k1:v1,k2:v2", "metric09,k1=v1,k2=v2:8|h", "metric10.with.dots,k1=v1,k2=v2:9|h", + "metric11;k1=v1;k2=v2:10|c", } events := make([]*testUDPEvent, len(tests)) From afd1eb2eecd075c45f4e9afb47e24042456712a7 Mon Sep 17 00:00:00 2001 From: tehbooom Date: Mon, 20 May 2024 09:29:16 -0400 Subject: [PATCH 03/13] updated changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 054a62d2aec..df84f44d319 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -48,6 +48,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Metricbeat* - Setting period for counter cache for Prometheus remote_write at least to 60sec {pull}38553[38553] +- Added conditional to check if statsd metric contains , or ; and split accordingly {pull}39619[39619] *Osquerybeat* From c98e92022d7912a15a1ec4519f8ff59f68212612 Mon Sep 17 00:00:00 2001 From: tehbooom Date: Thu, 23 May 2024 08:12:40 -0400 Subject: [PATCH 04/13] Updated changelog --- CHANGELOG.next.asciidoc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index df84f44d319..175082e2ac0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -48,8 +48,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Metricbeat* - Setting period for counter cache for Prometheus remote_write at least to 60sec {pull}38553[38553] -- Added conditional to check if statsd metric contains , or ; and split accordingly {pull}39619[39619] - +- Added a conditional check to see if the statsd metric contains ',' or ';' and split accordingly. {pull}39619[39619] *Osquerybeat* From 190d3a51c2e37fe1e79ce5c02c817b28ff804f50 Mon Sep 17 00:00:00 2001 From: tehbooom Date: Thu, 23 May 2024 08:13:36 -0400 Subject: [PATCH 05/13] Fixed spacing between headers --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 175082e2ac0..b17a311b2a9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -49,6 +49,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Setting period for counter cache for Prometheus remote_write at least to 60sec {pull}38553[38553] - Added a conditional check to see if the statsd metric contains ',' or ';' and split accordingly. {pull}39619[39619] + *Osquerybeat* From 7af50938fa2ea6ccc3946edda1d28a3e41322c62 Mon Sep 17 00:00:00 2001 From: Alec Carpenter <57996746+tehbooom@users.noreply.github.com> Date: Thu, 23 May 2024 11:53:36 -0400 Subject: [PATCH 06/13] Update x-pack/metricbeat/module/statsd/server/data.go Co-authored-by: subham sarkar --- x-pack/metricbeat/module/statsd/server/data.go | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/metricbeat/module/statsd/server/data.go b/x-pack/metricbeat/module/statsd/server/data.go index 289d200d18f..1691fc12227 100644 --- a/x-pack/metricbeat/module/statsd/server/data.go +++ b/x-pack/metricbeat/module/statsd/server/data.go @@ -86,7 +86,6 @@ func parseSingle(b []byte) (statsdMetric, error) { } var nameTagsSplit [][]byte - if bytes.Contains(nameSplit[0], []byte(",")) { nameTagsSplit = bytes.SplitN(nameSplit[0], []byte(","), 2) } else { From bdbbf83e546ab05f9fa0abeef1386ec9e867427d Mon Sep 17 00:00:00 2001 From: tehbooom Date: Thu, 23 May 2024 13:45:36 -0400 Subject: [PATCH 07/13] Updated changelog --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b17a311b2a9..358044fe4df 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -48,7 +48,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Metricbeat* - Setting period for counter cache for Prometheus remote_write at least to 60sec {pull}38553[38553] -- Added a conditional check to see if the statsd metric contains ',' or ';' and split accordingly. {pull}39619[39619] +- Add support of Graphite series 1.1.0+ tagging extension for statsd module. {pull}39619[39619] *Osquerybeat* From 6f6a3c759c6af47403416804f2cda353f3274306 Mon Sep 17 00:00:00 2001 From: tehbooom Date: Thu, 23 May 2024 13:50:06 -0400 Subject: [PATCH 08/13] Fixed code repetition and added comments --- .../metricbeat/module/statsd/server/data.go | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/x-pack/metricbeat/module/statsd/server/data.go b/x-pack/metricbeat/module/statsd/server/data.go index 1691fc12227..d985464f6ff 100644 --- a/x-pack/metricbeat/module/statsd/server/data.go +++ b/x-pack/metricbeat/module/statsd/server/data.go @@ -33,24 +33,21 @@ type statsdMetric struct { func splitTags(rawTags, kvSep []byte) map[string]string { tags := map[string]string{} + var tagSplit [][]byte + if bytes.Contains(rawTags, []byte(",")) { - for _, kv := range bytes.Split(rawTags, []byte(",")) { - kvSplit := bytes.SplitN(kv, kvSep, 2) - if len(kvSplit) != 2 { - logger.Warn("could not parse tags") - continue - } - tags[string(kvSplit[0])] = string(kvSplit[1]) - } + tagSplit = bytes.Split(rawTags, []byte(",")) } else { - for _, kv := range bytes.Split(rawTags, []byte(";")) { - kvSplit := bytes.SplitN(kv, kvSep, 2) - if len(kvSplit) != 2 { - logger.Warn("could not parse tags") - continue - } - tags[string(kvSplit[0])] = string(kvSplit[1]) + tagSplit = bytes.Split(rawTags, []byte(";")) + } + + for _, kv := range tagSplit { + kvSplit := bytes.SplitN(kv, kvSep, 2) + if len(kvSplit) != 2 { + logger.Warn("could not parse tags") + continue } + tags[string(kvSplit[0])] = string(kvSplit[1]) } return tags } @@ -85,6 +82,8 @@ func parseSingle(b []byte) (statsdMetric, error) { return s, errInvalidPacket } + // Metric tags could be separated by `,` or `;` + // We split here based on the separator var nameTagsSplit [][]byte if bytes.Contains(nameSplit[0], []byte(",")) { nameTagsSplit = bytes.SplitN(nameSplit[0], []byte(","), 2) From 1229766898f250c1cfc28bf6df678df1904d3743 Mon Sep 17 00:00:00 2001 From: tehbooom Date: Thu, 23 May 2024 13:50:44 -0400 Subject: [PATCH 09/13] Updated README to show the valid extensions for the module --- .../module/statsd/_meta/docs.asciidoc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/x-pack/metricbeat/module/statsd/_meta/docs.asciidoc b/x-pack/metricbeat/module/statsd/_meta/docs.asciidoc index 34ccee20d3f..f6e6d48e91d 100644 --- a/x-pack/metricbeat/module/statsd/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/statsd/_meta/docs.asciidoc @@ -16,6 +16,23 @@ The module supports the following types of metrics: *Set (s)*:: Measurement which counts unique occurrences until flushed (value set to 0). +[float] +=== Suported tag extensions + +The `statsd` module supports the following tags: + +https://docs.datadoghq.com/developers/dogstatsd/datagram_shell/?tab=metrics#the-dogstatsd-protocol[DogStatsD-style tags] + +`:||@samplerate|#:,:` + +https://github.com/influxdata/telegraf/blob/master/plugins/inputs/statsd/README.md#influx-statsd[InfluxDB-style tags] + +`,=,=:||@samplerate` + +https://graphite.readthedocs.io/en/latest/tags.html#graphite-tag-support[Graphite_1.1.x-style tags] + +`;=;=:||@samplerate` + [float] === Module-specific configuration notes From 4654853ffb9414ebe9241deee3794ec1ef186a6f Mon Sep 17 00:00:00 2001 From: tehbooom Date: Thu, 23 May 2024 14:18:09 -0400 Subject: [PATCH 10/13] Updated module doc --- metricbeat/docs/modules/statsd.asciidoc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/metricbeat/docs/modules/statsd.asciidoc b/metricbeat/docs/modules/statsd.asciidoc index 467093a9c63..be98b2b8b16 100644 --- a/metricbeat/docs/modules/statsd.asciidoc +++ b/metricbeat/docs/modules/statsd.asciidoc @@ -28,6 +28,23 @@ The module supports the following types of metrics: *Set (s)*:: Measurement which counts unique occurrences until flushed (value set to 0). +[float] +=== Suported tag extensions + +The `statsd` module supports the following tags: + +https://docs.datadoghq.com/developers/dogstatsd/datagram_shell/?tab=metrics#the-dogstatsd-protocol[DogStatsD-style tags] + +`:||@samplerate|#:,:` + +https://github.com/influxdata/telegraf/blob/master/plugins/inputs/statsd/README.md#influx-statsd[InfluxDB-style tags] + +`,=,=:||@samplerate` + +https://graphite.readthedocs.io/en/latest/tags.html#graphite-tag-support[Graphite_1.1.x-style tags] + +`;=;=:||@samplerate` + [float] === Module-specific configuration notes From 82e71b8bc8fa97524caa80caeb9818d2e1dba427 Mon Sep 17 00:00:00 2001 From: tehbooom Date: Fri, 24 May 2024 08:24:07 -0400 Subject: [PATCH 11/13] Updated module documentation showing supported tags --- metricbeat/docs/modules/statsd.asciidoc | 8 ++++---- x-pack/metricbeat/module/statsd/_meta/docs.asciidoc | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/metricbeat/docs/modules/statsd.asciidoc b/metricbeat/docs/modules/statsd.asciidoc index be98b2b8b16..47890f0ed44 100644 --- a/metricbeat/docs/modules/statsd.asciidoc +++ b/metricbeat/docs/modules/statsd.asciidoc @@ -31,17 +31,17 @@ The module supports the following types of metrics: [float] === Suported tag extensions -The `statsd` module supports the following tags: +Example of tag styles supported by the `statsd` module: -https://docs.datadoghq.com/developers/dogstatsd/datagram_shell/?tab=metrics#the-dogstatsd-protocol[DogStatsD-style tags] +https://docs.datadoghq.com/developers/dogstatsd/datagram_shell/?tab=metrics#the-dogstatsd-protocol[DogStatsD] `:||@samplerate|#:,:` -https://github.com/influxdata/telegraf/blob/master/plugins/inputs/statsd/README.md#influx-statsd[InfluxDB-style tags] +https://github.com/influxdata/telegraf/blob/master/plugins/inputs/statsd/README.md#influx-statsd[InfluxDB] `,=,=:||@samplerate` -https://graphite.readthedocs.io/en/latest/tags.html#graphite-tag-support[Graphite_1.1.x-style tags] +https://graphite.readthedocs.io/en/latest/tags.html#graphite-tag-support[Graphite_1.1.x] `;=;=:||@samplerate` diff --git a/x-pack/metricbeat/module/statsd/_meta/docs.asciidoc b/x-pack/metricbeat/module/statsd/_meta/docs.asciidoc index f6e6d48e91d..a65e50c3281 100644 --- a/x-pack/metricbeat/module/statsd/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/statsd/_meta/docs.asciidoc @@ -19,17 +19,17 @@ The module supports the following types of metrics: [float] === Suported tag extensions -The `statsd` module supports the following tags: +Example of tag styles supported by the `statsd` module: -https://docs.datadoghq.com/developers/dogstatsd/datagram_shell/?tab=metrics#the-dogstatsd-protocol[DogStatsD-style tags] +https://docs.datadoghq.com/developers/dogstatsd/datagram_shell/?tab=metrics#the-dogstatsd-protocol[DogStatsD] `:||@samplerate|#:,:` -https://github.com/influxdata/telegraf/blob/master/plugins/inputs/statsd/README.md#influx-statsd[InfluxDB-style tags] +https://github.com/influxdata/telegraf/blob/master/plugins/inputs/statsd/README.md#influx-statsd[InfluxDB] `,=,=:||@samplerate` -https://graphite.readthedocs.io/en/latest/tags.html#graphite-tag-support[Graphite_1.1.x-style tags] +https://graphite.readthedocs.io/en/latest/tags.html#graphite-tag-support[Graphite_1.1.x] `;=;=:||@samplerate` From 636dc35b56bb0654eecce0c16cbd314d85bd2e83 Mon Sep 17 00:00:00 2001 From: subham sarkar Date: Wed, 29 May 2024 13:07:09 +0530 Subject: [PATCH 12/13] Update metricbeat/docs/modules/statsd.asciidoc Co-authored-by: Aliabbas Attarwala <124054599+aliabbas-elastic@users.noreply.github.com> --- metricbeat/docs/modules/statsd.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/docs/modules/statsd.asciidoc b/metricbeat/docs/modules/statsd.asciidoc index 47890f0ed44..1f3a961d15f 100644 --- a/metricbeat/docs/modules/statsd.asciidoc +++ b/metricbeat/docs/modules/statsd.asciidoc @@ -29,7 +29,7 @@ The module supports the following types of metrics: *Set (s)*:: Measurement which counts unique occurrences until flushed (value set to 0). [float] -=== Suported tag extensions +=== Supported tag extensions Example of tag styles supported by the `statsd` module: From 71b89041fae287c9f6c7d5a26f374abcd2332f3e Mon Sep 17 00:00:00 2001 From: tehbooom Date: Wed, 29 May 2024 08:01:17 -0400 Subject: [PATCH 13/13] spelling --- x-pack/metricbeat/module/statsd/_meta/docs.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/statsd/_meta/docs.asciidoc b/x-pack/metricbeat/module/statsd/_meta/docs.asciidoc index a65e50c3281..d3b6d385964 100644 --- a/x-pack/metricbeat/module/statsd/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/statsd/_meta/docs.asciidoc @@ -17,7 +17,7 @@ The module supports the following types of metrics: *Set (s)*:: Measurement which counts unique occurrences until flushed (value set to 0). [float] -=== Suported tag extensions +=== Supported tag extensions Example of tag styles supported by the `statsd` module: