From 6199043588c5a12507f38fb50f7c4445b58d1c0b Mon Sep 17 00:00:00 2001 From: ndrpnt <22328659+ndrpnt@users.noreply.github.com> Date: Thu, 7 Dec 2023 15:21:06 +0100 Subject: [PATCH 01/17] operator: fix production templates namespaces (#3257) Co-authored-by: Craig Peterson <192540+captncraig@users.noreply.github.com> --- .../build/templates/operator/main.jsonnet | 6 +-- .../operator/templates/agent-operator.yaml | 48 +++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/production/kubernetes/build/templates/operator/main.jsonnet b/production/kubernetes/build/templates/operator/main.jsonnet index dda322bac503..fe49426c4fa4 100644 --- a/production/kubernetes/build/templates/operator/main.jsonnet +++ b/production/kubernetes/build/templates/operator/main.jsonnet @@ -101,7 +101,7 @@ local ksm = import 'kube-state-metrics/kube-state-metrics.libsonnet'; name='kubelet-monitor', namespace=this._config.namespace, monitorLabels={ instance: 'primary' }, - targetNamespace=this._config.namespace, + targetNamespace='default', targetLabels={ 'app.kubernetes.io/name': 'kubelet' }, jobLabel=this._config.kubelet_job, metricsPath='/metrics', @@ -110,9 +110,9 @@ local ksm = import 'kube-state-metrics/kube-state-metrics.libsonnet'; ), mon_util.newKubernetesMonitor( name='cadvisor-monitor', - namespace='default', + namespace=this._config.namespace, monitorLabels={ instance: 'primary' }, - targetNamespace=this._config.namespace, + targetNamespace='default', targetLabels={ 'app.kubernetes.io/name': 'kubelet' }, jobLabel=this._config.cadvisor_job, metricsPath='/metrics/cadvisor', diff --git a/production/operator/templates/agent-operator.yaml b/production/operator/templates/agent-operator.yaml index 797ce35fa972..ba8c08e75e6f 100644 --- a/production/operator/templates/agent-operator.yaml +++ b/production/operator/templates/agent-operator.yaml @@ -563,83 +563,83 @@ kind: ServiceMonitor metadata: labels: instance: primary - name: ksm-monitor + name: cadvisor-monitor namespace: ${NAMESPACE} spec: endpoints: - - honorLabels: true + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: true interval: 60s - path: /metrics - port: http-metrics + path: /metrics/cadvisor + port: https-metrics relabelings: + - sourceLabels: + - __metrics_path__ + targetLabel: metrics_path - action: replace - replacement: kube-state-metrics + replacement: cadvisor targetLabel: job + scheme: https + tlsConfig: + insecureSkipVerify: true namespaceSelector: matchNames: - - ${NAMESPACE} + - default selector: matchLabels: - app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/name: kubelet --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: labels: instance: primary - name: kubelet-monitor + name: ksm-monitor namespace: ${NAMESPACE} spec: endpoints: - - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - honorLabels: true + - honorLabels: true interval: 60s path: /metrics - port: https-metrics + port: http-metrics relabelings: - - sourceLabels: - - __metrics_path__ - targetLabel: metrics_path - action: replace - replacement: kubelet + replacement: kube-state-metrics targetLabel: job - scheme: https - tlsConfig: - insecureSkipVerify: true namespaceSelector: matchNames: - ${NAMESPACE} selector: matchLabels: - app.kubernetes.io/name: kubelet + app.kubernetes.io/name: kube-state-metrics --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: labels: instance: primary - name: cadvisor-monitor - namespace: default + name: kubelet-monitor + namespace: ${NAMESPACE} spec: endpoints: - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token honorLabels: true interval: 60s - path: /metrics/cadvisor + path: /metrics port: https-metrics relabelings: - sourceLabels: - __metrics_path__ targetLabel: metrics_path - action: replace - replacement: cadvisor + replacement: kubelet targetLabel: job scheme: https tlsConfig: insecureSkipVerify: true namespaceSelector: matchNames: - - ${NAMESPACE} + - default selector: matchLabels: app.kubernetes.io/name: kubelet From 394721eb04c5b216ed5fc8d9c265ab7eed7fd019 Mon Sep 17 00:00:00 2001 From: Erik Baranowski <39704712+erikbaranowski@users.noreply.github.com> Date: Thu, 7 Dec 2023 18:14:25 -0500 Subject: [PATCH 02/17] Fix windows tests 2 (#5932) --- component/loki/source/windowsevent/bookmark.go | 4 +++- docs/docs_updated_test.go | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/component/loki/source/windowsevent/bookmark.go b/component/loki/source/windowsevent/bookmark.go index bd42f2532ccd..1997427ce50b 100644 --- a/component/loki/source/windowsevent/bookmark.go +++ b/component/loki/source/windowsevent/bookmark.go @@ -8,11 +8,12 @@ package windowsevent import ( "bytes" "errors" - "github.com/natefinch/atomic" "io" "io/fs" "os" + "github.com/natefinch/atomic" + "github.com/grafana/loki/clients/pkg/promtail/targets/windows/win_eventlog" ) @@ -55,6 +56,7 @@ func newBookMark(path string) (*bookMark, error) { if err != nil { return nil, err } + defer file.Close() fileContent, err := io.ReadAll(file) if err != nil { return nil, err diff --git a/docs/docs_updated_test.go b/docs/docs_updated_test.go index 21e9081c9493..a3c17e98b393 100644 --- a/docs/docs_updated_test.go +++ b/docs/docs_updated_test.go @@ -1,3 +1,5 @@ +//go:build !windows + package docs import ( From bd1992652e9173ed3a1c84be325621bbb1a2050d Mon Sep 17 00:00:00 2001 From: Domas Date: Fri, 8 Dec 2023 11:26:18 +0200 Subject: [PATCH 03/17] chore(deps): remove minio dependency (#5933) * remove minio dependency * gofmt --- component/faro/receiver/sourcemaps.go | 2 +- go.mod | 5 +- go.sum | 2 - .../v2/app_agent_receiver/utils.go | 2 +- pkg/util/wildcard/match.go | 71 +++ pkg/util/wildcard/match_test.go | 542 ++++++++++++++++++ 6 files changed, 617 insertions(+), 7 deletions(-) create mode 100644 pkg/util/wildcard/match.go create mode 100644 pkg/util/wildcard/match_test.go diff --git a/component/faro/receiver/sourcemaps.go b/component/faro/receiver/sourcemaps.go index 7f1183c27c27..0c679052b6a9 100644 --- a/component/faro/receiver/sourcemaps.go +++ b/component/faro/receiver/sourcemaps.go @@ -18,7 +18,7 @@ import ( "github.com/go-sourcemap/sourcemap" "github.com/grafana/agent/component/faro/receiver/internal/payload" "github.com/grafana/agent/pkg/flow/logging/level" - "github.com/minio/pkg/wildcard" + "github.com/grafana/agent/pkg/util/wildcard" "github.com/prometheus/client_golang/prometheus" "github.com/vincent-petithory/dataurl" ) diff --git a/go.mod b/go.mod index ab84d4303dcd..3c16f9b4ba83 100644 --- a/go.mod +++ b/go.mod @@ -90,7 +90,6 @@ require ( github.com/lib/pq v1.10.7 github.com/mackerelio/go-osstat v0.2.3 github.com/miekg/dns v1.1.55 - github.com/minio/pkg v1.5.8 github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 github.com/mitchellh/reflectwalk v1.0.2 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f @@ -474,8 +473,6 @@ require ( github.com/mdlayher/socket v0.4.1 // indirect github.com/mdlayher/wifi v0.1.0 // indirect github.com/microsoft/go-mssqldb v0.19.0 // indirect - github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect - github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -632,6 +629,8 @@ require ( github.com/lightstep/go-expohisto v1.0.0 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a // indirect + github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect + github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig v0.87.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/kafka v0.87.0 // indirect github.com/openshift/api v3.9.0+incompatible // indirect diff --git a/go.sum b/go.sum index 65e785069389..ac3ae21e9703 100644 --- a/go.sum +++ b/go.sum @@ -1582,8 +1582,6 @@ github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcs github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= -github.com/minio/pkg v1.5.8 h1:ryx23f28havoidUezmYRNgaZpbyn4y3m2yp/vfasFy0= -github.com/minio/pkg v1.5.8/go.mod h1:EiGlHS2xaooa2VMxhJsxxAZHDObHVUB3HwtuoEXOCVE= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= diff --git a/pkg/integrations/v2/app_agent_receiver/utils.go b/pkg/integrations/v2/app_agent_receiver/utils.go index e7ef135b7601..9ea0d19d95f5 100644 --- a/pkg/integrations/v2/app_agent_receiver/utils.go +++ b/pkg/integrations/v2/app_agent_receiver/utils.go @@ -4,7 +4,7 @@ import ( "fmt" "sort" - "github.com/minio/pkg/wildcard" + "github.com/grafana/agent/pkg/util/wildcard" om "github.com/wk8/go-ordered-map" ) diff --git a/pkg/util/wildcard/match.go b/pkg/util/wildcard/match.go new file mode 100644 index 000000000000..c9c4b97e2004 --- /dev/null +++ b/pkg/util/wildcard/match.go @@ -0,0 +1,71 @@ +/* + * MinIO Cloud Storage, (C) 2015, 2016 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wildcard + +// MatchSimple - finds whether the text matches/satisfies the pattern string. +// supports only '*' wildcard in the pattern. +// considers a file system path as a flat name space. +func MatchSimple(pattern, name string) bool { + if pattern == "" { + return name == pattern + } + + if pattern == "*" { + return true + } + // Does only wildcard '*' match. + return deepMatchRune([]rune(name), []rune(pattern), true) +} + +// Match - finds whether the text matches/satisfies the pattern string. +// supports '*' and '?' wildcards in the pattern string. +// unlike path.Match(), considers a path as a flat name space while matching the pattern. +// The difference is illustrated in the example here https://play.golang.org/p/Ega9qgD4Qz . +func Match(pattern, name string) (matched bool) { + if pattern == "" { + return name == pattern + } + + if pattern == "*" { + return true + } + // Does extended wildcard '*' and '?' match. + return deepMatchRune([]rune(name), []rune(pattern), false) +} + +func deepMatchRune(str, pattern []rune, simple bool) bool { + for len(pattern) > 0 { + switch pattern[0] { + default: + if len(str) == 0 || str[0] != pattern[0] { + return false + } + case '?': + if len(str) == 0 && !simple { + return false + } + case '*': + return deepMatchRune(str, pattern[1:], simple) || + (len(str) > 0 && deepMatchRune(str[1:], pattern, simple)) + } + + str = str[1:] + pattern = pattern[1:] + } + + return len(str) == 0 && len(pattern) == 0 +} diff --git a/pkg/util/wildcard/match_test.go b/pkg/util/wildcard/match_test.go new file mode 100644 index 000000000000..ea9caf59e3ed --- /dev/null +++ b/pkg/util/wildcard/match_test.go @@ -0,0 +1,542 @@ +/* + * MinIO Cloud Storage, (C) 2015, 2016 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wildcard_test + +import ( + "testing" + + "github.com/grafana/agent/pkg/util/wildcard" +) + +// TestMatch - Tests validate the logic of wild card matching. +// `Match` supports '*' and '?' wildcards. +// Sample usage: In resource matching for bucket policy validation. +func TestMatch(t *testing.T) { + testCases := []struct { + pattern string + text string + matched bool + }{ + // Test case - 1. + // Test case with pattern "*". Expected to match any text. + { + pattern: "*", + text: "s3:GetObject", + matched: true, + }, + // Test case - 2. + // Test case with empty pattern. This only matches empty string. + { + pattern: "", + text: "s3:GetObject", + matched: false, + }, + // Test case - 3. + // Test case with empty pattern. This only matches empty string. + { + pattern: "", + text: "", + matched: true, + }, + // Test case - 4. + // Test case with single "*" at the end. + { + pattern: "s3:*", + text: "s3:ListMultipartUploadParts", + matched: true, + }, + // Test case - 5. + // Test case with a no "*". In this case the pattern and text should be the same. + { + pattern: "s3:ListBucketMultipartUploads", + text: "s3:ListBucket", + matched: false, + }, + // Test case - 6. + // Test case with a no "*". In this case the pattern and text should be the same. + { + pattern: "s3:ListBucket", + text: "s3:ListBucket", + matched: true, + }, + // Test case - 7. + // Test case with a no "*". In this case the pattern and text should be the same. + { + pattern: "s3:ListBucketMultipartUploads", + text: "s3:ListBucketMultipartUploads", + matched: true, + }, + // Test case - 8. + // Test case with pattern containing key name with a prefix. Should accept the same text without a "*". + { + pattern: "my-bucket/oo*", + text: "my-bucket/oo", + matched: true, + }, + // Test case - 9. + // Test case with "*" at the end of the pattern. + { + pattern: "my-bucket/In*", + text: "my-bucket/India/Karnataka/", + matched: true, + }, + // Test case - 10. + // Test case with prefixes shuffled. + // This should fail. + { + pattern: "my-bucket/In*", + text: "my-bucket/Karnataka/India/", + matched: false, + }, + // Test case - 11. + // Test case with text expanded to the wildcards in the pattern. + { + pattern: "my-bucket/In*/Ka*/Ban", + text: "my-bucket/India/Karnataka/Ban", + matched: true, + }, + // Test case - 12. + // Test case with the keyname part is repeated as prefix several times. + // This is valid. + { + pattern: "my-bucket/In*/Ka*/Ban", + text: "my-bucket/India/Karnataka/Ban/Ban/Ban/Ban/Ban", + matched: true, + }, + // Test case - 13. + // Test case to validate that `*` can be expanded into multiple prefixes. + { + pattern: "my-bucket/In*/Ka*/Ban", + text: "my-bucket/India/Karnataka/Area1/Area2/Area3/Ban", + matched: true, + }, + // Test case - 14. + // Test case to validate that `*` can be expanded into multiple prefixes. + { + pattern: "my-bucket/In*/Ka*/Ban", + text: "my-bucket/India/State1/State2/Karnataka/Area1/Area2/Area3/Ban", + matched: true, + }, + // Test case - 15. + // Test case where the keyname part of the pattern is expanded in the text. + { + pattern: "my-bucket/In*/Ka*/Ban", + text: "my-bucket/India/Karnataka/Bangalore", + matched: false, + }, + // Test case - 16. + // Test case with prefixes and wildcard expanded for all "*". + { + pattern: "my-bucket/In*/Ka*/Ban*", + text: "my-bucket/India/Karnataka/Bangalore", + matched: true, + }, + // Test case - 17. + // Test case with keyname part being a wildcard in the pattern. + { + pattern: "my-bucket/*", + text: "my-bucket/India", + matched: true, + }, + // Test case - 18. + { + pattern: "my-bucket/oo*", + text: "my-bucket/odo", + matched: false, + }, + + // Test case with pattern containing wildcard '?'. + // Test case - 19. + // "my-bucket?/" matches "my-bucket1/", "my-bucket2/", "my-bucket3" etc... + // doesn't match "mybucket/". + { + pattern: "my-bucket?/abc*", + text: "mybucket/abc", + matched: false, + }, + // Test case - 20. + { + pattern: "my-bucket?/abc*", + text: "my-bucket1/abc", + matched: true, + }, + // Test case - 21. + { + pattern: "my-?-bucket/abc*", + text: "my--bucket/abc", + matched: false, + }, + // Test case - 22. + { + pattern: "my-?-bucket/abc*", + text: "my-1-bucket/abc", + matched: true, + }, + // Test case - 23. + { + pattern: "my-?-bucket/abc*", + text: "my-k-bucket/abc", + matched: true, + }, + // Test case - 24. + { + pattern: "my??bucket/abc*", + text: "mybucket/abc", + matched: false, + }, + // Test case - 25. + { + pattern: "my??bucket/abc*", + text: "my4abucket/abc", + matched: true, + }, + // Test case - 26. + { + pattern: "my-bucket?abc*", + text: "my-bucket/abc", + matched: true, + }, + // Test case 27-28. + // '?' matches '/' too. (works with s3). + // This is because the namespace is considered flat. + // "abc?efg" matches both "abcdefg" and "abc/efg". + { + pattern: "my-bucket/abc?efg", + text: "my-bucket/abcdefg", + matched: true, + }, + { + pattern: "my-bucket/abc?efg", + text: "my-bucket/abc/efg", + matched: true, + }, + // Test case - 29. + { + pattern: "my-bucket/abc????", + text: "my-bucket/abc", + matched: false, + }, + // Test case - 30. + { + pattern: "my-bucket/abc????", + text: "my-bucket/abcde", + matched: false, + }, + // Test case - 31. + { + pattern: "my-bucket/abc????", + text: "my-bucket/abcdefg", + matched: true, + }, + // Test case 32-34. + // test case with no '*'. + { + pattern: "my-bucket/abc?", + text: "my-bucket/abc", + matched: false, + }, + { + pattern: "my-bucket/abc?", + text: "my-bucket/abcd", + matched: true, + }, + { + pattern: "my-bucket/abc?", + text: "my-bucket/abcde", + matched: false, + }, + // Test case 35. + { + pattern: "my-bucket/mnop*?", + text: "my-bucket/mnop", + matched: false, + }, + // Test case 36. + { + pattern: "my-bucket/mnop*?", + text: "my-bucket/mnopqrst/mnopqr", + matched: true, + }, + // Test case 37. + { + pattern: "my-bucket/mnop*?", + text: "my-bucket/mnopqrst/mnopqrs", + matched: true, + }, + // Test case 38. + { + pattern: "my-bucket/mnop*?", + text: "my-bucket/mnop", + matched: false, + }, + // Test case 39. + { + pattern: "my-bucket/mnop*?", + text: "my-bucket/mnopq", + matched: true, + }, + // Test case 40. + { + pattern: "my-bucket/mnop*?", + text: "my-bucket/mnopqr", + matched: true, + }, + // Test case 41. + { + pattern: "my-bucket/mnop*?and", + text: "my-bucket/mnopqand", + matched: true, + }, + // Test case 42. + { + pattern: "my-bucket/mnop*?and", + text: "my-bucket/mnopand", + matched: false, + }, + // Test case 43. + { + pattern: "my-bucket/mnop*?and", + text: "my-bucket/mnopqand", + matched: true, + }, + // Test case 44. + { + pattern: "my-bucket/mnop*?", + text: "my-bucket/mn", + matched: false, + }, + // Test case 45. + { + pattern: "my-bucket/mnop*?", + text: "my-bucket/mnopqrst/mnopqrs", + matched: true, + }, + // Test case 46. + { + pattern: "my-bucket/mnop*??", + text: "my-bucket/mnopqrst", + matched: true, + }, + // Test case 47. + { + pattern: "my-bucket/mnop*qrst", + text: "my-bucket/mnopabcdegqrst", + matched: true, + }, + // Test case 48. + { + pattern: "my-bucket/mnop*?and", + text: "my-bucket/mnopqand", + matched: true, + }, + // Test case 49. + { + pattern: "my-bucket/mnop*?and", + text: "my-bucket/mnopand", + matched: false, + }, + // Test case 50. + { + pattern: "my-bucket/mnop*?and?", + text: "my-bucket/mnopqanda", + matched: true, + }, + // Test case 51. + { + pattern: "my-bucket/mnop*?and", + text: "my-bucket/mnopqanda", + matched: false, + }, + // Test case 52. + + { + pattern: "my-?-bucket/abc*", + text: "my-bucket/mnopqanda", + matched: false, + }, + } + // Iterating over the test cases, call the function under test and asert the output. + for i, testCase := range testCases { + actualResult := wildcard.Match(testCase.pattern, testCase.text) + if testCase.matched != actualResult { + t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.matched, actualResult) + } + } +} + +// TestMatchSimple - Tests validate the logic of wild card matching. +// `MatchSimple` supports matching for only '*' in the pattern string. +func TestMatchSimple(t *testing.T) { + testCases := []struct { + pattern string + text string + matched bool + }{ + // Test case - 1. + // Test case with pattern "*". Expected to match any text. + { + pattern: "*", + text: "s3:GetObject", + matched: true, + }, + // Test case - 2. + // Test case with empty pattern. This only matches empty string. + { + pattern: "", + text: "s3:GetObject", + matched: false, + }, + // Test case - 3. + // Test case with empty pattern. This only matches empty string. + { + pattern: "", + text: "", + matched: true, + }, + // Test case - 4. + // Test case with single "*" at the end. + { + pattern: "s3:*", + text: "s3:ListMultipartUploadParts", + matched: true, + }, + // Test case - 5. + // Test case with a no "*". In this case the pattern and text should be the same. + { + pattern: "s3:ListBucketMultipartUploads", + text: "s3:ListBucket", + matched: false, + }, + // Test case - 6. + // Test case with a no "*". In this case the pattern and text should be the same. + { + pattern: "s3:ListBucket", + text: "s3:ListBucket", + matched: true, + }, + // Test case - 7. + // Test case with a no "*". In this case the pattern and text should be the same. + { + pattern: "s3:ListBucketMultipartUploads", + text: "s3:ListBucketMultipartUploads", + matched: true, + }, + // Test case - 8. + // Test case with pattern containing key name with a prefix. Should accept the same text without a "*". + { + pattern: "my-bucket/oo*", + text: "my-bucket/oo", + matched: true, + }, + // Test case - 9. + // Test case with "*" at the end of the pattern. + { + pattern: "my-bucket/In*", + text: "my-bucket/India/Karnataka/", + matched: true, + }, + // Test case - 10. + // Test case with prefixes shuffled. + // This should fail. + { + pattern: "my-bucket/In*", + text: "my-bucket/Karnataka/India/", + matched: false, + }, + // Test case - 11. + // Test case with text expanded to the wildcards in the pattern. + { + pattern: "my-bucket/In*/Ka*/Ban", + text: "my-bucket/India/Karnataka/Ban", + matched: true, + }, + // Test case - 12. + // Test case with the keyname part is repeated as prefix several times. + // This is valid. + { + pattern: "my-bucket/In*/Ka*/Ban", + text: "my-bucket/India/Karnataka/Ban/Ban/Ban/Ban/Ban", + matched: true, + }, + // Test case - 13. + // Test case to validate that `*` can be expanded into multiple prefixes. + { + pattern: "my-bucket/In*/Ka*/Ban", + text: "my-bucket/India/Karnataka/Area1/Area2/Area3/Ban", + matched: true, + }, + // Test case - 14. + // Test case to validate that `*` can be expanded into multiple prefixes. + { + pattern: "my-bucket/In*/Ka*/Ban", + text: "my-bucket/India/State1/State2/Karnataka/Area1/Area2/Area3/Ban", + matched: true, + }, + // Test case - 15. + // Test case where the keyname part of the pattern is expanded in the text. + { + pattern: "my-bucket/In*/Ka*/Ban", + text: "my-bucket/India/Karnataka/Bangalore", + matched: false, + }, + // Test case - 16. + // Test case with prefixes and wildcard expanded for all "*". + { + pattern: "my-bucket/In*/Ka*/Ban*", + text: "my-bucket/India/Karnataka/Bangalore", + matched: true, + }, + // Test case - 17. + // Test case with keyname part being a wildcard in the pattern. + { + pattern: "my-bucket/*", + text: "my-bucket/India", + matched: true, + }, + // Test case - 18. + { + pattern: "my-bucket/oo*", + text: "my-bucket/odo", + matched: false, + }, + // Test case - 11. + { + pattern: "my-bucket/oo?*", + text: "my-bucket/oo???", + matched: true, + }, + // Test case - 12: + { + pattern: "my-bucket/oo??*", + text: "my-bucket/odo", + matched: false, + }, + // Test case - 13: + { + pattern: "?h?*", + text: "?h?hello", + matched: true, + }, + } + // Iterating over the test cases, call the function under test and asert the output. + for i, testCase := range testCases { + actualResult := wildcard.MatchSimple(testCase.pattern, testCase.text) + if testCase.matched != actualResult { + t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.matched, actualResult) + } + } +} From c0798fbe2b471baf7fbfced02333c268741deed5 Mon Sep 17 00:00:00 2001 From: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> Date: Fri, 8 Dec 2023 05:32:32 -0800 Subject: [PATCH 04/17] Update the style and syntax used in Agent Monitoring topics (#5929) * Minor updates and tweaks * Update docs/sources/flow/monitoring/debugging.md Co-authored-by: Jack Baldry --------- Co-authored-by: Jack Baldry --- .../flow/monitoring/component_metrics.md | 39 ++++----- .../flow/monitoring/controller_metrics.md | 31 +++---- docs/sources/flow/monitoring/debugging.md | 82 +++++++------------ 3 files changed, 56 insertions(+), 96 deletions(-) diff --git a/docs/sources/flow/monitoring/component_metrics.md b/docs/sources/flow/monitoring/component_metrics.md index 33c9d5c02e98..90c3769572c9 100644 --- a/docs/sources/flow/monitoring/component_metrics.md +++ b/docs/sources/flow/monitoring/component_metrics.md @@ -13,30 +13,21 @@ weight: 200 # Component metrics -{{< param "PRODUCT_NAME" >}} [components][] may optionally expose Prometheus metrics -which can be used to investigate the behavior of that component. These -component-specific metrics are only generated when an instance of that -component is running. - -> Component-specific metrics are different than any metrics being processed by -> the component. Component-specific metrics are used to expose the state of a -> component for observability, alerting, and debugging. - -Component-specific metrics are exposed at the `/metrics` HTTP endpoint of the -{{< param "PRODUCT_NAME" >}} HTTP server, which defaults to listening on -`http://localhost:12345`. - -> The documentation for the [`grafana-agent run`][grafana-agent run] command describes how to -> modify the address {{< param "PRODUCT_NAME" >}} listens on for HTTP traffic. - -Component-specific metrics will have a `component_id` label matching the -component ID generating those metrics. For example, component-specific metrics -for a `prometheus.remote_write` component labeled `production` will have a -`component_id` label with the value `prometheus.remote_write.production`. - -The [reference documentation][] for each component will describe the list of -component-specific metrics that component exposes. Not all components will -expose metrics. +{{< param "PRODUCT_NAME" >}} [components][] may optionally expose Prometheus metrics which can be used to investigate the behavior of that component. +These component-specific metrics are only generated when an instance of that component is running. + +> Component-specific metrics are different than any metrics being processed by the component. +> Component-specific metrics are used to expose the state of a component for observability, alerting, and debugging. + +Component-specific metrics are exposed at the `/metrics` HTTP endpoint of the {{< param "PRODUCT_NAME" >}} HTTP server, which defaults to listening on `http://localhost:12345`. + +> The documentation for the [`grafana-agent run`][grafana-agent run] command describes how to > modify the address {{< param "PRODUCT_NAME" >}} listens on for HTTP traffic. + +Component-specific metrics have a `component_id` label matching the component ID generating those metrics. +For example, component-specific metrics for a `prometheus.remote_write` component labeled `production` will have a `component_id` label with the value `prometheus.remote_write.production`. + +The [reference documentation][] for each component described the list of component-specific metrics that the component exposes. +Not all components expose metrics. {{% docs/reference %}} [components]: "/docs/agent/ -> /docs/agent//flow/concepts/components.md" diff --git a/docs/sources/flow/monitoring/controller_metrics.md b/docs/sources/flow/monitoring/controller_metrics.md index a785909d91fe..e0f17a2edba9 100644 --- a/docs/sources/flow/monitoring/controller_metrics.md +++ b/docs/sources/flow/monitoring/controller_metrics.md @@ -13,32 +13,21 @@ weight: 100 # Controller metrics -The {{< param "PRODUCT_NAME" >}} [component controller][] exposes Prometheus metrics -which can be used to investigate the controller state. +The {{< param "PRODUCT_NAME" >}} [component controller][] exposes Prometheus metrics which you can use to investigate the controller state. -Metrics for the controller are exposed at the `/metrics` HTTP endpoint of the -{{< param "PRODUCT_NAME" >}} HTTP server, which defaults to listening on -`http://localhost:12345`. +Metrics for the controller are exposed at the `/metrics` HTTP endpoint of the {{< param "PRODUCT_NAME" >}} HTTP server, which defaults to listening on `http://localhost:12345`. -> The documentation for the [`grafana-agent run`][grafana-agent run] command -> describes how to modify the address {{< param "PRODUCT_NAME" >}} listens on for HTTP -> traffic. +> The documentation for the [`grafana-agent run`][grafana-agent run] command describes how to modify the address {{< param "PRODUCT_NAME" >}} listens on for HTTP traffic. The controller exposes the following metrics: -* `agent_component_controller_evaluating` (Gauge): Set to `1` whenever the - component controller is currently evaluating components. Note that this value - may be misrepresented depending on how fast evaluations complete or how often - evaluations occur. -* `agent_component_controller_running_components` (Gauge): The current - number of running components by health. The health is represented in the - `health_type` label. -* `agent_component_evaluation_seconds` (Histogram): The time it takes to - evaluate components after one of their dependencies is updated. -* `agent_component_dependencies_wait_seconds` (Histogram): Time spent by - components waiting to be evaluated after one of their dependencies is updated. -* `agent_component_evaluation_queue_size` (Gauge): The current number of - component evaluations waiting to be performed. +* `agent_component_controller_evaluating` (Gauge): Set to `1` whenever the component controller is currently evaluating components. + This value may be misrepresented depending on how fast evaluations complete or how often evaluations occur. +* `agent_component_controller_running_components` (Gauge): The current number of running components by health. + The health is represented in the `health_type` label. +* `agent_component_evaluation_seconds` (Histogram): The time it takes to evaluate components after one of their dependencies is updated. +* `agent_component_dependencies_wait_seconds` (Histogram): Time spent by components waiting to be evaluated after one of their dependencies is updated. +* `agent_component_evaluation_queue_size` (Gauge): The current number of component evaluations waiting to be performed. {{% docs/reference %}} [component controller]: "/docs/agent/ -> /docs/agent//flow/concepts/component_controller.md" diff --git a/docs/sources/flow/monitoring/debugging.md b/docs/sources/flow/monitoring/debugging.md index eb1a87ef65d3..36b2f2636b97 100644 --- a/docs/sources/flow/monitoring/debugging.md +++ b/docs/sources/flow/monitoring/debugging.md @@ -15,36 +15,28 @@ weight: 300 Follow these steps to debug issues with {{< param "PRODUCT_NAME" >}}: 1. Use the {{< param "PRODUCT_NAME" >}} UI to debug issues. -2. If the {{< param "PRODUCT_NAME" >}} UI doesn't help with debugging an issue, logs can be examined - instead. +1. If the {{< param "PRODUCT_NAME" >}} UI doesn't help with debugging an issue, logs can be examined instead. ## {{< param "PRODUCT_NAME" >}} UI -{{< param "PRODUCT_NAME" >}} includes an embedded UI viewable from the {{< param "PRODUCT_ROOT_NAME" >}} HTTP -server, which defaults to listening at `http://localhost:12345`. +{{< param "PRODUCT_NAME" >}} includes an embedded UI viewable from the {{< param "PRODUCT_ROOT_NAME" >}} HTTP server, which defaults to listening at `http://localhost:12345`. -> **NOTE**: For security reasons, installations of {{< param "PRODUCT_NAME" >}} on -> non-containerized platforms default to listening on `localhost`. default -> prevents other machines on the network from being able to view the UI. +> **NOTE**: For security reasons, installations of {{< param "PRODUCT_NAME" >}} on non-containerized platforms default to listening on `localhost`. +> This default prevents other machines on the network from being able to view the UI. > -> To expose the UI to other machines on the network on non-containerized -> platforms, refer to the documentation for how you [installed][install] -> {{< param "PRODUCT_NAME" >}}. +> To expose the UI to other machines on the network on non-containerized platforms, refer to the documentation for how you [installed][install] {{< param "PRODUCT_NAME" >}}. > -> If you are running a custom installation of {{< param "PRODUCT_NAME" >}}, refer to the -> documentation for [the `grafana-agent run` command][grafana-agent run] to -> learn how to change the HTTP listen address, and pass the appropriate flag -> when running {{< param "PRODUCT_NAME" >}}. +> If you are running a custom installation of {{< param "PRODUCT_NAME" >}}, +> refer to the documentation for [the `grafana-agent run` command][grafana-agent run] to learn how to change the HTTP listen address, +> and pass the appropriate flag when running {{< param "PRODUCT_NAME" >}}. ### Home page ![](../../../assets/ui_home_page.png) -The home page shows a table of components defined in the configuration file along with -their health. +The home page shows a table of components defined in the configuration file and their health. -Click **View** on a row in the table to navigate to the [Component detail page](#component-detail-page) -for that component. +Click **View** on a row in the table to navigate to the [Component detail page](#component-detail-page) for that component. Click the {{< param "PRODUCT_ROOT_NAME" >}} logo to navigate back to the home page. @@ -52,9 +44,8 @@ Click the {{< param "PRODUCT_ROOT_NAME" >}} logo to navigate back to the home pa ![](../../../assets/ui_graph_page.png) -The **Graph** page shows a graph view of components defined in the configuration file -along with their health. Clicking a component in the graph navigates to the -[Component detail page](#component-detail-page) for that component. +The **Graph** page shows a graph view of components defined in the configuration file and their health. +Clicking a component in the graph navigates to the [Component detail page](#component-detail-page) for that component. ### Component detail page @@ -67,8 +58,7 @@ The component detail page shows the following information for each component: * The current exports for the component. * The current debug info for the component (if the component has debug info). -> Values marked as a [secret][] are obfuscated and will display as the text -> `(secret)`. +> Values marked as a [secret][] are obfuscated and display as the text `(secret)`. ### Clustering page @@ -86,45 +76,35 @@ The clustering page shows the following information for each cluster node: To debug using the UI: * Ensure that no component is reported as unhealthy. -* Ensure that the arguments and exports for misbehaving components appear - correct. +* Ensure that the arguments and exports for misbehaving components appear correct. ## Examining logs Logs may also help debug issues with {{< param "PRODUCT_NAME" >}}. -To reduce logging noise, many components hide debugging info behind debug-level -log lines. It is recommended that you configure the [`logging` block][logging] -to show debug-level log lines when debugging issues with {{< param "PRODUCT_NAME" >}}. +To reduce logging noise, many components hide debugging info behind debug-level log lines. +It is recommended that you configure the [`logging` block][logging] to show debug-level log lines when debugging issues with {{< param "PRODUCT_NAME" >}}. -The location of {{< param "PRODUCT_NAME" >}} logs is different based on how it is deployed. -Refer to the [`logging` block][logging] page to see how to find logs for your -system. +The location of {{< param "PRODUCT_NAME" >}} logs is different based on how it's deployed. +Refer to the [`logging` block][logging] page to see how to find logs for your system. ## Debugging clustering issues To debug issues when using [clustering][], check for the following symptoms. -- **Cluster not converging**: The cluster peers are not converging on the same - view of their peers' status. This is most likely due to network connectivity -issues between the cluster nodes. Use the Flow UI of each running peer to -understand which nodes are not being picked up correctly. -- **Cluster split brain**: The cluster peers are not aware of one another, - thinking they’re the only node present. Again, check for network connectivity -issues. Check that the addresses or DNS names given to the node to join are -correctly formatted and reachable. -- **Configuration drift**: Clustering assumes that all nodes are running with - the same configuration file at roughly the same time. Check the logs for -issues with the reloaded configuration file as well as the graph page to verify -changes have been applied. -- **Node name conflicts**: Clustering assumes all nodes have unique names; - nodes with conflicting names are rejected and will not join the cluster. Look -at the clustering UI page for the list of current peers with their names, and -check the logs for any reported name conflict events. -- **Node stuck in terminating state**: The node attempted to gracefully shut -down and set its state to Terminating, but it has not completely gone away. Check -the clustering page to view the state of the peers and verify that the -terminating {{< param "PRODUCT_ROOT_NAME" >}} has been shut down. +- **Cluster not converging**: The cluster peers aren't converging on the same view of their peers' status. + This is most likely due to network connectivity issues between the cluster nodes. + Use the {{< param "PRODUCT_NAME" >}} UI of each running peer to understand which nodes aren't being picked up correctly. +- **Cluster split brain**: The cluster peers aren't aware of one another, thinking they’re the only node present. + Again, check for network connectivity issues. + Check that the addresses or DNS names given to the node to join are correctly formatted and reachable. +- **Configuration drift**: Clustering assumes that all nodes are running with the same configuration file at roughly the same time. + Check the logs for issues with the reloaded configuration file as well as the graph page to verify changes have been applied. +- **Node name conflicts**: Clustering assumes all nodes have unique names. + Nodes with conflicting names are rejected and won't join the cluster. + Look at the clustering UI page for the list of current peers with their names, and check the logs for any reported name conflict events. +- **Node stuck in terminating state**: The node attempted to gracefully shut down and set its state to Terminating, but it has not completely gone away. + Check the clustering page to view the state of the peers and verify that the terminating {{< param "PRODUCT_ROOT_NAME" >}} has been shut down. {{% docs/reference %}} [logging]: "/docs/agent/ -> /docs/agent//flow/reference/config-blocks/logging.md" From 220187106bbee4996d880e875d5de509fea968d2 Mon Sep 17 00:00:00 2001 From: arunkg2787 <131352769+arunkg2787@users.noreply.github.com> Date: Fri, 8 Dec 2023 19:48:32 +0530 Subject: [PATCH 05/17] Add sharding example to operator docs (#4642) Signed-off-by: Paschalis Tsilias Co-authored-by: Paschalis Tsilias Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> Co-authored-by: Paschalis Tsilias --- docs/sources/operator/architecture.md | 66 +++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/docs/sources/operator/architecture.md b/docs/sources/operator/architecture.md index a76903ff3422..ba0b5c97fd06 100644 --- a/docs/sources/operator/architecture.md +++ b/docs/sources/operator/architecture.md @@ -138,3 +138,69 @@ Two labels are added by default to every metric: The shard number is not added as a label, as sharding is designed to be transparent on the receiver end. + +## Enable sharding and replication + +To enable sharding and replication, you must set the `shards` and `replicas` properties in the Grafana Agent configuration file. For example, the following configuration file would shard the data into three shards and replicate each shard to two other Grafana Agent instances: + +``` +shards: 3 +replicas: 2 +``` + +You can also enable sharding and replication by setting the `shards` and `replicas` arguments when you start the Grafana Agent. + +### Examples + +The following examples show you how to enable sharding and replication in a Kubernetes environment. + +* To shard the data into three shards and replicate each shard to two other Grafana Agent instances, you would use the following deployment manifest: + + ``` + apiVersion: apps/v1 + kind: Deployment + metadata: + name: grafana-agent + spec: + replicas: 3 + selector: + matchLabels: + app: grafana-agent + template: + metadata: + labels: + app: grafana-agent + spec: + containers: + - name: grafana-agent + image: grafana/agent:latest + args: + - "--shards=3" + - "--replicas=2" + ``` + +* To shard the data into 10 shards and replicate each shard to three other Grafana Agent instances, you would use the following deployment manifest: + + ``` + apiVersion: apps/v1 + kind: Deployment + metadata: + name: grafana-agent + spec: + replicas: 10 + selector: + matchLabels: + app: grafana-agent + template: + metadata: + labels: + app: grafana-agent + spec: + containers: + - name: grafana-agent + image: grafana/agent:latest + args: + - "--shards=10" + - "--replicas=3" + ``` + From 2899c9deb39b8496e67c9ec5657fe79ffdeac081 Mon Sep 17 00:00:00 2001 From: Paulin Todev Date: Fri, 8 Dec 2023 14:20:22 +0000 Subject: [PATCH 06/17] Upgrade Docker images to use ubuntu:mantic (#5537) --- cmd/grafana-agent-operator/Dockerfile | 3 +-- cmd/grafana-agent/Dockerfile | 3 +-- cmd/grafana-agentctl/Dockerfile | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/cmd/grafana-agent-operator/Dockerfile b/cmd/grafana-agent-operator/Dockerfile index 17847af8456b..a86af13209bb 100644 --- a/cmd/grafana-agent-operator/Dockerfile +++ b/cmd/grafana-agent-operator/Dockerfile @@ -22,7 +22,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ RELEASE_BUILD=${RELEASE_BUILD} VERSION=${VERSION} \ make operator -FROM ubuntu:lunar +FROM ubuntu:mantic LABEL org.opencontainers.image.source="https://github.com/grafana/agent" @@ -30,7 +30,6 @@ LABEL org.opencontainers.image.source="https://github.com/grafana/agent" RUN < Date: Sat, 9 Dec 2023 01:26:17 +1100 Subject: [PATCH 07/17] Add HTTPClientConfig to discovery.ec2 (#5767) Co-authored-by: Paschalis Tsilias --- CHANGELOG.md | 2 ++ component/discovery/aws/ec2.go | 20 ++++++++----- component/discovery/aws/ec2_test.go | 29 +++++++++++++++++++ .../reference/components/discovery.ec2.md | 3 ++ 4 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 component/discovery/aws/ec2_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e532812bb40c..ab30936cc866 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ Main (unreleased) - Added links between compatible components in the documentation to make it easier to discover them. (@thampiotr) + +- Allow defining `HTTPClientConfig` for `discovery.ec2`. (@cmbrad) ### Bugfixes diff --git a/component/discovery/aws/ec2.go b/component/discovery/aws/ec2.go index 566527d2f67e..7672165e05e0 100644 --- a/component/discovery/aws/ec2.go +++ b/component/discovery/aws/ec2.go @@ -7,6 +7,7 @@ import ( "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" "github.com/grafana/agent/component" + "github.com/grafana/agent/component/common/config" "github.com/grafana/agent/component/discovery" "github.com/grafana/river/rivertypes" promcfg "github.com/prometheus/common/config" @@ -42,18 +43,21 @@ type EC2Arguments struct { RefreshInterval time.Duration `river:"refresh_interval,attr,optional"` Port int `river:"port,attr,optional"` Filters []*EC2Filter `river:"filter,block,optional"` + + HTTPClientConfig config.HTTPClientConfig `river:",squash"` } func (args EC2Arguments) Convert() *promaws.EC2SDConfig { cfg := &promaws.EC2SDConfig{ - Endpoint: args.Endpoint, - Region: args.Region, - AccessKey: args.AccessKey, - SecretKey: promcfg.Secret(args.SecretKey), - Profile: args.Profile, - RoleARN: args.RoleARN, - RefreshInterval: model.Duration(args.RefreshInterval), - Port: args.Port, + Endpoint: args.Endpoint, + Region: args.Region, + AccessKey: args.AccessKey, + SecretKey: promcfg.Secret(args.SecretKey), + Profile: args.Profile, + RoleARN: args.RoleARN, + RefreshInterval: model.Duration(args.RefreshInterval), + Port: args.Port, + HTTPClientConfig: *args.HTTPClientConfig.Convert(), } for _, f := range args.Filters { cfg.Filters = append(cfg.Filters, &promaws.EC2Filter{ diff --git a/component/discovery/aws/ec2_test.go b/component/discovery/aws/ec2_test.go new file mode 100644 index 000000000000..7696d750a4ff --- /dev/null +++ b/component/discovery/aws/ec2_test.go @@ -0,0 +1,29 @@ +package aws + +import ( + "net/url" + "testing" + + "github.com/grafana/agent/component/common/config" + "github.com/stretchr/testify/require" + "gotest.tools/assert" +) + +func TestConvert(t *testing.T) { + // parse example proxy + u, err := url.Parse("http://example:8080") + require.NoError(t, err) + httpClientConfig := config.DefaultHTTPClientConfig + httpClientConfig.ProxyURL = config.URL{URL: u} + + // example configuration + riverArgs := EC2Arguments{ + Region: "us-east-1", + HTTPClientConfig: httpClientConfig, + } + + // ensure values are set + promArgs := riverArgs.Convert() + assert.Equal(t, "us-east-1", promArgs.Region) + assert.Equal(t, "http://example:8080", promArgs.HTTPClientConfig.ProxyURL.String()) +} diff --git a/docs/sources/flow/reference/components/discovery.ec2.md b/docs/sources/flow/reference/components/discovery.ec2.md index 98f8b033b873..e2964d2df0a3 100644 --- a/docs/sources/flow/reference/components/discovery.ec2.md +++ b/docs/sources/flow/reference/components/discovery.ec2.md @@ -36,6 +36,9 @@ Name | Type | Description | Default | Required `role_arn` | `string` | AWS Role Amazon Resource Name (ARN), an alternative to using AWS API keys. | | no `refresh_interval` | `string` | Refresh interval to re-read the instance list. | 60s | no `port` | `int` | The port to scrape metrics from. If using the public IP address, this must instead be specified in the relabeling rule. | 80 | no +`proxy_url` | `string` | HTTP proxy to proxy requests through. | | no +`follow_redirects` | `bool` | Whether redirects returned by the server should be followed. | `true` | no +`enable_http2` | `bool` | Whether HTTP2 is supported for requests. | `true` | no ## Blocks From e961c1705382615375a3b00f5e1d215f79f0cff5 Mon Sep 17 00:00:00 2001 From: Ferdinand von Hagen Date: Fri, 8 Dec 2023 15:36:40 +0100 Subject: [PATCH 08/17] loki.source.docker: fix partial syncing of targets (#5798) * fix: partial docker syncTargets syncTargets for docker is called within a loop which leads to an incomplete targets slice to be sent to the manager. Moving the syncTargets after the loop ensures that always all targets are correctly registered. --------- Co-authored-by: Paschalis Tsilias --- CHANGELOG.md | 2 ++ component/loki/source/docker/docker.go | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab30936cc866..a03838a3252c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,8 @@ Main (unreleased) - Update `pyroscope.ebpf` to produce more optimal pprof profiles for python processes https://github.com/grafana/pyroscope/pull/2788 (@korniltsev) +- Fixes `loki.source.docker` a behavior that synced an incomplete list of targets to the tailer manager. (@FerdinandvHagen) + v0.38.1 (2023-11-30) -------------------- diff --git a/component/loki/source/docker/docker.go b/component/loki/source/docker/docker.go index 53a25b85da47..400b1d30b5e6 100644 --- a/component/loki/source/docker/docker.go +++ b/component/loki/source/docker/docker.go @@ -242,11 +242,11 @@ func (c *Component) Update(args component.Arguments) error { return err } targets = append(targets, tgt) - - // This will never fail because it only fails if the context gets canceled. - _ = c.manager.syncTargets(context.Background(), targets) } + // This will never fail because it only fails if the context gets canceled. + _ = c.manager.syncTargets(context.Background(), targets) + c.args = newArgs return nil } From 7424b9275f255c8c74853cf6350024dbd1d043b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90=E1=BB=97=20Tr=E1=BB=8Dng=20H=E1=BA=A3i?= <41283691+hainenber@users.noreply.github.com> Date: Fri, 8 Dec 2023 22:11:19 +0700 Subject: [PATCH 09/17] mimir.rules.kubernetes: retry startup process on failure (#5906) Signed-off-by: hainenber --- CHANGELOG.md | 2 ++ component/mimir/rules/kubernetes/rules.go | 24 +++++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a03838a3252c..a3e03364979d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ Main (unreleased) - `pyroscope.ebpf` support python on arm64 platforms. (@korniltsev) +- `mimir.rules.kubernetes` may now retry its startup on failure. (@hainenber) + - Added links between compatible components in the documentation to make it easier to discover them. (@thampiotr) diff --git a/component/mimir/rules/kubernetes/rules.go b/component/mimir/rules/kubernetes/rules.go index 16deaaa74b42..016a888d9104 100644 --- a/component/mimir/rules/kubernetes/rules.go +++ b/component/mimir/rules/kubernetes/rules.go @@ -10,6 +10,7 @@ import ( "github.com/grafana/agent/component" "github.com/grafana/agent/pkg/flow/logging/level" mimirClient "github.com/grafana/agent/pkg/mimir/client" + "github.com/grafana/dskit/backoff" "github.com/grafana/dskit/instrument" promListers "github.com/prometheus-operator/prometheus-operator/pkg/client/listers/monitoring/v1" "github.com/prometheus/client_golang/prometheus" @@ -153,10 +154,22 @@ func New(o component.Options, args Arguments) (*Component, error) { } func (c *Component) Run(ctx context.Context) error { - err := c.startup(ctx) - if err != nil { - level.Error(c.log).Log("msg", "starting up component failed", "err", err) - c.reportUnhealthy(err) + startupBackoff := backoff.New( + ctx, + backoff.Config{ + MinBackoff: 1 * time.Second, + MaxBackoff: 10 * time.Second, + MaxRetries: 0, // infinite retries + }, + ) + for { + if err := c.startup(ctx); err != nil { + level.Error(c.log).Log("msg", "starting up component failed", "err", err) + c.reportUnhealthy(err) + } else { + break + } + startupBackoff.Wait() } for { @@ -205,8 +218,7 @@ func (c *Component) startup(ctx context.Context) error { if err := c.startRuleInformer(); err != nil { return err } - err := c.syncMimir(ctx) - if err != nil { + if err := c.syncMimir(ctx); err != nil { return err } go c.eventLoop(ctx) From d7fbffadc84951222917f4f3f4e527db846e1002 Mon Sep 17 00:00:00 2001 From: Paulin Todev Date: Fri, 8 Dec 2023 15:27:57 +0000 Subject: [PATCH 10/17] Generate spanmetrics after load balancing. (#5889) Co-authored-by: Paschalis Tsilias --- CHANGELOG.md | 4 ++++ pkg/traces/config.go | 7 +++++-- pkg/traces/config_test.go | 9 +++++---- pkg/traces/instance.go | 9 +++++++-- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3e03364979d..b268bdc443a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,10 @@ Main (unreleased) - Update `pyroscope.ebpf` to produce more optimal pprof profiles for python processes https://github.com/grafana/pyroscope/pull/2788 (@korniltsev) +- In Static mode's `traces` subsystem, `spanmetrics` used to be generated prior to load balancing. + This could lead to inaccurate metrics. This issue only affects Agents using both `spanmetrics` and + `load_balancing`, when running in a load balanced cluster with more than one Agent instance. (@ptodev) + - Fixes `loki.source.docker` a behavior that synced an incomplete list of targets to the tailer manager. (@FerdinandvHagen) v0.38.1 (2023-11-30) diff --git a/pkg/traces/config.go b/pkg/traces/config.go index 69d7792a599f..e76b0d89e682 100644 --- a/pkg/traces/config.go +++ b/pkg/traces/config.go @@ -581,7 +581,7 @@ func (c *InstanceConfig) extensions() (map[string]interface{}, error) { func resolver(config map[string]interface{}) (map[string]interface{}, error) { if len(config) == 0 { - return nil, fmt.Errorf("must configure one resolver (dns or static)") + return nil, fmt.Errorf("must configure one resolver (dns, static, or kubernetes)") } resolverCfg := make(map[string]interface{}) for typ, cfg := range config { @@ -950,7 +950,9 @@ func tracingFactories() (otelcol.Factories, error) { // sets: before and after load balancing func orderProcessors(processors []string, splitPipelines bool) [][]string { order := map[string]int{ - "attributes": 0, + "attributes": 0, + // Spanmetrics should be before tail_sampling so that + // metrics are generated using as many spans as possible. "spanmetrics": 1, "service_graphs": 2, "tail_sampling": 3, @@ -978,6 +980,7 @@ func orderProcessors(processors []string, splitPipelines bool) [][]string { if processor == "batch" || processor == "tail_sampling" || processor == "automatic_logging" || + processor == "spanmetrics" || processor == "service_graphs" { foundAt = i diff --git a/pkg/traces/config_test.go b/pkg/traces/config_test.go index 9ee9e0b32abf..0781a94bd56b 100644 --- a/pkg/traces/config_test.go +++ b/pkg/traces/config_test.go @@ -1664,9 +1664,9 @@ service_graphs: expectedProcessors: map[component.ID][]component.ID{ component.NewIDWithName("traces", "0"): { component.NewID("attributes"), - component.NewID("spanmetrics"), }, component.NewIDWithName("traces", "1"): { + component.NewID("spanmetrics"), component.NewID("service_graphs"), component.NewID("tail_sampling"), component.NewID("automatic_logging"), @@ -1715,9 +1715,9 @@ load_balancing: expectedProcessors: map[component.ID][]component.ID{ component.NewIDWithName("traces", "0"): { component.NewID("attributes"), - component.NewID("spanmetrics"), }, component.NewIDWithName("traces", "1"): { + component.NewID("spanmetrics"), component.NewID("automatic_logging"), component.NewID("batch"), }, @@ -1819,9 +1819,9 @@ func TestOrderProcessors(t *testing.T) { expected: [][]string{ { "attributes", - "spanmetrics", }, { + "spanmetrics", "tail_sampling", "automatic_logging", "batch", @@ -1853,9 +1853,10 @@ func TestOrderProcessors(t *testing.T) { expected: [][]string{ { "attributes", + }, + { "spanmetrics", }, - {}, }, }, } diff --git a/pkg/traces/instance.go b/pkg/traces/instance.go index bf35e31ccb14..85c69a6450e4 100644 --- a/pkg/traces/instance.go +++ b/pkg/traces/instance.go @@ -112,8 +112,13 @@ func (i *Instance) buildAndStartPipeline(ctx context.Context, cfg InstanceConfig } if cfg.LoadBalancing == nil && (cfg.TailSampling != nil || cfg.ServiceGraphs != nil) { - i.logger.Warn("Configuring tail_sampling and/or service_graphs without load_balance." + - "Load balancing is required for those features to properly work in multi agent deployments") + i.logger.Warn("Configuring tail_sampling and/or service_graphs without load_balancing." + + "Load balancing via trace ID is required for those features to work properly in multi agent deployments") + } + + if cfg.LoadBalancing == nil && cfg.SpanMetrics != nil { + i.logger.Warn("Configuring spanmetrics without load_balancing." + + "Load balancing via service name is required for spanmetrics to work properly in multi agent deployments") } if cfg.AutomaticLogging != nil && cfg.AutomaticLogging.Backend != automaticloggingprocessor.BackendStdout { From 505217691b5761541beb756346dd75f72b475b09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Dec 2023 17:33:31 +0200 Subject: [PATCH 11/17] build(deps): bump helm/chart-testing-action from 2.6.0 to 2.6.1 (#5721) Bumps [helm/chart-testing-action](https://github.com/helm/chart-testing-action) from 2.6.0 to 2.6.1. - [Release notes](https://github.com/helm/chart-testing-action/releases) - [Commits](https://github.com/helm/chart-testing-action/compare/v2.6.0...v2.6.1) --- updated-dependencies: - dependency-name: helm/chart-testing-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/helm-release.yml | 2 +- .github/workflows/helm-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml index 76b1727e2514..b5a310bd4163 100644 --- a/.github/workflows/helm-release.yml +++ b/.github/workflows/helm-release.yml @@ -22,7 +22,7 @@ jobs: path: source - name: Install chart-testing - uses: helm/chart-testing-action@v2.6.0 + uses: helm/chart-testing-action@v2.6.1 - name: List changed charts id: list-changed diff --git a/.github/workflows/helm-test.yml b/.github/workflows/helm-test.yml index 321b3ecd93e7..20a3745c8fe1 100644 --- a/.github/workflows/helm-test.yml +++ b/.github/workflows/helm-test.yml @@ -57,7 +57,7 @@ jobs: check-latest: true - name: Install chart-testing - uses: helm/chart-testing-action@v2.6.0 + uses: helm/chart-testing-action@v2.6.1 - name: Determine changed charts id: list-changed From 5f62b01ecd65878144ae9309553034cd3830549a Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Fri, 8 Dec 2023 17:39:35 +0200 Subject: [PATCH 12/17] remote/http: add an optional request body (#5918) Signed-off-by: Paschalis Tsilias --- CHANGELOG.md | 2 ++ component/remote/http/http.go | 8 +++++++- component/remote/http/http_test.go | 7 ++++++- docs/sources/flow/reference/components/remote.http.md | 1 + 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b268bdc443a3..faff9f9f4814 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,8 @@ Main (unreleased) - Allow defining `HTTPClientConfig` for `discovery.ec2`. (@cmbrad) +- The `remote.http` component can optionally define a request body. (@tpaschalis) + ### Bugfixes - Update `pyroscope.ebpf` to fix a logical bug causing to profile to many kthreads instead of regular processes https://github.com/grafana/pyroscope/pull/2778 (@korniltsev) diff --git a/component/remote/http/http.go b/component/remote/http/http.go index c45833ba81df..254f8f90f3c3 100644 --- a/component/remote/http/http.go +++ b/component/remote/http/http.go @@ -41,6 +41,7 @@ type Arguments struct { Method string `river:"method,attr,optional"` Headers map[string]string `river:"headers,attr,optional"` + Body string `river:"body,attr,optional"` Client common_config.HTTPClientConfig `river:"client,block,optional"` } @@ -193,7 +194,12 @@ func (c *Component) pollError() error { ctx, cancel := context.WithTimeout(context.Background(), c.args.PollTimeout) defer cancel() - req, err := http.NewRequest(c.args.Method, c.args.URL, nil) + var body io.Reader + if c.args.Body != "" { + body = strings.NewReader(c.args.Body) + } + + req, err := http.NewRequest(c.args.Method, c.args.URL, body) if err != nil { level.Error(c.log).Log("msg", "failed to build request", "err", err) return fmt.Errorf("building request: %w", err) diff --git a/component/remote/http/http_test.go b/component/remote/http/http_test.go index f8ef2214394f..e59d7c6e74c9 100644 --- a/component/remote/http/http_test.go +++ b/component/remote/http/http_test.go @@ -3,6 +3,7 @@ package http_test import ( "context" "fmt" + "io" "net/http" "net/http/httptest" "sync" @@ -27,6 +28,9 @@ func Test(t *testing.T) { defer srv.Close() handler.SetHandler(func(w http.ResponseWriter, r *http.Request) { + b, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.Equal(t, string(b), "hello there!") fmt.Fprintln(w, "Hello, world!") }) @@ -40,10 +44,11 @@ func Test(t *testing.T) { "x-custom" = "value", "User-Agent" = "custom_useragent", } + body = "%s" poll_frequency = "50ms" poll_timeout = "25ms" - `, srv.URL, http.MethodPut) + `, srv.URL, http.MethodPut, "hello there!") var args http_component.Arguments require.NoError(t, river.Unmarshal([]byte(cfg), &args)) diff --git a/docs/sources/flow/reference/components/remote.http.md b/docs/sources/flow/reference/components/remote.http.md index 1060ff78b54d..7428cb62d18d 100644 --- a/docs/sources/flow/reference/components/remote.http.md +++ b/docs/sources/flow/reference/components/remote.http.md @@ -37,6 +37,7 @@ Name | Type | Description | Default | Required `url` | `string` | URL to poll. | | yes `method` | `string` | Define HTTP method for the request | `"GET"` | no `headers` | `map(string)` | Custom headers for the request. | `{}` | no +`body` | `string` | The request body. | `""` | no `poll_frequency` | `duration` | Frequency to poll the URL. | `"1m"` | no `poll_timeout` | `duration` | Timeout when polling the URL. | `"10s"` | no `is_secret` | `bool` | Whether the response body should be treated as a secret. | false | no From c670ca02285caadf4c1922ff3935d406784096cd Mon Sep 17 00:00:00 2001 From: Aksel Skaar Leirvaag <52233080+akselleirv@users.noreply.github.com> Date: Fri, 8 Dec 2023 16:40:47 +0100 Subject: [PATCH 13/17] chore(deps): bump github.com/IBM/sarama from v1.41.2 to v1.42.1 (#5912) Co-authored-by: Paschalis Tsilias --- CHANGELOG.md | 4 ++++ go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index faff9f9f4814..e08d1682dbd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,10 @@ Main (unreleased) - Fixes `loki.source.docker` a behavior that synced an incomplete list of targets to the tailer manager. (@FerdinandvHagen) +### Other changes + +- Bump github.com/IBM/sarama from v1.41.2 to v1.42.1 + v0.38.1 (2023-11-30) -------------------- diff --git a/go.mod b/go.mod index 3c16f9b4ba83..0bbd11eae324 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/Azure/go-autorest/autorest v0.11.29 - github.com/IBM/sarama v1.41.2 + github.com/IBM/sarama v1.42.1 github.com/Lusitaniae/apache_exporter v0.11.1-0.20220518131644-f9522724dab4 github.com/Masterminds/sprig/v3 v3.2.3 github.com/PuerkitoBio/rehttp v1.1.0 diff --git a/go.sum b/go.sum index ac3ae21e9703..8040fffff98f 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,8 @@ github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 h1:KeNholpO2xKjgaa github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962/go.mod h1:kC29dT1vFpj7py2OvG1khBdQpo3kInWP+6QipLbdngo= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= -github.com/IBM/sarama v1.41.2 h1:ZDBZfGPHAD4uuAtSv4U22fRZBgst0eEwGFzLj0fb85c= -github.com/IBM/sarama v1.41.2/go.mod h1:xdpu7sd6OE1uxNdjYTSKUfY8FaKkJES9/+EyjSgiGQk= +github.com/IBM/sarama v1.42.1 h1:wugyWa15TDEHh2kvq2gAy1IHLjEjuYOYgXz/ruC/OSQ= +github.com/IBM/sarama v1.42.1/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= github.com/Jeffail/gabs v1.1.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= From c147c5aef0e45afd7f7484c2ba6c06d0cc0c4f67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Dec 2023 17:47:24 +0200 Subject: [PATCH 14/17] build(deps): bump github.com/go-jose/go-jose/v3 from 3.0.0 to 3.0.1 (#5834) Bumps [github.com/go-jose/go-jose/v3](https://github.com/go-jose/go-jose) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/go-jose/go-jose/releases) - [Changelog](https://github.com/go-jose/go-jose/blob/v3/CHANGELOG.md) - [Commits](https://github.com/go-jose/go-jose/compare/v3.0.0...v3.0.1) --- updated-dependencies: - dependency-name: github.com/go-jose/go-jose/v3 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 0bbd11eae324..6c82f970e4b7 100644 --- a/go.mod +++ b/go.mod @@ -621,7 +621,7 @@ require ( github.com/Shopify/sarama v1.38.1 // indirect github.com/Workiva/go-datastructures v1.1.0 // indirect github.com/drone/envsubst v1.0.3 // indirect - github.com/go-jose/go-jose/v3 v3.0.0 // indirect + github.com/go-jose/go-jose/v3 v3.0.1 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/hetznercloud/hcloud-go/v2 v2.0.0 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect diff --git a/go.sum b/go.sum index 8040fffff98f..648c487e8ed2 100644 --- a/go.sum +++ b/go.sum @@ -715,8 +715,9 @@ github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6 github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= +github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA= +github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= From 9a10fffb7e48f44000e9ebe33212722c58273dcd Mon Sep 17 00:00:00 2001 From: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> Date: Fri, 8 Dec 2023 12:31:18 -0800 Subject: [PATCH 15/17] Fix missing codefence (#5938) --- .../flow/reference/components/otelcol.receiver.vcenter.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/sources/flow/reference/components/otelcol.receiver.vcenter.md b/docs/sources/flow/reference/components/otelcol.receiver.vcenter.md index 20618018e850..2bc175e99d3e 100644 --- a/docs/sources/flow/reference/components/otelcol.receiver.vcenter.md +++ b/docs/sources/flow/reference/components/otelcol.receiver.vcenter.md @@ -219,6 +219,8 @@ otelcol.exporter.otlp "default" { endpoint = env("OTLP_ENDPOINT") } } +``` + ## Compatible components From 28ba4f3a122d6488a03d61b63d99dd287f22da82 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Mon, 11 Dec 2023 10:48:35 +0100 Subject: [PATCH 16/17] rename EvaluateDependencies to EvaluateDependants (#5894) --- pkg/flow/flow.go | 2 +- pkg/flow/internal/controller/loader.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/flow/flow.go b/pkg/flow/flow.go index ec16333568ee..76d62bdb9cb2 100644 --- a/pkg/flow/flow.go +++ b/pkg/flow/flow.go @@ -240,7 +240,7 @@ func (f *Flow) Run(ctx context.Context) { // throughput - it prevents the situation where two components have the same dependency, and the first time // it's picked up by the worker pool and the second time it's enqueued again, resulting in more evaluations. all := f.updateQueue.DequeueAll() - f.loader.EvaluateDependencies(ctx, all) + f.loader.EvaluateDependants(ctx, all) case <-f.loadFinished: level.Info(f.log).Log("msg", "scheduling loaded components and services") diff --git a/pkg/flow/internal/controller/loader.go b/pkg/flow/internal/controller/loader.go index e968e8102fe0..10a6f37965ab 100644 --- a/pkg/flow/internal/controller/loader.go +++ b/pkg/flow/internal/controller/loader.go @@ -32,7 +32,7 @@ type Loader struct { componentReg ComponentRegistry workerPool worker.Pool // backoffConfig is used to backoff when an updated component's dependencies cannot be submitted to worker - // pool for evaluation in EvaluateDependencies, because the queue is full. This is an unlikely scenario, but when + // pool for evaluation in EvaluateDependants, because the queue is full. This is an unlikely scenario, but when // it happens we should avoid retrying too often to give other goroutines a chance to progress. Having a backoff // also prevents log spamming with errors. backoffConfig backoff.Config @@ -543,13 +543,13 @@ func (l *Loader) OriginalGraph() *dag.Graph { return l.originalGraph.Clone() } -// EvaluateDependencies sends components which depend directly on components in updatedNodes for evaluation to the +// EvaluateDependants sends components which depend directly on components in updatedNodes for evaluation to the // workerPool. It should be called whenever components update their exports. -// It is beneficial to call EvaluateDependencies with a batch of components, as it will enqueue the entire batch before +// It is beneficial to call EvaluateDependants with a batch of components, as it will enqueue the entire batch before // the worker pool starts to evaluate them, resulting in smaller number of total evaluations when -// node updates are frequent. If the worker pool's queue is full, EvaluateDependencies will retry with a backoff until +// node updates are frequent. If the worker pool's queue is full, EvaluateDependants will retry with a backoff until // it succeeds or until the ctx is cancelled. -func (l *Loader) EvaluateDependencies(ctx context.Context, updatedNodes []*ComponentNode) { +func (l *Loader) EvaluateDependants(ctx context.Context, updatedNodes []*ComponentNode) { if len(updatedNodes) == 0 { return } @@ -578,7 +578,7 @@ func (l *Loader) EvaluateDependencies(ctx context.Context, updatedNodes []*Compo // Submit all dependencies for asynchronous evaluation. // During evaluation, if a node's exports change, Flow will add it to updated nodes queue (controller.Queue) and - // the Flow controller will call EvaluateDependencies on it again. This results in a concurrent breadth-first + // the Flow controller will call EvaluateDependants on it again. This results in a concurrent breadth-first // traversal of the nodes that need to be evaluated. for n, parent := range dependenciesToParentsMap { dependantCtx, span := tracer.Start(spanCtx, "SubmitForEvaluation", trace.WithSpanKind(trace.SpanKindInternal)) From a2badce44a1fbfb6b95aa6f20f0f762e92dbc4ca Mon Sep 17 00:00:00 2001 From: Robert Lankford Date: Mon, 11 Dec 2023 03:02:07 -0800 Subject: [PATCH 17/17] otelcol.connector.servicegraph: update default store ttl value (#5941) * otelcol.connector.servicegraph: update default store ttl value Signed-off-by: Robbie Lankford * update test --------- Signed-off-by: Robbie Lankford --- CHANGELOG.md | 2 ++ component/otelcol/connector/servicegraph/servicegraph.go | 2 +- component/otelcol/connector/servicegraph/servicegraph_test.go | 2 +- .../reference/components/otelcol.connector.servicegraph.md | 4 ++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e08d1682dbd1..40432c8ed847 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,8 @@ Main (unreleased) - Fixes `loki.source.docker` a behavior that synced an incomplete list of targets to the tailer manager. (@FerdinandvHagen) +- Fixes `otelcol.connector.servicegraph` store ttl default value from 2ms to 2s. (@rlankfo) + ### Other changes - Bump github.com/IBM/sarama from v1.41.2 to v1.42.1 diff --git a/component/otelcol/connector/servicegraph/servicegraph.go b/component/otelcol/connector/servicegraph/servicegraph.go index e5370d89f620..c1713cca5ad5 100644 --- a/component/otelcol/connector/servicegraph/servicegraph.go +++ b/component/otelcol/connector/servicegraph/servicegraph.go @@ -91,7 +91,7 @@ var DefaultArguments = Arguments{ Dimensions: []string{}, Store: StoreConfig{ MaxItems: 1000, - TTL: 2 * time.Millisecond, + TTL: 2 * time.Second, }, CacheLoop: 1 * time.Minute, StoreExpirationLoop: 2 * time.Second, diff --git a/component/otelcol/connector/servicegraph/servicegraph_test.go b/component/otelcol/connector/servicegraph/servicegraph_test.go index 5f7204b2bb6b..952ac8fc06d5 100644 --- a/component/otelcol/connector/servicegraph/servicegraph_test.go +++ b/component/otelcol/connector/servicegraph/servicegraph_test.go @@ -44,7 +44,7 @@ func TestArguments_UnmarshalRiver(t *testing.T) { Dimensions: []string{}, Store: servicegraphprocessor.StoreConfig{ MaxItems: 1000, - TTL: 2 * time.Millisecond, + TTL: 2 * time.Second, }, CacheLoop: 1 * time.Minute, StoreExpirationLoop: 2 * time.Second, diff --git a/docs/sources/flow/reference/components/otelcol.connector.servicegraph.md b/docs/sources/flow/reference/components/otelcol.connector.servicegraph.md index e7aad20d8e29..ab3e55b5521f 100644 --- a/docs/sources/flow/reference/components/otelcol.connector.servicegraph.md +++ b/docs/sources/flow/reference/components/otelcol.connector.servicegraph.md @@ -135,7 +135,7 @@ The `store` block configures the in-memory store for spans. Name | Type | Description | Default | Required ---- | ---- | ----------- | ------- | -------- `max_items` | `number` | Maximum number of items to keep in the store. | `1000` | no -`ttl` | `duration` | The time to live for spans in the store. | `"2ms"` | no +`ttl` | `duration` | The time to live for spans in the store. | `"2s"` | no ### output block @@ -238,4 +238,4 @@ connection work correctly. Refer to the linked documentation for more details. {{% /admonition %}} - \ No newline at end of file +