From 9ad592ca6f8db0ae965a8ea7d1a5b4d7e0c4c60e Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Tue, 11 Jul 2023 09:21:40 -0400 Subject: [PATCH 1/2] breaking change: rename discovery.file to local.file_match This commit renames discovery.file to local.file_match so that: * The purpose of the component is more clear (i.e., to find files on the local filesystem which match a given pattern). * To remove a point of confusion where `discovery.file` was thought to be Prometheus' file SD. * To allow a component for Prometheus' file SD to be named discovery.file, which otherwise would've collided with the now-named local.file_match. A redirect from the old documentation page is explicitly not configured because we anticipate creating a new page called discovery.file, which will have a warning about the rename. Related to #3607. --- CHANGELOG.md | 8 ++- component/all/all.go | 2 +- .../file => local/file_match}/file.go | 10 +-- .../file => local/file_match}/file_test.go | 2 +- .../file => local/file_match}/watch.go | 2 +- ...{discovery.file.md => local.file_match.md} | 30 ++++---- .../flow/reference/components/loki.echo.md | 4 +- .../reference/components/loki.source.file.md | 6 +- docs/sources/flow/upgrade-guide.md | 68 +++++++++++++++++++ 9 files changed, 102 insertions(+), 30 deletions(-) rename component/{discovery/file => local/file_match}/file.go (92%) rename component/{discovery/file => local/file_match}/file_test.go (99%) rename component/{discovery/file => local/file_match}/watch.go (98%) rename docs/sources/flow/reference/components/{discovery.file.md => local.file_match.md} (81%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39df222d9d8a..2cc9ce31663f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,10 @@ Main (unreleased) - The field `version` and `auth` struct block from `walk_params` in `prometheus.exporter.snmp` and SNMP integration have been removed. The auth block now can be configured at top level, together with `modules` (@marctc) +- Rename `discovery.file` to `local.file_match` to make it more clear that it + discovers file on the local filesystem, and so it doesn't get confused with + Prometheus' file discovery. (@rfratto) + ### Features - The Pyroscope scrape component computes and sends delta profiles automatically when required to reduce bandwidth usage. (@cyriltovena) @@ -122,8 +126,8 @@ Main (unreleased) - Fix bug where `stage.timestamp` in `loki.process` wasn't able to correctly parse timezones. This issue only impacts the dedicated `grafana-agent-flow` binary. (@rfratto) - -- Fix bug where JSON requests to `loki.source.api` would not be handled correctly. This adds `/loki/api/v1/raw` and `/loki/api/v1/push` endpoints to `loki.source.api` and maps the `/api/v1/push` and `/api/v1/raw` to + +- Fix bug where JSON requests to `loki.source.api` would not be handled correctly. This adds `/loki/api/v1/raw` and `/loki/api/v1/push` endpoints to `loki.source.api` and maps the `/api/v1/push` and `/api/v1/raw` to the `/loki` prefixed endpoints. (@mattdurham) ### Other changes diff --git a/component/all/all.go b/component/all/all.go index fca605a14a74..3ad51b47363d 100644 --- a/component/all/all.go +++ b/component/all/all.go @@ -8,12 +8,12 @@ import ( _ "github.com/grafana/agent/component/discovery/digitalocean" // Import discovery.digitalocean _ "github.com/grafana/agent/component/discovery/dns" // Import discovery.dns _ "github.com/grafana/agent/component/discovery/docker" // Import discovery.docker - _ "github.com/grafana/agent/component/discovery/file" // Import discovery.file _ "github.com/grafana/agent/component/discovery/gce" // Import discovery.gce _ "github.com/grafana/agent/component/discovery/kubelet" // Import discovery.kubelet _ "github.com/grafana/agent/component/discovery/kubernetes" // Import discovery.kubernetes _ "github.com/grafana/agent/component/discovery/relabel" // Import discovery.relabel _ "github.com/grafana/agent/component/local/file" // Import local.file + _ "github.com/grafana/agent/component/local/file_match" // Import local.file_match _ "github.com/grafana/agent/component/loki/echo" // Import loki.echo _ "github.com/grafana/agent/component/loki/process" // Import loki.process _ "github.com/grafana/agent/component/loki/relabel" // Import loki.relabel diff --git a/component/discovery/file/file.go b/component/local/file_match/file.go similarity index 92% rename from component/discovery/file/file.go rename to component/local/file_match/file.go index b43d1309c56c..a3bdeeb15b6f 100644 --- a/component/discovery/file/file.go +++ b/component/local/file_match/file.go @@ -1,4 +1,4 @@ -package file +package file_match import ( "context" @@ -13,7 +13,7 @@ import ( func init() { component.Register(component.Registration{ - Name: "discovery.file", + Name: "local.file_match", Args: Arguments{}, Exports: discovery.Exports{}, Build: func(opts component.Options, args component.Arguments) (component.Component, error) { @@ -22,7 +22,7 @@ func init() { }) } -// Arguments holds values which are used to configure the discovery.file +// Arguments holds values which are used to configure the local.file_match // component. type Arguments struct { PathTargets []discovery.Target `river:"path_targets,attr"` @@ -31,7 +31,7 @@ type Arguments struct { var _ component.Component = (*Component)(nil) -// Component implements the discovery.file component. +// Component implements the local.file_match component. type Component struct { opts component.Options @@ -41,7 +41,7 @@ type Component struct { watchDog *time.Ticker } -// New creates a new discovery.file component. +// New creates a new local.file_match component. func New(o component.Options, args Arguments) (*Component, error) { c := &Component{ opts: o, diff --git a/component/discovery/file/file_test.go b/component/local/file_match/file_test.go similarity index 99% rename from component/discovery/file/file_test.go rename to component/local/file_match/file_test.go index 361d4e67156e..f8bc95fc9b5c 100644 --- a/component/discovery/file/file_test.go +++ b/component/local/file_match/file_test.go @@ -1,7 +1,7 @@ //go:build !windows // This should run on windows but windows does not like the tight timing of file creation and deletion. -package file +package file_match import ( "os" diff --git a/component/discovery/file/watch.go b/component/local/file_match/watch.go similarity index 98% rename from component/discovery/file/watch.go rename to component/local/file_match/watch.go index f7f0b633f32f..5b8b650af982 100644 --- a/component/discovery/file/watch.go +++ b/component/local/file_match/watch.go @@ -1,4 +1,4 @@ -package file +package file_match import ( "os" diff --git a/docs/sources/flow/reference/components/discovery.file.md b/docs/sources/flow/reference/components/local.file_match.md similarity index 81% rename from docs/sources/flow/reference/components/discovery.file.md rename to docs/sources/flow/reference/components/local.file_match.md index a6c78046aaee..4a56f313a4cd 100644 --- a/docs/sources/flow/reference/components/discovery.file.md +++ b/docs/sources/flow/reference/components/local.file_match.md @@ -1,17 +1,17 @@ --- -title: discovery.file +title: local.file_match --- -# discovery.file +# local.file_match -`discovery.file` discovers files on the local filesystem using glob patterns and the [doublestar][] library. +`local.file_match` discovers files on the local filesystem using glob patterns and the [doublestar][] library. [doublestar]: https://github.com/bmatcuk/doublestar ## Usage ```river -discovery.file "LABEL" { +local.file_match "LABEL" { path_targets = [{"__path__" = "DOUBLESTAR_PATH"}] } ``` @@ -45,30 +45,30 @@ Each target includes the following labels: ## Component health -`discovery.file` is only reported as unhealthy when given an invalid +`local.file_match` is only reported as unhealthy when given an invalid configuration. In those cases, exported fields retain their last healthy values. ## Debug information -`discovery.file` does not expose any component-specific debug information. +`local.file_match` does not expose any component-specific debug information. ### Debug metrics -`discovery.file` does not expose any component-specific debug metrics. +`local.file_match` does not expose any component-specific debug metrics. ## Examples -This example discovers all files and folders under `/tmp/logs`. The absolute paths are +This example discovers all files and folders under `/tmp/logs`. The absolute paths are used by `loki.source.file.files` targets. ```river -discovery.file "tmp" { +local.file_match "tmp" { path_targets = [{"__path__" = "/tmp/logs/**/*.log"}] } loki.source.file "files" { - targets = discovery.file.tmp.targets + targets = local.file_match.tmp.targets forward_to = [ /* ... */ ] } ``` @@ -84,7 +84,7 @@ discovery.kubernetes "k8s" { discovery.relabel "k8s" { targets = discovery.kubernetes.k8s.targets - + rule { source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_name"] target_label = "job" @@ -94,17 +94,17 @@ discovery.relabel "k8s" { rule { source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] target_label = "__path__" - separator = "/" + separator = "/" replacement = "/var/log/pods/*$1/*.log" - } + } } -discovery.file "pods" { +local.file_match "pods" { path_targets = discovery.relabel.k8s.output } loki.source.file "pods" { - targets = discovery.file.pods.targets + targets = local.file_match.pods.targets forward_to = [loki.write.endpoint.receiver] } diff --git a/docs/sources/flow/reference/components/loki.echo.md b/docs/sources/flow/reference/components/loki.echo.md index e2f932835830..24745fa1c6ea 100644 --- a/docs/sources/flow/reference/components/loki.echo.md +++ b/docs/sources/flow/reference/components/loki.echo.md @@ -46,7 +46,7 @@ This example creates a pipeline that reads log files from `/var/log` and prints log lines to echo: ```river -discovery.file "varlog" { +local.file_match "varlog" { path_targets = [{ __path__ = "/var/log/*log", job = "varlog", @@ -54,7 +54,7 @@ discovery.file "varlog" { } loki.source.file "logs" { - targets = discovery.file.varlog.targets + targets = local.file_match.varlog.targets forward_to = [loki.echo.example.receiver] } diff --git a/docs/sources/flow/reference/components/loki.source.file.md b/docs/sources/flow/reference/components/loki.source.file.md index 7c477bdadb86..d09aa12f0db7 100644 --- a/docs/sources/flow/reference/components/loki.source.file.md +++ b/docs/sources/flow/reference/components/loki.source.file.md @@ -64,7 +64,7 @@ _labels_. The set of targets can either be _static_, or dynamically provided periodically by a service discovery component. The special label `__path__` _must always_ be present and must point to the absolute path of the file to read from. - + The `__path__` value is available as the `filename` label to each log entry the component reads. All other labels starting with a double underscore are @@ -74,7 +74,7 @@ passed to other `loki.*` components. The component uses its data path (a directory named after the domain's fully qualified name) to store its _positions file_. The positions file is used to store read offsets, so that in case of a component or Agent restart, -`loki.source.file` can pick up tailing from the same spot. +`loki.source.file` can pick up tailing from the same spot. If a file is removed from the `targets` list, its positions file entry is also removed. When it's added back on, `loki.source.file` starts reading it from the @@ -91,7 +91,7 @@ loki.source.file "tmpfiles" { {__path__ = "/tmp/foo.txt", "color" = "pink"}, {__path__ = "/tmp/bar.txt", "color" = "blue"}, {__path__ = "/tmp/baz.txt", "color" = "grey"}, - ] + ] forward_to = [loki.write.local.receiver] } diff --git a/docs/sources/flow/upgrade-guide.md b/docs/sources/flow/upgrade-guide.md index 4f9de5edcbc4..8847d4b90da4 100644 --- a/docs/sources/flow/upgrade-guide.md +++ b/docs/sources/flow/upgrade-guide.md @@ -69,6 +69,74 @@ prometheus.exporter.snmp "example" { See [Module and Auth Split Migration](https://github.com/prometheus/snmp_exporter/blob/main/auth-split-migration.md) for more details. +### Breaking change: `discovery.file` has been renamed to `local.file_match` + +The `discovery.file` component has been renamed to `local.file_match` to make +its purpose more clear: to find files on the local filesystem matching a +pattern. + +Renaming `discovery.file` to `local.file_match` also resolves a point of +confusion where `discovery.file` was thought to implement Prometheus' file +service discovery. + +Old configuration example: + +```river +discovery.kubernetes "k8s" { + role = "pod" +} + +discovery.relabel "k8s" { + targets = discovery.kubernetes.k8s.targets + + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_name"] + target_label = "job" + separator = "/" + } + + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + target_label = "__path__" + separator = "/" + replacement = "/var/log/pods/*$1/*.log" + } +} + +discovery.file "pods" { + path_targets = discovery.relabel.k8s.output +} +``` + +New configuration example: + +```river +discovery.kubernetes "k8s" { + role = "pod" +} + +discovery.relabel "k8s" { + targets = discovery.kubernetes.k8s.targets + + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_name"] + target_label = "job" + separator = "/" + } + + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + target_label = "__path__" + separator = "/" + replacement = "/var/log/pods/*$1/*.log" + } +} + +local.file_match "pods" { + path_targets = discovery.relabel.k8s.output +} +``` + ## v0.34 ### Breaking change: `phlare.scrape` and `phlare.write` have been renamed to `pyroscope.scrape` and `pyroscope.scrape` From 0e77c3d50c3f74f352b70eab6fde463a3c237651 Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Tue, 11 Jul 2023 09:25:15 -0400 Subject: [PATCH 2/2] fix changelog --- CHANGELOG.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cc9ce31663f..a93064bf6b5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,6 @@ internal API changes are not present. Main (unreleased) ----------------- -### Features -- `pyroscope.ebpf` collects system-wide performance profiles from the current host (@korniltsev) - > **BREAKING CHANGES**: This release has breaking changes. Please read entries > carefully and consult the [upgrade guide][] for specific instructions. @@ -45,14 +42,15 @@ Main (unreleased) - New Grafana Agent Flow components: - `discovery.kubelet` collect scrape targets from the Kubelet API. (@gcampbell12) - - `prometheus.exporter.kafka` collects metrics from Kafka Server. (@oliver-zhang) + - `module.http` runs a Grafana Agent Flow module loaded from a remote HTTP endpoint. (@spartan0x117) - `otelcol.processor.attributes` accepts telemetry data from other `otelcol` components and modifies attributes of a span, log, or metric. (@ptodev) - - `prometheus.exporter.squid` collects metrics from a squid server. (@armstrmi) - - `prometheus.exporter.elasticsearch` collects metrics from Elasticsearch. (@marctc) - `prometheus.exporter.cloudwatch` - scrape AWS CloudWatch metrics (@thepalbi) + - `prometheus.exporter.elasticsearch` collects metrics from Elasticsearch. (@marctc) + - `prometheus.exporter.kafka` collects metrics from Kafka Server. (@oliver-zhang) - `prometheus.exporter.mongodb` collects metrics from MongoDB. (@marctc) - - `module.http` runs a Grafana Agent Flow module loaded from a remote HTTP endpoint. (@spartan0x117) + - `prometheus.exporter.squid` collects metrics from a squid server. (@armstrmi) + - `pyroscope.ebpf` collects system-wide performance profiles from the current host (@korniltsev) - Added json_path function to river stdlib. (@jkroepke)