Merge branch 'main' into feature/http-client-config-discovery-ec2

.github/workflows/integration-tests.yml

@@ -16,8 +16,8 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v4
       with:
-        go-version: "1.21.0"
+        go-version: "1.21"
     - name: Set OTEL Exporter Endpoint
-      run: echo "OTEL_EXPORTER_ENDPOINT=http://172.17.0.1:8080" >> $GITHUB_ENV
+      run: echo "OTEL_EXPORTER_ENDPOINT=172.17.0.1:4318" >> $GITHUB_ENV
     - name: Run tests
-      run: make integration-test
+      run: make integration-test

.github/workflows/test.yml

@@ -12,15 +12,15 @@ jobs:
     name: Test
     strategy:
       matrix:
-        platform: [macos-latest]
+        platform: [macos-latest-xlarge]
     runs-on: ${{ matrix.platform }}
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
     - name: Set up Go 1.21
       uses: actions/setup-go@v4
       with:
-        go-version: "1.21.0"
+        go-version: "1.21"
         cache: true
     - name: Test
       run: make GO_TAGS="nodocker" test

CHANGELOG.md

@@ -10,6 +10,74 @@ internal API changes are not present.
 Main (unreleased)
 -----------------
 
+### Enhancements
+
+- Flow Windows service: Support environment variables. (@jkroepke)
+
+- Allow disabling collection of root Cgroup stats in
+  `prometheus.exporter.cadvisor` (flow mode) and the `cadvisor` integration
+  (static mode). (@hainenber)
+
+- Grafana Agent on Windows now automatically restarts on failure. (@hainenber)
+
+- Added metrics, alerts and dashboard visualisations to help diagnose issues
+  with unhealthy components and components that take too long to evaluate. (@thampiotr)
+
+- The `http` config block may now reference exports from any component.
+  Previously, only `remote.*` and `local.*` components could be referenced
+  without a circular dependency. (@rfratto)
+
+- Add a `resource_to_telemetry_conversion` argument to `otelcol.exporter.prometheus`
+  for converting resource attributes to Prometheus labels. (@hainenber)
+
+- `pyroscope.ebpf` support python on arm64 platforms. (@korniltsev)
+
+- Added links between compatible components in the documentation to make it
+  easier to discover them. (@thampiotr)
+
+- Allow defining `HTTPClientConfig` for `discovery.ec2`. (@cmbrad)
+
+### Bugfixes
+
+- Update `pyroscope.ebpf` to fix a logical bug causing to profile to many kthreads instead of regular processes https://github.com/grafana/pyroscope/pull/2778 (@korniltsev)
+
+- Update `pyroscope.ebpf` to produce more optimal pprof profiles for python processes https://github.com/grafana/pyroscope/pull/2788 (@korniltsev)
+
+v0.38.1 (2023-11-30)
+--------------------
+
+### Security fixes
+
+- Fix CVE-2023-47108 by updating `otelgrpc` from v0.45.0 to v0.46.0. (@hainenber)
+
+### Features
+
+- Agent Management: Introduce support for templated configuration. (@jcreixell)
+
+### Bugfixes
+
+- Permit `X-Faro-Session-ID` header in CORS requests for the `faro.receiver`
+  component (flow mode) and the `app_agent_receiver` integration (static mode).
+  (@cedricziel)
+
+- Fix issue with windows_exporter defaults not being set correctly. (@mattdurham)
+
+- Fix agent crash when process null OTel's fan out consumers. (@hainenber)
+
+- Fix issue in `prometheus.operator.*` where targets would be dropped if two crds share a common prefix in their names. (@Paul424, @captncraig)
+
+- Fix issue where `convert` command would generate incorrect Flow Mode config
+  when provided `promtail` configuration that uses `docker_sd_configs` (@thampiotr)
+
+- Fix converter issue with `loki.relabel` and `max_cache_size` being set to 0 instead of default (10_000). (@mattdurham)
+
+### Other changes
+
+- Add Agent Deploy Mode to usage report. (@captncraig)
+
+v0.38.0 (2023-11-21)
+--------------------
+
 ### Breaking changes
 
 - Remove `otelcol.exporter.jaeger` component (@hainenber)
@@ -26,11 +94,6 @@ Main (unreleased)
     - renamed 3 metrics starting with `mysql_perf_schema_transaction_` to start with `mysql_perf_schema_transactions_` to be consistent with column names.
     - exposing only server's own stats by matching `MEMBER_ID` with `@@server_uuid` resulting "member_id" label to be dropped.
 
-### Other changes
-
-- Bump `mysqld_exporter` version to v0.15.0. (@marctc)
-- Bump `github-exporter` version to 1.0.6. (@marctc)
-
 ### Features
 
 - Added a new `stage.decolorize` stage to `loki.process` component which
@@ -62,6 +125,9 @@ Main (unreleased)
 
   - `otelcol.processor.filter` - filters OTLP telemetry data using OpenTelemetry
     Transformation Language (OTTL). (@hainenber)
+  - `otelcol.receiver.vcenter` - receives metrics telemetry data from vCenter. (@marctc)
+
+- Agent Management: Introduce support for remotely managed external labels for logs. (@jcreixell)
 
 ### Enhancements
 
@@ -90,19 +156,19 @@ Main (unreleased)
 - Make component list sortable in web UI. (@hainenber)
 
 - Adds new metrics (`mssql_server_total_memory_bytes`, `mssql_server_target_memory_bytes`,
-  and `mssql_available_commit_memory_bytes`) for `mssql` integration.
+  and `mssql_available_commit_memory_bytes`) for `mssql` integration (@StefanKurek).
 
 - Grafana Agent Operator: `config-reloader` container no longer runs as root.
   (@rootmout)
 
 - Added support for replaying not sent data for `loki.write` when WAL is enabled. (@thepalbi)
 
+- Make the result of 'discovery.kubelet' support pods that without ports, such as k8s control plane static pods. (@masonmei)
+
 - Added support for unicode strings in `pyroscope.ebpf` python profiles. (@korniltsev)
 
 - Improved resilience of graph evaluation in presence of slow components. (@thampiotr)
 
-- Allow overriding `HTTPClientConfig` fields such as `proxy_url` in `discovery.ec2`. (@cmbrad)
-
 ### Bugfixes
 
 - Set exit code 1 on grafana-agentctl non-runnable command. (@fgouteroux)
@@ -141,6 +207,38 @@ Main (unreleased)
 
 - Updating configuration for `loki.write` no longer drops data. (@thepalbi)
 
+- Fixed a bug in WAL where exemplars were recorded before the first native histogram samples for new series,
+  resulting in remote write sending the exemplar first and Prometheus failing to ingest it due to missing
+  series. (@krajorama)
+
+- Fixed an issue in the static config converter where exporter instance values
+  were not being mapped when translating to flow. (@erikbaranowski)
+
+- Fix a bug which prevented Agent from running `otelcol.exporter.loadbalancing`
+  with a `routing_key` of `traceID`. (@ptodev)
+
+- Added Kubernetes service resolver to static node's loadbalancing exporter
+  and to Flow's `otelcol.exporter.loadbalancing`. (@ptodev)
+
+- Fix default configuration file `grafana-agent-flow.river` used in downstream
+  packages. (@bricewge)
+
+- Fix converter output for prometheus.exporter.windows to not unnecessarily add
+  empty blocks. (@erikbaranowski)
+
+### Other changes
+
+- Bump `mysqld_exporter` version to v0.15.0. (@marctc)
+
+- Bump `github-exporter` version to 1.0.6. (@marctc)
+
+- Use Go 1.21.4 for builds. (@rfratto)
+
+- Change User-Agent header for outbound requests to include agent-mode, goos, and deployment mode. Example `GrafanaAgent/v0.38.0 (flow; linux; docker)` (@captncraig)
+
+- `loki.source.windowsevent` and `loki.source.*` changed to use a more robust positions file to prevent corruption on reboots when writing
+  the positions file. (@mattdurham)
+
 v0.37.4 (2023-11-06)
 -----------------
 
@@ -154,6 +252,9 @@ v0.37.4 (2023-11-06)
 - Fix a bug where reloading the configuration of a `loki.write` component lead
   to a panic. (@tpaschalis)
 
+- Added Kubernetes service resolver to static node's loadbalancing exporter
+  and to Flow's `otelcol.exporter.loadbalancing`. (@ptodev)
+
 v0.37.3 (2023-10-26)
 -----------------
 

CODEOWNERS

@@ -1,25 +1,10 @@
-# The following groups are used to refer to a changing set of users:
+# The CODEOWNERS file is used to define ownership of individuals or teams
+# outside of the core set of Grafana Agent maintainers.
 #
-# * @grafana/grafana-agent-core-maintainers: maintainers of type/core issues.
-# * @grafana/grafana-agent-signals-maintainers: maintainers of type/signals issues.
-# * @grafana/grafana-agent-operator-maintainers: maintainers of type/operator issues.
-# * @grafana/grafana-agent-infrastructure-maintainers: maintainers of type/infrastructure issues.
-#
-# Other users may be listed explicitly if maintainership does not fall into one
-# of the above groups.
-
-# The default owners for everything in the repo. Unless a later match takes
-# precedence, these owners are requested for review whenever someone opens a
-# pull request.
-* @grafana/grafana-agent-core-maintainers
-
-# Some directories have shared ownership with the respective owners of the
-# specific code for the PR being opened, so there's no CODEOWNERS.
-/CHANGELOG.md
-/component/all
-
-# Binaries:
-/cmd/grafana-agent-operator/  @grafana/grafana-agent-operator-maintainers
+# If a directory is not listed here, it is assumed to be owned by the
+# @grafana/grafana-agent-maintainers; they are not explicitly listed as
+# CODEOWNERS as a GitHub project board is used instead for PR tracking, which
+# helps reduce notification noise of the members of that team.
 
 # `make docs` procedure and related workflows are owned by @jdbaldry.
 /.github/workflows/publish-technical-documentation-next.yml    @jdbaldry
@@ -30,25 +15,7 @@
 /docs/variables.mk                                             @jdbaldry
 
 # Documentation:
-/docs/sources/  @clayton-cornell
+/docs/sources/ @clayton-cornell
 
 # Components:
-/component/discovery/               @grafana/grafana-agent-infrastructure-maintainers
-/component/local/                   @grafana/grafana-agent-infrastructure-maintainers
-/component/loki/                    @grafana/grafana-agent-signals-maintainers
-/component/loki/source/podlogs/     @grafana/grafana-agent-infrastructure-maintainers
-/component/mimir/rules/kubernetes/  @grafana/grafana-agent-infrastructure-maintainers
-/component/otelcol/                 @grafana/grafana-agent-signals-maintainers
-/component/prometheus/              @grafana/grafana-agent-signals-maintainers
-/component/prometheus/exporter/     @grafana/grafana-agent-infrastructure-maintainers
-/component/prometheus/operator/     @grafana/grafana-agent-operator-maintainers
-/component/pyroscope/               @grafana/grafana-agent-profiling-maintainers
-/component/remote/                  @grafana/grafana-agent-infrastructure-maintainers
-
-# Static mode packages:
-/pkg/integrations/  @grafana/grafana-agent-infrastructure-maintainers
-/pkg/logs/          @grafana/grafana-agent-signals-maintainers
-/pkg/metrics/       @grafana/grafana-agent-signals-maintainers
-/pkg/mimir/client/  @grafana/grafana-agent-infrastructure-maintainers
-/pkg/operator/      @grafana/grafana-agent-operator-maintainers
-/pkg/traces/        @grafana/grafana-agent-signals-maintainers
+/component/pyroscope/ @grafana/grafana-agent-profiling-maintainers

Makefile

@@ -287,7 +287,7 @@ smoke-image:
 #
 
 .PHONY: generate generate-crds generate-drone generate-helm-docs generate-helm-tests generate-manifests generate-dashboards generate-protos generate-ui generate-versioned-files
-generate: generate-crds generate-drone generate-helm-docs generate-helm-tests generate-manifests generate-dashboards generate-protos generate-ui generate-versioned-files
+generate: generate-crds generate-drone generate-helm-docs generate-helm-tests generate-manifests generate-dashboards generate-protos generate-ui generate-versioned-files generate-docs
 
 generate-crds:
 ifeq ($(USE_CONTAINER),1)
@@ -350,6 +350,12 @@ else
 	sh ./tools/gen-versioned-files/gen-versioned-files.sh
 endif
 
+generate-docs:
+ifeq ($(USE_CONTAINER),1)
+	$(RERUN_IN_CONTAINER)
+else
+	go generate ./docs
+endif
 #
 # Other targets
 #

build-image/Dockerfile

@@ -23,7 +23,7 @@ FROM alpine:3.17 as helm
 RUN apk add --no-cache helm
 
 # Dependency: Go and Go dependencies
-FROM golang:1.21.3-bullseye as golang
+FROM golang:1.21.4-bullseye as golang
 
 # Keep in sync with cmd/grafana-agent-operator/DEVELOPERS.md
 ENV CONTROLLER_GEN_VERSION v0.9.2

build-image/windows/Dockerfile

@@ -1,4 +1,4 @@
-FROM library/golang:1.21.3-windowsservercore-1809
+FROM library/golang:1.21.4-windowsservercore-1809
 
 SHELL ["powershell", "-command"]
 

cmd/grafana-agent-operator/Dockerfile

@@ -4,7 +4,7 @@
 # default when running `docker buildx build` or when DOCKER_BUILDKIT=1 is set
 # in environment variables.
 
-FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.30.3 as build
+FROM --platform=$BUILDPLATFORM grafana/agent-build-image:0.30.4 as build
 ARG BUILDPLATFORM
 ARG TARGETPLATFORM
 ARG TARGETOS

cmd/grafana-agent-service/config_windows.go

@@ -16,6 +16,11 @@ type config struct {
 	// not included.
 	Args []string
 
+	// Environment holds environment variables for the Grafana Agent service.
+	// Each item represents an environment variable in form "key=value".
+	// All environments variables from the current process with be merged into Environment
+	Environment []string
+
 	// WorkingDirectory points to the working directory to run the Grafana Agent
 	// binary from.
 	WorkingDirectory string
@@ -42,9 +47,15 @@ func loadConfig() (*config, error) {
 		return nil, fmt.Errorf("failed to retrieve key Arguments: %w", err)
 	}
 
+	env, _, err := agentKey.GetStringsValue("Environment")
+	if err != nil {
+		return nil, fmt.Errorf("failed to retrieve key Environment: %w", err)
+	}
+
 	return &config{
 		ServicePath:      servicePath,
 		Args:             args,
+		Environment:      env,
 		WorkingDirectory: filepath.Dir(servicePath),
 	}, nil
 }

cmd/grafana-agent-service/main_windows.go

@@ -29,9 +29,10 @@ func main() {
 	}
 
 	cfg := serviceManagerConfig{
-		Path: managerConfig.ServicePath,
-		Args: managerConfig.Args,
-		Dir:  managerConfig.WorkingDirectory,
+		Path:        managerConfig.ServicePath,
+		Args:        managerConfig.Args,
+		Environment: managerConfig.Environment,
+		Dir:         managerConfig.WorkingDirectory,
 
 		// Send logs directly to the event logger.
 		Stdout: logger,

cmd/grafana-agent-service/service.go

@@ -24,6 +24,9 @@ type serviceManagerConfig struct {
 	// Args of the binary to run, not including the command itself.
 	Args []string
 
+	// Environment of the binary to run, including the command environment itself.
+	Environment []string
+
 	// Dir specifies the working directory to run the binary from. If Dir is
 	// empty, the working directory of the current process is used.
 	Dir string
@@ -84,5 +87,7 @@ func (svc *serviceManager) buildCommand(ctx context.Context) *exec.Cmd {
 	cmd.Dir = svc.cfg.Dir
 	cmd.Stdout = svc.cfg.Stdout
 	cmd.Stderr = svc.cfg.Stderr
+	cmd.Env = os.Environ()
+	cmd.Env = append(cmd.Env, svc.cfg.Environment...)
 	return cmd
 }

cmd/grafana-agent-service/service_test.go

@@ -30,8 +30,9 @@ func Test_serviceManager(t *testing.T) {
 		listenHost := getListenHost(t)
 
 		mgr := newServiceManager(l, serviceManagerConfig{
-			Path: serviceBinary,
-			Args: []string{"-listen-addr", listenHost},
+			Path:        serviceBinary,
+			Args:        []string{"-listen-addr", listenHost},
+			Environment: []string{"LISTEN=" + listenHost},
 		})
 		go mgr.Run(componenttest.TestContext(t))
 
@@ -40,6 +41,12 @@ func Test_serviceManager(t *testing.T) {
 			require.NoError(t, err)
 			require.Equal(t, []byte("Hello, world!"), resp)
 		})
+
+		util.Eventually(t, func(t require.TestingT) {
+			resp, err := makeServiceRequest(listenHost, "/echo/env", nil)
+			require.NoError(t, err)
+			require.Contains(t, string(resp), "LISTEN="+listenHost)
+		})
 	})
 
 	t.Run("terminates service binary", func(t *testing.T) {