Skip to content

Commit

Permalink
[exporter/loadbalancing] Fix memory leaks (open-telemetry#31050)
Browse files Browse the repository at this point in the history
**Description:** <Describe what has changed.>
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
This fixes a few goroutine leaks in the loadbalancing exporter.

1. `metrics`, `traces`, and `logs` exporters were starting their
respective load balancers, but were not shutting them down. This adds
each respective shutdown call.
2. The `loadbalancer` was starting the resolver but never shutting it
down. This adds a shutdown call to the resolver.
3. The static resolver was starting resolvers for each passed in
exporter, but never shut them down. This adds a shutdown call for each
resolver in the static resolver.

Also added a couple missing `Shutdown` calls from tests.

**Link to tracking Issue:** <Issue number if applicable>
open-telemetry#30438

**Testing:** <Describe what testing was performed and which tests were
added.>
All existing tests are passing as well as added goleak checks.
  • Loading branch information
crobert-1 authored and XinRanZhAWS committed Mar 13, 2024
1 parent 229e1ee commit 1ccd1fc
Show file tree
Hide file tree
Showing 10 changed files with 66 additions and 9 deletions.
27 changes: 27 additions & 0 deletions .chloggen/goleak_loadbalancingexp.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Use this changelog template to create an entry for release notes.

# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: bug_fix

# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
component: loadbalancingexporter

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: Fix memory leaks on shutdown

# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
issues: [31050]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext:

# If your change doesn't affect end users or the exported elements of any package,
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
# Optional: The change log or logs in which this entry should be included.
# e.g. '[user]' or '[user, api]'
# Include 'user' if the change is relevant to end users.
# Include 'api' if there is a change to a library API.
# Default: '[user]'
change_logs: [user]
1 change: 1 addition & 0 deletions exporter/loadbalancingexporter/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ require (
go.opentelemetry.io/collector/semconv v0.96.1-0.20240306115632-b2693620eff6
go.opentelemetry.io/otel/metric v1.24.0
go.opentelemetry.io/otel/trace v1.24.0
go.uber.org/goleak v1.3.0
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.27.0
k8s.io/api v0.29.2
Expand Down
5 changes: 3 additions & 2 deletions exporter/loadbalancingexporter/loadbalancer.go
Original file line number Diff line number Diff line change
Expand Up @@ -168,9 +168,10 @@ func endpointFound(endpoint string, endpoints []string) bool {
return false
}

func (lb *loadBalancer) Shutdown(context.Context) error {
func (lb *loadBalancer) Shutdown(ctx context.Context) error {
err := lb.res.shutdown(ctx)
lb.stopped = true
return nil
return err
}

// exporterAndEndpoint returns the exporter and the endpoint for the given identifier.
Expand Down
1 change: 1 addition & 0 deletions exporter/loadbalancingexporter/loadbalancer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ func TestWithDNSResolverNoEndpoints(t *testing.T) {

err = p.Start(context.Background(), componenttest.NewNopHost())
require.NoError(t, err)
defer func() { assert.NoError(t, p.Shutdown(context.Background())) }()

// test
_, e, _ := p.exporterAndEndpoint([]byte{128, 128, 0, 0})
Expand Down
5 changes: 3 additions & 2 deletions exporter/loadbalancingexporter/log_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,14 @@ func (e *logExporterImp) Start(ctx context.Context, host component.Host) error {
return e.loadBalancer.Start(ctx, host)
}

func (e *logExporterImp) Shutdown(context.Context) error {
func (e *logExporterImp) Shutdown(ctx context.Context) error {
if !e.started {
return nil
}
err := e.loadBalancer.Shutdown(ctx)
e.started = false
e.shutdownWg.Wait()
return nil
return err
}

func (e *logExporterImp) ConsumeLogs(ctx context.Context, ld plog.Logs) error {
Expand Down
5 changes: 3 additions & 2 deletions exporter/loadbalancingexporter/metrics_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,11 @@ func (e *metricExporterImp) Start(ctx context.Context, host component.Host) erro
return e.loadBalancer.Start(ctx, host)
}

func (e *metricExporterImp) Shutdown(context.Context) error {
func (e *metricExporterImp) Shutdown(ctx context.Context) error {
err := e.loadBalancer.Shutdown(ctx)
e.stopped = true
e.shutdownWg.Wait()
return nil
return err
}

func (e *metricExporterImp) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error {
Expand Down
17 changes: 17 additions & 0 deletions exporter/loadbalancingexporter/package_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
// Copyright The OpenTelemetry Authors
// SPDX-License-Identifier: Apache-2.0

package loadbalancingexporter

import (
"testing"

"go.uber.org/goleak"
)

// The IgnoreTopFunction call prevents catching the leak generated by opencensus
// defaultWorker.Start which at this time is part of the package's init call.
// See https://github.com/census-instrumentation/opencensus-go/issues/1191 for more information.
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m, goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"))
}
1 change: 1 addition & 0 deletions exporter/loadbalancingexporter/resolver_dns_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ func TestCantResolve(t *testing.T) {

// verify
assert.NoError(t, err)
assert.NoError(t, res.shutdown(context.Background()))
}

func TestOnChange(t *testing.T) {
Expand Down
8 changes: 7 additions & 1 deletion exporter/loadbalancingexporter/resolver_static.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,13 @@ func (r *staticResolver) start(ctx context.Context) error {
return err
}

func (r *staticResolver) shutdown(_ context.Context) error {
func (r *staticResolver) shutdown(context.Context) error {
r.endpoints = nil

for _, callback := range r.onChangeCallbacks {
callback(r.endpoints)
}

return nil
}

Expand Down
5 changes: 3 additions & 2 deletions exporter/loadbalancingexporter/trace_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,11 @@ func (e *traceExporterImp) Start(ctx context.Context, host component.Host) error
return e.loadBalancer.Start(ctx, host)
}

func (e *traceExporterImp) Shutdown(context.Context) error {
func (e *traceExporterImp) Shutdown(ctx context.Context) error {
err := e.loadBalancer.Shutdown(ctx)
e.stopped = true
e.shutdownWg.Wait()
return nil
return err
}

func (e *traceExporterImp) ConsumeTraces(ctx context.Context, td ptrace.Traces) error {
Expand Down

0 comments on commit 1ccd1fc

Please sign in to comment.