[SLT-141] feat(metrics): multiple exports

core/go.mod

@@ -63,6 +63,7 @@ require (
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/sync v0.8.0
+	google.golang.org/grpc v1.64.0
 	gorm.io/driver/sqlite v1.5.6
 	gorm.io/gorm v1.25.10
 	k8s.io/apimachinery v0.29.3
@@ -189,7 +190,6 @@ require (
 	golang.org/x/tools v0.24.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
-	google.golang.org/grpc v1.64.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect

core/metrics/README.md

@@ -12,7 +12,27 @@ There's also a `NAME_PREFIX` environment variable that will prefix all the metri
 
 ## OTLP
 
-We do our best to support enviornment variables specified in the [Otel Spec](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/) and have added a few of our own. Key ones to note are:
+We do our best to support enviornment variables specified in the [Otel Spec](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/) and the [OTLP Spec](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/) and have added a few of our own. This was to allow for multiple exporter backends for traces, as otel clients only allow for one URL. The relevant multi exporter code is in `multiexporter.go`, and simply wraps multiple otel clients.
+
+The additional environment variables to note are:
+| Enviornment Variable                     | Description                               | Default |
+|------------------------------------------|-------------------------------------------|---------|
+| `OTEL_EXPORTER_OTLP_ENDPOINT`            | The endpoint for the primary OTLP exporter | None    |
+| `OTEL_EXPORTER_OTLP_ENDPOINT_1`          | The endpoint for the first additional OTLP exporter | None    |
+| `OTEL_EXPORTER_OTLP_ENDPOINT_2`          | The endpoint for the second additional OTLP exporter | None    |
+| `OTEL_EXPORTER_OTLP_ENDPOINT_3`          | The endpoint for the third additional OTLP exporter | None    |
+| ...                                      | Additional endpoints can be added by incrementing the number | None    |
+| `OTEL_EXPORTER_OTLP_TRANSPORT`           | The transport protocol for the primary OTLP exporter | `http` |
+| `OTEL_EXPORTER_OTLP_TRANSPORT_1`         | The transport protocol for the first additional OTLP exporter | `http` |
+| `OTEL_EXPORTER_OTLP_TRANSPORT_2`         | The transport protocol for the second additional OTLP exporter | `http` |
+| `OTEL_EXPORTER_OTLP_TRANSPORT_3`         | The transport protocol for the third additional OTLP exporter | `http` |
+| ...                                      | Additional transports can be specified by incrementing the number | `http` |
+
+You can do the same thing for `OTEL_EXPORTER_OTLP_SECURE_MODE` and `OTEL_EXPORTER_OTLP_HEADERS`
+
+<!-- TODO: fully document these optins-->
+
+Note: The OTLP exporter endpoints and transports can be specified for multiple exporters by using incrementing numbers (1, 2, 3, etc.) in the environment variable names. This allows for configuration of multiple OTLP exporters. The primary exporter uses the base names without numbers.
 
 
 ## Jaeger

core/metrics/export_test.go

@@ -0,0 +1,6 @@
+package metrics
+
+// HeadersToMap converts a string of headers to a map.
+func HeadersToMap(val string) map[string]string {
+	return headersToMap(val)
+}

core/metrics/multiexporter.go

@@ -0,0 +1,83 @@
+package metrics
+
+import (
+	"context"
+	"fmt"
+	"go.uber.org/multierr"
+	"sync"
+	"time"
+
+	"go.opentelemetry.io/otel/sdk/trace"
+)
+
+// MultiExporter is an interface that allows exporting spans to multiple OTLP trace exporters.
+type MultiExporter interface {
+	trace.SpanExporter
+	AddExporter(exporter trace.SpanExporter)
+}
+
+type multiExporter struct {
+	exporters []trace.SpanExporter
+}
+
+// NewMultiExporter creates a new multi exporter that forwards spans to multiple OTLP trace exporters.
+// It takes in one or more trace.SpanExporter instances and ensures that spans are sent to all of them.
+// This is useful when you need to send trace data to multiple backends or endpoints.
+func NewMultiExporter(exporters ...trace.SpanExporter) MultiExporter {
+	return &multiExporter{
+		exporters: exporters,
+	}
+}
+
+const defaultTimeout = 30 * time.Second
+
+// ExportSpans exports a batch of spans.
+func (m *multiExporter) ExportSpans(parentCtx context.Context, ss []trace.ReadOnlySpan) error {
+	return m.doParallel(parentCtx, func(ctx context.Context, exporter trace.SpanExporter) error {
+		return exporter.ExportSpans(ctx, ss)
+	})
+}
+
+func (m *multiExporter) doParallel(parentCtx context.Context, fn func(context.Context, trace.SpanExporter) error) error {
+	ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout)
+	defer cancel()
+
+	var wg sync.WaitGroup
+	var errors []error
+	var mu sync.Mutex
+
+	wg.Add(len(m.exporters))
+	for _, exporter := range m.exporters {
+		go func(exporter trace.SpanExporter) {
+			defer wg.Done()
+			err := fn(ctx, exporter)
+			if err != nil {
+				mu.Lock()
+				errors = append(errors, fmt.Errorf("error in doMultiple: %w", err))
+				mu.Unlock()
+			}
+		}(exporter)
+	}
+
+	wg.Wait()
+	if len(errors) > 0 {
+		// nolint: wrapcheck
+		return multierr.Combine(errors...)
+	}
+
+	return nil
+}
+
+// Shutdown notifies the exporter of a pending halt to operations.
+func (m *multiExporter) Shutdown(ctx context.Context) error {
+	return m.doParallel(ctx, func(ctx context.Context, exporter trace.SpanExporter) error {
+		return exporter.Shutdown(ctx)
+	})
+}
+
+// AddExporter adds an exporter to the multi exporter.
+func (m *multiExporter) AddExporter(exporter trace.SpanExporter) {
+	m.exporters = append(m.exporters, exporter)
+}
+
+var _ trace.SpanExporter = &multiExporter{}

core/metrics/multiexporter_test.go

@@ -0,0 +1,45 @@
+package metrics_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/synapsecns/sanguine/core/metrics"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+)
+
+func TestMultiExporter(t *testing.T) {
+	// Create in-memory exporters
+	exporter1 := tracetest.NewInMemoryExporter()
+	exporter2 := tracetest.NewInMemoryExporter()
+
+	// Create multi-exporter
+	multiExporter := metrics.NewMultiExporter(exporter1, exporter2)
+
+	// Create test spans
+	spans := []sdktrace.ReadOnlySpan{
+		tracetest.SpanStub{}.Snapshot(),
+		tracetest.SpanStub{}.Snapshot(),
+	}
+
+	// Test ExportSpans
+	err := multiExporter.ExportSpans(context.Background(), spans)
+	require.NoError(t, err)
+
+	// Verify that spans were exported to both exporters
+	assert.Equal(t, 2, len(exporter1.GetSpans()))
+	assert.Equal(t, 2, len(exporter2.GetSpans()))
+
+	// Test Shutdown
+	err = multiExporter.Shutdown(context.Background())
+	require.NoError(t, err)
+
+	// Verify that both exporters were shut down
+	// Note: InMemoryExporter doesn't have a Stopped() method, so we can't check this directly
+	// Instead, we can try to export spans again and check for an error
+	err = multiExporter.ExportSpans(context.Background(), spans)
+	assert.NoError(t, err, "Expected no error after shutdown")
+}