[SLT-141] feat(metrics): multiple exports

core/metrics/README.md

@@ -12,8 +12,13 @@ There's also a `NAME_PREFIX` environment variable that will prefix all the metri
 
 ## OTLP
 
-We do our best to support enviornment variables specified in the [Otel Spec](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/) and have added a few of our own. Key ones to note are:
+We do our best to support enviornment variables specified in the [Otel Spec](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/) and have added a few of our own. This was to allow for multiple exporter backends for traces, as otel clients only allow for one URL. The relevant multi exporter code is in `multiexporter.go`, and simply wraps multiple otel clients.
 
+The additional environment variables to note are:
+| Enviornment Variable                     | Description                               | Default |
+|------------------------------------------|-------------------------------------------|---------|
+| `OTEL_EXPORTER_OTLP_TRANSPORT_PRIMARY`   | Primary exporter URL to send traces to.   | `true`  |
+| `OTEL_EXPORTER_OTLP_TRANSPORT_SECONDARY` | Secondary exporter URL to send traces to. | `8080`  |
 
 ## Jaeger
 

core/metrics/multiexporter.go

@@ -0,0 +1,46 @@
+package metrics
+
+import (
+	"context"
+	"fmt"
+
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	tracesdk "go.opentelemetry.io/otel/sdk/trace"
+)
+
+type multiExporter struct {
+	exporters []*otlptrace.Exporter
+}
+
+// NewMultiExporter creates a new multi exporter that forwards spans to multiple OTLP trace exporters.
+// It takes in one or more otlptrace.Exporter instances and ensures that spans are sent to all of them.
+// This is useful when you need to send trace data to multiple backends or endpoints.
+func NewMultiExporter(exporters ...*otlptrace.Exporter) tracesdk.SpanExporter {
+	return &multiExporter{
+		exporters: exporters,
+	}
+}
+
+// ExportSpans exports a batch of spans.
+func (m *multiExporter) ExportSpans(ctx context.Context, ss []tracesdk.ReadOnlySpan) error {
+	for _, exporter := range m.exporters {
+		err := exporter.ExportSpans(ctx, ss)
+		if err != nil {
+			return fmt.Errorf("could not export spans: %w", err)
+		}
+	}
+	return nil
+}
+
+// Shutdown notifies the exporter of a pending halt to operations.
+func (m *multiExporter) Shutdown(ctx context.Context) error {
+	for _, exporter := range m.exporters {
+		err := exporter.Shutdown(ctx)
+		if err != nil {
+			return fmt.Errorf("could not stop exporter: %w", err)
+		}
+	}
+	return nil
+}
+
+var _ tracesdk.SpanExporter = &multiExporter{}

core/metrics/otlp.go

@@ -3,15 +3,15 @@
 import (
 	"context"
 	"fmt"
+	"strings"
+	"time"
+
 	"github.com/synapsecns/sanguine/core"
 	"github.com/synapsecns/sanguine/core/config"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
 	tracesdk "go.opentelemetry.io/otel/sdk/trace"
-	"os"
-	"strings"
-	"time"
 )
 
 type otlpHandler struct {
@@ -28,23 +28,53 @@
 }
 
 func (n *otlpHandler) Start(ctx context.Context) (err error) {
-	var client otlptrace.Client
-	transport := transportFromString(core.GetEnv(otlpTransportEnv, otlpTransportGRPC.String()))
-	switch transport {
-	case otlpTransportHTTP:
-		client = otlptracehttp.NewClient()
-	case otlpTransportGRPC:
-		client = otlptracegrpc.NewClient()
-	default:
-		return fmt.Errorf("unknown transport type: %s", os.Getenv(otlpTransportEnv))
+	isDefaultConfig := core.GetEnv(otlpTransportEnvDefault, "") != ""
+
+	var client, secondaryClient otlptrace.Client
+	if isDefaultConfig {
+		client, err = getClient(otlpTransportEnvDefault)
+		if err != nil {
+			return fmt.Errorf("could not create default client: %w", err)
+		}
+	} else {
+		// instantiate both the primary and secondary clients
+		client, err = getClient(otlpTransportEnvPrimary)
+		if err != nil {
+			return fmt.Errorf("could not create primary client: %w", err)
+		}
+
+		secondaryClient, err = getClient(otlpTransportEnvSecondary)
+		if err != nil {
+			return fmt.Errorf("could not create secondary client: %w", err)
+		}
 	}
 
 	exporter, err := otlptrace.New(ctx, client)
 	if err != nil {
 		return fmt.Errorf("failed to create otlp exporter: %w", err)
 	}
 
-	n.baseHandler = newBaseHandler(n.buildInfo, tracesdk.WithBatcher(exporter, tracesdk.WithMaxQueueSize(1000000), tracesdk.WithMaxExportBatchSize(2000)), tracesdk.WithSampler(tracesdk.AlwaysSample()))
+	// create the multi-exporter with optional secondary exporter
+	var multiExporter tracesdk.SpanExporter
+	if secondaryClient != nil {
+		secondaryExporter, err := otlptrace.New(ctx, secondaryClient)
+		if err != nil {
+			return fmt.Errorf("failed to create secondary otlp exporter: %w", err)
+		}
+		multiExporter = NewMultiExporter(exporter, secondaryExporter)
+	} else {
+		multiExporter = NewMultiExporter(exporter)
+	}
+
+	n.baseHandler = newBaseHandler(
+		n.buildInfo,
+		tracesdk.WithBatcher(
+			multiExporter,
+			tracesdk.WithMaxQueueSize(defaultMaxQueueSize),
+			tracesdk.WithMaxExportBatchSize(defaultMaxExportBatch),
+		),
+		tracesdk.WithSampler(tracesdk.AlwaysSample()),
+	)
 
 	// start the new parent
 	err = n.baseHandler.Start(ctx)
@@ -90,7 +120,9 @@
 }
 
 const (
-	otlpTransportEnv = "OTEL_EXPORTER_OTLP_TRANSPORT"
+	otlpTransportEnvDefault   = "OTEL_EXPORTER_OTLP_TRANSPORT"
+	otlpTransportEnvPrimary   = "OTEL_EXPORTER_OTLP_TRANSPORT_PRIMARY"
+	otlpTransportEnvSecondary = "OTEL_EXPORTER_OTLP_TRANSPORT_SECONDARY"
 )
 
 //go:generate go run golang.org/x/tools/cmd/stringer -type=otlpTransportType -linecomment
@@ -101,6 +133,21 @@
 	otlpTransportGRPC                              // grpc
 )
 
+func buildClientFromTransport(transport otlpTransportType, url string) (otlptrace.Client, error) {
+	if url == "" {
+		return nil, fmt.Errorf("no url specified")
+	}
+
+	switch transport {
+	case otlpTransportHTTP:
+		return otlptracehttp.NewClient(otlptracehttp.WithEndpointURL(url)), nil
+	case otlpTransportGRPC:
+		return otlptracegrpc.NewClient(otlptracegrpc.WithEndpointURL(url)), nil
+	default:
+		return nil, fmt.Errorf("unknown transport type: %s", transport.String())
+	}
+}
+
 // transportFromString converts a string to a transport type.
 // Defaults to http if the string is not recognized.
 func transportFromString(transport string) otlpTransportType {
@@ -114,3 +161,18 @@
 	// (see uber's go stye guide for details)
 	return otlpTransportType(0)
 }
+
+func getClient(transportEnv string) (otlptrace.Client, error) {
+	transport := transportFromString(core.GetEnv(transportEnv, otlpTransportHTTP.String()))
+	url := core.GetEnv(transportEnv, "")
+
+	return buildClientFromTransport(
+		transport,
+		url,
+	)
+}
+
+const (
+	defaultMaxQueueSize   = 1000000
+	defaultMaxExportBatch = 2000
+)

core/metrics/rookout.go

@@ -1,12 +1,15 @@
+//go:build go1.16 && !go1.23
+
 package metrics
 
 import (
+	"os"
+
 	rookout "github.com/Rookout/GoSDK"
 	"github.com/Rookout/GoSDK/pkg/config"
 	"github.com/synapsecns/sanguine/core"
 	synconfig "github.com/synapsecns/sanguine/core/config"
 	"github.com/synapsecns/sanguine/core/metrics/internal"
-	"os"
 )
 
 // DefaultGitRepo is the default git repo for sanguine.