Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: DataDog POC #250

Merged
merged 2 commits into from
Jul 5, 2024
Merged

feat: DataDog POC #250

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
89e5b8c
feat: DataDog POC
p0mvn Jul 4, 2024
1ee8b2b
updates
p0mvn Jul 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 29 additions & 63 deletions app/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,17 @@ import (
"os"
"os/signal"
"syscall"
"time"

"github.com/getsentry/sentry-go"
sentryotel "github.com/getsentry/sentry-go/otel"
"github.com/osmosis-labs/sqs/chaininfo/client"
"github.com/osmosis-labs/sqs/domain"
sqslog "github.com/osmosis-labs/sqs/log"
"github.com/spf13/viper"
_ "github.com/swaggo/echo-swagger"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"go.opentelemetry.io/otel/propagation"
"go.uber.org/zap"

"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
Expand Down Expand Up @@ -74,62 +74,36 @@ func main() {
}
}()

if config.OTEL.DSN != "" {
otelConfig := config.OTEL

var (
// sentryEndpointWhitelist is a map of endpoints and their respective sampling rates
sentryEndpointWhitelist = map[string]float64{
"/router/quote": otelConfig.CustomSampleRate.Quote,
"/custom-direct-quote": otelConfig.CustomSampleRate.Other,
"/tokens/prices": otelConfig.CustomSampleRate.Other,
"/pools": otelConfig.CustomSampleRate.Other,
}

// custom sampler that samples only the whitelisted endpoints per their configured rates.
traceSampler sentry.TracesSampler = func(ctx sentry.SamplingContext) float64 {
if ctx.Span == nil {
return 0
}

spanName := ctx.Span.Name

if samplerRate, ok := sentryEndpointWhitelist[spanName]; ok {
return samplerRate
}
// Use context for graceful shutdown
ctx, cancel := context.WithCancel(context.Background())

return 0
}
if config.OTEL.Enabled {
// resource.WithContainer() adds container.id which the agent will leverage to fetch container tags via the tagger.
res, err := resource.New(ctx, resource.WithContainer(),
resource.WithAttributes(semconv.ServiceNameKey.String(*hostName)),
resource.WithFromEnv(),
)

err = sentry.Init(sentry.ClientOptions{
ServerName: *hostName,
Dsn: otelConfig.DSN,
SampleRate: otelConfig.SampleRate,
EnableTracing: otelConfig.EnableTracing,
Debug: *isDebug,
TracesSampler: traceSampler,
ProfilesSampleRate: otelConfig.ProfilesSampleRate,
Environment: otelConfig.Environment,
})
if err != nil {
log.Fatalf("sentry.Init: %s", err)
panic(err)
}
defer sentry.Flush(2 * time.Second)

sentry.CaptureMessage("SQS started")
tp, err := initOTELTracer(ctx, res)
if err != nil {
panic(err)
}

initOTELTracer(*hostName)
defer func() {
if err := tp.Shutdown(ctx); err != nil {
log.Fatal("Error shutting down tracer provider: ", zap.Error(err))
}
}()
}

chainClient, err := client.NewClient(config.ChainID, config.ChainGRPCGatewayEndpoint)
chainClient, err := client.NewClient(config.ChainID, config.ChainTendermingRPCEndpoint)
if err != nil {
panic(err)
}

// Use context for graceful shutdown
ctx, cancel := context.WithCancel(context.Background())

// If fails, it means that the node is not reachable
if _, err := chainClient.GetLatestHeight(ctx); err != nil {
panic(err)
Expand Down Expand Up @@ -168,26 +142,18 @@ func main() {

// initOTELTracer initializes the OTEL tracer
// and wires it up with the Sentry exporter.
func initOTELTracer(hostName string) {
exporter, err := stdouttrace.New(stdouttrace.WithPrettyPrint())
if err != nil {
log.Fatalf("stdouttrace.New: %v", err)
}

resource, err := resource.New(context.Background(),
resource.WithAttributes(
semconv.ServiceNameKey.String(hostName),
),
)
func initOTELTracer(ctx context.Context, res *resource.Resource) (*sdktrace.TracerProvider, error) {
exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithInsecure())
if err != nil {
log.Fatalf("resource.New: %v", err)
log.Fatal("can't initialize grpc trace exporter", zap.Error(err))
return nil, err
}

tp := sdktrace.NewTracerProvider(
sdktrace.WithBatcher(exporter),
sdktrace.WithResource(resource),
sdktrace.WithSpanProcessor(sentryotel.NewSentrySpanProcessor()),
sdktrace.WithResource(res),
)
otel.SetTracerProvider(tp)
otel.SetTextMapPropagator(sentryotel.NewSentryPropagator())
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
return tp, nil
}
3 changes: 2 additions & 1 deletion app/sqs_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ var DefaultConfig = domain.Config{
LoggerIsProduction: true,
LoggerLevel: "info",

ChainGRPCGatewayEndpoint: "http://localhost:26657",
ChainTendermingRPCEndpoint: "http://localhost:26657",
ChainGRPCGatewayEndpoint: "http://localhost:9090",
ChainID: "osmosis-1",
ChainRegistryAssetsFileURL: "https://raw.githubusercontent.com/osmosis-labs/assetlists/main/osmosis-1/generated/frontend/assetlist.json",

Expand Down
8 changes: 3 additions & 5 deletions config-testnet.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
"logger-filename": "sqs.log",
"logger-is-production": true,
"logger-level": "info",
"grpc-gateway-endpoint": "http://localhost:26657",
"grpc-tendermint-rpc-endpoint": "http://localhost:26657",
"grpc-gateway-endpoint": "http://localhost:9090",
"chain-id": "osmosis-1",
"chain-registry-assets-url": "https://raw.githubusercontent.com/osmosis-labs/assetlists/main/osmo-test-5/osmo-test-5.assetlist.json",
"flight-recording": {
Expand Down Expand Up @@ -58,10 +59,7 @@
"server-connection-timeout-seconds": 10
},
"otel": {
"dsn": "",
"sample-rate": 1,
"enable-tracing": true,
"profiles-sample-rate": 1,
"enabled": true,
"environment": "production"
},
"cors": {
Expand Down
14 changes: 4 additions & 10 deletions config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
"logger-filename": "sqs.log",
"logger-is-production": true,
"logger-level": "info",
"grpc-gateway-endpoint": "http://localhost:26657",
"grpc-tendermint-rpc-endpoint": "http://localhost:26657",
"grpc-gateway-endpoint": "http://localhost:9090",
"chain-id": "osmosis-1",
"chain-registry-assets-url": "https://raw.githubusercontent.com/osmosis-labs/assetlists/main/osmosis-1/generated/frontend/assetlist.json",
"router": {
Expand Down Expand Up @@ -77,15 +78,8 @@
"server-connection-timeout-seconds": 10
},
"otel": {
"dsn": "",
"sample-rate": 0,
"enable-tracing": false,
"profiles-sample-rate": 0,
"environment": "sqs-dev",
"custom-sample-rate": {
"/router/quote": 0,
"other": 0
}
"enabled": true,
"environment": "sqs-dev"
},
"cors": {
"allowed-origin": "*",
Expand Down
20 changes: 5 additions & 15 deletions domain/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@ type Config struct {
LoggerIsProduction bool `mapstructure:"logger-is-production"`
LoggerLevel string `mapstructure:"logger-level"`

ChainGRPCGatewayEndpoint string `mapstructure:"grpc-gateway-endpoint"`
ChainID string `mapstructure:"chain-id"`
ChainTendermingRPCEndpoint string `mapstructure:"grpc-tendermint-rpc-endpoint"`
ChainGRPCGatewayEndpoint string `mapstructure:"grpc-gateway-endpoint"`
ChainID string `mapstructure:"chain-id"`

// Chain registry assets URL.
ChainRegistryAssetsFileURL string `mapstructure:"chain-registry-assets-url"`
Expand Down Expand Up @@ -45,19 +46,8 @@ type EndpointOTELConfig struct {

// OTELConfig represents OpenTelemetry configuration.
type OTELConfig struct {
// The DSN to use.
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note: removed Sentry config

DSN string `mapstructure:"dsn"`
// The sample rate for event submission in the range [0.0, 1.0].
// By default, all events are sent.
SampleRate float64 `mapstructure:"sample-rate"`
// Enable performance tracing.
EnableTracing bool `mapstructure:"enable-tracing"`
// The sample rate for profiling traces in the range [0.0, 1.0].
// This is relative to TracesSampleRate - it is a ratio of profiled traces out of all sampled traces.
ProfilesSampleRate float64 `mapstructure:"profiles-sample-rate"`
// The environment to be sent with events.
Environment string `mapstructure:"environment"`
CustomSampleRate EndpointOTELConfig `mapstructure:"custom-sample-rate"`
Enabled bool `mapstructure:"enabled"`
Environment string `mapstructure:"environment"`
}

// CORSConfig represents HTTP CORS headers configuration.
Expand Down
6 changes: 3 additions & 3 deletions domain/mocks/pools_usecase_mock.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ func (pm *PoolsUsecaseMock) StorePools(pools []sqsdomain.PoolI) error {
// GetCosmWasmPoolConfig implements mvc.PoolsUsecase.
func (pm *PoolsUsecaseMock) GetCosmWasmPoolConfig() domain.CosmWasmPoolRouterConfig {
return domain.CosmWasmPoolRouterConfig{
TransmuterCodeIDs: map[uint64]struct{}{},
GeneralCosmWasmCodeIDs: map[uint64]struct{}{},
NodeURI: "",
TransmuterCodeIDs: map[uint64]struct{}{},
GeneralCosmWasmCodeIDs: map[uint64]struct{}{},
ChainGRPCGatewayEndpoint: "",
}
}

Expand Down
4 changes: 2 additions & 2 deletions domain/pools.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ type CosmWasmPoolRouterConfig struct {
// code IDs for the generalized cosmwasm pool type
GeneralCosmWasmCodeIDs map[uint64]struct{}

// node URI
NodeURI string
// ChainGRPCGatewayEndpoint is the endpoint for the chain's gRPC gateway
ChainGRPCGatewayEndpoint string
}

// ScalingFactorGetterCb is a callback that is used to get the scaling factor for a given denom.
Expand Down
55 changes: 27 additions & 28 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ require (
github.com/alecthomas/assert/v2 v2.7.0
github.com/cometbft/cometbft v0.37.4
github.com/cosmos/cosmos-sdk v0.47.8
github.com/getsentry/sentry-go v0.27.0
github.com/labstack/echo/v4 v4.11.4
github.com/osmosis-labs/osmosis/osmomath v0.0.13
github.com/osmosis-labs/osmosis/osmoutils v0.0.13
Expand All @@ -20,17 +19,21 @@ require (
github.com/stretchr/testify v1.9.0
github.com/swaggo/echo-swagger v1.4.1
github.com/swaggo/swag v1.16.3
go.opentelemetry.io/otel v1.25.0
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.25.0
go.opentelemetry.io/otel/sdk v1.25.0
go.opentelemetry.io/otel v1.28.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0
go.opentelemetry.io/otel/sdk v1.28.0
go.uber.org/zap v1.26.0
google.golang.org/grpc v1.63.2
google.golang.org/grpc v1.64.0
)

require (
github.com/Masterminds/semver v1.5.0 // indirect
github.com/bits-and-blooms/bitset v1.8.0 // indirect
github.com/getsentry/sentry-go v0.27.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
github.com/skip-mev/block-sdk v1.4.2 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
Comment on lines +35 to +36
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tip

Codebase Verification

Ensure correct usage of indirect OpenTelemetry dependencies.

The go.opentelemetry.io/proto/otlp dependency is not used in the codebase and may need to be removed. The go.opentelemetry.io/otel/exporters/otlp/otlptrace dependency is indirectly used through otlptracegrpc.

  • Remove go.opentelemetry.io/proto/otlp if it is not required.
Analysis chain

Ensure correct usage of indirect OpenTelemetry dependencies.

The indirect OpenTelemetry-related packages (go.opentelemetry.io/otel/exporters/otlp/otlptrace, go.opentelemetry.io/proto/otlp) have been added. Verify that these dependencies are correctly used in the code.

Scripts executed

The following scripts were executed for the analysis:

Script:

#!/bin/bash
# Description: Verify that the indirect OpenTelemetry dependencies are correctly used in the code.

# Test: Search for the usage of indirect OpenTelemetry-related packages.
rg --type go $'go.opentelemetry.io/otel/exporters/otlp/otlptrace|go.opentelemetry.io/proto/otlp'

Length of output: 176

Script:

#!/bin/bash
# Description: Verify if go.opentelemetry.io/proto/otlp is used in the codebase

# Search for the usage of go.opentelemetry.io/proto/otlp package
rg --type go 'go.opentelemetry.io/proto/otlp'

Length of output: 45

)

require (
Expand All @@ -42,11 +45,10 @@ require (
)

require (
cloud.google.com/go v0.112.0 // indirect
cloud.google.com/go/compute v1.24.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go v0.112.1 // indirect
cloud.google.com/go/compute/metadata v0.3.0 // indirect
cloud.google.com/go/iam v1.1.6 // indirect
cloud.google.com/go/storage v1.37.0 // indirect
cloud.google.com/go/storage v1.38.0 // indirect
cosmossdk.io/api v0.3.1 // indirect
cosmossdk.io/core v0.5.1 // indirect
cosmossdk.io/depinject v1.0.0-alpha.4 // indirect
Expand Down Expand Up @@ -103,12 +105,11 @@ require (
github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/getsentry/sentry-go/otel v0.27.0
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-kit/kit v0.12.0 // indirect
github.com/go-kit/log v0.2.1 // indirect
github.com/go-logfmt/logfmt v0.6.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.20.2 // indirect
github.com/go-openapi/jsonreference v0.20.4 // indirect
Expand All @@ -130,7 +131,7 @@ require (
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/googleapis/gax-go/v2 v2.12.2 // indirect
github.com/gorilla/handlers v1.5.1 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/gorilla/websocket v1.5.1 // indirect
Expand Down Expand Up @@ -211,28 +212,26 @@ require (
github.com/zondax/ledger-go v0.14.3 // indirect
go.etcd.io/bbolt v1.3.8 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel/metric v1.25.0 // indirect
go.opentelemetry.io/otel/trace v1.25.0
go.uber.org/goleak v1.3.0 // indirect
go.opentelemetry.io/otel/metric v1.28.0 // indirect
go.opentelemetry.io/otel/trace v1.28.0
Comment on lines +215 to +218
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tip

Codebase Verification

Unused OpenTelemetry Dependencies Found

The following OpenTelemetry dependencies are not used in the codebase and can be removed from go.mod if they are not needed:

  • go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp
  • go.opentelemetry.io/otel/metric

Please review and remove these unnecessary dependencies to keep the go.mod file clean.

Analysis chain

Ensure correct usage of OpenTelemetry instrumentation dependencies.

The OpenTelemetry instrumentation packages (go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc, go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp, go.opentelemetry.io/otel/metric, go.opentelemetry.io/otel/trace) have been added. Verify that these dependencies are correctly used in the code.

Scripts executed

The following scripts were executed for the analysis:

Script:

#!/bin/bash
# Description: Verify that the OpenTelemetry instrumentation dependencies are correctly used in the code.

# Test: Search for the usage of OpenTelemetry instrumentation packages.
rg --type go $'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc|go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp|go.opentelemetry.io/otel/metric|go.opentelemetry.io/otel/trace'

Length of output: 467

Script:

#!/bin/bash
# Description: Verify that the OpenTelemetry instrumentation dependencies are correctly used in the code.

# Search for the usage of the specific OpenTelemetry instrumentation packages that were not found in the initial search.
rg --type go 'go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp|go.opentelemetry.io/otel/metric'

Length of output: 110

go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/crypto v0.24.0 // indirect
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
golang.org/x/net v0.25.0 // indirect
golang.org/x/oauth2 v0.17.0 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/oauth2 v0.20.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/term v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/term v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.21.0 // indirect
google.golang.org/api v0.162.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
google.golang.org/api v0.169.0 // indirect
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect
google.golang.org/protobuf v1.33.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand Down
Loading
Loading