[exporter/datadog] Respect confighttp configs and use better defaults

.chloggen/ddog-config-change.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datadogexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Remove config structs `LimitedClientConfig` and `LimitedTLSClientSettings`"
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [31733]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: This should have no impact to end users as long as they do not import Datadog exporter config structs in their source code.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [api]

.chloggen/ddog-http-defaults.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datadogexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Datadog exporter uses the same default HTTP settings as Datadog Agent HTTP transport
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [31733]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

.chloggen/ddog-respect-confighttp.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datadogexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Datadog exporter respects a subset of settings in confighttp client configs
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [31733]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: "Currently the following configs are respected: `read_buffer_size`, `write_buffer_size`, `timeout`, `max_idle_conns`, `max_idle_conns_per_host`, `max_conns_per_host`, `idle_conn_timeout`, `disable_keep_alives` and `tls.insecure_skip_verify`."
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

exporter/datadogexporter/config.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config/confighttp"
 	"go.opentelemetry.io/collector/config/confignet"
 	"go.opentelemetry.io/collector/config/configopaque"
 	"go.opentelemetry.io/collector/config/configretry"
@@ -390,24 +391,11 @@ type HostMetadataConfig struct {
 	Tags []string `mapstructure:"tags"`
 }
 
-// LimitedTLSClientSetting is a subset of TLSClientSetting, see LimitedClientConfig for more details
-type LimitedTLSClientSettings struct {
-	// InsecureSkipVerify controls whether a client verifies the server's
-	// certificate chain and host name.
-	InsecureSkipVerify bool `mapstructure:"insecure_skip_verify"`
-}
-
-type LimitedClientConfig struct {
-	TLSSetting LimitedTLSClientSettings `mapstructure:"tls,omitempty"`
-}
-
 // Config defines configuration for the Datadog exporter.
 type Config struct {
-	exporterhelper.TimeoutSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
-	exporterhelper.QueueSettings   `mapstructure:"sending_queue"`
-	configretry.BackOffConfig      `mapstructure:"retry_on_failure"`
-
-	LimitedClientConfig `mapstructure:",squash"`
+	confighttp.ClientConfig      `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
+	exporterhelper.QueueSettings `mapstructure:"sending_queue"`
+	configretry.BackOffConfig    `mapstructure:"retry_on_failure"`
 
 	TagsConfig `mapstructure:",squash"`
 

exporter/datadogexporter/config_test.go

@@ -5,12 +5,19 @@ package datadogexporter
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/collector/config/confighttp"
+	"go.opentelemetry.io/collector/config/configtls"
 	"go.opentelemetry.io/collector/confmap"
 )
 
 func TestValidate(t *testing.T) {
+	idleConnTimeout := 30 * time.Second
+	maxIdleConn := 300
+	maxIdleConnPerHost := 150
+	maxConnPerHost := 250
 
 	tests := []struct {
 		name string
@@ -94,8 +101,8 @@ func TestValidate(t *testing.T) {
 			name: "TLS settings are valid",
 			cfg: &Config{
 				API: APIConfig{Key: "notnull"},
-				LimitedClientConfig: LimitedClientConfig{
-					TLSSetting: LimitedTLSClientSettings{
+				ClientConfig: confighttp.ClientConfig{
+					TLSSetting: configtls.ClientConfig{
 						InsecureSkipVerify: true,
 					},
 				},
@@ -115,6 +122,23 @@ func TestValidate(t *testing.T) {
 				Traces: TracesConfig{PeerTags: []string{"tag1", "tag2"}},
 			},
 		},
+		{
+			name: "With confighttp client configs",
+			cfg: &Config{
+				API: APIConfig{Key: "notnull"},
+				ClientConfig: confighttp.ClientConfig{
+					ReadBufferSize:      100,
+					WriteBufferSize:     200,
+					Timeout:             10 * time.Second,
+					IdleConnTimeout:     &idleConnTimeout,
+					MaxIdleConns:        &maxIdleConn,
+					MaxIdleConnsPerHost: &maxIdleConnPerHost,
+					MaxConnsPerHost:     &maxConnPerHost,
+					DisableKeepAlives:   true,
+					TLSSetting:          configtls.ClientConfig{InsecureSkipVerify: true},
+				},
+			},
+		},
 	}
 	for _, testInstance := range tests {
 		t.Run(testInstance.name, func(t *testing.T) {
@@ -129,10 +153,26 @@ func TestValidate(t *testing.T) {
 }
 
 func TestUnmarshal(t *testing.T) {
+	cfgWithHTTPConfigs := NewFactory().CreateDefaultConfig().(*Config)
+	idleConnTimeout := 30 * time.Second
+	maxIdleConn := 300
+	maxIdleConnPerHost := 150
+	maxConnPerHost := 250
+	cfgWithHTTPConfigs.ReadBufferSize = 100
+	cfgWithHTTPConfigs.WriteBufferSize = 200
+	cfgWithHTTPConfigs.Timeout = 10 * time.Second
+	cfgWithHTTPConfigs.MaxIdleConns = &maxIdleConn
+	cfgWithHTTPConfigs.MaxIdleConnsPerHost = &maxIdleConnPerHost
+	cfgWithHTTPConfigs.MaxConnsPerHost = &maxConnPerHost
+	cfgWithHTTPConfigs.IdleConnTimeout = &idleConnTimeout
+	cfgWithHTTPConfigs.DisableKeepAlives = true
+	cfgWithHTTPConfigs.TLSSetting.InsecureSkipVerify = true
+	cfgWithHTTPConfigs.warnings = nil
+
 	tests := []struct {
 		name      string
 		configMap *confmap.Conf
-		cfg       Config
+		cfg       *Config
 		err       string
 	}{
 		{
@@ -264,6 +304,21 @@ func TestUnmarshal(t *testing.T) {
 			}),
 			err: "\"metrics::sums::initial_cumulative_monotonic_value\" can only be configured when \"metrics::sums::cumulative_monotonic_mode\" is set to \"to_delta\"",
 		},
+		{
+			name: "unmarshall confighttp client configs",
+			configMap: confmap.NewFromStringMap(map[string]any{
+				"read_buffer_size":        100,
+				"write_buffer_size":       200,
+				"timeout":                 "10s",
+				"max_idle_conns":          300,
+				"max_idle_conns_per_host": 150,
+				"max_conns_per_host":      250,
+				"disable_keep_alives":     true,
+				"idle_conn_timeout":       "30s",
+				"tls":                     map[string]any{"insecure_skip_verify": true},
+			}),
+			cfg: cfgWithHTTPConfigs,
+		},
 	}
 
 	f := NewFactory()

exporter/datadogexporter/examples/collector.yaml

@@ -214,6 +214,27 @@ exporters:
       ## @param tls - boolean - optional - default: false
       # insecure_skip_verify: false
 
+    ## @param read_buffer_size - integer - optional
+    ## @param write_buffer_size - integer - optional
+    ## @param timeout - duration - optional
+    ## @param max_idle_conns - integer - optional
+    ## @param max_idle_conns_per_host - integer - optional
+    ## @param max_conns_per_host - integer - optional
+    ## @param idle_conn_timeout - duration - optional
+    ## @param disable_keep_alives - boolean - optional
+    ##
+    ## Below are a subset of configs in confighttp that are respected by Datadog exporter.
+    ## See https://pkg.go.dev/go.opentelemetry.io/collector/config/confighttp for details on these configs.
+    ##
+    # read_buffer_size: 0
+    # write_buffer_size: 0
+    # timeout: 15s
+    # max_idle_conns: 100
+    # max_idle_conns_per_host: 0
+    # max_conns_per_host: 0
+    # idle_conn_timeout: 0s
+    # disable_keep_alives: false
+
     ## @param metrics - custom object - optional
     ## Metric exporter specific configuration.
     #

exporter/datadogexporter/examples/logs.yaml

@@ -23,6 +23,7 @@ processors:
 
 exporters:
   datadog:
+    idle_conn_timeout: 10s
     api:
       site: ${env:DD_SITE}
       key: ${env:DD_API_KEY}

exporter/datadogexporter/factory.go

@@ -19,6 +19,7 @@ import (
 	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes"
 	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/source"
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config/confighttp"
 	"go.opentelemetry.io/collector/config/confignet"
 	"go.opentelemetry.io/collector/config/configretry"
 	"go.opentelemetry.io/collector/consumer"
@@ -170,18 +171,19 @@ func NewFactory() exporter.Factory {
 	return newFactoryWithRegistry(featuregate.GlobalRegistry())
 }
 
-func defaulttimeoutSettings() exporterhelper.TimeoutSettings {
-	return exporterhelper.TimeoutSettings{
+func defaultClientConfig() confighttp.ClientConfig {
+	// do not use NewDefaultClientConfig for backwards-compatibility
+	return confighttp.ClientConfig{
 		Timeout: 15 * time.Second,
 	}
 }
 
 // createDefaultConfig creates the default exporter configuration
 func (f *factory) createDefaultConfig() component.Config {
 	return &Config{
-		TimeoutSettings: defaulttimeoutSettings(),
-		BackOffConfig:   configretry.NewDefaultBackOffConfig(),
-		QueueSettings:   exporterhelper.NewDefaultQueueSettings(),
+		ClientConfig:  defaultClientConfig(),
+		BackOffConfig: configretry.NewDefaultBackOffConfig(),
+		QueueSettings: exporterhelper.NewDefaultQueueSettings(),
 
 		API: APIConfig{
 			Site: "datadoghq.com",

exporter/datadogexporter/factory_test.go

@@ -71,9 +71,9 @@ func TestCreateDefaultConfig(t *testing.T) {
 	cfg := factory.CreateDefaultConfig()
 
 	assert.Equal(t, &Config{
-		TimeoutSettings: defaulttimeoutSettings(),
-		BackOffConfig:   configretry.NewDefaultBackOffConfig(),
-		QueueSettings:   exporterhelper.NewDefaultQueueSettings(),
+		ClientConfig:  defaultClientConfig(),
+		BackOffConfig: configretry.NewDefaultBackOffConfig(),
+		QueueSettings: exporterhelper.NewDefaultQueueSettings(),
 
 		API: APIConfig{
 			Site: "datadoghq.com",
@@ -132,9 +132,9 @@ func TestLoadConfig(t *testing.T) {
 		{
 			id: component.NewIDWithName(metadata.Type, "default"),
 			expected: &Config{
-				TimeoutSettings: defaulttimeoutSettings(),
-				BackOffConfig:   configretry.NewDefaultBackOffConfig(),
-				QueueSettings:   exporterhelper.NewDefaultQueueSettings(),
+				ClientConfig:  defaultClientConfig(),
+				BackOffConfig: configretry.NewDefaultBackOffConfig(),
+				QueueSettings: exporterhelper.NewDefaultQueueSettings(),
 				API: APIConfig{
 					Key:              "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 					Site:             "datadoghq.com",
@@ -180,9 +180,9 @@ func TestLoadConfig(t *testing.T) {
 		{
 			id: component.NewIDWithName(metadata.Type, "api"),
 			expected: &Config{
-				TimeoutSettings: defaulttimeoutSettings(),
-				BackOffConfig:   configretry.NewDefaultBackOffConfig(),
-				QueueSettings:   exporterhelper.NewDefaultQueueSettings(),
+				ClientConfig:  defaultClientConfig(),
+				BackOffConfig: configretry.NewDefaultBackOffConfig(),
+				QueueSettings: exporterhelper.NewDefaultQueueSettings(),
 				TagsConfig: TagsConfig{
 					Hostname: "customhostname",
 				},
@@ -235,9 +235,9 @@ func TestLoadConfig(t *testing.T) {
 		{
 			id: component.NewIDWithName(metadata.Type, "api2"),
 			expected: &Config{
-				TimeoutSettings: defaulttimeoutSettings(),
-				BackOffConfig:   configretry.NewDefaultBackOffConfig(),
-				QueueSettings:   exporterhelper.NewDefaultQueueSettings(),
+				ClientConfig:  defaultClientConfig(),
+				BackOffConfig: configretry.NewDefaultBackOffConfig(),
+				QueueSettings: exporterhelper.NewDefaultQueueSettings(),
 				TagsConfig: TagsConfig{
 					Hostname: "customhostname",
 				},
@@ -620,9 +620,9 @@ func TestOnlyMetadata(t *testing.T) {
 	factory := NewFactory()
 	ctx := context.Background()
 	cfg := &Config{
-		TimeoutSettings: defaulttimeoutSettings(),
-		BackOffConfig:   configretry.NewDefaultBackOffConfig(),
-		QueueSettings:   exporterhelper.NewDefaultQueueSettings(),
+		ClientConfig:  defaultClientConfig(),
+		BackOffConfig: configretry.NewDefaultBackOffConfig(),
+		QueueSettings: exporterhelper.NewDefaultQueueSettings(),
 
 		API:          APIConfig{Key: "notnull"},
 		Metrics:      MetricsConfig{TCPAddrConfig: confignet.TCPAddrConfig{Endpoint: server.URL}},

exporter/datadogexporter/go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.21.0
 	github.com/aws/aws-sdk-go v1.50.27
 	github.com/cenkalti/backoff/v4 v4.2.1
+	github.com/google/go-cmp v0.6.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector v0.96.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/ecsutil v0.96.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.96.0
@@ -35,9 +36,12 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.96.0
 	github.com/stretchr/testify v1.9.0
 	go.opentelemetry.io/collector/component v0.96.1-0.20240306115632-b2693620eff6
+	go.opentelemetry.io/collector/config/configcompression v0.96.1-0.20240306115632-b2693620eff6
+	go.opentelemetry.io/collector/config/confighttp v0.96.1-0.20240306115632-b2693620eff6
 	go.opentelemetry.io/collector/config/confignet v0.96.1-0.20240306115632-b2693620eff6
 	go.opentelemetry.io/collector/config/configopaque v1.3.1-0.20240306115632-b2693620eff6
 	go.opentelemetry.io/collector/config/configretry v0.96.1-0.20240306115632-b2693620eff6
+	go.opentelemetry.io/collector/config/configtls v0.96.1-0.20240306115632-b2693620eff6
 	go.opentelemetry.io/collector/confmap v0.96.1-0.20240306115632-b2693620eff6
 	go.opentelemetry.io/collector/connector v0.96.1-0.20240306115632-b2693620eff6
 	go.opentelemetry.io/collector/consumer v0.96.1-0.20240306115632-b2693620eff6
@@ -129,7 +133,6 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
@@ -233,11 +236,8 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/collector v0.96.1-0.20240306115632-b2693620eff6 // indirect
 	go.opentelemetry.io/collector/config/configauth v0.96.1-0.20240306115632-b2693620eff6 // indirect
-	go.opentelemetry.io/collector/config/configcompression v0.96.1-0.20240306115632-b2693620eff6 // indirect
 	go.opentelemetry.io/collector/config/configgrpc v0.96.1-0.20240306115632-b2693620eff6 // indirect
-	go.opentelemetry.io/collector/config/confighttp v0.96.1-0.20240306115632-b2693620eff6 // indirect
 	go.opentelemetry.io/collector/config/configtelemetry v0.96.1-0.20240306115632-b2693620eff6 // indirect
-	go.opentelemetry.io/collector/config/configtls v0.96.1-0.20240306115632-b2693620eff6 // indirect
 	go.opentelemetry.io/collector/config/internal v0.96.1-0.20240306115632-b2693620eff6 // indirect
 	go.opentelemetry.io/collector/confmap/converter/expandconverter v0.96.1-0.20240306115632-b2693620eff6 // indirect
 	go.opentelemetry.io/collector/confmap/provider/envprovider v0.96.1-0.20240306115632-b2693620eff6 // indirect