[exporter/datasetexporter] Upgrade to dataset-go v0.19.0 (#33675)

**Description:** This PR is upgrading dataset-go to version v0.19.0 -
https://github.com/scalyr/dataset-go/releases/tag/v0.19.0

This PR is also fixing:
* #33498 - config validation is not called during conversion, therefore
I had to remove tests from logs and trace exporter. These tests are now
moved to the factory.
* #32533 - fixing failing integration tests, that were never updated and
therefore the configuration contained invalid values

**Link to tracking Issue:** #33498, #32533, #33675

**Testing:** Unit tests and integration tests.

**Documentation:** <Describe the documentation added.>

fixes #32533

---------

Signed-off-by: Alex Boten <[email protected]>
Co-authored-by: Curtis Robert <[email protected]>
Co-authored-by: Dmitrii Anoshin <[email protected]>
Co-authored-by: Ishan Shanware <[email protected]>
Co-authored-by: Evan Bradley <[email protected]>
Co-authored-by: Tyler Helmuth <[email protected]>
Co-authored-by: Alex Boten <[email protected]>
Co-authored-by: Paulo Janotti <[email protected]>
Co-authored-by: Joshua MacDonald <[email protected]>
Co-authored-by: Rong Hu <[email protected]>

.chloggen/datasetexporter-update-to-0.19.0.yaml

@@ -0,0 +1,30 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: 'bug_fix'
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datasetexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Upgrade dataset-go to v0.19.0 and fix found issues
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [33498, 32533, 33675]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Upgrade `dataset-go` library from v0.18.0 to v0.19.0.
+  Enable skipped integration test and adjust the test so it is passing again.
+  Do not validate configuration, let the framework run the validation.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

cmd/otelcontribcol/go.mod

@@ -691,7 +691,7 @@ require (
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/samber/lo v1.38.1 // indirect
 	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25 // indirect
-	github.com/scalyr/dataset-go v0.18.0 // indirect
+	github.com/scalyr/dataset-go v0.19.0 // indirect
 	github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect
 	github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect

exporter/datasetexporter/config.go

@@ -202,39 +202,33 @@ func (c *Config) String() string {
 	return s
 }
 
-func (c *Config) convert() (*ExporterConfig, error) {
-	err := c.Validate()
-	if err != nil {
-		return nil, fmt.Errorf("config is not valid: %w", err)
-	}
-
+func (c *Config) convert() *ExporterConfig {
 	return &ExporterConfig{
-			datasetConfig: &datasetConfig.DataSetConfig{
-				Endpoint: c.DatasetURL,
-				Tokens:   datasetConfig.DataSetTokens{WriteLog: string(c.APIKey)},
-				BufferSettings: buffer_config.DataSetBufferSettings{
-					MaxLifetime:              c.BufferSettings.MaxLifetime,
-					PurgeOlderThan:           c.BufferSettings.PurgeOlderThan,
-					MaxSize:                  buffer.LimitBufferSize,
-					GroupBy:                  c.BufferSettings.GroupBy,
-					RetryInitialInterval:     c.BufferSettings.RetryInitialInterval,
-					RetryMaxInterval:         c.BufferSettings.RetryMaxInterval,
-					RetryMaxElapsedTime:      c.BufferSettings.RetryMaxElapsedTime,
-					RetryMultiplier:          backoff.DefaultMultiplier,
-					RetryRandomizationFactor: backoff.DefaultRandomizationFactor,
-					RetryShutdownTimeout:     c.BufferSettings.RetryShutdownTimeout,
-				},
-				ServerHostSettings: server_host_config.DataSetServerHostSettings{
-					UseHostName: c.ServerHostSettings.UseHostName,
-					ServerHost:  c.ServerHostSettings.ServerHost,
-				},
-				Debug: c.Debug,
+		datasetConfig: &datasetConfig.DataSetConfig{
+			Endpoint: c.DatasetURL,
+			Tokens:   datasetConfig.DataSetTokens{WriteLog: string(c.APIKey)},
+			BufferSettings: buffer_config.DataSetBufferSettings{
+				MaxLifetime:              c.BufferSettings.MaxLifetime,
+				PurgeOlderThan:           c.BufferSettings.PurgeOlderThan,
+				MaxSize:                  buffer.LimitBufferSize,
+				GroupBy:                  c.BufferSettings.GroupBy,
+				RetryInitialInterval:     c.BufferSettings.RetryInitialInterval,
+				RetryMaxInterval:         c.BufferSettings.RetryMaxInterval,
+				RetryMaxElapsedTime:      c.BufferSettings.RetryMaxElapsedTime,
+				RetryMultiplier:          backoff.DefaultMultiplier,
+				RetryRandomizationFactor: backoff.DefaultRandomizationFactor,
+				RetryShutdownTimeout:     c.BufferSettings.RetryShutdownTimeout,
 			},
-			tracesSettings:     c.TracesSettings,
-			logsSettings:       c.LogsSettings,
-			serverHostSettings: c.ServerHostSettings,
+			ServerHostSettings: server_host_config.DataSetServerHostSettings{
+				UseHostName: c.ServerHostSettings.UseHostName,
+				ServerHost:  c.ServerHostSettings.ServerHost,
+			},
+			Debug: c.Debug,
 		},
-		nil
+		tracesSettings:     c.TracesSettings,
+		logsSettings:       c.LogsSettings,
+		serverHostSettings: c.ServerHostSettings,
+	}
 }
 
 type ExporterConfig struct {

exporter/datasetexporter/datasetexporter.go

@@ -36,13 +36,7 @@ func newDatasetExporter(entity string, config *Config, set exporter.Settings) (*
 		zap.String("id.string", set.ID.String()),
 		zap.String("id.name", set.ID.Name()),
 	)
-	exporterCfg, err := config.convert()
-	if err != nil {
-		return nil, fmt.Errorf(
-			"cannot convert config: %s; %w",
-			config.String(), err,
-		)
-	}
+	exporterCfg := config.convert()
 	userAgent := fmt.Sprintf(
 		"%s;%s;%s",
 		"OtelCollector",

exporter/datasetexporter/factory_test.go

@@ -153,6 +153,17 @@ func TestLoadConfig(t *testing.T) {
 	}
 }
 
+func TestValidateConfigs(t *testing.T) {
+	tests := createExporterTests()
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(*testing.T) {
+			err := component.ValidateConfig(tt.config)
+			assert.Equal(t, tt.expectedError, err)
+		})
+	}
+}
+
 type CreateTest struct {
 	name          string
 	config        component.Config
@@ -169,7 +180,17 @@ func createExporterTests() []CreateTest {
 		{
 			name:          "broken",
 			config:        &Config{},
-			expectedError: fmt.Errorf("cannot get DataSetExporter: cannot convert config: DatasetURL: ; APIKey: [REDACTED] (0); Debug: false; BufferSettings: {MaxLifetime:0s PurgeOlderThan:0s GroupBy:[] RetryInitialInterval:0s RetryMaxInterval:0s RetryMaxElapsedTime:0s RetryShutdownTimeout:0s}; LogsSettings: {ExportResourceInfo:false ExportResourcePrefix: ExportScopeInfo:false ExportScopePrefix: DecomposeComplexMessageField:false DecomposedComplexMessagePrefix: exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; TracesSettings: {exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; ServerHostSettings: {UseHostName:false ServerHost:}; BackOffConfig: {Enabled:false InitialInterval:0s RandomizationFactor:0 Multiplier:0 MaxInterval:0s MaxElapsedTime:0s}; QueueSettings: {Enabled:false NumConsumers:0 QueueSize:0 StorageID:<nil>}; TimeoutSettings: {Timeout:0s}; config is not valid: api_key is required"),
+			expectedError: fmt.Errorf("api_key is required"),
+		},
+		{
+			name:          "missing-url",
+			config:        &Config{APIKey: "AAA"},
+			expectedError: fmt.Errorf("dataset_url is required"),
+		},
+		{
+			name:          "missing-key",
+			config:        &Config{DatasetURL: "bbb"},
+			expectedError: fmt.Errorf("api_key is required"),
 		},
 		{
 			name: "valid",

exporter/datasetexporter/go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/google/uuid v1.6.0
 	// github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage v0.103.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.103.0
-	github.com/scalyr/dataset-go v0.18.0
+	github.com/scalyr/dataset-go v0.19.0
 	github.com/stretchr/testify v1.9.0
 	go.opentelemetry.io/collector/component v0.103.0
 	go.opentelemetry.io/collector/confmap v0.103.0

exporter/datasetexporter/logs_exporter_stress_test.go

@@ -28,8 +28,6 @@ import (
 )
 
 func TestConsumeLogsManyLogsShouldSucceed(t *testing.T) {
-	t.Skip("TODO: Skipping due to https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/32533")
-
 	const maxDelay = 200 * time.Millisecond
 	createSettings := exportertest.NewNopSettings()
 
@@ -52,7 +50,7 @@ func TestConsumeLogsManyLogsShouldSucceed(t *testing.T) {
 
 		for _, ev := range cer.Events {
 			processedEvents.Add(1)
-			key, found := ev.Attrs["body.str"]
+			key, found := ev.Attrs["key"]
 			assert.True(t, found)
 			mutex.Lock()
 			sKey := key.(string)
@@ -82,11 +80,18 @@ func TestConsumeLogsManyLogsShouldSucceed(t *testing.T) {
 		BufferSettings: BufferSettings{
 			MaxLifetime:          maxDelay,
 			GroupBy:              []string{"attributes.container_id"},
+			RetryInitialInterval: maxDelay,
+			RetryMaxInterval:     10 * maxDelay,
+			RetryMaxElapsedTime:  50 * maxDelay,
 			RetryShutdownTimeout: time.Minute,
+			PurgeOlderThan:       100 * maxDelay,
 		},
 		BackOffConfig:   configretry.NewDefaultBackOffConfig(),
 		QueueSettings:   exporterhelper.NewDefaultQueueSettings(),
 		TimeoutSettings: exporterhelper.NewDefaultTimeoutSettings(),
+		ServerHostSettings: ServerHostSettings{
+			UseHostName: true,
+		},
 	}
 
 	logs, err := createLogsExporter(context.Background(), createSettings, config)

exporter/datasetexporter/logs_exporter_test.go

@@ -31,30 +31,6 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/testdata"
 )
 
-func TestCreateLogsExporter(t *testing.T) {
-	ctx := context.Background()
-	createSettings := exportertest.NewNopSettings()
-	tests := createExporterTests()
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(*testing.T) {
-			logs, err := createLogsExporter(ctx, createSettings, tt.config)
-
-			if err == nil {
-				assert.Nil(t, tt.expectedError, tt.name)
-				assert.NotNil(t, logs, tt.name)
-			} else {
-				if tt.expectedError == nil {
-					assert.Nil(t, err, tt.name)
-				} else {
-					assert.Equal(t, tt.expectedError.Error(), err.Error(), tt.name)
-					assert.Nil(t, logs, tt.name)
-				}
-			}
-		})
-	}
-}
-
 func TestBuildBody(t *testing.T) {
 	slice := pcommon.NewValueSlice()
 	err := slice.FromRaw([]any{1, 2, 3})

exporter/datasetexporter/traces_exporter_test.go

@@ -4,41 +4,19 @@
 package datasetexporter
 
 import (
-	"context"
 	"fmt"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/scalyr/dataset-go/pkg/api/add_events"
 	"github.com/stretchr/testify/assert"
-	"go.opentelemetry.io/collector/exporter/exportertest"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/ptrace"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/testdata"
 )
 
-func TestCreateTracesExporter(t *testing.T) {
-	ctx := context.Background()
-	createSettings := exportertest.NewNopSettings()
-	tests := createExporterTests()
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(*testing.T) {
-			logs, err := createTracesExporter(ctx, createSettings, tt.config)
-
-			if err == nil {
-				assert.Nil(t, tt.expectedError)
-				assert.NotNil(t, logs)
-			} else {
-				assert.Equal(t, tt.expectedError.Error(), err.Error())
-				assert.Nil(t, logs)
-			}
-		})
-	}
-}
-
 func generateTEvent1Raw() *add_events.Event {
 	return &add_events.Event{
 		Thread:     "TT",