[chore] [exporter/s3exporter] aws s3 exporter initial version (First …

…PR, config + exporter stub) (#9979)

* aws s3 exporter initial version

* fixup! Prerelease collector contrib v0.51.0 (#9975)

* [chore] update changelog.

* update dependabot.yml

* interfaces compliance

* fixing go.mod

* update dependabot.yml

* fixing issue templates

* update CODEOWNERS

* update go.mod

* update CHANGELOG

* fixing CHANGELOG.md

* Minor change to changelog

---------

Co-authored-by: Antoine Toulme <[email protected]>
Co-authored-by: Pablo Baeyens <[email protected]>

.chloggen/s3exporter-1.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: new_component
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: awss3exporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add new AWS S3 exporter.
+
+# One or more tracking issues related to the change
+issues: [9979]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

.github/CODEOWNERS

@@ -38,6 +38,7 @@ exporter/alibabacloudlogserviceexporter/                 @open-telemetry/collect
 exporter/awscloudwatchlogsexporter/                      @open-telemetry/collector-contrib-approvers @boostchicken
 exporter/awsemfexporter/                                 @open-telemetry/collector-contrib-approvers @Aneurysm9 @shaochengwang @mxiamxia
 exporter/awskinesisexporter/                             @open-telemetry/collector-contrib-approvers @Aneurysm9 @MovieStoreGuy
+exporter/awss3exporter/                                  @open-telemetry/collector-contrib-approvers @atoulme @pdelewski
 exporter/awsxrayexporter/                                @open-telemetry/collector-contrib-approvers @willarmiros
 exporter/azuremonitorexporter/                           @open-telemetry/collector-contrib-approvers @pcwiese
 exporter/azuredataexplorerexporter/                      @open-telemetry/collector-contrib-approvers @asaharan @ag-ramachandran

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -32,6 +32,7 @@ body:
       - exporter/awscloudwatchlogs
       - exporter/awsemf
       - exporter/awskinesis
+      - exporter/awss3
       - exporter/awsxray
       - exporter/azuredataexplorer
       - exporter/azuremonitor

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -26,6 +26,7 @@ body:
       - exporter/awscloudwatchlogs
       - exporter/awsemf
       - exporter/awskinesis
+      - exporter/awss3
       - exporter/awsxray
       - exporter/azuredataexplorer
       - exporter/azuremonitor

.github/ISSUE_TEMPLATE/other.yaml

@@ -26,6 +26,7 @@ body:
       - exporter/awscloudwatchlogs
       - exporter/awsemf
       - exporter/awskinesis
+      - exporter/awss3
       - exporter/awsxray
       - exporter/azuredataexplorer
       - exporter/azuremonitor

.github/dependabot.yml

@@ -92,6 +92,11 @@ updates:
     schedule:
       interval: "weekly"
       day: "wednesday"
+  - package-ecosystem: "gomod"
+    directory: "/exporter/awss3exporter"
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
   - package-ecosystem: "gomod"
     directory: "/exporter/awsxrayexporter"
     schedule:
@@ -1097,8 +1102,3 @@ updates:
     schedule:
       interval: "weekly"
       day: "wednesday"
-  - package-ecosystem: "gomod"
-    directory: "/receiver/zookeeperreceiver"
-    schedule:
-      interval: "weekly"
-      day: "wednesday"

exporter/awss3exporter/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

exporter/awss3exporter/README.md

@@ -0,0 +1,51 @@
+# AWS S3 Exporter for OpenTelemetry Collector
+
+| Status                   |                       |
+| ------------------------ |-----------------------|
+| Stability                | [in development]      |
+| Supported pipeline types | traces, logs          |
+| Distributions            | [contrib]             |
+
+## Schema supported
+This exporter targets to support proto/json and proto/binary format
+
+## Exporter Configuration
+
+The following exporter configuration parameters are supported. 
+
+| Name                   | Description                                                                        | Default |
+| :--------------------- | :--------------------------------------------------------------------------------- | ------- |
+| `region`               | AWS region.                                                                        |         |
+| `s3_bucket`            | S3 bucket                                                                          |         |
+| `s3_prefix`            | prefix for the S3 key (root directory inside bucket).                              |         |
+| `s3_partition`         | time granularity of S3 key: hour or minute                                         |"minute" |
+| `file_prefix`          | file prefix defined by user                                                        |         |
+| `marshaler_name`       | marshaler used to produce output data otlp_json or otlp_proto                      |         |
+
+# Example Configuration
+
+Following example configuration defines to store output in 'eu-central' region and bucket named 'databucket'.
+
+```yaml
+exporters:
+  awss3:
+    s3uploader:
+        region: 'eu-central-1'
+        s3_bucket: 'databucket'
+        s3_prefix: 'metric'
+        s3_partition: 'minute'
+```
+
+Logs and traces will be stored inside 'databucket' in the following path format.
+
+```console
+metric/year=XXXX/month=XX/day=XX/hour=XX/minute=XX
+```
+
+## AWS Credential Configuration
+
+This exporter follows default credential resolution for the
+[aws-sdk-go](https://docs.aws.amazon.com/sdk-for-go/api/index.html).
+
+Follow the [guidelines](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html) for the
+credential configuration.

exporter/awss3exporter/config.go

@@ -0,0 +1,42 @@
+// Copyright 2022 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awss3exporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter"
+
+import (
+	"go.uber.org/zap"
+)
+
+// S3UploaderConfig contains aws s3 uploader related config to controls things
+// like bucket, prefix, batching, connections, retries, etc.
+type S3UploaderConfig struct {
+	Region      string `mapstructure:"region"`
+	S3Bucket    string `mapstructure:"s3_bucket"`
+	S3Prefix    string `mapstructure:"s3_prefix"`
+	S3Partition string `mapstructure:"s3_partition"`
+	FilePrefix  string `mapstructure:"file_prefix"`
+}
+
+// Config contains the main configuration options for the awskinesis exporter
+type Config struct {
+	S3Uploader    S3UploaderConfig `mapstructure:"s3uploader"`
+	MarshalerName string           `mapstructure:"marshaler_name"`
+
+	// ResourceToTelemetrySettings is the option for converting resource attrihutes to telemetry attributes.
+	// "Enabled" - A boolean field to enable/disable this option. Default is `false`.
+	// If enabled, all the resource attributes will be converted to metric labels by default.
+	// exporterhelper.ResourceToTelemetrySettings `mapstructure:"resource_to_telemetry_conversion"`
+
+	logger *zap.Logger
+}

exporter/awss3exporter/config_test.go

@@ -0,0 +1,75 @@
+// Copyright 2021 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      shttp://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awss3exporter
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/otelcol/otelcoltest"
+)
+
+func TestLoadConfig(t *testing.T) {
+	factories, err := otelcoltest.NopFactories()
+	assert.NoError(t, err)
+
+	factory := NewFactory()
+	factories.Exporters[typeStr] = factory
+	cfg, err := otelcoltest.LoadConfigAndValidate(filepath.Join("testdata", "default.yaml"), factories)
+
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+
+	e := cfg.Exporters[component.NewID(typeStr)].(*Config)
+	assert.Equal(t, e,
+		&Config{
+			S3Uploader: S3UploaderConfig{
+				Region:      "us-east-1",
+				S3Partition: "minute",
+			},
+			MarshalerName: "otlp_json",
+		},
+	)
+}
+
+func TestConfig(t *testing.T) {
+	factories, err := otelcoltest.NopFactories()
+	assert.Nil(t, err)
+
+	factory := NewFactory()
+	factories.Exporters[factory.Type()] = factory
+	cfg, err := otelcoltest.LoadConfigAndValidate(
+		filepath.Join("testdata", "config.yaml"), factories)
+
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+
+	e := cfg.Exporters[component.NewID(typeStr)].(*Config)
+
+	assert.Equal(t, e,
+		&Config{
+			S3Uploader: S3UploaderConfig{
+				Region:      "us-east-1",
+				S3Bucket:    "foo",
+				S3Prefix:    "bar",
+				S3Partition: "minute",
+			},
+			MarshalerName: "otlp_json",
+		},
+	)
+}

exporter/awss3exporter/data_writer.go

@@ -0,0 +1,21 @@
+// Copyright 2022 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awss3exporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter"
+
+import "context"
+
+type DataWriter interface {
+	WriteBuffer(ctx context.Context, buf []byte, config *Config, metadata string, format string) error
+}

exporter/awss3exporter/exporter.go

@@ -0,0 +1,82 @@
+// Copyright 2022 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awss3exporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter"
+
+import (
+	"context"
+	"errors"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/consumer"
+	"go.opentelemetry.io/collector/exporter"
+	"go.opentelemetry.io/collector/pdata/plog"
+	"go.opentelemetry.io/collector/pdata/pmetric"
+	"go.opentelemetry.io/collector/pdata/ptrace"
+	"go.uber.org/zap"
+)
+
+type S3Exporter struct {
+	dataWriter DataWriter
+	logger     *zap.Logger
+	marshaler  Marshaler
+}
+
+func NewS3Exporter(config *Config,
+	params exporter.CreateSettings) (*S3Exporter, error) {
+
+	if config == nil {
+		return nil, errors.New("s3 exporter config is nil")
+	}
+
+	logger := params.Logger
+	expConfig := config
+	expConfig.logger = logger
+
+	marshaler, err := NewMarshaler(expConfig.MarshalerName, logger)
+	if err != nil {
+		return nil, errors.New("unknown marshaler")
+	}
+
+	s3Exporter := &S3Exporter{
+		dataWriter: &S3Writer{},
+		logger:     logger,
+		marshaler:  marshaler,
+	}
+	return s3Exporter, nil
+}
+
+func (e *S3Exporter) Capabilities() consumer.Capabilities {
+	return consumer.Capabilities{MutatesData: false}
+}
+
+func (e *S3Exporter) Start(ctx context.Context, host component.Host) error {
+	return nil
+}
+
+func (e *S3Exporter) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error {
+	return nil
+}
+
+func (e *S3Exporter) ConsumeLogs(ctx context.Context, logs plog.Logs) error {
+	return nil
+}
+
+func (e *S3Exporter) ConsumeTraces(ctx context.Context, traces ptrace.Traces) error {
+	return nil
+}
+
+func (e *S3Exporter) Shutdown(context.Context) error {
+	return nil
+}