aws s3 exporter initial version

exporter/awss3exporter/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

exporter/awss3exporter/README.md

@@ -0,0 +1,24 @@
+# AWS S3 Exporter for OpenTelemetry Collector
+This exporter converts OpenTelemetry metrics, logs and traces to supported format and upload to S3
+
+## Schema supported
+This exporter targets to support parquet/json format
+
+## Exporter Configuration
+
+The following exporter configuration parameters are supported. 
+
+| Name                   | Description                                                                        | Default |
+| :--------------------- | :--------------------------------------------------------------------------------- | ------- |
+| `region`               | AWS region.                                                                        |         |
+| `s3_bucket`            | S3 bucket                                                                          |         |
+| `s3_prefix`            | prefix for the S3 key.                                                             |         |
+| `s3_partition`         | time granularity of S3 key: hour or minute                                         |"minute" |
+
+## AWS Credential Configuration
+
+This exporter follows default credential resolution for the
+[aws-sdk-go](https://docs.aws.amazon.com/sdk-for-go/api/index.html).
+
+Follow the [guidelines](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html) for the
+credential configuration.

exporter/awss3exporter/config.go

@@ -0,0 +1,48 @@
+// Copyright 2022 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awss3exporter
+
+import (
+	"go.opentelemetry.io/collector/config"
+	"go.uber.org/zap"
+)
+
+// S3UploaderConfig contains aws s3 uploader related config to controls things
+// like bucket, prefix, batching, connections, retries, etc.
+type S3UploaderConfig struct {
+	Region              string `mapstructure:"region"`
+	S3Bucket            string `mapstructure:"s3_bucket"`
+	S3Prefix            string `mapstructure:"s3_prefix"`
+	S3Partition         string `mapstructure:"s3_partition"`
+	FilePrefix          string `mapstructure:"file_prefix"`
+}
+
+// Config contains the main configuration options for the awskinesis exporter
+type Config struct {
+	config.ExporterSettings `mapstructure:",squash"`
+
+	FileFormat string           `mapstructure:"file_format"`
+	S3Uploader S3UploaderConfig `mapstructure:"s3uploader"`
+
+	// MetricDescriptors is the list of override metric descriptors
+	MetricDescriptors []MetricDescriptor `mapstructure:"metric_descriptors"`
+
+	// ResourceToTelemetrySettings is the option for converting resource attrihutes to telemetry attributes.
+	// "Enabled" - A boolean field to enable/disable this option. Default is `false`.
+	// If enabled, all the resource attributes will be converted to metric labels by default.
+	// exporterhelper.ResourceToTelemetrySettings `mapstructure:"resource_to_telemetry_conversion"`
+
+	logger *zap.Logger
+}

exporter/awss3exporter/exporter.go

@@ -0,0 +1,82 @@
+// Copyright 2022 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awss3exporter
+
+import (
+	"context"
+	"errors"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config"
+	"go.opentelemetry.io/collector/consumer"
+	"go.opentelemetry.io/collector/pdata/plog"
+	"go.opentelemetry.io/collector/pdata/pmetric"
+	"go.opentelemetry.io/collector/pdata/ptrace"
+	"go.uber.org/zap"
+)
+
+var logsMarshaler = plog.NewJSONMarshaler()
+var traceMarshaler = ptrace.NewJSONMarshaler()
+
+type S3Exporter struct {
+	config           config.Exporter
+	metricTranslator metricTranslator
+	logger           *zap.Logger
+}
+
+func NewS3Exporter(config config.Exporter,
+	params component.ExporterCreateSettings) (*S3Exporter, error) {
+
+	if config == nil {
+		return nil, errors.New("s3 exporter config is nil")
+	}
+
+	logger := params.Logger
+	expConfig := config.(*Config)
+	expConfig.logger = logger
+
+	expConfig.Validate()
+
+	s3Exporter := &S3Exporter{
+		config:           config,
+		metricTranslator: newMetricTranslator(*expConfig),
+		logger:           logger,
+	}
+	return s3Exporter, nil
+}
+
+func (e *S3Exporter) Capabilities() consumer.Capabilities {
+	return consumer.Capabilities{MutatesData: false}
+}
+
+func (e *S3Exporter) Start(ctx context.Context, host component.Host) error {
+	return nil
+}
+
+func (e *S3Exporter) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error {
+	return nil
+}
+
+func (e *S3Exporter) ConsumeLogs(ctx context.Context, logs plog.Logs) error {
+	return nil
+}
+
+func (e *S3Exporter) ConsumeTraces(ctx context.Context, traces ptrace.Traces) error {
+	return nil
+}
+
+func (e *S3Exporter) Shutdown(context.Context) error {
+	return nil
+}

exporter/awss3exporter/factory.go

@@ -0,0 +1,97 @@
+// Copyright 2022 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awss3exporter
+
+import (
+	"context"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
+)
+
+const (
+	// The value of "type" key in configuration.
+	typeStr       = "awss3"
+)
+
+// NewFactory creates a factory for S3 exporter.
+func NewFactory() component.ExporterFactory {
+	return component.NewExporterFactory(
+		typeStr,
+		createDefaultConfig,
+		component.WithMetricsExporter(createMetricsExporter),
+		component.WithLogsExporter(createLogsExporter),
+		component.WithTracesExporter(createTracesExporter))
+}
+
+func createDefaultConfig() config.Exporter {
+	return &Config{
+		ExporterSettings: config.NewExporterSettings(config.NewComponentID(typeStr)),
+
+		S3Uploader: S3UploaderConfig{
+			Region:              "us-east-1",
+			S3Partition:         "minute",
+		},
+
+		MetricDescriptors: make([]MetricDescriptor, 0),
+		logger:            nil,
+	}
+}
+
+func createMetricsExporter(ctx context.Context,
+	params component.ExporterCreateSettings,
+	config config.Exporter) (component.MetricsExporter, error) {
+
+	s3Exporter, err := NewS3Exporter(config, params)
+	if err != nil {
+		return nil, err
+	}
+
+	return exporterhelper.NewMetricsExporter(
+		config,
+		params,
+		s3Exporter.ConsumeMetrics)
+}
+
+func createLogsExporter(ctx context.Context,
+	params component.ExporterCreateSettings,
+	config config.Exporter) (component.LogsExporter, error) {
+
+	s3Exporter, err := NewS3Exporter(config, params)
+	if err != nil {
+		return nil, err
+	}
+
+	return exporterhelper.NewLogsExporter(
+		config,
+		params,
+		s3Exporter.ConsumeLogs)
+}
+
+func createTracesExporter(ctx context.Context,
+	params component.ExporterCreateSettings,
+	config config.Exporter) (component.TracesExporter, error) {
+
+	s3Exporter, err := NewS3Exporter(config, params)
+	if err != nil {
+		return nil, err
+	}
+
+	return exporterhelper.NewTracesExporter(
+		config,
+		params,
+		s3Exporter.ConsumeTraces)
+}

exporter/awss3exporter/go.mod

@@ -0,0 +1,3 @@
+module github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter
+
+go 1.17