[exporter/awss3] Make file format suffix configurable when using enco…

…ding (#31833)

**Description:** Modify the behaviour of the exporter when using an
encoding extension to add a user defined file extension or no extension
at all if not configured.

**Link to tracking Issue:** #31818

**Testing:** Unit tests

**Documentation:** Added details of new configuration option.

.chloggen/awss3_encoding_file_extension.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: awss3exporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Add support for specifying the file extension for files uploaded to S3 when using an encoding extension."
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [31818]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/awss3exporter/README.md

@@ -32,6 +32,7 @@ The following exporter configuration parameters are supported.
 | `file_prefix`         | file prefix defined by user                                                                                                                |             |
 | `marshaler`           | marshaler used to produce output data                                                                                                      | `otlp_json` |
 | `encoding`            | Encoding extension to use to marshal data. Overrides the `marshaler` configuration option if set.                                          |             |
+| `encoding_file_extension` | file format extension suffix when using the `encoding` configuration option. May be left empty for no suffix to be appended.               |             |
 | `endpoint`            | overrides the endpoint used by the exporter instead of constructing it from `region` and `s3_bucket`                                       |             |
 | `s3_force_path_style` | [set this to `true` to force the request to use path-style addressing](http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html) | false       |
 | `disable_ssl`         | set this to `true` to disable SSL when sending requests                                                                                    | false       |

exporter/awss3exporter/config.go

@@ -40,9 +40,9 @@ type Config struct {
 	S3Uploader    S3UploaderConfig `mapstructure:"s3uploader"`
 	MarshalerName MarshalerType    `mapstructure:"marshaler"`
 
-	FileFormat string `mapstructure:"file_format"`
 	// Encoding to apply. If present, overrides the marshaler configuration option.
-	Encoding *component.ID `mapstructure:"encoding"`
+	Encoding              *component.ID `mapstructure:"encoding"`
+	EncodingFileExtension string        `mapstructure:"encoding_file_extension"`
 }
 
 func (c *Config) Validate() error {

exporter/awss3exporter/config_test.go

@@ -32,7 +32,8 @@ func TestLoadConfig(t *testing.T) {
 	encoding := component.MustNewIDWithName("foo", "bar")
 	assert.Equal(t, e,
 		&Config{
-			Encoding: &encoding,
+			Encoding:              &encoding,
+			EncodingFileExtension: "baz",
 			S3Uploader: S3UploaderConfig{
 				Region:      "us-east-1",
 				S3Bucket:    "foo",

exporter/awss3exporter/exporter.go

@@ -39,7 +39,7 @@ func (e *s3Exporter) start(_ context.Context, host component.Host) error {
 	var m marshaler
 	var err error
 	if e.config.Encoding != nil {
-		if m, err = newMarshalerFromEncoding(e.config.Encoding, host, e.logger); err != nil {
+		if m, err = newMarshalerFromEncoding(e.config.Encoding, e.config.EncodingFileExtension, host, e.logger); err != nil {
 			return err
 		}
 	} else {

exporter/awss3exporter/marshaler.go

@@ -25,7 +25,7 @@ var (
 	ErrUnknownMarshaler = errors.New("unknown marshaler")
 )
 
-func newMarshalerFromEncoding(encoding *component.ID, host component.Host, logger *zap.Logger) (marshaler, error) {
+func newMarshalerFromEncoding(encoding *component.ID, fileFormat string, host component.Host, logger *zap.Logger) (marshaler, error) {
 	marshaler := &s3Marshaler{logger: logger}
 	e, ok := host.GetExtensions()[*encoding]
 	if !ok {
@@ -35,7 +35,7 @@ func newMarshalerFromEncoding(encoding *component.ID, host component.Host, logge
 	marshaler.logsMarshaler, _ = e.(plog.Marshaler)
 	marshaler.metricsMarshaler, _ = e.(pmetric.Marshaler)
 	marshaler.tracesMarshaler, _ = e.(ptrace.Marshaler)
-	marshaler.fileFormat = encoding.String()
+	marshaler.fileFormat = fileFormat
 	return marshaler, nil
 }
 

exporter/awss3exporter/marshaler_test.go

@@ -90,13 +90,13 @@ func TestMarshalerFromEncoding(t *testing.T) {
 		host := hostWithExtensions{
 			encoding: encodingExtension{},
 		}
-		m, err := newMarshalerFromEncoding(&id, host, zap.NewNop())
+		m, err := newMarshalerFromEncoding(&id, "myext", host, zap.NewNop())
 		assert.NoError(t, err)
 		require.NotNil(t, m)
-		assert.Equal(t, "foo", m.format())
+		assert.Equal(t, "myext", m.format())
 	}
 	{
-		m, err := newMarshalerFromEncoding(&id, componenttest.NewNopHost(), zap.NewNop())
+		m, err := newMarshalerFromEncoding(&id, "", componenttest.NewNopHost(), zap.NewNop())
 		assert.EqualError(t, err, `unknown encoding "foo"`)
 		require.Nil(t, m)
 	}

exporter/awss3exporter/s3_writer.go

@@ -40,11 +40,15 @@ func randomInRange(low, hi int) int {
 	return low + rand.Intn(hi-low)
 }
 
-func getS3Key(time time.Time, keyPrefix string, partition string, filePrefix string, metadata string, fileformat string, compression configcompression.Type) string {
+func getS3Key(time time.Time, keyPrefix string, partition string, filePrefix string, metadata string, fileFormat string, compression configcompression.Type) string {
 	timeKey := getTimeKey(time, partition)
 	randomID := randomInRange(100000000, 999999999)
+	suffix := ""
+	if fileFormat != "" {
+		suffix = "." + fileFormat
+	}
 
-	s3Key := keyPrefix + "/" + timeKey + "/" + filePrefix + metadata + "_" + strconv.Itoa(randomID) + "." + fileformat
+	s3Key := keyPrefix + "/" + timeKey + "/" + filePrefix + metadata + "_" + strconv.Itoa(randomID) + suffix
 
 	// add ".gz" extension to files if compression is enabled
 	if compression == configcompression.TypeGzip {

exporter/awss3exporter/s3_writer_test.go

@@ -41,6 +41,20 @@ func TestS3Key(t *testing.T) {
 	assert.Equal(t, true, matched)
 }
 
+func TestS3KeyEmptyFileFormat(t *testing.T) {
+	const layout = "2006-01-02"
+
+	tm, err := time.Parse(layout, "2022-06-05")
+
+	assert.NoError(t, err)
+	require.NotNil(t, tm)
+
+	re := regexp.MustCompile(`keyprefix/year=2022/month=06/day=05/hour=00/minute=00/fileprefixlogs_([0-9]+)`)
+	s3Key := getS3Key(tm, "keyprefix", "minute", "fileprefix", "logs", "", "")
+	matched := re.MatchString(s3Key)
+	assert.Equal(t, true, matched)
+}
+
 func TestS3KeyOfCompressedFile(t *testing.T) {
 	const layout = "2006-01-02"
 
@@ -55,6 +69,20 @@ func TestS3KeyOfCompressedFile(t *testing.T) {
 	assert.Equal(t, true, matched)
 }
 
+func TestS3KeyOfCompressedFileEmptyFileFormat(t *testing.T) {
+	const layout = "2006-01-02"
+
+	tm, err := time.Parse(layout, "2022-06-05")
+
+	assert.NoError(t, err)
+	require.NotNil(t, tm)
+
+	re := regexp.MustCompile(`keyprefix/year=2022/month=06/day=05/hour=00/minute=00/fileprefixlogs_([0-9]+).gz`)
+	s3Key := getS3Key(tm, "keyprefix", "minute", "fileprefix", "logs", "", "gzip")
+	matched := re.MatchString(s3Key)
+	assert.Equal(t, true, matched)
+}
+
 func TestGetSessionConfigWithEndpoint(t *testing.T) {
 	const endpoint = "https://endpoint.com"
 	const region = "region"

exporter/awss3exporter/testdata/default.yaml

@@ -4,6 +4,7 @@ receivers:
 exporters:
   awss3:
     encoding: "foo/bar"
+    encoding_file_extension: "baz"
     s3uploader:
       s3_bucket: "foo"
       region: 'us-east-1'