Cherry-pick elastic#18370 to 7.7: [Filebeat] Unescape characters in s…

…3 file names (elastic#18413)

* [Filebeat] Unescape characters in s3 file names (elastic#18370)

* upescape characters in s3 file names

(cherry picked from commit ebddb94)

CHANGELOG.next.asciidoc

@@ -126,6 +126,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed `cloudfoundry.access` to have the correct `cloudfoundry.app.id` contents. {pull}17847[17847]
 - Fixing `ingress_controller.` fields to be of type keyword instead of text. {issue}17834[17834]
 - Fixed typo in log message. {pull}17897[17897]
+- Unescape file name from SQS message. {pull}18370[18370]
 
 *Heartbeat*
 

x-pack/filebeat/input/s3/input.go

@@ -13,6 +13,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net/url"
 	"strings"
 	"sync"
 	"time"
@@ -360,10 +361,16 @@ func handleSQSMessage(m sqs.Message) ([]s3Info, error) {
 	var s3Infos []s3Info
 	for _, record := range msg.Records {
 		if record.EventSource == "aws:s3" && strings.HasPrefix(record.EventName, "ObjectCreated:") {
+			// Unescape substrings from s3 log name. For example, convert "%3D" back to "="
+			filename, err := url.QueryUnescape(record.S3.object.Key)
+			if err != nil {
+				return nil, errors.Wrapf(err, "url.QueryUnescape failed")
+			}
+
 			s3Infos = append(s3Infos, s3Info{
 				region: record.AwsRegion,
 				name:   record.S3.bucket.Name,
-				key:    record.S3.object.Key,
+				key:    filename,
 				arn:    record.S3.bucket.Arn,
 			})
 		} else {
@@ -381,6 +388,7 @@ func (p *s3Input) handleS3Objects(svc s3iface.ClientAPI, s3Infos []s3Info, errC
 	defer s3Ctx.done()
 
 	for _, info := range s3Infos {
+		p.logger.Debugf("Processing file from s3 bucket \"%s\" with name \"%s\"", info.name, info.key)
 		err := p.createEventsFromS3Info(svc, info, s3Ctx)
 		if err != nil {
 			err = errors.Wrapf(err, "createEventsFromS3Info failed for %v", info.key)

x-pack/filebeat/input/s3/input_test.go

@@ -89,6 +89,30 @@ func TestHandleMessage(t *testing.T) {
 				},
 			},
 		},
+		{
+			"sqs message with event source aws:s3, event name ObjectCreated:Put and encoded filename",
+			sqs.Message{
+				Body: awssdk.String("{\"Records\":[{\"eventSource\":\"aws:s3\",\"awsRegion\":\"ap-southeast-1\",\"eventTime\":\"2019-06-21T16:16:54.629Z\",\"eventName\":\"ObjectCreated:Put\",\"s3\":{\"configurationId\":\"object-created-event\",\"bucket\":{\"name\":\"test-s3-ks-2\",\"arn\":\"arn:aws:s3:::test-s3-ks-2\"},\"object\":{\"key\":\"year%3D2020/month%3D05/test1.txt\"}}}]}"),
+			},
+			[]s3Info{
+				{
+					name: "test-s3-ks-2",
+					key:  "year=2020/month=05/test1.txt",
+				},
+			},
+		},
+		{
+			"sqs message with event source aws:s3, event name ObjectCreated:Put and gzip filename",
+			sqs.Message{
+				Body: awssdk.String("{\"Records\":[{\"eventSource\":\"aws:s3\",\"awsRegion\":\"ap-southeast-1\",\"eventTime\":\"2019-06-21T16:16:54.629Z\",\"eventName\":\"ObjectCreated:Put\",\"s3\":{\"configurationId\":\"object-created-event\",\"bucket\":{\"name\":\"test-s3-ks-2\",\"arn\":\"arn:aws:s3:::test-s3-ks-2\"},\"object\":{\"key\":\"428152502467_CloudTrail_us-east-2_20191219T1655Z_WXCas1PVnOaTpABD.json.gz\"}}}]}"),
+			},
+			[]s3Info{
+				{
+					name: "test-s3-ks-2",
+					key:  "428152502467_CloudTrail_us-east-2_20191219T1655Z_WXCas1PVnOaTpABD.json.gz",
+				},
+			},
+		},
 	}
 
 	for _, c := range casesPositive {