From e3c7d8f0c6fabc0a4a3ed2137c772eb9a8e6e2c2 Mon Sep 17 00:00:00 2001
From: Alex Resnick <adr8292@gmail.com>
Date: Fri, 9 Apr 2021 09:33:59 -0500
Subject: [PATCH] [Filebeat] Fix hardcoded amazonaws.com endpoint (#24861)

(cherry picked from commit 7b729da14738a5156ca62e8d8e47e3389f6bc18c)
---
 CHANGELOG.next.asciidoc                       |  1 +
 x-pack/filebeat/input/awss3/collector.go      | 16 +++--
 x-pack/filebeat/input/awss3/collector_test.go | 66 +++++++++++++++++--
 x-pack/filebeat/input/awss3/input.go          |  2 +-
 4 files changed, 75 insertions(+), 10 deletions(-)

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 10f24ddf1ac..9893f72aa62 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -206,6 +206,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix date parsing in GSuite/login and Google Workspace/login filesets. {issue}24694[24694]
 - Fix gcp/vpcflow module error where input type was defaulting to file. {pull}24719[24719]
 - Improve PanOS parsing and ingest pipeline. {issue}22413[22413] {issue}22748[22748] {pull}24799[24799]
+- Fix S3 input validation for non amazonaws.com domains. {issue}24420[24420] {pull}24861[24861]
 
 *Heartbeat*
 
diff --git a/x-pack/filebeat/input/awss3/collector.go b/x-pack/filebeat/input/awss3/collector.go
index ceec461797f..1ace01df882 100644
--- a/x-pack/filebeat/input/awss3/collector.go
+++ b/x-pack/filebeat/input/awss3/collector.go
@@ -226,14 +226,20 @@ func (c *s3Collector) changeVisibilityTimeout(queueURL string, visibilityTimeout
 	return err
 }
 
-func getRegionFromQueueURL(queueURL string) (string, error) {
+func getRegionFromQueueURL(queueURL string, endpoint string) (string, error) {
 	// get region from queueURL
 	// Example: https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs
-	queueURLSplit := strings.Split(queueURL, ".")
-	if queueURLSplit[0] == "https://sqs" && queueURLSplit[2] == "amazonaws" {
-		return queueURLSplit[1], nil
+	url, err := url.Parse(queueURL)
+	if err != nil {
+		return "", fmt.Errorf(queueURL + " is not a valid URL")
+	}
+	if url.Scheme == "https" && url.Host != "" {
+		queueHostSplit := strings.Split(url.Host, ".")
+		if len(queueHostSplit) > 2 && (strings.Join(queueHostSplit[2:], ".") == endpoint || (endpoint == "" && queueHostSplit[2] == "amazonaws")) {
+			return queueHostSplit[1], nil
+		}
 	}
-	return "", fmt.Errorf("queueURL is not in format: https://sqs.{REGION_ENDPOINT}.amazonaws.com/{ACCOUNT_NUMBER}/{QUEUE_NAME}")
+	return "", fmt.Errorf("QueueURL is not in format: https://sqs.{REGION_ENDPOINT}.{ENDPOINT}/{ACCOUNT_NUMBER}/{QUEUE_NAME}")
 }
 
 // handle message
diff --git a/x-pack/filebeat/input/awss3/collector_test.go b/x-pack/filebeat/input/awss3/collector_test.go
index 912b4cf076a..47313d7c877 100644
--- a/x-pack/filebeat/input/awss3/collector_test.go
+++ b/x-pack/filebeat/input/awss3/collector_test.go
@@ -58,10 +58,68 @@ func (m *MockS3Client) GetObjectRequest(input *s3.GetObjectInput) s3.GetObjectRe
 }
 
 func TestGetRegionFromQueueURL(t *testing.T) {
-	queueURL := "https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs"
-	regionName, err := getRegionFromQueueURL(queueURL)
-	assert.NoError(t, err)
-	assert.Equal(t, "us-east-1", regionName)
+	casesPositive := []struct {
+		title          string
+		queueURL       string
+		endpoint       string
+		expectedRegion string
+	}{
+		{
+			"QueueURL using amazonaws.com domain with blank Endpoint",
+			"https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs",
+			"",
+			"us-east-1",
+		},
+		{
+			"QueueURL using abc.xyz and domain with matching Endpoint",
+			"https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs",
+			"abc.xyz",
+			"us-east-1",
+		},
+	}
+
+	for _, c := range casesPositive {
+		t.Run(c.title, func(t *testing.T) {
+			regionName, err := getRegionFromQueueURL(c.queueURL, c.endpoint)
+			assert.NoError(t, err)
+			assert.Equal(t, c.expectedRegion, regionName)
+		})
+	}
+
+	casesNegative := []struct {
+		title          string
+		queueURL       string
+		endpoint       string
+		expectedRegion string
+	}{
+		{
+			"QueueURL using abc.xyz and domain with blank Endpoint",
+			"https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs",
+			"",
+			"",
+		},
+		{
+			"QueueURL using abc.xyz and domain with different Endpoint",
+			"https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs",
+			"googlecloud.com",
+			"",
+		},
+		{
+			"QueueURL is an invalid URL",
+			":foo",
+			"",
+			"",
+		},
+	}
+
+	for _, c := range casesNegative {
+		t.Run(c.title, func(t *testing.T) {
+			regionName, err := getRegionFromQueueURL(c.queueURL, c.endpoint)
+			assert.Error(t, err)
+			assert.Empty(t, regionName)
+		})
+	}
+
 }
 
 func TestHandleMessage(t *testing.T) {
diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go
index ccbe105974d..bdb5976bf2d 100644
--- a/x-pack/filebeat/input/awss3/input.go
+++ b/x-pack/filebeat/input/awss3/input.go
@@ -87,7 +87,7 @@ func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3C
 		return nil, err
 	}
 
-	regionName, err := getRegionFromQueueURL(in.config.QueueURL)
+	regionName, err := getRegionFromQueueURL(in.config.QueueURL, in.config.AwsConfig.Endpoint)
 	if err != nil {
 		err := fmt.Errorf("getRegionFromQueueURL failed: %w", err)
 		log.Error(err)