diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b7282a5bba2..ed72cfeeb9f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -383,6 +383,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve Cisco ASA/FTD parsing of messages - better support for identity FW messages. Change network.bytes, source.bytes, and destination.bytes to long from integer since value can exceed integer capacity. Add descriptions for various processors for easier pipeline editing in Kibana UI. {pull}23766[23766] - Updating Oauth2 flow for m365_defender fileset. {pull}24829[24829] - Improve PanOS parsing and ingest pipeline. {issue}22413[22413] {issue}22748[22748] {pull}24799[24799] +- Fix S3 input validation for non amazonaws.com domains. {issue}24420[24420] {pull}24861[24861] *Heartbeat* diff --git a/x-pack/filebeat/input/awss3/collector.go b/x-pack/filebeat/input/awss3/collector.go index 806bead57b2..d86220faf7d 100644 --- a/x-pack/filebeat/input/awss3/collector.go +++ b/x-pack/filebeat/input/awss3/collector.go @@ -226,14 +226,20 @@ func (c *s3Collector) changeVisibilityTimeout(queueURL string, visibilityTimeout return err } -func getRegionFromQueueURL(queueURL string) (string, error) { +func getRegionFromQueueURL(queueURL string, endpoint string) (string, error) { // get region from queueURL // Example: https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs - queueURLSplit := strings.Split(queueURL, ".") - if queueURLSplit[0] == "https://sqs" && queueURLSplit[2] == "amazonaws" { - return queueURLSplit[1], nil + url, err := url.Parse(queueURL) + if err != nil { + return "", fmt.Errorf(queueURL + " is not a valid URL") + } + if url.Scheme == "https" && url.Host != "" { + queueHostSplit := strings.Split(url.Host, ".") + if len(queueHostSplit) > 2 && (strings.Join(queueHostSplit[2:], ".") == endpoint || (endpoint == "" && queueHostSplit[2] == "amazonaws")) { + return queueHostSplit[1], nil + } } - return "", fmt.Errorf("queueURL is not in format: https://sqs.{REGION_ENDPOINT}.amazonaws.com/{ACCOUNT_NUMBER}/{QUEUE_NAME}") + return "", fmt.Errorf("QueueURL is not in format: https://sqs.{REGION_ENDPOINT}.{ENDPOINT}/{ACCOUNT_NUMBER}/{QUEUE_NAME}") } // handle message diff --git a/x-pack/filebeat/input/awss3/collector_test.go b/x-pack/filebeat/input/awss3/collector_test.go index fa613e29df8..e2a8bfca53e 100644 --- a/x-pack/filebeat/input/awss3/collector_test.go +++ b/x-pack/filebeat/input/awss3/collector_test.go @@ -57,10 +57,68 @@ func (m *MockS3Client) GetObjectRequest(input *s3.GetObjectInput) s3.GetObjectRe } func TestGetRegionFromQueueURL(t *testing.T) { - queueURL := "https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs" - regionName, err := getRegionFromQueueURL(queueURL) - assert.NoError(t, err) - assert.Equal(t, "us-east-1", regionName) + casesPositive := []struct { + title string + queueURL string + endpoint string + expectedRegion string + }{ + { + "QueueURL using amazonaws.com domain with blank Endpoint", + "https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs", + "", + "us-east-1", + }, + { + "QueueURL using abc.xyz and domain with matching Endpoint", + "https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs", + "abc.xyz", + "us-east-1", + }, + } + + for _, c := range casesPositive { + t.Run(c.title, func(t *testing.T) { + regionName, err := getRegionFromQueueURL(c.queueURL, c.endpoint) + assert.NoError(t, err) + assert.Equal(t, c.expectedRegion, regionName) + }) + } + + casesNegative := []struct { + title string + queueURL string + endpoint string + expectedRegion string + }{ + { + "QueueURL using abc.xyz and domain with blank Endpoint", + "https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs", + "", + "", + }, + { + "QueueURL using abc.xyz and domain with different Endpoint", + "https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs", + "googlecloud.com", + "", + }, + { + "QueueURL is an invalid URL", + ":foo", + "", + "", + }, + } + + for _, c := range casesNegative { + t.Run(c.title, func(t *testing.T) { + regionName, err := getRegionFromQueueURL(c.queueURL, c.endpoint) + assert.Error(t, err) + assert.Empty(t, regionName) + }) + } + } func TestHandleMessage(t *testing.T) { diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go index ccbe105974d..bdb5976bf2d 100644 --- a/x-pack/filebeat/input/awss3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -87,7 +87,7 @@ func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3C return nil, err } - regionName, err := getRegionFromQueueURL(in.config.QueueURL) + regionName, err := getRegionFromQueueURL(in.config.QueueURL, in.config.AwsConfig.Endpoint) if err != nil { err := fmt.Errorf("getRegionFromQueueURL failed: %w", err) log.Error(err)