Merge branch 'main' into test-queue-localstack

.ci/jobs/docker-compose.yml

@@ -6,18 +6,16 @@ services:
     image: busybox
     depends_on:
       localstack: { condition: service_healthy }
-
+      
   localstack:
-    image: localstack/localstack
-    hostname: localhost
+    container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}"
+    image: localstack/localstack:2.1.0
     ports:
-      - "4566:4566"            # LocalStack Gateway
+      - "127.0.0.1:4566:4566"            # LocalStack Gateway
     environment:
-      - SERVICES=s3,sqs,sns,secretsmanager
       - DEBUG=1
       - DOCKER_HOST=unix:///var/run/docker.sock
-      - HOST_TMP_FOLDER=${TMPDIR}
-      - HOSTNAME_EXTERNAL=localhost
+      - LOCALSTACK_HOST=localhost
       - S3_HOSTNAME=localhost
       - PROVIDER_OVERRIDE_S3=asf
     volumes:

CHANGELOG.next.asciidoc

@@ -145,6 +145,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix metric collection in GCPPubSub input. {pull}35773[35773]
 - Fix end point deregistration in http_endpoint input. {issue}35899[35899] {pull}35903[35903]
 - Fix duplicate ID panic in filestream metrics. {issue}35964[35964] {pull}35972[35972]
+- Improve error reporting and fix IPv6 handling of TCP and UDP metric collection. {pull}35996[35996]
 
 *Heartbeat*
 

Jenkinsfile

@@ -118,7 +118,7 @@ pipeline {
         }
       }
       steps {
-        runBuildAndTest(filterStage: 'mandatory')
+        runBuildAndTest(filterStage: 'extended')
       }
     }
     stage('Extended') {
@@ -597,6 +597,7 @@ def targetWithoutNode(Map args = [:]) {
         // make commands use -C <folder> while mage commands require the dir(folder)
         // let's support this scenario with the location variable.
         dir(isMage ? directory : '') {
+          unstash("terraform-${name}")
           if (enableRetry) {
             // Retry the same command to bypass any kind of flakiness.
             // Downside: genuine failures will be repeated.
@@ -940,7 +941,7 @@ def startCloudTestEnv(Map args = [:]) {
         // Archive terraform states in case manual cleanup is needed.
         archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate')
       }
-      stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**')
+      stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**,outputs*.yml')
     }
   }
 }

filebeat/input/tcp/input.go

@@ -338,6 +338,9 @@ func procNetTCP(path string, addr []string, hasUnspecified bool, addrIsUnspecifi
 	if len(addr) == 0 {
 		return 0, nil
 	}
+	if len(addr) != len(addrIsUnspecified) {
+		return 0, errors.New("mismatched address/unspecified lists: please report this")
+	}
 	b, err := os.ReadFile(path)
 	if err != nil {
 		return 0, err

filebeat/input/udp/input.go

@@ -272,7 +272,7 @@ func (m *inputMetrics) poll(addr, addr6 []string, each time.Duration, log *logp.
 	if err != nil {
 		log.Warnf("failed to get initial udp stats from /proc: %v", err)
 	}
-	rx6, drops6, err := procNetUDP("/proc/net/udp6", addr, hasUnspecified6, addrIsUnspecified6)
+	rx6, drops6, err := procNetUDP("/proc/net/udp6", addr6, hasUnspecified6, addrIsUnspecified6)
 	if err != nil {
 		log.Warnf("failed to get initial udp6 stats from /proc: %v", err)
 	}
@@ -288,7 +288,7 @@ func (m *inputMetrics) poll(addr, addr6 []string, each time.Duration, log *logp.
 				log.Warnf("failed to get udp stats from /proc: %v", err)
 				continue
 			}
-			rx6, drops6, err := procNetUDP("/proc/net/udp6", addr, hasUnspecified6, addrIsUnspecified6)
+			rx6, drops6, err := procNetUDP("/proc/net/udp6", addr6, hasUnspecified6, addrIsUnspecified6)
 			if err != nil {
 				log.Warnf("failed to get udp6 stats from /proc: %v", err)
 				continue
@@ -333,6 +333,9 @@ func procNetUDP(path string, addr []string, hasUnspecified bool, addrIsUnspecifi
 	if len(addr) == 0 {
 		return 0, 0, nil
 	}
+	if len(addr) != len(addrIsUnspecified) {
+		return 0, 0, errors.New("mismatched address/unspecified lists: please report this")
+	}
 	b, err := os.ReadFile(path)
 	if err != nil {
 		return 0, 0, err

x-pack/filebeat/docker-compose.yml

@@ -7,7 +7,6 @@ services:
     depends_on:
       elasticsearch: { condition: service_healthy }
       cometd: { condition: service_healthy }
-      localstack: { condition: service_healthy }
 
   elasticsearch:
     extends:
@@ -25,20 +24,3 @@ services:
     hostname: cometd
     ports:
       - 8080:8080
-
-  localstack:
-    image: localstack/localstack
-    hostname: localhost
-    ports:
-      - "4566:4566"            # LocalStack Gateway
-    environment:
-      - SERVICES=s3,sqs,sns,secretsmanager
-      - DEBUG=1
-      - DOCKER_HOST=unix:///var/run/docker.sock
-      - HOST_TMP_FOLDER=${TMPDIR}
-      - HOSTNAME_EXTERNAL=localhost
-      - S3_HOSTNAME=localhost
-      - PROVIDER_OVERRIDE_S3=asf
-    volumes:
-      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
-      - "/var/run/docker.sock:/var/run/docker.sock"

x-pack/filebeat/input/awss3/input_integration_test.go

@@ -63,9 +63,17 @@ func getTerraformOutputs(t *testing.T, isLocalStack bool) terraformOutputData {
 		outputFile = terraformOutputYML
 	}
 
+	files, err := ioutil.ReadDir(path.Join(path.Dir(filename), "_meta/terraform"))
+	if err != nil {
+		t.Fatalf("failed reading directory: %v", err)
+	}
+	for _, file := range files {
+		t.Logf(file.Name())
+	}
+
 	ymlData, err := ioutil.ReadFile(path.Join(path.Dir(filename), outputFile))
 	if os.IsNotExist(err) {
-		t.Skipf("Run 'terraform apply' in %v to setup S3 and SQS for the test.", filepath.Dir(terraformOutputYML))
+		t.Skipf("Run 'terraform apply' in %v to setup S3 and SQS for the test ------- %v", filepath.Dir(terraformOutputYML), path.Join(path.Dir(filename), outputFile))
 	}
 	if err != nil {
 		t.Fatalf("failed reading terraform output data: %v", err)