Merge remote-tracking branch 'elastic/8.3' into mergify/bp/8.3/pr-32421

.ci/packaging.groovy

@@ -400,6 +400,7 @@ def release(type){
       dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
       dir("${env.BEATS_FOLDER}") {
         sh(label: "mage package ${type} ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+        sh(label: "mage ironbank ${type} ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage ironbank')
         def folder = getBeatsName(env.BEATS_FOLDER)
         uploadPackagesToGoogleBucket(
           credentialsId: env.JOB_GCS_EXT_CREDENTIALS,

.github/workflows/macos-auditbeat.yml

@@ -29,3 +29,12 @@ jobs:
       run: cd ${{ env.BEAT_MODULE }} && mage build
     - name: Run test
       run: cd ${{ env.BEAT_MODULE }} && mage unitTest
+    - uses: v1v/otel-upload-test-artifact-action@v2
+      if: always()
+      continue-on-error: true
+      with:
+        jobName: "macos"
+        stepName: "Run test"
+        path: "${{ env.BEAT_MODULE }}/build/TEST*.xml"
+        type: "junit"
+        githubToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/macos-filebeat.yml

@@ -29,3 +29,12 @@ jobs:
       run: cd ${{ env.BEAT_MODULE }} && mage build
     - name: Run test
       run: cd ${{ env.BEAT_MODULE }} && mage unitTest
+    - uses: v1v/otel-upload-test-artifact-action@v2
+      if: always()
+      continue-on-error: true
+      with:
+        jobName: "macos"
+        stepName: "Run test"
+        path: "${{ env.BEAT_MODULE }}/build/TEST*.xml"
+        type: "junit"
+        githubToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/macos-heartbeat.yml

@@ -29,3 +29,12 @@ jobs:
       run: cd ${{ env.BEAT_MODULE }} && mage build
     - name: Run test
       run: cd ${{ env.BEAT_MODULE }} && mage unitTest
+    - uses: v1v/otel-upload-test-artifact-action@v2
+      if: always()
+      continue-on-error: true
+      with:
+        jobName: "macos"
+        stepName: "Run test"
+        path: "${{ env.BEAT_MODULE }}/build/TEST*.xml"
+        type: "junit"
+        githubToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/macos-metricbeat.yml

@@ -29,3 +29,12 @@ jobs:
       run: cd ${{ env.BEAT_MODULE }} && mage build
     - name: Run test
       run: cd ${{ env.BEAT_MODULE }} && echo "See https://github.com/elastic/beats/issues/29038"
+    - uses: v1v/otel-upload-test-artifact-action@v2
+      if: always()
+      continue-on-error: true
+      with:
+        jobName: "macos"
+        stepName: "Run test"
+        path: "${{ env.BEAT_MODULE }}/build/TEST*.xml"
+        type: "junit"
+        githubToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/macos-packetbeat.yml

@@ -29,3 +29,12 @@ jobs:
       run: cd ${{ env.BEAT_MODULE }} && mage build
     - name: Run test
       run: cd ${{ env.BEAT_MODULE }} && mage unitTest
+    - uses: v1v/otel-upload-test-artifact-action@v2
+      if: always()
+      continue-on-error: true
+      with:
+        jobName: "macos"
+        stepName: "Run test"
+        path: "${{ env.BEAT_MODULE }}/build/TEST*.xml"
+        type: "junit"
+        githubToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/macos-xpack-auditbeat.yml

@@ -29,3 +29,12 @@ jobs:
       run: cd ${{ env.BEAT_MODULE }} && mage build
     - name: Run test
       run: cd ${{ env.BEAT_MODULE }} && mage unitTest
+    - uses: v1v/otel-upload-test-artifact-action@v2
+      if: always()
+      continue-on-error: true
+      with:
+        jobName: "macos"
+        stepName: "Run test"
+        path: "${{ env.BEAT_MODULE }}/build/TEST*.xml"
+        type: "junit"
+        githubToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/macos-xpack-filebeat.yml

@@ -29,3 +29,12 @@ jobs:
       run: cd ${{ env.BEAT_MODULE }} && mage build
     - name: Run test
       run: cd ${{ env.BEAT_MODULE }} && mage unitTest
+    - uses: v1v/otel-upload-test-artifact-action@v2
+      if: always()
+      continue-on-error: true
+      with:
+        jobName: "macos"
+        stepName: "Run test"
+        path: "${{ env.BEAT_MODULE }}/build/TEST*.xml"
+        type: "junit"
+        githubToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/macos-xpack-functionbeat.yml

@@ -29,3 +29,12 @@ jobs:
       run: cd ${{ env.BEAT_MODULE }} && mage build
     - name: Run test
       run: cd ${{ env.BEAT_MODULE }} && mage unitTest
+    - uses: v1v/otel-upload-test-artifact-action@v2
+      if: always()
+      continue-on-error: true
+      with:
+        jobName: "macos"
+        stepName: "Run test"
+        path: "${{ env.BEAT_MODULE }}/build/TEST*.xml"
+        type: "junit"
+        githubToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/macos-xpack-heartbeat.yml

@@ -29,3 +29,12 @@ jobs:
       run: cd ${{ env.BEAT_MODULE }} && mage build
     - name: Run test
       run: cd ${{ env.BEAT_MODULE }} && mage unitTest
+    - uses: v1v/otel-upload-test-artifact-action@v2
+      if: always()
+      continue-on-error: true
+      with:
+        jobName: "macos"
+        stepName: "Run test"
+        path: "${{ env.BEAT_MODULE }}/build/TEST*.xml"
+        type: "junit"
+        githubToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/macos-xpack-metricbeat.yml

@@ -29,3 +29,12 @@ jobs:
       run: cd ${{ env.BEAT_MODULE }} && mage build
     - name: Run test
       run: cd ${{ env.BEAT_MODULE }} && mage unitTest
+    - uses: v1v/otel-upload-test-artifact-action@v2
+      if: always()
+      continue-on-error: true
+      with:
+        jobName: "macos"
+        stepName: "Run test"
+        path: "${{ env.BEAT_MODULE }}/build/TEST*.xml"
+        type: "junit"
+        githubToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/macos-xpack-osquerybeat.yml

@@ -29,3 +29,12 @@ jobs:
       run: cd ${{ env.BEAT_MODULE }} && mage build
     - name: Run test
       run: cd ${{ env.BEAT_MODULE }} && mage unitTest
+    - uses: v1v/otel-upload-test-artifact-action@v2
+      if: always()
+      continue-on-error: true
+      with:
+        jobName: "macos"
+        stepName: "Run test"
+        path: "${{ env.BEAT_MODULE }}/build/TEST*.xml"
+        type: "junit"
+        githubToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/macos-xpack-packetbeat.yml

@@ -29,3 +29,12 @@ jobs:
       run: cd ${{ env.BEAT_MODULE }} && mage build
     - name: Run test
       run: cd ${{ env.BEAT_MODULE }} && mage unitTest
+    - uses: v1v/otel-upload-test-artifact-action@v2
+      if: always()
+      continue-on-error: true
+      with:
+        jobName: "macos"
+        stepName: "Run test"
+        path: "${{ env.BEAT_MODULE }}/build/TEST*.xml"
+        type: "junit"
+        githubToken: ${{ secrets.GITHUB_TOKEN }}

CHANGELOG.next.asciidoc

@@ -40,12 +40,13 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 
 - Fix OS name reported by add_host_metadata on Windows 11. {issue}30833[30833] {pull}32259[32259]
 - Fix race condition when reloading runners {pull}32309[32309]
+- Fix race condition when stopping runners {pull}32433[32433]
 
 *Auditbeat*
 
 - auditd module: Fix parsing of audit rules where arguments are quoted (like file paths containing spaces). {pull}32421[32421]
 - auditd module: Fix minimum AuditStatus length so that library can support kernels from 2.6.32. {pull}32421[32421]
-
+- system/socket: Reduce memory usage of the dataset. {issue}32191[32191] {pull}32192[32192]
 
 *Filebeat*
 
@@ -60,6 +61,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 *Metricbeat*
 
 - update elasticsearch node_stats metricset to use keyword for cgroup memory instead of long {pull}32197[32197]
+- Fix to ARN parsing for Cloudwatch resource names with leading slashes {pull}32446[32446]
 
 *Packetbeat*
 
@@ -82,7 +84,6 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 
 *Auditbeat*
 
-
 *Filebeat*
 
 

Jenkinsfile

@@ -204,13 +204,18 @@ VERSION=${env.VERSION}-SNAPSHOT""")
         notifyBuildResult(prComment: true,
                           slackComment: true,
                           analyzeFlakey: !isTag(), jobName: getFlakyJobName(withBranch: getFlakyBranch()),
-                          githubIssue: isBranch() && currentBuild.currentResult != "SUCCESS",
+                          githubIssue: isGitHubIssueEnabled(),
                           githubLabels: 'Team:Elastic-Agent-Data-Plane')
       }
     }
   }
 }
 
+// When to create a GiHub issue
+def isGitHubIssueEnabled() {
+  return isBranch() && currentBuild.currentResult != "SUCCESS" && currentBuild.currentResult != "ABORTED"
+}
+
 def runChecks() {
   def mapParallelTasks = [:]
   def content = readYaml(file: 'Jenkinsfile.yml')

NOTICE.txt

@@ -6492,11 +6492,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-system-metrics
-Version: v0.4.3
+Version: v0.4.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].3/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].2/LICENSE.txt:
 
                                  Apache License
                            Version 2.0, January 2004

auditbeat/auditbeat.reference.yml

@@ -747,9 +747,6 @@ output.elasticsearch:
     # Strategy for fetching the topics metadata from the broker. Default is false.
     #full: false
 
-  # The number of concurrent load-balanced Kafka output workers.
-  #worker: 1
-
   # The number of times to retry publishing an event after a publishing failure.
   # After the specified number of retries, events are typically dropped.
   # Some Beats, such as Filebeat, ignore the max_retries setting and retry until

auditbeat/magefile.go

@@ -101,6 +101,14 @@ func Package() {
 	mg.SerialDeps(devtools.Package, TestPackages)
 }
 
+// Package packages the Beat for IronBank distribution.
+//
+// Use SNAPSHOT=true to build snapshots.
+func Ironbank() error {
+	fmt.Println(">> Ironbank: this module is not subscribed to the IronBank releases.")
+	return nil
+}
+
 // TestPackages tests the generated packages (i.e. file modes, owners, groups).
 func TestPackages() error {
 	return devtools.TestPackages()

deploy/kubernetes/auditbeat-kubernetes.yaml

@@ -109,7 +109,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: auditbeat
-        image: docker.elastic.co/beats/auditbeat:8.3.2
+        image: docker.elastic.co/beats/auditbeat:8.3.3
         args: [
           "-c", "/etc/auditbeat.yml",
           "-e",

deploy/kubernetes/filebeat-kubernetes.yaml

@@ -64,7 +64,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: filebeat
-        image: docker.elastic.co/beats/filebeat:8.3.2
+        image: docker.elastic.co/beats/filebeat:8.3.3
         args: [
           "-c", "/etc/filebeat.yml",
           "-e",

deploy/kubernetes/heartbeat-kubernetes.yaml

@@ -74,7 +74,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: heartbeat
-        image: docker.elastic.co/beats/heartbeat:8.3.2
+        image: docker.elastic.co/beats/heartbeat:8.3.3
         args: [
           "-c", "/etc/heartbeat.yml",
           "-e",

deploy/kubernetes/metricbeat-kubernetes.yaml

@@ -156,7 +156,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: metricbeat
-        image: docker.elastic.co/beats/metricbeat:8.3.2
+        image: docker.elastic.co/beats/metricbeat:8.3.3
         args: [
           "-c", "/etc/metricbeat.yml",
           "-e",

dev-tools/mage/common.go

@@ -374,8 +374,9 @@ func unzip(sourceFile, destinationDir string) error {
 	return nil
 }
 
-// Tar compress a directory using tar + gzip algorithms
-func Tar(src string, targetFile string) error {
+// Tar compress a directory using tar + gzip algorithms but without adding
+// the directory
+func TarWithOptions(src string, targetFile string, trimSource bool) error {
 	fmt.Printf(">> creating TAR file from directory: %s, target: %s\n", src, targetFile)
 
 	f, err := os.Create(targetFile)
@@ -409,6 +410,15 @@ func Tar(src string, targetFile string) error {
 		// must provide real name
 		// (see https://golang.org/src/archive/tar/common.go?#L626)
 		header.Name = filepath.ToSlash(file)
+		// Replace the source folder in the files to be compressed
+		if trimSource {
+			header.Name = strings.ReplaceAll(filepath.ToSlash(file), filepath.ToSlash(src), "")
+			header.Name = strings.TrimPrefix(header.Name, "/")
+			if header.Name == "" {
+				fmt.Print(">> skipping root directory\n")
+				return nil
+			}
+		}
 
 		// write header
 		if err := tw.WriteHeader(header); err != nil {
@@ -441,6 +451,11 @@ func Tar(src string, targetFile string) error {
 	return nil
 }
 
+// Tar compress a directory using tar + gzip algorithms
+func Tar(src string, targetFile string) error {
+	return TarWithOptions(src, targetFile, false)
+}
+
 func untar(sourceFile, destinationDir string) error {
 	file, err := os.Open(sourceFile)
 	if err != nil {

dev-tools/mage/copy.go

@@ -33,6 +33,16 @@ func Copy(src, dest string) error {
 	return copy.Execute()
 }
 
+// Copy copies a file and preserves the permissions.
+func CopyFile(src, dest string) error {
+	copy := &CopyTask{Source: src, Dest: dest}
+	info, err := os.Stat(src)
+	if err != nil {
+		return errors.Wrapf(err, "copy failed: cannot stat source file %v", src)
+	}
+	return copy.fileCopy(src, dest, info)
+}
+
 // CopyTask copies a file or directory (recursively) and preserves the permissions.
 type CopyTask struct {
 	Source   string           // Source directory or file.