Merge remote-tracking branch 'upstream/master' into feature/windows-10

* upstream/master: (29 commits) [CI] Enable build stage for arm architecture (elastic#21284) [BUILD][CI] fetch dependencies with retry (elastic#21614) Add proxy metricset (elastic#21751) [Filebeat][httpjson] Fix date_cursor validation (elastic#21756) Update endpoint-related terminology in Elastic Agent docs (elastic#21458) Move s3_daily_storage and s3_request metricsets to use cloudwatch input (elastic#21703) [Agent] Remove heartbeat from bundled beats in agent. (elastic#21602) [Elastic Agent] Don't perform install until after enroll (elastic#21772) [Elastic Agent] Update path from policy change for Kibana connection. (elastic#21804) Change cloud.provider from googlecloud to gcp (elastic#21775) Fix stress test package list (elastic#21719) [Elastic Agent] Remove the service installer scripts from packaging (elastic#21694) [CI] Support Windows-2012 in pipeline 2.0 (elastic#21338) Fix non-windows fields on system/filesystem (elastic#21758) disable TestReceiveEventsAndMetadata/TestSocketCleanup/TestReceiveNewEventsConcurrently in Windows (elastic#21750) Use badger code from upstream repository (elastic#21705) Disable writes sync in persistent cache (elastic#21754) Make API address and Shard ID required in Cloud Foundry settings (elastic#21759) [CI] Support skip-ci label (elastic#21377) Increase recommended memory when deploying in Cloud foundry (elastic#21755) ...
v1v · Oct 15, 2020 · 1997de1 · 1997de1
2 parents c011c4a + f8ff7ce
commit 1997de1
Show file tree

Hide file tree

Showing 115 changed files with 6,760 additions and 2,576 deletions.
diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -246,8 +246,12 @@ def runE2ETestForPackages(){
 
 def release(){
   withBeatsEnv(){
-    dir("${env.BEATS_FOLDER}") {
-      sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+    withEnv([
+      "DEV=true"
+    ]) {
+      dir("${env.BEATS_FOLDER}") {
+        sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+      }
     }
     publishPackages("${env.BEATS_FOLDER}")
   }

diff --git a/.ci/scripts/install-go.sh b/.ci/scripts/install-go.sh
@@ -5,7 +5,8 @@ MSG="environment variable missing"
 GO_VERSION=${GO_VERSION:?$MSG}
 PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"}
 HOME=${HOME:?$MSG}
-ARCH=$(uname -s| tr '[:upper:]' '[:lower:]')
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
 GVM_CMD="${HOME}/bin/gvm"
 
 if command -v go
@@ -19,10 +20,20 @@ then
     fi
 fi
 
+if [ "${ARCH}" == "aarch64" ] ; then
+    GVM_ARCH_SUFFIX=arm64
+elif [ "${ARCH}" == "x86_64" ] ; then
+    GVM_ARCH_SUFFIX=amd64
+elif [ "${ARCH}" == "i686" ] ; then
+    GVM_ARCH_SUFFIX=386
+else
+    GVM_ARCH_SUFFIX=arm
+fi
+
 echo "UNMET DEP: Installing Go"
 mkdir -p "${HOME}/bin"
 
-curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${ARCH}-amd64"
+curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${OS}-${GVM_ARCH_SUFFIX}"
 chmod +x "${GVM_CMD}"
 
 gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE}

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -25,6 +25,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Allow embedding of CAs, Certificate of private keys for anything that support TLS in ouputs and inputs https://github.com/elastic/beats/pull/21179
 - Update to Golang 1.12.1. {pull}11330[11330]
 - Disable Alibaba Cloud and Tencent Cloud metadata providers by default. {pull}13812[12812]
+- API address is a required setting in `add_cloudfoundry_metadata`. {pull}21759[21759]
 
 *Auditbeat*
 
@@ -78,6 +79,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Removed experimental modules `citrix`, `kaspersky`, `rapid7` and `tenable`. {pull}20706[20706]
 - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993]
 - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
+- API address and shard ID are required settings in the Cloud Foundry input. {pull}21759[21759]
 
 *Heartbeat*
 
@@ -95,6 +97,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix ECS compliance of user.id field in system/users  metricset {pull}19019[19019]
 - Rename googlecloud stackdriver metricset to metrics. {pull}19718[19718]
 - Remove "invalid zero" metrics on Windows and Darwin, don't report linux-only memory and diskio metrics when running under agent. {pull}21457[21457]
+- Change cloud.provider from googlecloud to gcp. {pull}21775[21775]
+- API address and shard ID are required settings in the Cloud Foundry module. {pull}21759[21759]
 
 *Packetbeat*
 
@@ -366,7 +370,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix remote_write flaky test. {pull}21173[21173]
 - Visualization title fixes in aws, azure and googlecloud compute dashboards. {pull}21098[21098]
 - Add a switch to the driver definition on SQL module to use pretty names {pull}17378[17378]
+- Fix retrieving resources by ID for the azure module. {pull}21711[21711] {issue}21707[21707]
 - Use timestamp from CloudWatch API when creating events. {pull}21498[21498]
+- Report the correct windows events for system/filesystem {pull}21758[21758]
 
 *Packetbeat*
 
@@ -458,6 +464,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add istiod metricset. {pull}21519[21519]
 - Release `add_cloudfoundry_metadata` as GA. {pull}21525[21525]
 - Add support for OpenStack SSL metadata APIs in `add_cloud_metadata`. {pull}21590[21590]
+- Add proxy metricset for istio module. {pull}21751[21751]
 
 *Auditbeat*
 
@@ -750,6 +757,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Move Prometheus query & remote_write to GA. {pull}21507[21507]
 - Expand unsupported option from namespace to metrics in the azure module. {pull}21486[21486]
 - Map cloud data filed `cloud.account.id` to azure subscription.  {pull}21483[21483] {issue}21381[21381]
+- Move s3_daily_storage and s3_request metricsets to use cloudwatch input. {pull}21703[21703]
 
 *Packetbeat*
 

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -17,7 +17,6 @@ pipeline {
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_COMPOSE_VERSION = "1.21.0"
     DOCKER_REGISTRY = 'docker.elastic.co'
-    GOX_FLAGS = "-arch amd64"
     JOB_GCS_BUCKET = 'beats-ci-temp'
     JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
     OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
@@ -29,7 +28,7 @@ pipeline {
     XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
   }
   options {
-    timeout(time: 2, unit: 'HOURS')
+    timeout(time: 3, unit: 'HOURS')
     buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
     timestamps()
     ansiColor('xterm')
@@ -46,6 +45,7 @@ pipeline {
     booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.')
     string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.')
     booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.')
+    booleanParam(name: 'armTest', defaultValue: false, description: 'Allow ARM stages.')
     booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.')
     string(name: 'PYTEST_ADDOPTS', defaultValue: '', description: 'Additional options to pass to pytest. Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. For retries you can use `--reruns 3 --reruns-delay 15`')
   }
@@ -103,13 +103,17 @@ pipeline {
           script {
             def mapParallelTasks = [:]
             def content = readYaml(file: 'Jenkinsfile.yml')
-            content['projects'].each { projectName ->
-              generateStages(project: projectName, changeset: content['changeset']).each { k,v ->
-                mapParallelTasks["${k}"] = v
+            if (content?.disabled?.when?.labels && beatsWhen(project: 'top-level', content: content?.disabled?.when)) {
+              error 'Pull Request has been configured to be disabled when there is a skip-ci label match'
+            } else {
+              content['projects'].each { projectName ->
+                generateStages(project: projectName, changeset: content['changeset']).each { k,v ->
+                  mapParallelTasks["${k}"] = v
+                }
               }
+              notifyBuildReason()
+              parallel(mapParallelTasks)
             }
-            notifyBuildReason()
-            parallel(mapParallelTasks)
           }
         }
       }
@@ -219,23 +223,32 @@ def withBeatsEnv(Map args = [:], Closure body) {
   def withModule = args.get('withModule', false)
   def directory = args.get('directory', '')
 
-  def goRoot, path, magefile, pythonEnv, testResults, artifacts
+  def goRoot, path, magefile, pythonEnv, testResults, artifacts, gox_flags
 
   if(isUnix()) {
-    goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64"
+    if (isArm() && is64arm()) {
+      // TODO: nodeOS() should support ARM
+      goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.linux.arm64"
+      gox_flags = '-arch arm'
+    } else {
+      goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64"
+      gox_flags = '-arch amd64'
+    }
     path = "${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}"
     magefile = "${WORKSPACE}/.magefile"
     pythonEnv = "${WORKSPACE}/python-env"
     testResults = '**/build/TEST*.xml'
     artifacts = '**/build/TEST*.out'
   } else {
     def chocoPath = 'C:\\ProgramData\\chocolatey\\bin'
+    def mingw64Path = 'C:\\tools\\mingw64\\bin'
     def chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts'
     goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64"
-    path = "${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH}"
+    path = "${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH};${mingw64Path}"
     magefile = "${env.WORKSPACE}\\.magefile"
     testResults = "**\\build\\TEST*.xml"
     artifacts = "**\\build\\TEST*.out"
+    gox_flags = '-arch amd64'
   }
 
   deleteDir()
@@ -253,7 +266,8 @@ def withBeatsEnv(Map args = [:], Closure body) {
     "PYTHON_ENV=${pythonEnv}",
     "RACE_DETECTOR=true",
     "TEST_COVERAGE=true",
-    "TEST_TAGS=${env.TEST_TAGS},oracle"
+    "TEST_TAGS=${env.TEST_TAGS},oracle",
+    "GOX_FLAGS=${gox_flags}"
   ]) {
     if(isDockerInstalled()) {
       dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
@@ -270,6 +284,13 @@ def withBeatsEnv(Map args = [:], Closure body) {
           fi''')
       }
       try {
+        // Add more stability when dependencies are not accessible temporarily
+        // See https://github.com/elastic/beats/issues/21609
+        // retry/try/catch approach reports errors, let's avoid it to keep the
+        // notifications cleaner.
+        if (cmd(label: 'Download modules to local cache', script: 'go mod download', returnStatus: true) > 0) {
+          cmd(label: 'Download modules to local cache - retry', script: 'go mod download', returnStatus: true)
+        }
         body()
       } finally {
         if (archive) {
@@ -357,7 +378,9 @@ def archiveTestOutput(Map args = [:]) {
       def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim()
       log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball")
       if (folder.trim()) {
-        def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + nodeOS()
+        // TODO: nodeOS() should support ARM
+        def os_suffix = isArm() ? 'linux' : nodeOS()
+        def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + os_suffix
         tar(file: "${name}.tgz", archive: true, dir: folder)
       }
     }

diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml
@@ -40,10 +40,9 @@ changeset:
         - "^testing/.*"
         - "^x-pack/libbeat/.*"
 
-## Proposal
-## TBC: This will allow to configure what to do based on the PR configuration
 disabled:
     when:
-        labels:      ## Skip the GitHub Pull Request builds if there is a GitHub label match
-            - "skip-ci"
-        draft: true  ## Skip the GitHub Pull Request builds with Draft PRs.
+        labels:      ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR.
+            - skip-ci
+        ## TODO: This will allow to configure what to do based on the PR configuration
+        draft: true  ## Skip the GitHub Pull Request builds with Draft PRs.
diff --git a/NOTICE.txt b/NOTICE.txt
@@ -4473,12 +4473,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 --------------------------------------------------------------------------------
-Dependency : github.com/elastic/badger/v2
-Version: v2.2007.2-beats
+Dependency : github.com/dgraph-io/badger/v2
+Version: v2.2007.3-0.20201012072640-f5a7e0a1c83b
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/badger/[email protected].2-beats/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/dgraph-io/badger/[email protected].3-0.20201012072640-f5a7e0a1c83b/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml
@@ -13,9 +13,22 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    arm:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "arm"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test auditbeat for arm"
+            labels:
+                - "arm"
+            parameters:
+                - "armTest"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     build:
         mage: "mage build test"
-    crosscompile: 
+    crosscompile:
         make: "make -C auditbeat crosscompile"
     macos:
         mage: "mage build unitTest"
@@ -45,6 +58,17 @@ stages:
                 - "windows-2016"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+    windows-2012:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-2012-r2"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test auditbeat for windows-2012"
+            labels:
+                - "windows-2012"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     windows-10:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.

diff --git a/deploy/cloudfoundry/filebeat/filebeat.yml b/deploy/cloudfoundry/filebeat/filebeat.yml
@@ -11,7 +11,7 @@ filebeat.inputs:
     #doppler_address: ${DOPPLER_ADDRESS}
     #uaa_address: ${UAA_ADDRESS}
     #rlp_address: ${RLP_ADDRESS}
-    #shard_id: ${SHARD_ID}
+    shard_id: ${SHARD_ID}
     #version: v1
 
 

diff --git a/deploy/cloudfoundry/filebeat/manifest.yml b/deploy/cloudfoundry/filebeat/manifest.yml
@@ -1,6 +1,6 @@
 applications:
 - name: filebeat
-  memory: 256M
+  memory: 512M
   instances: 1
   buildpacks:
   - binary_buildpack

diff --git a/deploy/cloudfoundry/metricbeat/manifest.yml b/deploy/cloudfoundry/metricbeat/manifest.yml
@@ -1,6 +1,6 @@
 applications:
   - name: metricbeat
-    memory: 256M
+    memory: 512M
     instances: 1
     buildpacks:
     - binary_buildpack

diff --git a/deploy/cloudfoundry/metricbeat/metricbeat.yml b/deploy/cloudfoundry/metricbeat/metricbeat.yml
@@ -11,7 +11,8 @@ metricbeat.modules:
     #doppler_address: ${DOPPLER_ADDRESS}
     #uaa_address: ${UAA_ADDRESS}
     #rlp_address: ${RLP_ADDRESS}
-    #shard_id: ${SHARD_ID}
+    shard_id: ${SHARD_ID}
+    #version: v1
 
 
 #================================ Outputs =====================================