Merge branch 'master' into auditbeat-fim-update

.ci/packaging.groovy

@@ -23,6 +23,7 @@ pipeline {
   }
   triggers {
     issueCommentTrigger('(?i)^\\/packaging$')
+    upstream('Beats/beats-beats-mbp/master')
   }
   parameters {
     booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
@@ -76,7 +77,19 @@ pipeline {
             }
             environment {
               HOME = "${env.WORKSPACE}"
-              PLATFORMS = "!darwin +linux/armv7 +linux/ppc64le +linux/s390x +linux/mips64"
+              PLATFORMS = [
+                '+all',
+                'linux/amd64',
+                'linux/386',
+                'linux/arm64',
+                'linux/armv7',
+                'linux/ppc64le',
+                'linux/mips64',
+                'linux/s390x',
+                'windows/amd64',
+                'windows/386',
+                (params.macos ? '' : 'darwin/amd64'),
+              ].join(' ')
             }
             steps {
               release()
@@ -94,7 +107,10 @@ pipeline {
             }
             environment {
               HOME = "${env.WORKSPACE}"
-              PLATFORMS = "!defaults +darwin/amd64"
+              PLATFORMS = [
+                '+all',
+                'darwin/amd64',
+              ].join(' ')
             }
             steps {
               withMacOSEnv(){
@@ -109,11 +125,51 @@ pipeline {
 }
 
 def pushCIDockerImages(){
-  sh(label: 'Push Docker image', script: '''
-    if [ -n "$(command -v docker)" ]; then
-      docker images || true
-    fi
-  ''')
+  catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') {
+    if ("${env.BEATS_FOLDER}" == "auditbeat"){
+      tagAndPush('auditbeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "filebeat") {
+      tagAndPush('filebeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "heartbeat"){
+      tagAndPush('heartbeat')
+      tagAndPush('heartbeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "journalbeat"){
+      tagAndPush('journalbeat')
+      tagAndPush('journalbeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "metricbeat"){
+      tagAndPush('metricbeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "packetbeat"){
+      tagAndPush('packetbeat')
+      tagAndPush('packetbeat-oss')
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/auditbeat"){
+      tagAndPush('auditbeat')
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
+      tagAndPush('elastic-agent')
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/filebeat"){
+      tagAndPush('filebeat')
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/metricbeat"){
+      tagAndPush('metricbeat')
+    }
+  }
+}
+
+def tagAndPush(name){
+  def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim()
+  if("${env.SNAPSHOT}" == "true"){
+    libbetaVer += "-SNAPSHOT"
+  }
+  def oldName = "${DOCKER_REGISTRY}/beats/${name}:${libbetaVer}"
+  def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${libbetaVer}"
+  def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}:${env.GIT_BASE_COMMIT}"
+  dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+  retry(3){
+    sh(label:'Change tag and push', script: """
+      docker tag ${oldName} ${newName}
+      docker push ${newName}
+      docker tag ${oldName} ${commitName}
+      docker push ${commitName}
+    """)
+  }
 }
 
 def release(){

.ci/scripts/install-kind.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+MSG="parameter missing."
+DEFAULT_HOME="/usr/local"
+KIND_VERSION=${KIND_VERSION:?$MSG}
+HOME=${HOME:?$DEFAULT_HOME}
+KIND_CMD="${HOME}/bin/kind"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${KIND_CMD}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
+chmod +x "${KIND_CMD}"

.ci/scripts/install-kubectl.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+MSG="parameter missing."
+DEFAULT_HOME="/usr/local"
+K8S_VERSION=${K8S_VERSION:?$MSG}
+HOME=${HOME:?$DEFAULT_HOME}
+KUBECTL_CMD="${HOME}/bin/kubectl"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${KUBECTL_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
+chmod +x "${KUBECTL_CMD}"
+

.ci/scripts/install-terraform.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+MSG="parameter missing."
+TERRAFORM_VERSION=${TERRAFORM_VERSION:?$MSG}
+HOME=${HOME:?$MSG}
+TERRAFORM_CMD="${HOME}/bin/terraform"
+
+OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo - "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${OS}_amd64.zip" > ${TERRAFORM_CMD}.zip
+unzip -o ${TERRAFORM_CMD}.zip -d $(dirname ${TERRAFORM_CMD})
+rm ${TERRAFORM_CMD}.zip
+
+chmod +x "${TERRAFORM_CMD}"

.ci/scripts/kind-setup.sh

@@ -1,18 +1,5 @@
 #!/usr/bin/env bash
 set -exuo pipefail
 
-MSG="parameter missing."
-K8S_VERSION=${K8S_VERSION:?$MSG}
-HOME=${HOME:?$MSG}
-KBC_CMD="${HOME}/bin/kubectl"
-
-mkdir -p "${HOME}/bin"
-
-curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
-chmod +x "${KBC_CMD}"
-
-GO111MODULE="on" go get sigs.k8s.io/[email protected]
 kind create cluster --image kindest/node:${K8S_VERSION}
-
-export KUBECONFIG="$(kind get kubeconfig-path)"
 kubectl cluster-info

.ci/scripts/terraform-cleanup.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+DIRECTORY=${1:-.}
+
+FAILED=0
+for tfstate in $(find $DIRECTORY -name terraform.tfstate); do
+  cd $(dirname $tfstate)
+  if ! terraform destroy -auto-approve; then
+    FAILED=1
+  fi
+  cd -
+done
+
+exit $FAILED

.gitignore

@@ -39,3 +39,7 @@ x-pack/dockerlogbeat/temproot.tar
 *.test
 *.prof
 *.pyc
+
+# Terraform
+*.terraform
+*.tfstate*

.travis.yml

@@ -111,23 +111,28 @@ jobs:
     # Metricbeat
     - os: linux
       before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
-      env: TARGETS="-C metricbeat unit-tests coverage-report"
+      env: TARGETS="-C metricbeat unit-tests"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
-      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
-      env: TARGETS="-C metricbeat integration-tests-environment coverage-report"
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
+      install: 
+        - .ci/scripts/install-kind.sh
+        - .ci/scripts/install-kubectl.sh
+      env:
+        - TARGETS="-C metricbeat integration-tests"
+        - K8S_VERSION=v1.17.2
+        - KIND_VERSION=v0.7.0
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
-      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
-      env: TARGETS="-C metricbeat update system-tests-environment coverage-report"
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C metricbeat system-tests"
       go: $TRAVIS_GO_VERSION
       stage: test
-
     - os: osx
       before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
-      env: TARGETS="TEST_ENVIRONMENT=0 -C metricbeat testsuite"
+      env: TARGETS="-C metricbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
@@ -137,6 +142,21 @@ jobs:
       stage: test
     - os: linux
       before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C x-pack/metricbeat unit-tests"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C x-pack/metricbeat integration-tests"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C x-pack/metricbeat system-tests"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: osx
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
       env: TARGETS="-C x-pack/metricbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test

CHANGELOG-developer.next.asciidoc

@@ -40,6 +40,12 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Extract Elasticsearch client logic from `outputs/elasticsearch` package into new `esclientleg` package. {pull}16150[16150]
 - Rename `queue.BufferConfig.Events` to `queue.BufferConfig.MaxEvents`. {pull}17622[17622]
 - Remove `queue.Feature` and replace `queue.RegisterType` with `queue.RegisterQueueType`. {pull}17666[17666]
+- Introduce APM libbeat instrumentation. `Publish` method on `Client` interface now takes a Context as first argument. {pull}17938[17938]
+- The way configuration files are generated has changed to make it easier to customize parts
+  of the config without requiring changes to libbeat config templates. Generation is now
+  fully based on Go text/template and no longer uses file concatenation to generate the config.
+  Your magefile.go will require a change to adapt the devtool API. See the pull request for
+  more details. {pull}18148[18148]
 
 ==== Bugfixes
 

CHANGELOG.asciidoc

@@ -30,6 +30,7 @@ https://github.com/elastic/beats/compare/v7.5.0...v7.5.1[View commits]
 - Fix docker network stats when multiple interfaces are configured. {issue}14586[14586] {pull}14825[14825]
 - Fix ListMetrics pagination in aws module. {issue}14926[14926] {pull}14942[14942]
 - Fix CPU count in docker/cpu in cases where no `online_cpus` are reported {pull}15070[15070]
+- Add domain state to kvm module {pull}17673[17673]
 
 [[release-notes-7.5.0]]
 === Beats version 7.5.0

CHANGELOG.next.asciidoc

@@ -14,13 +14,16 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Variable substitution from environment variables is not longer supported. {pull}15937{15937}
 - Change aws_elb autodiscover provider field name from elb_listener.* to aws.elb.*. {issue}16219[16219] {pull}16402{16402}
 - Remove `AddDockerMetadata` and `AddKubernetesMetadata` processors from the `script` processor. They can still be used as normal processors in the configuration. {issue}16349[16349] {pull}16514[16514]
+- Introduce APM libbeat instrumentation, active when running the beat with ELASTIC_APM_ACTIVE=true. {pull}17938[17938]
 
 *Auditbeat*
 
 - File integrity dataset (macOS): Replace unnecessary `file.origin.raw` (type keyword) with `file.origin.text` (type `text`). {issue}12423[12423] {pull}15630[15630]
 
 *Filebeat*
-
+- Improve ECS field mappings in panw module.  event.outcome now only contains success/failure per ECS specification. {issue}16025[16025] {pull}17910[17910]
+- Improve ECS categorization field mappings for nginx module. http.request.referrer is now lowercase & http.request.referrer only populated when nginx sets a value {issue}16174[16174] {pull}17844[17844]
+- Improve ECS field mappings in santa module. move hash.sha256 to process.hash.sha256 & move certificate fields to santa.certificate . {issue}16180[16180] {pull}17982[17982]
 
 *Heartbeat*
 
@@ -76,6 +79,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix building on FreeBSD by removing build flags from `add_cloudfoundry_metadata` processor. {pull}17486[17486]
 - Do not rotate log files on startup when interval is configured and rotateonstartup is disabled. {pull}17613[17613]
 - Fix goroutine leak and Elasticsearch output file descriptor leak when output reloading is in use. {issue}10491[10491] {pull}17381[17381]
+- Fix `setup.dashboards.index` setting not working. {pull}17749[17749]
+- Fix Elasticsearch license endpoint URL referenced in error message. {issue}17880[17880] {pull}18030[18030]
+- Fix panic when assigning a key to a `nil` value in an event. {pull}18143[18143]
+- Gives monitoring reporter hosts, if configured, total precedence over corresponding output hosts. {issue}17937[17937] {pull}17991[17991]
+- Arbitrary fields and metadata maps are now deep merged into event. {pull}17958[17958]
+- Change `decode_json_fields` processor, to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958]
 
 *Auditbeat*
 
@@ -113,6 +122,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove migrationVersion map 7.7.0 reference from Kibana dashboard file to fix backward compatibility issues. {pull}17425[17425]
 - Fix issue 17734 to retry on rate-limit error in the Filebeat httpjson input. {issue}17734[17734] {pull}17735[17735]
 - Fixed `cloudfoundry.access` to have the correct `cloudfoundry.app.id` contents. {pull}17847[17847]
+- Fixing `ingress_controller.` fields to be of type keyword instead of text. {issue}17834[17834]
+- Fixed typo in log message. {pull}17897[17897]
+- Fix Cisco ASA ASA 3020** and 106023 messages {pull}17964[17964]
 
 *Heartbeat*
 
@@ -196,9 +208,16 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update RPM packages contained in Beat Docker images. {issue}17035[17035]
 - Update supported versions of `redis` output. {pull}17198[17198]
 - Update documentation for system.process.memory fields to include clarification on Windows os's. {pull}17268[17268]
+- Add `replace` processor for replacing string values of fields. {pull}17342[17342]
 - Add optional regex based cid extractor to `add_kubernetes_metadata` processor. {pull}17360[17360]
 - Add `urldecode` processor to for decoding URL-encoded fields. {pull}17505[17505]
 - Add support for AWS IAM `role_arn` in credentials config. {pull}17658[17658] {issue}12464[12464]
+- Add keystore support for autodiscover static configurations. {pull]16306[16306]
+- Add Kerberos support to Elasticsearch output. {pull}17927[17927]
+- Add support for fixed length extraction in `dissect` processor. {pull}17191[17191]
+- Set `agent.name` to the hostname by default. {issue}16377[16377] {pull}18000[18000]
+- Add config example of how to skip the `add_host_metadata` processor when forwarding logs. {issue}13920[13920] {pull}18153[18153]
+- When using the `decode_json_fields` processor, decoded fields are now deep-merged into existing event. {pull}17958[17958]
 
 *Auditbeat*
 
@@ -207,6 +226,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Log to stderr when running using reference kubernetes manifests. {pull}17443[174443]
 - Fix syscall kprobe arguments for 32-bit systems in socket module. {pull}17500[17500]
 - Fix memory leak on when we miss socket close kprobe events. {pull}17500[17500]
+- Add system module process dataset ECS categorization fields. {pull}18032[18032]
 - Add file integrity module ECS categorization fields. {pull}18012[18012]
 - Add `file.mime_type`, `file.extension`, and `file.drive_letter` for file integrity module. {pull}18012[18012]
 
@@ -270,11 +290,22 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Enhance `elasticsearch/slowlog` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17729[17729]
 - Improve ECS categorization field mappings in misp module. {issue}16026[16026] {pull}17344[17344]
 - Added Unix stream socket support as an input source and a syslog input source. {pull}17492[17492]
+- Improve ECS categorization field mappings in postgresql module. {issue}16177[16177] {pull}17914[17914]
+- Improve ECS categorization field mappings in rabbitmq module. {issue}16178[16178] {pull}17916[17916]
+- Make `decode_cef` processor GA. {pull}17944[17944]
+- Improve ECS categorization field mappings in redis module. {issue}16179[16179] {pull}17918[17918]
+- Improve ECS categorization field mappings for zeek module. {issue}16029[16029] {pull}17738[17738]
+- Improve ECS categorization field mappings for netflow module. {issue}16135[16135] {pull}18108[18108]
+- Added an input option `publisher_pipeline.disable_host` to disable `host.name`
+  from being added to events by default. {pull}18159[18159]
+- Improve ECS categorization field mappings in system module. {issue}16031[16031] {pull}18065[18065]
+- When using the `json.*` setting available on some inputs, decoded fields are now deep-merged into existing event. {pull}17958[17958]
+- Change the `json.*` input settings implementation to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958]
 
 *Heartbeat*
 
 - Allow a list of status codes for HTTP checks. {pull}15587[15587]
-
+- Add additional ECS compatible fields for TLS information. {pull}17687[17687]
 
 *Journalbeat*
 
@@ -343,6 +374,15 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Reference kubernetes manifests mount data directory from the host when running metricbeat as daemonset, so data persist between executions in the same node. {pull}17429[17429]
 - Add more detailed error messages, system tests and small refactoring to the service metricset in windows. {pull}17725[17725]
 - Stack Monitoring modules now auto-configure required metricsets when `xpack.enabled: true` is set. {issue}16471[[16471] {pull}17609[17609]
+- Add Metricbeat IIS module dashboards. {pull}17966[17966]
+- Add dashboard for the azure database account metricset. {pull}17901[17901]
+- Allow partial region and zone name in googlecloud module config. {pull}17913[17913]
+- Add aggregation aligner as a config parameter for googlecloud stackdriver metricset. {issue}17141[[17141] {pull}17719[17719]
+- Move the perfmon metricset to GA. {issue}16608[16608] {pull}17879[17879]
+- Add static mapping for metricsets under aws module. {pull}17614[17614] {pull}17650[17650]
+- Collect new `bulk` indexing metrics from Elasticsearch when `xpack.enabled:true` is set. {issue} {pull}17992[17992]
+- Remove requirement to connect as sysdba in Oracle module {issue}15846[15846] {pull}18182[18182]
+- Update MSSQL module to fix some SSPI authentication and add brackets to USE statements {pull}17862[17862]]
 
 *Packetbeat*
 
@@ -353,6 +393,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - Add more DNS error codes to the Sysmon module. {issue}15685[15685]
 - Add experimental event log reader implementation that should be faster in most cases. {issue}6585[6585] {pull}16849[16849]
+- Set process.command_line and process.parent.command_line from Sysmon Event ID 1. {pull}17327[17327]
+- Add support for event IDs 4673,4674,4697,4698,4699,4700,4701,4702,4768,4769,4770,4771,4776,4778,4779,4964 to the Security module {pull}17517[17517]
 
 ==== Deprecated