Merge remote-tracking branch 'upstream/master' into feature/mergify

* upstream/master: (91 commits)
  [Filebeat] Change okta.target to nested field (elastic#24636)
  Add RFC5424 format support for syslog input  (elastic#23954)
  Fix links to Beats product pages (elastic#24821)
  [DOCS] Fix 'make setup' instructions for a new beat (elastic#24944)
  Remove duplicate decode_xml entry (elastic#24941)
  [libbeat] Add wineventlog schema to decode_xml processor (elastic#24726)
  [Elastic Agent] Add check for URL set when cert and cert key. (elastic#24904)
  feat: stage execution cache (elastic#24780)
  Fix error in Journalbeat commands (elastic#24880)
  Add baseline ECS 1.9.0 upgrade (elastic#24909)
  [Elastic Agent] Cloud container legacy apm files. (elastic#24896)
  [Elastic Agent]: Reduce allowed socket path length (elastic#24914)
  Add ability to destroy indices with wildcards in testing (elastic#24915)
  Add status subcommand to report status of running daemon. (elastic#24856)
  Fix types of fields GetHits and Ops in Metricbeat module for Couchbase (elastic#23287)
  Add support for Filestream input in elastic agent. (elastic#24820)
  Implement k8s secrets provider for Agent (elastic#24789)
  Sort processor list in docs (elastic#24874)
  Add support for SCRAM authentication in kafka metricbeat module (elastic#24810)
  Properly update offset in case of unparasable line (elastic#22685)
  ...

.ci/packaging.groovy

@@ -115,6 +115,7 @@ pipeline {
                    'x-pack/heartbeat',
                   // 'x-pack/journalbeat',
                   'x-pack/metricbeat',
+                  'x-pack/osquerybeat',
                   'x-pack/packetbeat',
                   'x-pack/winlogbeat'
                 )
@@ -290,6 +291,8 @@ def pushCIDockerImages(Map args = [:]) {
       tagAndPush(beatName: 'journalbeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('metricbeat')) {
       tagAndPush(beatName: 'metricbeat', arch: arch)
+    } else if (env?.BEATS_FOLDER?.endsWith('osquerybeat')) {
+      tagAndPush(beatName: 'osquerybeat', arch: arch)
     } else if ("${env.BEATS_FOLDER}" == "packetbeat"){
       tagAndPush(beatName: 'packetbeat', arch: arch)
     } else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {

.ci/scripts/install-go.sh

@@ -11,13 +11,15 @@ GVM_CMD="${HOME}/bin/gvm"
 
 if command -v go
 then
+    set +e
     echo "Found Go. Checking version.."
     FOUND_GO_VERSION=$(go version|awk '{print $3}'|sed s/go//)
-    if [ $FOUND_GO_VERSION == $GO_VERSION ]
+    if [ "$FOUND_GO_VERSION" == "$GO_VERSION" ]
     then
         echo "Versions match. No need to install Go. Exiting."
         exit 0
     fi
+    set -e
 fi
 
 if [ "${ARCH}" == "aarch64" ] ; then

.github/CODEOWNERS

@@ -13,20 +13,20 @@
 # /winlogbeat/ @elastic/beats
 
 # Auditbeat
-/auditbeat/module/ @elastic/siem
-/x-pack/auditbeat/ @elastic/siem
+/auditbeat/module/ @elastic/security-external-integrations
+/x-pack/auditbeat/ @elastic/security-external-integrations
 
 # Packetbeat
-/packetbeat/protos/ @elastic/siem
-/x-pack/packetbeat/ @elastic/siem
+/packetbeat/protos/ @elastic/security-external-integrations
+/x-pack/packetbeat/ @elastic/security-external-integrations
 
 # Filebeat
 # /filebeat/module/ @elastic/integrations
 # /filebeat/module/elasticsearch/ @elastic/stack-monitoring
 # /filebeat/module/kibana/ @elastic/stack-monitoring
 # /filebeat/module/logstash/ @elastic/stack-monitoring
 # /x-pack/filebeat/module/ @elastic/integrations
-# /x-pack/filebeat/module/suricata/ @elastic/secops
+# /x-pack/filebeat/module/suricata/ @elastic/security-external-integrations
 
 # Metricbeat
 # /metricbeat/module/ @elastic/integrations
@@ -40,7 +40,7 @@
 /heartbeat/ @elastic/uptime
 
 # Winlogbeat
-/x-pack/winlogbeat/ @elastic/siem
+/x-pack/winlogbeat/ @elastic/security-external-integrations
 
 # CI Specific
 /.ci/ @elastic/observablt-robots

CHANGELOG-developer.next.asciidoc

@@ -110,3 +110,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Go version to 1.15.7. {pull}22495[22495]
 - Update Go version to 1.15.8. {pull}23955[23955]
 - Update Go version to 1.15.9. {pull}24442[24442]
+- Update Go version to 1.15.10. {pull}24606[24606]

CHANGELOG.asciidoc

@@ -3,6 +3,166 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.12.0]]
+=== Beats version 7.12.0
+https://github.com/elastic/beats/compare/v7.11.2...v7.12.0[View commits]
+
+==== Breaking changes
+
+*Filebeat*
+
+- Rename `s3` input to `aws-s3` input. {pull}23469[23469]
+
+*Heartbeat*
+
+- Refactor synthetics configuration to new syntax. {pull}23467[23467]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Fix `nested` subfield handling in generated Elasticsearch templates. {issue}23178[23178] {pull}23183[23183]
+- Fix CPU usage metrics on VMs with dynamic CPU config {pull}23154[23154]
+- Allow configuring credential_profile_name and shared_credential_file when using role_arn. {pull}24174[24174]
+- Fix panic with inline SSL when the certificate or key was smaller than 256 bytes. {issue}23820[23820] {pull}23858[23858]
+
+*Auditbeat*
+
+- system/login: Fixed offset reset on inode reuse. {pull}24414[24414]
+- system/login: Add additional offset check for utmp files. {pull}24515[24515]
+
+*Filebeat*
+
+- CheckPoint Firewall module: Change event.severity JSON data type to a number because the field mapping is a `long`. {pull}23424[23424]
+- Cisco IOS: Change icmp.type/code and igmp.type JSON data types to strings because the fields mappings are `keyword`. {pull}23424[23424]
+- CrowdStrike Falcon: Change JSON field types to match the field mappings. {pull}23424[23424]
+- Fortinet Firewall: Drop `fortinet.firewall.assignip` when the value is "N/A". {pull}23424[23424]
+- Juniper SRX: Change JSON field types to match the field mappings. {pull}23424[23424]
+- Suricata EVE: Convert `suricata.eve.flow_id` to string because the field is a keyword in the mapping. {pull}23424[23424]
+- Zeek DNS: Ignore failures in data type conversions. And change `dns.id` JSON field to a string to match its `keyword` mapping. {pull}23424[23424]
+- Update `filestream` reader offset when a line is skipped. {pull}23417[23417]
+- Add check for empty values in azure module. {pull}24156[24156]
+- Change the `event.created` in Netflow events to be the time the event was created by Filebeat
+- Fix Zoom module parameters for basic auth and url path. {pull}23779[23779]
+- Use rfc6587 framing for fortinet firewall and clientendpoint filesets when transferring over tcp. {pull}23837[23837]
+- Fix httpjson input logging so it doesn't conflict with ECS. {pull}23972[23972]
+- Fix Logstash module handling of logstash.log.log_event.action field. {issue}20709[20709]
+- aws/s3access dataset was populating event.duration using the wrong unit. {pull}23920[23920]
+- Zoom module pipeline failed to ingest some chat_channel events. {pull}23904[23904]
+- Fix Netlow module issue with missing `internal_networks` config parameter. {issue}24094[24094] {pull}24110[24110]
+- in httpjson input using encode_as "application/x-www-form-urlencoded" now sets Content-Type correctly {issue}24331[24331] {pull}24336[24336]
+- Fix default `scope` in `add_nomad_metadata`. {issue}24559[24559]
+
+*Metricbeat*
+
+- Add stack monitoring section to elasticsearch module documentation {pull}#23286[23286]
+- Fix ec2 metricset fields.yml and the integration test {pull}23726[23726]
+- Unskip s3_request integration test. {pull}23887[23887]
+- Add system.hostfs configuration option for system module. {pull}23831[23831]
+
+==== Added
+
+*Affecting all Beats*
+
+- Honor kube event resysncs to handle missed watch events {pull}22668[22668]
+- Add autodiscover provider and metadata processor for Nomad. {pull}14954[14954] {pull}23324[23324]
+- Add `processors.rate_limit.n.dropped` monitoring counter metric for the `rate_limit` processor. {pull}23330[23330]
+- Deprecate aws_partition config parameter for AWS, use endpoint instead. {pull}23539[23539]
+- Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595]
+- Add kubernetes.volume.fs.used.pct field. {pull}23564[23564]
+- Add the `enable_krb5_fast` flag to the Kafka output to explicitly opt-in to FAST authentication. {pull}23629[23629]
+- Added new decode_xml processor to libbeat that is available to all beat types. {pull}23678[23678]
+- Add deployment name in pod's meta. {pull}23610[23610]
+- Added ECS 1.8 `host.os.type` field to `add_host_metadata` processor. {pull}23513[23513]
+- Add `selector` information in Kubernetes services' metadata. {pull}23730[23730]
+
+*Auditbeat*
+
+- Improve file_integrity monitoring when a file is created/deleted in quick succession. {issue}17347[17347] {pull}22170[22170]
+- system/host: Add new ECS 1.8 field `os.type` in `host.os.type`. {pull}23513[23513]
+- Update Auditbeat auditd module to ECS 1.8 {pull}23594[23594] {issue}23118[23118]
+
+*Filebeat*
+
+- Add parsing of tcp flags to AWS vpcflow fileset {issue}228020[22820] {pull}23157[23157]
+- Added support for first_event context in Filebeat httpjson input {pull}23437[23437]
+- Adding Threat Intel module {pull}21795[21795]
+- Added username parsing from Cisco ASA message 302013. {pull}21196[21196]
+- Added `encode_as` and `decode_as` options to httpjson along with pluggable encoders/decoders {pull}23478[23478]
+- Added feature to modules to adapt Ingest Node pipelines for compatibility with older Elasticsearch versions by removing unsupported processors. {pull}23763[23763]
+- Added support for Cisco AMP API as a new fileset. {pull}22768[22768]
+- Added RFC6587 framing option for tcp and unix inputs {issue}23663[23663] {pull}23724[23724]
+- Added `application/x-ndjson` as decode option for httpjson input {pull}23521[23521]
+- Added `application/x-www-form-urlencoded` as encode option for httpjson input {pull}23521[23521]
+- Move aws-s3 input to GA. {pull}23631[23631]
+- Populate `source.mac` and `destination.mac` for Suricata EVE events. {issue}23706[23706] {pull}23721[23721]
+- Added string splitting for httpjson input {pull}24022[24022]
+- Added Signatures fileset to Zeek module {pull}23772[23772]
+- Upgrade Cisco ASA/FTD/Umbrella to ECS 1.8.0. {pull}23819[23819]
+- Add new ECS user and categories features to google_workspace/gsuite {issue}23118[23118] {pull}23709[23709]
+- Move crowdstrike JS processor to ingest pipelines and upgrade to ECS 1.8.0 {issue}23118[23118] {pull}23875[23875]
+- Update Filebeat auditd dataset to ECS 1.8.0. {pull}23723[23723] {issue}23118[23118]
+- Updated microsoft defender_atp and m365_defender to ECS 1.8. {pull}23897[23897] {issue}23118[23118]
+- Updated o365 module to ECS 1.8. {issue}23118[23118] {pull}23896[23896]
+- Upgrade CEF module to ECS 1.8.0. {pull}23832[23832]
+- Upgrade fortinet/firewall to ECS 1.8 {issue}23118[23118] {pull}23902[23902]
+- Upgrade Zeek to ECS 1.8.0. {issue}23118[23118] {pull}23847[23847]
+- Updated azure module to ECS 1.8. {issue}23118[23118] {pull}23927[23927]
+- Update aws/s3access to ECS 1.8. {issue}23118[23118] {pull}23920[23920]
+- Upgrade panw module to ECS 1.8 {issue}23118[23118] {pull}23931[23931]
+- Updated aws/cloudtrail fileset to ECS 1.8. {issue}23118[23118] {pull}23911[23911]
+- Upgrade juniper/srx to ECS 1.8.0. {issue}23118[23118] {pull}23936[23936]
+- Update mysqlenterprise module to ECS 1.8. {issue}23118[23118] {pull}23978[23978]
+- Upgrade sophos/xg fileset to ECS 1.8.0. {issue}23118[23118] {pull}23967[23967]
+- Upgrade system/auth to ECS 1.8 {issue}23118[23118] {pull}23961[23961]
+- Upgrade elasticsearch/audit to ECS 1.8 {issue}23118[23118] {pull}24000[24000]
+- Upgrade okta to ECS 1.8.0 and move js processor to ingest pipeline {issue}23118[23118] {pull}23929[23929]
+- Update zoom module to ECS 1.8. {pull}23904[23904] {issue}23118[23118]
+- Add fileset to ingest PostgreSQL CSV logs. {pull}23334[23334]
+
+*Heartbeat*
+
+- Bundle synthetics dependencies with Heartbeat docker image. {pull}23274[23274]
+
+*Heartbeat*
+
+- Update Journalbeat to ECS 1.8. {pull}23737[23737]
+
+*Metricbeat*
+
+- Enrich events of `state_service` metricset with Kubernetes services' metadata. {pull}23730[23730]
+- Add support for Darwin/arm M1. {pull}24019[24019]
+- Check fields are documented in AWS metricsets. {pull}23887[23887]
+- Add container.image.name and containe.name ECS fields for state_container. {pull}23802[23802]
+- Add support for the MemoryPressure, DiskPressure, OutOfDisk and PIDPressure status conditions in state_node. {pull}23905[23905]
+
+*Packetbeat*
+
+- Upgrade to ECS 1.8.0. {pull}23783[23783]
+- Add `event.type: [connection]` to flow events and include `end` for final flows. {pull}24564[24564]
+
+*Functionbeat*
+
+- Provide more ways to set AWS credentials. {issue}12464[12464] {pull}23344[23344]
+- Add support for multiple regions {pull}21065[21065]
+
+*Heartbeat*
+
+- Add support for script processor. {pull}23229[23229]
+
+*Winlogbeat*
+
+- Add Audit and Authentication Policy Change Events and related.ip information {pull}20684[20684]
+- Add new ECS 1.8 improvements. {pull}23563[23563]
+- Remove deprecated eventlogging API that was used for Windows XP/2003 and associated unused code. {pull}24463[24463]
+
+==== Deprecated
+
+*Affecting all Beats*
+
+- Selecting `full` in `ssl.verification_mode` option will not treat CommonName field in x509 certificates as a hostname when Subject Alternative Name is not present from v8.0. Please update your certificates so it contains at least one DNSName instead of relying on CommonName in the new major version of Beats.
+
+
 [[release-notes-7.11.2]]
 === Beats version 7.11.2
 https://github.com/elastic/beats/compare/v7.11.1...v7.11.2[View commits]