Merge remote-tracking branch 'upstream/master' into update-stadanalon…

…e-agent-manifest-conditions

.ci/packaging.groovy

@@ -246,7 +246,8 @@ pipeline {
           agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           steps {
-            runE2ETests()
+            log(level: 'WARN', text: "E2E Tests for Beats are disabled until latest breaking changes in Kibana affecting Package Registry are resolved.")
+            //runE2ETests()
           }
         }
       }

.go-version

@@ -1 +1 @@
-1.17.1
+1.17.2

CHANGELOG.asciidoc

@@ -69,7 +69,6 @@ https://github.com/elastic/beats/compare/v7.14.2...v7.15.0[View commits]
 *Auditbeat*
 
 - File Integrity Module: Honor `include_files` when doing initial scan. {issue}27273[27273] {pull}27722[27722]
-- Fix handling of root and relative paths {issue}24430[24430] {pull}28354[28354]
 
 *Filebeat*
 

Jenkinsfile

@@ -617,7 +617,8 @@ def target(Map args = [:]) {
           pushCIDockerImages(beatsFolder: "${directory}", arch: dockerArch)
         }
         if(isE2E) {
-          e2e(args)
+          log(level: 'WARN', text: "E2E Tests for Beats are disabled until latest breaking changes in Kibana affecting Package Registry are resolved.")
+          //e2e(args)
         }
       }
     }

Jenkinsfile.yml

@@ -2,9 +2,9 @@ projects:
     - "auditbeat"
     - "deploy/kubernetes"
     - "filebeat"
-    - "generator"
+    # Skipping because they are failing, see https://github.com/elastic/beats/pull/28723
+    #- "generator"
     - "heartbeat"
-    - "journalbeat"
     - "libbeat"
     - "metricbeat"
     - "packetbeat"
@@ -21,7 +21,6 @@ projects:
     - "x-pack/packetbeat"
     - "x-pack/winlogbeat"
     - "dev-tools"
-    ##- "x-pack/journalbeat"  It is not yet in the 1.0 pipeline.
 
 ## Changeset macros that are defined here and used in each specific 2.0 pipeline.
 changeset:

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17.1
+FROM golang:1.17.2
 
 RUN \
     apt-get update \

dev-tools/notice/overrides.json

@@ -12,4 +12,5 @@
 {"name": "github.com/pelletier/go-buffruneio", "licenceType": "MIT"}
 {"name": "github.com/urso/magetools", "licenceType": "Apache-2.0"}
 {"name": "kernel.org/pub/linux/libs/security/libcap/cap", "licenceType": "BSD-3-Clause", "note": "dual licensed as GPL-v2 and BSD"}
-{"name": "kernel.org/pub/linux/libs/security/libcap/psx", "licenceType": "BSD-3-Clause", "note": "dual licensed as GPL-v2 and BSD"}
+{"name": "kernel.org/pub/linux/libs/security/libcap/psx", "licenceType": "BSD-3-Clause", "note": "dual licensed as GPL-v2 and BSD"}
+{"name": "github.com/awslabs/kinesis-aggregation/go", "licenceType": "Apache-2.0", "url": "https://github.com/awslabs/kinesis-aggregation/blob/master/LICENSE.txt"}

filebeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17.1
+FROM golang:1.17.2
 
 RUN \
     apt-get update \

filebeat/docs/modules/google_workspace.asciidoc

@@ -37,9 +37,9 @@ It is compatible with a subset of applications under the https://developers.goog
 
 In order for Filebeat to ingest data from the Google Reports API you must:
 
-- Have an *administrator account*.
+- Have an *administrator account*, as described https://developers.google.com/admin-sdk/reports/v1/guides/prerequisites[here].
 - https://support.google.com/workspacemigrate/answer/9222993?hl=en[Set up a ServiceAccount] using the administrator account.
-- https://support.google.com/workspacemigrate/answer/9222865?hl=en[Set up access to the Admin SDK API] for the ServiceAccount.
+- https://developers.google.com/admin-sdk/reports/v1/guides/authorizing[Set up access to the Admin SDK API] for the ServiceAccount.
 - https://developers.google.com/admin-sdk/reports/v1/guides/delegation[Enable Domain-Wide Delegation] for your ServiceAccount.
 
 This module will make use of the following *oauth2 scope*:

filebeat/docs/modules/okta.asciidoc

@@ -102,6 +102,17 @@ An initial interval can be defined. The first time the module starts, will fetch
     var.initial_interval: 24h # will fetch events starting 24h ago.
 ----
 
+*`input.request.rate_limit.early_limit`*::
+
+You can override the default rate-limiting behavior in <<filebeat-input-httpjson>>.
+The default for the Okta module is to use up to 89% of the Okta rate-limit,
+which should avoid Okta Warnings on rate-limit usage.
++
+[source.yaml]
+----
+    input.request.rate_limit.early_limit: 0.89
+----
+
 [float]
 === Example dashboard
 

filebeat/docs/modules/oracle.asciidoc

@@ -10,7 +10,6 @@ This file is generated! See scripts/docs_collector.py
 
 
 == Oracle module
-beta[]
 
 This is a module for ingesting Audit Trail logs from Oracle Databases.
 

filebeat/module/apache/access/test/darwin-2.4.23.log

@@ -1,6 +1,6 @@
 ::1 - - [26/Dec/2016:16:16:28 +0200] "GET / HTTP/1.1" 200 45
 ::1 - - [26/Dec/2016:16:16:29 +0200] "GET /favicon.ico HTTP/1.1" 404 209
 ::1 - - [26/Dec/2016:16:16:48 +0200] "-" 408 -
-77.179.66.156 - - [26/Dec/2016:18:23:35 +0200] "GET / HTTP/1.1" 200 45
-77.179.66.156 - - [26/Dec/2016:18:23:41 +0200] "GET /notfound HTTP/1.1" 404 206
-77.179.66.156 - - [26/Dec/2016:18:23:45 +0200] "GET /hmm HTTP/1.1" 404 201
+89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] "GET / HTTP/1.1" 200 45
+89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] "GET /notfound HTTP/1.1" 404 206
+89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] "GET /hmm HTTP/1.1" 404 201

filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json

@@ -67,7 +67,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
-        "event.original": "77.179.66.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45",
+        "event.original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45",
         "event.outcome": "success",
         "fileset.name": "access",
         "http.request.method": "GET",
@@ -77,18 +77,18 @@
         "input.type": "log",
         "log.offset": 181,
         "service.type": "apache",
-        "source.address": "77.179.66.156",
-        "source.as.number": 6805,
-        "source.as.organization.name": "Telefonica Germany",
-        "source.geo.city_name": "Germersheim",
+        "source.address": "89.160.20.156",
+        "source.as.number": 29518,
+        "source.as.organization.name": "Bredband2 AB",
+        "source.geo.city_name": "Link\u00f6ping",
         "source.geo.continent_name": "Europe",
-        "source.geo.country_iso_code": "DE",
-        "source.geo.country_name": "Germany",
-        "source.geo.location.lat": 49.2231,
-        "source.geo.location.lon": 8.3639,
-        "source.geo.region_iso_code": "DE-RP",
-        "source.geo.region_name": "Rheinland-Pfalz",
-        "source.ip": "77.179.66.156",
+        "source.geo.country_iso_code": "SE",
+        "source.geo.country_name": "Sweden",
+        "source.geo.location.lat": 58.4167,
+        "source.geo.location.lon": 15.6167,
+        "source.geo.region_iso_code": "SE-E",
+        "source.geo.region_name": "\u00d6sterg\u00f6tland County",
+        "source.ip": "89.160.20.156",
         "url.original": "/",
         "url.path": "/",
         "user.name": "-"
@@ -99,7 +99,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
-        "event.original": "77.179.66.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206",
+        "event.original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206",
         "event.outcome": "failure",
         "fileset.name": "access",
         "http.request.method": "GET",
@@ -109,18 +109,18 @@
         "input.type": "log",
         "log.offset": 252,
         "service.type": "apache",
-        "source.address": "77.179.66.156",
-        "source.as.number": 6805,
-        "source.as.organization.name": "Telefonica Germany",
-        "source.geo.city_name": "Germersheim",
+        "source.address": "89.160.20.156",
+        "source.as.number": 29518,
+        "source.as.organization.name": "Bredband2 AB",
+        "source.geo.city_name": "Link\u00f6ping",
         "source.geo.continent_name": "Europe",
-        "source.geo.country_iso_code": "DE",
-        "source.geo.country_name": "Germany",
-        "source.geo.location.lat": 49.2231,
-        "source.geo.location.lon": 8.3639,
-        "source.geo.region_iso_code": "DE-RP",
-        "source.geo.region_name": "Rheinland-Pfalz",
-        "source.ip": "77.179.66.156",
+        "source.geo.country_iso_code": "SE",
+        "source.geo.country_name": "Sweden",
+        "source.geo.location.lat": 58.4167,
+        "source.geo.location.lon": 15.6167,
+        "source.geo.region_iso_code": "SE-E",
+        "source.geo.region_name": "\u00d6sterg\u00f6tland County",
+        "source.ip": "89.160.20.156",
         "url.original": "/notfound",
         "url.path": "/notfound",
         "user.name": "-"
@@ -131,7 +131,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
-        "event.original": "77.179.66.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201",
+        "event.original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201",
         "event.outcome": "failure",
         "fileset.name": "access",
         "http.request.method": "GET",
@@ -141,18 +141,18 @@
         "input.type": "log",
         "log.offset": 332,
         "service.type": "apache",
-        "source.address": "77.179.66.156",
-        "source.as.number": 6805,
-        "source.as.organization.name": "Telefonica Germany",
-        "source.geo.city_name": "Germersheim",
+        "source.address": "89.160.20.156",
+        "source.as.number": 29518,
+        "source.as.organization.name": "Bredband2 AB",
+        "source.geo.city_name": "Link\u00f6ping",
         "source.geo.continent_name": "Europe",
-        "source.geo.country_iso_code": "DE",
-        "source.geo.country_name": "Germany",
-        "source.geo.location.lat": 49.2231,
-        "source.geo.location.lon": 8.3639,
-        "source.geo.region_iso_code": "DE-RP",
-        "source.geo.region_name": "Rheinland-Pfalz",
-        "source.ip": "77.179.66.156",
+        "source.geo.country_iso_code": "SE",
+        "source.geo.country_name": "Sweden",
+        "source.geo.location.lat": 58.4167,
+        "source.geo.location.lon": 15.6167,
+        "source.geo.region_iso_code": "SE-E",
+        "source.geo.region_name": "\u00d6sterg\u00f6tland County",
+        "source.ip": "89.160.20.156",
         "url.original": "/hmm",
         "url.path": "/hmm",
         "user.name": "-"

filebeat/module/apache/access/test/ssl-request.log

@@ -1,2 +1,2 @@
 [10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 "GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1" 1375
-[16/Oct/2019:11:53:47 +0200] 11.19.0.217 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 "GET /appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1" -
+[16/Oct/2019:11:53:47 +0200] 81.2.69.143 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 "GET /appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1" -

filebeat/module/apache/access/test/ssl-request.log-expected.json

@@ -33,20 +33,23 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
-        "event.original": "[16/Oct/2019:11:53:47 +0200] 11.19.0.217 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -",
+        "event.original": "[16/Oct/2019:11:53:47 +0200] 81.2.69.143 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -",
         "fileset.name": "access",
         "http.request.method": "GET",
         "http.version": "1.1",
         "input.type": "log",
         "log.offset": 276,
         "service.type": "apache",
-        "source.address": "11.19.0.217",
-        "source.geo.continent_name": "North America",
-        "source.geo.country_iso_code": "US",
-        "source.geo.country_name": "United States",
-        "source.geo.location.lat": 37.751,
-        "source.geo.location.lon": -97.822,
-        "source.ip": "11.19.0.217",
+        "source.address": "81.2.69.143",
+        "source.geo.city_name": "London",
+        "source.geo.continent_name": "Europe",
+        "source.geo.country_iso_code": "GB",
+        "source.geo.country_name": "United Kingdom",
+        "source.geo.location.lat": 51.5142,
+        "source.geo.location.lon": -0.0931,
+        "source.geo.region_iso_code": "GB-ENG",
+        "source.geo.region_name": "England",
+        "source.ip": "81.2.69.143",
         "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
         "tls.version": "1.2",
         "tls.version_protocol": "tls",

filebeat/module/apache/error/test/test.log

@@ -1,4 +1,4 @@
 [Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico
 [Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'
-[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 72.15.99.187] File does not exist: /usr/local/apache2/htdocs/favicon.ico
-[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 123.123.123.123:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html
+[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.112] File does not exist: /usr/local/apache2/htdocs/favicon.ico
+[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 67.43.156.12:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html

filebeat/module/apache/error/test/test.log-expected.json

@@ -43,7 +43,7 @@
         "event.dataset": "apache.error",
         "event.kind": "event",
         "event.module": "apache",
-        "event.original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 72.15.99.187] File does not exist: /usr/local/apache2/htdocs/favicon.ico",
+        "event.original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.112] File does not exist: /usr/local/apache2/htdocs/favicon.ico",
         "event.timezone": "-02:00",
         "event.type": "error",
         "file.path": "/usr/local/apache2/htdocs/favicon.ico",
@@ -55,18 +55,18 @@
         "process.pid": 35708,
         "process.thread.id": 4328636416,
         "service.type": "apache",
-        "source.address": "72.15.99.187",
-        "source.as.number": 11693,
-        "source.as.organization.name": "WideOpenWest Finance LLC",
-        "source.geo.city_name": "Newnan",
-        "source.geo.continent_name": "North America",
-        "source.geo.country_iso_code": "US",
-        "source.geo.country_name": "United States",
-        "source.geo.location.lat": 33.3708,
-        "source.geo.location.lon": -84.8154,
-        "source.geo.region_iso_code": "US-GA",
-        "source.geo.region_name": "Georgia",
-        "source.ip": "72.15.99.187"
+        "source.address": "89.160.20.112",
+        "source.as.number": 29518,
+        "source.as.organization.name": "Bredband2 AB",
+        "source.geo.city_name": "Link\u00f6ping",
+        "source.geo.continent_name": "Europe",
+        "source.geo.country_iso_code": "SE",
+        "source.geo.country_name": "Sweden",
+        "source.geo.location.lat": 58.4167,
+        "source.geo.location.lon": 15.6167,
+        "source.geo.region_iso_code": "SE-E",
+        "source.geo.region_name": "\u00d6sterg\u00f6tland County",
+        "source.ip": "89.160.20.112"
     },
     {
         "@timestamp": "2019-06-27T06:58:09.169-02:00",
@@ -75,28 +75,24 @@
         "event.dataset": "apache.error",
         "event.kind": "event",
         "event.module": "apache",
-        "event.original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 123.123.123.123:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html",
+        "event.original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 67.43.156.12:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html",
         "event.timezone": "-02:00",
         "event.type": "error",
         "fileset.name": "error",
         "input.type": "log",
         "log.level": "warn",
-        "log.offset": 384,
+        "log.offset": 385,
         "message": "AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html",
         "process.pid": 15934,
         "service.type": "apache",
-        "source.address": "123.123.123.123",
-        "source.as.number": 4808,
-        "source.as.organization.name": "China Unicom Beijing Province Network",
-        "source.geo.city_name": "Beijing",
+        "source.address": "67.43.156.12",
+        "source.as.number": 35908,
         "source.geo.continent_name": "Asia",
-        "source.geo.country_iso_code": "CN",
-        "source.geo.country_name": "China",
-        "source.geo.location.lat": 39.9288,
-        "source.geo.location.lon": 116.3889,
-        "source.geo.region_iso_code": "CN-BJ",
-        "source.geo.region_name": "Beijing",
-        "source.ip": "123.123.123.123",
+        "source.geo.country_iso_code": "BT",
+        "source.geo.country_name": "Bhutan",
+        "source.geo.location.lat": 27.5,
+        "source.geo.location.lon": 90.5,
+        "source.ip": "67.43.156.12",
         "source.port": "12345"
     }
 ]

filebeat/module/auditd/log/test/audit-rhel6.log

@@ -6,7 +6,7 @@ type=USER_START msg=audit(1489519256.193:19600331): user pid=4151 uid=0 auid=700
 type=MAC_IPSEC_EVENT msg=audit(1489519382.529:19600354): op=SPD-add auid=4294967295 ses=4294967295 res=1 src=10.100.0.0 src_prefixlen=16 dst=10.100.4.0 dst_prefixlen=22
 type=SYSCALL msg=audit(1489519382.529:19600354): arch=c000003e syscall=44 success=yes exit=184 a0=9 a1=7f564ee6d2a0 a2=b8 a3=0 items=0 ppid=1240 pid=1275 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm="charon" exe=2F7573722F6C6962657865632F7374726F6E677377616E2F636861726F6E202864656C6574656429 key=(null)
 type=LOGIN msg=audit(1489636960.072:19623791): pid=28281 uid=0 old auid=700 new auid=700 old ses=6793 new ses=12286
-type=CRYPTO_KEY_USER msg=audit(1489636960.070:19623788): user pid=28281 uid=0 auid=700 ses=6793 msg='op=destroy kind=session fp=? direction=both spid=28282 suid=74 rport=58994 laddr=107.170.139.210 lport=50022  exe="/usr/sbin/sshd" hostname=? addr=96.241.146.97 terminal=? res=success'
-type=USER_AUTH msg=audit(1489636960.072:19623789): user pid=28281 uid=0 auid=700 ses=6793 msg='op=success acct="admin" exe="/usr/sbin/sshd" hostname=? addr=96.241.146.97 terminal=ssh res=success'
+type=CRYPTO_KEY_USER msg=audit(1489636960.070:19623788): user pid=28281 uid=0 auid=700 ses=6793 msg='op=destroy kind=session fp=? direction=both spid=28282 suid=74 rport=58994 laddr=107.170.139.210 lport=50022  exe="/usr/sbin/sshd" hostname=? addr=216.160.83.61 terminal=? res=success'
+type=USER_AUTH msg=audit(1489636960.072:19623789): user pid=28281 uid=0 auid=700 ses=6793 msg='op=success acct="admin" exe="/usr/sbin/sshd" hostname=? addr=216.160.83.61 terminal=ssh res=success'
 type=USER_AUTH msg=audit(1489636977.804:19623807): user pid=28395 uid=0 auid=700 ses=12286 msg='op=PAM:authentication acct="root" exe="/bin/su" hostname=? addr=? terminal=pts/0 res=success'
 type=USER_ACCT msg=audit(1489636977.805:19623808): user pid=28395 uid=0 auid=700 ses=12286 msg='op=PAM:accounting acct="root" exe="/bin/su" hostname=? addr=? terminal=pts/0 res=success'