From 38b707746fb7b65e534957894601cc7e6255bf16 Mon Sep 17 00:00:00 2001 From: Denis Rechkunov Date: Wed, 23 Nov 2022 11:57:36 +0100 Subject: [PATCH 1/4] Update test versions of NGINX, Apache and MongoDB The old tests were based on Debian Jessie released in April 2015 (end-of-life on June 30, 2020). We started having problems with expired GPG keys when installing dependencies in our Docker files. Add the missing MPM module and the `ServerName` directive for Apache. --- metricbeat/docs/modules/apache.asciidoc | 2 +- metricbeat/docs/modules/nginx.asciidoc | 2 +- metricbeat/module/apache/_meta/Dockerfile | 3 +-- metricbeat/module/apache/_meta/docs.asciidoc | 2 +- metricbeat/module/apache/_meta/httpd.conf | 3 ++- metricbeat/module/apache/_meta/supported-versions.yml | 6 ++++-- metricbeat/module/apache/docker-compose.yml | 6 +++--- metricbeat/module/mongodb/_meta/Dockerfile | 3 +-- metricbeat/module/mongodb/docker-compose.yml | 4 ++-- metricbeat/module/nginx/_meta/Dockerfile | 3 +-- metricbeat/module/nginx/_meta/docs.asciidoc | 2 +- metricbeat/module/nginx/docker-compose.yml | 8 ++++---- 12 files changed, 22 insertions(+), 22 deletions(-) diff --git a/metricbeat/docs/modules/apache.asciidoc b/metricbeat/docs/modules/apache.asciidoc index e632bbe6313..0bb38c2fd3d 100644 --- a/metricbeat/docs/modules/apache.asciidoc +++ b/metricbeat/docs/modules/apache.asciidoc @@ -19,7 +19,7 @@ HTTPD] servers. The default metricset is `status`. [float] === Compatibility -The Apache metricsets were tested with Apache 2.4.12 and 2.4.20 and are expected to work with +The Apache metricsets were tested with Apache 2.4.12 and 2.4.54 and are expected to work with all versions >= 2.2.31 and >= 2.4.16. diff --git a/metricbeat/docs/modules/nginx.asciidoc b/metricbeat/docs/modules/nginx.asciidoc index 9a1b36b8d9d..8940a457170 100644 --- a/metricbeat/docs/modules/nginx.asciidoc +++ b/metricbeat/docs/modules/nginx.asciidoc @@ -21,7 +21,7 @@ The default metricset is `stubstatus`. [float] === Compatibility -The Nginx metricsets were tested with Nginx 1.9 and are expected to work with all version +The Nginx metricsets were tested with Nginx 1.23.2 and are expected to work with all version >= 1.9. [float] diff --git a/metricbeat/module/apache/_meta/Dockerfile b/metricbeat/module/apache/_meta/Dockerfile index 1af4077568e..e7206c3d40b 100644 --- a/metricbeat/module/apache/_meta/Dockerfile +++ b/metricbeat/module/apache/_meta/Dockerfile @@ -1,6 +1,5 @@ ARG APACHE_VERSION=${APACHE_VERSION} FROM httpd:$APACHE_VERSION -RUN sed -i "/jessie-updates/d" /etc/apt/sources.list -RUN apt-get update && apt-get install -y curl +RUN apt update && yes | apt install curl HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf diff --git a/metricbeat/module/apache/_meta/docs.asciidoc b/metricbeat/module/apache/_meta/docs.asciidoc index c47b8172603..7a265409fa4 100644 --- a/metricbeat/module/apache/_meta/docs.asciidoc +++ b/metricbeat/module/apache/_meta/docs.asciidoc @@ -8,7 +8,7 @@ HTTPD] servers. The default metricset is `status`. [float] === Compatibility -The Apache metricsets were tested with Apache 2.4.12 and 2.4.20 and are expected to work with +The Apache metricsets were tested with Apache 2.4.12 and 2.4.54 and are expected to work with all versions >= 2.2.31 and >= 2.4.16. diff --git a/metricbeat/module/apache/_meta/httpd.conf b/metricbeat/module/apache/_meta/httpd.conf index 504be256fbf..ef00e9b284c 100644 --- a/metricbeat/module/apache/_meta/httpd.conf +++ b/metricbeat/module/apache/_meta/httpd.conf @@ -63,6 +63,7 @@ Listen 80 # Example: # LoadModule foo_module modules/mod_foo.so # +LoadModule mpm_event_module modules/mod_mpm_event.so LoadModule authn_file_module modules/mod_authn_file.so #LoadModule authn_dbm_module modules/mod_authn_dbm.so #LoadModule authn_anon_module modules/mod_authn_anon.so @@ -190,7 +191,7 @@ ServerAdmin you@example.com # # If your host doesn't have a registered DNS name, enter its IP address here. # -#ServerName www.example.com:80 +ServerName localhost # # Deny access to the entirety of your server's filesystem. You must diff --git a/metricbeat/module/apache/_meta/supported-versions.yml b/metricbeat/module/apache/_meta/supported-versions.yml index cecc98367c8..a2a69c3725e 100644 --- a/metricbeat/module/apache/_meta/supported-versions.yml +++ b/metricbeat/module/apache/_meta/supported-versions.yml @@ -1,3 +1,5 @@ +# all the versions have to be the valid semantic version syntax x.x.x +# because of the test check for comparing against older field structure variants: - - APACHE_VERSION: 2.4.12 - - APACHE_VERSION: 2.4.20 + - APACHE_VERSION: "2.4.54" + - APACHE_VERSION: "2.4.12" diff --git a/metricbeat/module/apache/docker-compose.yml b/metricbeat/module/apache/docker-compose.yml index b88b86b3ca7..831ef622f10 100644 --- a/metricbeat/module/apache/docker-compose.yml +++ b/metricbeat/module/apache/docker-compose.yml @@ -1,11 +1,11 @@ -version: '2.3' +version: "2.3" services: apache: - image: docker.elastic.co/integrations-ci/beats-apache:${APACHE_VERSION:-2.4.20}-1 + image: docker.elastic.co/integrations-ci/beats-apache:${APACHE_VERSION:-2.4.54}-1 build: context: ./_meta args: - APACHE_VERSION: ${APACHE_VERSION:-2.4.20} + APACHE_VERSION: "${APACHE_VERSION:-2.4.54}" ports: - 80 diff --git a/metricbeat/module/mongodb/_meta/Dockerfile b/metricbeat/module/mongodb/_meta/Dockerfile index f103b04e4da..c9c4c776f2d 100644 --- a/metricbeat/module/mongodb/_meta/Dockerfile +++ b/metricbeat/module/mongodb/_meta/Dockerfile @@ -1,5 +1,4 @@ ARG MONGODB_VERSION FROM mongo:${MONGODB_VERSION} -RUN sed -i "/jessie-updates/d" /etc/apt/sources.list -RUN apt-get update && apt-get install -y netcat +RUN apt update && yes | apt install netcat HEALTHCHECK --interval=1s --retries=90 CMD nc -z localhost 27017 diff --git a/metricbeat/module/mongodb/docker-compose.yml b/metricbeat/module/mongodb/docker-compose.yml index b2252b8b716..cee7fdb13ad 100644 --- a/metricbeat/module/mongodb/docker-compose.yml +++ b/metricbeat/module/mongodb/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2.3' +version: "2.3" services: mongodb: @@ -6,7 +6,7 @@ services: build: context: ./_meta args: - MONGODB_VERSION: ${MONGODB_VERSION:-3.4} + MONGODB_VERSION: "${MONGODB_VERSION:-3.4}" command: mongod --replSet beats ports: - 27017 diff --git a/metricbeat/module/nginx/_meta/Dockerfile b/metricbeat/module/nginx/_meta/Dockerfile index 32b4cbc57db..c36f0332103 100755 --- a/metricbeat/module/nginx/_meta/Dockerfile +++ b/metricbeat/module/nginx/_meta/Dockerfile @@ -1,6 +1,5 @@ ARG NGINX_VERSION FROM nginx:${NGINX_VERSION} -RUN sed -i "/jessie-updates/d" /etc/apt/sources.list -RUN apt-get update && apt-get install -y curl +RUN apt update && apt install -y curl HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost/nginx_status COPY ./nginx.conf /etc/nginx/ diff --git a/metricbeat/module/nginx/_meta/docs.asciidoc b/metricbeat/module/nginx/_meta/docs.asciidoc index 6815c0a3c67..d80f74d73da 100644 --- a/metricbeat/module/nginx/_meta/docs.asciidoc +++ b/metricbeat/module/nginx/_meta/docs.asciidoc @@ -10,7 +10,7 @@ The default metricset is `stubstatus`. [float] === Compatibility -The Nginx metricsets were tested with Nginx 1.9 and are expected to work with all version +The Nginx metricsets were tested with Nginx 1.23.2 and are expected to work with all version >= 1.9. [float] diff --git a/metricbeat/module/nginx/docker-compose.yml b/metricbeat/module/nginx/docker-compose.yml index a9a445bf2d8..e5fe1b24982 100644 --- a/metricbeat/module/nginx/docker-compose.yml +++ b/metricbeat/module/nginx/docker-compose.yml @@ -1,11 +1,11 @@ -version: '2.3' +version: "2.3" services: nginx: - image: docker.elastic.co/integrations-ci/beats-nginx:${NGINX_VERSION:-1.9}-1 - build: + image: docker.elastic.co/integrations-ci/beats-nginx:${NGINX_VERSION:-1.23.2}-1 + build: context: ./_meta args: - NGINX_VERSION: ${NGINX_VERSION:-1.9} + NGINX_VERSION: "${NGINX_VERSION:-1.23.2}" ports: - 80 From 3ab1de6d2b709cb262d790f1bd742a470cf589ca Mon Sep 17 00:00:00 2001 From: Denis Rechkunov Date: Thu, 24 Nov 2022 17:56:46 +0100 Subject: [PATCH 2/4] Add retries to Logstash tests because they are flaky --- .../module/logstash/logstash_integration_test.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/logstash/logstash_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go index a884644075a..b6da26367c4 100644 --- a/metricbeat/module/logstash/logstash_integration_test.go +++ b/metricbeat/module/logstash/logstash_integration_test.go @@ -25,10 +25,12 @@ import ( "io/ioutil" "net/http" "testing" + "time" "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/tests/compose" + "github.com/elastic/beats/v7/metricbeat/mb" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/metricbeat/module/logstash" _ "github.com/elastic/beats/v7/metricbeat/module/logstash/node" @@ -47,7 +49,15 @@ func TestFetch(t *testing.T) { t.Run(metricSet, func(t *testing.T) { config := getConfig(metricSet, service.Host()) f := mbtest.NewReportingMetricSetV2Error(t, config) - events, errs := mbtest.ReportingFetchV2Error(f) + var ( + events []mb.Event + errs []error + ) + + require.Eventually(t, func() bool { + events, errs = mbtest.ReportingFetchV2Error(f) + return len(events) > 0 && len(errs) == 0 + }, 30*time.Second, time.Second, "events cannot be empty: %v, %v", events, errs) require.Empty(t, errs) require.NotEmpty(t, events) From 075e60f9c995720d44d10a72780c393208c6fe95 Mon Sep 17 00:00:00 2001 From: Denis Rechkunov Date: Mon, 28 Nov 2022 09:07:16 +0100 Subject: [PATCH 3/4] Do not install mage v1.14.0 --- .github/workflows/check-docs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/check-docs.yml b/.github/workflows/check-docs.yml index 4d56b80fa31..c9c397bfeeb 100644 --- a/.github/workflows/check-docs.yml +++ b/.github/workflows/check-docs.yml @@ -30,5 +30,4 @@ jobs: run: sudo apt-get install -y librpm-dev - name: Run check run: | - go install github.com/magefile/mage make check From 4048b6581f37c194d0e1717893aa05b0aada463d Mon Sep 17 00:00:00 2001 From: Denis Rechkunov Date: Mon, 28 Nov 2022 16:29:49 +0100 Subject: [PATCH 4/4] Update elastic-agent-client version to avoid broken agent dependency --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 2f7263410c7..6bb8fd69615 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -9867,11 +9867,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-a -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-client/v7 -Version: v7.0.0-20221121201703-4b23a52d0ebe +Version: v7.0.1 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.0.0-20221121201703-4b23a52d0ebe/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.0.1/LICENSE.txt: ELASTIC LICENSE AGREEMENT diff --git a/go.mod b/go.mod index 425b2d777d3..3280bfa3f38 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,7 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/eapache/go-resiliency v1.2.0 github.com/eclipse/paho.mqtt.golang v1.3.5 - github.com/elastic/elastic-agent-client/v7 v7.0.0-20221121201703-4b23a52d0ebe + github.com/elastic/elastic-agent-client/v7 v7.0.1 github.com/elastic/go-concert v0.2.0 github.com/elastic/go-libaudit/v2 v2.3.2 github.com/elastic/go-licenser v0.4.0 diff --git a/go.sum b/go.sum index df7214bef16..a903e59536b 100644 --- a/go.sum +++ b/go.sum @@ -617,8 +617,8 @@ github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqr github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/elastic-agent-autodiscover v0.4.0 h1:R1JMLHQpH2KP3GXY8zmgV4dj39uoe1asyPPWGQbGgSk= github.com/elastic/elastic-agent-autodiscover v0.4.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20221121201703-4b23a52d0ebe h1:fpM0Lw4JTA9XYbQMQhVrMzJ0GrwUaXHVSsrm5n/b23E= -github.com/elastic/elastic-agent-client/v7 v7.0.0-20221121201703-4b23a52d0ebe/go.mod h1:FEXUbFMfaV62S0CtJgD+FFHGY7+4o4fXkDicyONPSH8= +github.com/elastic/elastic-agent-client/v7 v7.0.1 h1:RDiozMRkjVfsn1vZ5tc1YzRC+g4w4QTsXIOPEsVDyIY= +github.com/elastic/elastic-agent-client/v7 v7.0.1/go.mod h1:cHviLpA5fAwMbfBIHBVNl16qp90bO7pKHMAQaG+9raU= github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-libs v0.2.13 h1:YQzhO8RaLosGlyt7IHtj/ZxigWiwLcXXlv3gS4QY9CA= github.com/elastic/elastic-agent-libs v0.2.13/go.mod h1:0J9lzJh+BjttIiVjYDLncKYCEWUUHiiqnuI64y6C6ss=