From a634ccefec1d3d682f082c4d07100b5ddf998cfd Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Mon, 16 Dec 2024 10:25:19 -0600 Subject: [PATCH] fix templates and docs to use correct `--` command line args --- CHANGELOG.next.asciidoc | 1 + deploy/kubernetes/metricbeat-kubernetes.yaml | 2 +- .../metricbeat/metricbeat-daemonset.yaml | 2 +- .../metricbeat/manifest.debug.multi.yaml | 4 +- .../kubernetes/metricbeat/manifest.debug.yaml | 2 +- .../kubernetes/metricbeat/manifest.run.yaml | 2 +- dev-tools/mage/pkg.go | 7 +- dev-tools/packaging/package_test.go | 129 ++++++++++++++++-- .../templates/docker/Dockerfile.tmpl | 2 +- .../templates/ironbank/auditbeat/Dockerfile | 2 +- .../templates/ironbank/filebeat/Dockerfile | 2 +- .../templates/ironbank/heartbeat/Dockerfile | 2 +- .../templates/ironbank/metricbeat/Dockerfile | 2 +- .../templates/ironbank/packetbeat/Dockerfile | 2 +- filebeat/tests/open-file-handlers/run.sh | 2 +- libbeat/docs/command-reference.asciidoc | 2 +- metricbeat/docs/running-on-docker.asciidoc | 2 +- .../_meta/remote-debugger/README.md | 2 +- 18 files changed, 138 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 05a6a09516c..9c899d63609 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -17,6 +17,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Drop support for Debian 10 and upgrade statically linked glibc from 2.28 to 2.31 {pull}41402[41402] - Fix metrics not being ingested, due to "Limit of total fields [10000] has been exceeded while adding new fields [...]". The total fields limit has been increased to 12500. No significant performance impact on Elasticsearch is anticipated. {pull}41640[41640] - Set default kafka version to 2.1.0 in kafka output and filebeat. {pull}41662[41662] +- Fix templates and docs to use correct `--` version of command line arguments. {issue}42038[42038] {pull}99999[99999] *Auditbeat* diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index 418c902bffc..0afc6438b65 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -295,7 +295,7 @@ spec: args: [ "-c", "/etc/metricbeat.yml", "-e", - "-system.hostfs=/hostfs", + "--system.hostfs=/hostfs", ] env: - name: ELASTICSEARCH_HOST diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml index e8c0074be6d..c89dd2b21d3 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml @@ -25,7 +25,7 @@ spec: args: [ "-c", "/etc/metricbeat.yml", "-e", - "-system.hostfs=/hostfs", + "--system.hostfs=/hostfs", ] env: - name: ELASTICSEARCH_HOST diff --git a/dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml b/dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml index 6dd492804c8..c266335baba 100644 --- a/dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml +++ b/dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml @@ -298,7 +298,7 @@ spec: args: [ "-c", "/etc/metricbeat.yml", "-e", - "-system.hostfs=/hostfs", + "--system.hostfs=/hostfs", ] env: - name: ELASTICSEARCH_HOST @@ -403,7 +403,7 @@ spec: args: [ "-c", "/etc/metricbeat.yml", "-e", - "-system.hostfs=/hostfs", + "--system.hostfs=/hostfs", ] ports: - containerPort: 56268 diff --git a/dev-tools/kubernetes/metricbeat/manifest.debug.yaml b/dev-tools/kubernetes/metricbeat/manifest.debug.yaml index 398d7fa8560..7bb9dd96336 100644 --- a/dev-tools/kubernetes/metricbeat/manifest.debug.yaml +++ b/dev-tools/kubernetes/metricbeat/manifest.debug.yaml @@ -298,7 +298,7 @@ spec: args: [ "-c", "/etc/metricbeat.yml", "-e", - "-system.hostfs=/hostfs", + "--system.hostfs=/hostfs", ] ports: - containerPort: 56268 diff --git a/dev-tools/kubernetes/metricbeat/manifest.run.yaml b/dev-tools/kubernetes/metricbeat/manifest.run.yaml index 21c9727d45e..6b42510754e 100644 --- a/dev-tools/kubernetes/metricbeat/manifest.run.yaml +++ b/dev-tools/kubernetes/metricbeat/manifest.run.yaml @@ -298,7 +298,7 @@ spec: args: [ "-c", "/etc/metricbeat.yml", "-e", - "-system.hostfs=/hostfs", + "--system.hostfs=/hostfs", ] env: - name: ELASTICSEARCH_HOST diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index 53783b29553..757f857265f 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -172,7 +172,6 @@ func prepareIronbankBuild() error { } return nil }) - if err != nil { return fmt.Errorf("cannot create templates for the IronBank: %w", err) } @@ -206,7 +205,7 @@ func saveIronbank() error { distributionsDir := "build/distributions" if _, err := os.Stat(distributionsDir); os.IsNotExist(err) { - err := os.MkdirAll(distributionsDir, 0750) + err := os.MkdirAll(distributionsDir, 0o750) if err != nil { return fmt.Errorf("cannot create folder for docker artifacts: %w", err) } @@ -348,9 +347,7 @@ func TestPackages(options ...TestPackagesOption) error { args = append(args, "-files", MustExpand("{{.PWD}}/build/distributions/*")) if out, err := goTest(args...); err != nil { - if !mg.Verbose() { - fmt.Println(out) - } + fmt.Println(out) return err } diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index cad23aec9e8..ccd6e632d30 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -26,7 +26,9 @@ import ( "bufio" "bytes" "compress/gzip" + "context" "encoding/json" + "errors" "flag" "fmt" "io" @@ -36,18 +38,20 @@ import ( "slices" "strings" "testing" - - "errors" + "time" "github.com/blakesmith/ar" rpm "github.com/cavaliergopher/rpm" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/strslice" + "github.com/docker/docker/client" ) const ( - expectedConfigMode = os.FileMode(0600) - expectedManifestMode = os.FileMode(0644) + expectedConfigMode = os.FileMode(0o600) + expectedManifestMode = os.FileMode(0o644) expectedModuleFileMode = expectedManifestMode - expectedModuleDirMode = os.FileMode(0755) + expectedModuleDirMode = os.FileMode(0o755) ) var ( @@ -234,15 +238,15 @@ func checkDocker(t *testing.T, file string) { t.Errorf("error reading file %v: %v", file, err) return } - checkDockerEntryPoint(t, p, info) checkDockerLabels(t, p, info, file) checkDockerUser(t, p, info, *rootUserContainer) - checkConfigPermissionsWithMode(t, p, os.FileMode(0644)) - checkManifestPermissionsWithMode(t, p, os.FileMode(0644)) + checkConfigPermissionsWithMode(t, p, os.FileMode(0o644)) + checkManifestPermissionsWithMode(t, p, os.FileMode(0o644)) checkModulesPresent(t, "", p) checkModulesDPresent(t, "", p) checkLicensesPresent(t, "licenses/", p) + checkDockerImageRun(t, p, file) } // Verify that the main configuration file is installed with a 0600 file mode. @@ -356,7 +360,7 @@ func checkModulesOwner(t *testing.T, p *packageFile, expectRoot bool) { // Verify that the systemd unit file has a mode of 0644. It should not be // executable. func checkSystemdUnitPermissions(t *testing.T, p *packageFile) { - const expectedMode = os.FileMode(0644) + const expectedMode = os.FileMode(0o644) t.Run(p.Name+" systemd unit file permissions", func(t *testing.T) { for _, entry := range p.Contents { if systemdUnitFilePattern.MatchString(entry.File) { @@ -443,7 +447,7 @@ func checkLicensesPresent(t *testing.T, prefix string, p *packageFile) { } func checkDockerEntryPoint(t *testing.T, p *packageFile, info *dockerInfo) { - expectedMode := os.FileMode(0755) + expectedMode := os.FileMode(0o755) t.Run(fmt.Sprintf("%s entrypoint", p.Name), func(t *testing.T) { if len(info.Config.Entrypoint) == 0 { @@ -511,6 +515,111 @@ func checkDockerUser(t *testing.T, p *packageFile, info *dockerInfo, expectRoot }) } +func checkDockerImageRun(t *testing.T, p *packageFile, imagePath string) { + t.Run(fmt.Sprintf("%s check docker images runs", p.Name), func(t *testing.T) { + var ctx context.Context + dl, ok := t.Deadline() + if !ok { + ctx = context.Background() + } else { + c, cancel := context.WithDeadline(context.Background(), dl) + ctx = c + defer cancel() + } + f, err := os.Open(imagePath) + if err != nil { + t.Fatalf("failed to open docker image %q: %s", imagePath, err) + } + defer f.Close() + + c, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + t.Fatalf("failed to get a Docker client: %s", err) + } + + loadResp, err := c.ImageLoad(ctx, f, true) + if err != nil { + t.Fatalf("error loading docker image: %s", err) + } + + loadRespBody, err := io.ReadAll(loadResp.Body) + if err != nil { + t.Fatalf("failed to read image load response: %s", err) + } + loadResp.Body.Close() + + _, after, found := strings.Cut(string(loadRespBody), "Loaded image: ") + if !found { + t.Fatalf("image load response was unexpected: %s", string(loadRespBody)) + } + imageId := strings.TrimRight(after, "\\n\"}\r\n") + + var caps strslice.StrSlice + if strings.Contains(imageId, "packetbeat") { + caps = append(caps, "NET_ADMIN") + } + + createResp, err := c.ContainerCreate(ctx, + &container.Config{ + Image: imageId, + }, + &container.HostConfig{ + CapAdd: caps, + }, + nil, + nil, + "") + if err != nil { + t.Fatalf("error creating container from image: %s", err) + } + defer func() { + err := c.ContainerRemove(ctx, createResp.ID, container.RemoveOptions{Force: true}) + if err != nil { + t.Errorf("error removing container: %s", err) + } + }() + + err = c.ContainerStart(ctx, createResp.ID, container.StartOptions{}) + if err != nil { + t.Fatalf("failed to start container: %s", err) + } + defer func() { + err := c.ContainerStop(ctx, createResp.ID, container.StopOptions{}) + if err != nil { + t.Errorf("error stopping container: %s", err) + } + }() + + timer := time.NewTimer(15 * time.Second) + defer timer.Stop() + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + var logs []byte + sentinelLog := "Beat ID: not really" + for { + select { + case <-timer.C: + t.Fatalf("logs:\n%s\nnever saw %q within timeout", string(logs), sentinelLog) + goto done + case <-ticker.C: + out, err := c.ContainerLogs(ctx, createResp.ID, container.LogsOptions{ShowStdout: true, ShowStderr: true}) + if err != nil { + t.Logf("could not get logs: %s", err) + } + logs, err = io.ReadAll(out) + if err != nil { + t.Logf("error reading logs: %s", err) + } + if bytes.Contains(logs, []byte(sentinelLog)) { + goto done + } + } + } + done: + }) +} + // ensureNoBuildIDLinks checks for regressions related to // https://github.com/elastic/beats/issues/12956. func ensureNoBuildIDLinks(t *testing.T, p *packageFile) { diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index d5696e9fa0e..1643e308c9a 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -244,4 +244,4 @@ ENV LIBBEAT_MONITORING_CGROUPS_HIERARCHY_OVERRIDE=/ WORKDIR {{ $beatHome }} ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/docker-entrypoint"] -CMD ["-environment", "container"] +CMD ["--environment", "container"] diff --git a/dev-tools/packaging/templates/ironbank/auditbeat/Dockerfile b/dev-tools/packaging/templates/ironbank/auditbeat/Dockerfile index 893e84d57e3..387b1c4f4f1 100644 --- a/dev-tools/packaging/templates/ironbank/auditbeat/Dockerfile +++ b/dev-tools/packaging/templates/ironbank/auditbeat/Dockerfile @@ -80,7 +80,7 @@ ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT} # TODO: eventually /tinit will be replaced by /usr/bin/tini ENTRYPOINT ["/tinit", "--", "/usr/share/auditbeat/auditbeat", "-E", "http.enabled=true", "-E", "http.host=unix:///usr/share/auditbeat/data/auditbeat.sock"] -CMD ["-environment", "container"] +CMD ["--environment", "container"] # see https://www.elastic.co/guide/en/beats/auditbeat/current/http-endpoint.html HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 --unix-socket '/usr/share/auditbeat/data/auditbeat.sock' 'http:/stats?pretty' diff --git a/dev-tools/packaging/templates/ironbank/filebeat/Dockerfile b/dev-tools/packaging/templates/ironbank/filebeat/Dockerfile index dc4f7bb49ea..5f8b5fafb65 100644 --- a/dev-tools/packaging/templates/ironbank/filebeat/Dockerfile +++ b/dev-tools/packaging/templates/ironbank/filebeat/Dockerfile @@ -80,7 +80,7 @@ ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT} # TODO: eventually /tinit will be replaced by /usr/bin/tini ENTRYPOINT ["/tinit", "--", "/usr/share/filebeat/filebeat", "-E", "http.enabled=true", "-E", "http.host=unix:///usr/share/filebeat/data/filebeat.sock"] -CMD ["-environment", "container"] +CMD ["--environment", "container"] # see https://www.elastic.co/guide/en/beats/filebeat/current/http-endpoint.html HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 --unix-socket '/usr/share/filebeat/data/filebeat.sock' 'http:/stats/?pretty' diff --git a/dev-tools/packaging/templates/ironbank/heartbeat/Dockerfile b/dev-tools/packaging/templates/ironbank/heartbeat/Dockerfile index c71357b8478..b24ce932c3c 100644 --- a/dev-tools/packaging/templates/ironbank/heartbeat/Dockerfile +++ b/dev-tools/packaging/templates/ironbank/heartbeat/Dockerfile @@ -80,7 +80,7 @@ ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT} # TODO: eventually /tinit will be replaced by /usr/bin/tini ENTRYPOINT ["/tinit", "--", "/usr/share/heartbeat/heartbeat", "-E", "http.enabled=true", "-E", "http.host=unix:///usr/share/heartbeat/data/heartbeat.sock"] -CMD ["-environment", "container"] +CMD ["--environment", "container"] # see https://www.elastic.co/guide/en/beats/heartbeat/current/http-endpoint.html HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 --unix-socket '/usr/share/heartbeat/data/heartbeat.sock' 'http:/stats/?pretty' diff --git a/dev-tools/packaging/templates/ironbank/metricbeat/Dockerfile b/dev-tools/packaging/templates/ironbank/metricbeat/Dockerfile index cee8fe1fe6f..e0f832d1ef5 100644 --- a/dev-tools/packaging/templates/ironbank/metricbeat/Dockerfile +++ b/dev-tools/packaging/templates/ironbank/metricbeat/Dockerfile @@ -81,7 +81,7 @@ ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT} # TODO: eventually /tinit will be replaced by /usr/bin/tini ENTRYPOINT ["/tinit", "--", "/usr/share/metricbeat/metricbeat", "-E", "http.enabled=true", "-E", "http.host=unix:///usr/share/metricbeat/data/metricbeat.sock"] -CMD ["-environment", "container"] +CMD ["--environment", "container"] # see https://www.elastic.co/guide/en/beats/metricbeat/current/http-endpoint.html HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 --unix-socket '/usr/share/metricbeat/data/metricbeat.sock' 'http:/stats/?pretty' diff --git a/dev-tools/packaging/templates/ironbank/packetbeat/Dockerfile b/dev-tools/packaging/templates/ironbank/packetbeat/Dockerfile index e7a025ef6aa..6d7155c80c0 100644 --- a/dev-tools/packaging/templates/ironbank/packetbeat/Dockerfile +++ b/dev-tools/packaging/templates/ironbank/packetbeat/Dockerfile @@ -82,7 +82,7 @@ ENV ELASTIC_PRODUCT=${ELASTIC_PRODUCT} # TODO: eventually /tinit will be replaced by /usr/bin/tini ENTRYPOINT ["/tinit", "--", "/usr/share/packetbeat/packetbeat", "-E", "http.enabled=true", "-E", "http.host=unix:///usr/share/packetbeat/data/packetbeat.sock"] -CMD ["-environment", "container"] +CMD ["--environment", "container"] # see https://www.elastic.co/guide/en/beats/packetbeat/current/http-endpoint.html HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 --unix-socket '/usr/share/packetbeat/data/packetbeat.sock' 'http:/stats/?pretty' diff --git a/filebeat/tests/open-file-handlers/run.sh b/filebeat/tests/open-file-handlers/run.sh index 4554660ab2f..1bcc545391d 100644 --- a/filebeat/tests/open-file-handlers/run.sh +++ b/filebeat/tests/open-file-handlers/run.sh @@ -1,3 +1,3 @@ /etc/init.d/metricbeat start cd /filebeat -./filebeat -httpprof :6060 +./filebeat --httpprof :6060 diff --git a/libbeat/docs/command-reference.asciidoc b/libbeat/docs/command-reference.asciidoc index 4766152f39f..b869d7e4956 100644 --- a/libbeat/docs/command-reference.asciidoc +++ b/libbeat/docs/command-reference.asciidoc @@ -926,7 +926,7 @@ messages. *`-e, --e`*:: Logs to stderr and disables syslog/file output. -*`-environment`*:: +*`--environment`*:: For logging purposes, specifies the environment that {beatname_uc} is running in. This setting is used to select a default log output when no log output is configured. Supported values are: `systemd`, `container`, `macos_service`, and `windows_service`. diff --git a/metricbeat/docs/running-on-docker.asciidoc b/metricbeat/docs/running-on-docker.asciidoc index 26ba27658b8..da012b2eb55 100644 --- a/metricbeat/docs/running-on-docker.asciidoc +++ b/metricbeat/docs/running-on-docker.asciidoc @@ -22,7 +22,7 @@ docker run \ --env DBUS_SYSTEM_BUS_ADDRESS='unix:path=/hostfs/var/run/dbus/system_bus_socket' \ <4> --net=host \ <5> --cgroupns=host \ <6> - {dockerimage} -e -system.hostfs=/hostfs + {dockerimage} -e --system.hostfs=/hostfs ---- <1> Metricbeat's <> collects much of its data through the Linux proc diff --git a/metricbeat/module/kubernetes/_meta/remote-debugger/README.md b/metricbeat/module/kubernetes/_meta/remote-debugger/README.md index cdebbe225c2..a62eca2a051 100644 --- a/metricbeat/module/kubernetes/_meta/remote-debugger/README.md +++ b/metricbeat/module/kubernetes/_meta/remote-debugger/README.md @@ -72,7 +72,7 @@ containers: args: [ "-c", "/etc/metricbeat.yml", "-e", - "-system.hostfs=/hostfs", + "--system.hostfs=/hostfs", ] ports: - containerPort: 56268