From 32fd76c1bcd8e8201fcd8ea2b80d905847490bd5 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 10 Aug 2020 18:22:23 +0200 Subject: [PATCH 01/15] Recover previous changes for UBI images --- dev-tools/packaging/packages.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index ea3ddfe76e0..d1889c46aa4 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -468,6 +468,11 @@ shared: mode: 0600 config: true + - &docker_ubi_spec + extra_vars: + image_name: '{{.BeatName}}-ubi8' + from: 'registry.access.redhat.com/ubi8/ubi:8.2' + - &elastic_docker_spec extra_vars: repository: 'docker.elastic.co/beats' @@ -637,6 +642,14 @@ specs: <<: *elastic_docker_spec <<: *elastic_license_for_binaries + - os: linux + types: [docker] + spec: + <<: *docker_spec + <<: *docker_ubi_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries + # Elastic Beat with Elastic License and binary taken the current directory. elastic_beat_xpack_reduced: ### @@ -721,6 +734,17 @@ specs: '{{.BeatName}}{{.BinaryExt}}': source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + - os: linux + types: [docker] + spec: + <<: *docker_spec + <<: *docker_ubi_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + # Elastic Beat with Elastic License and binary taken from the x-pack dir. elastic_beat_agent_binaries: ### From 0f033071aac28b4d4948a884aec946d114a56b96 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 10 Aug 2020 18:58:15 +0200 Subject: [PATCH 02/15] Copy licenses --- dev-tools/packaging/packages.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index d1889c46aa4..6397cf13b3b 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -472,6 +472,16 @@ shared: extra_vars: image_name: '{{.BeatName}}-ubi8' from: 'registry.access.redhat.com/ubi8/ubi:8.2' + files: + /licenses/ELASTIC-LICENSE.txt: + source: '{{ repo.RootDir }}/licenses/ELASTIC-LICENSE.txt' + mode: 0644 + /licenses/LICENSE.txt: + source: '{{ repo.RootDir }}/LICENSE.txt' + mode: 0644 + /licenses/NOTICE.txt: + source: '{{ repo.RootDir }}/NOTICE.txt' + mode: 0644 - &elastic_docker_spec extra_vars: From e136dccdb90b4c9c6f1e95ca6f51e2ba4258de42 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 12 Aug 2020 18:48:12 +0200 Subject: [PATCH 03/15] Use ubi-minimal --- dev-tools/mage/settings.go | 1 + dev-tools/packaging/packages.yml | 3 ++- dev-tools/packaging/templates/docker/Dockerfile.tmpl | 11 ++++++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index 2473202648e..f7de61e7db0 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -91,6 +91,7 @@ var ( "repo": GetProjectRepoInfo, "title": strings.Title, "tolower": strings.ToLower, + "contains": strings.Contains, } ) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 6397cf13b3b..64398a63567 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -460,6 +460,7 @@ shared: <<: *binary_spec extra_vars: from: 'centos:7' + buildFrom: 'centos:7' user: '{{ .BeatName }}' linux_capabilities: '' files: @@ -471,7 +472,7 @@ shared: - &docker_ubi_spec extra_vars: image_name: '{{.BeatName}}-ubi8' - from: 'registry.access.redhat.com/ubi8/ubi:8.2' + from: 'registry.access.redhat.com/ubi8/ubi-minimal:8.2' files: /licenses/ELASTIC-LICENSE.txt: source: '{{ repo.RootDir }}/licenses/ELASTIC-LICENSE.txt' diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 9eac254f822..b07259ebff7 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -4,7 +4,7 @@ # Prepare home in a different stage to avoid creating additional layers on # the final image because of permission changes. -FROM {{ .from }} AS home +FROM {{ .buildFrom }} AS home COPY beat {{ $beatHome }} @@ -23,8 +23,13 @@ RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \ FROM {{ .from }} -RUN yum -y --setopt=tsflags=nodocs update && \ - yum clean all +{{- if contains .from "ubi-minimal" }} +RUN microdnf -y --setopt=tsflags=nodocs update && \ + microdnf install shadow-utils && \ + microdnf clean all +{{- else }} +RUN yum -y --setopt=tsflags=nodocs update && yum clean all +{{- end }} LABEL \ org.label-schema.build-date="{{ date }}" \ From 655595f4b025a48612d9f68b748ab153cae75cd9 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 13 Aug 2020 21:01:37 +0200 Subject: [PATCH 04/15] Move licenses to the expected location --- dev-tools/packaging/packages.yml | 11 +++++------ dev-tools/packaging/templates/docker/Dockerfile.tmpl | 3 +++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 64398a63567..8188ace938f 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -468,12 +468,6 @@ shared: source: '{{.BeatName}}.docker.yml' mode: 0600 config: true - - - &docker_ubi_spec - extra_vars: - image_name: '{{.BeatName}}-ubi8' - from: 'registry.access.redhat.com/ubi8/ubi-minimal:8.2' - files: /licenses/ELASTIC-LICENSE.txt: source: '{{ repo.RootDir }}/licenses/ELASTIC-LICENSE.txt' mode: 0644 @@ -484,6 +478,11 @@ shared: source: '{{ repo.RootDir }}/NOTICE.txt' mode: 0644 + - &docker_ubi_spec + extra_vars: + image_name: '{{.BeatName}}-ubi8' + from: 'registry.access.redhat.com/ubi8/ubi-minimal:8.2' + - &elastic_docker_spec extra_vars: repository: 'docker.elastic.co/beats' diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index b07259ebff7..90f28b00b7a 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -21,6 +21,8 @@ RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \ {{- end }} chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs +RUN mv {{ $beatHome }}/licenses /licenses + FROM {{ .from }} {{- if contains .from "ubi-minimal" }} @@ -51,6 +53,7 @@ COPY docker-entrypoint /usr/local/bin/docker-entrypoint RUN chmod 755 /usr/local/bin/docker-entrypoint COPY --from=home {{ $beatHome }} {{ $beatHome }} +COPY --from=home /licenses /licenses {{- if ne .user "root" }} RUN groupadd --gid 1000 {{ .BeatName }} From ddd4f2f91d5cf7c208ae6d3bf486642e372996f0 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 13 Aug 2020 21:13:29 +0200 Subject: [PATCH 05/15] Add labels --- dev-tools/packaging/templates/docker/Dockerfile.tmpl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 90f28b00b7a..d46f3b73306 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -43,6 +43,12 @@ LABEL \ org.label-schema.url="{{ .BeatURL }}" \ org.label-schema.vcs-url="{{ $repoInfo.RootImportPath }}" \ org.label-schema.vcs-ref="{{ commit }}" \ + name="{{ .BeatName }}" \ + maintainer="infra@elastic.co" \ + vendor="{{ .BeatVendor }}" \ + version="{{ beat_version }}" \ + release="1" \ + summary="{{ .BeatName }}" \ license="{{ .License }}" \ description="{{ .BeatDescription }}" From 90a1b760bc45e476dc80223f5d7682555e1917e6 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 14 Aug 2020 10:52:23 +0200 Subject: [PATCH 06/15] Use latest ubi-minimal for UBI docker builds --- dev-tools/packaging/packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 8188ace938f..decb6d44c32 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -481,7 +481,7 @@ shared: - &docker_ubi_spec extra_vars: image_name: '{{.BeatName}}-ubi8' - from: 'registry.access.redhat.com/ubi8/ubi-minimal:8.2' + from: 'registry.access.redhat.com/ubi8/ubi-minimal' - &elastic_docker_spec extra_vars: From 489880079372bc26331ced35882e882b5a7c8eb7 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 14 Aug 2020 13:52:58 +0200 Subject: [PATCH 07/15] Add more labels --- dev-tools/packaging/templates/docker/Dockerfile.tmpl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index d46f3b73306..e81596abca7 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -43,11 +43,18 @@ LABEL \ org.label-schema.url="{{ .BeatURL }}" \ org.label-schema.vcs-url="{{ $repoInfo.RootImportPath }}" \ org.label-schema.vcs-ref="{{ commit }}" \ + io.k8s.description="{{ .BeatDescription }}" \ + io.k8s.display-name="{{ .BeatName | title }} image" \ + org.opencontainers.image.created="{{ date }}" \ + org.opencontainers.image.licenses="{{ .License }}" \ + org.opencontainers.image.title="{{ .BeatName | title }}" \ + org.opencontainers.image.vendor="{{ .BeatVendor }}" \ name="{{ .BeatName }}" \ maintainer="infra@elastic.co" \ vendor="{{ .BeatVendor }}" \ version="{{ beat_version }}" \ release="1" \ + url="{{ .BeatURL }}" \ summary="{{ .BeatName }}" \ license="{{ .License }}" \ description="{{ .BeatDescription }}" From 60b413cb6a9240c471dbf3620b7d55fbed870c5f Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 14 Aug 2020 14:39:51 +0200 Subject: [PATCH 08/15] Copy proper licenses and add test for them --- dev-tools/packaging/package_test.go | 29 +++++++++++++++++++ dev-tools/packaging/packages.yml | 9 ------ .../templates/docker/Dockerfile.tmpl | 7 +++-- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index 96173cde880..9024eb05787 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -55,6 +55,8 @@ var ( modulesDFilePattern = regexp.MustCompile(`modules.d/.+`) monitorsDFilePattern = regexp.MustCompile(`monitors.d/.+`) systemdUnitFilePattern = regexp.MustCompile(`/lib/systemd/system/.*\.service`) + + licenseFiles = []string{"LICENSE.txt", "NOTICE.txt"} ) var ( @@ -122,6 +124,7 @@ func checkRPM(t *testing.T, file string) { checkModulesPresent(t, "/usr/share", p) checkModulesDPresent(t, "/etc/", p) checkMonitorsDPresent(t, "/etc", p) + checkLicensesPresent(t, "/usr/share", p) checkSystemdUnitPermissions(t, p) ensureNoBuildIDLinks(t, p) } @@ -141,6 +144,7 @@ func checkDeb(t *testing.T, file string, buf *bytes.Buffer) { checkModulesPresent(t, "./usr/share", p) checkModulesDPresent(t, "./etc/", p) checkMonitorsDPresent(t, "./etc/", p) + checkLicensesPresent(t, "./usr/share", p) checkModulesOwner(t, p, true) checkModulesPermissions(t, p) checkSystemdUnitPermissions(t, p) @@ -160,6 +164,7 @@ func checkTar(t *testing.T, file string) { checkModulesDPresent(t, "", p) checkModulesPermissions(t, p) checkModulesOwner(t, p, true) + checkLicensesPresent(t, "", p) } func checkZip(t *testing.T, file string) { @@ -174,6 +179,7 @@ func checkZip(t *testing.T, file string) { checkModulesPresent(t, "", p) checkModulesDPresent(t, "", p) checkModulesPermissions(t, p) + checkLicensesPresent(t, "", p) } func checkDocker(t *testing.T, file string) { @@ -190,6 +196,7 @@ func checkDocker(t *testing.T, file string) { checkManifestPermissionsWithMode(t, p, os.FileMode(0640)) checkModulesPresent(t, "", p) checkModulesDPresent(t, "", p) + checkLicensesPresent(t, "licenses/", p) } // Verify that the main configuration file is installed with a 0600 file mode. @@ -373,6 +380,22 @@ func checkMonitors(t *testing.T, name, prefix string, r *regexp.Regexp, p *packa }) } +func checkLicensesPresent(t *testing.T, prefix string, p *packageFile) { + for _, licenseFile := range licenseFiles { + t.Run("License file "+licenseFile, func(t *testing.T) { + for _, entry := range p.Contents { + if strings.HasPrefix(entry.File, prefix) && strings.HasSuffix(entry.File, "/"+licenseFile) { + return + } + } + if prefix != "" { + t.Fatalf("not found under %s", prefix) + } + t.Fatal("not found") + }) + } +} + func checkDockerEntryPoint(t *testing.T, p *packageFile, info *dockerInfo) { expectedMode := os.FileMode(0755) @@ -657,6 +680,12 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { if strings.HasPrefix("/"+name, workingDir) || "/"+name == entrypoint { p.Contents[name] = entry } + // Add also licenses + for _, licenseFile := range licenseFiles { + if strings.Contains(name, licenseFile) { + p.Contents[name] = entry + } + } } } diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index decb6d44c32..d9e0f181cac 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -468,15 +468,6 @@ shared: source: '{{.BeatName}}.docker.yml' mode: 0600 config: true - /licenses/ELASTIC-LICENSE.txt: - source: '{{ repo.RootDir }}/licenses/ELASTIC-LICENSE.txt' - mode: 0644 - /licenses/LICENSE.txt: - source: '{{ repo.RootDir }}/LICENSE.txt' - mode: 0644 - /licenses/NOTICE.txt: - source: '{{ repo.RootDir }}/NOTICE.txt' - mode: 0644 - &docker_ubi_spec extra_vars: diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index e81596abca7..8b7eb80745c 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -21,8 +21,6 @@ RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \ {{- end }} chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs -RUN mv {{ $beatHome }}/licenses /licenses - FROM {{ .from }} {{- if contains .from "ubi-minimal" }} @@ -66,7 +64,10 @@ COPY docker-entrypoint /usr/local/bin/docker-entrypoint RUN chmod 755 /usr/local/bin/docker-entrypoint COPY --from=home {{ $beatHome }} {{ $beatHome }} -COPY --from=home /licenses /licenses + +RUN mkdir /licenses +COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses +COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses {{- if ne .user "root" }} RUN groupadd --gid 1000 {{ .BeatName }} From 1c5ee9791570b2ef9cfd1a76e5d9cedf0616a2d6 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 14 Aug 2020 14:50:39 +0200 Subject: [PATCH 09/15] Add checks for labels --- dev-tools/packaging/package_test.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index 9024eb05787..7f3d9222f40 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -425,7 +425,8 @@ func checkDockerLabels(t *testing.T, p *packageFile, info *dockerInfo, file stri if vendor != "Elastic" { return } - t.Run(fmt.Sprintf("%s labels", p.Name), func(t *testing.T) { + + t.Run(fmt.Sprintf("%s license labels", p.Name), func(t *testing.T) { expectedLicense := "Elastic License" ossPrefix := strings.Join([]string{ info.Config.Labels["org.label-schema.name"], @@ -435,8 +436,24 @@ func checkDockerLabels(t *testing.T, p *packageFile, info *dockerInfo, file stri if strings.HasPrefix(filepath.Base(file), ossPrefix) { expectedLicense = "ASL 2.0" } - if license, present := info.Config.Labels["license"]; !present || license != expectedLicense { - t.Errorf("unexpected license label: %s", license) + licenseLabels := []string{ + "license", + "org.label-schema.license", + } + for _, licenseLabel := range licenseLabels { + if license, present := info.Config.Labels[licenseLabel]; !present || license != expectedLicense { + t.Errorf("unexpected license label %s: %s", licenseLabel, license) + } + } + }) + + t.Run(fmt.Sprintf("%s required labels", p.Name), func(t *testing.T) { + // From https://redhat-connect.gitbook.io/partner-guide-for-red-hat-openshift-and-container/program-on-boarding/technical-prerequisites + requiredLabels := []string{"name", "vendor", "version", "release", "summary", "description"} + for _, label := range requiredLabels { + if value, present := info.Config.Labels[label]; !present || value == "" { + t.Errorf("missing required label %s", label) + } } }) } From 902f57437d630f4be53950cb87ad37d83c19f9b1 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 14 Aug 2020 16:46:36 +0200 Subject: [PATCH 10/15] Fix package tests for elastic-agent --- dev-tools/packaging/packages.yml | 15 +++++ .../docker/Dockerfile.elastic-agent.tmpl | 58 ++++++++++++++----- 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index d9e0f181cac..8a4834a34ab 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -186,6 +186,9 @@ shared: LICENSE.txt: source: '{{ repo.RootDir }}/LICENSE.txt' mode: 0644 + NOTICE.txt: + source: '{{ repo.RootDir }}/NOTICE.txt' + mode: 0644 README.md: template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl' mode: 0644 @@ -307,6 +310,7 @@ shared: <<: *agent_binary_spec extra_vars: from: 'centos:7' + buildFrom: 'centos:7' user: 'root' linux_capabilities: '' files: @@ -807,6 +811,17 @@ specs: '{{.BeatName}}{{.BinaryExt}}': source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + - os: linux + types: [docker] + spec: + <<: *agent_docker_spec + <<: *docker_ubi_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + # Elastic Beat with Elastic License and binary taken from the x-pack dir. elastic_beat_agent_demo_binaries: diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index a38ea8701a3..bc548e439c2 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -2,11 +2,35 @@ {{- $beatBinary := printf "%s/%s" $beatHome .BeatName }} {{- $repoInfo := repo }} +# Prepare home in a different stage to avoid creating additional layers on +# the final image because of permission changes. +FROM {{ .buildFrom }} AS home + +COPY beat {{ $beatHome }} + +RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \ + chown -R root:root {{ $beatHome }} && \ + find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \ + find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \ + chmod 0750 {{ $beatBinary }} && \ +{{- if .linux_capabilities }} + setcap {{ .linux_capabilities }} {{ $beatBinary }} && \ +{{- end }} +{{- range $i, $modulesd := .ModulesDirs }} + chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \ +{{- end }} + chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs + FROM {{ .from }} # Installing jq needs to be installed after epel-release and cannot be in the same yum install command. +{{- if contains .from "ubi-minimal" }} +RUN for iter in {1..10}; do microdnf update --setopt=tsflags=nodocs -y && microdnf install --setopt=tsflags=nodocs -y epel-release && microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) +RUN for iter in {1..10}; do microdnf update -y && microdnf install -y jq shadow-utils && microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) +{{- else }} RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && yum install --setopt=tsflags=nodocs -y epel-release && yum clean all && exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) RUN for iter in {1..10}; do yum update -y && yum install -y jq && yum clean all && exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) +{{- end }} LABEL \ org.label-schema.build-date="{{ date }}" \ @@ -18,33 +42,37 @@ LABEL \ org.label-schema.url="{{ .BeatURL }}" \ org.label-schema.vcs-url="{{ $repoInfo.RootImportPath }}" \ org.label-schema.vcs-ref="{{ commit }}" \ + io.k8s.description="{{ .BeatDescription }}" \ + io.k8s.display-name="{{ .BeatName | title }} image" \ + org.opencontainers.image.created="{{ date }}" \ + org.opencontainers.image.licenses="{{ .License }}" \ + org.opencontainers.image.title="{{ .BeatName | title }}" \ + org.opencontainers.image.vendor="{{ .BeatVendor }}" \ + name="{{ .BeatName }}" \ + maintainer="infra@elastic.co" \ + vendor="{{ .BeatVendor }}" \ + version="{{ beat_version }}" \ + release="1" \ + url="{{ .BeatURL }}" \ + summary="{{ .BeatName }}" \ license="{{ .License }}" \ description="{{ .BeatDescription }}" ENV ELASTIC_CONTAINER "true" ENV PATH={{ $beatHome }}:$PATH -COPY beat {{ $beatHome }} COPY docker-entrypoint /usr/local/bin/docker-entrypoint RUN chmod 755 /usr/local/bin/docker-entrypoint -RUN groupadd --gid 1000 {{ .BeatName }} +COPY --from=home {{ $beatHome }} {{ $beatHome }} -RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \ - chown -R root:{{ .BeatName }} {{ $beatHome }} && \ - find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \ - find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \ - chmod 0750 {{ $beatBinary }} && \ -{{- if .linux_capabilities }} - setcap {{ .linux_capabilities }} {{ $beatBinary }} && \ -{{- end }} -{{- range $i, $modulesd := .ModulesDirs }} - chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \ -{{- end }} - chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs +RUN mkdir /licenses +COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses +COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses {{- if ne .user "root" }} -RUN useradd -M --uid 1000 --gid 1000 --home {{ $beatHome }} {{ .user }} +RUN groupadd --gid 1000 {{ .BeatName }} +RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} {{- end }} USER {{ .user }} From d7933ff830bff729f9cf249feb1fe9d002745e28 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 17 Aug 2020 10:31:42 +0200 Subject: [PATCH 11/15] Explicitly select Dockerfile and entrypoint --- dev-tools/mage/dockerbuilder.go | 19 +++++++------------ dev-tools/packaging/packages.yml | 2 ++ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 2da79983775..90a99434884 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -151,19 +151,14 @@ func isDockerFile(path string) bool { } func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]interface{}) error { - // has specific dockerfile - dockerfile := fmt.Sprintf("Dockerfile.%s.tmpl", b.imageName) - _, err := os.Stat(filepath.Join(templatesDir, dockerfile)) - if err != nil { - // specific missing fallback to generic - dockerfile = "Dockerfile.tmpl" + dockerfile := "Dockerfile.tmpl" + if f, found := b.ExtraVars["dockerfile"]; found { + dockerfile = f } - entrypoint := fmt.Sprintf("docker-entrypoint.%s.tmpl", b.imageName) - _, err = os.Stat(filepath.Join(templatesDir, entrypoint)) - if err != nil { - // specific missing fallback to generic - entrypoint = "docker-entrypoint.tmpl" + entrypoint := "docker-entrypoint.tmpl" + if e, found := b.ExtraVars["docker_entrypoint"]; found { + entrypoint = e } type fileExpansion struct { @@ -176,7 +171,7 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in ".tmpl", ) path := filepath.Join(templatesDir, file.source) - err = b.ExpandFile(path, target, data) + err := b.ExpandFile(path, target, data) if err != nil { return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target) } diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 8a4834a34ab..d9a8bca897a 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -311,6 +311,8 @@ shared: extra_vars: from: 'centos:7' buildFrom: 'centos:7' + dockerfile: 'Dockerfile.elastic-agent.tmpl' + docker_entrypoint: 'docker-entrypoint.elastic-agent.tmpl' user: 'root' linux_capabilities: '' files: From 38085a8c75c0f09f658b3af59606c806ccb07e78 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 17 Aug 2020 10:51:16 +0200 Subject: [PATCH 12/15] Leverage the use of PLATFORMS when selecting required packages for agent --- x-pack/elastic-agent/magefile.go | 52 ++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index 2d634d6fce1..0674491a17f 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -272,14 +272,30 @@ func Package() { start := time.Now() defer func() { fmt.Println("package ran for", time.Since(start)) }() - packageAgent([]string{ - "darwin-x86_64.tar.gz", - "linux-x86.tar.gz", - "linux-x86_64.tar.gz", - "windows-x86.zip", - "windows-x86_64.zip", - "linux-arm64.tar.gz", - }, devtools.UseElasticAgentPackaging) + platformPackages := []struct { + platform string + packages string + }{ + {"darwin/amd64", "darwin-x86_64.tar.gz"}, + {"linux/386", "linux-x86.tar.gz"}, + {"linux/amd64", "linux-x86_64.tar.gz"}, + {"linux/arm64", "linux-arm64.tar.gz"}, + {"windows/386", "windows-x86.zip"}, + {"windows/amd64", "windows-x86_64.zip"}, + } + + var requiredPackages []string + for _, p := range platformPackages { + if _, enabled := devtools.Platforms.Get(p.platform); enabled { + requiredPackages = append(requiredPackages, p.packages) + } + } + + if len(requiredPackages) == 0 { + panic("elastic-agent package is expected to include other packages") + } + + packageAgent(requiredPackages, devtools.UseElasticAgentPackaging) } func requiredPackagesPresent(basePath, beat, version string, requiredPackages []string) bool { @@ -531,18 +547,16 @@ func packageAgent(requiredPackages []string, packagingFn func()) { panic(err) } - if requiredPackagesPresent(pwd, b, version, requiredPackages) { - continue - } + if !requiredPackagesPresent(pwd, b, version, requiredPackages) { + cmd := exec.Command("mage", "package") + cmd.Dir = pwd + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = append(os.Environ(), fmt.Sprintf("PWD=%s", pwd), "AGENT_PACKAGING=on") - cmd := exec.Command("mage", "package") - cmd.Dir = pwd - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Env = append(os.Environ(), fmt.Sprintf("PWD=%s", pwd), "AGENT_PACKAGING=on") - - if err := cmd.Run(); err != nil { - panic(err) + if err := cmd.Run(); err != nil { + panic(err) + } } // copy to new drop From f978f4a19f46aa8a418566e4834c5692ec7a39bd Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 17 Aug 2020 11:24:04 +0200 Subject: [PATCH 13/15] Fix package tests for elastic-agent --- dev-tools/packaging/package_test.go | 2 +- dev-tools/packaging/packages.yml | 6 ++++++ .../templates/docker/Dockerfile.elastic-agent.tmpl | 6 +++--- x-pack/elastic-agent/magefile.go | 4 ++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index 7f3d9222f40..2a74e80b7f4 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -48,7 +48,7 @@ const ( ) var ( - configFilePattern = regexp.MustCompile(`.*beat\.yml$|apm-server\.yml$`) + configFilePattern = regexp.MustCompile(`.*beat\.yml$|apm-server\.yml|elastic-agent\.yml$`) manifestFilePattern = regexp.MustCompile(`manifest.yml`) modulesDirPattern = regexp.MustCompile(`module/.+`) modulesDDirPattern = regexp.MustCompile(`modules.d/$`) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index d9a8bca897a..f4261945233 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -28,6 +28,9 @@ shared: /usr/share/{{.BeatName}}/LICENSE.txt: source: '{{ repo.RootDir }}/LICENSE.txt' mode: 0644 + /usr/share/{{.BeatName}}/NOTICE.txt: + source: '{{ repo.RootDir }}/NOTICE.txt' + mode: 0644 /usr/share/{{.BeatName}}/README.md: template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl' mode: 0644 @@ -117,6 +120,9 @@ shared: /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/LICENSE.txt: source: '{{ repo.RootDir }}/LICENSE.txt' mode: 0644 + /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/NOTICE.txt: + source: '{{ repo.RootDir }}/NOTICE.txt' + mode: 0644 /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/README.md: template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl' mode: 0644 diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index bc548e439c2..cc9f4ce7f4e 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -23,11 +23,11 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \ FROM {{ .from }} -# Installing jq needs to be installed after epel-release and cannot be in the same yum install command. {{- if contains .from "ubi-minimal" }} -RUN for iter in {1..10}; do microdnf update --setopt=tsflags=nodocs -y && microdnf install --setopt=tsflags=nodocs -y epel-release && microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) -RUN for iter in {1..10}; do microdnf update -y && microdnf install -y jq shadow-utils && microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) +# TODO Install or replace jq +RUN for iter in {1..10}; do microdnf update -y && microdnf install -y shadow-utils && microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) {{- else }} +# Installing jq needs to be installed after epel-release and cannot be in the same yum install command. RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && yum install --setopt=tsflags=nodocs -y epel-release && yum clean all && exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) RUN for iter in {1..10}; do yum update -y && yum install -y jq && yum clean all && exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) {{- end }} diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index 0674491a17f..5735470b54c 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -313,7 +313,7 @@ func requiredPackagesPresent(basePath, beat, version string, requiredPackages [] // TestPackages tests the generated packages (i.e. file modes, owners, groups). func TestPackages() error { - return devtools.TestPackages() + return devtools.TestPackages(devtools.WithRootUserContainer()) } // RunGo runs go command and output the feedback to the stdout and the stderr. @@ -572,7 +572,7 @@ func packageAgent(requiredPackages []string, packagingFn func()) { mg.Deps(Update) mg.Deps(CrossBuild, CrossBuildGoDaemon) - mg.SerialDeps(devtools.Package) + mg.SerialDeps(devtools.Package, TestPackages) } func copyAll(from, to string) error { From b7ef2b091d949319a7d298b1fe85f750a749dd83 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 17 Aug 2020 11:33:32 +0200 Subject: [PATCH 14/15] Install jq in ubi image for agent --- .../packaging/templates/docker/Dockerfile.elastic-agent.tmpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index cc9f4ce7f4e..a7242baa73b 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -24,8 +24,9 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \ FROM {{ .from }} {{- if contains .from "ubi-minimal" }} -# TODO Install or replace jq RUN for iter in {1..10}; do microdnf update -y && microdnf install -y shadow-utils && microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) +RUN curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o /usr/local/bin/jq && \ + chmod +x /usr/local/bin/jq {{- else }} # Installing jq needs to be installed after epel-release and cannot be in the same yum install command. RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && yum install --setopt=tsflags=nodocs -y epel-release && yum clean all && exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) From e1539d1f783dcbc32577d7049e9cadbf33043ef1 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 17 Aug 2020 11:45:28 +0200 Subject: [PATCH 15/15] Add changelog entry --- CHANGELOG-developer.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index bb2dcf96345..29734e17fa8 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -100,3 +100,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Added SQL helper that can be used from any Metricbeat module {pull}18955[18955] - Update Go version to 1.14.4. {pull}19753[19753] - Update Go version to 1.14.7. {pull}20508[20508] +- Add packaging for docker image based on UBI minimal 8. {pull}20576[20576]