diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 8b473f0d68b..f1daee33282 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -92,3 +92,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Added SQL helper that can be used from any Metricbeat module {pull}18955[18955] - Update Go version to 1.14.4. {pull}19753[19753] - Update Go version to 1.14.7. {pull}20508[20508] +- Add packaging for docker image based on UBI minimal 8. {pull}20576[20576] diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 2da79983775..90a99434884 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -151,19 +151,14 @@ func isDockerFile(path string) bool { } func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]interface{}) error { - // has specific dockerfile - dockerfile := fmt.Sprintf("Dockerfile.%s.tmpl", b.imageName) - _, err := os.Stat(filepath.Join(templatesDir, dockerfile)) - if err != nil { - // specific missing fallback to generic - dockerfile = "Dockerfile.tmpl" + dockerfile := "Dockerfile.tmpl" + if f, found := b.ExtraVars["dockerfile"]; found { + dockerfile = f } - entrypoint := fmt.Sprintf("docker-entrypoint.%s.tmpl", b.imageName) - _, err = os.Stat(filepath.Join(templatesDir, entrypoint)) - if err != nil { - // specific missing fallback to generic - entrypoint = "docker-entrypoint.tmpl" + entrypoint := "docker-entrypoint.tmpl" + if e, found := b.ExtraVars["docker_entrypoint"]; found { + entrypoint = e } type fileExpansion struct { @@ -176,7 +171,7 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in ".tmpl", ) path := filepath.Join(templatesDir, file.source) - err = b.ExpandFile(path, target, data) + err := b.ExpandFile(path, target, data) if err != nil { return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target) } diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index 2473202648e..f7de61e7db0 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -91,6 +91,7 @@ var ( "repo": GetProjectRepoInfo, "title": strings.Title, "tolower": strings.ToLower, + "contains": strings.Contains, } ) diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index 96173cde880..2a74e80b7f4 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -48,13 +48,15 @@ const ( ) var ( - configFilePattern = regexp.MustCompile(`.*beat\.yml$|apm-server\.yml$`) + configFilePattern = regexp.MustCompile(`.*beat\.yml$|apm-server\.yml|elastic-agent\.yml$`) manifestFilePattern = regexp.MustCompile(`manifest.yml`) modulesDirPattern = regexp.MustCompile(`module/.+`) modulesDDirPattern = regexp.MustCompile(`modules.d/$`) modulesDFilePattern = regexp.MustCompile(`modules.d/.+`) monitorsDFilePattern = regexp.MustCompile(`monitors.d/.+`) systemdUnitFilePattern = regexp.MustCompile(`/lib/systemd/system/.*\.service`) + + licenseFiles = []string{"LICENSE.txt", "NOTICE.txt"} ) var ( @@ -122,6 +124,7 @@ func checkRPM(t *testing.T, file string) { checkModulesPresent(t, "/usr/share", p) checkModulesDPresent(t, "/etc/", p) checkMonitorsDPresent(t, "/etc", p) + checkLicensesPresent(t, "/usr/share", p) checkSystemdUnitPermissions(t, p) ensureNoBuildIDLinks(t, p) } @@ -141,6 +144,7 @@ func checkDeb(t *testing.T, file string, buf *bytes.Buffer) { checkModulesPresent(t, "./usr/share", p) checkModulesDPresent(t, "./etc/", p) checkMonitorsDPresent(t, "./etc/", p) + checkLicensesPresent(t, "./usr/share", p) checkModulesOwner(t, p, true) checkModulesPermissions(t, p) checkSystemdUnitPermissions(t, p) @@ -160,6 +164,7 @@ func checkTar(t *testing.T, file string) { checkModulesDPresent(t, "", p) checkModulesPermissions(t, p) checkModulesOwner(t, p, true) + checkLicensesPresent(t, "", p) } func checkZip(t *testing.T, file string) { @@ -174,6 +179,7 @@ func checkZip(t *testing.T, file string) { checkModulesPresent(t, "", p) checkModulesDPresent(t, "", p) checkModulesPermissions(t, p) + checkLicensesPresent(t, "", p) } func checkDocker(t *testing.T, file string) { @@ -190,6 +196,7 @@ func checkDocker(t *testing.T, file string) { checkManifestPermissionsWithMode(t, p, os.FileMode(0640)) checkModulesPresent(t, "", p) checkModulesDPresent(t, "", p) + checkLicensesPresent(t, "licenses/", p) } // Verify that the main configuration file is installed with a 0600 file mode. @@ -373,6 +380,22 @@ func checkMonitors(t *testing.T, name, prefix string, r *regexp.Regexp, p *packa }) } +func checkLicensesPresent(t *testing.T, prefix string, p *packageFile) { + for _, licenseFile := range licenseFiles { + t.Run("License file "+licenseFile, func(t *testing.T) { + for _, entry := range p.Contents { + if strings.HasPrefix(entry.File, prefix) && strings.HasSuffix(entry.File, "/"+licenseFile) { + return + } + } + if prefix != "" { + t.Fatalf("not found under %s", prefix) + } + t.Fatal("not found") + }) + } +} + func checkDockerEntryPoint(t *testing.T, p *packageFile, info *dockerInfo) { expectedMode := os.FileMode(0755) @@ -402,7 +425,8 @@ func checkDockerLabels(t *testing.T, p *packageFile, info *dockerInfo, file stri if vendor != "Elastic" { return } - t.Run(fmt.Sprintf("%s labels", p.Name), func(t *testing.T) { + + t.Run(fmt.Sprintf("%s license labels", p.Name), func(t *testing.T) { expectedLicense := "Elastic License" ossPrefix := strings.Join([]string{ info.Config.Labels["org.label-schema.name"], @@ -412,8 +436,24 @@ func checkDockerLabels(t *testing.T, p *packageFile, info *dockerInfo, file stri if strings.HasPrefix(filepath.Base(file), ossPrefix) { expectedLicense = "ASL 2.0" } - if license, present := info.Config.Labels["license"]; !present || license != expectedLicense { - t.Errorf("unexpected license label: %s", license) + licenseLabels := []string{ + "license", + "org.label-schema.license", + } + for _, licenseLabel := range licenseLabels { + if license, present := info.Config.Labels[licenseLabel]; !present || license != expectedLicense { + t.Errorf("unexpected license label %s: %s", licenseLabel, license) + } + } + }) + + t.Run(fmt.Sprintf("%s required labels", p.Name), func(t *testing.T) { + // From https://redhat-connect.gitbook.io/partner-guide-for-red-hat-openshift-and-container/program-on-boarding/technical-prerequisites + requiredLabels := []string{"name", "vendor", "version", "release", "summary", "description"} + for _, label := range requiredLabels { + if value, present := info.Config.Labels[label]; !present || value == "" { + t.Errorf("missing required label %s", label) + } } }) } @@ -657,6 +697,12 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { if strings.HasPrefix("/"+name, workingDir) || "/"+name == entrypoint { p.Contents[name] = entry } + // Add also licenses + for _, licenseFile := range licenseFiles { + if strings.Contains(name, licenseFile) { + p.Contents[name] = entry + } + } } } diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index ea3ddfe76e0..f4261945233 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -28,6 +28,9 @@ shared: /usr/share/{{.BeatName}}/LICENSE.txt: source: '{{ repo.RootDir }}/LICENSE.txt' mode: 0644 + /usr/share/{{.BeatName}}/NOTICE.txt: + source: '{{ repo.RootDir }}/NOTICE.txt' + mode: 0644 /usr/share/{{.BeatName}}/README.md: template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl' mode: 0644 @@ -117,6 +120,9 @@ shared: /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/LICENSE.txt: source: '{{ repo.RootDir }}/LICENSE.txt' mode: 0644 + /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/NOTICE.txt: + source: '{{ repo.RootDir }}/NOTICE.txt' + mode: 0644 /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/README.md: template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl' mode: 0644 @@ -186,6 +192,9 @@ shared: LICENSE.txt: source: '{{ repo.RootDir }}/LICENSE.txt' mode: 0644 + NOTICE.txt: + source: '{{ repo.RootDir }}/NOTICE.txt' + mode: 0644 README.md: template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl' mode: 0644 @@ -307,6 +316,9 @@ shared: <<: *agent_binary_spec extra_vars: from: 'centos:7' + buildFrom: 'centos:7' + dockerfile: 'Dockerfile.elastic-agent.tmpl' + docker_entrypoint: 'docker-entrypoint.elastic-agent.tmpl' user: 'root' linux_capabilities: '' files: @@ -460,6 +472,7 @@ shared: <<: *binary_spec extra_vars: from: 'centos:7' + buildFrom: 'centos:7' user: '{{ .BeatName }}' linux_capabilities: '' files: @@ -468,6 +481,11 @@ shared: mode: 0600 config: true + - &docker_ubi_spec + extra_vars: + image_name: '{{.BeatName}}-ubi8' + from: 'registry.access.redhat.com/ubi8/ubi-minimal' + - &elastic_docker_spec extra_vars: repository: 'docker.elastic.co/beats' @@ -637,6 +655,14 @@ specs: <<: *elastic_docker_spec <<: *elastic_license_for_binaries + - os: linux + types: [docker] + spec: + <<: *docker_spec + <<: *docker_ubi_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries + # Elastic Beat with Elastic License and binary taken the current directory. elastic_beat_xpack_reduced: ### @@ -721,6 +747,17 @@ specs: '{{.BeatName}}{{.BinaryExt}}': source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + - os: linux + types: [docker] + spec: + <<: *docker_spec + <<: *docker_ubi_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + # Elastic Beat with Elastic License and binary taken from the x-pack dir. elastic_beat_agent_binaries: ### @@ -782,6 +819,17 @@ specs: '{{.BeatName}}{{.BinaryExt}}': source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + - os: linux + types: [docker] + spec: + <<: *agent_docker_spec + <<: *docker_ubi_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + # Elastic Beat with Elastic License and binary taken from the x-pack dir. elastic_beat_agent_demo_binaries: diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index 52b0870d719..f6b74dadb47 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -2,13 +2,38 @@ {{- $beatBinary := printf "%s/%s" $beatHome .BeatName }} {{- $repoInfo := repo }} +# Prepare home in a different stage to avoid creating additional layers on +# the final image because of permission changes. +FROM {{ .buildFrom }} AS home + +COPY beat {{ $beatHome }} + +RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \ + chown -R root:root {{ $beatHome }} && \ + find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \ + find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \ + chmod 0750 {{ $beatBinary }} && \ +{{- if .linux_capabilities }} + setcap {{ .linux_capabilities }} {{ $beatBinary }} && \ +{{- end }} +{{- range $i, $modulesd := .ModulesDirs }} + chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \ +{{- end }} + chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs + FROM {{ .from }} +{{- if contains .from "ubi-minimal" }} +RUN for iter in {1..10}; do microdnf update -y && microdnf install -y shadow-utils && microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) +RUN curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o /usr/local/bin/jq && \ + chmod +x /usr/local/bin/jq +{{- else }} # Installing jq needs to be installed after epel-release and cannot be in the same yum install command. RUN yum -y --setopt=tsflags=nodocs update && \ yum install epel-release -y && \ yum install jq -y && \ yum clean all +{{- end }} LABEL \ org.label-schema.build-date="{{ date }}" \ @@ -20,33 +45,37 @@ LABEL \ org.label-schema.url="{{ .BeatURL }}" \ org.label-schema.vcs-url="{{ $repoInfo.RootImportPath }}" \ org.label-schema.vcs-ref="{{ commit }}" \ + io.k8s.description="{{ .BeatDescription }}" \ + io.k8s.display-name="{{ .BeatName | title }} image" \ + org.opencontainers.image.created="{{ date }}" \ + org.opencontainers.image.licenses="{{ .License }}" \ + org.opencontainers.image.title="{{ .BeatName | title }}" \ + org.opencontainers.image.vendor="{{ .BeatVendor }}" \ + name="{{ .BeatName }}" \ + maintainer="infra@elastic.co" \ + vendor="{{ .BeatVendor }}" \ + version="{{ beat_version }}" \ + release="1" \ + url="{{ .BeatURL }}" \ + summary="{{ .BeatName }}" \ license="{{ .License }}" \ description="{{ .BeatDescription }}" ENV ELASTIC_CONTAINER "true" ENV PATH={{ $beatHome }}:$PATH -COPY beat {{ $beatHome }} COPY docker-entrypoint /usr/local/bin/docker-entrypoint RUN chmod 755 /usr/local/bin/docker-entrypoint -RUN groupadd --gid 1000 {{ .BeatName }} +COPY --from=home {{ $beatHome }} {{ $beatHome }} -RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \ - chown -R root:{{ .BeatName }} {{ $beatHome }} && \ - find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \ - find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \ - chmod 0750 {{ $beatBinary }} && \ -{{- if .linux_capabilities }} - setcap {{ .linux_capabilities }} {{ $beatBinary }} && \ -{{- end }} -{{- range $i, $modulesd := .ModulesDirs }} - chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \ -{{- end }} - chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs +RUN mkdir /licenses +COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses +COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses {{- if ne .user "root" }} -RUN useradd -M --uid 1000 --gid 1000 --home {{ $beatHome }} {{ .user }} +RUN groupadd --gid 1000 {{ .BeatName }} +RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} {{- end }} USER {{ .user }} diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 9eac254f822..8b7eb80745c 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -4,7 +4,7 @@ # Prepare home in a different stage to avoid creating additional layers on # the final image because of permission changes. -FROM {{ .from }} AS home +FROM {{ .buildFrom }} AS home COPY beat {{ $beatHome }} @@ -23,8 +23,13 @@ RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \ FROM {{ .from }} -RUN yum -y --setopt=tsflags=nodocs update && \ - yum clean all +{{- if contains .from "ubi-minimal" }} +RUN microdnf -y --setopt=tsflags=nodocs update && \ + microdnf install shadow-utils && \ + microdnf clean all +{{- else }} +RUN yum -y --setopt=tsflags=nodocs update && yum clean all +{{- end }} LABEL \ org.label-schema.build-date="{{ date }}" \ @@ -36,6 +41,19 @@ LABEL \ org.label-schema.url="{{ .BeatURL }}" \ org.label-schema.vcs-url="{{ $repoInfo.RootImportPath }}" \ org.label-schema.vcs-ref="{{ commit }}" \ + io.k8s.description="{{ .BeatDescription }}" \ + io.k8s.display-name="{{ .BeatName | title }} image" \ + org.opencontainers.image.created="{{ date }}" \ + org.opencontainers.image.licenses="{{ .License }}" \ + org.opencontainers.image.title="{{ .BeatName | title }}" \ + org.opencontainers.image.vendor="{{ .BeatVendor }}" \ + name="{{ .BeatName }}" \ + maintainer="infra@elastic.co" \ + vendor="{{ .BeatVendor }}" \ + version="{{ beat_version }}" \ + release="1" \ + url="{{ .BeatURL }}" \ + summary="{{ .BeatName }}" \ license="{{ .License }}" \ description="{{ .BeatDescription }}" @@ -47,6 +65,10 @@ RUN chmod 755 /usr/local/bin/docker-entrypoint COPY --from=home {{ $beatHome }} {{ $beatHome }} +RUN mkdir /licenses +COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses +COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses + {{- if ne .user "root" }} RUN groupadd --gid 1000 {{ .BeatName }} RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index cf3b3b37cb2..2975cfaba44 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -268,14 +268,30 @@ func Package() { start := time.Now() defer func() { fmt.Println("package ran for", time.Since(start)) }() - packageAgent([]string{ - "darwin-x86_64.tar.gz", - "linux-x86.tar.gz", - "linux-x86_64.tar.gz", - "windows-x86.zip", - "windows-x86_64.zip", - "linux-arm64.tar.gz", - }, devtools.UseElasticAgentPackaging) + platformPackages := []struct { + platform string + packages string + }{ + {"darwin/amd64", "darwin-x86_64.tar.gz"}, + {"linux/386", "linux-x86.tar.gz"}, + {"linux/amd64", "linux-x86_64.tar.gz"}, + {"linux/arm64", "linux-arm64.tar.gz"}, + {"windows/386", "windows-x86.zip"}, + {"windows/amd64", "windows-x86_64.zip"}, + } + + var requiredPackages []string + for _, p := range platformPackages { + if _, enabled := devtools.Platforms.Get(p.platform); enabled { + requiredPackages = append(requiredPackages, p.packages) + } + } + + if len(requiredPackages) == 0 { + panic("elastic-agent package is expected to include other packages") + } + + packageAgent(requiredPackages, devtools.UseElasticAgentPackaging) } func requiredPackagesPresent(basePath, beat, version string, requiredPackages []string) bool { @@ -293,7 +309,7 @@ func requiredPackagesPresent(basePath, beat, version string, requiredPackages [] // TestPackages tests the generated packages (i.e. file modes, owners, groups). func TestPackages() error { - return devtools.TestPackages() + return devtools.TestPackages(devtools.WithRootUserContainer()) } // RunGo runs go command and output the feedback to the stdout and the stderr. @@ -514,18 +530,16 @@ func packageAgent(requiredPackages []string, packagingFn func()) { panic(err) } - if requiredPackagesPresent(pwd, b, version, requiredPackages) { - continue - } + if !requiredPackagesPresent(pwd, b, version, requiredPackages) { + cmd := exec.Command("mage", "package") + cmd.Dir = pwd + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = append(os.Environ(), fmt.Sprintf("PWD=%s", pwd), "AGENT_PACKAGING=on") - cmd := exec.Command("mage", "package") - cmd.Dir = pwd - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Env = append(os.Environ(), fmt.Sprintf("PWD=%s", pwd), "AGENT_PACKAGING=on") - - if err := cmd.Run(); err != nil { - panic(err) + if err := cmd.Run(); err != nil { + panic(err) + } } // copy to new drop @@ -541,7 +555,7 @@ func packageAgent(requiredPackages []string, packagingFn func()) { mg.Deps(Update) mg.Deps(CrossBuild, CrossBuildGoDaemon) - mg.SerialDeps(devtools.Package) + mg.SerialDeps(devtools.Package, TestPackages) } func copyAll(from, to string) error {