From 35237284e95272580221b95104d7a40434ccd943 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 2 Nov 2021 09:58:06 +0100 Subject: [PATCH] [7.16](backport #28526) fix: use declarative style for complete variant of the elastic-agent (#28635) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: use declarative style for complete variant of the elastic-agent (#28526) * Revert "[elastic-agent] Fix docker tar.gz generation for complete image (#27621)" This reverts commit 89e415d049772424ca078e830d3f7565c6c0129e. * Revert "[elastic-agent] Use -complete in docker image name, not tag (#27399)" This reverts commit 954a250c6d9862735ec069a780fb7a1585b6b3d3. * chore: use declarative variant for complete * fix: pass agent dockerfile to complete spec * fix: do not append variant to the artifact name * docs: update reference to complete variant * chore: remove unused param * chore: remove Variants from PacakgeSpec struct As we prefer using the declarative way over the programmatic way, we do not need the variable. Instead, we'll use the YAML file for declaring the new variants * fix: align comment Co-authored-by: Jaime Soriano Pastor Co-authored-by: Jaime Soriano Pastor (cherry picked from commit 554399aa1e365e6ef313065bc1e9a908a62c35f6) # Conflicts: # dev-tools/packaging/packages.yml * fix: resolve conflicts in packages.yml file Co-authored-by: Manuel de la Peña --- dev-tools/mage/dockerbuilder.go | 60 ++++++++----------- dev-tools/mage/pkgtypes.go | 7 +-- dev-tools/packaging/packages.yml | 39 ++++++++++-- .../docker/Dockerfile.elastic-agent.tmpl | 9 +-- x-pack/elastic-agent/README.md | 4 +- 5 files changed, 65 insertions(+), 54 deletions(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index c1027e94837..b9c48c2c35f 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -66,29 +66,25 @@ func (b *dockerBuilder) Build() error { return err } - // We always have at least one default variant - variants := append([]string{""}, b.PackageSpec.Variants...) - for _, variant := range variants { - if err := b.prepareBuild(variant); err != nil { - return errors.Wrap(err, "failed to prepare build") - } + if err := b.prepareBuild(); err != nil { + return errors.Wrap(err, "failed to prepare build") + } - tag, err := b.dockerBuild(variant) - tries := 3 - for err != nil && tries != 0 { - fmt.Println(">> Building docker images again (after 10 s)") - // This sleep is to avoid hitting the docker build issues when resources are not available. - time.Sleep(time.Second * 10) - tag, err = b.dockerBuild(variant) - tries -= 1 - } - if err != nil { - return errors.Wrap(err, "failed to build docker") - } + tag, err := b.dockerBuild() + tries := 3 + for err != nil && tries != 0 { + fmt.Println(">> Building docker images again (after 10 s)") + // This sleep is to avoid hitting the docker build issues when resources are not available. + time.Sleep(time.Second * 10) + tag, err = b.dockerBuild() + tries -= 1 + } + if err != nil { + return errors.Wrap(err, "failed to build docker") + } - if err := b.dockerSave(tag, variant); err != nil { - return errors.Wrap(err, "failed to save docker as artifact") - } + if err := b.dockerSave(tag); err != nil { + return errors.Wrap(err, "failed to save docker as artifact") } return nil @@ -124,7 +120,7 @@ func (b *dockerBuilder) copyFiles() error { return nil } -func (b *dockerBuilder) prepareBuild(variant string) error { +func (b *dockerBuilder) prepareBuild() error { elasticBeatsDir, err := ElasticBeatsDir() if err != nil { return err @@ -134,7 +130,6 @@ func (b *dockerBuilder) prepareBuild(variant string) error { data := map[string]interface{}{ "ExposePorts": b.exposePorts(), "ModulesDirs": b.modulesDirs(), - "Variant": variant, } err = filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error { @@ -194,22 +189,18 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in return nil } -func (b *dockerBuilder) dockerBuild(variant string) (string, error) { - imageName := b.imageName - if variant != "" { - imageName = fmt.Sprintf("%s-%s", imageName, variant) - } - taggedImageName := fmt.Sprintf("%s:%s", imageName, b.Version) +func (b *dockerBuilder) dockerBuild() (string, error) { + tag := fmt.Sprintf("%s:%s", b.imageName, b.Version) if b.Snapshot { - taggedImageName = taggedImageName + "-SNAPSHOT" + tag = tag + "-SNAPSHOT" } if repository, _ := b.ExtraVars["repository"]; repository != "" { - taggedImageName = fmt.Sprintf("%s/%s", repository, taggedImageName) + tag = fmt.Sprintf("%s/%s", repository, tag) } - return taggedImageName, sh.Run("docker", "build", "-t", taggedImageName, b.buildDir) + return tag, sh.Run("docker", "build", "-t", tag, b.buildDir) } -func (b *dockerBuilder) dockerSave(tag string, variant string) error { +func (b *dockerBuilder) dockerSave(tag string) error { if _, err := os.Stat(distributionsDir); os.IsNotExist(err) { err := os.MkdirAll(distributionsDir, 0750) if err != nil { @@ -220,8 +211,7 @@ func (b *dockerBuilder) dockerSave(tag string, variant string) error { outputFile := b.OutputFile if outputFile == "" { outputTar, err := b.Expand(defaultBinaryName+".docker.tar.gz", map[string]interface{}{ - "Name": b.imageName, - "Variant": variant, + "Name": b.imageName, }) if err != nil { return err diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index 3a011131501..c2c454c873d 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -48,7 +48,7 @@ const ( packageStagingDir = "build/package" // defaultBinaryName specifies the output file for zip and tar.gz. - defaultBinaryName = "{{.Name}}-{{if .Variant}}{{.Variant}}-{{end}}{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}" + defaultBinaryName = "{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}" ) // PackageType defines the file format of the package (e.g. zip, rpm, etc). @@ -90,7 +90,6 @@ type PackageSpec struct { Files map[string]PackageFile `yaml:"files"` OutputFile string `yaml:"output_file,omitempty"` // Optional ExtraVars map[string]string `yaml:"extra_vars,omitempty"` // Optional - Variants []string `yaml:"variants"` // Optional evalContext map[string]interface{} packageDir string @@ -329,10 +328,8 @@ func (s *PackageSpec) ExtraVar(key, value string) { // Expand expands a templated string using data from the spec. func (s PackageSpec) Expand(in string, args ...map[string]interface{}) (string, error) { - // Assign a default value for variant since it's not always passed in return expandTemplate("inline", in, FuncMap, - - EnvMap(append([]map[string]interface{}{s.evalContext, {"Variant": ""}, s.toMap()}, args...)...)) + EnvMap(append([]map[string]interface{}{s.evalContext, s.toMap()}, args...)...)) } // MustExpand expands a templated string using data from the spec. It panics if diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 27b345d692d..7c4ab7c8525 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -451,6 +451,11 @@ shared: from: 'arm64v8/centos:7' buildFrom: 'arm64v8/centos:7' + - &agent_docker_complete_spec + <<: *agent_docker_spec + extra_vars: + image_name: '{{.BeatName}}-complete' + # Deb/RPM spec for community beats. - &deb_rpm_spec <<: *common @@ -1002,9 +1007,20 @@ specs: <<: *agent_docker_spec <<: *elastic_docker_spec <<: *elastic_license_for_binaries - # This image gets a 'complete' variant for synthetics and other large - # packages too big to fit in the main image - variants: ["complete"] + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + + # Complete image gets a 'complete' variant for synthetics and other large + # packages too big to fit in the main image + - os: linux + arch: amd64 + types: [docker] + spec: + <<: *agent_docker_spec + <<: *agent_docker_complete_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries files: '{{.BeatName}}{{.BinaryExt}}': source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} @@ -1028,9 +1044,20 @@ specs: <<: *agent_docker_arm_spec <<: *elastic_docker_spec <<: *elastic_license_for_binaries - # This image gets a 'complete' variant for synthetics and other large - # packages too big to fit in the main image - variants: ["complete"] + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + + # Complete image gets a 'complete' variant for synthetics and other large + # packages too big to fit in the main image + - os: linux + arch: arm64 + types: [docker] + spec: + <<: *agent_docker_arm_spec + <<: *agent_docker_complete_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries files: '{{.BeatName}}{{.BinaryExt}}': source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index 60b76cd5518..1f4d78002ea 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -32,9 +32,6 @@ RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }} FROM {{ .from }} -# Contains the elastic agent image variant, an empty string for the standard variant -# or "complete" for the bigger one. -ENV ELASTIC_AGENT_IMAGE_VARIANT={{.Variant}} ENV BEAT_SETUID_AS={{ .user }} {{- if contains .from "ubi-minimal" }} @@ -54,7 +51,7 @@ RUN case $(arch) in aarch64) YUM_FLAGS="-x bind-license";; esac; \ (exit $exit_code) {{- end }} -{{- if (and (eq .Variant "complete") (not (contains .from "ubi-minimal"))) }} +{{- if (and (contains .image_name "-complete") (not (contains .from "ubi-minimal"))) }} RUN for iter in {1..10}; do \ yum -y install atk cups gtk gdk xrandr pango libXcomposite libXcursor libXdamage \ libXext libXi libXtst cups-libs libXScrnSaver libXrandr GConf2 \ @@ -143,13 +140,13 @@ COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses {{- if ne .user "root" }} RUN groupadd --gid 1000 {{ .BeatName }} RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} -{{- if (and (eq .Variant "complete") (not (contains .from "ubi-minimal"))) }} +{{- if (and (contains .image_name "-complete") (not (contains .from "ubi-minimal"))) }} RUN chown {{ .user }} $NODE_PATH {{- end }} {{- end }} USER {{ .user }} -{{- if (and (eq .Variant "complete") (not (contains .from "ubi-minimal"))) }} +{{- if (and (contains .image_name "-complete") (not (contains .from "ubi-minimal"))) }} # Setup synthetics env vars ENV ELASTIC_SYNTHETICS_CAPABLE=true ENV SUITES_DIR={{ $beatHome }}/suites diff --git a/x-pack/elastic-agent/README.md b/x-pack/elastic-agent/README.md index 1483a206475..8d70b9b8420 100644 --- a/x-pack/elastic-agent/README.md +++ b/x-pack/elastic-agent/README.md @@ -17,6 +17,6 @@ If you are in the 7.13 branch, this will create the `docker.elastic.co/beats/ela elastic-package stack up --version=7.13.0-SNAPSHOT -v ``` -Please note that the docker container is built in both standard and 'offline' variants. -The 'offline' variant contains extra files, like the chromium browser, that are too large +Please note that the docker container is built in both standard and 'complete' variants. +The 'complete' variant contains extra files, like the chromium browser, that are too large for the standard variant. \ No newline at end of file