From 08c6c2097f508be9d876696893f9a6b3ec6010ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Tue, 19 Oct 2021 11:29:52 +0200 Subject: [PATCH 1/9] Revert "[elastic-agent] Fix docker tar.gz generation for complete image (#27621)" This reverts commit 89e415d049772424ca078e830d3f7565c6c0129e. --- dev-tools/mage/dockerbuilder.go | 7 +++---- dev-tools/mage/pkgtypes.go | 6 ++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index c1027e94837..0fba00b4e26 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -86,7 +86,7 @@ func (b *dockerBuilder) Build() error { return errors.Wrap(err, "failed to build docker") } - if err := b.dockerSave(tag, variant); err != nil { + if err := b.dockerSave(tag); err != nil { return errors.Wrap(err, "failed to save docker as artifact") } } @@ -209,7 +209,7 @@ func (b *dockerBuilder) dockerBuild(variant string) (string, error) { return taggedImageName, sh.Run("docker", "build", "-t", taggedImageName, b.buildDir) } -func (b *dockerBuilder) dockerSave(tag string, variant string) error { +func (b *dockerBuilder) dockerSave(tag string) error { if _, err := os.Stat(distributionsDir); os.IsNotExist(err) { err := os.MkdirAll(distributionsDir, 0750) if err != nil { @@ -220,8 +220,7 @@ func (b *dockerBuilder) dockerSave(tag string, variant string) error { outputFile := b.OutputFile if outputFile == "" { outputTar, err := b.Expand(defaultBinaryName+".docker.tar.gz", map[string]interface{}{ - "Name": b.imageName, - "Variant": variant, + "Name": b.imageName, }) if err != nil { return err diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index 3a011131501..9a8ebc6d2ad 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -48,7 +48,7 @@ const ( packageStagingDir = "build/package" // defaultBinaryName specifies the output file for zip and tar.gz. - defaultBinaryName = "{{.Name}}-{{if .Variant}}{{.Variant}}-{{end}}{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}" + defaultBinaryName = "{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}" ) // PackageType defines the file format of the package (e.g. zip, rpm, etc). @@ -329,10 +329,8 @@ func (s *PackageSpec) ExtraVar(key, value string) { // Expand expands a templated string using data from the spec. func (s PackageSpec) Expand(in string, args ...map[string]interface{}) (string, error) { - // Assign a default value for variant since it's not always passed in return expandTemplate("inline", in, FuncMap, - - EnvMap(append([]map[string]interface{}{s.evalContext, {"Variant": ""}, s.toMap()}, args...)...)) + EnvMap(append([]map[string]interface{}{s.evalContext, s.toMap()}, args...)...)) } // MustExpand expands a templated string using data from the spec. It panics if From 467f58f2665f4e498e379c3cde489ee5560da463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Tue, 19 Oct 2021 11:30:21 +0200 Subject: [PATCH 2/9] Revert "[elastic-agent] Use -complete in docker image name, not tag (#27399)" This reverts commit 954a250c6d9862735ec069a780fb7a1585b6b3d3. --- dev-tools/mage/dockerbuilder.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 0fba00b4e26..986209609d2 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -195,18 +195,17 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in } func (b *dockerBuilder) dockerBuild(variant string) (string, error) { - imageName := b.imageName + tag := fmt.Sprintf("%s:%s", b.imageName, b.Version) if variant != "" { - imageName = fmt.Sprintf("%s-%s", imageName, variant) + tag = fmt.Sprintf("%s-%s", tag, variant) } - taggedImageName := fmt.Sprintf("%s:%s", imageName, b.Version) if b.Snapshot { - taggedImageName = taggedImageName + "-SNAPSHOT" + tag = tag + "-SNAPSHOT" } if repository, _ := b.ExtraVars["repository"]; repository != "" { - taggedImageName = fmt.Sprintf("%s/%s", repository, taggedImageName) + tag = fmt.Sprintf("%s/%s", repository, tag) } - return taggedImageName, sh.Run("docker", "build", "-t", taggedImageName, b.buildDir) + return tag, sh.Run("docker", "build", "-t", tag, b.buildDir) } func (b *dockerBuilder) dockerSave(tag string) error { From c8f542f6575181f4123dd56891afc56f7f6af3bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 26 Aug 2021 14:14:07 +0200 Subject: [PATCH 3/9] chore: use declarative variant for complete --- dev-tools/packaging/packages.yml | 38 +++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index a9f5f245eba..08af32adbda 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -456,6 +456,10 @@ shared: image_name: '{{.BeatName}}-cloud' repository: 'docker.elastic.co/beats-ci' + - &agent_docker_complete_spec + extra_vars: + image_name: '{{.BeatName}}-complete' + # Deb/RPM spec for community beats. - &deb_rpm_spec <<: *common @@ -1007,9 +1011,20 @@ specs: <<: *agent_docker_spec <<: *elastic_docker_spec <<: *elastic_license_for_binaries - # This image gets a 'complete' variant for synthetics and other large - # packages too big to fit in the main image - variants: ["complete"] + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + + # Complete image gets a 'complete' variant for synthetics and other large + # packages too big to fit in the main image + - os: linux + arch: amd64 + types: [docker] + spec: + <<: *agent_docker_spec + <<: *agent_docker_complete_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries files: '{{.BeatName}}{{.BinaryExt}}': source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} @@ -1046,9 +1061,20 @@ specs: <<: *agent_docker_arm_spec <<: *elastic_docker_spec <<: *elastic_license_for_binaries - # This image gets a 'complete' variant for synthetics and other large - # packages too big to fit in the main image - variants: ["complete"] + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + + # Complete image gets a 'complete' variant for synthetics and other large + # packages too big to fit in the main image + - os: linux + arch: arm64 + types: [docker] + spec: + <<: *agent_docker_arm_spec + <<: *agent_docker_complete_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries files: '{{.BeatName}}{{.BinaryExt}}': source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} From 5a5e107783656ae81341fe71cb48fdf332001489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 9 Sep 2021 12:15:45 +0200 Subject: [PATCH 4/9] fix: pass agent dockerfile to complete spec --- dev-tools/packaging/packages.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 08af32adbda..c9472e8901f 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -457,6 +457,7 @@ shared: repository: 'docker.elastic.co/beats-ci' - &agent_docker_complete_spec + <<: *agent_docker_spec extra_vars: image_name: '{{.BeatName}}-complete' From 4b2ad3a00bbfdbbac10515d2af285653ed070601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Tue, 19 Oct 2021 11:47:33 +0200 Subject: [PATCH 5/9] fix: do not append variant to the artifact name --- dev-tools/mage/dockerbuilder.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 986209609d2..8400ecfde2a 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -196,9 +196,6 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in func (b *dockerBuilder) dockerBuild(variant string) (string, error) { tag := fmt.Sprintf("%s:%s", b.imageName, b.Version) - if variant != "" { - tag = fmt.Sprintf("%s-%s", tag, variant) - } if b.Snapshot { tag = tag + "-SNAPSHOT" } From 84f3271d4711e7754996e36cdc74f5194e21c0b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 9 Sep 2021 17:29:13 +0200 Subject: [PATCH 6/9] docs: update reference to complete variant --- x-pack/elastic-agent/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/elastic-agent/README.md b/x-pack/elastic-agent/README.md index 1483a206475..8d70b9b8420 100644 --- a/x-pack/elastic-agent/README.md +++ b/x-pack/elastic-agent/README.md @@ -17,6 +17,6 @@ If you are in the 7.13 branch, this will create the `docker.elastic.co/beats/ela elastic-package stack up --version=7.13.0-SNAPSHOT -v ``` -Please note that the docker container is built in both standard and 'offline' variants. -The 'offline' variant contains extra files, like the chromium browser, that are too large +Please note that the docker container is built in both standard and 'complete' variants. +The 'complete' variant contains extra files, like the chromium browser, that are too large for the standard variant. \ No newline at end of file From 8bb207963dd3016db8062cabd0e3ed87e8105902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Tue, 19 Oct 2021 11:49:18 +0200 Subject: [PATCH 7/9] chore: remove unused param --- dev-tools/mage/dockerbuilder.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 8400ecfde2a..7d1a1376ad6 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -73,13 +73,13 @@ func (b *dockerBuilder) Build() error { return errors.Wrap(err, "failed to prepare build") } - tag, err := b.dockerBuild(variant) + tag, err := b.dockerBuild() tries := 3 for err != nil && tries != 0 { fmt.Println(">> Building docker images again (after 10 s)") // This sleep is to avoid hitting the docker build issues when resources are not available. time.Sleep(time.Second * 10) - tag, err = b.dockerBuild(variant) + tag, err = b.dockerBuild() tries -= 1 } if err != nil { @@ -194,7 +194,7 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in return nil } -func (b *dockerBuilder) dockerBuild(variant string) (string, error) { +func (b *dockerBuilder) dockerBuild() (string, error) { tag := fmt.Sprintf("%s:%s", b.imageName, b.Version) if b.Snapshot { tag = tag + "-SNAPSHOT" From 7a0d9bc5566e7803397fe1dea20d7f3da511645f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Tue, 19 Oct 2021 12:02:44 +0200 Subject: [PATCH 8/9] chore: remove Variants from PacakgeSpec struct As we prefer using the declarative way over the programmatic way, we do not need the variable. Instead, we'll use the YAML file for declaring the new variants --- dev-tools/mage/dockerbuilder.go | 41 ++++++++----------- dev-tools/mage/pkgtypes.go | 1 - .../docker/Dockerfile.elastic-agent.tmpl | 9 ++-- 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 7d1a1376ad6..b9c48c2c35f 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -66,29 +66,25 @@ func (b *dockerBuilder) Build() error { return err } - // We always have at least one default variant - variants := append([]string{""}, b.PackageSpec.Variants...) - for _, variant := range variants { - if err := b.prepareBuild(variant); err != nil { - return errors.Wrap(err, "failed to prepare build") - } + if err := b.prepareBuild(); err != nil { + return errors.Wrap(err, "failed to prepare build") + } - tag, err := b.dockerBuild() - tries := 3 - for err != nil && tries != 0 { - fmt.Println(">> Building docker images again (after 10 s)") - // This sleep is to avoid hitting the docker build issues when resources are not available. - time.Sleep(time.Second * 10) - tag, err = b.dockerBuild() - tries -= 1 - } - if err != nil { - return errors.Wrap(err, "failed to build docker") - } + tag, err := b.dockerBuild() + tries := 3 + for err != nil && tries != 0 { + fmt.Println(">> Building docker images again (after 10 s)") + // This sleep is to avoid hitting the docker build issues when resources are not available. + time.Sleep(time.Second * 10) + tag, err = b.dockerBuild() + tries -= 1 + } + if err != nil { + return errors.Wrap(err, "failed to build docker") + } - if err := b.dockerSave(tag); err != nil { - return errors.Wrap(err, "failed to save docker as artifact") - } + if err := b.dockerSave(tag); err != nil { + return errors.Wrap(err, "failed to save docker as artifact") } return nil @@ -124,7 +120,7 @@ func (b *dockerBuilder) copyFiles() error { return nil } -func (b *dockerBuilder) prepareBuild(variant string) error { +func (b *dockerBuilder) prepareBuild() error { elasticBeatsDir, err := ElasticBeatsDir() if err != nil { return err @@ -134,7 +130,6 @@ func (b *dockerBuilder) prepareBuild(variant string) error { data := map[string]interface{}{ "ExposePorts": b.exposePorts(), "ModulesDirs": b.modulesDirs(), - "Variant": variant, } err = filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error { diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index 9a8ebc6d2ad..c2c454c873d 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -90,7 +90,6 @@ type PackageSpec struct { Files map[string]PackageFile `yaml:"files"` OutputFile string `yaml:"output_file,omitempty"` // Optional ExtraVars map[string]string `yaml:"extra_vars,omitempty"` // Optional - Variants []string `yaml:"variants"` // Optional evalContext map[string]interface{} packageDir string diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index 0044852b149..614062c3f02 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -37,9 +37,6 @@ RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }} FROM {{ .from }} -# Contains the elastic agent image variant, an empty string for the standard variant -# or "complete" for the bigger one. -ENV ELASTIC_AGENT_IMAGE_VARIANT={{.Variant}} ENV BEAT_SETUID_AS={{ .user }} {{- if contains .from "ubi-minimal" }} @@ -58,7 +55,7 @@ RUN case $(arch) in aarch64) YUM_FLAGS="-x bind-license";; esac; \ (exit $exit_code) {{- end }} -{{- if (and (eq .Variant "complete") (not (contains .from "ubi-minimal"))) }} +{{- if (and (contains .image_name "-complete") (not (contains .from "ubi-minimal"))) }} RUN for iter in {1..10}; do \ yum -y install atk cups gtk gdk xrandr pango libXcomposite libXcursor libXdamage \ libXext libXi libXtst cups-libs libXScrnSaver libXrandr GConf2 \ @@ -156,7 +153,7 @@ RUN mkdir /app {{- else }} RUN groupadd --gid 1000 {{ .BeatName }} RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} -{{- if (and (eq .Variant "complete") (not (contains .from "ubi-minimal"))) }} +{{- if (and (contains .image_name "-complete") (not (contains .from "ubi-minimal"))) }} RUN chown {{ .user }} $NODE_PATH {{- end }} {{- if contains .image_name "-cloud" }} @@ -167,7 +164,7 @@ RUN chown {{ .user }} /app {{- end }} USER {{ .user }} -{{- if (and (eq .Variant "complete") (not (contains .from "ubi-minimal"))) }} +{{- if (and (contains .image_name "-complete") (not (contains .from "ubi-minimal"))) }} # Setup synthetics env vars ENV ELASTIC_SYNTHETICS_CAPABLE=true ENV SUITES_DIR={{ $beatHome }}/suites From c74f6aef0505cc541cd5fd5541fd81c96a8fd7f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Tue, 19 Oct 2021 21:59:45 +0200 Subject: [PATCH 9/9] fix: align comment Co-authored-by: Jaime Soriano Pastor --- dev-tools/packaging/packages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index c9472e8901f..284e5fd1f06 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -1066,8 +1066,8 @@ specs: '{{.BeatName}}{{.BinaryExt}}': source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} - # Complete image gets a 'complete' variant for synthetics and other large - # packages too big to fit in the main image + # Complete image gets a 'complete' variant for synthetics and other large + # packages too big to fit in the main image - os: linux arch: arm64 types: [docker]