From 6df4e77e0885c028609052658a87828ebd8ed04b Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 14 Mar 2022 18:35:33 +0100 Subject: [PATCH 1/6] * builds the darwin/arm64 binary * docs small changes * add arm64 to darwin architectures * docs/comments wee changes * stop using deprecated funcations * it works * now it works * add darwin-binary-arm64 to packageArchMap * add darwin/arm64 to ci packaging * use aarch instead of arm64 for package names --- dev-tools/mage/common.go | 26 +++++++++++++++- dev-tools/mage/crossbuild.go | 34 ++++++++++++++++++-- dev-tools/mage/pkg.go | 18 ++++++++++- dev-tools/mage/pkgtypes.go | 43 +++++++++++++++++++++---- dev-tools/mage/platforms.go | 6 ++-- dev-tools/mage/settings.go | 4 +-- internal/pkg/artifact/artifact.go | 16 +++++----- internal/pkg/artifact/config.go | 4 +-- magefile.go | 52 +++++++++++++++++++++++++------ 9 files changed, 169 insertions(+), 34 deletions(-) diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go index b4063f61785..272795581ea 100644 --- a/dev-tools/mage/common.go +++ b/dev-tools/mage/common.go @@ -222,6 +222,25 @@ func HaveKubectl() error { return nil } +// IsDarwinUniversal indicates whether ot not the darwin/universal should be +// assembled. If both platforms darwin/adm64 and darwin/arm64 are listed, then +// IsDarwinUniversal returns true. +// Note: Platforms might be edited at different moments, therefore it's necessary +// to perform this check on the fly. +func IsDarwinUniversal() bool { + var darwinAMD64, darwinARM64 bool + for _, p := range Platforms { + if p.Name == "darwin/arm64" { + darwinARM64 = true + } + if p.Name == "darwin/amd64" { + darwinAMD64 = true + } + } + + return darwinAMD64 && darwinARM64 +} + // FindReplace reads a file, performs a find/replace operation, then writes the // output to the same file path. func FindReplace(file string, re *regexp.Regexp, repl string) error { @@ -573,7 +592,12 @@ func ParallelCtx(ctx context.Context, fns ...interface{}) { // Parallel runs the given functions in parallel with an upper limit set based // on GOMAXPROCS. func Parallel(fns ...interface{}) { - ParallelCtx(context.Background(), fns...) + // Question: if Parallel is just hiding the need for a context, does it mean + // ParallelCtx should be used instead and a refactor is need to remove calls + // Parallel? If so, context.TODO() is more appropriated as it makes explicit + // a context should came from the caller and the current context is just a + // temporary solution until the needed refactor is made. + ParallelCtx(context.TODO(), fns...) } // funcTypeWrap wraps a valid FuncType to FuncContextError diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index b43ad3fee01..dd59b47cab3 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -159,7 +159,7 @@ func CrossBuild(options ...CrossBuildOption) error { mg.Deps(func() error { return gotool.Mod.Download() }) } - // Build the magefile for Linux so we can run it inside the container. + // Build the magefile for Linux, so we can run it inside the container. mg.Deps(buildMage) log.Println("crossBuild: Platform list =", params.Platforms) @@ -181,6 +181,33 @@ func CrossBuild(options ...CrossBuildOption) error { // Each build runs in parallel. Parallel(deps...) + + // It needs to run after all the builds, as it needs the darwin binaries. + if err := assembleDarwinUniversal(params); err != nil { + return err + } + + return nil +} + +// assembleDarwinUniversal checks if darwin/amd64 and darwin/arm64 were build, +// if so, it generates a darwin/universal binary that is the merge fo them two. +func assembleDarwinUniversal(params crossBuildParams) error { + if IsDarwinUniversal() { + builder := GolangCrossBuilder{ + // the docker image for darwin/arm64 is the one capable of merging the binaries. + Platform: "darwin/arm64", + Target: "assembleDarwinUniversal", + InDir: params.InDir, + ImageSelector: params.ImageSelector} + if err := builder.Build(); err != nil { + return errors.Wrapf(err, + "failed merging darwin/amd64 and darwin/arm64 into darwin/universal target=%v for platform=%v", + builder.Target, + builder.Platform) + } + } + return nil } @@ -211,6 +238,8 @@ func CrossBuildImage(platform string) (string, error) { tagSuffix = "darwin-debian10" case platform == "darwin/arm64": tagSuffix = "darwin-arm64-debian10" + case platform == "darwin/universal": + tagSuffix = "darwin-arm64-debian10" case platform == "linux/arm64": tagSuffix = "arm" // when it runs on a ARM64 host/worker. @@ -270,7 +299,8 @@ func (b GolangCrossBuilder) Build() error { workDir := filepath.ToSlash(filepath.Join(mountPoint, cwd)) builderArch := runtime.GOARCH - buildCmd, err := filepath.Rel(workDir, filepath.Join(mountPoint, repoInfo.SubDir, "build/mage-linux-"+builderArch)) + buildCmd, err := filepath.Rel(workDir, + filepath.Join(mountPoint, repoInfo.SubDir, "build/mage-linux-"+builderArch)) if err != nil { return errors.Wrap(err, "failed to determine mage-linux-"+builderArch+" relative path") } diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index 9b5ad7c6cb3..6be12c24b3d 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -30,8 +30,10 @@ func Package() error { "UseCommunityBeatPackaging, UseElasticBeatPackaging or USeElasticBeatWithoutXPackPackaging first.") } + platforms := updateWithDarwinUniversal(Platforms) + var tasks []interface{} - for _, target := range Platforms { + for _, target := range platforms { for _, pkg := range Packages { if pkg.OS != target.GOOS() || pkg.Arch != "" && pkg.Arch != target.Arch() { continue @@ -99,6 +101,20 @@ func Package() error { return nil } +// updateWithDarwinUniversal checks if darwin/amd64 and darwin/arm64, are listed +// if so, the universal binary was built, then we need to package it as well. +func updateWithDarwinUniversal(platforms BuildPlatformList) BuildPlatformList { + if IsDarwinUniversal() { + platforms = append(platforms, + BuildPlatform{ + Name: "darwin/universal", + Flags: CGOSupported | CrossBuildSupported | Default, + }) + } + + return platforms +} + // isPackageTypeSelected returns true if SelectedPackageTypes is empty or if // pkgType is present on SelectedPackageTypes. It returns false otherwise. func isPackageTypeSelected(pkgType PackageType) bool { diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index eaa2a2aff4e..65356955068 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -108,8 +108,10 @@ var OSArchNames = map[string]map[PackageType]map[string]string{ }, "darwin": map[PackageType]map[string]string{ TarGz: map[string]string{ - "386": "x86", - "amd64": "x86_64", + "386": "x86", + "amd64": "x86_64", + "arm64": "aarch64", + "universal": "universal", }, }, "linux": map[PackageType]map[string]string{ @@ -410,12 +412,12 @@ func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec { } f.Source = filepath.Join(s.packageDir, filepath.Base(f.Target)) - if err = ioutil.WriteFile(createDir(f.Source), []byte(content), 0644); err != nil { + if err = ioutil.WriteFile(CreateDir(f.Source), []byte(content), 0644); err != nil { panic(errors.Wrapf(err, "failed to write file containing content for target=%v", target)) } case f.Template != "": f.Source = filepath.Join(s.packageDir, filepath.Base(f.Template)) - if err := s.ExpandFile(f.Template, createDir(f.Source)); err != nil { + if err := s.ExpandFile(f.Template, CreateDir(f.Source)); err != nil { panic(errors.Wrapf(err, "failed to expand template file for target=%v", target)) } default: @@ -553,7 +555,7 @@ func PackageZip(spec PackageSpec) error { spec.OutputFile = Zip.AddFileExtension(spec.OutputFile) // Write the zip file. - if err := ioutil.WriteFile(createDir(spec.OutputFile), buf.Bytes(), 0644); err != nil { + if err := ioutil.WriteFile(CreateDir(spec.OutputFile), buf.Bytes(), 0644); err != nil { return errors.Wrap(err, "failed to write zip file") } @@ -575,6 +577,27 @@ func PackageTarGz(spec PackageSpec) error { w := tar.NewWriter(buf) baseDir := spec.rootDir() + // Replace the darwin-universal by darwin-x86_64 and darwin-arm64. Also + // keep the other files. + if spec.Name == "elastic-agent" && spec.OS == "darwin" && spec.Arch == "universal" { + newFiles := map[string]PackageFile{} + for filename, pkgFile := range spec.Files { + if strings.Contains(pkgFile.Target, "darwin-universal") && + strings.Contains(pkgFile.Target, "downloads") { + + amdFilename, amdpkgFile := replaceFileArch(filename, pkgFile, "x86_64") + armFilename, armpkgFile := replaceFileArch(filename, pkgFile, "aarch64") + + newFiles[amdFilename] = amdpkgFile + newFiles[armFilename] = armpkgFile + } else { + newFiles[filename] = pkgFile + } + } + + spec.Files = newFiles + } + // Add files to tar. for _, pkgFile := range spec.Files { if pkgFile.Symlink { @@ -619,7 +642,7 @@ func PackageTarGz(spec PackageSpec) error { // Open the output file. log.Println("Creating output file at", spec.OutputFile) - outFile, err := os.Create(createDir(spec.OutputFile)) + outFile, err := os.Create(CreateDir(spec.OutputFile)) if err != nil { return err } @@ -645,6 +668,14 @@ func PackageTarGz(spec PackageSpec) error { return errors.Wrap(CreateSHA512File(spec.OutputFile), "failed to create .sha512 file") } +func replaceFileArch(filename string, pkgFile PackageFile, arch string) (string, PackageFile) { + filename = strings.ReplaceAll(filename, "universal", arch) + pkgFile.Source = strings.ReplaceAll(pkgFile.Source, "universal", arch) + pkgFile.Target = strings.ReplaceAll(pkgFile.Target, "universal", arch) + + return filename, pkgFile +} + // PackageDeb packages a deb file. This requires Docker to execute FPM. func PackageDeb(spec PackageSpec) error { return runFPM(spec, Deb) diff --git a/dev-tools/mage/platforms.go b/dev-tools/mage/platforms.go index 7e8bce45723..5450f5a4094 100644 --- a/dev-tools/mage/platforms.go +++ b/dev-tools/mage/platforms.go @@ -22,7 +22,7 @@ var BuildPlatforms = BuildPlatformList{ {"darwin/386", CGOSupported | CrossBuildSupported}, {"darwin/amd64", CGOSupported | CrossBuildSupported | Default}, {"darwin/arm", CGOSupported}, - {"darwin/arm64", CGOSupported}, + {"darwin/arm64", CGOSupported | CrossBuildSupported | Default}, {"dragonfly/amd64", CGOSupported}, {"freebsd/386", CGOSupported}, {"freebsd/amd64", CGOSupported}, @@ -313,13 +313,13 @@ func newPlatformExpression(expr string) (*platformExpression, error) { // NewPlatformList returns a new BuildPlatformList based on given expression. // -// By default the initial set include only the platforms designated as defaults. +// By default, the initial set include only the platforms designated as defaults. // To add additional platforms to list use an addition term that is designated // with a plug sign (e.g. "+netbsd" or "+linux/armv7"). Or you may use "+all" // to change the initial set to include all possible platforms then filter // from there (e.g. "+all linux windows"). // -// The expression can consists of selections (e.g. "linux") and/or +// The expression can consist of selections (e.g. "linux") and/or // removals (e.g."!windows"). Each term can be valid GOOS or a valid GOOS/Arch // pair. // diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index d9ffbaa8223..7a2cccab8f9 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -51,8 +51,8 @@ var ( PACKAGES = EnvOr("PACKAGES", "") CI = EnvOr("CI", "") - // CrossBuildMountModcache CrossBuildMountModcache, if true, mounts $GOPATH/pkg/mod into - // the crossbuild images at /go/pkg/mod, read-only. + // CrossBuildMountModcache mounts $GOPATH/pkg/mod into + // the crossbuild images at /go/pkg/mod, read-only, when set to true. CrossBuildMountModcache = true BeatName = EnvOr("BEAT_NAME", filepath.Base(CWD())) diff --git a/internal/pkg/artifact/artifact.go b/internal/pkg/artifact/artifact.go index 0fb526f7258..63ae2366a58 100644 --- a/internal/pkg/artifact/artifact.go +++ b/internal/pkg/artifact/artifact.go @@ -13,13 +13,15 @@ import ( ) var packageArchMap = map[string]string{ - "linux-binary-32": "linux-x86.tar.gz", - "linux-binary-64": "linux-x86_64.tar.gz", - "linux-binary-arm64": "linux-arm64.tar.gz", - "windows-binary-32": "windows-x86.zip", - "windows-binary-64": "windows-x86_64.zip", - "darwin-binary-32": "darwin-x86_64.tar.gz", - "darwin-binary-64": "darwin-x86_64.tar.gz", + "linux-binary-32": "linux-x86.tar.gz", + "linux-binary-64": "linux-x86_64.tar.gz", + "linux-binary-arm64": "linux-arm64.tar.gz", + "windows-binary-32": "windows-x86.zip", + "windows-binary-64": "windows-x86_64.zip", + "darwin-binary-32": "darwin-x86_64.tar.gz", + "darwin-binary-64": "darwin-x86_64.tar.gz", + "darwin-binary-arm64": "darwin-aarch64.tar.gz", + "darwin-binary-universal": "darwin-universal.tar.gz", } // GetArtifactName constructs a path to a downloaded artifact diff --git a/internal/pkg/artifact/config.go b/internal/pkg/artifact/config.go index 3dd2bb62685..20a6936d20f 100644 --- a/internal/pkg/artifact/config.go +++ b/internal/pkg/artifact/config.go @@ -55,7 +55,7 @@ func DefaultConfig() *Config { } } -// OS return configured operating system or falls back to runtime.GOOS +// OS returns the configured operating system or falls back to runtime.GOOS func (c *Config) OS() string { if c.OperatingSystem != "" { return c.OperatingSystem @@ -73,7 +73,7 @@ func (c *Config) OS() string { return c.OperatingSystem } -// Arch return configured architecture or falls back to 32bit +// Arch returns the configured architecture or falls back to 32bit func (c *Config) Arch() string { if c.Architecture != "" { return c.Architecture diff --git a/magefile.go b/magefile.go index f0391e470d7..000364ec47d 100644 --- a/magefile.go +++ b/magefile.go @@ -26,6 +26,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/e2e-testing/pkg/downloads" + devtools "github.com/elastic/elastic-agent/dev-tools/mage" // mage:import @@ -363,6 +364,31 @@ func (Format) License() error { ) } +// AssembleDarwinUniversal merges the darwin/amd64 and darwin/arm64 into a single +// universal binary using `lipo`. It's automatically invoked by CrossBuild whenever +// the darwin/amd64 and darwin/arm64 are present. +func AssembleDarwinUniversal() error { + cmd := "lipo" + + if _, err := exec.LookPath(cmd); err != nil { + return fmt.Errorf("'%s' is required to assemble the universal binary: %w", + cmd, err) + } + + var lipoArgs []string + args := []string{ + "build/golang-crossbuild/%s-darwin-universal", + "build/golang-crossbuild/%s-darwin-arm64", + "build/golang-crossbuild/%s-darwin-amd64"} + + for _, arg := range args { + lipoArgs = append(lipoArgs, fmt.Sprintf(arg, devtools.BeatName)) + } + + lipo := sh.RunCmd(cmd, "-create", "-output") + return lipo(lipoArgs...) +} + // Package packages the Beat for distribution. // Use SNAPSHOT=true to build snapshots. // Use PLATFORMS to control the target platforms. @@ -376,6 +402,7 @@ func Package() { packages string }{ {"darwin/amd64", "darwin-x86_64.tar.gz"}, + {"darwin/arm64", "darwin-aarch64.tar.gz"}, {"linux/amd64", "linux-x86_64.tar.gz"}, {"linux/arm64", "linux-arm64.tar.gz"}, {"windows/amd64", "windows-x86_64.zip"}, @@ -548,7 +575,7 @@ func (Demo) Enroll() error { return runAgent(env) } -// Enroll runs agent which does not enroll before running. +// NoEnroll runs agent which does not enroll before running. func (Demo) NoEnroll() error { env := map[string]string{ "FLEET_ENROLL": "0", @@ -667,12 +694,12 @@ func packageAgent(requiredPackages []string, packagingFn func()) { newVersion, packageName := getPackageName(beat, version, reqPackage) err := fetchBinaryFromArtifactsApi(ctx, packageName, beat, newVersion, dropPath) if err != nil { - panic(err) + panic(fmt.Sprintf("fetchBinaryFromArtifactsApi failed: %v", err)) } } } } else { - //build from local repo, will assume beats repo is located on the same root level + // build from local repo, will assume beats repo is located on the same root level for _, b := range packedBeats { pwd, err := filepath.Abs(filepath.Join("../beats/x-pack", b)) if err != nil { @@ -701,7 +728,6 @@ func packageAgent(requiredPackages []string, packagingFn func()) { } } } - } // package agent @@ -713,12 +739,18 @@ func packageAgent(requiredPackages []string, packagingFn func()) { } func fetchBinaryFromArtifactsApi(ctx context.Context, packageName, artifact, version, downloadPath string) error { - location, err := downloads.FetchBeatsBinary(ctx, packageName, artifact, version, 3, false, downloadPath, true) - fmt.Println("downloaded binaries on location ", location) - if err != nil { - panic(err) - } - return nil + location, err := downloads.FetchBeatsBinary( + ctx, + packageName, + artifact, + version, + 3, + false, + downloadPath, + true) + fmt.Println("downloaded binaries on location:", location) + + return err } func selectedPackageTypes() string { From 69a76928433e0076341bc3a3452a28b68bb3b827 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 15 Mar 2022 15:35:56 +0100 Subject: [PATCH 2/6] fix lint issue --- dev-tools/mage/common.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go index 272795581ea..6992686f117 100644 --- a/dev-tools/mage/common.go +++ b/dev-tools/mage/common.go @@ -229,6 +229,8 @@ func HaveKubectl() error { // to perform this check on the fly. func IsDarwinUniversal() bool { var darwinAMD64, darwinARM64 bool + + //nolint:goconst // Consistency: there are constants for platforms. for _, p := range Platforms { if p.Name == "darwin/arm64" { darwinARM64 = true From f1680febd11499db447475e52e2fca33bed6853f Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 16 Mar 2022 11:06:50 +0100 Subject: [PATCH 3/6] add changelog --- CHANGELOG.next.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 39368c63c69..b027907694e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -164,4 +164,5 @@ - Install command will skip install/uninstall steps when installation via package is detected on Linux distros. {pull}30289[30289] - Update docker/distribution dependency library to fix a security issues concerning OCI Manifest Type Confusion Issue. {pull}30462[30462] - Add action_input_type for the .fleet-actions-results {pull}30562[30562] - +- Agent can be built for `darwin/arm64`. When it's built for both `darwin/arm64` and `darwin/adm64` +a universal binary is also built and packaged. {pull}29585[29585] From 14b42349a5e58d60a10e8ecd8902481ab8581c68 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 16 Mar 2022 11:07:16 +0100 Subject: [PATCH 4/6] fix lint issues --- dev-tools/mage/pkgtypes.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index 65356955068..a8f9d885fcf 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -412,6 +412,7 @@ func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec { } f.Source = filepath.Join(s.packageDir, filepath.Base(f.Target)) + //nolint:gosec,G306 // 0644 is fine. if err = ioutil.WriteFile(CreateDir(f.Source), []byte(content), 0644); err != nil { panic(errors.Wrapf(err, "failed to write file containing content for target=%v", target)) } @@ -555,6 +556,7 @@ func PackageZip(spec PackageSpec) error { spec.OutputFile = Zip.AddFileExtension(spec.OutputFile) // Write the zip file. + //nolint:gosec,G306 // 0644 is fine. if err := ioutil.WriteFile(CreateDir(spec.OutputFile), buf.Bytes(), 0644); err != nil { return errors.Wrap(err, "failed to write zip file") } From 25a60159a520773b7f61af41686b5130e87a7332 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 21 Mar 2022 17:31:40 +0100 Subject: [PATCH 5/6] add darwin/arm64 to ci build --- .ci/Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 4de02ec0773..07962d9dfc7 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -319,7 +319,8 @@ def withPackageArmEnv(Closure body) { def withPackageDarwinEnv(Closure body) { // Copied from https://github.com/elastic/beats/blob/e6e65aa92fe355c95789691ebf5a3bcecaf5b4ea/.ci/packaging.groovy#L126-L142 def PLATFORMS = [ '+all', - 'darwin/amd64' + 'darwin/amd64', + 'darwin/arm64', ].join(' ') withEnv([ "PLATFORMS=${PLATFORMS}" From 122e1fbfa00c381ff4354ea3384d3d5ec73f2529 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 22 Mar 2022 10:31:28 +0100 Subject: [PATCH 6/6] remove question --- dev-tools/mage/common.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go index 6992686f117..5abec22cd1f 100644 --- a/dev-tools/mage/common.go +++ b/dev-tools/mage/common.go @@ -594,11 +594,6 @@ func ParallelCtx(ctx context.Context, fns ...interface{}) { // Parallel runs the given functions in parallel with an upper limit set based // on GOMAXPROCS. func Parallel(fns ...interface{}) { - // Question: if Parallel is just hiding the need for a context, does it mean - // ParallelCtx should be used instead and a refactor is need to remove calls - // Parallel? If so, context.TODO() is more appropriated as it makes explicit - // a context should came from the caller and the current context is just a - // temporary solution until the needed refactor is made. ParallelCtx(context.TODO(), fns...) }