build and package the beats for darwin/arm64 (#29585)

* build and packages beats for darwin/arm64. The tar.gz package is called darwin-aarch64

* Osquerybeat: Add darwin/arm64 packaging support (#30935)

Co-authored-by: Aleksandr Maus <[email protected]>

.ci/packaging.groovy

@@ -211,6 +211,8 @@ def linuxPlatforms() {
             'windows/amd64',
             'windows/386',
             'darwin/amd64'
+            // TODO(AndersonQ): comment in after the tests pass
+            // 'darwin/arm64'
           ].join(' ')
 }
 

auditbeat/magefile.go

@@ -28,6 +28,7 @@ import (
 
 	auditbeat "github.com/elastic/beats/v7/auditbeat/scripts/mage"
 	devtools "github.com/elastic/beats/v7/dev-tools/mage"
+	"github.com/elastic/beats/v7/dev-tools/mage/target/build"
 
 	// mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/common"
@@ -74,6 +75,13 @@ func CrossBuildGoDaemon() error {
 	return devtools.CrossBuildGoDaemon()
 }
 
+// AssembleDarwinUniversal merges the darwin/amd64 and darwin/arm64 into a single
+// universal binary using `lipo`. It assumes the darwin/amd64 and darwin/arm64
+// were built and only performs the merge.
+func AssembleDarwinUniversal() error {
+	return build.AssembleDarwinUniversal()
+}
+
 // Package packages the Beat for distribution.
 // Use SNAPSHOT=true to build snapshots.
 // Use PLATFORMS to control the target platforms.

dev-tools/mage/common.go

@@ -235,6 +235,25 @@ func HaveKubectl() error {
 	return nil
 }
 
+// IsDarwinUniversal indicates whether ot not the darwin/universal should be
+// assembled. If both platforms darwin/adm64 and darwin/arm64 are listed, then
+// IsDarwinUniversal returns true.
+// Note: Platforms might be edited at different moments, therefore it's necessary
+// to perform this check on the fly.
+func IsDarwinUniversal() bool {
+	var darwinAMD64, darwinARM64 bool
+	for _, p := range Platforms {
+		if p.Name == "darwin/arm64" {
+			darwinARM64 = true
+		}
+		if p.Name == "darwin/amd64" {
+			darwinAMD64 = true
+		}
+	}
+
+	return darwinAMD64 && darwinARM64
+}
+
 // FindReplace reads a file, performs a find/replace operation, then writes the
 // output to the same file path.
 func FindReplace(file string, re *regexp.Regexp, repl string) error {
@@ -586,7 +605,7 @@ func ParallelCtx(ctx context.Context, fns ...interface{}) {
 // Parallel runs the given functions in parallel with an upper limit set based
 // on GOMAXPROCS.
 func Parallel(fns ...interface{}) {
-	ParallelCtx(context.Background(), fns...)
+	ParallelCtx(context.TODO(), fns...)
 }
 
 // funcTypeWrap wraps a valid FuncType to FuncContextError

dev-tools/mage/crossbuild.go

@@ -172,7 +172,7 @@ func CrossBuild(options ...CrossBuildOption) error {
 		mg.Deps(func() error { return gotool.Mod.Download() })
 	}
 
-	// Build the magefile for Linux so we can run it inside the container.
+	// Build the magefile for Linux, so we can run it inside the container.
 	mg.Deps(buildMage)
 
 	log.Println("crossBuild: Platform list =", params.Platforms)
@@ -194,6 +194,33 @@ func CrossBuild(options ...CrossBuildOption) error {
 
 	// Each build runs in parallel.
 	Parallel(deps...)
+
+	// It needs to run after all the builds, as it needs the darwin binaries.
+	if err := assembleDarwinUniversal(params); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// assembleDarwinUniversal checks if darwin/amd64 and darwin/arm64 were build,
+// if so, it generates a darwin/universal binary that is the merge fo them two.
+func assembleDarwinUniversal(params crossBuildParams) error {
+	if IsDarwinUniversal() {
+		builder := GolangCrossBuilder{
+			// the docker image for darwin/arm64 is the one capable of merging the binaries.
+			Platform:      "darwin/arm64",
+			Target:        "assembleDarwinUniversal",
+			InDir:         params.InDir,
+			ImageSelector: params.ImageSelector}
+		if err := builder.Build(); err != nil {
+			return errors.Wrapf(err,
+				"failed merging darwin/amd64 and darwin/arm64 into darwin/universal target=%v for platform=%v",
+				builder.Target,
+				builder.Platform)
+		}
+	}
+
 	return nil
 }
 
@@ -223,6 +250,8 @@ func CrossBuildImage(platform string) (string, error) {
 		tagSuffix = "darwin-debian10"
 	case platform == "darwin/arm64":
 		tagSuffix = "darwin-arm64-debian10"
+	case platform == "darwin/universal":
+		tagSuffix = "darwin-arm64-debian10"
 	case platform == "linux/arm64":
 		tagSuffix = "arm"
 		// when it runs on a ARM64 host/worker.

dev-tools/mage/pkg.go

@@ -43,8 +43,10 @@ func Package() error {
 			"UseCommunityBeatPackaging, UseElasticBeatPackaging or USeElasticBeatWithoutXPackPackaging first.")
 	}
 
+	platforms := updateWithDarwinUniversal(Platforms)
+
 	var tasks []interface{}
-	for _, target := range Platforms {
+	for _, target := range platforms {
 		for _, pkg := range Packages {
 			if pkg.OS != target.GOOS() || pkg.Arch != "" && pkg.Arch != target.Arch() {
 				continue
@@ -112,6 +114,20 @@ func Package() error {
 	return nil
 }
 
+// updateWithDarwinUniversal checks if darwin/amd64 and darwin/arm64, are listed
+// if so, the universal binary was built, then we need to package it as well.
+func updateWithDarwinUniversal(platforms BuildPlatformList) BuildPlatformList {
+	if IsDarwinUniversal() {
+		platforms = append(platforms,
+			BuildPlatform{
+				Name:  "darwin/universal",
+				Flags: CGOSupported | CrossBuildSupported | Default,
+			})
+	}
+
+	return platforms
+}
+
 // isPackageTypeSelected returns true if SelectedPackageTypes is empty or if
 // pkgType is present on SelectedPackageTypes. It returns false otherwise.
 func isPackageTypeSelected(pkgType PackageType) bool {

dev-tools/mage/pkgtypes.go

@@ -121,8 +121,10 @@ var OSArchNames = map[string]map[PackageType]map[string]string{
 	},
 	"darwin": map[PackageType]map[string]string{
 		TarGz: map[string]string{
-			"386":   "x86",
-			"amd64": "x86_64",
+			"386":       "x86",
+			"amd64":     "x86_64",
+			"arm64":     "aarch64",
+			"universal": "universal",
 		},
 	},
 	"linux": map[PackageType]map[string]string{
@@ -423,12 +425,12 @@ func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec {
 			}
 
 			f.Source = filepath.Join(s.packageDir, filepath.Base(f.Target))
-			if err = ioutil.WriteFile(createDir(f.Source), []byte(content), 0644); err != nil {
+			if err = ioutil.WriteFile(CreateDir(f.Source), []byte(content), 0644); err != nil {
 				panic(errors.Wrapf(err, "failed to write file containing content for target=%v", target))
 			}
 		case f.Template != "":
 			f.Source = filepath.Join(s.packageDir, filepath.Base(f.Template))
-			if err := s.ExpandFile(f.Template, createDir(f.Source)); err != nil {
+			if err := s.ExpandFile(f.Template, CreateDir(f.Source)); err != nil {
 				panic(errors.Wrapf(err, "failed to expand template file for target=%v", target))
 			}
 		default:
@@ -566,7 +568,7 @@ func PackageZip(spec PackageSpec) error {
 	spec.OutputFile = Zip.AddFileExtension(spec.OutputFile)
 
 	// Write the zip file.
-	if err := ioutil.WriteFile(createDir(spec.OutputFile), buf.Bytes(), 0644); err != nil {
+	if err := ioutil.WriteFile(CreateDir(spec.OutputFile), buf.Bytes(), 0644); err != nil {
 		return errors.Wrap(err, "failed to write zip file")
 	}
 
@@ -588,6 +590,27 @@ func PackageTarGz(spec PackageSpec) error {
 	w := tar.NewWriter(buf)
 	baseDir := spec.rootDir()
 
+	// Replace the darwin-universal by darwin-x86_64 and darwin-arm64. Also
+	// keep the other files.
+	if spec.Name == "elastic-agent" && spec.OS == "darwin" && spec.Arch == "universal" {
+		newFiles := map[string]PackageFile{}
+		for filename, pkgFile := range spec.Files {
+			if strings.Contains(pkgFile.Target, "darwin-universal") &&
+				strings.Contains(pkgFile.Target, "downloads") {
+
+				amdFilename, amdpkgFile := replaceFileArch(filename, pkgFile, "x86_64")
+				armFilename, armpkgFile := replaceFileArch(filename, pkgFile, "aarch64")
+
+				newFiles[amdFilename] = amdpkgFile
+				newFiles[armFilename] = armpkgFile
+			} else {
+				newFiles[filename] = pkgFile
+			}
+		}
+
+		spec.Files = newFiles
+	}
+
 	// Add files to tar.
 	for _, pkgFile := range spec.Files {
 		if pkgFile.Symlink {
@@ -632,7 +655,7 @@ func PackageTarGz(spec PackageSpec) error {
 
 	// Open the output file.
 	log.Println("Creating output file at", spec.OutputFile)
-	outFile, err := os.Create(createDir(spec.OutputFile))
+	outFile, err := os.Create(CreateDir(spec.OutputFile))
 	if err != nil {
 		return err
 	}
@@ -658,6 +681,14 @@ func PackageTarGz(spec PackageSpec) error {
 	return errors.Wrap(CreateSHA512File(spec.OutputFile), "failed to create .sha512 file")
 }
 
+func replaceFileArch(filename string, pkgFile PackageFile, arch string) (string, PackageFile) {
+	filename = strings.ReplaceAll(filename, "universal", arch)
+	pkgFile.Source = strings.ReplaceAll(pkgFile.Source, "universal", arch)
+	pkgFile.Target = strings.ReplaceAll(pkgFile.Target, "universal", arch)
+
+	return filename, pkgFile
+}
+
 // PackageDeb packages a deb file. This requires Docker to execute FPM.
 func PackageDeb(spec PackageSpec) error {
 	return runFPM(spec, Deb)

dev-tools/mage/platforms.go

@@ -35,7 +35,7 @@ var BuildPlatforms = BuildPlatformList{
 	{"darwin/386", CGOSupported | CrossBuildSupported},
 	{"darwin/amd64", CGOSupported | CrossBuildSupported | Default},
 	{"darwin/arm", CGOSupported},
-	{"darwin/arm64", CGOSupported},
+	{"darwin/arm64", CGOSupported | CrossBuildSupported | Default},
 	{"dragonfly/amd64", CGOSupported},
 	{"freebsd/386", CGOSupported},
 	{"freebsd/amd64", CGOSupported},
@@ -326,13 +326,13 @@ func newPlatformExpression(expr string) (*platformExpression, error) {
 
 // NewPlatformList returns a new BuildPlatformList based on given expression.
 //
-// By default the initial set include only the platforms designated as defaults.
+// By default, the initial set include only the platforms designated as defaults.
 // To add additional platforms to list use an addition term that is designated
 // with a plug sign (e.g. "+netbsd" or "+linux/armv7"). Or you may use "+all"
 // to change the initial set to include all possible platforms then filter
 // from there (e.g. "+all linux windows").
 //
-// The expression can consists of selections (e.g. "linux") and/or
+// The expression can consist of selections (e.g. "linux") and/or
 // removals (e.g."!windows"). Each term can be valid GOOS or a valid GOOS/Arch
 // pair.
 //

dev-tools/mage/settings.go

@@ -63,8 +63,8 @@ var (
 	PLATFORMS    = EnvOr("PLATFORMS", "")
 	PACKAGES     = EnvOr("PACKAGES", "")
 
-	// CrossBuildMountModcache, if true, mounts $GOPATH/pkg/mod into
-	// the crossbuild images at /go/pkg/mod, read-only.
+	// CrossBuildMountModcache mounts $GOPATH/pkg/mod into
+	// the crossbuild images at /go/pkg/mod, read-only,  when set to true.
 	CrossBuildMountModcache = true
 
 	BeatName        = EnvOr("BEAT_NAME", filepath.Base(CWD()))

dev-tools/mage/target/build/build.go

@@ -18,6 +18,11 @@
 package build
 
 import (
+	"fmt"
+	"os/exec"
+
+	"github.com/magefile/mage/sh"
+
 	devtools "github.com/elastic/beats/v7/dev-tools/mage"
 )
 
@@ -46,3 +51,28 @@ func CrossBuild() error {
 func CrossBuildGoDaemon() error {
 	return devtools.CrossBuildGoDaemon()
 }
+
+// AssembleDarwinUniversal merges the darwin/amd64 and darwin/arm64 into a single
+// universal binary using `lipo`. It's automatically invoked by CrossBuild whenever
+// the darwin/amd64 and darwin/arm64 are present.
+func AssembleDarwinUniversal() error {
+	cmd := "lipo"
+
+	if _, err := exec.LookPath(cmd); err != nil {
+		return fmt.Errorf("'%s' is required to assemble the universal binary: %w",
+			cmd, err)
+	}
+
+	var lipoArgs []string
+	args := []string{
+		"build/golang-crossbuild/%s-darwin-universal",
+		"build/golang-crossbuild/%s-darwin-arm64",
+		"build/golang-crossbuild/%s-darwin-amd64"}
+
+	for _, arg := range args {
+		lipoArgs = append(lipoArgs, fmt.Sprintf(arg, devtools.BeatName))
+	}
+
+	lipo := sh.RunCmd(cmd, "-create", "-output")
+	return lipo(lipoArgs...)
+}

filebeat/magefile.go

@@ -28,6 +28,7 @@ import (
 	"github.com/magefile/mage/mg"
 
 	devtools "github.com/elastic/beats/v7/dev-tools/mage"
+	"github.com/elastic/beats/v7/dev-tools/mage/target/build"
 	filebeat "github.com/elastic/beats/v7/filebeat/scripts/mage"
 
 	// mage:import
@@ -73,6 +74,13 @@ func CrossBuildGoDaemon() error {
 	return devtools.CrossBuildGoDaemon()
 }
 
+// AssembleDarwinUniversal merges the darwin/amd64 and darwin/arm64 into a single
+// universal binary using `lipo`. It assumes the darwin/amd64 and darwin/arm64
+// were built and only performs the merge.
+func AssembleDarwinUniversal() error {
+	return build.AssembleDarwinUniversal()
+}
+
 // Package packages the Beat for distribution.
 // Use SNAPSHOT=true to build snapshots.
 // Use PLATFORMS to control the target platforms.

libbeat/magefile.go

@@ -22,6 +22,7 @@ package main
 
 import (
 	devtools "github.com/elastic/beats/v7/dev-tools/mage"
+	"github.com/elastic/beats/v7/dev-tools/mage/target/build"
 
 	// mage:import
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/common"
@@ -53,3 +54,10 @@ func Fields() error {
 func Config() error {
 	return devtools.Config(devtools.ShortConfigType|devtools.ReferenceConfigType, devtools.DefaultConfigFileParams(), ".")
 }
+
+// AssembleDarwinUniversal merges the darwin/amd64 and darwin/arm64 into a single
+// universal binary using `lipo`. It assumes the darwin/amd64 and darwin/arm64
+// were built and only performs the merge.
+func AssembleDarwinUniversal() error {
+	return build.AssembleDarwinUniversal()
+}

packetbeat/magefile.go

@@ -27,6 +27,7 @@ import (
 	"github.com/magefile/mage/mg"
 
 	devtools "github.com/elastic/beats/v7/dev-tools/mage"
+	"github.com/elastic/beats/v7/dev-tools/mage/target/build"
 	packetbeat "github.com/elastic/beats/v7/packetbeat/scripts/mage"
 
 	// mage:import
@@ -72,6 +73,13 @@ func CrossBuildGoDaemon() error {
 	return devtools.CrossBuildGoDaemon()
 }
 
+// AssembleDarwinUniversal merges the darwin/amd64 and darwin/arm64 into a single
+// universal binary using `lipo`. It assumes the darwin/amd64 and darwin/arm64
+// were built and only performs the merge.
+func AssembleDarwinUniversal() error {
+	return build.AssembleDarwinUniversal()
+}
+
 // Package packages the Beat for distribution.
 // Use SNAPSHOT=true to build snapshots.
 // Use PLATFORMS to control the target platforms.