Build darwin/aarch64 and universal binary

CHANGELOG.next.asciidoc

@@ -164,4 +164,5 @@
 - Install command will skip install/uninstall steps when installation via package is detected on Linux distros. {pull}30289[30289]
 - Update docker/distribution dependency library to fix a security issues concerning OCI Manifest Type Confusion Issue. {pull}30462[30462]
 - Add action_input_type for the .fleet-actions-results {pull}30562[30562]
-
+- Agent can be built for `darwin/arm64`. When it's built for both `darwin/arm64` and `darwin/adm64`
+a universal binary is also built and packaged. {pull}29585[29585]

dev-tools/mage/common.go

@@ -222,6 +222,27 @@ func HaveKubectl() error {
 	return nil
 }
 
+// IsDarwinUniversal indicates whether ot not the darwin/universal should be
+// assembled. If both platforms darwin/adm64 and darwin/arm64 are listed, then
+// IsDarwinUniversal returns true.
+// Note: Platforms might be edited at different moments, therefore it's necessary
+// to perform this check on the fly.
+func IsDarwinUniversal() bool {
+	var darwinAMD64, darwinARM64 bool
+
+	//nolint:goconst // Consistency: there are constants for platforms.
+	for _, p := range Platforms {
+		if p.Name == "darwin/arm64" {
+			darwinARM64 = true
+		}
+		if p.Name == "darwin/amd64" {
+			darwinAMD64 = true
+		}
+	}
+
+	return darwinAMD64 && darwinARM64
+}
+
 // FindReplace reads a file, performs a find/replace operation, then writes the
 // output to the same file path.
 func FindReplace(file string, re *regexp.Regexp, repl string) error {
@@ -573,7 +594,12 @@ func ParallelCtx(ctx context.Context, fns ...interface{}) {
 // Parallel runs the given functions in parallel with an upper limit set based
 // on GOMAXPROCS.
 func Parallel(fns ...interface{}) {
-	ParallelCtx(context.Background(), fns...)
+	// Question: if Parallel is just hiding the need for a context, does it mean
+	// ParallelCtx should be used instead and a refactor is need to remove calls
+	// Parallel? If so, context.TODO() is more appropriated as it makes explicit
+	// a context should came from the caller and the current context is just a
+	// temporary solution until the needed refactor is made.
+	ParallelCtx(context.TODO(), fns...)
 }
 
 // funcTypeWrap wraps a valid FuncType to FuncContextError

dev-tools/mage/crossbuild.go

@@ -159,7 +159,7 @@ func CrossBuild(options ...CrossBuildOption) error {
 		mg.Deps(func() error { return gotool.Mod.Download() })
 	}
 
-	// Build the magefile for Linux so we can run it inside the container.
+	// Build the magefile for Linux, so we can run it inside the container.
 	mg.Deps(buildMage)
 
 	log.Println("crossBuild: Platform list =", params.Platforms)
@@ -181,6 +181,33 @@ func CrossBuild(options ...CrossBuildOption) error {
 
 	// Each build runs in parallel.
 	Parallel(deps...)
+
+	// It needs to run after all the builds, as it needs the darwin binaries.
+	if err := assembleDarwinUniversal(params); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// assembleDarwinUniversal checks if darwin/amd64 and darwin/arm64 were build,
+// if so, it generates a darwin/universal binary that is the merge fo them two.
+func assembleDarwinUniversal(params crossBuildParams) error {
+	if IsDarwinUniversal() {
+		builder := GolangCrossBuilder{
+			// the docker image for darwin/arm64 is the one capable of merging the binaries.
+			Platform:      "darwin/arm64",
+			Target:        "assembleDarwinUniversal",
+			InDir:         params.InDir,
+			ImageSelector: params.ImageSelector}
+		if err := builder.Build(); err != nil {
+			return errors.Wrapf(err,
+				"failed merging darwin/amd64 and darwin/arm64 into darwin/universal target=%v for platform=%v",
+				builder.Target,
+				builder.Platform)
+		}
+	}
+
 	return nil
 }
 
@@ -211,6 +238,8 @@ func CrossBuildImage(platform string) (string, error) {
 		tagSuffix = "darwin-debian10"
 	case platform == "darwin/arm64":
 		tagSuffix = "darwin-arm64-debian10"
+	case platform == "darwin/universal":
+		tagSuffix = "darwin-arm64-debian10"
 	case platform == "linux/arm64":
 		tagSuffix = "arm"
 		// when it runs on a ARM64 host/worker.
@@ -270,7 +299,8 @@ func (b GolangCrossBuilder) Build() error {
 	workDir := filepath.ToSlash(filepath.Join(mountPoint, cwd))
 
 	builderArch := runtime.GOARCH
-	buildCmd, err := filepath.Rel(workDir, filepath.Join(mountPoint, repoInfo.SubDir, "build/mage-linux-"+builderArch))
+	buildCmd, err := filepath.Rel(workDir,
+		filepath.Join(mountPoint, repoInfo.SubDir, "build/mage-linux-"+builderArch))
 	if err != nil {
 		return errors.Wrap(err, "failed to determine mage-linux-"+builderArch+" relative path")
 	}

dev-tools/mage/pkg.go

@@ -30,8 +30,10 @@ func Package() error {
 			"UseCommunityBeatPackaging, UseElasticBeatPackaging or USeElasticBeatWithoutXPackPackaging first.")
 	}
 
+	platforms := updateWithDarwinUniversal(Platforms)
+
 	var tasks []interface{}
-	for _, target := range Platforms {
+	for _, target := range platforms {
 		for _, pkg := range Packages {
 			if pkg.OS != target.GOOS() || pkg.Arch != "" && pkg.Arch != target.Arch() {
 				continue
@@ -99,6 +101,20 @@ func Package() error {
 	return nil
 }
 
+// updateWithDarwinUniversal checks if darwin/amd64 and darwin/arm64, are listed
+// if so, the universal binary was built, then we need to package it as well.
+func updateWithDarwinUniversal(platforms BuildPlatformList) BuildPlatformList {
+	if IsDarwinUniversal() {
+		platforms = append(platforms,
+			BuildPlatform{
+				Name:  "darwin/universal",
+				Flags: CGOSupported | CrossBuildSupported | Default,
+			})
+	}
+
+	return platforms
+}
+
 // isPackageTypeSelected returns true if SelectedPackageTypes is empty or if
 // pkgType is present on SelectedPackageTypes. It returns false otherwise.
 func isPackageTypeSelected(pkgType PackageType) bool {

dev-tools/mage/pkgtypes.go

@@ -108,8 +108,10 @@ var OSArchNames = map[string]map[PackageType]map[string]string{
 	},
 	"darwin": map[PackageType]map[string]string{
 		TarGz: map[string]string{
-			"386":   "x86",
-			"amd64": "x86_64",
+			"386":       "x86",
+			"amd64":     "x86_64",
+			"arm64":     "aarch64",
+			"universal": "universal",
 		},
 	},
 	"linux": map[PackageType]map[string]string{
@@ -410,12 +412,13 @@ func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec {
 			}
 
 			f.Source = filepath.Join(s.packageDir, filepath.Base(f.Target))
-			if err = ioutil.WriteFile(createDir(f.Source), []byte(content), 0644); err != nil {
+			//nolint:gosec,G306 // 0644 is fine.
+			if err = ioutil.WriteFile(CreateDir(f.Source), []byte(content), 0644); err != nil {
 				panic(errors.Wrapf(err, "failed to write file containing content for target=%v", target))
 			}
 		case f.Template != "":
 			f.Source = filepath.Join(s.packageDir, filepath.Base(f.Template))
-			if err := s.ExpandFile(f.Template, createDir(f.Source)); err != nil {
+			if err := s.ExpandFile(f.Template, CreateDir(f.Source)); err != nil {
 				panic(errors.Wrapf(err, "failed to expand template file for target=%v", target))
 			}
 		default:
@@ -553,7 +556,8 @@ func PackageZip(spec PackageSpec) error {
 	spec.OutputFile = Zip.AddFileExtension(spec.OutputFile)
 
 	// Write the zip file.
-	if err := ioutil.WriteFile(createDir(spec.OutputFile), buf.Bytes(), 0644); err != nil {
+	//nolint:gosec,G306 // 0644 is fine.
+	if err := ioutil.WriteFile(CreateDir(spec.OutputFile), buf.Bytes(), 0644); err != nil {
 		return errors.Wrap(err, "failed to write zip file")
 	}
 
@@ -575,6 +579,27 @@ func PackageTarGz(spec PackageSpec) error {
 	w := tar.NewWriter(buf)
 	baseDir := spec.rootDir()
 
+	// Replace the darwin-universal by darwin-x86_64 and darwin-arm64. Also
+	// keep the other files.
+	if spec.Name == "elastic-agent" && spec.OS == "darwin" && spec.Arch == "universal" {
+		newFiles := map[string]PackageFile{}
+		for filename, pkgFile := range spec.Files {
+			if strings.Contains(pkgFile.Target, "darwin-universal") &&
+				strings.Contains(pkgFile.Target, "downloads") {
+
+				amdFilename, amdpkgFile := replaceFileArch(filename, pkgFile, "x86_64")
+				armFilename, armpkgFile := replaceFileArch(filename, pkgFile, "aarch64")
+
+				newFiles[amdFilename] = amdpkgFile
+				newFiles[armFilename] = armpkgFile
+			} else {
+				newFiles[filename] = pkgFile
+			}
+		}
+
+		spec.Files = newFiles
+	}
+
 	// Add files to tar.
 	for _, pkgFile := range spec.Files {
 		if pkgFile.Symlink {
@@ -619,7 +644,7 @@ func PackageTarGz(spec PackageSpec) error {
 
 	// Open the output file.
 	log.Println("Creating output file at", spec.OutputFile)
-	outFile, err := os.Create(createDir(spec.OutputFile))
+	outFile, err := os.Create(CreateDir(spec.OutputFile))
 	if err != nil {
 		return err
 	}
@@ -645,6 +670,14 @@ func PackageTarGz(spec PackageSpec) error {
 	return errors.Wrap(CreateSHA512File(spec.OutputFile), "failed to create .sha512 file")
 }
 
+func replaceFileArch(filename string, pkgFile PackageFile, arch string) (string, PackageFile) {
+	filename = strings.ReplaceAll(filename, "universal", arch)
+	pkgFile.Source = strings.ReplaceAll(pkgFile.Source, "universal", arch)
+	pkgFile.Target = strings.ReplaceAll(pkgFile.Target, "universal", arch)
+
+	return filename, pkgFile
+}
+
 // PackageDeb packages a deb file. This requires Docker to execute FPM.
 func PackageDeb(spec PackageSpec) error {
 	return runFPM(spec, Deb)

dev-tools/mage/platforms.go

@@ -22,7 +22,7 @@ var BuildPlatforms = BuildPlatformList{
 	{"darwin/386", CGOSupported | CrossBuildSupported},
 	{"darwin/amd64", CGOSupported | CrossBuildSupported | Default},
 	{"darwin/arm", CGOSupported},
-	{"darwin/arm64", CGOSupported},
+	{"darwin/arm64", CGOSupported | CrossBuildSupported | Default},
 	{"dragonfly/amd64", CGOSupported},
 	{"freebsd/386", CGOSupported},
 	{"freebsd/amd64", CGOSupported},
@@ -313,13 +313,13 @@ func newPlatformExpression(expr string) (*platformExpression, error) {
 
 // NewPlatformList returns a new BuildPlatformList based on given expression.
 //
-// By default the initial set include only the platforms designated as defaults.
+// By default, the initial set include only the platforms designated as defaults.
 // To add additional platforms to list use an addition term that is designated
 // with a plug sign (e.g. "+netbsd" or "+linux/armv7"). Or you may use "+all"
 // to change the initial set to include all possible platforms then filter
 // from there (e.g. "+all linux windows").
 //
-// The expression can consists of selections (e.g. "linux") and/or
+// The expression can consist of selections (e.g. "linux") and/or
 // removals (e.g."!windows"). Each term can be valid GOOS or a valid GOOS/Arch
 // pair.
 //

dev-tools/mage/settings.go

@@ -51,8 +51,8 @@ var (
 	PACKAGES     = EnvOr("PACKAGES", "")
 	CI           = EnvOr("CI", "")
 
-	// CrossBuildMountModcache CrossBuildMountModcache, if true, mounts $GOPATH/pkg/mod into
-	// the crossbuild images at /go/pkg/mod, read-only.
+	// CrossBuildMountModcache mounts $GOPATH/pkg/mod into
+	// the crossbuild images at /go/pkg/mod, read-only,  when set to true.
 	CrossBuildMountModcache = true
 
 	BeatName        = EnvOr("BEAT_NAME", filepath.Base(CWD()))

internal/pkg/artifact/artifact.go

@@ -13,13 +13,15 @@ import (
 )
 
 var packageArchMap = map[string]string{
-	"linux-binary-32":    "linux-x86.tar.gz",
-	"linux-binary-64":    "linux-x86_64.tar.gz",
-	"linux-binary-arm64": "linux-arm64.tar.gz",
-	"windows-binary-32":  "windows-x86.zip",
-	"windows-binary-64":  "windows-x86_64.zip",
-	"darwin-binary-32":   "darwin-x86_64.tar.gz",
-	"darwin-binary-64":   "darwin-x86_64.tar.gz",
+	"linux-binary-32":         "linux-x86.tar.gz",
+	"linux-binary-64":         "linux-x86_64.tar.gz",
+	"linux-binary-arm64":      "linux-arm64.tar.gz",
+	"windows-binary-32":       "windows-x86.zip",
+	"windows-binary-64":       "windows-x86_64.zip",
+	"darwin-binary-32":        "darwin-x86_64.tar.gz",
+	"darwin-binary-64":        "darwin-x86_64.tar.gz",
+	"darwin-binary-arm64":     "darwin-aarch64.tar.gz",
+	"darwin-binary-universal": "darwin-universal.tar.gz",
 }
 
 // GetArtifactName constructs a path to a downloaded artifact

internal/pkg/artifact/config.go

@@ -55,7 +55,7 @@ func DefaultConfig() *Config {
 	}
 }
 
-// OS return configured operating system or falls back to runtime.GOOS
+// OS returns the configured operating system or falls back to runtime.GOOS
 func (c *Config) OS() string {
 	if c.OperatingSystem != "" {
 		return c.OperatingSystem
@@ -73,7 +73,7 @@ func (c *Config) OS() string {
 	return c.OperatingSystem
 }
 
-// Arch return configured architecture or falls back to 32bit
+// Arch returns the configured architecture or falls back to 32bit
 func (c *Config) Arch() string {
 	if c.Architecture != "" {
 		return c.Architecture