automate the ironbank generation (#8537)

(cherry picked from commit 682a0e5)

# Conflicts:
#	Makefile
#	magefile.go

Makefile

@@ -305,8 +305,14 @@ release-manager-release: release
 
 .PHONY: release
 release: export PATH:=$(dir $(BIN_MAGE)):$(PATH)
+<<<<<<< HEAD
 release: $(MAGE) build/dependencies.csv
 	$(MAGE) package
+=======
+release: $(MAGE) $(PYTHON) build/$(JAVA_ATTACHER_JAR) build/dependencies.csv $(APM_SERVER_BINARIES)
+	@$(MAGE) package
+	@$(MAGE) ironbank
+>>>>>>> 682a0e5b (automate the ironbank generation (#8537))
 
 build/dependencies.csv: $(PYTHON) go.mod
 	$(PYTHON) script/generate_notice.py ./x-pack/apm-server --csv $@

magefile.go

@@ -29,6 +29,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"time"
 
@@ -161,7 +162,31 @@ func filterPackages(types string) {
 	mage.Packages = packages
 }
 
+<<<<<<< HEAD
 // Package packages the Beat for distribution.
+=======
+// Ironbank packages apm-server for the Ironbank distribution, relying on the
+// binaries having already been built.
+//
+// Use SNAPSHOT=true to build snapshots.
+func Ironbank() error {
+	if runtime.GOARCH != "amd64" {
+		fmt.Printf(">> Ironbank images are only supported for amd64 arch (%s is not supported)\n", runtime.GOARCH)
+		return nil
+	}
+	if err := prepareIronbankBuild(); err != nil {
+		return errors.Wrap(err, "failed to prepare the ironbank context")
+	}
+	if err := saveIronbank(); err != nil {
+		return errors.Wrap(err, "failed to save artifacts for ironbank")
+	}
+	return nil
+}
+
+// Package packages apm-server for distribution, relying on the
+// binaries having already been built.
+//
+>>>>>>> 682a0e5b (automate the ironbank generation (#8537))
 // Use SNAPSHOT=true to build snapshots.
 // Use PLATFORMS to control the target platforms. eg linux/amd64
 // Use TYPES to control the target types. eg docker
@@ -198,6 +223,7 @@ func Update() error {
 	return nil
 }
 
+<<<<<<< HEAD
 func Fields() error {
 	fieldsInclude := "include/fields.go"
 	xpackFieldsInclude := mage.XPackBeatDir(fieldsInclude)
@@ -297,6 +323,9 @@ func generateFieldsYAML(output string, modules ...string) error {
 	return ioutil.WriteFile(output, contents, 0644)
 }
 
+=======
+// GoTestUnit runs the go test unit.
+>>>>>>> 682a0e5b (automate the ironbank generation (#8537))
 // Use RACE_DETECTOR=true to enable the race detector.
 func GoTestUnit(ctx context.Context) error {
 	return mage.GoTest(ctx, mage.DefaultGoTestUnitArgs())
@@ -426,6 +455,92 @@ func DumpVariables() error {
 	return mage.DumpVariables()
 }
 
+func saveIronbank() error {
+	fmt.Println(">> saveIronbank: save the IronBank container context.")
+
+	ironbank := getIronbankContextName()
+	buildDir := filepath.Join("build", ironbank)
+	if _, err := os.Stat(buildDir); os.IsNotExist(err) {
+		return fmt.Errorf("cannot find the folder with the ironbank context: %+v", err)
+	}
+
+	distributionsDir := "build/distributions"
+	if _, err := os.Stat(distributionsDir); os.IsNotExist(err) {
+		err := os.MkdirAll(distributionsDir, 0750)
+		if err != nil {
+			return fmt.Errorf("cannot create folder for docker artifacts: %+v", err)
+		}
+	}
+
+	// change dir to the buildDir location where the ironbank folder exists
+	// this will generate a tar.gz without some nested folders.
+	wd, _ := os.Getwd()
+	os.Chdir(buildDir)
+	defer os.Chdir(wd)
+
+	// move the folder to the parent folder, there are two parent folder since
+	// buildDir contains a two folders dir.
+	tarGzFile := filepath.Join("..", "..", distributionsDir, ironbank+".tar.gz")
+
+	// Save the build context as tar.gz artifact
+	err := mage.Tar("./", tarGzFile)
+	if err != nil {
+		return fmt.Errorf("cannot compress the tar.gz file: %+v", err)
+	}
+
+	return errors.Wrap(mage.CreateSHA512File(tarGzFile), "failed to create .sha512 file")
+}
+
+func getIronbankContextName() string {
+	version, _ := mage.BeatQualifiedVersion()
+	defaultBinaryName := "{{.Name}}-ironbank-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}"
+	outputDir, _ := mage.Expand(defaultBinaryName+"-docker-build-context", map[string]interface{}{
+		"Name":    "apm-server",
+		"Version": version,
+	})
+	return outputDir
+}
+
+func prepareIronbankBuild() error {
+	fmt.Println(">> prepareIronbankBuild: prepare the IronBank container context.")
+	ironbank := getIronbankContextName()
+	buildDir := filepath.Join("build", ironbank)
+	templatesDir := filepath.Join("packaging", "ironbank")
+
+	data := map[string]interface{}{
+		"MajorMinor": majorMinor(),
+	}
+
+	err := filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error {
+		if !info.IsDir() {
+			target := strings.TrimSuffix(
+				filepath.Join(buildDir, filepath.Base(path)),
+				".tmpl",
+			)
+
+			err := mage.ExpandFile(path, target, data)
+			if err != nil {
+				return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target)
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func majorMinor() string {
+	if v, _ := mage.BeatQualifiedVersion(); v != "" {
+		parts := strings.SplitN(v, ".", 3)
+		return parts[0] + "." + parts[1]
+	}
+	return ""
+}
+
+// Check checks the source code for common problems.
 func Check() error {
 	fmt.Println(">> check: Checking source code for common problems")
 

packaging/ironbank/Dockerfile.tmpl

@@ -0,0 +1,74 @@
+################################################################################
+# Build stage 0
+# Extract APM Server and make various file manipulations.
+################################################################################
+ARG BASE_REGISTRY=registry1.dsop.io
+ARG BASE_IMAGE=ironbank/redhat/ubi/ubi8
+ARG BASE_TAG=8.6
+
+FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} AS builder
+
+ARG ELASTIC_STACK={{ beat_version }}
+ARG ELASTIC_PRODUCT=apm-server
+ARG OS_AND_ARCH=linux-x86_64
+
+RUN mkdir /usr/share/${ELASTIC_PRODUCT}
+WORKDIR /usr/share/${ELASTIC_PRODUCT}
+COPY --chown=1000:0 ${ELASTIC_PRODUCT}-${ELASTIC_STACK}-${OS_AND_ARCH}.tar.gz  .
+RUN tar --strip-components=1 -zxf ${ELASTIC_PRODUCT}-${ELASTIC_STACK}-${OS_AND_ARCH}.tar.gz
+# Support arbitrary user ids
+# Ensure that group permissions are the same as user permissions.
+# This will help when relying on GID-0 to run Kibana, rather than UID-1000.
+# OpenShift does this, for example.
+# REF: https://docs.okd.io/latest/openshift_images/create-images.html
+RUN chmod -R g=u /usr/share/${ELASTIC_PRODUCT}
+
+# Create auxiliar folders and assing default permissions.
+RUN mkdir /usr/share/${ELASTIC_PRODUCT}/data /usr/share/${ELASTIC_PRODUCT}/logs && \
+    chown -R root:root /usr/share/${ELASTIC_PRODUCT} && \
+    find /usr/share/${ELASTIC_PRODUCT} -type d -exec chmod 0750 {} \; && \
+    find /usr/share/${ELASTIC_PRODUCT} -type f -exec chmod 0640 {} \; && \
+    chmod 0750 /usr/share/${ELASTIC_PRODUCT}/${ELASTIC_PRODUCT} && \
+    chmod 0770 /usr/share/${ELASTIC_PRODUCT}/data /usr/share/${ELASTIC_PRODUCT}/logs
+
+################################################################################
+# Build stage 1
+# Copy prepared files from the previous stage and complete the image.
+################################################################################
+FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}
+
+ARG ELASTIC_PRODUCT=apm-server
+
+COPY LICENSE /licenses/elastic-${ELASTIC_PRODUCT}
+
+# Add a dumb init process
+COPY tinit /tinit
+RUN chmod +x /tinit
+
+# Bring in product from the initial stage.
+COPY --from=prep_files --chown=1000:0 /usr/share/${ELASTIC_PRODUCT} /usr/share/${ELASTIC_PRODUCT}
+WORKDIR /usr/share/${ELASTIC_PRODUCT}
+RUN ln -s /usr/share/${ELASTIC_PRODUCT} /opt/${ELASTIC_PRODUCT}
+
+ENV ELASTIC_CONTAINER true
+RUN ln -s /usr/share/${ELASTIC_PRODUCT}/${ELASTIC_PRODUCT} /usr/bin/${ELASTIC_PRODUCT}
+
+# Support arbitrary user ids
+# Ensure gid 0 write permissions for OpenShift.
+RUN chmod -R g+w /usr/share/${ELASTIC_PRODUCT}
+
+# config file ("${ELASTIC_PRODUCT}.yml") can only be writable by the owner
+RUN chmod go-w /usr/share/${ELASTIC_PRODUCT}/${ELASTIC_PRODUCT}.yml
+
+# Remove the suid bit everywhere to mitigate "Stack Clash"
+RUN find / -xdev -perm -4000 -exec chmod u-s {} +
+
+# Provide a non-root user to run the process.
+RUN groupadd --gid 1000 ${ELASTIC_PRODUCT} && useradd --uid 1000 --gid 1000 --home-dir /usr/share/${ELASTIC_PRODUCT} --no-create-home ${ELASTIC_PRODUCT}
+USER ${ELASTIC_PRODUCT}
+
+EXPOSE 8200
+ENTRYPOINT ["/tinit", "--", "/usr/share/apm-server/apm-server"]
+CMD ["-environment", "container"]
+
+HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD (curl -I -f --max-time 5 https://localhost:8200 || curl -I -f --max-time 5 http://localhost:8200 || exit 1)