chore: refactor build artifacts methods (#689) (#710)

* feat: use local beats directory for elastic-agent installers

* docs: document BEATS_LOCAL_PATH variable

* chore: check that the local file exists

* chore: add unit tests for the BEATS_LOCAL_PATH variable

* chore: simplify logic after test coverage

* chore: enrich test name

* chore: rename test

* chore: move default values to the end of the descriptions

* chore: add a helper method to build artifact names

* chore: build artifact name only once

* chore: pass the entire filename to calculate the GCP bucket coordinates

* chore: simplify method signature, as the file name is not changed within the method

* chore: use buildArtifactName method

* chore: pass filename to helper methods to avoid unnecessary calculations

* chore: rename variable for consistency

* chore: add license headers

* chore: use lowercase comparison for artifact name

* chore: build artifact changes from Beats CI to Elastic's artifactory

* fix: support building artifact names from local

Local builds uses the same as the CI

* WIP

* fix: rename docker-ubi8 installer key

* chore: pass a fallback version when building artifact name

In the case we are using a PR , where the version is "pr-12345", then we
need to use the base version of the product

* chore: remove log

* fix: do not override artifact name in Docker installer

* feat: support consuming the docker images from snapshots or local in standalone mode

* chore: add unit tests for fetching docker images from local Beats repo

* chore: apply version to Docker installer

* chore: support loading metricbeat image from local repository

* chore: exrtact loadImage method to docker helper

* feat: support consuming CI artifacts in metricbeat

* fix: remove non-existent field from logrus log

* chore: extract download logic to helper methods

We are discarding the stale parameter, because the stale version is set
when the installer is required, so we simply download what is requested

* chore: fix typo

* chore: check agent version before setting it

* chore: use the docker client to load the image instead of the docker binary

* chore: add unit tests for check method

* chore: pass fallback version to check method

* fix: properly calculate version for PRs

* chore: move PR check logic to a method

* chore: make test independent of maintenance branch
# Conflicts:
#	e2e/_suites/fleet/services.go
#	e2e/utils.go

cli/docker/docker.go

@@ -6,7 +6,10 @@ package docker
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
+	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/docker/docker/api/types"
@@ -172,6 +175,37 @@ func RemoveContainer(containerName string) error {
 	return nil
 }
 
+// LoadImage loads a TAR file in the local docker engine
+func LoadImage(imagePath string) error {
+	fileNamePath, err := filepath.Abs(imagePath)
+	if err != nil {
+		return err
+	}
+
+	_, err = os.Stat(fileNamePath)
+	if err != nil || os.IsNotExist(err) {
+		return err
+	}
+
+	dockerClient := getDockerClient()
+	file, err := os.Open(imagePath)
+
+	input, err := gzip.NewReader(file)
+	imageLoadResponse, err := dockerClient.ImageLoad(context.Background(), input, false)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error": err,
+			"image": fileNamePath,
+		}).Error("Could not load the Docker image.")
+		return err
+	}
+
+	log.WithFields(log.Fields{
+		"response": imageLoadResponse,
+	}).Debug("Docker image loaded successfully")
+	return nil
+}
+
 // RemoveDevNetwork removes the developer network
 func RemoveDevNetwork() error {
 	dockerClient := getDockerClient()

e2e/_suites/fleet/fleet.go

@@ -167,7 +167,7 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver
 
 	// prepare installer for stale version
 	if fts.Version != agentVersionBackup {
-		i := GetElasticAgentInstaller(image, installerType, fts.Version, true)
+		i := GetElasticAgentInstaller(image, installerType, fts.Version)
 		fts.Installers[fmt.Sprintf("%s-%s-%s", image, installerType, version)] = i
 	}
 
@@ -379,11 +379,10 @@ func (fts *FleetTestSuite) theAgentIsListedInFleetWithStatus(desiredStatus strin
 			// the agent is not listed in Fleet
 			if desiredStatus == "offline" || desiredStatus == "inactive" {
 				log.WithFields(log.Fields{
-					"isAgentInStatus": isAgentInStatus,
-					"elapsedTime":     exp.GetElapsedTime(),
-					"hostname":        fts.Hostname,
-					"retries":         retryCount,
-					"status":          desiredStatus,
+					"elapsedTime": exp.GetElapsedTime(),
+					"hostname":    fts.Hostname,
+					"retries":     retryCount,
+					"status":      desiredStatus,
 				}).Info("The Agent is not present in Fleet, as expected")
 				return nil
 			} else if desiredStatus == "online" {

e2e/_suites/fleet/ingest-manager_test.go

@@ -96,10 +96,12 @@ func setUpSuite() {
 	imts = IngestManagerTestSuite{
 		Fleet: &FleetTestSuite{
 			Installers: map[string]ElasticAgentInstaller{
-				"centos-systemd-" + agentVersion: GetElasticAgentInstaller("centos", "systemd", agentVersion, false),
-				"centos-tar-" + agentVersion:     GetElasticAgentInstaller("centos", "tar", agentVersion, false),
-				"debian-systemd-" + agentVersion: GetElasticAgentInstaller("debian", "systemd", agentVersion, false),
-				"debian-tar-" + agentVersion:     GetElasticAgentInstaller("debian", "tar", agentVersion, false),
+				"centos-systemd-" + agentVersion: GetElasticAgentInstaller("centos", "systemd", agentVersion),
+				"centos-tar-" + agentVersion:     GetElasticAgentInstaller("centos", "tar", agentVersion),
+				"debian-systemd-" + agentVersion: GetElasticAgentInstaller("debian", "systemd", agentVersion),
+				"debian-tar-" + agentVersion:     GetElasticAgentInstaller("debian", "tar", agentVersion),
+				"docker-default-" + agentVersion: GetElasticAgentInstaller("docker", "default", agentVersion),
+				"docker-ubi8-" + agentVersion:    GetElasticAgentInstaller("docker", "ubi8", agentVersion),
 			},
 		},
 		StandAlone: &StandAloneTestSuite{},
@@ -231,21 +233,6 @@ func (imts *IngestManagerTestSuite) processStateOnTheHost(process string, state
 	return checkProcessStateOnTheHost(containerName, process, state)
 }
 
-// checkElasticAgentVersion returns a fallback version (agentVersionBase) if the version set by the environment is empty
-func checkElasticAgentVersion(version string) string {
-	environmentVersion := os.Getenv("ELASTIC_AGENT_VERSION")
-
-	if environmentVersion == "" {
-		return agentVersionBase
-	}
-
-	if strings.HasPrefix(strings.ToLower(environmentVersion), "pr-") {
-		return agentVersionBase
-	}
-
-	return version
-}
-
 // name of the container for the service:
 // we are using the Docker client instead of docker-compose
 // because it does not support returning the output of a

e2e/_suites/fleet/installers.go

@@ -1,8 +1,13 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package main
 
 import (
 	"fmt"
 
+	"github.com/elastic/e2e-testing/cli/docker"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -98,6 +103,85 @@ func (i *DEBPackage) Uninstall() error {
 	return nil
 }
 
+// DockerPackage implements operations for a DEB installer
+type DockerPackage struct {
+	BasePackage
+	installerPath string
+	ubi8          bool
+	// optional fields
+	arch     string
+	artifact string
+	OS       string
+	version  string
+}
+
+// NewDockerPackage creates an instance for the Docker installer
+func NewDockerPackage(binaryName string, profile string, image string, service string, installerPath string, ubi8 bool) *DockerPackage {
+	return &DockerPackage{
+		BasePackage: BasePackage{
+			binaryName: binaryName,
+			image:      image,
+			profile:    profile,
+			service:    service,
+		},
+		installerPath: installerPath,
+		ubi8:          ubi8,
+	}
+}
+
+// Install installs a Docker package
+func (i *DockerPackage) Install(containerName string, token string) error {
+	log.Trace("No install commands for Docker packages")
+	return nil
+}
+
+// InstallCerts installs the certificates for a Docker package
+func (i *DockerPackage) InstallCerts() error {
+	log.Trace("No install certs commands for Docker packages")
+	return nil
+}
+
+// Preinstall executes operations before installing a Docker package
+func (i *DockerPackage) Preinstall() error {
+	return docker.LoadImage(i.installerPath)
+}
+
+// Postinstall executes operations after installing a Docker package
+func (i *DockerPackage) Postinstall() error {
+	log.Trace("No postinstall commands for Docker packages")
+	return nil
+}
+
+// Uninstall uninstalls a Docker package
+func (i *DockerPackage) Uninstall() error {
+	log.Trace("No uninstall commands for Docker packages")
+	return nil
+}
+
+// WithArch sets the architecture
+func (i *DockerPackage) WithArch(arch string) *DockerPackage {
+	i.arch = arch
+	return i
+}
+
+// WithArtifact sets the artifact
+func (i *DockerPackage) WithArtifact(artifact string) *DockerPackage {
+	i.artifact = artifact
+	return i
+}
+
+// WithOS sets the OS
+func (i *DockerPackage) WithOS(OS string) *DockerPackage {
+	i.OS = OS
+	return i
+}
+
+// WithVersion sets the version
+func (i *DockerPackage) WithVersion(version string) *DockerPackage {
+	i.version = version
+	return i
+}
+
 // RPMPackage implements operations for a RPM installer
 type RPMPackage struct {
 	BasePackage
@@ -157,7 +241,6 @@ type TARPackage struct {
 	arch     string
 	artifact string
 	OS       string
-	stale    bool
 	version  string
 }
 
@@ -215,15 +298,10 @@ func (i *TARPackage) Preinstall() error {
 		return err
 	}
 
-	version := i.version
-	if !i.stale {
-		version = checkElasticAgentVersion(i.version)
-	}
-
 	// simplify layout
 	cmds := [][]string{
 		[]string{"rm", "-fr", "/elastic-agent"},
-		[]string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, version, i.OS, i.arch), "/elastic-agent"},
+		[]string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, i.version, i.OS, i.arch), "/elastic-agent"},
 	}
 	for _, cmd := range cmds {
 		err = execCommandInService(i.profile, i.image, i.service, cmd, false)
@@ -233,7 +311,7 @@ func (i *TARPackage) Preinstall() error {
 				"error":   err,
 				"image":   i.image,
 				"service": i.service,
-				"version": version,
+				"version": i.version,
 			}).Error("Could not extract agent package in the box")
 
 			return err
@@ -243,12 +321,6 @@ func (i *TARPackage) Preinstall() error {
 	return nil
 }
 
-// Stale sets the stale state
-func (i *TARPackage) Stale(stale bool) *TARPackage {
-	i.stale = stale
-	return i
-}
-
 // Uninstall uninstalls a TAR package
 func (i *TARPackage) Uninstall() error {
 	args := []string{"-f"}