chore: refactor Fleet upgrade tests (#671) | chore: refactor build artifacts methods (#689) backport for 6.8.x

.ci/Jenkinsfile

@@ -37,7 +37,7 @@ pipeline {
     string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel(s) where errors will be posted. For multiple channels, use a comma-separated list of channels')
     string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz')
     string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '8.0.0-SNAPSHOT', description: 'SemVer version of the stand-alone elastic-agent to be used for Fleet tests. You can use here the tag of your PR to test your changes')
-    string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.10.1', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.')
+    string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.10.2', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.')
     booleanParam(name: "BEATS_USE_CI_SNAPSHOTS", defaultValue: false, description: "If it's needed to use the binary snapshots produced by Beats CI instead of the official releases")
     choice(name: 'LOG_LEVEL', choices: ['DEBUG', 'INFO'], description: 'Log level to be used')
     choice(name: 'TIMEOUT_FACTOR', choices: ['5', '3', '7', '11'], description: 'Max number of minutes for timeout backoff strategies')

cli/docker/docker.go

@@ -6,7 +6,10 @@ package docker
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
+	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/docker/docker/api/types"
@@ -172,6 +175,37 @@ func RemoveContainer(containerName string) error {
 	return nil
 }
 
+// LoadImage loads a TAR file in the local docker engine
+func LoadImage(imagePath string) error {
+	fileNamePath, err := filepath.Abs(imagePath)
+	if err != nil {
+		return err
+	}
+
+	_, err = os.Stat(fileNamePath)
+	if err != nil || os.IsNotExist(err) {
+		return err
+	}
+
+	dockerClient := getDockerClient()
+	file, err := os.Open(imagePath)
+
+	input, err := gzip.NewReader(file)
+	imageLoadResponse, err := dockerClient.ImageLoad(context.Background(), input, false)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error": err,
+			"image": fileNamePath,
+		}).Error("Could not load the Docker image.")
+		return err
+	}
+
+	log.WithFields(log.Fields{
+		"response": imageLoadResponse,
+	}).Debug("Docker image loaded successfully")
+	return nil
+}
+
 // RemoveDevNetwork removes the developer network
 func RemoveDevNetwork() error {
 	dockerClient := getDockerClient()

e2e/Makefile

@@ -91,3 +91,25 @@ sync-integrations:
 unit-test:
 	gotestsum --format testname -- -count=1 -timeout=$(TEST_TIMEOUT) ./...
 	cd _suites && gotestsum --format testname -- -count=1 -timeout=$(TEST_TIMEOUT) ./...
+
+## Test examples
+
+.PHONY: fleet-fleet
+fleet-fleet:
+	SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test
+
+.PHONY: fleet-fleet-ci-snapshots
+fleet-fleet-ci-snapshots:
+	SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test
+
+.PHONY: fleet-fleet-pr-ci-snapshots
+fleet-fleet-pr-ci-snapshots:
+	SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true ELASTIC_AGENT_VERSION=pr-14954 $(MAKE) functional-test
+
+.PHONY: fleet-nightly
+fleet-nightly:
+	SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test
+
+.PHONY: fleet-nightly-ci-snapshots
+fleet-nightly-ci-snapshots:
+	SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test

e2e/_suites/fleet/features/fleet_mode_agent.feature

@@ -40,7 +40,7 @@ Examples:
 | debian |
 
 # @upgrade-agent
-@skip
+@nightly
 Scenario Outline: Upgrading the installed <os> agent
   Given a "<os>" agent "stale" is deployed to Fleet with "tar" installer
     And certs for "<os>" are installed

e2e/_suites/fleet/fleet.go

@@ -42,6 +42,7 @@ type FleetTestSuite struct {
 	CurrentToken   string // current enrollment token
 	CurrentTokenID string // current enrollment tokenID
 	Hostname       string // the hostname of the container
+	Version        string // current elastic-agent version
 	// integrations
 	Integration     IntegrationPackage // the installed integration
 	PolicyUpdatedAt string             // the moment the policy was updated
@@ -107,6 +108,8 @@ func (fts *FleetTestSuite) afterScenario() {
 func (fts *FleetTestSuite) beforeScenario() {
 	fts.Cleanup = false
 
+	fts.Version = agentVersion
+
 	// create policy with system monitoring enabled
 	defaultPolicy, err := getAgentDefaultPolicy()
 	if err != nil {
@@ -148,8 +151,8 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) {
 }
 
 func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, version, installerType string) error {
-	agentVersionBackup := agentVersion
-	defer func() { agentVersion = agentVersionBackup }()
+	agentVersionBackup := fts.Version
+	defer func() { fts.Version = agentVersionBackup }()
 
 	switch version {
 	case "stale":
@@ -160,13 +163,12 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver
 		version = agentStaleVersion
 	}
 
-	agentVersion = version
+	fts.Version = version
 
 	// prepare installer for stale version
-	if agentVersion != agentVersionBackup {
-		i := GetElasticAgentInstaller(image, installerType)
-		installerType = fmt.Sprintf("%s-%s", installerType, version)
-		fts.Installers[fmt.Sprintf("%s-%s", image, installerType)] = i
+	if fts.Version != agentVersionBackup {
+		i := GetElasticAgentInstaller(image, installerType, fts.Version)
+		fts.Installers[fmt.Sprintf("%s-%s-%s", image, installerType, version)] = i
 	}
 
 	return fts.anAgentIsDeployedToFleetWithInstaller(image, installerType)
@@ -175,6 +177,12 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver
 func (fts *FleetTestSuite) installCerts(targetOS string) error {
 	installer := fts.getInstaller()
 	if installer.InstallCertsFn == nil {
+		log.WithFields(log.Fields{
+			"installer":         installer,
+			"version":           fts.Version,
+			"agentVersion":      agentVersion,
+			"agentStaleVersion": agentStaleVersion,
+		}).Error("No installer found")
 		return errors.New("no installer found")
 	}
 
@@ -291,7 +299,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(image string, i
 }
 
 func (fts *FleetTestSuite) getInstaller() ElasticAgentInstaller {
-	return fts.Installers[fts.Image+"-"+fts.InstallerType]
+	return fts.Installers[fts.Image+"-"+fts.InstallerType+"-"+fts.Version]
 }
 
 func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state string) error {
@@ -371,11 +379,10 @@ func (fts *FleetTestSuite) theAgentIsListedInFleetWithStatus(desiredStatus strin
 			// the agent is not listed in Fleet
 			if desiredStatus == "offline" || desiredStatus == "inactive" {
 				log.WithFields(log.Fields{
-					"isAgentInStatus": isAgentInStatus,
-					"elapsedTime":     exp.GetElapsedTime(),
-					"hostname":        fts.Hostname,
-					"retries":         retryCount,
-					"status":          desiredStatus,
+					"elapsedTime": exp.GetElapsedTime(),
+					"hostname":    fts.Hostname,
+					"retries":     retryCount,
+					"status":      desiredStatus,
 				}).Info("The Agent is not present in Fleet, as expected")
 				return nil
 			} else if desiredStatus == "online" {

e2e/_suites/fleet/ingest-manager_test.go

@@ -45,7 +45,7 @@ var agentVersion = agentVersionBase
 
 // agentStaleVersion is the version of the agent to use as a base during upgrade
 // It can be overriden by ELASTIC_AGENT_STALE_VERSION env var. Using latest GA as a default.
-var agentStaleVersion = "7.10.0"
+var agentStaleVersion = "7.10.2"
 
 // stackVersion is the version of the stack to use
 // It can be overriden by STACK_VERSION env var
@@ -83,6 +83,11 @@ func setUpSuite() {
 	agentVersion = shell.GetEnv("ELASTIC_AGENT_VERSION", agentVersionBase)
 	agentStaleVersion = shell.GetEnv("ELASTIC_AGENT_STALE_VERSION", agentStaleVersion)
 
+	useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS")
+	if useCISnapshots && !strings.HasSuffix(agentStaleVersion, "-SNAPSHOT") {
+		agentStaleVersion += "-SNAPSHOT"
+	}
+
 	// check if version is an alias
 	agentVersion = e2e.GetElasticArtifactVersion(agentVersion)
 
@@ -91,10 +96,12 @@ func setUpSuite() {
 	imts = IngestManagerTestSuite{
 		Fleet: &FleetTestSuite{
 			Installers: map[string]ElasticAgentInstaller{
-				"centos-systemd": GetElasticAgentInstaller("centos", "systemd"),
-				"centos-tar":     GetElasticAgentInstaller("centos", "tar"),
-				"debian-systemd": GetElasticAgentInstaller("debian", "systemd"),
-				"debian-tar":     GetElasticAgentInstaller("debian", "tar"),
+				"centos-systemd-" + agentVersion: GetElasticAgentInstaller("centos", "systemd", agentVersion),
+				"centos-tar-" + agentVersion:     GetElasticAgentInstaller("centos", "tar", agentVersion),
+				"debian-systemd-" + agentVersion: GetElasticAgentInstaller("debian", "systemd", agentVersion),
+				"debian-tar-" + agentVersion:     GetElasticAgentInstaller("debian", "tar", agentVersion),
+				"docker-default-" + agentVersion: GetElasticAgentInstaller("docker", "default", agentVersion),
+				"docker-ubi8-" + agentVersion:    GetElasticAgentInstaller("docker", "ubi8", agentVersion),
 			},
 		},
 		StandAlone: &StandAloneTestSuite{},
@@ -226,21 +233,6 @@ func (imts *IngestManagerTestSuite) processStateOnTheHost(process string, state
 	return checkProcessStateOnTheHost(containerName, process, state)
 }
 
-// checkElasticAgentVersion returns a fallback version (agentVersionBase) if the version set by the environment is empty
-func checkElasticAgentVersion(version string) string {
-	environmentVersion := os.Getenv("ELASTIC_AGENT_VERSION")
-
-	if environmentVersion == "" {
-		return agentVersionBase
-	}
-
-	if strings.HasPrefix(strings.ToLower(environmentVersion), "pr-") {
-		return agentVersionBase
-	}
-
-	return version
-}
-
 // name of the container for the service:
 // we are using the Docker client instead of docker-compose
 // because it does not support returning the output of a

e2e/_suites/fleet/installers.go

@@ -1,8 +1,13 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package main
 
 import (
 	"fmt"
 
+	"github.com/elastic/e2e-testing/cli/docker"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -98,6 +103,85 @@ func (i *DEBPackage) Uninstall() error {
 	return nil
 }
 
+// DockerPackage implements operations for a DEB installer
+type DockerPackage struct {
+	BasePackage
+	installerPath string
+	ubi8          bool
+	// optional fields
+	arch     string
+	artifact string
+	OS       string
+	version  string
+}
+
+// NewDockerPackage creates an instance for the Docker installer
+func NewDockerPackage(binaryName string, profile string, image string, service string, installerPath string, ubi8 bool) *DockerPackage {
+	return &DockerPackage{
+		BasePackage: BasePackage{
+			binaryName: binaryName,
+			image:      image,
+			profile:    profile,
+			service:    service,
+		},
+		installerPath: installerPath,
+		ubi8:          ubi8,
+	}
+}
+
+// Install installs a Docker package
+func (i *DockerPackage) Install(containerName string, token string) error {
+	log.Trace("No install commands for Docker packages")
+	return nil
+}
+
+// InstallCerts installs the certificates for a Docker package
+func (i *DockerPackage) InstallCerts() error {
+	log.Trace("No install certs commands for Docker packages")
+	return nil
+}
+
+// Preinstall executes operations before installing a Docker package
+func (i *DockerPackage) Preinstall() error {
+	return docker.LoadImage(i.installerPath)
+}
+
+// Postinstall executes operations after installing a Docker package
+func (i *DockerPackage) Postinstall() error {
+	log.Trace("No postinstall commands for Docker packages")
+	return nil
+}
+
+// Uninstall uninstalls a Docker package
+func (i *DockerPackage) Uninstall() error {
+	log.Trace("No uninstall commands for Docker packages")
+	return nil
+}
+
+// WithArch sets the architecture
+func (i *DockerPackage) WithArch(arch string) *DockerPackage {
+	i.arch = arch
+	return i
+}
+
+// WithArtifact sets the artifact
+func (i *DockerPackage) WithArtifact(artifact string) *DockerPackage {
+	i.artifact = artifact
+	return i
+}
+
+// WithOS sets the OS
+func (i *DockerPackage) WithOS(OS string) *DockerPackage {
+	i.OS = OS
+	return i
+}
+
+// WithVersion sets the version
+func (i *DockerPackage) WithVersion(version string) *DockerPackage {
+	i.version = version
+	return i
+}
+
 // RPMPackage implements operations for a RPM installer
 type RPMPackage struct {
 	BasePackage
@@ -214,20 +298,24 @@ func (i *TARPackage) Preinstall() error {
 		return err
 	}
 
-	version := checkElasticAgentVersion(i.version)
-
 	// simplify layout
-	cmds := []string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, version, i.OS, i.arch), "/elastic-agent"}
-	err = execCommandInService(i.profile, i.image, i.service, cmds, false)
-	if err != nil {
-		log.WithFields(log.Fields{
-			"command": cmds,
-			"error":   err,
-			"image":   i.image,
-			"service": i.service,
-		}).Error("Could not extract agent package in the box")
-
-		return err
+	cmds := [][]string{
+		[]string{"rm", "-fr", "/elastic-agent"},
+		[]string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, i.version, i.OS, i.arch), "/elastic-agent"},
+	}
+	for _, cmd := range cmds {
+		err = execCommandInService(i.profile, i.image, i.service, cmd, false)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"command": cmd,
+				"error":   err,
+				"image":   i.image,
+				"service": i.service,
+				"version": i.version,
+			}).Error("Could not extract agent package in the box")
+
+			return err
+		}
 	}
 
 	return nil