chore: refactor Fleet upgrade tests (elastic#671)

* chore: use nightly annotation for the Upgrade tests

* chore: add two make goals for the nightly use cases

- e2e-fleet-nightly: run the nightly tests not using CI snapshots.
It should download the binaries from the official artifactory.
- e2e-fleet-nightly-ci-snapshots: run the nightly tests using the CI
snapshots for a specific SHA commit from Beats, downloading them from
a GCP bucket

* chore: bump elastic-agent stale version

* chore: pass version and state state when creating an installer

This will allow selecting the proper binary, depending if we are using
a stale agent or a regular one.

* fix: append snapshot to the stale version when using CI snapshots

* fix: check for version aliases with non-stale versions

* chore: store current agent version in the test suite struct

* chore: make sure the layout is properly created for TAR installer

* chore: add make goals for testing fleet use cases

* chore: move Make goals to the e2e Makefile
# Conflicts:
#	.ci/Jenkinsfile

.ci/Jenkinsfile

@@ -37,7 +37,7 @@ pipeline {
     string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel(s) where errors will be posted. For multiple channels, use a comma-separated list of channels')
     string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/7.11.0-326bd5a6/downloads/beats/elastic-agent/elastic-agent-7.11.0-SNAPSHOT-linux-x86_64.tar.gz')
     string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '7.11.0-SNAPSHOT', description: 'SemVer version of the stand-alone elastic-agent to be used for Fleet tests. You can use here the tag of your PR to test your changes')
-    string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.10.1', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.')
+    string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.10.2', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.')
     booleanParam(name: "BEATS_USE_CI_SNAPSHOTS", defaultValue: false, description: "If it's needed to use the binary snapshots produced by Beats CI instead of the official releases")
     choice(name: 'LOG_LEVEL', choices: ['DEBUG', 'INFO'], description: 'Log level to be used')
     choice(name: 'TIMEOUT_FACTOR', choices: ['5', '3', '7', '11'], description: 'Max number of minutes for timeout backoff strategies')

e2e/Makefile

@@ -91,3 +91,25 @@ sync-integrations:
 unit-test:
 	gotestsum --format testname -- -count=1 -timeout=$(TEST_TIMEOUT) ./...
 	cd _suites && gotestsum --format testname -- -count=1 -timeout=$(TEST_TIMEOUT) ./...
+
+## Test examples
+
+.PHONY: fleet-fleet
+fleet-fleet:
+	SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test
+
+.PHONY: fleet-fleet-ci-snapshots
+fleet-fleet-ci-snapshots:
+	SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test
+
+.PHONY: fleet-fleet-pr-ci-snapshots
+fleet-fleet-pr-ci-snapshots:
+	SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true ELASTIC_AGENT_VERSION=pr-14954 $(MAKE) functional-test
+
+.PHONY: fleet-nightly
+fleet-nightly:
+	SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test
+
+.PHONY: fleet-nightly-ci-snapshots
+fleet-nightly-ci-snapshots:
+	SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test

e2e/_suites/fleet/features/fleet_mode_agent.feature

@@ -40,7 +40,7 @@ Examples:
 | debian |
 
 # @upgrade-agent
-@skip
+@nightly
 Scenario Outline: Upgrading the installed <os> agent
   Given a "<os>" agent "stale" is deployed to Fleet with "tar" installer
     And certs for "<os>" are installed

e2e/_suites/fleet/fleet.go

@@ -42,6 +42,7 @@ type FleetTestSuite struct {
 	CurrentToken   string // current enrollment token
 	CurrentTokenID string // current enrollment tokenID
 	Hostname       string // the hostname of the container
+	Version        string // current elastic-agent version
 	// integrations
 	Integration     IntegrationPackage // the installed integration
 	PolicyUpdatedAt string             // the moment the policy was updated
@@ -107,6 +108,8 @@ func (fts *FleetTestSuite) afterScenario() {
 func (fts *FleetTestSuite) beforeScenario() {
 	fts.Cleanup = false
 
+	fts.Version = agentVersion
+
 	// create policy with system monitoring enabled
 	defaultPolicy, err := getAgentDefaultPolicy()
 	if err != nil {
@@ -148,8 +151,8 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) {
 }
 
 func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, version, installerType string) error {
-	agentVersionBackup := agentVersion
-	defer func() { agentVersion = agentVersionBackup }()
+	agentVersionBackup := fts.Version
+	defer func() { fts.Version = agentVersionBackup }()
 
 	switch version {
 	case "stale":
@@ -160,13 +163,12 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver
 		version = agentStaleVersion
 	}
 
-	agentVersion = version
+	fts.Version = version
 
 	// prepare installer for stale version
-	if agentVersion != agentVersionBackup {
-		i := GetElasticAgentInstaller(image, installerType)
-		installerType = fmt.Sprintf("%s-%s", installerType, version)
-		fts.Installers[fmt.Sprintf("%s-%s", image, installerType)] = i
+	if fts.Version != agentVersionBackup {
+		i := GetElasticAgentInstaller(image, installerType, fts.Version, true)
+		fts.Installers[fmt.Sprintf("%s-%s-%s", image, installerType, version)] = i
 	}
 
 	return fts.anAgentIsDeployedToFleetWithInstaller(image, installerType)
@@ -175,6 +177,12 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver
 func (fts *FleetTestSuite) installCerts(targetOS string) error {
 	installer := fts.getInstaller()
 	if installer.InstallCertsFn == nil {
+		log.WithFields(log.Fields{
+			"installer":         installer,
+			"version":           fts.Version,
+			"agentVersion":      agentVersion,
+			"agentStaleVersion": agentStaleVersion,
+		}).Error("No installer found")
 		return errors.New("no installer found")
 	}
 
@@ -291,7 +299,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(image string, i
 }
 
 func (fts *FleetTestSuite) getInstaller() ElasticAgentInstaller {
-	return fts.Installers[fts.Image+"-"+fts.InstallerType]
+	return fts.Installers[fts.Image+"-"+fts.InstallerType+"-"+fts.Version]
 }
 
 func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state string) error {

e2e/_suites/fleet/ingest-manager_test.go

@@ -45,7 +45,7 @@ var agentVersion = agentVersionBase
 
 // agentStaleVersion is the version of the agent to use as a base during upgrade
 // It can be overriden by ELASTIC_AGENT_STALE_VERSION env var. Using latest GA as a default.
-var agentStaleVersion = "7.10.0"
+var agentStaleVersion = "7.10.2"
 
 // stackVersion is the version of the stack to use
 // It can be overriden by STACK_VERSION env var
@@ -83,6 +83,11 @@ func setUpSuite() {
 	agentVersion = shell.GetEnv("ELASTIC_AGENT_VERSION", agentVersionBase)
 	agentStaleVersion = shell.GetEnv("ELASTIC_AGENT_STALE_VERSION", agentStaleVersion)
 
+	useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS")
+	if useCISnapshots && !strings.HasSuffix(agentStaleVersion, "-SNAPSHOT") {
+		agentStaleVersion += "-SNAPSHOT"
+	}
+
 	// check if version is an alias
 	agentVersion = e2e.GetElasticArtifactVersion(agentVersion)
 
@@ -91,10 +96,10 @@ func setUpSuite() {
 	imts = IngestManagerTestSuite{
 		Fleet: &FleetTestSuite{
 			Installers: map[string]ElasticAgentInstaller{
-				"centos-systemd": GetElasticAgentInstaller("centos", "systemd"),
-				"centos-tar":     GetElasticAgentInstaller("centos", "tar"),
-				"debian-systemd": GetElasticAgentInstaller("debian", "systemd"),
-				"debian-tar":     GetElasticAgentInstaller("debian", "tar"),
+				"centos-systemd-" + agentVersion: GetElasticAgentInstaller("centos", "systemd", agentVersion, false),
+				"centos-tar-" + agentVersion:     GetElasticAgentInstaller("centos", "tar", agentVersion, false),
+				"debian-systemd-" + agentVersion: GetElasticAgentInstaller("debian", "systemd", agentVersion, false),
+				"debian-tar-" + agentVersion:     GetElasticAgentInstaller("debian", "tar", agentVersion, false),
 			},
 		},
 		StandAlone: &StandAloneTestSuite{},

e2e/_suites/fleet/installers.go

@@ -157,6 +157,7 @@ type TARPackage struct {
 	arch     string
 	artifact string
 	OS       string
+	stale    bool
 	version  string
 }
 
@@ -214,25 +215,40 @@ func (i *TARPackage) Preinstall() error {
 		return err
 	}
 
-	version := checkElasticAgentVersion(i.version)
+	version := i.version
+	if !i.stale {
+		version = checkElasticAgentVersion(i.version)
+	}
 
 	// simplify layout
-	cmds := []string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, version, i.OS, i.arch), "/elastic-agent"}
-	err = execCommandInService(i.profile, i.image, i.service, cmds, false)
-	if err != nil {
-		log.WithFields(log.Fields{
-			"command": cmds,
-			"error":   err,
-			"image":   i.image,
-			"service": i.service,
-		}).Error("Could not extract agent package in the box")
-
-		return err
+	cmds := [][]string{
+		[]string{"rm", "-fr", "/elastic-agent"},
+		[]string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, version, i.OS, i.arch), "/elastic-agent"},
+	}
+	for _, cmd := range cmds {
+		err = execCommandInService(i.profile, i.image, i.service, cmd, false)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"command": cmd,
+				"error":   err,
+				"image":   i.image,
+				"service": i.service,
+				"version": version,
+			}).Error("Could not extract agent package in the box")
+
+			return err
+		}
 	}
 
 	return nil
 }
 
+// Stale sets the stale state
+func (i *TARPackage) Stale(stale bool) *TARPackage {
+	i.stale = stale
+	return i
+}
+
 // Uninstall uninstalls a TAR package
 func (i *TARPackage) Uninstall() error {
 	args := []string{"-f"}

e2e/_suites/fleet/services.go

@@ -166,7 +166,7 @@ func runElasticAgentCommand(profile string, image string, service string, proces
 // to be used will be defined by the local snapshot produced by the local build.
 // Else, if the environment variable BEATS_USE_CI_SNAPSHOTS is set, then the artifact
 // to be downloaded will be defined by the latest snapshot produced by the Beats CI.
-func downloadAgentBinary(artifact string, version string, OS string, arch string, extension string) (string, string, error) {
+func downloadAgentBinary(artifact string, version string, OS string, arch string, extension string, stale bool) (string, string, error) {
 	fileName := fmt.Sprintf("%s-%s-%s.%s", artifact, version, arch, extension)
 
 	beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "")
@@ -217,7 +217,7 @@ func downloadAgentBinary(artifact string, version string, OS string, arch string
 	if useCISnapshots {
 		log.Debug("Using CI snapshots for the Elastic Agent")
 
-		bucketFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension)
+		bucketFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, stale)
 
 		maxTimeout := time.Duration(timeoutFactor) * time.Minute
 
@@ -229,7 +229,12 @@ func downloadAgentBinary(artifact string, version string, OS string, arch string
 		return handleDownload(downloadURL, bucketFileName)
 	}
 
-	downloadURL, err = e2e.GetElasticArtifactURL(artifact, checkElasticAgentVersion(version), OS, arch, extension)
+	downloadVersion := version
+	if !stale {
+		downloadVersion = checkElasticAgentVersion(version)
+	}
+
+	downloadURL, err = e2e.GetElasticArtifactURL(artifact, downloadVersion, OS, arch, extension)
 	if err != nil {
 		return "", "", err
 	}
@@ -238,7 +243,7 @@ func downloadAgentBinary(artifact string, version string, OS string, arch string
 }
 
 // GetElasticAgentInstaller returns an installer from a docker image
-func GetElasticAgentInstaller(image string, installerType string) ElasticAgentInstaller {
+func GetElasticAgentInstaller(image string, installerType string, version string, stale bool) ElasticAgentInstaller {
 	log.WithFields(log.Fields{
 		"image":     image,
 		"installer": installerType,
@@ -247,13 +252,13 @@ func GetElasticAgentInstaller(image string, installerType string) ElasticAgentIn
 	var installer ElasticAgentInstaller
 	var err error
 	if "centos" == image && "tar" == installerType {
-		installer, err = newTarInstaller("centos", "latest")
+		installer, err = newTarInstaller("centos", "latest", version, stale)
 	} else if "centos" == image && "systemd" == installerType {
-		installer, err = newCentosInstaller("centos", "latest")
+		installer, err = newCentosInstaller("centos", "latest", version, stale)
 	} else if "debian" == image && "tar" == installerType {
-		installer, err = newTarInstaller("debian", "stretch")
+		installer, err = newTarInstaller("debian", "stretch", version, stale)
 	} else if "debian" == image && "systemd" == installerType {
-		installer, err = newDebianInstaller("debian", "stretch")
+		installer, err = newDebianInstaller("debian", "stretch", version, stale)
 	} else {
 		log.WithFields(log.Fields{
 			"image":     image,
@@ -273,7 +278,7 @@ func GetElasticAgentInstaller(image string, installerType string) ElasticAgentIn
 }
 
 // getGCPBucketCoordinates it calculates the bucket path in GCP
-func getGCPBucketCoordinates(fileName string, artifact string, version string, OS string, arch string, extension string) (string, string, string, string) {
+func getGCPBucketCoordinates(fileName string, artifact string, version string, OS string, arch string, extension string, stale bool) (string, string, string, string) {
 	if extension == "tar.gz" {
 		fileName = fmt.Sprintf("%s-%s-%s-%s.%s", artifact, version, OS, arch, extension)
 	}
@@ -307,27 +312,31 @@ func getGCPBucketCoordinates(fileName string, artifact string, version string, O
 		object = fmt.Sprintf("%s/%s", artifact, newFileName)
 	}
 
+	if stale {
+		prefix = fmt.Sprintf("snapshots/%s", artifact)
+		object = newFileName
+	}
+
 	return newFileName, bucket, prefix, object
 }
 
 func isSystemdBased(image string) bool {
 	return strings.HasSuffix(image, "-systemd")
 }
 
-// newCentosInstaller returns an instance of the Centos installer
-func newCentosInstaller(image string, tag string) (ElasticAgentInstaller, error) {
+// newCentosInstaller returns an instance of the Centos installer for a specific version
+func newCentosInstaller(image string, tag string, version string, stale bool) (ElasticAgentInstaller, error) {
 	image = image + "-systemd" // we want to consume systemd boxes
 	service := image
 	profile := FleetProfileName
 
 	// extract the agent in the box, as it's mounted as a volume
 	artifact := "elastic-agent"
-	version := agentVersion
 	os := "linux"
 	arch := "x86_64"
 	extension := "rpm"
 
-	binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension)
+	binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension, stale)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"artifact":  artifact,
@@ -379,20 +388,19 @@ func newCentosInstaller(image string, tag string) (ElasticAgentInstaller, error)
 	}, nil
 }
 
-// newDebianInstaller returns an instance of the Debian installer
-func newDebianInstaller(image string, tag string) (ElasticAgentInstaller, error) {
+// newDebianInstaller returns an instance of the Debian installer for a specific version
+func newDebianInstaller(image string, tag string, version string, stale bool) (ElasticAgentInstaller, error) {
 	image = image + "-systemd" // we want to consume systemd boxes
 	service := image
 	profile := FleetProfileName
 
 	// extract the agent in the box, as it's mounted as a volume
 	artifact := "elastic-agent"
-	version := agentVersion
 	os := "linux"
 	arch := "amd64"
 	extension := "deb"
 
-	binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension)
+	binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension, stale)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"artifact":  artifact,
@@ -444,20 +452,19 @@ func newDebianInstaller(image string, tag string) (ElasticAgentInstaller, error)
 	}, nil
 }
 
-// newTarInstaller returns an instance of the Debian installer
-func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) {
+// newTarInstaller returns an instance of the Debian installer for a specific version
+func newTarInstaller(image string, tag string, version string, stale bool) (ElasticAgentInstaller, error) {
 	image = image + "-systemd" // we want to consume systemd boxes
 	service := image
 	profile := FleetProfileName
 
 	// extract the agent in the box, as it's mounted as a volume
 	artifact := "elastic-agent"
-	version := agentVersion
 	os := "linux"
 	arch := "x86_64"
 	extension := "tar.gz"
 
-	tarFile, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension)
+	tarFile, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension, stale)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"artifact":  artifact,
@@ -482,6 +489,7 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) {
 
 	//
 	installerPackage := NewTARPackage(tarFile, profile, image, service).
+		Stale(stale).
 		WithArch(arch).
 		WithArtifact(artifact).
 		WithOS(os).