Introduce tests for elastic-agent upgrade procedure

Introduce tests for elastic-agent upgrade procedure

.ci/Jenkinsfile

@@ -35,6 +35,7 @@ pipeline {
     string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel(s) where errors will be posted. For multiple channels, use a comma-separated list of channels')
     string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz')
     string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '8.0.0-SNAPSHOT', description: 'SemVer version of the stand-alone elastic-agent to be used for Fleet tests. You can use here the tag of your PR to test your changes')
+    string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.10.0', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.')
     booleanParam(name: "ELASTIC_AGENT_USE_CI_SNAPSHOTS", defaultValue: false, description: "If it's needed to use the binary snapshots produced by Beats CI instead of the official releases")
     choice(name: 'LOG_LEVEL', choices: ['DEBUG', 'INFO'], description: 'Log level to be used')
     choice(name: 'TIMEOUT_FACTOR', choices: ['3', '5', '7', '11'], description: 'Max number of minutes for timeout backoff strategies')

e2e/_suites/fleet/features/fleet_mode_agent.feature

@@ -40,6 +40,16 @@ Examples:
 | centos |
 | debian |
 
+@upgrade-agent
+Scenario Outline: Upgrading the installed <os> agent
+  Given a "<os>" agent "stale" is deployed to Fleet with "tar" installer
+    And certs for "<os>" are installed
+  When agent is upgraded to version "latest"
+  Then agent is in version "latest"
+Examples:
+| os     | 
+| debian | 
+
 @restart-agent
 Scenario Outline: Restarting the installed <os> agent
   Given a "<os>" agent is deployed to Fleet with "tar" installer

e2e/_suites/fleet/fleet.go

@@ -16,12 +16,14 @@ import (
 	curl "github.com/elastic/e2e-testing/cli/shell"
 	"github.com/elastic/e2e-testing/e2e"
 	"github.com/google/uuid"
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 )
 
 const fleetAgentsURL = kibanaBaseURL + "/api/fleet/agents"
 const fleetAgentEventsURL = kibanaBaseURL + "/api/fleet/agents/%s/events"
 const fleetAgentsUnEnrollURL = kibanaBaseURL + "/api/fleet/agents/%s/unenroll"
+const fleetAgentUpgradeURL = kibanaBaseURL + "/api/fleet/agents/%s/upgrade"
 const fleetEnrollmentTokenURL = kibanaBaseURL + "/api/fleet/enrollment-api-keys"
 const fleetSetupURL = kibanaBaseURL + "/api/fleet/agents/setup"
 const ingestManagerAgentPoliciesURL = kibanaBaseURL + "/api/fleet/agent_policies"
@@ -117,6 +119,9 @@ func (fts *FleetTestSuite) beforeScenario() {
 
 func (fts *FleetTestSuite) contributeSteps(s *godog.Suite) {
 	s.Step(`^a "([^"]*)" agent is deployed to Fleet with "([^"]*)" installer$`, fts.anAgentIsDeployedToFleetWithInstaller)
+	s.Step(`^a "([^"]*)" agent "([^"]*)" is deployed to Fleet with "([^"]*)" installer$`, fts.anStaleAgentIsDeployedToFleetWithInstaller)
+	s.Step(`^agent is in version "([^"]*)"$`, fts.agentInVersion)
+	s.Step(`^agent is upgraded to version "([^"]*)"$`, fts.anAgentIsUpgraded)
 	s.Step(`^the agent is listed in Fleet as "([^"]*)"$`, fts.theAgentIsListedInFleetWithStatus)
 	s.Step(`^the host is restarted$`, fts.theHostIsRestarted)
 	s.Step(`^system package dashboards are listed in Fleet$`, fts.systemPackageDashboardsAreListedInFleet)
@@ -126,6 +131,7 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.Suite) {
 	s.Step(`^an attempt to enroll a new agent fails$`, fts.anAttemptToEnrollANewAgentFails)
 	s.Step(`^the "([^"]*)" process is "([^"]*)" on the host$`, fts.processStateChangedOnTheHost)
 	s.Step(`^the file system Agent folder is empty$`, fts.theFileSystemAgentFolderIsEmpty)
+	s.Step(`^certs for "([^"]*)" are installed$`, fts.installCerts)
 
 	// endpoint steps
 	s.Step(`^the "([^"]*)" integration is "([^"]*)" in the policy$`, fts.theIntegrationIsOperatedInThePolicy)
@@ -138,6 +144,98 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.Suite) {
 	s.Step(`^the policy will reflect the change in the Security App$`, fts.thePolicyWillReflectTheChangeInTheSecurityApp)
 }
 
+func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, version, installerType string) error {
+	agentVersionBackup := agentVersion
+	defer func() { agentVersion = agentVersionBackup }()
+
+	switch version {
+	case "stale":
+		version = agentStaleVersion
+	case "latest":
+		version = agentVersion
+	default:
+		version = agentStaleVersion
+	}
+
+	agentVersion = version
+
+	// prepare installer for stale version
+	if agentVersion != agentVersionBackup {
+		i := GetElasticAgentInstaller(image, installerType)
+		installerType = fmt.Sprintf("%s-%s", installerType, version)
+		fts.Installers[fmt.Sprintf("%s-%s", image, installerType)] = i
+	}
+
+	return fts.anAgentIsDeployedToFleetWithInstaller(image, installerType)
+}
+
+func (fts *FleetTestSuite) installCerts(targetOS string) error {
+	installer := fts.getInstaller()
+	if installer.InstallCertsFn == nil {
+		return errors.New("no installer found")
+	}
+
+	return installer.InstallCertsFn()
+}
+
+func (fts *FleetTestSuite) anAgentIsUpgraded(desiredVersion string) error {
+	switch desiredVersion {
+	case "stale":
+		desiredVersion = agentStaleVersion
+	case "latest":
+		desiredVersion = agentVersion
+	default:
+		desiredVersion = agentVersion
+	}
+
+	return fts.upgradeAgent(desiredVersion)
+}
+
+func (fts *FleetTestSuite) agentInVersion(version string) error {
+	switch version {
+	case "stale":
+		version = agentStaleVersion
+	case "latest":
+		version = agentVersion
+	}
+
+	agentInVersionFn := func() error {
+		agentID, err := getAgentID(fts.Hostname)
+		if err != nil {
+			return err
+		}
+
+		r := createDefaultHTTPRequest(fleetAgentsURL + "/" + agentID)
+		body, err := curl.Get(r)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"body":  body,
+				"error": err,
+				"url":   r.GetURL(),
+			}).Error("Could not get agent in Fleet")
+			return err
+		}
+
+		jsonResponse, err := gabs.ParseJSON([]byte(body))
+
+		retrievedVersion := jsonResponse.Path("item.local_metadata.elastic.agent.version").Data().(string)
+		if isSnapshot := jsonResponse.Path("item.local_metadata.elastic.agent.snapshot").Data().(bool); isSnapshot {
+			retrievedVersion += "-SNAPSHOT"
+		}
+
+		if retrievedVersion != version {
+			return fmt.Errorf("version mismatch required '%s' retrieved '%s'", version, retrievedVersion)
+		}
+
+		return nil
+	}
+
+	maxTimeout := time.Duration(timeoutFactor) * time.Minute * 2
+	exp := e2e.GetExponentialBackOff(maxTimeout)
+
+	return backoff.Retry(agentInVersionFn, exp)
+}
+
 // supported installers: tar, systemd
 func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(image string, installerType string) error {
 	log.WithFields(log.Fields{
@@ -951,6 +1049,30 @@ func (fts *FleetTestSuite) unenrollHostname(force bool) error {
 	return nil
 }
 
+func (fts *FleetTestSuite) upgradeAgent(version string) error {
+	agentID, err := getAgentID(fts.Hostname)
+	if err != nil {
+		return err
+	}
+
+	upgradeReq := curl.HTTPRequest{
+		BasicAuthUser:     "elastic",
+		BasicAuthPassword: "changeme",
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+			"kbn-xsrf":     "true",
+		},
+		URL:     fmt.Sprintf(fleetAgentUpgradeURL, agentID),
+		Payload: `{"version":"` + version + `", "force": true}`,
+	}
+
+	if content, err := curl.Post(upgradeReq); err != nil {
+		return errors.Wrap(err, content)
+	}
+
+	return nil
+}
+
 // checkFleetConfiguration checks that Fleet configuration is not missing
 // any requirements and is read. To achieve it, a GET request is executed
 func checkFleetConfiguration() error {

e2e/_suites/fleet/ingest-manager_test.go

@@ -43,6 +43,10 @@ var agentVersionBase = "8.0.0-SNAPSHOT"
 // It can be overriden by ELASTIC_AGENT_VERSION env var
 var agentVersion = agentVersionBase
 
+// agentStaleVersion is the version of the agent to use as a base during upgrade
+// It can be overriden by ELASTIC_AGENT_STALE_VERSION env var. Using latest GA as a default.
+var agentStaleVersion = "7.10.0"
+
 // stackVersion is the version of the stack to use
 // It can be overriden by STACK_VERSION env var
 var stackVersion = agentVersionBase
@@ -75,6 +79,7 @@ func init() {
 
 	timeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", timeoutFactor)
 	agentVersion = shell.GetEnv("ELASTIC_AGENT_VERSION", agentVersionBase)
+	agentStaleVersion = shell.GetEnv("ELASTIC_AGENT_STALE_VERSION", agentStaleVersion)
 
 	// check if version is an alias
 	agentVersion = e2e.GetElasticArtifactVersion(agentVersion)

e2e/_suites/fleet/services.go

@@ -32,6 +32,7 @@ type ElasticAgentInstaller struct {
 	image             string // docker image
 	installerType     string
 	InstallFn         func(containerName string, token string) error
+	InstallCertsFn    func() error
 	logFile           string // the name of the log file
 	logsDir           string // location of the logs
 	name              string // the name for the binary
@@ -112,8 +113,11 @@ func (i *ElasticAgentInstaller) getElasticAgentLogs(hostname string) error {
 	}
 
 	logFile := i.logsDir + i.logFile
+	if strings.Contains(logFile, "%s") {
+		logFile = fmt.Sprintf(logFile, hash)
+	}
 	cmd := []string{
-		"cat", fmt.Sprintf(logFile, hash),
+		"cat", logFile,
 	}
 
 	err = execCommandInService(i.profile, i.image, i.service, cmd, false)
@@ -319,6 +323,22 @@ func newCentosInstaller(image string, tag string) (ElasticAgentInstaller, error)
 		log.Trace("No uninstall commands for Centos + systemd")
 		return nil
 	}
+	installCertsFn := func() error {
+		if err := execCommandInService(profile, image, service, []string{"yum", "check-update"}, false); err != nil {
+			return err
+		}
+		if err := execCommandInService(profile, image, service, []string{"yum", "install", "ca-certificates", "-y"}, false); err != nil {
+			return err
+		}
+		if err := execCommandInService(profile, image, service, []string{"update-ca-trust", "force-enable"}, false); err != nil {
+			return err
+		}
+		if err := execCommandInService(profile, image, service, []string{"update-ca-trust", "extract"}, false); err != nil {
+			return err
+		}
+
+		return nil
+	}
 
 	binDir := "/var/lib/elastic-agent/data/elastic-agent-%s/"
 
@@ -334,6 +354,7 @@ func newCentosInstaller(image string, tag string) (ElasticAgentInstaller, error)
 		homeDir:           "/etc/elastic-agent/",
 		image:             image,
 		InstallFn:         installFn,
+		InstallCertsFn:    installCertsFn,
 		installerType:     "rpm",
 		logFile:           "elastic-agent-json.log",
 		logsDir:           binDir + "logs/",
@@ -400,6 +421,18 @@ func newDebianInstaller(image string, tag string) (ElasticAgentInstaller, error)
 		log.Trace("No uninstall commands for Debian + systemd")
 		return nil
 	}
+	installCertsFn := func() error {
+		if err := execCommandInService(profile, image, service, []string{"apt-get", "update"}, false); err != nil {
+			return err
+		}
+		if err := execCommandInService(profile, image, service, []string{"apt", "install", "ca-certificates", "-y"}, false); err != nil {
+			return err
+		}
+		if err := execCommandInService(profile, image, service, []string{"update-ca-certificates"}, false); err != nil {
+			return err
+		}
+		return nil
+	}
 
 	binDir := "/var/lib/elastic-agent/data/elastic-agent-%s/"
 
@@ -415,6 +448,7 @@ func newDebianInstaller(image string, tag string) (ElasticAgentInstaller, error)
 		homeDir:           "/etc/elastic-agent/",
 		image:             image,
 		InstallFn:         installFn,
+		InstallCertsFn:    installCertsFn,
 		installerType:     "deb",
 		logFile:           "elastic-agent-json.log",
 		logsDir:           binDir + "logs/",
@@ -490,6 +524,18 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) {
 
 		return runElasticAgentCommand(profile, image, service, ElasticAgentProcessName, "uninstall", args)
 	}
+	installCertsFn := func() error {
+		if err := execCommandInService(profile, image, service, []string{"apt-get", "update"}, false); err != nil {
+			return err
+		}
+		if err := execCommandInService(profile, image, service, []string{"apt", "install", "ca-certificates", "-y"}, false); err != nil {
+			return err
+		}
+		if err := execCommandInService(profile, image, service, []string{"update-ca-certificates"}, false); err != nil {
+			return err
+		}
+		return nil
+	}
 
 	return ElasticAgentInstaller{
 		artifactArch:      arch,
@@ -503,9 +549,10 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) {
 		homeDir:           homeDir,
 		image:             image,
 		InstallFn:         installFn,
+		InstallCertsFn:    installCertsFn,
 		installerType:     "tar",
 		logFile:           "elastic-agent.log",
-		logsDir:           "/opt/Elastic/Agent/data/elastic-agent-%s/",
+		logsDir:           "/opt/Elastic/Agent/",
 		name:              tarFile,
 		path:              binaryPath,
 		PostInstallFn:     postInstallFn,

e2e/go.mod

@@ -6,9 +6,11 @@ require (
 	github.com/Jeffail/gabs/v2 v2.5.1
 	github.com/cenkalti/backoff/v4 v4.0.2
 	github.com/cucumber/godog v0.10.0
+	github.com/cucumber/messages-go/v10 v10.0.3
 	github.com/elastic/e2e-testing/cli v0.0.0-20200717181709-15d2db53ded7
 	github.com/elastic/go-elasticsearch/v8 v8.0.0-20190731061900-ea052088db25
 	github.com/google/uuid v1.1.1
+	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.4.2
 )