diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index ac0a8855ca..b763e58fbc 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -35,6 +35,7 @@ pipeline { string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel(s) where errors will be posted. For multiple channels, use a comma-separated list of channels') string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz') string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '8.0.0-SNAPSHOT', description: 'SemVer version of the stand-alone elastic-agent to be used for Fleet tests. You can use here the tag of your PR to test your changes') + string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.10.0', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.') booleanParam(name: "ELASTIC_AGENT_USE_CI_SNAPSHOTS", defaultValue: false, description: "If it's needed to use the binary snapshots produced by Beats CI instead of the official releases") choice(name: 'LOG_LEVEL', choices: ['DEBUG', 'INFO'], description: 'Log level to be used') choice(name: 'TIMEOUT_FACTOR', choices: ['3', '5', '7', '11'], description: 'Max number of minutes for timeout backoff strategies') diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index b1963c6f86..70e525a4ed 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -40,6 +40,16 @@ Examples: | centos | | debian | +@upgrade-agent +Scenario Outline: Upgrading the installed agent + Given a "" agent "stale" is deployed to Fleet with "tar" installer + And certs for "" are installed + When agent is upgraded to version "latest" + Then agent is in version "latest" +Examples: +| os | +| debian | + @restart-agent Scenario Outline: Restarting the installed agent Given a "" agent is deployed to Fleet with "tar" installer diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index 490286e56f..8b6491fe8f 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -16,12 +16,14 @@ import ( curl "github.com/elastic/e2e-testing/cli/shell" "github.com/elastic/e2e-testing/e2e" "github.com/google/uuid" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) const fleetAgentsURL = kibanaBaseURL + "/api/fleet/agents" const fleetAgentEventsURL = kibanaBaseURL + "/api/fleet/agents/%s/events" const fleetAgentsUnEnrollURL = kibanaBaseURL + "/api/fleet/agents/%s/unenroll" +const fleetAgentUpgradeURL = kibanaBaseURL + "/api/fleet/agents/%s/upgrade" const fleetEnrollmentTokenURL = kibanaBaseURL + "/api/fleet/enrollment-api-keys" const fleetSetupURL = kibanaBaseURL + "/api/fleet/agents/setup" const ingestManagerAgentPoliciesURL = kibanaBaseURL + "/api/fleet/agent_policies" @@ -117,6 +119,9 @@ func (fts *FleetTestSuite) beforeScenario() { func (fts *FleetTestSuite) contributeSteps(s *godog.Suite) { s.Step(`^a "([^"]*)" agent is deployed to Fleet with "([^"]*)" installer$`, fts.anAgentIsDeployedToFleetWithInstaller) + s.Step(`^a "([^"]*)" agent "([^"]*)" is deployed to Fleet with "([^"]*)" installer$`, fts.anStaleAgentIsDeployedToFleetWithInstaller) + s.Step(`^agent is in version "([^"]*)"$`, fts.agentInVersion) + s.Step(`^agent is upgraded to version "([^"]*)"$`, fts.anAgentIsUpgraded) s.Step(`^the agent is listed in Fleet as "([^"]*)"$`, fts.theAgentIsListedInFleetWithStatus) s.Step(`^the host is restarted$`, fts.theHostIsRestarted) s.Step(`^system package dashboards are listed in Fleet$`, fts.systemPackageDashboardsAreListedInFleet) @@ -126,6 +131,7 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.Suite) { s.Step(`^an attempt to enroll a new agent fails$`, fts.anAttemptToEnrollANewAgentFails) s.Step(`^the "([^"]*)" process is "([^"]*)" on the host$`, fts.processStateChangedOnTheHost) s.Step(`^the file system Agent folder is empty$`, fts.theFileSystemAgentFolderIsEmpty) + s.Step(`^certs for "([^"]*)" are installed$`, fts.installCerts) // endpoint steps s.Step(`^the "([^"]*)" integration is "([^"]*)" in the policy$`, fts.theIntegrationIsOperatedInThePolicy) @@ -138,6 +144,98 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.Suite) { s.Step(`^the policy will reflect the change in the Security App$`, fts.thePolicyWillReflectTheChangeInTheSecurityApp) } +func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, version, installerType string) error { + agentVersionBackup := agentVersion + defer func() { agentVersion = agentVersionBackup }() + + switch version { + case "stale": + version = agentStaleVersion + case "latest": + version = agentVersion + default: + version = agentStaleVersion + } + + agentVersion = version + + // prepare installer for stale version + if agentVersion != agentVersionBackup { + i := GetElasticAgentInstaller(image, installerType) + installerType = fmt.Sprintf("%s-%s", installerType, version) + fts.Installers[fmt.Sprintf("%s-%s", image, installerType)] = i + } + + return fts.anAgentIsDeployedToFleetWithInstaller(image, installerType) +} + +func (fts *FleetTestSuite) installCerts(targetOS string) error { + installer := fts.getInstaller() + if installer.InstallCertsFn == nil { + return errors.New("no installer found") + } + + return installer.InstallCertsFn() +} + +func (fts *FleetTestSuite) anAgentIsUpgraded(desiredVersion string) error { + switch desiredVersion { + case "stale": + desiredVersion = agentStaleVersion + case "latest": + desiredVersion = agentVersion + default: + desiredVersion = agentVersion + } + + return fts.upgradeAgent(desiredVersion) +} + +func (fts *FleetTestSuite) agentInVersion(version string) error { + switch version { + case "stale": + version = agentStaleVersion + case "latest": + version = agentVersion + } + + agentInVersionFn := func() error { + agentID, err := getAgentID(fts.Hostname) + if err != nil { + return err + } + + r := createDefaultHTTPRequest(fleetAgentsURL + "/" + agentID) + body, err := curl.Get(r) + if err != nil { + log.WithFields(log.Fields{ + "body": body, + "error": err, + "url": r.GetURL(), + }).Error("Could not get agent in Fleet") + return err + } + + jsonResponse, err := gabs.ParseJSON([]byte(body)) + + retrievedVersion := jsonResponse.Path("item.local_metadata.elastic.agent.version").Data().(string) + if isSnapshot := jsonResponse.Path("item.local_metadata.elastic.agent.snapshot").Data().(bool); isSnapshot { + retrievedVersion += "-SNAPSHOT" + } + + if retrievedVersion != version { + return fmt.Errorf("version mismatch required '%s' retrieved '%s'", version, retrievedVersion) + } + + return nil + } + + maxTimeout := time.Duration(timeoutFactor) * time.Minute * 2 + exp := e2e.GetExponentialBackOff(maxTimeout) + + return backoff.Retry(agentInVersionFn, exp) +} + // supported installers: tar, systemd func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(image string, installerType string) error { log.WithFields(log.Fields{ @@ -951,6 +1049,30 @@ func (fts *FleetTestSuite) unenrollHostname(force bool) error { return nil } +func (fts *FleetTestSuite) upgradeAgent(version string) error { + agentID, err := getAgentID(fts.Hostname) + if err != nil { + return err + } + + upgradeReq := curl.HTTPRequest{ + BasicAuthUser: "elastic", + BasicAuthPassword: "changeme", + Headers: map[string]string{ + "Content-Type": "application/json", + "kbn-xsrf": "true", + }, + URL: fmt.Sprintf(fleetAgentUpgradeURL, agentID), + Payload: `{"version":"` + version + `", "force": true}`, + } + + if content, err := curl.Post(upgradeReq); err != nil { + return errors.Wrap(err, content) + } + + return nil +} + // checkFleetConfiguration checks that Fleet configuration is not missing // any requirements and is read. To achieve it, a GET request is executed func checkFleetConfiguration() error { diff --git a/e2e/_suites/fleet/ingest-manager_test.go b/e2e/_suites/fleet/ingest-manager_test.go index 1860bd883f..511e9b4f53 100644 --- a/e2e/_suites/fleet/ingest-manager_test.go +++ b/e2e/_suites/fleet/ingest-manager_test.go @@ -43,6 +43,10 @@ var agentVersionBase = "8.0.0-SNAPSHOT" // It can be overriden by ELASTIC_AGENT_VERSION env var var agentVersion = agentVersionBase +// agentStaleVersion is the version of the agent to use as a base during upgrade +// It can be overriden by ELASTIC_AGENT_STALE_VERSION env var. Using latest GA as a default. +var agentStaleVersion = "7.10.0" + // stackVersion is the version of the stack to use // It can be overriden by STACK_VERSION env var var stackVersion = agentVersionBase @@ -75,6 +79,7 @@ func init() { timeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", timeoutFactor) agentVersion = shell.GetEnv("ELASTIC_AGENT_VERSION", agentVersionBase) + agentStaleVersion = shell.GetEnv("ELASTIC_AGENT_STALE_VERSION", agentStaleVersion) // check if version is an alias agentVersion = e2e.GetElasticArtifactVersion(agentVersion) diff --git a/e2e/_suites/fleet/services.go b/e2e/_suites/fleet/services.go index e6ae0d7cfa..60852f6ade 100644 --- a/e2e/_suites/fleet/services.go +++ b/e2e/_suites/fleet/services.go @@ -32,6 +32,7 @@ type ElasticAgentInstaller struct { image string // docker image installerType string InstallFn func(containerName string, token string) error + InstallCertsFn func() error logFile string // the name of the log file logsDir string // location of the logs name string // the name for the binary @@ -112,8 +113,11 @@ func (i *ElasticAgentInstaller) getElasticAgentLogs(hostname string) error { } logFile := i.logsDir + i.logFile + if strings.Contains(logFile, "%s") { + logFile = fmt.Sprintf(logFile, hash) + } cmd := []string{ - "cat", fmt.Sprintf(logFile, hash), + "cat", logFile, } err = execCommandInService(i.profile, i.image, i.service, cmd, false) @@ -319,6 +323,22 @@ func newCentosInstaller(image string, tag string) (ElasticAgentInstaller, error) log.Trace("No uninstall commands for Centos + systemd") return nil } + installCertsFn := func() error { + if err := execCommandInService(profile, image, service, []string{"yum", "check-update"}, false); err != nil { + return err + } + if err := execCommandInService(profile, image, service, []string{"yum", "install", "ca-certificates", "-y"}, false); err != nil { + return err + } + if err := execCommandInService(profile, image, service, []string{"update-ca-trust", "force-enable"}, false); err != nil { + return err + } + if err := execCommandInService(profile, image, service, []string{"update-ca-trust", "extract"}, false); err != nil { + return err + } + + return nil + } binDir := "/var/lib/elastic-agent/data/elastic-agent-%s/" @@ -334,6 +354,7 @@ func newCentosInstaller(image string, tag string) (ElasticAgentInstaller, error) homeDir: "/etc/elastic-agent/", image: image, InstallFn: installFn, + InstallCertsFn: installCertsFn, installerType: "rpm", logFile: "elastic-agent-json.log", logsDir: binDir + "logs/", @@ -400,6 +421,18 @@ func newDebianInstaller(image string, tag string) (ElasticAgentInstaller, error) log.Trace("No uninstall commands for Debian + systemd") return nil } + installCertsFn := func() error { + if err := execCommandInService(profile, image, service, []string{"apt-get", "update"}, false); err != nil { + return err + } + if err := execCommandInService(profile, image, service, []string{"apt", "install", "ca-certificates", "-y"}, false); err != nil { + return err + } + if err := execCommandInService(profile, image, service, []string{"update-ca-certificates"}, false); err != nil { + return err + } + return nil + } binDir := "/var/lib/elastic-agent/data/elastic-agent-%s/" @@ -415,6 +448,7 @@ func newDebianInstaller(image string, tag string) (ElasticAgentInstaller, error) homeDir: "/etc/elastic-agent/", image: image, InstallFn: installFn, + InstallCertsFn: installCertsFn, installerType: "deb", logFile: "elastic-agent-json.log", logsDir: binDir + "logs/", @@ -490,6 +524,18 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) { return runElasticAgentCommand(profile, image, service, ElasticAgentProcessName, "uninstall", args) } + installCertsFn := func() error { + if err := execCommandInService(profile, image, service, []string{"apt-get", "update"}, false); err != nil { + return err + } + if err := execCommandInService(profile, image, service, []string{"apt", "install", "ca-certificates", "-y"}, false); err != nil { + return err + } + if err := execCommandInService(profile, image, service, []string{"update-ca-certificates"}, false); err != nil { + return err + } + return nil + } return ElasticAgentInstaller{ artifactArch: arch, @@ -503,9 +549,10 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) { homeDir: homeDir, image: image, InstallFn: installFn, + InstallCertsFn: installCertsFn, installerType: "tar", logFile: "elastic-agent.log", - logsDir: "/opt/Elastic/Agent/data/elastic-agent-%s/", + logsDir: "/opt/Elastic/Agent/", name: tarFile, path: binaryPath, PostInstallFn: postInstallFn, diff --git a/e2e/go.mod b/e2e/go.mod index f83c3aa7be..417b5458c1 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -6,9 +6,11 @@ require ( github.com/Jeffail/gabs/v2 v2.5.1 github.com/cenkalti/backoff/v4 v4.0.2 github.com/cucumber/godog v0.10.0 + github.com/cucumber/messages-go/v10 v10.0.3 github.com/elastic/e2e-testing/cli v0.0.0-20200717181709-15d2db53ded7 github.com/elastic/go-elasticsearch/v8 v8.0.0-20190731061900-ea052088db25 github.com/google/uuid v1.1.1 + github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.4.2 )