From 22986b037e09e87ae644a82ee3044569125c13bd Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 2 Dec 2020 11:12:36 +0100 Subject: [PATCH 01/20] upgrade --- .../fleet/features/fleet_mode_agent.feature | 10 ++ e2e/_suites/fleet/fleet.go | 116 ++++++++++++++++++ e2e/_suites/fleet/ingest-manager_test.go | 5 + e2e/_suites/fleet/services.go | 7 +- 4 files changed, 135 insertions(+), 3 deletions(-) diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index 1c765373a6..53c332e22f 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -26,6 +26,16 @@ Examples: | centos | tar | | debian | tar | +@upgrade-agent +Scenario Outline: Upgrading the installed agent + Given a "" agent "stale" is deployed to Fleet with "" installer + When agent is upgraded to version "latest" + Then agent is in version "stack" + And the agent is listed in Fleet as "online" +Examples: +| os | installer | +| debian | tar | + @restart-agent Scenario Outline: Restarting the installed agent Given a "" agent is deployed to Fleet with "" installer diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index 490286e56f..960dbd8865 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -16,12 +16,14 @@ import ( curl "github.com/elastic/e2e-testing/cli/shell" "github.com/elastic/e2e-testing/e2e" "github.com/google/uuid" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) const fleetAgentsURL = kibanaBaseURL + "/api/fleet/agents" const fleetAgentEventsURL = kibanaBaseURL + "/api/fleet/agents/%s/events" const fleetAgentsUnEnrollURL = kibanaBaseURL + "/api/fleet/agents/%s/unenroll" +const fleetAgentUpgradeURL = kibanaBaseURL + "/api/fleet/agents/%s/upgrade" const fleetEnrollmentTokenURL = kibanaBaseURL + "/api/fleet/enrollment-api-keys" const fleetSetupURL = kibanaBaseURL + "/api/fleet/agents/setup" const ingestManagerAgentPoliciesURL = kibanaBaseURL + "/api/fleet/agent_policies" @@ -117,6 +119,9 @@ func (fts *FleetTestSuite) beforeScenario() { func (fts *FleetTestSuite) contributeSteps(s *godog.Suite) { s.Step(`^a "([^"]*)" agent is deployed to Fleet with "([^"]*)" installer$`, fts.anAgentIsDeployedToFleetWithInstaller) + s.Step(`^a "([^"]*)" agent "([^"]*)" is deployed to Fleet with "([^"]*)" installer$`, fts.anStaleAgentIsDeployedToFleetWithInstaller) + s.Step(`^agent is in version "([^"]*)"$`, fts.agentInVersion) + s.Step(`^agent is upgraded to version "([^"]*)"$`, fts.anAgentIsUpgraded) s.Step(`^the agent is listed in Fleet as "([^"]*)"$`, fts.theAgentIsListedInFleetWithStatus) s.Step(`^the host is restarted$`, fts.theHostIsRestarted) s.Step(`^system package dashboards are listed in Fleet$`, fts.systemPackageDashboardsAreListedInFleet) @@ -138,6 +143,93 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.Suite) { s.Step(`^the policy will reflect the change in the Security App$`, fts.thePolicyWillReflectTheChangeInTheSecurityApp) } +func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, version, installerType string) error { + agentVersionBackup := agentVersion + defer func() { agentVersion = agentVersionBackup }() + + switch version { + case "stale": + version = agentStaleVersion + case "latest": + version = agentVersion + default: + version = agentStaleVersion + } + + agentVersion = version + + // prepare installer for stale version + if agentVersion != agentVersionBackup { + i := GetElasticAgentInstaller(image, installerType) + installerType = fmt.Sprintf("%s-%s", installerType, version) + fts.Installers[fmt.Sprintf("%s-%s", image, installerType)] = i + } + + return fts.anAgentIsDeployedToFleetWithInstaller(image, installerType) +} + +func (fts *FleetTestSuite) anAgentIsUpgraded(desiredVersion string) error { + switch desiredVersion { + case "stale": + desiredVersion = agentStaleVersion + case "latest": + desiredVersion = agentVersion + default: + desiredVersion = agentVersion + } + + // prepare installer for stale version + // installerType := fts.InstallerType + // if desiredVersion != agentVersion { + // fts.InstallerType = fmt.Sprintf("%s-%s", installerType, desiredVersion) + // } + + // installer := fts.getInstaller() + // defer func() { fts.InstallerType = installerType }() + + return fts.upgradeAgent(desiredVersion) +} + +func (fts *FleetTestSuite) agentInVersion(version string) error { + switch version { + case "stale": + version = agentStaleVersion + case "latest": + version = agentVersion + case "stack": + version = stackVersion + } + + agentID, err := getAgentID(fts.Hostname) + if err != nil { + return err + } + + r := createDefaultHTTPRequest(fleetAgentsURL + "/" + agentID) + body, err := curl.Get(r) + if err != nil { + log.WithFields(log.Fields{ + "body": body, + "error": err, + "url": r.GetURL(), + }).Error("Could not get agent in Fleet") + return err + } + + jsonResponse, err := gabs.ParseJSON([]byte(body)) + + retrievedVersion := jsonResponse.Path("item.local_metadata.elastic.agent.version").Data().(string) + if isSnapshot := jsonResponse.Path("item.local_metadata.elastic.agent.version").Data().(string); isSnapshot == "true" { + retrievedVersion += "-SNAPSHOT" + } + + if retrievedVersion != version { + return fmt.Errorf("version mismatch required '%s' desired '%s'", version, retrievedVersion) + } + + return nil +} + // supported installers: tar, systemd func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(image string, installerType string) error { log.WithFields(log.Fields{ @@ -951,6 +1043,30 @@ func (fts *FleetTestSuite) unenrollHostname(force bool) error { return nil } +func (fts *FleetTestSuite) upgradeAgent(version string) error { + agentID, err := getAgentID(fts.Hostname) + if err != nil { + return err + } + + upgradeReq := curl.HTTPRequest{ + BasicAuthUser: "elastic", + BasicAuthPassword: "changeme", + Headers: map[string]string{ + "Content-Type": "application/json", + "kbn-xsrf": "true", + }, + URL: fmt.Sprintf(fleetAgentUpgradeURL, agentID), + Payload: `{"version":"` + version + `"}`, + } + + if content, err := curl.Post(upgradeReq); err != nil { + return errors.Wrap(err, content) + } + + return nil +} + // checkFleetConfiguration checks that Fleet configuration is not missing // any requirements and is read. To achieve it, a GET request is executed func checkFleetConfiguration() error { diff --git a/e2e/_suites/fleet/ingest-manager_test.go b/e2e/_suites/fleet/ingest-manager_test.go index 8be12e2e30..2115f50db2 100644 --- a/e2e/_suites/fleet/ingest-manager_test.go +++ b/e2e/_suites/fleet/ingest-manager_test.go @@ -42,6 +42,10 @@ var agentVersionBase = "8.0.0-SNAPSHOT" // It can be overriden by ELASTIC_AGENT_VERSION env var var agentVersion = agentVersionBase +// agentStaleVersion is the version of the agent to use as a base during upgrade +// It can be overriden by ELASTIC_AGENT_STALE_VERSION env var. Using latest GA as a default. +var agentStaleVersion = "7.10.0" + // stackVersion is the version of the stack to use // It can be overriden by STACK_VERSION env var var stackVersion = agentVersionBase @@ -71,6 +75,7 @@ func init() { timeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", timeoutFactor) agentVersion = shell.GetEnv("ELASTIC_AGENT_VERSION", agentVersionBase) + agentStaleVersion = shell.GetEnv("ELASTIC_AGENT_STALE_VERSION", agentStaleVersion) stackVersion = shell.GetEnv("STACK_VERSION", stackVersion) } diff --git a/e2e/_suites/fleet/services.go b/e2e/_suites/fleet/services.go index 9450a39305..7dff82360b 100644 --- a/e2e/_suites/fleet/services.go +++ b/e2e/_suites/fleet/services.go @@ -215,7 +215,7 @@ func downloadAgentBinary(artifact string, version string, OS string, arch string return handleDownload(downloadURL, fileName) } - downloadURL, err = e2e.GetElasticArtifactURL(artifact, agentVersionBase, OS, arch, extension) + downloadURL, err = e2e.GetElasticArtifactURL(artifact, version, OS, arch, extension) if err != nil { return "", "", err } @@ -456,7 +456,7 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) { preInstallFn := func() error { commitFile := homeDir + commitFile - return installFromTar(profile, image, service, tarFile, commitFile, artifact, agentVersionBase, os, arch) + return installFromTar(profile, image, service, tarFile, commitFile, artifact, version, os, arch) } installFn := func(containerName string, token string) error { // install the elastic-agent to /usr/bin/elastic-agent using command @@ -479,6 +479,7 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) { return nil } unInstallFn := func() error { + return nil args := []string{"-f"} return runElasticAgentCommand(profile, image, service, ElasticAgentProcessName, "uninstall", args) @@ -497,7 +498,7 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) { image: image, InstallFn: installFn, installerType: "tar", - logFile: "elastic-agent.log", + logFile: "elastic-agent-json.log", logsDir: "/opt/Elastic/Agent/data/elastic-agent-%s/logs/", name: tarFile, path: binaryPath, From 28b7b3682def0b158542f79f43be15434e32e97d Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 2 Dec 2020 11:22:12 +0100 Subject: [PATCH 02/20] may the force --- e2e/_suites/fleet/fleet.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index 960dbd8865..dabb9ba07f 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -178,15 +178,6 @@ func (fts *FleetTestSuite) anAgentIsUpgraded(desiredVersion string) error { desiredVersion = agentVersion } - // prepare installer for stale version - // installerType := fts.InstallerType - // if desiredVersion != agentVersion { - // fts.InstallerType = fmt.Sprintf("%s-%s", installerType, desiredVersion) - // } - - // installer := fts.getInstaller() - // defer func() { fts.InstallerType = installerType }() - return fts.upgradeAgent(desiredVersion) } @@ -1057,7 +1048,7 @@ func (fts *FleetTestSuite) upgradeAgent(version string) error { "kbn-xsrf": "true", }, URL: fmt.Sprintf(fleetAgentUpgradeURL, agentID), - Payload: `{"version":"` + version + `"}`, + Payload: `{"version":"` + version + `", "force": true}`, } if content, err := curl.Post(upgradeReq); err != nil { From 276810c5d7f390c6a1c9ffd8b3d6464649081b61 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 2 Dec 2020 12:45:29 +0100 Subject: [PATCH 03/20] working yay --- e2e/_suites/fleet/features/fleet_mode_agent.feature | 2 +- e2e/_suites/fleet/fleet.go | 4 +--- e2e/go.mod | 2 ++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index 53c332e22f..341a41b707 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -30,7 +30,7 @@ Examples: Scenario Outline: Upgrading the installed agent Given a "" agent "stale" is deployed to Fleet with "" installer When agent is upgraded to version "latest" - Then agent is in version "stack" + Then agent is in version "latest" And the agent is listed in Fleet as "online" Examples: | os | installer | diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index dabb9ba07f..1d39bfb708 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -187,8 +187,6 @@ func (fts *FleetTestSuite) agentInVersion(version string) error { version = agentStaleVersion case "latest": version = agentVersion - case "stack": - version = stackVersion } agentID, err := getAgentID(fts.Hostname) @@ -210,7 +208,7 @@ func (fts *FleetTestSuite) agentInVersion(version string) error { jsonResponse, err := gabs.ParseJSON([]byte(body)) retrievedVersion := jsonResponse.Path("item.local_metadata.elastic.agent.version").Data().(string) - if isSnapshot := jsonResponse.Path("item.local_metadata.elastic.agent.version").Data().(string); isSnapshot == "true" { + if isSnapshot := jsonResponse.Path("item.local_metadata.elastic.agent.snapshot").Data().(bool); isSnapshot { retrievedVersion += "-SNAPSHOT" } diff --git a/e2e/go.mod b/e2e/go.mod index f83c3aa7be..417b5458c1 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -6,9 +6,11 @@ require ( github.com/Jeffail/gabs/v2 v2.5.1 github.com/cenkalti/backoff/v4 v4.0.2 github.com/cucumber/godog v0.10.0 + github.com/cucumber/messages-go/v10 v10.0.3 github.com/elastic/e2e-testing/cli v0.0.0-20200717181709-15d2db53ded7 github.com/elastic/go-elasticsearch/v8 v8.0.0-20190731061900-ea052088db25 github.com/google/uuid v1.1.1 + github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.4.2 ) From 56595fda3d9943e6ae4a4fe3a702ab6c64cb26d1 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 2 Dec 2020 12:46:49 +0100 Subject: [PATCH 04/20] jenkins var --- .ci/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 5f5d64857e..9f8cf7bfc7 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -34,6 +34,7 @@ pipeline { string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel(s) where errors will be posted. For multiple channels, use a comma-separated list of channels') string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz') string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '8.0.0-SNAPSHOT', description: 'SemVer version of the stand-alone elastic-agent to be used for Fleet tests. You can use here the tag of your PR to test your changes') + string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.10.0', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.') booleanParam(name: "ELASTIC_AGENT_USE_CI_SNAPSHOTS", defaultValue: false, description: "If it's needed to use the binary snapshots produced by Beats CI instead of the official releases") choice(name: 'LOG_LEVEL', choices: ['DEBUG', 'INFO'], description: 'Log level to be used') choice(name: 'TIMEOUT_FACTOR', choices: ['3', '5', '7', '11'], description: 'Max number of minutes for timeout backoff strategies') From 08bbe75c6d7445ba2b186e40080497c7e6bd2f2d Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 2 Dec 2020 12:48:05 +0100 Subject: [PATCH 05/20] revert json logs --- e2e/_suites/fleet/services.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/_suites/fleet/services.go b/e2e/_suites/fleet/services.go index ed97e417f5..d14551b653 100644 --- a/e2e/_suites/fleet/services.go +++ b/e2e/_suites/fleet/services.go @@ -498,8 +498,8 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) { image: image, InstallFn: installFn, installerType: "tar", - logFile: "elastic-agent-json.log", - logsDir: "/opt/Elastic/Agent/data/elastic-agent-%s/logs/", + logFile: "elastic-agent.log", + logsDir: "/opt/Elastic/Agent/data/elastic-agent-%s/", name: tarFile, path: binaryPath, PostInstallFn: postInstallFn, From 7695b235fabb0f0a62ab584ab42efa68dbc5049d Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 2 Dec 2020 12:49:09 +0100 Subject: [PATCH 06/20] uninstall revert --- e2e/_suites/fleet/services.go | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e/_suites/fleet/services.go b/e2e/_suites/fleet/services.go index d14551b653..e6a1588f26 100644 --- a/e2e/_suites/fleet/services.go +++ b/e2e/_suites/fleet/services.go @@ -479,7 +479,6 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) { return nil } unInstallFn := func() error { - return nil args := []string{"-f"} return runElasticAgentCommand(profile, image, service, ElasticAgentProcessName, "uninstall", args) From a82cec151c524b10f841d0842a0cb99e2443edf7 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 2 Dec 2020 13:32:23 +0100 Subject: [PATCH 07/20] looogs --- e2e/_suites/fleet/services.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/e2e/_suites/fleet/services.go b/e2e/_suites/fleet/services.go index e6a1588f26..905e4accb2 100644 --- a/e2e/_suites/fleet/services.go +++ b/e2e/_suites/fleet/services.go @@ -111,8 +111,11 @@ func (i *ElasticAgentInstaller) getElasticAgentLogs(hostname string) error { } logFile := i.logsDir + i.logFile + if strings.Contains(logFile, "%s") { + logFile = fmt.Sprintf(logFile, hash) + } cmd := []string{ - "cat", fmt.Sprintf(logFile, hash), + "cat", logFile, } err = execCommandInService(i.profile, i.image, i.service, cmd, false) @@ -498,7 +501,7 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) { InstallFn: installFn, installerType: "tar", logFile: "elastic-agent.log", - logsDir: "/opt/Elastic/Agent/data/elastic-agent-%s/", + logsDir: "/opt/Elastic/Agent/", name: tarFile, path: binaryPath, PostInstallFn: postInstallFn, From 0a3ade84e9ac7aaa59e486ee6fd2d1b555610a49 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 2 Dec 2020 14:16:19 +0100 Subject: [PATCH 08/20] added wait --- e2e/_suites/fleet/features/fleet_mode_agent.feature | 1 + e2e/_suites/fleet/fleet.go | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index 341a41b707..e42c41de86 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -30,6 +30,7 @@ Examples: Scenario Outline: Upgrading the installed agent Given a "" agent "stale" is deployed to Fleet with "" installer When agent is upgraded to version "latest" + Then wait for "2m" Then agent is in version "latest" And the agent is listed in Fleet as "online" Examples: diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index 1d39bfb708..d9ac1e2c8d 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -131,6 +131,7 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.Suite) { s.Step(`^an attempt to enroll a new agent fails$`, fts.anAttemptToEnrollANewAgentFails) s.Step(`^the "([^"]*)" process is "([^"]*)" on the host$`, fts.processStateChangedOnTheHost) s.Step(`^the file system Agent folder is empty$`, fts.theFileSystemAgentFolderIsEmpty) + s.Step(`^wait for "([^"]*)"$`, fts.waitForTime) // endpoint steps s.Step(`^the "([^"]*)" integration is "([^"]*)" in the policy$`, fts.theIntegrationIsOperatedInThePolicy) @@ -168,6 +169,16 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver return fts.anAgentIsDeployedToFleetWithInstaller(image, installerType) } +func (fts *FleetTestSuite) waitForTime(durationString string) error { + duration, err := time.ParseDuration(durationString) + if err != nil { + return err + } + + <-time.After(duration) + return nil +} + func (fts *FleetTestSuite) anAgentIsUpgraded(desiredVersion string) error { switch desiredVersion { case "stale": @@ -213,7 +224,7 @@ func (fts *FleetTestSuite) agentInVersion(version string) error { } if retrievedVersion != version { - return fmt.Errorf("version mismatch required '%s' desired '%s'", version, retrievedVersion) + return fmt.Errorf("version mismatch required '%s' retrieved '%s'", version, retrievedVersion) } return nil From ce0c704717a8b5188eafc4e7b8c1986427a3c082 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 2 Dec 2020 14:30:30 +0100 Subject: [PATCH 09/20] then and --- e2e/_suites/fleet/features/fleet_mode_agent.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index e42c41de86..35947db01b 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -31,7 +31,7 @@ Scenario Outline: Upgrading the installed agent Given a "" agent "stale" is deployed to Fleet with "" installer When agent is upgraded to version "latest" Then wait for "2m" - Then agent is in version "latest" + And agent is in version "latest" And the agent is listed in Fleet as "online" Examples: | os | installer | From df9e936a6b7f143a53ca3f29bedec418e3789c1f Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 2 Dec 2020 15:53:44 +0100 Subject: [PATCH 10/20] no online check --- e2e/_suites/fleet/features/fleet_mode_agent.feature | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index 35947db01b..cd608dc2cf 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -32,7 +32,6 @@ Scenario Outline: Upgrading the installed agent When agent is upgraded to version "latest" Then wait for "2m" And agent is in version "latest" - And the agent is listed in Fleet as "online" Examples: | os | installer | | debian | tar | From 9d38c55c94d6b963c771918084399b49aad69e87 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 2 Dec 2020 18:08:37 +0100 Subject: [PATCH 11/20] updated --- e2e/_suites/fleet/features/fleet_mode_agent.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index 0059ee65eb..0a6c6322af 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -46,8 +46,8 @@ Scenario Outline: Upgrading the installed agent Then wait for "2m" And agent is in version "latest" Examples: -| os | installer | -| debian | tar | +| os | +| debian | @restart-agent Scenario Outline: Restarting the installed agent From 6002c19abf975792af95fe0e1dbefb116a068b79 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 3 Dec 2020 07:58:02 +0100 Subject: [PATCH 12/20] rephrased --- e2e/_suites/fleet/features/fleet_mode_agent.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index 0a6c6322af..cdaac51ed2 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -43,8 +43,8 @@ Examples: Scenario Outline: Upgrading the installed agent Given a "" agent "stale" is deployed to Fleet with "" installer When agent is upgraded to version "latest" - Then wait for "2m" - And agent is in version "latest" + And wait for "2m" + Then agent is in version "latest" Examples: | os | | debian | From c5b4dd6fdd2d636c8dc20829d9fdfe53fa0b7da5 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 3 Dec 2020 08:30:00 +0100 Subject: [PATCH 13/20] tar installer --- e2e/_suites/fleet/features/fleet_mode_agent.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index 8099fa6914..1ffa793225 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -42,7 +42,7 @@ Examples: @upgrade-agent Scenario Outline: Upgrading the installed agent - Given a "" agent "stale" is deployed to Fleet with "" installer + Given a "" agent "stale" is deployed to Fleet with "tar" installer When agent is upgraded to version "latest" And wait for "2m" Then agent is in version "latest" From 539c88f4e4895de4c312b78784a96d470182a1cc Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 3 Dec 2020 10:40:50 +0100 Subject: [PATCH 14/20] install certs --- .../fleet/features/fleet_mode_agent.feature | 1 + e2e/_suites/fleet/fleet.go | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index 1ffa793225..6af9db2c75 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -43,6 +43,7 @@ Examples: @upgrade-agent Scenario Outline: Upgrading the installed agent Given a "" agent "stale" is deployed to Fleet with "tar" installer + And certs for "" are installed When agent is upgraded to version "latest" And wait for "2m" Then agent is in version "latest" diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index d9ac1e2c8d..ae8a518212 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -132,6 +132,7 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.Suite) { s.Step(`^the "([^"]*)" process is "([^"]*)" on the host$`, fts.processStateChangedOnTheHost) s.Step(`^the file system Agent folder is empty$`, fts.theFileSystemAgentFolderIsEmpty) s.Step(`^wait for "([^"]*)"$`, fts.waitForTime) + s.Step(`^certs for "([^"]*)" are installed$`, fts.installCerts) // endpoint steps s.Step(`^the "([^"]*)" integration is "([^"]*)" in the policy$`, fts.theIntegrationIsOperatedInThePolicy) @@ -169,6 +170,37 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver return fts.anAgentIsDeployedToFleetWithInstaller(image, installerType) } +func (fts *FleetTestSuite) installCerts(targetOS string) error { + service := targetOS + "-systemd" + image := targetOS + "-systemd" + if targetOS == "debian" { + if err := execCommandInService(FleetProfileName, image, service, []string{"apt-get", "update"}, false); err != nil { + return err + } + if err := execCommandInService(FleetProfileName, image, service, []string{"apt", "install", "ca-certificates", "-y"}, false); err != nil { + return err + } + if err := execCommandInService(FleetProfileName, image, service, []string{"update-ca-certificates"}, false); err != nil { + return err + } + } + if targetOS == "centos" { + if err := execCommandInService(FleetProfileName, image, service, []string{"yum", "check-update"}, false); err != nil { + return err + } + if err := execCommandInService(FleetProfileName, image, service, []string{"yum", "install", "ca-certificates", "-y"}, false); err != nil { + return err + } + if err := execCommandInService(FleetProfileName, image, service, []string{"update-ca-trust", "force-enable"}, false); err != nil { + return err + } + if err := execCommandInService(FleetProfileName, image, service, []string{"update-ca-trust", "extract"}, false); err != nil { + return err + } + } + return nil +} + func (fts *FleetTestSuite) waitForTime(durationString string) error { duration, err := time.ParseDuration(durationString) if err != nil { From 9c1749e56550b78a8b55d9bf273892f7493b910b Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 3 Dec 2020 14:29:44 +0100 Subject: [PATCH 15/20] be declarative --- e2e/_suites/fleet/features/fleet_mode_agent.feature | 4 ++-- e2e/_suites/fleet/fleet.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index 6af9db2c75..5c384ec4c7 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -45,8 +45,8 @@ Scenario Outline: Upgrading the installed agent Given a "" agent "stale" is deployed to Fleet with "tar" installer And certs for "" are installed When agent is upgraded to version "latest" - And wait for "2m" - Then agent is in version "latest" + Then process waits for "2m" + And agent is in version "latest" Examples: | os | | debian | diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index ae8a518212..d532ba1272 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -131,7 +131,7 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.Suite) { s.Step(`^an attempt to enroll a new agent fails$`, fts.anAttemptToEnrollANewAgentFails) s.Step(`^the "([^"]*)" process is "([^"]*)" on the host$`, fts.processStateChangedOnTheHost) s.Step(`^the file system Agent folder is empty$`, fts.theFileSystemAgentFolderIsEmpty) - s.Step(`^wait for "([^"]*)"$`, fts.waitForTime) + s.Step(`^process waits for "([^"]*)"$`, fts.waitForTime) s.Step(`^certs for "([^"]*)" are installed$`, fts.installCerts) // endpoint steps From 9de13af607d291289b171fedbd9252b4831ebf95 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Fri, 4 Dec 2020 09:05:40 +0100 Subject: [PATCH 16/20] conflicts --- e2e/_suites/fleet/ingest-manager_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/e2e/_suites/fleet/ingest-manager_test.go b/e2e/_suites/fleet/ingest-manager_test.go index 948286cf9e..511e9b4f53 100644 --- a/e2e/_suites/fleet/ingest-manager_test.go +++ b/e2e/_suites/fleet/ingest-manager_test.go @@ -79,14 +79,11 @@ func init() { timeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", timeoutFactor) agentVersion = shell.GetEnv("ELASTIC_AGENT_VERSION", agentVersionBase) -<<<<<<< HEAD agentStaleVersion = shell.GetEnv("ELASTIC_AGENT_STALE_VERSION", agentStaleVersion) -======= // check if version is an alias agentVersion = e2e.GetElasticArtifactVersion(agentVersion) ->>>>>>> f98f8022fb70462227a4d5601d710f74b4042291 stackVersion = shell.GetEnv("STACK_VERSION", stackVersion) } From 07a221930fb58504458643a7c3a078cd80b40f30 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 8 Dec 2020 11:00:10 +0100 Subject: [PATCH 17/20] make it nice --- .../fleet/features/fleet_mode_agent.feature | 7 +- e2e/_suites/fleet/fleet.go | 83 ++++++++----------- e2e/_suites/fleet/services.go | 44 ++++++++++ 3 files changed, 81 insertions(+), 53 deletions(-) diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index 5c384ec4c7..e45540473c 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -42,11 +42,10 @@ Examples: @upgrade-agent Scenario Outline: Upgrading the installed agent - Given a "" agent "stale" is deployed to Fleet with "tar" installer - And certs for "" are installed + Given certs for "" are installed + And a "" agent "stale" is deployed to Fleet with "tar" installer When agent is upgraded to version "latest" - Then process waits for "2m" - And agent is in version "latest" + Then agent is in version "latest" Examples: | os | | debian | diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index d532ba1272..067a41b98c 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -171,34 +171,12 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver } func (fts *FleetTestSuite) installCerts(targetOS string) error { - service := targetOS + "-systemd" - image := targetOS + "-systemd" - if targetOS == "debian" { - if err := execCommandInService(FleetProfileName, image, service, []string{"apt-get", "update"}, false); err != nil { - return err - } - if err := execCommandInService(FleetProfileName, image, service, []string{"apt", "install", "ca-certificates", "-y"}, false); err != nil { - return err - } - if err := execCommandInService(FleetProfileName, image, service, []string{"update-ca-certificates"}, false); err != nil { - return err - } - } - if targetOS == "centos" { - if err := execCommandInService(FleetProfileName, image, service, []string{"yum", "check-update"}, false); err != nil { - return err - } - if err := execCommandInService(FleetProfileName, image, service, []string{"yum", "install", "ca-certificates", "-y"}, false); err != nil { - return err - } - if err := execCommandInService(FleetProfileName, image, service, []string{"update-ca-trust", "force-enable"}, false); err != nil { - return err - } - if err := execCommandInService(FleetProfileName, image, service, []string{"update-ca-trust", "extract"}, false); err != nil { - return err - } + installer := fts.getInstaller() + if installer.InstallCertsFn == nil { + return nil } - return nil + + return installer.InstallCertsFn() } func (fts *FleetTestSuite) waitForTime(durationString string) error { @@ -232,34 +210,41 @@ func (fts *FleetTestSuite) agentInVersion(version string) error { version = agentVersion } - agentID, err := getAgentID(fts.Hostname) - if err != nil { - return err - } + agentInVersionFn := func() error { + agentID, err := getAgentID(fts.Hostname) + if err != nil { + return err + } - r := createDefaultHTTPRequest(fleetAgentsURL + "/" + agentID) - body, err := curl.Get(r) - if err != nil { - log.WithFields(log.Fields{ - "body": body, - "error": err, - "url": r.GetURL(), - }).Error("Could not get agent in Fleet") - return err - } + r := createDefaultHTTPRequest(fleetAgentsURL + "/" + agentID) + body, err := curl.Get(r) + if err != nil { + log.WithFields(log.Fields{ + "body": body, + "error": err, + "url": r.GetURL(), + }).Error("Could not get agent in Fleet") + return err + } - jsonResponse, err := gabs.ParseJSON([]byte(body)) + jsonResponse, err := gabs.ParseJSON([]byte(body)) - retrievedVersion := jsonResponse.Path("item.local_metadata.elastic.agent.version").Data().(string) - if isSnapshot := jsonResponse.Path("item.local_metadata.elastic.agent.snapshot").Data().(bool); isSnapshot { - retrievedVersion += "-SNAPSHOT" - } + retrievedVersion := jsonResponse.Path("item.local_metadata.elastic.agent.version").Data().(string) + if isSnapshot := jsonResponse.Path("item.local_metadata.elastic.agent.snapshot").Data().(bool); isSnapshot { + retrievedVersion += "-SNAPSHOT" + } + + if retrievedVersion != version { + return fmt.Errorf("version mismatch required '%s' retrieved '%s'", version, retrievedVersion) + } - if retrievedVersion != version { - return fmt.Errorf("version mismatch required '%s' retrieved '%s'", version, retrievedVersion) + return nil } - return nil + maxTimeout := time.Duration(timeoutFactor) * time.Minute * 2 + exp := e2e.GetExponentialBackOff(maxTimeout) + + return backoff.Retry(agentInVersionFn, exp) } // supported installers: tar, systemd diff --git a/e2e/_suites/fleet/services.go b/e2e/_suites/fleet/services.go index 4cdc709d94..60852f6ade 100644 --- a/e2e/_suites/fleet/services.go +++ b/e2e/_suites/fleet/services.go @@ -32,6 +32,7 @@ type ElasticAgentInstaller struct { image string // docker image installerType string InstallFn func(containerName string, token string) error + InstallCertsFn func() error logFile string // the name of the log file logsDir string // location of the logs name string // the name for the binary @@ -322,6 +323,22 @@ func newCentosInstaller(image string, tag string) (ElasticAgentInstaller, error) log.Trace("No uninstall commands for Centos + systemd") return nil } + installCertsFn := func() error { + if err := execCommandInService(profile, image, service, []string{"yum", "check-update"}, false); err != nil { + return err + } + if err := execCommandInService(profile, image, service, []string{"yum", "install", "ca-certificates", "-y"}, false); err != nil { + return err + } + if err := execCommandInService(profile, image, service, []string{"update-ca-trust", "force-enable"}, false); err != nil { + return err + } + if err := execCommandInService(profile, image, service, []string{"update-ca-trust", "extract"}, false); err != nil { + return err + } + + return nil + } binDir := "/var/lib/elastic-agent/data/elastic-agent-%s/" @@ -337,6 +354,7 @@ func newCentosInstaller(image string, tag string) (ElasticAgentInstaller, error) homeDir: "/etc/elastic-agent/", image: image, InstallFn: installFn, + InstallCertsFn: installCertsFn, installerType: "rpm", logFile: "elastic-agent-json.log", logsDir: binDir + "logs/", @@ -403,6 +421,18 @@ func newDebianInstaller(image string, tag string) (ElasticAgentInstaller, error) log.Trace("No uninstall commands for Debian + systemd") return nil } + installCertsFn := func() error { + if err := execCommandInService(profile, image, service, []string{"apt-get", "update"}, false); err != nil { + return err + } + if err := execCommandInService(profile, image, service, []string{"apt", "install", "ca-certificates", "-y"}, false); err != nil { + return err + } + if err := execCommandInService(profile, image, service, []string{"update-ca-certificates"}, false); err != nil { + return err + } + return nil + } binDir := "/var/lib/elastic-agent/data/elastic-agent-%s/" @@ -418,6 +448,7 @@ func newDebianInstaller(image string, tag string) (ElasticAgentInstaller, error) homeDir: "/etc/elastic-agent/", image: image, InstallFn: installFn, + InstallCertsFn: installCertsFn, installerType: "deb", logFile: "elastic-agent-json.log", logsDir: binDir + "logs/", @@ -493,6 +524,18 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) { return runElasticAgentCommand(profile, image, service, ElasticAgentProcessName, "uninstall", args) } + installCertsFn := func() error { + if err := execCommandInService(profile, image, service, []string{"apt-get", "update"}, false); err != nil { + return err + } + if err := execCommandInService(profile, image, service, []string{"apt", "install", "ca-certificates", "-y"}, false); err != nil { + return err + } + if err := execCommandInService(profile, image, service, []string{"update-ca-certificates"}, false); err != nil { + return err + } + return nil + } return ElasticAgentInstaller{ artifactArch: arch, @@ -506,6 +549,7 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) { homeDir: homeDir, image: image, InstallFn: installFn, + InstallCertsFn: installCertsFn, installerType: "tar", logFile: "elastic-agent.log", logsDir: "/opt/Elastic/Agent/", From 1df2679c21c2511ecdf0deeaf8071e6c517afa1a Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 8 Dec 2020 11:38:55 +0100 Subject: [PATCH 18/20] reverse order --- e2e/_suites/fleet/features/fleet_mode_agent.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index e45540473c..70e525a4ed 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -42,8 +42,8 @@ Examples: @upgrade-agent Scenario Outline: Upgrading the installed agent - Given certs for "" are installed - And a "" agent "stale" is deployed to Fleet with "tar" installer + Given a "" agent "stale" is deployed to Fleet with "tar" installer + And certs for "" are installed When agent is upgraded to version "latest" Then agent is in version "latest" Examples: From 44b1e62ff1f895d670705e2046f98c15592e8c8e Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 8 Dec 2020 11:39:38 +0100 Subject: [PATCH 19/20] reverse order --- e2e/_suites/fleet/fleet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index 067a41b98c..0efead32f1 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -173,7 +173,7 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver func (fts *FleetTestSuite) installCerts(targetOS string) error { installer := fts.getInstaller() if installer.InstallCertsFn == nil { - return nil + return errors.New("no installer found") } return installer.InstallCertsFn() From 6c69884151cfbecf29cdb7edc4d97dc75379b930 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 8 Dec 2020 12:21:12 +0100 Subject: [PATCH 20/20] removed wait --- e2e/_suites/fleet/fleet.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index 0efead32f1..8b6491fe8f 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -131,7 +131,6 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.Suite) { s.Step(`^an attempt to enroll a new agent fails$`, fts.anAttemptToEnrollANewAgentFails) s.Step(`^the "([^"]*)" process is "([^"]*)" on the host$`, fts.processStateChangedOnTheHost) s.Step(`^the file system Agent folder is empty$`, fts.theFileSystemAgentFolderIsEmpty) - s.Step(`^process waits for "([^"]*)"$`, fts.waitForTime) s.Step(`^certs for "([^"]*)" are installed$`, fts.installCerts) // endpoint steps @@ -179,16 +178,6 @@ func (fts *FleetTestSuite) installCerts(targetOS string) error { return installer.InstallCertsFn() } -func (fts *FleetTestSuite) waitForTime(durationString string) error { - duration, err := time.ParseDuration(durationString) - if err != nil { - return err - } - - <-time.After(duration) - return nil -} - func (fts *FleetTestSuite) anAgentIsUpgraded(desiredVersion string) error { switch desiredVersion { case "stale":