Skip to content
This repository has been archived by the owner on Sep 17, 2024. It is now read-only.

chore: refactor Fleet upgrade tests #671

Merged
merged 12 commits into from
Feb 4, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .ci/Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ pipeline {
string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel(s) where errors will be posted. For multiple channels, use a comma-separated list of channels')
string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz')
string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '8.0.0-SNAPSHOT', description: 'SemVer version of the stand-alone elastic-agent to be used for Fleet tests. You can use here the tag of your PR to test your changes')
string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.10.1', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.')
string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.10.2', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.')
booleanParam(name: "BEATS_USE_CI_SNAPSHOTS", defaultValue: false, description: "If it's needed to use the binary snapshots produced by Beats CI instead of the official releases")
choice(name: 'LOG_LEVEL', choices: ['DEBUG', 'INFO'], description: 'Log level to be used')
choice(name: 'TIMEOUT_FACTOR', choices: ['5', '3', '7', '11'], description: 'Max number of minutes for timeout backoff strategies')
Expand Down
22 changes: 22 additions & 0 deletions e2e/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -91,3 +91,25 @@ sync-integrations:
unit-test:
gotestsum --format testname -- -count=1 -timeout=$(TEST_TIMEOUT) ./...
cd _suites && gotestsum --format testname -- -count=1 -timeout=$(TEST_TIMEOUT) ./...

## Test examples

.PHONY: fleet-fleet
fleet-fleet:
SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test

.PHONY: fleet-fleet-ci-snapshots
fleet-fleet-ci-snapshots:
SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test

.PHONY: fleet-fleet-pr-ci-snapshots
fleet-fleet-pr-ci-snapshots:
SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true ELASTIC_AGENT_VERSION=pr-14954 $(MAKE) functional-test

.PHONY: fleet-nightly
fleet-nightly:
SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test

.PHONY: fleet-nightly-ci-snapshots
fleet-nightly-ci-snapshots:
SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test
2 changes: 1 addition & 1 deletion e2e/_suites/fleet/features/fleet_mode_agent.feature
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ Examples:
| debian |

# @upgrade-agent
@skip
@nightly
Scenario Outline: Upgrading the installed <os> agent
Given a "<os>" agent "stale" is deployed to Fleet with "tar" installer
And certs for "<os>" are installed
Expand Down
24 changes: 16 additions & 8 deletions e2e/_suites/fleet/fleet.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ type FleetTestSuite struct {
CurrentToken string // current enrollment token
CurrentTokenID string // current enrollment tokenID
Hostname string // the hostname of the container
Version string // current elastic-agent version
// integrations
Integration IntegrationPackage // the installed integration
PolicyUpdatedAt string // the moment the policy was updated
Expand Down Expand Up @@ -107,6 +108,8 @@ func (fts *FleetTestSuite) afterScenario() {
func (fts *FleetTestSuite) beforeScenario() {
fts.Cleanup = false

fts.Version = agentVersion

// create policy with system monitoring enabled
defaultPolicy, err := getAgentDefaultPolicy()
if err != nil {
Expand Down Expand Up @@ -148,8 +151,8 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) {
}

func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, version, installerType string) error {
agentVersionBackup := agentVersion
defer func() { agentVersion = agentVersionBackup }()
agentVersionBackup := fts.Version
defer func() { fts.Version = agentVersionBackup }()

switch version {
case "stale":
Expand All @@ -160,13 +163,12 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver
version = agentStaleVersion
}

agentVersion = version
fts.Version = version

// prepare installer for stale version
if agentVersion != agentVersionBackup {
i := GetElasticAgentInstaller(image, installerType)
installerType = fmt.Sprintf("%s-%s", installerType, version)
fts.Installers[fmt.Sprintf("%s-%s", image, installerType)] = i
if fts.Version != agentVersionBackup {
i := GetElasticAgentInstaller(image, installerType, fts.Version, true)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i like that

fts.Installers[fmt.Sprintf("%s-%s-%s", image, installerType, version)] = i
}

return fts.anAgentIsDeployedToFleetWithInstaller(image, installerType)
Expand All @@ -175,6 +177,12 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver
func (fts *FleetTestSuite) installCerts(targetOS string) error {
installer := fts.getInstaller()
if installer.InstallCertsFn == nil {
log.WithFields(log.Fields{
"installer": installer,
"version": fts.Version,
"agentVersion": agentVersion,
"agentStaleVersion": agentStaleVersion,
}).Error("No installer found")
return errors.New("no installer found")
}

Expand Down Expand Up @@ -291,7 +299,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(image string, i
}

func (fts *FleetTestSuite) getInstaller() ElasticAgentInstaller {
return fts.Installers[fts.Image+"-"+fts.InstallerType]
return fts.Installers[fts.Image+"-"+fts.InstallerType+"-"+fts.Version]
}

func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state string) error {
Expand Down
15 changes: 10 additions & 5 deletions e2e/_suites/fleet/ingest-manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ var agentVersion = agentVersionBase

// agentStaleVersion is the version of the agent to use as a base during upgrade
// It can be overriden by ELASTIC_AGENT_STALE_VERSION env var. Using latest GA as a default.
var agentStaleVersion = "7.10.0"
var agentStaleVersion = "7.10.2"
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pairing version with Jenkinsfile


// stackVersion is the version of the stack to use
// It can be overriden by STACK_VERSION env var
Expand Down Expand Up @@ -83,6 +83,11 @@ func setUpSuite() {
agentVersion = shell.GetEnv("ELASTIC_AGENT_VERSION", agentVersionBase)
agentStaleVersion = shell.GetEnv("ELASTIC_AGENT_STALE_VERSION", agentStaleVersion)

useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS")
if useCISnapshots && !strings.HasSuffix(agentStaleVersion, "-SNAPSHOT") {
agentStaleVersion += "-SNAPSHOT"
}

// check if version is an alias
agentVersion = e2e.GetElasticArtifactVersion(agentVersion)

Expand All @@ -91,10 +96,10 @@ func setUpSuite() {
imts = IngestManagerTestSuite{
Fleet: &FleetTestSuite{
Installers: map[string]ElasticAgentInstaller{
"centos-systemd": GetElasticAgentInstaller("centos", "systemd"),
"centos-tar": GetElasticAgentInstaller("centos", "tar"),
"debian-systemd": GetElasticAgentInstaller("debian", "systemd"),
"debian-tar": GetElasticAgentInstaller("debian", "tar"),
"centos-systemd-" + agentVersion: GetElasticAgentInstaller("centos", "systemd", agentVersion, false),
"centos-tar-" + agentVersion: GetElasticAgentInstaller("centos", "tar", agentVersion, false),
"debian-systemd-" + agentVersion: GetElasticAgentInstaller("debian", "systemd", agentVersion, false),
"debian-tar-" + agentVersion: GetElasticAgentInstaller("debian", "tar", agentVersion, false),
},
},
StandAlone: &StandAloneTestSuite{},
Expand Down
40 changes: 28 additions & 12 deletions e2e/_suites/fleet/installers.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ type TARPackage struct {
arch string
artifact string
OS string
stale bool
version string
}

Expand Down Expand Up @@ -214,25 +215,40 @@ func (i *TARPackage) Preinstall() error {
return err
}

version := checkElasticAgentVersion(i.version)
version := i.version
if !i.stale {
version = checkElasticAgentVersion(i.version)
}

// simplify layout
cmds := []string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, version, i.OS, i.arch), "/elastic-agent"}
err = execCommandInService(i.profile, i.image, i.service, cmds, false)
if err != nil {
log.WithFields(log.Fields{
"command": cmds,
"error": err,
"image": i.image,
"service": i.service,
}).Error("Could not extract agent package in the box")

return err
cmds := [][]string{
[]string{"rm", "-fr", "/elastic-agent"},
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When running all Fleet tests in my local machine, I saw multiple errors when moving the agent, because mv will refuse to rename a directory to another directory if the target directory contains files.

It does not happen on CI but in my local machine happens after multiple runs. I'm adding it defensively, as it's harmless.

[]string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, version, i.OS, i.arch), "/elastic-agent"},
}
for _, cmd := range cmds {
err = execCommandInService(i.profile, i.image, i.service, cmd, false)
if err != nil {
log.WithFields(log.Fields{
"command": cmd,
"error": err,
"image": i.image,
"service": i.service,
"version": version,
}).Error("Could not extract agent package in the box")

return err
}
}

return nil
}

// Stale sets the stale state
func (i *TARPackage) Stale(stale bool) *TARPackage {
i.stale = stale
return i
}

// Uninstall uninstalls a TAR package
func (i *TARPackage) Uninstall() error {
args := []string{"-f"}
Expand Down
50 changes: 29 additions & 21 deletions e2e/_suites/fleet/services.go
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ func runElasticAgentCommand(profile string, image string, service string, proces
// be defined by that value
// Else, if the environment variable BEATS_USE_CI_SNAPSHOTS is set, then the artifact
// to be downloaded will be defined by the latest snapshot produced by the Beats CI.
func downloadAgentBinary(artifact string, version string, OS string, arch string, extension string) (string, string, error) {
func downloadAgentBinary(artifact string, version string, OS string, arch string, extension string, stale bool) (string, string, error) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As an improvement in a follow-up PR, I'm interested in creating a BinaryRequest struct to hold the references needed to download the binary, which would simplify the signature of the method.

fileName := fmt.Sprintf("%s-%s-%s.%s", artifact, version, arch, extension)

handleDownload := func(URL string, fileName string) (string, string, error) {
Expand Down Expand Up @@ -196,7 +196,7 @@ func downloadAgentBinary(artifact string, version string, OS string, arch string
if useCISnapshots {
log.Debug("Using CI snapshots for the Elastic Agent")

bucketFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension)
bucketFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, stale)

maxTimeout := time.Duration(timeoutFactor) * time.Minute

Expand All @@ -208,7 +208,12 @@ func downloadAgentBinary(artifact string, version string, OS string, arch string
return handleDownload(downloadURL, bucketFileName)
}

downloadURL, err = e2e.GetElasticArtifactURL(artifact, checkElasticAgentVersion(version), OS, arch, extension)
downloadVersion := version
if !stale {
downloadVersion = checkElasticAgentVersion(version)
}

downloadURL, err = e2e.GetElasticArtifactURL(artifact, downloadVersion, OS, arch, extension)
if err != nil {
return "", "", err
}
Expand All @@ -217,7 +222,7 @@ func downloadAgentBinary(artifact string, version string, OS string, arch string
}

// GetElasticAgentInstaller returns an installer from a docker image
func GetElasticAgentInstaller(image string, installerType string) ElasticAgentInstaller {
func GetElasticAgentInstaller(image string, installerType string, version string, stale bool) ElasticAgentInstaller {
log.WithFields(log.Fields{
"image": image,
"installer": installerType,
Expand All @@ -226,13 +231,13 @@ func GetElasticAgentInstaller(image string, installerType string) ElasticAgentIn
var installer ElasticAgentInstaller
var err error
if "centos" == image && "tar" == installerType {
installer, err = newTarInstaller("centos", "latest")
installer, err = newTarInstaller("centos", "latest", version, stale)
} else if "centos" == image && "systemd" == installerType {
installer, err = newCentosInstaller("centos", "latest")
installer, err = newCentosInstaller("centos", "latest", version, stale)
} else if "debian" == image && "tar" == installerType {
installer, err = newTarInstaller("debian", "stretch")
installer, err = newTarInstaller("debian", "stretch", version, stale)
} else if "debian" == image && "systemd" == installerType {
installer, err = newDebianInstaller("debian", "stretch")
installer, err = newDebianInstaller("debian", "stretch", version, stale)
} else {
log.WithFields(log.Fields{
"image": image,
Expand All @@ -252,7 +257,7 @@ func GetElasticAgentInstaller(image string, installerType string) ElasticAgentIn
}

// getGCPBucketCoordinates it calculates the bucket path in GCP
func getGCPBucketCoordinates(fileName string, artifact string, version string, OS string, arch string, extension string) (string, string, string, string) {
func getGCPBucketCoordinates(fileName string, artifact string, version string, OS string, arch string, extension string, stale bool) (string, string, string, string) {
if extension == "tar.gz" {
fileName = fmt.Sprintf("%s-%s-%s-%s.%s", artifact, version, OS, arch, extension)
}
Expand Down Expand Up @@ -286,27 +291,31 @@ func getGCPBucketCoordinates(fileName string, artifact string, version string, O
object = fmt.Sprintf("%s/%s", artifact, newFileName)
}

if stale {
prefix = fmt.Sprintf("snapshots/%s", artifact)
object = newFileName
}

return newFileName, bucket, prefix, object
}

func isSystemdBased(image string) bool {
return strings.HasSuffix(image, "-systemd")
}

// newCentosInstaller returns an instance of the Centos installer
func newCentosInstaller(image string, tag string) (ElasticAgentInstaller, error) {
// newCentosInstaller returns an instance of the Centos installer for a specific version
func newCentosInstaller(image string, tag string, version string, stale bool) (ElasticAgentInstaller, error) {
image = image + "-systemd" // we want to consume systemd boxes
service := image
profile := FleetProfileName

// extract the agent in the box, as it's mounted as a volume
artifact := "elastic-agent"
version := agentVersion
os := "linux"
arch := "x86_64"
extension := "rpm"

binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension)
binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension, stale)
if err != nil {
log.WithFields(log.Fields{
"artifact": artifact,
Expand Down Expand Up @@ -358,20 +367,19 @@ func newCentosInstaller(image string, tag string) (ElasticAgentInstaller, error)
}, nil
}

// newDebianInstaller returns an instance of the Debian installer
func newDebianInstaller(image string, tag string) (ElasticAgentInstaller, error) {
// newDebianInstaller returns an instance of the Debian installer for a specific version
func newDebianInstaller(image string, tag string, version string, stale bool) (ElasticAgentInstaller, error) {
image = image + "-systemd" // we want to consume systemd boxes
service := image
profile := FleetProfileName

// extract the agent in the box, as it's mounted as a volume
artifact := "elastic-agent"
version := agentVersion
os := "linux"
arch := "amd64"
extension := "deb"

binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension)
binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension, stale)
if err != nil {
log.WithFields(log.Fields{
"artifact": artifact,
Expand Down Expand Up @@ -423,20 +431,19 @@ func newDebianInstaller(image string, tag string) (ElasticAgentInstaller, error)
}, nil
}

// newTarInstaller returns an instance of the Debian installer
func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) {
// newTarInstaller returns an instance of the Debian installer for a specific version
func newTarInstaller(image string, tag string, version string, stale bool) (ElasticAgentInstaller, error) {
image = image + "-systemd" // we want to consume systemd boxes
service := image
profile := FleetProfileName

// extract the agent in the box, as it's mounted as a volume
artifact := "elastic-agent"
version := agentVersion
os := "linux"
arch := "x86_64"
extension := "tar.gz"

tarFile, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension)
tarFile, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension, stale)
if err != nil {
log.WithFields(log.Fields{
"artifact": artifact,
Expand All @@ -461,6 +468,7 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) {

//
installerPackage := NewTARPackage(tarFile, profile, image, service).
Stale(stale).
WithArch(arch).
WithArtifact(artifact).
WithOS(os).
Expand Down
Loading