Skip to content
This repository has been archived by the owner on Sep 17, 2024. It is now read-only.

Commit

Permalink
chore: refactor Fleet upgrade tests (#671)
Browse files Browse the repository at this point in the history 
* chore: use nightly annotation for the Upgrade tests

* chore: add two make goals for the nightly use cases

- e2e-fleet-nightly: run the nightly tests not using CI snapshots.
It should download the binaries from the official artifactory.
- e2e-fleet-nightly-ci-snapshots: run the nightly tests using the CI
snapshots for a specific SHA commit from Beats, downloading them from
a GCP bucket

* chore: bump elastic-agent stale version

* chore: pass version and state state when creating an installer

This will allow selecting the proper binary, depending if we are using
a stale agent or a regular one.

* fix: append snapshot to the stale version when using CI snapshots

* fix: check for version aliases with non-stale versions

* chore: store current agent version in the test suite struct

* chore: make sure the layout is properly created for TAR installer

* chore: add make goals for testing fleet use cases

* chore: move Make goals to the e2e Makefile
# Conflicts:
#	.ci/Jenkinsfile
  • Loading branch information
@mdelapenya
mdelapenya committed Feb 4, 2021
1 parent 37bc58e commit bcc9599
Show file tree
Hide file tree
Showing 8 changed files with 269 additions and 60 deletions.
2 changes: 1 addition & 1 deletion .ci/Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ pipeline {
string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel where errors will be posted. For multiple channels, use a comma-separated list of channels')
string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/7.10.0-9362413a/downloads/beats/elastic-agent/elastic-agent-7.10.0-SNAPSHOT-linux-x86_64.tar.gz')
string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '7.10-SNAPSHOT', description: 'SemVer version of the stand-alone elastic-agent to be used for Fleet tests. You can use here the tag of your PR to test your changes')
string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.10.1', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.')
string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.9.3', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.')
booleanParam(name: "BEATS_USE_CI_SNAPSHOTS", defaultValue: false, description: "If it's needed to use the binary snapshots produced by Beats CI instead of the official releases")
choice(name: 'LOG_LEVEL', choices: ['DEBUG', 'INFO'], description: 'Log level to be used')
choice(name: 'TIMEOUT_FACTOR', choices: ['5', '3', '7', '11'], description: 'Max number of minutes for timeout backoff strategies')
Expand Down
22 changes: 22 additions & 0 deletions e2e/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,3 +91,25 @@ sync-integrations:
unit-test:
gotestsum --format testname -- -count=1 -timeout=$(TEST_TIMEOUT) ./...
cd _suites && gotestsum --format testname -- -count=1 -timeout=$(TEST_TIMEOUT) ./...

## Test examples

.PHONY: fleet-fleet
fleet-fleet:
SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test

.PHONY: fleet-fleet-ci-snapshots
fleet-fleet-ci-snapshots:
SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test

.PHONY: fleet-fleet-pr-ci-snapshots
fleet-fleet-pr-ci-snapshots:
SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true ELASTIC_AGENT_VERSION=pr-14954 $(MAKE) functional-test

.PHONY: fleet-nightly
fleet-nightly:
SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test

.PHONY: fleet-nightly-ci-snapshots
fleet-nightly-ci-snapshots:
SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test
2 changes: 1 addition & 1 deletion e2e/_suites/fleet/features/fleet_mode_agent.feature
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ Examples:
| debian |

# @upgrade-agent
@skip
@nightly
Scenario Outline: Upgrading the installed <os> agent
Given a "<os>" agent "stale" is deployed to Fleet with "tar" installer
And certs for "<os>" are installed
Expand Down
24 changes: 16 additions & 8 deletions e2e/_suites/fleet/fleet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ type FleetTestSuite struct {
CurrentToken string // current enrollment token
CurrentTokenID string // current enrollment tokenID
Hostname string // the hostname of the container
Version string // current elastic-agent version
// integrations
Integration IntegrationPackage // the installed integration
PolicyUpdatedAt string // the moment the policy was updated
Expand Down Expand Up @@ -107,6 +108,8 @@ func (fts *FleetTestSuite) afterScenario() {
func (fts *FleetTestSuite) beforeScenario() {
fts.Cleanup = false

fts.Version = agentVersion

// create policy with system monitoring enabled
defaultPolicy, err := getAgentDefaultPolicy()
if err != nil {
Expand Down Expand Up @@ -148,8 +151,8 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) {
}

func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, version, installerType string) error {
agentVersionBackup := agentVersion
defer func() { agentVersion = agentVersionBackup }()
agentVersionBackup := fts.Version
defer func() { fts.Version = agentVersionBackup }()

switch version {
case "stale":
Expand All @@ -160,13 +163,12 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver
version = agentStaleVersion
}

agentVersion = version
fts.Version = version

// prepare installer for stale version
if agentVersion != agentVersionBackup {
i := GetElasticAgentInstaller(image, installerType)
installerType = fmt.Sprintf("%s-%s", installerType, version)
fts.Installers[fmt.Sprintf("%s-%s", image, installerType)] = i
if fts.Version != agentVersionBackup {
i := GetElasticAgentInstaller(image, installerType, fts.Version, true)
fts.Installers[fmt.Sprintf("%s-%s-%s", image, installerType, version)] = i
}

return fts.anAgentIsDeployedToFleetWithInstaller(image, installerType)
Expand All @@ -175,6 +177,12 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver
func (fts *FleetTestSuite) installCerts(targetOS string) error {
installer := fts.getInstaller()
if installer.InstallCertsFn == nil {
log.WithFields(log.Fields{
"installer": installer,
"version": fts.Version,
"agentVersion": agentVersion,
"agentStaleVersion": agentStaleVersion,
}).Error("No installer found")
return errors.New("no installer found")
}

Expand Down Expand Up @@ -291,7 +299,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(image string, i
}

func (fts *FleetTestSuite) getInstaller() ElasticAgentInstaller {
return fts.Installers[fts.Image+"-"+fts.InstallerType]
return fts.Installers[fts.Image+"-"+fts.InstallerType+"-"+fts.Version]
}

func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state string) error {
Expand Down
15 changes: 10 additions & 5 deletions e2e/_suites/fleet/ingest-manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ var agentVersion = agentVersionBase

// agentStaleVersion is the version of the agent to use as a base during upgrade
// It can be overriden by ELASTIC_AGENT_STALE_VERSION env var. Using latest GA as a default.
var agentStaleVersion = "7.10.0"
var agentStaleVersion = "7.10.2"

// stackVersion is the version of the stack to use
// It can be overriden by STACK_VERSION env var
Expand Down Expand Up @@ -81,6 +81,11 @@ func setUpSuite() {

timeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", timeoutFactor)

useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS")
if useCISnapshots && !strings.HasSuffix(agentStaleVersion, "-SNAPSHOT") {
agentStaleVersion += "-SNAPSHOT"
}

// check if version is an alias
agentVersion = e2e.GetElasticArtifactVersion(agentVersion)
agentStaleVersion = shell.GetEnv("ELASTIC_AGENT_STALE_VERSION", agentStaleVersion)
Expand All @@ -89,10 +94,10 @@ func setUpSuite() {
imts = IngestManagerTestSuite{
Fleet: &FleetTestSuite{
Installers: map[string]ElasticAgentInstaller{
"centos-systemd": GetElasticAgentInstaller("centos", "systemd"),
"centos-tar": GetElasticAgentInstaller("centos", "tar"),
"debian-systemd": GetElasticAgentInstaller("debian", "systemd"),
"debian-tar": GetElasticAgentInstaller("debian", "tar"),
"centos-systemd-" + agentVersion: GetElasticAgentInstaller("centos", "systemd", agentVersion, false),
"centos-tar-" + agentVersion: GetElasticAgentInstaller("centos", "tar", agentVersion, false),
"debian-systemd-" + agentVersion: GetElasticAgentInstaller("debian", "systemd", agentVersion, false),
"debian-tar-" + agentVersion: GetElasticAgentInstaller("debian", "tar", agentVersion, false),
},
},
StandAlone: &StandAloneTestSuite{},
Expand Down
40 changes: 28 additions & 12 deletions e2e/_suites/fleet/installers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ type TARPackage struct {
arch string
artifact string
OS string
stale bool
version string
}

Expand Down Expand Up @@ -214,25 +215,40 @@ func (i *TARPackage) Preinstall() error {
return err
}

version := checkElasticAgentVersion(i.version)
version := i.version
if !i.stale {
version = checkElasticAgentVersion(i.version)
}

// simplify layout
cmds := []string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, version, i.OS, i.arch), "/elastic-agent"}
err = execCommandInService(i.profile, i.image, i.service, cmds, false)
if err != nil {
log.WithFields(log.Fields{
"command": cmds,
"error": err,
"image": i.image,
"service": i.service,
}).Error("Could not extract agent package in the box")

return err
cmds := [][]string{
[]string{"rm", "-fr", "/elastic-agent"},
[]string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, version, i.OS, i.arch), "/elastic-agent"},
}
for _, cmd := range cmds {
err = execCommandInService(i.profile, i.image, i.service, cmd, false)
if err != nil {
log.WithFields(log.Fields{
"command": cmd,
"error": err,
"image": i.image,
"service": i.service,
"version": version,
}).Error("Could not extract agent package in the box")

return err
}
}

return nil
}

// Stale sets the stale state
func (i *TARPackage) Stale(stale bool) *TARPackage {
i.stale = stale
return i
}

// Uninstall uninstalls a TAR package
func (i *TARPackage) Uninstall() error {
args := []string{"-f"}
Expand Down
50 changes: 29 additions & 21 deletions e2e/_suites/fleet/services.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ func runElasticAgentCommand(profile string, image string, service string, proces
// to be used will be defined by the local snapshot produced by the local build.
// Else, if the environment variable BEATS_USE_CI_SNAPSHOTS is set, then the artifact
// to be downloaded will be defined by the latest snapshot produced by the Beats CI.
func downloadAgentBinary(artifact string, version string, OS string, arch string, extension string) (string, string, error) {
func downloadAgentBinary(artifact string, version string, OS string, arch string, extension string, stale bool) (string, string, error) {
fileName := fmt.Sprintf("%s-%s-%s.%s", artifact, version, arch, extension)

beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "")
Expand Down Expand Up @@ -217,7 +217,7 @@ func downloadAgentBinary(artifact string, version string, OS string, arch string
if useCISnapshots {
log.Debug("Using CI snapshots for the Elastic Agent")

bucketFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension)
bucketFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, stale)

maxTimeout := time.Duration(timeoutFactor) * time.Minute

Expand All @@ -229,7 +229,12 @@ func downloadAgentBinary(artifact string, version string, OS string, arch string
return handleDownload(downloadURL, bucketFileName)
}

downloadURL, err = e2e.GetElasticArtifactURL(artifact, checkElasticAgentVersion(version), OS, arch, extension)
downloadVersion := version
if !stale {
downloadVersion = checkElasticAgentVersion(version)
}

downloadURL, err = e2e.GetElasticArtifactURL(artifact, downloadVersion, OS, arch, extension)
if err != nil {
return "", "", err
}
Expand All @@ -238,7 +243,7 @@ func downloadAgentBinary(artifact string, version string, OS string, arch string
}

// GetElasticAgentInstaller returns an installer from a docker image
func GetElasticAgentInstaller(image string, installerType string) ElasticAgentInstaller {
func GetElasticAgentInstaller(image string, installerType string, version string, stale bool) ElasticAgentInstaller {
log.WithFields(log.Fields{
"image": image,
"installer": installerType,
Expand All @@ -247,13 +252,13 @@ func GetElasticAgentInstaller(image string, installerType string) ElasticAgentIn
var installer ElasticAgentInstaller
var err error
if "centos" == image && "tar" == installerType {
installer, err = newTarInstaller("centos", "latest")
installer, err = newTarInstaller("centos", "latest", version, stale)
} else if "centos" == image && "systemd" == installerType {
installer, err = newCentosInstaller("centos", "latest")
installer, err = newCentosInstaller("centos", "latest", version, stale)
} else if "debian" == image && "tar" == installerType {
installer, err = newTarInstaller("debian", "stretch")
installer, err = newTarInstaller("debian", "stretch", version, stale)
} else if "debian" == image && "systemd" == installerType {
installer, err = newDebianInstaller("debian", "stretch")
installer, err = newDebianInstaller("debian", "stretch", version, stale)
} else {
log.WithFields(log.Fields{
"image": image,
Expand All @@ -273,7 +278,7 @@ func GetElasticAgentInstaller(image string, installerType string) ElasticAgentIn
}

// getGCPBucketCoordinates it calculates the bucket path in GCP
func getGCPBucketCoordinates(fileName string, artifact string, version string, OS string, arch string, extension string) (string, string, string, string) {
func getGCPBucketCoordinates(fileName string, artifact string, version string, OS string, arch string, extension string, stale bool) (string, string, string, string) {
if extension == "tar.gz" {
fileName = fmt.Sprintf("%s-%s-%s-%s.%s", artifact, version, OS, arch, extension)
}
Expand Down Expand Up @@ -307,27 +312,31 @@ func getGCPBucketCoordinates(fileName string, artifact string, version string, O
object = fmt.Sprintf("%s/%s", artifact, newFileName)
}

if stale {
prefix = fmt.Sprintf("snapshots/%s", artifact)
object = newFileName
}

return newFileName, bucket, prefix, object
}

func isSystemdBased(image string) bool {
return strings.HasSuffix(image, "-systemd")
}

// newCentosInstaller returns an instance of the Centos installer
func newCentosInstaller(image string, tag string) (ElasticAgentInstaller, error) {
// newCentosInstaller returns an instance of the Centos installer for a specific version
func newCentosInstaller(image string, tag string, version string, stale bool) (ElasticAgentInstaller, error) {
image = image + "-systemd" // we want to consume systemd boxes
service := image
profile := FleetProfileName

// extract the agent in the box, as it's mounted as a volume
artifact := "elastic-agent"
version := agentVersion
os := "linux"
arch := "x86_64"
extension := "rpm"

binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension)
binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension, stale)
if err != nil {
log.WithFields(log.Fields{
"artifact": artifact,
Expand Down Expand Up @@ -379,20 +388,19 @@ func newCentosInstaller(image string, tag string) (ElasticAgentInstaller, error)
}, nil
}

// newDebianInstaller returns an instance of the Debian installer
func newDebianInstaller(image string, tag string) (ElasticAgentInstaller, error) {
// newDebianInstaller returns an instance of the Debian installer for a specific version
func newDebianInstaller(image string, tag string, version string, stale bool) (ElasticAgentInstaller, error) {
image = image + "-systemd" // we want to consume systemd boxes
service := image
profile := FleetProfileName

// extract the agent in the box, as it's mounted as a volume
artifact := "elastic-agent"
version := agentVersion
os := "linux"
arch := "amd64"
extension := "deb"

binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension)
binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension, stale)
if err != nil {
log.WithFields(log.Fields{
"artifact": artifact,
Expand Down Expand Up @@ -444,20 +452,19 @@ func newDebianInstaller(image string, tag string) (ElasticAgentInstaller, error)
}, nil
}

// newTarInstaller returns an instance of the Debian installer
func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) {
// newTarInstaller returns an instance of the Debian installer for a specific version
func newTarInstaller(image string, tag string, version string, stale bool) (ElasticAgentInstaller, error) {
image = image + "-systemd" // we want to consume systemd boxes
service := image
profile := FleetProfileName

// extract the agent in the box, as it's mounted as a volume
artifact := "elastic-agent"
version := agentVersion
os := "linux"
arch := "x86_64"
extension := "tar.gz"

tarFile, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension)
tarFile, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension, stale)
if err != nil {
log.WithFields(log.Fields{
"artifact": artifact,
Expand All @@ -482,6 +489,7 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) {

//
installerPackage := NewTARPackage(tarFile, profile, image, service).
Stale(stale).
WithArch(arch).
WithArtifact(artifact).
WithOS(os).
Expand Down
Loading

0 comments on commit bcc9599

Please sign in to comment.