Skip to content
This repository has been archived by the owner on Sep 17, 2024. It is now read-only.

Commit

Permalink
chore: refactor Fleet upgrade tests (#671) | chore: refactor build ar…
Browse files Browse the repository at this point in the history
…tifacts methods (#689) backport for 6.8.x (#713)

* chore: refactor Fleet upgrade tests (#671)

* chore: use nightly annotation for the Upgrade tests

* chore: add two make goals for the nightly use cases

- e2e-fleet-nightly: run the nightly tests not using CI snapshots.
It should download the binaries from the official artifactory.
- e2e-fleet-nightly-ci-snapshots: run the nightly tests using the CI
snapshots for a specific SHA commit from Beats, downloading them from
a GCP bucket

* chore: bump elastic-agent stale version

* chore: pass version and state state when creating an installer

This will allow selecting the proper binary, depending if we are using
a stale agent or a regular one.

* fix: append snapshot to the stale version when using CI snapshots

* fix: check for version aliases with non-stale versions

* chore: store current agent version in the test suite struct

* chore: make sure the layout is properly created for TAR installer

* chore: add make goals for testing fleet use cases

* chore: move Make goals to the e2e Makefile

* chore: refactor build artifacts methods (#689)

* feat: use local beats directory for elastic-agent installers

* docs: document BEATS_LOCAL_PATH variable

* chore: check that the local file exists

* chore: add unit tests for the BEATS_LOCAL_PATH variable

* chore: simplify logic after test coverage

* chore: enrich test name

* chore: rename test

* chore: move default values to the end of the descriptions

* chore: add a helper method to build artifact names

* chore: build artifact name only once

* chore: pass the entire filename to calculate the GCP bucket coordinates

* chore: simplify method signature, as the file name is not changed within the method

* chore: use buildArtifactName method

* chore: pass filename to helper methods to avoid unnecessary calculations

* chore: rename variable for consistency

* chore: add license headers

* chore: use lowercase comparison for artifact name

* chore: build artifact changes from Beats CI to Elastic's artifactory

* fix: support building artifact names from local

Local builds uses the same as the CI

* WIP

* fix: rename docker-ubi8 installer key

* chore: pass a fallback version when building artifact name

In the case we are using a PR , where the version is "pr-12345", then we
need to use the base version of the product

* chore: remove log

* fix: do not override artifact name in Docker installer

* feat: support consuming the docker images from snapshots or local in standalone mode

* chore: add unit tests for fetching docker images from local Beats repo

* chore: apply version to Docker installer

* chore: support loading metricbeat image from local repository

* chore: exrtact loadImage method to docker helper

* feat: support consuming CI artifacts in metricbeat

* fix: remove non-existent field from logrus log

* chore: extract download logic to helper methods

We are discarding the stale parameter, because the stale version is set
when the installer is required, so we simply download what is requested

* chore: fix typo

* chore: check agent version before setting it

* chore: use the docker client to load the image instead of the docker binary

* chore: add unit tests for check method

* chore: pass fallback version to check method

* fix: properly calculate version for PRs

* chore: move PR check logic to a method

* chore: make test independent of maintenance branch
# Conflicts:
#	e2e/_suites/fleet/services_test.go
  • Loading branch information
mdelapenya authored Feb 8, 2021
1 parent a960848 commit eabc001
Showing 15 changed files with 840 additions and 414 deletions.
2 changes: 1 addition & 1 deletion .ci/Jenkinsfile
Original file line number Diff line number Diff line change
@@ -37,7 +37,7 @@ pipeline {
string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel(s) where errors will be posted. For multiple channels, use a comma-separated list of channels')
string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz')
string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '8.0.0-SNAPSHOT', description: 'SemVer version of the stand-alone elastic-agent to be used for Fleet tests. You can use here the tag of your PR to test your changes')
string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.10.1', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.')
string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.10.2', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.')
booleanParam(name: "BEATS_USE_CI_SNAPSHOTS", defaultValue: false, description: "If it's needed to use the binary snapshots produced by Beats CI instead of the official releases")
choice(name: 'LOG_LEVEL', choices: ['DEBUG', 'INFO'], description: 'Log level to be used')
choice(name: 'TIMEOUT_FACTOR', choices: ['5', '3', '7', '11'], description: 'Max number of minutes for timeout backoff strategies')
34 changes: 34 additions & 0 deletions cli/docker/docker.go
Original file line number Diff line number Diff line change
@@ -6,7 +6,10 @@ package docker

import (
"bytes"
"compress/gzip"
"context"
"os"
"path/filepath"
"strings"

"github.com/docker/docker/api/types"
@@ -172,6 +175,37 @@ func RemoveContainer(containerName string) error {
return nil
}

// LoadImage loads a TAR file in the local docker engine
func LoadImage(imagePath string) error {
fileNamePath, err := filepath.Abs(imagePath)
if err != nil {
return err
}

_, err = os.Stat(fileNamePath)
if err != nil || os.IsNotExist(err) {
return err
}

dockerClient := getDockerClient()
file, err := os.Open(imagePath)

input, err := gzip.NewReader(file)
imageLoadResponse, err := dockerClient.ImageLoad(context.Background(), input, false)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"image": fileNamePath,
}).Error("Could not load the Docker image.")
return err
}

log.WithFields(log.Fields{
"response": imageLoadResponse,
}).Debug("Docker image loaded successfully")
return nil
}

// RemoveDevNetwork removes the developer network
func RemoveDevNetwork() error {
dockerClient := getDockerClient()
22 changes: 22 additions & 0 deletions e2e/Makefile
Original file line number Diff line number Diff line change
@@ -91,3 +91,25 @@ sync-integrations:
unit-test:
gotestsum --format testname -- -count=1 -timeout=$(TEST_TIMEOUT) ./...
cd _suites && gotestsum --format testname -- -count=1 -timeout=$(TEST_TIMEOUT) ./...

## Test examples

.PHONY: fleet-fleet
fleet-fleet:
SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test

.PHONY: fleet-fleet-ci-snapshots
fleet-fleet-ci-snapshots:
SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test

.PHONY: fleet-fleet-pr-ci-snapshots
fleet-fleet-pr-ci-snapshots:
SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true ELASTIC_AGENT_VERSION=pr-14954 $(MAKE) functional-test

.PHONY: fleet-nightly
fleet-nightly:
SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test

.PHONY: fleet-nightly-ci-snapshots
fleet-nightly-ci-snapshots:
SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test
2 changes: 1 addition & 1 deletion e2e/_suites/fleet/features/fleet_mode_agent.feature
Original file line number Diff line number Diff line change
@@ -40,7 +40,7 @@ Examples:
| debian |

# @upgrade-agent
@skip
@nightly
Scenario Outline: Upgrading the installed <os> agent
Given a "<os>" agent "stale" is deployed to Fleet with "tar" installer
And certs for "<os>" are installed
33 changes: 20 additions & 13 deletions e2e/_suites/fleet/fleet.go
Original file line number Diff line number Diff line change
@@ -42,6 +42,7 @@ type FleetTestSuite struct {
CurrentToken string // current enrollment token
CurrentTokenID string // current enrollment tokenID
Hostname string // the hostname of the container
Version string // current elastic-agent version
// integrations
Integration IntegrationPackage // the installed integration
PolicyUpdatedAt string // the moment the policy was updated
@@ -107,6 +108,8 @@ func (fts *FleetTestSuite) afterScenario() {
func (fts *FleetTestSuite) beforeScenario() {
fts.Cleanup = false

fts.Version = agentVersion

// create policy with system monitoring enabled
defaultPolicy, err := getAgentDefaultPolicy()
if err != nil {
@@ -148,8 +151,8 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) {
}

func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, version, installerType string) error {
agentVersionBackup := agentVersion
defer func() { agentVersion = agentVersionBackup }()
agentVersionBackup := fts.Version
defer func() { fts.Version = agentVersionBackup }()

switch version {
case "stale":
@@ -160,13 +163,12 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver
version = agentStaleVersion
}

agentVersion = version
fts.Version = version

// prepare installer for stale version
if agentVersion != agentVersionBackup {
i := GetElasticAgentInstaller(image, installerType)
installerType = fmt.Sprintf("%s-%s", installerType, version)
fts.Installers[fmt.Sprintf("%s-%s", image, installerType)] = i
if fts.Version != agentVersionBackup {
i := GetElasticAgentInstaller(image, installerType, fts.Version)
fts.Installers[fmt.Sprintf("%s-%s-%s", image, installerType, version)] = i
}

return fts.anAgentIsDeployedToFleetWithInstaller(image, installerType)
@@ -175,6 +177,12 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver
func (fts *FleetTestSuite) installCerts(targetOS string) error {
installer := fts.getInstaller()
if installer.InstallCertsFn == nil {
log.WithFields(log.Fields{
"installer": installer,
"version": fts.Version,
"agentVersion": agentVersion,
"agentStaleVersion": agentStaleVersion,
}).Error("No installer found")
return errors.New("no installer found")
}

@@ -291,7 +299,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(image string, i
}

func (fts *FleetTestSuite) getInstaller() ElasticAgentInstaller {
return fts.Installers[fts.Image+"-"+fts.InstallerType]
return fts.Installers[fts.Image+"-"+fts.InstallerType+"-"+fts.Version]
}

func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state string) error {
@@ -371,11 +379,10 @@ func (fts *FleetTestSuite) theAgentIsListedInFleetWithStatus(desiredStatus strin
// the agent is not listed in Fleet
if desiredStatus == "offline" || desiredStatus == "inactive" {
log.WithFields(log.Fields{
"isAgentInStatus": isAgentInStatus,
"elapsedTime": exp.GetElapsedTime(),
"hostname": fts.Hostname,
"retries": retryCount,
"status": desiredStatus,
"elapsedTime": exp.GetElapsedTime(),
"hostname": fts.Hostname,
"retries": retryCount,
"status": desiredStatus,
}).Info("The Agent is not present in Fleet, as expected")
return nil
} else if desiredStatus == "online" {
32 changes: 12 additions & 20 deletions e2e/_suites/fleet/ingest-manager_test.go
Original file line number Diff line number Diff line change
@@ -45,7 +45,7 @@ var agentVersion = agentVersionBase

// agentStaleVersion is the version of the agent to use as a base during upgrade
// It can be overriden by ELASTIC_AGENT_STALE_VERSION env var. Using latest GA as a default.
var agentStaleVersion = "7.10.0"
var agentStaleVersion = "7.10.2"

// stackVersion is the version of the stack to use
// It can be overriden by STACK_VERSION env var
@@ -83,6 +83,11 @@ func setUpSuite() {
agentVersion = shell.GetEnv("ELASTIC_AGENT_VERSION", agentVersionBase)
agentStaleVersion = shell.GetEnv("ELASTIC_AGENT_STALE_VERSION", agentStaleVersion)

useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS")
if useCISnapshots && !strings.HasSuffix(agentStaleVersion, "-SNAPSHOT") {
agentStaleVersion += "-SNAPSHOT"
}

// check if version is an alias
agentVersion = e2e.GetElasticArtifactVersion(agentVersion)

@@ -91,10 +96,12 @@ func setUpSuite() {
imts = IngestManagerTestSuite{
Fleet: &FleetTestSuite{
Installers: map[string]ElasticAgentInstaller{
"centos-systemd": GetElasticAgentInstaller("centos", "systemd"),
"centos-tar": GetElasticAgentInstaller("centos", "tar"),
"debian-systemd": GetElasticAgentInstaller("debian", "systemd"),
"debian-tar": GetElasticAgentInstaller("debian", "tar"),
"centos-systemd-" + agentVersion: GetElasticAgentInstaller("centos", "systemd", agentVersion),
"centos-tar-" + agentVersion: GetElasticAgentInstaller("centos", "tar", agentVersion),
"debian-systemd-" + agentVersion: GetElasticAgentInstaller("debian", "systemd", agentVersion),
"debian-tar-" + agentVersion: GetElasticAgentInstaller("debian", "tar", agentVersion),
"docker-default-" + agentVersion: GetElasticAgentInstaller("docker", "default", agentVersion),
"docker-ubi8-" + agentVersion: GetElasticAgentInstaller("docker", "ubi8", agentVersion),
},
},
StandAlone: &StandAloneTestSuite{},
@@ -226,21 +233,6 @@ func (imts *IngestManagerTestSuite) processStateOnTheHost(process string, state
return checkProcessStateOnTheHost(containerName, process, state)
}

// checkElasticAgentVersion returns a fallback version (agentVersionBase) if the version set by the environment is empty
func checkElasticAgentVersion(version string) string {
environmentVersion := os.Getenv("ELASTIC_AGENT_VERSION")

if environmentVersion == "" {
return agentVersionBase
}

if strings.HasPrefix(strings.ToLower(environmentVersion), "pr-") {
return agentVersionBase
}

return version
}

// name of the container for the service:
// we are using the Docker client instead of docker-compose
// because it does not support returning the output of a
114 changes: 101 additions & 13 deletions e2e/_suites/fleet/installers.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package main

import (
"fmt"

"github.com/elastic/e2e-testing/cli/docker"
log "github.com/sirupsen/logrus"
)

@@ -98,6 +103,85 @@ func (i *DEBPackage) Uninstall() error {
return nil
}

// DockerPackage implements operations for a DEB installer
type DockerPackage struct {
BasePackage
installerPath string
ubi8 bool
// optional fields
arch string
artifact string
OS string
version string
}

// NewDockerPackage creates an instance for the Docker installer
func NewDockerPackage(binaryName string, profile string, image string, service string, installerPath string, ubi8 bool) *DockerPackage {
return &DockerPackage{
BasePackage: BasePackage{
binaryName: binaryName,
image: image,
profile: profile,
service: service,
},
installerPath: installerPath,
ubi8: ubi8,
}
}

// Install installs a Docker package
func (i *DockerPackage) Install(containerName string, token string) error {
log.Trace("No install commands for Docker packages")
return nil
}

// InstallCerts installs the certificates for a Docker package
func (i *DockerPackage) InstallCerts() error {
log.Trace("No install certs commands for Docker packages")
return nil
}

// Preinstall executes operations before installing a Docker package
func (i *DockerPackage) Preinstall() error {
return docker.LoadImage(i.installerPath)
}

// Postinstall executes operations after installing a Docker package
func (i *DockerPackage) Postinstall() error {
log.Trace("No postinstall commands for Docker packages")
return nil
}

// Uninstall uninstalls a Docker package
func (i *DockerPackage) Uninstall() error {
log.Trace("No uninstall commands for Docker packages")
return nil
}

// WithArch sets the architecture
func (i *DockerPackage) WithArch(arch string) *DockerPackage {
i.arch = arch
return i
}

// WithArtifact sets the artifact
func (i *DockerPackage) WithArtifact(artifact string) *DockerPackage {
i.artifact = artifact
return i
}

// WithOS sets the OS
func (i *DockerPackage) WithOS(OS string) *DockerPackage {
i.OS = OS
return i
}

// WithVersion sets the version
func (i *DockerPackage) WithVersion(version string) *DockerPackage {
i.version = version
return i
}

// RPMPackage implements operations for a RPM installer
type RPMPackage struct {
BasePackage
@@ -214,20 +298,24 @@ func (i *TARPackage) Preinstall() error {
return err
}

version := checkElasticAgentVersion(i.version)

// simplify layout
cmds := []string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, version, i.OS, i.arch), "/elastic-agent"}
err = execCommandInService(i.profile, i.image, i.service, cmds, false)
if err != nil {
log.WithFields(log.Fields{
"command": cmds,
"error": err,
"image": i.image,
"service": i.service,
}).Error("Could not extract agent package in the box")

return err
cmds := [][]string{
[]string{"rm", "-fr", "/elastic-agent"},
[]string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, i.version, i.OS, i.arch), "/elastic-agent"},
}
for _, cmd := range cmds {
err = execCommandInService(i.profile, i.image, i.service, cmd, false)
if err != nil {
log.WithFields(log.Fields{
"command": cmd,
"error": err,
"image": i.image,
"service": i.service,
"version": i.version,
}).Error("Could not extract agent package in the box")

return err
}
}

return nil
Loading

0 comments on commit eabc001

Please sign in to comment.