From 83ea2177195441199ed9574a8599e6a3bfd47b99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 4 Feb 2021 16:38:06 +0100 Subject: [PATCH 1/2] chore: refactor Fleet upgrade tests (#671) * chore: use nightly annotation for the Upgrade tests * chore: add two make goals for the nightly use cases - e2e-fleet-nightly: run the nightly tests not using CI snapshots. It should download the binaries from the official artifactory. - e2e-fleet-nightly-ci-snapshots: run the nightly tests using the CI snapshots for a specific SHA commit from Beats, downloading them from a GCP bucket * chore: bump elastic-agent stale version * chore: pass version and state state when creating an installer This will allow selecting the proper binary, depending if we are using a stale agent or a regular one. * fix: append snapshot to the stale version when using CI snapshots * fix: check for version aliases with non-stale versions * chore: store current agent version in the test suite struct * chore: make sure the layout is properly created for TAR installer * chore: add make goals for testing fleet use cases * chore: move Make goals to the e2e Makefile --- .ci/Jenkinsfile | 2 +- e2e/Makefile | 22 +++ .../fleet/features/fleet_mode_agent.feature | 2 +- e2e/_suites/fleet/fleet.go | 24 ++- e2e/_suites/fleet/ingest-manager_test.go | 15 +- e2e/_suites/fleet/installers.go | 40 ++-- e2e/_suites/fleet/services.go | 50 ++--- e2e/_suites/fleet/services_test.go | 174 ++++++++++++++++-- 8 files changed, 269 insertions(+), 60 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index a4aee18507..264a7c5ec0 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -37,7 +37,7 @@ pipeline { string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel(s) where errors will be posted. For multiple channels, use a comma-separated list of channels') string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz') string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '8.0.0-SNAPSHOT', description: 'SemVer version of the stand-alone elastic-agent to be used for Fleet tests. You can use here the tag of your PR to test your changes') - string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.10.1', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.') + string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.10.2', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.') booleanParam(name: "BEATS_USE_CI_SNAPSHOTS", defaultValue: false, description: "If it's needed to use the binary snapshots produced by Beats CI instead of the official releases") choice(name: 'LOG_LEVEL', choices: ['DEBUG', 'INFO'], description: 'Log level to be used') choice(name: 'TIMEOUT_FACTOR', choices: ['5', '3', '7', '11'], description: 'Max number of minutes for timeout backoff strategies') diff --git a/e2e/Makefile b/e2e/Makefile index 8223e897ef..3fd8b194ff 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -91,3 +91,25 @@ sync-integrations: unit-test: gotestsum --format testname -- -count=1 -timeout=$(TEST_TIMEOUT) ./... cd _suites && gotestsum --format testname -- -count=1 -timeout=$(TEST_TIMEOUT) ./... + +## Test examples + +.PHONY: fleet-fleet +fleet-fleet: + SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test + +.PHONY: fleet-fleet-ci-snapshots +fleet-fleet-ci-snapshots: + SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test + +.PHONY: fleet-fleet-pr-ci-snapshots +fleet-fleet-pr-ci-snapshots: + SUITE="fleet" TAGS="fleet_mode_agent" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true ELASTIC_AGENT_VERSION=pr-14954 $(MAKE) functional-test + +.PHONY: fleet-nightly +fleet-nightly: + SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE DEVELOPER_MODE=true $(MAKE) functional-test + +.PHONY: fleet-nightly-ci-snapshots +fleet-nightly-ci-snapshots: + SUITE="fleet" TAGS="fleet_mode_agent && nightly" TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE BEATS_USE_CI_SNAPSHOTS=true DEVELOPER_MODE=true GITHUB_CHECK_SHA1=a1962c8864016010adcde9f35bd8378debb4fbf7 $(MAKE) functional-test diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index edc457fd9c..2af294f985 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -40,7 +40,7 @@ Examples: | debian | # @upgrade-agent -@skip +@nightly Scenario Outline: Upgrading the installed agent Given a "" agent "stale" is deployed to Fleet with "tar" installer And certs for "" are installed diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index 1140a91f1b..a92c8e2518 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -42,6 +42,7 @@ type FleetTestSuite struct { CurrentToken string // current enrollment token CurrentTokenID string // current enrollment tokenID Hostname string // the hostname of the container + Version string // current elastic-agent version // integrations Integration IntegrationPackage // the installed integration PolicyUpdatedAt string // the moment the policy was updated @@ -107,6 +108,8 @@ func (fts *FleetTestSuite) afterScenario() { func (fts *FleetTestSuite) beforeScenario() { fts.Cleanup = false + fts.Version = agentVersion + // create policy with system monitoring enabled defaultPolicy, err := getAgentDefaultPolicy() if err != nil { @@ -148,8 +151,8 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) { } func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, version, installerType string) error { - agentVersionBackup := agentVersion - defer func() { agentVersion = agentVersionBackup }() + agentVersionBackup := fts.Version + defer func() { fts.Version = agentVersionBackup }() switch version { case "stale": @@ -160,13 +163,12 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver version = agentStaleVersion } - agentVersion = version + fts.Version = version // prepare installer for stale version - if agentVersion != agentVersionBackup { - i := GetElasticAgentInstaller(image, installerType) - installerType = fmt.Sprintf("%s-%s", installerType, version) - fts.Installers[fmt.Sprintf("%s-%s", image, installerType)] = i + if fts.Version != agentVersionBackup { + i := GetElasticAgentInstaller(image, installerType, fts.Version, true) + fts.Installers[fmt.Sprintf("%s-%s-%s", image, installerType, version)] = i } return fts.anAgentIsDeployedToFleetWithInstaller(image, installerType) @@ -175,6 +177,12 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver func (fts *FleetTestSuite) installCerts(targetOS string) error { installer := fts.getInstaller() if installer.InstallCertsFn == nil { + log.WithFields(log.Fields{ + "installer": installer, + "version": fts.Version, + "agentVersion": agentVersion, + "agentStaleVersion": agentStaleVersion, + }).Error("No installer found") return errors.New("no installer found") } @@ -291,7 +299,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(image string, i } func (fts *FleetTestSuite) getInstaller() ElasticAgentInstaller { - return fts.Installers[fts.Image+"-"+fts.InstallerType] + return fts.Installers[fts.Image+"-"+fts.InstallerType+"-"+fts.Version] } func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state string) error { diff --git a/e2e/_suites/fleet/ingest-manager_test.go b/e2e/_suites/fleet/ingest-manager_test.go index 4ad2c93612..fe61fa2341 100644 --- a/e2e/_suites/fleet/ingest-manager_test.go +++ b/e2e/_suites/fleet/ingest-manager_test.go @@ -45,7 +45,7 @@ var agentVersion = agentVersionBase // agentStaleVersion is the version of the agent to use as a base during upgrade // It can be overriden by ELASTIC_AGENT_STALE_VERSION env var. Using latest GA as a default. -var agentStaleVersion = "7.10.0" +var agentStaleVersion = "7.10.2" // stackVersion is the version of the stack to use // It can be overriden by STACK_VERSION env var @@ -83,6 +83,11 @@ func setUpSuite() { agentVersion = shell.GetEnv("ELASTIC_AGENT_VERSION", agentVersionBase) agentStaleVersion = shell.GetEnv("ELASTIC_AGENT_STALE_VERSION", agentStaleVersion) + useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS") + if useCISnapshots && !strings.HasSuffix(agentStaleVersion, "-SNAPSHOT") { + agentStaleVersion += "-SNAPSHOT" + } + // check if version is an alias agentVersion = e2e.GetElasticArtifactVersion(agentVersion) @@ -91,10 +96,10 @@ func setUpSuite() { imts = IngestManagerTestSuite{ Fleet: &FleetTestSuite{ Installers: map[string]ElasticAgentInstaller{ - "centos-systemd": GetElasticAgentInstaller("centos", "systemd"), - "centos-tar": GetElasticAgentInstaller("centos", "tar"), - "debian-systemd": GetElasticAgentInstaller("debian", "systemd"), - "debian-tar": GetElasticAgentInstaller("debian", "tar"), + "centos-systemd-" + agentVersion: GetElasticAgentInstaller("centos", "systemd", agentVersion, false), + "centos-tar-" + agentVersion: GetElasticAgentInstaller("centos", "tar", agentVersion, false), + "debian-systemd-" + agentVersion: GetElasticAgentInstaller("debian", "systemd", agentVersion, false), + "debian-tar-" + agentVersion: GetElasticAgentInstaller("debian", "tar", agentVersion, false), }, }, StandAlone: &StandAloneTestSuite{}, diff --git a/e2e/_suites/fleet/installers.go b/e2e/_suites/fleet/installers.go index c992e0f7dd..75589ea2e5 100644 --- a/e2e/_suites/fleet/installers.go +++ b/e2e/_suites/fleet/installers.go @@ -157,6 +157,7 @@ type TARPackage struct { arch string artifact string OS string + stale bool version string } @@ -214,25 +215,40 @@ func (i *TARPackage) Preinstall() error { return err } - version := checkElasticAgentVersion(i.version) + version := i.version + if !i.stale { + version = checkElasticAgentVersion(i.version) + } // simplify layout - cmds := []string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, version, i.OS, i.arch), "/elastic-agent"} - err = execCommandInService(i.profile, i.image, i.service, cmds, false) - if err != nil { - log.WithFields(log.Fields{ - "command": cmds, - "error": err, - "image": i.image, - "service": i.service, - }).Error("Could not extract agent package in the box") - - return err + cmds := [][]string{ + []string{"rm", "-fr", "/elastic-agent"}, + []string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, version, i.OS, i.arch), "/elastic-agent"}, + } + for _, cmd := range cmds { + err = execCommandInService(i.profile, i.image, i.service, cmd, false) + if err != nil { + log.WithFields(log.Fields{ + "command": cmd, + "error": err, + "image": i.image, + "service": i.service, + "version": version, + }).Error("Could not extract agent package in the box") + + return err + } } return nil } +// Stale sets the stale state +func (i *TARPackage) Stale(stale bool) *TARPackage { + i.stale = stale + return i +} + // Uninstall uninstalls a TAR package func (i *TARPackage) Uninstall() error { args := []string{"-f"} diff --git a/e2e/_suites/fleet/services.go b/e2e/_suites/fleet/services.go index a4dd83afe8..d9b5b2eecf 100644 --- a/e2e/_suites/fleet/services.go +++ b/e2e/_suites/fleet/services.go @@ -166,7 +166,7 @@ func runElasticAgentCommand(profile string, image string, service string, proces // to be used will be defined by the local snapshot produced by the local build. // Else, if the environment variable BEATS_USE_CI_SNAPSHOTS is set, then the artifact // to be downloaded will be defined by the latest snapshot produced by the Beats CI. -func downloadAgentBinary(artifact string, version string, OS string, arch string, extension string) (string, string, error) { +func downloadAgentBinary(artifact string, version string, OS string, arch string, extension string, stale bool) (string, string, error) { fileName := fmt.Sprintf("%s-%s-%s.%s", artifact, version, arch, extension) beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") @@ -217,7 +217,7 @@ func downloadAgentBinary(artifact string, version string, OS string, arch string if useCISnapshots { log.Debug("Using CI snapshots for the Elastic Agent") - bucketFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension) + bucketFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, stale) maxTimeout := time.Duration(timeoutFactor) * time.Minute @@ -229,7 +229,12 @@ func downloadAgentBinary(artifact string, version string, OS string, arch string return handleDownload(downloadURL, bucketFileName) } - downloadURL, err = e2e.GetElasticArtifactURL(artifact, checkElasticAgentVersion(version), OS, arch, extension) + downloadVersion := version + if !stale { + downloadVersion = checkElasticAgentVersion(version) + } + + downloadURL, err = e2e.GetElasticArtifactURL(artifact, downloadVersion, OS, arch, extension) if err != nil { return "", "", err } @@ -238,7 +243,7 @@ func downloadAgentBinary(artifact string, version string, OS string, arch string } // GetElasticAgentInstaller returns an installer from a docker image -func GetElasticAgentInstaller(image string, installerType string) ElasticAgentInstaller { +func GetElasticAgentInstaller(image string, installerType string, version string, stale bool) ElasticAgentInstaller { log.WithFields(log.Fields{ "image": image, "installer": installerType, @@ -247,13 +252,13 @@ func GetElasticAgentInstaller(image string, installerType string) ElasticAgentIn var installer ElasticAgentInstaller var err error if "centos" == image && "tar" == installerType { - installer, err = newTarInstaller("centos", "latest") + installer, err = newTarInstaller("centos", "latest", version, stale) } else if "centos" == image && "systemd" == installerType { - installer, err = newCentosInstaller("centos", "latest") + installer, err = newCentosInstaller("centos", "latest", version, stale) } else if "debian" == image && "tar" == installerType { - installer, err = newTarInstaller("debian", "stretch") + installer, err = newTarInstaller("debian", "stretch", version, stale) } else if "debian" == image && "systemd" == installerType { - installer, err = newDebianInstaller("debian", "stretch") + installer, err = newDebianInstaller("debian", "stretch", version, stale) } else { log.WithFields(log.Fields{ "image": image, @@ -273,7 +278,7 @@ func GetElasticAgentInstaller(image string, installerType string) ElasticAgentIn } // getGCPBucketCoordinates it calculates the bucket path in GCP -func getGCPBucketCoordinates(fileName string, artifact string, version string, OS string, arch string, extension string) (string, string, string, string) { +func getGCPBucketCoordinates(fileName string, artifact string, version string, OS string, arch string, extension string, stale bool) (string, string, string, string) { if extension == "tar.gz" { fileName = fmt.Sprintf("%s-%s-%s-%s.%s", artifact, version, OS, arch, extension) } @@ -307,6 +312,11 @@ func getGCPBucketCoordinates(fileName string, artifact string, version string, O object = fmt.Sprintf("%s/%s", artifact, newFileName) } + if stale { + prefix = fmt.Sprintf("snapshots/%s", artifact) + object = newFileName + } + return newFileName, bucket, prefix, object } @@ -314,20 +324,19 @@ func isSystemdBased(image string) bool { return strings.HasSuffix(image, "-systemd") } -// newCentosInstaller returns an instance of the Centos installer -func newCentosInstaller(image string, tag string) (ElasticAgentInstaller, error) { +// newCentosInstaller returns an instance of the Centos installer for a specific version +func newCentosInstaller(image string, tag string, version string, stale bool) (ElasticAgentInstaller, error) { image = image + "-systemd" // we want to consume systemd boxes service := image profile := FleetProfileName // extract the agent in the box, as it's mounted as a volume artifact := "elastic-agent" - version := agentVersion os := "linux" arch := "x86_64" extension := "rpm" - binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension) + binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension, stale) if err != nil { log.WithFields(log.Fields{ "artifact": artifact, @@ -379,20 +388,19 @@ func newCentosInstaller(image string, tag string) (ElasticAgentInstaller, error) }, nil } -// newDebianInstaller returns an instance of the Debian installer -func newDebianInstaller(image string, tag string) (ElasticAgentInstaller, error) { +// newDebianInstaller returns an instance of the Debian installer for a specific version +func newDebianInstaller(image string, tag string, version string, stale bool) (ElasticAgentInstaller, error) { image = image + "-systemd" // we want to consume systemd boxes service := image profile := FleetProfileName // extract the agent in the box, as it's mounted as a volume artifact := "elastic-agent" - version := agentVersion os := "linux" arch := "amd64" extension := "deb" - binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension) + binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension, stale) if err != nil { log.WithFields(log.Fields{ "artifact": artifact, @@ -444,20 +452,19 @@ func newDebianInstaller(image string, tag string) (ElasticAgentInstaller, error) }, nil } -// newTarInstaller returns an instance of the Debian installer -func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) { +// newTarInstaller returns an instance of the Debian installer for a specific version +func newTarInstaller(image string, tag string, version string, stale bool) (ElasticAgentInstaller, error) { image = image + "-systemd" // we want to consume systemd boxes service := image profile := FleetProfileName // extract the agent in the box, as it's mounted as a volume artifact := "elastic-agent" - version := agentVersion os := "linux" arch := "x86_64" extension := "tar.gz" - tarFile, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension) + tarFile, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension, stale) if err != nil { log.WithFields(log.Fields{ "artifact": artifact, @@ -482,6 +489,7 @@ func newTarInstaller(image string, tag string) (ElasticAgentInstaller, error) { // installerPackage := NewTARPackage(tarFile, profile, image, service). + Stale(stale). WithArch(arch). WithArtifact(artifact). WithOS(os). diff --git a/e2e/_suites/fleet/services_test.go b/e2e/_suites/fleet/services_test.go index e48629157c..cbaf1abdf7 100644 --- a/e2e/_suites/fleet/services_test.go +++ b/e2e/_suites/fleet/services_test.go @@ -84,12 +84,26 @@ func TestGetGCPBucketCoordinates_Commits(t *testing.T) { extension := "rpm" fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension) + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) assert.Equal(t, bucket, "beats-ci-artifacts") assert.Equal(t, prefix, "commits/0123456789") assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-x86_64.rpm") }) + t.Run("Fetching commits bucket for stale RPM package in the snapshots folder", func(t *testing.T) { + defer os.Unsetenv("GITHUB_CHECK_SHA1") + os.Setenv("GITHUB_CHECK_SHA1", "0123456789") + + arch := "x86_64" + extension := "rpm" + fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" + + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/elastic-agent") + assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") + assert.Equal(t, object, "elastic-agent-"+testVersion+"-x86_64.rpm") + }) t.Run("Fetching commits bucket for DEB package", func(t *testing.T) { defer os.Unsetenv("GITHUB_CHECK_SHA1") @@ -99,12 +113,26 @@ func TestGetGCPBucketCoordinates_Commits(t *testing.T) { extension := "deb" fileName := "elastic-agent-" + testVersion + "-amd64.deb" - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension) + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) assert.Equal(t, bucket, "beats-ci-artifacts") assert.Equal(t, prefix, "commits/0123456789") assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-amd64.deb") }) + t.Run("Fetching commits bucket for stale DEB package in the snapshots folder", func(t *testing.T) { + defer os.Unsetenv("GITHUB_CHECK_SHA1") + os.Setenv("GITHUB_CHECK_SHA1", "0123456789") + + arch := "amd64" + extension := "deb" + fileName := "elastic-agent-" + testVersion + "-amd64.deb" + + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/elastic-agent") + assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") + assert.Equal(t, object, "elastic-agent-"+testVersion+"-amd64.deb") + }) t.Run("Fetching commits bucket for TAR package adds OS to fileName and object", func(t *testing.T) { defer os.Unsetenv("GITHUB_CHECK_SHA1") @@ -114,12 +142,26 @@ func TestGetGCPBucketCoordinates_Commits(t *testing.T) { extension := "tar.gz" fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension) + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) assert.Equal(t, bucket, "beats-ci-artifacts") assert.Equal(t, prefix, "commits/0123456789") assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") }) + t.Run("Fetching commits bucket for stale TAR package adds OS to fileName and object in the snapshots folder", func(t *testing.T) { + defer os.Unsetenv("GITHUB_CHECK_SHA1") + os.Setenv("GITHUB_CHECK_SHA1", "0123456789") + + arch := "x86_64" + extension := "tar.gz" + fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" + + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/elastic-agent") + assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") + assert.Equal(t, object, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") + }) } func TestGetGCPBucketCoordinates_CommitsForAPullRequest(t *testing.T) { @@ -135,12 +177,26 @@ func TestGetGCPBucketCoordinates_CommitsForAPullRequest(t *testing.T) { extension := "rpm" fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension) + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) assert.Equal(t, bucket, "beats-ci-artifacts") assert.Equal(t, prefix, "pull-requests/pr-23456") assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-x86_64.rpm") }) + t.Run("Fetching commits bucket for stale RPM package in the snapshots folder", func(t *testing.T) { + defer os.Unsetenv("GITHUB_CHECK_SHA1") + os.Setenv("GITHUB_CHECK_SHA1", "0123456789") + + arch := "x86_64" + extension := "rpm" + fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" + + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/elastic-agent") + assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") + assert.Equal(t, object, "elastic-agent-"+testVersion+"-x86_64.rpm") + }) t.Run("Fetching commits bucket for DEB package", func(t *testing.T) { defer os.Unsetenv("GITHUB_CHECK_SHA1") @@ -150,12 +206,26 @@ func TestGetGCPBucketCoordinates_CommitsForAPullRequest(t *testing.T) { extension := "deb" fileName := "elastic-agent-" + testVersion + "-amd64.deb" - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension) + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) assert.Equal(t, bucket, "beats-ci-artifacts") assert.Equal(t, prefix, "pull-requests/pr-23456") assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-amd64.deb") }) + t.Run("Fetching commits bucket for stale DEB package in the snapshots folder", func(t *testing.T) { + defer os.Unsetenv("GITHUB_CHECK_SHA1") + os.Setenv("GITHUB_CHECK_SHA1", "0123456789") + + arch := "amd64" + extension := "deb" + fileName := "elastic-agent-" + testVersion + "-amd64.deb" + + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/elastic-agent") + assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") + assert.Equal(t, object, "elastic-agent-"+testVersion+"-amd64.deb") + }) t.Run("Fetching commits bucket for TAR package adds OS to fileName and object", func(t *testing.T) { defer os.Unsetenv("GITHUB_CHECK_SHA1") @@ -165,12 +235,26 @@ func TestGetGCPBucketCoordinates_CommitsForAPullRequest(t *testing.T) { extension := "tar.gz" fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension) + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) assert.Equal(t, bucket, "beats-ci-artifacts") assert.Equal(t, prefix, "pull-requests/pr-23456") assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") }) + t.Run("Fetching commits bucket for stale TAR package adds OS to fileName and object in the snapshots folder", func(t *testing.T) { + defer os.Unsetenv("GITHUB_CHECK_SHA1") + os.Setenv("GITHUB_CHECK_SHA1", "0123456789") + + arch := "x86_64" + extension := "tar.gz" + fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" + + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/elastic-agent") + assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") + assert.Equal(t, object, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") + }) } func TestGetGCPBucketCoordinates_PullRequests(t *testing.T) { @@ -183,36 +267,69 @@ func TestGetGCPBucketCoordinates_PullRequests(t *testing.T) { extension := "rpm" fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension) + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") assert.Equal(t, bucket, "beats-ci-artifacts") assert.Equal(t, prefix, "pull-requests/pr-23456") assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-x86_64.rpm") }) + t.Run("Fetching commits bucket for stale RPM package in the snapshots folder", func(t *testing.T) { + arch := "x86_64" + extension := "rpm" + fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" + + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) + assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/elastic-agent") + assert.Equal(t, object, "elastic-agent-"+testVersion+"-x86_64.rpm") + }) t.Run("Fetching commits bucket for DEB package", func(t *testing.T) { arch := "amd64" extension := "deb" fileName := "elastic-agent-" + testVersion + "-amd64.deb" - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension) + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") assert.Equal(t, bucket, "beats-ci-artifacts") assert.Equal(t, prefix, "pull-requests/pr-23456") assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-amd64.deb") }) + t.Run("Fetching commits bucket for stale DEB package in the snapshots folder", func(t *testing.T) { + arch := "amd64" + extension := "deb" + fileName := "elastic-agent-" + testVersion + "-amd64.deb" + + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) + assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/elastic-agent") + assert.Equal(t, object, "elastic-agent-"+testVersion+"-amd64.deb") + }) t.Run("Fetching commits bucket for TAR package adds OS to fileName and object", func(t *testing.T) { arch := "x86_64" extension := "tar.gz" fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension) + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") assert.Equal(t, bucket, "beats-ci-artifacts") assert.Equal(t, prefix, "pull-requests/pr-23456") assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") }) + t.Run("Fetching commits bucket for stale TAR package adds OS to fileName and object in the snapshots folder", func(t *testing.T) { + arch := "x86_64" + extension := "tar.gz" + fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" + + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) + assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/elastic-agent") + assert.Equal(t, object, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") + }) } func TestGetGCPBucketCoordinates_Snapshots(t *testing.T) { @@ -225,7 +342,18 @@ func TestGetGCPBucketCoordinates_Snapshots(t *testing.T) { extension := "rpm" fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension) + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/elastic-agent") + assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") + assert.Equal(t, object, "elastic-agent-"+testVersion+"-x86_64.rpm") + }) + t.Run("Fetching commits bucket for stale RPM package", func(t *testing.T) { + arch := "x86_64" + extension := "rpm" + fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" + + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) assert.Equal(t, bucket, "beats-ci-artifacts") assert.Equal(t, prefix, "snapshots/elastic-agent") assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") @@ -237,7 +365,18 @@ func TestGetGCPBucketCoordinates_Snapshots(t *testing.T) { extension := "deb" fileName := "elastic-agent-" + testVersion + "-amd64.deb" - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension) + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/elastic-agent") + assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") + assert.Equal(t, object, "elastic-agent-"+testVersion+"-amd64.deb") + }) + t.Run("Fetching commits bucket for stale DEB package", func(t *testing.T) { + arch := "amd64" + extension := "deb" + fileName := "elastic-agent-" + testVersion + "-amd64.deb" + + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) assert.Equal(t, bucket, "beats-ci-artifacts") assert.Equal(t, prefix, "snapshots/elastic-agent") assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") @@ -249,7 +388,18 @@ func TestGetGCPBucketCoordinates_Snapshots(t *testing.T) { extension := "tar.gz" fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension) + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/elastic-agent") + assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") + assert.Equal(t, object, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") + }) + t.Run("Fetching commits bucket for stale TAR package adds OS to fileName and object", func(t *testing.T) { + arch := "x86_64" + extension := "tar.gz" + fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" + + newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) assert.Equal(t, bucket, "beats-ci-artifacts") assert.Equal(t, prefix, "snapshots/elastic-agent") assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") From f695d29ae1b78a0a635f0efc101e5f2f17877926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 8 Feb 2021 17:38:08 +0100 Subject: [PATCH 2/2] chore: refactor build artifacts methods (#689) * feat: use local beats directory for elastic-agent installers * docs: document BEATS_LOCAL_PATH variable * chore: check that the local file exists * chore: add unit tests for the BEATS_LOCAL_PATH variable * chore: simplify logic after test coverage * chore: enrich test name * chore: rename test * chore: move default values to the end of the descriptions * chore: add a helper method to build artifact names * chore: build artifact name only once * chore: pass the entire filename to calculate the GCP bucket coordinates * chore: simplify method signature, as the file name is not changed within the method * chore: use buildArtifactName method * chore: pass filename to helper methods to avoid unnecessary calculations * chore: rename variable for consistency * chore: add license headers * chore: use lowercase comparison for artifact name * chore: build artifact changes from Beats CI to Elastic's artifactory * fix: support building artifact names from local Local builds uses the same as the CI * WIP * fix: rename docker-ubi8 installer key * chore: pass a fallback version when building artifact name In the case we are using a PR , where the version is "pr-12345", then we need to use the base version of the product * chore: remove log * fix: do not override artifact name in Docker installer * feat: support consuming the docker images from snapshots or local in standalone mode * chore: add unit tests for fetching docker images from local Beats repo * chore: apply version to Docker installer * chore: support loading metricbeat image from local repository * chore: exrtact loadImage method to docker helper * feat: support consuming CI artifacts in metricbeat * fix: remove non-existent field from logrus log * chore: extract download logic to helper methods We are discarding the stale parameter, because the stale version is set when the installer is required, so we simply download what is requested * chore: fix typo * chore: check agent version before setting it * chore: use the docker client to load the image instead of the docker binary * chore: add unit tests for check method * chore: pass fallback version to check method * fix: properly calculate version for PRs * chore: move PR check logic to a method * chore: make test independent of maintenance branch # Conflicts: # e2e/_suites/fleet/services_test.go --- cli/docker/docker.go | 34 ++ e2e/_suites/fleet/fleet.go | 11 +- e2e/_suites/fleet/ingest-manager_test.go | 25 +- e2e/_suites/fleet/installers.go | 100 ++++- e2e/_suites/fleet/services.go | 242 +++++------ e2e/_suites/fleet/services_test.go | 389 ++---------------- e2e/_suites/fleet/stand-alone.go | 12 + e2e/_suites/metricbeat/metricbeat_test.go | 21 +- ...t-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz | 0 ...8-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz | 0 e2e/utils.go | 185 +++++++-- e2e/utils_test.go | 318 ++++++++++++++ 12 files changed, 777 insertions(+), 560 deletions(-) create mode 100644 e2e/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz create mode 100644 e2e/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-ubi8-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz diff --git a/cli/docker/docker.go b/cli/docker/docker.go index c9463323a8..b7e5a0d40b 100644 --- a/cli/docker/docker.go +++ b/cli/docker/docker.go @@ -6,7 +6,10 @@ package docker import ( "bytes" + "compress/gzip" "context" + "os" + "path/filepath" "strings" "github.com/docker/docker/api/types" @@ -172,6 +175,37 @@ func RemoveContainer(containerName string) error { return nil } +// LoadImage loads a TAR file in the local docker engine +func LoadImage(imagePath string) error { + fileNamePath, err := filepath.Abs(imagePath) + if err != nil { + return err + } + + _, err = os.Stat(fileNamePath) + if err != nil || os.IsNotExist(err) { + return err + } + + dockerClient := getDockerClient() + file, err := os.Open(imagePath) + + input, err := gzip.NewReader(file) + imageLoadResponse, err := dockerClient.ImageLoad(context.Background(), input, false) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "image": fileNamePath, + }).Error("Could not load the Docker image.") + return err + } + + log.WithFields(log.Fields{ + "response": imageLoadResponse, + }).Debug("Docker image loaded successfully") + return nil +} + // RemoveDevNetwork removes the developer network func RemoveDevNetwork() error { dockerClient := getDockerClient() diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index a92c8e2518..68e246989a 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -167,7 +167,7 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver // prepare installer for stale version if fts.Version != agentVersionBackup { - i := GetElasticAgentInstaller(image, installerType, fts.Version, true) + i := GetElasticAgentInstaller(image, installerType, fts.Version) fts.Installers[fmt.Sprintf("%s-%s-%s", image, installerType, version)] = i } @@ -379,11 +379,10 @@ func (fts *FleetTestSuite) theAgentIsListedInFleetWithStatus(desiredStatus strin // the agent is not listed in Fleet if desiredStatus == "offline" || desiredStatus == "inactive" { log.WithFields(log.Fields{ - "isAgentInStatus": isAgentInStatus, - "elapsedTime": exp.GetElapsedTime(), - "hostname": fts.Hostname, - "retries": retryCount, - "status": desiredStatus, + "elapsedTime": exp.GetElapsedTime(), + "hostname": fts.Hostname, + "retries": retryCount, + "status": desiredStatus, }).Info("The Agent is not present in Fleet, as expected") return nil } else if desiredStatus == "online" { diff --git a/e2e/_suites/fleet/ingest-manager_test.go b/e2e/_suites/fleet/ingest-manager_test.go index fe61fa2341..89735cc08b 100644 --- a/e2e/_suites/fleet/ingest-manager_test.go +++ b/e2e/_suites/fleet/ingest-manager_test.go @@ -96,10 +96,12 @@ func setUpSuite() { imts = IngestManagerTestSuite{ Fleet: &FleetTestSuite{ Installers: map[string]ElasticAgentInstaller{ - "centos-systemd-" + agentVersion: GetElasticAgentInstaller("centos", "systemd", agentVersion, false), - "centos-tar-" + agentVersion: GetElasticAgentInstaller("centos", "tar", agentVersion, false), - "debian-systemd-" + agentVersion: GetElasticAgentInstaller("debian", "systemd", agentVersion, false), - "debian-tar-" + agentVersion: GetElasticAgentInstaller("debian", "tar", agentVersion, false), + "centos-systemd-" + agentVersion: GetElasticAgentInstaller("centos", "systemd", agentVersion), + "centos-tar-" + agentVersion: GetElasticAgentInstaller("centos", "tar", agentVersion), + "debian-systemd-" + agentVersion: GetElasticAgentInstaller("debian", "systemd", agentVersion), + "debian-tar-" + agentVersion: GetElasticAgentInstaller("debian", "tar", agentVersion), + "docker-default-" + agentVersion: GetElasticAgentInstaller("docker", "default", agentVersion), + "docker-ubi8-" + agentVersion: GetElasticAgentInstaller("docker", "ubi8", agentVersion), }, }, StandAlone: &StandAloneTestSuite{}, @@ -231,21 +233,6 @@ func (imts *IngestManagerTestSuite) processStateOnTheHost(process string, state return checkProcessStateOnTheHost(containerName, process, state) } -// checkElasticAgentVersion returns a fallback version (agentVersionBase) if the version set by the environment is empty -func checkElasticAgentVersion(version string) string { - environmentVersion := os.Getenv("ELASTIC_AGENT_VERSION") - - if environmentVersion == "" { - return agentVersionBase - } - - if strings.HasPrefix(strings.ToLower(environmentVersion), "pr-") { - return agentVersionBase - } - - return version -} - // name of the container for the service: // we are using the Docker client instead of docker-compose // because it does not support returning the output of a diff --git a/e2e/_suites/fleet/installers.go b/e2e/_suites/fleet/installers.go index 75589ea2e5..40e810b02c 100644 --- a/e2e/_suites/fleet/installers.go +++ b/e2e/_suites/fleet/installers.go @@ -1,8 +1,13 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package main import ( "fmt" + "github.com/elastic/e2e-testing/cli/docker" log "github.com/sirupsen/logrus" ) @@ -98,6 +103,85 @@ func (i *DEBPackage) Uninstall() error { return nil } +// DockerPackage implements operations for a DEB installer +type DockerPackage struct { + BasePackage + installerPath string + ubi8 bool + // optional fields + arch string + artifact string + OS string + version string +} + +// NewDockerPackage creates an instance for the Docker installer +func NewDockerPackage(binaryName string, profile string, image string, service string, installerPath string, ubi8 bool) *DockerPackage { + return &DockerPackage{ + BasePackage: BasePackage{ + binaryName: binaryName, + image: image, + profile: profile, + service: service, + }, + installerPath: installerPath, + ubi8: ubi8, + } +} + +// Install installs a Docker package +func (i *DockerPackage) Install(containerName string, token string) error { + log.Trace("No install commands for Docker packages") + return nil +} + +// InstallCerts installs the certificates for a Docker package +func (i *DockerPackage) InstallCerts() error { + log.Trace("No install certs commands for Docker packages") + return nil +} + +// Preinstall executes operations before installing a Docker package +func (i *DockerPackage) Preinstall() error { + return docker.LoadImage(i.installerPath) +} + +// Postinstall executes operations after installing a Docker package +func (i *DockerPackage) Postinstall() error { + log.Trace("No postinstall commands for Docker packages") + return nil +} + +// Uninstall uninstalls a Docker package +func (i *DockerPackage) Uninstall() error { + log.Trace("No uninstall commands for Docker packages") + return nil +} + +// WithArch sets the architecture +func (i *DockerPackage) WithArch(arch string) *DockerPackage { + i.arch = arch + return i +} + +// WithArtifact sets the artifact +func (i *DockerPackage) WithArtifact(artifact string) *DockerPackage { + i.artifact = artifact + return i +} + +// WithOS sets the OS +func (i *DockerPackage) WithOS(OS string) *DockerPackage { + i.OS = OS + return i +} + +// WithVersion sets the version +func (i *DockerPackage) WithVersion(version string) *DockerPackage { + i.version = version + return i +} + // RPMPackage implements operations for a RPM installer type RPMPackage struct { BasePackage @@ -157,7 +241,6 @@ type TARPackage struct { arch string artifact string OS string - stale bool version string } @@ -215,15 +298,10 @@ func (i *TARPackage) Preinstall() error { return err } - version := i.version - if !i.stale { - version = checkElasticAgentVersion(i.version) - } - // simplify layout cmds := [][]string{ []string{"rm", "-fr", "/elastic-agent"}, - []string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, version, i.OS, i.arch), "/elastic-agent"}, + []string{"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, i.version, i.OS, i.arch), "/elastic-agent"}, } for _, cmd := range cmds { err = execCommandInService(i.profile, i.image, i.service, cmd, false) @@ -233,7 +311,7 @@ func (i *TARPackage) Preinstall() error { "error": err, "image": i.image, "service": i.service, - "version": version, + "version": i.version, }).Error("Could not extract agent package in the box") return err @@ -243,12 +321,6 @@ func (i *TARPackage) Preinstall() error { return nil } -// Stale sets the stale state -func (i *TARPackage) Stale(stale bool) *TARPackage { - i.stale = stale - return i -} - // Uninstall uninstalls a TAR package func (i *TARPackage) Uninstall() error { args := []string{"-f"} diff --git a/e2e/_suites/fleet/services.go b/e2e/_suites/fleet/services.go index d9b5b2eecf..abb35479fa 100644 --- a/e2e/_suites/fleet/services.go +++ b/e2e/_suites/fleet/services.go @@ -1,24 +1,19 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package main import ( "context" "fmt" - "os" - "path" "strings" - "time" "github.com/elastic/e2e-testing/cli/docker" - "github.com/elastic/e2e-testing/cli/shell" "github.com/elastic/e2e-testing/e2e" log "github.com/sirupsen/logrus" ) -// to avoid downloading the same artifacts, we are adding this map to cache the URL of the downloaded binaries, using as key -// the URL of the artifact. If another installer is trying to download the same URL, it will return the location of the -// already downloaded artifact. -var binariesCache = map[string]string{} - // ElasticAgentInstaller represents how to install an agent, depending of the box type type ElasticAgentInstaller struct { artifactArch string // architecture of the artifact @@ -166,84 +161,17 @@ func runElasticAgentCommand(profile string, image string, service string, proces // to be used will be defined by the local snapshot produced by the local build. // Else, if the environment variable BEATS_USE_CI_SNAPSHOTS is set, then the artifact // to be downloaded will be defined by the latest snapshot produced by the Beats CI. -func downloadAgentBinary(artifact string, version string, OS string, arch string, extension string, stale bool) (string, string, error) { - fileName := fmt.Sprintf("%s-%s-%s.%s", artifact, version, arch, extension) - - beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") - if beatsLocalPath != "" { - distributions := path.Join(beatsLocalPath, "x-pack", "elastic-agent", "build", "distributions") - log.Debugf("Using local snapshots for the Elastic Agent: %s", distributions) - - if extension == "tar.gz" { - fileName = fmt.Sprintf("%s-%s-%s-%s.%s", artifact, version, OS, arch, extension) - } - - fileNamePath := path.Join(distributions, fileName) - _, err := os.Stat(fileNamePath) - if err != nil || os.IsNotExist(err) { - return fileName, fileNamePath, err - } - - return fileName, fileNamePath, err - } - - handleDownload := func(URL string, fileName string) (string, string, error) { - if val, ok := binariesCache[URL]; ok { - log.WithFields(log.Fields{ - "URL": URL, - "path": val, - }).Debug("Retrieving binary from local cache") - return fileName, val, nil - } - - filePath, err := e2e.DownloadFile(URL) - if err != nil { - return fileName, filePath, err - } - - binariesCache[URL] = filePath - - return fileName, filePath, nil - } - - if downloadURL, exists := os.LookupEnv("ELASTIC_AGENT_DOWNLOAD_URL"); exists { - return handleDownload(downloadURL, fileName) - } - - var downloadURL string - var err error - - useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS") - if useCISnapshots { - log.Debug("Using CI snapshots for the Elastic Agent") - - bucketFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, stale) - - maxTimeout := time.Duration(timeoutFactor) * time.Minute - - downloadURL, err = e2e.GetObjectURLFromBucket(bucket, prefix, object, maxTimeout) - if err != nil { - return "", "", err - } - - return handleDownload(downloadURL, bucketFileName) - } - - downloadVersion := version - if !stale { - downloadVersion = checkElasticAgentVersion(version) - } - - downloadURL, err = e2e.GetElasticArtifactURL(artifact, downloadVersion, OS, arch, extension) +func downloadAgentBinary(artifactName string, artifact string, version string) (string, error) { + imagePath, err := e2e.FetchBeatsBinary(artifactName, artifact, version, agentVersionBase, timeoutFactor, true) if err != nil { - return "", "", err + return "", err } - return handleDownload(downloadURL, fileName) + return imagePath, nil } // GetElasticAgentInstaller returns an installer from a docker image -func GetElasticAgentInstaller(image string, installerType string, version string, stale bool) ElasticAgentInstaller { +func GetElasticAgentInstaller(image string, installerType string, version string) ElasticAgentInstaller { log.WithFields(log.Fields{ "image": image, "installer": installerType, @@ -252,13 +180,17 @@ func GetElasticAgentInstaller(image string, installerType string, version string var installer ElasticAgentInstaller var err error if "centos" == image && "tar" == installerType { - installer, err = newTarInstaller("centos", "latest", version, stale) + installer, err = newTarInstaller("centos", "latest", version) } else if "centos" == image && "systemd" == installerType { - installer, err = newCentosInstaller("centos", "latest", version, stale) + installer, err = newCentosInstaller("centos", "latest", version) } else if "debian" == image && "tar" == installerType { - installer, err = newTarInstaller("debian", "stretch", version, stale) + installer, err = newTarInstaller("debian", "stretch", version) } else if "debian" == image && "systemd" == installerType { - installer, err = newDebianInstaller("debian", "stretch", version, stale) + installer, err = newDebianInstaller("debian", "stretch", version) + } else if "docker" == image && "default" == installerType { + installer, err = newDockerInstaller(false, version) + } else if "docker" == image && "ubi8" == installerType { + installer, err = newDockerInstaller(true, version) } else { log.WithFields(log.Fields{ "image": image, @@ -277,55 +209,12 @@ func GetElasticAgentInstaller(image string, installerType string, version string return installer } -// getGCPBucketCoordinates it calculates the bucket path in GCP -func getGCPBucketCoordinates(fileName string, artifact string, version string, OS string, arch string, extension string, stale bool) (string, string, string, string) { - if extension == "tar.gz" { - fileName = fmt.Sprintf("%s-%s-%s-%s.%s", artifact, version, OS, arch, extension) - } - - bucket := "beats-ci-artifacts" - prefix := fmt.Sprintf("snapshots/%s", artifact) - object := fileName - newFileName := fileName - - // the commit SHA will identify univocally the artifact in the GCP storage bucket - commitSHA := shell.GetEnv("GITHUB_CHECK_SHA1", "") - if commitSHA != "" { - prefix = fmt.Sprintf("commits/%s", commitSHA) - object = artifact + "/" + fileName - } - - // we are setting a version from a pull request: the version of the artifact will be kept as the base one - // i.e. /pull-requests/pr-21100/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-x86_64.rpm - // i.e. /pull-requests/pr-21100/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-amd64.deb - // i.e. /pull-requests/pr-21100/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz - if strings.HasPrefix(strings.ToLower(version), "pr-") { - newFileName = fmt.Sprintf("%s-%s-%s.%s", artifact, agentVersionBase, arch, extension) - if extension == "tar.gz" { - newFileName = fmt.Sprintf("%s-%s-%s-%s.%s", artifact, agentVersionBase, OS, arch, extension) - } - log.WithFields(log.Fields{ - "agentVersion": agentVersionBase, - "PR": version, - }).Debug("Using CI snapshots for a pull request") - prefix = fmt.Sprintf("pull-requests/%s", version) - object = fmt.Sprintf("%s/%s", artifact, newFileName) - } - - if stale { - prefix = fmt.Sprintf("snapshots/%s", artifact) - object = newFileName - } - - return newFileName, bucket, prefix, object -} - func isSystemdBased(image string) bool { return strings.HasSuffix(image, "-systemd") } // newCentosInstaller returns an instance of the Centos installer for a specific version -func newCentosInstaller(image string, tag string, version string, stale bool) (ElasticAgentInstaller, error) { +func newCentosInstaller(image string, tag string, version string) (ElasticAgentInstaller, error) { image = image + "-systemd" // we want to consume systemd boxes service := image profile := FleetProfileName @@ -336,7 +225,8 @@ func newCentosInstaller(image string, tag string, version string, stale bool) (E arch := "x86_64" extension := "rpm" - binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension, stale) + binaryName := e2e.BuildArtifactName(artifact, version, agentVersionBase, os, arch, extension, false) + binaryPath, err := downloadAgentBinary(binaryName, artifact, version) if err != nil { log.WithFields(log.Fields{ "artifact": artifact, @@ -389,7 +279,7 @@ func newCentosInstaller(image string, tag string, version string, stale bool) (E } // newDebianInstaller returns an instance of the Debian installer for a specific version -func newDebianInstaller(image string, tag string, version string, stale bool) (ElasticAgentInstaller, error) { +func newDebianInstaller(image string, tag string, version string) (ElasticAgentInstaller, error) { image = image + "-systemd" // we want to consume systemd boxes service := image profile := FleetProfileName @@ -400,7 +290,8 @@ func newDebianInstaller(image string, tag string, version string, stale bool) (E arch := "amd64" extension := "deb" - binaryName, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension, stale) + binaryName := e2e.BuildArtifactName(artifact, version, agentVersionBase, os, arch, extension, false) + binaryPath, err := downloadAgentBinary(binaryName, artifact, version) if err != nil { log.WithFields(log.Fields{ "artifact": artifact, @@ -452,8 +343,83 @@ func newDebianInstaller(image string, tag string, version string, stale bool) (E }, nil } +// newDockerInstaller returns an instance of the Docker installer +func newDockerInstaller(ubi8 bool, version string) (ElasticAgentInstaller, error) { + image := "elastic-agent" + service := image + profile := FleetProfileName + + // extract the agent in the box, as it's mounted as a volume + artifact := "elastic-agent" + + artifactName := artifact + if ubi8 { + artifactName = "elastic-agent-ubi8" + } + + os := "linux" + arch := "amd64" + extension := "tar.gz" + + binaryName := e2e.BuildArtifactName(artifactName, version, agentVersionBase, os, arch, extension, true) + binaryPath, err := downloadAgentBinary(binaryName, artifact, version) + if err != nil { + log.WithFields(log.Fields{ + "artifact": artifact, + "version": version, + "os": os, + "arch": arch, + "extension": extension, + "error": err, + }).Error("Could not download the binary for the agent") + return ElasticAgentInstaller{}, err + } + + commitFile := ".elastic-agent.active.commit" + homeDir := "/usr/share/elastic-agent" + binDir := "/usr/share/elastic-agent/data/elastic-agent-%s/" + + enrollFn := func(token string) error { + return nil + } + + installerPackage := NewDockerPackage(binaryName, profile, image, service, binaryPath, ubi8). + WithArch(arch). + WithArtifact(artifact). + WithOS(os). + WithVersion(e2e.CheckPRVersion(version, agentVersionBase)) // sanitize version + + return ElasticAgentInstaller{ + artifactArch: arch, + artifactExtension: extension, + artifactName: artifact, + artifactOS: os, + artifactVersion: version, + binDir: binDir, + commitFile: commitFile, + EnrollFn: enrollFn, + homeDir: homeDir, + image: image, + InstallFn: installerPackage.Install, + InstallCertsFn: installerPackage.InstallCerts, + installerType: "docker", + logFile: "elastic-agent-json.log", + logsDir: binDir + "logs/", + name: binaryName, + path: binaryPath, + PostInstallFn: installerPackage.Postinstall, + PreInstallFn: installerPackage.Preinstall, + processName: ElasticAgentProcessName, + profile: profile, + service: service, + tag: version, + UninstallFn: installerPackage.Uninstall, + workingDir: "/usr/share/elastic-agent/", + }, nil +} + // newTarInstaller returns an instance of the Debian installer for a specific version -func newTarInstaller(image string, tag string, version string, stale bool) (ElasticAgentInstaller, error) { +func newTarInstaller(image string, tag string, version string) (ElasticAgentInstaller, error) { image = image + "-systemd" // we want to consume systemd boxes service := image profile := FleetProfileName @@ -464,7 +430,8 @@ func newTarInstaller(image string, tag string, version string, stale bool) (Elas arch := "x86_64" extension := "tar.gz" - tarFile, binaryPath, err := downloadAgentBinary(artifact, version, os, arch, extension, stale) + binaryName := e2e.BuildArtifactName(artifact, version, agentVersionBase, os, arch, extension, false) + binaryPath, err := downloadAgentBinary(binaryName, artifact, version) if err != nil { log.WithFields(log.Fields{ "artifact": artifact, @@ -488,12 +455,11 @@ func newTarInstaller(image string, tag string, version string, stale bool) (Elas } // - installerPackage := NewTARPackage(tarFile, profile, image, service). - Stale(stale). + installerPackage := NewTARPackage(binaryName, profile, image, service). WithArch(arch). WithArtifact(artifact). WithOS(os). - WithVersion(version) + WithVersion(e2e.CheckPRVersion(version, agentVersionBase)) // sanitize version return ElasticAgentInstaller{ artifactArch: arch, @@ -511,7 +477,7 @@ func newTarInstaller(image string, tag string, version string, stale bool) (Elas installerType: "tar", logFile: "elastic-agent.log", logsDir: "/opt/Elastic/Agent/", - name: tarFile, + name: binaryName, path: binaryPath, PostInstallFn: installerPackage.Postinstall, PreInstallFn: installerPackage.Preinstall, diff --git a/e2e/_suites/fleet/services_test.go b/e2e/_suites/fleet/services_test.go index cbaf1abdf7..12e4d87f43 100644 --- a/e2e/_suites/fleet/services_test.go +++ b/e2e/_suites/fleet/services_test.go @@ -1,30 +1,29 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package main import ( "os" "path" + "path/filepath" "testing" "github.com/stretchr/testify/assert" ) -var testVersion = agentVersionBase - func TestDownloadAgentBinary(t *testing.T) { artifact := "elastic-agent" beatsDir := path.Join("..", "..", "_testresources", "beats") - distributionsDir := path.Join(beatsDir, "x-pack", "elastic-agent", "build", "distributions") - OS := "linux" + distributionsDir, _ := filepath.Abs(path.Join(beatsDir, "x-pack", "elastic-agent", "build", "distributions")) version := "8.0.0-SNAPSHOT" t.Run("Fetching non-existent binary from local Beats dir throws an error", func(t *testing.T) { defer os.Unsetenv("BEATS_LOCAL_PATH") os.Setenv("BEATS_LOCAL_PATH", beatsDir) - arch := "foo_arch" - extension := "foo_ext" - - _, _, err := downloadAgentBinary(artifact, version, OS, arch, extension) + _, err := downloadAgentBinary("foo_fileName", artifact, version) assert.NotNil(t, err) }) @@ -32,377 +31,59 @@ func TestDownloadAgentBinary(t *testing.T) { defer os.Unsetenv("BEATS_LOCAL_PATH") os.Setenv("BEATS_LOCAL_PATH", beatsDir) - arch := "x86_64" - extension := "rpm" - expectedFileName := "elastic-agent-8.0.0-SNAPSHOT-x86_64.rpm" + artifactName := "elastic-agent-8.0.0-SNAPSHOT-x86_64.rpm" + expectedFilePath := path.Join(distributionsDir, artifactName) - newFileName, downloadedFilePath, err := downloadAgentBinary(artifact, version, OS, arch, extension) + downloadedFilePath, err := downloadAgentBinary(artifactName, artifact, version) assert.Nil(t, err) - assert.Equal(t, newFileName, expectedFileName) - assert.Equal(t, downloadedFilePath, path.Join(distributionsDir, expectedFileName)) + assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching DEB binary from local Beats dir", func(t *testing.T) { defer os.Unsetenv("BEATS_LOCAL_PATH") os.Setenv("BEATS_LOCAL_PATH", beatsDir) - arch := "amd64" - extension := "deb" - expectedFileName := "elastic-agent-8.0.0-SNAPSHOT-amd64.deb" + artifactName := "elastic-agent-8.0.0-SNAPSHOT-amd64.deb" + expectedFilePath := path.Join(distributionsDir, artifactName) - newFileName, downloadedFilePath, err := downloadAgentBinary(artifact, version, OS, arch, extension) + downloadedFilePath, err := downloadAgentBinary(artifactName, artifact, version) assert.Nil(t, err) - assert.Equal(t, newFileName, expectedFileName) - assert.Equal(t, downloadedFilePath, path.Join(distributionsDir, expectedFileName)) + assert.Equal(t, downloadedFilePath, expectedFilePath) }) t.Run("Fetching TAR binary from local Beats dir", func(t *testing.T) { defer os.Unsetenv("BEATS_LOCAL_PATH") os.Setenv("BEATS_LOCAL_PATH", beatsDir) - arch := "amd64" - extension := "tar.gz" - expectedFileName := "elastic-agent-8.0.0-SNAPSHOT-linux-amd64.tar.gz" + artifactName := "elastic-agent-8.0.0-SNAPSHOT-linux-amd64.tar.gz" + expectedFilePath := path.Join(distributionsDir, artifactName) - newFileName, downloadedFilePath, err := downloadAgentBinary(artifact, version, OS, arch, extension) + downloadedFilePath, err := downloadAgentBinary(artifactName, artifact, version) assert.Nil(t, err) - assert.Equal(t, newFileName, expectedFileName) - assert.Equal(t, downloadedFilePath, path.Join(distributionsDir, expectedFileName)) - }) -} - -func TestGetGCPBucketCoordinates_Commits(t *testing.T) { - artifact := "elastic-agent" - version := testVersion - OS := "linux" - - t.Run("Fetching commits bucket for RPM package", func(t *testing.T) { - defer os.Unsetenv("GITHUB_CHECK_SHA1") - os.Setenv("GITHUB_CHECK_SHA1", "0123456789") - - arch := "x86_64" - extension := "rpm" - fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "commits/0123456789") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") - assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-x86_64.rpm") - }) - t.Run("Fetching commits bucket for stale RPM package in the snapshots folder", func(t *testing.T) { - defer os.Unsetenv("GITHUB_CHECK_SHA1") - os.Setenv("GITHUB_CHECK_SHA1", "0123456789") - - arch := "x86_64" - extension := "rpm" - fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-x86_64.rpm") - }) - - t.Run("Fetching commits bucket for DEB package", func(t *testing.T) { - defer os.Unsetenv("GITHUB_CHECK_SHA1") - os.Setenv("GITHUB_CHECK_SHA1", "0123456789") - - arch := "amd64" - extension := "deb" - fileName := "elastic-agent-" + testVersion + "-amd64.deb" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "commits/0123456789") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") - assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-amd64.deb") + assert.Equal(t, downloadedFilePath, expectedFilePath) }) - t.Run("Fetching commits bucket for stale DEB package in the snapshots folder", func(t *testing.T) { - defer os.Unsetenv("GITHUB_CHECK_SHA1") - os.Setenv("GITHUB_CHECK_SHA1", "0123456789") - - arch := "amd64" - extension := "deb" - fileName := "elastic-agent-" + testVersion + "-amd64.deb" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-amd64.deb") - }) - - t.Run("Fetching commits bucket for TAR package adds OS to fileName and object", func(t *testing.T) { - defer os.Unsetenv("GITHUB_CHECK_SHA1") - os.Setenv("GITHUB_CHECK_SHA1", "0123456789") - - arch := "x86_64" - extension := "tar.gz" - fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "commits/0123456789") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - }) - t.Run("Fetching commits bucket for stale TAR package adds OS to fileName and object in the snapshots folder", func(t *testing.T) { - defer os.Unsetenv("GITHUB_CHECK_SHA1") - os.Setenv("GITHUB_CHECK_SHA1", "0123456789") - - arch := "x86_64" - extension := "tar.gz" - fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - }) -} - -func TestGetGCPBucketCoordinates_CommitsForAPullRequest(t *testing.T) { - artifact := "elastic-agent" - version := "pr-23456" - OS := "linux" - - t.Run("Fetching commits bucket for RPM package", func(t *testing.T) { - defer os.Unsetenv("GITHUB_CHECK_SHA1") - os.Setenv("GITHUB_CHECK_SHA1", "0123456789") - - arch := "x86_64" - extension := "rpm" - fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "pull-requests/pr-23456") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") - assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-x86_64.rpm") - }) - t.Run("Fetching commits bucket for stale RPM package in the snapshots folder", func(t *testing.T) { - defer os.Unsetenv("GITHUB_CHECK_SHA1") - os.Setenv("GITHUB_CHECK_SHA1", "0123456789") - - arch := "x86_64" - extension := "rpm" - fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-x86_64.rpm") - }) - - t.Run("Fetching commits bucket for DEB package", func(t *testing.T) { - defer os.Unsetenv("GITHUB_CHECK_SHA1") - os.Setenv("GITHUB_CHECK_SHA1", "0123456789") - - arch := "amd64" - extension := "deb" - fileName := "elastic-agent-" + testVersion + "-amd64.deb" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "pull-requests/pr-23456") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") - assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-amd64.deb") - }) - t.Run("Fetching commits bucket for stale DEB package in the snapshots folder", func(t *testing.T) { - defer os.Unsetenv("GITHUB_CHECK_SHA1") - os.Setenv("GITHUB_CHECK_SHA1", "0123456789") - - arch := "amd64" - extension := "deb" - fileName := "elastic-agent-" + testVersion + "-amd64.deb" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-amd64.deb") - }) - - t.Run("Fetching commits bucket for TAR package adds OS to fileName and object", func(t *testing.T) { - defer os.Unsetenv("GITHUB_CHECK_SHA1") - os.Setenv("GITHUB_CHECK_SHA1", "0123456789") - - arch := "x86_64" - extension := "tar.gz" - fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "pull-requests/pr-23456") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - }) - t.Run("Fetching commits bucket for stale TAR package adds OS to fileName and object in the snapshots folder", func(t *testing.T) { - defer os.Unsetenv("GITHUB_CHECK_SHA1") - os.Setenv("GITHUB_CHECK_SHA1", "0123456789") - - arch := "x86_64" - extension := "tar.gz" - fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - }) -} - -func TestGetGCPBucketCoordinates_PullRequests(t *testing.T) { - artifact := "elastic-agent" - version := "pr-23456" - OS := "linux" - - t.Run("Fetching commits bucket for RPM package", func(t *testing.T) { - arch := "x86_64" - extension := "rpm" - fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "pull-requests/pr-23456") - assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-x86_64.rpm") - }) - t.Run("Fetching commits bucket for stale RPM package in the snapshots folder", func(t *testing.T) { - arch := "x86_64" - extension := "rpm" - fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-x86_64.rpm") - }) - - t.Run("Fetching commits bucket for DEB package", func(t *testing.T) { - arch := "amd64" - extension := "deb" - fileName := "elastic-agent-" + testVersion + "-amd64.deb" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "pull-requests/pr-23456") - assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-amd64.deb") - }) - t.Run("Fetching commits bucket for stale DEB package in the snapshots folder", func(t *testing.T) { - arch := "amd64" - extension := "deb" - fileName := "elastic-agent-" + testVersion + "-amd64.deb" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-amd64.deb") - }) - - t.Run("Fetching commits bucket for TAR package adds OS to fileName and object", func(t *testing.T) { - arch := "x86_64" - extension := "tar.gz" - fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "pull-requests/pr-23456") - assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - }) - t.Run("Fetching commits bucket for stale TAR package adds OS to fileName and object in the snapshots folder", func(t *testing.T) { - arch := "x86_64" - extension := "tar.gz" - fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - }) -} - -func TestGetGCPBucketCoordinates_Snapshots(t *testing.T) { - artifact := "elastic-agent" - version := testVersion - OS := "linux" - - t.Run("Fetching commits bucket for RPM package", func(t *testing.T) { - arch := "x86_64" - extension := "rpm" - fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-x86_64.rpm") - }) - t.Run("Fetching commits bucket for stale RPM package", func(t *testing.T) { - arch := "x86_64" - extension := "rpm" - fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" - - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-x86_64.rpm") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-x86_64.rpm") - }) - - t.Run("Fetching commits bucket for DEB package", func(t *testing.T) { - arch := "amd64" - extension := "deb" - fileName := "elastic-agent-" + testVersion + "-amd64.deb" + t.Run("Fetching Docker binary from local Beats dir", func(t *testing.T) { + defer os.Unsetenv("BEATS_LOCAL_PATH") + os.Setenv("BEATS_LOCAL_PATH", beatsDir) - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-amd64.deb") - }) - t.Run("Fetching commits bucket for stale DEB package", func(t *testing.T) { - arch := "amd64" - extension := "deb" - fileName := "elastic-agent-" + testVersion + "-amd64.deb" + artifactName := "elastic-agent-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz" + expectedFilePath := path.Join(distributionsDir, artifactName) - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-amd64.deb") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-amd64.deb") + downloadedFilePath, err := downloadAgentBinary(artifactName, artifact, version) + assert.Nil(t, err) + assert.Equal(t, downloadedFilePath, expectedFilePath) }) - t.Run("Fetching commits bucket for TAR package adds OS to fileName and object", func(t *testing.T) { - arch := "x86_64" - extension := "tar.gz" - fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" + t.Run("Fetching ubi8 Docker binary from local Beats dir", func(t *testing.T) { + defer os.Unsetenv("BEATS_LOCAL_PATH") + os.Setenv("BEATS_LOCAL_PATH", beatsDir) - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, false) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - }) - t.Run("Fetching commits bucket for stale TAR package adds OS to fileName and object", func(t *testing.T) { - arch := "x86_64" - extension := "tar.gz" - fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" + artifactName := "elastic-agent-ubi8-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz" + expectedFilePath := path.Join(distributionsDir, artifactName) - newFileName, bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, OS, arch, extension, true) - assert.Equal(t, bucket, "beats-ci-artifacts") - assert.Equal(t, prefix, "snapshots/elastic-agent") - assert.Equal(t, newFileName, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") - assert.Equal(t, object, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") + downloadedFilePath, err := downloadAgentBinary(artifactName, artifact, version) + assert.Nil(t, err) + assert.Equal(t, downloadedFilePath, expectedFilePath) }) } diff --git a/e2e/_suites/fleet/stand-alone.go b/e2e/_suites/fleet/stand-alone.go index 42dbfc43bf..08475eb336 100644 --- a/e2e/_suites/fleet/stand-alone.go +++ b/e2e/_suites/fleet/stand-alone.go @@ -14,6 +14,7 @@ import ( "github.com/cucumber/godog" "github.com/elastic/e2e-testing/cli/docker" "github.com/elastic/e2e-testing/cli/services" + shell "github.com/elastic/e2e-testing/cli/shell" "github.com/elastic/e2e-testing/e2e" log "github.com/sirupsen/logrus" ) @@ -62,6 +63,17 @@ func (sats *StandAloneTestSuite) contributeSteps(s *godog.ScenarioContext) { func (sats *StandAloneTestSuite) aStandaloneAgentIsDeployed(image string) error { log.Trace("Deploying an agent to Fleet") + useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS") + beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") + if useCISnapshots || beatsLocalPath != "" { + // load the docker images that were already: + // a. downloaded from the GCP bucket + // b. fetched from the local beats binaries + dockerInstaller := GetElasticAgentInstaller("docker", image, agentVersion) + + dockerInstaller.PreInstallFn() + } + serviceManager := services.NewServiceManager() profileEnv["elasticAgentDockerImageSuffix"] = "" diff --git a/e2e/_suites/metricbeat/metricbeat_test.go b/e2e/_suites/metricbeat/metricbeat_test.go index cb3cc77258..ee8b6d20cf 100644 --- a/e2e/_suites/metricbeat/metricbeat_test.go +++ b/e2e/_suites/metricbeat/metricbeat_test.go @@ -15,6 +15,7 @@ import ( "github.com/cucumber/godog" messages "github.com/cucumber/messages-go/v10" "github.com/elastic/e2e-testing/cli/config" + "github.com/elastic/e2e-testing/cli/docker" "github.com/elastic/e2e-testing/cli/services" "github.com/elastic/e2e-testing/cli/shell" "github.com/elastic/e2e-testing/e2e" @@ -281,9 +282,7 @@ func (mts *MetricbeatTestSuite) installedUsingConfiguration(configuration string mts.Version = metricbeatVersion mts.setIndexName() - if strings.HasPrefix(metricbeatVersion, "pr-") { - metricbeatVersion = metricbeatVersionBase - } + metricbeatVersion = e2e.CheckPRVersion(metricbeatVersion, metricbeatVersionBase) // use master branch for snapshots tag := "v" + metricbeatVersion @@ -313,6 +312,22 @@ func (mts *MetricbeatTestSuite) installedUsingConfiguration(configuration string // runMetricbeatService runs a metricbeat service entity for a service to monitor it func (mts *MetricbeatTestSuite) runMetricbeatService() error { + useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS") + beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") + if useCISnapshots || beatsLocalPath != "" { + artifactName := e2e.BuildArtifactName("metricbeat", mts.Version, metricbeatVersionBase, "linux", "amd64", "tar.gz", true) + + imagePath, err := e2e.FetchBeatsBinary(artifactName, "metricbeat", mts.Version, metricbeatVersionBase, timeoutFactor, false) + if err != nil { + return err + } + + err = docker.LoadImage(imagePath) + if err != nil { + return err + } + } + // this is needed because, in general, the target service (apache, mysql, redis) does not have a healthcheck waitForService := time.Duration(timeoutFactor) * 10 * time.Second if mts.ServiceName == "ceph" { diff --git a/e2e/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz b/e2e/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz new file mode 100644 index 0000000000..e69de29bb2 diff --git a/e2e/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-ubi8-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz b/e2e/_testresources/beats/x-pack/elastic-agent/build/distributions/elastic-agent-ubi8-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz new file mode 100644 index 0000000000..e69de29bb2 diff --git a/e2e/utils.go b/e2e/utils.go index 350cd708fb..001aa4163e 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -13,6 +13,7 @@ import ( "net/http" "os" "path" + "path/filepath" "strings" "time" @@ -24,12 +25,158 @@ import ( log "github.com/sirupsen/logrus" ) +// to avoid downloading the same artifacts, we are adding this map to cache the URL of the downloaded binaries, using as key +// the URL of the artifact. If another installer is trying to download the same URL, it will return the location of the +// already downloaded artifact. +var binariesCache = map[string]string{} + //nolint:unused const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" //nolint:unused var seededRand = rand.New(rand.NewSource(time.Now().UnixNano())) +// BuildArtifactName builds the artifact name from the different coordinates for the artifact +func BuildArtifactName(artifact string, version string, fallbackVersion string, OS string, arch string, extension string, isDocker bool) string { + dockerString := "" + if isDocker { + dockerString = ".docker" + } + + artifactVersion := CheckPRVersion(version, fallbackVersion) + + lowerCaseExtension := strings.ToLower(extension) + + artifactName := fmt.Sprintf("%s-%s-%s-%s%s.%s", artifact, artifactVersion, OS, arch, dockerString, lowerCaseExtension) + if lowerCaseExtension == "deb" || lowerCaseExtension == "rpm" { + artifactName = fmt.Sprintf("%s-%s-%s%s.%s", artifact, artifactVersion, arch, dockerString, lowerCaseExtension) + } + + beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") + if beatsLocalPath != "" && isDocker { + return fmt.Sprintf("%s-%s-%s-%s%s.%s", artifact, artifactVersion, OS, arch, dockerString, lowerCaseExtension) + } + + useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS") + // we detected that the docker name on CI is using a different structure + // CI snapshots on GCP: elastic-agent-$VERSION-linux-amd64.docker.tar.gz + // Elastic's snapshots: elastic-agent-$VERSION-docker-image-linux-amd64.tar.gz + if !useCISnapshots && isDocker { + dockerString = "docker-image" + artifactName = fmt.Sprintf("%s-%s-%s-%s-%s.%s", artifact, artifactVersion, dockerString, OS, arch, lowerCaseExtension) + } + + return artifactName +} + +// CheckPRVersion returns a fallback version if the version comes from a Pull Request (PR) +func CheckPRVersion(version string, fallbackVersion string) string { + if strings.HasPrefix(strings.ToLower(version), "pr-") { + return fallbackVersion + } + + return version +} + +// FetchBeatsBinary it downloads the binary and returns the location of the downloaded file +// If the environment variable BEATS_LOCAL_PATH is set, then the artifact +// to be used will be defined by the local snapshot produced by the local build. +// Else, if the environment variable BEATS_USE_CI_SNAPSHOTS is set, then the artifact +// to be downloaded will be defined by the latest snapshot produced by the Beats CI. +func FetchBeatsBinary(artifactName string, artifact string, version string, fallbackVersion string, timeoutFactor int, xpack bool) (string, error) { + beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") + if beatsLocalPath != "" { + distributions := path.Join(beatsLocalPath, artifact, "build", "distributions") + if xpack { + distributions = path.Join(beatsLocalPath, "x-pack", artifact, "build", "distributions") + } + + log.Debugf("Using local snapshots for the %s: %s", artifact, distributions) + + fileNamePath, _ := filepath.Abs(path.Join(distributions, artifactName)) + _, err := os.Stat(fileNamePath) + if err != nil || os.IsNotExist(err) { + return fileNamePath, err + } + + return fileNamePath, err + } + + handleDownload := func(URL string) (string, error) { + if val, ok := binariesCache[URL]; ok { + log.WithFields(log.Fields{ + "URL": URL, + "path": val, + }).Debug("Retrieving binary from local cache") + return val, nil + } + + filePath, err := DownloadFile(URL) + if err != nil { + return filePath, err + } + + binariesCache[URL] = filePath + + return filePath, nil + } + + var downloadURL string + var err error + + useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS") + if useCISnapshots { + log.Debugf("Using CI snapshots for %s", artifact) + + bucket, prefix, object := getGCPBucketCoordinates(artifactName, artifact, version, fallbackVersion) + + maxTimeout := time.Duration(timeoutFactor) * time.Minute + + downloadURL, err = GetObjectURLFromBucket(bucket, prefix, object, maxTimeout) + if err != nil { + return "", err + } + + return handleDownload(downloadURL) + } + + downloadURL, err = GetElasticArtifactURL(artifactName, artifact, version) + if err != nil { + return "", err + } + + return handleDownload(downloadURL) +} + +// getGCPBucketCoordinates it calculates the bucket path in GCP +func getGCPBucketCoordinates(fileName string, artifact string, version string, fallbackVersion string) (string, string, string) { + bucket := "beats-ci-artifacts" + prefix := fmt.Sprintf("snapshots/%s", artifact) + object := fileName + + // the commit SHA will identify univocally the artifact in the GCP storage bucket + commitSHA := shell.GetEnv("GITHUB_CHECK_SHA1", "") + if commitSHA != "" { + prefix = fmt.Sprintf("commits/%s", commitSHA) + object = artifact + "/" + fileName + } + + // we are setting a version from a pull request: the version of the artifact will be kept as the base one + // i.e. /pull-requests/pr-21100/$THE_BEAT/$THE_BEAT-$VERSION-x86_64.rpm + // i.e. /pull-requests/pr-21100/$THE_BEAT/$THE_BEAT-$VERSION-amd64.deb + // i.e. /pull-requests/pr-21100/$THE_BEAT/$THE_BEAT-$VERSION-linux-x86_64.tar.gz + if strings.HasPrefix(strings.ToLower(version), "pr-") { + log.WithFields(log.Fields{ + "version": fallbackVersion, + "PR": version, + }).Debug("Using CI snapshots for a pull request") + prefix = fmt.Sprintf("pull-requests/%s", version) + object = fmt.Sprintf("%s/%s", artifact, fileName) + } + + return bucket, prefix, object +} + // GetExponentialBackOff returns a preconfigured exponential backoff instance func GetExponentialBackOff(elapsedTime time.Duration) *backoff.ExponentialBackOff { var ( @@ -53,7 +200,7 @@ func GetExponentialBackOff(elapsedTime time.Duration) *backoff.ExponentialBackOf // GetElasticArtifactVersion returns the current version: // 1. Elastic's artifact repository, building the JSON path query based // If the version is a PR, then it will return the version without checking the artifacts API -// i.e. GetElasticArtifactVersion("8.0.0-SNAPSHOT") +// i.e. GetElasticArtifactVersion("$VERSION") // i.e. GetElasticArtifactVersion("pr-22000") func GetElasticArtifactVersion(version string) string { if strings.HasPrefix(strings.ToLower(version), "pr-") { @@ -127,14 +274,12 @@ func GetElasticArtifactVersion(version string) string { return latestVersion } -// GetElasticArtifactURL returns the URL of a released artifact from two possible sources -// on the desired OS, architecture and file extension: -// 1. Observability CI Storage bucket -// 2. Elastic's artifact repository, building the JSON path query based -// i.e. GetElasticArtifactURL("elastic-agent", "8.0.0-SNAPSHOT", "linux", "x86_64", "tar.gz") -// i.e. GetElasticArtifactURL("elastic-agent", "8.0.0-SNAPSHOT", "x86_64", "rpm") -// i.e. GetElasticArtifactURL("elastic-agent", "8.0.0-SNAPSHOT", "amd64", "deb") -func GetElasticArtifactURL(artifact string, version string, operativeSystem string, arch string, extension string) (string, error) { +// GetElasticArtifactURL returns the URL of a released artifact, which its full name is defined in the first argument, +// from Elastic's artifact repository, building the JSON path query based on the full name +// i.e. GetElasticArtifactURL("elastic-agent-$VERSION-amd64.deb", "elastic-agent", "$VERSION") +// i.e. GetElasticArtifactURL("elastic-agent-$VERSION-x86_64.rpm", "elastic-agent","$VERSION") +// i.e. GetElasticArtifactURL("elastic-agent-$VERSION-linux-amd64.tar.gz", "elastic-agent","$VERSION") +func GetElasticArtifactURL(artifactName string, artifact string, version string) (string, error) { exp := GetExponentialBackOff(time.Minute) retryCount := 1 @@ -150,10 +295,8 @@ func GetElasticArtifactURL(artifact string, version string, operativeSystem stri if err != nil { log.WithFields(log.Fields{ "artifact": artifact, + "artifactName": artifactName, "version": version, - "os": operativeSystem, - "arch": arch, - "extension": extension, "error": err, "retry": retryCount, "statusEndpoint": r.URL, @@ -183,26 +326,16 @@ func GetElasticArtifactURL(artifact string, version string, operativeSystem stri jsonParsed, err := gabs.ParseJSON([]byte(body)) if err != nil { log.WithFields(log.Fields{ - "artifact": artifact, - "version": version, - "os": operativeSystem, - "arch": arch, - "extension": extension, + "artifact": artifact, + "artifactName": artifactName, + "version": version, }).Error("Could not parse the response body for the artifact") return "", err } - // elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz - artifactPath := fmt.Sprintf("%s-%s-%s-%s.%s", artifact, version, operativeSystem, arch, extension) - if extension == "deb" || extension == "rpm" { - // elastic-agent-8.0.0-SNAPSHOT-x86_64.rpm - // elastic-agent-8.0.0-SNAPSHOT-amd64.deb - artifactPath = fmt.Sprintf("%s-%s-%s.%s", artifact, version, arch, extension) - } - packagesObject := jsonParsed.Path("packages") // we need to get keys with dots using Search instead of Path - downloadObject := packagesObject.Search(artifactPath) + downloadObject := packagesObject.Search(artifactName) downloadURL := downloadObject.Path("url").Data().(string) return downloadURL, nil diff --git a/e2e/utils_test.go b/e2e/utils_test.go index ec2521149d..0f6ae28a58 100644 --- a/e2e/utils_test.go +++ b/e2e/utils_test.go @@ -10,6 +10,8 @@ import ( gabs "github.com/Jeffail/gabs/v2" ) +var testVersion = "8.0.0-SNAPSHOT" + const bucket = "beats-ci-artifacts" const pullRequests = "pull-requests" const snapshots = "snapshots" @@ -38,6 +40,176 @@ func init() { snapshotsJSON, _ = gabs.ParseJSON([]byte(snapshotsContent)) } +func TestBuildArtifactName(t *testing.T) { + artifact := "elastic-agent" + OS := "linux" + version := "8.0.0-SNAPSHOT" + + t.Run("For RPM", func(t *testing.T) { + arch := "x86_64" + extension := "rpm" + expectedFileName := "elastic-agent-8.0.0-SNAPSHOT-x86_64.rpm" + + artifactName := BuildArtifactName(artifact, version, version, OS, arch, extension, false) + assert.Equal(t, expectedFileName, artifactName) + + artifactName = BuildArtifactName(artifact, version, version, OS, arch, "RPM", false) + assert.Equal(t, expectedFileName, artifactName) + }) + + t.Run("For DEB", func(t *testing.T) { + arch := "amd64" + extension := "deb" + expectedFileName := "elastic-agent-8.0.0-SNAPSHOT-amd64.deb" + + artifactName := BuildArtifactName(artifact, version, version, OS, arch, extension, false) + assert.Equal(t, expectedFileName, artifactName) + + artifactName = BuildArtifactName(artifact, version, version, OS, arch, "DEB", false) + assert.Equal(t, expectedFileName, artifactName) + }) + + t.Run("For TAR", func(t *testing.T) { + arch := "amd64" + extension := "tar.gz" + expectedFileName := "elastic-agent-8.0.0-SNAPSHOT-linux-amd64.tar.gz" + + artifactName := BuildArtifactName(artifact, version, version, OS, arch, extension, false) + assert.Equal(t, expectedFileName, artifactName) + + artifactName = BuildArtifactName(artifact, version, version, OS, arch, "TAR.GZ", false) + assert.Equal(t, expectedFileName, artifactName) + }) + + t.Run("For Docker from Elastic's repository", func(t *testing.T) { + defer os.Unsetenv("BEATS_USE_CI_SNAPSHOTS") + os.Setenv("BEATS_USE_CI_SNAPSHOTS", "false") + + artifact = "elastic-agent" + arch := "amd64" + extension := "tar.gz" + expectedFileName := "elastic-agent-8.0.0-SNAPSHOT-docker-image-linux-amd64.tar.gz" + + artifactName := BuildArtifactName(artifact, version, version, OS, arch, extension, true) + assert.Equal(t, expectedFileName, artifactName) + + artifactName = BuildArtifactName(artifact, version, version, OS, arch, "TAR.GZ", true) + assert.Equal(t, expectedFileName, artifactName) + }) + + t.Run("For Docker UBI8 from Elastic's repository", func(t *testing.T) { + defer os.Unsetenv("BEATS_USE_CI_SNAPSHOTS") + os.Setenv("BEATS_USE_CI_SNAPSHOTS", "false") + + artifact = "elastic-agent-ubi8" + arch := "amd64" + extension := "tar.gz" + expectedFileName := "elastic-agent-ubi8-8.0.0-SNAPSHOT-docker-image-linux-amd64.tar.gz" + + artifactName := BuildArtifactName(artifact, version, version, OS, arch, extension, true) + assert.Equal(t, expectedFileName, artifactName) + + artifactName = BuildArtifactName(artifact, version, version, OS, arch, "TAR.GZ", true) + assert.Equal(t, expectedFileName, artifactName) + }) + + t.Run("For Docker from local repository", func(t *testing.T) { + defer os.Unsetenv("BEATS_LOCAL_PATH") + os.Setenv("BEATS_LOCAL_PATH", "/tmp") + + artifact = "elastic-agent" + arch := "amd64" + extension := "tar.gz" + expectedFileName := "elastic-agent-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz" + + artifactName := BuildArtifactName(artifact, version, version, OS, arch, extension, true) + assert.Equal(t, expectedFileName, artifactName) + + artifactName = BuildArtifactName(artifact, version, version, OS, arch, "TAR.GZ", true) + assert.Equal(t, expectedFileName, artifactName) + }) + + t.Run("For Docker UBI8 from local repository", func(t *testing.T) { + defer os.Unsetenv("BEATS_LOCAL_PATH") + os.Setenv("BEATS_LOCAL_PATH", "/tmp") + + artifact = "elastic-agent-ubi8" + arch := "amd64" + extension := "tar.gz" + expectedFileName := "elastic-agent-ubi8-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz" + + artifactName := BuildArtifactName(artifact, version, version, OS, arch, extension, true) + assert.Equal(t, expectedFileName, artifactName) + + artifactName = BuildArtifactName(artifact, version, version, OS, arch, "TAR.GZ", true) + assert.Equal(t, expectedFileName, artifactName) + }) + + t.Run("For Docker from GCP", func(t *testing.T) { + defer os.Unsetenv("BEATS_USE_CI_SNAPSHOTS") + os.Setenv("BEATS_USE_CI_SNAPSHOTS", "true") + + artifact = "elastic-agent" + arch := "amd64" + extension := "tar.gz" + expectedFileName := "elastic-agent-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz" + + artifactName := BuildArtifactName(artifact, version, version, OS, arch, extension, true) + assert.Equal(t, expectedFileName, artifactName) + + artifactName = BuildArtifactName(artifact, version, version, OS, arch, "TAR.GZ", true) + assert.Equal(t, expectedFileName, artifactName) + }) + + t.Run("For Docker UBI8 from GCP", func(t *testing.T) { + defer os.Unsetenv("BEATS_USE_CI_SNAPSHOTS") + os.Setenv("BEATS_USE_CI_SNAPSHOTS", "true") + + artifact = "elastic-agent-ubi8" + arch := "amd64" + extension := "tar.gz" + expectedFileName := "elastic-agent-ubi8-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz" + + artifactName := BuildArtifactName(artifact, version, version, OS, arch, extension, true) + assert.Equal(t, expectedFileName, artifactName) + + artifactName = BuildArtifactName(artifact, version, version, OS, arch, "TAR.GZ", true) + assert.Equal(t, expectedFileName, artifactName) + }) + + t.Run("For Docker for a Pull Request", func(t *testing.T) { + defer os.Unsetenv("ELASTIC_AGENT_VERSION") + os.Setenv("ELASTIC_AGENT_VERSION", "pr-12345") + + artifact = "elastic-agent" + arch := "amd64" + extension := "tar.gz" + fallbackVersion := "8.0.0-SNAPSHOT" + expectedFileName := "elastic-agent-8.0.0-SNAPSHOT-docker-image-linux-amd64.tar.gz" + + artifactName := BuildArtifactName(artifact, version, fallbackVersion, OS, arch, extension, true) + assert.Equal(t, expectedFileName, artifactName) + + artifactName = BuildArtifactName(artifact, version, fallbackVersion, OS, arch, "TAR.GZ", true) + assert.Equal(t, expectedFileName, artifactName) + }) +} + +func TestCheckPRVersion(t *testing.T) { + t.Run("Checking a version should return the version", func(t *testing.T) { + v := CheckPRVersion(testVersion, testVersion) + + assert.Equal(t, testVersion, v) + }) + + t.Run("A PR should return base version", func(t *testing.T) { + prVersion := "pr-12345" + v := CheckPRVersion(prVersion, testVersion) + + assert.Equal(t, testVersion, v) + }) +} + func TestGetBucketSearchNextPageParam_HasMorePages(t *testing.T) { expectedParam := "&pageToken=foo" @@ -51,6 +223,152 @@ func TestGetBucketSearchNextPageParam_HasNoMorePages(t *testing.T) { assert.True(t, param == "") } +func TestGetGCPBucketCoordinates_Commits(t *testing.T) { + artifact := "elastic-agent" + version := testVersion + + t.Run("Fetching commits bucket for RPM package", func(t *testing.T) { + defer os.Unsetenv("GITHUB_CHECK_SHA1") + os.Setenv("GITHUB_CHECK_SHA1", "0123456789") + + fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" + + bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, testVersion) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "commits/0123456789") + assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-x86_64.rpm") + }) + + t.Run("Fetching commits bucket for DEB package", func(t *testing.T) { + defer os.Unsetenv("GITHUB_CHECK_SHA1") + os.Setenv("GITHUB_CHECK_SHA1", "0123456789") + + fileName := "elastic-agent-" + testVersion + "-amd64.deb" + + bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, testVersion) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "commits/0123456789") + assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-amd64.deb") + }) + + t.Run("Fetching commits bucket for TAR package adds OS to fileName and object", func(t *testing.T) { + defer os.Unsetenv("GITHUB_CHECK_SHA1") + os.Setenv("GITHUB_CHECK_SHA1", "0123456789") + + fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" + + bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, testVersion) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "commits/0123456789") + assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") + }) +} + +func TestGetGCPBucketCoordinates_CommitsForAPullRequest(t *testing.T) { + artifact := "elastic-agent" + version := "pr-23456" + + t.Run("Fetching pull request bucket for RPM package", func(t *testing.T) { + defer os.Unsetenv("GITHUB_CHECK_SHA1") + os.Setenv("GITHUB_CHECK_SHA1", "0123456789") + + fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" + + bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, testVersion) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "pull-requests/pr-23456") + assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-x86_64.rpm") + }) + + t.Run("Fetching pull request bucket for DEB package", func(t *testing.T) { + defer os.Unsetenv("GITHUB_CHECK_SHA1") + os.Setenv("GITHUB_CHECK_SHA1", "0123456789") + + fileName := "elastic-agent-" + testVersion + "-amd64.deb" + + bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, testVersion) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "pull-requests/pr-23456") + assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-amd64.deb") + }) + + t.Run("Fetching pull request bucket for TAR package adds OS to fileName and object", func(t *testing.T) { + defer os.Unsetenv("GITHUB_CHECK_SHA1") + os.Setenv("GITHUB_CHECK_SHA1", "0123456789") + + fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" + + bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, testVersion) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "pull-requests/pr-23456") + assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") + }) +} + +func TestGetGCPBucketCoordinates_PullRequests(t *testing.T) { + artifact := "elastic-agent" + version := "pr-23456" + + t.Run("Fetching pull request bucket for RPM package", func(t *testing.T) { + fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" + + bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, testVersion) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "pull-requests/pr-23456") + assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-x86_64.rpm") + }) + + t.Run("Fetching pull request bucket for DEB package", func(t *testing.T) { + fileName := "elastic-agent-" + testVersion + "-amd64.deb" + + bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, testVersion) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "pull-requests/pr-23456") + assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-amd64.deb") + }) + + t.Run("Fetching pull request bucket for TAR package adds OS to fileName and object", func(t *testing.T) { + fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" + + bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, testVersion) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "pull-requests/pr-23456") + assert.Equal(t, object, "elastic-agent/elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") + }) +} + +func TestGetGCPBucketCoordinates_Snapshots(t *testing.T) { + artifact := "elastic-agent" + version := testVersion + + t.Run("Fetching snapshots bucket for RPM package", func(t *testing.T) { + fileName := "elastic-agent-" + testVersion + "-x86_64.rpm" + + bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, testVersion) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/elastic-agent") + assert.Equal(t, object, "elastic-agent-"+testVersion+"-x86_64.rpm") + }) + + t.Run("Fetching snapshots bucket for DEB package", func(t *testing.T) { + fileName := "elastic-agent-" + testVersion + "-amd64.deb" + + bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, testVersion) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/elastic-agent") + assert.Equal(t, object, "elastic-agent-"+testVersion+"-amd64.deb") + }) + + t.Run("Fetching snapshots bucket for TAR package adds OS to fileName and object", func(t *testing.T) { + fileName := "elastic-agent-" + testVersion + "-linux-x86_64.tar.gz" + + bucket, prefix, object := getGCPBucketCoordinates(fileName, artifact, version, testVersion) + assert.Equal(t, bucket, "beats-ci-artifacts") + assert.Equal(t, prefix, "snapshots/elastic-agent") + assert.Equal(t, object, "elastic-agent-"+testVersion+"-linux-x86_64.tar.gz") + }) +} + func TestProcessBucketSearchPage_PullRequestsFound(t *testing.T) { // retrieving last element in pull-requests.json object := "pr-22495/filebeat/filebeat-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz"