feat: export Fetch&Download methods in the /pkg directory (#1943)

* chore: copy internal's version files into the exported pkg public package

* chore: run unit tests from top-level goal

* chore: use public package instead of the internal one

* chore: remove old internal files for habdling downloads

* fix: update missing references

Makefile

@@ -37,7 +37,7 @@ notice:
 		-depsOut ""
 
 .PHONY: unit-test
-unit-test: test-report-setup unit-test-dir-cli unit-test-dir-internal
+unit-test: test-report-setup unit-test-dir-cli unit-test-dir-internal unit-test-dir-pkg
 
 # See https://pkg.go.dev/gotest.tools/gotestsum/#readme-junit-xml-output
 .PHONY: unit-test-suite-%

e2e/_suites/fleet/fleet.go

@@ -20,14 +20,14 @@ import (
 
 	"github.com/cenkalti/backoff/v4"
 	"github.com/cucumber/godog"
-	elasticversion "github.com/elastic/e2e-testing/internal"
 	"github.com/elastic/e2e-testing/internal/common"
 	"github.com/elastic/e2e-testing/internal/deploy"
 	"github.com/elastic/e2e-testing/internal/elasticsearch"
 	"github.com/elastic/e2e-testing/internal/installer"
 	"github.com/elastic/e2e-testing/internal/kibana"
 	"github.com/elastic/e2e-testing/internal/shell"
 	"github.com/elastic/e2e-testing/internal/utils"
+	"github.com/elastic/e2e-testing/pkg/downloads"
 
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -331,7 +331,7 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(version, i
 
 	common.AgentStaleVersion = shell.GetEnv("ELASTIC_AGENT_STALE_VERSION", common.AgentStaleVersion)
 	// check if stale version is an alias
-	v, err := elasticversion.GetElasticArtifactVersion(common.AgentStaleVersion)
+	v, err := downloads.GetElasticArtifactVersion(common.AgentStaleVersion)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"error":   err,
@@ -341,7 +341,7 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(version, i
 	}
 	common.AgentStaleVersion = v
 
-	useCISnapshots := elasticversion.GithubCommitSha1 != ""
+	useCISnapshots := downloads.GithubCommitSha1 != ""
 	if useCISnapshots && !strings.HasSuffix(common.AgentStaleVersion, "-SNAPSHOT") {
 		common.AgentStaleVersion += "-SNAPSHOT"
 	}
@@ -399,7 +399,7 @@ func (fts *FleetTestSuite) agentInVersion(version string) error {
 	case "stale":
 		version = common.AgentStaleVersion
 	case "latest":
-		version = elasticversion.GetSnapshotVersion(common.BeatVersion)
+		version = downloads.GetSnapshotVersion(common.BeatVersion)
 	}
 
 	agentInVersionFn := func() error {

e2e/_suites/fleet/stand-alone.go

@@ -10,12 +10,12 @@ import (
 	"time"
 
 	"github.com/cenkalti/backoff/v4"
-	elasticversion "github.com/elastic/e2e-testing/internal"
 	"github.com/elastic/e2e-testing/internal/common"
 	"github.com/elastic/e2e-testing/internal/deploy"
 	"github.com/elastic/e2e-testing/internal/installer"
 	"github.com/elastic/e2e-testing/internal/kibana"
 	"github.com/elastic/e2e-testing/internal/utils"
+	"github.com/elastic/e2e-testing/pkg/downloads"
 
 	"github.com/elastic/e2e-testing/internal/elasticsearch"
 	log "github.com/sirupsen/logrus"
@@ -90,9 +90,9 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, flavour string, en
 		common.ProfileEnv["elasticAgentDockerImageSuffix"] = "-" + image
 	}
 
-	useCISnapshots := elasticversion.GithubCommitSha1 != ""
+	useCISnapshots := downloads.GithubCommitSha1 != ""
 
-	if useCISnapshots || elasticversion.BeatsLocalPath != "" {
+	if useCISnapshots || downloads.BeatsLocalPath != "" {
 		// load the docker images that were already:
 		// a. downloaded from the GCP bucket
 		// b. fetched from the local beats binaries

e2e/_suites/kubernetes-autodiscover/autodiscover_test.go

@@ -28,12 +28,12 @@ import (
 	"go.elastic.co/apm"
 
 	"github.com/elastic/e2e-testing/cli/config"
-	elasticversion "github.com/elastic/e2e-testing/internal"
 	"github.com/elastic/e2e-testing/internal/common"
 	"github.com/elastic/e2e-testing/internal/deploy"
 	"github.com/elastic/e2e-testing/internal/kubernetes"
 	"github.com/elastic/e2e-testing/internal/shell"
 	"github.com/elastic/e2e-testing/internal/utils"
+	"github.com/elastic/e2e-testing/pkg/downloads"
 )
 
 var beatVersions = map[string]string{}
@@ -133,13 +133,13 @@ func (m *podsManager) configureDockerImage(podName string) error {
 		return nil
 	}
 
-	beatVersion := elasticversion.GetSnapshotVersion(common.BeatVersion) + "-amd64"
+	beatVersion := downloads.GetSnapshotVersion(common.BeatVersion) + "-amd64"
 
-	useCISnapshots := elasticversion.GithubCommitSha1 != ""
-	if useCISnapshots || elasticversion.BeatsLocalPath != "" {
+	useCISnapshots := downloads.GithubCommitSha1 != ""
+	if useCISnapshots || downloads.BeatsLocalPath != "" {
 		log.Debugf("Configuring Docker image for %s", podName)
 
-		_, imagePath, err := elasticversion.FetchElasticArtifact(m.ctx, podName, common.BeatVersion, "linux", "amd64", "tar.gz", true, true)
+		_, imagePath, err := downloads.FetchElasticArtifact(m.ctx, podName, common.BeatVersion, "linux", "amd64", "tar.gz", true, true)
 		if err != nil {
 			return err
 		}
@@ -152,7 +152,7 @@ func (m *podsManager) configureDockerImage(podName string) error {
 
 		// tag the image with the proper docker tag, including platform
 		err = deploy.TagImage(
-			"docker.elastic.co/beats/"+podName+":"+elasticversion.GetSnapshotVersion(common.BeatVersionBase),
+			"docker.elastic.co/beats/"+podName+":"+downloads.GetSnapshotVersion(common.BeatVersionBase),
 			"docker.elastic.co/observability-ci/"+podName+":"+beatVersion,
 		)
 		if err != nil {

internal/common/defaults.go

@@ -5,8 +5,8 @@
 package common
 
 import (
-	elasticversion "github.com/elastic/e2e-testing/internal"
 	"github.com/elastic/e2e-testing/internal/shell"
+	"github.com/elastic/e2e-testing/pkg/downloads"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -81,7 +81,7 @@ func init() {
 // supporting lazy-loading the versions when needed. Basically, the CLI part does not
 // need to load them
 func InitVersions() {
-	v, err := elasticversion.GetElasticArtifactVersion(BeatVersionBase)
+	v, err := downloads.GetElasticArtifactVersion(BeatVersionBase)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"error":   err,
@@ -93,7 +93,7 @@ func InitVersions() {
 	BeatVersion = shell.GetEnv("BEAT_VERSION", BeatVersionBase)
 
 	// check if version is an alias
-	v, err = elasticversion.GetElasticArtifactVersion(BeatVersion)
+	v, err = downloads.GetElasticArtifactVersion(BeatVersion)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"error":   err,
@@ -111,10 +111,10 @@ func InitVersions() {
 		}).Trace("Beat Version provided: will be used as fallback")
 		fallbackVersion = BeatVersion
 	}
-	BeatVersion = elasticversion.CheckPRVersion(BeatVersion, fallbackVersion)
+	BeatVersion = downloads.CheckPRVersion(BeatVersion, fallbackVersion)
 
 	StackVersion = shell.GetEnv("STACK_VERSION", BeatVersionBase)
-	v, err = elasticversion.GetElasticArtifactVersion(StackVersion)
+	v, err = downloads.GetElasticArtifactVersion(StackVersion)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"error":   err,
@@ -127,7 +127,7 @@ func InitVersions() {
 	if KibanaVersion == "" {
 		// we want to deploy a released version for Kibana
 		// if not set, let's use StackVersion
-		KibanaVersion, err = elasticversion.GetElasticArtifactVersion(StackVersion)
+		KibanaVersion, err = downloads.GetElasticArtifactVersion(StackVersion)
 		if err != nil {
 			log.WithFields(log.Fields{
 				"error":   err,

internal/deploy/docker.go

@@ -10,8 +10,8 @@ import (
 	"path/filepath"
 	"strings"
 
-	elasticversion "github.com/elastic/e2e-testing/internal"
 	"github.com/elastic/e2e-testing/internal/shell"
+	"github.com/elastic/e2e-testing/pkg/downloads"
 	log "github.com/sirupsen/logrus"
 	"go.elastic.co/apm"
 )
@@ -278,8 +278,8 @@ func (c *dockerDeploymentManifest) Stop(ctx context.Context, service ServiceRequ
 // the images produced by local Beats build, or not.
 // If an error occurred reading the environment, will return the passed namespace as fallback
 func GetDockerNamespaceEnvVar(fallback string) string {
-	useCISnapshots := elasticversion.GithubCommitSha1 != ""
-	if useCISnapshots || elasticversion.BeatsLocalPath != "" {
+	useCISnapshots := downloads.GithubCommitSha1 != ""
+	if useCISnapshots || downloads.BeatsLocalPath != "" {
 		return "observability-ci"
 	}
 	return fallback

internal/deploy/docker_test.go

@@ -7,8 +7,7 @@ package deploy
 import (
 	"testing"
 
-	elasticversion "github.com/elastic/e2e-testing/internal"
-
+	"github.com/elastic/e2e-testing/pkg/downloads"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -19,8 +18,8 @@ func TestGetDockerNamespaceEnvVar(t *testing.T) {
 	})
 
 	t.Run("Returns Observability CI when environment variable is set", func(t *testing.T) {
-		elasticversion.GithubCommitSha1 = "0123456789"
-		defer func() { elasticversion.GithubCommitSha1 = "" }()
+		downloads.GithubCommitSha1 = "0123456789"
+		defer func() { downloads.GithubCommitSha1 = "" }()
 
 		namespace := GetDockerNamespaceEnvVar("beats")
 		assert.True(t, namespace == "observability-ci")

internal/installer/elasticagent_deb.go

@@ -9,11 +9,11 @@ import (
 	"fmt"
 	"strings"
 
-	elasticversion "github.com/elastic/e2e-testing/internal"
 	"github.com/elastic/e2e-testing/internal/common"
 	"github.com/elastic/e2e-testing/internal/deploy"
 	"github.com/elastic/e2e-testing/internal/kibana"
 	"github.com/elastic/e2e-testing/internal/utils"
+	"github.com/elastic/e2e-testing/pkg/downloads"
 	log "github.com/sirupsen/logrus"
 	"go.elastic.co/apm"
 )
@@ -142,7 +142,7 @@ func (i *elasticAgentDEBPackage) Preinstall(ctx context.Context) error {
 		arch := utils.GetArchitecture()
 		extension := "deb"
 
-		binaryName, binaryPath, err := elasticversion.FetchElasticArtifact(ctx, artifact, common.BeatVersion, os, arch, extension, false, true)
+		binaryName, binaryPath, err := downloads.FetchElasticArtifact(ctx, artifact, common.BeatVersion, os, arch, extension, false, true)
 		if err != nil {
 			log.WithFields(log.Fields{
 				"artifact":  artifact,

internal/installer/elasticagent_docker.go

@@ -8,10 +8,10 @@ import (
 	"context"
 	"fmt"
 
-	elasticversion "github.com/elastic/e2e-testing/internal"
 	"github.com/elastic/e2e-testing/internal/common"
 	"github.com/elastic/e2e-testing/internal/deploy"
 	"github.com/elastic/e2e-testing/internal/utils"
+	"github.com/elastic/e2e-testing/pkg/downloads"
 	log "github.com/sirupsen/logrus"
 	"go.elastic.co/apm"
 )
@@ -99,7 +99,7 @@ func (i *elasticAgentDockerPackage) Preinstall(ctx context.Context) error {
 	arch := utils.GetArchitecture()
 	extension := "tar.gz"
 
-	_, binaryPath, err := elasticversion.FetchElasticArtifact(ctx, artifact, common.BeatVersion, os, arch, extension, true, true)
+	_, binaryPath, err := downloads.FetchElasticArtifact(ctx, artifact, common.BeatVersion, os, arch, extension, true, true)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"artifact":  artifact,
@@ -119,10 +119,10 @@ func (i *elasticAgentDockerPackage) Preinstall(ctx context.Context) error {
 
 	// we need to tag the loaded image because its tag relates to the target branch
 	return deploy.TagImage(
-		fmt.Sprintf("docker.elastic.co/beats/%s:%s", artifact, elasticversion.GetSnapshotVersion(common.BeatVersionBase)),
-		fmt.Sprintf("docker.elastic.co/observability-ci/%s:%s-%s", artifact, elasticversion.GetSnapshotVersion(common.BeatVersion), arch),
+		fmt.Sprintf("docker.elastic.co/beats/%s:%s", artifact, downloads.GetSnapshotVersion(common.BeatVersionBase)),
+		fmt.Sprintf("docker.elastic.co/observability-ci/%s:%s-%s", artifact, downloads.GetSnapshotVersion(common.BeatVersion), arch),
 		// tagging including git commit and snapshot
-		fmt.Sprintf("docker.elastic.co/observability-ci/%s:%s-%s", artifact, elasticversion.GetFullVersion(common.BeatVersion), arch),
+		fmt.Sprintf("docker.elastic.co/observability-ci/%s:%s-%s", artifact, downloads.GetFullVersion(common.BeatVersion), arch),
 	)
 }
 

internal/installer/elasticagent_rpm.go

@@ -9,11 +9,11 @@ import (
 	"fmt"
 	"strings"
 
-	elasticversion "github.com/elastic/e2e-testing/internal"
 	"github.com/elastic/e2e-testing/internal/common"
 	"github.com/elastic/e2e-testing/internal/deploy"
 	"github.com/elastic/e2e-testing/internal/kibana"
 	"github.com/elastic/e2e-testing/internal/utils"
+	"github.com/elastic/e2e-testing/pkg/downloads"
 	log "github.com/sirupsen/logrus"
 	"go.elastic.co/apm"
 )
@@ -146,7 +146,7 @@ func (i *elasticAgentRPMPackage) Preinstall(ctx context.Context) error {
 		}
 		extension := "rpm"
 
-		binaryName, binaryPath, err := elasticversion.FetchElasticArtifact(ctx, artifact, common.BeatVersion, os, arch, extension, false, true)
+		binaryName, binaryPath, err := downloads.FetchElasticArtifact(ctx, artifact, common.BeatVersion, os, arch, extension, false, true)
 		if err != nil {
 			log.WithFields(log.Fields{
 				"artifact":  artifact,

internal/installer/elasticagent_tar.go

@@ -10,12 +10,12 @@ import (
 	"os"
 	"strings"
 
-	elasticversion "github.com/elastic/e2e-testing/internal"
 	"github.com/elastic/e2e-testing/internal/common"
 	"github.com/elastic/e2e-testing/internal/deploy"
 	"github.com/elastic/e2e-testing/internal/io"
 	"github.com/elastic/e2e-testing/internal/kibana"
 	"github.com/elastic/e2e-testing/internal/utils"
+	"github.com/elastic/e2e-testing/pkg/downloads"
 	log "github.com/sirupsen/logrus"
 	"go.elastic.co/apm"
 )
@@ -129,7 +129,7 @@ func (i *elasticAgentTARPackage) Preinstall(ctx context.Context) error {
 			}
 			log.Trace("Cleared previously downloaded artifacts")
 		}
-		_, binaryPath, err := elasticversion.FetchElasticArtifact(ctx, artifact, common.BeatVersion, runningOS, arch, extension, false, true)
+		_, binaryPath, err := downloads.FetchElasticArtifact(ctx, artifact, common.BeatVersion, runningOS, arch, extension, false, true)
 		if err != nil {
 			log.WithFields(log.Fields{
 				"artifact":  artifact,
@@ -147,7 +147,7 @@ func (i *elasticAgentTARPackage) Preinstall(ctx context.Context) error {
 			return err
 		}
 
-		output, _ := i.Exec(ctx, []string{"mv", fmt.Sprintf("%s-%s-%s-%s", artifact, elasticversion.GetSnapshotVersion(common.BeatVersion), runningOS, arch), artifact})
+		output, _ := i.Exec(ctx, []string{"mv", fmt.Sprintf("%s-%s-%s-%s", artifact, downloads.GetSnapshotVersion(common.BeatVersion), runningOS, arch), artifact})
 		log.WithFields(log.Fields{
 			"output":   output,
 			"artifact": artifact,

internal/installer/elasticagent_tar_macos.go

@@ -9,11 +9,11 @@ import (
 	"fmt"
 	"runtime"
 
-	elasticversion "github.com/elastic/e2e-testing/internal"
 	"github.com/elastic/e2e-testing/internal/common"
 	"github.com/elastic/e2e-testing/internal/deploy"
 	"github.com/elastic/e2e-testing/internal/kibana"
 	"github.com/elastic/e2e-testing/internal/utils"
+	"github.com/elastic/e2e-testing/pkg/downloads"
 	log "github.com/sirupsen/logrus"
 	"go.elastic.co/apm"
 )
@@ -117,7 +117,7 @@ func (i *elasticAgentTARDarwinPackage) Preinstall(ctx context.Context) error {
 	}
 	extension := "tar.gz"
 
-	_, binaryPath, err := elasticversion.FetchElasticArtifact(ctx, artifact, common.BeatVersion, os, arch, extension, false, true)
+	_, binaryPath, err := downloads.FetchElasticArtifact(ctx, artifact, common.BeatVersion, os, arch, extension, false, true)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"artifact":  artifact,
@@ -135,7 +135,7 @@ func (i *elasticAgentTARDarwinPackage) Preinstall(ctx context.Context) error {
 		return err
 	}
 
-	output, _ := i.Exec(ctx, []string{"mv", fmt.Sprintf("/%s-%s-%s-%s", artifact, elasticversion.GetSnapshotVersion(common.BeatVersion), os, arch), "/elastic-agent"})
+	output, _ := i.Exec(ctx, []string{"mv", fmt.Sprintf("/%s-%s-%s-%s", artifact, downloads.GetSnapshotVersion(common.BeatVersion), os, arch), "/elastic-agent"})
 	log.WithField("output", output).Trace("Moved elastic-agent")
 	return nil
 }

internal/installer/elasticagent_zip.go

@@ -9,10 +9,10 @@ import (
 	"fmt"
 	"strings"
 
-	elasticversion "github.com/elastic/e2e-testing/internal"
 	"github.com/elastic/e2e-testing/internal/common"
 	"github.com/elastic/e2e-testing/internal/deploy"
 	"github.com/elastic/e2e-testing/internal/kibana"
+	"github.com/elastic/e2e-testing/pkg/downloads"
 	log "github.com/sirupsen/logrus"
 	"go.elastic.co/apm"
 )
@@ -110,7 +110,7 @@ func (i *elasticAgentZIPPackage) Preinstall(ctx context.Context) error {
 	arch := "x86_64"
 	extension := "zip"
 
-	_, binaryPath, err := elasticversion.FetchElasticArtifact(ctx, artifact, common.BeatVersion, os, arch, extension, false, true)
+	_, binaryPath, err := downloads.FetchElasticArtifact(ctx, artifact, common.BeatVersion, os, arch, extension, false, true)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"artifact":  artifact,
@@ -135,7 +135,7 @@ func (i *elasticAgentZIPPackage) Preinstall(ctx context.Context) error {
 			return err
 		}
 
-		output, _ := i.Exec(ctx, []string{"powershell.exe", "Move-Item", "-Force", "-Path", fmt.Sprintf("C:\\%s-%s-%s-%s", artifact, elasticversion.GetSnapshotVersion(common.BeatVersion), os, arch), "-Destination", "C:\\elastic-agent"})
+		output, _ := i.Exec(ctx, []string{"powershell.exe", "Move-Item", "-Force", "-Path", fmt.Sprintf("C:\\%s-%s-%s-%s", artifact, downloads.GetSnapshotVersion(common.BeatVersion), os, arch), "-Destination", "C:\\elastic-agent"})
 		log.WithField("output", output).Trace("Moved elastic-agent")
 		return nil
 	}

internal/versions.go → pkg/downloads/versions.go

@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package internal
+package downloads
 
 import (
 	"context"