feat: support using local Beats repository in the tests (#688) (#694)

* feat: use local beats directory for elastic-agent installers

* docs: document BEATS_LOCAL_PATH variable

* chore: check that the local file exists

* chore: add unit tests for the BEATS_LOCAL_PATH variable

* chore: simplify logic after test coverage

* chore: enrich test name

* chore: rename test

* chore: move default values to the end of the descriptions

* docs: clarify local path for Beats

e2e/README.md

@@ -135,7 +135,8 @@ We are going to enumerate the variables that will affect the product versions us
 The following environment variables affect how the tests are run in both the CI and a local machine.
 
 - `SKIP_SCENARIOS`: Set this environment variable to `false` if it's needed to include the scenarios annotated as `@skip` in the current test execution. Default value: `true`.
-- `BEATS_USE_CI_SNAPSHOTS`: Set this environment variable to `true` if it's needed to use the binary snapshots produced by Beats CI instead of the official releases. The snapshots will be downloaded from a bucket in Google Cloud Storage. Default: `false`. This variable is used by the Beats repository, when testing the artifacts generated by the packaging job.
+- `BEATS_LOCAL_PATH`: Set this environment variable to the base path to your local clone of Beats if it's needed to use the binary snapshots produced by your local build instead of the official releases. The snapshots will be fetched from the `${BEATS_LOCAL_PATH}/${THE_BEAT}/build/distributions` local directory. This variable is intended to be used by Beats developers, when testing locally the artifacts generated its own build. Default: empty.
+- `BEATS_USE_CI_SNAPSHOTS`: Set this environment variable to `true` if it's needed to use the binary snapshots produced by Beats CI instead of the official releases. The snapshots will be downloaded from a bucket in Google Cloud Storage. This variable is used by the Beats repository, when testing the artifacts generated by the packaging job. Default: `false`.
 - `LOG_LEVEL`: Set this environment variable to `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR` or `FATAL` to set the log level in the project. Default: `INFO`.
 - `DEVELOPER_MODE`: Set this environment variable to `true` to activate developer mode, which means not destroying the services provisioned by the test framework. Default: `false`.
 - `STACK_VERSION`. Set this environment variable to the proper version of the Elastic Stack (Elasticsearch and Kibana) to be used in the current execution. The default value depens on the branch you are targeting your work.

e2e/_suites/fleet/services.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path"
 	"strings"
 	"time"
 
@@ -161,11 +162,31 @@ func runElasticAgentCommand(profile string, image string, service string, proces
 // into the installer struct, to be used else where
 // If the environment variable ELASTIC_AGENT_DOWNLOAD_URL exists, then the artifact to be downloaded will
 // be defined by that value
+// Else if the environment variable BEATS_LOCAL_PATH is set, then the artifact
+// to be used will be defined by the local snapshot produced by the local build.
 // Else, if the environment variable BEATS_USE_CI_SNAPSHOTS is set, then the artifact
 // to be downloaded will be defined by the latest snapshot produced by the Beats CI.
 func downloadAgentBinary(artifact string, version string, OS string, arch string, extension string) (string, string, error) {
 	fileName := fmt.Sprintf("%s-%s-%s.%s", artifact, version, arch, extension)
 
+	beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "")
+	if beatsLocalPath != "" {
+		distributions := path.Join(beatsLocalPath, "x-pack", "elastic-agent", "build", "distributions")
+		log.Debugf("Using local snapshots for the Elastic Agent: %s", distributions)
+
+		if extension == "tar.gz" {
+			fileName = fmt.Sprintf("%s-%s-%s-%s.%s", artifact, version, OS, arch, extension)
+		}
+
+		fileNamePath := path.Join(distributions, fileName)
+		_, err := os.Stat(fileNamePath)
+		if err != nil || os.IsNotExist(err) {
+			return fileName, fileNamePath, err
+		}
+
+		return fileName, fileNamePath, err
+	}
+
 	handleDownload := func(URL string, fileName string) (string, string, error) {
 		if val, ok := binariesCache[URL]; ok {
 			log.WithFields(log.Fields{

e2e/_suites/fleet/services_test.go

@@ -2,13 +2,75 @@ package main
 
 import (
 	"os"
+	"path"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
 var testVersion = agentVersionBase
 
+func TestDownloadAgentBinary(t *testing.T) {
+	artifact := "elastic-agent"
+	beatsDir := path.Join("..", "..", "_testresources", "beats")
+	distributionsDir := path.Join(beatsDir, "x-pack", "elastic-agent", "build", "distributions")
+	OS := "linux"
+	version := "8.0.0-SNAPSHOT"
+
+	t.Run("Fetching non-existent binary from local Beats dir throws an error", func(t *testing.T) {
+		defer os.Unsetenv("BEATS_LOCAL_PATH")
+		os.Setenv("BEATS_LOCAL_PATH", beatsDir)
+
+		arch := "foo_arch"
+		extension := "foo_ext"
+
+		_, _, err := downloadAgentBinary(artifact, version, OS, arch, extension)
+		assert.NotNil(t, err)
+	})
+
+	t.Run("Fetching RPM binary from local Beats dir", func(t *testing.T) {
+		defer os.Unsetenv("BEATS_LOCAL_PATH")
+		os.Setenv("BEATS_LOCAL_PATH", beatsDir)
+
+		arch := "x86_64"
+		extension := "rpm"
+		expectedFileName := "elastic-agent-8.0.0-SNAPSHOT-x86_64.rpm"
+
+		newFileName, downloadedFilePath, err := downloadAgentBinary(artifact, version, OS, arch, extension)
+		assert.Nil(t, err)
+		assert.Equal(t, newFileName, expectedFileName)
+		assert.Equal(t, downloadedFilePath, path.Join(distributionsDir, expectedFileName))
+	})
+
+	t.Run("Fetching DEB binary from local Beats dir", func(t *testing.T) {
+		defer os.Unsetenv("BEATS_LOCAL_PATH")
+		os.Setenv("BEATS_LOCAL_PATH", beatsDir)
+
+		arch := "amd64"
+		extension := "deb"
+		expectedFileName := "elastic-agent-8.0.0-SNAPSHOT-amd64.deb"
+
+		newFileName, downloadedFilePath, err := downloadAgentBinary(artifact, version, OS, arch, extension)
+		assert.Nil(t, err)
+		assert.Equal(t, newFileName, expectedFileName)
+		assert.Equal(t, downloadedFilePath, path.Join(distributionsDir, expectedFileName))
+	})
+
+	t.Run("Fetching TAR binary from local Beats dir", func(t *testing.T) {
+		defer os.Unsetenv("BEATS_LOCAL_PATH")
+		os.Setenv("BEATS_LOCAL_PATH", beatsDir)
+
+		arch := "amd64"
+		extension := "tar.gz"
+		expectedFileName := "elastic-agent-8.0.0-SNAPSHOT-linux-amd64.tar.gz"
+
+		newFileName, downloadedFilePath, err := downloadAgentBinary(artifact, version, OS, arch, extension)
+		assert.Nil(t, err)
+		assert.Equal(t, newFileName, expectedFileName)
+		assert.Equal(t, downloadedFilePath, path.Join(distributionsDir, expectedFileName))
+	})
+}
+
 func TestGetGCPBucketCoordinates_Commits(t *testing.T) {
 	artifact := "elastic-agent"
 	version := testVersion