[Elastic Agent] Fix artifact downloading to allow endpoint-security t…

…o be downloaded (elastic#19503)

* Fix downloading so its not hard coded to beats repo name.

* Add artifact to spec file, use artifact for download.

* Fix tar installer.

* Fix downloading.

* Fix tests.

* Add changelog.

* Revert some un-needed changes.

x-pack/elastic-agent/CHANGELOG.asciidoc

@@ -74,3 +74,4 @@
 - Agent now runs the GRPC server and spawned application connect by to Agent {pull}18973[18973]
 - Rename input.type logs to logfile {pull}19360[19360]
 - Agent now installs/uninstalls Elastic Endpoint {pull}19248[19248]
+- Agent now downloads Elastic Endpoint {pull}19503[19503]

x-pack/elastic-agent/pkg/agent/operation/common_test.go

@@ -144,7 +144,7 @@ func waitFor(t *testing.T, check func() error) {
 
 type DummyDownloader struct{}
 
-func (*DummyDownloader) Download(_ context.Context, p, v string) (string, error) {
+func (*DummyDownloader) Download(_ context.Context, p, a, v string) (string, error) {
 	return "", nil
 }
 

x-pack/elastic-agent/pkg/agent/operation/monitoring.go

@@ -5,6 +5,8 @@
 package operation
 
 import (
+	"fmt"
+
 	"github.com/hashicorp/go-multierror"
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configrequest"
@@ -19,6 +21,7 @@ const (
 	monitoringEnabledSubkey = "enabled"
 	logsProcessName         = "filebeat"
 	metricsProcessName      = "metricbeat"
+	artifactPrefix          = "beats"
 )
 
 func (o *Operator) handleStartSidecar(s configrequest.Step) (result error) {
@@ -133,8 +136,9 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [
 			ID:      stepID,
 			Version: version,
 			ProgramSpec: program.Spec{
-				Name: logsProcessName,
-				Cmd:  logsProcessName,
+				Name:     logsProcessName,
+				Cmd:      logsProcessName,
+				Artifact: fmt.Sprintf("%s/%s", artifactPrefix, logsProcessName),
 			},
 			Meta: map[string]interface{}{
 				configrequest.MetaConfigKey: fbConfig,
@@ -154,8 +158,9 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [
 			ID:      stepID,
 			Version: version,
 			ProgramSpec: program.Spec{
-				Name: metricsProcessName,
-				Cmd:  metricsProcessName,
+				Name:     metricsProcessName,
+				Cmd:      metricsProcessName,
+				Artifact: fmt.Sprintf("%s/%s", artifactPrefix, logsProcessName),
 			},
 			Meta: map[string]interface{}{
 				configrequest.MetaConfigKey: mbConfig,

x-pack/elastic-agent/pkg/agent/operation/operation.go

@@ -50,6 +50,7 @@ type Application interface {
 type Descriptor interface {
 	ServicePort() int
 	BinaryName() string
+	ArtifactName() string
 	Version() string
 	ID() string
 	Directory() string

x-pack/elastic-agent/pkg/agent/operation/operation_fetch.go

@@ -70,7 +70,7 @@ func (o *operationFetch) Run(ctx context.Context, application Application) (err
 		}
 	}()
 
-	fullPath, err := o.downloader.Download(ctx, o.program.BinaryName(), o.program.Version())
+	fullPath, err := o.downloader.Download(ctx, o.program.BinaryName(), o.program.ArtifactName(), o.program.Version())
 	if err == nil {
 		o.logger.Infof("operation '%s' downloaded %s.%s into %s", o.Name(), o.program.BinaryName(), o.program.Version(), fullPath)
 	}

x-pack/elastic-agent/pkg/agent/program/spec.go

@@ -31,6 +31,7 @@ type Spec struct {
 	ServicePort       int                  `yaml:"service,omitempty"`
 	Cmd               string               `yaml:"cmd"`
 	Args              []string             `yaml:"args"`
+	Artifact          string               `yaml:"artifact"`
 	Rules             *transpiler.RuleList `yaml:"rules"`
 	CheckInstallSteps *transpiler.StepList `yaml:"check_install"`
 	PostInstallSteps  *transpiler.StepList `yaml:"post_install"`

x-pack/elastic-agent/pkg/agent/program/spec_test.go

@@ -21,9 +21,10 @@ import (
 
 func TestSerialization(t *testing.T) {
 	spec := Spec{
-		Name: "hello",
-		Cmd:  "hellocmd",
-		Args: []string{"-c", "first"},
+		Name:     "hello",
+		Cmd:      "hellocmd",
+		Args:     []string{"-c", "first"},
+		Artifact: "nested/hellocmd",
 		Rules: transpiler.NewRuleList(
 			transpiler.Copy("inputs", "filebeat"),
 			transpiler.Filter("filebeat", "output", "keystore"),
@@ -60,6 +61,7 @@ cmd: hellocmd
 args:
 - -c
 - first
+artifact: nested/hellocmd
 rules:
 - copy:
     from: inputs

x-pack/elastic-agent/pkg/artifact/artifact.go

@@ -21,14 +21,14 @@ var packageArchMap = map[string]string{
 }
 
 // GetArtifactName constructs a path to a downloaded artifact
-func GetArtifactName(program, version, operatingSystem, arch string) (string, error) {
+func GetArtifactName(programName, version, operatingSystem, arch string) (string, error) {
 	key := fmt.Sprintf("%s-binary-%s", operatingSystem, arch)
 	suffix, found := packageArchMap[key]
 	if !found {
 		return "", errors.New(fmt.Sprintf("'%s' is not a valid combination for a package", key), errors.TypeConfig)
 	}
 
-	return fmt.Sprintf("%s-%s-%s", program, version, suffix), nil
+	return fmt.Sprintf("%s-%s-%s", programName, version, suffix), nil
 }
 
 // GetArtifactPath returns a full path of artifact for a program in specific version

x-pack/elastic-agent/pkg/artifact/config.go

@@ -21,8 +21,8 @@ type Config struct {
 	// Architecture: target architecture [32, 64]
 	Architecture string `json:"-" config:",ignore"`
 
-	// BeatsSourceURI: source of the artifacts, e.g https://artifacts.elastic.co/downloads/beats/
-	BeatsSourceURI string `json:"sourceURI" config:"sourceURI"`
+	// SourceURI: source of the artifacts, e.g https://artifacts.elastic.co/downloads/
+	SourceURI string `json:"sourceURI" config:"sourceURI"`
 
 	// TargetDirectory: path to the directory containing downloaded packages
 	TargetDirectory string `json:"targetDirectory" config:"target_directory"`
@@ -49,7 +49,7 @@ type Config struct {
 func DefaultConfig() *Config {
 	dataPath := paths.Data()
 	return &Config{
-		BeatsSourceURI:  "https://artifacts.elastic.co/downloads/beats/",
+		SourceURI:       "https://artifacts.elastic.co/downloads/",
 		TargetDirectory: filepath.Join(dataPath, "downloads"),
 		Timeout:         30 * time.Second,
 		PgpFile:         filepath.Join(dataPath, "elastic.pgp"),

x-pack/elastic-agent/pkg/artifact/download/composed/downloader.go

@@ -32,11 +32,11 @@ func NewDownloader(downloaders ...download.Downloader) *Downloader {
 
 // Download fetches the package from configured source.
 // Returns absolute path to downloaded package and an error.
-func (e *Downloader) Download(ctx context.Context, programName, version string) (string, error) {
+func (e *Downloader) Download(ctx context.Context, programName, artifactName, version string) (string, error) {
 	var err error
 
 	for _, d := range e.dd {
-		s, e := d.Download(ctx, programName, version)
+		s, e := d.Download(ctx, programName, artifactName, version)
 		if e == nil {
 			return s, nil
 		}

x-pack/elastic-agent/pkg/artifact/download/composed/downloader_test.go

@@ -18,7 +18,7 @@ type FailingDownloader struct {
 	called bool
 }
 
-func (d *FailingDownloader) Download(ctx context.Context, a, b string) (string, error) {
+func (d *FailingDownloader) Download(ctx context.Context, a, b, c string) (string, error) {
 	d.called = true
 	return "", errors.New("failing")
 }
@@ -29,7 +29,7 @@ type SuccDownloader struct {
 	called bool
 }
 
-func (d *SuccDownloader) Download(ctx context.Context, a, b string) (string, error) {
+func (d *SuccDownloader) Download(ctx context.Context, a, b, c string) (string, error) {
 	d.called = true
 	return "succ", nil
 }
@@ -58,7 +58,7 @@ func TestComposed(t *testing.T) {
 
 	for _, tc := range testCases {
 		d := NewDownloader(tc.downloaders[0], tc.downloaders[1])
-		r, _ := d.Download(nil, "a", "b")
+		r, _ := d.Download(nil, "a", "b", "c")
 
 		assert.Equal(t, tc.expectedResult, r == "succ")
 

x-pack/elastic-agent/pkg/artifact/download/downloader.go

@@ -8,5 +8,5 @@ import "context"
 
 // Downloader is an interface allowing download of an artifact
 type Downloader interface {
-	Download(ctx context.Context, programName, version string) (string, error)
+	Download(ctx context.Context, programName, artifactName, version string) (string, error)
 }

x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go

@@ -39,7 +39,7 @@ func NewDownloader(config *artifact.Config) *Downloader {
 
 // Download fetches the package from configured source.
 // Returns absolute path to downloaded package and an error.
-func (e *Downloader) Download(_ context.Context, programName, version string) (_ string, err error) {
+func (e *Downloader) Download(_ context.Context, programName, artifactName, version string) (_ string, err error) {
 	downloadedFiles := make([]string, 0, 2)
 	defer func() {
 		if err != nil {
@@ -50,18 +50,18 @@ func (e *Downloader) Download(_ context.Context, programName, version string) (_
 	}()
 
 	// download from source to dest
-	path, err := e.download(e.config.OS(), programName, version)
+	path, err := e.download(e.config.OS(), programName, artifactName, version)
 	downloadedFiles = append(downloadedFiles, path)
 	if err != nil {
 		return "", err
 	}
 
-	hashPath, err := e.downloadHash(e.config.OS(), programName, version)
+	hashPath, err := e.downloadHash(e.config.OS(), programName, artifactName, version)
 	downloadedFiles = append(downloadedFiles, hashPath)
 	return path, err
 }
 
-func (e *Downloader) download(operatingSystem, programName, version string) (string, error) {
+func (e *Downloader) download(operatingSystem, programName, _, version string) (string, error) {
 	filename, err := artifact.GetArtifactName(programName, version, operatingSystem, e.config.Arch())
 	if err != nil {
 		return "", errors.New(err, "generating package name failed")
@@ -75,7 +75,7 @@ func (e *Downloader) download(operatingSystem, programName, version string) (str
 	return e.downloadFile(filename, fullPath)
 }
 
-func (e *Downloader) downloadHash(operatingSystem, programName, version string) (string, error) {
+func (e *Downloader) downloadHash(operatingSystem, programName, _, version string) (string, error) {
 	filename, err := artifact.GetArtifactName(programName, version, operatingSystem, e.config.Arch())
 	if err != nil {
 		return "", errors.New(err, "generating package name failed")

x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go

@@ -20,9 +20,9 @@ import (
 )
 
 const (
-	beatName      = "filebeat"
-	version       = "7.5.1"
-	sourcePattern = "/downloads/beats/filebeat/"
+	beatName     = "filebeat"
+	artifactName = "beats/filebeat"
+	version      = "7.5.1"
 )
 
 type testCase struct {
@@ -37,6 +37,7 @@ func TestFetchVerify(t *testing.T) {
 	targetPath := filepath.Join("testdata", "download")
 	ctx := context.Background()
 	programName := "beat"
+	artifactName := "beats/beat"
 	version := "8.0.0"
 
 	targetFilePath := filepath.Join(targetPath, "beat-8.0.0-darwin-x86_64.tar.gz")
@@ -77,7 +78,7 @@ func TestFetchVerify(t *testing.T) {
 	// second one should pass
 	// download not skipped: package missing
 	// verify passes because hash is not correct
-	_, err = downloader.Download(ctx, programName, version)
+	_, err = downloader.Download(ctx, programName, artifactName, version)
 	assert.NoError(t, err)
 
 	// file downloaded ok
@@ -146,7 +147,7 @@ func TestVerify(t *testing.T) {
 	}
 
 	testClient := NewDownloader(config)
-	artifact, err := testClient.Download(context.Background(), beatName, version)
+	artifact, err := testClient.Download(context.Background(), beatName, artifactName, version)
 	if err != nil {
 		t.Fatal(err)
 	}

x-pack/elastic-agent/pkg/artifact/download/http/downloader.go

@@ -51,7 +51,7 @@ func NewDownloaderWithClient(config *artifact.Config, client http.Client) *Downl
 
 // Download fetches the package from configured source.
 // Returns absolute path to downloaded package and an error.
-func (e *Downloader) Download(ctx context.Context, programName, version string) (_ string, err error) {
+func (e *Downloader) Download(ctx context.Context, programName, artifactName, version string) (_ string, err error) {
 	downloadedFiles := make([]string, 0, 2)
 	defer func() {
 		if err != nil {
@@ -62,19 +62,19 @@ func (e *Downloader) Download(ctx context.Context, programName, version string)
 	}()
 
 	// download from source to dest
-	path, err := e.download(ctx, e.config.OS(), programName, version)
+	path, err := e.download(ctx, e.config.OS(), programName, artifactName, version)
 	downloadedFiles = append(downloadedFiles, path)
 	if err != nil {
 		return "", err
 	}
 
-	hashPath, err := e.downloadHash(ctx, e.config.OS(), programName, version)
+	hashPath, err := e.downloadHash(ctx, e.config.OS(), programName, artifactName, version)
 	downloadedFiles = append(downloadedFiles, hashPath)
 	return path, err
 }
 
-func (e *Downloader) composeURI(programName, packageName string) (string, error) {
-	upstream := e.config.BeatsSourceURI
+func (e *Downloader) composeURI(artifactName, packageName string) (string, error) {
+	upstream := e.config.SourceURI
 	if !strings.HasPrefix(upstream, "http") && !strings.HasPrefix(upstream, "file") && !strings.HasPrefix(upstream, "/") {
 		// always default to https
 		upstream = fmt.Sprintf("https://%s", upstream)
@@ -86,11 +86,11 @@ func (e *Downloader) composeURI(programName, packageName string) (string, error)
 		return "", errors.New(err, "invalid upstream URI", errors.TypeConfig)
 	}
 
-	uri.Path = path.Join(uri.Path, programName, packageName)
+	uri.Path = path.Join(uri.Path, artifactName, packageName)
 	return uri.String(), nil
 }
 
-func (e *Downloader) download(ctx context.Context, operatingSystem, programName, version string) (string, error) {
+func (e *Downloader) download(ctx context.Context, operatingSystem, programName, artifactName, version string) (string, error) {
 	filename, err := artifact.GetArtifactName(programName, version, operatingSystem, e.config.Arch())
 	if err != nil {
 		return "", errors.New(err, "generating package name failed")
@@ -101,10 +101,10 @@ func (e *Downloader) download(ctx context.Context, operatingSystem, programName,
 		return "", errors.New(err, "generating package path failed")
 	}
 
-	return e.downloadFile(ctx, programName, filename, fullPath)
+	return e.downloadFile(ctx, artifactName, filename, fullPath)
 }
 
-func (e *Downloader) downloadHash(ctx context.Context, operatingSystem, programName, version string) (string, error) {
+func (e *Downloader) downloadHash(ctx context.Context, operatingSystem, programName, artifactName, version string) (string, error) {
 	filename, err := artifact.GetArtifactName(programName, version, operatingSystem, e.config.Arch())
 	if err != nil {
 		return "", errors.New(err, "generating package name failed")
@@ -118,11 +118,11 @@ func (e *Downloader) downloadHash(ctx context.Context, operatingSystem, programN
 	filename = filename + ".sha512"
 	fullPath = fullPath + ".sha512"
 
-	return e.downloadFile(ctx, programName, filename, fullPath)
+	return e.downloadFile(ctx, artifactName, filename, fullPath)
 }
 
-func (e *Downloader) downloadFile(ctx context.Context, programName, filename, fullPath string) (string, error) {
-	sourceURI, err := e.composeURI(programName, filename)
+func (e *Downloader) downloadFile(ctx context.Context, artifactName, filename, fullPath string) (string, error) {
+	sourceURI, err := e.composeURI(artifactName, filename)
 	if err != nil {
 		return "", err
 	}