From 25e223f61504cdfdcebb01af01eaf3042d2e4a61 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Tue, 30 Jun 2020 08:37:02 -0400 Subject: [PATCH] [Elastic Agent] Fix artifact downloading to allow endpoint-security to be downloaded (#19503) * Fix downloading so its not hard coded to beats repo name. * Add artifact to spec file, use artifact for download. * Fix tar installer. * Fix downloading. * Fix tests. * Add changelog. * Revert some un-needed changes. --- x-pack/elastic-agent/CHANGELOG.asciidoc | 1 + .../pkg/agent/operation/common_test.go | 2 +- .../pkg/agent/operation/monitoring.go | 13 ++++++---- .../pkg/agent/operation/operation.go | 1 + .../pkg/agent/operation/operation_fetch.go | 2 +- .../elastic-agent/pkg/agent/program/spec.go | 1 + .../pkg/agent/program/spec_test.go | 8 ++++--- .../pkg/agent/program/supported.go | 2 +- x-pack/elastic-agent/pkg/artifact/artifact.go | 4 ++-- x-pack/elastic-agent/pkg/artifact/config.go | 6 ++--- .../artifact/download/composed/downloader.go | 4 ++-- .../download/composed/downloader_test.go | 6 ++--- .../pkg/artifact/download/downloader.go | 2 +- .../pkg/artifact/download/fs/downloader.go | 10 ++++---- .../pkg/artifact/download/fs/verifier_test.go | 11 +++++---- .../pkg/artifact/download/http/downloader.go | 24 +++++++++---------- .../artifact/download/http/elastic_test.go | 11 +++++---- .../pkg/artifact/download/http/verifier.go | 4 ++-- .../artifact/download/snapshot/downloader.go | 4 ++-- .../pkg/artifact/install/tar/tar_installer.go | 5 ++-- .../elastic-agent/pkg/core/app/descriptor.go | 7 ++++++ x-pack/elastic-agent/spec/endpoint.yml | 1 + x-pack/elastic-agent/spec/filebeat.yml | 1 + x-pack/elastic-agent/spec/metricbeat.yml | 1 + 24 files changed, 77 insertions(+), 54 deletions(-) diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index d7b2248799f..b926202727a 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -74,3 +74,4 @@ - Agent now runs the GRPC server and spawned application connect by to Agent {pull}18973[18973] - Rename input.type logs to logfile {pull}19360[19360] - Agent now installs/uninstalls Elastic Endpoint {pull}19248[19248] +- Agent now downloads Elastic Endpoint {pull}19503[19503] diff --git a/x-pack/elastic-agent/pkg/agent/operation/common_test.go b/x-pack/elastic-agent/pkg/agent/operation/common_test.go index c314a5d7fdb..1f323284405 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/common_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/common_test.go @@ -144,7 +144,7 @@ func waitFor(t *testing.T, check func() error) { type DummyDownloader struct{} -func (*DummyDownloader) Download(_ context.Context, p, v string) (string, error) { +func (*DummyDownloader) Download(_ context.Context, p, a, v string) (string, error) { return "", nil } diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index 6aad94a2e62..10947f4d7df 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -5,6 +5,8 @@ package operation import ( + "fmt" + "github.com/hashicorp/go-multierror" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configrequest" @@ -19,6 +21,7 @@ const ( monitoringEnabledSubkey = "enabled" logsProcessName = "filebeat" metricsProcessName = "metricbeat" + artifactPrefix = "beats" ) func (o *Operator) handleStartSidecar(s configrequest.Step) (result error) { @@ -133,8 +136,9 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [ ID: stepID, Version: version, ProgramSpec: program.Spec{ - Name: logsProcessName, - Cmd: logsProcessName, + Name: logsProcessName, + Cmd: logsProcessName, + Artifact: fmt.Sprintf("%s/%s", artifactPrefix, logsProcessName), }, Meta: map[string]interface{}{ configrequest.MetaConfigKey: fbConfig, @@ -154,8 +158,9 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [ ID: stepID, Version: version, ProgramSpec: program.Spec{ - Name: metricsProcessName, - Cmd: metricsProcessName, + Name: metricsProcessName, + Cmd: metricsProcessName, + Artifact: fmt.Sprintf("%s/%s", artifactPrefix, logsProcessName), }, Meta: map[string]interface{}{ configrequest.MetaConfigKey: mbConfig, diff --git a/x-pack/elastic-agent/pkg/agent/operation/operation.go b/x-pack/elastic-agent/pkg/agent/operation/operation.go index 3f5201f2fed..ebe6bc3065d 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operation.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operation.go @@ -50,6 +50,7 @@ type Application interface { type Descriptor interface { ServicePort() int BinaryName() string + ArtifactName() string Version() string ID() string Directory() string diff --git a/x-pack/elastic-agent/pkg/agent/operation/operation_fetch.go b/x-pack/elastic-agent/pkg/agent/operation/operation_fetch.go index bb01890347b..fc72915ee60 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operation_fetch.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operation_fetch.go @@ -70,7 +70,7 @@ func (o *operationFetch) Run(ctx context.Context, application Application) (err } }() - fullPath, err := o.downloader.Download(ctx, o.program.BinaryName(), o.program.Version()) + fullPath, err := o.downloader.Download(ctx, o.program.BinaryName(), o.program.ArtifactName(), o.program.Version()) if err == nil { o.logger.Infof("operation '%s' downloaded %s.%s into %s", o.Name(), o.program.BinaryName(), o.program.Version(), fullPath) } diff --git a/x-pack/elastic-agent/pkg/agent/program/spec.go b/x-pack/elastic-agent/pkg/agent/program/spec.go index ed5fc61047c..5646a3652e9 100644 --- a/x-pack/elastic-agent/pkg/agent/program/spec.go +++ b/x-pack/elastic-agent/pkg/agent/program/spec.go @@ -31,6 +31,7 @@ type Spec struct { ServicePort int `yaml:"service,omitempty"` Cmd string `yaml:"cmd"` Args []string `yaml:"args"` + Artifact string `yaml:"artifact"` Rules *transpiler.RuleList `yaml:"rules"` CheckInstallSteps *transpiler.StepList `yaml:"check_install"` PostInstallSteps *transpiler.StepList `yaml:"post_install"` diff --git a/x-pack/elastic-agent/pkg/agent/program/spec_test.go b/x-pack/elastic-agent/pkg/agent/program/spec_test.go index df847de7aff..df1fb082f7a 100644 --- a/x-pack/elastic-agent/pkg/agent/program/spec_test.go +++ b/x-pack/elastic-agent/pkg/agent/program/spec_test.go @@ -21,9 +21,10 @@ import ( func TestSerialization(t *testing.T) { spec := Spec{ - Name: "hello", - Cmd: "hellocmd", - Args: []string{"-c", "first"}, + Name: "hello", + Cmd: "hellocmd", + Args: []string{"-c", "first"}, + Artifact: "nested/hellocmd", Rules: transpiler.NewRuleList( transpiler.Copy("inputs", "filebeat"), transpiler.Filter("filebeat", "output", "keystore"), @@ -60,6 +61,7 @@ cmd: hellocmd args: - -c - first +artifact: nested/hellocmd rules: - copy: from: inputs diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go index bdc9c951fbc..343466f7e49 100644 --- a/x-pack/elastic-agent/pkg/agent/program/supported.go +++ b/x-pack/elastic-agent/pkg/agent/program/supported.go @@ -20,7 +20,7 @@ func init() { // spec/endpoint.yml // spec/filebeat.yml // spec/metricbeat.yml - unpacked := packer.MustUnpack("eJzsWEuTq7gV3udn3G1SCYi2J6RqFoYeXnbT17hbEtohyQZsganGL0jlv6cEGIPdM3PvzSSrLLrsxnqcx3e+8x3++aUs1uxv65wX+zQ//LXKxJd/fKGZdSBv+zhAkx2z9YLmy/gdwC3HXsGd3TwE6u41NQTNgjMF4shNtSbIV1kmlPWyqAn2QIR88ZrOgF/HT2bm7wnyPzAQR6YFCbXPUzNVYoITEap6FqGLeI33sWvOYmZbSvS8j90MHoljnCI0URbYF6EGqwgHE7dbF2IvZ/V+Lr+7phK7OREsE9l65bbPnKCgCJ44Xk7d55ezmRsnpgWbCE1qblslfd7PFytDrG24xYAU1H6fmqk8yziHONi/prOU3Pw/sJv/abcuYRmvr3YvVrO0P9t0b3alyoFjI2eZtSOr8XOWwZpqsAoBrN3BOb9y72j9Ii8KZn7m6y/Vy6w4swxuOJoUNPeVEF3K13h/cG34RJC/IZkoyVsXO8dIuB1PXdP7PJ9Xm22rItpLF6NrzOHxwedHWyqOLkLmarEysggnG6YFFUHW4TU1ftosu+foohDsbXhmlRzBPq4MwJIgX6GaO3VtcWQOVNi5/S0CcPKaGsoaG6LLCQjRRSW4t1P63OPIbGJACuoEggn9bq1RE3QRoRac2HYfR2hy5jiou98+CN5N5Vkh8kqClp2vJKEOFEMccsdTycgn9/H8fGgfKQmCyu3/sY+LlXHiODhzvJybOd8T9DR1f0kSpljH9SrOSXYRBC/z+WqWeanhhtj3uA0PrIrFWj0UNDdU7vh8s9rFXhrGK9uq32R9Y/8cIl/MTX49P1+Ys5ygS8K0oAg1X4TY20YmK8z455+//KWlik0q1nQdPVCFhBTyRIiXV3poyjrMYMJnRVsqqUHdVLXc9By7uS+4A8+LTJR0NRE0s1Jqw91XJEPqi2bN/do8EBQbZYgDscj6VOgks0oG3tOFOUsX7+0nRdYxRFxQBI/cnBwoCMRXHB+YbW2jSs2oDQU33dI13UOwkp/eIUSThAB4IJJyBuc3KV2N1pYU8DxCk3yRXQTPYPkVBSLMYe4KZc5yWLawUmKSiSes+QrLYELf9vFaUyQ0CpoVEgqSjnYEx1eINFB+TQ15ft3CVRyjDG65pdfc8USI1A1zvJOkAAb06lomFEw2IdCPJLsUobZsSwXAilt6QvJAsG4dd8RZ5ovaes7ODUwPIZ7dUZ/Rwealg7mnUhsOqM7YUs2YYGCV1NIVquplhH3l+ntHA6fX1LjaXN9o4zNbJyeqzeb9XlPSsi+Ys5y6Vple6YJVTewL0uTN6zHTYsSoKPAF03xZXile3tmqBScMLgXTltOhLREOhMzLMKbX0vt+P/qYpwSRE8ve5xIDrMGS2rc4knsneecdxSi0vj3rfT53bcoJqhD5SutDsGnsvlHXXd78O3uNQlLCox/jO1u8qQl7vmsNd5R0w3dHWY1tk5raltK3FXNk14Ciu9gh68xsWI0wI59LXNYtPofrXbOpiVYqNPXB5sN9rg1BU7ddK3fty4mcx2cTTATNl1IG7CJJo46vEFsc7+6R9Xzkpq6E2mzqOsGWzT4/h6Pg/JoaKnFmd7ZcTqTSdxT4H5L+XTs4heDQ1+DQ14XmCWKLGmt+STUu/Zq6Tvvs0X92Ypqo5T6ZyzX2B3FocHagGhEYHMR6nIe2fTmwJlDv28k377ODhNtWTX5sz5EO2s0NM56gSAcE6s36UU22bX0X4iBhACqLbJJQ1NaSlC7chhXW7lqv4wtqwy239eqBv+zGxpb7Ol64lxmLlTGOZYuRW1tHwXmA9x4Ti5VxzfMwNrsIvwzOanN22/9S39r9RHCblFS73R05gcLsoqbgdj+tX4CPLJXYYiAVrCnHnmjxGOw5utkQgovEZ0Y1KX+8yW0PP1HAS4INNcx9Nbz5/xEi8hH+53Ks5+VflWROUPEunyHQzz1+bFJRoDycy+2/TxvZZsOUIGss47aP+R/KMTLQJX0sNajI828xOYg1lj55YiyxhEKQWs9N+BMCt/sXWWefquoD2fUqOZ5pRhKC970HdJU7hspNVromF9S2am6LLQMwYZm/96pz7HW171VnaVseASuLwC+NDGt4UJP4j/P5ct/Lr2x9+EjZJwLsDUGFZWLbCa4tRbIhqYI7XhGCTpi1M1OMql7I1AQHKjMnBbWVI0GTJJR68lnNQnSp70TPda0Myrkh+98Tbpma0MzKCVIlYR4p0nfkTX1aYBmk8tBp198SbrfzcQOa8VpbzwkQR1JNyqYIntUdQZ5KKo9/Nu+1yVZPPIPjGWg0d6on4jQN7khMXZKsQpByXCO1dHsCNxQZSyJnHNOT4rIRSIts2RC0bLaLXByoOdlF2G9jarp/hDA8EBxUEfL/u+Kww1InPiqJC4r6+ThjmX54FImNUHoUIL8nKq4xwP8Xm3+E2CQ4UVhmSUw0ccKgJbH79yufCM2xjXfPWxLuifnTJoE1XnA72bAM5gQn5+udn8zplcQZTp8+FqDjKO1l95tz+P92dv+RoUjG/cjAJeE23HAglMjSK4K4WDuzEQ6uORgKku5MWePr+bO+/OoEE2a//3mRlsVjjNq/5o7nfey1a4cCZDRwjJrraO1tCJDczNHlJtBsNYkA3ITYq8J7gdZhpOcJ0DfTAVauNsvh/eU7hONg301kVQS1vN1gaeRXK+763vU9e2xLCfF33nPdY06aGP+4oIU1s60tWX773Rz7Al+FxazBacM33zJY0pE++E1ev/aYsdgZ8UDfI0ei/FEkesP6GJ17w1nvTy9ECZY9VK9kLXKQCLrdN++tVu3Qu/fwgY78aV/wNPWIsVK4Zpi1Am4Q51WcX4VfI74ALEPsK7KXEmRVIYjzRWMDrwgKClY1wq0Vxavme/NSRg41/Tuxf/3p3wEAAP//5uHUJw==") + unpacked := packer.MustUnpack("eJy8mFuToziWx9/3Y9TrbuyCKOc2E9EPhmxudpJlnCkJvSHJBmwJuwt8gYn57hPiYoOzqrurZ6YfMjJNCulc/uecn/z3T+Vxw/5vU/DjIS+q/62l+PS3T1Q6FXk7pBGa7ZlrHmmxSt8B3HEcHLm3X8RA37/mlqAyulAgTtzWG4JCnUmhbVbHjBXRkUhnx58PKbnvUREXArsIBSvIMQbvT/5zbLw+p4sYZCIG1TZBs4a7TkmfD4vl2hIbF+4wIEfqvj/Z+Tz1besS4+jwms/z8b7sblver8uY5M1rekh9e54u1/OcS1gniMz8/hl3RUWQqSsbX5r5grlmwx21X6jF6Fq+pofKd+FngsItkaIkb4eFes/3rIy76ZNvB9/2f+1361ynJsZLb/e88u3gtrc/smu51nXm8jpGkXh4XhMcnjkOdgS/5KN9vnPuZP1pI8Xlm76++Re7sGoCTZ1KcWJGlFH38mTnWkpwJmLdlAm6iiF2zHW05PmQ+hKeiGedEzTTljgUsQHrBEe3eMY4KFjTx2iIOZp98PmjLYFOXdh08SbHjWM23AtEjLQn36tMu39OvUgwYYIYXXWCh7haDUFXERvRme0OaYJmF46jpv/fV4L3T74XzZj73ueOZNSD4m6nNtbnoo2BFCV3YY2Nh7VeKKgLd9w169fcOtLC0rn30ue6Epu3VutZLK+CzHtfpVNyBEc6tDRWQDHxafdx/4l9aj+8un2e+jivfNfUuWfpfH5YcJAJujuka9dpVmg2W9hcUhcKbrOjb8fS/yXLmCY0gvRmYcP/R0CcmAc1puvmYj2XQW75MQ5fY6QLZlhZDN4PAej3t1np21xQ12m4K3YMwIzJ8BDU+8Wn/+nayDYXG7pJPrQRVT4oEDFeDa2jlVgsYcbnx65Uc4v6ue74+SX1i1BwD16WUpR0PRNUOjl14f4LUikJRbvmcW0RCYqtMsaRWEp4ilFQErQyiXRKBt7zpT3Pl+/db4qcU4y4oAieuD2rKIjEF5xWzHV2Sa33IfNL3/araK1+B1WMZhkBsCJK/qP9uRfoZD1ZW1LAiwTNiqW8Ci5h+QVFIi5g4QttEeNASxDJYmP15LsqJlGzbFsczAlytN8th7yVwK8EhRoG4kRc+HmQF/fERcWbumbBLm0LPVJ5VOWxZUZUE+RU2LBqCkLBjPB8k55rnjAIz1SSMkGh1slbteloGyOiEdy3tK6Unnz3eibGS1suFDmXx1bxUIY1R9dJycXAvGygmVH3uuWuuaWuaPjzvXX4tqXR5pAONrPLIb3976OtJwrMS18O7Q/B2Y5gS2s1VYQakzCj+KXNfYJW7e9bqbZ5Di5MmjuCw4YaQaPy9GCrRnWzTHCoDb52tqhSUnkZxbR4+bN+3GMuoaRG0LUJT7X7to76XJGaAu3Jd/vSvQxt9af7M+Pm86Jv6xpT49jpfMCgawHfy9ujvQmOBH376MfkzE5vVYznD6P6oVV6N33fW6J0TgxcM34b8/OJXa2uV+PY6RnzrHvrvD1vddnpcxxre97WRD/G27/H5/i2per0xO1+9OVWSQF72BuCtvaNUCOuOL3mVsPd6EjT6TkEE0GL1Zm74eU1t7TYmH9nH7hnyjcUXR5sUWefiWvuEgDVqNlTEH691eDY19qsCYqOTDcb6pqG8us1t7pn3/B/aYQzBq5n0mpSNKM4dDpDJiDQbMfm2KYEwNlrbukMwC01oKby/QPv7WMcZQz8uXeWcpZRBBvVmwcNLNfKR/3MJdy26yc12Y9i19FiHApuz05qnP2nMWIay04j989wz+Z3vd81oaVDnsexIa5oRnt1Obvc3m9e7n+fCI6kyv0dBzKNe9avDJj3892fmnAHM46iI7/bkW0KWBPU6THx4B0x3bCkBtwTqRAxqjf3uBcUmAV14YWjoOHIv2MJcGQCfvmLESo8M2/QjyWZNKuP/SY6T3mj7VPqs5prH/NfjJDdHc3lj7g12POVYNEon8jqaCQuVL1BYVTAXVixOhUbvbqfb896+0K+Xe/TII9bXHtTPR6HlxiFYmHzM8fRheNVsbTnBUHXjBnRMTZCEeNgl3Qo1tV5zUplWwCqjMgqC+pLGii9GKHS/wTP5Kb6mrNvANobghqTYtcD2Y4iBSe64F5wjEEPbh3fp6i+gU5DcKQz5ZCrnW6Je9ZljK7NAxQNa9XguVDX0cjvgZ3UMyqdgiBdNdQTReaevOmfl1hxaVn1XPxbYHffH0c1Rw8QqAaCAqh6VrZN+1nfExTopA64LZ2au0LG3WBuBcBqsyI4qhMU9qBmnZkRTe6snUC6xjS5M07uUfqZeLBUdzRim80GhwoiThuk3+5IChxUvAleqXumAtR2aC7l6qwatxLWshAVtWf7BIcDyCxY0e771BfhZzyAwNsh3RjaBAgTNNsTnA4FoG2wpQbf4GPTFZg4JRIqcOghVN8yLzjHADYMmPVQlBTMtjEwT0Rejx3YihMDsOaOmZEiEuwRUHu99dBQK+1Q9P5HCnkonsGe3k49Y88P9/NvwNB3AGRHDWuGgVNS5zug1519P3M0sD76PjvTydC3xMYNBfNWT75T5jeIqtu6OPYAetNqd6mZAGWOVw+2GtEZg+uRGavp8BxAbZSjCQj8kB+3HOYEkTOT7381TPZNOtgOQwAb/MjdbMskLAjObhePHhpaDQ+XDqUznH/+ugR9HzNe9r857P9VQHBvl8LvQkJ7WVz9O0AZbjkQWuIoAORi483/GDh7garxzeLZXH3pgOa/l3l5/Bij7qc94/mQBmP4sbWUtRdgXUzBq7+YTNbeL0Gqf6uL4H2Q6lkC4DbGQR0/QmCvkVufuMPjSCuDzaG4xayzraIGERi0QDG2o8/T6L35DwDlA5j8EIT+mXcGcP0LoZnjUGBw/z5p6DfDWlIEZ5WXB/33+R0zxG/29ckcXcpvXWRvM3IClLceA2BJUKipy/qoPqbz+Q8B2/Q7s7XSHX45BLiiE3+6L4naesRYG32XNorzOi2Gy1EHYbCMcaipWUqQU8cgLZatDXdw823+NUbka7xu/y4p4IolmsRmRzv9+edP//ivfwYAAP//Yq/3CA==") SupportedMap = make(map[string]Spec) for f, v := range unpacked { diff --git a/x-pack/elastic-agent/pkg/artifact/artifact.go b/x-pack/elastic-agent/pkg/artifact/artifact.go index 72c9ad06a2a..b1dfb4d8a5e 100644 --- a/x-pack/elastic-agent/pkg/artifact/artifact.go +++ b/x-pack/elastic-agent/pkg/artifact/artifact.go @@ -21,14 +21,14 @@ var packageArchMap = map[string]string{ } // GetArtifactName constructs a path to a downloaded artifact -func GetArtifactName(program, version, operatingSystem, arch string) (string, error) { +func GetArtifactName(programName, version, operatingSystem, arch string) (string, error) { key := fmt.Sprintf("%s-binary-%s", operatingSystem, arch) suffix, found := packageArchMap[key] if !found { return "", errors.New(fmt.Sprintf("'%s' is not a valid combination for a package", key), errors.TypeConfig) } - return fmt.Sprintf("%s-%s-%s", program, version, suffix), nil + return fmt.Sprintf("%s-%s-%s", programName, version, suffix), nil } // GetArtifactPath returns a full path of artifact for a program in specific version diff --git a/x-pack/elastic-agent/pkg/artifact/config.go b/x-pack/elastic-agent/pkg/artifact/config.go index aefe35c26e1..a8a09de8e48 100644 --- a/x-pack/elastic-agent/pkg/artifact/config.go +++ b/x-pack/elastic-agent/pkg/artifact/config.go @@ -21,8 +21,8 @@ type Config struct { // Architecture: target architecture [32, 64] Architecture string `json:"-" config:",ignore"` - // BeatsSourceURI: source of the artifacts, e.g https://artifacts.elastic.co/downloads/beats/ - BeatsSourceURI string `json:"sourceURI" config:"sourceURI"` + // SourceURI: source of the artifacts, e.g https://artifacts.elastic.co/downloads/ + SourceURI string `json:"sourceURI" config:"sourceURI"` // TargetDirectory: path to the directory containing downloaded packages TargetDirectory string `json:"targetDirectory" config:"target_directory"` @@ -49,7 +49,7 @@ type Config struct { func DefaultConfig() *Config { dataPath := paths.Data() return &Config{ - BeatsSourceURI: "https://artifacts.elastic.co/downloads/beats/", + SourceURI: "https://artifacts.elastic.co/downloads/", TargetDirectory: filepath.Join(dataPath, "downloads"), Timeout: 30 * time.Second, PgpFile: filepath.Join(dataPath, "elastic.pgp"), diff --git a/x-pack/elastic-agent/pkg/artifact/download/composed/downloader.go b/x-pack/elastic-agent/pkg/artifact/download/composed/downloader.go index c8760861269..9209c8e84c1 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/composed/downloader.go +++ b/x-pack/elastic-agent/pkg/artifact/download/composed/downloader.go @@ -32,11 +32,11 @@ func NewDownloader(downloaders ...download.Downloader) *Downloader { // Download fetches the package from configured source. // Returns absolute path to downloaded package and an error. -func (e *Downloader) Download(ctx context.Context, programName, version string) (string, error) { +func (e *Downloader) Download(ctx context.Context, programName, artifactName, version string) (string, error) { var err error for _, d := range e.dd { - s, e := d.Download(ctx, programName, version) + s, e := d.Download(ctx, programName, artifactName, version) if e == nil { return s, nil } diff --git a/x-pack/elastic-agent/pkg/artifact/download/composed/downloader_test.go b/x-pack/elastic-agent/pkg/artifact/download/composed/downloader_test.go index 9f720fae235..34b4e658041 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/composed/downloader_test.go +++ b/x-pack/elastic-agent/pkg/artifact/download/composed/downloader_test.go @@ -18,7 +18,7 @@ type FailingDownloader struct { called bool } -func (d *FailingDownloader) Download(ctx context.Context, a, b string) (string, error) { +func (d *FailingDownloader) Download(ctx context.Context, a, b, c string) (string, error) { d.called = true return "", errors.New("failing") } @@ -29,7 +29,7 @@ type SuccDownloader struct { called bool } -func (d *SuccDownloader) Download(ctx context.Context, a, b string) (string, error) { +func (d *SuccDownloader) Download(ctx context.Context, a, b, c string) (string, error) { d.called = true return "succ", nil } @@ -58,7 +58,7 @@ func TestComposed(t *testing.T) { for _, tc := range testCases { d := NewDownloader(tc.downloaders[0], tc.downloaders[1]) - r, _ := d.Download(nil, "a", "b") + r, _ := d.Download(nil, "a", "b", "c") assert.Equal(t, tc.expectedResult, r == "succ") diff --git a/x-pack/elastic-agent/pkg/artifact/download/downloader.go b/x-pack/elastic-agent/pkg/artifact/download/downloader.go index 0b7010062e9..7bfbd4b7019 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/downloader.go +++ b/x-pack/elastic-agent/pkg/artifact/download/downloader.go @@ -8,5 +8,5 @@ import "context" // Downloader is an interface allowing download of an artifact type Downloader interface { - Download(ctx context.Context, programName, version string) (string, error) + Download(ctx context.Context, programName, artifactName, version string) (string, error) } diff --git a/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go b/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go index 87740908069..df289ae03ad 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go +++ b/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go @@ -39,7 +39,7 @@ func NewDownloader(config *artifact.Config) *Downloader { // Download fetches the package from configured source. // Returns absolute path to downloaded package and an error. -func (e *Downloader) Download(_ context.Context, programName, version string) (_ string, err error) { +func (e *Downloader) Download(_ context.Context, programName, artifactName, version string) (_ string, err error) { downloadedFiles := make([]string, 0, 2) defer func() { if err != nil { @@ -50,18 +50,18 @@ func (e *Downloader) Download(_ context.Context, programName, version string) (_ }() // download from source to dest - path, err := e.download(e.config.OS(), programName, version) + path, err := e.download(e.config.OS(), programName, artifactName, version) downloadedFiles = append(downloadedFiles, path) if err != nil { return "", err } - hashPath, err := e.downloadHash(e.config.OS(), programName, version) + hashPath, err := e.downloadHash(e.config.OS(), programName, artifactName, version) downloadedFiles = append(downloadedFiles, hashPath) return path, err } -func (e *Downloader) download(operatingSystem, programName, version string) (string, error) { +func (e *Downloader) download(operatingSystem, programName, _, version string) (string, error) { filename, err := artifact.GetArtifactName(programName, version, operatingSystem, e.config.Arch()) if err != nil { return "", errors.New(err, "generating package name failed") @@ -75,7 +75,7 @@ func (e *Downloader) download(operatingSystem, programName, version string) (str return e.downloadFile(filename, fullPath) } -func (e *Downloader) downloadHash(operatingSystem, programName, version string) (string, error) { +func (e *Downloader) downloadHash(operatingSystem, programName, _, version string) (string, error) { filename, err := artifact.GetArtifactName(programName, version, operatingSystem, e.config.Arch()) if err != nil { return "", errors.New(err, "generating package name failed") diff --git a/x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go index 25c5df2efda..4787656d8b0 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go +++ b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go @@ -20,9 +20,9 @@ import ( ) const ( - beatName = "filebeat" - version = "7.5.1" - sourcePattern = "/downloads/beats/filebeat/" + beatName = "filebeat" + artifactName = "beats/filebeat" + version = "7.5.1" ) type testCase struct { @@ -37,6 +37,7 @@ func TestFetchVerify(t *testing.T) { targetPath := filepath.Join("testdata", "download") ctx := context.Background() programName := "beat" + artifactName := "beats/beat" version := "8.0.0" targetFilePath := filepath.Join(targetPath, "beat-8.0.0-darwin-x86_64.tar.gz") @@ -77,7 +78,7 @@ func TestFetchVerify(t *testing.T) { // second one should pass // download not skipped: package missing // verify passes because hash is not correct - _, err = downloader.Download(ctx, programName, version) + _, err = downloader.Download(ctx, programName, artifactName, version) assert.NoError(t, err) // file downloaded ok @@ -146,7 +147,7 @@ func TestVerify(t *testing.T) { } testClient := NewDownloader(config) - artifact, err := testClient.Download(context.Background(), beatName, version) + artifact, err := testClient.Download(context.Background(), beatName, artifactName, version) if err != nil { t.Fatal(err) } diff --git a/x-pack/elastic-agent/pkg/artifact/download/http/downloader.go b/x-pack/elastic-agent/pkg/artifact/download/http/downloader.go index 63d093a6647..358b793fccf 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/http/downloader.go +++ b/x-pack/elastic-agent/pkg/artifact/download/http/downloader.go @@ -51,7 +51,7 @@ func NewDownloaderWithClient(config *artifact.Config, client http.Client) *Downl // Download fetches the package from configured source. // Returns absolute path to downloaded package and an error. -func (e *Downloader) Download(ctx context.Context, programName, version string) (_ string, err error) { +func (e *Downloader) Download(ctx context.Context, programName, artifactName, version string) (_ string, err error) { downloadedFiles := make([]string, 0, 2) defer func() { if err != nil { @@ -62,19 +62,19 @@ func (e *Downloader) Download(ctx context.Context, programName, version string) }() // download from source to dest - path, err := e.download(ctx, e.config.OS(), programName, version) + path, err := e.download(ctx, e.config.OS(), programName, artifactName, version) downloadedFiles = append(downloadedFiles, path) if err != nil { return "", err } - hashPath, err := e.downloadHash(ctx, e.config.OS(), programName, version) + hashPath, err := e.downloadHash(ctx, e.config.OS(), programName, artifactName, version) downloadedFiles = append(downloadedFiles, hashPath) return path, err } -func (e *Downloader) composeURI(programName, packageName string) (string, error) { - upstream := e.config.BeatsSourceURI +func (e *Downloader) composeURI(artifactName, packageName string) (string, error) { + upstream := e.config.SourceURI if !strings.HasPrefix(upstream, "http") && !strings.HasPrefix(upstream, "file") && !strings.HasPrefix(upstream, "/") { // always default to https upstream = fmt.Sprintf("https://%s", upstream) @@ -86,11 +86,11 @@ func (e *Downloader) composeURI(programName, packageName string) (string, error) return "", errors.New(err, "invalid upstream URI", errors.TypeConfig) } - uri.Path = path.Join(uri.Path, programName, packageName) + uri.Path = path.Join(uri.Path, artifactName, packageName) return uri.String(), nil } -func (e *Downloader) download(ctx context.Context, operatingSystem, programName, version string) (string, error) { +func (e *Downloader) download(ctx context.Context, operatingSystem, programName, artifactName, version string) (string, error) { filename, err := artifact.GetArtifactName(programName, version, operatingSystem, e.config.Arch()) if err != nil { return "", errors.New(err, "generating package name failed") @@ -101,10 +101,10 @@ func (e *Downloader) download(ctx context.Context, operatingSystem, programName, return "", errors.New(err, "generating package path failed") } - return e.downloadFile(ctx, programName, filename, fullPath) + return e.downloadFile(ctx, artifactName, filename, fullPath) } -func (e *Downloader) downloadHash(ctx context.Context, operatingSystem, programName, version string) (string, error) { +func (e *Downloader) downloadHash(ctx context.Context, operatingSystem, programName, artifactName, version string) (string, error) { filename, err := artifact.GetArtifactName(programName, version, operatingSystem, e.config.Arch()) if err != nil { return "", errors.New(err, "generating package name failed") @@ -118,11 +118,11 @@ func (e *Downloader) downloadHash(ctx context.Context, operatingSystem, programN filename = filename + ".sha512" fullPath = fullPath + ".sha512" - return e.downloadFile(ctx, programName, filename, fullPath) + return e.downloadFile(ctx, artifactName, filename, fullPath) } -func (e *Downloader) downloadFile(ctx context.Context, programName, filename, fullPath string) (string, error) { - sourceURI, err := e.composeURI(programName, filename) +func (e *Downloader) downloadFile(ctx context.Context, artifactName, filename, fullPath string) (string, error) { + sourceURI, err := e.composeURI(artifactName, filename) if err != nil { return "", err } diff --git a/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go b/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go index 8087fc28823..a1e1ced0b66 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go +++ b/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go @@ -23,9 +23,10 @@ import ( const ( beatName = "filebeat" + artifactName = "beats/filebeat" version = "7.5.1" sourcePattern = "/downloads/beats/filebeat/" - source = "http://artifacts.elastic.co/downloads/beats/" + source = "http://artifacts.elastic.co/downloads/" ) type testCase struct { @@ -44,7 +45,7 @@ func TestDownload(t *testing.T) { elasticClient := getElasticCoClient() config := &artifact.Config{ - BeatsSourceURI: source, + SourceURI: source, TargetDirectory: targetDir, Timeout: timeout, } @@ -56,7 +57,7 @@ func TestDownload(t *testing.T) { config.Architecture = testCase.arch testClient := NewDownloaderWithClient(config, elasticClient) - artifactPath, err := testClient.Download(context.Background(), beatName, version) + artifactPath, err := testClient.Download(context.Background(), beatName, artifactName, version) if err != nil { t.Fatal(err) } @@ -82,7 +83,7 @@ func TestVerify(t *testing.T) { elasticClient := getElasticCoClient() config := &artifact.Config{ - BeatsSourceURI: source, + SourceURI: source, TargetDirectory: targetDir, Timeout: timeout, } @@ -94,7 +95,7 @@ func TestVerify(t *testing.T) { config.Architecture = testCase.arch testClient := NewDownloaderWithClient(config, elasticClient) - artifact, err := testClient.Download(context.Background(), beatName, version) + artifact, err := testClient.Download(context.Background(), beatName, artifactName, version) if err != nil { t.Fatal(err) } diff --git a/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go b/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go index 486f222c8e5..171b3363aec 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go +++ b/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go @@ -173,7 +173,7 @@ func (v *Verifier) verifyAsc(programName, version string) (bool, error) { } func (v *Verifier) composeURI(programName, filename string) (string, error) { - upstream := v.config.BeatsSourceURI + upstream := v.config.SourceURI if !strings.HasPrefix(upstream, "http") && !strings.HasPrefix(upstream, "file") && !strings.HasPrefix(upstream, "/") { // always default to https upstream = fmt.Sprintf("https://%s", upstream) @@ -185,7 +185,7 @@ func (v *Verifier) composeURI(programName, filename string) (string, error) { return "", errors.New(err, "invalid upstream URI", errors.TypeNetwork, errors.M(errors.MetaKeyURI, upstream)) } - uri.Path = path.Join(uri.Path, programName, filename+ascSuffix) + uri.Path = path.Join(uri.Path, "beats", programName, filename+ascSuffix) return uri.String(), nil } diff --git a/x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go b/x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go index 022cda0ffff..f45a38d3f97 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go +++ b/x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go @@ -35,7 +35,7 @@ func snapshotConfig(config *artifact.Config) (*artifact.Config, error) { return &artifact.Config{ OperatingSystem: config.OperatingSystem, Architecture: config.Architecture, - BeatsSourceURI: snapshotURI, + SourceURI: snapshotURI, TargetDirectory: config.TargetDirectory, Timeout: config.Timeout, PgpFile: config.PgpFile, @@ -81,7 +81,7 @@ func snapshotURI() (string, error) { return "", fmt.Errorf("uri is not a string") } - index := strings.Index(uri, "/elastic-agent/") + index := strings.Index(uri, "/beats/elastic-agent/") if index == -1 { return "", fmt.Errorf("not an agent uri: '%s'", uri) } diff --git a/x-pack/elastic-agent/pkg/artifact/install/tar/tar_installer.go b/x-pack/elastic-agent/pkg/artifact/install/tar/tar_installer.go index f73ace5765a..5c7f0f593a3 100644 --- a/x-pack/elastic-agent/pkg/artifact/install/tar/tar_installer.go +++ b/x-pack/elastic-agent/pkg/artifact/install/tar/tar_installer.go @@ -50,8 +50,9 @@ func (i *Installer) Install(_ context.Context, programName, version, installDir os.RemoveAll(installDir) } - return unpack(f, installDir) - + // unpack must occur in directory that holds the installation directory + // or the extraction will be double nested + return unpack(f, filepath.Dir(installDir)) } func unpack(r io.Reader, dir string) error { diff --git a/x-pack/elastic-agent/pkg/core/app/descriptor.go b/x-pack/elastic-agent/pkg/core/app/descriptor.go index 5f556405971..330b0da11e7 100644 --- a/x-pack/elastic-agent/pkg/core/app/descriptor.go +++ b/x-pack/elastic-agent/pkg/core/app/descriptor.go @@ -16,6 +16,7 @@ import ( // Descriptor defines a program which needs to be run. // Is passed around operator operations. type Descriptor struct { + artifactName string executionCtx ExecutionContext directory string spec ProcessSpec @@ -27,6 +28,7 @@ func NewDescriptor(pSpec program.Spec, version string, config *artifact.Config, dir := directory(binaryName, version, config) return &Descriptor{ + artifactName: pSpec.Artifact, directory: dir, executionCtx: NewExecutionContext(pSpec.ServicePort, binaryName, version, tags), spec: spec(dir, binaryName), @@ -39,6 +41,11 @@ func (p *Descriptor) ServicePort() int { return p.executionCtx.ServicePort } +// ArtifactName is the name of the artifact to download from the artifact store. E.g beats/filebeat. +func (p *Descriptor) ArtifactName() string { + return p.artifactName +} + // BinaryName is the name of the binary. E.g filebeat. func (p *Descriptor) BinaryName() string { return p.executionCtx.BinaryName diff --git a/x-pack/elastic-agent/spec/endpoint.yml b/x-pack/elastic-agent/spec/endpoint.yml index e864c182e6e..18e56cf82ad 100644 --- a/x-pack/elastic-agent/spec/endpoint.yml +++ b/x-pack/elastic-agent/spec/endpoint.yml @@ -1,5 +1,6 @@ name: Endpoint Security cmd: endpoint-security +artifact: endpoint-dev service: 6788 check_install: - exec_file: diff --git a/x-pack/elastic-agent/spec/filebeat.yml b/x-pack/elastic-agent/spec/filebeat.yml index 20305373674..81c82e276ae 100644 --- a/x-pack/elastic-agent/spec/filebeat.yml +++ b/x-pack/elastic-agent/spec/filebeat.yml @@ -1,6 +1,7 @@ name: Filebeat cmd: filebeat args: ["-E", "setup.ilm.enabled=false", "-E", "setup.template.enabled=false", "-E", "management.mode=x-pack-fleet", "-E", "management.enabled=true", "-E", "logging.level=debug"] +artifact: beats/filebeat rules: - fix_stream: {} - inject_index: diff --git a/x-pack/elastic-agent/spec/metricbeat.yml b/x-pack/elastic-agent/spec/metricbeat.yml index 14552838b75..fcc8428517a 100644 --- a/x-pack/elastic-agent/spec/metricbeat.yml +++ b/x-pack/elastic-agent/spec/metricbeat.yml @@ -1,6 +1,7 @@ name: Metricbeat cmd: metricbeat args: ["-E", "setup.ilm.enabled=false", "-E", "setup.template.enabled=false", "-E", "management.mode=x-pack-fleet", "-E", "management.enabled=true", "-E", "logging.level=debug"] +artifact: beats/metricbeat post_install: - move_file: path: "modules.d/system.yml"