From 817041191cb77f12f0ba5b54f670d07576ff1cdf Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Thu, 28 Jul 2022 01:42:46 -0400 Subject: [PATCH 1/3] [Automation] Update elastic stack version to 8.4.0-60171339 for testing (#799) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index b8b6792b912..3b747702258 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-d058e92f-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-60171339-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -42,7 +42,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-d058e92f-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-60171339-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 7a88635a793b3e2ae5301fe9fd5463fb1261c01f Mon Sep 17 00:00:00 2001 From: Florian Lehner Date: Fri, 29 Jul 2022 15:08:05 +0200 Subject: [PATCH 2/3] update dependency elastic/go-structform from v0.0.9 to v0.0.10 (#802) Signed-off-by: Florian Lehner --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 8b5c887cbe4..3949233c361 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -8227,11 +8227,11 @@ Contents of probable licence file $GOMODCACHE/github.com/docker/go-connections@v -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-structform -Version: v0.0.9 +Version: v0.0.10 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v0.0.9/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v0.0.10/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 95044f69abc..dd1d29345e3 100644 --- a/go.mod +++ b/go.mod @@ -72,7 +72,7 @@ require ( github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/docker v20.10.12+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect - github.com/elastic/go-structform v0.0.9 // indirect + github.com/elastic/go-structform v0.0.10 // indirect github.com/elastic/go-windows v1.0.1 // indirect github.com/elastic/gosigar v0.14.2 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect diff --git a/go.sum b/go.sum index 5026651a90b..4b63a456695 100644 --- a/go.sum +++ b/go.sum @@ -397,8 +397,9 @@ github.com/elastic/go-elasticsearch/v8 v8.0.0-20210317102009-a9d74cec0186/go.mod github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ= github.com/elastic/go-licenser v0.4.0 h1:jLq6A5SilDS/Iz1ABRkO6BHy91B9jBora8FwGRsDqUI= github.com/elastic/go-licenser v0.4.0/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= -github.com/elastic/go-structform v0.0.9 h1:HpcS7xljL4kSyUfDJ8cXTJC6rU5ChL1wYb6cx3HLD+o= github.com/elastic/go-structform v0.0.9/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= +github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gnCJPDo1w= +github.com/elastic/go-structform v0.0.10/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-sysinfo v1.7.1 h1:Wx4DSARcKLllpKT2TnFVdSUJOsybqMYCNQZq1/wO+s0= github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= From 74ce2ba8f7643280e2d5441018e44c10c93dd4e9 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Mon, 1 Aug 2022 08:50:04 +0200 Subject: [PATCH 3/3] Fix unpacking of artifact config (#776) Fix unpacking of artifact config (#776) --- internal/pkg/agent/application/local_mode.go | 4 +- internal/pkg/artifact/config.go | 84 +++++- internal/pkg/artifact/config_test.go | 247 ++++++++++++++++++ .../artifact/download/snapshot/downloader.go | 13 +- 4 files changed, 337 insertions(+), 11 deletions(-) create mode 100644 internal/pkg/artifact/config_test.go diff --git a/internal/pkg/agent/application/local_mode.go b/internal/pkg/agent/application/local_mode.go index 29f311fe582..f06949bcba1 100644 --- a/internal/pkg/agent/application/local_mode.go +++ b/internal/pkg/agent/application/local_mode.go @@ -22,6 +22,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/configuration" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" "github.com/elastic/elastic-agent/internal/pkg/agent/operation" + "github.com/elastic/elastic-agent/internal/pkg/artifact" "github.com/elastic/elastic-agent/internal/pkg/capabilities" "github.com/elastic/elastic-agent/internal/pkg/composable" "github.com/elastic/elastic-agent/internal/pkg/config" @@ -131,6 +132,7 @@ func newLocal( }, caps, monitor, + artifact.NewReloader(cfg.Settings.DownloadConfig, log), ) if err != nil { return nil, err @@ -203,7 +205,7 @@ func (l *Local) AgentInfo() *info.AgentInfo { } func discoverer(patterns ...string) discoverFunc { - var p []string + p := make([]string, 0, len(patterns)) for _, newP := range patterns { if len(newP) == 0 { continue diff --git a/internal/pkg/artifact/config.go b/internal/pkg/artifact/config.go index fa57ca06870..d88031e5de5 100644 --- a/internal/pkg/artifact/config.go +++ b/internal/pkg/artifact/config.go @@ -9,6 +9,7 @@ import ( "strings" "time" + c "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/transport/httpcommon" "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" @@ -64,6 +65,42 @@ func NewReloader(cfg *Config, log *logger.Logger) *Reloader { } func (r *Reloader) Reload(rawConfig *config.Config) error { + if err := r.reloadConfig(rawConfig); err != nil { + return errors.New(err, "failed to reload config") + } + + if err := r.reloadSourceURI(rawConfig); err != nil { + return errors.New(err, "failed to reload source URI") + } + + return nil +} + +func (r *Reloader) reloadConfig(rawConfig *config.Config) error { + type reloadConfig struct { + C *Config `json:"agent.download" config:"agent.download"` + } + tmp := &reloadConfig{ + C: DefaultConfig(), + } + if err := rawConfig.Unpack(&tmp); err != nil { + return err + } + + *(r.cfg) = Config{ + OperatingSystem: tmp.C.OperatingSystem, + Architecture: tmp.C.Architecture, + SourceURI: tmp.C.SourceURI, + TargetDirectory: tmp.C.TargetDirectory, + InstallPath: tmp.C.InstallPath, + DropPath: tmp.C.DropPath, + HTTPTransportSettings: tmp.C.HTTPTransportSettings, + } + + return nil +} + +func (r *Reloader) reloadSourceURI(rawConfig *config.Config) error { type reloadConfig struct { // SourceURI: source of the artifacts, e.g https://artifacts.elastic.co/downloads/ SourceURI string `json:"agent.download.sourceURI" config:"agent.download.sourceURI"` @@ -78,11 +115,11 @@ func (r *Reloader) Reload(rawConfig *config.Config) error { } var newSourceURI string - if cfg.FleetSourceURI != "" { + if fleetURI := strings.TrimSpace(cfg.FleetSourceURI); fleetURI != "" { // fleet configuration takes precedence - newSourceURI = cfg.FleetSourceURI - } else if cfg.SourceURI != "" { - newSourceURI = cfg.SourceURI + newSourceURI = fleetURI + } else if sourceURI := strings.TrimSpace(cfg.SourceURI); sourceURI != "" { + newSourceURI = sourceURI } if newSourceURI != "" { @@ -148,3 +185,42 @@ func (c *Config) Arch() string { c.Architecture = arch return c.Architecture } + +// Unpack reads a config object into the settings. +func (c *Config) Unpack(cfg *c.C) error { + tmp := struct { + OperatingSystem string `json:"-" config:",ignore"` + Architecture string `json:"-" config:",ignore"` + SourceURI string `json:"sourceURI" config:"sourceURI"` + TargetDirectory string `json:"targetDirectory" config:"target_directory"` + InstallPath string `yaml:"installPath" config:"install_path"` + DropPath string `yaml:"dropPath" config:"drop_path"` + }{ + OperatingSystem: c.OperatingSystem, + Architecture: c.Architecture, + SourceURI: c.SourceURI, + TargetDirectory: c.TargetDirectory, + InstallPath: c.InstallPath, + DropPath: c.DropPath, + } + + if err := cfg.Unpack(&tmp); err != nil { + return err + } + + transport := DefaultConfig().HTTPTransportSettings + if err := cfg.Unpack(&transport); err != nil { + return err + } + + *c = Config{ + OperatingSystem: tmp.OperatingSystem, + Architecture: tmp.Architecture, + SourceURI: tmp.SourceURI, + TargetDirectory: tmp.TargetDirectory, + InstallPath: tmp.InstallPath, + DropPath: tmp.DropPath, + HTTPTransportSettings: transport, + } + return nil +} diff --git a/internal/pkg/artifact/config_test.go b/internal/pkg/artifact/config_test.go new file mode 100644 index 00000000000..3a9a694b757 --- /dev/null +++ b/internal/pkg/artifact/config_test.go @@ -0,0 +1,247 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package artifact + +import ( + "testing" + "time" + + "github.com/elastic/elastic-agent/internal/pkg/config" + "github.com/elastic/elastic-agent/pkg/core/logger" + "github.com/stretchr/testify/require" +) + +func TestReload(t *testing.T) { + type testCase struct { + input string + initialConfig *Config + expectedSourceURI string + expectedTargetDirectory string + expectedInstallDirectory string + expectedDropDirectory string + expectedFingerprint string + expectedTLS bool + expectedTLSEnabled bool + expectedDisableProxy bool + expectedTimeout time.Duration + } + defaultValues := DefaultConfig() + testCases := []testCase{ + { + input: `agent.download: + sourceURI: "testing.uri" + target_directory: "a/b/c" + install_path: "i/p" + drop_path: "d/p" + proxy_disable: true + timeout: 33s + ssl.enabled: true + ssl.ca_trusted_fingerprint: "my_finger_print" +`, + initialConfig: DefaultConfig(), + expectedSourceURI: "testing.uri", + expectedTargetDirectory: "a/b/c", + expectedInstallDirectory: "i/p", + expectedDropDirectory: "d/p", + expectedFingerprint: "my_finger_print", + expectedTLS: true, + expectedTLSEnabled: true, + expectedDisableProxy: true, + expectedTimeout: 33 * time.Second, + }, + { + input: `agent.download: + sourceURI: "testing.uri" +`, + initialConfig: DefaultConfig(), + expectedSourceURI: "testing.uri", + expectedTargetDirectory: defaultValues.TargetDirectory, + expectedInstallDirectory: defaultValues.InstallPath, + expectedDropDirectory: defaultValues.DropPath, + expectedFingerprint: "", + expectedTLS: defaultValues.TLS != nil, + expectedTLSEnabled: false, + expectedDisableProxy: defaultValues.Proxy.Disable, + expectedTimeout: defaultValues.Timeout, + }, + { + input: `agent.download: + sourceURI: "" +`, + initialConfig: &Config{ + SourceURI: "testing.uri", + HTTPTransportSettings: defaultValues.HTTPTransportSettings, + }, + expectedSourceURI: defaultValues.SourceURI, // fallback to default when set to empty + expectedTargetDirectory: defaultValues.TargetDirectory, + expectedInstallDirectory: defaultValues.InstallPath, + expectedDropDirectory: defaultValues.DropPath, + expectedFingerprint: "", + expectedTLS: defaultValues.TLS != nil, + expectedTLSEnabled: false, + expectedDisableProxy: defaultValues.Proxy.Disable, + expectedTimeout: defaultValues.Timeout, + }, + { + input: ``, + initialConfig: &Config{ + SourceURI: "testing.uri", + HTTPTransportSettings: defaultValues.HTTPTransportSettings, + }, + expectedSourceURI: defaultValues.SourceURI, // fallback to default when not set + expectedTargetDirectory: defaultValues.TargetDirectory, + expectedInstallDirectory: defaultValues.InstallPath, + expectedDropDirectory: defaultValues.DropPath, + expectedFingerprint: "", + expectedTLS: defaultValues.TLS != nil, + expectedTLSEnabled: false, + expectedDisableProxy: defaultValues.Proxy.Disable, + expectedTimeout: defaultValues.Timeout, + }, + { + input: `agent.download: + sourceURI: " " +`, + initialConfig: &Config{ + SourceURI: "testing.uri", + HTTPTransportSettings: defaultValues.HTTPTransportSettings, + }, + expectedSourceURI: defaultValues.SourceURI, // fallback to default when set to whitespace + expectedTargetDirectory: defaultValues.TargetDirectory, + expectedInstallDirectory: defaultValues.InstallPath, + expectedDropDirectory: defaultValues.DropPath, + expectedFingerprint: "", + expectedTLS: defaultValues.TLS != nil, + expectedTLSEnabled: false, + expectedDisableProxy: defaultValues.Proxy.Disable, + expectedTimeout: defaultValues.Timeout, + }, + { + input: `agent.download: + source_uri: " " +`, + initialConfig: &Config{ + SourceURI: "testing.uri", + HTTPTransportSettings: defaultValues.HTTPTransportSettings, + }, + expectedSourceURI: defaultValues.SourceURI, // fallback to default when set to whitespace + expectedTargetDirectory: defaultValues.TargetDirectory, + expectedInstallDirectory: defaultValues.InstallPath, + expectedDropDirectory: defaultValues.DropPath, + expectedFingerprint: "", + expectedTLS: defaultValues.TLS != nil, + expectedTLSEnabled: false, + expectedDisableProxy: defaultValues.Proxy.Disable, + expectedTimeout: defaultValues.Timeout, + }, + { + input: `agent.download: + source_uri: " " + sourceURI: " " +`, + initialConfig: DefaultConfig(), + expectedSourceURI: defaultValues.SourceURI, // fallback to default when set to whitespace + expectedTargetDirectory: defaultValues.TargetDirectory, + expectedInstallDirectory: defaultValues.InstallPath, + expectedDropDirectory: defaultValues.DropPath, + expectedFingerprint: "", + expectedTLS: defaultValues.TLS != nil, + expectedTLSEnabled: false, + expectedDisableProxy: defaultValues.Proxy.Disable, + expectedTimeout: defaultValues.Timeout, + }, + { + input: ``, + initialConfig: &Config{ + SourceURI: "testing.uri", + HTTPTransportSettings: defaultValues.HTTPTransportSettings, + }, + expectedSourceURI: defaultValues.SourceURI, + expectedTargetDirectory: defaultValues.TargetDirectory, + expectedInstallDirectory: defaultValues.InstallPath, + expectedDropDirectory: defaultValues.DropPath, + expectedFingerprint: "", + expectedTLS: defaultValues.TLS != nil, + expectedTLSEnabled: false, + expectedDisableProxy: defaultValues.Proxy.Disable, + expectedTimeout: defaultValues.Timeout, + }, + { + input: `agent.download: + source_uri: " " + sourceURI: "testing.uri" +`, + initialConfig: DefaultConfig(), + expectedSourceURI: "testing.uri", + expectedTargetDirectory: defaultValues.TargetDirectory, + expectedInstallDirectory: defaultValues.InstallPath, + expectedDropDirectory: defaultValues.DropPath, + expectedFingerprint: "", + expectedTLS: defaultValues.TLS != nil, + expectedTLSEnabled: false, + expectedDisableProxy: defaultValues.Proxy.Disable, + expectedTimeout: defaultValues.Timeout, + }, + { + input: `agent.download: + source_uri: "testing.uri" + sourceURI: " " +`, + initialConfig: DefaultConfig(), + expectedSourceURI: "testing.uri", + expectedTargetDirectory: defaultValues.TargetDirectory, + expectedInstallDirectory: defaultValues.InstallPath, + expectedDropDirectory: defaultValues.DropPath, + expectedFingerprint: "", + expectedTLS: defaultValues.TLS != nil, + expectedTLSEnabled: false, + expectedDisableProxy: defaultValues.Proxy.Disable, + expectedTimeout: defaultValues.Timeout, + }, + { + input: `agent.download: + source_uri: "testing.uri" + sourceURI: "another.uri" +`, + initialConfig: DefaultConfig(), + expectedSourceURI: "testing.uri", + expectedTargetDirectory: defaultValues.TargetDirectory, + expectedInstallDirectory: defaultValues.InstallPath, + expectedDropDirectory: defaultValues.DropPath, + expectedFingerprint: "", + expectedTLS: defaultValues.TLS != nil, + expectedTLSEnabled: false, + expectedDisableProxy: defaultValues.Proxy.Disable, + expectedTimeout: defaultValues.Timeout, + }, + } + + l, _ := logger.NewTesting("t") + for _, tc := range testCases { + cfg := tc.initialConfig + reloader := NewReloader(cfg, l) + + c, err := config.NewConfigFrom(tc.input) + require.NoError(t, err) + + require.NoError(t, reloader.Reload(c)) + + require.Equal(t, tc.expectedSourceURI, cfg.SourceURI) + require.Equal(t, tc.expectedTargetDirectory, cfg.TargetDirectory) + require.Equal(t, tc.expectedInstallDirectory, cfg.InstallPath) + require.Equal(t, tc.expectedDropDirectory, cfg.DropPath) + require.Equal(t, tc.expectedTimeout, cfg.Timeout) + + require.Equal(t, tc.expectedDisableProxy, cfg.Proxy.Disable) + + if tc.expectedTLS { + require.NotNil(t, cfg.TLS) + require.Equal(t, tc.expectedTLSEnabled, *cfg.TLS.Enabled) + require.Equal(t, tc.expectedFingerprint, cfg.TLS.CATrustedFingerprint) + } else { + require.Nil(t, cfg.TLS) + } + } +} diff --git a/internal/pkg/artifact/download/snapshot/downloader.go b/internal/pkg/artifact/download/snapshot/downloader.go index 2fbe027ae4b..c3680147927 100644 --- a/internal/pkg/artifact/download/snapshot/downloader.go +++ b/internal/pkg/artifact/download/snapshot/downloader.go @@ -34,12 +34,13 @@ func snapshotConfig(config *artifact.Config, versionOverride string) (*artifact. } return &artifact.Config{ - OperatingSystem: config.OperatingSystem, - Architecture: config.Architecture, - SourceURI: snapshotURI, - TargetDirectory: config.TargetDirectory, - InstallPath: config.InstallPath, - DropPath: config.DropPath, + OperatingSystem: config.OperatingSystem, + Architecture: config.Architecture, + SourceURI: snapshotURI, + TargetDirectory: config.TargetDirectory, + InstallPath: config.InstallPath, + DropPath: config.DropPath, + HTTPTransportSettings: config.HTTPTransportSettings, }, nil }