From cedd26ba0bf576a10f2fa2e0a2bd5489c666fb20 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Mon, 19 Oct 2020 09:15:27 +0200 Subject: [PATCH 1/4] use snapshot as a backup always for upgrade --- .../application/upgrade/step_download.go | 54 +++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go index 3aea96da0ab..cf4c8b77ad2 100644 --- a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go +++ b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go @@ -9,7 +9,12 @@ import ( "strings" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" - downloader "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/localremote" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/composed" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/fs" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/http" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/snapshot" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" ) @@ -26,13 +31,12 @@ func (u *Upgrader) downloadArtifact(ctx context.Context, version, sourceURI stri } } - allowEmptyPgp, pgp := release.PGP() - verifier, err := downloader.NewVerifier(u.log, &settings, allowEmptyPgp, pgp) + verifier, err := u.verifier(&settings) if err != nil { return "", errors.New(err, "initiating verifier") } - fetcher := downloader.NewDownloader(u.log, &settings) + fetcher := u.downloader(&settings) path, err := fetcher.Download(ctx, agentName, agentArtifactName, version) if err != nil { return "", errors.New(err, "failed upgrade of agent binary") @@ -48,3 +52,45 @@ func (u *Upgrader) downloadArtifact(ctx context.Context, version, sourceURI stri return path, nil } + +// gets a downloader for local, official, snapshot in that order +func (u *Upgrader) downloader(settings *artifact.Config) download.Downloader { + downloaders := make([]download.Downloader, 0, 3) + downloaders = append(downloaders, fs.NewDownloader(settings), http.NewDownloader(settings)) + + snapDownloader, err := snapshot.NewDownloader(settings) + if err != nil { + u.log.Error(err) + } else { + downloaders = append(downloaders, snapDownloader) + } + + return composed.NewDownloader(downloaders...) +} + +// gets a verifier for local, official, snapshot in that order +func (u *Upgrader) verifier(settings *artifact.Config) (download.Verifier, error) { + allowEmptyPgp, pgp := release.PGP() + verifiers := make([]download.Verifier, 0, 3) + + fsVer, err := fs.NewVerifier(settings, allowEmptyPgp, pgp) + if err != nil { + return nil, err + } + verifiers = append(verifiers, fsVer) + + remoteVer, err := http.NewVerifier(settings, allowEmptyPgp, pgp) + if err != nil { + return nil, err + } + verifiers = append(verifiers, remoteVer) + + snapshotVerifier, err := snapshot.NewVerifier(settings, allowEmptyPgp, pgp) + if err != nil { + u.log.Error(err) + } else { + verifiers = append(verifiers, snapshotVerifier) + } + + return composed.NewVerifier(verifiers...), nil +} From 8fe8a14a733d058e0a0b5fa452acefb8ca4ec1d3 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Mon, 19 Oct 2020 09:19:15 +0200 Subject: [PATCH 2/4] arg approach --- .../pkg/agent/application/stream.go | 4 +- .../application/upgrade/step_download.go | 54 ++----------------- .../download/localremote/downloader.go | 4 +- .../artifact/download/localremote/verifier.go | 4 +- 4 files changed, 10 insertions(+), 56 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/application/stream.go b/x-pack/elastic-agent/pkg/agent/application/stream.go index 41999fcb832..784038e77ab 100644 --- a/x-pack/elastic-agent/pkg/agent/application/stream.go +++ b/x-pack/elastic-agent/pkg/agent/application/stream.go @@ -56,9 +56,9 @@ func streamFactory(ctx context.Context, cfg *configuration.SettingsConfig, srv * } func newOperator(ctx context.Context, log *logger.Logger, id routingKey, config *configuration.SettingsConfig, srv *server.Server, r state.Reporter, m monitoring.Monitor) (*operation.Operator, error) { - fetcher := downloader.NewDownloader(log, config.DownloadConfig) + fetcher := downloader.NewDownloader(log, config.DownloadConfig, false) allowEmptyPgp, pgp := release.PGP() - verifier, err := downloader.NewVerifier(log, config.DownloadConfig, allowEmptyPgp, pgp) + verifier, err := downloader.NewVerifier(log, config.DownloadConfig, allowEmptyPgp, pgp, false) if err != nil { return nil, errors.New(err, "initiating verifier") } diff --git a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go index cf4c8b77ad2..58af508e1aa 100644 --- a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go +++ b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go @@ -9,12 +9,7 @@ import ( "strings" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/composed" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/fs" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/http" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/snapshot" + download "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/localremote" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" ) @@ -31,12 +26,13 @@ func (u *Upgrader) downloadArtifact(ctx context.Context, version, sourceURI stri } } - verifier, err := u.verifier(&settings) + allowEmptyPgp, pgp := release.PGP() + verifier, err := download.NewVerifier(u.log, &settings, allowEmptyPgp, pgp, true) if err != nil { return "", errors.New(err, "initiating verifier") } - fetcher := u.downloader(&settings) + fetcher := download.NewDownloader(u.log, &settings, true) path, err := fetcher.Download(ctx, agentName, agentArtifactName, version) if err != nil { return "", errors.New(err, "failed upgrade of agent binary") @@ -52,45 +48,3 @@ func (u *Upgrader) downloadArtifact(ctx context.Context, version, sourceURI stri return path, nil } - -// gets a downloader for local, official, snapshot in that order -func (u *Upgrader) downloader(settings *artifact.Config) download.Downloader { - downloaders := make([]download.Downloader, 0, 3) - downloaders = append(downloaders, fs.NewDownloader(settings), http.NewDownloader(settings)) - - snapDownloader, err := snapshot.NewDownloader(settings) - if err != nil { - u.log.Error(err) - } else { - downloaders = append(downloaders, snapDownloader) - } - - return composed.NewDownloader(downloaders...) -} - -// gets a verifier for local, official, snapshot in that order -func (u *Upgrader) verifier(settings *artifact.Config) (download.Verifier, error) { - allowEmptyPgp, pgp := release.PGP() - verifiers := make([]download.Verifier, 0, 3) - - fsVer, err := fs.NewVerifier(settings, allowEmptyPgp, pgp) - if err != nil { - return nil, err - } - verifiers = append(verifiers, fsVer) - - remoteVer, err := http.NewVerifier(settings, allowEmptyPgp, pgp) - if err != nil { - return nil, err - } - verifiers = append(verifiers, remoteVer) - - snapshotVerifier, err := snapshot.NewVerifier(settings, allowEmptyPgp, pgp) - if err != nil { - u.log.Error(err) - } else { - verifiers = append(verifiers, snapshotVerifier) - } - - return composed.NewVerifier(verifiers...), nil -} diff --git a/x-pack/elastic-agent/pkg/artifact/download/localremote/downloader.go b/x-pack/elastic-agent/pkg/artifact/download/localremote/downloader.go index 6448af25aca..ba82195ffbd 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/localremote/downloader.go +++ b/x-pack/elastic-agent/pkg/artifact/download/localremote/downloader.go @@ -17,12 +17,12 @@ import ( // NewDownloader creates a downloader which first checks local directory // and then fallbacks to remote if configured. -func NewDownloader(log *logger.Logger, config *artifact.Config) download.Downloader { +func NewDownloader(log *logger.Logger, config *artifact.Config, forceSnapshot bool) download.Downloader { downloaders := make([]download.Downloader, 0, 3) downloaders = append(downloaders, fs.NewDownloader(config)) // try snapshot repo before official - if release.Snapshot() { + if release.Snapshot() || forceSnapshot { snapDownloader, err := snapshot.NewDownloader(config) if err != nil { log.Error(err) diff --git a/x-pack/elastic-agent/pkg/artifact/download/localremote/verifier.go b/x-pack/elastic-agent/pkg/artifact/download/localremote/verifier.go index 4f33cbbdb8e..30517d12d3d 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/localremote/verifier.go +++ b/x-pack/elastic-agent/pkg/artifact/download/localremote/verifier.go @@ -17,7 +17,7 @@ import ( // NewVerifier creates a downloader which first checks local directory // and then fallbacks to remote if configured. -func NewVerifier(log *logger.Logger, config *artifact.Config, allowEmptyPgp bool, pgp []byte) (download.Verifier, error) { +func NewVerifier(log *logger.Logger, config *artifact.Config, allowEmptyPgp bool, pgp []byte, forceSnapshot bool) (download.Verifier, error) { verifiers := make([]download.Verifier, 0, 3) fsVer, err := fs.NewVerifier(config, allowEmptyPgp, pgp) @@ -27,7 +27,7 @@ func NewVerifier(log *logger.Logger, config *artifact.Config, allowEmptyPgp bool verifiers = append(verifiers, fsVer) // try snapshot repo before official - if release.Snapshot() { + if release.Snapshot() || forceSnapshot { snapshotVerifier, err := snapshot.NewVerifier(config, allowEmptyPgp, pgp) if err != nil { log.Error(err) From 3a484020ffeb3bcca6360059900e7f9ae55a7eed Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Mon, 19 Oct 2020 12:04:45 +0200 Subject: [PATCH 3/4] download downloader,tomatos tomatoes --- .../pkg/agent/application/upgrade/step_download.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go index 58af508e1aa..0294308ff3a 100644 --- a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go +++ b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" - download "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/localremote" + downloader "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/localremote" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" ) @@ -27,12 +27,12 @@ func (u *Upgrader) downloadArtifact(ctx context.Context, version, sourceURI stri } allowEmptyPgp, pgp := release.PGP() - verifier, err := download.NewVerifier(u.log, &settings, allowEmptyPgp, pgp, true) + verifier, err := downloader.NewVerifier(u.log, &settings, allowEmptyPgp, pgp, true) if err != nil { return "", errors.New(err, "initiating verifier") } - fetcher := download.NewDownloader(u.log, &settings, true) + fetcher := downloader.NewDownloader(u.log, &settings, true) path, err := fetcher.Download(ctx, agentName, agentArtifactName, version) if err != nil { return "", errors.New(err, "failed upgrade of agent binary") From 73f01d7845f4a65759307ac5bd76daa00bb8666b Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Mon, 19 Oct 2020 14:37:33 +0200 Subject: [PATCH 4/4] dont require uri for snapshot --- x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go b/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go index 1a21bc154a1..d7e69fc3972 100644 --- a/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go +++ b/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go @@ -183,9 +183,6 @@ func (u *Upgrader) Ack(ctx context.Context) error { } func (u *Upgrader) sourceURI(version, retrievedURI string) (string, error) { - if strings.HasSuffix(version, "-SNAPSHOT") && retrievedURI == "" { - return "", errors.New("snapshot upgrade requires source uri", errors.TypeConfig) - } if retrievedURI != "" { return retrievedURI, nil }