From 34d8a976ad82e2a08f4a238cc64579400b64bbc1 Mon Sep 17 00:00:00 2001 From: narph Date: Mon, 10 Jan 2022 15:25:23 +0100 Subject: [PATCH 1/8] update func --- internal/utils/utils.go | 9 +++++++-- internal/utils/utils_test.go | 2 +- pkg/downloads/versions.go | 8 ++++---- pkg/downloads/versions_test.go | 24 ++++++++++++------------ 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/internal/utils/utils.go b/internal/utils/utils.go index f0adc405b2..e73841de7e 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -40,7 +40,8 @@ func GetArchitecture() string { // DownloadFile will download a url and store it in a temporary path. // It writes to the destination file as it downloads it, without // loading the entire file into memory. -func DownloadFile(url string) (string, error) { +func DownloadFile(url string, downloadPath string) (string, error) { + var filepathFull string tempParentDir := filepath.Join(os.TempDir(), uuid.NewString()) internalio.MkdirAll(tempParentDir) @@ -54,7 +55,11 @@ func DownloadFile(url string) (string, error) { } defer tempFile.Close() - filepathFull := tempFile.Name() + if downloadPath != "" { + filepathFull = downloadPath + } else { + filepathFull = tempFile.Name() + } exp := GetExponentialBackOff(3) diff --git a/internal/utils/utils_test.go b/internal/utils/utils_test.go index 82098c5459..dde701cb75 100644 --- a/internal/utils/utils_test.go +++ b/internal/utils/utils_test.go @@ -9,7 +9,7 @@ import ( ) func TestDownloadFile(t *testing.T) { - f, err := DownloadFile("https://www.elastic.co/robots.txt") + f, err := DownloadFile("https://www.elastic.co/robots.txt", "") assert.Nil(t, err) defer os.Remove(filepath.Dir(f)) } diff --git a/pkg/downloads/versions.go b/pkg/downloads/versions.go index e58fe837b7..8305f33c0e 100644 --- a/pkg/downloads/versions.go +++ b/pkg/downloads/versions.go @@ -83,7 +83,7 @@ func CheckPRVersion(version string, fallbackVersion string) string { // FetchElasticArtifact fetches an artifact from the right repository, returning binary name, path and error func FetchElasticArtifact(ctx context.Context, artifact string, version string, os string, arch string, extension string, isDocker bool, xpack bool) (string, string, error) { binaryName := buildArtifactName(artifact, version, os, arch, extension, isDocker) - binaryPath, err := fetchBeatsBinary(ctx, binaryName, artifact, version, utils.TimeoutFactor, xpack) + binaryPath, err := FetchBeatsBinary(ctx, binaryName, artifact, version, utils.TimeoutFactor, xpack) if err != nil { log.WithFields(log.Fields{ "artifact": artifact, @@ -347,12 +347,12 @@ func buildArtifactName(artifact string, artifactVersion string, OS string, arch } -// fetchBeatsBinary it downloads the binary and returns the location of the downloaded file +// FetchBeatsBinary it downloads the binary and returns the location of the downloaded file // If the environment variable BEATS_LOCAL_PATH is set, then the artifact // to be used will be defined by the local snapshot produced by the local build. // Else, if the environment variable GITHUB_CHECK_SHA1 is set, then the artifact // to be downloaded will be defined by the snapshot produced by the Beats CI for that commit. -func fetchBeatsBinary(ctx context.Context, artifactName string, artifact string, version string, timeoutFactor int, xpack bool) (string, error) { +func FetchBeatsBinary(ctx context.Context, artifactName string, artifact string, version string, timeoutFactor int, xpack bool, downloadPath string) (string, error) { if BeatsLocalPath != "" { span, _ := apm.StartSpanOptions(ctx, "Fetching Beats binary", "beats.local.fetch-binary", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), @@ -389,7 +389,7 @@ func fetchBeatsBinary(ctx context.Context, artifactName string, artifact string, return val, nil } - filePathFull, err := utils.DownloadFile(URL) + filePathFull, err := utils.DownloadFile(URL, downloadPath) if err != nil { return filePathFull, err } diff --git a/pkg/downloads/versions_test.go b/pkg/downloads/versions_test.go index b7f0c41339..18d9e2349e 100644 --- a/pkg/downloads/versions_test.go +++ b/pkg/downloads/versions_test.go @@ -387,7 +387,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { defer func() { BeatsLocalPath = "" }() BeatsLocalPath = beatsDir - _, err := fetchBeatsBinary(ctx, "foo_fileName", artifact, version, utils.TimeoutFactor, true) + _, err := FetchBeatsBinary(ctx, "foo_fileName", artifact, version, utils.TimeoutFactor, true,"") assert.NotNil(t, err) }) @@ -398,7 +398,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-x86_64.rpm" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -409,7 +409,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-aarch64.rpm" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -421,7 +421,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-amd64.deb" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -432,7 +432,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-arm64.deb" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -444,7 +444,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-linux-amd64.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -455,7 +455,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-linux-x86_64.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -466,7 +466,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-linux-arm64.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -478,7 +478,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-linux-amd64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -489,7 +489,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-linux-arm64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -501,7 +501,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := ubi8VersionPrefix + "-linux-amd64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -512,7 +512,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := ubi8VersionPrefix + "-linux-arm64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := fetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true) + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) From 4b938d45fa6c341751beb70982544f49e3c0f53d Mon Sep 17 00:00:00 2001 From: narph Date: Mon, 10 Jan 2022 15:33:45 +0100 Subject: [PATCH 2/8] fix path --- pkg/downloads/versions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/downloads/versions.go b/pkg/downloads/versions.go index 8305f33c0e..b1c9a5b0ab 100644 --- a/pkg/downloads/versions.go +++ b/pkg/downloads/versions.go @@ -83,7 +83,7 @@ func CheckPRVersion(version string, fallbackVersion string) string { // FetchElasticArtifact fetches an artifact from the right repository, returning binary name, path and error func FetchElasticArtifact(ctx context.Context, artifact string, version string, os string, arch string, extension string, isDocker bool, xpack bool) (string, string, error) { binaryName := buildArtifactName(artifact, version, os, arch, extension, isDocker) - binaryPath, err := FetchBeatsBinary(ctx, binaryName, artifact, version, utils.TimeoutFactor, xpack) + binaryPath, err := FetchBeatsBinary(ctx, binaryName, artifact, version, utils.TimeoutFactor, xpack,"") if err != nil { log.WithFields(log.Fields{ "artifact": artifact, From 84b08649609423509873b2134b552782b7a904ac Mon Sep 17 00:00:00 2001 From: narph Date: Tue, 11 Jan 2022 12:57:08 +0100 Subject: [PATCH 3/8] work on download --- internal/utils/utils.go | 42 +++++++++++++++++++--------------- internal/utils/utils_test.go | 6 ++++- pkg/downloads/versions.go | 39 +++++++++++++++++++++---------- pkg/downloads/versions_test.go | 24 +++++++++---------- 4 files changed, 68 insertions(+), 43 deletions(-) diff --git a/internal/utils/utils.go b/internal/utils/utils.go index e73841de7e..613572c19c 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -26,6 +26,12 @@ const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" //nolint:unused var seededRand = rand.New(rand.NewSource(time.Now().UnixNano())) + +type DownloadRequest struct { + URL string + DownloadPath string +} + // GetArchitecture retrieves if the underlying system platform is arm64 or amd64 func GetArchitecture() string { arch, present := os.LookupEnv("GOARCH") @@ -40,41 +46,41 @@ func GetArchitecture() string { // DownloadFile will download a url and store it in a temporary path. // It writes to the destination file as it downloads it, without // loading the entire file into memory. -func DownloadFile(url string, downloadPath string) (string, error) { - var filepathFull string - tempParentDir := filepath.Join(os.TempDir(), uuid.NewString()) - internalio.MkdirAll(tempParentDir) +func DownloadFile(downloadRequest *DownloadRequest) (string, error) { + var filePath string + if downloadRequest.DownloadPath == "" { + tempParentDir := filepath.Join(os.TempDir(), uuid.NewString()) + internalio.MkdirAll(tempParentDir) + filePath= filepath.Join(tempParentDir, uuid.NewString()) + + }else { + filePath = filepath.Join(downloadRequest.DownloadPath, uuid.NewString()) + } - tempFile, err := os.Create(filepath.Join(tempParentDir, uuid.NewString())) + tempFile, err := os.Create(filePath) if err != nil { log.WithFields(log.Fields{ "error": err, - "url": url, + "url": downloadRequest.URL, }).Error("Error creating file") return "", err } defer tempFile.Close() - if downloadPath != "" { - filepathFull = downloadPath - } else { - filepathFull = tempFile.Name() - } - + filepathFull := tempFile.Name() exp := GetExponentialBackOff(3) retryCount := 1 var fileReader io.ReadCloser - download := func() error { - resp, err := http.Get(url) + resp, err := http.Get(downloadRequest.URL) if err != nil { log.WithFields(log.Fields{ "elapsedTime": exp.GetElapsedTime(), "error": err, "path": filepathFull, "retry": retryCount, - "url": url, + "url": downloadRequest.URL, }).Warn("Could not download the file") retryCount++ @@ -86,7 +92,7 @@ func DownloadFile(url string, downloadPath string) (string, error) { "elapsedTime": exp.GetElapsedTime(), "retries": retryCount, "path": filepathFull, - "url": url, + "url": downloadRequest.URL, }).Trace("File downloaded") fileReader = resp.Body @@ -95,7 +101,7 @@ func DownloadFile(url string, downloadPath string) (string, error) { } log.WithFields(log.Fields{ - "url": url, + "url": downloadRequest.URL, "path": filepathFull, }).Trace("Downloading file") @@ -109,7 +115,7 @@ func DownloadFile(url string, downloadPath string) (string, error) { if err != nil { log.WithFields(log.Fields{ "error": err, - "url": url, + "url": downloadRequest.URL, "path": filepathFull, }).Error("Could not write file") diff --git a/internal/utils/utils_test.go b/internal/utils/utils_test.go index dde701cb75..78edd297e6 100644 --- a/internal/utils/utils_test.go +++ b/internal/utils/utils_test.go @@ -9,7 +9,11 @@ import ( ) func TestDownloadFile(t *testing.T) { - f, err := DownloadFile("https://www.elastic.co/robots.txt", "") + var dRequest = DownloadRequest{ + URL: "https://www.elastic.co/robots.txt", + DownloadPath: "", + } + f, err := DownloadFile(&dRequest) assert.Nil(t, err) defer os.Remove(filepath.Dir(f)) } diff --git a/pkg/downloads/versions.go b/pkg/downloads/versions.go index b1c9a5b0ab..2af030295e 100644 --- a/pkg/downloads/versions.go +++ b/pkg/downloads/versions.go @@ -83,7 +83,7 @@ func CheckPRVersion(version string, fallbackVersion string) string { // FetchElasticArtifact fetches an artifact from the right repository, returning binary name, path and error func FetchElasticArtifact(ctx context.Context, artifact string, version string, os string, arch string, extension string, isDocker bool, xpack bool) (string, string, error) { binaryName := buildArtifactName(artifact, version, os, arch, extension, isDocker) - binaryPath, err := FetchBeatsBinary(ctx, binaryName, artifact, version, utils.TimeoutFactor, xpack,"") + binaryPath, err := FetchBeatsBinary(ctx, binaryName, artifact, version, utils.TimeoutFactor, xpack, "", false) if err != nil { log.WithFields(log.Fields{ "artifact": artifact, @@ -109,7 +109,7 @@ func GetCommitVersion(version string) string { // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-$ARCH.deb", "elastic-agent", "$VERSION") // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-x86_64.rpm", "elastic-agent","$VERSION") // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-linux-$ARCH.tar.gz", "elastic-agent","$VERSION") -func GetElasticArtifactURL(artifactName string, artifact string, version string) (string, error) { +func GetElasticArtifactURL(artifactName string, artifact string, version string) (string, string, error) { exp := utils.GetExponentialBackOff(time.Minute) retryCount := 1 @@ -161,7 +161,7 @@ func GetElasticArtifactURL(artifactName string, artifact string, version string) err := backoff.Retry(apiStatus, exp) if err != nil { - return "", err + return "", "", err } jsonParsed, err := gabs.ParseJSON([]byte(body)) @@ -171,7 +171,7 @@ func GetElasticArtifactURL(artifactName string, artifact string, version string) "artifactName": artifactName, "version": tmpVersion, }).Error("Could not parse the response body for the artifact") - return "", err + return "", "", err } log.WithFields(log.Fields{ @@ -191,8 +191,9 @@ func GetElasticArtifactURL(artifactName string, artifact string, version string) // we need to get keys with dots using Search instead of Path downloadObject := packagesObject.Search(artifactName) downloadURL := downloadObject.Path("url").Data().(string) + downloadshaURL := downloadObject.Path("sha_url").Data().(string) - return downloadURL, nil + return downloadURL, downloadshaURL, nil } // GetElasticArtifactVersion returns the current version: @@ -352,7 +353,7 @@ func buildArtifactName(artifact string, artifactVersion string, OS string, arch // to be used will be defined by the local snapshot produced by the local build. // Else, if the environment variable GITHUB_CHECK_SHA1 is set, then the artifact // to be downloaded will be defined by the snapshot produced by the Beats CI for that commit. -func FetchBeatsBinary(ctx context.Context, artifactName string, artifact string, version string, timeoutFactor int, xpack bool, downloadPath string) (string, error) { +func FetchBeatsBinary(ctx context.Context, artifactName string, artifact string, version string, timeoutFactor int, xpack bool, downloadPath string, downloadSHAFIle bool) (string, error) { if BeatsLocalPath != "" { span, _ := apm.StartSpanOptions(ctx, "Fetching Beats binary", "beats.local.fetch-binary", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), @@ -376,6 +377,11 @@ func FetchBeatsBinary(ctx context.Context, artifactName string, artifact string, } handleDownload := func(URL string) (string, error) { + name := artifactName + downloadRequest := utils.DownloadRequest{ + DownloadPath: downloadPath, + URL: URL, + } span, _ := apm.StartSpanOptions(ctx, "Fetching Beats binary", "beats.url.fetch-binary", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) @@ -389,13 +395,16 @@ func FetchBeatsBinary(ctx context.Context, artifactName string, artifact string, return val, nil } - filePathFull, err := utils.DownloadFile(URL, downloadPath) + filePathFull, err := utils.DownloadFile(&downloadRequest) if err != nil { return filePathFull, err } + if strings.HasSuffix(URL, ".sha512") { + name = fmt.Sprintf("%s.sha512", name) + } // use artifact name as file name to avoid having URL params in the name - sanitizedFilePath := filepath.Join(path.Dir(filePathFull), artifactName) + sanitizedFilePath := filepath.Join(path.Dir(filePathFull), name) err = os.Rename(filePathFull, sanitizedFilePath) if err != nil { log.WithFields(log.Fields{ @@ -410,7 +419,7 @@ func FetchBeatsBinary(ctx context.Context, artifactName string, artifact string, return sanitizedFilePath, nil } - var downloadURL string + var downloadURL, downloadShaURL string var err error useCISnapshots := GithubCommitSha1 != "" @@ -434,12 +443,18 @@ func FetchBeatsBinary(ctx context.Context, artifactName string, artifact string, return handleDownload(downloadURL) } - downloadURL, err = GetElasticArtifactURL(artifactName, artifact, version) + downloadURL, downloadShaURL, err = GetElasticArtifactURL(artifactName, artifact, version) if err != nil { return "", err } - - return handleDownload(downloadURL) + downloadLocation, err := handleDownload(downloadURL) + if err != nil { + return "", err + } + if downloadSHAFIle == true && downloadShaURL != "" { + downloadLocation, err = handleDownload(downloadShaURL) + } + return downloadLocation, err } func getBucketSearchNextPageParam(jsonParsed *gabs.Container) string { diff --git a/pkg/downloads/versions_test.go b/pkg/downloads/versions_test.go index 18d9e2349e..5a08548644 100644 --- a/pkg/downloads/versions_test.go +++ b/pkg/downloads/versions_test.go @@ -387,7 +387,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { defer func() { BeatsLocalPath = "" }() BeatsLocalPath = beatsDir - _, err := FetchBeatsBinary(ctx, "foo_fileName", artifact, version, utils.TimeoutFactor, true,"") + _, err := FetchBeatsBinary(ctx, "foo_fileName", artifact, version, utils.TimeoutFactor, true, "", false) assert.NotNil(t, err) }) @@ -398,7 +398,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-x86_64.rpm" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -409,7 +409,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-aarch64.rpm" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -421,7 +421,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-amd64.deb" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -432,7 +432,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-arm64.deb" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -444,7 +444,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-linux-amd64.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -455,7 +455,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-linux-x86_64.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -466,7 +466,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-linux-arm64.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -478,7 +478,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-linux-amd64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -489,7 +489,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := versionPrefix + "-linux-arm64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -501,7 +501,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := ubi8VersionPrefix + "-linux-amd64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) @@ -512,7 +512,7 @@ func TestFetchBeatsBinaryFromLocalPath(t *testing.T) { artifactName := ubi8VersionPrefix + "-linux-arm64.docker.tar.gz" expectedFilePath := path.Join(distributionsDir, artifactName) - downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true,"") + downloadedFilePath, err := FetchBeatsBinary(ctx, artifactName, artifact, version, utils.TimeoutFactor, true, "", false) assert.Nil(t, err) assert.Equal(t, downloadedFilePath, expectedFilePath) }) From 2f19ed7a4b7f3405b872d5d26215690ea1595432 Mon Sep 17 00:00:00 2001 From: narph Date: Tue, 11 Jan 2022 14:28:03 +0100 Subject: [PATCH 4/8] small fix --- internal/utils/utils.go | 31 +++++++++++++++---------------- internal/utils/utils_test.go | 5 +++-- pkg/downloads/versions.go | 12 ++++++------ pkg/downloads/versions_test.go | 6 ++++++ 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 613572c19c..25e3397821 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -26,10 +26,10 @@ const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" //nolint:unused var seededRand = rand.New(rand.NewSource(time.Now().UnixNano())) - type DownloadRequest struct { - URL string - DownloadPath string + URL string + DownloadPath string + UnsanitizedFilePath string } // GetArchitecture retrieves if the underlying system platform is arm64 or amd64 @@ -46,14 +46,13 @@ func GetArchitecture() string { // DownloadFile will download a url and store it in a temporary path. // It writes to the destination file as it downloads it, without // loading the entire file into memory. -func DownloadFile(downloadRequest *DownloadRequest) (string, error) { +func DownloadFile(downloadRequest *DownloadRequest) error { var filePath string if downloadRequest.DownloadPath == "" { tempParentDir := filepath.Join(os.TempDir(), uuid.NewString()) internalio.MkdirAll(tempParentDir) - filePath= filepath.Join(tempParentDir, uuid.NewString()) - - }else { + filePath = filepath.Join(tempParentDir, uuid.NewString()) + } else { filePath = filepath.Join(downloadRequest.DownloadPath, uuid.NewString()) } @@ -63,11 +62,11 @@ func DownloadFile(downloadRequest *DownloadRequest) (string, error) { "error": err, "url": downloadRequest.URL, }).Error("Error creating file") - return "", err + return err } defer tempFile.Close() - filepathFull := tempFile.Name() + downloadRequest.UnsanitizedFilePath = tempFile.Name() exp := GetExponentialBackOff(3) retryCount := 1 @@ -78,7 +77,7 @@ func DownloadFile(downloadRequest *DownloadRequest) (string, error) { log.WithFields(log.Fields{ "elapsedTime": exp.GetElapsedTime(), "error": err, - "path": filepathFull, + "path": downloadRequest.UnsanitizedFilePath, "retry": retryCount, "url": downloadRequest.URL, }).Warn("Could not download the file") @@ -91,7 +90,7 @@ func DownloadFile(downloadRequest *DownloadRequest) (string, error) { log.WithFields(log.Fields{ "elapsedTime": exp.GetElapsedTime(), "retries": retryCount, - "path": filepathFull, + "path": downloadRequest.UnsanitizedFilePath, "url": downloadRequest.URL, }).Trace("File downloaded") @@ -102,12 +101,12 @@ func DownloadFile(downloadRequest *DownloadRequest) (string, error) { log.WithFields(log.Fields{ "url": downloadRequest.URL, - "path": filepathFull, + "path": downloadRequest.UnsanitizedFilePath, }).Trace("Downloading file") err = backoff.Retry(download, exp) if err != nil { - return "", err + return err } defer fileReader.Close() @@ -116,15 +115,15 @@ func DownloadFile(downloadRequest *DownloadRequest) (string, error) { log.WithFields(log.Fields{ "error": err, "url": downloadRequest.URL, - "path": filepathFull, + "path": downloadRequest.UnsanitizedFilePath, }).Error("Could not write file") - return filepathFull, err + return err } _ = os.Chmod(tempFile.Name(), 0666) - return filepathFull, nil + return nil } // IsCommit returns true if the string matches commit format diff --git a/internal/utils/utils_test.go b/internal/utils/utils_test.go index 78edd297e6..44fbc18043 100644 --- a/internal/utils/utils_test.go +++ b/internal/utils/utils_test.go @@ -13,9 +13,10 @@ func TestDownloadFile(t *testing.T) { URL: "https://www.elastic.co/robots.txt", DownloadPath: "", } - f, err := DownloadFile(&dRequest) + err := DownloadFile(&dRequest) assert.Nil(t, err) - defer os.Remove(filepath.Dir(f)) + assert.NotEmpty(t, dRequest.UnsanitizedFilePath) + defer os.Remove(filepath.Dir(dRequest.UnsanitizedFilePath)) } func TestGetArchitecture(t *testing.T) { diff --git a/pkg/downloads/versions.go b/pkg/downloads/versions.go index 2af030295e..58646f5f4b 100644 --- a/pkg/downloads/versions.go +++ b/pkg/downloads/versions.go @@ -395,23 +395,23 @@ func FetchBeatsBinary(ctx context.Context, artifactName string, artifact string, return val, nil } - filePathFull, err := utils.DownloadFile(&downloadRequest) + err := utils.DownloadFile(&downloadRequest) if err != nil { - return filePathFull, err + return downloadRequest.UnsanitizedFilePath, err } if strings.HasSuffix(URL, ".sha512") { name = fmt.Sprintf("%s.sha512", name) } // use artifact name as file name to avoid having URL params in the name - sanitizedFilePath := filepath.Join(path.Dir(filePathFull), name) - err = os.Rename(filePathFull, sanitizedFilePath) + sanitizedFilePath := filepath.Join(path.Dir(downloadRequest.UnsanitizedFilePath), name) + err = os.Rename(downloadRequest.UnsanitizedFilePath, sanitizedFilePath) if err != nil { log.WithFields(log.Fields{ - "fileName": filePathFull, + "fileName": downloadRequest.UnsanitizedFilePath, "sanitizedFileName": sanitizedFilePath, }).Warn("Could not sanitize downloaded file name. Keeping old name") - sanitizedFilePath = filePathFull + sanitizedFilePath = downloadRequest.UnsanitizedFilePath } binariesCache[URL] = sanitizedFilePath diff --git a/pkg/downloads/versions_test.go b/pkg/downloads/versions_test.go index 5a08548644..c657063626 100644 --- a/pkg/downloads/versions_test.go +++ b/pkg/downloads/versions_test.go @@ -52,6 +52,12 @@ func init() { snapshotsJSON, _ = gabs.ParseJSON([]byte(snapshotsContent)) } +func TestFetch(t *testing.T) { + ctx := context.Background() + _, err := FetchBeatsBinary(ctx, "filebeat-8.1.0-SNAPSHOT-windows-x86_64.zip", "filebeat", "8.1.0-SNAPSHOT", utils.TimeoutFactor, false, "C:\\Work\\e2e-testing\\pkg\\downloads\\hello", true) + assert.Nil(t, err) +} + func TestBuildArtifactName(t *testing.T) { OS := "linux" version := testVersion From 6596b0bf105a1b6410f88d42f139d2c95408b375 Mon Sep 17 00:00:00 2001 From: narph Date: Tue, 11 Jan 2022 14:28:50 +0100 Subject: [PATCH 5/8] remove test --- pkg/downloads/versions_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/downloads/versions_test.go b/pkg/downloads/versions_test.go index c657063626..5a08548644 100644 --- a/pkg/downloads/versions_test.go +++ b/pkg/downloads/versions_test.go @@ -52,12 +52,6 @@ func init() { snapshotsJSON, _ = gabs.ParseJSON([]byte(snapshotsContent)) } -func TestFetch(t *testing.T) { - ctx := context.Background() - _, err := FetchBeatsBinary(ctx, "filebeat-8.1.0-SNAPSHOT-windows-x86_64.zip", "filebeat", "8.1.0-SNAPSHOT", utils.TimeoutFactor, false, "C:\\Work\\e2e-testing\\pkg\\downloads\\hello", true) - assert.Nil(t, err) -} - func TestBuildArtifactName(t *testing.T) { OS := "linux" version := testVersion From 2296bf5ce8e67bf241e023ca827d3484794708ab Mon Sep 17 00:00:00 2001 From: narph Date: Tue, 11 Jan 2022 15:40:27 +0100 Subject: [PATCH 6/8] add sha to google --- pkg/downloads/versions.go | 15 +++++++++++++-- pkg/downloads/versions_test.go | 7 +++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/pkg/downloads/versions.go b/pkg/downloads/versions.go index 58646f5f4b..ba497608ac 100644 --- a/pkg/downloads/versions.go +++ b/pkg/downloads/versions.go @@ -431,15 +431,26 @@ func FetchBeatsBinary(ctx context.Context, artifactName string, artifact string, log.Debugf("Using CI snapshots for %s", artifact) - bucket, prefix, object := getGCPBucketCoordinates(artifactName, artifact) - maxTimeout := time.Duration(timeoutFactor) * time.Minute + bucket, prefix, object := getGCPBucketCoordinates(artifactName, artifact) + downloadURL, err = getObjectURLFromBucket(bucket, prefix, object, maxTimeout) if err != nil { return "", err } + downloadLocation, err := handleDownload(downloadURL) + + // check if sha file should be downloaded, else return + if downloadSHAFIle == false { + return downloadLocation, err + } + bucket, prefix, object = getGCPBucketCoordinates(fmt.Sprintf("%s.sha512", artifactName), artifact) + downloadURL, err = getObjectURLFromBucket(bucket, prefix, object, maxTimeout) + if err != nil { + return "", err + } return handleDownload(downloadURL) } diff --git a/pkg/downloads/versions_test.go b/pkg/downloads/versions_test.go index 5a08548644..26b997312a 100644 --- a/pkg/downloads/versions_test.go +++ b/pkg/downloads/versions_test.go @@ -6,15 +6,14 @@ package downloads import ( "context" + "github.com/Jeffail/gabs/v2" + "github.com/elastic/e2e-testing/internal/utils" + "github.com/stretchr/testify/assert" "io/ioutil" "os" "path" "path/filepath" "testing" - - "github.com/Jeffail/gabs/v2" - "github.com/elastic/e2e-testing/internal/utils" - "github.com/stretchr/testify/assert" ) var artifact = "elastic-agent" From b3778ed822fc28bd787bd54507e18b1d6b10f496 Mon Sep 17 00:00:00 2001 From: narph Date: Mon, 17 Jan 2022 11:18:35 +0100 Subject: [PATCH 7/8] fix typo --- internal/utils/utils.go | 1 + pkg/downloads/versions.go | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 25e3397821..56ba522271 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -52,6 +52,7 @@ func DownloadFile(downloadRequest *DownloadRequest) error { tempParentDir := filepath.Join(os.TempDir(), uuid.NewString()) internalio.MkdirAll(tempParentDir) filePath = filepath.Join(tempParentDir, uuid.NewString()) + downloadRequest.DownloadPath = filePath } else { filePath = filepath.Join(downloadRequest.DownloadPath, uuid.NewString()) } diff --git a/pkg/downloads/versions.go b/pkg/downloads/versions.go index ba497608ac..244ab43404 100644 --- a/pkg/downloads/versions.go +++ b/pkg/downloads/versions.go @@ -106,6 +106,7 @@ func GetCommitVersion(version string) string { // GetElasticArtifactURL returns the URL of a released artifact, which its full name is defined in the first argument, // from Elastic's artifact repository, building the JSON path query based on the full name +// It also returns the URL of the sha512 file of the released artifact. // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-$ARCH.deb", "elastic-agent", "$VERSION") // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-x86_64.rpm", "elastic-agent","$VERSION") // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-linux-$ARCH.tar.gz", "elastic-agent","$VERSION") @@ -353,7 +354,7 @@ func buildArtifactName(artifact string, artifactVersion string, OS string, arch // to be used will be defined by the local snapshot produced by the local build. // Else, if the environment variable GITHUB_CHECK_SHA1 is set, then the artifact // to be downloaded will be defined by the snapshot produced by the Beats CI for that commit. -func FetchBeatsBinary(ctx context.Context, artifactName string, artifact string, version string, timeoutFactor int, xpack bool, downloadPath string, downloadSHAFIle bool) (string, error) { +func FetchBeatsBinary(ctx context.Context, artifactName string, artifact string, version string, timeoutFactor int, xpack bool, downloadPath string, downloadSHAFile bool) (string, error) { if BeatsLocalPath != "" { span, _ := apm.StartSpanOptions(ctx, "Fetching Beats binary", "beats.local.fetch-binary", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), @@ -442,7 +443,7 @@ func FetchBeatsBinary(ctx context.Context, artifactName string, artifact string, downloadLocation, err := handleDownload(downloadURL) // check if sha file should be downloaded, else return - if downloadSHAFIle == false { + if downloadSHAFile == false { return downloadLocation, err } @@ -462,7 +463,7 @@ func FetchBeatsBinary(ctx context.Context, artifactName string, artifact string, if err != nil { return "", err } - if downloadSHAFIle == true && downloadShaURL != "" { + if downloadSHAFile == true && downloadShaURL != "" { downloadLocation, err = handleDownload(downloadShaURL) } return downloadLocation, err From 2e3ecea4f732261bd6f2c7d0f605e42ecf3b4268 Mon Sep 17 00:00:00 2001 From: narph Date: Mon, 17 Jan 2022 15:59:22 +0100 Subject: [PATCH 8/8] add comment --- internal/utils/utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 56ba522271..13c28262c8 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -26,6 +26,7 @@ const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" //nolint:unused var seededRand = rand.New(rand.NewSource(time.Now().UnixNano())) +// DownloadRequest struct contains download details ad path and URL type DownloadRequest struct { URL string DownloadPath string