-
Notifications
You must be signed in to change notification settings - Fork 4.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Heartbeat] Zip URL Support (#24714)
Adds support for remote zips to browser monitors, letting users just automatically pull from a remote repo. Fixes elastic/uptime#271
- Loading branch information
Showing
7 changed files
with
399 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
28 changes: 28 additions & 0 deletions
28
x-pack/heartbeat/monitors/browser/source/fixtures/testhelp.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| // or more contributor license agreements. Licensed under the Elastic License; | ||
| // you may not use this file except in compliance with the Elastic License. | ||
|
|
||
| package fixtures | ||
|
|
||
| import ( | ||
| "os" | ||
| "path" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func TestTodosFiles(t *testing.T, dir string) { | ||
| expected := []string{ | ||
| "node_modules", | ||
| "package.json", | ||
| "helpers.ts", | ||
| "add-remove.journey.ts", | ||
| "basics.journey.ts", | ||
| } | ||
| for _, file := range expected { | ||
| _, err := os.Stat(path.Join(dir, file)) | ||
| // assert, not require, because we want to proceed to the close bit | ||
| assert.NoError(t, err) | ||
| } | ||
| } |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,236 @@ | ||
| // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| // or more contributor license agreements. Licensed under the Elastic License; | ||
| // you may not use this file except in compliance with the Elastic License. | ||
|
|
||
| package source | ||
|
|
||
| import ( | ||
| "archive/zip" | ||
| "fmt" | ||
| "io" | ||
| "io/ioutil" | ||
| "net/http" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
| "time" | ||
| ) | ||
|
|
||
| type ZipURLSource struct { | ||
| URL string `config:"url" json:"url"` | ||
| Folder string `config:"folder" json:"folder"` | ||
| Username string `config:"username" json:"username"` | ||
| Password string `config:"password" json:"password"` | ||
| Retries int `config:"retries" default:"3" json:"retries"` | ||
| BaseSource | ||
| // Etag from last successful fetch | ||
| etag string | ||
| TargetDirectory string `config:"target_directory" json:"target_directory"` | ||
| } | ||
|
|
||
| var ErrNoEtag = fmt.Errorf("No ETag header in zip file response. Heartbeat requires an etag to efficiently cache downloaded code") | ||
|
|
||
| func (z *ZipURLSource) Fetch() error { | ||
| changed, err := checkIfChanged(z) | ||
| if err != nil { | ||
| return fmt.Errorf("could not check if zip source changed for %s: %w", z.URL, err) | ||
| } | ||
| if !changed { | ||
| return nil | ||
| } | ||
|
|
||
| // remove target directory if etag changed | ||
| if z.TargetDirectory != "" { | ||
| os.RemoveAll(z.TargetDirectory) | ||
| } | ||
|
|
||
| tf, err := ioutil.TempFile("/tmp", "elastic-synthetics-zip-") | ||
| if err != nil { | ||
| return fmt.Errorf("could not create tmpfile for zip source: %w", err) | ||
| } | ||
| defer os.Remove(tf.Name()) | ||
|
|
||
| newEtag, err := download(z, tf) | ||
| if err != nil { | ||
| return fmt.Errorf("could not download %s: %w", z.URL, err) | ||
| } | ||
| // We are guaranteed an etag | ||
| z.etag = newEtag | ||
|
|
||
| if z.TargetDirectory != "" { | ||
| os.MkdirAll(z.TargetDirectory, 0755) | ||
| } else { | ||
| z.TargetDirectory, err = ioutil.TempDir("/tmp", "elastic-synthetics-unzip-") | ||
| if err != nil { | ||
| return fmt.Errorf("could not make temp dir for zip download: %w", err) | ||
| } | ||
| } | ||
|
|
||
| err = unzip(tf, z.TargetDirectory, z.Folder) | ||
| if err != nil { | ||
| z.Close() | ||
| return err | ||
| } | ||
|
|
||
| // run as the local job after extracting the files | ||
| if !Offline() { | ||
| err = setupOnlineDir(z.TargetDirectory) | ||
| if err != nil { | ||
| os.RemoveAll(z.TargetDirectory) | ||
| return fmt.Errorf("failed to install dependencies at: '%s' %w", z.TargetDirectory, err) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func unzip(tf *os.File, targetDir string, folder string) error { | ||
| stat, err := tf.Stat() | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| rdr, err := zip.NewReader(tf, stat.Size()) | ||
| if err != nil { | ||
| return fmt.Errorf("could not read file %s as zip: %w", tf.Name(), err) | ||
| } | ||
|
|
||
| for _, f := range rdr.File { | ||
| err = unzipFile(targetDir, folder, f) | ||
| if err != nil { | ||
| // TODO: err handlers | ||
| os.RemoveAll(targetDir) | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func unzipFile(workdir string, folder string, f *zip.File) error { | ||
| folderPaths := strings.Split(folder, string(filepath.Separator)) | ||
| var folderDepth = 1 | ||
| for _, path := range folderPaths { | ||
| if path != "" { | ||
| folderDepth++ | ||
| } | ||
| } | ||
| splitZipFileName := strings.Split(f.Name, string(filepath.Separator)) | ||
| root := splitZipFileName[0] | ||
|
|
||
| prefix := filepath.Join(root, folder) | ||
| if !strings.HasPrefix(f.Name, prefix) { | ||
| return nil | ||
| } | ||
|
|
||
| sansFolder := splitZipFileName[folderDepth:] | ||
| destPath := filepath.Join(workdir, filepath.Join(sansFolder...)) | ||
|
|
||
| // Never unpack node modules | ||
| if strings.HasPrefix(destPath, "node_modules/") { | ||
| return nil | ||
| } | ||
|
|
||
| if f.FileInfo().IsDir() { | ||
| err := os.MkdirAll(destPath, 0755) | ||
| if err != nil { | ||
| return fmt.Errorf("could not make dest zip dir '%s': %w", destPath, err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| dest, err := os.Create(destPath) | ||
| if err != nil { | ||
| return fmt.Errorf("could not create dest file for zip '%s': %w", destPath, err) | ||
| } | ||
| defer dest.Close() | ||
|
|
||
| rdr, err := f.Open() | ||
| if err != nil { | ||
| return fmt.Errorf("could not open source zip file '%s': %w", f.Name, err) | ||
| } | ||
| defer rdr.Close() | ||
|
|
||
| _, err = io.Copy(dest, rdr) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func retryingZipRequest(method string, z *ZipURLSource) (resp *http.Response, err error) { | ||
| if z.Retries < 1 { | ||
| z.Retries = 1 | ||
| } | ||
| for i := z.Retries; i > 0; i-- { | ||
| resp, err = zipRequest(method, z) | ||
| // If the request is successful | ||
| // Retry server errors, but not non-retryable 4xx errors | ||
| if err == nil && resp.StatusCode >= 200 && resp.StatusCode < 500 { | ||
| break | ||
| } | ||
| if err == nil { | ||
| resp.Body.Close() | ||
| } | ||
| time.Sleep(time.Second) | ||
| } | ||
| if resp.StatusCode > 300 { | ||
| return nil, fmt.Errorf("failed to retrieve zip, received status of %d requesting zip URL", resp.StatusCode) | ||
| } | ||
| return resp, err | ||
| } | ||
|
|
||
| func zipRequest(method string, z *ZipURLSource) (*http.Response, error) { | ||
| req, err := http.NewRequest(method, z.URL, nil) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("could not issue request to: %s %w", z.URL, err) | ||
| } | ||
| if z.Username != "" && z.Password != "" { | ||
| req.SetBasicAuth(z.Username, z.Password) | ||
| } | ||
| return http.DefaultClient.Do(req) | ||
| } | ||
|
|
||
| func download(z *ZipURLSource, tf *os.File) (etag string, err error) { | ||
| resp, err := retryingZipRequest("GET", z) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| defer resp.Body.Close() | ||
|
|
||
| etag = resp.Header.Get("ETag") | ||
| if etag == "" { | ||
| return "", ErrNoEtag | ||
| } | ||
|
|
||
| io.Copy(tf, resp.Body) | ||
|
|
||
| return | ||
| } | ||
|
|
||
| func checkIfChanged(z *ZipURLSource) (bool, error) { | ||
| resp, err := retryingZipRequest("HEAD", z) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| defer resp.Body.Close() | ||
|
|
||
| // If the etag matches what we already have on file, skip this | ||
| if resp.Header.Get("ETag") == "" { | ||
| return false, ErrNoEtag | ||
| } | ||
| // Nothing has changed since the last fetch, so we can just abort | ||
| if resp.Header.Get("ETag") == z.etag { | ||
| return false, nil | ||
| } | ||
|
|
||
| return true, nil | ||
| } | ||
|
|
||
| func (z *ZipURLSource) Workdir() string { | ||
| return z.TargetDirectory | ||
| } | ||
|
|
||
| func (z *ZipURLSource) Close() error { | ||
| return os.RemoveAll(z.TargetDirectory) | ||
| } |
Oops, something went wrong.