From b60cd4b5e7c426c684fff6079ca1845eb664e616 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 14 Sep 2020 17:27:28 +0200 Subject: [PATCH] Add vendoring to Google Cloud Functions again (#21070) ## What does this PR do? This PR adds the vendor folder back the Cloud Functions of Functionbeat, so they can be used on GCP. This PR basically reverts the removal of the vendor builder functionality. ## Why is it important? The functions are currently broken, as there is no go.mod file and vendor folder. However, we cannot use go modules, because the Golang environment on GCP runs go version 1.13. Thus, we have to fall back to vendoring. ## Checklist - [x] My code follows the style guidelines of this project - [x] I have commented my code, particularly in hard-to-understand areas ~~- [ ] I have made corresponding changes to the documentation~~ ~~- [ ] I have made corresponding change to the default configuration files~~ ~~- [ ] I have added tests that prove my fix is effective or that my feature works~~ - [x] I have added an entry in `CHANGELOG.next.asciidoc` or `CHANGELOG-developer.next.asciidoc`. ## Related issues Closes #20830 (cherry picked from commit 50ea11a32e99b99f50e8344f3131c78d9b19afe2) --- CHANGELOG.next.asciidoc | 1 + dev-tools/mage/gotool/go.go | 19 +++++++++ .../dev-tools/packaging/packages.yml | 6 +++ x-pack/functionbeat/magefile.go | 8 ++-- .../manager/gcp/template_builder.go | 3 +- x-pack/functionbeat/scripts/mage/update.go | 42 ++++++++++++++++++- 6 files changed, 74 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 77d1ea6c93c8..0ea45e376b2a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -404,6 +404,7 @@ field. You can revert this change by configuring tags for the module and omittin - Fix timeout option of GCP functions. {issue}16282[16282] {pull}16287[16287] - Do not need Google credentials if not required for the operation. {issue}17329[17329] {pull}21072[21072] +- Fix dependency issues of GCP functions. {issue}20830[20830] {pull}21070[21070] ==== Added diff --git a/dev-tools/mage/gotool/go.go b/dev-tools/mage/gotool/go.go index 9022e0c01377..bb7066a4f3e5 100644 --- a/dev-tools/mage/gotool/go.go +++ b/dev-tools/mage/gotool/go.go @@ -85,6 +85,25 @@ func ListDeps(pkg string) ([]string, error) { return getLines(callGo(nil, "list", "-deps", "-f", tmpl, pkg)) } +// ListDepsLocation calls `go list -dep` for every package spec given. +func ListDepsLocation(pkg string) (map[string]string, error) { + const tmpl = `{{if not .Standard}}{{.ImportPath}};{{.Dir}}{{end}}` + + lines, err := getLines(callGo(nil, "list", "-deps", "-f", tmpl, pkg)) + if err != nil { + return nil, err + } + deps := make(map[string]string, len(lines)) + for _, l := range lines { + parts := strings.Split(l, ";") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid number of parts") + } + deps[parts[0]] = parts[1] + } + return deps, nil +} + // ListTestFiles lists all go and cgo test files available in a package. func ListTestFiles(pkg string) ([]string, error) { const tmpl = `{{ range .TestGoFiles }}{{ printf "%s\n" . }}{{ end }}` + diff --git a/x-pack/functionbeat/dev-tools/packaging/packages.yml b/x-pack/functionbeat/dev-tools/packaging/packages.yml index 47379c8793e2..e6c3346bf1ab 100644 --- a/x-pack/functionbeat/dev-tools/packaging/packages.yml +++ b/x-pack/functionbeat/dev-tools/packaging/packages.yml @@ -65,6 +65,12 @@ shared: pkg/functionbeat-aws: source: 'provider/aws/build/golang-crossbuild/aws-linux-amd64' mode: 0755 + pkg/pubsub/vendor: + source: 'provider/gcp/build/pubsub/vendor' + mode: 0644 + pkg/storage/vendor: + source: 'provider/gcp/build/storage/vendor' + mode: 0644 pkg/pubsub/pubsub.go: source: 'provider/gcp/pubsub/pubsub.go' mode: 0655 diff --git a/x-pack/functionbeat/magefile.go b/x-pack/functionbeat/magefile.go index e1982b66400e..a317bd0cb71f 100644 --- a/x-pack/functionbeat/magefile.go +++ b/x-pack/functionbeat/magefile.go @@ -159,9 +159,11 @@ func BuildPkgForFunctions() error { err := os.RemoveAll("pkg") filesToCopy := map[string]string{ - filepath.Join("provider", "aws", "functionbeat-aws"): filepath.Join("pkg", "functionbeat-aws"), - filepath.Join("provider", "gcp", "pubsub", "pubsub.go"): filepath.Join("pkg", "pubsub", "pubsub.go"), - filepath.Join("provider", "gcp", "storage", "storage.go"): filepath.Join("pkg", "storage", "storage.go"), + filepath.Join("provider", "aws", "functionbeat-aws"): filepath.Join("pkg", "functionbeat-aws"), + filepath.Join("provider", "gcp", "pubsub", "pubsub.go"): filepath.Join("pkg", "pubsub", "pubsub.go"), + filepath.Join("provider", "gcp", "storage", "storage.go"): filepath.Join("pkg", "storage", "storage.go"), + filepath.Join("provider", "gcp", "build", "pubsub", "vendor"): filepath.Join("pkg", "pubsub", "vendor"), + filepath.Join("provider", "gcp", "build", "storage", "vendor"): filepath.Join("pkg", "storage", "vendor"), } for src, dest := range filesToCopy { c := &devtools.CopyTask{ diff --git a/x-pack/functionbeat/manager/gcp/template_builder.go b/x-pack/functionbeat/manager/gcp/template_builder.go index da468abee303..c6f1292ce154 100644 --- a/x-pack/functionbeat/manager/gcp/template_builder.go +++ b/x-pack/functionbeat/manager/gcp/template_builder.go @@ -190,5 +190,6 @@ func zipResources() map[string][]bundle.Resource { func zipResourcesOfFunc(typeName string) []bundle.Resource { root := filepath.Join("pkg", typeName) - return []bundle.Resource{&bundle.LocalFile{Path: filepath.Join(root, typeName+".go"), FileMode: 0755}} + vendor := bundle.Folder(filepath.Join("pkg", typeName, "vendor"), filepath.Join("pkg", typeName), 0644) + return append(vendor, &bundle.LocalFile{Path: filepath.Join(root, typeName+".go"), FileMode: 0755}) } diff --git a/x-pack/functionbeat/scripts/mage/update.go b/x-pack/functionbeat/scripts/mage/update.go index 468bdafbe0fd..59b56cb6bed5 100644 --- a/x-pack/functionbeat/scripts/mage/update.go +++ b/x-pack/functionbeat/scripts/mage/update.go @@ -5,9 +5,13 @@ package mage import ( + "os" + "path/filepath" + "github.com/magefile/mage/mg" devtools "github.com/elastic/beats/v7/dev-tools/mage" + "github.com/elastic/beats/v7/dev-tools/mage/gotool" ) // Update target namespace. @@ -20,7 +24,7 @@ var Aliases = map[string]interface{}{ // All updates all generated content. func (Update) All() { - mg.Deps(Update.Fields, Update.IncludeFields, Update.Config, Update.FieldDocs) + mg.Deps(Update.Fields, Update.IncludeFields, Update.Config, Update.FieldDocs, Update.VendorBeats) } // Config generates both the short and reference configs. @@ -46,3 +50,39 @@ func (Update) IncludeFields() error { return devtools.GenerateAllInOneFieldsGo() } + +// VendorBeats collects the vendor folder required to deploy the function for GCP. +func (Update) VendorBeats() error { + for _, f := range []string{"pubsub", "storage"} { + gcpVendorPath := filepath.Join("provider", "gcp", "build", f, "vendor") + err := os.RemoveAll(gcpVendorPath) + if err != nil { + return err + } + + deps, err := gotool.ListDepsLocation("github.com/elastic/beats/v7/x-pack/functionbeat/provider/gcp/" + f) + if err != nil { + return err + } + + for importPath, location := range deps { + cp := &devtools.CopyTask{ + Source: location, + Dest: filepath.Join(gcpVendorPath, importPath), + Mode: 0600, + DirMode: os.ModeDir | 0750, + Exclude: []string{ + ".*_test.go$", + ".*.yml", + }, + } + err = cp.Execute() + if err != nil { + return err + } + } + + } + + return nil +}