From bdef31fabeb7320ceffc8c2b46c43848a4138118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 12 Feb 2020 19:44:52 +0100 Subject: [PATCH 1/6] Beat generators new dependency management: go modules --- dev-tools/mage/gomod.go | 2 - dev-tools/mage/gotool/get.go | 1 + dev-tools/mage/gotool/go.go | 2 +- dev-tools/mage/gotool/modules.go | 5 + dev-tools/mage/settings.go | 23 +-- docs/devguide/newbeat.asciidoc | 41 +++++- generator/_templates/beat/{beat}/Makefile | 11 +- .../beat/{beat}/_meta/beat.reference.yml | 7 + .../_templates/beat/{beat}/include/include.go | 1 + generator/_templates/beat/{beat}/magefile.go | 2 +- .../_templates/beat/{beat}/tools/tools.go | 14 ++ .../_templates/metricbeat/{beat}/Makefile | 11 +- .../_templates/metricbeat/{beat}/README.md | 11 +- .../metricbeat/{beat}/include/include.go | 1 + .../metricbeat/{beat}/tools/tools.go | 14 ++ generator/common/Makefile | 14 +- generator/common/beatgen/beatgen.go | 35 ++++- generator/common/beatgen/setup/creator.go | 2 +- generator/common/beatgen/setup/setup.go | 137 +++++++++--------- 19 files changed, 216 insertions(+), 118 deletions(-) create mode 100644 generator/_templates/beat/{beat}/_meta/beat.reference.yml create mode 100644 generator/_templates/beat/{beat}/include/include.go create mode 100644 generator/_templates/beat/{beat}/tools/tools.go create mode 100644 generator/_templates/metricbeat/{beat}/include/include.go create mode 100644 generator/_templates/metricbeat/{beat}/tools/tools.go diff --git a/dev-tools/mage/gomod.go b/dev-tools/mage/gomod.go index 17f4e8917f6a..7cc6b7d8c681 100644 --- a/dev-tools/mage/gomod.go +++ b/dev-tools/mage/gomod.go @@ -18,7 +18,6 @@ package mage import ( - "fmt" "os" "path/filepath" @@ -80,7 +79,6 @@ func Vendor() error { if err != nil { return err } - fmt.Println(path) for _, f := range p.filesToCopy { from := filepath.Join(path, f) diff --git a/dev-tools/mage/gotool/get.go b/dev-tools/mage/gotool/get.go index 6a147ca93921..43046019fdd8 100644 --- a/dev-tools/mage/gotool/get.go +++ b/dev-tools/mage/gotool/get.go @@ -27,5 +27,6 @@ func runGoGet(opts ...ArgOpt) error { return runVGo("get", args) } +func (goGet) Download() ArgOpt { return flagBoolIf("-d", true) } func (goGet) Update() ArgOpt { return flagBoolIf("-u", true) } func (goGet) Package(pkg string) ArgOpt { return posArg(pkg) } diff --git a/dev-tools/mage/gotool/go.go b/dev-tools/mage/gotool/go.go index 30eaa6ff87d6..9e6cc08a4719 100644 --- a/dev-tools/mage/gotool/go.go +++ b/dev-tools/mage/gotool/go.go @@ -105,7 +105,7 @@ func ListModulePath(pkg string) (string, error) { return "", err } if n := len(lines); n != 1 { - return "", fmt.Errorf("expected 1 line, got %d", n) + return "", fmt.Errorf("expected 1 line, got %d while looking for %s", n, pkg) } return lines[0], nil } diff --git a/dev-tools/mage/gotool/modules.go b/dev-tools/mage/gotool/modules.go index 239425ee3217..ebaf2869ec5b 100644 --- a/dev-tools/mage/gotool/modules.go +++ b/dev-tools/mage/gotool/modules.go @@ -19,6 +19,7 @@ package gotool // Mod is the command go mod. var Mod = goMod{ + Init: modCommand{"init"}.run, Tidy: modCommand{"tidy"}.run, Verify: modCommand{"verify"}.run, Vendor: modCommand{"vendor"}.run, @@ -39,11 +40,15 @@ func (cmd modCommand) run(opts ...ArgOpt) error { } type goMod struct { + Init modInit Tidy modTidy Verify modVerify Vendor modVendor } +// modInit initializes a new go module in folder. +type modInit func(opts ...ArgOpt) error + // modTidy cleans the go.mod file type modTidy func(opts ...ArgOpt) error diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index cc1a42f86ef4..5b96e58e9f08 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -23,9 +23,7 @@ import ( "io/ioutil" "log" "os" - "path" "path/filepath" - "reflect" "regexp" "strconv" "strings" @@ -275,20 +273,15 @@ func ElasticBeatsDir() (string, error) { // it will return the root directory of the module from within the module cache or vendor // directory. func findElasticBeatsDir() (string, error) { - // Find the import path for the package containing this file. - type foo struct{} - typ := reflect.TypeOf(foo{}) - magepkgpath := typ.PkgPath() - - // Walk up the import path until we find the elastic/beats module path. - pkgpath := magepkgpath - for extractCanonicalRootImportPath(pkgpath) != elasticBeatsImportPath { - pkgpath = path.Dir(pkgpath) - if pkgpath == "." { - return "", errors.Errorf("failed to find %q from %q", elasticBeatsImportPath, magepkgpath) - } + repo, err := GetProjectRepoInfo() + if err != nil { + return "", err } - return gotool.ListModulePath(pkgpath) + if repo.IsElasticBeats() { + return repo.RootDir, nil + } + + return gotool.ListModulePath("github.com/elastic/beats/v7") } var ( diff --git a/docs/devguide/newbeat.asciidoc b/docs/devguide/newbeat.asciidoc index 22728fde8746..34261be27668 100644 --- a/docs/devguide/newbeat.asciidoc +++ b/docs/devguide/newbeat.asciidoc @@ -26,6 +26,10 @@ The following topics describe how to build a new Beat: * <> * <> +The following topic provide information about moving generated Beats to go modules: + +* <> + [[newbeat-getting-ready]] === Getting Ready @@ -128,7 +132,7 @@ mage GenerateCustomBeat -------------------- The mage script will prompt you to enter information about your Beat. For the `project_name`, enter `Countbeat`. -For the `github_name`, enter your github id. The `beat` and `beat_path` are set to the correct values automatically (just press Enter to accept each default). For the `full_name`, enter your firstname and lastname. +For the `github_name`, enter your github id. The `beat` and `beat_path` are set to the correct values automatically (just press Enter to accept each default). For the `full_name`, enter your firstname and lastname. Finally, pick the revision of elastic/beats you would like to depend on. [source,shell] --------- @@ -136,10 +140,13 @@ Enter a project name [examplebeat]: Countbeat Enter a github name [your-github-name]: {username} Enter a beat path [github.com/{username}/countbeat]: Enter a full name [Firstname Lastname]: {Full Name} +Enter the github.com/elastic/beats revision [master]: --------- The Beat generator creates a directory called `countbeat` inside of your project folder (e.g. {project folder}/github.com/{github id}/countbeat). +Please note that it is not possible to use revisions of Beats which does not support go modules. + You now have a raw template of the Beat, but you still need to <>. [[setting-up-beat]] @@ -482,3 +489,35 @@ func main() { When you're done with your new Beat, how about letting everyone know? Open a pull request to add your link to the {beats-ref}/community-beats.html[Community Beats] list. + +[[newbeat-migrate-gomodules]] +=== Migrate existing Beat to go modules + +Get started by making sure the contents of the vendored beats folder is +pushed to a remote repository. Otherwise, go will overwrite the folder +with the selected revision of `github.com/elastic/beats`. + +The following command will initialize a new module in your repo. + +[source,shell] +---------------------------------------------------------------------- +go mod init +---------------------------------------------------------------------- + +Make sure to add the approprite `replace` directives from the `go.mod` file +of Beats: https://github.com/elastic/beats/tree/master/go.mod +Unfortunately, this workaround is needed to make sure your Beat can be compiled. + +To depend on the latest `master` of `github.com/elastic/beats` run the following command: + +[source,shell] +---------------------------------------------------------------------- +go get github.com/elastic/beats@master +---------------------------------------------------------------------- + +To move the dependencies to vendor, you need to manually fetch the new +`magefile.go` for newly generated Beats from the dev-tools of `elastic/beats`. + +We suggest you read the following section to learn about maintaining dependencies +using go modules: +* https://github.com/golang/go/wiki/Modules#how-to-upgrade-and-downgrade-dependencies[How to upgrade and downgrade dependencies] diff --git a/generator/_templates/beat/{beat}/Makefile b/generator/_templates/beat/{beat}/Makefile index 0246f73dc98e..6ad41d814564 100644 --- a/generator/_templates/beat/{beat}/Makefile +++ b/generator/_templates/beat/{beat}/Makefile @@ -3,11 +3,12 @@ BEAT_PATH={beat_path} BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH})) SYSTEM_TESTS=false TEST_ENVIRONMENT=false -ES_BEATS?=./vendor/github.com/elastic/beats +ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7 +ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH} LIBBEAT_MAKEFILE=$(ES_BEATS)/libbeat/scripts/Makefile -GOPACKAGES=$(shell govendor list -no-status +local) -GOBUILD_FLAGS=-i -ldflags "-X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)" -MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage +GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools) +GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)" +MAGE_IMPORT_PATH=github.com/magefile/mage NO_COLLECT=true CHECK_HEADERS_DISABLED=true @@ -16,4 +17,4 @@ CHECK_HEADERS_DISABLED=true .PHONY: copy-vendor copy-vendor: - mage vendorUpdate \ No newline at end of file + mage vendorUpdate diff --git a/generator/_templates/beat/{beat}/_meta/beat.reference.yml b/generator/_templates/beat/{beat}/_meta/beat.reference.yml new file mode 100644 index 000000000000..7d9bb3942c30 --- /dev/null +++ b/generator/_templates/beat/{beat}/_meta/beat.reference.yml @@ -0,0 +1,7 @@ +################### {Beat} Configuration Example ######################### + +############################# {Beat} ###################################### + +{beat}: + # Defines how often an event is sent to the output + period: 1s diff --git a/generator/_templates/beat/{beat}/include/include.go b/generator/_templates/beat/{beat}/include/include.go new file mode 100644 index 000000000000..8e731cdfc380 --- /dev/null +++ b/generator/_templates/beat/{beat}/include/include.go @@ -0,0 +1 @@ +package include diff --git a/generator/_templates/beat/{beat}/magefile.go b/generator/_templates/beat/{beat}/magefile.go index b79448c2aaa5..d924f3c79461 100644 --- a/generator/_templates/beat/{beat}/magefile.go +++ b/generator/_templates/beat/{beat}/magefile.go @@ -25,7 +25,7 @@ func init() { devtools.BeatProjectType = devtools.CommunityProject } -// VendorUpdate updates elastic/beats/v7 in the vendor dir +// VendorUpdate updates the vendor dir func VendorUpdate() error { return beatgen.VendorUpdate() } diff --git a/generator/_templates/beat/{beat}/tools/tools.go b/generator/_templates/beat/{beat}/tools/tools.go new file mode 100644 index 000000000000..d816e58647fd --- /dev/null +++ b/generator/_templates/beat/{beat}/tools/tools.go @@ -0,0 +1,14 @@ +// +build tools + +// This package contains the tool dependencies of the project. + +package tools + +import ( + _ "github.com/pierrre/gotestcover" + _ "golang.org/x/tools/cmd/goimports" + + _ "github.com/mitchellh/gox" + _ "github.com/reviewdog/reviewdog/cmd/reviewdog" + _ "golang.org/x/lint/golint" +) diff --git a/generator/_templates/metricbeat/{beat}/Makefile b/generator/_templates/metricbeat/{beat}/Makefile index 932b05083789..453344c8feb1 100644 --- a/generator/_templates/metricbeat/{beat}/Makefile +++ b/generator/_templates/metricbeat/{beat}/Makefile @@ -3,10 +3,11 @@ BEAT_PATH={beat_path} BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH})) SYSTEM_TESTS=false TEST_ENVIRONMENT=false -ES_BEATS?=./vendor/github.com/elastic/beats -GOPACKAGES=$(shell govendor list -no-status +local) -GOBUILD_FLAGS=-i -ldflags "-X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)" -MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage +ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7 +ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH} +GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools) +GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)" +MAGE_IMPORT_PATH=github.com/magefile/mage CHECK_HEADERS_DISABLED=true # Path to the libbeat Makefile @@ -14,4 +15,4 @@ CHECK_HEADERS_DISABLED=true .PHONY: copy-vendor copy-vendor: - mage vendorUpdate \ No newline at end of file + mage vendorUpdate diff --git a/generator/_templates/metricbeat/{beat}/README.md b/generator/_templates/metricbeat/{beat}/README.md index cf71741f5fe6..10e88d88b515 100644 --- a/generator/_templates/metricbeat/{beat}/README.md +++ b/generator/_templates/metricbeat/{beat}/README.md @@ -33,14 +33,11 @@ make collect This updates all fields and docs with the most recent changes. -## Use vendoring +## Dependency management -We recommend to use vendoring for your beat. This means the dependencies are put into your beat folder. The beats team currently uses [govendor](https://github.com/kardianos/govendor) for vendoring. +Beats uses go modules for dependency management. -``` -govendor init -govendor update +e -``` +In order to maintain your `vendor` folder, run `mage copyVendor`. This will create a directory `vendor` inside your repository. To make sure all dependencies for the Makefile commands are loaded from the vendor directory, find the following line in your Makefile: @@ -50,7 +47,7 @@ ES_BEATS=${GOPATH}/src/github.com/elastic/beats Replace it with: ``` -ES_BEATS=./vendor/github.com/elastic/beats +ES_BEATS=./vendor/github.com/elastic/beats/v7 ``` diff --git a/generator/_templates/metricbeat/{beat}/include/include.go b/generator/_templates/metricbeat/{beat}/include/include.go new file mode 100644 index 000000000000..8e731cdfc380 --- /dev/null +++ b/generator/_templates/metricbeat/{beat}/include/include.go @@ -0,0 +1 @@ +package include diff --git a/generator/_templates/metricbeat/{beat}/tools/tools.go b/generator/_templates/metricbeat/{beat}/tools/tools.go new file mode 100644 index 000000000000..d816e58647fd --- /dev/null +++ b/generator/_templates/metricbeat/{beat}/tools/tools.go @@ -0,0 +1,14 @@ +// +build tools + +// This package contains the tool dependencies of the project. + +package tools + +import ( + _ "github.com/pierrre/gotestcover" + _ "golang.org/x/tools/cmd/goimports" + + _ "github.com/mitchellh/gox" + _ "github.com/reviewdog/reviewdog/cmd/reviewdog" + _ "golang.org/x/lint/golint" +) diff --git a/generator/common/Makefile b/generator/common/Makefile index d19c6e737788..b01320c7e436 100644 --- a/generator/common/Makefile +++ b/generator/common/Makefile @@ -5,6 +5,8 @@ BEAT_PATH=${GOPATH}/src/${BEAT_NAME} ES_BEATS=${GOPATH}/src/github.com/elastic/beats PREPARE_COMMAND?= +-include ${ES_BEATS}/dev-tools/make/mage.mk + # Runs test build for mock beat .PHONY: test test: prepare-test @@ -23,36 +25,28 @@ test-package: test mage package .PHONY: prepare-test -prepare-test:: ${GOPATH}/bin/mage +prepare-test:: mage rm -fr ${BEAT_PATH} mkdir -p ${BEAT_PATH} cd ${GOPATH}/src/github.com/elastic/beats/ ; \ export MODULE=elastic ; \ export METRICSET=test ; \ - export GO111MODULE=off; \ export NEWBEAT_PROJECT_NAME=Testbeat ; \ export NEWBEAT_GITHUB_NAME=ruflin ; \ export NEWBEAT_BEAT_PATH=${BEAT_NAME} ; \ export NEWBEAT_FULL_NAME="Nicolas Ruflin" ; \ export NEWBEAT_TYPE=${BEAT_TYPE} ; \ export NEWBEAT_DEV=1 ; \ + export NEWBEAT_BEATS_REVISION=master ; \ mage GenerateCustomBeat # Runs test build for the created beat .PHONY: test-build test-build: test - # Copy dev tools - cp -r ${PWD}/../../../dev-tools ${BEAT_PATH}/vendor/github.com/elastic/beats/ - cd ${BEAT_PATH}/dev-tools/packer ; \ $(MAKE) deps ; \ $(MAKE) images -${GOPATH}/bin/mage: - go get -u -d github.com/magefile/mage - cd $${GOPATH}/src/github.com/magefile/mage; \ - go run bootstrap.go - # Cleans up environment .PHONY: clean clean: diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go index cd1d1900e883..7060e4a142b8 100644 --- a/generator/common/beatgen/beatgen.go +++ b/generator/common/beatgen/beatgen.go @@ -30,6 +30,7 @@ import ( "github.com/pkg/errors" devtools "github.com/elastic/beats/v7/dev-tools/mage" + "github.com/elastic/beats/v7/dev-tools/mage/gotool" "github.com/elastic/beats/v7/generator/common/beatgen/setup" ) @@ -80,17 +81,29 @@ var configList = []ConfigItem{ return "beat" }, }, + { + Key: "beats_revision", + Help: "Enter the github.com/elastic/beats revision", + Default: func(cfg map[string]string) string { + return "master" + }, + }, } // Generate generates a new custom beat func Generate() error { cfg, err := getConfig() if err != nil { - return errors.Wrap(err, "Error getting config") + return errors.Wrap(err, "error getting config") } + beatsModuleName, err := gotool.GetModuleName() + if err != nil { + return err + } + err = setup.GenNewBeat(cfg) if err != nil { - return errors.Wrap(err, "Error generating new beat") + return errors.Wrap(err, "error generating new beat") } absBeatPath := filepath.Join(build.Default.GOPATH, "src", cfg["beat_path"]) @@ -100,8 +113,14 @@ func Generate() error { return errors.Wrap(err, "error changing directory") } + mg.Deps(setup.InitModule) + + err = getConfiguredBeatsRevision(beatsModuleName, cfg["beats_revision"]) + if err != nil { + return errors.Wrap(err, "error while getting required beats version") + } + mg.Deps(setup.CopyVendor) - mg.Deps(setup.RunSetup) mg.Deps(setup.GitInit) if cfg["type"] == "metricbeat" { @@ -122,7 +141,15 @@ func Generate() error { return nil } -// VendorUpdate updates the beat vendor directory +func getConfiguredBeatsRevision(beatsModule, revision string) error { + beatsPkg := beatsModule + "@" + revision + return gotool.Get( + gotool.Get.Download(), + gotool.Get.Package(beatsPkg), + ) +} + +// VendorUpdate updates the vendor directory func VendorUpdate() error { err := sh.Rm("./vendor/github.com/elastic/beats") if err != nil { diff --git a/generator/common/beatgen/setup/creator.go b/generator/common/beatgen/setup/creator.go index 8656bf5a5f1f..d6e07d2228f5 100644 --- a/generator/common/beatgen/setup/creator.go +++ b/generator/common/beatgen/setup/creator.go @@ -40,7 +40,7 @@ func GenNewBeat(config map[string]string) error { return fmt.Errorf("%s is not a valid custom beat type. Valid types are 'beat' and 'metricbeat'", config["type"]) } - genPath := devtools.OSSBeatDir("generator", config["type"], "{beat}") + genPath := devtools.OSSBeatDir("generator", "_templates", config["type"], "{beat}") err := filepath.Walk(genPath, func(path string, info os.FileInfo, err error) error { newBase := filepath.Join(build.Default.GOPATH, "src", config["beat_path"]) replacePath := strings.Replace(path, genPath, newBase, -1) diff --git a/generator/common/beatgen/setup/setup.go b/generator/common/beatgen/setup/setup.go index b0bb5427e22d..c4ef95c76540 100644 --- a/generator/common/beatgen/setup/setup.go +++ b/generator/common/beatgen/setup/setup.go @@ -18,7 +18,7 @@ package setup import ( - "fmt" + "bufio" "os" "path/filepath" @@ -26,103 +26,108 @@ import ( "github.com/pkg/errors" devtools "github.com/elastic/beats/v7/dev-tools/mage" + "github.com/elastic/beats/v7/dev-tools/mage/gotool" ) -// RunSetup runs any remaining setup commands after the vendor directory has been setup -func RunSetup() error { - vendorPath := "./vendor/github.com/" - - //Copy mage stuff - toMkdir := filepath.Join(vendorPath, "magefile") - err := os.MkdirAll(toMkdir, 0755) - if err != nil { - return errors.Wrapf(err, "error making mage directory at %s", toMkdir) +var ( + makefileDeps = []string{ + "dev-tools", + "libbeat", + "licenses", + "metricbeat", + "script", } +) - err = sh.Run("cp", "-R", filepath.Join(vendorPath, "elastic/beats/vendor/github.com/magefile/mage"), filepath.Join(vendorPath, "magefile")) +func InitModule() error { + err := gotool.Mod.Init() if err != nil { - return errors.Wrapf(err, "error copying vendored magefile to %s", filepath.Join(vendorPath, "magefile")) + return errors.Wrap(err, "error initializing a module for the Beat") } - //Copy the pkg helper - err = sh.Run("cp", "-R", filepath.Join(vendorPath, "elastic/beats/vendor/github.com/pkg"), vendorPath) - if err != nil { - return errors.Wrapf(err, "error copying pkg to %s", vendorPath) - } - return nil + return copyReplacedModules() } -// CopyVendor copies a new version of beats into the vendor directory of PWD -// By default this uses git archive, meaning any uncommitted changes will not be copied. -// Set the NEWBEAT_DEV env variable to use a slow `cp` copy that will catch uncommited changes -func CopyVendor() error { - vendorPath := "./vendor/github.com/elastic/" +func copyReplacedModules() error { + const goModPath = "go.mod" + beatPath, err := devtools.ElasticBeatsDir() if err != nil { - return errors.Wrap(err, "Could not find ElasticBeatsDir") + return errors.Wrap(err, "error determining path to github.com/elastic/beats") } - err = os.MkdirAll(vendorPath, 0755) + + inMod, err := os.Open(filepath.Join(beatPath, goModPath)) if err != nil { - return errors.Wrap(err, "error creating vendor dir") + return errors.Wrap(err, "error opening go.mod file of github.com/elastic/beats") } + defer inMod.Close() - isClean, err := checkBeatsDirClean() - if err != nil { - return errors.Wrap(err, "error in checkIfBeatsDirClean") + s := bufio.NewScanner(inMod) + inReplaceSection := false + replacedLines := []string{ + "", + "replace (", } + for s.Scan() { + if err := s.Err(); err != nil { + return errors.Wrap(err, "error reading go.mod file") + } - if !isClean { - //Dev mode. Use CP. - fmt.Printf("You have uncommited changes in elastic/beats. Running CopyVendor running in dev mode, elastic/beats will be copied into the vendor directory with cp\n") - vendorPath = filepath.Join(vendorPath, "beats") + line := s.Text() + if inReplaceSection { + replacedLines = append(replacedLines, line) - err = sh.Run("cp", "-R", beatPath, vendorPath) - if err != nil { - return errors.Wrap(err, "error copying vendor dir") - } - err = sh.Rm(filepath.Join(vendorPath, ".git")) - if err != nil { - return errors.Wrap(err, "error removing vendor git directory") - } - err = sh.Rm(filepath.Join(vendorPath, "x-pack")) - if err != nil { - return errors.Wrap(err, "error removing x-pack directory") + if line == ")" { + break + } + continue } - } else { - //not dev mode. Use git archive - vendorPath = filepath.Join(vendorPath, "beats") - err = os.MkdirAll(vendorPath, 0755) - if err != nil { - return errors.Wrap(err, "error creating vendor dir") + + if line == "replace (" { + inReplaceSection = true } - err = sh.Run("sh", - "-c", - "git archive --remote "+beatPath+" HEAD | tar -x --exclude=x-pack -C "+vendorPath) + } + + outMod, err := os.OpenFile(goModPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) + if err != nil { + return errors.Wrap(err, "error opening the go.mod file of the generated Beat") + } + defer outMod.Close() + + w := bufio.NewWriter(outMod) + for _, rep := range replacedLines { + _, err = w.WriteString(rep + "\n") if err != nil { - return errors.Wrap(err, "error running git archive") + return errors.Wrap(err, "error writing replace lines to go.mod file") } } - return nil - + return w.Flush() } -// checkIfBeatsDirClean checks to see if the working elastic/beats dir is modified. -// If it is, we'll use a different method to copy beats to vendor/ -func checkBeatsDirClean() (bool, error) { - beatPath, err := devtools.ElasticBeatsDir() +// CopyVendor copies a new version of the dependencies to the vendor folder +func CopyVendor() error { + err := gotool.Mod.Vendor() if err != nil { - return false, errors.Wrap(err, "Could not find ElasticBeatsDir") + return err } - out, err := sh.Output("git", "-C", beatPath, "status", "--porcelain") + + path, err := gotool.ListModulePath("github.com/elastic/beats/v7") if err != nil { - return false, errors.Wrap(err, "Error checking status of elastic/beats repo") + return err } - if len(out) == 0 { - return true, nil + vendorPath := "./vendor/github.com/elastic/beats/v7" + for _, d := range makefileDeps { + from := filepath.Join(path, d) + to := filepath.Join(vendorPath, d) + copyTask := &devtools.CopyTask{Source: from, Dest: to, Mode: 0640, DirMode: os.ModeDir | 0750} + err = copyTask.Execute() + if err != nil { + return err + } } - return false, nil + return nil } // GitInit initializes a new git repo in the current directory From 3f0775db91219c4a07b3032ccbe18482b5fd0d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 3 Mar 2020 09:38:08 +0100 Subject: [PATCH 2/6] Enable CI jobs for generators again --- .travis.yml | 33 +++++++++++----------- Jenkinsfile | 80 ++++++++++++++++++++++++++--------------------------- 2 files changed, 56 insertions(+), 57 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9c5eae3e17c8..82f87c1e08e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -148,24 +148,23 @@ jobs: stage: test # Generators - # Temporarily disable generator jobs - #- os: linux - # env: TARGETS="-C generator/_templates/metricbeat test test-package" - # go: $TRAVIS_GO_VERSION - # stage: test - #- os: linux - # env: TARGETS="-C generator/_templates/beat test test-package" - # go: $TRAVIS_GO_VERSION - # stage: test + - os: linux + env: TARGETS="-C generator/_templates/metricbeat test test-package" + go: $TRAVIS_GO_VERSION + stage: test + - os: linux + env: TARGETS="-C generator/_templates/beat test test-package" + go: $TRAVIS_GO_VERSION + stage: test - #- os: osx - # env: TARGETS="-C generator/_templates/metricbeat test" - # go: $TRAVIS_GO_VERSION - # stage: test - #- os: osx - # env: TARGETS="-C generator/_templates/beat test" - # go: $TRAVIS_GO_VERSION - # stage: test + - os: osx + env: TARGETS="-C generator/_templates/metricbeat test" + go: $TRAVIS_GO_VERSION + stage: test + - os: osx + env: TARGETS="-C generator/_templates/beat test" + go: $TRAVIS_GO_VERSION + stage: test # Kubernetes - os: linux diff --git a/Jenkinsfile b/Jenkinsfile index 79145a83e41f..3789de360967 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -459,45 +459,45 @@ pipeline { } } } - // Temporarily disable generator jobs - //stage('Generators'){ - // agent { label 'ubuntu && immutable' } - // options { skipDefaultCheckout() } - // when { - // beforeAgent true - // expression { - // return env.BUILD_GENERATOR != "false" - // } - // } - // stages { - // stage('Generators Metricbeat Linux'){ - // steps { - // makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test") - // makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test-package") - // } - // } - // stage('Generators Beat Linux'){ - // steps { - // makeTarget("Generators Beat Linux", "-C generator/_templates/beat test") - // makeTarget("Generators Beat Linux", "-C generator/_templates/beat test-package") - // } - // } - // stage('Generators Metricbeat Mac OS X'){ - // agent { label 'macosx' } - // options { skipDefaultCheckout() } - // steps { - // makeTarget("Generators Metricbeat Mac OS X", "-C generator/_templates/metricbeat test") - // } - // } - // stage('Generators Beat Mac OS X'){ - // agent { label 'macosx' } - // options { skipDefaultCheckout() } - // steps { - // makeTarget("Generators Beat Mac OS X", "-C generator/_templates/beat test") - // } - // } - // } - //} + Temporarily disable generator jobs + stage('Generators'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_GENERATOR != "false" + } + } + stages { + stage('Generators Metricbeat Linux'){ + steps { + makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test") + makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test-package") + } + } + stage('Generators Beat Linux'){ + steps { + makeTarget("Generators Beat Linux", "-C generator/_templates/beat test") + makeTarget("Generators Beat Linux", "-C generator/_templates/beat test-package") + } + } + stage('Generators Metricbeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + steps { + makeTarget("Generators Metricbeat Mac OS X", "-C generator/_templates/metricbeat test") + } + } + stage('Generators Beat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + steps { + makeTarget("Generators Beat Mac OS X", "-C generator/_templates/beat test") + } + } + } + } stage('Kubernetes'){ agent { label 'ubuntu && immutable' } options { skipDefaultCheckout() } @@ -673,7 +673,7 @@ def loadConfigEnvVars(){ "^x-pack/functionbeat/.*", "^x-pack/libbeat/.*", ]) - //env.BUILD_GENERATOR = isChanged(["^generator/.*"]) + env.BUILD_GENERATOR = isChanged(["^generator/.*"]) env.BUILD_HEARTBEAT = isChanged(["^heartbeat/.*"]) env.BUILD_HEARTBEAT_XPACK = isChanged([ "^heartbeat/.*", From 67e880721c0ccfcfad10d2ace424daaa446f8db7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 3 Mar 2020 12:52:58 +0100 Subject: [PATCH 3/6] remove leftover from comment --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3789de360967..446dfe576d80 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -459,7 +459,6 @@ pipeline { } } } - Temporarily disable generator jobs stage('Generators'){ agent { label 'ubuntu && immutable' } options { skipDefaultCheckout() } From e8dc81a95492be7a9debfccbaec3ab9d2d585f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 4 Mar 2020 16:09:07 +0100 Subject: [PATCH 4/6] Adjust path to generators in Makefile --- generator/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/generator/Makefile b/generator/Makefile index 92e1eddc61bc..6f0122cc3601 100644 --- a/generator/Makefile +++ b/generator/Makefile @@ -1,11 +1,11 @@ # Runs test build for the count metricset .PHONY: test test: - $(MAKE) -C beat test - $(MAKE) -C metricbeat test + $(MAKE) -C _templates/beat test + $(MAKE) -C _templates/metricbeat test # Cleans up environment .PHONY: clean clean: - @$(MAKE) -C beat clean - @$(MAKE) -C metricbeat clean + @$(MAKE) -C _templates/beat clean + @$(MAKE) -C _templates/metricbeat clean From 971da5e859ae367108c64462d8cc50eb72351930 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 4 Mar 2020 17:18:13 +0100 Subject: [PATCH 5/6] add -u to go get --- generator/common/beatgen/beatgen.go | 1 + 1 file changed, 1 insertion(+) diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go index 7060e4a142b8..a4a1f436a13a 100644 --- a/generator/common/beatgen/beatgen.go +++ b/generator/common/beatgen/beatgen.go @@ -145,6 +145,7 @@ func getConfiguredBeatsRevision(beatsModule, revision string) error { beatsPkg := beatsModule + "@" + revision return gotool.Get( gotool.Get.Download(), + gotool.Get.Update(), gotool.Get.Package(beatsPkg), ) } From 99fe80b7d9621fe9d4aed041e95fc4f1fc13800b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 4 Mar 2020 17:49:07 +0100 Subject: [PATCH 6/6] find the last commit of master --- generator/common/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/common/Makefile b/generator/common/Makefile index b01320c7e436..0a7b3608daea 100644 --- a/generator/common/Makefile +++ b/generator/common/Makefile @@ -37,7 +37,7 @@ prepare-test:: mage export NEWBEAT_FULL_NAME="Nicolas Ruflin" ; \ export NEWBEAT_TYPE=${BEAT_TYPE} ; \ export NEWBEAT_DEV=1 ; \ - export NEWBEAT_BEATS_REVISION=master ; \ + export NEWBEAT_BEATS_REVISION=$(shell git rev-parse origin/master) ; \ mage GenerateCustomBeat # Runs test build for the created beat