Fix generator jobs for go modules

.travis.yml

@@ -148,24 +148,23 @@ jobs:
       stage: test
 
     # Generators
-    # Temporarily disable generator jobs
-    #- os: linux
-    #  env: TARGETS="-C generator/_templates/metricbeat test test-package"
-    #  go: $TRAVIS_GO_VERSION
-    #  stage: test
-    #- os: linux
-    #  env: TARGETS="-C generator/_templates/beat test test-package"
-    #  go: $TRAVIS_GO_VERSION
-    #  stage: test
+    - os: linux
+      env: TARGETS="-C generator/_templates/metricbeat test test-package"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: linux
+      env: TARGETS="-C generator/_templates/beat test test-package"
+      go: $TRAVIS_GO_VERSION
+      stage: test
 
-    #- os: osx
-    #  env: TARGETS="-C generator/_templates/metricbeat test"
-    #  go: $TRAVIS_GO_VERSION
-    #  stage: test
-    #- os: osx
-    #  env: TARGETS="-C generator/_templates/beat test"
-    #  go: $TRAVIS_GO_VERSION
-    #  stage: test
+    - os: osx
+      env: TARGETS="-C generator/_templates/metricbeat test"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: osx
+      env: TARGETS="-C generator/_templates/beat test"
+      go: $TRAVIS_GO_VERSION
+      stage: test
 
     # Kubernetes
     - os: linux

Jenkinsfile

@@ -459,45 +459,44 @@ pipeline {
             }
           }
         }
-        // Temporarily disable generator jobs
-        //stage('Generators'){
-        //  agent { label 'ubuntu && immutable' }
-        //  options { skipDefaultCheckout() }
-        //  when {
-        //    beforeAgent true
-        //    expression {
-        //      return env.BUILD_GENERATOR != "false"
-        //    }
-        //  }
-        //  stages {
-        //    stage('Generators Metricbeat Linux'){
-        //      steps {
-        //        makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test")
-        //        makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test-package")
-        //      }
-        //    }
-        //    stage('Generators Beat Linux'){
-        //      steps {
-        //        makeTarget("Generators Beat Linux", "-C generator/_templates/beat test")
-        //        makeTarget("Generators Beat Linux", "-C generator/_templates/beat test-package")
-        //      }
-        //    }
-        //    stage('Generators Metricbeat Mac OS X'){
-        //      agent { label 'macosx' }
-        //      options { skipDefaultCheckout() }
-        //      steps {
-        //        makeTarget("Generators Metricbeat Mac OS X", "-C generator/_templates/metricbeat test")
-        //      }
-        //    }
-        //    stage('Generators Beat Mac OS X'){
-        //      agent { label 'macosx' }
-        //      options { skipDefaultCheckout() }
-        //      steps {
-        //        makeTarget("Generators Beat Mac OS X", "-C generator/_templates/beat test")
-        //      }
-        //    }
-        //  }
-        //}
+        stage('Generators'){
+          agent { label 'ubuntu && immutable' }
+          options { skipDefaultCheckout() }
+          when {
+            beforeAgent true
+            expression {
+              return env.BUILD_GENERATOR != "false"
+            }
+          }
+          stages {
+            stage('Generators Metricbeat Linux'){
+              steps {
+                makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test")
+                makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test-package")
+              }
+            }
+            stage('Generators Beat Linux'){
+              steps {
+                makeTarget("Generators Beat Linux", "-C generator/_templates/beat test")
+                makeTarget("Generators Beat Linux", "-C generator/_templates/beat test-package")
+              }
+            }
+            stage('Generators Metricbeat Mac OS X'){
+              agent { label 'macosx' }
+              options { skipDefaultCheckout() }
+              steps {
+                makeTarget("Generators Metricbeat Mac OS X", "-C generator/_templates/metricbeat test")
+              }
+            }
+            stage('Generators Beat Mac OS X'){
+              agent { label 'macosx' }
+              options { skipDefaultCheckout() }
+              steps {
+                makeTarget("Generators Beat Mac OS X", "-C generator/_templates/beat test")
+              }
+            }
+          }
+        }
         stage('Kubernetes'){
           agent { label 'ubuntu && immutable' }
           options { skipDefaultCheckout() }
@@ -673,7 +672,7 @@ def loadConfigEnvVars(){
     "^x-pack/functionbeat/.*",
     "^x-pack/libbeat/.*",
   ])
-  //env.BUILD_GENERATOR = isChanged(["^generator/.*"])
+  env.BUILD_GENERATOR = isChanged(["^generator/.*"])
   env.BUILD_HEARTBEAT = isChanged(["^heartbeat/.*"])
   env.BUILD_HEARTBEAT_XPACK = isChanged([
     "^heartbeat/.*",

dev-tools/mage/gomod.go

@@ -18,7 +18,6 @@
 package mage
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
 
@@ -80,7 +79,6 @@ func Vendor() error {
 		if err != nil {
 			return err
 		}
-		fmt.Println(path)
 
 		for _, f := range p.filesToCopy {
 			from := filepath.Join(path, f)

dev-tools/mage/gotool/get.go

@@ -27,5 +27,6 @@ func runGoGet(opts ...ArgOpt) error {
 	return runVGo("get", args)
 }
 
+func (goGet) Download() ArgOpt          { return flagBoolIf("-d", true) }
 func (goGet) Update() ArgOpt            { return flagBoolIf("-u", true) }
 func (goGet) Package(pkg string) ArgOpt { return posArg(pkg) }

dev-tools/mage/gotool/go.go

@@ -105,7 +105,7 @@ func ListModulePath(pkg string) (string, error) {
 		return "", err
 	}
 	if n := len(lines); n != 1 {
-		return "", fmt.Errorf("expected 1 line, got %d", n)
+		return "", fmt.Errorf("expected 1 line, got %d while looking for %s", n, pkg)
 	}
 	return lines[0], nil
 }

dev-tools/mage/gotool/modules.go

@@ -19,6 +19,7 @@ package gotool
 
 // Mod is the command go mod.
 var Mod = goMod{
+	Init:   modCommand{"init"}.run,
 	Tidy:   modCommand{"tidy"}.run,
 	Verify: modCommand{"verify"}.run,
 	Vendor: modCommand{"vendor"}.run,
@@ -39,11 +40,15 @@ func (cmd modCommand) run(opts ...ArgOpt) error {
 }
 
 type goMod struct {
+	Init   modInit
 	Tidy   modTidy
 	Verify modVerify
 	Vendor modVendor
 }
 
+// modInit initializes a new go module in folder.
+type modInit func(opts ...ArgOpt) error
+
 // modTidy cleans the go.mod file
 type modTidy func(opts ...ArgOpt) error
 

dev-tools/mage/settings.go

@@ -23,9 +23,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-	"path"
 	"path/filepath"
-	"reflect"
 	"regexp"
 	"strconv"
 	"strings"
@@ -275,20 +273,15 @@ func ElasticBeatsDir() (string, error) {
 // it will return the root directory of the module from within the module cache or vendor
 // directory.
 func findElasticBeatsDir() (string, error) {
-	// Find the import path for the package containing this file.
-	type foo struct{}
-	typ := reflect.TypeOf(foo{})
-	magepkgpath := typ.PkgPath()
-
-	// Walk up the import path until we find the elastic/beats module path.
-	pkgpath := magepkgpath
-	for extractCanonicalRootImportPath(pkgpath) != elasticBeatsImportPath {
-		pkgpath = path.Dir(pkgpath)
-		if pkgpath == "." {
-			return "", errors.Errorf("failed to find %q from %q", elasticBeatsImportPath, magepkgpath)
-		}
+	repo, err := GetProjectRepoInfo()
+	if err != nil {
+		return "", err
 	}
-	return gotool.ListModulePath(pkgpath)
+	if repo.IsElasticBeats() {
+		return repo.RootDir, nil
+	}
+
+	return gotool.ListModulePath("github.com/elastic/beats/v7")
 }
 
 var (

docs/devguide/newbeat.asciidoc

@@ -26,6 +26,10 @@ The following topics describe how to build a new Beat:
 * <<newbeat-sharing>>
 * <<event-conventions>>
 
+The following topic provide information about moving generated Beats to go modules:
+
+* <<newbeat-migrate-gomodules>>
+
 [[newbeat-getting-ready]]
 === Getting Ready
 
@@ -128,18 +132,21 @@ mage GenerateCustomBeat
 --------------------
 
 The mage script will prompt you to enter information about your Beat. For the `project_name`, enter `Countbeat`.
-For the `github_name`, enter your github id. The `beat` and `beat_path` are set to the correct values automatically (just press Enter to accept each default). For the `full_name`, enter your firstname and lastname.
+For the `github_name`, enter your github id. The `beat` and `beat_path` are set to the correct values automatically (just press Enter to accept each default). For the `full_name`, enter your firstname and lastname. Finally, pick the revision of elastic/beats you would like to depend on.
 
 [source,shell]
 ---------
 Enter a project name [examplebeat]:  Countbeat
 Enter a github name [your-github-name]: {username}
 Enter a beat path [github.com/{username}/countbeat]: 
 Enter a full name [Firstname Lastname]: {Full Name}
+Enter the github.com/elastic/beats revision [master]:
 ---------
 
 The Beat generator creates a directory called `countbeat` inside of your project folder (e.g. {project folder}/github.com/{github id}/countbeat).
 
+Please note that it is not possible to use revisions of Beats which does not support go modules.
+
 You now have a raw template of the Beat, but you still need to <<setting-up-beat,fetch dependencies and set up the Beat>>.
 
 [[setting-up-beat]]
@@ -482,3 +489,35 @@ func main() {
 
 When you're done with your new Beat, how about letting everyone know? Open
 a pull request to add your link to the {beats-ref}/community-beats.html[Community Beats] list.
+
+[[newbeat-migrate-gomodules]]
+=== Migrate existing Beat to go modules
+
+Get started by making sure the contents of the vendored beats folder is
+pushed to a remote repository. Otherwise, go will overwrite the folder
+with the selected revision of `github.com/elastic/beats`.
+
+The following command will initialize a new module in your repo.
+
+[source,shell]
+----------------------------------------------------------------------
+go mod init
+----------------------------------------------------------------------
+
+Make sure to add the approprite `replace` directives from the `go.mod` file
+of Beats: https://github.com/elastic/beats/tree/master/go.mod
+Unfortunately, this workaround is needed to make sure your Beat can be compiled.
+
+To depend on the latest `master` of `github.com/elastic/beats` run the following command:
+
+[source,shell]
+----------------------------------------------------------------------
+go get github.com/elastic/beats@master
+----------------------------------------------------------------------
+
+To move the dependencies to vendor, you need to manually fetch the new
+`magefile.go` for newly generated Beats from the dev-tools of `elastic/beats`.
+
+We suggest you read the following section to learn about maintaining dependencies
+using go modules:
+* https://github.com/golang/go/wiki/Modules#how-to-upgrade-and-downgrade-dependencies[How to upgrade and downgrade dependencies]

generator/Makefile

@@ -1,11 +1,11 @@
 # Runs test build for the count metricset
 .PHONY: test
 test:
-	$(MAKE) -C beat test
-	$(MAKE) -C metricbeat test
+	$(MAKE) -C _templates/beat test
+	$(MAKE) -C _templates/metricbeat test
 
 # Cleans up environment
 .PHONY: clean
 clean:
-	@$(MAKE) -C beat clean
-	@$(MAKE) -C metricbeat clean
+	@$(MAKE) -C _templates/beat clean
+	@$(MAKE) -C _templates/metricbeat clean

generator/_templates/beat/{beat}/Makefile

@@ -3,11 +3,12 @@ BEAT_PATH={beat_path}
 BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
-ES_BEATS?=./vendor/github.com/elastic/beats
+ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7
+ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH}
 LIBBEAT_MAKEFILE=$(ES_BEATS)/libbeat/scripts/Makefile
-GOPACKAGES=$(shell govendor list -no-status +local)
-GOBUILD_FLAGS=-i -ldflags "-X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)"
-MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage
+GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools)
+GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)"
+MAGE_IMPORT_PATH=github.com/magefile/mage
 NO_COLLECT=true
 CHECK_HEADERS_DISABLED=true
 
@@ -16,4 +17,4 @@ CHECK_HEADERS_DISABLED=true
 
 .PHONY: copy-vendor
 copy-vendor:
-	mage vendorUpdate
+	mage vendorUpdate

generator/_templates/beat/{beat}/_meta/beat.reference.yml

@@ -0,0 +1,7 @@
+################### {Beat} Configuration Example #########################
+
+############################# {Beat} ######################################
+
+{beat}:
+  # Defines how often an event is sent to the output
+  period: 1s

generator/_templates/beat/{beat}/include/include.go

@@ -0,0 +1 @@
+package include

generator/_templates/beat/{beat}/magefile.go

@@ -25,7 +25,7 @@ func init() {
 	devtools.BeatProjectType = devtools.CommunityProject
 }
 
-// VendorUpdate updates elastic/beats/v7 in the vendor dir
+// VendorUpdate updates the vendor dir
 func VendorUpdate() error {
 	return beatgen.VendorUpdate()
 }

generator/_templates/beat/{beat}/tools/tools.go

@@ -0,0 +1,14 @@
+// +build tools
+
+// This package contains the tool dependencies of the project.
+
+package tools
+
+import (
+	_ "github.com/pierrre/gotestcover"
+	_ "golang.org/x/tools/cmd/goimports"
+
+	_ "github.com/mitchellh/gox"
+	_ "github.com/reviewdog/reviewdog/cmd/reviewdog"
+	_ "golang.org/x/lint/golint"
+)

generator/_templates/metricbeat/{beat}/Makefile

@@ -3,15 +3,16 @@ BEAT_PATH={beat_path}
 BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
-ES_BEATS?=./vendor/github.com/elastic/beats
-GOPACKAGES=$(shell govendor list -no-status +local)
-GOBUILD_FLAGS=-i -ldflags "-X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)"
-MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage
+ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7
+ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH}
+GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools)
+GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)"
+MAGE_IMPORT_PATH=github.com/magefile/mage
 CHECK_HEADERS_DISABLED=true
 
 # Path to the libbeat Makefile
 -include $(ES_BEATS)/metricbeat/Makefile
 
 .PHONY: copy-vendor
 copy-vendor:
-	mage vendorUpdate
+	mage vendorUpdate