Converting generator.py to mage (#13610)

Converting generator.py to mage

CHANGELOG-developer.next.asciidoc

@@ -24,6 +24,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - For "metricbeat style" generated custom beats, the mage target `GoTestIntegration` has changed to `GoIntegTest` and `GoTestUnit` has changed to `GoUnitTest`. {pull}13341[13341]
 - Build docker and kubernetes features only on supported platforms. {pull}13509[13509]
 - Need to register new processors to be used in the JS processor in their `init` functions. {pull}13509[13509]
+- The custom beat generator now uses mage instead of python, `mage GenerateCustomBeat` can be used to create a new beat, and `mage vendorUpdate` to update the vendored libbeat in a custom beat. {pull}13610[13610]
 
 ==== Bugfixes
 

dev-tools/mage/settings.go

@@ -235,6 +235,14 @@ var (
 	elasticBeatsDirLock  sync.Mutex
 )
 
+// SetElasticBeatsDir sets the internal elastic beats dir to a preassigned value
+func SetElasticBeatsDir(path string) {
+	elasticBeatsDirLock.Lock()
+	defer elasticBeatsDirLock.Unlock()
+
+	elasticBeatsDirValue = path
+}
+
 // ElasticBeatsDir returns the path to Elastic beats dir.
 func ElasticBeatsDir() (string, error) {
 	elasticBeatsDirLock.Lock()

docs/devguide/creating-beat-from-metricbeat.asciidoc

@@ -12,6 +12,7 @@ must be set up correctly. In addition, the following tools are required:
 
 * https://www.python.org/downloads/[python]
 * https://virtualenv.pypa.io/en/stable/[virtualenv]
+* https://github.com/magefile/mage[mage]
 
 Virtualenv is easiest installed with your package manager or https://pip.pypa.io/en/stable/[pip].
 
@@ -43,34 +44,20 @@ This directory is normally located under `$GOPATH/src/github.com/{your-github-na
 [float]
 ==== Step 2 - Create the Beat
 
-Run the command:
+Run the command from the root `beats` directory:
 
 [source,bash]
 ----
-python ${GOPATH}/src/github.com/elastic/beats/script/generate.py --type=metricbeat
+NEWBEAT_TYPE=metricbeat mage GenerateCustomBeat
 ----
 
-When prompted, enter the Beat name and path.
+When prompted, enter the Beat name and path, along with an initial module and metricset name for your beat. For more details about creating a metricset, see the docs <<creating-metricsets>>.
 
-
-[float]
-==== Step 3 - Init and create the metricset
-
-After creating the Beat, change the directory to `$GOPATH/src/github.com/{your-github-name}/{beat}` and run:
-
-[source,bash]
-----
-make setup
-----
-
-This will do the initial setup for your Beat and also run `make create-metricset`, which will ask you for the
-module name and metricset name of your Beat.
-
-For more details about creating a metricset, see the docs <<creating-metricsets>>.
+After creating the Beat, change the directory to `$GOPATH/src/github.com/{your-github-name}/{beat}`
 
 
 [float]
-==== Step 4 - Build & Run
+==== Step 3 - Build & Run
 
 To create a binary run the `make` command. This will create the binary in your beats directory.
 
@@ -84,7 +71,7 @@ To run it, execute the binary. This will automatically load the default configur
 This will run the beat with debug output enabled to the console to directly see what is happening. Stop the beat with `CTRL-C`.
 
 [float]
-==== Step 5 - Package
+==== Step 4 - Package
 
 To create packages and binaries for different platforms, https://www.docker.com/[docker] is required.
 The first step is to get the most recent packaging tools into your beat:

docs/devguide/newbeat.asciidoc

@@ -122,24 +122,23 @@ mkdir ${GOPATH}/src/github.com/{user}
 cd ${GOPATH}/src/github.com/{user}
 --------------------
 
-Run python and specify the path to the Beat generator:
+Run the mage script to generate the custom beat:
 
-NOTE: Python 2 is required (Python 3 will not work).
 
 [source,shell]
 --------------------
-python $GOPATH/src/github.com/elastic/beats/script/generate.py
+mage GenerateCustomBeat
 --------------------
 
-Python will prompt you to enter information about your Beat. For the `project_name`, enter `Countbeat`.
+The mage script will prompt you to enter information about your Beat. For the `project_name`, enter `Countbeat`.
 For the `github_name`, enter your github id. The `beat` and `beat_path` are set to the correct values automatically (just press Enter to accept each default). For the `full_name`, enter your firstname and lastname.
 
 [source,shell]
 ---------
-Beat Name [Examplebeat]: Countbeat
-Your Github Name [your-github-name]: {username}
-Beat Path [github.com/{github id}/{beat name}]:
-Firstname Lastname: {Full Name}
+Enter a project name [examplebeat]:  Countbeat
+Enter a github name [your-github-name]: {username}
+Enter a beat path [github.com/{username}/countbeat]: 
+Enter a full name [Firstname Lastname]: {Full Name}
 ---------
 
 The Beat generator creates a directory called `countbeat` inside of your project folder (e.g. {project folder}/github.com/{github id}/countbeat).

generator/beat/{beat}/Makefile

@@ -14,32 +14,6 @@ CHECK_HEADERS_DISABLED=true
 # Path to the libbeat Makefile
 -include $(LIBBEAT_MAKEFILE)
 
-# Initial beat setup
-.PHONY: setup
-setup: pre-setup git-add
-
-pre-setup: copy-vendor git-init
-	$(MAKE) -f $(LIBBEAT_MAKEFILE) mage ES_BEATS=$(ES_BEATS)
-	$(MAKE) -f $(LIBBEAT_MAKEFILE) update BEAT_NAME=$(BEAT_NAME) ES_BEATS=$(ES_BEATS) NO_COLLECT=$(NO_COLLECT)
-
-# Copy beats into vendor directory
 .PHONY: copy-vendor
-copy-vendor: vendor-check
-	mkdir -p vendor/github.com/elastic/beats
-	git archive --remote ${BEAT_GOPATH}/src/github.com/elastic/beats HEAD | tar -x --exclude=x-pack -C vendor/github.com/elastic/beats
-	mkdir -p vendor/github.com/magefile
-	cp -R vendor/github.com/elastic/beats/vendor/github.com/magefile/mage vendor/github.com/magefile
-
-.PHONY: git-init
-git-init:
-	git init
-
-.PHONY: git-add
-git-add:
-	git add -A
-	git commit -q -m "Add generated {beat} files"
-
-
-.PHONY: vendor-check
-vendor-check:
-	@if output=$$(git -C ${BEAT_GOPATH}/src/github.com/elastic/beats status --porcelain) && [ ! -z "$${output}" ]; then printf "\033[31mWARNING: elastic/beats has uncommitted changes, these will not be in the vendor directory!\033[0m\n"; fi
+copy-vendor:
+	mage vendorUpdate

generator/beat/{beat}/beater/{beat}.go

@@ -11,8 +11,8 @@ import (
 	"{beat_path}/config"
 )
 
-// {Beat} configuration.
-type {Beat} struct {
+// {beat} configuration.
+type {beat} struct {
 	done   chan struct{}
 	config config.Config
 	client beat.Client
@@ -25,15 +25,15 @@ func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) {
 		return nil, fmt.Errorf("Error reading config file: %v", err)
 	}
 
-	bt := &{Beat}{
+	bt := &{beat}{
 		done:   make(chan struct{}),
 		config: c,
 	}
 	return bt, nil
 }
 
 // Run starts {beat}.
-func (bt *{Beat}) Run(b *beat.Beat) error {
+func (bt *{beat}) Run(b *beat.Beat) error {
 	logp.Info("{beat} is running! Hit CTRL-C to stop it.")
 
 	var err error
@@ -65,7 +65,7 @@ func (bt *{Beat}) Run(b *beat.Beat) error {
 }
 
 // Stop stops {beat}.
-func (bt *{Beat}) Stop() {
+func (bt *{beat}) Stop() {
 	bt.client.Close()
 	close(bt.done)
 }

generator/beat/{beat}/magefile.go

@@ -7,21 +7,23 @@ import (
 	"time"
 
 	"github.com/magefile/mage/mg"
+	"github.com/magefile/mage/sh"
 
 	devtools "github.com/elastic/beats/dev-tools/mage"
+	"github.com/elastic/beats/generator/common/beatgen"
 
 	// mage:import
-	_ "github.com/elastic/beats/dev-tools/mage/target/common"
+	"github.com/elastic/beats/dev-tools/mage/target/pkg"
 	// mage:import
 	"github.com/elastic/beats/dev-tools/mage/target/build"
 	// mage:import
-	"github.com/elastic/beats/dev-tools/mage/target/update"
+	_ "github.com/elastic/beats/dev-tools/mage/target/common"
 	// mage:import
 	_ "github.com/elastic/beats/dev-tools/mage/target/test"
 	// mage:import
 	_ "github.com/elastic/beats/dev-tools/mage/target/unittest"
 	// mage:import
-	"github.com/elastic/beats/dev-tools/mage/target/pkg"
+	_ "github.com/elastic/beats/dev-tools/mage/target/integtest"
 )
 
 func init() {
@@ -31,6 +33,11 @@ func init() {
 	devtools.BeatVendor = "{full_name}"
 }
 
+// VendorUpdate updates elastic/beats in the vendor dir
+func VendorUpdate() error {
+	return beatgen.VendorUpdate()
+}
+
 // Package packages the Beat for distribution.
 // Use SNAPSHOT=true to build snapshots.
 // Use PLATFORMS to control the target platforms.
@@ -40,17 +47,22 @@ func Package() {
 
 	devtools.UseCommunityBeatPackaging()
 
-	mg.Deps(update.Update)
+	mg.Deps(Update)
 	mg.Deps(build.CrossBuild, build.CrossBuildGoDaemon)
 	mg.SerialDeps(devtools.Package, pkg.PackageTest)
 }
 
-// Config generates both the short/reference/docker configs.
-func Config() error {
-	return devtools.Config(devtools.AllConfigTypes, devtools.ConfigFileParams{}, ".")
+// Update updates the generated files (aka make update).
+func Update() error {
+	return sh.Run("make", "update")
 }
 
-//Fields generates a fields.yml for the Beat.
+// Fields generates a fields.yml for the Beat.
 func Fields() error {
 	return devtools.GenerateFieldsYAML()
 }
+
+// Config generates both the short/reference/docker configs.
+func Config() error {
+	return devtools.Config(devtools.AllConfigTypes, devtools.ConfigFileParams{}, ".")
+}

generator/common/Makefile

@@ -1,45 +1,36 @@
-BUILD_DIR?=build
 PWD=$(shell pwd)
-PYTHON_ENV?=${BUILD_DIR}/python-env
 BEAT_TYPE?=beat
-BEAT_PATH=${BUILD_DIR}/src/beatpath/testbeat
+BEAT_NAME?=beatpath/test${BEAT_TYPE}
+BEAT_PATH=${GOPATH}/src/${BEAT_NAME}
 ES_BEATS=${GOPATH}/src/github.com/elastic/beats
 PREPARE_COMMAND?=
 
 # Runs test build for mock beat
 .PHONY: test
 test: prepare-test
-	. ${PYTHON_ENV}/bin/activate; \
-	export GOPATH=${PWD}/build ; \
-	export PATH=$${GOPATH}/bin:${PATH}; \
 	cd ${BEAT_PATH} ; \
-	$(MAKE) copy-vendor || exit 1  ; \
-	${PREPARE_COMMAND} \
-	$(MAKE) git-init || exit 1 ; \
-	$(MAKE) update fmt || exit 1 ; \
+	export PATH=$${GOPATH}/bin:$${PATH}; \
 	git config user.email "[email protected]" || exit 1 ; \
 	git config user.name "beats-jenkins" || exit 1 ; \
-	$(MAKE) git-add || exit 1 ; \
 	$(MAKE) check CHECK_HEADERS_DISABLED=y || exit 1 ; \
 	$(MAKE) || exit 1 ; \
 	$(MAKE) unit
 
 .PHONY: prepare-test
-prepare-test:: python-env
-	# Makes sure to use current version of beats for testing
-	rm -fr ${BUILD_DIR}/src/github.com/elastic/beats/
-	git clone -s ${PWD}/../../ ${BUILD_DIR}/src/github.com/elastic/beats/
-
+prepare-test:: ${GOPATH}/bin/mage
 	rm -fr ${BEAT_PATH}
 	mkdir -p ${BEAT_PATH}
-	export GOPATH=${PWD}/build ; \
-	. ${PYTHON_ENV}/bin/activate && \
-	  python ${PWD}/build/src/github.com/elastic/beats/script/generate.py \
-	    --type=${BEAT_TYPE} \
-	    --project_name=Testbeat \
-	    --github_name=ruflin \
-	    --beat_path=beatpath/testbeat \
-	    --full_name="Nicolas Ruflin"
+	cd ${GOPATH}/src/github.com/elastic/beats/ ; \
+	export MODULE=elastic ; \
+	export METRICSET=test ; \
+	export GO111MODULE=off; \
+	export NEWBEAT_PROJECT_NAME=Testbeat ; \
+	export NEWBEAT_GITHUB_NAME=ruflin ; \
+	export NEWBEAT_BEAT_PATH=${BEAT_NAME} ; \
+	export NEWBEAT_FULL_NAME="Nicolas Ruflin" ; \
+	export NEWBEAT_TYPE=${BEAT_TYPE} ; \
+	export NEWBEAT_DEV=1 ; \
+	mage GenerateCustomBeat
 
 # Runs test build for the created beat
 .PHONY: test-build
@@ -51,15 +42,12 @@ test-build: test
 	$(MAKE) deps ; \
 	$(MAKE) images
 
-# Sets up the virtual python environment
-.PHONY: python-env
-python-env:
-	@test -d ${PYTHON_ENV} || virtualenv ${PYTHON_ENV}
-	@${PYTHON_ENV}/bin/pip install --upgrade pip PyYAML
-	@# Work around pip bug. See: https://github.com/pypa/pip/issues/4464
-	@find $(PYTHON_ENV) -type d -name dist-packages -exec sh -c "echo dist-packages > {}.pth" ';'
+${GOPATH}/bin/mage:
+	go get -u -d github.com/magefile/mage
+	cd $${GOPATH}/src/github.com/magefile/mage;  \
+	go run bootstrap.go
 
 # Cleans up environment
 .PHONY: clean
 clean:
-	@rm -rf build
+	rm -rf ${BEAT_PATH}