diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 233432fda3e..dcefdc09d7e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -109,6 +109,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add dashboard for AWS vpcflow fileset. {pull}16007[16007] - Add ECS tls fields to zeek:smtp,rdp,ssl and aws:s3access,elb {issue}15757[15757] {pull}15935[15936] - Add custom string mapping to CEF module to support Forcepoint NGFW {issue}14663[14663] {pull}15910[15910] +- move create-[module,fileset,fields] to mage and enable in x-pack/filebeat {pull}15836[15836] *Heartbeat* diff --git a/filebeat/Makefile b/filebeat/Makefile index 6e37e664325..940077b04cd 100644 --- a/filebeat/Makefile +++ b/filebeat/Makefile @@ -14,15 +14,15 @@ update: mage # Creates a new module. Requires the params MODULE. .PHONY: create-module -create-module: - @go run ${ES_BEATS}/filebeat/scripts/generator/module/main.go --path=$(PWD) --beats_path=$(BEAT_GOPATH)/src/$(BEAT_PATH) --module=$(MODULE) +create-module: mage + mage generate:module # Creates a new fileset. Requires the params MODULE and FILESET. .PHONY: create-fileset -create-fileset: - @go run ${ES_BEATS}/filebeat/scripts/generator/fileset/main.go --path=$(PWD) --beats_path=$(BEAT_GOPATH)/src/$(BEAT_PATH) --module=$(MODULE) --fileset=$(FILESET) +create-fileset: mage + mage generate:fileset # Creates a fields.yml based on a pipeline.json file. Requires the params MODULE and FILESET. .PHONY: create-fields -create-fields: - @go run ${ES_BEATS}/filebeat/scripts/generator/fields/main.go --beats_path=$(BEAT_GOPATH)/src/$(BEAT_PATH) --module=$(MODULE) --fileset=$(FILESET) +create-fields: mage + mage generate:fields diff --git a/filebeat/magefile.go b/filebeat/magefile.go index d4ffea0ee45..e82403d361c 100644 --- a/filebeat/magefile.go +++ b/filebeat/magefile.go @@ -31,6 +31,8 @@ import ( // mage:import "github.com/elastic/beats/dev-tools/mage/target/common" + // mage:import generate + _ "github.com/elastic/beats/filebeat/scripts/mage/generate" ) func init() { diff --git a/filebeat/scripts/generator/fileset/main.go b/filebeat/scripts/generator/fileset/main.go deleted file mode 100644 index 320ddbc55ef..00000000000 --- a/filebeat/scripts/generator/fileset/main.go +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package main - -import ( - "flag" - "fmt" - "os" - - "github.com/elastic/beats/filebeat/generator/fileset" -) - -func main() { - module := flag.String("module", "", "Name of the module") - filesetName := flag.String("fileset", "", "Name of the fileset") - modulesPath := flag.String("path", ".", "Path to the generated fileset") - beatsPath := flag.String("beats_path", ".", "Path to elastic/beats") - flag.Parse() - - if *module == "" { - fmt.Println("Missing parameter: module") - os.Exit(1) - } - - if *filesetName == "" { - fmt.Println("Missing parameter: fileset") - os.Exit(1) - } - - err := fileset.Generate(*module, *filesetName, *modulesPath, *beatsPath) - if err != nil { - fmt.Printf("Cannot generate fileset: %v\n", err) - os.Exit(3) - } - - fmt.Println("New fileset was generated, please check that module.yml file have proper fileset dashboard settings. After setting up Grok pattern in pipeline.json, please generate fields.yml") -} diff --git a/filebeat/scripts/generator/module/main.go b/filebeat/scripts/mage/generate/fields.go similarity index 54% rename from filebeat/scripts/generator/module/main.go rename to filebeat/scripts/mage/generate/fields.go index 737650d918d..9bc8a8af40a 100644 --- a/filebeat/scripts/generator/module/main.go +++ b/filebeat/scripts/mage/generate/fields.go @@ -15,32 +15,28 @@ // specific language governing permissions and limitations // under the License. -package main +package generate import ( - "flag" "fmt" "os" - "github.com/elastic/beats/filebeat/generator/module" + devtools "github.com/elastic/beats/dev-tools/mage" + genfields "github.com/elastic/beats/filebeat/generator/fields" ) -func main() { - moduleName := flag.String("module", "", "Name of the module") - modulePath := flag.String("path", ".", "Path to the generated fileset") - beatsPath := flag.String("beats_path", ".", "Path to elastic/beats") - flag.Parse() +// Fields creates a new fields.yml for an existing Filebeat fileset. +// Use MODULE=module to specify the name of the existing module +// Use FILESET=fileset to specify the name of the existing fileset +func Fields() error { + targetModule := os.Getenv("MODULE") + targetFileset := os.Getenv("FILESET") - if *moduleName == "" { - fmt.Println("Missing parameter: module") - os.Exit(1) + if targetModule == "" || targetFileset == "" { + return fmt.Errorf("you must specify the module and fileset: MODULE=module FILESET=fileset mage generate:fields") } - err := module.Generate(*moduleName, *modulePath, *beatsPath) - if err != nil { - fmt.Printf("Cannot generate module: %v\n", err) - os.Exit(2) - } + curDir := devtools.CWD() - fmt.Println("New module was generated, now you can start creating filesets by create-fileset command.") + return genfields.Generate(curDir, targetModule, targetFileset, false) } diff --git a/filebeat/scripts/mage/generate/fileset.go b/filebeat/scripts/mage/generate/fileset.go new file mode 100644 index 00000000000..612567a1104 --- /dev/null +++ b/filebeat/scripts/mage/generate/fileset.go @@ -0,0 +1,50 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package generate + +import ( + "fmt" + "os" + + devtools "github.com/elastic/beats/dev-tools/mage" + genfileset "github.com/elastic/beats/filebeat/generator/fileset" +) + +// Fileset creates a new fileset for an existing Filebeat module. +// Use MODULE=module to specify the name of the existing module +// Use FILESET=fileset to specify the name of the new fileset +func Fileset() error { + targetModule := os.Getenv("MODULE") + targetFileset := os.Getenv("FILESET") + + if targetModule == "" || targetFileset == "" { + return fmt.Errorf("you must specify the module and fileset: MODULE=module FILESET=fileset createFileset") + } + + ossDir := devtools.OSSBeatDir() + xPackDir := devtools.XPackBeatDir() + + switch devtools.CWD() { + case ossDir: + return genfileset.Generate(targetModule, targetFileset, ossDir, ossDir) + case xPackDir: + return genfileset.Generate(targetModule, targetFileset, xPackDir, ossDir) + default: + return fmt.Errorf("you must be in a filebeat directory") + } +} diff --git a/filebeat/scripts/generator/fields/main.go b/filebeat/scripts/mage/generate/module.go similarity index 52% rename from filebeat/scripts/generator/fields/main.go rename to filebeat/scripts/mage/generate/module.go index 523ebf82249..5d14e3ec09f 100644 --- a/filebeat/scripts/generator/fields/main.go +++ b/filebeat/scripts/mage/generate/module.go @@ -15,38 +15,33 @@ // specific language governing permissions and limitations // under the License. -package main +package generate import ( - "flag" "fmt" "os" - "github.com/elastic/beats/filebeat/generator/fields" + devtools "github.com/elastic/beats/dev-tools/mage" + genmod "github.com/elastic/beats/filebeat/generator/module" ) -func main() { - module := flag.String("module", "", "Name of the module") - fileset := flag.String("fileset", "", "Name of the fileset") - beatsPath := flag.String("beats_path", ".", "Path to elastic/beats") - noDoc := flag.Bool("nodoc", false, "Generate documentation for fields") - flag.Parse() - - if *module == "" { - fmt.Println("Missing parameter: module") - os.Exit(1) +// Module creates a new Filebeat module. +// Use MODULE=module to specify the name of the new module +func Module() error { + targetModule := os.Getenv("MODULE") + if targetModule == "" { + return fmt.Errorf("you must specify the module: MODULE=name mage generate:module") } - if *fileset == "" { - fmt.Println("Missing parameter: fileset") - os.Exit(1) - } + ossDir := devtools.OSSBeatDir() + xPackDir := devtools.XPackBeatDir() - err := fields.Generate(*beatsPath, *module, *fileset, *noDoc) - if err != nil { - fmt.Printf("Error while generating fields.yml: %v\n", err) - os.Exit(2) + switch devtools.CWD() { + case ossDir: + return genmod.Generate(targetModule, ossDir, ossDir) + case xPackDir: + return genmod.Generate(targetModule, xPackDir, ossDir) + default: + return fmt.Errorf("you must be in a filebeat directory") } - - fmt.Printf("Fields.yml generated for %s/%s\n", *module, *fileset) } diff --git a/x-pack/filebeat/magefile.go b/x-pack/filebeat/magefile.go index d40cb140796..a8ffd9d6875 100644 --- a/x-pack/filebeat/magefile.go +++ b/x-pack/filebeat/magefile.go @@ -18,6 +18,8 @@ import ( // mage:import "github.com/elastic/beats/dev-tools/mage/target/common" + // mage:import generate + _ "github.com/elastic/beats/filebeat/scripts/mage/generate" ) func init() {