Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cherry-pick #7713 to 6.x: Simplify fields.yml generator #7778

Merged
merged 1 commit into from
Jul 30, 2018
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion auditbeat/Makefile
Original file line number Diff line number Diff line change
@@ -12,7 +12,7 @@ include ${ES_BEATS}/libbeat/scripts/Makefile

# Collects all dependencies and then calls update
.PHONY: collect
collect: fields collect-docs configs kibana
collect: collect-docs configs kibana

# Collects all module configs
.PHONY: configs
2 changes: 1 addition & 1 deletion auditbeat/include/fields.go

Large diffs are not rendered by default.

58 changes: 42 additions & 16 deletions dev-tools/cmd/asset/asset.go
Original file line number Diff line number Diff line change
@@ -20,6 +20,7 @@
package main

import (
"bufio"
"bytes"
"flag"
"fmt"
@@ -30,33 +31,54 @@ import (
"github.com/elastic/beats/libbeat/asset"
)

var pkg *string
var (
pkg string
input string
output string
)

func init() {
pkg = flag.String("pkg", "", "Package name")
flag.StringVar(&pkg, "pkg", "", "Package name")
flag.StringVar(&input, "in", "-", "Source of input. \"-\" means reading from stdin")
flag.StringVar(&output, "out", "-", "Output path. \"-\" means writing to stdout")
}

func main() {
flag.Parse()

args := flag.Args()
if len(args) != 2 {
fmt.Fprintln(os.Stderr, "File path must be set")
os.Exit(1)
}

file := args[0]
beatName := args[1]
var (
file, beatName string
data []byte
err error
)
if input == "-" {
if len(args) != 2 {
fmt.Fprintln(os.Stderr, "File path must be set")
os.Exit(1)
}
file = args[0]
beatName = args[1]

data, err := ioutil.ReadFile(file)
if err != nil {
fmt.Fprintln(os.Stderr, "Invalid file path: %s", args[0])
os.Exit(1)
r := bufio.NewReader(os.Stdin)
data, err = ioutil.ReadAll(r)
if err != nil {
fmt.Fprintf(os.Stderr, "Error while reading from stdin: %v\n", err)
os.Exit(1)
}
} else {
file = input
beatName = args[0]
data, err = ioutil.ReadFile(input)
if err != nil {
fmt.Fprintf(os.Stderr, "Invalid file path: %s\n", input)
os.Exit(1)
}
}

encData, err := asset.EncodeData(string(data))
if err != nil {
fmt.Fprintln(os.Stderr, "Error encoding the data: %s", err)
fmt.Fprintf(os.Stderr, "Error encoding the data: %s\n", err)
os.Exit(1)
}

@@ -65,13 +87,17 @@ func main() {
Beat: beatName,
Name: file,
Data: encData,
Package: *pkg,
Package: pkg,
})

bs, err := format.Source(buf.Bytes())
if err != nil {
panic(err)
}

os.Stdout.Write(bs)
if output == "-" {
os.Stdout.Write(bs)
} else {
ioutil.WriteFile(output, bs, 0640)
}
}
5 changes: 0 additions & 5 deletions dev-tools/jenkins_ci.ps1
Original file line number Diff line number Diff line change
@@ -24,11 +24,6 @@ $env:MAGEFILE_CACHE = "$env:WORKSPACE\.magefile"
exec { go install github.com/elastic/beats/vendor/github.com/magefile/mage }
exec { go get -u github.com/jstemmer/go-junit-report }

echo "Building libbeat fields.yml"
cd libbeat
exec { mage fields }
cd ..

if (Test-Path "$env:beat") {
cd "$env:beat"
} else {
1 change: 1 addition & 0 deletions dev-tools/mage/fields.go
Original file line number Diff line number Diff line change
@@ -42,6 +42,7 @@ func GenerateFieldsYAML(fieldsFiles ...string) error {
filepath.Join(beatsDir, globalFieldsCmdPath),
"-es_beats_path", beatsDir,
"-beat_path", CWD(),
"-out", "fields.yml",
)

return globalFieldsCmd(fieldsFiles...)
2 changes: 1 addition & 1 deletion filebeat/Makefile
Original file line number Diff line number Diff line change
@@ -42,7 +42,7 @@ imports: python-env

# Runs all collection steps and updates afterwards
.PHONY: collect
collect: fields kibana configs collect-docs imports
collect: kibana configs collect-docs imports

# Creates a new module. Requires the params MODULE
.PHONY: create-module
2 changes: 1 addition & 1 deletion filebeat/include/fields.go

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion generator/beat/{beat}/Makefile
Original file line number Diff line number Diff line change
@@ -35,4 +35,4 @@ git-add:

# Collects all dependencies and then calls update
.PHONY: collect
collect: fields
collect:
3 changes: 1 addition & 2 deletions generator/metricbeat/Makefile
Original file line number Diff line number Diff line change
@@ -10,5 +10,4 @@ prepare-test:: python-env

# Collects all dependencies and then calls update
.PHONY: collect
collect: fields

collect:
2 changes: 1 addition & 1 deletion heartbeat/Makefile
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@ TEST_ENVIRONMENT=false

# Collects all dependencies and then calls update
.PHONY: collect
collect: fields imports kibana
collect: imports kibana

# Generate imports for all monitors
.PHONY: imports
2 changes: 1 addition & 1 deletion heartbeat/include/fields.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion libbeat/Makefile
Original file line number Diff line number Diff line change
@@ -6,4 +6,4 @@ include scripts/Makefile

# Collects all dependencies and then calls update
.PHONY: collect
collect: libbeat_fields
collect:
142 changes: 49 additions & 93 deletions libbeat/generator/fields/fields.go
Original file line number Diff line number Diff line change
@@ -20,17 +20,9 @@ package fields
import (
"bufio"
"bytes"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"

"github.com/pkg/errors"
)

var (
generatedFieldsYml = filepath.Join("_meta", "fields.generated.yml")
)

// YmlFile holds the info on files and how to write them into the global fields.yml
@@ -39,37 +31,67 @@ type YmlFile struct {
Indent int
}

func collectBeatFiles(beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) {
commonFields := filepath.Join(beatPath, "_meta", "fields.common.yml")
_, err := os.Stat(commonFields)
if os.IsNotExist(err) {
return fieldFiles, nil
} else if err != nil {
return nil, err
func collectCommonFiles(esBeatsPath, beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) {
commonFields := []string{
filepath.Join(beatPath, "_meta/fields.common.yml"),
}

var libbeatFieldFiles []*YmlFile
var err error
if !isLibbeat(beatPath) {
commonFields = append(commonFields,
filepath.Join(esBeatsPath, "libbeat/_meta/fields.common.yml"),
)

libbeatModulesPath := filepath.Join(esBeatsPath, "libbeat/processors")
libbeatFieldFiles, err = CollectModuleFiles(libbeatModulesPath)
if err != nil {
return nil, err
}
}

files := []*YmlFile{
{
Path: commonFields,
var files []*YmlFile
for _, cf := range commonFields {
_, err := os.Stat(cf)
if os.IsNotExist(err) {
continue
} else if err != nil {
return nil, err
}
files = append(files, &YmlFile{
Path: cf,
Indent: 0,
},
})
}

files = append(files, libbeatFieldFiles...)

return append(files, fieldFiles...), nil
}

func writeGeneratedFieldsYml(beatsPath string, fieldFiles []*YmlFile) error {
outPath := path.Join(beatsPath, generatedFieldsYml)
f, err := os.Create(outPath)
func isLibbeat(beatPath string) bool {
return filepath.Base(beatPath) == "libbeat"
}

func writeGeneratedFieldsYml(beatPath string, fieldFiles []*YmlFile, output string) error {
data, err := GenerateFieldsYml(fieldFiles)
if err != nil {
return err
}
defer f.Close()

data, err := GenerateFieldsYml(fieldFiles)
if output == "-" {
fw := bufio.NewWriter(os.Stdout)
fw.Write(data)
return fw.Flush()
}

outPath := filepath.Join(beatPath, output)
f, err := os.Create(outPath)
if err != nil {
return err
}
defer f.Close()

fw := bufio.NewWriter(f)
fw.Write(data)
return fw.Flush()
@@ -106,77 +128,11 @@ func writeIndentedLine(buf *bytes.Buffer, line string, indent int) error {
}

// Generate collects fields.yml files and concatenates them into one global file.
func Generate(esBeatsPath, beatPath string, files []*YmlFile) error {
files, err := collectBeatFiles(beatPath, files)
if err != nil {
return err
}

err = writeGeneratedFieldsYml(beatPath, files)
if err != nil {
return err
}

return AppendFromLibbeat(esBeatsPath, beatPath)
}

// AppendFromLibbeat appends fields.yml of libbeat to the fields.yml
func AppendFromLibbeat(esBeatsPath, beatPath string) error {
fieldsMetaPath := path.Join(beatPath, "_meta", "fields.yml")
generatedPath := path.Join(beatPath, generatedFieldsYml)

err := createIfNotExists(fieldsMetaPath, generatedPath)
if err != nil {
return err
}

if isLibbeat(beatPath) {
out := filepath.Join(esBeatsPath, "libbeat", "fields.yml")
return copyFileWithFlag(generatedPath, out, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
}

libbeatPath := filepath.Join(esBeatsPath, "libbeat", generatedFieldsYml)
out := filepath.Join(beatPath, "fields.yml")
err = copyFileWithFlag(libbeatPath, out, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
func Generate(esBeatsPath, beatPath string, files []*YmlFile, output string) error {
files, err := collectCommonFiles(esBeatsPath, beatPath, files)
if err != nil {
return err
}
return copyFileWithFlag(generatedPath, out, os.O_WRONLY|os.O_APPEND)
}

func isLibbeat(beatPath string) bool {
return filepath.Base(beatPath) == "libbeat"
}

func createIfNotExists(inPath, outPath string) error {
_, err := os.Stat(outPath)
if os.IsNotExist(err) {
err := copyFileWithFlag(inPath, outPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
if err != nil {
return err
}
return nil
}
return err
}

func copyFileWithFlag(in, out string, flag int) error {
input, err := ioutil.ReadFile(in)
if err != nil {
return errors.Wrap(err, "failed to read source in copy")
}

if err := os.MkdirAll(filepath.Dir(out), 0755); err != nil {
return errors.Wrapf(err, "failed to create destination dir for copy "+
"at %v", filepath.Dir(out))
}

output, err := os.OpenFile(out, flag, 0644)
if err != nil {
return errors.Wrap(err, "failed to open destination file for copy")
}
defer output.Close()

_, err = output.Write(input)
return err
return writeGeneratedFieldsYml(beatPath, files, output)
}
4 changes: 2 additions & 2 deletions libbeat/generator/fields/module_fields_collector.go
Original file line number Diff line number Diff line change
@@ -71,7 +71,7 @@ func CollectModuleFiles(modulesDir string) ([]*YmlFile, error) {
func CollectFiles(module string, modulesPath string) ([]*YmlFile, error) {

var files []*YmlFile
fieldsYmlPath := filepath.Join(modulesPath, module, "_meta", "fields.yml")
fieldsYmlPath := filepath.Join(modulesPath, module, "_meta/fields.yml")
if _, err := os.Stat(fieldsYmlPath); !os.IsNotExist(err) {
files = append(files, &YmlFile{
Path: fieldsYmlPath,
@@ -91,7 +91,7 @@ func CollectFiles(module string, modulesPath string) ([]*YmlFile, error) {
if !s.IsDir() {
continue
}
fieldsYmlPath = filepath.Join(modulesPath, module, s.Name(), "_meta", "fields.yml")
fieldsYmlPath = filepath.Join(modulesPath, module, s.Name(), "_meta/fields.yml")
if _, err = os.Stat(fieldsYmlPath); !os.IsNotExist(err) {
files = append(files, &YmlFile{
Path: fieldsYmlPath,
2 changes: 1 addition & 1 deletion libbeat/kibana/fields_transformer.go
Original file line number Diff line number Diff line change
@@ -80,7 +80,7 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str
}

if t.keys[f.Path] != nil {
msg := fmt.Sprintf("ERROR: Field <%s> is duplicated. Please update and try again.", f.Path)
msg := fmt.Sprintf("ERROR: Field <%s> is duplicated. Please update and try again.\n", f.Path)
panic(errors.New(msg))
}

Loading