Feature/3484 - set metric name with 'measurement' modifier for grok parser

docs/DATA_FORMATS_INPUT.md

@@ -663,6 +663,28 @@ For more information about the dropwizard json format see
 #### Grok
 Parse logstash-style "grok" patterns. Patterns can be added to patterns, or custom patterns read from custom_pattern_files.
 
+Modifiers can be appended to the end of a grok field to specify how that field should be handled. 
+There are also timestamp modifiers, which can be used to specify the format of time data.
+Available modifiers can be found below.
+
+The 'measurement' modifier has two seperate use cases, one for static measurement names and one for
+dynamic measurement names. 
+
+For setting a static measurement name, apply the 'measurement' modifier to the a single pattern in the 
+'patterns' field.  If grok matches the pattern, the measurement name will be changed to the specified name.
+So the config: `patterns = ["%{TEST:test_name:measurement}"]` would output a metric named "test_name" if grok
+matches the pattern. It is important to only specify one pattern per element in the patterns array field
+or an error will be thrown.
+So the config: `patterns = ["%{TEST:test_name:measurement}|%{TEST2:test2_name:measurement}"]` would need to be changed
+to: `patterns = ["%{TEST:test_name:measurement}","%{TEST2:test2_name:measurement}"]`
+
+For setting a dynamic measurement name, apply the 'measurement' modifier to a value in a custom pattern.
+If the pattern is matched, the measurement name will be set to the value of the field it was applied to.
+Each pattern should only have one 'measurement' modifier applied to it. The modifier should only apply to fields 
+of a single value type, not to another grok pattern.  The name of the field with the measurement modifier applied
+will be ignored, so these formats are the same: `custom_patterns = {"TEST %{WORD:ignored_name:measurement}"}`
+`custom_patterns = {"TEST %{WORD::measurement}"}`
+
 # View logstash grok pattern docs here:
 #   https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html
 # All default logstash patterns are supported, these can be viewed here:
@@ -675,6 +697,7 @@ Parse logstash-style "grok" patterns. Patterns can be added to patterns, or cust
 #   duration (ie, 5.23ms gets converted to int nanoseconds)
 #   tag      (converts the field into a tag)
 #   drop     (drops the field completely)
+#   measurement (sets the metric name to designated field)
 # Timestamp modifiers:
 #   ts-ansic         ("Mon Jan _2 15:04:05 2006")
 #   ts-unix          ("Mon Jan _2 15:04:05 MST 2006")

plugins/inputs/all/all.go

@@ -95,6 +95,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/puppetagent"
 	_ "github.com/influxdata/telegraf/plugins/inputs/rabbitmq"
 	_ "github.com/influxdata/telegraf/plugins/inputs/raindrops"
+	_ "github.com/influxdata/telegraf/plugins/inputs/reader"
 	_ "github.com/influxdata/telegraf/plugins/inputs/redis"
 	_ "github.com/influxdata/telegraf/plugins/inputs/rethinkdb"
 	_ "github.com/influxdata/telegraf/plugins/inputs/riak"

plugins/inputs/logparser/logparser.go

@@ -22,12 +22,12 @@ const (
 
 // LogParser in the primary interface for the plugin
 type GrokConfig struct {
-	MeasurementName    string `toml:"measurement"`
 	Patterns           []string
 	NamedPatterns      []string
 	CustomPatterns     string
 	CustomPatternFiles []string
 	TimeZone           string
+	MeasurementName    string `toml:"measurement"`
 }
 
 type logEntry struct {

plugins/inputs/logparser/logparser_test.go

@@ -44,9 +44,9 @@ func TestGrokParseLogFiles(t *testing.T) {
 
 	logparser := &LogParserPlugin{
 		GrokConfig: GrokConfig{
-			MeasurementName:    "logparser_grok",
 			Patterns:           []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}"},
 			CustomPatternFiles: []string{thisdir + "grok/testdata/test-patterns"},
+			MeasurementName:    "logparser_grok",
 		},
 		FromBeginning: true,
 		Files:         []string{thisdir + "grok/testdata/*.log"},

plugins/inputs/reader/README.md

@@ -0,0 +1,23 @@
+# Reader Input Plugin
+
+The Reader Plugin updates a list of files every interval and parses the data inside.
+Files will always be read from the beginning.  
+This plugin can parse any "data_format" formats.
+
+### Configuration:
+```toml
+[[inputs.reader]]
+## Files to parse each interval.
+## These accept standard unix glob matching rules, but with the addition of
+## ** as a "super asterisk". ie:
+##   /var/log/**.log     -> recursively find all .log files in /var/log
+##   /var/log/*/*.log    -> find all .log files with a parent dir in /var/log
+##   /var/log/apache.log -> only tail the apache log file
+files = ["/var/log/apache/access.log"]
+
+## The dataformat to be read from files
+## Each data format has its own unique set of configuration options, read
+## more about them here:
+## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+data_format = ""
+```

plugins/inputs/reader/dev/docker-compose.yml

@@ -0,0 +1,13 @@
+version: '3'
+
+services:
+  telegraf:
+      image: glinton/scratch
+      volumes:
+        - ./telegraf.conf:/telegraf.conf
+        - ../../../../telegraf:/telegraf
+        - ./json_a.log:/var/log/test.log
+      entrypoint:
+        - /telegraf
+        - --config
+        - /telegraf.conf

plugins/inputs/reader/dev/json_a.log

@@ -0,0 +1,14 @@
+{
+"parent": {
+	"child": 3.0,
+	"ignored_child": "hi"
+},
+"ignored_null": null,
+"integer": 4,
+"list": [3, 4],
+"ignored_parent": {
+	"another_ignored_null": null,
+	"ignored_string": "hello, world!"
+},
+"another_list": [4]
+}

plugins/inputs/reader/dev/telegraf.conf

@@ -0,0 +1,7 @@
+[[inputs.reader]]
+  files = ["/var/log/test.log"]
+  data_format = "json"
+  name_override = "json_reader"
+
+[[outputs.file]]
+  files = ["stdout"]

plugins/inputs/reader/reader.go

@@ -0,0 +1,95 @@
+package reader
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal/globpath"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/plugins/parsers"
+)
+
+type Reader struct {
+	Files         []string `toml:"files"`
+	FromBeginning bool
+	parser        parsers.Parser
+
+	filenames []string
+}
+
+const sampleConfig = `## Files to parse each interval.
+## These accept standard unix glob matching rules, but with the addition of
+## ** as a "super asterisk". ie:
+##   /var/log/**.log     -> recursively find all .log files in /var/log
+##   /var/log/*/*.log    -> find all .log files with a parent dir in /var/log
+##   /var/log/apache.log -> only tail the apache log file
+files = ["/var/log/apache/access.log"]
+
+## The dataformat to be read from files
+## Each data format has its own unique set of configuration options, read
+## more about them here:
+## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+data_format = ""
+`
+
+// SampleConfig returns the default configuration of the Input
+func (r *Reader) SampleConfig() string {
+	return sampleConfig
+}
+
+func (r *Reader) Description() string {
+	return "reload and gather from file[s] on telegraf's interval"
+}
+
+func (r *Reader) Gather(acc telegraf.Accumulator) error {
+	r.refreshFilePaths()
+	for _, k := range r.filenames {
+		metrics, err := r.readMetric(k)
+		if err != nil {
+			return err
+		}
+
+		for _, m := range metrics {
+			acc.AddFields(m.Name(), m.Fields(), m.Tags(), m.Time())
+		}
+	}
+	return nil
+}
+
+func (r *Reader) SetParser(p parsers.Parser) {
+	r.parser = p
+}
+
+func (r *Reader) refreshFilePaths() error {
+	var allFiles []string
+	for _, filepath := range r.Files {
+		g, err := globpath.Compile(filepath)
+		if err != nil {
+			return fmt.Errorf("could not compile glob %v: %v", filepath, err)
+		}
+		files := g.Match()
+
+		for k := range files {
+			allFiles = append(allFiles, k)
+		}
+	}
+
+	r.filenames = allFiles
+	return nil
+}
+
+func (r *Reader) readMetric(filename string) ([]telegraf.Metric, error) {
+	fileContents, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, fmt.Errorf("E! Error file: %v could not be read, %s", filename, err)
+	}
+	return r.parser.Parse(fileContents)
+
+}
+
+func init() {
+	inputs.Add("reader", func() telegraf.Input {
+		return &Reader{}
+	})
+}

plugins/inputs/reader/reader_test.go

@@ -0,0 +1,64 @@
+package reader
+
+import (
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/influxdata/telegraf/plugins/parsers"
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRefreshFilePaths(t *testing.T) {
+	testDir := getPluginDir()
+	r := Reader{
+		Files: []string{testDir + "/reader/testfiles/**.log"},
+	}
+
+	r.refreshFilePaths()
+	assert.Equal(t, len(r.filenames), 2)
+}
+func TestJSONParserCompile(t *testing.T) {
+	testDir := getPluginDir()
+	var acc testutil.Accumulator
+	r := Reader{
+		Files: []string{testDir + "/reader/testfiles/json_a.log"},
+	}
+	parserConfig := parsers.Config{
+		DataFormat: "json",
+		TagKeys:    []string{"parent_ignored_child"},
+	}
+	nParser, err := parsers.NewParser(&parserConfig)
+	r.parser = nParser
+	assert.NoError(t, err)
+
+	r.Gather(&acc)
+	assert.Equal(t, map[string]string{"parent_ignored_child": "hi"}, acc.Metrics[0].Tags)
+	assert.Equal(t, 5, len(acc.Metrics[0].Fields))
+}
+
+func TestGrokParser(t *testing.T) {
+	testDir := getPluginDir()
+	var acc testutil.Accumulator
+	r := Reader{
+		Files: []string{testDir + "/reader/testfiles/grok_a.log"},
+	}
+
+	parserConfig := parsers.Config{
+		DataFormat:   "grok",
+		GrokPatterns: []string{"%{COMMON_LOG_FORMAT}"},
+	}
+
+	nParser, err := parsers.NewParser(&parserConfig)
+	r.parser = nParser
+	assert.NoError(t, err)
+
+	err = r.Gather(&acc)
+	assert.Equal(t, 2, len(acc.Metrics))
+}
+
+func getPluginDir() string {
+	_, filename, _, _ := runtime.Caller(1)
+	return strings.Replace(filename, "/reader/reader_test.go", "", 1)
+}

plugins/inputs/reader/testfiles/grok_a.log

@@ -0,0 +1,2 @@
+127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
+128.0.0.1 user-identifier tony [10/Oct/2000:13:55:36 -0800] "GET /apache_pb.gif HTTP/1.0" 300 45

plugins/inputs/reader/testfiles/json_a.log

@@ -0,0 +1,14 @@
+{
+		"parent": {
+			"child": 3.0,
+			"ignored_child": "hi"
+		},
+		"ignored_null": null,
+		"integer": 4,
+		"list": [3, 4],
+		"ignored_parent": {
+			"another_ignored_null": null,
+			"ignored_string": "hello, world!"
+		},
+		"another_list": [4]
+	}