Merge pull request #4096 from tsg/automatic_merge_from_5.3_to_5.4_branch

Automatic merge from 5.3 to 5.4 branch

filebeat/docs/getting-started.asciidoc

@@ -24,7 +24,7 @@ After installing the Elastic Stack, read the following topics to learn how to in
 [[filebeat-installation]]
 === Step 1: Installing Filebeat
 
-Before running Filebeat, you need to install and configure the Elastic stack. See 
+Before running Filebeat, you need to install and configure the Elastic stack. See
 {libbeat}/getting-started.html[Getting Started with Beats and the Elastic Stack].
 
 To download and install Filebeat, use the commands that work with your system
@@ -248,7 +248,7 @@ sudo ./filebeat -e -c filebeat.yml -d "publish"
 ----------------------------------------------------------------------
 <1> You'll be running Filebeat as root, so you need to change ownership
 of the configuration file (see
-{libbeat}config-file-permissions.html[Config File Ownership and Permissions]
+{libbeat}/config-file-permissions.html[Config File Ownership and Permissions]
 in the _Beats Platform Reference_).
 
 *win:*

filebeat/module/apache2/access/ingest/default.json

@@ -1,10 +1,10 @@
 {
-  "description": "Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins.",
+  "description": "Pipeline for parsing Apache2 access logs. Requires the geoip and user_agent plugins.",
   "processors": [{
     "grok": {
       "field": "message",
       "patterns":[
-        "%{IPORHOST:apache2.access.remote_ip} - %{DATA:apache2.access.user_name} \\[%{HTTPDATE:apache2.access.time}\\] \"%{WORD:apache2.access.method} %{DATA:apache2.access.url} HTTP/%{NUMBER:apache2.access.http_version}\" %{NUMBER:apache2.access.response_code} %{NUMBER:apache2.access.body_sent.bytes}( \"%{DATA:apache2.access.referrer}\")?( \"%{DATA:apache2.access.agent}\")?",
+        "%{IPORHOST:apache2.access.remote_ip} - %{DATA:apache2.access.user_name} \\[%{HTTPDATE:apache2.access.time}\\] \"%{WORD:apache2.access.method} %{DATA:apache2.access.url} HTTP/%{NUMBER:apache2.access.http_version}\" %{NUMBER:apache2.access.response_code} (?:%{NUMBER:apache2.access.body_sent.bytes}|-)( \"%{DATA:apache2.access.referrer}\")?( \"%{DATA:apache2.access.agent}\")?",
         "%{IPORHOST:apache2.access.remote_ip} - %{DATA:apache2.access.user_name} \\[%{HTTPDATE:apache2.access.time}\\] \"-\" %{NUMBER:apache2.access.response_code} -"
         ],
       "ignore_missing": true

heartbeat/docs/getting-started.asciidoc

@@ -7,7 +7,7 @@ related products:
  * Elasticsearch for storage and indexing the data.
  * Kibana for the UI.
  * Logstash (optional) for inserting data into Elasticsearch.
- 
+
 See {libbeat}/getting-started.html[Getting Started with Beats and the Elastic Stack]
 for more information.
 
@@ -28,9 +28,9 @@ install, configure, and run Heartbeat:
 Unlike most Beats, which you install on edge nodes, you typically install
 Heartbeat as part of monitoring service that runs on a separate machine
 and possibly even outside of the network where the services that you want to
-monitor are running. 
+monitor are running.
 
-//TODO: Add a separate topic that explores deployment scenarios in more detail (like installing on a sub-network where there's a firewall etc. 
+//TODO: Add a separate topic that explores deployment scenarios in more detail (like installing on a sub-network where there's a firewall etc.
 
 To download and install Heartbeat, use the commands that work with your
 system (<<deb, deb>> for Debian/Ubuntu, <<rpm, rpm>> for Redhat/Centos/Fedora,
@@ -165,8 +165,8 @@ monitor:
 [source,yaml]
 ----------------------------------------------------------------------
 heartbeat.monitors:
-- type: icmp 
-  schedule: '*/5 * * * * * *' 
+- type: icmp
+  schedule: '*/5 * * * * * *'
   hosts: ["myhost"]
 output.elasticsearch:
   hosts: ["myhost:9200"]
@@ -184,7 +184,7 @@ heartbeat.monitors:
 - type: icmp
   schedule: '*/5 * * * * * *' <1>
   hosts: ["myhost"]
-- type: tcp 
+- type: tcp
   schedule: '@every 5s' <2>
   hosts: ["myhost:12345"]
   mode: any <3>
@@ -258,7 +258,7 @@ sudo ./heartbeat -e -c heartbeat.yml -d "publish"
 ----------------------------------------------------------------------
 <1> You'll be running Heartbeat as root, so you need to change ownership
 of the configuration file (see
-{libbeat}config-file-permissions.html[Config File Ownership and Permissions]
+{libbeat}/config-file-permissions.html[Config File Ownership and Permissions]
 in the _Beats Platform Reference_).
 
 *win:*

libbeat/common/match/cmp.go

@@ -90,7 +90,7 @@ func isPrefixNumDate(r *syntax.Regexp) bool {
 		i++
 	}
 
-	// check digits
+	// check starts with digits `\d{n}` or `[0-9]{n}`
 	if !isMultiDigits(r.Sub[i]) {
 		return false
 	}
@@ -103,6 +103,11 @@ func isPrefixNumDate(r *syntax.Regexp) bool {
 		}
 		i++
 
+		// regex has 'OpLiteral' suffix, without any more digits/patterns following
+		if i == len(r.Sub) {
+			return true
+		}
+
 		// check digits
 		if !isMultiDigits(r.Sub[i]) {
 			return false

libbeat/common/match/compile.go

@@ -88,14 +88,21 @@ func compilePrefixNumDate(r *syntax.Regexp) (stringMatcher, error) {
 	i++
 
 	for i < len(r.Sub) {
-		seps = append(seps, []byte(string(r.Sub[i].Rune)))
+		lit := []byte(string(r.Sub[i].Rune))
 		i++
 
+		// capture literal suffix
+		if i == len(r.Sub) {
+			m.suffix = lit
+			break
+		}
+
+		seps = append(seps, lit)
 		digits = append(digits, digitLen(r.Sub[i]))
 		i++
 	}
 
-	minLen := len(m.prefix)
+	minLen := len(m.prefix) + len(m.suffix)
 	for _, d := range digits {
 		minLen += d
 	}

libbeat/common/match/matcher_bench_test.go

@@ -89,7 +89,9 @@ func BenchmarkPatterns(b *testing.B) {
 		{"startsWith ' '", `^ `},
 		{"startsWithDate", `^\d{2}-\d{2}-\d{4}`},
 		{"startsWithDate2", `^\d{4}-\d{2}-\d{2}`},
-		{"startsWithDate3", `^20\d{2}-\d{2}-\d{2}`},
+		{"startsWithDate3", `^\d\d\d\d-\d\d-\d\d`},
+		{"startsWithDate4", `^20\d{2}-\d{2}-\d{2}`},
+		{"startsWithDateAndSpace", `^\d{4}-\d{2}-\d{2} `},
 		{"startsWithLevel", `^(DEBUG|INFO|WARN|ERR|CRIT)`},
 		{"hasLevel", `(DEBUG|INFO|WARN|ERR|CRIT)`},
 		{"contains 'PATTERN'", `PATTERN`},

libbeat/common/match/matcher_test.go

@@ -120,6 +120,18 @@ func TestMatchers(t *testing.T) {
 				"This should not match",
 			},
 		},
+		{
+			`^\d\d\d\d-\d\d-\d\d`,
+			typeOf((*prefixNumDate)(nil)),
+			[]string{
+				"2017-01-02 should match",
+				"2017-01-03 should also match",
+			},
+			[]string{
+				"- 2017-01-02 should not match",
+				"fail",
+			},
+		},
 		{
 			`^\d{4}-\d{2}-\d{2}`,
 			typeOf((*prefixNumDate)(nil)),
@@ -132,6 +144,30 @@ func TestMatchers(t *testing.T) {
 				"fail",
 			},
 		},
+		{
+			`^(\d{2}){2}-\d{2}-\d{2}`,
+			typeOf((*prefixNumDate)(nil)),
+			[]string{
+				"2017-01-02 should match",
+				"2017-01-03 should also match",
+			},
+			[]string{
+				"- 2017-01-02 should not match",
+				"fail",
+			},
+		},
+		{
+			`^\d{4}-\d{2}-\d{2} - `,
+			typeOf((*prefixNumDate)(nil)),
+			[]string{
+				"2017-01-02 - should match",
+				"2017-01-03 - should also match",
+			},
+			[]string{
+				"- 2017-01-02 should not match",
+				"fail",
+			},
+		},
 		{
 			`^20\d{2}-\d{2}-\d{2}`,
 			typeOf((*prefixNumDate)(nil)),

libbeat/common/match/matchers.go

@@ -36,9 +36,10 @@ type altPrefixMatcher struct {
 
 type prefixNumDate struct {
 	minLen int
-	digits []int
 	prefix []byte
+	digits []int
 	seps   [][]byte
+	suffix []byte
 }
 
 type emptyStringMatcher struct{}
@@ -182,6 +183,12 @@ func (m *prefixNumDate) Match(in []byte) bool {
 		}
 	}
 
+	if sfx := m.suffix; len(sfx) > 0 {
+		if !bytes.HasPrefix(in[pos:], sfx) {
+			return false
+		}
+	}
+
 	return true
 }
 

libbeat/common/match/optimize.go

@@ -11,6 +11,7 @@ var transformations = []trans{
 	trimRight,
 	unconcat,
 	concatRepetition,
+	flattenRepetition,
 }
 
 // optimize runs minimal regular expression optimizations
@@ -112,8 +113,8 @@ func unconcat(r *syntax.Regexp) (bool, *syntax.Regexp) {
 	return false, r
 }
 
-// concatRepetition concatenates multiple repeated sub-patterns into
-// a repetition of exactly N.
+// concatRepetition concatenates 2 consecutive repeated sub-patterns into a
+// repetition of length 2.
 func concatRepetition(r *syntax.Regexp) (bool, *syntax.Regexp) {
 
 	if r.Op != syntax.OpConcat {
@@ -204,3 +205,54 @@ func concatRepetition(r *syntax.Regexp) (bool, *syntax.Regexp) {
 	}
 	return changed, r
 }
+
+// flattenRepetition flattens nested repetitions
+func flattenRepetition(r *syntax.Regexp) (bool, *syntax.Regexp) {
+	if r.Op != syntax.OpConcat {
+		// don't iterate sub-expressions if top-level is no OpConcat
+		return false, r
+	}
+
+	sub := r.Sub
+	inRepetition := false
+	if isConcatRepetition(r) {
+		sub = sub[:1]
+		inRepetition = true
+
+		// create flattened regex repetition mulitplying count
+		// if nexted expression is also a repetition
+		if s := sub[0]; isConcatRepetition(s) {
+			count := len(s.Sub) * len(r.Sub)
+			return true, &syntax.Regexp{
+				Op:    syntax.OpRepeat,
+				Sub:   s.Sub[:1],
+				Min:   count,
+				Max:   count,
+				Flags: r.Flags | s.Flags,
+			}
+		}
+	}
+
+	// recursively check if we can flatten sub-expressions
+	changed := false
+	for i, s := range sub {
+		upd, tmp := flattenRepetition(s)
+		changed = changed || upd
+		sub[i] = tmp
+	}
+
+	if !changed {
+		return false, r
+	}
+
+	// fix up top-level repetition with modified one
+	tmp := *r
+	if inRepetition {
+		for i := range r.Sub {
+			tmp.Sub[i] = sub[0]
+		}
+	} else {
+		tmp.Sub = sub
+	}
+	return changed, &tmp
+}

libbeat/docs/newbeat.asciidoc

@@ -188,6 +188,8 @@ To compile the Beat, make sure you are in the Beat directory (`$GOPATH/src/githu
 make
 ---------
 
+NOTE: we don't support the `-j` option for make at the moment.
+
 Running this command creates the binary called `countbeat` in `$GOPATH/src/github.com/{user}/countbeat`.
 
 Now run the Beat:

libbeat/docs/release.asciidoc

@@ -6,6 +6,7 @@
 --
 This section summarizes the changes in each release.
 
+* <<release-notes-5.3.1>>
 * <<release-notes-5.3.0>>
 * <<release-notes-5.2.2>>
 * <<release-notes-5.2.1>>

libbeat/docs/version.asciidoc

@@ -1,4 +1,4 @@
 :stack-version: 5.4.0
-:doc-branch: 5.x
+:doc-branch: 5.4
 :go-version: 1.7.4
 :release-state: unreleased

metricbeat/docs/developer-guide/creating-beat-from-metricbeat.asciidoc

@@ -8,7 +8,7 @@ own metricsets.
 === Requirements
 
 To create your own Beat, you must have Golang {go-version} or later installed, and the `$GOPATH`
-must be set up correctly. In addition, the following tools are quired:
+must be set up correctly. In addition, the following tools are required:
 
 * https://www.python.org/downloads/[python]
 * https://virtualenv.pypa.io/en/stable/[virtualenv]

metricbeat/docs/gettingstarted.asciidoc

@@ -11,7 +11,7 @@ related products:
  * Kibana for the UI.
  * Logstash (optional) for inserting data into Elasticsearch.
 
-See {libbeat}/getting-started.html[Getting Started with Beats and the Elastic Stack] for more information. 
+See {libbeat}/getting-started.html[Getting Started with Beats and the Elastic Stack] for more information.
 
 After installing the Elastic Stack, read the following topics to learn how to install, configure, and run Metricbeat:
 
@@ -179,7 +179,7 @@ metricbeat.modules:
   enabled: true
   period: 10s
   processes: ['.*']
-  cpu_ticks: false  
+  cpu_ticks: false
 -------------------------------------
 
 The following example shows how to configure two modules: the system module
@@ -198,7 +198,7 @@ metricbeat.modules:
   enabled: true
   period: 10s
   processes: ['.*']
-  cpu_ticks: false  
+  cpu_ticks: false
 - module: apache
   metricsets: ["status"]
   enabled: true
@@ -264,7 +264,7 @@ sudo ./metricbeat -e -c metricbeat.yml -d "publish"
 ----------------------------------------------------------------------
 <1> You'll be running Metricbeat as root, so you need to change ownership
 of the configuration file (see
-{libbeat}config-file-permissions.html[Config File Ownership and Permissions]
+{libbeat}/config-file-permissions.html[Config File Ownership and Permissions]
 in the _Beats Platform Reference_).
 
 *win:*

packetbeat/docs/gettingstarted.asciidoc

@@ -280,8 +280,8 @@ sudo chown root packetbeat.yml <1>
 sudo ./packetbeat -e -c packetbeat.yml -d "publish"
 ----------------------------------------------------------------------
 <1> You'll be running Packetbeat as root, so you need to change ownership
-of the configuration file (see 
-{libbeat}config-file-permissions.html[Config File Ownership and Permissions]
+of the configuration file (see
+{libbeat}/config-file-permissions.html[Config File Ownership and Permissions]
 in the _Beats Platform Reference_).
 
 *win:*