Journalbeat matches support && minor additions (elastic#8324)

### Matching support
From now on it's possible to match for journal entry fields in Journalbeat using the new option `matches`. This requires a list of key value pairs separated by "=". The key has to be a journalbeat event key (e.g systemd.unit) and the value is the exact value journal reader needs to find in the entries.

Example configuration which returns NGINX and dhclient entries from the journal:
```yml
include_matches:
  - "systemd.unit=nginx"
  - "process.name=dhclient"
```
### Docker fields
Added docker fields from: https://docs.docker.com/config/containers/logging/journald/
- `container.id`
- `container.id_truncated`
- `container.name`
- `container.image.tag`
- `container.partial`

### Parse timestamp of entries
Journalbeat parses the timestamp of the entry and adds it to the event as `@timestamp`. The time of reading by Journalbeat is saved in `read_timestamp`.

### Save custom fields
Custom fields by various sources are stored under `custom`. Field names are normalized, meaning `"_"` prefix is removed and every letter is lowercase.

### Fields && processors
From now on it is possible to configure `processors` and `fields`, etc on `input` level.

### Metrics

The size of each open reader is reporting in bytes:
```
{
  "journalbeat": {
    "journals": {
      "journal_1": {
         "path": "system.journal",
         "size_in_bytes": 123124214,
       }
    }
}
```

journalbeat/Dockerfile

@@ -0,0 +1,17 @@
+FROM golang:1.10.3
+MAINTAINER Noémi Ványi <[email protected]>
+
+RUN set -x && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends \
+      python-pip virtualenv libsystemd-dev libc6-dev-i386 gcc-arm-linux-gnueabi && \
+    apt-get clean
+
+RUN pip install --upgrade setuptools
+
+# Setup work environment
+ENV JOURNALBEAT_PATH /go/src/github.com/elastic/beats/journalbeat
+
+RUN mkdir -p $JOURNALBEAT_PATH/build/coverage
+WORKDIR $JOURNALBEAT_PATH
+HEALTHCHECK CMD exit 0

journalbeat/_meta/beat.yml

@@ -28,6 +28,17 @@ journalbeat.inputs:
   # Position to start reading from journal. Valid values: head, tail, cursor
   seek: tail
 
+  # Exact matching for field values of events.
+  # Matching for nginx entries: "systemd.unit=nginx"
+  #include_matches: []
+
+  # Optional fields that you can specify to add additional information to the
+  # output. Fields can be scalar values, arrays, dictionaries, or any nested
+  # combination of these.
+  #fields:
+  #  env: staging
+
+
 #========================= Journalbeat global options ============================
 #journalbeat:
   # Name of the registry file. If a relative path is used, it is considered relative to the
@@ -43,3 +54,7 @@ journalbeat.inputs:
 
   # Position to start reading from all journal. Possible values: head, tail, cursor
   #seek: head
+
+  # Exact matching for field values of events.
+  # Matching for nginx entries: "systemd.unit=nginx"
+  #matches: []

journalbeat/_meta/fields.common.yml

@@ -3,6 +3,9 @@
   description: >
     Contains common fields available in all event types.
   fields:
+    - name: read_timestamp
+      description: >
+          The time when Journalbeat read the journal entry.
     - name: coredump
       type: group
       description: >
@@ -16,117 +19,145 @@
           type: keyword
           description: >
             Annotations of messages containing coredumps from user units.
-    - name: object
+    - name: journald
       type: group
       description: >
-          Fields to log on behalf of a different program.
+          Fields provided by journald.
       fields:
-        - name: audit
+        - name: object
           type: group
           description: >
-              Audit fields of event.
+              Fields to log on behalf of a different program.
           fields:
-            - name: loginuid
+            - name: audit
+              type: group
+              description: >
+                  Audit fields of event.
+              fields:
+                - name: loginuid
+                  type: long
+                  required: false
+                  example: 1000
+                  description: >
+                    The login UID of the object process.
+                - name: session
+                  type: long
+                  required: false
+                  example: 3
+                  description: >
+                    The audit session of the object process.
+            - name: cmd
+              type: keyword
+              required: false
+              example: "/lib/systemd/systemd --user"
+              description: >
+                The command line of the process.
+            - name: name
+              type: keyword
+              required: false
+              example: "/lib/systemd/systemd"
+              description: >
+                Name of the executable.
+            - name: executable
+              type: keyword
+              required: false
+              description: >
+                Path to the the executable.
+              example: "/lib/systemd/systemd"
+            - name: uid
               type: long
               required: false
-              example: 1000
               description: >
-                The login UID of the object process.
-            - name: session
+                UID of the object process.
+            - name: gid
               type: long
               required: false
-              example: 3
               description: >
-                The audit session of the object process.
-        - name: cmd
-          type: keyword
-          required: false
-          example: "/lib/systemd/systemd --user"
-          description: >
-            The command line of the process.
-        - name: name
-          type: keyword
-          required: false
-          example: "/lib/systemd/systemd"
-          description: >
-            Name of the executable.
-        - name: executable
-          type: keyword
-          required: false
-          description: >
-            Path to the the executable.
-          example: "/lib/systemd/systemd"
-        - name: uid
-          type: long
-          required: false
-          description: >
-            UID of the object process.
-        - name: gid
-          type: long
-          required: false
-          description: >
-            GID of the object process.
-        - name: pid
-          type: long
-          required: false
-          description: >
-            PID of the object process.
-        - name: systemd
+                GID of the object process.
+            - name: pid
+              type: long
+              required: false
+              description: >
+                PID of the object process.
+            - name: systemd
+              type: group
+              description: >
+                  Systemd fields of event.
+              fields:
+                - name: owner_uid
+                  type: long
+                  required: false
+                  description: >
+                    The UID of the owner.
+                - name: session
+                  type: keyword
+                  required: false
+                  description: >
+                    The ID of the systemd session.
+                - name: unit
+                  type: keyword
+                  required: false
+                  description: >
+                    The name of the systemd unit.
+                - name: user_unit
+                  type: keyword
+                  required: false
+                  description: >
+                    The name of the systemd user unit.
+        - name: kernel
           type: group
           description: >
-              Systemd fields of event.
+              Fields to log on behalf of a different program.
           fields:
-            - name: owner_uid
-              type: long
+            - name: device
+              type: keyword
               required: false
               description: >
-                The UID of the owner.
-            - name: session
+                The kernel device name.
+            - name: subsystem
               type: keyword
               required: false
               description: >
-                The ID of the systemd session.
-            - name: unit
-              type: keyword
+                The kernel subsystem name.
+            - name: device_symlinks
+              type: text
               required: false
               description: >
-                The name of the systemd unit.
-            - name: user_unit
-              type: keyword
+                Additional symlink names pointing to the device node in /dev.
+            - name: device_node_path
+              type: text
               required: false
               description: >
-                The name of the systemd user unit.
-    - name: kernel
-      type: group
-      description: >
-          Fields to log on behalf of a different program.
-      fields:
-        - name: device
-          type: keyword
-          required: false
-          description: >
-            The kernel device name.
-        - name: subsystem
-          type: keyword
-          required: false
-          description: >
-            The kernel subsystem name.
-        - name: device_symlinks
-          type: text
-          required: false
-          description: >
-            Additional symlink names pointing to the device node in /dev.
-        - name: device_node_path
-          type: text
-          required: false
-          description: >
-            The device node path of this device in /dev.
-        - name: device_name
-          type: text
-          required: false
+                The device node path of this device in /dev.
+            - name: device_name
+              type: text
+              required: false
+              description: >
+                The kernel device name as it shows up in the device tree below /sys.
+        - name: code
+          type: group
           description: >
-            The kernel device name as it shows up in the device tree below /sys.
-    - name: process
+              Fields of the code generating the event.
+          fields:
+            - name: file
+              type: text
+              required: false
+              example: "../src/core/manager.c"
+              description: >
+                The name of the source file where the log is generated.
+            - name: function
+              type: text
+              required: false
+              example: "job_log_status_message"
+              description: >
+                The name of the function which generated the log message.
+            - name: line
+              type: long
+              required: false
+              example: 123
+              description: >
+                The line number of the code which generated the log message.
+        - name: process
       type: group
       description: >
           Fields to log on behalf of a different program.
@@ -184,6 +215,10 @@
           example: 1
           description: >
             The ID of the user which runs the process.
+        - name: capabilites
+          required: false
+          description: >
+            The effective capabilites of the process.
     - name: systemd
       type: group
       description: >
@@ -217,6 +252,11 @@
           example: "user-1234.slice"
           description: >
             The systemd slice unit.
+        - name: user_slice
+          type: keyword
+          required: false
+          description: >
+            The systemd user slice unit.
         - name: unit
           type: keyword
           required: false
@@ -246,29 +286,6 @@
           example: "dd8c974asdf01dbe2ef26d7fasdf264c9"
           description: >
             The boot ID for the boot the log was generated in.
-    - name: code
-      type: group
-      description: >
-          Fields of the code generating the event.
-      fields:
-        - name: file
-          type: text
-          required: false
-          example: "../src/core/manager.c"
-          description: >
-            The name of the source file where the log is generated.
-        - name: function
-          type: text
-          required: false
-          example: "job_log_status_message"
-          description: >
-            The name of the function which generated the log message.
-        - name: line
-          type: long
-          required: false
-          example: 123
-          description: >
-            The line number of the code which generated the log message.
     - name: syslog
       type: group
       description: >

journalbeat/beater/journalbeat.go

@@ -23,9 +23,11 @@ import (
 	"time"
 
 	"github.com/elastic/beats/journalbeat/checkpoint"
+	"github.com/elastic/beats/journalbeat/cmd/instance"
 	"github.com/elastic/beats/journalbeat/input"
 	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/common/cfgwarn"
 	"github.com/elastic/beats/libbeat/logp"
 
 	"github.com/elastic/beats/journalbeat/config"
@@ -44,6 +46,8 @@ type Journalbeat struct {
 
 // New returns a new Journalbeat instance
 func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) {
+	cfgwarn.Experimental("Journalbeat is experimental.")
+
 	config := config.DefaultConfig
 	if err := cfg.Unpack(&config); err != nil {
 		return nil, fmt.Errorf("error reading config file: %v", err)
@@ -55,6 +59,8 @@ func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) {
 		return nil, err
 	}
 
+	instance.SetupJournalMetrics()
+
 	var inputs []*input.Input
 	for _, c := range config.Inputs {
 		i, err := input.New(c, b.Publisher, done, cp.States())