From 47796b8872bf774ae5d98cf2dcced8041039ce05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20St=C3=BCrmer?= <stuermer@weltenwort.de>
Date: Thu, 7 Feb 2019 14:36:56 +0100
Subject: [PATCH 1/7] Add icinga filebeat log message rules

---
 .../builtin_rules/filebeat_icinga.test.ts     | 147 ++++++++++++++++++
 .../builtin_rules/filebeat_icinga.ts          |  86 ++++++++++
 .../log_entries_domain/builtin_rules/index.ts |   2 +
 3 files changed, 235 insertions(+)
 create mode 100644 x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.test.ts
 create mode 100644 x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.ts

diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.test.ts
new file mode 100644
index 0000000000000..f7b74358ae1c0
--- /dev/null
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.test.ts
@@ -0,0 +1,147 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { compileFormattingRules } from '../message';
+import { filebeatIcingaRules } from './filebeat_icinga';
+
+const { format } = compileFormattingRules(filebeatIcingaRules);
+
+describe('Filebeat Rules', () => {
+  describe('in pre-ECS format', () => {
+    test('icinga debug log', () => {
+      const flattenedDocument = {
+        '@timestamp': '2017-04-04T11:43:09.000Z',
+        'event.dataset': 'icinga.debug',
+        'fileset.module': 'icinga',
+        'fileset.name': 'debug',
+        'icinga.debug.facility': 'GraphiteWriter',
+        'icinga.debug.message':
+          "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.",
+        'icinga.debug.severity': 'debug',
+        'input.type': 'log',
+        offset: 0,
+        'prospector.type': 'log',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[Icinga][",
+  },
+  Object {
+    "field": "icinga.debug.facility",
+    "highlights": Array [],
+    "value": "GraphiteWriter",
+  },
+  Object {
+    "constant": "][",
+  },
+  Object {
+    "field": "icinga.debug.severity",
+    "highlights": Array [],
+    "value": "debug",
+  },
+  Object {
+    "constant": "] ",
+  },
+  Object {
+    "field": "icinga.debug.message",
+    "highlights": Array [],
+    "value": "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.",
+  },
+]
+`);
+    });
+
+    test('icinga main log', () => {
+      const flattenedDocument = {
+        '@timestamp': '2017-04-04T09:16:34.000Z',
+        'event.dataset': 'icinga.main',
+        'fileset.module': 'icinga',
+        'fileset.name': 'main',
+        'icinga.main.facility': 'Notification',
+        'icinga.main.message':
+          "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'",
+        'icinga.main.severity': 'information',
+        'input.type': 'log',
+        offset: 0,
+        'prospector.type': 'log',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[Icinga][",
+  },
+  Object {
+    "field": "icinga.main.facility",
+    "highlights": Array [],
+    "value": "Notification",
+  },
+  Object {
+    "constant": "][",
+  },
+  Object {
+    "field": "icinga.main.severity",
+    "highlights": Array [],
+    "value": "information",
+  },
+  Object {
+    "constant": "] ",
+  },
+  Object {
+    "field": "icinga.main.message",
+    "highlights": Array [],
+    "value": "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'",
+  },
+]
+`);
+    });
+
+    test('icinga startup log', () => {
+      const flattenedDocument = {
+        'event.dataset': 'icinga.startup',
+        'fileset.module': 'icinga',
+        'fileset.name': 'startup',
+        'icinga.startup.facility': 'cli',
+        'icinga.startup.message': 'Icinga application loader (version: r2.6.3-1)',
+        'icinga.startup.severity': 'information',
+        'input.type': 'log',
+        offset: 0,
+        'prospector.type': 'log',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[Icinga][",
+  },
+  Object {
+    "field": "icinga.startup.facility",
+    "highlights": Array [],
+    "value": "cli",
+  },
+  Object {
+    "constant": "][",
+  },
+  Object {
+    "field": "icinga.startup.severity",
+    "highlights": Array [],
+    "value": "information",
+  },
+  Object {
+    "constant": "] ",
+  },
+  Object {
+    "field": "icinga.startup.message",
+    "highlights": Array [],
+    "value": "Icinga application loader (version: r2.6.3-1)",
+  },
+]
+`);
+    });
+  });
+});
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.ts
new file mode 100644
index 0000000000000..c04a746e6bf41
--- /dev/null
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.ts
@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const filebeatIcingaRules = [
+  {
+    // pre-ECS
+    when: {
+      exists: ['icinga.main.message'],
+    },
+    format: [
+      {
+        constant: '[Icinga][',
+      },
+      {
+        field: 'icinga.main.facility',
+      },
+      {
+        constant: '][',
+      },
+      {
+        field: 'icinga.main.severity',
+      },
+      {
+        constant: '] ',
+      },
+      {
+        field: 'icinga.main.message',
+      },
+    ],
+  },
+  {
+    // pre-ECS
+    when: {
+      exists: ['icinga.debug.message'],
+    },
+    format: [
+      {
+        constant: '[Icinga][',
+      },
+      {
+        field: 'icinga.debug.facility',
+      },
+      {
+        constant: '][',
+      },
+      {
+        field: 'icinga.debug.severity',
+      },
+      {
+        constant: '] ',
+      },
+      {
+        field: 'icinga.debug.message',
+      },
+    ],
+  },
+  {
+    // pre-ECS
+    when: {
+      exists: ['icinga.startup.message'],
+    },
+    format: [
+      {
+        constant: '[Icinga][',
+      },
+      {
+        field: 'icinga.startup.facility',
+      },
+      {
+        constant: '][',
+      },
+      {
+        field: 'icinga.startup.severity',
+      },
+      {
+        constant: '] ',
+      },
+      {
+        field: 'icinga.startup.message',
+      },
+    ],
+  },
+];
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
index 5ee8286e77f58..e82a2f27722be 100644
--- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
@@ -6,6 +6,7 @@
 
 import { filebeatApache2Rules } from './filebeat_apache2';
 import { filebeatAuditdRules } from './filebeat_auditd';
+import { filebeatIcingaRules } from './filebeat_icinga';
 import { filebeatMySQLRules } from './filebeat_mysql';
 import { filebeatNginxRules } from './filebeat_nginx';
 import { filebeatRedisRules } from './filebeat_redis';
@@ -20,6 +21,7 @@ export const builtinRules = [
   ...filebeatSystemRules,
   ...filebeatMySQLRules,
   ...filebeatAuditdRules,
+  ...filebeatIcingaRules,
   ...genericRules,
   {
     when: {

From 9a43ddd9bb1b934d7ea4f7a7ad42e1a1b04522d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20St=C3=BCrmer?= <stuermer@weltenwort.de>
Date: Thu, 7 Feb 2019 18:20:58 +0100
Subject: [PATCH 2/7] Add haproxy filebeat log message rules

---
 .../builtin_rules/filebeat_haproxy.test.ts    | 400 ++++++++++++++++++
 .../builtin_rules/filebeat_haproxy.ts         | 200 +++++++++
 .../log_entries_domain/builtin_rules/index.ts |   2 +
 3 files changed, 602 insertions(+)
 create mode 100644 x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.test.ts
 create mode 100644 x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.ts

diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.test.ts
new file mode 100644
index 0000000000000..ab39af6359769
--- /dev/null
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.test.ts
@@ -0,0 +1,400 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { compileFormattingRules } from '../message';
+import { filebeatHaproxyRules } from './filebeat_haproxy';
+
+const { format } = compileFormattingRules(filebeatHaproxyRules);
+
+describe('Filebeat Rules', () => {
+  describe('in pre-ECS format', () => {
+    test('haproxy default log', () => {
+      const flattenedDocument = {
+        'event.dataset': 'haproxy.log',
+        'fileset.module': 'haproxy',
+        'fileset.name': 'log',
+        'haproxy.client.ip': '1.2.3.4',
+        'haproxy.client.port': '40780',
+        'haproxy.destination.ip': '1.2.3.4',
+        'haproxy.destination.port': '5000',
+        'haproxy.frontend_name': 'main',
+        'haproxy.geoip.continent_name': 'North America',
+        'haproxy.geoip.country_iso_code': 'US',
+        'haproxy.geoip.location.lat': 37.751,
+        'haproxy.geoip.location.lon': -97.822,
+        'haproxy.mode': 'HTTP',
+        'haproxy.pid': '24551',
+        'haproxy.process_name': 'haproxy',
+        'haproxy.source': '1.2.3.4',
+        'input.type': 'log',
+        offset: 0,
+        'prospector.type': 'log',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[HAProxy] ",
+  },
+  Object {
+    "field": "haproxy.client.ip",
+    "highlights": Array [],
+    "value": "1.2.3.4",
+  },
+  Object {
+    "constant": ":",
+  },
+  Object {
+    "field": "haproxy.client.port",
+    "highlights": Array [],
+    "value": "40780",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "haproxy.frontend_name",
+    "highlights": Array [],
+    "value": "main",
+  },
+  Object {
+    "constant": " ",
+  },
+]
+`);
+    });
+
+    test('haproxy tcp log', () => {
+      const flattenedDocument = {
+        'event.dataset': 'haproxy.log',
+        'fileset.module': 'haproxy',
+        'fileset.name': 'log',
+        'haproxy.backend_name': 'app',
+        'haproxy.backend_queue': 0,
+        'haproxy.bytes_read': 212,
+        'haproxy.client.ip': '127.0.0.1',
+        'haproxy.client.port': 40962,
+        'haproxy.connection_wait_time_ms': -1,
+        'haproxy.connections.active': 1,
+        'haproxy.connections.backend': 0,
+        'haproxy.connections.frontend': 1,
+        'haproxy.connections.retries': 0,
+        'haproxy.connections.server': 0,
+        'haproxy.frontend_name': 'main',
+        'haproxy.pid': 25457,
+        'haproxy.process_name': 'haproxy',
+        'haproxy.server_name': '<NOSRV>',
+        'haproxy.server_queue': 0,
+        'haproxy.source': '127.0.0.1',
+        'haproxy.tcp.processing_time_ms': 0,
+        'haproxy.termination_state': 'SC',
+        'haproxy.total_waiting_time_ms': -1,
+        'input.type': 'log',
+        offset: 0,
+        'prospector.type': 'log',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[HAProxy][tcp] ",
+  },
+  Object {
+    "field": "haproxy.client.ip",
+    "highlights": Array [],
+    "value": "127.0.0.1",
+  },
+  Object {
+    "constant": ":",
+  },
+  Object {
+    "field": "haproxy.client.port",
+    "highlights": Array [],
+    "value": "40962",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "haproxy.frontend_name",
+    "highlights": Array [],
+    "value": "main",
+  },
+  Object {
+    "constant": " -> ",
+  },
+  Object {
+    "field": "haproxy.backend_name",
+    "highlights": Array [],
+    "value": "app",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.server_name",
+    "highlights": Array [],
+    "value": "<NOSRV>",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "haproxy.connections.active",
+    "highlights": Array [],
+    "value": "1",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.connections.frontend",
+    "highlights": Array [],
+    "value": "1",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.connections.backend",
+    "highlights": Array [],
+    "value": "0",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.connections.server",
+    "highlights": Array [],
+    "value": "0",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.connections.retries",
+    "highlights": Array [],
+    "value": "0",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "haproxy.server_queue",
+    "highlights": Array [],
+    "value": "0",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.backend_queue",
+    "highlights": Array [],
+    "value": "0",
+  },
+]
+`);
+    });
+
+    test('haproxy http log', () => {
+      const flattenedDocument = {
+        'event.dataset': 'haproxy.log',
+        'fileset.module': 'haproxy',
+        'fileset.name': 'log',
+        'haproxy.backend_name': 'docs_microservice',
+        'haproxy.backend_queue': 0,
+        'haproxy.bytes_read': 168,
+        'haproxy.client.ip': '1.2.3.4',
+        'haproxy.client.port': 38862,
+        'haproxy.connection_wait_time_ms': 1,
+        'haproxy.connections.active': 6,
+        'haproxy.connections.backend': 0,
+        'haproxy.connections.frontend': 6,
+        'haproxy.connections.retries': 0,
+        'haproxy.connections.server': 0,
+        'haproxy.frontend_name': 'incoming~',
+        'haproxy.geoip.continent_name': 'North America',
+        'haproxy.geoip.country_iso_code': 'US',
+        'haproxy.geoip.location.lat': 37.751,
+        'haproxy.geoip.location.lon': -97.822,
+        'haproxy.http.request.captured_cookie': '-',
+        'haproxy.http.request.raw_request_line':
+          'GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1',
+        'haproxy.http.request.time_active_ms': 2,
+        'haproxy.http.request.time_wait_ms': 0,
+        'haproxy.http.request.time_wait_without_data_ms': 0,
+        'haproxy.http.response.captured_cookie': '-',
+        'haproxy.http.response.status_code': 304,
+        'haproxy.pid': 32450,
+        'haproxy.process_name': 'haproxy',
+        'haproxy.server_name': 'docs',
+        'haproxy.server_queue': 0,
+        'haproxy.termination_state': '----',
+        'haproxy.total_waiting_time_ms': 0,
+        'input.type': 'log',
+        offset: 0,
+        'prospector.type': 'log',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[HAProxy][http] ",
+  },
+  Object {
+    "field": "haproxy.client.ip",
+    "highlights": Array [],
+    "value": "1.2.3.4",
+  },
+  Object {
+    "constant": ":",
+  },
+  Object {
+    "field": "haproxy.client.port",
+    "highlights": Array [],
+    "value": "38862",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "haproxy.frontend_name",
+    "highlights": Array [],
+    "value": "incoming~",
+  },
+  Object {
+    "constant": " -> ",
+  },
+  Object {
+    "field": "haproxy.backend_name",
+    "highlights": Array [],
+    "value": "docs_microservice",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.server_name",
+    "highlights": Array [],
+    "value": "docs",
+  },
+  Object {
+    "constant": " \\"",
+  },
+  Object {
+    "field": "haproxy.http.request.raw_request_line",
+    "highlights": Array [],
+    "value": "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1",
+  },
+  Object {
+    "constant": "\\" ",
+  },
+  Object {
+    "field": "haproxy.http.response.status_code",
+    "highlights": Array [],
+    "value": "304",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "haproxy.http.request.time_wait_ms",
+    "highlights": Array [],
+    "value": "0",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.total_waiting_time_ms",
+    "highlights": Array [],
+    "value": "0",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.connection_wait_time_ms",
+    "highlights": Array [],
+    "value": "1",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.http.request.time_wait_without_data_ms",
+    "highlights": Array [],
+    "value": "0",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.http.request.time_active_ms",
+    "highlights": Array [],
+    "value": "2",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "haproxy.connections.active",
+    "highlights": Array [],
+    "value": "6",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.connections.frontend",
+    "highlights": Array [],
+    "value": "6",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.connections.backend",
+    "highlights": Array [],
+    "value": "0",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.connections.server",
+    "highlights": Array [],
+    "value": "0",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.connections.retries",
+    "highlights": Array [],
+    "value": "0",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "haproxy.server_queue",
+    "highlights": Array [],
+    "value": "0",
+  },
+  Object {
+    "constant": "/",
+  },
+  Object {
+    "field": "haproxy.backend_queue",
+    "highlights": Array [],
+    "value": "0",
+  },
+]
+`);
+    });
+  });
+});
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.ts
new file mode 100644
index 0000000000000..79b778e9448ee
--- /dev/null
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.ts
@@ -0,0 +1,200 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+const commonFrontendFields = [
+  {
+    field: 'haproxy.client.ip',
+  },
+  {
+    constant: ':',
+  },
+  {
+    field: 'haproxy.client.port',
+  },
+  {
+    constant: ' ',
+  },
+  {
+    field: 'haproxy.frontend_name',
+  },
+];
+
+const commonBackendFields = [
+  {
+    constant: ' -> ',
+  },
+  {
+    field: 'haproxy.backend_name',
+  },
+  {
+    constant: '/',
+  },
+  {
+    field: 'haproxy.server_name',
+  },
+];
+
+const commonConnectionStatsFields = [
+  {
+    field: 'haproxy.connections.active',
+  },
+  {
+    constant: '/',
+  },
+  {
+    field: 'haproxy.connections.frontend',
+  },
+  {
+    constant: '/',
+  },
+  {
+    field: 'haproxy.connections.backend',
+  },
+  {
+    constant: '/',
+  },
+  {
+    field: 'haproxy.connections.server',
+  },
+  {
+    constant: '/',
+  },
+  {
+    field: 'haproxy.connections.retries',
+  },
+];
+
+const commonQueueStatsFields = [
+  {
+    field: 'haproxy.server_queue',
+  },
+  {
+    constant: '/',
+  },
+  {
+    field: 'haproxy.backend_queue',
+  },
+];
+
+export const filebeatHaproxyRules = [
+  {
+    // pre-ECS
+    when: {
+      exists: ['haproxy.http.request.raw_request_line'],
+    },
+    format: [
+      {
+        constant: '[HAProxy][http] ',
+      },
+      ...commonFrontendFields,
+      ...commonBackendFields,
+      {
+        constant: ' "',
+      },
+      {
+        field: 'haproxy.http.request.raw_request_line',
+      },
+      {
+        constant: '" ',
+      },
+      {
+        field: 'haproxy.http.response.status_code',
+      },
+      {
+        constant: ' ',
+      },
+      {
+        field: 'haproxy.http.request.time_wait_ms',
+      },
+      {
+        constant: '/',
+      },
+      {
+        field: 'haproxy.total_waiting_time_ms',
+      },
+      {
+        constant: '/',
+      },
+      {
+        field: 'haproxy.connection_wait_time_ms',
+      },
+      {
+        constant: '/',
+      },
+      {
+        field: 'haproxy.http.request.time_wait_without_data_ms',
+      },
+      {
+        constant: '/',
+      },
+      {
+        field: 'haproxy.http.request.time_active_ms',
+      },
+      {
+        constant: ' ',
+      },
+      ...commonConnectionStatsFields,
+      {
+        constant: ' ',
+      },
+      ...commonQueueStatsFields,
+    ],
+  },
+  {
+    // pre-ECS
+    when: {
+      exists: ['haproxy.connections.active'],
+    },
+    format: [
+      {
+        constant: '[HAProxy][tcp] ',
+      },
+      ...commonFrontendFields,
+      ...commonBackendFields,
+      {
+        constant: ' ',
+      },
+      ...commonConnectionStatsFields,
+      {
+        constant: ' ',
+      },
+      ...commonQueueStatsFields,
+    ],
+  },
+  {
+    // pre-ECS
+    when: {
+      exists: ['haproxy.error_message'],
+    },
+    format: [
+      {
+        constant: '[HAProxy] ',
+      },
+      ...commonFrontendFields,
+      {
+        constant: ' ',
+      },
+      {
+        field: 'haproxy.error_message',
+      },
+    ],
+  },
+  {
+    // pre-ECS
+    when: {
+      exists: ['haproxy.frontend_name'],
+    },
+    format: [
+      {
+        constant: '[HAProxy] ',
+      },
+      ...commonFrontendFields,
+      {
+        constant: ' ',
+      },
+    ],
+  },
+];
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
index e82a2f27722be..e3950a386653e 100644
--- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
@@ -6,6 +6,7 @@
 
 import { filebeatApache2Rules } from './filebeat_apache2';
 import { filebeatAuditdRules } from './filebeat_auditd';
+import { filebeatHaproxyRules } from './filebeat_haproxy';
 import { filebeatIcingaRules } from './filebeat_icinga';
 import { filebeatMySQLRules } from './filebeat_mysql';
 import { filebeatNginxRules } from './filebeat_nginx';
@@ -21,6 +22,7 @@ export const builtinRules = [
   ...filebeatSystemRules,
   ...filebeatMySQLRules,
   ...filebeatAuditdRules,
+  ...filebeatHaproxyRules,
   ...filebeatIcingaRules,
   ...genericRules,
   {

From a25d8f253dfba608611320be053b6c3809438d76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20St=C3=BCrmer?= <stuermer@weltenwort.de>
Date: Thu, 7 Feb 2019 18:30:15 +0100
Subject: [PATCH 3/7] Add mongodb filebeat log message rules

---
 .../builtin_rules/filebeat_mongodb.test.ts    | 52 +++++++++++++++++++
 .../builtin_rules/filebeat_mongodb.ts         | 28 ++++++++++
 .../log_entries_domain/builtin_rules/index.ts |  2 +
 3 files changed, 82 insertions(+)
 create mode 100644 x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.test.ts
 create mode 100644 x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.ts

diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.test.ts
new file mode 100644
index 0000000000000..63e8efd0ca427
--- /dev/null
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.test.ts
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { compileFormattingRules } from '../message';
+import { filebeatMongodbRules } from './filebeat_mongodb';
+
+const { format } = compileFormattingRules(filebeatMongodbRules);
+
+describe('Filebeat Rules', () => {
+  describe('in pre-ECS format', () => {
+    test('mongodb log', () => {
+      const flattenedDocument = {
+        '@timestamp': '2018-02-05T12:44:56.677Z',
+        'event.dataset': 'mongodb.log',
+        'fileset.module': 'mongodb',
+        'fileset.name': 'log',
+        'input.type': 'log',
+        'mongodb.log.component': 'STORAGE',
+        'mongodb.log.context': 'initandlisten',
+        'mongodb.log.message':
+          'wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),',
+        'mongodb.log.severity': 'I',
+        offset: 281,
+        'prospector.type': 'log',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[MongoDB][",
+  },
+  Object {
+    "field": "mongodb.log.component",
+    "highlights": Array [],
+    "value": "STORAGE",
+  },
+  Object {
+    "constant": "] ",
+  },
+  Object {
+    "field": "mongodb.log.message",
+    "highlights": Array [],
+    "value": "wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),",
+  },
+]
+`);
+    });
+  });
+});
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.ts
new file mode 100644
index 0000000000000..06a4964875898
--- /dev/null
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.ts
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const filebeatMongodbRules = [
+  {
+    // pre-ECS
+    when: {
+      exists: ['mongodb.log.message'],
+    },
+    format: [
+      {
+        constant: '[MongoDB][',
+      },
+      {
+        field: 'mongodb.log.component',
+      },
+      {
+        constant: '] ',
+      },
+      {
+        field: 'mongodb.log.message',
+      },
+    ],
+  },
+];
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
index e3950a386653e..a27aa89ac4465 100644
--- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
@@ -8,6 +8,7 @@ import { filebeatApache2Rules } from './filebeat_apache2';
 import { filebeatAuditdRules } from './filebeat_auditd';
 import { filebeatHaproxyRules } from './filebeat_haproxy';
 import { filebeatIcingaRules } from './filebeat_icinga';
+import { filebeatMongodbRules } from './filebeat_mongodb';
 import { filebeatMySQLRules } from './filebeat_mysql';
 import { filebeatNginxRules } from './filebeat_nginx';
 import { filebeatRedisRules } from './filebeat_redis';
@@ -24,6 +25,7 @@ export const builtinRules = [
   ...filebeatAuditdRules,
   ...filebeatHaproxyRules,
   ...filebeatIcingaRules,
+  ...filebeatMongodbRules,
   ...genericRules,
   {
     when: {

From 4459913c8cf4423f5b70271c619fbed83ebbcce1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20St=C3=BCrmer?= <stuermer@weltenwort.de>
Date: Fri, 8 Feb 2019 13:25:38 +0100
Subject: [PATCH 4/7] Add iis filebeat log message rules

---
 .../builtin_rules/filebeat_iis.test.ts        | 279 ++++++++++++++++++
 .../builtin_rules/filebeat_iis.ts             | 122 ++++++++
 .../log_entries_domain/builtin_rules/index.ts |   2 +
 3 files changed, 403 insertions(+)
 create mode 100644 x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.test.ts
 create mode 100644 x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.ts

diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.test.ts
new file mode 100644
index 0000000000000..192736a138d49
--- /dev/null
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.test.ts
@@ -0,0 +1,279 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { compileFormattingRules } from '../message';
+import { filebeatIisRules } from './filebeat_iis';
+
+const { format } = compileFormattingRules(filebeatIisRules);
+
+describe('Filebeat Rules', () => {
+  describe('in pre-ECS format', () => {
+    test('iis access log', () => {
+      const flattenedDocument = {
+        '@timestamp': '2018-01-01T08:09:10.000Z',
+        'event.dataset': 'iis.access',
+        'fileset.module': 'iis',
+        'fileset.name': 'access',
+        'iis.access.geoip.city_name': 'Berlin',
+        'iis.access.geoip.continent_name': 'Europe',
+        'iis.access.geoip.country_iso_code': 'DE',
+        'iis.access.geoip.location.lat': 52.4908,
+        'iis.access.geoip.location.lon': 13.3275,
+        'iis.access.geoip.region_iso_code': 'DE-BE',
+        'iis.access.geoip.region_name': 'Land Berlin',
+        'iis.access.method': 'GET',
+        'iis.access.port': '80',
+        'iis.access.query_string': 'q=100',
+        'iis.access.referrer': '-',
+        'iis.access.remote_ip': '85.181.35.98',
+        'iis.access.request_time_ms': '123',
+        'iis.access.response_code': '200',
+        'iis.access.server_ip': '127.0.0.1',
+        'iis.access.sub_status': '0',
+        'iis.access.url': '/',
+        'iis.access.user_agent.device': 'Other',
+        'iis.access.user_agent.major': '57',
+        'iis.access.user_agent.minor': '0',
+        'iis.access.user_agent.name': 'Firefox',
+        'iis.access.user_agent.original':
+          'Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0',
+        'iis.access.user_agent.os': 'Windows',
+        'iis.access.user_agent.os_name': 'Windows',
+        'iis.access.user_name': '-',
+        'iis.access.win32_status': '0',
+        'input.type': 'log',
+        offset: 257,
+        'prospector.type': 'log',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[IIS][access] ",
+  },
+  Object {
+    "field": "iis.access.remote_ip",
+    "highlights": Array [],
+    "value": "85.181.35.98",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "iis.access.user_name",
+    "highlights": Array [],
+    "value": "-",
+  },
+  Object {
+    "constant": " \\"",
+  },
+  Object {
+    "field": "iis.access.method",
+    "highlights": Array [],
+    "value": "GET",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "iis.access.url",
+    "highlights": Array [],
+    "value": "/",
+  },
+  Object {
+    "constant": " HTTP/",
+  },
+  Object {
+    "field": "iis.access.http_version",
+    "highlights": Array [],
+    "value": "undefined",
+  },
+  Object {
+    "constant": "\\" ",
+  },
+  Object {
+    "field": "iis.access.response_code",
+    "highlights": Array [],
+    "value": "200",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "iis.access.body_sent.bytes",
+    "highlights": Array [],
+    "value": "undefined",
+  },
+]
+`);
+    });
+
+    test('iis 7.5 access log', () => {
+      const flattenedDocument = {
+        '@timestamp': '2018-08-28T18:24:25.000Z',
+        'event.dataset': 'iis.access',
+        'fileset.module': 'iis',
+        'fileset.name': 'access',
+        'iis.access.method': 'GET',
+        'iis.access.port': '80',
+        'iis.access.query_string': '-',
+        'iis.access.remote_ip': '10.100.118.31',
+        'iis.access.request_time_ms': '792',
+        'iis.access.response_code': '404',
+        'iis.access.server_ip': '10.100.220.70',
+        'iis.access.sub_status': '4',
+        'iis.access.url': '/',
+        'iis.access.user_agent.device': 'Other',
+        'iis.access.user_agent.name': 'Other',
+        'iis.access.user_agent.original':
+          'Mozilla/4.0+(compatible;+MSIE+7.0;+Windows+NT+6.3;+WOW64;+Trident/7.0;+.NET4.0E;+.NET4.0C;+.NET+CLR+3.5.30729;+.NET+CLR[+2.0.50727](tel:+2050727);+.NET+CLR+3.0.30729)',
+        'iis.access.user_agent.os': 'Windows',
+        'iis.access.user_agent.os_name': 'Windows',
+        'iis.access.user_name': '-',
+        'iis.access.win32_status': '2',
+        'input.type': 'log',
+        offset: 244,
+        'prospector.type': 'log',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[IIS][access] ",
+  },
+  Object {
+    "field": "iis.access.remote_ip",
+    "highlights": Array [],
+    "value": "10.100.118.31",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "iis.access.user_name",
+    "highlights": Array [],
+    "value": "-",
+  },
+  Object {
+    "constant": " \\"",
+  },
+  Object {
+    "field": "iis.access.method",
+    "highlights": Array [],
+    "value": "GET",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "iis.access.url",
+    "highlights": Array [],
+    "value": "/",
+  },
+  Object {
+    "constant": " HTTP/",
+  },
+  Object {
+    "field": "iis.access.http_version",
+    "highlights": Array [],
+    "value": "undefined",
+  },
+  Object {
+    "constant": "\\" ",
+  },
+  Object {
+    "field": "iis.access.response_code",
+    "highlights": Array [],
+    "value": "404",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "iis.access.body_sent.bytes",
+    "highlights": Array [],
+    "value": "undefined",
+  },
+]
+`);
+    });
+
+    test('iis error log', () => {
+      const flattenedDocument = {
+        '@timestamp': '2018-01-01T08:09:10.000Z',
+        'event.dataset': 'iis.error',
+        'fileset.module': 'iis',
+        'fileset.name': 'error',
+        'iis.error.http_version': '1.1',
+        'iis.error.method': 'GET',
+        'iis.error.queue_name': '-',
+        'iis.error.reason_phrase': 'ConnLimit',
+        'iis.error.remote_ip': '172.31.77.6',
+        'iis.error.remote_port': '2094',
+        'iis.error.response_code': '503',
+        'iis.error.server_ip': '172.31.77.6',
+        'iis.error.server_port': '80',
+        'iis.error.url': '/qos/1kbfile.txt',
+        'input.type': 'log',
+        offset: 186,
+        'prospector.type': 'log',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[IIS][error] ",
+  },
+  Object {
+    "field": "iis.error.remote_ip",
+    "highlights": Array [],
+    "value": "172.31.77.6",
+  },
+  Object {
+    "constant": " \\"",
+  },
+  Object {
+    "field": "iis.error.method",
+    "highlights": Array [],
+    "value": "GET",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "iis.error.url",
+    "highlights": Array [],
+    "value": "/qos/1kbfile.txt",
+  },
+  Object {
+    "constant": " HTTP/",
+  },
+  Object {
+    "field": "iis.error.http_version",
+    "highlights": Array [],
+    "value": "1.1",
+  },
+  Object {
+    "constant": "\\" ",
+  },
+  Object {
+    "field": "iis.error.response_code",
+    "highlights": Array [],
+    "value": "503",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "iis.error.reason_phrase",
+    "highlights": Array [],
+    "value": "ConnLimit",
+  },
+]
+`);
+    });
+  });
+});
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.ts
new file mode 100644
index 0000000000000..e8d447349e1f3
--- /dev/null
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.ts
@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const filebeatIisRules = [
+  {
+    // pre-ECS
+    when: {
+      exists: ['iis.access.method'],
+    },
+    format: [
+      {
+        constant: '[IIS][access] ',
+      },
+      {
+        field: 'iis.access.remote_ip',
+      },
+      {
+        constant: ' ',
+      },
+      {
+        field: 'iis.access.user_name',
+      },
+      {
+        constant: ' "',
+      },
+      {
+        field: 'iis.access.method',
+      },
+      {
+        constant: ' ',
+      },
+      {
+        field: 'iis.access.url',
+      },
+      {
+        constant: ' HTTP/',
+      },
+      {
+        field: 'iis.access.http_version',
+      },
+      {
+        constant: '" ',
+      },
+      {
+        field: 'iis.access.response_code',
+      },
+      {
+        constant: ' ',
+      },
+      {
+        field: 'iis.access.body_sent.bytes',
+      },
+    ],
+  },
+  {
+    // pre-ECS
+    when: {
+      exists: ['iis.error.url'],
+    },
+    format: [
+      {
+        constant: '[IIS][error] ',
+      },
+      {
+        field: 'iis.error.remote_ip',
+      },
+      {
+        constant: ' "',
+      },
+      {
+        field: 'iis.error.method',
+      },
+      {
+        constant: ' ',
+      },
+      {
+        field: 'iis.error.url',
+      },
+      {
+        constant: ' HTTP/',
+      },
+      {
+        field: 'iis.error.http_version',
+      },
+      {
+        constant: '" ',
+      },
+      {
+        field: 'iis.error.response_code',
+      },
+      {
+        constant: ' ',
+      },
+      {
+        field: 'iis.error.reason_phrase',
+      },
+    ],
+  },
+  {
+    // pre-ECS
+    when: {
+      exists: ['iis.error.reason_phrase'],
+    },
+    format: [
+      {
+        constant: '[IIS][error] ',
+      },
+      {
+        field: 'iis.error.remote_ip',
+      },
+      {
+        constant: ' ',
+      },
+      {
+        field: 'iis.error.reason_phrase',
+      },
+    ],
+  },
+];
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
index a27aa89ac4465..c0dab0dcccf6f 100644
--- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
@@ -8,6 +8,7 @@ import { filebeatApache2Rules } from './filebeat_apache2';
 import { filebeatAuditdRules } from './filebeat_auditd';
 import { filebeatHaproxyRules } from './filebeat_haproxy';
 import { filebeatIcingaRules } from './filebeat_icinga';
+import { filebeatIisRules } from './filebeat_iis';
 import { filebeatMongodbRules } from './filebeat_mongodb';
 import { filebeatMySQLRules } from './filebeat_mysql';
 import { filebeatNginxRules } from './filebeat_nginx';
@@ -25,6 +26,7 @@ export const builtinRules = [
   ...filebeatAuditdRules,
   ...filebeatHaproxyRules,
   ...filebeatIcingaRules,
+  ...filebeatIisRules,
   ...filebeatMongodbRules,
   ...genericRules,
   {

From 579b29b831287fd54c074eba6315c544a72e2f19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20St=C3=BCrmer?= <stuermer@weltenwort.de>
Date: Fri, 8 Feb 2019 13:54:25 +0100
Subject: [PATCH 5/7] Add logstash filebeat log message rules

---
 .../builtin_rules/filebeat_logstash.test.ts   | 112 ++++++++++++++++++
 .../builtin_rules/filebeat_logstash.ts        |  60 ++++++++++
 .../log_entries_domain/builtin_rules/index.ts |   2 +
 3 files changed, 174 insertions(+)
 create mode 100644 x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.test.ts
 create mode 100644 x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.ts

diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.test.ts
new file mode 100644
index 0000000000000..fc438d323b577
--- /dev/null
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.test.ts
@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { compileFormattingRules } from '../message';
+import { filebeatLogstashRules } from './filebeat_logstash';
+
+const { format } = compileFormattingRules(filebeatLogstashRules);
+
+describe('Filebeat Rules', () => {
+  describe('in pre-ECS format', () => {
+    test('logstash log', () => {
+      const flattenedDocument = {
+        '@timestamp': '2017-10-23T14:20:12.046Z',
+        'event.dataset': 'logstash.log',
+        'fileset.module': 'logstash',
+        'fileset.name': 'log',
+        'input.type': 'log',
+        'logstash.log.level': 'INFO',
+        'logstash.log.message':
+          'Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}',
+        'logstash.log.module': 'logstash.modules.scaffold',
+        offset: 0,
+        'prospector.type': 'log',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[Logstash][",
+  },
+  Object {
+    "field": "logstash.log.level",
+    "highlights": Array [],
+    "value": "INFO",
+  },
+  Object {
+    "constant": "] ",
+  },
+  Object {
+    "field": "logstash.log.module",
+    "highlights": Array [],
+    "value": "logstash.modules.scaffold",
+  },
+  Object {
+    "constant": " - ",
+  },
+  Object {
+    "field": "logstash.log.message",
+    "highlights": Array [],
+    "value": "Initializing module {:module_name=>\\"fb_apache\\", :directory=>\\"/usr/share/logstash/modules/fb_apache/configuration\\"}",
+  },
+]
+`);
+    });
+
+    test('logstash slowlog', () => {
+      const flattenedDocument = {
+        '@timestamp': '2017-10-30T09:57:58.243Z',
+        'event.dataset': 'logstash.slowlog',
+        'fileset.module': 'logstash',
+        'fileset.name': 'slowlog',
+        'input.type': 'log',
+        'logstash.slowlog.event':
+          '"{\\"@version\\":\\"1\\",\\"@timestamp\\":\\"2017-10-30T13:57:55.130Z\\",\\"host\\":\\"sashimi\\",\\"sequence\\":0,\\"message\\":\\"Hello world!\\"}"',
+        'logstash.slowlog.level': 'WARN',
+        'logstash.slowlog.message':
+          'event processing time {:plugin_params=>{"time"=>3, "id"=>"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c"}, :took_in_nanos=>3027675106, :took_in_millis=>3027, :event=>"{\\"@version\\":\\"1\\",\\"@timestamp\\":\\"2017-10-30T13:57:55.130Z\\",\\"host\\":\\"sashimi\\",\\"sequence\\":0,\\"message\\":\\"Hello world!\\"}"}',
+        'logstash.slowlog.module': 'slowlog.logstash.filters.sleep',
+        'logstash.slowlog.plugin_name': 'sleep',
+        'logstash.slowlog.plugin_params':
+          '{"time"=>3, "id"=>"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c"}',
+        'logstash.slowlog.plugin_type': 'filters',
+        'logstash.slowlog.took_in_millis': 3027,
+        'logstash.slowlog.took_in_nanos': 3027675106,
+        offset: 0,
+        'prospector.type': 'log',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[Logstash][",
+  },
+  Object {
+    "field": "logstash.slowlog.level",
+    "highlights": Array [],
+    "value": "WARN",
+  },
+  Object {
+    "constant": "] ",
+  },
+  Object {
+    "field": "logstash.slowlog.module",
+    "highlights": Array [],
+    "value": "slowlog.logstash.filters.sleep",
+  },
+  Object {
+    "constant": " - ",
+  },
+  Object {
+    "field": "logstash.slowlog.message",
+    "highlights": Array [],
+    "value": "event processing time {:plugin_params=>{\\"time\\"=>3, \\"id\\"=>\\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\\"}, :took_in_nanos=>3027675106, :took_in_millis=>3027, :event=>\\"{\\\\\\"@version\\\\\\":\\\\\\"1\\\\\\",\\\\\\"@timestamp\\\\\\":\\\\\\"2017-10-30T13:57:55.130Z\\\\\\",\\\\\\"host\\\\\\":\\\\\\"sashimi\\\\\\",\\\\\\"sequence\\\\\\":0,\\\\\\"message\\\\\\":\\\\\\"Hello world!\\\\\\"}\\"}",
+  },
+]
+`);
+    });
+  });
+});
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.ts
new file mode 100644
index 0000000000000..6e0c26cd68023
--- /dev/null
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.ts
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const filebeatLogstashRules = [
+  {
+    // pre-ECS
+    when: {
+      exists: ['logstash.log.message'],
+    },
+    format: [
+      {
+        constant: '[Logstash][',
+      },
+      {
+        field: 'logstash.log.level',
+      },
+      {
+        constant: '] ',
+      },
+      {
+        field: 'logstash.log.module',
+      },
+      {
+        constant: ' - ',
+      },
+      {
+        field: 'logstash.log.message',
+      },
+    ],
+  },
+  {
+    // pre-ECS
+    when: {
+      exists: ['logstash.slowlog.message'],
+    },
+    format: [
+      {
+        constant: '[Logstash][',
+      },
+      {
+        field: 'logstash.slowlog.level',
+      },
+      {
+        constant: '] ',
+      },
+      {
+        field: 'logstash.slowlog.module',
+      },
+      {
+        constant: ' - ',
+      },
+      {
+        field: 'logstash.slowlog.message',
+      },
+    ],
+  },
+];
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
index c0dab0dcccf6f..2bd0d93866169 100644
--- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
@@ -9,6 +9,7 @@ import { filebeatAuditdRules } from './filebeat_auditd';
 import { filebeatHaproxyRules } from './filebeat_haproxy';
 import { filebeatIcingaRules } from './filebeat_icinga';
 import { filebeatIisRules } from './filebeat_iis';
+import { filebeatLogstashRules } from './filebeat_logstash';
 import { filebeatMongodbRules } from './filebeat_mongodb';
 import { filebeatMySQLRules } from './filebeat_mysql';
 import { filebeatNginxRules } from './filebeat_nginx';
@@ -27,6 +28,7 @@ export const builtinRules = [
   ...filebeatHaproxyRules,
   ...filebeatIcingaRules,
   ...filebeatIisRules,
+  ...filebeatLogstashRules,
   ...filebeatMongodbRules,
   ...genericRules,
   {

From 268ea338c376412f91eeb067faabe674437bcb7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20St=C3=BCrmer?= <stuermer@weltenwort.de>
Date: Fri, 8 Feb 2019 17:32:23 +0100
Subject: [PATCH 6/7] Add osquery filebeat log message rules

---
 .../log_text_stream/item_message_field.tsx    |  1 +
 .../builtin_rules/filebeat_osquery.test.ts    | 77 +++++++++++++++++++
 .../builtin_rules/filebeat_osquery.ts         | 34 ++++++++
 .../log_entries_domain/builtin_rules/index.ts |  2 +
 .../lib/domains/log_entries_domain/message.ts | 21 +++--
 5 files changed, 127 insertions(+), 8 deletions(-)
 create mode 100644 x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.test.ts
 create mode 100644 x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.ts

diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/item_message_field.tsx b/x-pack/plugins/infra/public/components/logging/log_text_stream/item_message_field.tsx
index 39f75d5b7ab64..8d5ea413dffa9 100644
--- a/x-pack/plugins/infra/public/components/logging/log_text_stream/item_message_field.tsx
+++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/item_message_field.tsx
@@ -82,6 +82,7 @@ const hoveredFieldStyle = css`
 const wrappedFieldStyle = css`
   overflow: visible;
   white-space: pre-wrap;
+  word-break: break-all;
 `;
 
 const unwrappedFieldStyle = css`
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.test.ts
new file mode 100644
index 0000000000000..45726285f0c29
--- /dev/null
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.test.ts
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { compileFormattingRules } from '../message';
+import { filebeatOsqueryRules } from './filebeat_osquery';
+
+const { format } = compileFormattingRules(filebeatOsqueryRules);
+
+describe('Filebeat Rules', () => {
+  describe('in pre-ECS format', () => {
+    test('osquery result log', () => {
+      const flattenedDocument = {
+        '@timestamp': '2017-12-28T14:40:08.000Z',
+        'event.dataset': 'osquery.result',
+        'fileset.module': 'osquery',
+        'fileset.name': 'result',
+        'input.type': 'log',
+        offset: 0,
+        'osquery.result.action': 'removed',
+        'osquery.result.calendar_time': 'Thu Dec 28 14:40:08 2017 UTC',
+        'osquery.result.columns': {
+          blocks: '122061322',
+          blocks_available: '75966945',
+          blocks_free: '121274885',
+          blocks_size: '4096',
+          device: '/dev/disk1s4',
+          device_alias: '/dev/disk1s4',
+          flags: '345018372',
+          inodes: '9223372036854775807',
+          inodes_free: '9223372036854775804',
+          path: '/private/var/vm',
+          type: 'apfs',
+        },
+        'osquery.result.counter': '1',
+        'osquery.result.decorations.host_uuid': '4AB2906D-5516-5794-AF54-86D1D7F533F3',
+        'osquery.result.decorations.username': 'tsg',
+        'osquery.result.epoch': '0',
+        'osquery.result.host_identifier': '192-168-0-4.rdsnet.ro',
+        'osquery.result.name': 'pack_it-compliance_mounts',
+        'osquery.result.unix_time': '1514472008',
+        'prospector.type': 'log',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[Osquery][",
+  },
+  Object {
+    "field": "osquery.result.action",
+    "highlights": Array [],
+    "value": "removed",
+  },
+  Object {
+    "constant": "] ",
+  },
+  Object {
+    "field": "osquery.result.host_identifier",
+    "highlights": Array [],
+    "value": "192-168-0-4.rdsnet.ro",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "osquery.result.columns",
+    "highlights": Array [],
+    "value": "{\\"blocks\\":\\"122061322\\",\\"blocks_available\\":\\"75966945\\",\\"blocks_free\\":\\"121274885\\",\\"blocks_size\\":\\"4096\\",\\"device\\":\\"/dev/disk1s4\\",\\"device_alias\\":\\"/dev/disk1s4\\",\\"flags\\":\\"345018372\\",\\"inodes\\":\\"9223372036854775807\\",\\"inodes_free\\":\\"9223372036854775804\\",\\"path\\":\\"/private/var/vm\\",\\"type\\":\\"apfs\\"}",
+  },
+]
+`);
+    });
+  });
+});
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.ts
new file mode 100644
index 0000000000000..b3a6ee8c5cb47
--- /dev/null
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.ts
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const filebeatOsqueryRules = [
+  {
+    // pre-ECS
+    when: {
+      exists: ['osquery.result.name'],
+    },
+    format: [
+      {
+        constant: '[Osquery][',
+      },
+      {
+        field: 'osquery.result.action',
+      },
+      {
+        constant: '] ',
+      },
+      {
+        field: 'osquery.result.host_identifier',
+      },
+      {
+        constant: ' ',
+      },
+      {
+        field: 'osquery.result.columns',
+      },
+    ],
+  },
+];
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
index 2bd0d93866169..6e3e8fcfcaab2 100644
--- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
@@ -13,6 +13,7 @@ import { filebeatLogstashRules } from './filebeat_logstash';
 import { filebeatMongodbRules } from './filebeat_mongodb';
 import { filebeatMySQLRules } from './filebeat_mysql';
 import { filebeatNginxRules } from './filebeat_nginx';
+import { filebeatOsqueryRules } from './filebeat_osquery';
 import { filebeatRedisRules } from './filebeat_redis';
 import { filebeatSystemRules } from './filebeat_system';
 
@@ -30,6 +31,7 @@ export const builtinRules = [
   ...filebeatIisRules,
   ...filebeatLogstashRules,
   ...filebeatMongodbRules,
+  ...filebeatOsqueryRules,
   ...genericRules,
   {
     when: {
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/message.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/message.ts
index 1a96f4c17e918..0186e11245294 100644
--- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/message.ts
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/message.ts
@@ -4,6 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import stringify from 'json-stable-stringify';
+
 import { InfraLogMessageSegment } from '../../../graphql/types';
 
 export function compileFormattingRules(rules: LogMessageFormattingRule[]) {
@@ -125,13 +127,16 @@ const compileFieldReferenceFormattingInstruction = (
   'field' in formattingInstruction
     ? {
         formattingFields: [formattingInstruction.field],
-        format: (fields: Fields) => [
-          {
-            field: formattingInstruction.field,
-            value: `${fields[formattingInstruction.field]}`,
-            highlights: [],
-          },
-        ],
+        format: (fields: Fields) => {
+          const value = fields[formattingInstruction.field];
+          return [
+            {
+              field: formattingInstruction.field,
+              value: typeof value === 'object' ? stringify(value) : `${value}`,
+              highlights: [],
+            },
+          ];
+        },
       }
     : null;
 
@@ -150,7 +155,7 @@ const compileConstantFormattingInstruction = (
     : null;
 
 interface Fields {
-  [fieldName: string]: string | number | boolean | null;
+  [fieldName: string]: string | number | object | boolean | null;
 }
 
 interface LogMessageFormattingRule {

From a0e1bb33b57f3a9212d01cf45d0434863a4ae3ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20St=C3=BCrmer?= <stuermer@weltenwort.de>
Date: Mon, 11 Feb 2019 12:17:36 +0100
Subject: [PATCH 7/7] Add traefik filebeat log message rules

---
 .../builtin_rules/filebeat_traefik.test.ts    | 124 ++++++++++++++++++
 .../builtin_rules/filebeat_traefik.ts         |  64 +++++++++
 .../log_entries_domain/builtin_rules/index.ts |   2 +
 3 files changed, 190 insertions(+)
 create mode 100644 x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.test.ts
 create mode 100644 x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.ts

diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.test.ts
new file mode 100644
index 0000000000000..c8a13b54dfae9
--- /dev/null
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.test.ts
@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { compileFormattingRules } from '../message';
+import { filebeatTraefikRules } from './filebeat_traefik';
+
+const { format } = compileFormattingRules(filebeatTraefikRules);
+
+describe('Filebeat Rules', () => {
+  describe('in pre-ECS format', () => {
+    test('traefik access log', () => {
+      const flattenedDocument = {
+        '@timestamp': '2017-10-02T20:22:08.000Z',
+        'event.dataset': 'traefik.access',
+        'fileset.module': 'traefik',
+        'fileset.name': 'access',
+        'input.type': 'log',
+        offset: 280,
+        'prospector.type': 'log',
+        'traefik.access.backend_url': 'http://172.19.0.3:5601',
+        'traefik.access.body_sent.bytes': 0,
+        'traefik.access.duration': 3,
+        'traefik.access.frontend_name': 'Host-host1',
+        'traefik.access.geoip.city_name': 'Berlin',
+        'traefik.access.geoip.continent_name': 'Europe',
+        'traefik.access.geoip.country_iso_code': 'DE',
+        'traefik.access.geoip.location.lat': 52.4908,
+        'traefik.access.geoip.location.lon': 13.3275,
+        'traefik.access.geoip.region_iso_code': 'DE-BE',
+        'traefik.access.geoip.region_name': 'Land Berlin',
+        'traefik.access.http_version': '1.1',
+        'traefik.access.method': 'GET',
+        'traefik.access.referrer': 'http://example.com/login',
+        'traefik.access.remote_ip': '85.181.35.98',
+        'traefik.access.request_count': 271,
+        'traefik.access.response_code': '304',
+        'traefik.access.url': '/ui/favicons/favicon.ico',
+        'traefik.access.user_agent.device': 'Other',
+        'traefik.access.user_agent.major': '61',
+        'traefik.access.user_agent.minor': '0',
+        'traefik.access.user_agent.name': 'Chrome',
+        'traefik.access.user_agent.original':
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
+        'traefik.access.user_agent.os': 'Linux',
+        'traefik.access.user_agent.os_name': 'Linux',
+        'traefik.access.user_agent.patch': '3163',
+        'traefik.access.user_identifier': '-',
+        'traefik.access.user_name': '-',
+      };
+
+      expect(format(flattenedDocument)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "constant": "[Traefik][access] ",
+  },
+  Object {
+    "field": "traefik.access.remote_ip",
+    "highlights": Array [],
+    "value": "85.181.35.98",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "traefik.access.frontend_name",
+    "highlights": Array [],
+    "value": "Host-host1",
+  },
+  Object {
+    "constant": " -> ",
+  },
+  Object {
+    "field": "traefik.access.backend_url",
+    "highlights": Array [],
+    "value": "http://172.19.0.3:5601",
+  },
+  Object {
+    "constant": " \\"",
+  },
+  Object {
+    "field": "traefik.access.method",
+    "highlights": Array [],
+    "value": "GET",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "traefik.access.url",
+    "highlights": Array [],
+    "value": "/ui/favicons/favicon.ico",
+  },
+  Object {
+    "constant": " HTTP/",
+  },
+  Object {
+    "field": "traefik.access.http_version",
+    "highlights": Array [],
+    "value": "1.1",
+  },
+  Object {
+    "constant": "\\" ",
+  },
+  Object {
+    "field": "traefik.access.response_code",
+    "highlights": Array [],
+    "value": "304",
+  },
+  Object {
+    "constant": " ",
+  },
+  Object {
+    "field": "traefik.access.body_sent.bytes",
+    "highlights": Array [],
+    "value": "0",
+  },
+]
+`);
+    });
+  });
+});
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.ts
new file mode 100644
index 0000000000000..9b0244900364a
--- /dev/null
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.ts
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const filebeatTraefikRules = [
+  {
+    // pre-ECS
+    when: {
+      exists: ['traefik.access.method'],
+    },
+    format: [
+      {
+        constant: '[Traefik][access] ',
+      },
+      {
+        field: 'traefik.access.remote_ip',
+      },
+      {
+        constant: ' ',
+      },
+      {
+        field: 'traefik.access.frontend_name',
+      },
+      {
+        constant: ' -> ',
+      },
+      {
+        field: 'traefik.access.backend_url',
+      },
+      {
+        constant: ' "',
+      },
+      {
+        field: 'traefik.access.method',
+      },
+      {
+        constant: ' ',
+      },
+      {
+        field: 'traefik.access.url',
+      },
+      {
+        constant: ' HTTP/',
+      },
+      {
+        field: 'traefik.access.http_version',
+      },
+      {
+        constant: '" ',
+      },
+      {
+        field: 'traefik.access.response_code',
+      },
+      {
+        constant: ' ',
+      },
+      {
+        field: 'traefik.access.body_sent.bytes',
+      },
+    ],
+  },
+];
diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
index 6e3e8fcfcaab2..cc0cf96cf1ea6 100644
--- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
+++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
@@ -16,6 +16,7 @@ import { filebeatNginxRules } from './filebeat_nginx';
 import { filebeatOsqueryRules } from './filebeat_osquery';
 import { filebeatRedisRules } from './filebeat_redis';
 import { filebeatSystemRules } from './filebeat_system';
+import { filebeatTraefikRules } from './filebeat_traefik';
 
 import { genericRules } from './generic';
 
@@ -32,6 +33,7 @@ export const builtinRules = [
   ...filebeatLogstashRules,
   ...filebeatMongodbRules,
   ...filebeatOsqueryRules,
+  ...filebeatTraefikRules,
   ...genericRules,
   {
     when: {