Add system test for iptables syslog input (elastic#587)

Add system test for the syslog input (udp) in the iptables log data stream.
Pipeline tests and a system test for the logfile input already existed.

Fixes

- Handle missing iptables.raw_date when beats syslog input is used.
- Add missing fields used by the beats syslog input.

packages/iptables/_dev/deploy/docker/docker-compose.yml

@@ -1,8 +1,13 @@
 version: '2.3'
 services:
-  iptables:
-    tty: true
-    build: .
+  iptables-logfile:
+    image: alpine
     volumes:
+      - ./sample_logs:/sample_logs:ro
       - ${SERVICE_LOGS_DIR}:/var/log
-    command: -c "cp /iptables.log /var/log"
+    command: /bin/sh -c "cp /sample_logs/* /var/log/"
+  iptables-log-syslog:
+    image: akroh/stream:v0.0.1
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9514 -p=udp /sample_logs/iptables.log

packages/iptables/data_stream/log/_dev/test/system/test-default-config.yml → packages/iptables/data_stream/log/_dev/test/system/test-logfile-config.yml

@@ -1,5 +1,5 @@
+service: iptables-logfile
 input: logfile
-vars: ~
 data_stream:
   vars:
     paths:

packages/iptables/data_stream/log/_dev/test/system/test-syslog-config.yml

@@ -0,0 +1,7 @@
+service: iptables-log-syslog
+service_notify_signal: SIGHUP
+input: syslog
+data_stream:
+  vars:
+    syslog_host: 0.0.0.0
+    syslog_port: 9514

packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -60,7 +60,7 @@ processors:
     pattern_definitions:
       UBIQUITI_FIELD: '[^-]*'
 - date:
-    if: ctx.event.timezone == null
+    if: ctx?.event?.timezone == null && ctx?.iptables?.raw_date != null
     field: iptables.raw_date
     formats:
     - MMM  d HH:mm:ss
@@ -70,7 +70,7 @@ processors:
         field: error.message
         value: '{{ _ingest.on_failure_message }}'
 - date:
-    if: ctx.event.timezone != null
+    if: ctx?.event?.timezone != null && ctx?.iptables?.raw_date != null
     field: iptables.raw_date
     formats:
     - MMM  d HH:mm:ss

packages/iptables/data_stream/log/fields/agent.yml

@@ -196,3 +196,12 @@
       description: >
         OS codename, if any.
 
+- name: hostname
+  type: keyword
+  description: Hostname from syslog header.
+- name: log.source.address
+  type: keyword
+  description: Source address of the syslog message.
+- name: process.program
+  type: keyword
+  description: Process from syslog header.

packages/iptables/docs/README.md

@@ -165,6 +165,7 @@ An example event for `log` looks as following:
 | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword |
 | host.os.version | Operating system version as a raw string. | keyword |
 | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |
+| hostname | Hostname from syslog header. | keyword |
 | input.type | Input type | keyword |
 | iptables.ether_type | Value of the ethernet type field identifying the network layer protocol. | long |
 | iptables.flow_label | IPv6 flow label. | integer |
@@ -197,12 +198,14 @@ An example event for `log` looks as following:
 | log.file.path | Full path to the log file this event came from, including the file name. | keyword |
 | log.offset | Log offset | long |
 | log.original | This is the original log message and contains the full log message before splitting it up in multiple parts. | keyword |
+| log.source.address | Source address of the syslog message. | keyword |
 | network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. | keyword |
 | network.forwarded_ip | Host IP address when the source IP address is the proxy. | ip |
 | network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) | keyword |
 | network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc | keyword |
 | observer.egress.zone | Network zone of outbound traffic as reported by the observer to categorize the destination area of egress  traffic, e.g. Internal, External, DMZ, HR, Legal, etc. | keyword |
 | observer.ingress.zone | Network zone of incoming traffic as reported by the observer to categorize the source area of ingress  traffic. e.g. internal, External, DMZ, HR, Legal, etc. | keyword |
+| process.program | Process from syslog header. | keyword |
 | related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword |
 | related.ip | All of the IPs seen on your event. | ip |
 | rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword |

packages/iptables/manifest.yml

@@ -1,6 +1,6 @@
 name: iptables
 title: Iptables
-version: 0.0.1
+version: 0.0.2
 release: experimental
 description: Iptables Integration
 type: integration