[iptables,journald] Re-enable system tests for journald inputs (#5984)

system tests for the journald input have been disabled to a segfault. This uses a workaround
to avoid that segfault so we can continue testing.

While performing that testing I discovered that neither iptables nor journald were
aligned with the current ECS definition of the log.syslog.* fields. ECS added
numerous log.syslog fields that should be used by journald/iptables
instead of syslog.*.

And because journald is an input package this needs to be done without
an Ingest Pipeline so that users with custom pipelines can benefit.

Bump stack version for the iptables integration to get journald input fixes.

Closes #2602
Relates elastic/elastic-package#1236

packages/iptables/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.6.0"
+  changes:
+    - description: Align journald syslog fields with ECS.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/5984
 - version: "1.5.0"
   changes:
     - description: Update package to ECS 8.7.0.

packages/iptables/data_stream/log/_dev/test/pipeline/test-journald.json-expected.json

@@ -71,10 +71,10 @@
             },
             "log": {
                 "syslog": {
+                    "appname": "kernel",
                     "facility": {
                         "code": 0
                     },
-                    "identifier": "kernel",
                     "priority": 7
                 }
             },

packages/iptables/data_stream/log/_dev/test/system/test-journald-config.yml

@@ -1,9 +1,6 @@
 service: iptables-log-journald
-skip:
-  reason: "A bug on the host journald causes our journald input to panic"
-  link: https://github.com/elastic/integrations/issues/2602
 input: journald
 data_stream:
   vars:
     paths:
-      - "{{SERVICE_LOGS_DIR}}/iptables.journal"
+      - "/run/service_logs/iptables.journal"

packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -23,11 +23,11 @@ processors:
       ignore_missing: true
   - rename:
       field: syslog.pid
-      target_field: log.syslog.pid
+      target_field: log.syslog.procid
       ignore_missing: true
   - rename:
       field: syslog.identifier
-      target_field: log.syslog.identifier
+      target_field: log.syslog.appname
       ignore_missing: true
   - remove:
       description: Cleanup an empty syslog object.

packages/iptables/data_stream/log/fields/ecs.yml

@@ -32,10 +32,14 @@
   name: event.outcome
 - external: ecs
   name: log.file.path
+- external: ecs
+  name: log.syslog.appname
 - external: ecs
   name: log.syslog.facility.code
 - external: ecs
   name: log.syslog.priority
+- external: ecs
+  name: log.syslog.procid
 - external: ecs
   name: message
 - external: ecs

packages/iptables/data_stream/log/fields/journald-input.yml

@@ -5,15 +5,5 @@
 
 - name: systemd.transport
   type: keyword
-  description: >
-    How the entry was received by the journal service.
-
-- name: log.syslog.identifier
-  type: keyword
-  description: >
-    Identifier (usually process) contained in the syslog header.
-
-- name: log.syslog.pid
-  type: long
   description: >-
-    PID contained in the syslog header.
+    How the entry was received by the journal service.

packages/iptables/data_stream/log/sample_event.json

@@ -1,11 +1,11 @@
 {
     "@timestamp": "2021-03-12T14:10:18.000Z",
     "agent": {
-        "ephemeral_id": "fe763653-ca99-4a13-b01e-f49e33946306",
-        "id": "660f37cf-e109-4766-b85b-8150ca4cd173",
+        "ephemeral_id": "9d70b3da-b816-48af-9c86-8e6c6a5bf0fb",
+        "id": "4e644293-3984-48e7-a63c-00be2338b58d",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.4.1"
+        "version": "8.8.0"
     },
     "data_stream": {
         "dataset": "iptables.log",
@@ -21,19 +21,19 @@
         "version": "8.7.0"
     },
     "elastic_agent": {
-        "id": "660f37cf-e109-4766-b85b-8150ca4cd173",
-        "snapshot": false,
-        "version": "8.4.1"
+        "id": "4e644293-3984-48e7-a63c-00be2338b58d",
+        "snapshot": true,
+        "version": "8.8.0"
     },
     "event": {
         "action": "drop",
         "agent_id_status": "verified",
         "category": [
             "network"
         ],
-        "created": "2022-10-20T04:11:20.974Z",
+        "created": "2023-04-25T19:13:39.793Z",
         "dataset": "iptables.log",
-        "ingested": "2022-10-20T04:11:22Z",
+        "ingested": "2023-04-25T19:13:40Z",
         "kind": "event",
         "timezone": "+00:00",
         "type": [
@@ -72,7 +72,7 @@
     },
     "log": {
         "source": {
-            "address": "172.18.0.4:54943"
+            "address": "172.18.0.5:39990"
         },
         "syslog": {
             "priority": 6

packages/iptables/docs/README.md

@@ -21,11 +21,11 @@ An example event for `log` looks as following:
 {
     "@timestamp": "2021-03-12T14:10:18.000Z",
     "agent": {
-        "ephemeral_id": "fe763653-ca99-4a13-b01e-f49e33946306",
-        "id": "660f37cf-e109-4766-b85b-8150ca4cd173",
+        "ephemeral_id": "9d70b3da-b816-48af-9c86-8e6c6a5bf0fb",
+        "id": "4e644293-3984-48e7-a63c-00be2338b58d",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.4.1"
+        "version": "8.8.0"
     },
     "data_stream": {
         "dataset": "iptables.log",
@@ -41,19 +41,19 @@ An example event for `log` looks as following:
         "version": "8.7.0"
     },
     "elastic_agent": {
-        "id": "660f37cf-e109-4766-b85b-8150ca4cd173",
-        "snapshot": false,
-        "version": "8.4.1"
+        "id": "4e644293-3984-48e7-a63c-00be2338b58d",
+        "snapshot": true,
+        "version": "8.8.0"
     },
     "event": {
         "action": "drop",
         "agent_id_status": "verified",
         "category": [
             "network"
         ],
-        "created": "2022-10-20T04:11:20.974Z",
+        "created": "2023-04-25T19:13:39.793Z",
         "dataset": "iptables.log",
-        "ingested": "2022-10-20T04:11:22Z",
+        "ingested": "2023-04-25T19:13:40Z",
         "kind": "event",
         "timezone": "+00:00",
         "type": [
@@ -92,7 +92,7 @@ An example event for `log` looks as following:
     },
     "log": {
         "source": {
-            "address": "172.18.0.4:54943"
+            "address": "172.18.0.5:39990"
         },
         "syslog": {
             "priority": 6
@@ -238,10 +238,10 @@ An example event for `log` looks as following:
 | log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword |
 | log.offset | Log offset | long |
 | log.source.address | Source address of the syslog message. | keyword |
+| log.syslog.appname | The device or application that originated the Syslog message, if available. | keyword |
 | log.syslog.facility.code | The Syslog numeric facility of the log event, if available. According to RFCs 5424 and 3164, this value should be an integer between 0 and 23. | long |
-| log.syslog.identifier | Identifier (usually process) contained in the syslog header. | keyword |
-| log.syslog.pid | PID contained in the syslog header. | long |
 | log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long |
+| log.syslog.procid | The process name or ID that originated the Syslog message, if available. | keyword |
 | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text |
 | network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword |
 | network.forwarded_ip | Host IP address when the source IP address is the proxy. | ip |

packages/iptables/manifest.yml

@@ -1,6 +1,6 @@
 name: iptables
 title: Iptables
-version: "1.5.0"
+version: "1.6.0"
 release: ga
 description: Collect logs from Iptables with Elastic Agent.
 type: integration
@@ -15,7 +15,7 @@ categories:
   - network
   - security
 conditions:
-  kibana.version: ^8.1.0
+  kibana.version: ^8.7.0
 screenshots:
   - src: /img/kibana-iptables.png
     title: kibana iptables

packages/journald/_dev/deploy/docker/Dockerfile

@@ -0,0 +1,5 @@
+FROM debian:stable-slim
+
+RUN apt-get update \
+      && apt install -y systemd-journal-remote \
+      && rm -rf /var/lib/apt/lists/*

packages/journald/_dev/deploy/docker/docker-compose.yml

@@ -1,8 +1,11 @@
 version: '2.3'
 services:
   journald:
-    image: alpine
+    build: .
     volumes:
       - ./sample_logs:/sample_logs:ro
       - ${SERVICE_LOGS_DIR}:/var/log
-    command: /bin/sh -c "cp /sample_logs/* /var/log/"
+    # Use journalctl -o export > test.journal.export
+    # to write logs to journald export format. Then this creates a new binary journal
+    # file from those logs to use in testing.
+    command: /bin/sh -c "/lib/systemd/systemd-journal-remote -o /var/log/test.journal /sample_logs/*.export"