[iptables,journald] Re-enable system tests for journald inputs

packages/iptables/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.6.0"
+  changes:
+    - description: Align journald syslog fields with ECS.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/5984
 - version: "1.5.0"
   changes:
     - description: Update package to ECS 8.7.0.

packages/iptables/data_stream/log/_dev/test/pipeline/test-journald.json-expected.json

@@ -71,10 +71,10 @@
             },
             "log": {
                 "syslog": {
+                    "appname": "kernel",
                     "facility": {
                         "code": 0
                     },
-                    "identifier": "kernel",
                     "priority": 7
                 }
             },

packages/iptables/data_stream/log/_dev/test/system/test-journald-config.yml

@@ -1,9 +1,6 @@
 service: iptables-log-journald
-skip:
-  reason: "A bug on the host journald causes our journald input to panic"
-  link: https://github.com/elastic/integrations/issues/2602
 input: journald
 data_stream:
   vars:
     paths:
-      - "{{SERVICE_LOGS_DIR}}/iptables.journal"
+      - "/run/service_logs/iptables.journal"

packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -23,11 +23,11 @@ processors:
       ignore_missing: true
   - rename:
       field: syslog.pid
-      target_field: log.syslog.pid
+      target_field: log.syslog.procid
       ignore_missing: true
   - rename:
       field: syslog.identifier
-      target_field: log.syslog.identifier
+      target_field: log.syslog.appname
       ignore_missing: true
   - remove:
       description: Cleanup an empty syslog object.

packages/iptables/data_stream/log/fields/ecs.yml

@@ -32,10 +32,14 @@
   name: event.outcome
 - external: ecs
   name: log.file.path
+- external: ecs
+  name: log.syslog.appname
 - external: ecs
   name: log.syslog.facility.code
 - external: ecs
   name: log.syslog.priority
+- external: ecs
+  name: log.syslog.procid
 - external: ecs
   name: message
 - external: ecs

packages/iptables/data_stream/log/fields/journald-input.yml

@@ -5,15 +5,5 @@
 
 - name: systemd.transport
   type: keyword
-  description: >
-    How the entry was received by the journal service.
-
-- name: log.syslog.identifier
-  type: keyword
-  description: >
-    Identifier (usually process) contained in the syslog header.
-
-- name: log.syslog.pid
-  type: long
   description: >-
-    PID contained in the syslog header.
+    How the entry was received by the journal service.

packages/iptables/data_stream/log/sample_event.json

@@ -1,11 +1,11 @@
 {
     "@timestamp": "2021-03-12T14:10:18.000Z",
     "agent": {
-        "ephemeral_id": "fe763653-ca99-4a13-b01e-f49e33946306",
-        "id": "660f37cf-e109-4766-b85b-8150ca4cd173",
+        "ephemeral_id": "9d70b3da-b816-48af-9c86-8e6c6a5bf0fb",
+        "id": "4e644293-3984-48e7-a63c-00be2338b58d",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.4.1"
+        "version": "8.8.0"
     },
     "data_stream": {
         "dataset": "iptables.log",
@@ -21,19 +21,19 @@
         "version": "8.7.0"
     },
     "elastic_agent": {
-        "id": "660f37cf-e109-4766-b85b-8150ca4cd173",
-        "snapshot": false,
-        "version": "8.4.1"
+        "id": "4e644293-3984-48e7-a63c-00be2338b58d",
+        "snapshot": true,
+        "version": "8.8.0"
     },
     "event": {
         "action": "drop",
         "agent_id_status": "verified",
         "category": [
             "network"
         ],
-        "created": "2022-10-20T04:11:20.974Z",
+        "created": "2023-04-25T19:13:39.793Z",
         "dataset": "iptables.log",
-        "ingested": "2022-10-20T04:11:22Z",
+        "ingested": "2023-04-25T19:13:40Z",
         "kind": "event",
         "timezone": "+00:00",
         "type": [
@@ -72,7 +72,7 @@
     },
     "log": {
         "source": {
-            "address": "172.18.0.4:54943"
+            "address": "172.18.0.5:39990"
         },
         "syslog": {
             "priority": 6

packages/iptables/docs/README.md

@@ -21,11 +21,11 @@ An example event for `log` looks as following:
 {
     "@timestamp": "2021-03-12T14:10:18.000Z",
     "agent": {
-        "ephemeral_id": "fe763653-ca99-4a13-b01e-f49e33946306",
-        "id": "660f37cf-e109-4766-b85b-8150ca4cd173",
+        "ephemeral_id": "9d70b3da-b816-48af-9c86-8e6c6a5bf0fb",
+        "id": "4e644293-3984-48e7-a63c-00be2338b58d",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.4.1"
+        "version": "8.8.0"
     },
     "data_stream": {
         "dataset": "iptables.log",
@@ -41,19 +41,19 @@ An example event for `log` looks as following:
         "version": "8.7.0"
     },
     "elastic_agent": {
-        "id": "660f37cf-e109-4766-b85b-8150ca4cd173",
-        "snapshot": false,
-        "version": "8.4.1"
+        "id": "4e644293-3984-48e7-a63c-00be2338b58d",
+        "snapshot": true,
+        "version": "8.8.0"
     },
     "event": {
         "action": "drop",
         "agent_id_status": "verified",
         "category": [
             "network"
         ],
-        "created": "2022-10-20T04:11:20.974Z",
+        "created": "2023-04-25T19:13:39.793Z",
         "dataset": "iptables.log",
-        "ingested": "2022-10-20T04:11:22Z",
+        "ingested": "2023-04-25T19:13:40Z",
         "kind": "event",
         "timezone": "+00:00",
         "type": [
@@ -92,7 +92,7 @@ An example event for `log` looks as following:
     },
     "log": {
         "source": {
-            "address": "172.18.0.4:54943"
+            "address": "172.18.0.5:39990"
         },
         "syslog": {
             "priority": 6
@@ -238,10 +238,10 @@ An example event for `log` looks as following:
 | log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword |
 | log.offset | Log offset | long |
 | log.source.address | Source address of the syslog message. | keyword |
+| log.syslog.appname | The device or application that originated the Syslog message, if available. | keyword |
 | log.syslog.facility.code | The Syslog numeric facility of the log event, if available. According to RFCs 5424 and 3164, this value should be an integer between 0 and 23. | long |
-| log.syslog.identifier | Identifier (usually process) contained in the syslog header. | keyword |
-| log.syslog.pid | PID contained in the syslog header. | long |
 | log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long |
+| log.syslog.procid | The process name or ID that originated the Syslog message, if available. | keyword |
 | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text |
 | network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword |
 | network.forwarded_ip | Host IP address when the source IP address is the proxy. | ip |

packages/iptables/manifest.yml

@@ -1,6 +1,6 @@
 name: iptables
 title: Iptables
-version: "1.5.0"
+version: "1.6.0"
 release: ga
 description: Collect logs from Iptables with Elastic Agent.
 type: integration
@@ -15,7 +15,7 @@ categories:
   - network
   - security
 conditions:
-  kibana.version: ^8.1.0
+  kibana.version: ^8.7.0
 screenshots:
   - src: /img/kibana-iptables.png
     title: kibana iptables

packages/journald/_dev/deploy/docker/Dockerfile

@@ -0,0 +1,5 @@
+FROM debian:stable-slim
+
+RUN apt-get update \
+      && apt install -y systemd-journal-remote \
+      && rm -rf /var/lib/apt/lists/*

packages/journald/_dev/deploy/docker/docker-compose.yml

@@ -1,8 +1,11 @@
 version: '2.3'
 services:
   journald:
-    image: alpine
+    build: .
     volumes:
       - ./sample_logs:/sample_logs:ro
       - ${SERVICE_LOGS_DIR}:/var/log
-    command: /bin/sh -c "cp /sample_logs/* /var/log/"
+    # Use journalctl -o export > test.journal.export
+    # to write logs to journald export format. Then this creates a new binary journal
+    # file from those logs to use in testing.
+    command: /bin/sh -c "/lib/systemd/systemd-journal-remote -o /var/log/test.journal /sample_logs/*.export"