Fix "source" field for beats version >= 8 (#13895) (#13897)

* Fix "source" field for beats version >= 8 In beats v8 the "agent.hostname" field got renamed to "agent.name". Fixes #13254 * Add changelog entry (cherry picked from commit 63bc4fe)
Graylog2 · Nov 14, 2022 · 2ce300b · 2ce300b
1 parent 4ad8f5d
commit 2ce300b
Show file tree

Hide file tree

Showing 4 changed files with 109 additions and 1 deletion.
diff --git a/changelog/unreleased/issue-13254.toml b/changelog/unreleased/issue-13254.toml
@@ -0,0 +1,5 @@
+type = "fixed"
+message = "Fix `source` field extraction for Beats version 8 and later."
+
+issues = ["13254"]
+pulls = ["13895"]
diff --git a/graylog2-server/src/main/java/org/graylog/plugins/beats/Beats2Codec.java b/graylog2-server/src/main/java/org/graylog/plugins/beats/Beats2Codec.java
@@ -93,7 +93,14 @@ private Message parseEvent(JsonNode event) {
         if (agentOrBeat.isMissingNode()) {
             agentOrBeat = event.path("beat");
         }
-        final String hostname = agentOrBeat.path("hostname").asText(BEATS_UNKNOWN);
+
+        JsonNode agentName = agentOrBeat.path("hostname");
+        if (agentName.isMissingNode()) {
+            // Compatibility for beats >= 8.0
+            agentName = agentOrBeat.path("name");
+        }
+
+        final String hostname = agentName.asText(BEATS_UNKNOWN);
 
         final Message gelfMessage = new Message(message, hostname, timestamp);
         gelfMessage.addField("beats_type", beatsType);

diff --git a/graylog2-server/src/test/java/org/graylog/plugins/beats/Beats2CodecTest.java b/graylog2-server/src/test/java/org/graylog/plugins/beats/Beats2CodecTest.java
@@ -109,6 +109,24 @@ public void decodeMessagesHandlesPacketbeatMessages() throws Exception {
         assertThat(message.getField("packetbeat_dns_flags_recursion_allowed")).isEqualTo(true);
     }
 
+    @Test
+    public void decodeMessagesHandlesPacketbeatV8Messages() throws Exception {
+        final Message message = codec.decode(messageFromJson("packetbeat-mongodb-v8.json"));
+        assertThat(message).isNotNull();
+        assertThat(message.getSource()).isEqualTo("example.local");
+        assertThat(message.getTimestamp()).isEqualTo(new DateTime(2022, 11, 7, 9, 26, 10, 579, DateTimeZone.UTC));
+        assertThat(message.getField("beats_type")).isEqualTo("packetbeat");
+        assertThat(message.getField("packetbeat_type")).isEqualTo("mongodb");
+        assertThat(message.getField("packetbeat_status")).isEqualTo("OK");
+        assertThat(message.getField("packetbeat_method")).isEqualTo("msg");
+        assertThat(message.getField("packetbeat_network_bytes")).isEqualTo(557);
+        assertThat(message.getField("packetbeat_network_type")).isEqualTo("ipv4");
+        assertThat(message.getField("packetbeat_source_ip")).isEqualTo("10.0.55.1");
+        assertThat(message.getField("packetbeat_destination_ip")).isEqualTo("10.0.55.2");
+        assertThat(message.getField("packetbeat_destination_port")).isEqualTo(27017);
+        assertThat(message.getField("packetbeat_host_containerized")).isEqualTo(false);
+    }
+
     @Test
     public void decodeMessagesHandlesTopbeatMessages() throws Exception {
         final Message message = codec.decode(messageFromJson("topbeat-system.json"));

diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/beats/packetbeat-mongodb-v8.json b/graylog2-server/src/test/resources/org/graylog/plugins/beats/packetbeat-mongodb-v8.json
@@ -0,0 +1,78 @@
+{
+  "@timestamp" : "2022-11-07T09:26:10.579Z",
+  "@metadata" : {
+    "beat" : "packetbeat",
+    "type" : "_doc",
+    "version" : "8.5.0"
+  },
+  "related" : {
+    "ip" : [ "10.0.55.1", "10.0.55.2" ]
+  },
+  "status" : "OK",
+  "method" : "msg",
+  "mongodb" : { },
+  "event" : {
+    "kind" : "event",
+    "category" : [ "network" ],
+    "type" : [ "connection", "protocol" ],
+    "dataset" : "mongodb",
+    "start" : "2022-11-07T09:26:10.579Z"
+  },
+  "host" : {
+    "architecture" : "x86_64",
+    "os" : {
+      "codename" : "focal",
+      "type" : "linux",
+      "platform" : "ubuntu",
+      "version" : "20.04.5 LTS (Focal Fossa)",
+      "family" : "debian",
+      "name" : "Ubuntu",
+      "kernel" : "5.15.0-52-generic"
+    },
+    "id" : "3d758250c1e88241a6d2037786a23bcf",
+    "name" : "example.local",
+    "containerized" : false,
+    "ip" : [ "10.0.1.1", "10.0.55.1" ],
+    "mac" : [ "02-42-11-7A-FD-80", "02-42-77-6C-EC-7E" ],
+    "hostname" : "example.local"
+  },
+  "type" : "mongodb",
+  "network" : {
+    "community_id" : "1:NlfSWdQ8jRZXj4S/I200ox1MJQc=",
+    "bytes" : 557,
+    "type" : "ipv4",
+    "transport" : "tcp",
+    "protocol" : "mongodb",
+    "direction" : "egress"
+  },
+  "query" : ".msg()",
+  "source" : {
+    "ip" : "10.0.55.1",
+    "port" : 41076,
+    "bytes" : 557
+  },
+  "resource" : "",
+  "agent" : {
+    "name" : "example.local",
+    "type" : "packetbeat",
+    "version" : "8.5.0",
+    "ephemeral_id" : "ca453a1e-e8b7-43f5-8b4a-89cc71570dc1",
+    "id" : "506b1236-0c1e-4d1e-92ea-8fc344b4aaf0"
+  },
+  "server" : {
+    "port" : 27017,
+    "ip" : "10.0.55.2"
+  },
+  "destination" : {
+    "ip" : "10.0.55.2",
+    "port" : 27017
+  },
+  "client" : {
+    "ip" : "10.0.55.1",
+    "port" : 41076,
+    "bytes" : 557
+  },
+  "ecs" : {
+    "version" : "8.0.0"
+  }
+}