From ba73bdc4f97ae346a6271702178d907984c20818 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Mon, 7 Nov 2022 15:33:41 +0100 Subject: [PATCH] Fix "source" field for beats version >= 8 (#13895) * Fix "source" field for beats version >= 8 In beats v8 the "agent.hostname" field got renamed to "agent.name". Fixes #13254 * Add changelog entry (cherry picked from commit 63bc4fe56844194969cfa9023d61e6ae9fbb9dfc) --- changelog/unreleased/issue-13254.toml | 5 ++ .../graylog/plugins/beats/Beats2Codec.java | 9 ++- .../plugins/beats/Beats2CodecTest.java | 18 +++++ .../plugins/beats/packetbeat-mongodb-v8.json | 78 +++++++++++++++++++ 4 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/issue-13254.toml create mode 100644 graylog2-server/src/test/resources/org/graylog/plugins/beats/packetbeat-mongodb-v8.json diff --git a/changelog/unreleased/issue-13254.toml b/changelog/unreleased/issue-13254.toml new file mode 100644 index 000000000000..a8d606f6c87c --- /dev/null +++ b/changelog/unreleased/issue-13254.toml @@ -0,0 +1,5 @@ +type = "fixed" +message = "Fix `source` field extraction for Beats version 8 and later." + +issues = ["13254"] +pulls = ["13895"] diff --git a/graylog2-server/src/main/java/org/graylog/plugins/beats/Beats2Codec.java b/graylog2-server/src/main/java/org/graylog/plugins/beats/Beats2Codec.java index 9b3c88e92f3b..a6c5d845c9d8 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/beats/Beats2Codec.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/beats/Beats2Codec.java @@ -93,7 +93,14 @@ private Message parseEvent(JsonNode event) { if (agentOrBeat.isMissingNode()) { agentOrBeat = event.path("beat"); } - final String hostname = agentOrBeat.path("hostname").asText(BEATS_UNKNOWN); + + JsonNode agentName = agentOrBeat.path("hostname"); + if (agentName.isMissingNode()) { + // Compatibility for beats >= 8.0 + agentName = agentOrBeat.path("name"); + } + + final String hostname = agentName.asText(BEATS_UNKNOWN); final Message gelfMessage = new Message(message, hostname, timestamp); gelfMessage.addField("beats_type", beatsType); diff --git a/graylog2-server/src/test/java/org/graylog/plugins/beats/Beats2CodecTest.java b/graylog2-server/src/test/java/org/graylog/plugins/beats/Beats2CodecTest.java index 53ba897b0699..35ae8c86e75a 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/beats/Beats2CodecTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/beats/Beats2CodecTest.java @@ -109,6 +109,24 @@ public void decodeMessagesHandlesPacketbeatMessages() throws Exception { assertThat(message.getField("packetbeat_dns_flags_recursion_allowed")).isEqualTo(true); } + @Test + public void decodeMessagesHandlesPacketbeatV8Messages() throws Exception { + final Message message = codec.decode(messageFromJson("packetbeat-mongodb-v8.json")); + assertThat(message).isNotNull(); + assertThat(message.getSource()).isEqualTo("example.local"); + assertThat(message.getTimestamp()).isEqualTo(new DateTime(2022, 11, 7, 9, 26, 10, 579, DateTimeZone.UTC)); + assertThat(message.getField("beats_type")).isEqualTo("packetbeat"); + assertThat(message.getField("packetbeat_type")).isEqualTo("mongodb"); + assertThat(message.getField("packetbeat_status")).isEqualTo("OK"); + assertThat(message.getField("packetbeat_method")).isEqualTo("msg"); + assertThat(message.getField("packetbeat_network_bytes")).isEqualTo(557); + assertThat(message.getField("packetbeat_network_type")).isEqualTo("ipv4"); + assertThat(message.getField("packetbeat_source_ip")).isEqualTo("10.0.55.1"); + assertThat(message.getField("packetbeat_destination_ip")).isEqualTo("10.0.55.2"); + assertThat(message.getField("packetbeat_destination_port")).isEqualTo(27017); + assertThat(message.getField("packetbeat_host_containerized")).isEqualTo(false); + } + @Test public void decodeMessagesHandlesTopbeatMessages() throws Exception { final Message message = codec.decode(messageFromJson("topbeat-system.json")); diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/beats/packetbeat-mongodb-v8.json b/graylog2-server/src/test/resources/org/graylog/plugins/beats/packetbeat-mongodb-v8.json new file mode 100644 index 000000000000..425fa866b969 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/beats/packetbeat-mongodb-v8.json @@ -0,0 +1,78 @@ +{ + "@timestamp" : "2022-11-07T09:26:10.579Z", + "@metadata" : { + "beat" : "packetbeat", + "type" : "_doc", + "version" : "8.5.0" + }, + "related" : { + "ip" : [ "10.0.55.1", "10.0.55.2" ] + }, + "status" : "OK", + "method" : "msg", + "mongodb" : { }, + "event" : { + "kind" : "event", + "category" : [ "network" ], + "type" : [ "connection", "protocol" ], + "dataset" : "mongodb", + "start" : "2022-11-07T09:26:10.579Z" + }, + "host" : { + "architecture" : "x86_64", + "os" : { + "codename" : "focal", + "type" : "linux", + "platform" : "ubuntu", + "version" : "20.04.5 LTS (Focal Fossa)", + "family" : "debian", + "name" : "Ubuntu", + "kernel" : "5.15.0-52-generic" + }, + "id" : "3d758250c1e88241a6d2037786a23bcf", + "name" : "example.local", + "containerized" : false, + "ip" : [ "10.0.1.1", "10.0.55.1" ], + "mac" : [ "02-42-11-7A-FD-80", "02-42-77-6C-EC-7E" ], + "hostname" : "example.local" + }, + "type" : "mongodb", + "network" : { + "community_id" : "1:NlfSWdQ8jRZXj4S/I200ox1MJQc=", + "bytes" : 557, + "type" : "ipv4", + "transport" : "tcp", + "protocol" : "mongodb", + "direction" : "egress" + }, + "query" : ".msg()", + "source" : { + "ip" : "10.0.55.1", + "port" : 41076, + "bytes" : 557 + }, + "resource" : "", + "agent" : { + "name" : "example.local", + "type" : "packetbeat", + "version" : "8.5.0", + "ephemeral_id" : "ca453a1e-e8b7-43f5-8b4a-89cc71570dc1", + "id" : "506b1236-0c1e-4d1e-92ea-8fc344b4aaf0" + }, + "server" : { + "port" : 27017, + "ip" : "10.0.55.2" + }, + "destination" : { + "ip" : "10.0.55.2", + "port" : 27017 + }, + "client" : { + "ip" : "10.0.55.1", + "port" : 41076, + "bytes" : 557 + }, + "ecs" : { + "version" : "8.0.0" + } +}