From abd075e7ebc72593bcf69cd9769d8d972c936d2e Mon Sep 17 00:00:00 2001
From: Andrew Kroh <andrew.kroh@elastic.co>
Date: Thu, 29 Oct 2020 21:34:56 -0400
Subject: [PATCH 1/2] Refactor the Suricata module pipeline to use Ingest Node
 more

Refactor the Suricata module pipeline to use Ingest Node more. I moved most
processing from Beat processors to Ingest Node. I created a DNS and TLS
pipeline to house all of the processing for those protocols respectively.

Other changes
- Added DNS response IPs to `related.ip`.
- Some DNS events were missing `dns.header_flags`.
---
 CHANGELOG.next.asciidoc                       |   1 +
 .../module/suricata/eve/config/eve.yml        | 365 +-----------------
 .../suricata/eve/ingest/dns-answer-v1.yml     |  39 ++
 .../suricata/eve/ingest/dns-answer-v2.yml     |  42 ++
 .../module/suricata/eve/ingest/dns.yml        |  93 +++++
 .../module/suricata/eve/ingest/pipeline.yml   | 325 ++++++++++------
 .../module/suricata/eve/ingest/tls.yml        | 188 +++++++++
 .../filebeat/module/suricata/eve/manifest.yml |   7 +-
 .../eve/test/eve-dns-4.1.4.log-expected.json  |  90 +++++
 .../eve/test/eve-small.log-expected.json      |   7 +
 10 files changed, 684 insertions(+), 473 deletions(-)
 create mode 100644 x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v1.yml
 create mode 100644 x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v2.yml
 create mode 100644 x-pack/filebeat/module/suricata/eve/ingest/dns.yml
 create mode 100644 x-pack/filebeat/module/suricata/eve/ingest/tls.yml

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 6a211d61fd8..8b659990624 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -649,6 +649,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add max_number_of_messages config into s3 input. {pull}21993[21993]
 - Update Okta documentation for new stateful restarts. {pull}22091[22091]
 - Copy tag names from MISP data into events. {pull}21664[21664]
+- Added DNS response IP addresses to `related.ip` in Suricata module. {pull}22291[22291]
 
 *Heartbeat*
 
diff --git a/x-pack/filebeat/module/suricata/eve/config/eve.yml b/x-pack/filebeat/module/suricata/eve/config/eve.yml
index 879bcdd8e35..231127914c2 100644
--- a/x-pack/filebeat/module/suricata/eve/config/eve.yml
+++ b/x-pack/filebeat/module/suricata/eve/config/eve.yml
@@ -42,363 +42,16 @@ processors:
 {{ if .community_id }}
   - community_id:
 {{ end }}
-  - if:
-      equals:
-        suricata.eve.event_type: dns
-    then:
-      - convert:
-          ignore_missing: true
-          fail_on_error: false
-          mode: copy
-          fields:
-            - {from: suricata.eve.dns.id, to: dns.id, type: string}
-            - {from: suricata.eve.dns.rcode, to: dns.response_code}
-            - {from: suricata.eve.dns.type, to: dns.type}
-      - convert:
-          when.equals.dns.type: query
-          ignore_missing: true
-          fail_on_error: false
-          mode: copy
-          fields:
-            - {from: suricata.eve.dns.rrname, to: dns.question.name}
-            - {from: suricata.eve.dns.rrtype, to: dns.question.type}
-      - if:
-          and:
-            - equals.dns.type: answer
-            - equals.suricata.eve.dns.version: 2
-        then:
-          - convert:
-              ignore_missing: true
-              fail_on_error: false
-              mode: copy
-              fields:
-                - {from: suricata.eve.dns.rrname, to: dns.question.name}
-                - {from: suricata.eve.dns.rrtype, to: dns.question.type}
-      - registered_domain:
-          ignore_missing: true
-          ignore_failure: true
-          field: dns.question.name
-          target_field: dns.question.registered_domain
-  - script:
-      id: eve_process
-      lang: javascript
-      source: >-
-        function addEcsCategorization(evt) {
-          var event_type = evt.Get("suricata.eve.event_type");
-          if (event_type == null) {
-            return;
-          }
-          var catArray = [];
-          var typeArray = [];
-          evt.Put("suricata.eve.event_type", event_type.toLowerCase());
-          switch (event_type.toLowerCase()) {
-            case "alert":
-              evt.Put("event.kind", "alert");
-              catArray.push("network");
-              catArray.push("intrusion_detection");
-              break;
-            case "anomaly":
-              evt.Put("event.kind", "event");
-              catArray.push("network");
-              break;
-            case "http":
-              evt.Put("event.kind", "event");
-              catArray.push("network");
-              catArray.push("web");
-              typeArray.push("access");
-              typeArray.push("protocol");
-              evt.Put("network.protocol", "http");
-              var status = evt.Get("suricata.eve.http.status");
-              if (status == null) {
-                break;
-              }
-              if (status < 400) {
-                evt.Put("event.outcome", "success");
-              }
-              if (status >= 400 ) {
-                evt.Put("event.outcome", "failure");
-              }
-              break;
-            case "dns":
-              evt.Put("event.kind", "event");
-              catArray.push("network");
-              typeArray.push("protocol");
-              evt.Put("network.protocol", "dns");
-              break;
-            case "ftp":
-              evt.Put("event.kind", "event");
-              catArray.push("network");
-              typeArray.push("protocol");
-              evt.Put("network.protocol", "ftp");
-              break;
-            case "ftp_data":
-              evt.Put("event.kind", "event");
-              catArray.push("network");
-              typeArray.push("protocol");
-              evt.Put("network.protocol", "ftp");
-              break;
-            case "tls":
-              evt.Put("event.kind", "event");
-              catArray.push("network");
-              typeArray.push("protocol");
-              evt.Put("network.protocol", "tls");
-              break;
-            case "tftp":
-              evt.Put("event.kind", "event");
-              catArray.push("network");
-              typeArray.push("protocol");
-              evt.Put("network.protocol", "tftp");
-              break;
-            case "smb":
-              evt.Put("event.kind", "event");
-              catArray.push("network");
-              typeArray.push("protocol");
-              evt.Put("network.protocol", "smb");
-              break;
-            case "ssh":
-              evt.Put("event.kind", "event");
-              catArray.push("network");
-              typeArray.push("protocol");
-              evt.Put("network.protocol", "ssh");
-              break;
-            case "flow":
-              evt.Put("event.kind", "event");
-              catArray.push("network");
-              typeArray.push("connection");
-              var state = evt.Get("suricata.eve.flow.state");
-              if (state == null) {
-                break;
-              }
-              switch (state) {
-                case "new":
-                  typeArray.push("start");
-                  break;
-                case "closed":
-                  typeArray.push("end");
-                  break;
-              }
-              break;
-            case "rdp":
-              evt.Put("event.kind", "event");
-              catArray.push("network");
-              typeArray.push("protocol");
-              evt.Put("network.protocol", "rdp");
-              break;
-            case "stats":
-              evt.Put("event.kind", "metric");
-              break;
-            default:
-              evt.Put("event.kind", "event");
-              catArray.push("network");
-          }
-          if (catArray.length > 0) {
-            evt.Put("event.category", catArray);
-          }
-          if (typeArray.length > 0) {
-            evt.Put("event.type", typeArray);
-          }
-        }
-        function setDnsV1Answers(evt) {
-          var dns_type = evt.Get("dns.type")
-          if (dns_type != "answer") {
-            return;
-          }
-          var version = evt.Get("suricata.eve.dns.version")
-          if (version == "2") {
-            return;
-          }
-          var name = evt.Get("suricata.eve.dns.rrname");
-          var data = evt.Get("suricata.eve.dns.rdata");
-          var type = evt.Get("suricata.eve.dns.rrtype");
-          var ttl = evt.Get("suricata.eve.dns.ttl");
-          var answer = {};
-          if (name) {
-            answer.name = name;
-          }
-          if (data) {
-            answer.data = data;
-          }
-          if (type) {
-            answer.type = type;
-          }
-          if (ttl) {
-            answer.ttl = ttl;
-          }
-          if (Object.keys(answer).length === 0) {
-            return;
-          }
-          evt.Put("dns.answers", [answer]);
-        }
-        function addDnsV2Answers(evt) {
-          var type = evt.Get("dns.type")
-          if (type != "answer") {
-            return;
-          }
-          var version = evt.Get("suricata.eve.dns.version")
-          if (version != 2) {
-            return;
-          }
-          var answers = evt.Get("suricata.eve.dns.answers");
-          if (!answers) {
-            return;
-          }
-          evt.Delete("suricata.eve.dns.answers");
-          var resolvedIps = [];
-          for (var i = 0; i < answers.length; i++) {
-            var answer = answers[i];
-
-            // Rename properties.
-            var name = answer["rrname"];
-            delete answer["rrname"];
-            var type = answer["rrtype"];
-            delete answer["rrtype"];
-            var data = answer["rdata"];
-            delete answer["rdata"];
-
-            answer["name"] = name;
-            answer["type"] = type;
-            answer["data"] = data;
-
-            // Append IP addresses to dns.resolved_ip.
-            if (type === "A" || type === "AAAA") {
-              resolvedIps.push(data);
-            }
-          }
-          evt.Put("dns.answers", answers);
-          if (resolvedIps.length > 0) {
-            evt.Put("dns.resolved_ip", resolvedIps);
-          }
-        }
-        function addDnsV2HeaderFlags(evt) {
-          var type = evt.Get("dns.type")
-          if (type != "answer") {
-            return;
-          }
-          var version = evt.Get("suricata.eve.dns.version")
-          if (version != 2) {
-            return;
-          }
-          var flag = evt.Get("suricata.eve.dns.aa");
-          if (flag === true) {
-            evt.AppendTo("dns.header_flags", "AA");
-          }
-
-          flag = evt.Get("suricata.eve.dns.tc");
-          if (flag === true) {
-            evt.AppendTo("dns.header_flags", "TC");
-          }
-
-          flag = evt.Get("suricata.eve.dns.rd");
-          if (flag === true) {
-            evt.AppendTo("dns.header_flags", "RD");
-          }
-
-          flag = evt.Get("suricata.eve.dns.ra");
-          if (flag === true) {
-            evt.AppendTo("dns.header_flags", "RA");
-          }
-        }
-        function addTopLevelDomain(evt) {
-          var rd = evt.Get("dns.question.registered_domain");
-          if (rd == null) {
-            return;
-          }
-          var firstPeriod = rd.indexOf(".");
-          if (firstPeriod == -1) {
-            return;
-          }
-          evt.Put("dns.question.top_level_domain", rd.substr(firstPeriod + 1));
-        }
-        function cleanupAppProto(evt) {
-          var proto = evt.Get("suricata.eve.app_proto");
-          if (proto == null){
-            return;
-          }
-          switch (proto.toLowerCase()) {
-            case "failed":
-            case "template":
-            case "template-rust":
-              break;
-            case "ftp-data":
-              evt.Put("network.protocol", "ftp");
-              break;
-            default:
-              evt.Put("network.protocol", proto.toLowerCase());
-          }
-          evt.Delete("suricata.eve.app_proto");
-        }
-        function addRelatedIps(evt) {
-          var src_ip = evt.Get("source.ip");
-          if (src_ip != null) {
-            evt.AppendTo("related.ip", src_ip);
-          }
-          var dst_ip = evt.Get("destination.ip");
-          if (dst_ip != null) {
-            evt.AppendTo("related.ip", dst_ip);
-          }
-        }
-        function addTlsVersion(evt) {
-          var tls_version = evt.Get("suricata.eve.tls.version");
-          if (tls_version == null) {
-            return;
-          }
-          var parts = tls_version.split(" ");
-          if (parts.length <  2) {
-            return;
-          }
-          evt.Put("tls.version_protocol", parts[0].toLowerCase());
-          evt.Put("tls.version", parts[1]);
-        }
-        function cleanupTlsSni(evt) {
-          var sni = evt.Get("suricata.eve.tls.sni");
-          if (sni == null) {
-            return;
-          }
-          if ("." == sni.charAt(sni.length - 1)) {
-            evt.Put("suricata.eve.tls.sni", sni.substring(0, sni.length - 1));
-          }
-        }
-        function process(evt) {
-          var event_type = evt.Get("suricata.eve.event_type");
-
-          addEcsCategorization(evt);
-          if (event_type == "dns") {
-            setDnsV1Answers(evt);
-            addDnsV2Answers(evt);
-            addDnsV2HeaderFlags(evt);
-            addTopLevelDomain(evt);
-          }
-          cleanupAppProto(evt);
-          addRelatedIps(evt);
-          addTlsVersion(evt);
-          cleanupTlsSni(evt);
-        }
-  - convert:
+  - registered_domain:
+      when:
+        or:
+          - equals.suricata.eve.dns.type: query
+          # V2 events always include the query data.
+          - equals.suricata.eve.dns.version: 2
       ignore_missing: true
-      fail_on_error: false
-      mode: copy
-      fields:
-        - {from: suricata.eve.tls.subject, to: tls.server.subject}
-        - {from: suricata.eve.tls.issuerdn, to: tls.server.issuer}
-        - {from: suricata.eve.tls.session_resumed, to: tls.resumed, type: boolean}
-        - {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1}
-        - {from: suricata.eve.tls.sni, to: tls.client.server_name}
-        - {from: suricata.eve.tls.sni, to: destination.domain}
-        - {from: suricata.eve.tls.ja3s.hash, to: tls.server.ja3s}
-        - {from: suricata.eve.tls.ja3.hash, to: tls.client.ja3}
-        - {from: suricata.eve.tls.certificate, to: tls.server.certificate}
-        - {from: suricata.eve.tls.chain, to: tls.server.certificate_chain}
-  - drop_fields:
-      ignore_missing: true
-      fields:
-        - suricata.eve.dns.aa
-        - suricata.eve.dns.tc
-        - suricata.eve.dns.rd
-        - suricata.eve.dns.ra
-        - suricata.eve.dns.qr
-        - suricata.eve.dns.version
-        - suricata.eve.dns.flags
-        - suricata.eve.dns.grouped
+      ignore_failure: true
+      field: suricata.eve.dns.rrname
+      target_field: dns.question.registered_domain
   - add_fields:
       target: ''
       fields:
diff --git a/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v1.yml b/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v1.yml
new file mode 100644
index 00000000000..e915537365d
--- /dev/null
+++ b/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v1.yml
@@ -0,0 +1,39 @@
+---
+description: Pipeline for Suricata DNS answers v1
+
+# Suricata DNS v1 events contain a single answer. Multiple events are created
+# to represent all of the answers.
+processors:
+  - script:
+      lang: painless
+      tag: suricata_dns_answer_v1
+      source: |
+        def name = ctx?.suricata?.eve?.dns?.rrname;
+        def data = ctx?.suricata?.eve?.dns?.rdata;
+        def type = ctx?.suricata?.eve?.dns?.rrtype;
+        def ttl = ctx?.suricata?.eve?.dns?.ttl;
+
+        def answer = [:];
+        if (name != null) {
+            answer["name"] = name;
+        }
+        if (data != null) {
+            answer["data"] = data;
+        }
+        if (type != null) {
+            answer["type"] = type;
+        }
+        if (ttl != null) {
+            answer["ttl"] = ttl;
+        }
+        if (!answer.isEmpty()) {
+            ctx.dns.answers = [answer];
+        }
+
+        if (type == "A" || type == "AAAA") {
+            ctx.dns.resolved_ip = [data];
+        }
+on_failure:
+  - append:
+        field: error.message
+        value: "{{ _ingest.on_failure_message }}"
diff --git a/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v2.yml b/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v2.yml
new file mode 100644
index 00000000000..a9e77c28549
--- /dev/null
+++ b/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v2.yml
@@ -0,0 +1,42 @@
+---
+description: Pipeline for Suricata DNS answers v2
+
+# Suricata DNS v2 events contain all answers in a single event.
+processors:
+  - rename:
+      field: suricata.eve.dns.answers
+      target_field: dns.answers
+      ignore_missing: true
+  - script:
+      if: ctx?.dns?.answers != null
+      lang: painless
+      tag: suricata_dns_answers_v2
+      source: |
+        def resolvedIps = new ArrayList();
+        for (def answer : ctx?.dns?.answers) {
+            // Normalize field names to match ECS.
+            def name = answer.remove("rrname");
+            if (name != null) {
+                answer["name"] = name;
+            }
+            def type = answer.remove("rrtype");
+            if (type != null) {
+                answer["type"] = type;
+            }
+            def data = answer.remove("rdata");
+            if (data != null) {
+                answer["data"] = data;
+            }
+
+            if (type == "A" || type == "AAAA") {
+                resolvedIps.add(data);
+            }
+        }
+
+        if (resolvedIps.size() > 0) {
+            ctx.dns.resolved_ip = resolvedIps;
+        }
+on_failure:
+  - append:
+        field: error.message
+        value: "{{ _ingest.on_failure_message }}"
diff --git a/x-pack/filebeat/module/suricata/eve/ingest/dns.yml b/x-pack/filebeat/module/suricata/eve/ingest/dns.yml
new file mode 100644
index 00000000000..f2c1127ddee
--- /dev/null
+++ b/x-pack/filebeat/module/suricata/eve/ingest/dns.yml
@@ -0,0 +1,93 @@
+---
+description: Pipeline for Suricata DNS Events
+
+processors:
+  - set:
+      field: dns.id
+      value: '{{suricata.eve.dns.id}}'
+      ignore_empty_value: true
+  - set:
+      field: dns.response_code
+      value: '{{suricata.eve.dns.rcode}}'
+      ignore_empty_value: true
+  - set:
+      field: dns.type
+      value: '{{suricata.eve.dns.type}}'
+      ignore_empty_value: true
+  - set:
+      # V2 events always include the query data.
+      if: >-
+        ctx?.dns?.type == "query" ||
+        ctx?.suricata?.eve?.dns?.version == 2
+      field: dns.question.name
+      value: '{{suricata.eve.dns.rrname}}'
+      ignore_empty_value: true
+  - set:
+      # V2 events always include the query data.
+      if: >-
+        ctx?.dns?.type == "query" ||
+        ctx?.suricata?.eve?.dns?.version == 2
+      field: dns.question.type
+      value: '{{suricata.eve.dns.rrtype}}'
+      ignore_empty_value: true
+  - pipeline:
+      if: >-
+        ctx?.dns?.type == "answer" &&
+        ctx?.suricata?.eve?.dns?.version == null
+      name: '{< IngestPipeline "dns-answer-v1" >}'
+  - pipeline:
+      if: >-
+        ctx?.dns?.type == "answer" &&
+        ctx?.suricata?.eve?.dns?.version == 2
+      name: '{< IngestPipeline "dns-answer-v2" >}'
+  - foreach:
+      field: dns.resolved_ip
+      ignore_missing: true
+      processor:
+        append:
+          field: related.ip
+          value:
+            - '{{_ingest._value}}'
+          allow_duplicates: false
+  - script:
+      if: ctx?.dns?.question?.registered_domain != null
+      tag: suricata_dns_top_level_domain
+      lang: painless
+      source: |
+        def rd = ctx.dns.question.registered_domain;
+        def firstDot = rd.indexOf(".");
+        if (firstDot == -1) {
+            return;
+        }
+        ctx.dns.question.top_level_domain = rd.substring(firstDot + 1);
+  - append:
+      if: ctx?.suricata?.eve?.dns?.aa == true
+      field: dns.header_flags
+      value: AA
+  - append:
+      if: ctx?.suricata?.eve?.dns?.tc == true
+      field: dns.header_flags
+      value: TC
+  - append:
+      if: ctx?.suricata?.eve?.dns?.rd == true
+      field: dns.header_flags
+      value: RD
+  - append:
+      if: ctx?.suricata?.eve?.dns?.ra == true
+      field: dns.header_flags
+      value: RA
+  - remove:
+      field:
+        - suricata.eve.dns.aa
+        - suricata.eve.dns.tc
+        - suricata.eve.dns.rd
+        - suricata.eve.dns.ra
+        - suricata.eve.dns.qr
+        - suricata.eve.dns.version
+        - suricata.eve.dns.flags
+        - suricata.eve.dns.grouped
+      ignore_missing: true
+on_failure:
+  - append:
+        field: error.message
+        value: "{{ _ingest.on_failure_message }}"
diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml
index e132a8acdde..3aab0956fda 100644
--- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml
+++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml
@@ -5,6 +5,203 @@ processors:
   - set:
       field: event.ingested
       value: '{{_ingest.timestamp}}'
+
+  # Handle the different Suricata event types.
+  - lowercase:
+      field: suricata.eve.event_type
+      ignore_missing: true
+  - set:
+      field: event.kind
+      value: event
+  - append:
+      field: event.category
+      value:
+        - network
+  ## Alert
+  - set:
+      if: ctx?.suricata?.eve?.event_type == "alert"
+      field: event.kind
+      value: alert
+  - append:
+      if: ctx?.suricata?.eve?.event_type == "alert"
+      field: event.category
+      value:
+        - intrusion_detection
+  ## Anomaly and Alert
+  - lowercase:
+      field: suricata.eve.app_proto
+      ignore_missing: true
+  - set:
+      if: ctx?.suricata?.eve?.app_proto == "ftp-data"
+      field: network.protocol
+      value: ftp
+  - set:
+      if: >-
+        ctx?.suricata?.eve?.app_proto != "failed" &&
+        ctx?.suricata?.eve?.app_proto != "template" &&
+        ctx?.suricata?.eve?.app_proto != "template-rust"
+      field: network.protocol
+      value: '{{suricata.eve.app_proto}}'
+      ignore_empty_value: true
+  ## HTTP
+  - append:
+      if: ctx?.suricata?.eve?.event_type == "http"
+      field: event.category
+      value:
+        - web
+  - append:
+      if: ctx?.suricata?.eve?.event_type == "http"
+      field: event.type
+      value:
+        - access
+        - protocol
+  - set:
+      if: ctx?.suricata?.eve?.event_type == "http"
+      field: network.protocol
+      value: http
+  - set:
+      if: ctx?.suricata?.eve?.event_type == "http" && ctx?.suricata?.eve?.http?.status < 400
+      field: event.outcome
+      value: success
+  - set:
+      if: ctx?.suricata?.eve?.event_type == "http" && ctx?.suricata?.eve?.http?.status >= 400
+      field: event.outcome
+      value: failure
+  ## DNS
+  - append:
+      if: ctx?.suricata?.eve?.event_type == "dns"
+      field: event.type
+      value:
+        - protocol
+  - set:
+      if: ctx?.suricata?.eve?.event_type == "dns"
+      field: network.protocol
+      value: dns
+  - pipeline:
+      if: >-
+        ctx?.network?.protocol == "dns"
+      name: '{< IngestPipeline "dns" >}'
+  ## FTP or FTP Data
+  - append:
+      if: ctx?.suricata?.eve?.event_type == "ftp" || ctx?.suricata?.eve?.event_type == "ftp_data"
+      field: event.type
+      value:
+        - protocol
+  - set:
+      if: ctx?.suricata?.eve?.event_type == "ftp" || ctx?.suricata?.eve?.event_type == "ftp_data"
+      field: network.protocol
+      value: ftp
+  ## TLS
+  - append:
+      if: ctx?.suricata?.eve?.event_type == "tls"
+      field: event.type
+      value:
+        - protocol
+  - set:
+      if: ctx?.suricata?.eve?.event_type == "tls"
+      field: network.protocol
+      value: tls
+  - pipeline:
+      if: ctx?.network?.protocol == "tls"
+      name: '{< IngestPipeline "tls" >}'
+  ## TFTP
+  - append:
+      if: ctx?.suricata?.eve?.event_type == "tftp"
+      field: event.type
+      value:
+        - protocol
+  - set:
+      if: ctx?.suricata?.eve?.event_type == "tftp"
+      field: network.protocol
+      value: tftp
+  ## SMB
+  - append:
+      if: ctx?.suricata?.eve?.event_type == "smb"
+      field: event.type
+      value:
+        - protocol
+  - set:
+      if: ctx?.suricata?.eve?.event_type == "smb"
+      field: network.protocol
+      value: smb
+  ## SSH
+  - append:
+      if: ctx?.suricata?.eve?.event_type == "ssh"
+      field: event.type
+      value:
+        - protocol
+  - set:
+      if: ctx?.suricata?.eve?.event_type == "ssh"
+      field: network.protocol
+      value: ssh
+  ## Flow
+  - append:
+      if: ctx?.suricata?.eve?.event_type == "flow"
+      field: event.type
+      value:
+        - connection
+  - append:
+      if: ctx?.suricata?.eve?.flow?.state == "new"
+      field: event.type
+      value:
+        - start
+  - append:
+      if: ctx?.suricata?.eve?.flow?.state == "closed"
+      field: event.type
+      value:
+        - end
+  ## RDP
+  - append:
+      if: ctx?.suricata?.eve?.event_type == "rdp"
+      field: event.type
+      value:
+        - protocol
+  - set:
+      if: ctx?.suricata?.eve?.event_type == "rdp"
+      field: network.protocol
+      value: rdp
+  ## RFB (Remote Framebuffer Protocol)
+  - append:
+      if: ctx?.suricata?.eve?.event_type == "rfb"
+      field: event.type
+      value:
+        - protocol
+  - set:
+      if: ctx?.suricata?.eve?.event_type == "rfb"
+      field: network.protocol
+      value: rfb
+  ## MQTT
+  - append:
+      if: ctx?.suricata?.eve?.event_type == "mqtt"
+      field: event.type
+      value:
+        - protocol
+  - set:
+      if: ctx?.suricata?.eve?.event_type == "mqtt"
+      field: network.protocol
+      value: mqtt
+  ## HTTP2
+  - append:
+      if: ctx?.suricata?.eve?.event_type == "http2"
+      field: event.category
+      value:
+        - web
+  - append:
+      if: ctx?.suricata?.eve?.event_type == "http2"
+      field: event.type
+      value:
+        - access
+        - protocol
+  - set:
+      if: ctx?.suricata?.eve?.event_type == "http2"
+      field: network.protocol
+      value: http2
+  ## Stats
+  - set:
+      if: ctx?.suricata?.eve?.event_type == "stats"
+      field: event.kind
+      value: metric
+
   - set:
       value: "{{suricata.eve.http.http_method}}"
       field: http.request.method
@@ -17,11 +214,13 @@ processors:
       if: ctx.suricata?.eve?.http?.hostname != null
       value: '{{suricata.eve.http.hostname}}'
       field: destination.domain
+      allow_duplicates: false
   - remove:
       field: suricata.eve.http.hostname
       ignore_failure: true
   - script:
       lang: painless
+      tag: suricata_deduplicate_dest_domain
       source: >
         def domain = ctx.destination?.domain;
         if (domain instanceof Collection) {
@@ -219,125 +418,21 @@ processors:
       field: destination.as.organization_name
       target_field: destination.as.organization.name
       ignore_missing: true
-  - uppercase:
-      field: tls.server.hash.sha1
-      ignore_missing: true
-  - split:
-      field: tls.server.hash.sha1
-      separator: ":"
-      ignore_missing: true
-  - join:
-      field: tls.server.hash.sha1
-      separator: ""
-      ignore_failure: true
-  - append:
-      field: related.hash
-      value: "{{tls.server.hash.sha1}}"
-      if: "ctx?.tls?.server?.hash?.sha1 != null"
-  - gsub:
-      field: suricata.eve.tls.issuerdn
-      pattern: \\,
-      replacement: ""
-      ignore_missing: true
-  - kv:
-      field: suricata.eve.tls.issuerdn
-      field_split: ', '
-      value_split: '='
-      target_field: suricata.eve.tls.kv_issuerdn
-      ignore_missing: true
-  - rename:
-      field: suricata.eve.tls.kv_issuerdn.C
-      target_field: tls.server.x509.issuer.country
-      ignore_missing: true
-  - rename:
-      field: suricata.eve.tls.kv_issuerdn.CN
-      target_field: tls.server.x509.issuer.common_name
-      ignore_missing: true
-  - rename:
-      field: suricata.eve.tls.kv_issuerdn.L
-      target_field: tls.server.x509.issuer.locality
-      ignore_missing: true
-  - rename:
-      field: suricata.eve.tls.kv_issuerdn.O
-      target_field: tls.server.x509.issuer.organization
-      ignore_missing: true
-  - rename:
-      field: suricata.eve.tls.kv_issuerdn.OU
-      target_field: tls.server.x509.issuer.organizational_unit
-      ignore_missing: true
-  - rename:
-      field: suricata.eve.tls.kv_issuerdn.ST
-      target_field: tls.server.x509.issuer.state_or_province
-      ignore_missing: true
-  - gsub:
-      field: suricata.eve.tls.subject
-      pattern: \\,
-      replacement: ""
-      ignore_missing: true
-  - kv:
-      field: suricata.eve.tls.subject
-      field_split: ', '
-      value_split: '='
-      target_field: suricata.eve.tls.kv_subject
-      ignore_missing: true
-  - rename:
-      field: suricata.eve.tls.kv_subject.C
-      target_field: tls.server.x509.subject.country
-      ignore_missing: true
-  - rename:
-      field: suricata.eve.tls.kv_subject.CN
-      target_field: tls.server.x509.subject.common_name
-      ignore_missing: true
-  - rename:
-      field: suricata.eve.tls.kv_subject.L
-      target_field: tls.server.x509.subject.locality
-      ignore_missing: true
-  - rename:
-      field: suricata.eve.tls.kv_subject.O
-      target_field: tls.server.x509.subject.organization
-      ignore_missing: true
-  - rename:
-      field: suricata.eve.tls.kv_subject.OU
-      target_field: tls.server.x509.subject.organizational_unit
-      ignore_missing: true
-  - rename:
-      field: suricata.eve.tls.kv_subject.ST
-      target_field: tls.server.x509.subject.state_or_province
-      ignore_missing: true
-  - set:
-      field: tls.server.x509.serial_number
-      value: '{{suricata.eve.tls.serial}}'
-      ignore_empty_value: true
-  - gsub:
-      field: tls.server.x509.serial_number
-      pattern: ':'
-      replacement: ''
-      ignore_missing: true
-  - date:
-      field: suricata.eve.tls.notafter
-      target_field: tls.server.not_after
-      formats:
-        - ISO8601
-      if: ctx.suricata?.eve?.tls?.notafter != null
-  - date:
-      field: suricata.eve.tls.notbefore
-      target_field: tls.server.not_before
-      formats:
-        - ISO8601
-      if: ctx.suricata?.eve?.tls?.notbefore != null
-  - set:
-      field: tls.server.x509.not_after
-      value: '{{tls.server.not_after}}'
-      ignore_empty_value: true
-  - set:
-      field: tls.server.x509.not_before
-      value: '{{tls.server.not_before}}'
-      ignore_empty_value: true
   - append:
       field: related.hosts
       value: '{{url.domain}}'
       if: ctx.url?.domain != null && ctx.url?.domain != ''
       allow_duplicates: false
+  - append:
+      if: ctx?.source?.ip != null
+      field: related.ip
+      value: '{{source.ip}}'
+      allow_duplicates: false
+  - append:
+      if: ctx?.destination?.ip != null
+      field: related.ip
+      value: '{{destination.ip}}'
+      allow_duplicates: false
   - remove:
       field:
         - suricata.eve.app_proto
@@ -345,8 +440,6 @@ processors:
         - suricata.eve.flow.start
         - suricata.eve.http.http_method
         - suricata.eve.http.http_user_agent
-        - suricata.eve.tls.kv_issuerdn
-        - suricata.eve.tls.kv_subject
       ignore_missing: true
 on_failure:
   - set:
diff --git a/x-pack/filebeat/module/suricata/eve/ingest/tls.yml b/x-pack/filebeat/module/suricata/eve/ingest/tls.yml
new file mode 100644
index 00000000000..2c84e0c1cb7
--- /dev/null
+++ b/x-pack/filebeat/module/suricata/eve/ingest/tls.yml
@@ -0,0 +1,188 @@
+---
+description: Pipeline for Suricata TLS Events
+
+processors:
+  - dissect:
+      field: suricata.eve.tls.version
+      pattern: '%{tls.version_protocol} %{tls.version}'
+      ignore_missing: true
+  - lowercase:
+      field: tls.version_protocol
+      ignore_missing: true
+  - script:
+      if: ctx?.suricata?.eve?.tls?.sni != null
+      tag: suricata_trim_tls_sni
+      lang: painless
+      source: |
+        def sni = ctx.suricata.eve.tls.sni;
+        if (!sni.endsWith(".")) {
+            return;
+        }
+        ctx.suricata.eve.tls.sni = sni.substring(0, sni.length() - 1);
+  # Subject
+  - set:
+      field: tls.server.subject
+      value: '{{suricata.eve.tls.subject}}'
+      ignore_empty_value: true
+  - gsub:
+      field: suricata.eve.tls.subject
+      pattern: \\,
+      replacement: ""
+      ignore_missing: true
+  - kv:
+      field: suricata.eve.tls.subject
+      field_split: ', '
+      value_split: '='
+      target_field: suricata.eve.tls.kv_subject
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_subject.C
+      target_field: tls.server.x509.subject.country
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_subject.CN
+      target_field: tls.server.x509.subject.common_name
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_subject.L
+      target_field: tls.server.x509.subject.locality
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_subject.O
+      target_field: tls.server.x509.subject.organization
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_subject.OU
+      target_field: tls.server.x509.subject.organizational_unit
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_subject.ST
+      target_field: tls.server.x509.subject.state_or_province
+      ignore_missing: true
+  # Issuer
+  - set:
+      field: tls.server.issuer
+      value: '{{suricata.eve.tls.issuerdn}}'
+      ignore_empty_value: true
+  - gsub:
+      field: suricata.eve.tls.issuerdn
+      pattern: \\,
+      replacement: ""
+      ignore_missing: true
+  - kv:
+      field: suricata.eve.tls.issuerdn
+      field_split: ', '
+      value_split: '='
+      target_field: suricata.eve.tls.kv_issuerdn
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_issuerdn.C
+      target_field: tls.server.x509.issuer.country
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_issuerdn.CN
+      target_field: tls.server.x509.issuer.common_name
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_issuerdn.L
+      target_field: tls.server.x509.issuer.locality
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_issuerdn.O
+      target_field: tls.server.x509.issuer.organization
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_issuerdn.OU
+      target_field: tls.server.x509.issuer.organizational_unit
+      ignore_missing: true
+  - rename:
+      field: suricata.eve.tls.kv_issuerdn.ST
+      target_field: tls.server.x509.issuer.state_or_province
+      ignore_missing: true
+
+  - convert:
+      field: suricata.eve.tls.session_resumed
+      target_field: tls.resumed
+      type: boolean
+      ignore_missing: true
+  - set:
+      field: tls.server.hash.sha1
+      value: '{{suricata.eve.tls.fingerprint}}'
+      ignore_empty_value: true
+  - uppercase:
+      field: tls.server.hash.sha1
+      ignore_missing: true
+  - split:
+      field: tls.server.hash.sha1
+      separator: ":"
+      ignore_missing: true
+  - join:
+      field: tls.server.hash.sha1
+      separator: ""
+      ignore_failure: true
+  - append:
+      field: related.hash
+      value: "{{tls.server.hash.sha1}}"
+      if: "ctx?.tls?.server?.hash?.sha1 != null"
+  - set:
+      field: tls.client.server_name
+      value: '{{suricata.eve.tls.sni}}'
+      ignore_empty_value: true
+  - set:
+      field: destination.domain
+      value: '{{suricata.eve.tls.sni}}'
+      ignore_empty_value: true
+  - set:
+      field: tls.server.ja3s
+      value: '{{suricata.eve.tls.ja3s.hash}}'
+      ignore_empty_value: true
+  - set:
+      field: tls.client.ja3
+      value: '{{suricata.eve.tls.ja3.hash}}'
+      ignore_empty_value: true
+  - set:
+      field: tls.server.certificate
+      value: '{{suricata.eve.tls.certificate}}'
+      ignore_empty_value: true
+  - set:
+      field: tls.server.certificate_chain
+      value: '{{suricata.eve.tls.chain}}'
+      ignore_empty_value: true
+  - set:
+      field: tls.server.x509.serial_number
+      value: '{{suricata.eve.tls.serial}}'
+      ignore_empty_value: true
+  - gsub:
+      field: tls.server.x509.serial_number
+      pattern: ':'
+      replacement: ''
+      ignore_missing: true
+  - date:
+      field: suricata.eve.tls.notafter
+      target_field: tls.server.not_after
+      formats:
+        - ISO8601
+      if: ctx.suricata?.eve?.tls?.notafter != null
+  - date:
+      field: suricata.eve.tls.notbefore
+      target_field: tls.server.not_before
+      formats:
+        - ISO8601
+      if: ctx.suricata?.eve?.tls?.notbefore != null
+  - set:
+      field: tls.server.x509.not_after
+      value: '{{tls.server.not_after}}'
+      ignore_empty_value: true
+  - set:
+      field: tls.server.x509.not_before
+      value: '{{tls.server.not_before}}'
+      ignore_empty_value: true
+  - remove:
+      field:
+        - suricata.eve.tls.kv_issuerdn
+        - suricata.eve.tls.kv_subject
+      ignore_missing: true
+on_failure:
+  - append:
+        field: error.message
+        value: "{{ _ingest.on_failure_message }}"
diff --git a/x-pack/filebeat/module/suricata/eve/manifest.yml b/x-pack/filebeat/module/suricata/eve/manifest.yml
index 99efdd65ed4..8d7f87d8373 100644
--- a/x-pack/filebeat/module/suricata/eve/manifest.yml
+++ b/x-pack/filebeat/module/suricata/eve/manifest.yml
@@ -13,7 +13,12 @@ var:
   - name: community_id
     default: true
 
-ingest_pipeline: ingest/pipeline.yml
+ingest_pipeline:
+  - ingest/pipeline.yml
+  - ingest/dns.yml
+  - ingest/dns-answer-v1.yml
+  - ingest/dns-answer-v2.yml
+  - ingest/tls.yml
 input: config/eve.yml
 
 requires.processors:
diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-dns-4.1.4.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-dns-4.1.4.log-expected.json
index a36d9d951ad..7201aa1509c 100644
--- a/x-pack/filebeat/module/suricata/eve/test/eve-dns-4.1.4.log-expected.json
+++ b/x-pack/filebeat/module/suricata/eve/test/eve-dns-4.1.4.log-expected.json
@@ -137,6 +137,7 @@
         "network.protocol": "dns",
         "network.transport": "udp",
         "related.ip": [
+            "2607:f8b0:4006:0805:0000:0000:0000:200e",
             "10.0.2.3",
             "10.0.2.15"
         ],
@@ -200,6 +201,7 @@
         "network.protocol": "dns",
         "network.transport": "udp",
         "related.ip": [
+            "172.217.11.46",
             "10.0.2.3",
             "10.0.2.15"
         ],
@@ -384,6 +386,10 @@
         "network.protocol": "dns",
         "network.transport": "udp",
         "related.ip": [
+            "151.101.130.217",
+            "151.101.194.217",
+            "151.101.2.217",
+            "151.101.66.217",
             "10.0.2.3",
             "10.0.2.15"
         ],
@@ -474,6 +480,10 @@
         "network.protocol": "dns",
         "network.transport": "udp",
         "related.ip": [
+            "2a04:4e42:0600:0000:0000:0000:0000:0729",
+            "2a04:4e42:0000:0000:0000:0000:0000:0729",
+            "2a04:4e42:0200:0000:0000:0000:0000:0729",
+            "2a04:4e42:0400:0000:0000:0000:0000:0729",
             "10.0.2.3",
             "10.0.2.15"
         ],
@@ -600,6 +610,10 @@
                 "type": "CNAME"
             }
         ],
+        "dns.header_flags": [
+            "RD",
+            "RA"
+        ],
         "dns.id": "28329",
         "dns.response_code": "NOERROR",
         "dns.type": "answer",
@@ -654,7 +668,14 @@
                 "type": "A"
             }
         ],
+        "dns.header_flags": [
+            "RD",
+            "RA"
+        ],
         "dns.id": "28329",
+        "dns.resolved_ip": [
+            "98.138.219.232"
+        ],
         "dns.response_code": "NOERROR",
         "dns.type": "answer",
         "event.category": [
@@ -674,6 +695,7 @@
         "network.protocol": "dns",
         "network.transport": "udp",
         "related.ip": [
+            "98.138.219.232",
             "10.0.2.3",
             "10.0.2.15"
         ],
@@ -708,7 +730,14 @@
                 "type": "A"
             }
         ],
+        "dns.header_flags": [
+            "RD",
+            "RA"
+        ],
         "dns.id": "28329",
+        "dns.resolved_ip": [
+            "98.138.219.231"
+        ],
         "dns.response_code": "NOERROR",
         "dns.type": "answer",
         "event.category": [
@@ -728,6 +757,7 @@
         "network.protocol": "dns",
         "network.transport": "udp",
         "related.ip": [
+            "98.138.219.231",
             "10.0.2.3",
             "10.0.2.15"
         ],
@@ -762,7 +792,14 @@
                 "type": "A"
             }
         ],
+        "dns.header_flags": [
+            "RD",
+            "RA"
+        ],
         "dns.id": "28329",
+        "dns.resolved_ip": [
+            "72.30.35.10"
+        ],
         "dns.response_code": "NOERROR",
         "dns.type": "answer",
         "event.category": [
@@ -782,6 +819,7 @@
         "network.protocol": "dns",
         "network.transport": "udp",
         "related.ip": [
+            "72.30.35.10",
             "10.0.2.3",
             "10.0.2.15"
         ],
@@ -816,7 +854,14 @@
                 "type": "A"
             }
         ],
+        "dns.header_flags": [
+            "RD",
+            "RA"
+        ],
         "dns.id": "28329",
+        "dns.resolved_ip": [
+            "72.30.35.9"
+        ],
         "dns.response_code": "NOERROR",
         "dns.type": "answer",
         "event.category": [
@@ -836,6 +881,7 @@
         "network.protocol": "dns",
         "network.transport": "udp",
         "related.ip": [
+            "72.30.35.9",
             "10.0.2.3",
             "10.0.2.15"
         ],
@@ -870,6 +916,10 @@
                 "type": "CNAME"
             }
         ],
+        "dns.header_flags": [
+            "RD",
+            "RA"
+        ],
         "dns.id": "7050",
         "dns.response_code": "NOERROR",
         "dns.type": "answer",
@@ -924,7 +974,14 @@
                 "type": "AAAA"
             }
         ],
+        "dns.header_flags": [
+            "RD",
+            "RA"
+        ],
         "dns.id": "7050",
+        "dns.resolved_ip": [
+            "2001:4998:0058:1836:0000:0000:0000:0010"
+        ],
         "dns.response_code": "NOERROR",
         "dns.type": "answer",
         "event.category": [
@@ -944,6 +1001,7 @@
         "network.protocol": "dns",
         "network.transport": "udp",
         "related.ip": [
+            "2001:4998:0058:1836:0000:0000:0000:0010",
             "10.0.2.3",
             "10.0.2.15"
         ],
@@ -978,7 +1036,14 @@
                 "type": "AAAA"
             }
         ],
+        "dns.header_flags": [
+            "RD",
+            "RA"
+        ],
         "dns.id": "7050",
+        "dns.resolved_ip": [
+            "2001:4998:0044:041d:0000:0000:0000:0003"
+        ],
         "dns.response_code": "NOERROR",
         "dns.type": "answer",
         "event.category": [
@@ -998,6 +1063,7 @@
         "network.protocol": "dns",
         "network.transport": "udp",
         "related.ip": [
+            "2001:4998:0044:041d:0000:0000:0000:0003",
             "10.0.2.3",
             "10.0.2.15"
         ],
@@ -1032,7 +1098,14 @@
                 "type": "AAAA"
             }
         ],
+        "dns.header_flags": [
+            "RD",
+            "RA"
+        ],
         "dns.id": "7050",
+        "dns.resolved_ip": [
+            "2001:4998:0058:1836:0000:0000:0000:0011"
+        ],
         "dns.response_code": "NOERROR",
         "dns.type": "answer",
         "event.category": [
@@ -1052,6 +1125,7 @@
         "network.protocol": "dns",
         "network.transport": "udp",
         "related.ip": [
+            "2001:4998:0058:1836:0000:0000:0000:0011",
             "10.0.2.3",
             "10.0.2.15"
         ],
@@ -1086,7 +1160,14 @@
                 "type": "AAAA"
             }
         ],
+        "dns.header_flags": [
+            "RD",
+            "RA"
+        ],
         "dns.id": "7050",
+        "dns.resolved_ip": [
+            "2001:4998:0044:041d:0000:0000:0000:0004"
+        ],
         "dns.response_code": "NOERROR",
         "dns.type": "answer",
         "event.category": [
@@ -1106,6 +1187,7 @@
         "network.protocol": "dns",
         "network.transport": "udp",
         "related.ip": [
+            "2001:4998:0044:041d:0000:0000:0000:0004",
             "10.0.2.3",
             "10.0.2.15"
         ],
@@ -1292,6 +1374,10 @@
         "network.protocol": "dns",
         "network.transport": "udp",
         "related.ip": [
+            "151.101.194.217",
+            "151.101.2.217",
+            "151.101.66.217",
+            "151.101.130.217",
             "10.0.2.3",
             "10.0.2.15"
         ],
@@ -1382,6 +1468,10 @@
         "network.protocol": "dns",
         "network.transport": "udp",
         "related.ip": [
+            "2a04:4e42:0000:0000:0000:0000:0000:0729",
+            "2a04:4e42:0200:0000:0000:0000:0000:0729",
+            "2a04:4e42:0400:0000:0000:0000:0000:0729",
+            "2a04:4e42:0600:0000:0000:0000:0000:0729",
             "10.0.2.3",
             "10.0.2.15"
         ],
diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json
index 2db09a8ee38..e4de4700be8 100644
--- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json
+++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json
@@ -236,6 +236,9 @@
             }
         ],
         "dns.id": "12308",
+        "dns.resolved_ip": [
+            "172.217.13.110"
+        ],
         "dns.response_code": "NOERROR",
         "dns.type": "answer",
         "event.category": [
@@ -255,6 +258,7 @@
         "network.protocol": "dns",
         "network.transport": "udp",
         "related.ip": [
+            "172.217.13.110",
             "192.168.86.1",
             "192.168.86.85"
         ],
@@ -278,6 +282,9 @@
     },
     {
         "@timestamp": "2018-07-05T19:51:23.009Z",
+        "event.category": [
+            "network"
+        ],
         "event.dataset": "suricata.eve",
         "event.kind": "metric",
         "event.module": "suricata",

From c86b906f442cbbfd70c93af3ceeda0c2bc1521f7 Mon Sep 17 00:00:00 2001
From: Andrew Kroh <andrew.kroh@elastic.co>
Date: Thu, 5 Nov 2020 12:33:53 -0500
Subject: [PATCH 2/2] Use script with params

---
 .../module/suricata/eve/ingest/pipeline.yml   | 237 +++++++-----------
 1 file changed, 91 insertions(+), 146 deletions(-)

diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml
index 3aab0956fda..206f6b06e46 100644
--- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml
+++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml
@@ -10,23 +10,96 @@ processors:
   - lowercase:
       field: suricata.eve.event_type
       ignore_missing: true
-  - set:
-      field: event.kind
-      value: event
-  - append:
-      field: event.category
-      value:
-        - network
-  ## Alert
-  - set:
-      if: ctx?.suricata?.eve?.event_type == "alert"
-      field: event.kind
-      value: alert
-  - append:
-      if: ctx?.suricata?.eve?.event_type == "alert"
-      field: event.category
-      value:
-        - intrusion_detection
+  - script:
+      lang: painless
+      ignore_failure: true
+      params:
+        alert:
+          kind: alert
+          category:
+            - network
+            - intrusion_detection
+        dns:
+          type:
+            - protocol
+          network_protocol: dns
+        flow:
+          type:
+            - connection
+        ftp:
+          type:
+            - protocol
+          network_protocol: ftp
+        ftp_data:
+          type:
+            - protocol
+          network_protocol: ftp
+        http:
+          category:
+            - network
+            - web
+          type:
+            - access
+            - protocol
+          network_protocol: http
+        http2:
+          category:
+            - network
+            - web
+          type:
+            - access
+            - protocol
+          network_protocol: http
+        mqtt:
+          type:
+            - protocol
+          network_protocol: mqtt
+        smb:
+          type:
+            - protocol
+          network_protocol: smb
+        ssh:
+          type:
+            - protocol
+          network_protocol: ssh
+        stats:
+          kind: metric
+        tftp:
+          type:
+            - protocol
+          network_protocol: tftp
+        tls:
+          type:
+            - protocol
+          network_protocol: tls
+        rdp:
+          type:
+            - protocol
+          network_protocol: rdp
+        rfb: # RFB (Remote Framebuffer Protocol)
+          type:
+            - protocol
+          network_protocol: rdp
+
+      source: |
+        ctx.event.kind = 'event';
+        ctx.event.category = ['network'];
+        def type_params = params.get(ctx?.suricata?.eve?.event_type);
+        if (type_params == null) {
+            return;
+        }
+        type_params.forEach((k, v) -> {
+            if ('network_protocol' == k) {
+                if (ctx.network == null) {
+                    ctx.network = ['protocol': v];
+                } else {
+                    ctx.network.protocol = v;
+                }
+            } else {
+                ctx.event[k] = v;
+            }
+        });
+
   ## Anomaly and Alert
   - lowercase:
       field: suricata.eve.app_proto
@@ -44,21 +117,6 @@ processors:
       value: '{{suricata.eve.app_proto}}'
       ignore_empty_value: true
   ## HTTP
-  - append:
-      if: ctx?.suricata?.eve?.event_type == "http"
-      field: event.category
-      value:
-        - web
-  - append:
-      if: ctx?.suricata?.eve?.event_type == "http"
-      field: event.type
-      value:
-        - access
-        - protocol
-  - set:
-      if: ctx?.suricata?.eve?.event_type == "http"
-      field: network.protocol
-      value: http
   - set:
       if: ctx?.suricata?.eve?.event_type == "http" && ctx?.suricata?.eve?.http?.status < 400
       field: event.outcome
@@ -68,78 +126,15 @@ processors:
       field: event.outcome
       value: failure
   ## DNS
-  - append:
-      if: ctx?.suricata?.eve?.event_type == "dns"
-      field: event.type
-      value:
-        - protocol
-  - set:
-      if: ctx?.suricata?.eve?.event_type == "dns"
-      field: network.protocol
-      value: dns
   - pipeline:
       if: >-
         ctx?.network?.protocol == "dns"
       name: '{< IngestPipeline "dns" >}'
-  ## FTP or FTP Data
-  - append:
-      if: ctx?.suricata?.eve?.event_type == "ftp" || ctx?.suricata?.eve?.event_type == "ftp_data"
-      field: event.type
-      value:
-        - protocol
-  - set:
-      if: ctx?.suricata?.eve?.event_type == "ftp" || ctx?.suricata?.eve?.event_type == "ftp_data"
-      field: network.protocol
-      value: ftp
   ## TLS
-  - append:
-      if: ctx?.suricata?.eve?.event_type == "tls"
-      field: event.type
-      value:
-        - protocol
-  - set:
-      if: ctx?.suricata?.eve?.event_type == "tls"
-      field: network.protocol
-      value: tls
   - pipeline:
       if: ctx?.network?.protocol == "tls"
       name: '{< IngestPipeline "tls" >}'
-  ## TFTP
-  - append:
-      if: ctx?.suricata?.eve?.event_type == "tftp"
-      field: event.type
-      value:
-        - protocol
-  - set:
-      if: ctx?.suricata?.eve?.event_type == "tftp"
-      field: network.protocol
-      value: tftp
-  ## SMB
-  - append:
-      if: ctx?.suricata?.eve?.event_type == "smb"
-      field: event.type
-      value:
-        - protocol
-  - set:
-      if: ctx?.suricata?.eve?.event_type == "smb"
-      field: network.protocol
-      value: smb
-  ## SSH
-  - append:
-      if: ctx?.suricata?.eve?.event_type == "ssh"
-      field: event.type
-      value:
-        - protocol
-  - set:
-      if: ctx?.suricata?.eve?.event_type == "ssh"
-      field: network.protocol
-      value: ssh
   ## Flow
-  - append:
-      if: ctx?.suricata?.eve?.event_type == "flow"
-      field: event.type
-      value:
-        - connection
   - append:
       if: ctx?.suricata?.eve?.flow?.state == "new"
       field: event.type
@@ -150,57 +145,7 @@ processors:
       field: event.type
       value:
         - end
-  ## RDP
-  - append:
-      if: ctx?.suricata?.eve?.event_type == "rdp"
-      field: event.type
-      value:
-        - protocol
-  - set:
-      if: ctx?.suricata?.eve?.event_type == "rdp"
-      field: network.protocol
-      value: rdp
-  ## RFB (Remote Framebuffer Protocol)
-  - append:
-      if: ctx?.suricata?.eve?.event_type == "rfb"
-      field: event.type
-      value:
-        - protocol
-  - set:
-      if: ctx?.suricata?.eve?.event_type == "rfb"
-      field: network.protocol
-      value: rfb
-  ## MQTT
-  - append:
-      if: ctx?.suricata?.eve?.event_type == "mqtt"
-      field: event.type
-      value:
-        - protocol
-  - set:
-      if: ctx?.suricata?.eve?.event_type == "mqtt"
-      field: network.protocol
-      value: mqtt
-  ## HTTP2
-  - append:
-      if: ctx?.suricata?.eve?.event_type == "http2"
-      field: event.category
-      value:
-        - web
-  - append:
-      if: ctx?.suricata?.eve?.event_type == "http2"
-      field: event.type
-      value:
-        - access
-        - protocol
-  - set:
-      if: ctx?.suricata?.eve?.event_type == "http2"
-      field: network.protocol
-      value: http2
-  ## Stats
-  - set:
-      if: ctx?.suricata?.eve?.event_type == "stats"
-      field: event.kind
-      value: metric
+
 
   - set:
       value: "{{suricata.eve.http.http_method}}"