Cherry-pick #17910 to 7.x: [Filebeat] Improve ECS field mappings in p…

…anw module (#17943)

* Improve ECS field mappings in panw module (#17910)

- panw.panos.action
- event.outcome, limit to succes/failure
- event.kind
- event.category, make array
- event.type, make array
- rule.name
- related.user
- mage fmt update

Closes #16025

(cherry picked from commit e174441)

CHANGELOG.next.asciidoc

@@ -32,6 +32,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
 - CEF extensions are now mapped to the data types defined in the CEF guide. {pull}14342[14342]
+- Improve ECS field mappings in panw module.  event.outcome now only contains success/failure per ECS specification. {issue}16025[16025] {pull}17910[17910]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -28240,6 +28240,15 @@ type: keyword
 Palo Alto Networks name for the threat.
 
 
+type: keyword
+
+--
+
+*`panw.panos.action`*::
++
+--
+Action taken for the session.
+
 type: keyword
 
 --

x-pack/filebeat/module/panw/panos/_meta/fields.yml

@@ -127,3 +127,7 @@
       type: keyword
       description: >
         Palo Alto Networks name for the threat.
+    - name: action
+      type: keyword
+      description: >-
+        Action taken for the session.

x-pack/filebeat/module/panw/panos/config/input.yml

@@ -70,7 +70,7 @@ processors:
         destination.nat.port: 27
         _temp_.labels: 28
         network.transport: 29
-        event.outcome: 30
+        panw.panos.action: 30
         network.bytes: 31
         client.bytes: 32
         destination.bytes: 32
@@ -123,7 +123,7 @@ processors:
         destination.nat.port: 27
         _temp_.labels: 28
         network.transport: 29
-        event.outcome: 30
+        panw.panos.action: 30
         panw.panos.threat.resource: 31
         url.original: 31
         panw.panos.threat.name: 32

x-pack/filebeat/module/panw/panos/ingest/pipeline.yml

@@ -175,34 +175,82 @@ processors:
 
   # Set event.category depending on log type.
  - set:
+     field: event.kind
+     value: event
+     if: 'ctx?._temp_?.message_type == "TRAFFIC"'
+ - append:
      field: event.category
-     value: network_traffic
+     value:
+       - network_traffic
+       - network
      if: 'ctx?._temp_?.message_type == "TRAFFIC"'
  - set:
+     field: event.kind
+     value: alert
+     if: 'ctx?._temp_?.message_type == "THREAT"'
+ - append:
      field: event.category
-     value: security_threat
+     value:
+       - security_threat
+       - intrusion_detection
+       - network
      if: 'ctx?._temp_?.message_type == "THREAT"'
-
  - drop:
      if: 'ctx?.event?.category == null'
+ - append:
+     field: event.type
+     value: allowed
+     if: "ctx?.panw?.panos?.action != null && ['alert', 'allow', 'continue'].contains(ctx.panw.panos.action)"
+ - append:
+     field: event.type
+     value: denied
+     if: "ctx?.panw?.panos?.action != null && ['deny', 'drop', 'reset-client', 'reset-server', 'reset-both', 'block-url', 'block-ip', 'random-drop', 'sinkhole', 'block'].contains(ctx.panw.panos.action)"
+ - set:
+     field: event.outcome
+     value: success
+
 
 # event.action for traffic logs.
  - set:
      field: event.action
      value: flow_started
      if: 'ctx?._temp_?.message_subtype == "start"'
+ - append:
+     field: event.type
+     value:
+       - start
+       - connection
+     if: 'ctx?._temp_?.message_subtype == "start"'
  - set:
      field: event.action
      value: flow_terminated
      if: 'ctx?._temp_?.message_subtype == "end"'
+ - append:
+     field: event.type
+     value:
+       - end
+       - connection
+     if: 'ctx?._temp_?.message_subtype == "end"'
  - set:
      field: event.action
      value: flow_dropped
      if: 'ctx?._temp_?.message_subtype == "drop"'
+ - append:
+     field: event.type
+     value:
+       - denied
+       - connection
+     if: 'ctx?._temp_?.message_subtype == "drop"'
  - set:
      field: event.action
      value: flow_denied
      if: 'ctx?._temp_?.message_subtype == "deny"'
+ - append:
+     field: event.type
+     value:
+       - denied
+       - connection
+     if: 'ctx?._temp_?.message_subtype == "deny"'
 
 # event.action for threat logs.
  - set:
@@ -276,21 +324,21 @@ processors:
 # Normalize event.outcome.
 # These values appear in the TRAFFIC docs but look like a mistake.
  - set:
-     field: event.outcome
+     field: panw.panos.action
      value: 'drop-icmp'
-     if: 'ctx?.event?.outcome == "drop icmp" || ctx?.event?.outcome == "drop ICMP"'
+     if: 'ctx?.panw?.panos?.action == "drop icmp" || ctx?.panw?.panos?.action == "drop ICMP"'
  - set:
-     field: event.outcome
+     field: panw.panos.action
      value: 'reset-both'
-     if: 'ctx?.event?.outcome == "reset both"'
+     if: 'ctx?.panw?.panos?.action == "reset both"'
  - set:
-     field: event.outcome
+     field: panw.panos.action
      value: 'reset-client'
-     if: 'ctx?.event?.outcome == "reset client"'
+     if: 'ctx?.panw?.panos?.action == "reset client"'
  - set:
-     field: event.outcome
+     field: panw.panos.action
      value: 'reset-server'
-     if: 'ctx?.event?.outcome == "reset server"'
+     if: 'ctx?.panw?.panos?.action == "reset server"'
 
 # Build related.ip array from src/dest/NAT IPs.
  - append:
@@ -391,6 +439,36 @@ processors:
      value: 'URL-filtering'
      if: 'ctx?.panw?.panos?.threat?.id == "9999"'
 
+ - set:
+     field: rule.name
+     value: "{{panw.panos.ruleset}}"
+     if: "ctx?.panw?.panos?.ruleset != null"
+
+ - append:
+     field: related.user
+     value: "{{client.user.name}}"
+     if: "ctx?.client?.user?.name != null"
+
+ - append:
+     field: related.user
+     value: "{{source.user.name}}"
+     if: "ctx?.source?.user?.name != null"
+
+ - append:
+     field: related.user
+     value: "{{server.user.name}}"
+     if: "ctx?.server?.user?.name != null"
+
+ - append:
+     field: related.user
+     value: "{{destination.user.name}}"
+     if: "ctx?.destination?.user?.name != null"
+
+ - append:
+     field: related.hash
+     value: "{{panw.panos.file.hash}}"
+     if: "ctx?.panw?.panos?.file?.hash != null"
+
 # Remove temporary fields.
  - remove:
      field:

x-pack/filebeat/module/panw/panos/test/pan_inc_other.log-expected.json

@@ -23,14 +23,23 @@
         "destination.packets": 1,
         "destination.port": 80,
         "event.action": "flow_started",
-        "event.category": "network_traffic",
+        "event.category": [
+            "network_traffic",
+            "network"
+        ],
         "event.dataset": "panw.panos",
         "event.duration": 0,
         "event.end": "2012-04-10T04:39:56.000-02:00",
+        "event.kind": "event",
         "event.module": "panw",
-        "event.outcome": "allow",
+        "event.outcome": "success",
         "event.start": "2012-04-10T04:39:56.000-02:00",
         "event.timezone": "-02:00",
+        "event.type": [
+            "allowed",
+            "start",
+            "connection"
+        ],
         "fileset.name": "panos",
         "input.type": "log",
         "labels.captive_portal": true,
@@ -44,6 +53,7 @@
         "network.transport": "tcp",
         "network.type": "ipv4",
         "observer.serial_number": "01606001116",
+        "panw.panos.action": "allow",
         "panw.panos.destination.interface": "ethernet1/1",
         "panw.panos.destination.nat.ip": "0.0.0.0",
         "panw.panos.destination.nat.port": 0,
@@ -62,6 +72,11 @@
             "0.0.0.0",
             "0.0.0.0"
         ],
+        "related.user": [
+            "crusher",
+            "crusher"
+        ],
+        "rule.name": "rule1",
         "server.bytes": 0,
         "server.ip": "204.232.231.46",
         "server.packets": 1,