Set ECS NAT fields in PAN-OS fileset (#13330)

This updates the panw.panos fileset to populate the ECS NAT fields. The original non-ECS fields are still being populated to not break backwards compatibility. Relates #13320
elastic · Aug 26, 2019 · 3e34fb6 · 3e34fb6
1 parent c961365
commit 3e34fb6
Show file tree

Hide file tree

Showing 8 changed files with 1,537 additions and 18 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -274,6 +274,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add module for ingesting IBM MQ logs. {pull}8782[8782]
 - Add S3 input to retrieve logs from AWS S3 buckets. {pull}12640[12640] {issue}12582[12582]
 - Add aws module s3access metricset. {pull}13170[13170] {issue}12880[12880]
+- Update PAN-OS fileset to use the ECS NAT fields. {issue}13320[13320] {pull}13330[13330]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/panw/panos/config/input.yml b/x-pack/filebeat/module/panw/panos/config/input.yml
@@ -49,8 +49,8 @@ processors:
         server.ip: 8
         destination.ip: 8
         destination.address: 8
-        panw.panos.source.nat.ip: 9
-        panw.panos.destination.nat.ip: 10
+        source.nat.ip: 9
+        destination.nat.ip: 10
         panw.panos.ruleset: 11
         client.user.name: 12
         source.user.name: 12
@@ -66,8 +66,8 @@ processors:
         source.port: 24
         destination.port: 25
         server.port: 25
-        panw.panos.source.nat.port: 26
-        panw.panos.destination.nat.port: 27
+        source.nat.port: 26
+        destination.nat.port: 27
         _temp_.labels: 28
         network.transport: 29
         event.outcome: 30
@@ -102,8 +102,8 @@ processors:
         server.ip: 8
         destination.ip: 8
         destination.address: 8
-        panw.panos.source.nat.ip: 9
-        panw.panos.destination.nat.ip: 10
+        source.nat.ip: 9
+        destination.nat.ip: 10
         panw.panos.ruleset: 11
         client.user.name: 12
         source.user.name: 12
@@ -119,8 +119,8 @@ processors:
         source.port: 24
         destination.port: 25
         server.port: 25
-        panw.panos.source.nat.port: 26
-        panw.panos.destination.nat.port: 27
+        source.nat.port: 26
+        destination.nat.port: 27
         _temp_.labels: 28
         network.transport: 29
         event.outcome: 30
@@ -152,7 +152,17 @@ processors:
   - community_id:
       target: panw.panos.network.nat.community_id
       fields:
-        source_ip: panw.panos.source.nat.ip
-        source_port: panw.panos.source.nat.port
-        destination_ip: panw.panos.destination.nat.ip
-        destination_port: panw.panos.destination.nat.port
+        source_ip: source.nat.ip
+        source_port: source.nat.port
+        destination_ip: destination.nat.ip
+        destination_port: destination.nat.port
+
+  # Copy NAT data from ECS fields to the original non-ECS fields to retain
+  # backward compatibility. This should be removed for 8.0.
+  - convert:
+      ignore_missing: true
+      fields:
+        - {from: source.nat.ip, to: panw.panos.source.nat.ip, type: ip}
+        - {from: destination.nat.ip, to: panw.panos.destination.nat.ip, type: ip}
+        - {from: source.nat.port, to: panw.panos.source.nat.port, type: long}
+        - {from: destination.nat.port, to: panw.panos.destination.nat.port, type: long}
diff --git a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml
@@ -67,8 +67,8 @@ processors:
  - convert: { type: long, ignore_missing: true, field: event.duration }
  - convert: { type: long, ignore_missing: true, field: _temp_.labels }
  - convert: { type: long, ignore_missing: true, field: panw.panos.sequence_number }
- - convert: { type: long, ignore_missing: true, field: panw.panos.source.nat.port }
- - convert: { type: long, ignore_missing: true, field: panw.panos.destination.nat.port }
+ - convert: { type: long, ignore_missing: true, field: source.nat.port }
+ - convert: { type: long, ignore_missing: true, field: destination.nat.port }
 
 # Remove PCAP ID when zero (no packet capture).
  - remove:
@@ -298,15 +298,15 @@ processors:
      value:
        - '{{destination.ip}}'
  - append:
-     if: 'ctx?.panw?.panos?.source?.nat?.ip != null'
+     if: 'ctx?.source?.nat?.ip != null'
      field: related.ip
      value:
-       - '{{panw.panos.source.nat.ip}}'
+       - '{{source.nat.ip}}'
  - append:
-     if: 'ctx?.panw?.panos?.destination?.nat?.ip != null'
+     if: 'ctx?.destination?.nat?.ip != null'
      field: related.ip
      value:
-       - '{{panw.panos.destination.nat.ip}}'
+       - '{{destination.nat.ip}}'
 
 # Geolocation for source.
  - geoip:

diff --git a/x-pack/filebeat/module/panw/panos/test/pan_inc_other.log-expected.json b/x-pack/filebeat/module/panw/panos/test/pan_inc_other.log-expected.json
@@ -18,6 +18,8 @@
         "destination.geo.region_iso_code": "US-NC",
         "destination.geo.region_name": "North Carolina",
         "destination.ip": "204.232.231.46",
+        "destination.nat.ip": "0.0.0.0",
+        "destination.nat.port": 0,
         "destination.packets": 1,
         "destination.port": 80,
         "event.action": "flow_started",
@@ -68,6 +70,8 @@
         "source.address": "192.168.0.2",
         "source.bytes": 0,
         "source.ip": "192.168.0.2",
+        "source.nat.ip": "0.0.0.0",
+        "source.nat.port": 0,
         "source.packets": 0,
         "source.port": 59309,
         "source.user.name": "crusher",