Update CheckPoint package with RFC 5424 parsing fix (elastic#351)

* Update CheckPoint package with RFC 5424 parsing fix

Sync the change from elastic/beats#21854 to use the UDP input instead of syslog input
to allow for RFC 5424 syslog parsing.

* paths is required for logfile

packages/checkpoint/data_stream/firewall/agent/stream/syslog.yml.hbs → packages/checkpoint/data_stream/firewall/agent/stream/udp.yml.hbs

@@ -1,5 +1,4 @@
-protocol.udp:
-  host: "{{syslog_host}}:{{syslog_port}}"
+host: "{{syslog_host}}:{{syslog_port}}"
 tags:
 {{#each tags as |tag i|}}
 - {{tag}}

packages/checkpoint/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml

@@ -1,6 +1,9 @@
 ---
 description: Pipeline for parsing checkpoint firewall logs
 processors:
+- set:
+    field: event.ingested
+    value: '{{_ingest.timestamp}}'
 - grok:
     field: message
     patterns:
@@ -38,10 +41,14 @@ processors:
     - message
     - host
     ignore_missing: true
-- set:
-    field: '@timestamp'
-    value: '{{syslog5424_ts}}'
-    if: ctx.checkpoint?.time == null
+- rename:
+    field: "@timestamp"
+    target_field: "event.created"
+    ignore_missing: true
+- date:
+    field: "syslog5424_ts"
+    formats: ["ISO8601", "UNIX"]
+    if: "ctx.checkpoint?.time == null"
 - set:
     field: event.module
     value: checkpoint
@@ -158,7 +165,7 @@ processors:
     target_field: source.nat.port
     type: long
     ignore_failure: true
-    ignore_missing: true 
+    ignore_missing: true
     if: "ctx.checkpoint?.xlatesport != '0'"
 - rename:
     field: checkpoint.mac_source_address
@@ -576,10 +583,10 @@ processors:
     field: checkpoint.industry_reference
     target_field: vulnerability.id
     ignore_missing: true
-- rename:
-    field: checkpoint.time
-    target_field: '@timestamp'
-    ignore_missing: true
+- date:
+    field: "checkpoint.time"
+    formats: ["ISO8601", "UNIX"]
+    if: "ctx.checkpoint?.time != null"
 - rename:
     field: checkpoint.message
     target_field: message
@@ -692,7 +699,7 @@ processors:
     field: client.nat.port
     type: long
     ignore_failure: true
-    ignore_missing: true 
+    ignore_missing: true
 - convert:
     field: client.bytes
     type: long
@@ -712,7 +719,7 @@ processors:
     field: server.nat.port
     type: long
     ignore_failure: true
-    ignore_missing: true 
+    ignore_missing: true
 - convert:
     field: server.bytes
     type: long
@@ -722,7 +729,7 @@ processors:
     field: server.packets
     type: long
     ignore_failure: true
-    ignore_missing: true 
+    ignore_missing: true
 - script:
     lang: painless
     source: "ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes"
@@ -793,9 +800,10 @@ processors:
     - checkpoint.xlatesrc
     - checkpoint.xlatedst
     - checkpoint.uid
+    - checkpoint.time
     - syslog5424_ts
     ignore_missing: true
 on_failure:
 - set:
     field: error.message
-    value: '{{ _ingest.on_failure_message }}'
+    value: '{{ _ingest.on_failure_message }}'

packages/checkpoint/data_stream/firewall/fields/base-fields.yml

@@ -1,12 +1,12 @@
 - name: data_stream.type
   type: constant_keyword
-  description: Datastream type.
+  description: Data stream type.
 - name: data_stream.dataset
   type: constant_keyword
-  description: Datastream dataset name.
+  description: Data stream dataset.
 - name: data_stream.namespace
   type: constant_keyword
-  description: Datastream namespace.
-- name: "@timestamp"
+  description: Data stream namespace.
+- name: '@timestamp'
   type: date
   description: Event timestamp.

packages/checkpoint/data_stream/firewall/fields/ecs.yml

@@ -124,6 +124,12 @@
 - description: Event category.
   name: event.category
   type: keyword
+- description: Time when the event was first read by an agent or by your pipeline.
+  name: event.created
+  type: date
+- description: Timestamp when an event arrived in the central data store.
+  name: event.ingested
+  type: date
 - description: Contains the date when the event ended.
   name: event.end
   type: date

packages/checkpoint/data_stream/firewall/manifest.yml

@@ -2,10 +2,10 @@ type: logs
 title: Check Point firewall logs
 release: experimental
 streams:
-  - input: syslog
-    template_path: syslog.yml.hbs
-    title: Check Point firewall logs (syslog)
-    description: Collect Check Point firewall logs using syslog input
+  - input: udp
+    template_path: udp.yml.hbs
+    title: Check Point firewall logs (syslog over UDP)
+    description: Collect Check Point firewall logs using udp input
   - input: logfile
     template_path: log.yml.hbs
     title: Check Point firewall logs (log)

packages/checkpoint/docs/README.md

@@ -435,9 +435,9 @@ Consists of log entries from the Log Exporter in the Syslog format.
 | client.user.id | Unique identifier of the user. | keyword |
 | client.user.name | Short name or login of the user. | keyword |
 | container.id | Unique container id. | keyword |
-| data_stream.dataset | Datastream dataset name. | constant_keyword |
-| data_stream.namespace | Datastream namespace. | constant_keyword |
-| data_stream.type | Datastream type. | constant_keyword |
+| data_stream.dataset | Data stream dataset. | constant_keyword |
+| data_stream.namespace | Data stream namespace. | constant_keyword |
+| data_stream.type | Data stream type. | constant_keyword |
 | destination.as.number | Unique number allocated to the autonomous system. | long |
 | destination.as.organization.name | Organization name. | keyword |
 | destination.bytes | Bytes sent from the destination to the source. | long |
@@ -468,8 +468,10 @@ Consists of log entries from the Log Exporter in the Syslog format.
 | error.message | Error message. | text |
 | event.action | The action captured by the event. | keyword |
 | event.category | Event category. | keyword |
+| event.created | Time when the event was first read by an agent or by your pipeline. | date |
 | event.end | Contains the date when the event ended. | date |
 | event.id | Unique ID to describe the event. | keyword |
+| event.ingested | Timestamp when an event arrived in the central data store. | date |
 | event.kind | The kind of the event. | keyword |
 | event.module | Name of the module this data is coming from. | keyword |
 | event.outcome | The outcome of the event. | keyword |

packages/checkpoint/manifest.yml

@@ -1,6 +1,6 @@
 name: checkpoint
 title: Check Point
-version: 0.2.1
+version: 0.2.2
 release: experimental
 description: Check Point Integration
 type: integration
@@ -36,9 +36,9 @@ policy_templates:
             type: text
             title: Paths
             multi: true
-            required: false
+            required: true
             show_user: true
-      - type: syslog
+      - type: udp
         vars:
           - name: syslog_host
             type: text
@@ -63,7 +63,7 @@ policy_templates:
             required: true
             show_user: true
             default: 9001
-        title: "Collect Check Point firewall logs (input: syslog)"
-        description: "Collecting firewall logs from Check Point instances (input: syslog)"
+        title: "Collect Check Point firewall logs (input: udp)"
+        description: "Collecting firewall logs from Check Point instances (input: udp)"
 owner:
   github: elastic/security-external-integrations