[Barracuda CloudGen] Add initial Barracuda CloudGen Firewall integrat…

…ion (#3796)

Add initial Barracuda CloudGen Firewall integration for receiving Firewall Insight logs as described at https://campus.barracuda.com/product/cloudgenfirewall/doc/96025953/how-to-enable-filebeat-stream-to-a-logstash-pipeline. Elastic Agent starts a server to receive data sent over the Lumberjack protocol by CloudGen firewall. (This is the same protocol used between Beats and Logstash.)

Co-authored-by: Andrew Kroh <[email protected]>

.github/CODEOWNERS

@@ -26,6 +26,7 @@
 /packages/azure @elastic/obs-cloud-monitoring
 /packages/azure_metrics @elastic/obs-cloud-monitoring
 /packages/barracuda @elastic/security-external-integrations
+/packages/barracuda_cloudgen_firewall @elastic/security-external-integrations
 /packages/bluecoat @elastic/security-external-integrations
 /packages/box_events @elastic/security-external-integrations
 /packages/carbon_black_cloud @elastic/security-external-integrations

packages/barracuda_cloudgen_firewall/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: [email protected]

packages/barracuda_cloudgen_firewall/_dev/build/docs/README.md

@@ -0,0 +1,28 @@
+# Barracuda CloudGen Firewall integration
+
+This integration ingests and parses logs from
+[Barracuda CloudGen Firewalls](https://www.barracuda.com/products/cloudgenfirewall).
+
+Barracuda CloudGen Firewall allows you to stream event logs from Firewall
+Insights to Elastic Agent. This provides information on firewall activity,
+threat logs, and information related to network, version, and location of
+managed firewall units. Data is sent to Elastic Agent over a TCP connection
+using CloudGen Firewall's built-in generic Logstash output.
+
+### Setup
+
+For a detailed walk-through of the setup steps the see
+[How to Enable Filebeat Stream to a Logstash Pipeline](https://campus.barracuda.com/product/cloudgenfirewall/doc/96025953/how-to-enable-filebeat-stream-to-a-logstash-pipeline/).
+These steps were written with a Logstash server as the intended destination, and
+where it references the "Hostname" use the address and port of the Elastic Agent
+that is running this integration. Logstash is not used as part of this
+integration.
+
+## Logs
+
+This is the Barracuda CloudGen Firewall `log` dataset. Below is a sample
+event and a list of fields that can be produced.
+
+{{event "log"}}
+
+{{fields "log"}}

packages/barracuda_cloudgen_firewall/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,17 @@
+version: '2.3'
+services:
+  barracuda-cloudgen-lumberjack:
+    image: docker.elastic.co/observability/stream:v0.8.0
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    environment:
+      - STREAM_PROTOCOL=lumberjack
+      - STREAM_LUMBERJACK_PARSE_JSON=true
+      - STREAM_ADDR=tcp://elastic-agent:5044
+      - STREAM_DELAY=5s
+      - STREAM_START_SIGNAL=SIGHUP
+    # The ndjson files contain data that simulates the format of the Logstash
+    # output from the embedded Filebeat instance in Barracuda CloudGen.
+    # It contains a JSON string in the 'message' field and some additional
+    # firewall metadata (like serial number) stored in root level fields.
+    command: log /sample_logs/*.ndjson

packages/barracuda_cloudgen_firewall/_dev/deploy/docker/sample_logs/firewall.ndjson

@@ -0,0 +1 @@
+{"beat": {"hostname": "cgf-scout-int"}, "message":"{\"version\":1,\"timestamp\":1606230141,\"action\":\"End\",\"duration\":8436,\"src_iface\":\"eth0\",\"src_ip\":\"10.17.35.171\",\"src_port\":40532,\"src_mac\":\"00:0c:29:9a:0a:78\",\"dst_iface\":\"eth0\",\"dst_ip\":\"67.43.156.78\",\"dst_port\":443,\"dst_mac\":\"00:0c:29:00:d6:00\",\"fw_rule\":\"BOX-LAN-2-INTERNET\",\"app_rule\":\"<App>:ALL-APPS\",\"fw_info\":2007,\"src_ip_nat\":\"10.17.35.175\",\"dst_ip_nat\":\"67.43.156.100\",\"fwd_bytes\":7450,\"rev_bytes\":561503,\"fwd_packets\":129,\"rev_packets\":439,\"ip_proto\":6,\"protos\":[\"HTTPS direct\",\"HTTPS\",\"All HTTP protocols\"],\"apps\":[\"Web browsing\"]}","product":"ngfw","sn":"4f94abdf7a8c465fa2cd76f680ecafd1","type":"ngfw-act"}

packages/barracuda_cloudgen_firewall/_dev/deploy/docker/sample_logs/thread.ndjson

@@ -0,0 +1,3 @@
+{"beat": {"hostname": "cgf-scout-int"}, "message":"{\"app_target\":\"eicar.exe\",\"component\":\"firewall\",\"date\":\"2018 05 15\",\"description\":\"Eicar-Test-Signature\",\"dst_ip\":\"10.0.6.96\",\"operation\":\"Block\",\"port\":\"443\",\"severity\":\"Warning\",\"src_ip\":\"10.17.35.169\",\"threat_severity\":\"3\",\"time\":\"15:42:27\",\"timestamp\":\"2018-05-15T15:42:27+00:00\",\"timezone\":\"+00:00\",\"trans_proto\":\"TCP\",\"type\":\"Virus\",\"user\":\"user42\"}","product":"ngfw","sn":"4f94abdf7a8c465fa2cd76f680ecafd1","type":"ngfw-threat"}
+{"beat": {"hostname": "cgf-scout-int"}, "message":"{\"app_target\":\"boese.pdf\",\"component\":\"firewall\",\"date\":\"2018 05 15\",\"description\":\"ad43f5fc1d679c8d766824abb41b2b28b364c3c8;.pdf\",\"dst_ip\":\"89.160.20.129\",\"operation\":\"Block\",\"port\":\"80\",\"severity\":\"Warning\",\"src_ip\":\"10.17.35.169\",\"threat_severity\":\"3\",\"time\":\"15:42:32\",\"timestamp\":\"2018-05-15T15:42:32+00:00\",\"timezone\":\"+00:00\",\"trans_proto\":\"TCP\",\"type\":\"ATD\",\"user\":\"user42\"}","product":"ngfw","sn":"4f94abdf7a8c465fa2cd76f680ecafd1","type":"ngfw-threat"}
+{"beat": {"hostname": "cgf-scout-int"}, "message":"{\"component\":\"firewall\",\"date\":\"2018 05 15\",\"description\":\"ID: 1054837 WEB Remote File Inclusion /etc/passwd\",\"dst_ip\":\"89.160.20.130\",\"ips_category\":\"Web Attack\",\"operation\":\"Block\",\"port\":\"80\",\"severity\":\"Warning\",\"src_ip\":\"10.17.35.169\",\"threat_severity\":\"3\",\"time\":\"15:46:06\",\"timestamp\":\"2018-05-15T15:46:06+00:00\",\"timezone\":\"+00:00\",\"trans_proto\":\"TCP\",\"type\":\"IPS\",\"user\":\"user45\"}","product":"ngfw","sn":"4f94abdf7a8c465fa2cd76f680ecafd1","type":"ngfw-threat"}

packages/barracuda_cloudgen_firewall/_dev/deploy/docker/sample_logs/web.ndjson

@@ -0,0 +1,2 @@
+{"beat": {"hostname": "cgf-scout-int"}, "message":"{\"timestamp\":1526383397000,\"traffic_type\":0,\"action\":0,\"source_ip\":\"192.168.42.124\",\"source_port\":\"50646\",\"destination_ip\":\"175.16.199.12\",\"destination_port\":\"443\",\"method\":\"GET\",\"status_code\":\"200\",\"user_agent\":\"wget/1.19.2 (linux-gnu)\",\"content_type\":\"text/html; charset=UTF-8\",\"name\":\"https://www.heise.de/\",\"size\":59558,\"domain\":\"www.heise.de\",\"category\":[\"79\"],\"user\":\"192.168.42.124\",\"user_type\":0,\"fw_rule\":\"LAN-2-INTERNET\",\"app_rule\":\"<App>:<pass-no-match>\"}","product":"ngfw","sn":"4f94abdf7a8c465fa2cd76f680ecafd1","type":"ngfw-wf"}
+{"beat": {"hostname": "cgf-scout-int"}, "message":"{\"timestamp\":1526377804000,\"traffic_type\":0,\"action\":0,\"source_ip\":\"192.168.42.105\",\"source_port\":\"50159\",\"destination_ip\":\"89.160.20.114\",\"destination_port\":\"443\",\"method\":\"GET\",\"status_code\":\"200\",\"user_agent\":\"mozilla/5.0 (windows nt 6.1) applewebkit/537.36 (khtml, like gecko) chrome/66.0.3359.139 safari/537.36\",\"content_type\":\"\",\"name\":\"https://clientservices.googleapis.com/chrome-variations/seed?osname=win&channel=stable&milestone=66\",\"size\":0,\"domain\":\"clientservices.googleapis.com\",\"category\":[],\"user\":\"192.168.42.105\",\"user_type\":0,\"fw_rule\":\"LAN-2-INTERNET\",\"app_rule\":\"<App>:<pass-no-match>\"}","product":"ngfw","sn":"4f94abdf7a8c465fa2cd76f680ecafd1","type":"ngfw-wf"}

packages/barracuda_cloudgen_firewall/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.1.0"
+  changes:
+    - description: initial release
+      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/package-storage/pull/3796

packages/barracuda_cloudgen_firewall/data_stream/log/_dev/test/pipeline/test-firewall.log

@@ -0,0 +1 @@
+{"version":1,"timestamp":1606230141,"action":"End","duration":8436,"src_iface":"eth0","src_ip":"10.17.35.171","src_port":40532,"src_mac":"00:0c:29:9a:0a:78","dst_iface":"eth0","dst_ip":"67.43.156.78","dst_port":443,"dst_mac":"00:0c:29:00:d6:00","fw_rule":"BOX-LAN-2-INTERNET","app_rule":"<App>:ALL-APPS","fw_info":2007,"src_ip_nat":"10.17.35.175","dst_ip_nat":"67.43.156.100","fwd_bytes":7450,"rev_bytes":561503,"fwd_packets":129,"rev_packets":439,"ip_proto":6,"protos":["HTTPS direct","HTTPS","All HTTP protocols"],"apps":["Web browsing"]}

packages/barracuda_cloudgen_firewall/data_stream/log/_dev/test/pipeline/test-firewall.log-config.yml

@@ -0,0 +1,9 @@
+fields:
+  tags:
+    - preserve_original_event
+  lumberjack:
+    type: ngfw-act
+    sn: 4f94abdf7a8c465fa2cd76f680ecafd1
+    product: ngfw
+    beat:
+      hostname: cgf-scout-int

packages/barracuda_cloudgen_firewall/data_stream/log/_dev/test/pipeline/test-firewall.log-expected.json

@@ -0,0 +1,97 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2020-11-24T15:02:21.000Z",
+            "barracuda_cloudgen_firewall": {
+                "log": {
+                    "app_rule": "\u003cApp\u003e:ALL-APPS",
+                    "fw_info": 2007
+                }
+            },
+            "destination": {
+                "address": "67.43.156.78",
+                "as": {
+                    "number": 35908
+                },
+                "bytes": 561503,
+                "geo": {
+                    "continent_name": "Asia",
+                    "country_iso_code": "BT",
+                    "country_name": "Bhutan",
+                    "location": {
+                        "lat": 27.5,
+                        "lon": 90.5
+                    }
+                },
+                "ip": "67.43.156.78",
+                "mac": "00-0C-29-00-D6-00",
+                "nat": {
+                    "ip": "67.43.156.100"
+                },
+                "packets": 439,
+                "port": 443
+            },
+            "ecs": {
+                "version": "8.4.0"
+            },
+            "event": {
+                "action": "End",
+                "category": [
+                    "network"
+                ],
+                "duration": 8436000000,
+                "kind": "event",
+                "original": "{\"version\":1,\"timestamp\":1606230141,\"action\":\"End\",\"duration\":8436,\"src_iface\":\"eth0\",\"src_ip\":\"10.17.35.171\",\"src_port\":40532,\"src_mac\":\"00:0c:29:9a:0a:78\",\"dst_iface\":\"eth0\",\"dst_ip\":\"67.43.156.78\",\"dst_port\":443,\"dst_mac\":\"00:0c:29:00:d6:00\",\"fw_rule\":\"BOX-LAN-2-INTERNET\",\"app_rule\":\"\u003cApp\u003e:ALL-APPS\",\"fw_info\":2007,\"src_ip_nat\":\"10.17.35.175\",\"dst_ip_nat\":\"67.43.156.100\",\"fwd_bytes\":7450,\"rev_bytes\":561503,\"fwd_packets\":129,\"rev_packets\":439,\"ip_proto\":6,\"protos\":[\"HTTPS direct\",\"HTTPS\",\"All HTTP protocols\"],\"apps\":[\"Web browsing\"]}",
+                "type": [
+                    "end"
+                ]
+            },
+            "network": {
+                "community_id": "1:HGU1tX9W2VUF5ND2ey3X6Niv/AQ=",
+                "iana_number": "6",
+                "transport": "tcp",
+                "type": "ipv4"
+            },
+            "observer": {
+                "egress": {
+                    "interface": {
+                        "name": "eth0"
+                    }
+                },
+                "hostname": "cgf-scout-int",
+                "ingress": {
+                    "interface": {
+                        "name": "eth0"
+                    }
+                },
+                "product": "ngfw",
+                "serial_number": "4f94abdf7a8c465fa2cd76f680ecafd1",
+                "type": "firewall",
+                "vendor": "Barracuda"
+            },
+            "related": {
+                "ip": [
+                    "10.17.35.171",
+                    "67.43.156.78"
+                ]
+            },
+            "rule": {
+                "name": "BOX-LAN-2-INTERNET"
+            },
+            "source": {
+                "address": "10.17.35.171",
+                "bytes": 7450,
+                "ip": "10.17.35.171",
+                "mac": "00-0C-29-9A-0A-78",
+                "nat": {
+                    "ip": "10.17.35.175"
+                },
+                "packets": 129,
+                "port": 40532
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        }
+    ]
+}

packages/barracuda_cloudgen_firewall/data_stream/log/_dev/test/pipeline/test-threat.log

@@ -0,0 +1,3 @@
+{"app_target":"eicar.exe","component":"firewall","date":"2018 05 15","description":"Eicar-Test-Signature","dst_ip":"10.0.6.96","operation":"Block","port":"443","severity":"Warning","src_ip":"10.17.35.169","threat_severity":"3","time":"15:42:27","timestamp":"2018-05-15T15:42:27+00:00","timezone":"+00:00","trans_proto":"TCP","type":"Virus","user":"user42"}
+{"app_target":"boese.pdf","component":"firewall","date":"2018 05 15","description":"ad43f5fc1d679c8d766824abb41b2b28b364c3c8;.pdf","dst_ip":"89.160.20.129","operation":"Block","port":"80","severity":"Warning","src_ip":"10.17.35.169","threat_severity":"3","time":"15:42:32","timestamp":"2018-05-15T15:42:32+00:00","timezone":"+00:00","trans_proto":"TCP","type":"ATD","user":"user42"}
+{"component":"firewall","date":"2018 05 15","description":"ID: 1054837 WEB Remote File Inclusion /etc/passwd","dst_ip":"89.160.20.130","ips_category":"Web Attack","operation":"Block","port":"80","severity":"Warning","src_ip":"10.17.35.169","threat_severity":"3","time":"15:46:06","timestamp":"2018-05-15T15:46:06+00:00","timezone":"+00:00","trans_proto":"TCP","type":"IPS","user":"user45"}

packages/barracuda_cloudgen_firewall/data_stream/log/_dev/test/pipeline/test-threat.log-config.yml

@@ -0,0 +1,9 @@
+fields:
+  tags:
+    - preserve_original_event
+  lumberjack:
+    type: ngfw-threat
+    sn: 4f94abdf7a8c465fa2cd76f680ecafd1
+    product: ngfw
+    beat:
+      hostname: cgf-scout-int