From d07d890932c339c6968d940f6b5a9bab6bca2d8c Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 14 Dec 2020 14:45:43 -0500 Subject: [PATCH] Add system and pipeline tests for Suricata EVE (#457) This adds tests and update the Suricata pipeline. - Sync the pipeline from beats e9d12e2119ff58. - Convert suricata.eve.flow_id to string because the field is a keyword in the mapping. - Add missing ECS field definitions. --- .../suricata/_dev/deploy/docker/Dockerfile | 5 + .../_dev/deploy/docker/docker-compose.yml | 8 + .../_dev/deploy/docker/eve-alerts.ndjson | 22 ++ .../_dev/deploy/docker/eve-dns-4.1.4.ndjson | 24 ++ .../_dev/deploy/docker/eve-small.ndjson | 8 + .../eve/_dev/test/pipeline/test-events.json | 173 +++++++++ .../pipeline/test-events.json-config.json | 5 + .../pipeline/test-events.json-expected.json | 284 ++++++++++++++ .../eve/_dev/test/system/config.yml | 6 + .../data_stream/eve/agent/stream/log.yml.hbs | 363 +----------------- .../elasticsearch/ingest_pipeline/default.yml | 270 +++++++------ .../ingest_pipeline/dns-answer-v1.yml | 39 ++ .../ingest_pipeline/dns-answer-v2.yml | 42 ++ .../eve/elasticsearch/ingest_pipeline/dns.yml | 93 +++++ .../eve/elasticsearch/ingest_pipeline/tls.yml | 188 +++++++++ .../suricata/data_stream/eve/fields/ecs.yml | 111 ++++++ packages/suricata/docs/README.md | 39 +- packages/suricata/manifest.yml | 2 +- 18 files changed, 1214 insertions(+), 468 deletions(-) create mode 100644 packages/suricata/_dev/deploy/docker/Dockerfile create mode 100644 packages/suricata/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/suricata/_dev/deploy/docker/eve-alerts.ndjson create mode 100644 packages/suricata/_dev/deploy/docker/eve-dns-4.1.4.ndjson create mode 100644 packages/suricata/_dev/deploy/docker/eve-small.ndjson create mode 100644 packages/suricata/data_stream/eve/_dev/test/pipeline/test-events.json create mode 100644 packages/suricata/data_stream/eve/_dev/test/pipeline/test-events.json-config.json create mode 100644 packages/suricata/data_stream/eve/_dev/test/pipeline/test-events.json-expected.json create mode 100644 packages/suricata/data_stream/eve/_dev/test/system/config.yml create mode 100644 packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/dns-answer-v1.yml create mode 100644 packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/dns-answer-v2.yml create mode 100644 packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/dns.yml create mode 100644 packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/tls.yml diff --git a/packages/suricata/_dev/deploy/docker/Dockerfile b/packages/suricata/_dev/deploy/docker/Dockerfile new file mode 100644 index 00000000000..9b2fe3d99bc --- /dev/null +++ b/packages/suricata/_dev/deploy/docker/Dockerfile @@ -0,0 +1,5 @@ +FROM alpine + +COPY ./*.ndjson /sample_logs/ + +ENTRYPOINT [ "/bin/sh" ] \ No newline at end of file diff --git a/packages/suricata/_dev/deploy/docker/docker-compose.yml b/packages/suricata/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 00000000000..e58bbc250d8 --- /dev/null +++ b/packages/suricata/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,8 @@ +version: '2.3' +services: + suricata: + tty: true + build: . + volumes: + - ${SERVICE_LOGS_DIR}:/logs + command: -c "cp /sample_logs/*.ndjson /logs/" diff --git a/packages/suricata/_dev/deploy/docker/eve-alerts.ndjson b/packages/suricata/_dev/deploy/docker/eve-alerts.ndjson new file mode 100644 index 00000000000..915a6facbba --- /dev/null +++ b/packages/suricata/_dev/deploy/docker/eve-alerts.ndjson @@ -0,0 +1,22 @@ +{"timestamp":"2018-10-03T14:42:44.836744+0000","flow_id":2191386088856669,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":32858,"dest_ip":"93.184.216.34","dest_port":80,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":2013028,"rev":4,"signature":"ET POLICY curl User-Agent Outbound","category":"Attempted Information Leak","severity":2},"http":{"hostname":"example.net","url":"\/","http_user_agent":"curl\/7.58.0","http_content_type":"text\/html","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1121},"app_proto":"http","flow":{"pkts_toserver":4,"pkts_toclient":3,"bytes_toserver":347,"bytes_toclient":1654,"start":"2018-10-03T14:42:44.613469+0000"}} +{"timestamp":"2018-10-03T16:16:26.711841+0000","flow_id":678269478904081,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":32864,"dest_ip":"93.184.216.34","dest_port":80,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":2013028,"rev":4,"signature":"ET POLICY curl User-Agent Outbound","category":"Attempted Information Leak","severity":2},"http":{"hostname":"example.net","url":"\/","http_user_agent":"curl\/7.58.0","http_content_type":"text\/html","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1121},"app_proto":"http","flow":{"pkts_toserver":4,"pkts_toclient":3,"bytes_toserver":347,"bytes_toclient":1654,"start":"2018-10-03T16:16:26.467217+0000"}} +{"timestamp":"2018-10-03T16:44:50.813100+0000","flow_id":1170030461115650,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":32870,"dest_ip":"93.184.216.34","dest_port":80,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":2013028,"rev":4,"signature":"ET POLICY curl User-Agent Outbound","category":"Attempted Information Leak","severity":2},"http":{"hostname":"example.net","url":"\/","http_user_agent":"curl\/7.58.0","http_content_type":"text\/html","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1126},"app_proto":"http","flow":{"pkts_toserver":4,"pkts_toclient":3,"bytes_toserver":347,"bytes_toclient":1654,"start":"2018-10-03T16:44:50.580866+0000"}} +{"timestamp":"2018-10-03T16:45:09.267308+0000","flow_id":49628113637132,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":32872,"dest_ip":"93.184.216.34","dest_port":80,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":2013028,"rev":4,"signature":"ET POLICY curl User-Agent Outbound","category":"Attempted Information Leak","severity":2},"http":{"hostname":"example.org","url":"\/","http_user_agent":"curl\/7.58.0","http_content_type":"text\/html","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1121},"app_proto":"http","flow":{"pkts_toserver":4,"pkts_toclient":3,"bytes_toserver":347,"bytes_toclient":1654,"start":"2018-10-03T16:45:09.036620+0000"}} +{"timestamp":"2018-10-03T16:45:34.481113+0000","flow_id":116307482565223,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":32876,"dest_ip":"93.184.216.34","dest_port":80,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":2013028,"rev":4,"signature":"ET POLICY curl User-Agent Outbound","category":"Attempted Information Leak","severity":2},"http":{"hostname":"example.org","url":"\/","http_user_agent":"curl\/7.58.0","http_content_type":"text\/html","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1121},"app_proto":"http","flow":{"pkts_toserver":4,"pkts_toclient":3,"bytes_toserver":347,"bytes_toclient":1654,"start":"2018-10-03T16:45:34.252519+0000"}} +{"timestamp":"2018-10-03T17:02:38.900976+0000","flow_id":1205867738178946,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":32892,"dest_ip":"93.184.216.34","dest_port":80,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":2013028,"rev":4,"signature":"ET POLICY curl User-Agent Outbound","category":"Attempted Information Leak","severity":2},"http":{"hostname":"example.org","url":"\/","http_user_agent":"curl\/7.58.0","http_content_type":"text\/html","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1126},"app_proto":"http","flow":{"pkts_toserver":4,"pkts_toclient":3,"bytes_toserver":347,"bytes_toclient":1654,"start":"2018-10-03T17:02:38.599426+0000"}} +{"timestamp":"2018-10-04T09:34:59.009897+0000","flow_id":764842923400056,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":37742,"dest_ip":"91.189.88.152","dest_port":80,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"security.ubuntu.com","url":"\/ubuntu\/dists\/bionic-security\/InRelease","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1138},"app_proto":"http","flow":{"pkts_toserver":4,"pkts_toclient":3,"bytes_toserver":497,"bytes_toclient":1654,"start":"2018-10-04T09:34:58.924536+0000"}} +{"timestamp":"2018-10-04T09:34:59.168340+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic\/InRelease","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":304,"length":0},"app_proto":"http","flow":{"pkts_toserver":4,"pkts_toclient":3,"bytes_toserver":487,"bytes_toclient":417,"start":"2018-10-04T09:34:58.926006+0000"}} +{"timestamp":"2018-10-04T09:34:59.288862+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":1,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/InRelease","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":2601},"app_proto":"http","flow":{"pkts_toserver":6,"pkts_toclient":5,"bytes_toserver":842,"bytes_toclient":3445,"start":"2018-10-04T09:34:58.926006+0000"}} +{"timestamp":"2018-10-04T09:34:59.289324+0000","flow_id":764842923400056,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":37742,"dest_ip":"91.189.88.152","dest_port":80,"proto":"TCP","tx_id":1,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"security.ubuntu.com","url":"\/ubuntu\/dists\/bionic-security\/main\/source\/by-hash\/SHA256\/f5ec03d97ca76c98162d9233c8b7c578c52897e2136428277baf2e7b633a8e72","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1241},"app_proto":"http","flow":{"pkts_toserver":64,"pkts_toclient":62,"bytes_toserver":4810,"bytes_toclient":90543,"start":"2018-10-04T09:34:58.924536+0000"}} +{"timestamp":"2018-10-04T09:34:59.356132+0000","flow_id":764842923400056,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":37742,"dest_ip":"91.189.88.152","dest_port":80,"proto":"TCP","tx_id":2,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"security.ubuntu.com","url":"\/ubuntu\/dists\/bionic-security\/main\/binary-amd64\/by-hash\/SHA256\/c5b8346a3221bc9a23a79ba4dc4e730a6319a77fc9d63872dfc56539a0810015","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":2687},"app_proto":"http","flow":{"pkts_toserver":87,"pkts_toclient":98,"bytes_toserver":6591,"bytes_toclient":145014,"start":"2018-10-04T09:34:58.924536+0000"}} +{"timestamp":"2018-10-04T09:34:59.456919+0000","flow_id":764842923400056,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":37742,"dest_ip":"91.189.88.152","dest_port":80,"proto":"TCP","tx_id":3,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"security.ubuntu.com","url":"\/ubuntu\/dists\/bionic-security\/universe\/binary-amd64\/by-hash\/SHA256\/e5cc957139a25a0fee47cbf2c0fac8ad5cab50346d6a74abe031748924c5b558","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":2688},"app_proto":"http","flow":{"pkts_toserver":156,"pkts_toclient":221,"bytes_toserver":11460,"bytes_toclient":330525,"start":"2018-10-04T09:34:58.924536+0000"}} +{"timestamp":"2018-10-04T09:34:59.747122+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":2,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-backports\/InRelease","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":2601},"app_proto":"http","flow":{"pkts_toserver":64,"pkts_toclient":67,"bytes_toserver":4895,"bytes_toclient":96554,"start":"2018-10-04T09:34:58.926006+0000"}} +{"timestamp":"2018-10-04T09:34:59.953886+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":3,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/main\/source\/by-hash\/SHA256\/65f2e3a4e9d89d9d4b5e3d42e586bc96f48a24466b0ad0b4a707255e44a26b03","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":2687},"app_proto":"http","flow":{"pkts_toserver":91,"pkts_toclient":119,"bytes_toserver":6932,"bytes_toclient":174843,"start":"2018-10-04T09:34:58.926006+0000"}} +{"timestamp":"2018-10-04T09:35:00.250560+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":4,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/universe\/source\/by-hash\/SHA256\/56cfd9cc2efa61dff7428dddf921c3cd6047ab8e6484a7f1888e4c3f7252f1ef","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":2688},"app_proto":"http","flow":{"pkts_toserver":159,"pkts_toclient":253,"bytes_toserver":11679,"bytes_toclient":376452,"start":"2018-10-04T09:34:58.926006+0000"}} +{"timestamp":"2018-10-04T09:35:00.401788+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":5,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/main\/binary-amd64\/by-hash\/SHA256\/4360137dc8f98b47648da1fef5472ef234fb02115bc2b29873bcaeee62637e70","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":2687},"app_proto":"http","flow":{"pkts_toserver":190,"pkts_toclient":314,"bytes_toserver":13986,"bytes_toclient":468170,"start":"2018-10-04T09:34:58.926006+0000"}} +{"timestamp":"2018-10-04T09:35:00.776438+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":6,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/restricted\/binary-amd64\/by-hash\/SHA256\/c93fdc7f10cad1263349fd7b5bdd6a7f7163165b96ad263b3e12022e319d0d12","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":2691},"app_proto":"http","flow":{"pkts_toserver":328,"pkts_toclient":588,"bytes_toserver":23361,"bytes_toclient":880323,"start":"2018-10-04T09:34:58.926006+0000"}} +{"timestamp":"2018-10-04T09:35:00.897009+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":7,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/universe\/binary-amd64\/by-hash\/SHA256\/5190f7afbee38b3cb32225db478fdbabd46f76eaa9c5921a13091891bf3e9bbc","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":2687},"app_proto":"http","flow":{"pkts_toserver":330,"pkts_toclient":591,"bytes_toserver":23758,"bytes_toclient":884342,"start":"2018-10-04T09:34:58.926006+0000"}} +{"timestamp":"2018-10-04T09:35:01.362208+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":8,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/universe\/i18n\/by-hash\/SHA256\/9fe539b7036e51327cd85ca5e0a4dd4eb47f69168875de2ac9842a5e36ebd4a4","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","length":0},"app_proto":"http","flow":{"pkts_toserver":524,"pkts_toclient":979,"bytes_toserver":36819,"bytes_toclient":1467603,"start":"2018-10-04T09:34:58.926006+0000"}} +{"timestamp":"2018-10-04T09:35:01.575088+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":9,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/multiverse\/binary-amd64\/by-hash\/SHA256\/8ab8cb220c0e50521c589acc2bc2b43a3121210f0b035a0605972bcffd73dd16","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","length":0},"app_proto":"http","flow":{"pkts_toserver":575,"pkts_toclient":1079,"bytes_toserver":40452,"bytes_toclient":1618380,"start":"2018-10-04T09:34:58.926006+0000"}} +{"tls":{"ja3s":{"string":"333,55555,66666-22","hash":"0993626a07ad09e1ce91293be7aa5721"},"ja3":{"string":"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0","hash":"d92325c876e7279f4eb8c62415e3a6b7"},"notafter":"2024-07-16T14:52:35","notbefore":"2019-07-17T14:52:35","version":"TLS 1.2","sni":"hostname.domain.net","fingerprint":"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33","serial":"00:11:22:33:44:55:66:77:88","issuerdn":"C=US, O=Google Inc, CN=Google Internet Authority G2","subject":"C=US, ST=California, L=Mountain View, O=Google Inc, CN=*.google.com"},"proto":"TCP","dest_port":9080,"dest_ip":"10.232.0.237","src_port":45884,"src_ip":"10.126.2.140","event_type":"tls","in_iface":"enp5s0","flow_id":1091813059495729,"timestamp":"2018-10-04T09:35:02.796615+0000"} +{"flow":{"start":"2020-06-26T11:00:02.970011-0400","bytes_toclient":4660,"bytes_toserver":1074,"pkts_toclient":8,"pkts_toserver":7},"app_proto":"tls","tls":{"ja3s":{"string":"742,48172,30210-30","hash":"391231ba5675e42807b9e1f457b2614e"},"ja3":{"string":"718,4682-2687-2686-41992-41911-53292-53297-41969-22905-41926-41924-94181-94711-15-23-95-12-11-205,0-33-50-53-6-61-39-23-34-85-81,93-04-52,3-9-3","hash":"3f1ea03f5822e8021b60cc3e4b233181"},"notafter":"2026-06-25T17:36:29","notbefore":"2016-06-27T17:36:29","version":"TLS 1.2","sni":"host.domain.net","fingerprint":"36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc","serial":"72:A9:2C:51","issuerdn":"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown","subject":"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown"},"alert":{"severity":3,"category":"","signature":"SURICATA TLS on unusual port","rev":1,"signature_id":2610003,"gid":1,"action":"allowed"},"proto":"TCP","dest_port":8443,"dest_ip":"10.128.2.48","src_port":64389,"src_ip":"10.137.3.54","event_type":"alert","in_iface":"enp0s31f6","flow_id":991192778198299,"timestamp":"2020-06-26T11:00:03.342282-0400"} diff --git a/packages/suricata/_dev/deploy/docker/eve-dns-4.1.4.ndjson b/packages/suricata/_dev/deploy/docker/eve-dns-4.1.4.ndjson new file mode 100644 index 00000000000..4f625ae98f8 --- /dev/null +++ b/packages/suricata/_dev/deploy/docker/eve-dns-4.1.4.ndjson @@ -0,0 +1,24 @@ +{"timestamp":"2019-08-22T23:48:27.924120+0000","flow_id":885455453886936,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.15","src_port":46686,"dest_ip":"10.0.2.3","dest_port":53,"proto":"UDP","dns":{"type":"query","id":51803,"rrname":"google.com","rrtype":"A","tx_id":0}} +{"timestamp":"2019-08-22T23:48:27.924282+0000","flow_id":1418448010418810,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.15","src_port":36993,"dest_ip":"10.0.2.3","dest_port":53,"proto":"UDP","dns":{"type":"query","id":39523,"rrname":"google.com","rrtype":"AAAA","tx_id":0}} +{"timestamp":"2019-08-22T23:48:27.950946+0000","flow_id":1418448010418810,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":36993,"proto":"UDP","dns":{"version":2,"type":"answer","id":39523,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"google.com","rrtype":"AAAA","answers":[{"rrname":"google.com","rrtype":"AAAA","ttl":272,"rdata":"2607:f8b0:4006:0805:0000:0000:0000:200e"}],"grouped":{"AAAA":["2607:f8b0:4006:0805:0000:0000:0000:200e"]}}} +{"timestamp":"2019-08-22T23:48:27.957906+0000","flow_id":885455453886936,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":46686,"proto":"UDP","dns":{"version":2,"type":"answer","id":51803,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"google.com","rrtype":"A","answers":[{"rrname":"google.com","rrtype":"A","ttl":299,"rdata":"172.217.11.46"}],"grouped":{"A":["172.217.11.46"]}}} +{"timestamp":"2019-08-22T23:48:48.839495+0000","flow_id":40074894954311,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.15","src_port":50720,"dest_ip":"10.0.2.3","dest_port":53,"proto":"UDP","dns":{"type":"query","id":60273,"rrname":"www.elastic.co","rrtype":"A","tx_id":0}} +{"timestamp":"2019-08-22T23:48:48.839714+0000","flow_id":2130691028471842,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.15","src_port":41979,"dest_ip":"10.0.2.3","dest_port":53,"proto":"UDP","dns":{"type":"query","id":4210,"rrname":"www.elastic.co","rrtype":"AAAA","tx_id":0}} +{"timestamp":"2019-08-22T23:48:48.901548+0000","flow_id":40074894954311,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":50720,"proto":"UDP","dns":{"version":2,"type":"answer","id":60273,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"www.elastic.co","rrtype":"A","answers":[{"rrname":"www.elastic.co","rrtype":"CNAME","ttl":270,"rdata":"dualstack.r2.shared.global.fastly.net"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"A","ttl":29,"rdata":"151.101.130.217"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"A","ttl":29,"rdata":"151.101.194.217"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"A","ttl":29,"rdata":"151.101.2.217"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"A","ttl":29,"rdata":"151.101.66.217"}],"grouped":{"A":["151.101.130.217","151.101.194.217","151.101.2.217","151.101.66.217"],"CNAME":["dualstack.r2.shared.global.fastly.net"]}}} +{"timestamp":"2019-08-22T23:48:48.902685+0000","flow_id":2130691028471842,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":41979,"proto":"UDP","dns":{"version":2,"type":"answer","id":4210,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"www.elastic.co","rrtype":"AAAA","answers":[{"rrname":"www.elastic.co","rrtype":"CNAME","ttl":299,"rdata":"dualstack.r2.shared.global.fastly.net"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"AAAA","ttl":29,"rdata":"2a04:4e42:0600:0000:0000:0000:0000:0729"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"AAAA","ttl":29,"rdata":"2a04:4e42:0000:0000:0000:0000:0000:0729"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"AAAA","ttl":29,"rdata":"2a04:4e42:0200:0000:0000:0000:0000:0729"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"AAAA","ttl":29,"rdata":"2a04:4e42:0400:0000:0000:0000:0000:0729"}],"grouped":{"AAAA":["2a04:4e42:0600:0000:0000:0000:0000:0729","2a04:4e42:0000:0000:0000:0000:0000:0729","2a04:4e42:0200:0000:0000:0000:0000:0729","2a04:4e42:0400:0000:0000:0000:0000:0729"],"CNAME":["dualstack.r2.shared.global.fastly.net"]}}} +{"timestamp":"2019-08-23T01:22:31.812655+0000","flow_id":814378410010223,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.15","src_port":44773,"dest_ip":"10.0.2.3","dest_port":53,"proto":"UDP","dns":{"type":"query","id":28329,"rrname":"www.yahoo.com","rrtype":"A","tx_id":0}} +{"timestamp":"2019-08-23T01:22:31.812828+0000","flow_id":1887239765714716,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.15","src_port":55246,"dest_ip":"10.0.2.3","dest_port":53,"proto":"UDP","dns":{"type":"query","id":7050,"rrname":"www.yahoo.com","rrtype":"AAAA","tx_id":0}} +{"timestamp":"2019-08-23T01:22:31.846575+0000","flow_id":814378410010223,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":44773,"proto":"UDP","dns":{"type":"answer","id":28329,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"www.yahoo.com","rrtype":"CNAME","ttl":1315,"rdata":"atsv2-fp-shed.wg1.b.yahoo.com"}} +{"timestamp":"2019-08-23T01:22:31.846575+0000","flow_id":814378410010223,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":44773,"proto":"UDP","dns":{"type":"answer","id":28329,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"atsv2-fp-shed.wg1.b.yahoo.com","rrtype":"A","ttl":15,"rdata":"98.138.219.232"}} +{"timestamp":"2019-08-23T01:22:31.846575+0000","flow_id":814378410010223,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":44773,"proto":"UDP","dns":{"type":"answer","id":28329,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"atsv2-fp-shed.wg1.b.yahoo.com","rrtype":"A","ttl":15,"rdata":"98.138.219.231"}} +{"timestamp":"2019-08-23T01:22:31.846575+0000","flow_id":814378410010223,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":44773,"proto":"UDP","dns":{"type":"answer","id":28329,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"atsv2-fp-shed.wg1.b.yahoo.com","rrtype":"A","ttl":15,"rdata":"72.30.35.10"}} +{"timestamp":"2019-08-23T01:22:31.846575+0000","flow_id":814378410010223,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":44773,"proto":"UDP","dns":{"type":"answer","id":28329,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"atsv2-fp-shed.wg1.b.yahoo.com","rrtype":"A","ttl":15,"rdata":"72.30.35.9"}} +{"timestamp":"2019-08-23T01:22:31.847379+0000","flow_id":1887239765714716,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":55246,"proto":"UDP","dns":{"type":"answer","id":7050,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"www.yahoo.com","rrtype":"CNAME","ttl":1268,"rdata":"atsv2-fp-shed.wg1.b.yahoo.com"}} +{"timestamp":"2019-08-23T01:22:31.847379+0000","flow_id":1887239765714716,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":55246,"proto":"UDP","dns":{"type":"answer","id":7050,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"atsv2-fp-shed.wg1.b.yahoo.com","rrtype":"AAAA","ttl":53,"rdata":"2001:4998:0058:1836:0000:0000:0000:0010"}} +{"timestamp":"2019-08-23T01:22:31.847379+0000","flow_id":1887239765714716,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":55246,"proto":"UDP","dns":{"type":"answer","id":7050,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"atsv2-fp-shed.wg1.b.yahoo.com","rrtype":"AAAA","ttl":53,"rdata":"2001:4998:0044:041d:0000:0000:0000:0003"}} +{"timestamp":"2019-08-23T01:22:31.847379+0000","flow_id":1887239765714716,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":55246,"proto":"UDP","dns":{"type":"answer","id":7050,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"atsv2-fp-shed.wg1.b.yahoo.com","rrtype":"AAAA","ttl":53,"rdata":"2001:4998:0058:1836:0000:0000:0000:0011"}} +{"timestamp":"2019-08-23T01:22:31.847379+0000","flow_id":1887239765714716,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":55246,"proto":"UDP","dns":{"type":"answer","id":7050,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"atsv2-fp-shed.wg1.b.yahoo.com","rrtype":"AAAA","ttl":53,"rdata":"2001:4998:0044:041d:0000:0000:0000:0004"}} +{"timestamp":"2019-08-23T02:03:36.578089+0000","flow_id":2181951993205289,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.15","src_port":48288,"dest_ip":"10.0.2.3","dest_port":53,"proto":"UDP","dns":{"type":"query","id":9104,"rrname":"www.elastic.co","rrtype":"A","tx_id":0}} +{"timestamp":"2019-08-23T02:03:36.578262+0000","flow_id":928596784370390,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.15","src_port":59203,"dest_ip":"10.0.2.3","dest_port":53,"proto":"UDP","dns":{"type":"query","id":12859,"rrname":"www.elastic.co","rrtype":"AAAA","tx_id":0}} +{"timestamp":"2019-08-23T02:03:36.619381+0000","flow_id":2181951993205289,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":48288,"proto":"UDP","dns":{"version":2,"type":"answer","id":9104,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"www.elastic.co","rrtype":"A","answers":[{"rrname":"www.elastic.co","rrtype":"CNAME","ttl":150,"rdata":"dualstack.r2.shared.global.fastly.net"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"A","ttl":29,"rdata":"151.101.194.217"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"A","ttl":29,"rdata":"151.101.2.217"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"A","ttl":29,"rdata":"151.101.66.217"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"A","ttl":29,"rdata":"151.101.130.217"}]}} +{"timestamp":"2019-08-23T02:03:36.626559+0000","flow_id":928596784370390,"in_iface":"enp0s3","event_type":"dns","src_ip":"10.0.2.3","src_port":53,"dest_ip":"10.0.2.15","dest_port":59203,"proto":"UDP","dns":{"version":2,"type":"answer","id":12859,"flags":"8180","qr":true,"rd":true,"ra":true,"rcode":"NOERROR","rrname":"www.elastic.co","rrtype":"AAAA","answers":[{"rrname":"www.elastic.co","rrtype":"CNAME","ttl":269,"rdata":"dualstack.r2.shared.global.fastly.net"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"AAAA","ttl":29,"rdata":"2a04:4e42:0000:0000:0000:0000:0000:0729"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"AAAA","ttl":29,"rdata":"2a04:4e42:0200:0000:0000:0000:0000:0729"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"AAAA","ttl":29,"rdata":"2a04:4e42:0400:0000:0000:0000:0000:0729"},{"rrname":"dualstack.r2.shared.global.fastly.net","rrtype":"AAAA","ttl":29,"rdata":"2a04:4e42:0600:0000:0000:0000:0000:0729"}]}} diff --git a/packages/suricata/_dev/deploy/docker/eve-small.ndjson b/packages/suricata/_dev/deploy/docker/eve-small.ndjson new file mode 100644 index 00000000000..45163a617e9 --- /dev/null +++ b/packages/suricata/_dev/deploy/docker/eve-small.ndjson @@ -0,0 +1,8 @@ +{"timestamp":"2018-07-05T15:01:09.820360-0400","flow_id":298824096901438,"in_iface":"en0","event_type":"ssh","src_ip":"192.168.86.85","src_port":55406,"dest_ip":"192.168.253.112","dest_port":22,"proto":"TCP","ssh":{"client":{"proto_version":"2.0","software_version":"OpenSSH_7.6"},"server":{"proto_version":"2.0","software_version":"libssh_0.7.0"}}} +{"timestamp":"2018-07-05T15:07:20.910626-0400","flow_id":904992230150281,"in_iface":"en0","event_type":"alert","src_ip":"192.168.86.85","src_port":55641,"dest_ip":"192.168.156.70","dest_port":443,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":2024833,"rev":3,"signature":"ET POLICY Observed IP Lookup Domain (l2 .io in TLS SNI)","category":"Potential Corporate Privacy Violation","severity":1},"tls":{"session_resumed":true,"sni":"l2.io","version":"TLS 1.2"},"app_proto":"tls","flow":{"pkts_toserver":4,"pkts_toclient":3,"bytes_toserver":793,"bytes_toclient":343,"start":"2018-07-05T15:07:19.659593-0400"}} +{"timestamp":"2018-07-05T15:43:47.690014-0400","flow_id":2115002772430095,"in_iface":"en0","event_type":"http","src_ip":"192.168.86.85","src_port":56119,"dest_ip":"192.168.86.28","dest_port":63963,"proto":"TCP","tx_id":0,"http":{"hostname":"192.168.86.28","url":"\/dd.xml","http_user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/67.0.3396.99 Safari\/537.36","http_content_type":"text\/xml","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1155}} +{"timestamp":"2018-07-05T15:44:33.222441-0400","flow_id":2211411903323127,"in_iface":"en0","event_type":"fileinfo","src_ip":"192.168.86.28","src_port":8008,"dest_ip":"192.168.86.85","dest_port":56118,"proto":"TCP","http":{"hostname":"192.168.86.28","url":"\/ssdp\/device-desc.xml","http_user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/67.0.3396.99 Safari\/537.36","http_content_type":"application\/xml","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1071},"app_proto":"http","fileinfo":{"filename":"\/ssdp\/device-desc.xml","gaps":false,"state":"CLOSED","md5":"427b7337ff37eeb24d74f47d8e04cf21","sha1":"313573490192c685e9e53abef25453ed0d5e2aee","sha256":"f610428ebddf6f8cf9e39322e672583c45fcdcf885efad0ab48fd53a3dfc2c4b","stored":false,"size":1071,"tx_id":0}} +{"timestamp":"2018-07-05T15:51:20.213418-0400","flow_id":1684780223079543,"in_iface":"en0","event_type":"dns","src_ip":"192.168.86.1","src_port":53,"dest_ip":"192.168.86.85","dest_port":39464,"proto":"UDP","dns":{"type":"answer","id":12308,"rcode":"NOERROR","rrname":"clients.l.google.com","rrtype":"A","ttl":299,"rdata":"172.217.13.110"}} +{"timestamp":"2018-07-05T15:51:23.009510-0400","event_type":"stats","stats":{"uptime":5400,"capture":{"kernel_packets":430313,"kernel_drops":0,"kernel_ifdrops":0},"decoder":{"pkts":430313,"bytes":335138381,"invalid":2,"ipv4":425873,"ipv6":3785,"ethernet":430313,"raw":0,"null":0,"sll":0,"tcp":370093,"udp":58337,"sctp":0,"icmpv4":186,"icmpv6":1019,"ppp":0,"pppoe":0,"gre":0,"vlan":0,"vlan_qinq":0,"ieee8021ah":0,"teredo":1,"ipv4_in_ipv6":0,"ipv6_in_ipv6":0,"mpls":0,"avg_pkt_size":778,"max_pkt_size":1514,"erspan":0,"ipraw":{"invalid_ip_version":0},"ltnull":{"pkt_too_small":0,"unsupported_type":0},"dce":{"pkt_too_small":0}},"flow":{"memcap":0,"tcp":1113,"udp":1881,"icmpv4":0,"icmpv6":677,"spare":10000,"emerg_mode_entered":0,"emerg_mode_over":0,"tcp_reuse":0,"memuse":11537312},"defrag":{"ipv4":{"fragments":0,"reassembled":0,"timeouts":0},"ipv6":{"fragments":0,"reassembled":0,"timeouts":0},"max_frag_hits":0},"tcp":{"sessions":842,"ssn_memcap_drop":0,"pseudo":0,"pseudo_failed":0,"invalid_checksum":0,"no_flow":0,"syn":1138,"synack":656,"rst":1165,"segment_memcap_drop":0,"stream_depth_reached":63,"reassembly_gap":0,"overlap":5979,"overlap_diff_data":0,"insert_data_normal_fail":0,"insert_data_overlap_fail":0,"insert_list_fail":0,"memuse":4587520,"reassembly_memuse":768000},"detect":{"alert":2},"app_layer":{"flow":{"http":22,"ftp":0,"smtp":0,"tls":560,"ssh":4,"imap":0,"msn":0,"smb":0,"dcerpc_tcp":0,"dns_tcp":0,"failed_tcp":2,"dcerpc_udp":0,"dns_udp":762,"failed_udp":1119},"tx":{"http":25,"ftp":0,"smtp":0,"tls":0,"ssh":0,"smb":0,"dcerpc_tcp":0,"dns_tcp":0,"dcerpc_udp":0,"dns_udp":762}},"flow_mgr":{"closed_pruned":729,"new_pruned":1879,"est_pruned":975,"bypassed_pruned":0,"flows_checked":8,"flows_notimeout":8,"flows_timeout":0,"flows_timeout_inuse":0,"flows_removed":0,"rows_checked":65536,"rows_skipped":65530,"rows_empty":0,"rows_busy":0,"rows_maxlen":2},"file_store":{"open_files":0},"dns":{"memuse":7749,"memcap_state":0,"memcap_global":0},"http":{"memuse":17861,"memcap":0}}} +{"timestamp":"2018-07-05T15:51:50.666597-0400","flow_id":89751777876473,"in_iface":"en0","event_type":"tls","src_ip":"192.168.86.85","src_port":56187,"dest_ip":"17.142.164.13","dest_port":443,"proto":"TCP","tls":{"subject":"CN=*.icloud.com, OU=management:idms.group.506364, O=Apple Inc., ST=California, C=US","issuerdn":"CN=Apple IST CA 2 - G1, OU=Certification Authority, O=Apple Inc., C=US","serial":"5C:9C:E1:09:78:87:F8:07","fingerprint":"6a:ff:ac:a6:5f:8a:05:e7:a9:8c:76:29:b9:08:c7:69:ad:dc:72:47","sni":"p33-btmmdns.icloud.com.","version":"TLS 1.2","notbefore":"2017-02-27T17:54:31","notafter":"2019-03-29T17:54:31"}} +{"timestamp":"2018-07-05T15:51:54.001329-0400","flow_id":1828507008887644,"event_type":"flow","src_ip":"fe80:0000:0000:0000:fada:0cff:fedc:87f1","src_port":546,"dest_ip":"ff02:0000:0000:0000:0000:0000:0001:0002","dest_port":547,"proto":"UDP","app_proto":"failed","flow":{"pkts_toserver":1,"pkts_toclient":0,"bytes_toserver":110,"bytes_toclient":0,"start":"2018-07-05T15:51:23.453468-0400","end":"2018-07-05T15:51:23.453468-0400","age":0,"state":"new","reason":"timeout","alerted":false}} diff --git a/packages/suricata/data_stream/eve/_dev/test/pipeline/test-events.json b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-events.json new file mode 100644 index 00000000000..ac9944c54c5 --- /dev/null +++ b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-events.json @@ -0,0 +1,173 @@ +{ + "events": [ + { + "@timestamp": "2019-08-22T23:48:27.924Z", + "destination": { + "address": "10.0.2.3", + "ip": "10.0.2.3", + "port": 53 + }, + "dns": { + "question": { + "registered_domain": "google.com" + } + }, + "event": { + "created": "2020-11-10T19:08:35.841Z", + "dataset": "suricata.eve", + "module": "suricata", + "original": "{\"timestamp\":\"2019-08-22T23:48:27.924120+0000\",\"flow_id\":885455453886936,\"in_iface\":\"enp0s3\",\"event_type\":\"dns\",\"src_ip\":\"10.0.2.15\",\"src_port\":46686,\"dest_ip\":\"10.0.2.3\",\"dest_port\":53,\"proto\":\"UDP\",\"dns\":{\"type\":\"query\",\"id\":51803,\"rrname\":\"google.com\",\"rrtype\":\"A\",\"tx_id\":0}}" + }, + "host": { + "name": "731280f894fa" + }, + "network": { + "community_id": "1:HActqwgIaYeC8fc4sfMGrL8jjaI=", + "transport": "UDP" + }, + "source": { + "address": "10.0.2.15", + "ip": "10.0.2.15", + "port": 46686 + }, + "suricata": { + "eve": { + "dns": { + "id": 51803, + "rrname": "google.com", + "rrtype": "A", + "tx_id": 0, + "type": "query" + }, + "event_type": "dns", + "flow_id": "885455453886936", + "in_iface": "enp0s3" + } + } + }, + { + "@timestamp": "2018-07-05T19:07:20.910Z", + "destination": { + "address": "192.168.156.70", + "ip": "192.168.156.70", + "port": 443 + }, + "event": { + "created": "2020-11-10T19:08:41.847Z", + "dataset": "suricata.eve", + "module": "suricata", + "original": "{\"timestamp\":\"2018-07-05T15:07:20.910626-0400\",\"flow_id\":904992230150281,\"in_iface\":\"en0\",\"event_type\":\"alert\",\"src_ip\":\"192.168.86.85\",\"src_port\":55641,\"dest_ip\":\"192.168.156.70\",\"dest_port\":443,\"proto\":\"TCP\",\"tx_id\":0,\"alert\":{\"action\":\"allowed\",\"gid\":1,\"signature_id\":2024833,\"rev\":3,\"signature\":\"ET POLICY Observed IP Lookup Domain (l2 .io in TLS SNI)\",\"category\":\"Potential Corporate Privacy Violation\",\"severity\":1},\"tls\":{\"session_resumed\":true,\"sni\":\"l2.io\",\"version\":\"TLS 1.2\"},\"app_proto\":\"tls\",\"flow\":{\"pkts_toserver\":4,\"pkts_toclient\":3,\"bytes_toserver\":793,\"bytes_toclient\":343,\"start\":\"2018-07-05T15:07:19.659593-0400\"}}" + }, + "host": { + "name": "731280f894fa" + }, + "network": { + "community_id": "1:BWtsS+4pk477zAwfzve3Nm+x1Ms=", + "transport": "TCP" + }, + "source": { + "address": "192.168.86.85", + "ip": "192.168.86.85", + "port": 55641 + }, + "suricata": { + "eve": { + "alert": { + "action": "allowed", + "category": "Potential Corporate Privacy Violation", + "gid": 1, + "rev": 3, + "severity": 1, + "signature": "ET POLICY Observed IP Lookup Domain (l2 .io in TLS SNI)", + "signature_id": 2024833 + }, + "app_proto": "tls", + "event_type": "alert", + "flow": { + "bytes_toclient": 343, + "bytes_toserver": 793, + "pkts_toclient": 3, + "pkts_toserver": 4, + "start": "2018-07-05T15:07:19.659593-0400" + }, + "flow_id": "904992230150281", + "in_iface": "en0", + "tls": { + "session_resumed": true, + "sni": "l2.io", + "version": "TLS 1.2" + }, + "tx_id": 0 + } + } + }, + { + "@timestamp": "2018-10-03T16:44:50.813Z", + "agent": { + "ephemeral_id": "7c9260d7-4405-43f6-8723-d9051ca01d9e", + "id": "158306f7-ae3a-4ff8-baa1-1c4569ee20c2", + "name": "731280f894fa", + "type": "filebeat", + "version": "8.0.0" + }, + "destination": { + "address": "93.184.216.34", + "ip": "93.184.216.34", + "port": 80 + }, + "event": { + "created": "2020-11-10T19:08:03.782Z", + "dataset": "suricata.eve", + "module": "suricata", + "original": "{\"timestamp\":\"2018-10-03T16:44:50.813100+0000\",\"flow_id\":1170030461115650,\"in_iface\":\"enp0s3\",\"event_type\":\"alert\",\"src_ip\":\"192.168.1.146\",\"src_port\":32870,\"dest_ip\":\"93.184.216.34\",\"dest_port\":80,\"proto\":\"TCP\",\"tx_id\":0,\"alert\":{\"action\":\"allowed\",\"gid\":1,\"signature_id\":2013028,\"rev\":4,\"signature\":\"ET POLICY curl User-Agent Outbound\",\"category\":\"Attempted Information Leak\",\"severity\":2},\"http\":{\"hostname\":\"example.net\",\"url\":\"\\/\",\"http_user_agent\":\"curl\\/7.58.0\",\"http_content_type\":\"text\\/html\",\"http_method\":\"GET\",\"protocol\":\"HTTP\\/1.1\",\"status\":200,\"length\":1126},\"app_proto\":\"http\",\"flow\":{\"pkts_toserver\":4,\"pkts_toclient\":3,\"bytes_toserver\":347,\"bytes_toclient\":1654,\"start\":\"2018-10-03T16:44:50.580866+0000\"}}" + }, + "host": { + "name": "731280f894fa" + }, + "network": { + "community_id": "1:QI9ZBw/ltPo2cnzG5ne3IrgSdhw=", + "transport": "TCP" + }, + "source": { + "address": "192.168.1.146", + "ip": "192.168.1.146", + "port": 32870 + }, + "suricata": { + "eve": { + "alert": { + "action": "allowed", + "category": "Attempted Information Leak", + "gid": 1, + "rev": 4, + "severity": 2, + "signature": "ET POLICY curl User-Agent Outbound", + "signature_id": 2013028 + }, + "app_proto": "http", + "event_type": "alert", + "flow": { + "bytes_toclient": 1654, + "bytes_toserver": 347, + "pkts_toclient": 3, + "pkts_toserver": 4, + "start": "2018-10-03T16:44:50.580866+0000" + }, + "flow_id": "1170030461115650", + "http": { + "hostname": "example.net", + "http_content_type": "text/html", + "http_method": "GET", + "http_user_agent": "curl/7.58.0", + "length": 1126, + "protocol": "HTTP/1.1", + "status": 200, + "url": "/" + }, + "in_iface": "enp0s3", + "tx_id": 0 + } + } + } + ] +} \ No newline at end of file diff --git a/packages/suricata/data_stream/eve/_dev/test/pipeline/test-events.json-config.json b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-events.json-config.json new file mode 100644 index 00000000000..f71947c2f04 --- /dev/null +++ b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-events.json-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/suricata/data_stream/eve/_dev/test/pipeline/test-events.json-expected.json b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-events.json-expected.json new file mode 100644 index 00000000000..a9499c962d9 --- /dev/null +++ b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-events.json-expected.json @@ -0,0 +1,284 @@ +{ + "expected": [ + { + "@timestamp": "2019-08-22T23:48:27.924Z", + "related": { + "ip": [ + "10.0.2.15", + "10.0.2.3" + ] + }, + "destination": { + "port": 53, + "address": "10.0.2.3", + "ip": "10.0.2.3" + }, + "dns": { + "question": { + "name": "google.com", + "registered_domain": "google.com", + "type": "A", + "top_level_domain": "com" + }, + "type": "query", + "id": "51803" + }, + "host": { + "name": "731280f894fa" + }, + "suricata": { + "eve": { + "in_iface": "enp0s3", + "dns": { + "rrname": "google.com", + "id": 51803, + "tx_id": 0, + "type": "query", + "rrtype": "A" + }, + "event_type": "dns", + "flow_id": "885455453886936" + } + }, + "source": { + "port": 46686, + "address": "10.0.2.15", + "ip": "10.0.2.15" + }, + "event": { + "ingested": "2020-12-10T19:18:41.313982300Z", + "original": "{\"timestamp\":\"2019-08-22T23:48:27.924120+0000\",\"flow_id\":885455453886936,\"in_iface\":\"enp0s3\",\"event_type\":\"dns\",\"src_ip\":\"10.0.2.15\",\"src_port\":46686,\"dest_ip\":\"10.0.2.3\",\"dest_port\":53,\"proto\":\"UDP\",\"dns\":{\"type\":\"query\",\"id\":51803,\"rrname\":\"google.com\",\"rrtype\":\"A\",\"tx_id\":0}}", + "created": "2020-11-10T19:08:35.841Z", + "kind": "event", + "module": "suricata", + "category": [ + "network" + ], + "type": [ + "protocol" + ], + "dataset": "suricata.eve" + }, + "network": { + "community_id": "1:HActqwgIaYeC8fc4sfMGrL8jjaI=", + "protocol": "dns", + "transport": "udp" + } + }, + { + "destination": { + "address": "192.168.156.70", + "port": 443, + "bytes": 343, + "packets": 3, + "ip": "192.168.156.70", + "domain": "l2.io" + }, + "rule": { + "name": "ET POLICY Observed IP Lookup Domain (l2 .io in TLS SNI)", + "category": "Potential Corporate Privacy Violation", + "id": "2024833" + }, + "source": { + "address": "192.168.86.85", + "port": 55641, + "bytes": 793, + "packets": 4, + "ip": "192.168.86.85" + }, + "message": "Potential Corporate Privacy Violation", + "network": { + "community_id": "1:BWtsS+4pk477zAwfzve3Nm+x1Ms=", + "protocol": "tls", + "transport": "tcp", + "bytes": 1136, + "packets": 7 + }, + "@timestamp": "2018-07-05T19:07:20.910Z", + "related": { + "ip": [ + "192.168.86.85", + "192.168.156.70" + ] + }, + "host": { + "name": "731280f894fa" + }, + "suricata": { + "eve": { + "in_iface": "en0", + "event_type": "alert", + "alert": { + "rev": 3, + "signature_id": 2024833, + "gid": 1, + "category": "Potential Corporate Privacy Violation", + "signature": "ET POLICY Observed IP Lookup Domain (l2 .io in TLS SNI)" + }, + "flow_id": "904992230150281", + "tls": { + "session_resumed": true, + "version": "TLS 1.2", + "sni": "l2.io" + }, + "tx_id": 0, + "flow": {} + } + }, + "tls": { + "client": { + "server_name": "l2.io" + }, + "resumed": true, + "version": "1.2", + "version_protocol": "tls" + }, + "event": { + "severity": 1, + "ingested": "2020-12-10T19:18:41.313997500Z", + "original": "{\"timestamp\":\"2018-07-05T15:07:20.910626-0400\",\"flow_id\":904992230150281,\"in_iface\":\"en0\",\"event_type\":\"alert\",\"src_ip\":\"192.168.86.85\",\"src_port\":55641,\"dest_ip\":\"192.168.156.70\",\"dest_port\":443,\"proto\":\"TCP\",\"tx_id\":0,\"alert\":{\"action\":\"allowed\",\"gid\":1,\"signature_id\":2024833,\"rev\":3,\"signature\":\"ET POLICY Observed IP Lookup Domain (l2 .io in TLS SNI)\",\"category\":\"Potential Corporate Privacy Violation\",\"severity\":1},\"tls\":{\"session_resumed\":true,\"sni\":\"l2.io\",\"version\":\"TLS 1.2\"},\"app_proto\":\"tls\",\"flow\":{\"pkts_toserver\":4,\"pkts_toclient\":3,\"bytes_toserver\":793,\"bytes_toclient\":343,\"start\":\"2018-07-05T15:07:19.659593-0400\"}}", + "created": "2020-11-10T19:08:41.847Z", + "kind": "alert", + "module": "suricata", + "start": "2018-07-05T19:07:19.659Z", + "category": [ + "network", + "intrusion_detection" + ], + "type": [ + "allowed" + ], + "dataset": "suricata.eve" + } + }, + { + "agent": { + "name": "731280f894fa", + "id": "158306f7-ae3a-4ff8-baa1-1c4569ee20c2", + "ephemeral_id": "7c9260d7-4405-43f6-8723-d9051ca01d9e", + "type": "filebeat", + "version": "8.0.0" + }, + "destination": { + "geo": { + "continent_name": "North America", + "region_iso_code": "US-MA", + "city_name": "Norwell", + "country_iso_code": "US", + "country_name": "United States", + "region_name": "Massachusetts", + "location": { + "lon": -70.8217, + "lat": 42.1596 + } + }, + "as": { + "number": 15133, + "organization": { + "name": "MCI Communications Services, Inc. d/b/a Verizon Business" + } + }, + "address": "93.184.216.34", + "port": 80, + "bytes": 1654, + "ip": "93.184.216.34", + "domain": "example.net", + "packets": 3 + }, + "rule": { + "name": "ET POLICY curl User-Agent Outbound", + "category": "Attempted Information Leak", + "id": "2013028" + }, + "source": { + "address": "192.168.1.146", + "port": 32870, + "bytes": 347, + "packets": 4, + "ip": "192.168.1.146" + }, + "message": "Attempted Information Leak", + "url": { + "path": "/", + "original": "/", + "domain": "example.net" + }, + "network": { + "community_id": "1:QI9ZBw/ltPo2cnzG5ne3IrgSdhw=", + "protocol": "http", + "transport": "tcp", + "bytes": 2001, + "packets": 7 + }, + "@timestamp": "2018-10-03T16:44:50.813Z", + "related": { + "hosts": [ + "example.net" + ], + "ip": [ + "192.168.1.146", + "93.184.216.34" + ] + }, + "host": { + "name": "731280f894fa" + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "body": { + "bytes": 1126 + }, + "status_code": 200 + } + }, + "suricata": { + "eve": { + "in_iface": "enp0s3", + "event_type": "alert", + "alert": { + "rev": 4, + "signature_id": 2013028, + "gid": 1, + "category": "Attempted Information Leak", + "signature": "ET POLICY curl User-Agent Outbound" + }, + "flow_id": "1170030461115650", + "http": { + "http_content_type": "text/html", + "protocol": "HTTP/1.1" + }, + "tx_id": 0, + "flow": {} + } + }, + "event": { + "severity": 2, + "ingested": "2020-12-10T19:18:41.314009500Z", + "original": "{\"timestamp\":\"2018-10-03T16:44:50.813100+0000\",\"flow_id\":1170030461115650,\"in_iface\":\"enp0s3\",\"event_type\":\"alert\",\"src_ip\":\"192.168.1.146\",\"src_port\":32870,\"dest_ip\":\"93.184.216.34\",\"dest_port\":80,\"proto\":\"TCP\",\"tx_id\":0,\"alert\":{\"action\":\"allowed\",\"gid\":1,\"signature_id\":2013028,\"rev\":4,\"signature\":\"ET POLICY curl User-Agent Outbound\",\"category\":\"Attempted Information Leak\",\"severity\":2},\"http\":{\"hostname\":\"example.net\",\"url\":\"\\/\",\"http_user_agent\":\"curl\\/7.58.0\",\"http_content_type\":\"text\\/html\",\"http_method\":\"GET\",\"protocol\":\"HTTP\\/1.1\",\"status\":200,\"length\":1126},\"app_proto\":\"http\",\"flow\":{\"pkts_toserver\":4,\"pkts_toclient\":3,\"bytes_toserver\":347,\"bytes_toclient\":1654,\"start\":\"2018-10-03T16:44:50.580866+0000\"}}", + "created": "2020-11-10T19:08:03.782Z", + "kind": "alert", + "module": "suricata", + "start": "2018-10-03T16:44:50.580Z", + "category": [ + "network", + "intrusion_detection" + ], + "type": [ + "allowed" + ], + "dataset": "suricata.eve" + }, + "user_agent": { + "name": "curl", + "original": "curl/7.58.0", + "device": { + "name": "Other" + }, + "version": "7.58.0" + } + } + ] +} \ No newline at end of file diff --git a/packages/suricata/data_stream/eve/_dev/test/system/config.yml b/packages/suricata/data_stream/eve/_dev/test/system/config.yml new file mode 100644 index 00000000000..ab7acf408b6 --- /dev/null +++ b/packages/suricata/data_stream/eve/_dev/test/system/config.yml @@ -0,0 +1,6 @@ +input: logfile +vars: ~ +data_stream: + vars: + paths: + - "{{SERVICE_LOGS_DIR}}/eve*.ndjson" diff --git a/packages/suricata/data_stream/eve/agent/stream/log.yml.hbs b/packages/suricata/data_stream/eve/agent/stream/log.yml.hbs index bdad039dc0a..96e2fc04672 100644 --- a/packages/suricata/data_stream/eve/agent/stream/log.yml.hbs +++ b/packages/suricata/data_stream/eve/agent/stream/log.yml.hbs @@ -27,6 +27,7 @@ processors: - {from: suricata.eve.dest_ip, to: destination.address} - {from: suricata.eve.dest_port, to: destination.port, type: long} - {from: suricata.eve.proto, to: network.transport} + - {from: suricata.eve.flow_id, type: string} - convert: ignore_missing: true fail_on_error: false @@ -43,357 +44,19 @@ processors: fields: - suricata.eve.timestamp - community_id: - - if: - equals: - suricata.eve.event_type: dns - then: - - convert: - ignore_missing: true - fail_on_error: false - mode: copy - fields: - - {from: suricata.eve.dns.id, to: dns.id, type: string} - - {from: suricata.eve.dns.rcode, to: dns.response_code} - - {from: suricata.eve.dns.type, to: dns.type} - - convert: - when.equals.dns.type: query - ignore_missing: true - fail_on_error: false - mode: copy - fields: - - {from: suricata.eve.dns.rrname, to: dns.question.name} - - {from: suricata.eve.dns.rrtype, to: dns.question.type} - - if: - and: - - equals.dns.type: answer - - equals.suricata.eve.dns.version: 2 - then: - - convert: - ignore_missing: true - fail_on_error: false - mode: copy - fields: - - {from: suricata.eve.dns.rrname, to: dns.question.name} - - {from: suricata.eve.dns.rrtype, to: dns.question.type} - - registered_domain: - ignore_missing: true - ignore_failure: true - field: dns.question.name - target_field: dns.question.registered_domain - - script: - id: eve_process - lang: javascript - source: >- - function addEcsCategorization(evt) { - var event_type = evt.Get("suricata.eve.event_type"); - if (event_type == null) { - return; - } - var catArray = []; - var typeArray = []; - evt.Put("suricata.eve.event_type", event_type.toLowerCase()); - switch (event_type.toLowerCase()) { - case "alert": - evt.Put("event.kind", "alert"); - catArray.push("network"); - catArray.push("intrusion_detection"); - break; - case "anomaly": - evt.Put("event.kind", "event"); - catArray.push("network"); - break; - case "http": - evt.Put("event.kind", "event"); - catArray.push("network"); - catArray.push("web"); - typeArray.push("access"); - typeArray.push("protocol"); - evt.Put("network.protocol", "http"); - var status = evt.Get("suricata.eve.http.status"); - if (status == null) { - break; - } - if (status < 400) { - evt.Put("event.outcome", "success"); - } - if (status >= 400 ) { - evt.Put("event.outcome", "failure"); - } - break; - case "dns": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "dns"); - break; - case "ftp": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "ftp"); - break; - case "ftp_data": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "ftp"); - break; - case "tls": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "tls"); - break; - case "tftp": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "tftp"); - break; - case "smb": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "smb"); - break; - case "ssh": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "ssh"); - break; - case "flow": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("connection"); - var state = evt.Get("suricata.eve.flow.state"); - if (state == null) { - break; - } - switch (state) { - case "new": - typeArray.push("start"); - break; - case "closed": - typeArray.push("end"); - break; - } - break; - case "rdp": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "rdp"); - break; - case "stats": - evt.Put("event.kind", "metric"); - break; - default: - evt.Put("event.kind", "event"); - catArray.push("network"); - } - if (catArray.length > 0) { - evt.Put("event.category", catArray); - } - if (typeArray.length > 0) { - evt.Put("event.type", typeArray); - } - } - function setDnsV1Answers(evt) { - var dns_type = evt.Get("dns.type") - if (dns_type != "answer") { - return; - } - var version = evt.Get("suricata.eve.dns.version") - if (version == "2") { - return; - } - var name = evt.Get("suricata.eve.dns.rrname"); - var data = evt.Get("suricata.eve.dns.rdata"); - var type = evt.Get("suricata.eve.dns.rrtype"); - var ttl = evt.Get("suricata.eve.dns.ttl"); - var answer = {}; - if (name) { - answer.name = name; - } - if (data) { - answer.data = data; - } - if (type) { - answer.type = type; - } - if (ttl) { - answer.ttl = ttl; - } - if (Object.keys(answer).length === 0) { - return; - } - evt.Put("dns.answers", [answer]); - } - function addDnsV2Answers(evt) { - var type = evt.Get("dns.type") - if (type != "answer") { - return; - } - var version = evt.Get("suricata.eve.dns.version") - if (version != 2) { - return; - } - var answers = evt.Get("suricata.eve.dns.answers"); - if (!answers) { - return; - } - evt.Delete("suricata.eve.dns.answers"); - var resolvedIps = []; - for (var i = 0; i < answers.length; i++) { - var answer = answers[i]; - // Rename properties. - var name = answer["rrname"]; - delete answer["rrname"]; - var type = answer["rrtype"]; - delete answer["rrtype"]; - var data = answer["rdata"]; - delete answer["rdata"]; - answer["name"] = name; - answer["type"] = type; - answer["data"] = data; - // Append IP addresses to dns.resolved_ip. - if (type === "A" || type === "AAAA") { - resolvedIps.push(data); - } - } - evt.Put("dns.answers", answers); - if (resolvedIps.length > 0) { - evt.Put("dns.resolved_ip", resolvedIps); - } - } - function addDnsV2HeaderFlags(evt) { - var type = evt.Get("dns.type") - if (type != "answer") { - return; - } - var version = evt.Get("suricata.eve.dns.version") - if (version != 2) { - return; - } - var flag = evt.Get("suricata.eve.dns.aa"); - if (flag === true) { - evt.AppendTo("dns.header_flags", "AA"); - } - flag = evt.Get("suricata.eve.dns.tc"); - if (flag === true) { - evt.AppendTo("dns.header_flags", "TC"); - } - flag = evt.Get("suricata.eve.dns.rd"); - if (flag === true) { - evt.AppendTo("dns.header_flags", "RD"); - } - flag = evt.Get("suricata.eve.dns.ra"); - if (flag === true) { - evt.AppendTo("dns.header_flags", "RA"); - } - } - function addTopLevelDomain(evt) { - var rd = evt.Get("dns.question.registered_domain"); - if (rd == null) { - return; - } - var firstPeriod = rd.indexOf("."); - if (firstPeriod == -1) { - return; - } - evt.Put("dns.question.top_level_domain", rd.substr(firstPeriod + 1)); - } - function cleanupAppProto(evt) { - var proto = evt.Get("suricata.eve.app_proto"); - if (proto == null){ - return; - } - switch (proto.toLowerCase()) { - case "failed": - case "template": - case "template-rust": - break; - case "ftp-data": - evt.Put("network.protocol", "ftp"); - break; - default: - evt.Put("network.protocol", proto.toLowerCase()); - } - evt.Delete("suricata.eve.app_proto"); - } - function addRelatedIps(evt) { - var src_ip = evt.Get("source.ip"); - if (src_ip != null) { - evt.AppendTo("related.ip", src_ip); - } - var dst_ip = evt.Get("destination.ip"); - if (dst_ip != null) { - evt.AppendTo("related.ip", dst_ip); - } - } - function addTlsVersion(evt) { - var tls_version = evt.Get("suricata.eve.tls.version"); - if (tls_version == null) { - return; - } - var parts = tls_version.split(" "); - if (parts.length < 2) { - return; - } - evt.Put("tls.version_protocol", parts[0].toLowerCase()); - evt.Put("tls.version", parts[1]); - } - function cleanupTlsSni(evt) { - var sni = evt.Get("suricata.eve.tls.sni"); - if (sni == null) { - return; - } - if ("." == sni.charAt(sni.length - 1)) { - evt.Put("suricata.eve.tls.sni", sni.substring(0, sni.length - 1)); - } - } - function process(evt) { - var event_type = evt.Get("suricata.eve.event_type"); - addEcsCategorization(evt); - if (event_type == "dns") { - setDnsV1Answers(evt); - addDnsV2Answers(evt); - addDnsV2HeaderFlags(evt); - addTopLevelDomain(evt); - } - cleanupAppProto(evt); - addRelatedIps(evt); - addTlsVersion(evt); - cleanupTlsSni(evt); - } - - convert: + - registered_domain: + when: + or: + - equals.suricata.eve.dns.type: query + # V2 events always include the query data. + - equals.suricata.eve.dns.version: 2 ignore_missing: true - fail_on_error: false - mode: copy - fields: - - {from: suricata.eve.tls.subject, to: tls.server.subject} - - {from: suricata.eve.tls.issuerdn, to: tls.server.issuer} - - {from: suricata.eve.tls.session_resumed, to: tls.resumed, type: boolean} - - {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1} - - {from: suricata.eve.tls.sni, to: tls.client.server_name} - - {from: suricata.eve.tls.sni, to: destination.domain} - - {from: suricata.eve.tls.ja3s.hash, to: tls.server.ja3s} - - {from: suricata.eve.tls.ja3.hash, to: tls.client.ja3} - - {from: suricata.eve.tls.certificate, to: tls.server.certificate} - - {from: suricata.eve.tls.chain, to: tls.server.certificate_chain} - - drop_fields: - ignore_missing: true - fields: - - suricata.eve.dns.aa - - suricata.eve.dns.tc - - suricata.eve.dns.rd - - suricata.eve.dns.ra - - suricata.eve.dns.qr - - suricata.eve.dns.version - - suricata.eve.dns.flags - - suricata.eve.dns.grouped + ignore_failure: true + field: suricata.eve.dns.rrname + target_field: dns.question.registered_domain + target_subdomain_field: dns.question.subdomain + target_etld_field: dns.question.top_level_domain - add_fields: target: '' fields: - ecs.version: 1.6.0 \ No newline at end of file + ecs.version: 1.7.0 \ No newline at end of file diff --git a/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/default.yml b/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/default.yml index e132a8acdde..28636268456 100644 --- a/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/default.yml +++ b/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/default.yml @@ -5,6 +5,148 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' + + # Handle the different Suricata event types. + - lowercase: + field: suricata.eve.event_type + ignore_missing: true + - script: + lang: painless + ignore_failure: true + params: + alert: + kind: alert + category: + - network + - intrusion_detection + dns: + type: + - protocol + network_protocol: dns + flow: + type: + - connection + ftp: + type: + - protocol + network_protocol: ftp + ftp_data: + type: + - protocol + network_protocol: ftp + http: + category: + - network + - web + type: + - access + - protocol + network_protocol: http + http2: + category: + - network + - web + type: + - access + - protocol + network_protocol: http + mqtt: + type: + - protocol + network_protocol: mqtt + smb: + type: + - protocol + network_protocol: smb + ssh: + type: + - protocol + network_protocol: ssh + stats: + kind: metric + tftp: + type: + - protocol + network_protocol: tftp + tls: + type: + - protocol + network_protocol: tls + rdp: + type: + - protocol + network_protocol: rdp + rfb: # RFB (Remote Framebuffer Protocol) + type: + - protocol + network_protocol: rdp + + source: | + ctx.event.kind = 'event'; + ctx.event.category = ['network']; + def type_params = params.get(ctx?.suricata?.eve?.event_type); + if (type_params == null) { + return; + } + type_params.forEach((k, v) -> { + if ('network_protocol' == k) { + if (ctx.network == null) { + ctx.network = ['protocol': v]; + } else { + ctx.network.protocol = v; + } + } else { + ctx.event[k] = v; + } + }); + + ## Anomaly and Alert + - lowercase: + field: suricata.eve.app_proto + ignore_missing: true + - set: + if: ctx?.suricata?.eve?.app_proto == "ftp-data" + field: network.protocol + value: ftp + - set: + if: >- + ctx?.suricata?.eve?.app_proto != "failed" && + ctx?.suricata?.eve?.app_proto != "template" && + ctx?.suricata?.eve?.app_proto != "template-rust" + field: network.protocol + value: '{{suricata.eve.app_proto}}' + ignore_empty_value: true + ## HTTP + - set: + if: ctx?.suricata?.eve?.event_type == "http" && ctx?.suricata?.eve?.http?.status < 400 + field: event.outcome + value: success + - set: + if: ctx?.suricata?.eve?.event_type == "http" && ctx?.suricata?.eve?.http?.status >= 400 + field: event.outcome + value: failure + ## DNS + - pipeline: + if: >- + ctx?.network?.protocol == "dns" + name: '{{ IngestPipeline "dns" }}' + ## TLS + - pipeline: + if: ctx?.network?.protocol == "tls" + name: '{{ IngestPipeline "tls" }}' + ## Flow + - append: + if: ctx?.suricata?.eve?.flow?.state == "new" + field: event.type + value: + - start + - append: + if: ctx?.suricata?.eve?.flow?.state == "closed" + field: event.type + value: + - end + + - set: value: "{{suricata.eve.http.http_method}}" field: http.request.method @@ -17,11 +159,13 @@ processors: if: ctx.suricata?.eve?.http?.hostname != null value: '{{suricata.eve.http.hostname}}' field: destination.domain + allow_duplicates: false - remove: field: suricata.eve.http.hostname ignore_failure: true - script: lang: painless + tag: suricata_deduplicate_dest_domain source: > def domain = ctx.destination?.domain; if (domain instanceof Collection) { @@ -219,125 +363,21 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - - uppercase: - field: tls.server.hash.sha1 - ignore_missing: true - - split: - field: tls.server.hash.sha1 - separator: ":" - ignore_missing: true - - join: - field: tls.server.hash.sha1 - separator: "" - ignore_failure: true - - append: - field: related.hash - value: "{{tls.server.hash.sha1}}" - if: "ctx?.tls?.server?.hash?.sha1 != null" - - gsub: - field: suricata.eve.tls.issuerdn - pattern: \\, - replacement: "" - ignore_missing: true - - kv: - field: suricata.eve.tls.issuerdn - field_split: ', ' - value_split: '=' - target_field: suricata.eve.tls.kv_issuerdn - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_issuerdn.C - target_field: tls.server.x509.issuer.country - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_issuerdn.CN - target_field: tls.server.x509.issuer.common_name - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_issuerdn.L - target_field: tls.server.x509.issuer.locality - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_issuerdn.O - target_field: tls.server.x509.issuer.organization - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_issuerdn.OU - target_field: tls.server.x509.issuer.organizational_unit - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_issuerdn.ST - target_field: tls.server.x509.issuer.state_or_province - ignore_missing: true - - gsub: - field: suricata.eve.tls.subject - pattern: \\, - replacement: "" - ignore_missing: true - - kv: - field: suricata.eve.tls.subject - field_split: ', ' - value_split: '=' - target_field: suricata.eve.tls.kv_subject - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_subject.C - target_field: tls.server.x509.subject.country - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_subject.CN - target_field: tls.server.x509.subject.common_name - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_subject.L - target_field: tls.server.x509.subject.locality - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_subject.O - target_field: tls.server.x509.subject.organization - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_subject.OU - target_field: tls.server.x509.subject.organizational_unit - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_subject.ST - target_field: tls.server.x509.subject.state_or_province - ignore_missing: true - - set: - field: tls.server.x509.serial_number - value: '{{suricata.eve.tls.serial}}' - ignore_empty_value: true - - gsub: - field: tls.server.x509.serial_number - pattern: ':' - replacement: '' - ignore_missing: true - - date: - field: suricata.eve.tls.notafter - target_field: tls.server.not_after - formats: - - ISO8601 - if: ctx.suricata?.eve?.tls?.notafter != null - - date: - field: suricata.eve.tls.notbefore - target_field: tls.server.not_before - formats: - - ISO8601 - if: ctx.suricata?.eve?.tls?.notbefore != null - - set: - field: tls.server.x509.not_after - value: '{{tls.server.not_after}}' - ignore_empty_value: true - - set: - field: tls.server.x509.not_before - value: '{{tls.server.not_before}}' - ignore_empty_value: true - append: field: related.hosts value: '{{url.domain}}' if: ctx.url?.domain != null && ctx.url?.domain != '' allow_duplicates: false + - append: + if: ctx?.source?.ip != null + field: related.ip + value: '{{source.ip}}' + allow_duplicates: false + - append: + if: ctx?.destination?.ip != null + field: related.ip + value: '{{destination.ip}}' + allow_duplicates: false - remove: field: - suricata.eve.app_proto @@ -345,8 +385,6 @@ processors: - suricata.eve.flow.start - suricata.eve.http.http_method - suricata.eve.http.http_user_agent - - suricata.eve.tls.kv_issuerdn - - suricata.eve.tls.kv_subject ignore_missing: true on_failure: - set: diff --git a/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/dns-answer-v1.yml b/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/dns-answer-v1.yml new file mode 100644 index 00000000000..e915537365d --- /dev/null +++ b/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/dns-answer-v1.yml @@ -0,0 +1,39 @@ +--- +description: Pipeline for Suricata DNS answers v1 + +# Suricata DNS v1 events contain a single answer. Multiple events are created +# to represent all of the answers. +processors: + - script: + lang: painless + tag: suricata_dns_answer_v1 + source: | + def name = ctx?.suricata?.eve?.dns?.rrname; + def data = ctx?.suricata?.eve?.dns?.rdata; + def type = ctx?.suricata?.eve?.dns?.rrtype; + def ttl = ctx?.suricata?.eve?.dns?.ttl; + + def answer = [:]; + if (name != null) { + answer["name"] = name; + } + if (data != null) { + answer["data"] = data; + } + if (type != null) { + answer["type"] = type; + } + if (ttl != null) { + answer["ttl"] = ttl; + } + if (!answer.isEmpty()) { + ctx.dns.answers = [answer]; + } + + if (type == "A" || type == "AAAA") { + ctx.dns.resolved_ip = [data]; + } +on_failure: + - append: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/dns-answer-v2.yml b/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/dns-answer-v2.yml new file mode 100644 index 00000000000..a9e77c28549 --- /dev/null +++ b/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/dns-answer-v2.yml @@ -0,0 +1,42 @@ +--- +description: Pipeline for Suricata DNS answers v2 + +# Suricata DNS v2 events contain all answers in a single event. +processors: + - rename: + field: suricata.eve.dns.answers + target_field: dns.answers + ignore_missing: true + - script: + if: ctx?.dns?.answers != null + lang: painless + tag: suricata_dns_answers_v2 + source: | + def resolvedIps = new ArrayList(); + for (def answer : ctx?.dns?.answers) { + // Normalize field names to match ECS. + def name = answer.remove("rrname"); + if (name != null) { + answer["name"] = name; + } + def type = answer.remove("rrtype"); + if (type != null) { + answer["type"] = type; + } + def data = answer.remove("rdata"); + if (data != null) { + answer["data"] = data; + } + + if (type == "A" || type == "AAAA") { + resolvedIps.add(data); + } + } + + if (resolvedIps.size() > 0) { + ctx.dns.resolved_ip = resolvedIps; + } +on_failure: + - append: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/dns.yml b/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/dns.yml new file mode 100644 index 00000000000..2e108a88648 --- /dev/null +++ b/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/dns.yml @@ -0,0 +1,93 @@ +--- +description: Pipeline for Suricata DNS Events + +processors: + - set: + field: dns.id + value: '{{suricata.eve.dns.id}}' + ignore_empty_value: true + - set: + field: dns.response_code + value: '{{suricata.eve.dns.rcode}}' + ignore_empty_value: true + - set: + field: dns.type + value: '{{suricata.eve.dns.type}}' + ignore_empty_value: true + - set: + # V2 events always include the query data. + if: >- + ctx?.dns?.type == "query" || + ctx?.suricata?.eve?.dns?.version == 2 + field: dns.question.name + value: '{{suricata.eve.dns.rrname}}' + ignore_empty_value: true + - set: + # V2 events always include the query data. + if: >- + ctx?.dns?.type == "query" || + ctx?.suricata?.eve?.dns?.version == 2 + field: dns.question.type + value: '{{suricata.eve.dns.rrtype}}' + ignore_empty_value: true + - pipeline: + if: >- + ctx?.dns?.type == "answer" && + ctx?.suricata?.eve?.dns?.version == null + name: '{{ IngestPipeline "dns-answer-v1" }}' + - pipeline: + if: >- + ctx?.dns?.type == "answer" && + ctx?.suricata?.eve?.dns?.version == 2 + name: '{{ IngestPipeline "dns-answer-v2" }}' + - foreach: + field: dns.resolved_ip + ignore_missing: true + processor: + append: + field: related.ip + value: + - '{{_ingest._value}}' + allow_duplicates: false + - script: + if: ctx?.dns?.question?.registered_domain != null + tag: suricata_dns_top_level_domain + lang: painless + source: | + def rd = ctx.dns.question.registered_domain; + def firstDot = rd.indexOf("."); + if (firstDot == -1) { + return; + } + ctx.dns.question.top_level_domain = rd.substring(firstDot + 1); + - append: + if: ctx?.suricata?.eve?.dns?.aa == true + field: dns.header_flags + value: AA + - append: + if: ctx?.suricata?.eve?.dns?.tc == true + field: dns.header_flags + value: TC + - append: + if: ctx?.suricata?.eve?.dns?.rd == true + field: dns.header_flags + value: RD + - append: + if: ctx?.suricata?.eve?.dns?.ra == true + field: dns.header_flags + value: RA + - remove: + field: + - suricata.eve.dns.aa + - suricata.eve.dns.tc + - suricata.eve.dns.rd + - suricata.eve.dns.ra + - suricata.eve.dns.qr + - suricata.eve.dns.version + - suricata.eve.dns.flags + - suricata.eve.dns.grouped + ignore_missing: true +on_failure: + - append: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/tls.yml b/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/tls.yml new file mode 100644 index 00000000000..2c84e0c1cb7 --- /dev/null +++ b/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/tls.yml @@ -0,0 +1,188 @@ +--- +description: Pipeline for Suricata TLS Events + +processors: + - dissect: + field: suricata.eve.tls.version + pattern: '%{tls.version_protocol} %{tls.version}' + ignore_missing: true + - lowercase: + field: tls.version_protocol + ignore_missing: true + - script: + if: ctx?.suricata?.eve?.tls?.sni != null + tag: suricata_trim_tls_sni + lang: painless + source: | + def sni = ctx.suricata.eve.tls.sni; + if (!sni.endsWith(".")) { + return; + } + ctx.suricata.eve.tls.sni = sni.substring(0, sni.length() - 1); + # Subject + - set: + field: tls.server.subject + value: '{{suricata.eve.tls.subject}}' + ignore_empty_value: true + - gsub: + field: suricata.eve.tls.subject + pattern: \\, + replacement: "" + ignore_missing: true + - kv: + field: suricata.eve.tls.subject + field_split: ', ' + value_split: '=' + target_field: suricata.eve.tls.kv_subject + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_subject.C + target_field: tls.server.x509.subject.country + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_subject.CN + target_field: tls.server.x509.subject.common_name + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_subject.L + target_field: tls.server.x509.subject.locality + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_subject.O + target_field: tls.server.x509.subject.organization + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_subject.OU + target_field: tls.server.x509.subject.organizational_unit + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_subject.ST + target_field: tls.server.x509.subject.state_or_province + ignore_missing: true + # Issuer + - set: + field: tls.server.issuer + value: '{{suricata.eve.tls.issuerdn}}' + ignore_empty_value: true + - gsub: + field: suricata.eve.tls.issuerdn + pattern: \\, + replacement: "" + ignore_missing: true + - kv: + field: suricata.eve.tls.issuerdn + field_split: ', ' + value_split: '=' + target_field: suricata.eve.tls.kv_issuerdn + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_issuerdn.C + target_field: tls.server.x509.issuer.country + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_issuerdn.CN + target_field: tls.server.x509.issuer.common_name + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_issuerdn.L + target_field: tls.server.x509.issuer.locality + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_issuerdn.O + target_field: tls.server.x509.issuer.organization + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_issuerdn.OU + target_field: tls.server.x509.issuer.organizational_unit + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_issuerdn.ST + target_field: tls.server.x509.issuer.state_or_province + ignore_missing: true + + - convert: + field: suricata.eve.tls.session_resumed + target_field: tls.resumed + type: boolean + ignore_missing: true + - set: + field: tls.server.hash.sha1 + value: '{{suricata.eve.tls.fingerprint}}' + ignore_empty_value: true + - uppercase: + field: tls.server.hash.sha1 + ignore_missing: true + - split: + field: tls.server.hash.sha1 + separator: ":" + ignore_missing: true + - join: + field: tls.server.hash.sha1 + separator: "" + ignore_failure: true + - append: + field: related.hash + value: "{{tls.server.hash.sha1}}" + if: "ctx?.tls?.server?.hash?.sha1 != null" + - set: + field: tls.client.server_name + value: '{{suricata.eve.tls.sni}}' + ignore_empty_value: true + - set: + field: destination.domain + value: '{{suricata.eve.tls.sni}}' + ignore_empty_value: true + - set: + field: tls.server.ja3s + value: '{{suricata.eve.tls.ja3s.hash}}' + ignore_empty_value: true + - set: + field: tls.client.ja3 + value: '{{suricata.eve.tls.ja3.hash}}' + ignore_empty_value: true + - set: + field: tls.server.certificate + value: '{{suricata.eve.tls.certificate}}' + ignore_empty_value: true + - set: + field: tls.server.certificate_chain + value: '{{suricata.eve.tls.chain}}' + ignore_empty_value: true + - set: + field: tls.server.x509.serial_number + value: '{{suricata.eve.tls.serial}}' + ignore_empty_value: true + - gsub: + field: tls.server.x509.serial_number + pattern: ':' + replacement: '' + ignore_missing: true + - date: + field: suricata.eve.tls.notafter + target_field: tls.server.not_after + formats: + - ISO8601 + if: ctx.suricata?.eve?.tls?.notafter != null + - date: + field: suricata.eve.tls.notbefore + target_field: tls.server.not_before + formats: + - ISO8601 + if: ctx.suricata?.eve?.tls?.notbefore != null + - set: + field: tls.server.x509.not_after + value: '{{tls.server.not_after}}' + ignore_empty_value: true + - set: + field: tls.server.x509.not_before + value: '{{tls.server.not_before}}' + ignore_empty_value: true + - remove: + field: + - suricata.eve.tls.kv_issuerdn + - suricata.eve.tls.kv_subject + ignore_missing: true +on_failure: + - append: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/suricata/data_stream/eve/fields/ecs.yml b/packages/suricata/data_stream/eve/fields/ecs.yml index e58e8ae514f..3542e27dbd9 100644 --- a/packages/suricata/data_stream/eve/fields/ecs.yml +++ b/packages/suricata/data_stream/eve/fields/ecs.yml @@ -254,3 +254,114 @@ - name: tls.server.not_before description: Timestamp indicating when server certificate is first considered valid. type: date +- name: destination.address + type: keyword + description: Destination network address. +- name: destination.as.number + type: long + description: Unique number allocated to the autonomous system. +- name: destination.as.organization.name + type: keyword + description: Organization name. +- name: destination.domain + type: keyword + description: Destination domain. +- name: destination.geo.city_name + type: keyword + description: City name. +- name: destination.geo.continent_name + type: keyword + description: Name of the continent. +- name: destination.geo.country_iso_code + type: keyword + description: Country ISO code. +- name: destination.geo.country_name + type: keyword + description: Country name. +- name: destination.geo.location + type: geo_point + description: Longitude and latitude. +- name: destination.geo.region_iso_code + type: keyword + description: Region ISO code. +- name: destination.geo.region_name + type: keyword + description: Region name. +- name: message + type: text + description: Log message optimized for viewing in a log viewer. +- name: network.bytes + type: long + description: Total bytes transferred in both directions. +- name: network.community_id + type: keyword + description: A hash of source and destination IPs and ports. +- name: network.packets + type: long + description: Total packets transferred in both directions. +- name: rule.category + type: keyword + description: Rule category +- name: rule.id + type: keyword + description: Rule ID +- name: rule.name + type: keyword + description: Rule name +- name: source.address + type: keyword + description: Source network address. +- name: tls.client.server_name + type: keyword + description: Hostname the client is trying to connect to. Also called the SNI. +- name: tls.resumed + type: boolean + description: Boolean flag indicating if this TLS connection was resumed from an existing TLS negotiation. +- name: tls.version + type: keyword + description: Numeric part of the version parsed from the original string. +- name: tls.version_protocol + type: keyword + description: Normalized lowercase protocol name parsed from original string. +- name: url.path + type: keyword + description: Path of the request, such as "/search". +- name: user_agent.device.name + type: keyword + description: Name of the device. +- name: user_agent.name + type: keyword + description: Name of the user agent. +- name: user_agent.version + type: keyword + description: Version of the user agent. +- name: ecs.version + type: keyword + description: ECS version this event conforms to. +- name: related.hash + type: keyword + description: All the hashes seen on your event. +- name: tls.client.ja3 + type: keyword + description: A hash that identifies clients based on how they perform an SSL/TLS handshake. +- name: tls.server.hash.sha1 + type: keyword + description: Certificate fingerprint using the SHA1 digest of DER-encoded version of certificate offered by the server. +- name: tls.server.issuer + type: keyword + description: Subject of the issuer of the x.509 certificate presented by the server. +- name: tls.server.ja3s + type: keyword + description: A hash that identifies servers based on how they perform an SSL/TLS handshake. +- name: tls.server.subject + type: keyword + description: Subject of the x.509 certificate presented by the server. +- name: user_agent.os.full + type: keyword + description: Operating system name, including the version or code name. +- name: user_agent.os.name + type: keyword + description: Operating system name, without the version. +- name: user_agent.os.version + type: keyword + description: Operating system version as a raw string. diff --git a/packages/suricata/docs/README.md b/packages/suricata/docs/README.md index 55959c90851..adf00f6bc35 100644 --- a/packages/suricata/docs/README.md +++ b/packages/suricata/docs/README.md @@ -15,7 +15,7 @@ with other versions of Suricata. | Field | Description | Type | |---|---|---| -| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| @timestamp | Event timestamp. | date | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | @@ -32,7 +32,18 @@ with other versions of Suricata. | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| destination.address | Destination network address. | keyword | +| destination.as.number | Unique number allocated to the autonomous system. | long | +| destination.as.organization.name | Organization name. | keyword | | destination.bytes | Bytes sent from the destination to the source. | long | +| destination.domain | Destination domain. | keyword | +| destination.geo.city_name | City name. | keyword | +| destination.geo.continent_name | Name of the continent. | keyword | +| destination.geo.country_iso_code | Country ISO code. | keyword | +| destination.geo.country_name | Country name. | keyword | +| destination.geo.location | Longitude and latitude. | geo_point | +| destination.geo.region_iso_code | Region ISO code. | keyword | +| destination.geo.region_name | Region name. | keyword | | destination.ip | IP address of the destination (IPv4 or IPv6). | ip | | destination.packets | Packets sent from the destination to the source. | long | | destination.port | Port of the destination. | long | @@ -54,6 +65,7 @@ with other versions of Suricata. | dns.resolved_ip | Array containing all IPs seen in `answers.data`. The `answers` array can be difficult to use, because of the variety of data formats it can contain. Extracting all IP addresses seen in there to `dns.resolved_ip` makes it possible to index them as IP addresses, and makes them easier to visualize and query for. | ip | | dns.response_code | The DNS response code. | keyword | | dns.type | The type of DNS event captured, query or answer. If your source of DNS events only gives you DNS queries, you should only create dns events of type `dns.type:query`. If your source of DNS events gives you answers as well, you should create one event per query (optionally as soon as the query is seen). And a second event containing all query details as well as an array of answers. | keyword | +| ecs.version | ECS version this event conforms to. | keyword | | event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.duration | Duration of the event in nanoseconds. | long | | event.end | event.end contains the date when the event ended or when the activity was last observed. | date | @@ -87,10 +99,19 @@ with other versions of Suricata. | input.type | Filebeat input type used to collect the log. | keyword | | log.file.path | The file from which the line was read. This field contains the absolute path to the file. For example: `/var/log/system.log`. | keyword | | log.offset | The file offset the reported line starts at. | long | +| message | Log message optimized for viewing in a log viewer. | text | +| network.bytes | Total bytes transferred in both directions. | long | +| network.community_id | A hash of source and destination IPs and ports. | keyword | +| network.packets | Total packets transferred in both directions. | long | | network.protocol | L7 Network protocol name. ex. http, lumberjack, transport protocol. The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | | network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | +| related.hash | All the hashes seen on your event. | keyword | | related.hosts | All the host identifiers seen on your event. | keyword | | related.ip | All of the IPs seen on your event. | ip | +| rule.category | Rule category | keyword | +| rule.id | Rule ID | keyword | +| rule.name | Rule name | keyword | +| source.address | Source network address. | keyword | | source.bytes | Bytes sent from the source to the destination. | long | | source.ip | IP address of the source (IPv4 or IPv6). | ip | | source.packets | Packets sent from the source to the destination. | long | @@ -281,8 +302,15 @@ with other versions of Suricata. | suricata.eve.tls.version | | keyword | | suricata.eve.tx_id | | long | | tags | List of keywords used to tag each event. | keyword | +| tls.client.ja3 | A hash that identifies clients based on how they perform an SSL/TLS handshake. | keyword | +| tls.client.server_name | Hostname the client is trying to connect to. Also called the SNI. | keyword | +| tls.resumed | Boolean flag indicating if this TLS connection was resumed from an existing TLS negotiation. | boolean | +| tls.server.hash.sha1 | Certificate fingerprint using the SHA1 digest of DER-encoded version of certificate offered by the server. | keyword | +| tls.server.issuer | Subject of the issuer of the x.509 certificate presented by the server. | keyword | +| tls.server.ja3s | A hash that identifies servers based on how they perform an SSL/TLS handshake. | keyword | | tls.server.not_after | Timestamp indicating when server certificate is no longer considered valid. | date | | tls.server.not_before | Timestamp indicating when server certificate is first considered valid. | date | +| tls.server.subject | Subject of the x.509 certificate presented by the server. | keyword | | tls.server.x509.issuer.common_name | List of common name (CN) of issuing certificate authority. | keyword | | tls.server.x509.issuer.country | List of country (C) codes | keyword | | tls.server.x509.issuer.locality | List of locality names (L) | keyword | @@ -298,7 +326,16 @@ with other versions of Suricata. | tls.server.x509.subject.organization | List of organizations (O) of subject. | keyword | | tls.server.x509.subject.organizational_unit | List of organizational units (OU) of subject. | keyword | | tls.server.x509.subject.state_or_province | List of state or province names (ST, S, or P) | keyword | +| tls.version | Numeric part of the version parsed from the original string. | keyword | +| tls.version_protocol | Normalized lowercase protocol name parsed from original string. | keyword | | url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. | keyword | | url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | keyword | +| url.path | Path of the request, such as "/search". | keyword | +| user_agent.device.name | Name of the device. | keyword | +| user_agent.name | Name of the user agent. | keyword | | user_agent.original | Unparsed user_agent string. | keyword | +| user_agent.os.full | Operating system name, including the version or code name. | keyword | +| user_agent.os.name | Operating system name, without the version. | keyword | +| user_agent.os.version | Operating system version as a raw string. | keyword | +| user_agent.version | Version of the user agent. | keyword | diff --git a/packages/suricata/manifest.yml b/packages/suricata/manifest.yml index b5f822b45aa..53240eb08fc 100644 --- a/packages/suricata/manifest.yml +++ b/packages/suricata/manifest.yml @@ -1,6 +1,6 @@ name: suricata title: Suricata -version: 0.3.3 +version: 0.3.4 release: experimental description: Suricata Integration type: integration