Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Filebeat] Refactor Suricata pipeline to use Ingest Node more #22291

Merged
merged 4 commits into from
Nov 5, 2020
Merged

[Filebeat] Refactor Suricata pipeline to use Ingest Node more #22291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
abd075e
Refactor the Suricata module pipeline to use Ingest Node more
andrewkroh Oct 30, 2020
9e5a614
Merge branch 'master' into feature/suricata-ingest-node
andrewkroh Nov 2, 2020
c86b906
Use script with params
andrewkroh Nov 5, 2020
2b0cd1d
Merge branch 'feature/suricata-ingest-node' of github.com:andrewkroh/…
andrewkroh Nov 5, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -650,6 +650,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add max_number_of_messages config into s3 input. {pull}21993[21993]
- Update Okta documentation for new stateful restarts. {pull}22091[22091]
- Copy tag names from MISP data into events. {pull}21664[21664]
- Added DNS response IP addresses to `related.ip` in Suricata module. {pull}22291[22291]
- Added TLS JA3 fingerprint, certificate not_before/not_after, certificate SHA1 hash, and certificate subject fields to Zeek SSL dataset. {pull}21696[21696]

*Heartbeat*
Expand Down
365 changes: 9 additions & 356 deletions x-pack/filebeat/module/suricata/eve/config/eve.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,363 +42,16 @@ processors:
{{ if .community_id }}
- community_id:
{{ end }}
- if:
equals:
suricata.eve.event_type: dns
then:
- convert:
ignore_missing: true
fail_on_error: false
mode: copy
fields:
- {from: suricata.eve.dns.id, to: dns.id, type: string}
- {from: suricata.eve.dns.rcode, to: dns.response_code}
- {from: suricata.eve.dns.type, to: dns.type}
- convert:
when.equals.dns.type: query
ignore_missing: true
fail_on_error: false
mode: copy
fields:
- {from: suricata.eve.dns.rrname, to: dns.question.name}
- {from: suricata.eve.dns.rrtype, to: dns.question.type}
- if:
and:
- equals.dns.type: answer
- equals.suricata.eve.dns.version: 2
then:
- convert:
ignore_missing: true
fail_on_error: false
mode: copy
fields:
- {from: suricata.eve.dns.rrname, to: dns.question.name}
- {from: suricata.eve.dns.rrtype, to: dns.question.type}
- registered_domain:
ignore_missing: true
ignore_failure: true
field: dns.question.name
target_field: dns.question.registered_domain
- script:
id: eve_process
lang: javascript
source: >-
function addEcsCategorization(evt) {
var event_type = evt.Get("suricata.eve.event_type");
if (event_type == null) {
return;
}
var catArray = [];
var typeArray = [];
evt.Put("suricata.eve.event_type", event_type.toLowerCase());
switch (event_type.toLowerCase()) {
case "alert":
evt.Put("event.kind", "alert");
catArray.push("network");
catArray.push("intrusion_detection");
break;
case "anomaly":
evt.Put("event.kind", "event");
catArray.push("network");
break;
case "http":
evt.Put("event.kind", "event");
catArray.push("network");
catArray.push("web");
typeArray.push("access");
typeArray.push("protocol");
evt.Put("network.protocol", "http");
var status = evt.Get("suricata.eve.http.status");
if (status == null) {
break;
}
if (status < 400) {
evt.Put("event.outcome", "success");
}
if (status >= 400 ) {
evt.Put("event.outcome", "failure");
}
break;
case "dns":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "dns");
break;
case "ftp":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "ftp");
break;
case "ftp_data":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "ftp");
break;
case "tls":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "tls");
break;
case "tftp":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "tftp");
break;
case "smb":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "smb");
break;
case "ssh":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "ssh");
break;
case "flow":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("connection");
var state = evt.Get("suricata.eve.flow.state");
if (state == null) {
break;
}
switch (state) {
case "new":
typeArray.push("start");
break;
case "closed":
typeArray.push("end");
break;
}
break;
case "rdp":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "rdp");
break;
case "stats":
evt.Put("event.kind", "metric");
break;
default:
evt.Put("event.kind", "event");
catArray.push("network");
}
if (catArray.length > 0) {
evt.Put("event.category", catArray);
}
if (typeArray.length > 0) {
evt.Put("event.type", typeArray);
}
}
function setDnsV1Answers(evt) {
var dns_type = evt.Get("dns.type")
if (dns_type != "answer") {
return;
}
var version = evt.Get("suricata.eve.dns.version")
if (version == "2") {
return;
}
var name = evt.Get("suricata.eve.dns.rrname");
var data = evt.Get("suricata.eve.dns.rdata");
var type = evt.Get("suricata.eve.dns.rrtype");
var ttl = evt.Get("suricata.eve.dns.ttl");
var answer = {};
if (name) {
answer.name = name;
}
if (data) {
answer.data = data;
}
if (type) {
answer.type = type;
}
if (ttl) {
answer.ttl = ttl;
}
if (Object.keys(answer).length === 0) {
return;
}
evt.Put("dns.answers", [answer]);
}
function addDnsV2Answers(evt) {
var type = evt.Get("dns.type")
if (type != "answer") {
return;
}
var version = evt.Get("suricata.eve.dns.version")
if (version != 2) {
return;
}
var answers = evt.Get("suricata.eve.dns.answers");
if (!answers) {
return;
}
evt.Delete("suricata.eve.dns.answers");
var resolvedIps = [];
for (var i = 0; i < answers.length; i++) {
var answer = answers[i];

// Rename properties.
var name = answer["rrname"];
delete answer["rrname"];
var type = answer["rrtype"];
delete answer["rrtype"];
var data = answer["rdata"];
delete answer["rdata"];

answer["name"] = name;
answer["type"] = type;
answer["data"] = data;

// Append IP addresses to dns.resolved_ip.
if (type === "A" || type === "AAAA") {
resolvedIps.push(data);
}
}
evt.Put("dns.answers", answers);
if (resolvedIps.length > 0) {
evt.Put("dns.resolved_ip", resolvedIps);
}
}
function addDnsV2HeaderFlags(evt) {
var type = evt.Get("dns.type")
if (type != "answer") {
return;
}
var version = evt.Get("suricata.eve.dns.version")
if (version != 2) {
return;
}
var flag = evt.Get("suricata.eve.dns.aa");
if (flag === true) {
evt.AppendTo("dns.header_flags", "AA");
}

flag = evt.Get("suricata.eve.dns.tc");
if (flag === true) {
evt.AppendTo("dns.header_flags", "TC");
}

flag = evt.Get("suricata.eve.dns.rd");
if (flag === true) {
evt.AppendTo("dns.header_flags", "RD");
}

flag = evt.Get("suricata.eve.dns.ra");
if (flag === true) {
evt.AppendTo("dns.header_flags", "RA");
}
}
function addTopLevelDomain(evt) {
var rd = evt.Get("dns.question.registered_domain");
if (rd == null) {
return;
}
var firstPeriod = rd.indexOf(".");
if (firstPeriod == -1) {
return;
}
evt.Put("dns.question.top_level_domain", rd.substr(firstPeriod + 1));
}
function cleanupAppProto(evt) {
var proto = evt.Get("suricata.eve.app_proto");
if (proto == null){
return;
}
switch (proto.toLowerCase()) {
case "failed":
case "template":
case "template-rust":
break;
case "ftp-data":
evt.Put("network.protocol", "ftp");
break;
default:
evt.Put("network.protocol", proto.toLowerCase());
}
evt.Delete("suricata.eve.app_proto");
}
function addRelatedIps(evt) {
var src_ip = evt.Get("source.ip");
if (src_ip != null) {
evt.AppendTo("related.ip", src_ip);
}
var dst_ip = evt.Get("destination.ip");
if (dst_ip != null) {
evt.AppendTo("related.ip", dst_ip);
}
}
function addTlsVersion(evt) {
var tls_version = evt.Get("suricata.eve.tls.version");
if (tls_version == null) {
return;
}
var parts = tls_version.split(" ");
if (parts.length < 2) {
return;
}
evt.Put("tls.version_protocol", parts[0].toLowerCase());
evt.Put("tls.version", parts[1]);
}
function cleanupTlsSni(evt) {
var sni = evt.Get("suricata.eve.tls.sni");
if (sni == null) {
return;
}
if ("." == sni.charAt(sni.length - 1)) {
evt.Put("suricata.eve.tls.sni", sni.substring(0, sni.length - 1));
}
}
function process(evt) {
var event_type = evt.Get("suricata.eve.event_type");

addEcsCategorization(evt);
if (event_type == "dns") {
setDnsV1Answers(evt);
addDnsV2Answers(evt);
addDnsV2HeaderFlags(evt);
addTopLevelDomain(evt);
}
cleanupAppProto(evt);
addRelatedIps(evt);
addTlsVersion(evt);
cleanupTlsSni(evt);
}
- convert:
- registered_domain:
when:
or:
- equals.suricata.eve.dns.type: query
# V2 events always include the query data.
- equals.suricata.eve.dns.version: 2
ignore_missing: true
fail_on_error: false
mode: copy
fields:
- {from: suricata.eve.tls.subject, to: tls.server.subject}
- {from: suricata.eve.tls.issuerdn, to: tls.server.issuer}
- {from: suricata.eve.tls.session_resumed, to: tls.resumed, type: boolean}
- {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1}
- {from: suricata.eve.tls.sni, to: tls.client.server_name}
- {from: suricata.eve.tls.sni, to: destination.domain}
- {from: suricata.eve.tls.ja3s.hash, to: tls.server.ja3s}
- {from: suricata.eve.tls.ja3.hash, to: tls.client.ja3}
- {from: suricata.eve.tls.certificate, to: tls.server.certificate}
- {from: suricata.eve.tls.chain, to: tls.server.certificate_chain}
- drop_fields:
ignore_missing: true
fields:
- suricata.eve.dns.aa
- suricata.eve.dns.tc
- suricata.eve.dns.rd
- suricata.eve.dns.ra
- suricata.eve.dns.qr
- suricata.eve.dns.version
- suricata.eve.dns.flags
- suricata.eve.dns.grouped
ignore_failure: true
field: suricata.eve.dns.rrname
target_field: dns.question.registered_domain
- add_fields:
target: ''
fields:
Expand Down
Loading