Skip to content

Commit

Permalink
Refactor the Suricata module pipeline to use Ingest Node more
Browse files Browse the repository at this point in the history 
Refactor the Suricata module pipeline to use Ingest Node more. I moved most
processing from Beat processors to Ingest Node. I created a DNS and TLS
pipeline to house all of the processing for those protocols respectively.

Other changes
- Added DNS response IPs to `related.ip`.
- Some DNS events were missing `dns.header_flags`.
  • Loading branch information
@andrewkroh
andrewkroh committed Nov 2, 2020
1 parent 0626c2a commit abd075e
Show file tree
Hide file tree
Showing 10 changed files with 684 additions and 473 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -649,6 +649,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add max_number_of_messages config into s3 input. {pull}21993[21993]
- Update Okta documentation for new stateful restarts. {pull}22091[22091]
- Copy tag names from MISP data into events. {pull}21664[21664]
- Added DNS response IP addresses to `related.ip` in Suricata module. {pull}22291[22291]

*Heartbeat*

Expand Down
365 changes: 9 additions & 356 deletions x-pack/filebeat/module/suricata/eve/config/eve.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,363 +42,16 @@ processors:
{{ if .community_id }}
- community_id:
{{ end }}
- if:
equals:
suricata.eve.event_type: dns
then:
- convert:
ignore_missing: true
fail_on_error: false
mode: copy
fields:
- {from: suricata.eve.dns.id, to: dns.id, type: string}
- {from: suricata.eve.dns.rcode, to: dns.response_code}
- {from: suricata.eve.dns.type, to: dns.type}
- convert:
when.equals.dns.type: query
ignore_missing: true
fail_on_error: false
mode: copy
fields:
- {from: suricata.eve.dns.rrname, to: dns.question.name}
- {from: suricata.eve.dns.rrtype, to: dns.question.type}
- if:
and:
- equals.dns.type: answer
- equals.suricata.eve.dns.version: 2
then:
- convert:
ignore_missing: true
fail_on_error: false
mode: copy
fields:
- {from: suricata.eve.dns.rrname, to: dns.question.name}
- {from: suricata.eve.dns.rrtype, to: dns.question.type}
- registered_domain:
ignore_missing: true
ignore_failure: true
field: dns.question.name
target_field: dns.question.registered_domain
- script:
id: eve_process
lang: javascript
source: >-
function addEcsCategorization(evt) {
var event_type = evt.Get("suricata.eve.event_type");
if (event_type == null) {
return;
}
var catArray = [];
var typeArray = [];
evt.Put("suricata.eve.event_type", event_type.toLowerCase());
switch (event_type.toLowerCase()) {
case "alert":
evt.Put("event.kind", "alert");
catArray.push("network");
catArray.push("intrusion_detection");
break;
case "anomaly":
evt.Put("event.kind", "event");
catArray.push("network");
break;
case "http":
evt.Put("event.kind", "event");
catArray.push("network");
catArray.push("web");
typeArray.push("access");
typeArray.push("protocol");
evt.Put("network.protocol", "http");
var status = evt.Get("suricata.eve.http.status");
if (status == null) {
break;
}
if (status < 400) {
evt.Put("event.outcome", "success");
}
if (status >= 400 ) {
evt.Put("event.outcome", "failure");
}
break;
case "dns":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "dns");
break;
case "ftp":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "ftp");
break;
case "ftp_data":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "ftp");
break;
case "tls":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "tls");
break;
case "tftp":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "tftp");
break;
case "smb":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "smb");
break;
case "ssh":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "ssh");
break;
case "flow":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("connection");
var state = evt.Get("suricata.eve.flow.state");
if (state == null) {
break;
}
switch (state) {
case "new":
typeArray.push("start");
break;
case "closed":
typeArray.push("end");
break;
}
break;
case "rdp":
evt.Put("event.kind", "event");
catArray.push("network");
typeArray.push("protocol");
evt.Put("network.protocol", "rdp");
break;
case "stats":
evt.Put("event.kind", "metric");
break;
default:
evt.Put("event.kind", "event");
catArray.push("network");
}
if (catArray.length > 0) {
evt.Put("event.category", catArray);
}
if (typeArray.length > 0) {
evt.Put("event.type", typeArray);
}
}
function setDnsV1Answers(evt) {
var dns_type = evt.Get("dns.type")
if (dns_type != "answer") {
return;
}
var version = evt.Get("suricata.eve.dns.version")
if (version == "2") {
return;
}
var name = evt.Get("suricata.eve.dns.rrname");
var data = evt.Get("suricata.eve.dns.rdata");
var type = evt.Get("suricata.eve.dns.rrtype");
var ttl = evt.Get("suricata.eve.dns.ttl");
var answer = {};
if (name) {
answer.name = name;
}
if (data) {
answer.data = data;
}
if (type) {
answer.type = type;
}
if (ttl) {
answer.ttl = ttl;
}
if (Object.keys(answer).length === 0) {
return;
}
evt.Put("dns.answers", [answer]);
}
function addDnsV2Answers(evt) {
var type = evt.Get("dns.type")
if (type != "answer") {
return;
}
var version = evt.Get("suricata.eve.dns.version")
if (version != 2) {
return;
}
var answers = evt.Get("suricata.eve.dns.answers");
if (!answers) {
return;
}
evt.Delete("suricata.eve.dns.answers");
var resolvedIps = [];
for (var i = 0; i < answers.length; i++) {
var answer = answers[i];
// Rename properties.
var name = answer["rrname"];
delete answer["rrname"];
var type = answer["rrtype"];
delete answer["rrtype"];
var data = answer["rdata"];
delete answer["rdata"];
answer["name"] = name;
answer["type"] = type;
answer["data"] = data;
// Append IP addresses to dns.resolved_ip.
if (type === "A" || type === "AAAA") {
resolvedIps.push(data);
}
}
evt.Put("dns.answers", answers);
if (resolvedIps.length > 0) {
evt.Put("dns.resolved_ip", resolvedIps);
}
}
function addDnsV2HeaderFlags(evt) {
var type = evt.Get("dns.type")
if (type != "answer") {
return;
}
var version = evt.Get("suricata.eve.dns.version")
if (version != 2) {
return;
}
var flag = evt.Get("suricata.eve.dns.aa");
if (flag === true) {
evt.AppendTo("dns.header_flags", "AA");
}
flag = evt.Get("suricata.eve.dns.tc");
if (flag === true) {
evt.AppendTo("dns.header_flags", "TC");
}
flag = evt.Get("suricata.eve.dns.rd");
if (flag === true) {
evt.AppendTo("dns.header_flags", "RD");
}
flag = evt.Get("suricata.eve.dns.ra");
if (flag === true) {
evt.AppendTo("dns.header_flags", "RA");
}
}
function addTopLevelDomain(evt) {
var rd = evt.Get("dns.question.registered_domain");
if (rd == null) {
return;
}
var firstPeriod = rd.indexOf(".");
if (firstPeriod == -1) {
return;
}
evt.Put("dns.question.top_level_domain", rd.substr(firstPeriod + 1));
}
function cleanupAppProto(evt) {
var proto = evt.Get("suricata.eve.app_proto");
if (proto == null){
return;
}
switch (proto.toLowerCase()) {
case "failed":
case "template":
case "template-rust":
break;
case "ftp-data":
evt.Put("network.protocol", "ftp");
break;
default:
evt.Put("network.protocol", proto.toLowerCase());
}
evt.Delete("suricata.eve.app_proto");
}
function addRelatedIps(evt) {
var src_ip = evt.Get("source.ip");
if (src_ip != null) {
evt.AppendTo("related.ip", src_ip);
}
var dst_ip = evt.Get("destination.ip");
if (dst_ip != null) {
evt.AppendTo("related.ip", dst_ip);
}
}
function addTlsVersion(evt) {
var tls_version = evt.Get("suricata.eve.tls.version");
if (tls_version == null) {
return;
}
var parts = tls_version.split(" ");
if (parts.length < 2) {
return;
}
evt.Put("tls.version_protocol", parts[0].toLowerCase());
evt.Put("tls.version", parts[1]);
}
function cleanupTlsSni(evt) {
var sni = evt.Get("suricata.eve.tls.sni");
if (sni == null) {
return;
}
if ("." == sni.charAt(sni.length - 1)) {
evt.Put("suricata.eve.tls.sni", sni.substring(0, sni.length - 1));
}
}
function process(evt) {
var event_type = evt.Get("suricata.eve.event_type");
addEcsCategorization(evt);
if (event_type == "dns") {
setDnsV1Answers(evt);
addDnsV2Answers(evt);
addDnsV2HeaderFlags(evt);
addTopLevelDomain(evt);
}
cleanupAppProto(evt);
addRelatedIps(evt);
addTlsVersion(evt);
cleanupTlsSni(evt);
}
- convert:
- registered_domain:
when:
or:
- equals.suricata.eve.dns.type: query
# V2 events always include the query data.
- equals.suricata.eve.dns.version: 2
ignore_missing: true
fail_on_error: false
mode: copy
fields:
- {from: suricata.eve.tls.subject, to: tls.server.subject}
- {from: suricata.eve.tls.issuerdn, to: tls.server.issuer}
- {from: suricata.eve.tls.session_resumed, to: tls.resumed, type: boolean}
- {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1}
- {from: suricata.eve.tls.sni, to: tls.client.server_name}
- {from: suricata.eve.tls.sni, to: destination.domain}
- {from: suricata.eve.tls.ja3s.hash, to: tls.server.ja3s}
- {from: suricata.eve.tls.ja3.hash, to: tls.client.ja3}
- {from: suricata.eve.tls.certificate, to: tls.server.certificate}
- {from: suricata.eve.tls.chain, to: tls.server.certificate_chain}
- drop_fields:
ignore_missing: true
fields:
- suricata.eve.dns.aa
- suricata.eve.dns.tc
- suricata.eve.dns.rd
- suricata.eve.dns.ra
- suricata.eve.dns.qr
- suricata.eve.dns.version
- suricata.eve.dns.flags
- suricata.eve.dns.grouped
ignore_failure: true
field: suricata.eve.dns.rrname
target_field: dns.question.registered_domain
- add_fields:
target: ''
fields:
Expand Down
Loading

0 comments on commit abd075e

Please sign in to comment.