From 2c480149432d4fe92b7d6b84d996552230d02f30 Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 19 Feb 2024 18:59:29 +0200 Subject: [PATCH 01/33] catch problem json loading urlhaus responses --- modules/threat_intelligence/urlhaus.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/threat_intelligence/urlhaus.py b/modules/threat_intelligence/urlhaus.py index 58f81b1ab..9fe65cf1a 100644 --- a/modules/threat_intelligence/urlhaus.py +++ b/modules/threat_intelligence/urlhaus.py @@ -133,7 +133,10 @@ def urlhaus_lookup(self, ioc, type_of_ioc: str): if urlhaus_api_response.status_code != 200: return - response: dict = json.loads(urlhaus_api_response.text) + try: + response: dict = json.loads(urlhaus_api_response.text) + except json.decoder.JSONDecodeError: + return if response['query_status'] in ['no_results', 'invalid_url']: # no response or empty response From 493145b2941a18ed3cb7e6adbdf6af0635f6d654 Mon Sep 17 00:00:00 2001 From: alya Date: Tue, 20 Feb 2024 15:36:30 +0200 Subject: [PATCH 02/33] get client_ips from the config file --- config/slips.conf | 8 ++++++++ modules/flowalerts/flowalerts.py | 2 +- slips_files/common/parsers/config_parser.py | 21 ++++++++++++++++++++- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/config/slips.conf b/config/slips.conf index 24514146d..8ef2a9da8 100644 --- a/config/slips.conf +++ b/config/slips.conf @@ -137,6 +137,14 @@ export_labeled_flows = no # export_format can be tsv or json. this parameter is ignored if export_labeled_flows is set to no export_format = json +# These are the IPs that we see the majority of traffic going out of from. +# for example, this can be your own IP or some computer you’re monitoring +# when using slips on an interface, this client IP is automatically set as +# your own IP and is used to improve detections +# it would be useful to specify it when analyzing pcaps or zeek logs +#client_ips = [10.0.0.1, 172.16.0.9, 172.217.171.238] +client_ips = [] + ##################### # [2] Configuration for the detections [detection] diff --git a/modules/flowalerts/flowalerts.py b/modules/flowalerts/flowalerts.py index 2582a1ca1..b1e8c6a06 100644 --- a/modules/flowalerts/flowalerts.py +++ b/modules/flowalerts/flowalerts.py @@ -1096,7 +1096,7 @@ def check_invalid_dns_answers( for answer in answers: if answer in invalid_answers and domain != "localhost": - #blocked answer found + # blocked answer found self.set_evidence.invalid_dns_answer( domain, answer, daddr, profileid, twid, stime, uid ) diff --git a/slips_files/common/parsers/config_parser.py b/slips_files/common/parsers/config_parser.py index 91c3deabf..ac4da2ba7 100644 --- a/slips_files/common/parsers/config_parser.py +++ b/slips_files/common/parsers/config_parser.py @@ -1,6 +1,7 @@ from datetime import timedelta import sys import ipaddress +from typing import List import configparser from slips_files.common.parsers.arg_parser import ArgumentParser from slips_files.common.slips_utils import utils @@ -607,7 +608,25 @@ def rotation_period(self): 'parameters', 'rotation_period', '1 day' ) return utils.sanitize(rotation_period) - + + + def client_ips(self) -> List[str]: + client_ips: str = self.read_configuration( + 'parameters', 'client_ips', '[]' + ) + client_ips: str = utils.sanitize(client_ips) + client_ips: List[str] = (client_ips + .replace('[', '') + .replace(']', '') + .split(",") + ) + client_ips: List[str] = [client_ip.strip().strip("'") for client_ip + in client_ips] + # Remove empty strings if any + client_ips: List[str] = [client_ip for client_ip in client_ips if + client_ip] + return client_ips + def keep_rotated_files_for(self) -> int: """ returns period in seconds""" keep_rotated_files_for = self.read_configuration( From a3120bbd588fb63a1c8040d419e7779cd0ce246c Mon Sep 17 00:00:00 2001 From: alya Date: Tue, 20 Feb 2024 16:31:58 +0200 Subject: [PATCH 03/33] flowalerts: don't detect conn without dns if the daddr is in our client_ips --- modules/flowalerts/flowalerts.py | 51 +++++++++++++++++++------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/modules/flowalerts/flowalerts.py b/modules/flowalerts/flowalerts.py index b1e8c6a06..1265a0c82 100644 --- a/modules/flowalerts/flowalerts.py +++ b/modules/flowalerts/flowalerts.py @@ -1,11 +1,4 @@ import contextlib - -from slips_files.common.abstracts._module import IModule -from slips_files.common.imports import * -from .timer_thread import TimerThread -from .set_evidence import SetEvidnceHelper -from slips_files.core.helpers.whitelist import Whitelist -import multiprocessing import json import threading import ipaddress @@ -15,7 +8,13 @@ import collections import math import time + +from slips_files.common.imports import * +from .timer_thread import TimerThread +from .set_evidence import SetEvidnceHelper +from slips_files.core.helpers.whitelist import Whitelist from slips_files.common.slips_utils import utils +from typing import List class FlowAlerts(IModule): @@ -109,6 +108,7 @@ def read_configuration(self): self.pastebin_downloads_threshold = conf.get_pastebin_download_threshold() self.our_ips = utils.get_own_IPs() self.shannon_entropy_threshold = conf.get_entropy_threshold() + self.client_ips: List[str] = conf.client_ips() def check_connection_to_local_ip( self, @@ -364,8 +364,6 @@ def check_pastebin_download( """ Alerts on downloads from pastebin.com with more than 12000 bytes This function waits for the ssl.log flow to appear in conn.log before alerting - :param wait_time: the time we wait for the ssl conn to appear in conn.log in seconds - every time the timer is over, we wait extra 2 min and call the function again : param flow: this is the conn.log of the ssl flow we're currently checking """ @@ -425,7 +423,7 @@ def get_sent_bytes(all_flows: dict): mbs_uploaded = utils.convert_to_mb(bytes_uploaded) if mbs_uploaded < self.data_exfiltration_threshold: continue - + self.set_evidence.data_exfiltration( ip, mbs_uploaded, @@ -599,7 +597,26 @@ def is_well_known_org(self, ip): # (fb, twitter, microsoft, etc.) if self.whitelist.is_ip_in_org(ip, org): return True - + + def should_ignore_conn_without_dns(self, flow_type, appproto, daddr) \ + -> bool: + """ + checks for the cases that we should ignore the connection without dns + """ + # we should ignore this evidence if the ip is ours, whether it's a + # private ip or in the list of client_ips + return ( + flow_type != 'conn' + or appproto == 'dns' + or utils.is_ignored_ip(daddr) + # if the daddr is a client ip, it means that this is a conn + # from the internet to our ip, the dns res was probably + # made on their side before connecting to us, + # so we shouldn't be doing this detection on this ip + or daddr in self.client_ips + # because there's no dns.log to know if the dns was made + or self.db.get_input_type() == 'zeek_log_file' + ) def check_connection_without_dns_resolution( self, flow_type, appproto, daddr, twid, profileid, timestamp, uid @@ -611,18 +628,9 @@ def check_connection_without_dns_resolution( # 1- Do not check for DNS requests # 2- Ignore some IPs like private IPs, multicast, and broadcast - if ( - flow_type != 'conn' - or appproto == 'dns' - or utils.is_ignored_ip(daddr) - ): + if self.should_ignore_conn_without_dns(flow_type, appproto, daddr): return - # disable this alert when running on a zeek conn.log file - # because there's no dns.log to know if the dns was made - if self.db.get_input_type() == 'zeek_log_file': - return False - # Ignore some IP ## - All dhcp servers. Since is ok to connect to # them without a DNS request. @@ -652,6 +660,7 @@ def check_connection_without_dns_resolution( # search 24hs back for a dns resolution if self.db.is_ip_resolved(daddr, 24): return False + # self.print(f'No DNS resolution in {answers_dict}') # There is no DNS resolution, but it can be that Slips is # still reading it from the files. From e7a7e42bd8117a3ab57dd077bf2fd143da56530c Mon Sep 17 00:00:00 2001 From: alya Date: Tue, 20 Feb 2024 16:38:48 +0200 Subject: [PATCH 04/33] add type hinting --- modules/flowalerts/flowalerts.py | 11 +++++++---- slips_files/core/database/sqlite_db/database.py | 7 ++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/modules/flowalerts/flowalerts.py b/modules/flowalerts/flowalerts.py index 1265a0c82..d90967359 100644 --- a/modules/flowalerts/flowalerts.py +++ b/modules/flowalerts/flowalerts.py @@ -14,7 +14,9 @@ from .set_evidence import SetEvidnceHelper from slips_files.core.helpers.whitelist import Whitelist from slips_files.common.slips_utils import utils -from typing import List +from typing import List, \ + Tuple, \ + Dict class FlowAlerts(IModule): @@ -408,18 +410,19 @@ def get_sent_bytes(all_flows: dict): return bytes_sent - all_flows = self.db.get_all_flows_in_profileid( + all_flows: Dict[str, dict] = self.db.get_all_flows_in_profileid( profileid ) if not all_flows: return + bytes_sent: dict = get_sent_bytes(all_flows) for ip, ip_info in bytes_sent.items(): - # ip_info is a tuple (bytes_sent, [uids]) + ip_info: Tuple[int, List[str]] uids = ip_info[1] - bytes_uploaded = ip_info[0] + mbs_uploaded = utils.convert_to_mb(bytes_uploaded) if mbs_uploaded < self.data_exfiltration_threshold: continue diff --git a/slips_files/core/database/sqlite_db/database.py b/slips_files/core/database/sqlite_db/database.py index eec15a3cb..88b776f21 100644 --- a/slips_files/core/database/sqlite_db/database.py +++ b/slips_files/core/database/sqlite_db/database.py @@ -1,4 +1,5 @@ -from typing import List +from typing import List, \ + Dict import os.path import sqlite3 import json @@ -143,15 +144,15 @@ def get_all_flows_in_profileid_twid(self, profileid, twid): res[uid] = json.loads(flow) return res - def get_all_flows_in_profileid(self, profileid): + def get_all_flows_in_profileid(self, profileid) -> Dict[str, dict]: """ Return a list of all the flows in this profileid [{'uid':flow},...] """ condition = f'profileid = "{profileid}"' flows = self.select('flows', condition=condition) - all_flows = {} if flows: + all_flows: Dict[str, dict] = {} for flow in flows: uid = flow[0] flow: str = flow[1] From 014b1879602c2d3e23798e33c1f8b0107c4e41e5 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 14:15:36 +0200 Subject: [PATCH 05/33] flowalerts: use the dns answer of young domains to set evidence for the ip of that domain --- modules/flowalerts/flowalerts.py | 34 ++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/modules/flowalerts/flowalerts.py b/modules/flowalerts/flowalerts.py index 2582a1ca1..fbaf37aa0 100644 --- a/modules/flowalerts/flowalerts.py +++ b/modules/flowalerts/flowalerts.py @@ -1,6 +1,6 @@ import contextlib +from typing import List -from slips_files.common.abstracts._module import IModule from slips_files.common.imports import * from .timer_thread import TimerThread from .set_evidence import SetEvidnceHelper @@ -68,7 +68,8 @@ def init(self): # after this number of failed ssh logins, we alert pw guessing self.pw_guessing_threshold = 20 self.password_guessing_cache = {} - # in pastebin download detection, we wait for each conn.log flow of the seen ssl flow to appear + # in pastebin download detection, we wait for each conn.log flow + # of the seen ssl flow to appear # this is the dict of ssl flows we're waiting for self.pending_ssl_flows = multiprocessing.Queue() # thread that waits for ssl flows to appear in conn.log @@ -789,7 +790,7 @@ def check_dns_without_connection( # every dns answer is a list of ips that correspond to 1 query, # one of these ips should be present in the contacted ips # check each one of the resolutions of this domain - for ip in answers: + for ip in self.extract_ips_from_dns_answers(answers): # self.print(f'Checking if we have a connection to ip {ip}') if ( ip in contacted_ips @@ -1252,7 +1253,9 @@ def check_multiple_reconnection_attempts( profileid, twid, current_reconnections ) - def detect_young_domains(self, domain, stime, profileid, twid, uid): + def detect_young_domains(self, domain, answers: List[str], stime, + profileid, + twid, uid): """ Detect domains that are too young. The threshold is 60 days @@ -1278,12 +1281,27 @@ def detect_young_domains(self, domain, stime, profileid, twid, uid): age = domain_info['Age'] if age >= age_threshold: return False - + + + ips_returned_in_answer: List[str] = ( + self.extract_ips_from_dns_answers(answers) + ) + self.set_evidence.young_domain( - domain, age, stime, profileid, twid, uid + domain, age, stime, profileid, twid, uid, ips_returned_in_answer ) return True - + + def extract_ips_from_dns_answers(self, answers: List[str]) -> List[str]: + """ + extracts ipv4 and 6 from DNS answers + """ + ips = [] + for answer in answers: + if validators.ipv4(answer) or validators.ipv6(answer): + ips.append(answer) + return ips + def check_smtp_bruteforce( self, profileid, @@ -2129,7 +2147,7 @@ def main(self): # TODO: not sure how to make sure IP_info is # done adding domain age to the db or not self.detect_young_domains( - domain, stime, profileid, twid, uid + domain, answers, stime, profileid, twid, uid ) self.check_dns_arpa_scan( domain, stime, profileid, twid, uid From 37d6731d3d1b2bd350142ae271ad2b2c3a7ed84c Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 14:19:45 +0200 Subject: [PATCH 06/33] flowalerts: move weird http method detections to http analyzer module --- modules/flowalerts/flowalerts.py | 24 +----- modules/http_analyzer/http_analyzer.py | 82 ++++++++++++++++++- .../core/database/redis_db/database.py | 6 +- 3 files changed, 85 insertions(+), 27 deletions(-) diff --git a/modules/flowalerts/flowalerts.py b/modules/flowalerts/flowalerts.py index fbaf37aa0..f741d2f90 100644 --- a/modules/flowalerts/flowalerts.py +++ b/modules/flowalerts/flowalerts.py @@ -1603,25 +1603,6 @@ def check_malicious_ssl(self, ssl_info): ssl_info, ssl_info_from_db ) - def check_weird_http_method(self, msg): - """ - detect weird http methods in zeek's weird.log - """ - flow = msg['flow'] - profileid = msg['profileid'] - twid = msg['twid'] - - # what's the weird.log about - name = flow['name'] - - if 'unknown_HTTP_method' not in name: - return False - - self.set_evidence.weird_http_method( - profileid, - twid, - flow - ) def check_non_http_port_80_conns( self, @@ -2185,10 +2166,7 @@ def main(self): role='SSH::SERVER' ) - if msg := self.get_msg('new_weird'): - msg = json.loads(msg['data']) - self.check_weird_http_method(msg) - + if msg := self.get_msg('new_tunnel'): msg = json.loads(msg['data']) self.check_GRE_tunnel(msg) diff --git a/modules/http_analyzer/http_analyzer.py b/modules/http_analyzer/http_analyzer.py index f502387ed..bed6c5c5a 100644 --- a/modules/http_analyzer/http_analyzer.py +++ b/modules/http_analyzer/http_analyzer.py @@ -2,7 +2,8 @@ import json import urllib import requests -from typing import Union +from typing import Union, \ + Dict from slips_files.common.imports import * from slips_files.core.evidence_structure.evidence import \ @@ -29,8 +30,10 @@ class HTTPAnalyzer(IModule): def init(self): self.c1 = self.db.subscribe('new_http') + self.c2 = self.db.subscribe('new_weird') self.channels = { - 'new_http': self.c1 + 'new_http': self.c1, + 'new_weird': self.c2 } self.connections_counter = {} self.empty_connections_threshold = 4 @@ -637,8 +640,79 @@ def check_pastebin_downloads( self.db.set_evidence(evidence) return True + + def set_evidence_weird_http_method( + self, + profileid: str, + twid: str, + flow: dict + ) -> None: + daddr: str = flow['daddr'] + weird_method: str = flow['addl'] + uid: str = flow['uid'] + timestamp: str = flow['starttime'] + + confidence = 0.9 + threat_level: ThreatLevel = ThreatLevel.MEDIUM + saddr: str = profileid.split("_")[-1] + + attacker: Attacker = Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ) + victim: Victim = Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ) + + ip_identification: str = self.db.get_ip_identification(daddr) + description: str = f'Weird HTTP method "{weird_method}" to IP: ' \ + f'{daddr} {ip_identification}. by Zeek.' + + twid_number: int = int(twid.replace("timewindow", "")) + + evidence: Evidence = Evidence( + evidence_type=EvidenceType.WEIRD_HTTP_METHOD, + attacker=attacker, + victim=victim, + threat_level=threat_level, + category=IDEACategory.ANOMALY_TRAFFIC, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=timestamp, + conn_count=1, + confidence=confidence + ) + + self.db.set_evidence(evidence) + + + def check_weird_http_method(self, msg: Dict[str]): + """ + detect weird http methods in zeek's weird.log + """ + flow = msg['flow'] + profileid = msg['profileid'] + twid = msg['twid'] + + # what's the weird.log about + name = flow['name'] + + if 'unknown_HTTP_method' not in name: + return False + + self.set_evidence_weird_http_method( + profileid, + twid, + flow + ) + def pre_main(self): utils.drop_root_privs() @@ -736,3 +810,7 @@ def main(self): uid, timestamp ) + + if msg := self.get_msg('new_weird'): + msg = json.loads(msg['data']) + self.check_weird_http_method(msg) diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index e8d951cf7..59e0f20e1 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -628,10 +628,12 @@ def store_p2p_report(self, ip: str, report_data: dict): and last_report_about_this_ip['confidence'] == confidence ): report_time = report_data['report_time'] - # score and confidence are the same as the last report, only update the time + # score and confidence are the same as the last report, + # only update the time last_report_about_this_ip['report_time'] = report_time else: - # score and confidence are the different from the last report, add report to the list + # score and confidence are the different from the last + # report, add report to the list cached_p2p_reports[reporter].append(report_data) else: # ip was reported before, but not by the same peer From 587bcb2ef67585965d94315b4009bbd4593986cd Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 16:48:57 +0200 Subject: [PATCH 07/33] flowalerts: set 2 evidence for some detections, one for the saddr and one for the daddr --- modules/arp/arp.py | 2 +- modules/flowalerts/flowalerts.py | 11 +- modules/flowalerts/set_evidence.py | 811 +++++++++++++++-------------- 3 files changed, 414 insertions(+), 410 deletions(-) diff --git a/modules/arp/arp.py b/modules/arp/arp.py index b620439b9..c05d187fa 100644 --- a/modules/arp/arp.py +++ b/modules/arp/arp.py @@ -379,7 +379,7 @@ def detect_unsolicited_arp( # We're sure this is unsolicited arp # it may be arp spoofing confidence: float = 0.8 - threat_level: ThreatLevel = ThreatLevel.INFO + threat_level: ThreatLevel = ThreatLevel.LOW description: str = 'broadcasting unsolicited ARP' saddr: str = profileid.split('_')[-1] diff --git a/modules/flowalerts/flowalerts.py b/modules/flowalerts/flowalerts.py index f741d2f90..6ef76b63b 100644 --- a/modules/flowalerts/flowalerts.py +++ b/modules/flowalerts/flowalerts.py @@ -1483,7 +1483,6 @@ def detect_malicious_ja3( daddr, ja3, ja3s, - profileid, twid, uid, timestamp @@ -1496,16 +1495,16 @@ def detect_malicious_ja3( malicious_ja3_dict = self.db.get_ja3_in_IoC() if ja3 in malicious_ja3_dict: - self.set_evidence.malicious_ja3( + self.set_evidence.malicious_ja3s( malicious_ja3_dict, twid, uid, timestamp, - daddr, saddr, - type_='ja3', + daddr, ja3=ja3, - ) + ) + if ja3s in malicious_ja3_dict: self.set_evidence.malicious_ja3( @@ -1515,7 +1514,6 @@ def detect_malicious_ja3( timestamp, saddr, daddr, - type_='ja3s', ja3=ja3s, ) @@ -2064,7 +2062,6 @@ def main(self): daddr, ja3, ja3s, - profileid, twid, uid, timestamp diff --git a/modules/flowalerts/set_evidence.py b/modules/flowalerts/set_evidence.py index 8d69a2ed4..9f1edacfd 100644 --- a/modules/flowalerts/set_evidence.py +++ b/modules/flowalerts/set_evidence.py @@ -32,31 +32,46 @@ def young_domain( domain: str, age: int, stime: str, - profileid: ProfileID, + profileid: str, twid: str, - uid: str + uid: str, + answers: List[str] ): saddr: str = profileid.split("_")[-1] - victim = Victim( - direction=Direction.SRC, - victim_type=IoCType.IP, - value=saddr, - ) - attacker = Attacker( - direction=Direction.DST, - attacker_type=IoCType.DOMAIN, - value=domain, - ) twid_number: int = int(twid.replace("timewindow", "")) - description = f'connection to a young domain: {domain} ' \ - f'registered {age} days ago.', + description: str = (f'connection to a young domain: {domain} ' + f'registered {age} days ago.') + if answers: + attacker = answers[0] + evidence = Evidence( + evidence_type=EvidenceType.YOUNG_DOMAIN, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=attacker, + ), + threat_level=ThreatLevel.LOW, + category=IDEACategory.ANOMALY_TRAFFIC, + description=description, + profile=ProfileID(ip=attacker.value), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=stime, + conn_count=1, + confidence=1.0 + ) + self.db.set_evidence(evidence) + evidence = Evidence( evidence_type=EvidenceType.YOUNG_DOMAIN, - attacker=attacker, - threat_level=ThreatLevel.LOW, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr, + ), + threat_level=ThreatLevel.INFO, category=IDEACategory.ANOMALY_TRAFFIC, description=description, - victim=victim, profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=twid_number), uid=[uid], @@ -65,6 +80,7 @@ def young_domain( confidence=1.0 ) self.db.set_evidence(evidence) + def multiple_ssh_versions( self, @@ -82,22 +98,21 @@ def multiple_ssh_versions( :param role: can be 'SSH::CLIENT' or 'SSH::SERVER' as seen in zeek software.log flows """ - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) role = 'client' if 'CLIENT' in role.upper() else 'server' description = f'SSH {role} version changing from ' \ f'{cached_versions} to {current_versions}' evidence = Evidence( evidence_type=EvidenceType.MULTIPLE_SSH_VERSIONS, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), threat_level=ThreatLevel.MEDIUM, category=IDEACategory.ANOMALY_TRAFFIC, description=description, - profile=ProfileID(ip=attacker.value), + profile=ProfileID(ip=srcip), timewindow=TimeWindow(int(twid.replace("timewindow", ''))), uid=uid, timestamp=timestamp, @@ -187,22 +202,18 @@ def device_changing_ips( confidence = 0.8 threat_level = ThreatLevel.MEDIUM saddr: str = profileid.split("_")[-1] - - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - description = f'A device changing IPs. IP {saddr} was found ' \ f'with MAC address {smac} but the MAC belongs ' \ f'originally to IP: {old_ip}. ' - twid_number = int(twid.replace("timewindow", "")) evidence = Evidence( evidence_type=EvidenceType.DEVICE_CHANGING_IP, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), threat_level=threat_level, category=IDEACategory.ANOMALY_TRAFFIC, description=description, @@ -226,17 +237,8 @@ def non_http_port_80_conn( uid: str ) -> None: confidence = 0.8 - threat_level = ThreatLevel.MEDIUM saddr: str = profileid.split("_")[-1] - - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) - description: str = f'non-HTTP established connection to port 80. ' \ f'destination IP: {daddr} {ip_identification}' @@ -244,8 +246,12 @@ def non_http_port_80_conn( evidence: Evidence = Evidence( evidence_type=EvidenceType.NON_HTTP_PORT_80_CONNECTION, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, category=IDEACategory.ANOMALY_TRAFFIC, description=description, profile=ProfileID(ip=saddr), @@ -255,6 +261,25 @@ def non_http_port_80_conn( conn_count=1, confidence=confidence ) + self.db.set_evidence(evidence) + + evidence: Evidence = Evidence( + evidence_type=EvidenceType.NON_HTTP_PORT_80_CONNECTION, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.MEDIUM, + category=IDEACategory.ANOMALY_TRAFFIC, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=timestamp, + conn_count=1, + confidence=confidence + ) self.db.set_evidence(evidence) @@ -267,20 +292,7 @@ def non_ssl_port_443_conn( uid: str ) -> None: confidence: float = 0.8 - threat_level: ThreatLevel = ThreatLevel.MEDIUM saddr: str = profileid.split("_")[-1] - - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = f'non-SSL established connection to port 443. ' \ f'destination IP: {daddr} {ip_identification}' @@ -289,9 +301,17 @@ def non_ssl_port_443_conn( evidence: Evidence = Evidence( evidence_type=EvidenceType.NON_SSL_PORT_443_CONNECTION, - attacker=attacker, - victim=victim, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.MEDIUM, category=IDEACategory.ANOMALY_TRAFFIC, description=description, profile=ProfileID(ip=saddr), @@ -304,55 +324,7 @@ def non_ssl_port_443_conn( self.db.set_evidence(evidence) - def weird_http_method( - self, - profileid: str, - twid: str, - flow: dict - ) -> None: - daddr: str = flow['daddr'] - weird_method: str = flow['addl'] - uid: str = flow['uid'] - timestamp: str = flow['starttime'] - - confidence = 0.9 - threat_level: ThreatLevel = ThreatLevel.MEDIUM - saddr: str = profileid.split("_")[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - - ip_identification: str = self.db.get_ip_identification(daddr) - description: str = f'Weird HTTP method "{weird_method}" to IP: ' \ - f'{daddr} {ip_identification}. by Zeek.' - - twid_number: int = int(twid.replace("timewindow", "")) - - evidence: Evidence = Evidence( - evidence_type=EvidenceType.WEIRD_HTTP_METHOD, - attacker=attacker, - victim=victim, - threat_level=threat_level, - category=IDEACategory.ANOMALY_TRAFFIC, - description=description, - profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=twid_number), - uid=[uid], - timestamp=timestamp, - conn_count=1, - confidence=confidence - ) - self.db.set_evidence(evidence) def incompatible_CN( self, @@ -364,21 +336,7 @@ def incompatible_CN( uid: str ) -> None: confidence: float = 0.9 - threat_level: ThreatLevel = ThreatLevel.MEDIUM saddr: str = profileid.split("_")[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = f'Incompatible certificate CN to IP: {daddr} ' \ f'{ip_identification} claiming to ' \ @@ -387,9 +345,17 @@ def incompatible_CN( twid_number: int = int(twid.replace("timewindow", "")) evidence: Evidence = Evidence( evidence_type=EvidenceType.INCOMPATIBLE_CN, - attacker=attacker, - victim=victim, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.MEDIUM, category=IDEACategory.ANOMALY_TRAFFIC, description=description, profile=ProfileID(ip=saddr), @@ -415,21 +381,18 @@ def DGA( # +1 ensures that the minimum confidence score is 1. confidence: float = max(0, (1 / 100) * (nxdomains - 100) + 1) confidence = round(confidence, 2) # for readability - threat_level = ThreatLevel.HIGH saddr = profileid.split("_")[-1] description = f'Possible DGA or domain scanning. {saddr} ' \ f'failed to resolve {nxdomains} domains' - attacker = Attacker( + evidence: Evidence = Evidence( + evidence_type=EvidenceType.DGA_NXDOMAINS, + attacker= Attacker( direction=Direction.SRC, attacker_type=IoCType.IP, value=saddr - ) - - evidence: Evidence = Evidence( - evidence_type=EvidenceType.DGA_NXDOMAINS, - attacker=attacker, - threat_level=threat_level, + ), + threat_level=ThreatLevel.HIGH, category=IDEACategory.ANOMALY_BEHAVIOUR, description=description, profile=ProfileID(ip=saddr), @@ -452,23 +415,18 @@ def DNS_without_conn( uid: str ) -> None: confidence: float = 0.8 - threat_level: ThreatLevel = ThreatLevel.LOW saddr: str = profileid.split("_")[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - description: str = f'domain {domain} resolved with no connection' - twid_number: int = int(twid.replace("timewindow", "")) evidence: Evidence = Evidence( evidence_type=EvidenceType.DNS_WITHOUT_CONNECTION, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, category=IDEACategory.ANOMALY_TRAFFIC, description=description, profile=ProfileID(ip=saddr), @@ -490,15 +448,8 @@ def pastebin_download( uid: str ) -> bool: - threat_level: ThreatLevel = ThreatLevel.INFO confidence: float = 1.0 saddr: str = profileid.split("_")[-1] - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - response_body_len: float = utils.convert_to_mb(bytes_downloaded) description: str = f'A downloaded file from pastebin.com. ' \ f'size: {response_body_len} MBs' @@ -506,8 +457,12 @@ def pastebin_download( twid_number: int = int(twid.replace("timewindow", "")) evidence: Evidence = Evidence( evidence_type=EvidenceType.PASTEBIN_DOWNLOAD, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.INFO, category=IDEACategory.ANOMALY_BEHAVIOUR, description=description, profile=ProfileID(ip=saddr), @@ -531,7 +486,7 @@ def conn_without_dns( uid: str ) -> None: confidence: float = 0.8 - threat_level: ThreatLevel = ThreatLevel.HIGH + threat_level: ThreatLevel = ThreatLevel.INFO saddr: str = profileid.split("_")[-1] attacker: Attacker = Attacker( direction=Direction.SRC, @@ -587,18 +542,16 @@ def dns_arpa_scan( description = f"Doing DNS ARPA scan. Scanned {arpa_scan_threshold}" \ f" hosts within 2 seconds." - # Store attacker details in a local variable - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) # Create Evidence object using local variables evidence = Evidence( evidence_type=EvidenceType.DNS_ARPA_SCAN, description=description, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), threat_level=threat_level, category=IDEACategory.RECON_SCANNING, profile=ProfileID(ip=saddr), @@ -629,18 +582,6 @@ def unknown_port( twid_number: int = int(twid.replace("timewindow", "")) saddr = profileid.split('_')[-1] - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = ( f'Connection to unknown destination port {dport}/{proto.upper()} ' @@ -649,8 +590,16 @@ def unknown_port( evidence: Evidence = Evidence( evidence_type=EvidenceType.UNKNOWN_PORT, - attacker=attacker, - victim=victim, + attacker= Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=ThreatLevel.HIGH, category=IDEACategory.ANOMALY_CONNECTION, description=description, @@ -677,24 +626,20 @@ def pw_guessing( # confidence = 1 because this detection is comming # from a zeek file so we're sure it's accurate confidence: float = 1.0 - threat_level: ThreatLevel = ThreatLevel.HIGH twid_number: int = int(twid.replace("timewindow", "")) scanning_ip: str = msg.split(' appears')[0] description: str = f'password guessing. {msg}. by {by}.' - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=scanning_ip - ) - conn_count: int = int(msg.split('in ')[1].split('connections')[0]) evidence: Evidence = Evidence( evidence_type=EvidenceType.PASSWORD_GUESSING, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=scanning_ip + ), + threat_level=ThreatLevel.HIGH, category= IDEACategory.ATTEMPT_LOGIN, description=description, profile=ProfileID(ip=scanning_ip), @@ -718,7 +663,6 @@ def horizontal_portscan( uid: str ) -> None: confidence: float = 1.0 - threat_level: ThreatLevel = ThreatLevel.HIGH twid_number: int = int(twid.replace("timewindow", "")) saddr = profileid.split('_')[-1] @@ -726,16 +670,14 @@ def horizontal_portscan( # get the number of unique hosts scanned on a specific port conn_count: int = int(msg.split('least')[1].split('unique')[0]) - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - evidence: Evidence = Evidence( evidence_type=EvidenceType.HORIZONTAL_PORT_SCAN, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.HIGH, category=IDEACategory.RECON_SCANNING, description=description, profile=ProfileID(ip=saddr), @@ -761,7 +703,6 @@ def conn_to_private_ip( timestamp: str ) -> None: confidence: float = 1.0 - threat_level: ThreatLevel = ThreatLevel.INFO twid_number: int = int(twid.replace("timewindow", "")) description: str = f'Connecting to private IP: {daddr} ' @@ -772,21 +713,14 @@ def conn_to_private_ip( else: description += f'on destination port: {dport}' - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - evidence: Evidence = Evidence( evidence_type=EvidenceType.CONNECTION_TO_PRIVATE_IP, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.INFO, category=IDEACategory.RECON, description=description, profile=ProfileID(ip=saddr), @@ -795,7 +729,11 @@ def conn_to_private_ip( timestamp=timestamp, conn_count=1, confidence=confidence, - victim=victim + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ) ) self.db.set_evidence(evidence) @@ -825,22 +763,18 @@ def GRE_tunnel( f'to {daddr} {ip_identification} ' \ f'tunnel action: {action}' - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - evidence: Evidence = Evidence( evidence_type=EvidenceType.GRE_TUNNEL, - attacker=attacker, - victim=victim, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, category=IDEACategory.INFO, description=description, @@ -866,29 +800,24 @@ def vertical_portscan( # confidence = 1 because this detection is coming # from a Zeek file so we're sure it's accurate confidence: float = 1.0 - threat_level: ThreatLevel = ThreatLevel.HIGH twid: int = int(twid.replace("timewindow", "")) # msg example: 192.168.1.200 has scanned 60 ports of 192.168.1.102 description: str = f'vertical port scan by Zeek engine. {msg}' conn_count: int = int(msg.split('least ')[1].split(' unique')[0]) - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=scanning_ip - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=msg.split('ports of host ')[-1].split(" in")[0] - ) - + evidence: Evidence = Evidence( evidence_type=EvidenceType.VERTICAL_PORT_SCAN, - attacker=attacker, - victim=victim, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=scanning_ip + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=msg.split('ports of host ')[-1].split(" in")[0] + ), + threat_level=ThreatLevel.HIGH, category=IDEACategory.RECON_SCANNING, description=description, profile=ProfileID(ip=scanning_ip), @@ -924,17 +853,6 @@ def ssh_successful( threat_level: ThreatLevel = ThreatLevel.INFO twid: int = int(twid.replace("timewindow", "")) - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = ( f'SSH successful to IP {daddr}. {ip_identification}. ' @@ -944,8 +862,16 @@ def ssh_successful( evidence: Evidence = Evidence( evidence_type=EvidenceType.SSH_SUCCESSFUL, - attacker=attacker, - victim=victim, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, confidence=confidence, description=description, @@ -970,7 +896,6 @@ def long_connection( """ Set an evidence for a long connection. """ - threat_level: ThreatLevel = ThreatLevel.LOW twid: int = int(twid.replace("timewindow", "")) # Confidence depends on how long the connection. # Scale the confidence from 0 to 1; 1 means 24 hours long. @@ -980,18 +905,6 @@ def long_connection( duration_minutes: int = int(duration / 60) srcip: str = profileid.split('_')[1] - attacker_obj: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) - - victim_obj: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = ( f'Long Connection. Connection from {srcip} ' @@ -1001,8 +914,12 @@ def long_connection( evidence: Evidence = Evidence( evidence_type=EvidenceType.LONG_CONNECTION, - attacker=attacker_obj, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), + threat_level=ThreatLevel.LOW, confidence=confidence, description=description, profile=ProfileID(ip=srcip), @@ -1010,7 +927,11 @@ def long_connection( uid=[uid], timestamp=timestamp, category=IDEACategory.ANOMALY_CONNECTION, - victim=victim_obj + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ) ) self.db.set_evidence(evidence) @@ -1028,7 +949,6 @@ def self_signed_certificates( Set evidence for self-signed certificates. """ confidence: float = 0.5 - threat_level: ThreatLevel = ThreatLevel.LOW saddr: str = profileid.split("_")[-1] twid: int = int(twid.replace("timewindow", "")) @@ -1048,7 +968,7 @@ def self_signed_certificates( evidence: Evidence = Evidence( evidence_type=EvidenceType.SELF_SIGNED_CERTIFICATE, attacker=attacker, - threat_level=threat_level, + threat_level=ThreatLevel.LOW, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -1057,7 +977,25 @@ def self_signed_certificates( timestamp=timestamp, category=IDEACategory.ANOMALY_BEHAVIOUR ) + self.db.set_evidence(evidence) + attacker: Attacker = Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ) + evidence: Evidence = Evidence( + evidence_type=EvidenceType.SELF_SIGNED_CERTIFICATE, + attacker=attacker, + threat_level=ThreatLevel.LOW, + confidence=confidence, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid), + uid=[uid], + timestamp=timestamp, + category=IDEACategory.ANOMALY_BEHAVIOUR + ) self.db.set_evidence(evidence) def multiple_reconnection_attempts( @@ -1077,18 +1015,6 @@ def multiple_reconnection_attempts( saddr: str = profileid.split("_")[-1] twid: int = int(twid.replace("timewindow", "")) - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification = self.db.get_ip_identification(daddr) description = ( f'Multiple reconnection attempts to Destination IP:' @@ -1097,8 +1023,16 @@ def multiple_reconnection_attempts( ) evidence: Evidence = Evidence( evidence_type=EvidenceType.MULTIPLE_RECONNECTION_ATTEMPTS, - attacker=attacker, - victim = victim, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim = Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, confidence=confidence, description=description, @@ -1125,7 +1059,6 @@ def connection_to_multiple_ports( Set evidence for connection to multiple ports. """ confidence: float = 0.5 - threat_level: ThreatLevel = ThreatLevel.INFO twid: int = int(twid.replace("timewindow", "")) ip_identification = self.db.get_ip_identification(attacker) description = f'Connection to multiple ports {dstports} of ' \ @@ -1140,22 +1073,19 @@ def connection_to_multiple_ports( victim_direction = Direction.SRC profile_ip = victim - victim: Victim = Victim( + evidence = Evidence( + evidence_type=EvidenceType.CONNECTION_TO_MULTIPLE_PORTS, + attacker=Attacker( + direction=attacker_direction, + attacker_type=IoCType.IP, + value=attacker + ), + victim=Victim( direction=victim_direction, victim_type=IoCType.IP, value=victim - ) - attacker: Attacker = Attacker( - direction=attacker_direction, - attacker_type=IoCType.IP, - value=attacker - ) - - evidence = Evidence( - evidence_type=EvidenceType.CONNECTION_TO_MULTIPLE_PORTS, - attacker=attacker, - victim=victim, - threat_level=threat_level, + ), + threat_level=ThreatLevel.INFO, confidence=confidence, description=description, profile=ProfileID(ip=profile_ip), @@ -1179,30 +1109,21 @@ def suspicious_dns_answer( uid: str ) -> None: confidence: float = 0.6 - threat_level: ThreatLevel = ThreatLevel.MEDIUM twid: int = int(twid.replace("timewindow", "")) saddr: str = profileid.split("_")[-1] - attacker: Attacker = Attacker( - direction=Direction.DST, - attacker_type=IoCType.IP, - value=daddr - ) - victim: Victim = Victim( - direction=Direction.SRC, - victim_type=IoCType.IP, - value=saddr - ) - description: str = f'A DNS TXT answer with high entropy. ' \ f'query: {query} answer: "{answer}" ' \ f'entropy: {round(entropy, 2)} ' evidence: Evidence = Evidence( evidence_type=EvidenceType.HIGH_ENTROPY_DNS_ANSWER, - attacker=attacker, - victim=victim, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.MEDIUM, confidence=confidence, description=description, profile=ProfileID(ip=daddr), @@ -1213,40 +1134,49 @@ def suspicious_dns_answer( ) self.db.set_evidence(evidence) + + + evidence: Evidence = Evidence( + evidence_type=EvidenceType.HIGH_ENTROPY_DNS_ANSWER, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, + confidence=confidence, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid), + uid=[uid], + timestamp=stime, + category=IDEACategory.ANOMALY_TRAFFIC + ) + self.db.set_evidence(evidence) def invalid_dns_answer( self, query: str, answer: str, - daddr: str, profileid: str, twid: str, stime: str, - uid: str + uid: str, ) -> None: - threat_level: ThreatLevel = ThreatLevel.INFO confidence: float = 0.7 twid: int = int(twid.replace("timewindow", "")) saddr: str = profileid.split("_")[-1] - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - description: str = f"The DNS query {query} was resolved to {answer}" evidence: Evidence = Evidence( evidence_type=EvidenceType.INVALID_DNS_RESOLUTION, - attacker=attacker, - victim=victim, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.INFO, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -1257,7 +1187,7 @@ def invalid_dns_answer( ) self.db.set_evidence(evidence) - + def for_port_0_connection( self, @@ -1284,26 +1214,22 @@ def for_port_0_connection( victim_direction = Direction.SRC profile_ip = victim - victim: Victim = Victim( - direction=victim_direction, - victim_type=IoCType.IP, - value=victim - ) - attacker: Attacker = Attacker( - direction=attacker_direction, - attacker_type=IoCType.IP, - value=attacker - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = f'Connection on port 0 from {saddr}:{sport} ' \ f'to {daddr}:{dport}. {ip_identification}.' - evidence: Evidence = Evidence( evidence_type=EvidenceType.PORT_0_CONNECTION, - attacker=attacker, - victim=victim, + attacker=Attacker( + direction=attacker_direction, + attacker_type=IoCType.IP, + value=attacker + ), + victim=Victim( + direction=victim_direction, + victim_type=IoCType.IP, + value=victim + ), threat_level=threat_level, confidence=confidence, description=description, @@ -1317,7 +1243,76 @@ def for_port_0_connection( ) self.db.set_evidence(evidence) + + + def malicious_ja3s( + self, + malicious_ja3_dict: dict, + twid: str, + uid: str, + timestamp: str, + saddr: str, + daddr: str, + ja3: str = '', + ) -> None: + ja3_info: dict = json.loads(malicious_ja3_dict[ja3]) + + threat_level: str = ja3_info['threat_level'].upper() + threat_level: ThreatLevel = ThreatLevel[threat_level] + tags: str = ja3_info.get('tags', '') + ja3_description: str = ja3_info['description'] + + ip_identification: str = self.db.get_ip_identification(daddr) + description = ( + f'Malicious JA3s: (possible C&C server): {ja3} to server ' + f'{daddr} {ip_identification} ' + ) + if ja3_description != 'None': + description += f'description: {ja3_description} ' + description += f'tags: {tags}' + confidence: float = 1 + twid_number: int = int(twid.replace("timewindow", "")) + evidence: Evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_JA3S, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=timestamp, + category=IDEACategory.INTRUSION_BOTNET, + source_target_tag=Tag.CC + ) + + self.db.set_evidence(evidence) + attacker: Attacker = Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ) + evidence: Evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_JA3S, + attacker=attacker, + threat_level=ThreatLevel.LOW, + confidence=confidence, + description=description, + profile=ProfileID(ip=attacker.value), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=timestamp, + category=IDEACategory.INTRUSION_BOTNET, + source_target_tag=Tag.CC + ) + + self.db.set_evidence(evidence) + def malicious_ja3( self, @@ -1325,13 +1320,10 @@ def malicious_ja3( twid: str, uid: str, timestamp: str, - victim: str, - attacker: str, - type_: str = '', + daddr: str, + saddr: str, ja3: str = '', ) -> None: - """ - """ ja3_info: dict = json.loads(malicious_ja3_dict[ja3]) threat_level: str = ja3_info['threat_level'].upper() @@ -1340,47 +1332,26 @@ def malicious_ja3( tags: str = ja3_info.get('tags', '') ja3_description: str = ja3_info['description'] - if type_ == 'ja3': - description = f'Malicious JA3: {ja3} from source address ' \ - f'{attacker} ' - evidence_type: EvidenceType = EvidenceType.MALICIOUS_JA3 - source_target_tag: Tag = Tag.BOTNET - attacker_direction: Direction = Direction.SRC - victim_direction: Direction = Direction.DST - - elif type_ == 'ja3s': - description = ( - f'Malicious JA3s: (possible C&C server): {ja3} to server ' - f'{attacker} ' - ) - - evidence_type: EvidenceType = EvidenceType.MALICIOUS_JA3S - source_target_tag: Tag = Tag.CC - attacker_direction: Direction = Direction.DST - victim_direction: Direction = Direction.SRC - else: - return - - # append daddr identification to the description - ip_identification: str = self.db.get_ip_identification(attacker) - description += f'{ip_identification} ' + ip_identification: str = self.db.get_ip_identification(saddr) + description = f'Malicious JA3: {ja3} from source address ' \ + f'{saddr} {ip_identification}' if ja3_description != 'None': description += f'description: {ja3_description} ' description += f'tags: {tags}' attacker: Attacker = Attacker( - direction=attacker_direction, + direction=Direction.SRC, attacker_type=IoCType.IP, - value=attacker + value=saddr ) victim: Victim = Victim( - direction=victim_direction, + direction= Direction.DST, victim_type=IoCType.IP, - value=victim + value=daddr ) confidence: float = 1 evidence: Evidence = Evidence( - evidence_type=evidence_type, + evidence_type=EvidenceType.MALICIOUS_JA3, attacker=attacker, victim=victim, threat_level=threat_level, @@ -1391,7 +1362,7 @@ def malicious_ja3( uid=[uid], timestamp=timestamp, category=IDEACategory.INTRUSION_BOTNET, - source_target_tag=source_target_tag + source_target_tag=Tag.BOTNET ) self.db.set_evidence(evidence) @@ -1406,7 +1377,6 @@ def data_exfiltration( timestamp ) -> None: confidence: float = 0.6 - threat_level: ThreatLevel = ThreatLevel.HIGH saddr: str = profileid.split("_")[-1] attacker: Attacker = Attacker( direction=Direction.SRC, @@ -1421,7 +1391,28 @@ def data_exfiltration( evidence: Evidence = Evidence( evidence_type=EvidenceType.DATA_UPLOAD, attacker=attacker, - threat_level=threat_level, + threat_level=ThreatLevel.INFO, + confidence=confidence, + description=description, + profile=ProfileID(ip=attacker.value), + timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + uid=uid, + timestamp=timestamp, + category=IDEACategory.MALWARE, + source_target_tag=Tag.ORIGIN_MALWARE + ) + + self.db.set_evidence(evidence) + + attacker: Attacker = Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ) + evidence: Evidence = Evidence( + evidence_type=EvidenceType.DATA_UPLOAD, + attacker=attacker, + threat_level=ThreatLevel.HIGH, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -1445,24 +1436,22 @@ def bad_smtp_login( confidence: float = 1.0 threat_level: ThreatLevel = ThreatLevel.HIGH - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) ip_identification: str = self.db.get_ip_identification(daddr) description: str = f'doing bad SMTP login to {daddr} ' \ f'{ip_identification}' evidence: Evidence = Evidence( evidence_type=EvidenceType.BAD_SMTP_LOGIN, - attacker=attacker, - victim=victim, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, confidence=confidence, description=description, @@ -1532,6 +1521,7 @@ def malicious_ssl( flow: dict = ssl_info['flow'] ts: str = flow.get('starttime', '') daddr: str = flow.get('daddr', '') + saddr: str = flow.get('saddr', '') uid: str = flow.get('uid', '') twid: str = ssl_info.get('twid', '') @@ -1550,17 +1540,34 @@ def malicious_ssl( f'{ip_identification} description: ' \ f'{cert_description} {tags} ' - - attacker: Attacker = Attacker( - direction=Direction.DST, - attacker_type=IoCType.IP, - value=daddr + evidence: Evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_SSL_CERT, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + uid=[uid], + timestamp=ts, + category=IDEACategory.INTRUSION_BOTNET, + source_target_tag=Tag.CC ) + self.db.set_evidence(evidence) + evidence: Evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_SSL_CERT, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, confidence=confidence, description=description, profile=ProfileID(ip=daddr), From 437a4985d7561b5312038bfb860d2d0e4358ffc1 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 17:36:19 +0200 Subject: [PATCH 08/33] p2p: when the network reports a malicious ip, set an evidence for the src and the dst ips --- modules/p2ptrust/p2ptrust.py | 102 ++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/modules/p2ptrust/p2ptrust.py b/modules/p2ptrust/p2ptrust.py index 4c0ea7e38..5486164e4 100644 --- a/modules/p2ptrust/p2ptrust.py +++ b/modules/p2ptrust/p2ptrust.py @@ -144,7 +144,7 @@ def init(self, *args, **kwargs): self.sql_db_name = f'{self.data_dir}trustdb.db' if self.rename_sql_db_file: - self.sql_db_name += str(pigeon_port) + self.sql_db_name += str(self.pigeon_port) # todo don't duplicate this dict, move it to slips_utils # all evidence slips detects has threat levels of strings # each string should have a corresponding int value to be able to calculate @@ -287,12 +287,14 @@ def new_evidence_callback(self, msg: Dict): threat_level = data.get('threat_level', False) if not threat_level: self.print( - f"IP {attacker} doesn't have a threat_level. not sharing to the network.", 0,2, + f"IP {attacker} doesn't have a threat_level. " + f"not sharing to the network.", 0,2, ) return if not confidence: self.print( - f"IP {attacker} doesn't have a confidence. not sharing to the network.", 0, 2, + f"IP {attacker} doesn't have a confidence. " + f"not sharing to the network.", 0, 2, ) return @@ -316,7 +318,8 @@ def new_evidence_callback(self, msg: Dict): network_score, timestamp, ) = cached_opinion - # if we don't have info about this ip from the p2p network, report it to the p2p netwrok + # if we don't have info about this ip from the p2p network, + # report it to the p2p netwrok if not cached_score: data_already_reported = False except KeyError: @@ -325,7 +328,8 @@ def new_evidence_callback(self, msg: Dict): # data saved in local db have wrong structure, this is an invalid state return - # TODO: in the future, be smarter and share only when needed. For now, we will always share + # TODO: in the future, be smarter and share only when needed. + # For now, we will always share if not data_already_reported: # Take data and send it to a peer as report. p2p_utils.send_evaluation_to_go( @@ -413,6 +417,7 @@ def data_request_callback(self, msg: Dict): # # tell other peers that we're blocking this IP # utils.send_blame_to_go(ip_address, score, confidence, self.pygo_channel) + def set_evidence_malicious_ip(self, ip_info: dict, threat_level: str, @@ -420,70 +425,67 @@ def set_evidence_malicious_ip(self, """ Set an evidence for a malicious IP met in the timewindow ip_info format is json serialized { - # 'ip': the source/dst ip - # 'profileid' : profile where the alert was generated. It includes the src ip - # 'twid' : name of the timewindow when it happened. - # 'proto' : protocol - # 'ip_state' : is basically the answer to "which one is the - # blacklisted IP"?'can be 'srcip' or - # 'dstip', - # 'stime': Exact time when the evidence happened - # 'uid': Zeek uid of the flow that generated the evidence, - # 'cache_age': How old is the info about this ip - # } + 'ip': the source/dst ip + 'profileid' : profile where the alert was generated. + It includes the src ip + 'twid' : name of the timewindow when it happened. + 'proto' : protocol + 'ip_state' : is basically the answer to "which one is the + blacklisted IP"?'can be 'srcip' or + 'dstip', + 'stime': Exact time when the evidence happened + 'uid': Zeek uid of the flow that generated the evidence, + 'cache_age': How old is the info about this ip + } :param threat_level: the threat level we learned form the network :param confidence: how confident the network opinion is about this opinion """ - + attacker_ip: str = ip_info.get('ip') - ip_state = ip_info.get('ip_state') - uid = ip_info.get('uid') profileid = ip_info.get('profileid') - twid = ip_info.get('twid') - timestamp = str(ip_info.get('stime')) saddr = profileid.split("_")[-1] - - category = IDEACategory.ANOMALY_TRAFFIC - + + threat_level = utils.threat_level_to_string(threat_level) + threat_level = ThreatLevel[threat_level.upper()] + twid_int = int(ip_info.get('twid').replace("timewindow", "")) + + # add this ip to our MaliciousIPs hash in the database + self.db.set_malicious_ip(attacker_ip, profileid, ip_info.get('twid')) + ip_identification = self.db.get_ip_identification(attacker_ip) - if 'src' in ip_state: + + if 'src' in ip_info.get('ip_state'): description = ( f'Connection from blacklisted IP {attacker_ip} ' f'({ip_identification}) to {saddr} Source: Slips P2P network.' ) - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=attacker_ip - ) else: description = ( f'Connection to blacklisted IP {attacker_ip} ' f'({ip_identification}) ' f'from {saddr} Source: Slips P2P network.' ) - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr + + for ip in (saddr, attacker_ip): + evidence = Evidence( + evidence_type= EvidenceType.MALICIOUS_IP_FROM_P2P_NETWORK, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=ip + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=attacker_ip), + timewindow=TimeWindow(number=twid_int), + uid=[ip_info.get('uid')], + timestamp=str(ip_info.get('stime')), + category=IDEACategory.ANOMALY_TRAFFIC, ) + + self.db.set_evidence(evidence) - evidence = Evidence( - evidence_type= EvidenceType.MALICIOUS_IP_FROM_P2P_NETWORK, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, - description=description, - profile=ProfileID(ip=attacker.value), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), - uid=[uid], - timestamp=timestamp, - category=category, - ) - - self.db.set_evidence(evidence) - # add this ip to our MaliciousIPs hash in the database - self.db.set_malicious_ip(attacker, profileid, twid) def handle_data_request(self, message_data: str) -> None: """ From ee534825175d0b251aa6931b973c929a804dc145 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 17:37:00 +0200 Subject: [PATCH 09/33] p2p: remove the commented code for sending a blame report, handle_update() --- modules/p2ptrust/p2ptrust.py | 53 ------------------------------------ 1 file changed, 53 deletions(-) diff --git a/modules/p2ptrust/p2ptrust.py b/modules/p2ptrust/p2ptrust.py index 5486164e4..8ed1bdf3b 100644 --- a/modules/p2ptrust/p2ptrust.py +++ b/modules/p2ptrust/p2ptrust.py @@ -364,59 +364,6 @@ def data_request_callback(self, msg: Dict): except Exception as e: self.print(f'Exception {e} in data_request_callback', 0, 1) - # def handle_update(self, ip_address: str) -> None: - # """ - # Handle IP scores changing in Slips received from the ip_info_change channel - # - # This method checks if Slips has a new score that are different - # from the scores known to the network, and if so, it means that it is worth - # sharing and it will be shared. - # Additionally, if the score is serious, the node will be blamed(blocked) - # :param ip_address: The IP address sent through the ip_info_change channel (if it is not valid IP, it returns) - # """ - # - # # abort if the IP is not valid - # if not utils.validate_ip_address(ip_address): - # self.print("IP validation failed") - # return - # - # score, confidence = utils.get_ip_info_from_slips(ip_address) - # if score is None: - # self.print("IP doesn't have any score/confidence values in DB") - # return - # - # # insert data from slips to database - # self.trust_db.insert_slips_score(ip_address, score, confidence) - # - # # TODO: discuss - only share score if confidence is high enough? - # - # # compare slips data with data in go - # data_already_reported = True - # try: - # cached_opinion = self.trust_db.get_cached_network_opinion("ip", ip_address) - # cached_score, cached_confidence, network_score, timestamp = cached_opinion - # if cached_score is None: - # data_already_reported = False - # elif abs(score - cached_score) < 0.1: - # data_already_reported = False - # except KeyError: - # data_already_reported = False - # except IndexError: - # # data saved in local db have wrong structure, this is an invalid state - # return - # - # # TODO: in the future, be smarter and share only when needed. For now, we will always share - # if not data_already_reported: - # utils.send_evaluation_to_go(ip_address, score, confidence, "*", self.pygo_channel) - # - # # TODO: discuss - based on what criteria should we start blaming? - # # decide whether or not to block - # if score > 0.8 and confidence > 0.6: - # #todo finish the blocking logic and actually block the ip - # - # # tell other peers that we're blocking this IP - # utils.send_blame_to_go(ip_address, score, confidence, self.pygo_channel) - def set_evidence_malicious_ip(self, ip_info: dict, From acf9367e4d36718831b86142e360423a26297579 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 18:27:53 +0200 Subject: [PATCH 10/33] threat_intel: have a separate function for handling BLACKLISTED_DNS_ANSWER --- .../threat_intelligence.py | 191 +++++++++++++++--- .../core/evidence_structure/evidence.py | 1 + 2 files changed, 160 insertions(+), 32 deletions(-) diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index b3eeeea92..7293ef580 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -110,7 +110,7 @@ def __read_configuration(self): def set_evidence_malicious_asn( self, - attacker: str, + daddr: str, uid: str, timestamp: str, profileid: str, @@ -133,36 +133,142 @@ def set_evidence_malicious_asn( threat_level: ThreatLevel = ThreatLevel(threat_level) tags = asn_info.get('tags', '') - identification: str = self.db.get_ip_identification(attacker) + identification: str = self.db.get_ip_identification(daddr) description: str = ( - f'Connection to IP: {attacker} with blacklisted ASN: {asn} ' + f'Connection to IP: {daddr} with blacklisted ASN: {asn} ' f'Description: {asn_info["description"]}, ' f'Found in feed: {asn_info["source"]}, ' f'Confidence: {confidence}. Tags: {tags} {identification}' ) - attacker = Attacker( + twid_int = int(twid.replace("timewindow", "")) + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_ASN, + attacker=Attacker( direction=Direction.SRC, attacker_type=IoCType.IP, value=saddr - ) - evidence = Evidence( - evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_ASN, - attacker=attacker, + ), threat_level=threat_level, confidence=confidence, description=description, profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_int), uid=[uid], timestamp=utils.convert_format(timestamp, utils.alerts_format), category=IDEACategory.ANOMALY_TRAFFIC, source_target_tag=Tag.BLACKLISTED_ASN, ) + self.db.set_evidence(evidence) + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_ASN, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_int), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_ASN, + ) + + self.db.set_evidence(evidence) + + + def set_evidence_malicious_dns_response( + self, + ip: str, + uid: str, + timestamp: str, + ip_info: dict, + dns_query: str, + profileid: str, + twid: str, + ): + """ + Set an evidence for a blacklisted IP found in one of the TI files + :param ip: the ip source file + :param uid: Zeek uid of the flow that generated the evidence + :param timestamp: Exact time when the evidence happened + :param ip_info: is all the info we have about that IP + in the db source, confidence, description, etc. + :param profileid: profile where the alert was generated. It includes the src ip + :param twid: name of the timewindow when it happened. + """ + threat_level: float = utils.threat_levels[ + ip_info.get('threat_level', 'medium') + ] + threat_level: ThreatLevel = ThreatLevel(threat_level) + saddr = profileid.split("_")[-1] + + ip_identification: str = self.db.get_ip_identification( + ip, get_ti_data=False + ).strip() + description: str = (f'DNS answer with a blacklisted ' + f'IP: {ip} for query: {dns_query}' + f'{ip_identification} Description: ' + f'{ip_info["description"]}. ' + f'Source: {ip_info["source"]}.') + + twid_int = int(twid.replace("timewindow", "")) + evidence = Evidence( + evidence_type=EvidenceType + .THREAT_INTELLIGENCE_BLACKLISTED_DNS_ANSWER, + attacker= Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=ip + ), + threat_level=threat_level, + confidence=1.0, + description=description, + profile=ProfileID(ip=ip), + timewindow=TimeWindow(number=twid_int), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_IP, + ) + + self.db.set_evidence(evidence) + + evidence = Evidence( + evidence_type=EvidenceType + .THREAT_INTELLIGENCE_BLACKLISTED_DNS_ANSWER, + attacker= Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=threat_level, + confidence=1.0, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_int), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_IP, + ) + self.db.set_evidence(evidence) + # mark this ip as malicious in our database + ip_info = {'threatintelligence': ip_info} + self.db.setInfoForIPs(ip, ip_info) + + # add this ip to our MaliciousIPs hash in the database + self.db.set_malicious_ip(ip, profileid, twid) + + def set_evidence_malicious_ip( self, ip: str, @@ -179,7 +285,8 @@ def set_evidence_malicious_ip( :param ip: the ip source file :param uid: Zeek uid of the flow that generated the evidence :param timestamp: Exact time when the evidence happened - :param ip_info: is all the info we have about that IP in the db source, confidence, description, etc. + :param ip_info: is all the info we have about that IP + in the db source, confidence, description, etc. :param profileid: profile where the alert was generated. It includes the src ip :param twid: name of the timewindow when it happened. :param ip_state: is basically the answer to "which one is the @@ -201,12 +308,8 @@ def set_evidence_malicious_ip( value=ip ) elif 'dst' in ip_state: - if self.is_dns_response: - description: str = f'DNS answer with a blacklisted ' \ - f'IP: {ip} for query: {self.dns_query}' - else: - description: str = f'connection to blacklisted ' \ - f'IP: {ip} from {srcip}. ' + description: str = (f'connection to blacklisted ' + f'IP: {ip} from {srcip}. ') attacker = Attacker( direction=Direction.DST, @@ -224,7 +327,7 @@ def set_evidence_malicious_ip( f'{ip_info["description"]}. ' f'Source: {ip_info["source"]}.') - + twid_int = int(twid.replace("timewindow", "")) evidence = Evidence( evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_IP, attacker=attacker, @@ -232,7 +335,7 @@ def set_evidence_malicious_ip( confidence=confidence, description=description, profile=ProfileID(ip=attacker.value), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_int), uid=[uid], timestamp=utils.convert_format(timestamp, utils.alerts_format), category=IDEACategory.ANOMALY_TRAFFIC, @@ -911,10 +1014,18 @@ def is_malicious_ip(self, timestamp: str, profileid: str, twid: str, - ip_state: str) -> bool: + ip_state: str, + is_dns_response: bool=False, + dns_query: str=False + ) -> bool: """ Search for this IP in our database of IoC :param ip_state: is basically the answer to "which one is the + :param is_dns_response: set to true if the ip we're + looking up is a dns response + :param dns_query: is the dns query if the ip we're + looking up is a dns response + blacklisted IP"? can be 'srcip' or 'dstip' """ ip_info = self.search_offline_for_ip(ip) @@ -923,19 +1034,31 @@ def is_malicious_ip(self, if not ip_info: # not malicious return False + self.db.add_ips_to_IoC({ ip: json.dumps(ip_info) }) - self.set_evidence_malicious_ip( - ip, - uid, - daddr, - timestamp, - ip_info, - profileid, - twid, - ip_state, - ) + if is_dns_response: + self.set_evidence_malicious_dns_response( + ip, + uid, + timestamp, + ip_info, + dns_query, + profileid, + twid, + ) + else: + self.set_evidence_malicious_ip( + ip, + uid, + daddr, + timestamp, + ip_info, + profileid, + twid, + ip_state, + ) return True def is_malicious_hash(self, flow_info: dict): @@ -1073,7 +1196,7 @@ def main(self): # these 2 are only available when looking up dns answers # the query is needed when a malicious answer is found, # for more detailed description of the evidence - self.is_dns_response = data.get('is_dns_response') + is_dns_response = data.get('is_dns_response') self.dns_query = data.get('dns_query') # IP is the IP that we want the TI for. It can be a SRC or DST IP to_lookup = data.get('to_lookup', '') @@ -1092,13 +1215,17 @@ def main(self): or self.is_outgoing_icmp_packet(protocol, ip_state) ): self.is_malicious_ip( - ip, uid, daddr, timestamp, profileid, twid, ip_state + ip, uid, daddr, timestamp, profileid, twid, + ip_state, + dns_query=dns_query, + is_dns_response=is_dns_response, ) self.ip_belongs_to_blacklisted_range( ip, uid, daddr, timestamp, profileid, twid, ip_state ) self.ip_has_blacklisted_ASN( - ip, uid, timestamp, profileid, twid, ip_state + ip, uid, timestamp, profileid, twid, ip_state, + is_dns_response=is_dns_response ) elif type_ == 'domain': self.is_malicious_domain( diff --git a/slips_files/core/evidence_structure/evidence.py b/slips_files/core/evidence_structure/evidence.py index 8e8c23628..d68b7b7f5 100644 --- a/slips_files/core/evidence_structure/evidence.py +++ b/slips_files/core/evidence_structure/evidence.py @@ -83,6 +83,7 @@ class EvidenceType(Enum): COMMAND_AND_CONTROL_CHANNEL = auto() THREAT_INTELLIGENCE_BLACKLISTED_ASN = auto() THREAT_INTELLIGENCE_BLACKLISTED_IP = auto() + THREAT_INTELLIGENCE_BLACKLISTED_DNS_ANSWER = auto() THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN = auto() MALICIOUS_DOWNLOADED_FILE = auto() MALICIOUS_URL = auto() From 580720084c797b48cc4f70c3b70e1ca50be92ea9 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 18:45:40 +0200 Subject: [PATCH 11/33] threat_intel: have more descriptive evidence when there's an evidence of a malicious ASN for an IP in a DNS answer --- .../threat_intelligence.py | 89 ++++++++++++------- 1 file changed, 56 insertions(+), 33 deletions(-) diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index 7293ef580..3f813e79d 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -117,9 +117,11 @@ def set_evidence_malicious_asn( twid: str, asn: str, asn_info: dict, + is_dns_response: bool = False ): """ - :param asn_info: the malicious ASN info taken from own_malicious_iocs.csv + :param asn_info: the malicious ASN info taken from + own_malicious_iocs.csv """ confidence: float = 0.8 @@ -134,12 +136,20 @@ def set_evidence_malicious_asn( tags = asn_info.get('tags', '') identification: str = self.db.get_ip_identification(daddr) - - description: str = ( - f'Connection to IP: {daddr} with blacklisted ASN: {asn} ' + if is_dns_response: + description: str = ( + f'Connection to IP: {daddr} with blacklisted ASN: {asn} ' + ) + else: + description: str = ( + f'DNS response with IP: {daddr} with blacklisted ASN: {asn} ' + ) + + description += ( f'Description: {asn_info["description"]}, ' f'Found in feed: {asn_info["source"]}, ' - f'Confidence: {confidence}. Tags: {tags} {identification}' + f'Confidence: {confidence}. ' + f'Tags: {tags} {identification}' ) twid_int = int(twid.replace("timewindow", "")) evidence = Evidence( @@ -273,7 +283,7 @@ def set_evidence_malicious_ip( self, ip: str, uid: str, - dstip: str, + daddr: str, timestamp: str, ip_info: dict, profileid: str = '', @@ -285,6 +295,7 @@ def set_evidence_malicious_ip( :param ip: the ip source file :param uid: Zeek uid of the flow that generated the evidence :param timestamp: Exact time when the evidence happened + :param daddr: dst address of the flow :param ip_info: is all the info we have about that IP in the db source, confidence, description, etc. :param profileid: profile where the alert was generated. It includes the src ip @@ -296,26 +307,14 @@ def set_evidence_malicious_ip( ip_info.get('threat_level', 'medium') ] threat_level: ThreatLevel = ThreatLevel(threat_level) - confidence: float = 1.0 - srcip = profileid.split("_")[-1] + saddr = profileid.split("_")[-1] if 'src' in ip_state: description: str = f'connection from blacklisted ' \ - f'IP: {ip} to {dstip}. ' - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=ip - ) + f'IP: {ip} to {daddr}. ' elif 'dst' in ip_state: description: str = (f'connection to blacklisted ' - f'IP: {ip} from {srcip}. ') - - attacker = Attacker( - direction=Direction.DST, - attacker_type=IoCType.IP, - value=ip - ) + f'IP: {ip} from {saddr}. ') else: # ip_state is not specified? return @@ -330,20 +329,41 @@ def set_evidence_malicious_ip( twid_int = int(twid.replace("timewindow", "")) evidence = Evidence( evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_IP, - attacker=attacker, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, - confidence=confidence, + confidence=1.0, description=description, - profile=ProfileID(ip=attacker.value), + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_int), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_IP, + ) + self.db.set_evidence(evidence) + + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_IP, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=threat_level, + confidence=1.0, + description=description, + profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=twid_int), uid=[uid], timestamp=utils.convert_format(timestamp, utils.alerts_format), category=IDEACategory.ANOMALY_TRAFFIC, source_target_tag=Tag.BLACKLISTED_IP, ) - self.db.set_evidence(evidence) - # mark this ip as malicious in our database ip_info = {'threatintelligence': ip_info} @@ -351,6 +371,8 @@ def set_evidence_malicious_ip( # add this ip to our MaliciousIPs hash in the database self.db.set_malicious_ip(ip, profileid, twid) + + def set_evidence_malicious_domain( self, @@ -928,7 +950,8 @@ def search_online_for_ip(self, ip): return spamhaus_res def ip_has_blacklisted_ASN( - self, ip, uid, timestamp, profileid, twid, ip_state + self, ip, uid, timestamp, profileid, twid, + is_dns_response: bool = False, ): """ Check if this ip has any of our blacklisted ASNs. @@ -956,6 +979,7 @@ def ip_has_blacklisted_ASN( twid, asn, asn_info, + is_dns_response=is_dns_response, ) def ip_belongs_to_blacklisted_range( @@ -1182,11 +1206,9 @@ def pre_main(self): self.circllu_calls_thread.start() def main(self): - # The channel now can receive an IP address or a domain name + # The channel can receive an IP address or a domain name if msg:= self.get_msg('give_threat_intelligence'): - # Data is sent in the channel as a json dict so we need to deserialize it first data = json.loads(msg['data']) - # Extract data from dict profileid = data.get('profileid') twid = data.get('twid') timestamp = data.get('stime') @@ -1197,10 +1219,11 @@ def main(self): # the query is needed when a malicious answer is found, # for more detailed description of the evidence is_dns_response = data.get('is_dns_response') - self.dns_query = data.get('dns_query') + dns_query = data.get('dns_query') # IP is the IP that we want the TI for. It can be a SRC or DST IP to_lookup = data.get('to_lookup', '') - # detect the type given because sometimes, http.log host field has ips OR domains + # detect the type given because sometimes, + # http.log host field has ips OR domains type_ = utils.detect_data_type(to_lookup) # ip_state will say if it is a srcip or if it was a dst_ip @@ -1224,7 +1247,7 @@ def main(self): ip, uid, daddr, timestamp, profileid, twid, ip_state ) self.ip_has_blacklisted_ASN( - ip, uid, timestamp, profileid, twid, ip_state, + ip, uid, timestamp, profileid, twid, is_dns_response=is_dns_response ) elif type_ == 'domain': From 7fd416bb3b7a5ba4ea5fd53297346dda95dc6cb7 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 19:03:02 +0200 Subject: [PATCH 12/33] threat_intel: have a separate function to lookup cnames from dns answers --- .../threat_intelligence.py | 121 +++++++++++++----- 1 file changed, 88 insertions(+), 33 deletions(-) diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index 3f813e79d..67a1bb2da 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -372,6 +372,19 @@ def set_evidence_malicious_ip( # add this ip to our MaliciousIPs hash in the database self.db.set_malicious_ip(ip, profileid, twid) + def set_evidence_malicious_dns_query(self, + CNAME: str, + domain, + uid: str, + timestamp: str, + domain_info: dict, + is_subdomain: bool, + profileid: str = '', + twid: str = ''): + description: str = (f'DNS answer with a blacklisted ' + f'CNAME: {domain} ' + f'for query: {query} ') + def set_evidence_malicious_domain( @@ -385,7 +398,7 @@ def set_evidence_malicious_domain( twid: str = '', ): """ - Set an evidence for a malicious domain met in the timewindow + Set an evidence for a malicious domain :param source_file: is the domain source file :param domain_info: is all the info we have about this domain in the db source, confidence , description etc... @@ -406,18 +419,10 @@ def set_evidence_malicious_domain( domain_info.get('threat_level', 'high') ] threat_level: ThreatLevel = ThreatLevel(threat_level) - - - if self.is_dns_response: - description: str = (f'DNS answer with a blacklisted ' - f'CNAME: {domain} ' - f'for query: {self.dns_query} ') - else: - description: str = f'connection to a blacklisted domain {domain}. ' - - description += f'Description: {domain_info.get("description", "")},' \ - f' Found in feed: {domain_info["source"]}, ' \ - f'Confidence: {confidence}. ' + description: str = (f'connection to a blacklisted domain {domain}. ' + f'Description: {domain_info.get("description", "")},' + f'Found in feed: {domain_info["source"]}, ' + f'Confidence: {confidence}. ') tags = domain_info.get('tags', None) if tags: @@ -997,6 +1002,7 @@ def ip_belongs_to_blacklisted_range( ranges_starting_with_octet = self.cached_ipv6_ranges.get(first_octet, []) else: return False + for range in ranges_starting_with_octet: if ip_obj in ipaddress.ip_network(range): # ip was found in one of the blacklisted ranges @@ -1121,6 +1127,7 @@ def is_malicious_url( if not url_info: # not malicious return False + self.set_evidence_malicious_url( url_info, uid, @@ -1128,7 +1135,46 @@ def is_malicious_url( profileid, twid ) + + def is_malicious_cname(self, + dns_query, + cname, + uid, + timestamp, + profileid, + twid, + ): + """ + :param cname: is the dns answer we're looking up + :param dns_query: the query we asked the DNS server for when the + server returned the given sname + """ + + if self.is_ignored_domain(cname): + return False + + domain_info, is_subdomain = self.search_offline_for_domain(cname) + if not domain_info: + return False + + self.set_evidence_malicious_dns_query( + dns_query, + uid, + timestamp, + domain_info, + is_subdomain, + profileid, + twid, + ) + # mark this domain as malicious in our database + domain_info = { + 'threatintelligence': domain_info + } + self.db.setInfoForDomains(cname, domain_info) + # add this domain to our MaliciousDomains hash in the database + self.db.set_malicious_domain(cname, profileid, twid) + def is_malicious_domain( self, @@ -1136,7 +1182,7 @@ def is_malicious_domain( uid, timestamp, profileid, - twid + twid, ): if self.is_ignored_domain(domain): return False @@ -1144,7 +1190,7 @@ def is_malicious_domain( domain_info, is_subdomain = self.search_offline_for_domain(domain) if not domain_info: return False - + self.set_evidence_malicious_domain( domain, uid, @@ -1159,14 +1205,10 @@ def is_malicious_domain( domain_info = { 'threatintelligence': domain_info } - self.db.setInfoForDomains( - domain, domain_info - ) + self.db.setInfoForDomains(domain, domain_info) # add this domain to our MaliciousDomains hash in the database - self.db.set_malicious_domain( - domain, profileid, twid - ) + self.db.set_malicious_domain(domain, profileid, twid) def update_local_file(self, filename): @@ -1204,7 +1246,13 @@ def pre_main(self): self.update_local_file(local_file) self.circllu_calls_thread.start() - + + def should_lookup(self, ip: str, protocol: str, ip_state: str) \ + -> bool: + """return whther slips should lookup the given ip or notd""" + return (utils.is_ignored_ip(ip) or + self.is_outgoing_icmp_packet(protocol, ip_state)) + def main(self): # The channel can receive an IP address or a domain name if msg:= self.get_msg('give_threat_intelligence'): @@ -1231,12 +1279,10 @@ def main(self): # If given an IP, ask for it # Block only if the traffic isn't outgoing ICMP port unreachable packet + if type_ == 'ip': ip = to_lookup - if not ( - utils.is_ignored_ip(ip) - or self.is_outgoing_icmp_packet(protocol, ip_state) - ): + if not self.should_lookup(ip, protocol, ip_state): self.is_malicious_ip( ip, uid, daddr, timestamp, profileid, twid, ip_state, @@ -1251,13 +1297,22 @@ def main(self): is_dns_response=is_dns_response ) elif type_ == 'domain': - self.is_malicious_domain( - to_lookup, - uid, - timestamp, - profileid, - twid - ) + if is_dns_response: + self.is_malicious_cname( + dns_query, + to_lookup, + uid, + timestamp, + profileid, + twid) + else: + self.is_malicious_domain( + to_lookup, + uid, + timestamp, + profileid, + twid + ) elif type_ == 'url': self.is_malicious_url( to_lookup, From 6d64dad121cde7f236892cf3ceddac792cff14f5 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 19:46:45 +0200 Subject: [PATCH 13/33] set evidence for saddr and daddr for some evidence --- modules/arp/arp.py | 1 - modules/flowalerts/set_evidence.py | 10 +- modules/flowmldetection/flowmldetection.py | 16 +- modules/http_analyzer/http_analyzer.py | 131 +++++++------ modules/ip_info/ip_info.py | 51 +++-- modules/leak_detector/leak_detector.py | 148 ++++++++------- .../network_discovery/network_discovery.py | 44 ++--- modules/p2ptrust/p2ptrust.py | 39 ++-- modules/p2ptrust/utils/go_director.py | 11 +- modules/rnn_cc_detection/rnn_cc_detection.py | 40 ++-- .../threat_intelligence.py | 178 ++++++++++++++---- modules/threat_intelligence/urlhaus.py | 116 +++++++----- .../core/evidence_structure/evidence.py | 2 +- 13 files changed, 486 insertions(+), 301 deletions(-) diff --git a/modules/arp/arp.py b/modules/arp/arp.py index c05d187fa..468f1aa6f 100644 --- a/modules/arp/arp.py +++ b/modules/arp/arp.py @@ -1,4 +1,3 @@ -from slips_files.common.abstracts._module import IModule import json import ipaddress import time diff --git a/modules/flowalerts/set_evidence.py b/modules/flowalerts/set_evidence.py index 9f1edacfd..6488bbb8f 100644 --- a/modules/flowalerts/set_evidence.py +++ b/modules/flowalerts/set_evidence.py @@ -1,8 +1,8 @@ -import datetime import json import sys import time -from typing import List +from typing import List, \ + Dict from slips_files.common.slips_utils import utils from slips_files.core.evidence_structure.evidence import \ @@ -126,7 +126,7 @@ def different_localnet_usage( self, daddr: str, portproto: str, - profileid: ProfileID, + profileid: str, timestamp: str, twid: str, uid: str, @@ -741,7 +741,7 @@ def conn_to_private_ip( def GRE_tunnel( self, - tunnel_info: dict + tunnel_info: Dict[str, str] ) -> None: profileid: str = tunnel_info['profileid'] twid: str = tunnel_info['twid'] @@ -1516,7 +1516,7 @@ def smtp_bruteforce( def malicious_ssl( self, ssl_info: dict, - ssl_info_from_db: dict + ssl_info_from_db: str ) -> None: flow: dict = ssl_info['flow'] ts: str = flow.get('starttime', '') diff --git a/modules/flowmldetection/flowmldetection.py b/modules/flowmldetection/flowmldetection.py index f1d2c1af4..1f1d7ec3a 100644 --- a/modules/flowmldetection/flowmldetection.py +++ b/modules/flowmldetection/flowmldetection.py @@ -384,14 +384,6 @@ def set_evidence_malicious_flow( uid: str ): confidence: float = 0.1 - threat_level: ThreatLevel = ThreatLevel.LOW - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - ip_identification = self.db.get_ip_identification(daddr) description = f'Malicious flow by ML. Src IP {saddr}:{sport} to ' \ f'{daddr}:{dport} {ip_identification}' @@ -403,8 +395,12 @@ def set_evidence_malicious_flow( evidence: Evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_FLOW, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, confidence=confidence, description=description, profile=ProfileID(ip=saddr), diff --git a/modules/http_analyzer/http_analyzer.py b/modules/http_analyzer/http_analyzer.py index bed6c5c5a..c1ab18c36 100644 --- a/modules/http_analyzer/http_analyzer.py +++ b/modules/http_analyzer/http_analyzer.py @@ -1,4 +1,3 @@ -from slips_files.common.abstracts._module import IModule import json import urllib import requests @@ -92,21 +91,19 @@ def check_suspicious_user_agents( ) for suspicious_ua in suspicious_user_agents: if suspicious_ua.lower() in user_agent.lower(): - threat_level: ThreatLevel = ThreatLevel.HIGH confidence: float = 1 saddr = profileid.split('_')[1] description: str = (f'Suspicious user-agent: ' f'{user_agent} while ' f'connecting to {host}{uri}') - attacker = Attacker( + evidence: Evidence = Evidence( + evidence_type=EvidenceType.SUSPICIOUS_USER_AGENT, + attacker=Attacker( direction=Direction.SRC, attacker_type=IoCType.IP, value=saddr - ) - evidence: Evidence = Evidence( - evidence_type=EvidenceType.SUSPICIOUS_USER_AGENT, - attacker=attacker, - threat_level=threat_level, + ), + threat_level=ThreatLevel.HIGH, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -165,21 +162,18 @@ def check_multiple_empty_connections( uids, connections = self.connections_counter[host] if connections == self.empty_connections_threshold: - threat_level: ThreatLevel = ThreatLevel.MEDIUM confidence: float = 1 saddr: str = profileid.split('_')[-1] description: str = f'Multiple empty HTTP connections to {host}' - attacker = Attacker( + evidence: Evidence = Evidence( + evidence_type=EvidenceType.EMPTY_CONNECTIONS, + attacker=Attacker( direction=Direction.SRC, attacker_type=IoCType.IP, value=saddr - ) - - evidence: Evidence = Evidence( - evidence_type=EvidenceType.EMPTY_CONNECTIONS, - attacker=attacker, - threat_level=threat_level, + ), + threat_level=ThreatLevel.MEDIUM, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -206,9 +200,7 @@ def set_evidence_incompatible_user_agent( twid, uid: str ): - threat_level: ThreatLevel = ThreatLevel.HIGH saddr = profileid.split('_')[1] - confidence: float = 1 os_type: str = user_agent.get('os_type', '').lower() os_name: str = user_agent.get('os_name', '').lower() @@ -222,17 +214,15 @@ def set_evidence_incompatible_user_agent( f'IP has MAC vendor: {vendor.capitalize()}' ) - attacker: Attacker = Attacker( - direction= Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - evidence: Evidence = Evidence( evidence_type=EvidenceType.INCOMPATIBLE_USER_AGENT, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction= Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.HIGH, + confidence=1, description=description, profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), @@ -253,30 +243,46 @@ def set_evidence_executable_mime_type( timestamp: str, daddr: str ): - confidence: float = 1 - threat_level: ThreatLevel = ThreatLevel.LOW saddr: str = profileid.split('_')[1] - attacker_obj: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = ( f'Download of an executable with MIME type: {mime_type} ' f'by {saddr} from {daddr} {ip_identification}.' ) - + twid_number = int(twid.replace("timewindow", "")) evidence: Evidence = Evidence( evidence_type=EvidenceType.EXECUTABLE_MIME_TYPE, - attacker=attacker_obj, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, + confidence=1, description=description, profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=timestamp, + category=IDEACategory.ANOMALY_FILE, + source_target_tag=Tag.EXECUTABLE_MIME_TYPE + ) + + self.db.set_evidence(evidence) + + evidence: Evidence = Evidence( + evidence_type=EvidenceType.EXECUTABLE_MIME_TYPE, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.LOW, + confidence=1, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_number), uid=[uid], timestamp=timestamp, category=IDEACategory.ANOMALY_FILE, @@ -515,24 +521,20 @@ def check_multiple_UAs( # 'Linux' in both UAs, so we shouldn't alert return False - threat_level: ThreatLevel = ThreatLevel.INFO - confidence: float = 1 saddr: str = profileid.split('_')[1] - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - ua: str = cached_ua.get('user_agent', '') description: str = (f'Using multiple user-agents:' f' "{ua}" then "{user_agent}"') evidence: Evidence = Evidence( evidence_type=EvidenceType.MULTIPLE_USER_AGENT, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.INFO, + confidence=1, description=description, profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), @@ -556,26 +558,17 @@ def set_evidence_http_traffic( timestamp: str ): confidence: float = 1 - threat_level: ThreatLevel = ThreatLevel.LOW saddr = profileid.split('_')[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) description = f'Unencrypted HTTP traffic from {saddr} to {daddr}.' evidence: Evidence = Evidence( evidence_type=EvidenceType.HTTP_TRAFFIC, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -584,7 +577,11 @@ def set_evidence_http_traffic( timestamp=timestamp, category=IDEACategory.ANOMALY_TRAFFIC, source_target_tag=Tag.SENDING_UNENCRYPTED_DATA, - victim=victim + victim= Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ) ) self.db.set_evidence(evidence) diff --git a/modules/ip_info/ip_info.py b/modules/ip_info/ip_info.py index adefe6390..8bc5ba536 100644 --- a/modules/ip_info/ip_info.py +++ b/modules/ip_info/ip_info.py @@ -511,17 +511,9 @@ def set_evidence_malicious_jarm_hash( dport: int = flow['dport'] dstip: str = flow['daddr'] saddr: str = flow['saddr'] - timestamp: float = flow['starttime'] + timestamp = flow['starttime'] protocol: str = flow['proto'] - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - threat_level = ThreatLevel.MEDIUM - confidence = 0.7 - portproto = f'{dport}/{protocol}' port_info = self.db.get_port_info(portproto) or "" port_info = f'({port_info.upper()})' if port_info else "" @@ -529,17 +521,22 @@ def set_evidence_malicious_jarm_hash( dstip_id = self.db.get_ip_identification(dstip) description = ( f"Malicious JARM hash detected for destination IP: {dstip}" - f" on port: {portproto} {port_info}. {dstip_id}" + f" on port: {portproto} {port_info}. {dstip_id}" ) - + twid_number = int(twid.replace("timewindow", "")) + evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_JARM, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=dstip + ), + threat_level=ThreatLevel.MEDIUM, + confidence=0.7, description=description, - profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + profile=ProfileID(ip=dstip), + timewindow=TimeWindow(number=twid_number), uid=[flow['uid']], timestamp=timestamp, category=IDEACategory.ANOMALY_TRAFFIC, @@ -549,6 +546,28 @@ def set_evidence_malicious_jarm_hash( ) self.db.set_evidence(evidence) + + evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_JARM, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, + confidence=0.7, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_number), + uid=[flow['uid']], + timestamp=timestamp, + category=IDEACategory.ANOMALY_TRAFFIC, + proto=Proto(protocol.lower()), + port=dport, + source_target_tag=Tag.MALWARE + ) + + self.db.set_evidence(evidence) def pre_main(self): diff --git a/modules/leak_detector/leak_detector.py b/modules/leak_detector/leak_detector.py index 3471dd6bd..50792e851 100644 --- a/modules/leak_detector/leak_detector.py +++ b/modules/leak_detector/leak_detector.py @@ -163,78 +163,100 @@ def get_packet_info(self, offset: int): def set_evidence_yara_match(self, info: dict): """ This function is called when yara finds a match - :param info: a dict with info about the matched rule, example keys 'vars_matched', 'index', + :param info: a dict with info about the matched rule, + example keys 'vars_matched', 'index', 'rule', 'srings_matched' """ rule = info.get('rule').replace('_', ' ') offset = info.get('offset') # vars_matched = info.get('vars_matched') strings_matched = info.get('strings_matched') - # we now know there's a match at offset x, we need to know offset x belongs to which packet - if packet_info := self.get_packet_info(offset): - srcip, dstip, proto, sport, dport, ts = ( - packet_info[0], - packet_info[1], - packet_info[2], - packet_info[3], - packet_info[4], - packet_info[5], - ) - - portproto = f'{dport}/{proto}' - port_info = self.db.get_port_info(portproto) - - # generate a random uid - uid = base64.b64encode(binascii.b2a_hex(os.urandom(9))).decode( - 'utf-8' - ) - profileid = f'profile_{srcip}' - # sometimes this module tries to find the profile before it's created. so - # wait a while before alerting. - time.sleep(4) - - ip_identification = self.db.get_ip_identification(dstip) - description = f"{rule} to destination address: {dstip} " \ - f"{ip_identification} port: {portproto} " \ - f"{port_info or ''}. " \ - f"Leaked location: {strings_matched}" - - # in which tw is this ts? - twid = self.db.get_tw_of_ts(profileid, ts) - # convert ts to a readable format - ts = utils.convert_format(ts, utils.alerts_format) - - if twid: - twid = twid[0] - source_target_tag = Tag.CC - confidence = 0.9 - threat_level = ThreatLevel.HIGH - - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) - + # we now know there's a match at offset x, we need + # to know offset x belongs to which packet + packet_info = self.get_packet_info(offset) + if not packet_info: + return + + srcip, dstip, proto, sport, dport, ts = ( + packet_info[0], + packet_info[1], + packet_info[2], + packet_info[3], + packet_info[4], + packet_info[5], + ) - evidence = Evidence( - evidence_type=EvidenceType.NETWORK_GPS_LOCATION_LEAKED, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, - description=description, - profile=ProfileID(ip=srcip), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), - uid=[uid], - timestamp=ts, - proto=Proto(proto.lower()), - port=dport, - source_target_tag=source_target_tag, - category=IDEACategory.MALWARE - ) + portproto = f'{dport}/{proto}' + port_info = self.db.get_port_info(portproto) - self.db.set_evidence(evidence) + # generate a random uid + uid = base64.b64encode(binascii.b2a_hex(os.urandom(9))).decode( + 'utf-8' + ) + profileid = f'profile_{srcip}' + # sometimes this module tries to find the profile before it's created. so + # wait a while before alerting. + time.sleep(4) + + ip_identification = self.db.get_ip_identification(dstip) + description = f"{rule} to destination address: {dstip} " \ + f"{ip_identification} port: {portproto} " \ + f"{port_info or ''}. " \ + f"Leaked location: {strings_matched}" + + # in which tw is this ts? + twid = self.db.get_tw_of_ts(profileid, ts) + # convert ts to a readable format + ts = utils.convert_format(ts, utils.alerts_format) + + if not twid: + return + + twid_number = int(twid[0].replace("timewindow", "")) + evidence = Evidence( + evidence_type=EvidenceType.NETWORK_GPS_LOCATION_LEAKED, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), + threat_level=ThreatLevel.LOW, + confidence=0.9, + description=description, + profile=ProfileID(ip=srcip), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=ts, + proto=Proto(proto.lower()), + port=dport, + source_target_tag=Tag.CC, + category=IDEACategory.MALWARE + ) + self.db.set_evidence(evidence) + + evidence = Evidence( + evidence_type=EvidenceType.NETWORK_GPS_LOCATION_LEAKED, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=dstip + ), + threat_level=ThreatLevel.HIGH, + confidence=0.9, + description=description, + profile=ProfileID(ip=dstip), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=ts, + proto=Proto(proto.lower()), + port=dport, + source_target_tag=Tag.CC, + category=IDEACategory.MALWARE + ) + + self.db.set_evidence(evidence) + def compile_and_save_rules(self): """ diff --git a/modules/network_discovery/network_discovery.py b/modules/network_discovery/network_discovery.py index bcdebf0da..f89dc082c 100644 --- a/modules/network_discovery/network_discovery.py +++ b/modules/network_discovery/network_discovery.py @@ -74,7 +74,6 @@ def check_icmp_sweep( Use our own Zeek scripts to detect ICMP scans. Threshold is on the scripts and it is 25 ICMP flows """ - if 'TimestampScan' in note: evidence_type = EvidenceType.ICMP_TIMESTAMP_SCAN elif 'ICMPAddressScan' in note: @@ -88,21 +87,17 @@ def check_icmp_sweep( hosts_scanned = int(msg.split('on ')[1].split(' hosts')[0]) # get the confidence from 0 to 1 based on the number of hosts scanned confidence = 1 / (255 - 5) * (hosts_scanned - 255) + 1 - threat_level = ThreatLevel.MEDIUM saddr = profileid.split('_')[1] - # this is the last IP scanned - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - # this one is detected by Zeek, so we can't track the UIDs causing it evidence = Evidence( evidence_type=evidence_type, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.MEDIUM, confidence=confidence, description=msg, profile=ProfileID(ip=saddr), @@ -321,29 +316,26 @@ def set_evidence_dhcp_scan( uids, number_of_requested_addrs ): - threat_level = ThreatLevel.MEDIUM - confidence = 0.8 srcip = profileid.split('_')[-1] - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) description = ( f'Performing a DHCP scan by requesting ' f'{number_of_requested_addrs} different IP addresses. ' f'Threat Level: {threat_level}. ' f'Confidence: {confidence}. by Slips' ) - + twid_number = int(twid.replace("timewindow", "")) evidence = Evidence( evidence_type=EvidenceType.DHCP_SCAN, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), + threat_level=ThreatLevel.MEDIUM, + confidence=0.8, description=description, profile=ProfileID(ip=srcip), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_number), uid=uids, timestamp=timestamp, category=IDEACategory.RECON_SCANNING, @@ -363,7 +355,8 @@ def check_dhcp_scan(self, flow_info): flow = flow_info['flow'] requested_addr = flow['requested_addr'] if not requested_addr: - # we are only interested in DHCPREQUEST flows, where a client is requesting an IP + # we are only interested in DHCPREQUEST flows, + # where a client is requesting an IP return profileid = flow_info['profileid'] @@ -400,7 +393,8 @@ def check_dhcp_scan(self, flow_info): # we alert every 4,8,12, etc. requested IPs number_of_requested_addrs = len(dhcp_flows) if number_of_requested_addrs % self.minimum_requested_addrs == 0: - # get the uids of all the flows where this client was requesting an addr in this tw + # get the uids of all the flows where this client + # was requesting an addr in this tw for uids_list in dhcp_flows.values(): uids.append(uids_list[0]) diff --git a/modules/p2ptrust/p2ptrust.py b/modules/p2ptrust/p2ptrust.py index 8ed1bdf3b..4ffc36ee8 100644 --- a/modules/p2ptrust/p2ptrust.py +++ b/modules/p2ptrust/p2ptrust.py @@ -48,7 +48,8 @@ def validate_slips_data(message_data: str) -> (str, int): 'cache_age': cache_age } - If the message is correct, the two values are returned as a tuple (str, int). + If the message is correct, the two values are + returned as a tuple (str, int). If not, (None, None) is returned. :param message_data: data from slips request channel :return: the received msg or None tuple @@ -62,7 +63,8 @@ def validate_slips_data(message_data: str) -> (str, int): except ValueError: # message has wrong format print( - f'The message received from p2p_data_request channel has incorrect format: {message_data}' + f'The message received from p2p_data_request channel' + f' has incorrect format: {message_data}' ) return None @@ -78,7 +80,8 @@ class Trust(IModule): gopy_channel_raw='p2p_gopy' pygo_channel_raw='p2p_pygo' start_pigeon=True - pigeon_binary= os.path.join(os.getcwd(),'p2p4slips/p2p4slips') # or make sure the binary is in $PATH + # or make sure the binary is in $PATH + pigeon_binary= os.path.join(os.getcwd(),'p2p4slips/p2p4slips') pigeon_key_file='pigeon.keys' rename_redis_ip_info=False rename_sql_db_file=False @@ -122,7 +125,8 @@ def init(self, *args, **kwargs): if self.rename_redis_ip_info: self.storage_name += str(self.port) self.c1 = self.db.subscribe('report_to_peers') - # channel to send msgs to whenever slips needs info from other peers about an ip + # channel to send msgs to whenever slips needs + # info from other peers about an ip self.c2 = self.db.subscribe(self.p2p_data_request_channel) # this channel receives peers requests/updates self.c3 = self.db.subscribe(self.gopy_channel) @@ -200,8 +204,10 @@ def _configure(self): self.sql_db_name, drop_tables_on_startup=True ) - self.reputation_model = reputation_model.BaseModel(self.logger, self.trust_db) - # print(f"[DEBUGGING] Starting godirector with pygo_channel: {self.pygo_channel}") + self.reputation_model = reputation_model.BaseModel( + self.logger, self.trust_db) + # print(f"[DEBUGGING] Starting godirector with + # pygo_channel: {self.pygo_channel}") self.go_director = GoDirector( self.logger, self.trust_db, @@ -511,16 +517,20 @@ def handle_data_request(self, message_data: str) -> None: combined_confidence, ) = self.reputation_model.get_opinion_on_ip(ip_address) - # no data in db - this happens when testing, if there is not enough data on peers + # no data in db - this happens when testing, + # if there is not enough data on peers if combined_score is None: self.print( - f'No data received from the network about {ip_address}\n', 0, 2 + f'No data received from the' + f' network about {ip_address}\n', 0, 2 ) - # print(f"[DEBUGGING] No data received from the network about {ip_address}\n") + # print(f"[DEBUGGING] No data received + # from the network about {ip_address}\n") else: self.print( f'The Network shared some data about {ip_address}, ' - f'Shared data: score={combined_score}, confidence={combined_confidence} saving it to now!\n', + f'Shared data: score={combined_score}, ' + f'confidence={combined_confidence} saving it to now!\n', 0, 2, ) @@ -540,7 +550,8 @@ def handle_data_request(self, message_data: str) -> None: ) def respond_to_message_request(self, key, reporter): - # todo do you mean another peer is asking me about an ip? yes. in override mode + # todo do you mean another peer is asking me about + # an ip? yes. in override mode """ Handle data request from a peer (in overriding p2p mode) (set to false by defualt) :param key: The ip requested by the peer @@ -578,7 +589,8 @@ def pre_main(self): # check if it was possible to start up pigeon if self.start_pigeon and self.pigeon is None: self.print( - 'Module was supposed to start up pigeon but it was not possible to start pigeon! Exiting...' + 'Module was supposed to start up pigeon but it was not' + ' possible to start pigeon! Exiting...' ) return 1 @@ -604,7 +616,8 @@ def main(self): ret_code = self.pigeon.poll() if ret_code is not None: self.print( - f'Pigeon process suddenly terminated with return code {ret_code}. Stopping module.' + f'Pigeon process suddenly terminated with ' + f'return code {ret_code}. Stopping module.' ) return 1 diff --git a/modules/p2ptrust/utils/go_director.py b/modules/p2ptrust/utils/go_director.py index f15a8703e..9a3ec538c 100644 --- a/modules/p2ptrust/utils/go_director.py +++ b/modules/p2ptrust/utils/go_director.py @@ -488,11 +488,6 @@ def set_evidence_p2p_report( set evidence for the newly created attacker profile stating that it attacked another peer """ - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=ip - ) threat_level = utils.threat_level_to_string(score) # confidence depends on how long the connection @@ -521,7 +516,11 @@ def set_evidence_p2p_report( timestamp = utils.convert_format(timestamp, utils.alerts_format) evidence = Evidence( evidence_type=EvidenceType.P2P_REPORT, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=ip + ), threat_level=threat_level, confidence=confidence, description=description, diff --git a/modules/rnn_cc_detection/rnn_cc_detection.py b/modules/rnn_cc_detection/rnn_cc_detection.py index bebaa6e8f..e6f28f915 100644 --- a/modules/rnn_cc_detection/rnn_cc_detection.py +++ b/modules/rnn_cc_detection/rnn_cc_detection.py @@ -56,14 +56,6 @@ def set_evidence_cc_channel( tupleid = tupleid.split('-') dstip, port, proto = tupleid[0], tupleid[1], tupleid[2] srcip = profileid.split("_")[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) - - threat_level: ThreatLevel = ThreatLevel.HIGH portproto: str = f'{port}/{proto}' port_info: str = self.db.get_port_info(portproto) ip_identification: str = self.db.get_ip_identification(dstip) @@ -74,14 +66,19 @@ def set_evidence_cc_channel( ) timestamp: str = utils.convert_format(timestamp, utils.alerts_format) + twid_int = int(twid.replace("timewindow", "")) evidence: Evidence = Evidence( evidence_type=EvidenceType.COMMAND_AND_CONTROL_CHANNEL, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), + threat_level=ThreatLevel.HIGH, confidence=confidence, description=description, profile=ProfileID(ip=srcip), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_int), uid=[uid], timestamp=timestamp, category=IDEACategory.INTRUSION_BOTNET, @@ -89,7 +86,28 @@ def set_evidence_cc_channel( port=int(port), proto=Proto(proto.lower()) if proto else None, ) + self.db.set_evidence(evidence) + evidence: Evidence = Evidence( + evidence_type=EvidenceType.COMMAND_AND_CONTROL_CHANNEL, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=dstip + ), + threat_level=ThreatLevel.HIGH, + confidence=confidence, + description=description, + profile=ProfileID(ip=dstip), + timewindow=TimeWindow(number=twid_int), + uid=[uid], + timestamp=timestamp, + category=IDEACategory.INTRUSION_BOTNET, + source_target_tag=Tag.CC, + port=int(port), + proto=Proto(proto.lower()) if proto else None, + ) + self.db.set_evidence(evidence) diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index 67a1bb2da..1782620b7 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -192,7 +192,7 @@ def set_evidence_malicious_asn( self.db.set_evidence(evidence) - def set_evidence_malicious_dns_response( + def set_evidence_malicious_ip_in_dns_response( self, ip: str, uid: str, @@ -371,21 +371,6 @@ def set_evidence_malicious_ip( # add this ip to our MaliciousIPs hash in the database self.db.set_malicious_ip(ip, profileid, twid) - - def set_evidence_malicious_dns_query(self, - CNAME: str, - domain, - uid: str, - timestamp: str, - domain_info: dict, - is_subdomain: bool, - profileid: str = '', - twid: str = ''): - description: str = (f'DNS answer with a blacklisted ' - f'CNAME: {domain} ' - f'for query: {query} ') - - def set_evidence_malicious_domain( self, @@ -427,21 +412,19 @@ def set_evidence_malicious_domain( tags = domain_info.get('tags', None) if tags: description += f'with tags: {tags}. ' - - attacker = Attacker( + twid_number = int(twid.replace("timewindow", "")) + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN, + attacker=Attacker( direction=Direction.SRC, attacker_type=IoCType.IP, value=srcip - ) - - evidence = Evidence( - evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN, - attacker=attacker, + ), threat_level=threat_level, confidence=confidence, description=description, profile=ProfileID(ip=srcip), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_number), uid=[uid], timestamp=utils.convert_format(timestamp, utils.alerts_format), category=IDEACategory.ANOMALY_TRAFFIC, @@ -450,12 +433,35 @@ def set_evidence_malicious_domain( self.db.set_evidence(evidence) + domain_resolution: str = self.db.get_domain_resolution(domain) + if domain_resolution: + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.DOMAIN, + value=domain + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=domain_resolution), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_DOMAIN, + ) + + self.db.set_evidence(evidence) + def is_valid_threat_level(self, threat_level): return threat_level in utils.threat_levels def parse_local_ti_file(self, ti_file_path: str) -> bool: """ - Read all the files holding IP addresses and a description and store in the db. + Read all the files holding IP addresses and a description + and store in the db. This also helps in having unique ioc across files Returns nothing, but the dictionary should be filled :param ti_file_path: full path_to local threat intel file @@ -849,11 +855,6 @@ def set_evidence_malicious_hash(self, file_info: Dict[str, any]): f'Detected by: {file_info["blacklist"]}. ' f'Score: {confidence}. {ip_identification}' ) - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) ts = utils.convert_format( file_info['flow']["starttime"], utils.alerts_format ) @@ -862,7 +863,11 @@ def set_evidence_malicious_hash(self, file_info: Dict[str, any]): )) evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_DOWNLOADED_FILE, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), threat_level=threat_level, confidence=confidence, description=description, @@ -874,6 +879,25 @@ def set_evidence_malicious_hash(self, file_info: Dict[str, any]): ) self.db.set_evidence(evidence) + + evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_DOWNLOADED_FILE, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=srcip), + timewindow=twid, + uid=[file_info['flow']["uid"]], + timestamp=ts, + category=IDEACategory.MALWARE + ) + + self.db.set_evidence(evidence) def circl_lu(self, flow_info: dict): """ @@ -1069,7 +1093,7 @@ def is_malicious_ip(self, ip: json.dumps(ip_info) }) if is_dns_response: - self.set_evidence_malicious_dns_response( + self.set_evidence_malicious_ip_in_dns_response( ip, uid, timestamp, @@ -1097,9 +1121,12 @@ def is_malicious_hash(self, flow_info: dict): """ if not flow_info['flow']['md5']: # some lines in the zeek files.log doesn't have a hash for example - # {"ts":293.713187,"fuid":"FpvjEj3U0Qoj1fVCQc","tx_hosts":["94.127.78.125"],"rx_hosts":["10.0.2.19"], - # "conn_uids":["CY7bgw3KI8QyV67jqa","CZEkWx4wAvHJv0HTw9","CmM1ggccDvwnwPCl3","CBwoAH2RcIueFH4eu9","CZVfkc4BGLqRR7wwD5"], - # "source":"HTTP","depth":0,"analyzers":["SHA1","SHA256","MD5"] .. } + # {"ts":293.713187,"fuid":"FpvjEj3U0Qoj1fVCQc", + # "tx_hosts":["94.127.78.125"],"rx_hosts":["10.0.2.19"], + # "conn_uids":["CY7bgw3KI8QyV67jqa","CZEkWx4wAvHJv0HTw9", + # "CmM1ggccDvwnwPCl3","CBwoAH2RcIueFH4eu9","CZVfkc4BGLqRR7wwD5"], + # "source":"HTTP","depth":0,"analyzers":["SHA1","SHA256","MD5"] + # .. } return if blacklist_details := self.search_online_for_hash(flow_info): @@ -1119,6 +1146,7 @@ def is_malicious_url( url, uid, timestamp, + daddr, profileid, twid ): @@ -1128,13 +1156,79 @@ def is_malicious_url( # not malicious return False - self.set_evidence_malicious_url( + self.urlhaus.set_evidence_malicious_url( + daddr, url_info, uid, timestamp, profileid, twid ) + + def set_evidence_malicious_cname_in_dns_response(self, + cname: str, + dns_query: str, + uid: str, + timestamp: str, + cname_info: dict, + is_subdomain: bool, + profileid: str = '', + twid: str = '' + ): + """ + :param cname: the dns answer that we looked up and turned out to be + malicious + :param dns_query: the query we asked the DNS server for when the + server returned the given cname + """ + if not cname_info: + return + + srcip = profileid.split("_")[-1] + # in case of finding a subdomain in our blacklists + # print that in the description of the alert and change the + # confidence accordingly in case of a domain, confidence=1 + confidence: float = 0.7 if is_subdomain else 1 + + # when we comment ti_files and run slips, we + # get the error of not being able to get feed threat_level + threat_level: float = utils.threat_levels[ + cname_info.get('threat_level', 'high') + ] + threat_level: ThreatLevel = ThreatLevel(threat_level) + description: str = (f'blacklisted CNAME: {cname} when resolving ' + f'{dns_query}' + f'Description: {cname_info.get("description", "")},' + f'Found in feed: {cname_info["source"]}, ' + f'Confidence: {confidence}. ') + + tags = cname_info.get('tags', None) + if tags: + description += f'with tags: {tags}. ' + + attacker = Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ) + + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_DNS_ANSWER, + attacker=attacker, + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=srcip), + timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_DOMAIN, + ) + + self.db.set_evidence(evidence) + + def is_malicious_cname(self, dns_query, @@ -1145,9 +1239,10 @@ def is_malicious_cname(self, twid, ): """ + looks up the given CNAME :param cname: is the dns answer we're looking up :param dns_query: the query we asked the DNS server for when the - server returned the given sname + server returned the given cname """ if self.is_ignored_domain(cname): @@ -1157,7 +1252,8 @@ def is_malicious_cname(self, if not domain_info: return False - self.set_evidence_malicious_dns_query( + self.set_evidence_malicious_cname_in_dns_response( + cname, dns_query, uid, timestamp, @@ -1214,7 +1310,8 @@ def is_malicious_domain( def update_local_file(self, filename): """ Updates the given local ti file if the hash of it has changed - : param filename: local ti file, has to be plased in config/local_ti_files/ dir + : param filename: local ti file, has to be plased in + config/local_ti_files/ dir """ fullpath = os.path.join(self.path_to_local_ti_files, filename) if filehash := self.should_update_local_ti_file(fullpath): @@ -1318,11 +1415,12 @@ def main(self): to_lookup, uid, timestamp, + daddr, profileid, twid ) - if msg:= self.get_msg('new_downloaded_file'): + if msg := self.get_msg('new_downloaded_file'): file_info: dict = json.loads(msg['data']) # the format of file_info is as follows # { diff --git a/modules/threat_intelligence/urlhaus.py b/modules/threat_intelligence/urlhaus.py index 9fe65cf1a..89c35076d 100644 --- a/modules/threat_intelligence/urlhaus.py +++ b/modules/threat_intelligence/urlhaus.py @@ -80,7 +80,8 @@ def parse_urlhaus_url_response(self, response, url): threat_level = virustotal_percent # virustotal_result = virustotal_info.get("result", "") # virustotal_result.replace('\',''') - description += f'and was marked by {virustotal_percent}% of virustotal\'s AVs as malicious' + description += (f'and was marked by {virustotal_percent}% ' + f'of virustotal\'s AVs as malicious') except (KeyError, IndexError): # no payloads available @@ -148,7 +149,6 @@ def urlhaus_lookup(self, ioc, type_of_ioc: str): return self.parse_urlhaus_url_response(response, ioc) def set_evidence_malicious_hash(self, file_info: Dict[str, Any]) -> None: - flow: Dict[str, Any] = file_info['flow'] daddr: str = flow["daddr"] @@ -177,47 +177,77 @@ def set_evidence_malicious_hash(self, file_info: Dict[str, Any]) -> None: f" by URLhaus." ) - threat_level: float = file_info.get("threat_level", 0) + threat_level: float = file_info.get("threat_level") if threat_level: # Threat level here is the VT percentage from URLhaus description += f" Virustotal score: {threat_level}% malicious" threat_level: str = utils.threat_level_to_string(float( threat_level) / 100) + threat_level: ThreatLevel = ThreatLevel[threat_level.upper()] else: - threat_level = 'high' - - threat_level: ThreatLevel= ThreatLevel[threat_level.upper()] + threat_level: ThreatLevel = ThreatLevel.HIGH confidence: float = 0.7 saddr: str = file_info['profileid'].split("_")[-1] - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) timestamp: str = flow["starttime"] - twid: str = file_info["twid"] - - # Assuming you have an instance of the Evidence class in your class + twid_int = int(file_info["twid"].replace("timewindow", "")) evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_DOWNLOADED_FILE, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), threat_level=threat_level, confidence=confidence, description=description, timestamp=timestamp, category=IDEACategory.MALWARE, profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_int), uid=[flow["uid"]] ) self.db.set_evidence(evidence) - + + evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_DOWNLOADED_FILE, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=threat_level, + confidence=confidence, + description=description, + timestamp=timestamp, + category=IDEACategory.MALWARE, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_int), + uid=[flow["uid"]] + ) + + self.db.set_evidence(evidence) + + def get_threat_level(self, url_info: dict) -> ThreatLevel: + threat_level = url_info.get('threat_level', '') + if not threat_level: + return ThreatLevel.MEDIUM + + # Convert percentage reported by URLhaus (VirusTotal) to + # a valid SLIPS confidence + try: + threat_level = int(threat_level) / 100 + threat_level: str = utils.threat_level_to_string(threat_level) + return ThreatLevel[threat_level.upper()] + except ValueError: + return ThreatLevel.MEDIUM + def set_evidence_malicious_url( self, + daddr: str, url_info: Dict[str, Any], uid: str, timestamp: str, @@ -227,42 +257,42 @@ def set_evidence_malicious_url( """ Set evidence for a malicious URL based on the provided URL info """ - threat_level: str = url_info.get('threat_level', '') + threat_level: ThreatLevel = self.get_threat_level(url_info) description: str = url_info.get('description', '') - - confidence: float = 0.7 - - if not threat_level: - threat_level = 'medium' - else: - # Convert percentage reported by URLhaus (VirusTotal) to - # a valid SLIPS confidence - try: - threat_level = int(threat_level) / 100 - threat_level = utils.threat_level_to_string(threat_level) - except ValueError: - threat_level = 'medium' - - threat_level: ThreatLevel = ThreatLevel[threat_level.upper()] saddr: str = profileid.split("_")[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr + twid_int = int(twid.replace("timewindow", "")) + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_MALICIOUS_URL, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=threat_level, + confidence=0.7, + description=description, + timestamp=timestamp, + category=IDEACategory.MALWARE, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_int), + uid=[uid] ) + self.db.set_evidence(evidence) - # Assuming you have an instance of the Evidence class in your class evidence = Evidence( - evidence_type=EvidenceType.MALICIOUS_URL, - attacker=attacker, + evidence_type=EvidenceType.THREAT_INTELLIGENCE_MALICIOUS_URL, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, - confidence=confidence, + confidence=0.7, description=description, timestamp=timestamp, category=IDEACategory.MALWARE, profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_int), uid=[uid] ) diff --git a/slips_files/core/evidence_structure/evidence.py b/slips_files/core/evidence_structure/evidence.py index d68b7b7f5..92db93f67 100644 --- a/slips_files/core/evidence_structure/evidence.py +++ b/slips_files/core/evidence_structure/evidence.py @@ -86,7 +86,7 @@ class EvidenceType(Enum): THREAT_INTELLIGENCE_BLACKLISTED_DNS_ANSWER = auto() THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN = auto() MALICIOUS_DOWNLOADED_FILE = auto() - MALICIOUS_URL = auto() + THREAT_INTELLIGENCE_MALICIOUS_URL = auto() def __str__(self): return self.name From 92789283a7998a023e1705ce94c6089aff1f4bfd Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 23 Feb 2024 14:09:05 +0200 Subject: [PATCH 14/33] update unit tests --- modules/flowalerts/set_evidence.py | 78 +++++++++---------- modules/http_analyzer/http_analyzer.py | 7 +- .../core/database/redis_db/profile_handler.py | 1 - tests/module_factory.py | 2 +- tests/test_flowalerts.py | 20 ++++- tests/test_profiler.py | 7 +- 6 files changed, 61 insertions(+), 54 deletions(-) diff --git a/modules/flowalerts/set_evidence.py b/modules/flowalerts/set_evidence.py index 6488bbb8f..3235ea157 100644 --- a/modules/flowalerts/set_evidence.py +++ b/modules/flowalerts/set_evidence.py @@ -53,7 +53,7 @@ def young_domain( threat_level=ThreatLevel.LOW, category=IDEACategory.ANOMALY_TRAFFIC, description=description, - profile=ProfileID(ip=attacker.value), + profile=ProfileID(ip=attacker), timewindow=TimeWindow(number=twid_number), uid=[uid], timestamp=stime, @@ -1292,18 +1292,17 @@ def malicious_ja3s( ) self.db.set_evidence(evidence) - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) evidence: Evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_JA3S, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), threat_level=ThreatLevel.LOW, confidence=confidence, description=description, - profile=ProfileID(ip=attacker.value), + profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=twid_number), uid=[uid], timestamp=timestamp, @@ -1339,25 +1338,22 @@ def malicious_ja3( description += f'description: {ja3_description} ' description += f'tags: {tags}' - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim: Victim = Victim( - direction= Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - confidence: float = 1 evidence: Evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_JA3, - attacker=attacker, - victim=victim, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction= Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, - confidence=confidence, + confidence=1, description=description, - profile=ProfileID(ip=attacker.value), + profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), uid=[uid], timestamp=timestamp, @@ -1376,26 +1372,25 @@ def data_exfiltration( uid: List[str], timestamp ) -> None: - confidence: float = 0.6 saddr: str = profileid.split("_")[-1] - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) ip_identification: str = self.db.get_ip_identification(daddr) description: str = f'Large data upload. {src_mbs} MBs ' \ f'sent to {daddr} {ip_identification}' timestamp: str = utils.convert_format(timestamp, utils.alerts_format) - + twid_number = int(twid.replace("timewindow", "")) + evidence: Evidence = Evidence( evidence_type=EvidenceType.DATA_UPLOAD, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), threat_level=ThreatLevel.INFO, confidence=confidence, description=description, - profile=ProfileID(ip=attacker.value), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_number), uid=uid, timestamp=timestamp, category=IDEACategory.MALWARE, @@ -1404,19 +1399,18 @@ def data_exfiltration( self.db.set_evidence(evidence) - attacker: Attacker = Attacker( - direction=Direction.DST, - attacker_type=IoCType.IP, - value=daddr - ) evidence: Evidence = Evidence( evidence_type=EvidenceType.DATA_UPLOAD, - attacker=attacker, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), threat_level=ThreatLevel.HIGH, - confidence=confidence, + confidence=0.6, description=description, - profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_number), uid=uid, timestamp=timestamp, category=IDEACategory.MALWARE, diff --git a/modules/http_analyzer/http_analyzer.py b/modules/http_analyzer/http_analyzer.py index c1ab18c36..677a669ba 100644 --- a/modules/http_analyzer/http_analyzer.py +++ b/modules/http_analyzer/http_analyzer.py @@ -1,9 +1,10 @@ import json import urllib import requests -from typing import Union, \ +from typing import ( + Union, Dict - + ) from slips_files.common.imports import * from slips_files.core.evidence_structure.evidence import \ ( @@ -689,7 +690,7 @@ def set_evidence_weird_http_method( self.db.set_evidence(evidence) - def check_weird_http_method(self, msg: Dict[str]): + def check_weird_http_method(self, msg: Dict[str, str]): """ detect weird http methods in zeek's weird.log """ diff --git a/slips_files/core/database/redis_db/profile_handler.py b/slips_files/core/database/redis_db/profile_handler.py index ffc1a8f7c..53acd7939 100644 --- a/slips_files/core/database/redis_db/profile_handler.py +++ b/slips_files/core/database/redis_db/profile_handler.py @@ -1125,7 +1125,6 @@ def getT2ForProfileTW(self, profileid, twid, tupleid, tuple_key: str): ) self.print(type(e), 0, 1) self.print(e, 0, 1) - self.print(traceback.print_stack(), 0, 1) def has_profile(self, profileid): """Check if we have the given profile""" diff --git a/tests/module_factory.py b/tests/module_factory.py index dcbb5e0a7..815b8950b 100644 --- a/tests/module_factory.py +++ b/tests/module_factory.py @@ -55,7 +55,7 @@ def __init__(self): self.profiler_queue = Queue() self.input_queue = Queue() self.dummy_termination_event = Event() - self.logger = Output() + self.logger = Mock() # Output() def get_default_db(self): diff --git a/tests/test_flowalerts.py b/tests/test_flowalerts.py index 977fcd870..c6268dc50 100644 --- a/tests/test_flowalerts.py +++ b/tests/test_flowalerts.py @@ -1,5 +1,4 @@ """Unit test for modules/flowalerts/flowalerts.py""" -from slips_files.core.flows.zeek import Conn from tests.module_factory import ModuleFactory import json from numpy import arange @@ -161,15 +160,30 @@ def test_detect_young_domains( ): flowalerts = ModuleFactory().create_flowalerts_obj(mock_db) domain = 'example.com' + answers = ['192.168.1.1', '192.168.1.2', '192.168.1.3', 'CNAME_HERE.com'] # age in days mock_db.getDomainData.return_value = {'Age': 50} assert ( - flowalerts.detect_young_domains(domain, timestamp, profileid, twid, uid) is True + flowalerts.detect_young_domains( + domain, + answers, + timestamp, + profileid, + twid, + uid + ) is True ) # more than the age threshold mock_db.getDomainData.return_value = {'Age': 1000} assert ( - flowalerts.detect_young_domains(domain, timestamp, profileid, twid, uid) is False + flowalerts.detect_young_domains( + domain, + answers, + timestamp, + profileid, + twid, + uid + ) is False ) diff --git a/tests/test_profiler.py b/tests/test_profiler.py index 92f121a82..dae14a012 100644 --- a/tests/test_profiler.py +++ b/tests/test_profiler.py @@ -1,5 +1,5 @@ """Unit test for slips_files/core/performance_profiler.py""" -import ipaddress +from unittest.mock import Mock from tests.module_factory import ModuleFactory from tests.common_test_utils import do_nothing @@ -154,6 +154,7 @@ def test_define_separator_nfdump(nfdump_file, ) def test_process_line(file, flow_type, mock_db): profiler = ModuleFactory().create_profiler_obj(mock_db) + profiler.symbol = Mock() # we're testing another functionality here profiler.whitelist.is_whitelisted_flow = do_nothing profiler.input_type = 'zeek' @@ -172,11 +173,9 @@ def test_process_line(file, flow_type, mock_db): 'type': flow_type } - # process it profiler.flow = profiler.input_handler.process_line(sample_flow) - assert profiler.flow + assert profiler.flow is not None - # add to profile added_to_prof = profiler.add_flow_to_profile() assert added_to_prof is True From 85b20203ebf52777ad2c1f7ff3d1414ec71411b0 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 14:15:36 +0200 Subject: [PATCH 15/33] flowalerts: use the dns answer of young domains to set evidence for the ip of that domain --- modules/flowalerts/flowalerts.py | 36 +++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/modules/flowalerts/flowalerts.py b/modules/flowalerts/flowalerts.py index d90967359..fca8d47d8 100644 --- a/modules/flowalerts/flowalerts.py +++ b/modules/flowalerts/flowalerts.py @@ -1,4 +1,8 @@ import contextlib +from typing import List + +from slips_files.common.imports import * + import json import threading import ipaddress @@ -69,7 +73,8 @@ def init(self): # after this number of failed ssh logins, we alert pw guessing self.pw_guessing_threshold = 20 self.password_guessing_cache = {} - # in pastebin download detection, we wait for each conn.log flow of the seen ssl flow to appear + # in pastebin download detection, we wait for each conn.log flow + # of the seen ssl flow to appear # this is the dict of ssl flows we're waiting for self.pending_ssl_flows = multiprocessing.Queue() # thread that waits for ssl flows to appear in conn.log @@ -801,7 +806,7 @@ def check_dns_without_connection( # every dns answer is a list of ips that correspond to 1 query, # one of these ips should be present in the contacted ips # check each one of the resolutions of this domain - for ip in answers: + for ip in self.extract_ips_from_dns_answers(answers): # self.print(f'Checking if we have a connection to ip {ip}') if ( ip in contacted_ips @@ -1264,7 +1269,9 @@ def check_multiple_reconnection_attempts( profileid, twid, current_reconnections ) - def detect_young_domains(self, domain, stime, profileid, twid, uid): + def detect_young_domains(self, domain, answers: List[str], stime, + profileid, + twid, uid): """ Detect domains that are too young. The threshold is 60 days @@ -1290,12 +1297,27 @@ def detect_young_domains(self, domain, stime, profileid, twid, uid): age = domain_info['Age'] if age >= age_threshold: return False - + + + ips_returned_in_answer: List[str] = ( + self.extract_ips_from_dns_answers(answers) + ) + self.set_evidence.young_domain( - domain, age, stime, profileid, twid, uid + domain, age, stime, profileid, twid, uid, ips_returned_in_answer ) return True - + + def extract_ips_from_dns_answers(self, answers: List[str]) -> List[str]: + """ + extracts ipv4 and 6 from DNS answers + """ + ips = [] + for answer in answers: + if validators.ipv4(answer) or validators.ipv6(answer): + ips.append(answer) + return ips + def check_smtp_bruteforce( self, profileid, @@ -2141,7 +2163,7 @@ def main(self): # TODO: not sure how to make sure IP_info is # done adding domain age to the db or not self.detect_young_domains( - domain, stime, profileid, twid, uid + domain, answers, stime, profileid, twid, uid ) self.check_dns_arpa_scan( domain, stime, profileid, twid, uid From a95d46f4096bad6b51910dc5446982a4b2456da5 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 14:19:45 +0200 Subject: [PATCH 16/33] flowalerts: move weird http method detections to http analyzer module --- modules/flowalerts/flowalerts.py | 24 +----- modules/http_analyzer/http_analyzer.py | 82 ++++++++++++++++++- .../core/database/redis_db/database.py | 6 +- 3 files changed, 85 insertions(+), 27 deletions(-) diff --git a/modules/flowalerts/flowalerts.py b/modules/flowalerts/flowalerts.py index fca8d47d8..77b8bb810 100644 --- a/modules/flowalerts/flowalerts.py +++ b/modules/flowalerts/flowalerts.py @@ -1619,25 +1619,6 @@ def check_malicious_ssl(self, ssl_info): ssl_info, ssl_info_from_db ) - def check_weird_http_method(self, msg): - """ - detect weird http methods in zeek's weird.log - """ - flow = msg['flow'] - profileid = msg['profileid'] - twid = msg['twid'] - - # what's the weird.log about - name = flow['name'] - - if 'unknown_HTTP_method' not in name: - return False - - self.set_evidence.weird_http_method( - profileid, - twid, - flow - ) def check_non_http_port_80_conns( self, @@ -2201,10 +2182,7 @@ def main(self): role='SSH::SERVER' ) - if msg := self.get_msg('new_weird'): - msg = json.loads(msg['data']) - self.check_weird_http_method(msg) - + if msg := self.get_msg('new_tunnel'): msg = json.loads(msg['data']) self.check_GRE_tunnel(msg) diff --git a/modules/http_analyzer/http_analyzer.py b/modules/http_analyzer/http_analyzer.py index f502387ed..bed6c5c5a 100644 --- a/modules/http_analyzer/http_analyzer.py +++ b/modules/http_analyzer/http_analyzer.py @@ -2,7 +2,8 @@ import json import urllib import requests -from typing import Union +from typing import Union, \ + Dict from slips_files.common.imports import * from slips_files.core.evidence_structure.evidence import \ @@ -29,8 +30,10 @@ class HTTPAnalyzer(IModule): def init(self): self.c1 = self.db.subscribe('new_http') + self.c2 = self.db.subscribe('new_weird') self.channels = { - 'new_http': self.c1 + 'new_http': self.c1, + 'new_weird': self.c2 } self.connections_counter = {} self.empty_connections_threshold = 4 @@ -637,8 +640,79 @@ def check_pastebin_downloads( self.db.set_evidence(evidence) return True + + def set_evidence_weird_http_method( + self, + profileid: str, + twid: str, + flow: dict + ) -> None: + daddr: str = flow['daddr'] + weird_method: str = flow['addl'] + uid: str = flow['uid'] + timestamp: str = flow['starttime'] + + confidence = 0.9 + threat_level: ThreatLevel = ThreatLevel.MEDIUM + saddr: str = profileid.split("_")[-1] + + attacker: Attacker = Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ) + victim: Victim = Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ) + + ip_identification: str = self.db.get_ip_identification(daddr) + description: str = f'Weird HTTP method "{weird_method}" to IP: ' \ + f'{daddr} {ip_identification}. by Zeek.' + + twid_number: int = int(twid.replace("timewindow", "")) + + evidence: Evidence = Evidence( + evidence_type=EvidenceType.WEIRD_HTTP_METHOD, + attacker=attacker, + victim=victim, + threat_level=threat_level, + category=IDEACategory.ANOMALY_TRAFFIC, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=timestamp, + conn_count=1, + confidence=confidence + ) + + self.db.set_evidence(evidence) + + + def check_weird_http_method(self, msg: Dict[str]): + """ + detect weird http methods in zeek's weird.log + """ + flow = msg['flow'] + profileid = msg['profileid'] + twid = msg['twid'] + + # what's the weird.log about + name = flow['name'] + + if 'unknown_HTTP_method' not in name: + return False + + self.set_evidence_weird_http_method( + profileid, + twid, + flow + ) + def pre_main(self): utils.drop_root_privs() @@ -736,3 +810,7 @@ def main(self): uid, timestamp ) + + if msg := self.get_msg('new_weird'): + msg = json.loads(msg['data']) + self.check_weird_http_method(msg) diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index e8d951cf7..59e0f20e1 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -628,10 +628,12 @@ def store_p2p_report(self, ip: str, report_data: dict): and last_report_about_this_ip['confidence'] == confidence ): report_time = report_data['report_time'] - # score and confidence are the same as the last report, only update the time + # score and confidence are the same as the last report, + # only update the time last_report_about_this_ip['report_time'] = report_time else: - # score and confidence are the different from the last report, add report to the list + # score and confidence are the different from the last + # report, add report to the list cached_p2p_reports[reporter].append(report_data) else: # ip was reported before, but not by the same peer From 7e06dd7fc79e7c7ae7ecfa04999e0aee4173fdb1 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 16:48:57 +0200 Subject: [PATCH 17/33] flowalerts: set 2 evidence for some detections, one for the saddr and one for the daddr --- modules/arp/arp.py | 2 +- modules/flowalerts/flowalerts.py | 11 +- modules/flowalerts/set_evidence.py | 811 +++++++++++++++-------------- 3 files changed, 414 insertions(+), 410 deletions(-) diff --git a/modules/arp/arp.py b/modules/arp/arp.py index b620439b9..c05d187fa 100644 --- a/modules/arp/arp.py +++ b/modules/arp/arp.py @@ -379,7 +379,7 @@ def detect_unsolicited_arp( # We're sure this is unsolicited arp # it may be arp spoofing confidence: float = 0.8 - threat_level: ThreatLevel = ThreatLevel.INFO + threat_level: ThreatLevel = ThreatLevel.LOW description: str = 'broadcasting unsolicited ARP' saddr: str = profileid.split('_')[-1] diff --git a/modules/flowalerts/flowalerts.py b/modules/flowalerts/flowalerts.py index 77b8bb810..2147b5225 100644 --- a/modules/flowalerts/flowalerts.py +++ b/modules/flowalerts/flowalerts.py @@ -1499,7 +1499,6 @@ def detect_malicious_ja3( daddr, ja3, ja3s, - profileid, twid, uid, timestamp @@ -1512,16 +1511,16 @@ def detect_malicious_ja3( malicious_ja3_dict = self.db.get_ja3_in_IoC() if ja3 in malicious_ja3_dict: - self.set_evidence.malicious_ja3( + self.set_evidence.malicious_ja3s( malicious_ja3_dict, twid, uid, timestamp, - daddr, saddr, - type_='ja3', + daddr, ja3=ja3, - ) + ) + if ja3s in malicious_ja3_dict: self.set_evidence.malicious_ja3( @@ -1531,7 +1530,6 @@ def detect_malicious_ja3( timestamp, saddr, daddr, - type_='ja3s', ja3=ja3s, ) @@ -2080,7 +2078,6 @@ def main(self): daddr, ja3, ja3s, - profileid, twid, uid, timestamp diff --git a/modules/flowalerts/set_evidence.py b/modules/flowalerts/set_evidence.py index 8d69a2ed4..9f1edacfd 100644 --- a/modules/flowalerts/set_evidence.py +++ b/modules/flowalerts/set_evidence.py @@ -32,31 +32,46 @@ def young_domain( domain: str, age: int, stime: str, - profileid: ProfileID, + profileid: str, twid: str, - uid: str + uid: str, + answers: List[str] ): saddr: str = profileid.split("_")[-1] - victim = Victim( - direction=Direction.SRC, - victim_type=IoCType.IP, - value=saddr, - ) - attacker = Attacker( - direction=Direction.DST, - attacker_type=IoCType.DOMAIN, - value=domain, - ) twid_number: int = int(twid.replace("timewindow", "")) - description = f'connection to a young domain: {domain} ' \ - f'registered {age} days ago.', + description: str = (f'connection to a young domain: {domain} ' + f'registered {age} days ago.') + if answers: + attacker = answers[0] + evidence = Evidence( + evidence_type=EvidenceType.YOUNG_DOMAIN, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=attacker, + ), + threat_level=ThreatLevel.LOW, + category=IDEACategory.ANOMALY_TRAFFIC, + description=description, + profile=ProfileID(ip=attacker.value), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=stime, + conn_count=1, + confidence=1.0 + ) + self.db.set_evidence(evidence) + evidence = Evidence( evidence_type=EvidenceType.YOUNG_DOMAIN, - attacker=attacker, - threat_level=ThreatLevel.LOW, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr, + ), + threat_level=ThreatLevel.INFO, category=IDEACategory.ANOMALY_TRAFFIC, description=description, - victim=victim, profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=twid_number), uid=[uid], @@ -65,6 +80,7 @@ def young_domain( confidence=1.0 ) self.db.set_evidence(evidence) + def multiple_ssh_versions( self, @@ -82,22 +98,21 @@ def multiple_ssh_versions( :param role: can be 'SSH::CLIENT' or 'SSH::SERVER' as seen in zeek software.log flows """ - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) role = 'client' if 'CLIENT' in role.upper() else 'server' description = f'SSH {role} version changing from ' \ f'{cached_versions} to {current_versions}' evidence = Evidence( evidence_type=EvidenceType.MULTIPLE_SSH_VERSIONS, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), threat_level=ThreatLevel.MEDIUM, category=IDEACategory.ANOMALY_TRAFFIC, description=description, - profile=ProfileID(ip=attacker.value), + profile=ProfileID(ip=srcip), timewindow=TimeWindow(int(twid.replace("timewindow", ''))), uid=uid, timestamp=timestamp, @@ -187,22 +202,18 @@ def device_changing_ips( confidence = 0.8 threat_level = ThreatLevel.MEDIUM saddr: str = profileid.split("_")[-1] - - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - description = f'A device changing IPs. IP {saddr} was found ' \ f'with MAC address {smac} but the MAC belongs ' \ f'originally to IP: {old_ip}. ' - twid_number = int(twid.replace("timewindow", "")) evidence = Evidence( evidence_type=EvidenceType.DEVICE_CHANGING_IP, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), threat_level=threat_level, category=IDEACategory.ANOMALY_TRAFFIC, description=description, @@ -226,17 +237,8 @@ def non_http_port_80_conn( uid: str ) -> None: confidence = 0.8 - threat_level = ThreatLevel.MEDIUM saddr: str = profileid.split("_")[-1] - - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) - description: str = f'non-HTTP established connection to port 80. ' \ f'destination IP: {daddr} {ip_identification}' @@ -244,8 +246,12 @@ def non_http_port_80_conn( evidence: Evidence = Evidence( evidence_type=EvidenceType.NON_HTTP_PORT_80_CONNECTION, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, category=IDEACategory.ANOMALY_TRAFFIC, description=description, profile=ProfileID(ip=saddr), @@ -255,6 +261,25 @@ def non_http_port_80_conn( conn_count=1, confidence=confidence ) + self.db.set_evidence(evidence) + + evidence: Evidence = Evidence( + evidence_type=EvidenceType.NON_HTTP_PORT_80_CONNECTION, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.MEDIUM, + category=IDEACategory.ANOMALY_TRAFFIC, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=timestamp, + conn_count=1, + confidence=confidence + ) self.db.set_evidence(evidence) @@ -267,20 +292,7 @@ def non_ssl_port_443_conn( uid: str ) -> None: confidence: float = 0.8 - threat_level: ThreatLevel = ThreatLevel.MEDIUM saddr: str = profileid.split("_")[-1] - - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = f'non-SSL established connection to port 443. ' \ f'destination IP: {daddr} {ip_identification}' @@ -289,9 +301,17 @@ def non_ssl_port_443_conn( evidence: Evidence = Evidence( evidence_type=EvidenceType.NON_SSL_PORT_443_CONNECTION, - attacker=attacker, - victim=victim, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.MEDIUM, category=IDEACategory.ANOMALY_TRAFFIC, description=description, profile=ProfileID(ip=saddr), @@ -304,55 +324,7 @@ def non_ssl_port_443_conn( self.db.set_evidence(evidence) - def weird_http_method( - self, - profileid: str, - twid: str, - flow: dict - ) -> None: - daddr: str = flow['daddr'] - weird_method: str = flow['addl'] - uid: str = flow['uid'] - timestamp: str = flow['starttime'] - - confidence = 0.9 - threat_level: ThreatLevel = ThreatLevel.MEDIUM - saddr: str = profileid.split("_")[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - - ip_identification: str = self.db.get_ip_identification(daddr) - description: str = f'Weird HTTP method "{weird_method}" to IP: ' \ - f'{daddr} {ip_identification}. by Zeek.' - - twid_number: int = int(twid.replace("timewindow", "")) - - evidence: Evidence = Evidence( - evidence_type=EvidenceType.WEIRD_HTTP_METHOD, - attacker=attacker, - victim=victim, - threat_level=threat_level, - category=IDEACategory.ANOMALY_TRAFFIC, - description=description, - profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=twid_number), - uid=[uid], - timestamp=timestamp, - conn_count=1, - confidence=confidence - ) - self.db.set_evidence(evidence) def incompatible_CN( self, @@ -364,21 +336,7 @@ def incompatible_CN( uid: str ) -> None: confidence: float = 0.9 - threat_level: ThreatLevel = ThreatLevel.MEDIUM saddr: str = profileid.split("_")[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = f'Incompatible certificate CN to IP: {daddr} ' \ f'{ip_identification} claiming to ' \ @@ -387,9 +345,17 @@ def incompatible_CN( twid_number: int = int(twid.replace("timewindow", "")) evidence: Evidence = Evidence( evidence_type=EvidenceType.INCOMPATIBLE_CN, - attacker=attacker, - victim=victim, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.MEDIUM, category=IDEACategory.ANOMALY_TRAFFIC, description=description, profile=ProfileID(ip=saddr), @@ -415,21 +381,18 @@ def DGA( # +1 ensures that the minimum confidence score is 1. confidence: float = max(0, (1 / 100) * (nxdomains - 100) + 1) confidence = round(confidence, 2) # for readability - threat_level = ThreatLevel.HIGH saddr = profileid.split("_")[-1] description = f'Possible DGA or domain scanning. {saddr} ' \ f'failed to resolve {nxdomains} domains' - attacker = Attacker( + evidence: Evidence = Evidence( + evidence_type=EvidenceType.DGA_NXDOMAINS, + attacker= Attacker( direction=Direction.SRC, attacker_type=IoCType.IP, value=saddr - ) - - evidence: Evidence = Evidence( - evidence_type=EvidenceType.DGA_NXDOMAINS, - attacker=attacker, - threat_level=threat_level, + ), + threat_level=ThreatLevel.HIGH, category=IDEACategory.ANOMALY_BEHAVIOUR, description=description, profile=ProfileID(ip=saddr), @@ -452,23 +415,18 @@ def DNS_without_conn( uid: str ) -> None: confidence: float = 0.8 - threat_level: ThreatLevel = ThreatLevel.LOW saddr: str = profileid.split("_")[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - description: str = f'domain {domain} resolved with no connection' - twid_number: int = int(twid.replace("timewindow", "")) evidence: Evidence = Evidence( evidence_type=EvidenceType.DNS_WITHOUT_CONNECTION, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, category=IDEACategory.ANOMALY_TRAFFIC, description=description, profile=ProfileID(ip=saddr), @@ -490,15 +448,8 @@ def pastebin_download( uid: str ) -> bool: - threat_level: ThreatLevel = ThreatLevel.INFO confidence: float = 1.0 saddr: str = profileid.split("_")[-1] - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - response_body_len: float = utils.convert_to_mb(bytes_downloaded) description: str = f'A downloaded file from pastebin.com. ' \ f'size: {response_body_len} MBs' @@ -506,8 +457,12 @@ def pastebin_download( twid_number: int = int(twid.replace("timewindow", "")) evidence: Evidence = Evidence( evidence_type=EvidenceType.PASTEBIN_DOWNLOAD, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.INFO, category=IDEACategory.ANOMALY_BEHAVIOUR, description=description, profile=ProfileID(ip=saddr), @@ -531,7 +486,7 @@ def conn_without_dns( uid: str ) -> None: confidence: float = 0.8 - threat_level: ThreatLevel = ThreatLevel.HIGH + threat_level: ThreatLevel = ThreatLevel.INFO saddr: str = profileid.split("_")[-1] attacker: Attacker = Attacker( direction=Direction.SRC, @@ -587,18 +542,16 @@ def dns_arpa_scan( description = f"Doing DNS ARPA scan. Scanned {arpa_scan_threshold}" \ f" hosts within 2 seconds." - # Store attacker details in a local variable - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) # Create Evidence object using local variables evidence = Evidence( evidence_type=EvidenceType.DNS_ARPA_SCAN, description=description, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), threat_level=threat_level, category=IDEACategory.RECON_SCANNING, profile=ProfileID(ip=saddr), @@ -629,18 +582,6 @@ def unknown_port( twid_number: int = int(twid.replace("timewindow", "")) saddr = profileid.split('_')[-1] - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = ( f'Connection to unknown destination port {dport}/{proto.upper()} ' @@ -649,8 +590,16 @@ def unknown_port( evidence: Evidence = Evidence( evidence_type=EvidenceType.UNKNOWN_PORT, - attacker=attacker, - victim=victim, + attacker= Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=ThreatLevel.HIGH, category=IDEACategory.ANOMALY_CONNECTION, description=description, @@ -677,24 +626,20 @@ def pw_guessing( # confidence = 1 because this detection is comming # from a zeek file so we're sure it's accurate confidence: float = 1.0 - threat_level: ThreatLevel = ThreatLevel.HIGH twid_number: int = int(twid.replace("timewindow", "")) scanning_ip: str = msg.split(' appears')[0] description: str = f'password guessing. {msg}. by {by}.' - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=scanning_ip - ) - conn_count: int = int(msg.split('in ')[1].split('connections')[0]) evidence: Evidence = Evidence( evidence_type=EvidenceType.PASSWORD_GUESSING, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=scanning_ip + ), + threat_level=ThreatLevel.HIGH, category= IDEACategory.ATTEMPT_LOGIN, description=description, profile=ProfileID(ip=scanning_ip), @@ -718,7 +663,6 @@ def horizontal_portscan( uid: str ) -> None: confidence: float = 1.0 - threat_level: ThreatLevel = ThreatLevel.HIGH twid_number: int = int(twid.replace("timewindow", "")) saddr = profileid.split('_')[-1] @@ -726,16 +670,14 @@ def horizontal_portscan( # get the number of unique hosts scanned on a specific port conn_count: int = int(msg.split('least')[1].split('unique')[0]) - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - evidence: Evidence = Evidence( evidence_type=EvidenceType.HORIZONTAL_PORT_SCAN, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.HIGH, category=IDEACategory.RECON_SCANNING, description=description, profile=ProfileID(ip=saddr), @@ -761,7 +703,6 @@ def conn_to_private_ip( timestamp: str ) -> None: confidence: float = 1.0 - threat_level: ThreatLevel = ThreatLevel.INFO twid_number: int = int(twid.replace("timewindow", "")) description: str = f'Connecting to private IP: {daddr} ' @@ -772,21 +713,14 @@ def conn_to_private_ip( else: description += f'on destination port: {dport}' - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - evidence: Evidence = Evidence( evidence_type=EvidenceType.CONNECTION_TO_PRIVATE_IP, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.INFO, category=IDEACategory.RECON, description=description, profile=ProfileID(ip=saddr), @@ -795,7 +729,11 @@ def conn_to_private_ip( timestamp=timestamp, conn_count=1, confidence=confidence, - victim=victim + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ) ) self.db.set_evidence(evidence) @@ -825,22 +763,18 @@ def GRE_tunnel( f'to {daddr} {ip_identification} ' \ f'tunnel action: {action}' - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - evidence: Evidence = Evidence( evidence_type=EvidenceType.GRE_TUNNEL, - attacker=attacker, - victim=victim, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, category=IDEACategory.INFO, description=description, @@ -866,29 +800,24 @@ def vertical_portscan( # confidence = 1 because this detection is coming # from a Zeek file so we're sure it's accurate confidence: float = 1.0 - threat_level: ThreatLevel = ThreatLevel.HIGH twid: int = int(twid.replace("timewindow", "")) # msg example: 192.168.1.200 has scanned 60 ports of 192.168.1.102 description: str = f'vertical port scan by Zeek engine. {msg}' conn_count: int = int(msg.split('least ')[1].split(' unique')[0]) - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=scanning_ip - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=msg.split('ports of host ')[-1].split(" in")[0] - ) - + evidence: Evidence = Evidence( evidence_type=EvidenceType.VERTICAL_PORT_SCAN, - attacker=attacker, - victim=victim, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=scanning_ip + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=msg.split('ports of host ')[-1].split(" in")[0] + ), + threat_level=ThreatLevel.HIGH, category=IDEACategory.RECON_SCANNING, description=description, profile=ProfileID(ip=scanning_ip), @@ -924,17 +853,6 @@ def ssh_successful( threat_level: ThreatLevel = ThreatLevel.INFO twid: int = int(twid.replace("timewindow", "")) - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = ( f'SSH successful to IP {daddr}. {ip_identification}. ' @@ -944,8 +862,16 @@ def ssh_successful( evidence: Evidence = Evidence( evidence_type=EvidenceType.SSH_SUCCESSFUL, - attacker=attacker, - victim=victim, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, confidence=confidence, description=description, @@ -970,7 +896,6 @@ def long_connection( """ Set an evidence for a long connection. """ - threat_level: ThreatLevel = ThreatLevel.LOW twid: int = int(twid.replace("timewindow", "")) # Confidence depends on how long the connection. # Scale the confidence from 0 to 1; 1 means 24 hours long. @@ -980,18 +905,6 @@ def long_connection( duration_minutes: int = int(duration / 60) srcip: str = profileid.split('_')[1] - attacker_obj: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) - - victim_obj: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = ( f'Long Connection. Connection from {srcip} ' @@ -1001,8 +914,12 @@ def long_connection( evidence: Evidence = Evidence( evidence_type=EvidenceType.LONG_CONNECTION, - attacker=attacker_obj, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), + threat_level=ThreatLevel.LOW, confidence=confidence, description=description, profile=ProfileID(ip=srcip), @@ -1010,7 +927,11 @@ def long_connection( uid=[uid], timestamp=timestamp, category=IDEACategory.ANOMALY_CONNECTION, - victim=victim_obj + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ) ) self.db.set_evidence(evidence) @@ -1028,7 +949,6 @@ def self_signed_certificates( Set evidence for self-signed certificates. """ confidence: float = 0.5 - threat_level: ThreatLevel = ThreatLevel.LOW saddr: str = profileid.split("_")[-1] twid: int = int(twid.replace("timewindow", "")) @@ -1048,7 +968,7 @@ def self_signed_certificates( evidence: Evidence = Evidence( evidence_type=EvidenceType.SELF_SIGNED_CERTIFICATE, attacker=attacker, - threat_level=threat_level, + threat_level=ThreatLevel.LOW, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -1057,7 +977,25 @@ def self_signed_certificates( timestamp=timestamp, category=IDEACategory.ANOMALY_BEHAVIOUR ) + self.db.set_evidence(evidence) + attacker: Attacker = Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ) + evidence: Evidence = Evidence( + evidence_type=EvidenceType.SELF_SIGNED_CERTIFICATE, + attacker=attacker, + threat_level=ThreatLevel.LOW, + confidence=confidence, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid), + uid=[uid], + timestamp=timestamp, + category=IDEACategory.ANOMALY_BEHAVIOUR + ) self.db.set_evidence(evidence) def multiple_reconnection_attempts( @@ -1077,18 +1015,6 @@ def multiple_reconnection_attempts( saddr: str = profileid.split("_")[-1] twid: int = int(twid.replace("timewindow", "")) - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification = self.db.get_ip_identification(daddr) description = ( f'Multiple reconnection attempts to Destination IP:' @@ -1097,8 +1023,16 @@ def multiple_reconnection_attempts( ) evidence: Evidence = Evidence( evidence_type=EvidenceType.MULTIPLE_RECONNECTION_ATTEMPTS, - attacker=attacker, - victim = victim, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim = Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, confidence=confidence, description=description, @@ -1125,7 +1059,6 @@ def connection_to_multiple_ports( Set evidence for connection to multiple ports. """ confidence: float = 0.5 - threat_level: ThreatLevel = ThreatLevel.INFO twid: int = int(twid.replace("timewindow", "")) ip_identification = self.db.get_ip_identification(attacker) description = f'Connection to multiple ports {dstports} of ' \ @@ -1140,22 +1073,19 @@ def connection_to_multiple_ports( victim_direction = Direction.SRC profile_ip = victim - victim: Victim = Victim( + evidence = Evidence( + evidence_type=EvidenceType.CONNECTION_TO_MULTIPLE_PORTS, + attacker=Attacker( + direction=attacker_direction, + attacker_type=IoCType.IP, + value=attacker + ), + victim=Victim( direction=victim_direction, victim_type=IoCType.IP, value=victim - ) - attacker: Attacker = Attacker( - direction=attacker_direction, - attacker_type=IoCType.IP, - value=attacker - ) - - evidence = Evidence( - evidence_type=EvidenceType.CONNECTION_TO_MULTIPLE_PORTS, - attacker=attacker, - victim=victim, - threat_level=threat_level, + ), + threat_level=ThreatLevel.INFO, confidence=confidence, description=description, profile=ProfileID(ip=profile_ip), @@ -1179,30 +1109,21 @@ def suspicious_dns_answer( uid: str ) -> None: confidence: float = 0.6 - threat_level: ThreatLevel = ThreatLevel.MEDIUM twid: int = int(twid.replace("timewindow", "")) saddr: str = profileid.split("_")[-1] - attacker: Attacker = Attacker( - direction=Direction.DST, - attacker_type=IoCType.IP, - value=daddr - ) - victim: Victim = Victim( - direction=Direction.SRC, - victim_type=IoCType.IP, - value=saddr - ) - description: str = f'A DNS TXT answer with high entropy. ' \ f'query: {query} answer: "{answer}" ' \ f'entropy: {round(entropy, 2)} ' evidence: Evidence = Evidence( evidence_type=EvidenceType.HIGH_ENTROPY_DNS_ANSWER, - attacker=attacker, - victim=victim, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.MEDIUM, confidence=confidence, description=description, profile=ProfileID(ip=daddr), @@ -1213,40 +1134,49 @@ def suspicious_dns_answer( ) self.db.set_evidence(evidence) + + + evidence: Evidence = Evidence( + evidence_type=EvidenceType.HIGH_ENTROPY_DNS_ANSWER, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, + confidence=confidence, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid), + uid=[uid], + timestamp=stime, + category=IDEACategory.ANOMALY_TRAFFIC + ) + self.db.set_evidence(evidence) def invalid_dns_answer( self, query: str, answer: str, - daddr: str, profileid: str, twid: str, stime: str, - uid: str + uid: str, ) -> None: - threat_level: ThreatLevel = ThreatLevel.INFO confidence: float = 0.7 twid: int = int(twid.replace("timewindow", "")) saddr: str = profileid.split("_")[-1] - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - description: str = f"The DNS query {query} was resolved to {answer}" evidence: Evidence = Evidence( evidence_type=EvidenceType.INVALID_DNS_RESOLUTION, - attacker=attacker, - victim=victim, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.INFO, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -1257,7 +1187,7 @@ def invalid_dns_answer( ) self.db.set_evidence(evidence) - + def for_port_0_connection( self, @@ -1284,26 +1214,22 @@ def for_port_0_connection( victim_direction = Direction.SRC profile_ip = victim - victim: Victim = Victim( - direction=victim_direction, - victim_type=IoCType.IP, - value=victim - ) - attacker: Attacker = Attacker( - direction=attacker_direction, - attacker_type=IoCType.IP, - value=attacker - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = f'Connection on port 0 from {saddr}:{sport} ' \ f'to {daddr}:{dport}. {ip_identification}.' - evidence: Evidence = Evidence( evidence_type=EvidenceType.PORT_0_CONNECTION, - attacker=attacker, - victim=victim, + attacker=Attacker( + direction=attacker_direction, + attacker_type=IoCType.IP, + value=attacker + ), + victim=Victim( + direction=victim_direction, + victim_type=IoCType.IP, + value=victim + ), threat_level=threat_level, confidence=confidence, description=description, @@ -1317,7 +1243,76 @@ def for_port_0_connection( ) self.db.set_evidence(evidence) + + + def malicious_ja3s( + self, + malicious_ja3_dict: dict, + twid: str, + uid: str, + timestamp: str, + saddr: str, + daddr: str, + ja3: str = '', + ) -> None: + ja3_info: dict = json.loads(malicious_ja3_dict[ja3]) + + threat_level: str = ja3_info['threat_level'].upper() + threat_level: ThreatLevel = ThreatLevel[threat_level] + tags: str = ja3_info.get('tags', '') + ja3_description: str = ja3_info['description'] + + ip_identification: str = self.db.get_ip_identification(daddr) + description = ( + f'Malicious JA3s: (possible C&C server): {ja3} to server ' + f'{daddr} {ip_identification} ' + ) + if ja3_description != 'None': + description += f'description: {ja3_description} ' + description += f'tags: {tags}' + confidence: float = 1 + twid_number: int = int(twid.replace("timewindow", "")) + evidence: Evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_JA3S, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=timestamp, + category=IDEACategory.INTRUSION_BOTNET, + source_target_tag=Tag.CC + ) + + self.db.set_evidence(evidence) + attacker: Attacker = Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ) + evidence: Evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_JA3S, + attacker=attacker, + threat_level=ThreatLevel.LOW, + confidence=confidence, + description=description, + profile=ProfileID(ip=attacker.value), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=timestamp, + category=IDEACategory.INTRUSION_BOTNET, + source_target_tag=Tag.CC + ) + + self.db.set_evidence(evidence) + def malicious_ja3( self, @@ -1325,13 +1320,10 @@ def malicious_ja3( twid: str, uid: str, timestamp: str, - victim: str, - attacker: str, - type_: str = '', + daddr: str, + saddr: str, ja3: str = '', ) -> None: - """ - """ ja3_info: dict = json.loads(malicious_ja3_dict[ja3]) threat_level: str = ja3_info['threat_level'].upper() @@ -1340,47 +1332,26 @@ def malicious_ja3( tags: str = ja3_info.get('tags', '') ja3_description: str = ja3_info['description'] - if type_ == 'ja3': - description = f'Malicious JA3: {ja3} from source address ' \ - f'{attacker} ' - evidence_type: EvidenceType = EvidenceType.MALICIOUS_JA3 - source_target_tag: Tag = Tag.BOTNET - attacker_direction: Direction = Direction.SRC - victim_direction: Direction = Direction.DST - - elif type_ == 'ja3s': - description = ( - f'Malicious JA3s: (possible C&C server): {ja3} to server ' - f'{attacker} ' - ) - - evidence_type: EvidenceType = EvidenceType.MALICIOUS_JA3S - source_target_tag: Tag = Tag.CC - attacker_direction: Direction = Direction.DST - victim_direction: Direction = Direction.SRC - else: - return - - # append daddr identification to the description - ip_identification: str = self.db.get_ip_identification(attacker) - description += f'{ip_identification} ' + ip_identification: str = self.db.get_ip_identification(saddr) + description = f'Malicious JA3: {ja3} from source address ' \ + f'{saddr} {ip_identification}' if ja3_description != 'None': description += f'description: {ja3_description} ' description += f'tags: {tags}' attacker: Attacker = Attacker( - direction=attacker_direction, + direction=Direction.SRC, attacker_type=IoCType.IP, - value=attacker + value=saddr ) victim: Victim = Victim( - direction=victim_direction, + direction= Direction.DST, victim_type=IoCType.IP, - value=victim + value=daddr ) confidence: float = 1 evidence: Evidence = Evidence( - evidence_type=evidence_type, + evidence_type=EvidenceType.MALICIOUS_JA3, attacker=attacker, victim=victim, threat_level=threat_level, @@ -1391,7 +1362,7 @@ def malicious_ja3( uid=[uid], timestamp=timestamp, category=IDEACategory.INTRUSION_BOTNET, - source_target_tag=source_target_tag + source_target_tag=Tag.BOTNET ) self.db.set_evidence(evidence) @@ -1406,7 +1377,6 @@ def data_exfiltration( timestamp ) -> None: confidence: float = 0.6 - threat_level: ThreatLevel = ThreatLevel.HIGH saddr: str = profileid.split("_")[-1] attacker: Attacker = Attacker( direction=Direction.SRC, @@ -1421,7 +1391,28 @@ def data_exfiltration( evidence: Evidence = Evidence( evidence_type=EvidenceType.DATA_UPLOAD, attacker=attacker, - threat_level=threat_level, + threat_level=ThreatLevel.INFO, + confidence=confidence, + description=description, + profile=ProfileID(ip=attacker.value), + timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + uid=uid, + timestamp=timestamp, + category=IDEACategory.MALWARE, + source_target_tag=Tag.ORIGIN_MALWARE + ) + + self.db.set_evidence(evidence) + + attacker: Attacker = Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ) + evidence: Evidence = Evidence( + evidence_type=EvidenceType.DATA_UPLOAD, + attacker=attacker, + threat_level=ThreatLevel.HIGH, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -1445,24 +1436,22 @@ def bad_smtp_login( confidence: float = 1.0 threat_level: ThreatLevel = ThreatLevel.HIGH - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) ip_identification: str = self.db.get_ip_identification(daddr) description: str = f'doing bad SMTP login to {daddr} ' \ f'{ip_identification}' evidence: Evidence = Evidence( evidence_type=EvidenceType.BAD_SMTP_LOGIN, - attacker=attacker, - victim=victim, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, confidence=confidence, description=description, @@ -1532,6 +1521,7 @@ def malicious_ssl( flow: dict = ssl_info['flow'] ts: str = flow.get('starttime', '') daddr: str = flow.get('daddr', '') + saddr: str = flow.get('saddr', '') uid: str = flow.get('uid', '') twid: str = ssl_info.get('twid', '') @@ -1550,17 +1540,34 @@ def malicious_ssl( f'{ip_identification} description: ' \ f'{cert_description} {tags} ' - - attacker: Attacker = Attacker( - direction=Direction.DST, - attacker_type=IoCType.IP, - value=daddr + evidence: Evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_SSL_CERT, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + uid=[uid], + timestamp=ts, + category=IDEACategory.INTRUSION_BOTNET, + source_target_tag=Tag.CC ) + self.db.set_evidence(evidence) + evidence: Evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_SSL_CERT, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, confidence=confidence, description=description, profile=ProfileID(ip=daddr), From 6556ed5a5358beb02d078d98571ba545de0d8f2e Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 17:36:19 +0200 Subject: [PATCH 18/33] p2p: when the network reports a malicious ip, set an evidence for the src and the dst ips --- modules/p2ptrust/p2ptrust.py | 102 ++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/modules/p2ptrust/p2ptrust.py b/modules/p2ptrust/p2ptrust.py index 4c0ea7e38..5486164e4 100644 --- a/modules/p2ptrust/p2ptrust.py +++ b/modules/p2ptrust/p2ptrust.py @@ -144,7 +144,7 @@ def init(self, *args, **kwargs): self.sql_db_name = f'{self.data_dir}trustdb.db' if self.rename_sql_db_file: - self.sql_db_name += str(pigeon_port) + self.sql_db_name += str(self.pigeon_port) # todo don't duplicate this dict, move it to slips_utils # all evidence slips detects has threat levels of strings # each string should have a corresponding int value to be able to calculate @@ -287,12 +287,14 @@ def new_evidence_callback(self, msg: Dict): threat_level = data.get('threat_level', False) if not threat_level: self.print( - f"IP {attacker} doesn't have a threat_level. not sharing to the network.", 0,2, + f"IP {attacker} doesn't have a threat_level. " + f"not sharing to the network.", 0,2, ) return if not confidence: self.print( - f"IP {attacker} doesn't have a confidence. not sharing to the network.", 0, 2, + f"IP {attacker} doesn't have a confidence. " + f"not sharing to the network.", 0, 2, ) return @@ -316,7 +318,8 @@ def new_evidence_callback(self, msg: Dict): network_score, timestamp, ) = cached_opinion - # if we don't have info about this ip from the p2p network, report it to the p2p netwrok + # if we don't have info about this ip from the p2p network, + # report it to the p2p netwrok if not cached_score: data_already_reported = False except KeyError: @@ -325,7 +328,8 @@ def new_evidence_callback(self, msg: Dict): # data saved in local db have wrong structure, this is an invalid state return - # TODO: in the future, be smarter and share only when needed. For now, we will always share + # TODO: in the future, be smarter and share only when needed. + # For now, we will always share if not data_already_reported: # Take data and send it to a peer as report. p2p_utils.send_evaluation_to_go( @@ -413,6 +417,7 @@ def data_request_callback(self, msg: Dict): # # tell other peers that we're blocking this IP # utils.send_blame_to_go(ip_address, score, confidence, self.pygo_channel) + def set_evidence_malicious_ip(self, ip_info: dict, threat_level: str, @@ -420,70 +425,67 @@ def set_evidence_malicious_ip(self, """ Set an evidence for a malicious IP met in the timewindow ip_info format is json serialized { - # 'ip': the source/dst ip - # 'profileid' : profile where the alert was generated. It includes the src ip - # 'twid' : name of the timewindow when it happened. - # 'proto' : protocol - # 'ip_state' : is basically the answer to "which one is the - # blacklisted IP"?'can be 'srcip' or - # 'dstip', - # 'stime': Exact time when the evidence happened - # 'uid': Zeek uid of the flow that generated the evidence, - # 'cache_age': How old is the info about this ip - # } + 'ip': the source/dst ip + 'profileid' : profile where the alert was generated. + It includes the src ip + 'twid' : name of the timewindow when it happened. + 'proto' : protocol + 'ip_state' : is basically the answer to "which one is the + blacklisted IP"?'can be 'srcip' or + 'dstip', + 'stime': Exact time when the evidence happened + 'uid': Zeek uid of the flow that generated the evidence, + 'cache_age': How old is the info about this ip + } :param threat_level: the threat level we learned form the network :param confidence: how confident the network opinion is about this opinion """ - + attacker_ip: str = ip_info.get('ip') - ip_state = ip_info.get('ip_state') - uid = ip_info.get('uid') profileid = ip_info.get('profileid') - twid = ip_info.get('twid') - timestamp = str(ip_info.get('stime')) saddr = profileid.split("_")[-1] - - category = IDEACategory.ANOMALY_TRAFFIC - + + threat_level = utils.threat_level_to_string(threat_level) + threat_level = ThreatLevel[threat_level.upper()] + twid_int = int(ip_info.get('twid').replace("timewindow", "")) + + # add this ip to our MaliciousIPs hash in the database + self.db.set_malicious_ip(attacker_ip, profileid, ip_info.get('twid')) + ip_identification = self.db.get_ip_identification(attacker_ip) - if 'src' in ip_state: + + if 'src' in ip_info.get('ip_state'): description = ( f'Connection from blacklisted IP {attacker_ip} ' f'({ip_identification}) to {saddr} Source: Slips P2P network.' ) - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=attacker_ip - ) else: description = ( f'Connection to blacklisted IP {attacker_ip} ' f'({ip_identification}) ' f'from {saddr} Source: Slips P2P network.' ) - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr + + for ip in (saddr, attacker_ip): + evidence = Evidence( + evidence_type= EvidenceType.MALICIOUS_IP_FROM_P2P_NETWORK, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=ip + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=attacker_ip), + timewindow=TimeWindow(number=twid_int), + uid=[ip_info.get('uid')], + timestamp=str(ip_info.get('stime')), + category=IDEACategory.ANOMALY_TRAFFIC, ) + + self.db.set_evidence(evidence) - evidence = Evidence( - evidence_type= EvidenceType.MALICIOUS_IP_FROM_P2P_NETWORK, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, - description=description, - profile=ProfileID(ip=attacker.value), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), - uid=[uid], - timestamp=timestamp, - category=category, - ) - - self.db.set_evidence(evidence) - # add this ip to our MaliciousIPs hash in the database - self.db.set_malicious_ip(attacker, profileid, twid) def handle_data_request(self, message_data: str) -> None: """ From 8b019559c92f549604f8f98223ef36b9cf337c97 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 17:37:00 +0200 Subject: [PATCH 19/33] p2p: remove the commented code for sending a blame report, handle_update() --- modules/p2ptrust/p2ptrust.py | 53 ------------------------------------ 1 file changed, 53 deletions(-) diff --git a/modules/p2ptrust/p2ptrust.py b/modules/p2ptrust/p2ptrust.py index 5486164e4..8ed1bdf3b 100644 --- a/modules/p2ptrust/p2ptrust.py +++ b/modules/p2ptrust/p2ptrust.py @@ -364,59 +364,6 @@ def data_request_callback(self, msg: Dict): except Exception as e: self.print(f'Exception {e} in data_request_callback', 0, 1) - # def handle_update(self, ip_address: str) -> None: - # """ - # Handle IP scores changing in Slips received from the ip_info_change channel - # - # This method checks if Slips has a new score that are different - # from the scores known to the network, and if so, it means that it is worth - # sharing and it will be shared. - # Additionally, if the score is serious, the node will be blamed(blocked) - # :param ip_address: The IP address sent through the ip_info_change channel (if it is not valid IP, it returns) - # """ - # - # # abort if the IP is not valid - # if not utils.validate_ip_address(ip_address): - # self.print("IP validation failed") - # return - # - # score, confidence = utils.get_ip_info_from_slips(ip_address) - # if score is None: - # self.print("IP doesn't have any score/confidence values in DB") - # return - # - # # insert data from slips to database - # self.trust_db.insert_slips_score(ip_address, score, confidence) - # - # # TODO: discuss - only share score if confidence is high enough? - # - # # compare slips data with data in go - # data_already_reported = True - # try: - # cached_opinion = self.trust_db.get_cached_network_opinion("ip", ip_address) - # cached_score, cached_confidence, network_score, timestamp = cached_opinion - # if cached_score is None: - # data_already_reported = False - # elif abs(score - cached_score) < 0.1: - # data_already_reported = False - # except KeyError: - # data_already_reported = False - # except IndexError: - # # data saved in local db have wrong structure, this is an invalid state - # return - # - # # TODO: in the future, be smarter and share only when needed. For now, we will always share - # if not data_already_reported: - # utils.send_evaluation_to_go(ip_address, score, confidence, "*", self.pygo_channel) - # - # # TODO: discuss - based on what criteria should we start blaming? - # # decide whether or not to block - # if score > 0.8 and confidence > 0.6: - # #todo finish the blocking logic and actually block the ip - # - # # tell other peers that we're blocking this IP - # utils.send_blame_to_go(ip_address, score, confidence, self.pygo_channel) - def set_evidence_malicious_ip(self, ip_info: dict, From 1210179b9539caef77986001731b060b956212ef Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 18:27:53 +0200 Subject: [PATCH 20/33] threat_intel: have a separate function for handling BLACKLISTED_DNS_ANSWER --- .../threat_intelligence.py | 191 +++++++++++++++--- .../core/evidence_structure/evidence.py | 1 + 2 files changed, 160 insertions(+), 32 deletions(-) diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index b3eeeea92..7293ef580 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -110,7 +110,7 @@ def __read_configuration(self): def set_evidence_malicious_asn( self, - attacker: str, + daddr: str, uid: str, timestamp: str, profileid: str, @@ -133,36 +133,142 @@ def set_evidence_malicious_asn( threat_level: ThreatLevel = ThreatLevel(threat_level) tags = asn_info.get('tags', '') - identification: str = self.db.get_ip_identification(attacker) + identification: str = self.db.get_ip_identification(daddr) description: str = ( - f'Connection to IP: {attacker} with blacklisted ASN: {asn} ' + f'Connection to IP: {daddr} with blacklisted ASN: {asn} ' f'Description: {asn_info["description"]}, ' f'Found in feed: {asn_info["source"]}, ' f'Confidence: {confidence}. Tags: {tags} {identification}' ) - attacker = Attacker( + twid_int = int(twid.replace("timewindow", "")) + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_ASN, + attacker=Attacker( direction=Direction.SRC, attacker_type=IoCType.IP, value=saddr - ) - evidence = Evidence( - evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_ASN, - attacker=attacker, + ), threat_level=threat_level, confidence=confidence, description=description, profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_int), uid=[uid], timestamp=utils.convert_format(timestamp, utils.alerts_format), category=IDEACategory.ANOMALY_TRAFFIC, source_target_tag=Tag.BLACKLISTED_ASN, ) + self.db.set_evidence(evidence) + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_ASN, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_int), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_ASN, + ) + + self.db.set_evidence(evidence) + + + def set_evidence_malicious_dns_response( + self, + ip: str, + uid: str, + timestamp: str, + ip_info: dict, + dns_query: str, + profileid: str, + twid: str, + ): + """ + Set an evidence for a blacklisted IP found in one of the TI files + :param ip: the ip source file + :param uid: Zeek uid of the flow that generated the evidence + :param timestamp: Exact time when the evidence happened + :param ip_info: is all the info we have about that IP + in the db source, confidence, description, etc. + :param profileid: profile where the alert was generated. It includes the src ip + :param twid: name of the timewindow when it happened. + """ + threat_level: float = utils.threat_levels[ + ip_info.get('threat_level', 'medium') + ] + threat_level: ThreatLevel = ThreatLevel(threat_level) + saddr = profileid.split("_")[-1] + + ip_identification: str = self.db.get_ip_identification( + ip, get_ti_data=False + ).strip() + description: str = (f'DNS answer with a blacklisted ' + f'IP: {ip} for query: {dns_query}' + f'{ip_identification} Description: ' + f'{ip_info["description"]}. ' + f'Source: {ip_info["source"]}.') + + twid_int = int(twid.replace("timewindow", "")) + evidence = Evidence( + evidence_type=EvidenceType + .THREAT_INTELLIGENCE_BLACKLISTED_DNS_ANSWER, + attacker= Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=ip + ), + threat_level=threat_level, + confidence=1.0, + description=description, + profile=ProfileID(ip=ip), + timewindow=TimeWindow(number=twid_int), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_IP, + ) + + self.db.set_evidence(evidence) + + evidence = Evidence( + evidence_type=EvidenceType + .THREAT_INTELLIGENCE_BLACKLISTED_DNS_ANSWER, + attacker= Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=threat_level, + confidence=1.0, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_int), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_IP, + ) + self.db.set_evidence(evidence) + # mark this ip as malicious in our database + ip_info = {'threatintelligence': ip_info} + self.db.setInfoForIPs(ip, ip_info) + + # add this ip to our MaliciousIPs hash in the database + self.db.set_malicious_ip(ip, profileid, twid) + + def set_evidence_malicious_ip( self, ip: str, @@ -179,7 +285,8 @@ def set_evidence_malicious_ip( :param ip: the ip source file :param uid: Zeek uid of the flow that generated the evidence :param timestamp: Exact time when the evidence happened - :param ip_info: is all the info we have about that IP in the db source, confidence, description, etc. + :param ip_info: is all the info we have about that IP + in the db source, confidence, description, etc. :param profileid: profile where the alert was generated. It includes the src ip :param twid: name of the timewindow when it happened. :param ip_state: is basically the answer to "which one is the @@ -201,12 +308,8 @@ def set_evidence_malicious_ip( value=ip ) elif 'dst' in ip_state: - if self.is_dns_response: - description: str = f'DNS answer with a blacklisted ' \ - f'IP: {ip} for query: {self.dns_query}' - else: - description: str = f'connection to blacklisted ' \ - f'IP: {ip} from {srcip}. ' + description: str = (f'connection to blacklisted ' + f'IP: {ip} from {srcip}. ') attacker = Attacker( direction=Direction.DST, @@ -224,7 +327,7 @@ def set_evidence_malicious_ip( f'{ip_info["description"]}. ' f'Source: {ip_info["source"]}.') - + twid_int = int(twid.replace("timewindow", "")) evidence = Evidence( evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_IP, attacker=attacker, @@ -232,7 +335,7 @@ def set_evidence_malicious_ip( confidence=confidence, description=description, profile=ProfileID(ip=attacker.value), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_int), uid=[uid], timestamp=utils.convert_format(timestamp, utils.alerts_format), category=IDEACategory.ANOMALY_TRAFFIC, @@ -911,10 +1014,18 @@ def is_malicious_ip(self, timestamp: str, profileid: str, twid: str, - ip_state: str) -> bool: + ip_state: str, + is_dns_response: bool=False, + dns_query: str=False + ) -> bool: """ Search for this IP in our database of IoC :param ip_state: is basically the answer to "which one is the + :param is_dns_response: set to true if the ip we're + looking up is a dns response + :param dns_query: is the dns query if the ip we're + looking up is a dns response + blacklisted IP"? can be 'srcip' or 'dstip' """ ip_info = self.search_offline_for_ip(ip) @@ -923,19 +1034,31 @@ def is_malicious_ip(self, if not ip_info: # not malicious return False + self.db.add_ips_to_IoC({ ip: json.dumps(ip_info) }) - self.set_evidence_malicious_ip( - ip, - uid, - daddr, - timestamp, - ip_info, - profileid, - twid, - ip_state, - ) + if is_dns_response: + self.set_evidence_malicious_dns_response( + ip, + uid, + timestamp, + ip_info, + dns_query, + profileid, + twid, + ) + else: + self.set_evidence_malicious_ip( + ip, + uid, + daddr, + timestamp, + ip_info, + profileid, + twid, + ip_state, + ) return True def is_malicious_hash(self, flow_info: dict): @@ -1073,7 +1196,7 @@ def main(self): # these 2 are only available when looking up dns answers # the query is needed when a malicious answer is found, # for more detailed description of the evidence - self.is_dns_response = data.get('is_dns_response') + is_dns_response = data.get('is_dns_response') self.dns_query = data.get('dns_query') # IP is the IP that we want the TI for. It can be a SRC or DST IP to_lookup = data.get('to_lookup', '') @@ -1092,13 +1215,17 @@ def main(self): or self.is_outgoing_icmp_packet(protocol, ip_state) ): self.is_malicious_ip( - ip, uid, daddr, timestamp, profileid, twid, ip_state + ip, uid, daddr, timestamp, profileid, twid, + ip_state, + dns_query=dns_query, + is_dns_response=is_dns_response, ) self.ip_belongs_to_blacklisted_range( ip, uid, daddr, timestamp, profileid, twid, ip_state ) self.ip_has_blacklisted_ASN( - ip, uid, timestamp, profileid, twid, ip_state + ip, uid, timestamp, profileid, twid, ip_state, + is_dns_response=is_dns_response ) elif type_ == 'domain': self.is_malicious_domain( diff --git a/slips_files/core/evidence_structure/evidence.py b/slips_files/core/evidence_structure/evidence.py index 8e8c23628..d68b7b7f5 100644 --- a/slips_files/core/evidence_structure/evidence.py +++ b/slips_files/core/evidence_structure/evidence.py @@ -83,6 +83,7 @@ class EvidenceType(Enum): COMMAND_AND_CONTROL_CHANNEL = auto() THREAT_INTELLIGENCE_BLACKLISTED_ASN = auto() THREAT_INTELLIGENCE_BLACKLISTED_IP = auto() + THREAT_INTELLIGENCE_BLACKLISTED_DNS_ANSWER = auto() THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN = auto() MALICIOUS_DOWNLOADED_FILE = auto() MALICIOUS_URL = auto() From 6d071d3e61b93939b50c26dd35e5d7d5026032fc Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 18:45:40 +0200 Subject: [PATCH 21/33] threat_intel: have more descriptive evidence when there's an evidence of a malicious ASN for an IP in a DNS answer --- .../threat_intelligence.py | 89 ++++++++++++------- 1 file changed, 56 insertions(+), 33 deletions(-) diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index 7293ef580..3f813e79d 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -117,9 +117,11 @@ def set_evidence_malicious_asn( twid: str, asn: str, asn_info: dict, + is_dns_response: bool = False ): """ - :param asn_info: the malicious ASN info taken from own_malicious_iocs.csv + :param asn_info: the malicious ASN info taken from + own_malicious_iocs.csv """ confidence: float = 0.8 @@ -134,12 +136,20 @@ def set_evidence_malicious_asn( tags = asn_info.get('tags', '') identification: str = self.db.get_ip_identification(daddr) - - description: str = ( - f'Connection to IP: {daddr} with blacklisted ASN: {asn} ' + if is_dns_response: + description: str = ( + f'Connection to IP: {daddr} with blacklisted ASN: {asn} ' + ) + else: + description: str = ( + f'DNS response with IP: {daddr} with blacklisted ASN: {asn} ' + ) + + description += ( f'Description: {asn_info["description"]}, ' f'Found in feed: {asn_info["source"]}, ' - f'Confidence: {confidence}. Tags: {tags} {identification}' + f'Confidence: {confidence}. ' + f'Tags: {tags} {identification}' ) twid_int = int(twid.replace("timewindow", "")) evidence = Evidence( @@ -273,7 +283,7 @@ def set_evidence_malicious_ip( self, ip: str, uid: str, - dstip: str, + daddr: str, timestamp: str, ip_info: dict, profileid: str = '', @@ -285,6 +295,7 @@ def set_evidence_malicious_ip( :param ip: the ip source file :param uid: Zeek uid of the flow that generated the evidence :param timestamp: Exact time when the evidence happened + :param daddr: dst address of the flow :param ip_info: is all the info we have about that IP in the db source, confidence, description, etc. :param profileid: profile where the alert was generated. It includes the src ip @@ -296,26 +307,14 @@ def set_evidence_malicious_ip( ip_info.get('threat_level', 'medium') ] threat_level: ThreatLevel = ThreatLevel(threat_level) - confidence: float = 1.0 - srcip = profileid.split("_")[-1] + saddr = profileid.split("_")[-1] if 'src' in ip_state: description: str = f'connection from blacklisted ' \ - f'IP: {ip} to {dstip}. ' - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=ip - ) + f'IP: {ip} to {daddr}. ' elif 'dst' in ip_state: description: str = (f'connection to blacklisted ' - f'IP: {ip} from {srcip}. ') - - attacker = Attacker( - direction=Direction.DST, - attacker_type=IoCType.IP, - value=ip - ) + f'IP: {ip} from {saddr}. ') else: # ip_state is not specified? return @@ -330,20 +329,41 @@ def set_evidence_malicious_ip( twid_int = int(twid.replace("timewindow", "")) evidence = Evidence( evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_IP, - attacker=attacker, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, - confidence=confidence, + confidence=1.0, description=description, - profile=ProfileID(ip=attacker.value), + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_int), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_IP, + ) + self.db.set_evidence(evidence) + + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_IP, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=threat_level, + confidence=1.0, + description=description, + profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=twid_int), uid=[uid], timestamp=utils.convert_format(timestamp, utils.alerts_format), category=IDEACategory.ANOMALY_TRAFFIC, source_target_tag=Tag.BLACKLISTED_IP, ) - self.db.set_evidence(evidence) - # mark this ip as malicious in our database ip_info = {'threatintelligence': ip_info} @@ -351,6 +371,8 @@ def set_evidence_malicious_ip( # add this ip to our MaliciousIPs hash in the database self.db.set_malicious_ip(ip, profileid, twid) + + def set_evidence_malicious_domain( self, @@ -928,7 +950,8 @@ def search_online_for_ip(self, ip): return spamhaus_res def ip_has_blacklisted_ASN( - self, ip, uid, timestamp, profileid, twid, ip_state + self, ip, uid, timestamp, profileid, twid, + is_dns_response: bool = False, ): """ Check if this ip has any of our blacklisted ASNs. @@ -956,6 +979,7 @@ def ip_has_blacklisted_ASN( twid, asn, asn_info, + is_dns_response=is_dns_response, ) def ip_belongs_to_blacklisted_range( @@ -1182,11 +1206,9 @@ def pre_main(self): self.circllu_calls_thread.start() def main(self): - # The channel now can receive an IP address or a domain name + # The channel can receive an IP address or a domain name if msg:= self.get_msg('give_threat_intelligence'): - # Data is sent in the channel as a json dict so we need to deserialize it first data = json.loads(msg['data']) - # Extract data from dict profileid = data.get('profileid') twid = data.get('twid') timestamp = data.get('stime') @@ -1197,10 +1219,11 @@ def main(self): # the query is needed when a malicious answer is found, # for more detailed description of the evidence is_dns_response = data.get('is_dns_response') - self.dns_query = data.get('dns_query') + dns_query = data.get('dns_query') # IP is the IP that we want the TI for. It can be a SRC or DST IP to_lookup = data.get('to_lookup', '') - # detect the type given because sometimes, http.log host field has ips OR domains + # detect the type given because sometimes, + # http.log host field has ips OR domains type_ = utils.detect_data_type(to_lookup) # ip_state will say if it is a srcip or if it was a dst_ip @@ -1224,7 +1247,7 @@ def main(self): ip, uid, daddr, timestamp, profileid, twid, ip_state ) self.ip_has_blacklisted_ASN( - ip, uid, timestamp, profileid, twid, ip_state, + ip, uid, timestamp, profileid, twid, is_dns_response=is_dns_response ) elif type_ == 'domain': From 9a66a0c923b91db5ebcd60efc371432017d94073 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 19:03:02 +0200 Subject: [PATCH 22/33] threat_intel: have a separate function to lookup cnames from dns answers --- .../threat_intelligence.py | 121 +++++++++++++----- 1 file changed, 88 insertions(+), 33 deletions(-) diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index 3f813e79d..67a1bb2da 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -372,6 +372,19 @@ def set_evidence_malicious_ip( # add this ip to our MaliciousIPs hash in the database self.db.set_malicious_ip(ip, profileid, twid) + def set_evidence_malicious_dns_query(self, + CNAME: str, + domain, + uid: str, + timestamp: str, + domain_info: dict, + is_subdomain: bool, + profileid: str = '', + twid: str = ''): + description: str = (f'DNS answer with a blacklisted ' + f'CNAME: {domain} ' + f'for query: {query} ') + def set_evidence_malicious_domain( @@ -385,7 +398,7 @@ def set_evidence_malicious_domain( twid: str = '', ): """ - Set an evidence for a malicious domain met in the timewindow + Set an evidence for a malicious domain :param source_file: is the domain source file :param domain_info: is all the info we have about this domain in the db source, confidence , description etc... @@ -406,18 +419,10 @@ def set_evidence_malicious_domain( domain_info.get('threat_level', 'high') ] threat_level: ThreatLevel = ThreatLevel(threat_level) - - - if self.is_dns_response: - description: str = (f'DNS answer with a blacklisted ' - f'CNAME: {domain} ' - f'for query: {self.dns_query} ') - else: - description: str = f'connection to a blacklisted domain {domain}. ' - - description += f'Description: {domain_info.get("description", "")},' \ - f' Found in feed: {domain_info["source"]}, ' \ - f'Confidence: {confidence}. ' + description: str = (f'connection to a blacklisted domain {domain}. ' + f'Description: {domain_info.get("description", "")},' + f'Found in feed: {domain_info["source"]}, ' + f'Confidence: {confidence}. ') tags = domain_info.get('tags', None) if tags: @@ -997,6 +1002,7 @@ def ip_belongs_to_blacklisted_range( ranges_starting_with_octet = self.cached_ipv6_ranges.get(first_octet, []) else: return False + for range in ranges_starting_with_octet: if ip_obj in ipaddress.ip_network(range): # ip was found in one of the blacklisted ranges @@ -1121,6 +1127,7 @@ def is_malicious_url( if not url_info: # not malicious return False + self.set_evidence_malicious_url( url_info, uid, @@ -1128,7 +1135,46 @@ def is_malicious_url( profileid, twid ) + + def is_malicious_cname(self, + dns_query, + cname, + uid, + timestamp, + profileid, + twid, + ): + """ + :param cname: is the dns answer we're looking up + :param dns_query: the query we asked the DNS server for when the + server returned the given sname + """ + + if self.is_ignored_domain(cname): + return False + + domain_info, is_subdomain = self.search_offline_for_domain(cname) + if not domain_info: + return False + + self.set_evidence_malicious_dns_query( + dns_query, + uid, + timestamp, + domain_info, + is_subdomain, + profileid, + twid, + ) + # mark this domain as malicious in our database + domain_info = { + 'threatintelligence': domain_info + } + self.db.setInfoForDomains(cname, domain_info) + # add this domain to our MaliciousDomains hash in the database + self.db.set_malicious_domain(cname, profileid, twid) + def is_malicious_domain( self, @@ -1136,7 +1182,7 @@ def is_malicious_domain( uid, timestamp, profileid, - twid + twid, ): if self.is_ignored_domain(domain): return False @@ -1144,7 +1190,7 @@ def is_malicious_domain( domain_info, is_subdomain = self.search_offline_for_domain(domain) if not domain_info: return False - + self.set_evidence_malicious_domain( domain, uid, @@ -1159,14 +1205,10 @@ def is_malicious_domain( domain_info = { 'threatintelligence': domain_info } - self.db.setInfoForDomains( - domain, domain_info - ) + self.db.setInfoForDomains(domain, domain_info) # add this domain to our MaliciousDomains hash in the database - self.db.set_malicious_domain( - domain, profileid, twid - ) + self.db.set_malicious_domain(domain, profileid, twid) def update_local_file(self, filename): @@ -1204,7 +1246,13 @@ def pre_main(self): self.update_local_file(local_file) self.circllu_calls_thread.start() - + + def should_lookup(self, ip: str, protocol: str, ip_state: str) \ + -> bool: + """return whther slips should lookup the given ip or notd""" + return (utils.is_ignored_ip(ip) or + self.is_outgoing_icmp_packet(protocol, ip_state)) + def main(self): # The channel can receive an IP address or a domain name if msg:= self.get_msg('give_threat_intelligence'): @@ -1231,12 +1279,10 @@ def main(self): # If given an IP, ask for it # Block only if the traffic isn't outgoing ICMP port unreachable packet + if type_ == 'ip': ip = to_lookup - if not ( - utils.is_ignored_ip(ip) - or self.is_outgoing_icmp_packet(protocol, ip_state) - ): + if not self.should_lookup(ip, protocol, ip_state): self.is_malicious_ip( ip, uid, daddr, timestamp, profileid, twid, ip_state, @@ -1251,13 +1297,22 @@ def main(self): is_dns_response=is_dns_response ) elif type_ == 'domain': - self.is_malicious_domain( - to_lookup, - uid, - timestamp, - profileid, - twid - ) + if is_dns_response: + self.is_malicious_cname( + dns_query, + to_lookup, + uid, + timestamp, + profileid, + twid) + else: + self.is_malicious_domain( + to_lookup, + uid, + timestamp, + profileid, + twid + ) elif type_ == 'url': self.is_malicious_url( to_lookup, From caab6a5787990aa6ff6d88d44b6e470d5c53a846 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 22 Feb 2024 19:46:45 +0200 Subject: [PATCH 23/33] set evidence for saddr and daddr for some evidence --- modules/arp/arp.py | 1 - modules/flowalerts/set_evidence.py | 10 +- modules/flowmldetection/flowmldetection.py | 16 +- modules/http_analyzer/http_analyzer.py | 131 +++++++------ modules/ip_info/ip_info.py | 51 +++-- modules/leak_detector/leak_detector.py | 148 ++++++++------- .../network_discovery/network_discovery.py | 44 ++--- modules/p2ptrust/p2ptrust.py | 39 ++-- modules/p2ptrust/utils/go_director.py | 11 +- modules/rnn_cc_detection/rnn_cc_detection.py | 40 ++-- .../threat_intelligence.py | 178 ++++++++++++++---- modules/threat_intelligence/urlhaus.py | 116 +++++++----- .../core/evidence_structure/evidence.py | 2 +- 13 files changed, 486 insertions(+), 301 deletions(-) diff --git a/modules/arp/arp.py b/modules/arp/arp.py index c05d187fa..468f1aa6f 100644 --- a/modules/arp/arp.py +++ b/modules/arp/arp.py @@ -1,4 +1,3 @@ -from slips_files.common.abstracts._module import IModule import json import ipaddress import time diff --git a/modules/flowalerts/set_evidence.py b/modules/flowalerts/set_evidence.py index 9f1edacfd..6488bbb8f 100644 --- a/modules/flowalerts/set_evidence.py +++ b/modules/flowalerts/set_evidence.py @@ -1,8 +1,8 @@ -import datetime import json import sys import time -from typing import List +from typing import List, \ + Dict from slips_files.common.slips_utils import utils from slips_files.core.evidence_structure.evidence import \ @@ -126,7 +126,7 @@ def different_localnet_usage( self, daddr: str, portproto: str, - profileid: ProfileID, + profileid: str, timestamp: str, twid: str, uid: str, @@ -741,7 +741,7 @@ def conn_to_private_ip( def GRE_tunnel( self, - tunnel_info: dict + tunnel_info: Dict[str, str] ) -> None: profileid: str = tunnel_info['profileid'] twid: str = tunnel_info['twid'] @@ -1516,7 +1516,7 @@ def smtp_bruteforce( def malicious_ssl( self, ssl_info: dict, - ssl_info_from_db: dict + ssl_info_from_db: str ) -> None: flow: dict = ssl_info['flow'] ts: str = flow.get('starttime', '') diff --git a/modules/flowmldetection/flowmldetection.py b/modules/flowmldetection/flowmldetection.py index f1d2c1af4..1f1d7ec3a 100644 --- a/modules/flowmldetection/flowmldetection.py +++ b/modules/flowmldetection/flowmldetection.py @@ -384,14 +384,6 @@ def set_evidence_malicious_flow( uid: str ): confidence: float = 0.1 - threat_level: ThreatLevel = ThreatLevel.LOW - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - ip_identification = self.db.get_ip_identification(daddr) description = f'Malicious flow by ML. Src IP {saddr}:{sport} to ' \ f'{daddr}:{dport} {ip_identification}' @@ -403,8 +395,12 @@ def set_evidence_malicious_flow( evidence: Evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_FLOW, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, confidence=confidence, description=description, profile=ProfileID(ip=saddr), diff --git a/modules/http_analyzer/http_analyzer.py b/modules/http_analyzer/http_analyzer.py index bed6c5c5a..c1ab18c36 100644 --- a/modules/http_analyzer/http_analyzer.py +++ b/modules/http_analyzer/http_analyzer.py @@ -1,4 +1,3 @@ -from slips_files.common.abstracts._module import IModule import json import urllib import requests @@ -92,21 +91,19 @@ def check_suspicious_user_agents( ) for suspicious_ua in suspicious_user_agents: if suspicious_ua.lower() in user_agent.lower(): - threat_level: ThreatLevel = ThreatLevel.HIGH confidence: float = 1 saddr = profileid.split('_')[1] description: str = (f'Suspicious user-agent: ' f'{user_agent} while ' f'connecting to {host}{uri}') - attacker = Attacker( + evidence: Evidence = Evidence( + evidence_type=EvidenceType.SUSPICIOUS_USER_AGENT, + attacker=Attacker( direction=Direction.SRC, attacker_type=IoCType.IP, value=saddr - ) - evidence: Evidence = Evidence( - evidence_type=EvidenceType.SUSPICIOUS_USER_AGENT, - attacker=attacker, - threat_level=threat_level, + ), + threat_level=ThreatLevel.HIGH, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -165,21 +162,18 @@ def check_multiple_empty_connections( uids, connections = self.connections_counter[host] if connections == self.empty_connections_threshold: - threat_level: ThreatLevel = ThreatLevel.MEDIUM confidence: float = 1 saddr: str = profileid.split('_')[-1] description: str = f'Multiple empty HTTP connections to {host}' - attacker = Attacker( + evidence: Evidence = Evidence( + evidence_type=EvidenceType.EMPTY_CONNECTIONS, + attacker=Attacker( direction=Direction.SRC, attacker_type=IoCType.IP, value=saddr - ) - - evidence: Evidence = Evidence( - evidence_type=EvidenceType.EMPTY_CONNECTIONS, - attacker=attacker, - threat_level=threat_level, + ), + threat_level=ThreatLevel.MEDIUM, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -206,9 +200,7 @@ def set_evidence_incompatible_user_agent( twid, uid: str ): - threat_level: ThreatLevel = ThreatLevel.HIGH saddr = profileid.split('_')[1] - confidence: float = 1 os_type: str = user_agent.get('os_type', '').lower() os_name: str = user_agent.get('os_name', '').lower() @@ -222,17 +214,15 @@ def set_evidence_incompatible_user_agent( f'IP has MAC vendor: {vendor.capitalize()}' ) - attacker: Attacker = Attacker( - direction= Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - evidence: Evidence = Evidence( evidence_type=EvidenceType.INCOMPATIBLE_USER_AGENT, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction= Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.HIGH, + confidence=1, description=description, profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), @@ -253,30 +243,46 @@ def set_evidence_executable_mime_type( timestamp: str, daddr: str ): - confidence: float = 1 - threat_level: ThreatLevel = ThreatLevel.LOW saddr: str = profileid.split('_')[1] - attacker_obj: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = ( f'Download of an executable with MIME type: {mime_type} ' f'by {saddr} from {daddr} {ip_identification}.' ) - + twid_number = int(twid.replace("timewindow", "")) evidence: Evidence = Evidence( evidence_type=EvidenceType.EXECUTABLE_MIME_TYPE, - attacker=attacker_obj, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, + confidence=1, description=description, profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=timestamp, + category=IDEACategory.ANOMALY_FILE, + source_target_tag=Tag.EXECUTABLE_MIME_TYPE + ) + + self.db.set_evidence(evidence) + + evidence: Evidence = Evidence( + evidence_type=EvidenceType.EXECUTABLE_MIME_TYPE, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.LOW, + confidence=1, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_number), uid=[uid], timestamp=timestamp, category=IDEACategory.ANOMALY_FILE, @@ -515,24 +521,20 @@ def check_multiple_UAs( # 'Linux' in both UAs, so we shouldn't alert return False - threat_level: ThreatLevel = ThreatLevel.INFO - confidence: float = 1 saddr: str = profileid.split('_')[1] - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - ua: str = cached_ua.get('user_agent', '') description: str = (f'Using multiple user-agents:' f' "{ua}" then "{user_agent}"') evidence: Evidence = Evidence( evidence_type=EvidenceType.MULTIPLE_USER_AGENT, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.INFO, + confidence=1, description=description, profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), @@ -556,26 +558,17 @@ def set_evidence_http_traffic( timestamp: str ): confidence: float = 1 - threat_level: ThreatLevel = ThreatLevel.LOW saddr = profileid.split('_')[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) description = f'Unencrypted HTTP traffic from {saddr} to {daddr}.' evidence: Evidence = Evidence( evidence_type=EvidenceType.HTTP_TRAFFIC, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -584,7 +577,11 @@ def set_evidence_http_traffic( timestamp=timestamp, category=IDEACategory.ANOMALY_TRAFFIC, source_target_tag=Tag.SENDING_UNENCRYPTED_DATA, - victim=victim + victim= Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ) ) self.db.set_evidence(evidence) diff --git a/modules/ip_info/ip_info.py b/modules/ip_info/ip_info.py index adefe6390..8bc5ba536 100644 --- a/modules/ip_info/ip_info.py +++ b/modules/ip_info/ip_info.py @@ -511,17 +511,9 @@ def set_evidence_malicious_jarm_hash( dport: int = flow['dport'] dstip: str = flow['daddr'] saddr: str = flow['saddr'] - timestamp: float = flow['starttime'] + timestamp = flow['starttime'] protocol: str = flow['proto'] - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - threat_level = ThreatLevel.MEDIUM - confidence = 0.7 - portproto = f'{dport}/{protocol}' port_info = self.db.get_port_info(portproto) or "" port_info = f'({port_info.upper()})' if port_info else "" @@ -529,17 +521,22 @@ def set_evidence_malicious_jarm_hash( dstip_id = self.db.get_ip_identification(dstip) description = ( f"Malicious JARM hash detected for destination IP: {dstip}" - f" on port: {portproto} {port_info}. {dstip_id}" + f" on port: {portproto} {port_info}. {dstip_id}" ) - + twid_number = int(twid.replace("timewindow", "")) + evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_JARM, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=dstip + ), + threat_level=ThreatLevel.MEDIUM, + confidence=0.7, description=description, - profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + profile=ProfileID(ip=dstip), + timewindow=TimeWindow(number=twid_number), uid=[flow['uid']], timestamp=timestamp, category=IDEACategory.ANOMALY_TRAFFIC, @@ -549,6 +546,28 @@ def set_evidence_malicious_jarm_hash( ) self.db.set_evidence(evidence) + + evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_JARM, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, + confidence=0.7, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_number), + uid=[flow['uid']], + timestamp=timestamp, + category=IDEACategory.ANOMALY_TRAFFIC, + proto=Proto(protocol.lower()), + port=dport, + source_target_tag=Tag.MALWARE + ) + + self.db.set_evidence(evidence) def pre_main(self): diff --git a/modules/leak_detector/leak_detector.py b/modules/leak_detector/leak_detector.py index 3471dd6bd..50792e851 100644 --- a/modules/leak_detector/leak_detector.py +++ b/modules/leak_detector/leak_detector.py @@ -163,78 +163,100 @@ def get_packet_info(self, offset: int): def set_evidence_yara_match(self, info: dict): """ This function is called when yara finds a match - :param info: a dict with info about the matched rule, example keys 'vars_matched', 'index', + :param info: a dict with info about the matched rule, + example keys 'vars_matched', 'index', 'rule', 'srings_matched' """ rule = info.get('rule').replace('_', ' ') offset = info.get('offset') # vars_matched = info.get('vars_matched') strings_matched = info.get('strings_matched') - # we now know there's a match at offset x, we need to know offset x belongs to which packet - if packet_info := self.get_packet_info(offset): - srcip, dstip, proto, sport, dport, ts = ( - packet_info[0], - packet_info[1], - packet_info[2], - packet_info[3], - packet_info[4], - packet_info[5], - ) - - portproto = f'{dport}/{proto}' - port_info = self.db.get_port_info(portproto) - - # generate a random uid - uid = base64.b64encode(binascii.b2a_hex(os.urandom(9))).decode( - 'utf-8' - ) - profileid = f'profile_{srcip}' - # sometimes this module tries to find the profile before it's created. so - # wait a while before alerting. - time.sleep(4) - - ip_identification = self.db.get_ip_identification(dstip) - description = f"{rule} to destination address: {dstip} " \ - f"{ip_identification} port: {portproto} " \ - f"{port_info or ''}. " \ - f"Leaked location: {strings_matched}" - - # in which tw is this ts? - twid = self.db.get_tw_of_ts(profileid, ts) - # convert ts to a readable format - ts = utils.convert_format(ts, utils.alerts_format) - - if twid: - twid = twid[0] - source_target_tag = Tag.CC - confidence = 0.9 - threat_level = ThreatLevel.HIGH - - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) - + # we now know there's a match at offset x, we need + # to know offset x belongs to which packet + packet_info = self.get_packet_info(offset) + if not packet_info: + return + + srcip, dstip, proto, sport, dport, ts = ( + packet_info[0], + packet_info[1], + packet_info[2], + packet_info[3], + packet_info[4], + packet_info[5], + ) - evidence = Evidence( - evidence_type=EvidenceType.NETWORK_GPS_LOCATION_LEAKED, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, - description=description, - profile=ProfileID(ip=srcip), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), - uid=[uid], - timestamp=ts, - proto=Proto(proto.lower()), - port=dport, - source_target_tag=source_target_tag, - category=IDEACategory.MALWARE - ) + portproto = f'{dport}/{proto}' + port_info = self.db.get_port_info(portproto) - self.db.set_evidence(evidence) + # generate a random uid + uid = base64.b64encode(binascii.b2a_hex(os.urandom(9))).decode( + 'utf-8' + ) + profileid = f'profile_{srcip}' + # sometimes this module tries to find the profile before it's created. so + # wait a while before alerting. + time.sleep(4) + + ip_identification = self.db.get_ip_identification(dstip) + description = f"{rule} to destination address: {dstip} " \ + f"{ip_identification} port: {portproto} " \ + f"{port_info or ''}. " \ + f"Leaked location: {strings_matched}" + + # in which tw is this ts? + twid = self.db.get_tw_of_ts(profileid, ts) + # convert ts to a readable format + ts = utils.convert_format(ts, utils.alerts_format) + + if not twid: + return + + twid_number = int(twid[0].replace("timewindow", "")) + evidence = Evidence( + evidence_type=EvidenceType.NETWORK_GPS_LOCATION_LEAKED, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), + threat_level=ThreatLevel.LOW, + confidence=0.9, + description=description, + profile=ProfileID(ip=srcip), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=ts, + proto=Proto(proto.lower()), + port=dport, + source_target_tag=Tag.CC, + category=IDEACategory.MALWARE + ) + self.db.set_evidence(evidence) + + evidence = Evidence( + evidence_type=EvidenceType.NETWORK_GPS_LOCATION_LEAKED, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=dstip + ), + threat_level=ThreatLevel.HIGH, + confidence=0.9, + description=description, + profile=ProfileID(ip=dstip), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=ts, + proto=Proto(proto.lower()), + port=dport, + source_target_tag=Tag.CC, + category=IDEACategory.MALWARE + ) + + self.db.set_evidence(evidence) + def compile_and_save_rules(self): """ diff --git a/modules/network_discovery/network_discovery.py b/modules/network_discovery/network_discovery.py index bcdebf0da..f89dc082c 100644 --- a/modules/network_discovery/network_discovery.py +++ b/modules/network_discovery/network_discovery.py @@ -74,7 +74,6 @@ def check_icmp_sweep( Use our own Zeek scripts to detect ICMP scans. Threshold is on the scripts and it is 25 ICMP flows """ - if 'TimestampScan' in note: evidence_type = EvidenceType.ICMP_TIMESTAMP_SCAN elif 'ICMPAddressScan' in note: @@ -88,21 +87,17 @@ def check_icmp_sweep( hosts_scanned = int(msg.split('on ')[1].split(' hosts')[0]) # get the confidence from 0 to 1 based on the number of hosts scanned confidence = 1 / (255 - 5) * (hosts_scanned - 255) + 1 - threat_level = ThreatLevel.MEDIUM saddr = profileid.split('_')[1] - # this is the last IP scanned - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - # this one is detected by Zeek, so we can't track the UIDs causing it evidence = Evidence( evidence_type=evidence_type, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.MEDIUM, confidence=confidence, description=msg, profile=ProfileID(ip=saddr), @@ -321,29 +316,26 @@ def set_evidence_dhcp_scan( uids, number_of_requested_addrs ): - threat_level = ThreatLevel.MEDIUM - confidence = 0.8 srcip = profileid.split('_')[-1] - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) description = ( f'Performing a DHCP scan by requesting ' f'{number_of_requested_addrs} different IP addresses. ' f'Threat Level: {threat_level}. ' f'Confidence: {confidence}. by Slips' ) - + twid_number = int(twid.replace("timewindow", "")) evidence = Evidence( evidence_type=EvidenceType.DHCP_SCAN, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), + threat_level=ThreatLevel.MEDIUM, + confidence=0.8, description=description, profile=ProfileID(ip=srcip), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_number), uid=uids, timestamp=timestamp, category=IDEACategory.RECON_SCANNING, @@ -363,7 +355,8 @@ def check_dhcp_scan(self, flow_info): flow = flow_info['flow'] requested_addr = flow['requested_addr'] if not requested_addr: - # we are only interested in DHCPREQUEST flows, where a client is requesting an IP + # we are only interested in DHCPREQUEST flows, + # where a client is requesting an IP return profileid = flow_info['profileid'] @@ -400,7 +393,8 @@ def check_dhcp_scan(self, flow_info): # we alert every 4,8,12, etc. requested IPs number_of_requested_addrs = len(dhcp_flows) if number_of_requested_addrs % self.minimum_requested_addrs == 0: - # get the uids of all the flows where this client was requesting an addr in this tw + # get the uids of all the flows where this client + # was requesting an addr in this tw for uids_list in dhcp_flows.values(): uids.append(uids_list[0]) diff --git a/modules/p2ptrust/p2ptrust.py b/modules/p2ptrust/p2ptrust.py index 8ed1bdf3b..4ffc36ee8 100644 --- a/modules/p2ptrust/p2ptrust.py +++ b/modules/p2ptrust/p2ptrust.py @@ -48,7 +48,8 @@ def validate_slips_data(message_data: str) -> (str, int): 'cache_age': cache_age } - If the message is correct, the two values are returned as a tuple (str, int). + If the message is correct, the two values are + returned as a tuple (str, int). If not, (None, None) is returned. :param message_data: data from slips request channel :return: the received msg or None tuple @@ -62,7 +63,8 @@ def validate_slips_data(message_data: str) -> (str, int): except ValueError: # message has wrong format print( - f'The message received from p2p_data_request channel has incorrect format: {message_data}' + f'The message received from p2p_data_request channel' + f' has incorrect format: {message_data}' ) return None @@ -78,7 +80,8 @@ class Trust(IModule): gopy_channel_raw='p2p_gopy' pygo_channel_raw='p2p_pygo' start_pigeon=True - pigeon_binary= os.path.join(os.getcwd(),'p2p4slips/p2p4slips') # or make sure the binary is in $PATH + # or make sure the binary is in $PATH + pigeon_binary= os.path.join(os.getcwd(),'p2p4slips/p2p4slips') pigeon_key_file='pigeon.keys' rename_redis_ip_info=False rename_sql_db_file=False @@ -122,7 +125,8 @@ def init(self, *args, **kwargs): if self.rename_redis_ip_info: self.storage_name += str(self.port) self.c1 = self.db.subscribe('report_to_peers') - # channel to send msgs to whenever slips needs info from other peers about an ip + # channel to send msgs to whenever slips needs + # info from other peers about an ip self.c2 = self.db.subscribe(self.p2p_data_request_channel) # this channel receives peers requests/updates self.c3 = self.db.subscribe(self.gopy_channel) @@ -200,8 +204,10 @@ def _configure(self): self.sql_db_name, drop_tables_on_startup=True ) - self.reputation_model = reputation_model.BaseModel(self.logger, self.trust_db) - # print(f"[DEBUGGING] Starting godirector with pygo_channel: {self.pygo_channel}") + self.reputation_model = reputation_model.BaseModel( + self.logger, self.trust_db) + # print(f"[DEBUGGING] Starting godirector with + # pygo_channel: {self.pygo_channel}") self.go_director = GoDirector( self.logger, self.trust_db, @@ -511,16 +517,20 @@ def handle_data_request(self, message_data: str) -> None: combined_confidence, ) = self.reputation_model.get_opinion_on_ip(ip_address) - # no data in db - this happens when testing, if there is not enough data on peers + # no data in db - this happens when testing, + # if there is not enough data on peers if combined_score is None: self.print( - f'No data received from the network about {ip_address}\n', 0, 2 + f'No data received from the' + f' network about {ip_address}\n', 0, 2 ) - # print(f"[DEBUGGING] No data received from the network about {ip_address}\n") + # print(f"[DEBUGGING] No data received + # from the network about {ip_address}\n") else: self.print( f'The Network shared some data about {ip_address}, ' - f'Shared data: score={combined_score}, confidence={combined_confidence} saving it to now!\n', + f'Shared data: score={combined_score}, ' + f'confidence={combined_confidence} saving it to now!\n', 0, 2, ) @@ -540,7 +550,8 @@ def handle_data_request(self, message_data: str) -> None: ) def respond_to_message_request(self, key, reporter): - # todo do you mean another peer is asking me about an ip? yes. in override mode + # todo do you mean another peer is asking me about + # an ip? yes. in override mode """ Handle data request from a peer (in overriding p2p mode) (set to false by defualt) :param key: The ip requested by the peer @@ -578,7 +589,8 @@ def pre_main(self): # check if it was possible to start up pigeon if self.start_pigeon and self.pigeon is None: self.print( - 'Module was supposed to start up pigeon but it was not possible to start pigeon! Exiting...' + 'Module was supposed to start up pigeon but it was not' + ' possible to start pigeon! Exiting...' ) return 1 @@ -604,7 +616,8 @@ def main(self): ret_code = self.pigeon.poll() if ret_code is not None: self.print( - f'Pigeon process suddenly terminated with return code {ret_code}. Stopping module.' + f'Pigeon process suddenly terminated with ' + f'return code {ret_code}. Stopping module.' ) return 1 diff --git a/modules/p2ptrust/utils/go_director.py b/modules/p2ptrust/utils/go_director.py index f15a8703e..9a3ec538c 100644 --- a/modules/p2ptrust/utils/go_director.py +++ b/modules/p2ptrust/utils/go_director.py @@ -488,11 +488,6 @@ def set_evidence_p2p_report( set evidence for the newly created attacker profile stating that it attacked another peer """ - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=ip - ) threat_level = utils.threat_level_to_string(score) # confidence depends on how long the connection @@ -521,7 +516,11 @@ def set_evidence_p2p_report( timestamp = utils.convert_format(timestamp, utils.alerts_format) evidence = Evidence( evidence_type=EvidenceType.P2P_REPORT, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=ip + ), threat_level=threat_level, confidence=confidence, description=description, diff --git a/modules/rnn_cc_detection/rnn_cc_detection.py b/modules/rnn_cc_detection/rnn_cc_detection.py index bebaa6e8f..e6f28f915 100644 --- a/modules/rnn_cc_detection/rnn_cc_detection.py +++ b/modules/rnn_cc_detection/rnn_cc_detection.py @@ -56,14 +56,6 @@ def set_evidence_cc_channel( tupleid = tupleid.split('-') dstip, port, proto = tupleid[0], tupleid[1], tupleid[2] srcip = profileid.split("_")[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) - - threat_level: ThreatLevel = ThreatLevel.HIGH portproto: str = f'{port}/{proto}' port_info: str = self.db.get_port_info(portproto) ip_identification: str = self.db.get_ip_identification(dstip) @@ -74,14 +66,19 @@ def set_evidence_cc_channel( ) timestamp: str = utils.convert_format(timestamp, utils.alerts_format) + twid_int = int(twid.replace("timewindow", "")) evidence: Evidence = Evidence( evidence_type=EvidenceType.COMMAND_AND_CONTROL_CHANNEL, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), + threat_level=ThreatLevel.HIGH, confidence=confidence, description=description, profile=ProfileID(ip=srcip), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_int), uid=[uid], timestamp=timestamp, category=IDEACategory.INTRUSION_BOTNET, @@ -89,7 +86,28 @@ def set_evidence_cc_channel( port=int(port), proto=Proto(proto.lower()) if proto else None, ) + self.db.set_evidence(evidence) + evidence: Evidence = Evidence( + evidence_type=EvidenceType.COMMAND_AND_CONTROL_CHANNEL, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=dstip + ), + threat_level=ThreatLevel.HIGH, + confidence=confidence, + description=description, + profile=ProfileID(ip=dstip), + timewindow=TimeWindow(number=twid_int), + uid=[uid], + timestamp=timestamp, + category=IDEACategory.INTRUSION_BOTNET, + source_target_tag=Tag.CC, + port=int(port), + proto=Proto(proto.lower()) if proto else None, + ) + self.db.set_evidence(evidence) diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index 67a1bb2da..1782620b7 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -192,7 +192,7 @@ def set_evidence_malicious_asn( self.db.set_evidence(evidence) - def set_evidence_malicious_dns_response( + def set_evidence_malicious_ip_in_dns_response( self, ip: str, uid: str, @@ -371,21 +371,6 @@ def set_evidence_malicious_ip( # add this ip to our MaliciousIPs hash in the database self.db.set_malicious_ip(ip, profileid, twid) - - def set_evidence_malicious_dns_query(self, - CNAME: str, - domain, - uid: str, - timestamp: str, - domain_info: dict, - is_subdomain: bool, - profileid: str = '', - twid: str = ''): - description: str = (f'DNS answer with a blacklisted ' - f'CNAME: {domain} ' - f'for query: {query} ') - - def set_evidence_malicious_domain( self, @@ -427,21 +412,19 @@ def set_evidence_malicious_domain( tags = domain_info.get('tags', None) if tags: description += f'with tags: {tags}. ' - - attacker = Attacker( + twid_number = int(twid.replace("timewindow", "")) + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN, + attacker=Attacker( direction=Direction.SRC, attacker_type=IoCType.IP, value=srcip - ) - - evidence = Evidence( - evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN, - attacker=attacker, + ), threat_level=threat_level, confidence=confidence, description=description, profile=ProfileID(ip=srcip), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_number), uid=[uid], timestamp=utils.convert_format(timestamp, utils.alerts_format), category=IDEACategory.ANOMALY_TRAFFIC, @@ -450,12 +433,35 @@ def set_evidence_malicious_domain( self.db.set_evidence(evidence) + domain_resolution: str = self.db.get_domain_resolution(domain) + if domain_resolution: + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.DOMAIN, + value=domain + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=domain_resolution), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_DOMAIN, + ) + + self.db.set_evidence(evidence) + def is_valid_threat_level(self, threat_level): return threat_level in utils.threat_levels def parse_local_ti_file(self, ti_file_path: str) -> bool: """ - Read all the files holding IP addresses and a description and store in the db. + Read all the files holding IP addresses and a description + and store in the db. This also helps in having unique ioc across files Returns nothing, but the dictionary should be filled :param ti_file_path: full path_to local threat intel file @@ -849,11 +855,6 @@ def set_evidence_malicious_hash(self, file_info: Dict[str, any]): f'Detected by: {file_info["blacklist"]}. ' f'Score: {confidence}. {ip_identification}' ) - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) ts = utils.convert_format( file_info['flow']["starttime"], utils.alerts_format ) @@ -862,7 +863,11 @@ def set_evidence_malicious_hash(self, file_info: Dict[str, any]): )) evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_DOWNLOADED_FILE, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), threat_level=threat_level, confidence=confidence, description=description, @@ -874,6 +879,25 @@ def set_evidence_malicious_hash(self, file_info: Dict[str, any]): ) self.db.set_evidence(evidence) + + evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_DOWNLOADED_FILE, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=srcip), + timewindow=twid, + uid=[file_info['flow']["uid"]], + timestamp=ts, + category=IDEACategory.MALWARE + ) + + self.db.set_evidence(evidence) def circl_lu(self, flow_info: dict): """ @@ -1069,7 +1093,7 @@ def is_malicious_ip(self, ip: json.dumps(ip_info) }) if is_dns_response: - self.set_evidence_malicious_dns_response( + self.set_evidence_malicious_ip_in_dns_response( ip, uid, timestamp, @@ -1097,9 +1121,12 @@ def is_malicious_hash(self, flow_info: dict): """ if not flow_info['flow']['md5']: # some lines in the zeek files.log doesn't have a hash for example - # {"ts":293.713187,"fuid":"FpvjEj3U0Qoj1fVCQc","tx_hosts":["94.127.78.125"],"rx_hosts":["10.0.2.19"], - # "conn_uids":["CY7bgw3KI8QyV67jqa","CZEkWx4wAvHJv0HTw9","CmM1ggccDvwnwPCl3","CBwoAH2RcIueFH4eu9","CZVfkc4BGLqRR7wwD5"], - # "source":"HTTP","depth":0,"analyzers":["SHA1","SHA256","MD5"] .. } + # {"ts":293.713187,"fuid":"FpvjEj3U0Qoj1fVCQc", + # "tx_hosts":["94.127.78.125"],"rx_hosts":["10.0.2.19"], + # "conn_uids":["CY7bgw3KI8QyV67jqa","CZEkWx4wAvHJv0HTw9", + # "CmM1ggccDvwnwPCl3","CBwoAH2RcIueFH4eu9","CZVfkc4BGLqRR7wwD5"], + # "source":"HTTP","depth":0,"analyzers":["SHA1","SHA256","MD5"] + # .. } return if blacklist_details := self.search_online_for_hash(flow_info): @@ -1119,6 +1146,7 @@ def is_malicious_url( url, uid, timestamp, + daddr, profileid, twid ): @@ -1128,13 +1156,79 @@ def is_malicious_url( # not malicious return False - self.set_evidence_malicious_url( + self.urlhaus.set_evidence_malicious_url( + daddr, url_info, uid, timestamp, profileid, twid ) + + def set_evidence_malicious_cname_in_dns_response(self, + cname: str, + dns_query: str, + uid: str, + timestamp: str, + cname_info: dict, + is_subdomain: bool, + profileid: str = '', + twid: str = '' + ): + """ + :param cname: the dns answer that we looked up and turned out to be + malicious + :param dns_query: the query we asked the DNS server for when the + server returned the given cname + """ + if not cname_info: + return + + srcip = profileid.split("_")[-1] + # in case of finding a subdomain in our blacklists + # print that in the description of the alert and change the + # confidence accordingly in case of a domain, confidence=1 + confidence: float = 0.7 if is_subdomain else 1 + + # when we comment ti_files and run slips, we + # get the error of not being able to get feed threat_level + threat_level: float = utils.threat_levels[ + cname_info.get('threat_level', 'high') + ] + threat_level: ThreatLevel = ThreatLevel(threat_level) + description: str = (f'blacklisted CNAME: {cname} when resolving ' + f'{dns_query}' + f'Description: {cname_info.get("description", "")},' + f'Found in feed: {cname_info["source"]}, ' + f'Confidence: {confidence}. ') + + tags = cname_info.get('tags', None) + if tags: + description += f'with tags: {tags}. ' + + attacker = Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ) + + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_DNS_ANSWER, + attacker=attacker, + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=srcip), + timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_DOMAIN, + ) + + self.db.set_evidence(evidence) + + def is_malicious_cname(self, dns_query, @@ -1145,9 +1239,10 @@ def is_malicious_cname(self, twid, ): """ + looks up the given CNAME :param cname: is the dns answer we're looking up :param dns_query: the query we asked the DNS server for when the - server returned the given sname + server returned the given cname """ if self.is_ignored_domain(cname): @@ -1157,7 +1252,8 @@ def is_malicious_cname(self, if not domain_info: return False - self.set_evidence_malicious_dns_query( + self.set_evidence_malicious_cname_in_dns_response( + cname, dns_query, uid, timestamp, @@ -1214,7 +1310,8 @@ def is_malicious_domain( def update_local_file(self, filename): """ Updates the given local ti file if the hash of it has changed - : param filename: local ti file, has to be plased in config/local_ti_files/ dir + : param filename: local ti file, has to be plased in + config/local_ti_files/ dir """ fullpath = os.path.join(self.path_to_local_ti_files, filename) if filehash := self.should_update_local_ti_file(fullpath): @@ -1318,11 +1415,12 @@ def main(self): to_lookup, uid, timestamp, + daddr, profileid, twid ) - if msg:= self.get_msg('new_downloaded_file'): + if msg := self.get_msg('new_downloaded_file'): file_info: dict = json.loads(msg['data']) # the format of file_info is as follows # { diff --git a/modules/threat_intelligence/urlhaus.py b/modules/threat_intelligence/urlhaus.py index 9fe65cf1a..89c35076d 100644 --- a/modules/threat_intelligence/urlhaus.py +++ b/modules/threat_intelligence/urlhaus.py @@ -80,7 +80,8 @@ def parse_urlhaus_url_response(self, response, url): threat_level = virustotal_percent # virustotal_result = virustotal_info.get("result", "") # virustotal_result.replace('\',''') - description += f'and was marked by {virustotal_percent}% of virustotal\'s AVs as malicious' + description += (f'and was marked by {virustotal_percent}% ' + f'of virustotal\'s AVs as malicious') except (KeyError, IndexError): # no payloads available @@ -148,7 +149,6 @@ def urlhaus_lookup(self, ioc, type_of_ioc: str): return self.parse_urlhaus_url_response(response, ioc) def set_evidence_malicious_hash(self, file_info: Dict[str, Any]) -> None: - flow: Dict[str, Any] = file_info['flow'] daddr: str = flow["daddr"] @@ -177,47 +177,77 @@ def set_evidence_malicious_hash(self, file_info: Dict[str, Any]) -> None: f" by URLhaus." ) - threat_level: float = file_info.get("threat_level", 0) + threat_level: float = file_info.get("threat_level") if threat_level: # Threat level here is the VT percentage from URLhaus description += f" Virustotal score: {threat_level}% malicious" threat_level: str = utils.threat_level_to_string(float( threat_level) / 100) + threat_level: ThreatLevel = ThreatLevel[threat_level.upper()] else: - threat_level = 'high' - - threat_level: ThreatLevel= ThreatLevel[threat_level.upper()] + threat_level: ThreatLevel = ThreatLevel.HIGH confidence: float = 0.7 saddr: str = file_info['profileid'].split("_")[-1] - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) timestamp: str = flow["starttime"] - twid: str = file_info["twid"] - - # Assuming you have an instance of the Evidence class in your class + twid_int = int(file_info["twid"].replace("timewindow", "")) evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_DOWNLOADED_FILE, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), threat_level=threat_level, confidence=confidence, description=description, timestamp=timestamp, category=IDEACategory.MALWARE, profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_int), uid=[flow["uid"]] ) self.db.set_evidence(evidence) - + + evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_DOWNLOADED_FILE, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=threat_level, + confidence=confidence, + description=description, + timestamp=timestamp, + category=IDEACategory.MALWARE, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_int), + uid=[flow["uid"]] + ) + + self.db.set_evidence(evidence) + + def get_threat_level(self, url_info: dict) -> ThreatLevel: + threat_level = url_info.get('threat_level', '') + if not threat_level: + return ThreatLevel.MEDIUM + + # Convert percentage reported by URLhaus (VirusTotal) to + # a valid SLIPS confidence + try: + threat_level = int(threat_level) / 100 + threat_level: str = utils.threat_level_to_string(threat_level) + return ThreatLevel[threat_level.upper()] + except ValueError: + return ThreatLevel.MEDIUM + def set_evidence_malicious_url( self, + daddr: str, url_info: Dict[str, Any], uid: str, timestamp: str, @@ -227,42 +257,42 @@ def set_evidence_malicious_url( """ Set evidence for a malicious URL based on the provided URL info """ - threat_level: str = url_info.get('threat_level', '') + threat_level: ThreatLevel = self.get_threat_level(url_info) description: str = url_info.get('description', '') - - confidence: float = 0.7 - - if not threat_level: - threat_level = 'medium' - else: - # Convert percentage reported by URLhaus (VirusTotal) to - # a valid SLIPS confidence - try: - threat_level = int(threat_level) / 100 - threat_level = utils.threat_level_to_string(threat_level) - except ValueError: - threat_level = 'medium' - - threat_level: ThreatLevel = ThreatLevel[threat_level.upper()] saddr: str = profileid.split("_")[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr + twid_int = int(twid.replace("timewindow", "")) + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_MALICIOUS_URL, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=threat_level, + confidence=0.7, + description=description, + timestamp=timestamp, + category=IDEACategory.MALWARE, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_int), + uid=[uid] ) + self.db.set_evidence(evidence) - # Assuming you have an instance of the Evidence class in your class evidence = Evidence( - evidence_type=EvidenceType.MALICIOUS_URL, - attacker=attacker, + evidence_type=EvidenceType.THREAT_INTELLIGENCE_MALICIOUS_URL, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, - confidence=confidence, + confidence=0.7, description=description, timestamp=timestamp, category=IDEACategory.MALWARE, profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_int), uid=[uid] ) diff --git a/slips_files/core/evidence_structure/evidence.py b/slips_files/core/evidence_structure/evidence.py index d68b7b7f5..92db93f67 100644 --- a/slips_files/core/evidence_structure/evidence.py +++ b/slips_files/core/evidence_structure/evidence.py @@ -86,7 +86,7 @@ class EvidenceType(Enum): THREAT_INTELLIGENCE_BLACKLISTED_DNS_ANSWER = auto() THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN = auto() MALICIOUS_DOWNLOADED_FILE = auto() - MALICIOUS_URL = auto() + THREAT_INTELLIGENCE_MALICIOUS_URL = auto() def __str__(self): return self.name From 1f96b6d477c1f030f5cb0dfd60845f6c27f7cedc Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 23 Feb 2024 14:09:05 +0200 Subject: [PATCH 24/33] update unit tests --- modules/flowalerts/set_evidence.py | 78 +++++++++---------- modules/http_analyzer/http_analyzer.py | 7 +- .../core/database/redis_db/profile_handler.py | 1 - tests/module_factory.py | 2 +- tests/test_flowalerts.py | 20 ++++- tests/test_profiler.py | 7 +- 6 files changed, 61 insertions(+), 54 deletions(-) diff --git a/modules/flowalerts/set_evidence.py b/modules/flowalerts/set_evidence.py index 6488bbb8f..3235ea157 100644 --- a/modules/flowalerts/set_evidence.py +++ b/modules/flowalerts/set_evidence.py @@ -53,7 +53,7 @@ def young_domain( threat_level=ThreatLevel.LOW, category=IDEACategory.ANOMALY_TRAFFIC, description=description, - profile=ProfileID(ip=attacker.value), + profile=ProfileID(ip=attacker), timewindow=TimeWindow(number=twid_number), uid=[uid], timestamp=stime, @@ -1292,18 +1292,17 @@ def malicious_ja3s( ) self.db.set_evidence(evidence) - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) evidence: Evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_JA3S, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), threat_level=ThreatLevel.LOW, confidence=confidence, description=description, - profile=ProfileID(ip=attacker.value), + profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=twid_number), uid=[uid], timestamp=timestamp, @@ -1339,25 +1338,22 @@ def malicious_ja3( description += f'description: {ja3_description} ' description += f'tags: {tags}' - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim: Victim = Victim( - direction= Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - confidence: float = 1 evidence: Evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_JA3, - attacker=attacker, - victim=victim, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction= Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, - confidence=confidence, + confidence=1, description=description, - profile=ProfileID(ip=attacker.value), + profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), uid=[uid], timestamp=timestamp, @@ -1376,26 +1372,25 @@ def data_exfiltration( uid: List[str], timestamp ) -> None: - confidence: float = 0.6 saddr: str = profileid.split("_")[-1] - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) ip_identification: str = self.db.get_ip_identification(daddr) description: str = f'Large data upload. {src_mbs} MBs ' \ f'sent to {daddr} {ip_identification}' timestamp: str = utils.convert_format(timestamp, utils.alerts_format) - + twid_number = int(twid.replace("timewindow", "")) + evidence: Evidence = Evidence( evidence_type=EvidenceType.DATA_UPLOAD, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), threat_level=ThreatLevel.INFO, confidence=confidence, description=description, - profile=ProfileID(ip=attacker.value), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_number), uid=uid, timestamp=timestamp, category=IDEACategory.MALWARE, @@ -1404,19 +1399,18 @@ def data_exfiltration( self.db.set_evidence(evidence) - attacker: Attacker = Attacker( - direction=Direction.DST, - attacker_type=IoCType.IP, - value=daddr - ) evidence: Evidence = Evidence( evidence_type=EvidenceType.DATA_UPLOAD, - attacker=attacker, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), threat_level=ThreatLevel.HIGH, - confidence=confidence, + confidence=0.6, description=description, - profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_number), uid=uid, timestamp=timestamp, category=IDEACategory.MALWARE, diff --git a/modules/http_analyzer/http_analyzer.py b/modules/http_analyzer/http_analyzer.py index c1ab18c36..677a669ba 100644 --- a/modules/http_analyzer/http_analyzer.py +++ b/modules/http_analyzer/http_analyzer.py @@ -1,9 +1,10 @@ import json import urllib import requests -from typing import Union, \ +from typing import ( + Union, Dict - + ) from slips_files.common.imports import * from slips_files.core.evidence_structure.evidence import \ ( @@ -689,7 +690,7 @@ def set_evidence_weird_http_method( self.db.set_evidence(evidence) - def check_weird_http_method(self, msg: Dict[str]): + def check_weird_http_method(self, msg: Dict[str, str]): """ detect weird http methods in zeek's weird.log """ diff --git a/slips_files/core/database/redis_db/profile_handler.py b/slips_files/core/database/redis_db/profile_handler.py index ffc1a8f7c..53acd7939 100644 --- a/slips_files/core/database/redis_db/profile_handler.py +++ b/slips_files/core/database/redis_db/profile_handler.py @@ -1125,7 +1125,6 @@ def getT2ForProfileTW(self, profileid, twid, tupleid, tuple_key: str): ) self.print(type(e), 0, 1) self.print(e, 0, 1) - self.print(traceback.print_stack(), 0, 1) def has_profile(self, profileid): """Check if we have the given profile""" diff --git a/tests/module_factory.py b/tests/module_factory.py index dcbb5e0a7..815b8950b 100644 --- a/tests/module_factory.py +++ b/tests/module_factory.py @@ -55,7 +55,7 @@ def __init__(self): self.profiler_queue = Queue() self.input_queue = Queue() self.dummy_termination_event = Event() - self.logger = Output() + self.logger = Mock() # Output() def get_default_db(self): diff --git a/tests/test_flowalerts.py b/tests/test_flowalerts.py index 977fcd870..c6268dc50 100644 --- a/tests/test_flowalerts.py +++ b/tests/test_flowalerts.py @@ -1,5 +1,4 @@ """Unit test for modules/flowalerts/flowalerts.py""" -from slips_files.core.flows.zeek import Conn from tests.module_factory import ModuleFactory import json from numpy import arange @@ -161,15 +160,30 @@ def test_detect_young_domains( ): flowalerts = ModuleFactory().create_flowalerts_obj(mock_db) domain = 'example.com' + answers = ['192.168.1.1', '192.168.1.2', '192.168.1.3', 'CNAME_HERE.com'] # age in days mock_db.getDomainData.return_value = {'Age': 50} assert ( - flowalerts.detect_young_domains(domain, timestamp, profileid, twid, uid) is True + flowalerts.detect_young_domains( + domain, + answers, + timestamp, + profileid, + twid, + uid + ) is True ) # more than the age threshold mock_db.getDomainData.return_value = {'Age': 1000} assert ( - flowalerts.detect_young_domains(domain, timestamp, profileid, twid, uid) is False + flowalerts.detect_young_domains( + domain, + answers, + timestamp, + profileid, + twid, + uid + ) is False ) diff --git a/tests/test_profiler.py b/tests/test_profiler.py index 92f121a82..dae14a012 100644 --- a/tests/test_profiler.py +++ b/tests/test_profiler.py @@ -1,5 +1,5 @@ """Unit test for slips_files/core/performance_profiler.py""" -import ipaddress +from unittest.mock import Mock from tests.module_factory import ModuleFactory from tests.common_test_utils import do_nothing @@ -154,6 +154,7 @@ def test_define_separator_nfdump(nfdump_file, ) def test_process_line(file, flow_type, mock_db): profiler = ModuleFactory().create_profiler_obj(mock_db) + profiler.symbol = Mock() # we're testing another functionality here profiler.whitelist.is_whitelisted_flow = do_nothing profiler.input_type = 'zeek' @@ -172,11 +173,9 @@ def test_process_line(file, flow_type, mock_db): 'type': flow_type } - # process it profiler.flow = profiler.input_handler.process_line(sample_flow) - assert profiler.flow + assert profiler.flow is not None - # add to profile added_to_prof = profiler.add_flow_to_profile() assert added_to_prof is True From 1f9af7a2ce08aa9a18dd9485d75e014940d8006a Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 23 Feb 2024 14:55:42 +0200 Subject: [PATCH 25/33] db: use snake case for domain functions --- modules/flowalerts/flowalerts.py | 36 ++++++++++++------- modules/ip_info/ip_info.py | 5 ++- .../threat_intelligence.py | 4 +-- modules/virustotal/virustotal.py | 6 ++-- slips_files/core/database/database_manager.py | 8 ++--- .../core/database/redis_db/database.py | 2 +- .../core/database/redis_db/ioc_handler.py | 10 +++--- tests/test_database.py | 4 +-- tests/test_flowalerts.py | 4 +-- 9 files changed, 45 insertions(+), 34 deletions(-) diff --git a/modules/flowalerts/flowalerts.py b/modules/flowalerts/flowalerts.py index 2147b5225..4ad92de3f 100644 --- a/modules/flowalerts/flowalerts.py +++ b/modules/flowalerts/flowalerts.py @@ -779,7 +779,7 @@ def check_dns_without_connection( # with AAAA, and the computer chooses the A address. # Therefore, the 2nd DNS resolution # would be treated as 'without connection', but this is false. - if prev_domain_resolutions := self.db.getDomainData(domain): + if prev_domain_resolutions := self.db.get_domain_data(domain): prev_domain_resolutions = prev_domain_resolutions.get('IPs',[]) # if there's a domain in the cache # (prev_domain_resolutions) that is not in the @@ -1268,24 +1268,37 @@ def check_multiple_reconnection_attempts( self.db.setReconnections( profileid, twid, current_reconnections ) - - def detect_young_domains(self, domain, answers: List[str], stime, - profileid, - twid, uid): + + def should_detect_young_domain(self, domain): + """ + returns true if it's ok to detect young domains for the given + domain + """ + return ( + domain + and not domain.endswith(".local") + and not domain.endswith('.arpa') + ) + + def detect_young_domains( + self, + domain, + answers: List[str], + stime, + profileid, + twid, + uid + ): """ Detect domains that are too young. The threshold is 60 days """ - if not domain: + if not self.should_detect_young_domain(domain): return False age_threshold = 60 - # Ignore arpa and local domains - if domain.endswith('.arpa') or domain.endswith('.local'): - return False - - domain_info: dict = self.db.getDomainData(domain) + domain_info: dict = self.db.get_domain_data(domain) if not domain_info: return False @@ -1302,7 +1315,6 @@ def detect_young_domains(self, domain, answers: List[str], stime, ips_returned_in_answer: List[str] = ( self.extract_ips_from_dns_answers(answers) ) - self.set_evidence.young_domain( domain, age, stime, profileid, twid, uid, ips_returned_in_answer ) diff --git a/modules/ip_info/ip_info.py b/modules/ip_info/ip_info.py index 8bc5ba536..411aba2b3 100644 --- a/modules/ip_info/ip_info.py +++ b/modules/ip_info/ip_info.py @@ -358,7 +358,7 @@ def get_age(self, domain): # tld not supported return False - cached_data = self.db.getDomainData(domain) + cached_data = self.db.get_domain_data(domain) if cached_data and 'Age' in cached_data: # we already have age info about this domain return False @@ -385,8 +385,7 @@ def get_age(self, domain): today, return_type='days' ) - - self.db.setInfoForDomains(domain, {'Age': age}) + self.db.set_info_for_domains(domain, { 'Age': age}) return age def shutdown_gracefully(self): diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index 1782620b7..44d38648a 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -1266,7 +1266,7 @@ def is_malicious_cname(self, domain_info = { 'threatintelligence': domain_info } - self.db.setInfoForDomains(cname, domain_info) + self.db.set_info_for_domains(cname, domain_info) # add this domain to our MaliciousDomains hash in the database self.db.set_malicious_domain(cname, profileid, twid) @@ -1301,7 +1301,7 @@ def is_malicious_domain( domain_info = { 'threatintelligence': domain_info } - self.db.setInfoForDomains(domain, domain_info) + self.db.set_info_for_domains(domain, domain_info) # add this domain to our MaliciousDomains hash in the database self.db.set_malicious_domain(domain, profileid, twid) diff --git a/modules/virustotal/virustotal.py b/modules/virustotal/virustotal.py index 466989e58..c3a3de89d 100644 --- a/modules/virustotal/virustotal.py +++ b/modules/virustotal/virustotal.py @@ -188,7 +188,7 @@ def set_domain_data_in_DomainInfo(self, domain, cached_data): data['asn'] = { 'number': f'AS{as_owner}' } - self.db.setInfoForDomains(domain, data) + self.db.set_info_for_domains(domain, data) def API_calls_thread(self): """ @@ -226,7 +226,7 @@ def API_calls_thread(self): self.set_vt_data_in_IPInfo(ioc, cached_data) elif ioc_type == 'domain': - cached_data = self.db.getDomainData(ioc) + cached_data = self.db.get_domain_data(ioc) if not cached_data or 'VirusTotal' not in cached_data: self.set_domain_data_in_DomainInfo(ioc, cached_data) @@ -601,7 +601,7 @@ def main(self): ) # this is a dict {'uid':json flow data} domain = flow_data.get('query', False) - cached_data = self.db.getDomainData(domain) + cached_data = self.db.get_domain_data(domain) # If VT data of this domain is not in the DomainInfo, ask VT # If 'Virustotal' key is not in the DomainInfo if domain and ( diff --git a/slips_files/core/database/database_manager.py b/slips_files/core/database/database_manager.py index 9689c2cfa..4f3b2e5a0 100644 --- a/slips_files/core/database/database_manager.py +++ b/slips_files/core/database/database_manager.py @@ -534,14 +534,14 @@ def getURLData(self, *args, **kwargs): def setNewURL(self, *args, **kwargs): return self.rdb.setNewURL(*args, **kwargs) - def getDomainData(self, *args, **kwargs): - return self.rdb.getDomainData(*args, **kwargs) + def get_domain_data(self, *args, **kwargs): + return self.rdb.get_domain_data(*args, **kwargs) def setNewDomain(self, *args, **kwargs): return self.rdb.setNewDomain(*args, **kwargs) - def setInfoForDomains(self, *args, **kwargs): - return self.rdb.setInfoForDomains(*args, **kwargs) + def set_info_for_domains(self, *args, **kwargs): + return self.rdb.set_info_for_domains(*args, **kwargs) def setInfoForURLs(self, *args, **kwargs): return self.rdb.setInfoForURLs(*args, **kwargs) diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index 59e0f20e1..3b9515feb 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -799,7 +799,7 @@ def set_dns_resolution( # no CNAME came with this query pass - self.setInfoForDomains(query, domaindata, mode='add') + self.set_info_for_domains(query, domaindata, mode='add') self.set_domain_resolution(query, ips_to_add) def set_domain_resolution(self, domain, ips): diff --git a/slips_files/core/database/redis_db/ioc_handler.py b/slips_files/core/database/redis_db/ioc_handler.py index 7c17b3480..6c3ce7df0 100644 --- a/slips_files/core/database/redis_db/ioc_handler.py +++ b/slips_files/core/database/redis_db/ioc_handler.py @@ -348,7 +348,7 @@ def setNewURL(self, url: str): # We use the empty dictionary to find if an URL exists or not self.rcache.hset('URLsInfo', url, '{}') - def getDomainData(self, domain): + def get_domain_data(self, domain): """ Return information about this domain Returns a dictionary or False if there is no domain in the database @@ -367,7 +367,7 @@ def setNewDomain(self, domain: str): 2- Publishes in the channels that there is a new domain, and that we want data from the Threat Intelligence modules """ - data = self.getDomainData(domain) + data = self.get_domain_data(domain) if data is False: # If there is no data about this domain # Set this domain for the first time in the DomainsInfo @@ -376,7 +376,7 @@ def setNewDomain(self, domain: str): # We use the empty dictionary to find if a domain exists or not self.rcache.hset('DomainsInfo', domain, '{}') - def setInfoForDomains(self, domain: str, info_to_set: dict, mode='leave'): + def set_info_for_domains(self, domain: str, info_to_set: dict, mode= 'leave'): """ Store information for this domain :param info_to_set: a dictionary, such as {'geocountry': 'rumania'} that we are @@ -388,12 +388,12 @@ def setInfoForDomains(self, domain: str, info_to_set: dict, mode='leave'): """ # Get the previous info already stored - domain_data = self.getDomainData(domain) + domain_data = self.get_domain_data(domain) if not domain_data: # This domain is not in the dictionary, add it first: self.setNewDomain(domain) # Now get the data, which should be empty, but just in case - domain_data = self.getDomainData(domain) + domain_data = self.get_domain_data(domain) # Let's check each key stored for this domain for key in iter(info_to_set): diff --git a/tests/test_database.py b/tests/test_database.py index 8cce74288..f94762ad0 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -149,9 +149,9 @@ def test_setInfoForDomains(): """ tests setInfoForDomains, setNewDomain and getDomainData """ domain = 'www.google.com' domain_data = {'threatintelligence': 'sample data'} - db.setInfoForDomains(domain, domain_data) + db.set_info_for_domains(domain, domain_data) - stored_data = db.getDomainData(domain) + stored_data = db.get_domain_data(domain) assert 'threatintelligence' in stored_data assert stored_data['threatintelligence'] == 'sample data' diff --git a/tests/test_flowalerts.py b/tests/test_flowalerts.py index c6268dc50..d9dd59187 100644 --- a/tests/test_flowalerts.py +++ b/tests/test_flowalerts.py @@ -163,7 +163,7 @@ def test_detect_young_domains( answers = ['192.168.1.1', '192.168.1.2', '192.168.1.3', 'CNAME_HERE.com'] # age in days - mock_db.getDomainData.return_value = {'Age': 50} + mock_db.get_domain_data.return_value = { 'Age': 50} assert ( flowalerts.detect_young_domains( domain, @@ -176,7 +176,7 @@ def test_detect_young_domains( ) # more than the age threshold - mock_db.getDomainData.return_value = {'Age': 1000} + mock_db.get_domain_data.return_value = { 'Age': 1000} assert ( flowalerts.detect_young_domains( domain, From ec07f269284300783e888950f52e22fc3e7a0505 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 23 Feb 2024 15:36:55 +0200 Subject: [PATCH 26/33] fix problem printing traceback on exceptions --- managers/process_manager.py | 2 +- modules/flowalerts/flowalerts.py | 4 +- modules/flowalerts/set_evidence.py | 4 +- modules/flowmldetection/flowmldetection.py | 12 ++-- modules/timeline/timeline.py | 2 +- modules/update_manager/update_manager.py | 8 +-- modules/virustotal/virustotal.py | 4 +- slips_files/common/abstracts/_module.py | 10 +-- slips_files/common/abstracts/core.py | 6 +- slips_files/common/idea_format.py | 2 +- .../core/database/redis_db/profile_handler.py | 11 ++-- slips_files/core/evidencehandler.py | 4 +- slips_files/core/helpers/symbols_handler.py | 2 +- slips_files/core/input_profilers/argus.py | 2 +- slips_files/core/output.py | 62 +++++++++++-------- tests/common_test_utils.py | 10 +-- 16 files changed, 75 insertions(+), 70 deletions(-) diff --git a/managers/process_manager.py b/managers/process_manager.py index b36b0b6f0..59ff87855 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -291,7 +291,7 @@ def get_modules(self, to_ignore: list): except ImportError as e: print(f"Something wrong happened while " f"importing the module {module_name}: {e}") - print(traceback.print_stack()) + print(traceback.format_exc()) failed_to_load_modules += 1 continue diff --git a/modules/flowalerts/flowalerts.py b/modules/flowalerts/flowalerts.py index 4ad92de3f..acd920ed5 100644 --- a/modules/flowalerts/flowalerts.py +++ b/modules/flowalerts/flowalerts.py @@ -1523,7 +1523,7 @@ def detect_malicious_ja3( malicious_ja3_dict = self.db.get_ja3_in_IoC() if ja3 in malicious_ja3_dict: - self.set_evidence.malicious_ja3s( + self.set_evidence.malicious_ja3( malicious_ja3_dict, twid, uid, @@ -1535,7 +1535,7 @@ def detect_malicious_ja3( if ja3s in malicious_ja3_dict: - self.set_evidence.malicious_ja3( + self.set_evidence.malicious_ja3s( malicious_ja3_dict, twid, uid, diff --git a/modules/flowalerts/set_evidence.py b/modules/flowalerts/set_evidence.py index 3235ea157..5e5fffdf6 100644 --- a/modules/flowalerts/set_evidence.py +++ b/modules/flowalerts/set_evidence.py @@ -1335,8 +1335,8 @@ def malicious_ja3( description = f'Malicious JA3: {ja3} from source address ' \ f'{saddr} {ip_identification}' if ja3_description != 'None': - description += f'description: {ja3_description} ' - description += f'tags: {tags}' + description += f' description: {ja3_description} ' + description += f' tags: {tags}' evidence: Evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_JA3, diff --git a/modules/flowmldetection/flowmldetection.py b/modules/flowmldetection/flowmldetection.py index 1f1d7ec3a..e0195dc09 100644 --- a/modules/flowmldetection/flowmldetection.py +++ b/modules/flowmldetection/flowmldetection.py @@ -95,7 +95,7 @@ def train(self): ) except Exception: self.print('Error while calling clf.train()') - self.print(traceback.print_stack()) + self.print(traceback.format_exc(), 0, 1) # See score so far in training score = self.clf.score(X_flow, y_flow) @@ -115,7 +115,7 @@ def train(self): except Exception: self.print('Error in train()', 0 , 1) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) def process_features(self, dataset): @@ -216,7 +216,7 @@ def process_features(self, dataset): except Exception: # Stop the timer self.print('Error in process_features()') - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) def process_flows(self): """ @@ -295,7 +295,7 @@ def process_flows(self): except Exception: # Stop the timer self.print('Error in process_flows()') - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) def process_flow(self): """ @@ -312,7 +312,7 @@ def process_flow(self): except Exception: # Stop the timer self.print('Error in process_flow()') - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) def detect(self): """ @@ -333,7 +333,7 @@ def detect(self): # Stop the timer self.print('Error in detect() X_flow:') self.print(X_flow) - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) def store_model(self): """ diff --git a/modules/timeline/timeline.py b/modules/timeline/timeline.py index 591cac9f9..e1d73b166 100644 --- a/modules/timeline/timeline.py +++ b/modules/timeline/timeline.py @@ -358,7 +358,7 @@ def process_flow(self, profileid, twid, flow, timestamp: float): self.print( f'Problem on process_flow() line {exception_line}', 0, 1 ) - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) return True def pre_main(self): diff --git a/modules/update_manager/update_manager.py b/modules/update_manager/update_manager.py index 52dcbc2e3..94d23ee87 100644 --- a/modules/update_manager/update_manager.py +++ b/modules/update_manager/update_manager.py @@ -426,7 +426,7 @@ def check_if_update(self, file_to_download: str, update_period) -> bool: except Exception: exception_line = sys.exc_info()[2].tb_lineno self.print(f"Problem on update_TI_file() line {exception_line}", 0, 1) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) return False def get_e_tag(self, response): @@ -635,7 +635,7 @@ async def update_TI_file(self, link_to_download: str) -> bool: except Exception: exception_line = sys.exc_info()[2].tb_lineno self.print(f"Problem on update_TI_file() line {exception_line}", 0, 1) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) return False def update_riskiq_feed(self): @@ -865,7 +865,7 @@ def parse_ja3_feed(self, url, ja3_feed_path: str) -> bool: except Exception: self.print("Problem in parse_ja3_feed()", 0, 1) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) return False def parse_json_ti_feed(self, link_to_download, ti_file_path: str) -> bool: @@ -1347,7 +1347,7 @@ def parse_ti_feed(self, link_to_download, ti_file_path: str) -> bool: 0, 1, ) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) return False def check_if_update_org(self, file): diff --git a/modules/virustotal/virustotal.py b/modules/virustotal/virustotal.py index c3a3de89d..9d058ff13 100644 --- a/modules/virustotal/virustotal.py +++ b/modules/virustotal/virustotal.py @@ -286,7 +286,7 @@ def get_ip_vt_data(self, ip: str): self.print( f'Problem in the get_ip_vt_data() line {exception_line}', 0, 1 ) - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) def get_domain_vt_data(self, domain: str): """ @@ -313,7 +313,7 @@ def get_domain_vt_data(self, domain: str): f'Problem in the get_domain_vt_data() ' f'line {exception_line}',0,1, ) - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) return False def get_ioc_type(self, ioc): diff --git a/slips_files/common/abstracts/_module.py b/slips_files/common/abstracts/_module.py index 1b0ba2d0c..bbc817386 100644 --- a/slips_files/common/abstracts/_module.py +++ b/slips_files/common/abstracts/_module.py @@ -134,7 +134,7 @@ def run(self): except Exception: exception_line = sys.exc_info()[2].tb_lineno self.print(f'Problem in pre_main() line {exception_line}', 0, 1) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) return True try: @@ -150,10 +150,6 @@ def run(self): except KeyboardInterrupt: self.shutdown_gracefully() except Exception: - exception_line = sys.exc_info()[2].tb_lineno - self.print(f'Problem in {self.name}\'s main() ' - f'line {exception_line}', - 0, 1) - traceback.print_stack() - + self.print(f'Problem in {self.name}',0, 1) + self.print(traceback.format_exc(), 0, 1) return True diff --git a/slips_files/common/abstracts/core.py b/slips_files/common/abstracts/core.py index 2d36f2418..31f324eed 100644 --- a/slips_files/common/abstracts/core.py +++ b/slips_files/common/abstracts/core.py @@ -56,9 +56,7 @@ def run(self): except KeyboardInterrupt: self.shutdown_gracefully() except Exception: - exception_line = sys.exc_info()[2].tb_lineno - self.print(f'Problem in main() line {exception_line}', 0, 1) - self.print(traceback.print_stack(), 0, 1) - + self.print(f'Problem in {self.name}',0, 1) + self.print(traceback.format_exc(), 0, 1) return True diff --git a/slips_files/common/idea_format.py b/slips_files/common/idea_format.py index c1d796af8..fa3087367 100644 --- a/slips_files/common/idea_format.py +++ b/slips_files/common/idea_format.py @@ -163,4 +163,4 @@ def idea_format(evidence: Evidence): return idea_dict except Exception as e: print(f"Error in idea_format(): {e}") - print(traceback.print_stack()) + print(traceback.format_exc()) diff --git a/slips_files/core/database/redis_db/profile_handler.py b/slips_files/core/database/redis_db/profile_handler.py index 53acd7939..d2857c3e4 100644 --- a/slips_files/core/database/redis_db/profile_handler.py +++ b/slips_files/core/database/redis_db/profile_handler.py @@ -505,7 +505,7 @@ def getFinalStateFromFlags(self, state, pkts): 0, 1, ) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) def get_data_from_profile_tw( self, @@ -558,7 +558,7 @@ def get_data_from_profile_tw( self.print( f"Error in getDataFromProfileTW database.py line {exception_line}", 0, 1 ) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) def update_ip_info( self, @@ -1215,7 +1215,7 @@ def add_new_older_tw(self, profileid: str, tw_start_time: float, tw_number: int) self.print("error in addNewOlderTW in database.py", 0, 1) self.print(type(e), 0, 1) self.print(e, 0, 1) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) def add_new_tw(self, profileid, timewindow: str, startoftw: float): """ @@ -1245,8 +1245,7 @@ def add_new_tw(self, profileid, timewindow: str, startoftw: float): self.update_threat_level(profileid, "info", 0.5) except redis.exceptions.ResponseError as e: self.print("Error in addNewTW", 0, 1) - self.print(traceback.print_stack(), 0, 1) - self.print(e, 0, 1) + self.print(traceback.format_exc(), 0, 1) def get_tw_start_time(self, profileid, twid): """Return the time when this TW in this profile was created""" @@ -1748,7 +1747,7 @@ def add_tuple( except Exception: exception_line = sys.exc_info()[2].tb_lineno self.print(f"Error in add_tuple in database.py line {exception_line}", 0, 1) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) def get_tws_to_search(self, go_back): tws_to_search = float("inf") diff --git a/slips_files/core/evidencehandler.py b/slips_files/core/evidencehandler.py index 7d50e7178..35858276a 100644 --- a/slips_files/core/evidencehandler.py +++ b/slips_files/core/evidencehandler.py @@ -199,7 +199,7 @@ def add_to_json_log_file( return True except Exception: self.print('Error in add_to_json_log_file()') - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) def add_to_log_file(self, data): """ @@ -215,7 +215,7 @@ def add_to_log_file(self, data): return True except Exception: self.print('Error in add_to_log_file()') - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) def get_domains_of_flow(self, flow: dict): """ diff --git a/slips_files/core/helpers/symbols_handler.py b/slips_files/core/helpers/symbols_handler.py index 0b5fadafb..134aa4492 100644 --- a/slips_files/core/helpers/symbols_handler.py +++ b/slips_files/core/helpers/symbols_handler.py @@ -290,4 +290,4 @@ def compute_timechar(): # For some reason we can not use the output queue here.. check self.print('Error in compute_symbol in Profiler Process.', 0, 1) - self.print(traceback.print_stack(), 0, 1) \ No newline at end of file + self.print(traceback.format_exc(), 0, 1) \ No newline at end of file diff --git a/slips_files/core/input_profilers/argus.py b/slips_files/core/input_profilers/argus.py index 9a2cff195..4ea97541a 100644 --- a/slips_files/core/input_profilers/argus.py +++ b/slips_files/core/input_profilers/argus.py @@ -133,5 +133,5 @@ def define_columns(self, new_line: dict) -> dict: self.print( f'\tProblem in define_columns() line {exception_line}', 0, 1 ) - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) sys.exit(1) diff --git a/slips_files/core/output.py b/slips_files/core/output.py index 9af1003c2..c2d1097a9 100644 --- a/slips_files/core/output.py +++ b/slips_files/core/output.py @@ -286,6 +286,24 @@ def tell_pbar(self, msg: dict): def is_pbar_finished(self )-> bool: return self.pbar_finished.is_set() + + def forward_progress_bar_msgs(self, msg: dict): + """ + passes init and update msgs to pbar module + """ + pbar_event: str = msg['bar'] + if pbar_event == 'init': + self.tell_pbar({ + 'event': pbar_event, + 'total_flows': msg['bar_info']['total_flows'], + }) + return + + if pbar_event == 'update' and not self.is_pbar_finished(): + self.tell_pbar({ + 'event': 'update_bar', + }) + def update(self, msg: dict): """ gets called whenever any module need to print something @@ -303,28 +321,22 @@ def update(self, msg: dict): total_flows: int, } """ - try: - if 'init' in msg.get('bar', ''): - self.tell_pbar({ - 'event': 'init', - 'total_flows': msg['bar_info']['total_flows'], - }) - - elif ( - 'update' in msg.get('bar', '') - and not self.is_pbar_finished() - ): - # if pbar wasn't supported, inputproc won't send update msgs - self.tell_pbar({ - 'event': 'update_bar', - }) - else: - # output to terminal and logs or logs only? - if msg.get('log_to_logfiles_only', False): - self.log_line(msg) - else: - # output to terminal - self.output_line(msg) - except Exception as e: - print(f"Error in output.py: {e}") - print(traceback.print_stack()) + # if pbar wasn't supported, inputproc won't send update msgs + + # try: + if 'bar' in msg: + self.forward_progress_bar_msgs(msg) + return + + # output to terminal and logs or logs only? + if msg.get('log_to_logfiles_only', False): + self.log_line(msg) + else: + # output to terminal + self.output_line(msg) + + +# except Exception as e: +# print(f"Error in output.py: {e} {type(e)}") +# traceback.print_stack() + diff --git a/tests/common_test_utils.py b/tests/common_test_utils.py index 632e27be3..a29cfbdde 100644 --- a/tests/common_test_utils.py +++ b/tests/common_test_utils.py @@ -71,19 +71,19 @@ def check_for_text(txt, output_dir): return False -def check_error_keywords(line): +def has_error_keywords(line): """ these keywords indicate that an error needs to be fixed and should fail the integration tests when found """ error_keywords = (' Date: Fri, 23 Feb 2024 15:37:09 +0200 Subject: [PATCH 27/33] db: fix problem getting get_all_flows_in_profileid --- slips_files/core/database/sqlite_db/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slips_files/core/database/sqlite_db/database.py b/slips_files/core/database/sqlite_db/database.py index 88b776f21..766843a4b 100644 --- a/slips_files/core/database/sqlite_db/database.py +++ b/slips_files/core/database/sqlite_db/database.py @@ -151,8 +151,8 @@ def get_all_flows_in_profileid(self, profileid) -> Dict[str, dict]: """ condition = f'profileid = "{profileid}"' flows = self.select('flows', condition=condition) + all_flows: Dict[str, dict] = {} if flows: - all_flows: Dict[str, dict] = {} for flow in flows: uid = flow[0] flow: str = flow[1] From ec3caa6f34e106815467f6d62daef830a8629039 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 23 Feb 2024 15:42:27 +0200 Subject: [PATCH 28/33] db: properly handle domain resolutions from the db --- modules/threat_intelligence/threat_intelligence.py | 7 ++++--- slips_files/core/database/redis_db/database.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index 44d38648a..e53921275 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -6,7 +6,8 @@ import requests import threading import time -from typing import Dict +from typing import Dict, \ + List from slips_files.common.slips_utils import utils from slips_files.common.imports import * @@ -432,9 +433,9 @@ def set_evidence_malicious_domain( ) self.db.set_evidence(evidence) - - domain_resolution: str = self.db.get_domain_resolution(domain) + domain_resolution: List[str] = self.db.get_domain_resolution(domain) if domain_resolution: + domain_resolution: str = domain_resolution[0] evidence = Evidence( evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN, attacker=Attacker( diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index 3b9515feb..f3dc8ab11 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -1080,7 +1080,7 @@ def set_default_gateway(self, address_type: str, address: str): self.r.hset('default_gateway', address_type, address) - def get_domain_resolution(self, domain): + def get_domain_resolution(self, domain) -> List[str]: """ Returns the IPs resolved by this domain """ From 90b99878dfe35f8442d37500607c133057eb533f Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 23 Feb 2024 18:48:06 +0200 Subject: [PATCH 29/33] start progress bar before all modules so it doesn't miss any prints in its queue and slips wouldn't seem like it's frozen --- managers/process_manager.py | 248 +++++++++--------- modules/progress_bar/progress_bar.py | 2 - modules/update_manager/update_manager.py | 16 +- slips/main.py | 28 +- .../core/database/redis_db/profile_handler.py | 2 +- slips_files/core/helpers/checker.py | 3 +- 6 files changed, 155 insertions(+), 144 deletions(-) diff --git a/managers/process_manager.py b/managers/process_manager.py index 59ff87855..05f96dcdd 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -14,18 +14,19 @@ Process, Semaphore, Pipe, - ) +) from typing import ( List, Tuple, - ) +) from exclusiveprocess import ( Lock, CannotAcquireLock, - ) +) import modules +from modules.progress_bar.progress_bar import PBar from modules.update_manager.update_manager import UpdateManager from slips_files.common.imports import * from slips_files.common.style import green @@ -38,7 +39,6 @@ class ProcessManager: def __init__(self, main): self.main = main - self.module_objects = {} # this is the queue that will be used by the input proces # to pass flows to the profiler self.profiler_queue = Queue() @@ -58,6 +58,7 @@ def __init__(self, main): # and inout stops and renders the profiler queue useless and profiler # cant get more lines anymore! self.is_profiler_done_event = Event() + self.read_config() # for the communication between output.py and the progress bar # Pipe(False) means the pipe is unidirectional. # aka only msgs can go from output -> pbar and not vice versa @@ -65,7 +66,12 @@ def __init__(self, main): # send_pipe use donly for sending self.pbar_recv_pipe, self.output_send_pipe = Pipe(False) self.pbar_finished: Event = Event() - + + def read_config(self): + self.modules_to_ignore: list = self.main.conf.get_disabled_modules( + self.main.input_type + ) + def is_pbar_supported(self) -> bool: """ When running on a pcap, interface, or taking flows from an @@ -74,27 +80,26 @@ def is_pbar_supported(self) -> bool: """ # input type can be false whne using -S or in unit tests if ( - not self.main.input_type - or self.main.input_type in ('interface', 'pcap', 'stdin') - or self.main.mode == 'daemonized' + not self.main.input_type + or self.main.input_type in ("interface", "pcap", "stdin") + or self.main.mode == "daemonized" ): return False - - if self.main.stdout != '': + if self.main.stdout != "": # this means that stdout was redirected to a file, # no need to print the progress bar return False - + if ( - self.main.args.growing - or self.main.args.input_module - or self.main.args.testing + self.main.args.growing + or self.main.args.input_module + or self.main.args.testing ): return False - + return True - + def start_output_process(self, current_stdout, stderr, slips_logfile): output_process = Output( stdout=current_stdout, @@ -106,13 +111,13 @@ def start_output_process(self, current_stdout, stderr, slips_logfile): sender_pipe=self.output_send_pipe, has_pbar=self.is_pbar_supported(), pbar_finished=self.pbar_finished, - stop_daemon=self.main.args.stopdaemon + stop_daemon=self.main.args.stopdaemon, ) self.slips_logfile = output_process.slips_logfile return output_process - - def start_progress_bar(self, cls): - pbar = cls( + + def start_progress_bar(self): + pbar = PBar( self.main.logger, self.main.args.output, self.main.redis_port, @@ -122,8 +127,12 @@ def start_progress_bar(self, cls): slips_mode=self.main.mode, pbar_finished=self.pbar_finished, ) + pbar.start() + self.main.db.store_process_PID(pbar.name, int(pbar.pid)) + self.main.print(f"Started {green('PBar')} process [" + f"PID {green(pbar.pid)}]") return pbar - + def start_profiler_process(self): profiler_process = Profiler( self.main.logger, @@ -132,13 +141,15 @@ def start_profiler_process(self): self.termination_event, is_profiler_done=self.is_profiler_done, profiler_queue=self.profiler_queue, - is_profiler_done_event= self.is_profiler_done_event, + is_profiler_done_event=self.is_profiler_done_event, has_pbar=self.is_pbar_supported(), ) profiler_process.start() self.main.print( f'Started {green("Profiler Process")} ' - f"[PID {green(profiler_process.pid)}]", 1, 0, + f"[PID {green(profiler_process.pid)}]", + 1, + 0, ) self.main.db.store_process_PID("Profiler", int(profiler_process.pid)) return profiler_process @@ -178,15 +189,13 @@ def start_input_process(self): ) input_process.start() self.main.print( - f'Started {green("Input Process")} ' - f'[PID {green(input_process.pid)}]', + f'Started {green("Input Process")} ' f"[PID {green(input_process.pid)}]", 1, 0, ) self.main.db.store_process_PID("Input", int(input_process.pid)) return input_process - def kill_process_tree(self, pid: int): try: # Send SIGKILL signal to the process @@ -196,9 +205,7 @@ def kill_process_tree(self, pid: int): # Get the child processes of the current process try: - process_list = (os.popen(f'pgrep -P {pid}') - .read() - .splitlines()) + process_list = os.popen(f"pgrep -P {pid}").read().splitlines() except: process_list = [] @@ -222,28 +229,24 @@ def kill_all_children(self): self.kill_process_tree(process.pid) self.print_stopped_module(module_name) - def is_ignored_module( - self, module_name: str, to_ignore: list - )-> bool: + def is_ignored_module(self, module_name: str) -> bool: - for ignored_module in to_ignore: - ignored_module = (ignored_module - .replace(' ','') - .replace('_','') - .replace('-','') - .lower()) + for ignored_module in self.modules_to_ignore: + ignored_module = ( + ignored_module.replace(" ", "") + .replace("_", "") + .replace("-", "") + .lower() + ) # this version of the module name wont contain # _ or spaces so we can # easily match it with the ignored module name - curr_module_name = (module_name - .replace('_','') - .replace('-','') - .lower()) + curr_module_name = module_name.replace("_", "").replace("-", "").lower() if curr_module_name.__contains__(ignored_module): return True return False - def get_modules(self, to_ignore: list): + def get_modules(self): """ Get modules from the 'modules' folder. """ @@ -252,7 +255,6 @@ def get_modules(self, to_ignore: list): plugins = {} failed_to_load_modules = 0 - # __path__ is the current path of this python program look_for_modules_in = modules.__path__ prefix = f"{modules.__name__}." @@ -272,11 +274,9 @@ def get_modules(self, to_ignore: list): if dir_name != file_name: continue - - if self.is_ignored_module(module_name, to_ignore): + if self.is_ignored_module(module_name): continue - # Try to import the module, otherwise skip. try: # "level specifies whether to use absolute or relative imports. @@ -289,8 +289,10 @@ def get_modules(self, to_ignore: list): # module calling __import__()." module = importlib.import_module(module_name) except ImportError as e: - print(f"Something wrong happened while " - f"importing the module {module_name}: {e}") + print( + f"Something wrong happened while " + f"importing the module {module_name}: {e}" + ) print(traceback.format_exc()) failed_to_load_modules += 1 @@ -299,12 +301,8 @@ def get_modules(self, to_ignore: list): # Walk through all members of currently imported modules. for member_name, member_object in inspect.getmembers(module): # Check if current member is a class. - if ( - inspect.isclass(member_object) - and ( - issubclass(member_object, IModule) - and member_object is not IModule - ) + if inspect.isclass(member_object) and ( + issubclass(member_object, IModule) and member_object is not IModule ): plugins[member_object.name] = dict( obj=member_object, @@ -329,43 +327,44 @@ def get_modules(self, to_ignore: list): return plugins, failed_to_load_modules - def load_modules(self): - to_ignore: list = self.main.conf.get_disabled_modules( - self.main.input_type) + def print_disabled_modules(self): + print("-" * 27) + self.main.print(f"Disabled Modules: {self.modules_to_ignore}", 1, 0) - # Import all the modules - modules_to_call = self.get_modules(to_ignore)[0] - loaded_modules = [] + def load_modules(self): + """responsible for starting all the modules in the modules/ dir""" + modules_to_call = self.get_modules()[0] for module_name in modules_to_call: - if module_name in to_ignore: - continue - module_class = modules_to_call[module_name]["obj"] if module_name == "Progress Bar": - module = self.start_progress_bar(module_class) - else: - module = module_class( - self.main.logger, - self.main.args.output, - self.main.redis_port, - self.termination_event, - ) + # started it manually in main.py + # otherwise we miss some of the print right when slips + # starts, because when the pbar is supported, it handles + # all the printing + continue + + module = module_class( + self.main.logger, + self.main.args.output, + self.main.redis_port, + self.termination_event, + ) module.start() self.main.db.store_process_PID(module_name, int(module.pid)) - self.module_objects[module_name] = module # maps name -> object - description = modules_to_call[module_name]["description"] - self.main.print( - f"\t\tStarting the module {green(module_name)} " - f"({description}) " - f"[PID {green(module.pid)}]", - 1, 0, + self.print_started_module( + module_name, module.pid, modules_to_call[module_name]["description"] ) - loaded_modules.append(module_name) - # give outputprocess time to print all the started modules - time.sleep(0.5) - print("-" * 27) - self.main.print(f"Disabled Modules: {to_ignore}", 1, 0) - return loaded_modules + + def print_started_module( + self, module_name: str, module_pid: int, module_description: str + ) -> None: + self.main.print( + f"\t\tStarting the module {green(module_name)} " + f"({module_description}) " + f"[PID {green(module_pid)}]", + 1, + 0, + ) def print_stopped_module(self, module): self.stopped_modules.append(module) @@ -374,9 +373,9 @@ def print_stopped_module(self, module): # to vertically align them when printing module += " " * (20 - len(module)) - self.main.print(f"\t{green(module)} \tStopped. " - f"" f"{green(modules_left)} left.") - + self.main.print( + f"\t{green(module)} \tStopped. " f"" f"{green(modules_left)} left." + ) def start_update_manager(self, local_files=False, TI_feeds=False): """ @@ -399,7 +398,7 @@ def start_update_manager(self, local_files=False, TI_feeds=False): self.main.logger, self.main.args.output, self.main.redis_port, - multiprocessing.Event() + multiprocessing.Event(), ) if local_files: @@ -441,7 +440,6 @@ def warn_about_pending_modules(self, pending_modules: List[Process]): self.warning_printed_once = True return True - def get_hitlist_in_order(self) -> Tuple[List[Process], List[Process]]: """ returns a list of PIDs that slips should terminate first, @@ -516,9 +514,7 @@ def get_analysis_time(self): end_date = self.main.metadata_man.set_analysis_end_date() start_time = self.main.db.get_slips_start_time() - return utils.get_time_diff( - start_time, end_date, return_type="minutes" - ) + return utils.get_time_diff(start_time, end_date, return_type="minutes") def should_stop(self): """ @@ -526,16 +522,15 @@ def should_stop(self): """ message = self.main.c1.get_message(timeout=0.01) if ( - message - and utils.is_msg_intended_for(message, 'control_channel') - and message['data'] == 'stop_slips' + message + and utils.is_msg_intended_for(message, "control_channel") + and message["data"] == "stop_slips" ): return True - def is_debugger_active(self) -> bool: """Returns true if the debugger is currently active""" - gettrace = getattr(sys, 'gettrace', lambda: None) + gettrace = getattr(sys, "gettrace", lambda: None) return gettrace() is not None def should_run_non_stop(self) -> bool: @@ -547,9 +542,9 @@ def should_run_non_stop(self) -> bool: # when slips is reading from a special module other than the input process # this module should handle the stopping of slips if ( - self.is_debugger_active() - or self.main.input_type in ('stdin', 'cyst') - or self.main.is_interface + self.is_debugger_active() + or self.main.input_type in ("stdin", "cyst") + or self.main.is_interface ): return True return False @@ -590,7 +585,6 @@ def shutdown_interactive(self, to_kill_first, to_kill_last): # all of them are killed return None, None - def slips_is_done_receiving_new_flows(self) -> bool: """ this method will return True when the input and profiler release @@ -598,12 +592,8 @@ def slips_is_done_receiving_new_flows(self) -> bool: If they're still processing it will return False """ # try to acquire the semaphore without blocking - input_done_processing: bool = self.is_input_done.acquire( - block=False - ) - profiler_done_processing: bool = self.is_profiler_done.acquire( - block=False - ) + input_done_processing: bool = self.is_input_done.acquire(block=False) + profiler_done_processing: bool = self.is_profiler_done.acquire(block=False) if input_done_processing and profiler_done_processing: return True @@ -611,7 +601,6 @@ def slips_is_done_receiving_new_flows(self) -> bool: # can't acquire the semaphore, processes are still running return False - def shutdown_daemon(self): """ Shutdown slips modules in daemon mode @@ -636,7 +625,6 @@ def shutdown_gracefully(self): print("\n" + "-" * 27) self.main.print("Stopping Slips") - # by default, 15 mins from this time, all modules should be killed method_start_time = time.time() @@ -647,11 +635,13 @@ def shutdown_gracefully(self): # close all tws self.main.db.check_TW_to_close(close_all=True) analysis_time = self.get_analysis_time() - self.main.print(f"Analysis of {self.main.input_information} " - f"finished in {analysis_time:.2f} minutes") + self.main.print( + f"Analysis of {self.main.input_information} " + f"finished in {analysis_time:.2f} minutes" + ) graceful_shutdown = True - if self.main.mode == 'daemonized': + if self.main.mode == "daemonized": self.processes: dict = self.main.db.get_pids() self.shutdown_daemon() @@ -664,11 +654,13 @@ def shutdown_gracefully(self): else: flows_count: int = self.main.db.get_flows_count() - self.main.print(f"Total flows read (without altflows): " - f"{flows_count}", log_to_logfiles_only=True) + self.main.print( + f"Total flows read (without altflows): " f"{flows_count}", + log_to_logfiles_only=True, + ) hitlist: Tuple[List[Process], List[Process]] - hitlist = self.get_hitlist_in_order() + hitlist = self.get_hitlist_in_order() to_kill_first: List[Process] = hitlist[0] to_kill_last: List[Process] = hitlist[1] self.termination_event.set() @@ -677,13 +669,11 @@ def shutdown_gracefully(self): # modules self.warning_printed_once = False - try: # Wait timeout_seconds for all the processes to finish while time.time() - method_start_time < timeout_seconds: to_kill_first, to_kill_last = self.shutdown_interactive( - to_kill_first, - to_kill_last + to_kill_first, to_kill_last ) if not to_kill_first and not to_kill_last: # all modules are done @@ -704,8 +694,10 @@ def shutdown_gracefully(self): # getting here means we're killing them bc of the timeout # not getting here means we're killing them bc of double # ctr+c OR they terminated successfully - reason = (f"Killing modules that took more than {timeout}" - f" mins to finish.") + reason = ( + f"Killing modules that took more than {timeout}" + f" mins to finish." + ) self.main.print(reason) graceful_shutdown = False @@ -732,12 +724,16 @@ def shutdown_gracefully(self): self.main.db.close() if graceful_shutdown: - self.main.print("[Process Manager] Slips shutdown gracefully\n", - log_to_logfiles_only=True) + self.main.print( + "[Process Manager] Slips shutdown gracefully\n", + log_to_logfiles_only=True, + ) else: - self.main.print(f"[Process Manager] Slips didn't " - f"shutdown gracefully - {reason}\n", - log_to_logfiles_only=True) + self.main.print( + f"[Process Manager] Slips didn't " + f"shutdown gracefully - {reason}\n", + log_to_logfiles_only=True, + ) except KeyboardInterrupt: return False diff --git a/modules/progress_bar/progress_bar.py b/modules/progress_bar/progress_bar.py index 405c6c102..9f8eed432 100644 --- a/modules/progress_bar/progress_bar.py +++ b/modules/progress_bar/progress_bar.py @@ -124,8 +124,6 @@ def shutdown_gracefully(self): # to tell output.py to no longer send prints here self.pbar_finished.set() - - def main(self): """ keeps receiving events until pbar reaches 100% diff --git a/modules/update_manager/update_manager.py b/modules/update_manager/update_manager.py index 94d23ee87..ed9876a53 100644 --- a/modules/update_manager/update_manager.py +++ b/modules/update_manager/update_manager.py @@ -1509,18 +1509,22 @@ async def update(self) -> bool: # every function call to update_TI_file is now running concurrently instead of serially # so when a server's taking a while to give us the TI feed, we proceed # to download the next file instead of being idle - task = asyncio.create_task(self.update_TI_file(file_to_download)) + task = asyncio.create_task( + self.update_TI_file(file_to_download) + ) ####################################################### # in case of riskiq files, we don't have a link for them in ti_files, We update these files using their API # check if we have a username and api key and a week has passed since we last updated - if self.check_if_update("riskiq_domains", self.riskiq_update_period): + if self.check_if_update("riskiq_domains", + self.riskiq_update_period): self.update_riskiq_feed() # wait for all TI files to update try: await task except UnboundLocalError: - # in case all our files are updated, we don't have task defined, skip + # in case all our files are updated, we don't + # have task defined, skip pass self.db.set_loaded_ti_files(self.loaded_ti_files) @@ -1533,10 +1537,12 @@ async def update_ti_files(self): """ Update TI files and store them in database before slips starts """ - # create_task is used to run update() function concurrently instead of serially + # create_task is used to run update() function + # concurrently instead of serially self.update_finished = asyncio.create_task(self.update()) await self.update_finished - self.print(f"{self.db.get_loaded_ti_files()} TI files successfully loaded.") + self.print(f"{self.db.get_loaded_ti_files()} " + f"TI files successfully loaded.") def shutdown_gracefully(self): # terminating the timer for the process to be killed diff --git a/slips/main.py b/slips/main.py index 570141a1a..e57f581c9 100644 --- a/slips/main.py +++ b/slips/main.py @@ -37,7 +37,6 @@ def __init__(self, testing=False): self.redis_man = RedisManager(self) self.ui_man = UIManager(self) self.metadata_man = MetadataManager(self) - self.proc_man = ProcessManager(self) self.conf = ConfigParser() self.version = self.get_slips_version() # will be filled later @@ -45,6 +44,7 @@ def __init__(self, testing=False): self.branch = "None" self.last_updated_stats_time = datetime.now() self.input_type = False + self.proc_man = ProcessManager(self) # in testing mode we manually set the following params if not testing: self.args = self.conf.get_args() @@ -554,6 +554,7 @@ def start(self): current_stdout, stderr, slips_logfile ) self.add_observer(self.logger) + # get the port that is going to be used for this instance of slips if self.args.port: @@ -580,6 +581,17 @@ def start(self): "branch": self.branch, } ) + self.print( + f"Using redis server on " f"port: " + f"{green(self.redis_port)}", 1, 1 + ) + self.print( + f'Started {green("Main")} process ' f"[PID" + f" {green(self.pid)}]", 1, 1 + ) + # start progress bar before all modules so it doesn't miss + # any prints in its queue and slips wouldn't seem like it's frozen + self.proc_man.start_progress_bar() self.cpu_profiler_init() self.memory_profiler_init() @@ -593,7 +605,7 @@ def start(self): ) else: self.print( - f"Running on a growing zeek dir:" f" {self.input_information}" + f"Running on a growing zeek dir: {self.input_information}" ) self.db.set_growing_zeek_dir() @@ -620,13 +632,7 @@ def start(self): self.db.store_std_file(**std_files) - self.print( - f"Using redis server on " f"port: {green(self.redis_port)}", 1, 0 - ) - self.print( - f'Started {green("Main")} process ' f"[PID {green(self.pid)}]", 1, 0 - ) - self.print("Starting modules", 1, 0) + # if slips is given a .rdb file, don't load the # modules as we don't need them @@ -638,7 +644,11 @@ def start(self): self.proc_man.start_update_manager( local_files=True, TI_feeds=self.conf.wait_for_TI_to_finish() ) + self.print("Starting modules",1, 1) self.proc_man.load_modules() + # give outputprocess time to print all the started modules + time.sleep(0.5) + self.proc_man.print_disabled_modules() if self.args.webinterface: self.ui_man.start_webinterface() diff --git a/slips_files/core/database/redis_db/profile_handler.py b/slips_files/core/database/redis_db/profile_handler.py index d2857c3e4..221bfb475 100644 --- a/slips_files/core/database/redis_db/profile_handler.py +++ b/slips_files/core/database/redis_db/profile_handler.py @@ -1119,7 +1119,7 @@ def getT2ForProfileTW(self, profileid, twid, tupleid, tuple_key: str): except Exception as e: exception_line = sys.exc_info()[2].tb_lineno self.print( - f"Error in getT2ForProfileTW in database.py line " f"{exception_line}", + f"Error in getT2ForProfileTW in database.py line {exception_line}", 0, 1, ) diff --git a/slips_files/core/helpers/checker.py b/slips_files/core/helpers/checker.py index b8134474a..53d8d684f 100644 --- a/slips_files/core/helpers/checker.py +++ b/slips_files/core/helpers/checker.py @@ -1,6 +1,7 @@ import os import subprocess import sys +from typing import Tuple import psutil @@ -187,7 +188,7 @@ def input_module_exists(self, module): return True - def check_output_redirection(self) -> tuple: + def check_output_redirection(self) -> Tuple[str,str,str]: """ Determine where slips will place stdout, stderr and logfile based on slips mode From e66c86e4d1c617a938239eec371fba48d3b4cdcc Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 23 Feb 2024 19:13:38 +0200 Subject: [PATCH 30/33] update unit tests --- tests/test_slips.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_slips.py b/tests/test_slips.py index f4c82916c..3b95aeb35 100644 --- a/tests/test_slips.py +++ b/tests/test_slips.py @@ -5,9 +5,12 @@ def test_load_modules(): proc_manager = ModuleFactory().create_process_manager_obj() - failed_to_load_modules = proc_manager.get_modules( - ['template', 'mldetection-1', 'ensembling'] - )[1] + proc_manager.modules_to_ignore = [ + 'template', + 'mldetection-1', + 'ensembling' + ] + failed_to_load_modules = proc_manager.get_modules()[1] assert failed_to_load_modules == 0 # # @pytest.mark.skipif(IS_IN_A_DOCKER_CONTAINER, reason='This functionality is not supported in docker') From d3f986a25d7d6fde0bf51f77d86734edf306e4de Mon Sep 17 00:00:00 2001 From: Rosheen Naeem Date: Sun, 25 Feb 2024 01:52:35 +0100 Subject: [PATCH 31/33] Update installation.md to fix the image --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 8fe6a4875..22d82e141 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -212,7 +212,7 @@ The main limitation of running Slips in a Docker is that every time the containe First, you need to check which image is suitable for your architecture. - Before building the docker locally from the Dockerfile, first you should clone Slips repo or download the code directly: From 2b47b6f025dda6f91c1692ef45969a121d3cda59 Mon Sep 17 00:00:00 2001 From: Sebastian Garcia Date: Tue, 27 Feb 2024 14:25:05 +0100 Subject: [PATCH 32/33] Update installation.md image file --- docs/installation.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 22d82e141..b0baeb6c1 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -212,7 +212,8 @@ The main limitation of running Slips in a Docker is that every time the containe First, you need to check which image is suitable for your architecture. - +.. image:: /images/docker_images.png + :align: center Before building the docker locally from the Dockerfile, first you should clone Slips repo or download the code directly: From 8273de14cd89b8c2f76060377a5c4671d6d23a3a Mon Sep 17 00:00:00 2001 From: Sebastian Garcia Date: Tue, 27 Feb 2024 15:01:48 +0100 Subject: [PATCH 33/33] Update installation.md. Fix link to image that was not working --- docs/installation.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index b0baeb6c1..22d82e141 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -212,8 +212,7 @@ The main limitation of running Slips in a Docker is that every time the containe First, you need to check which image is suitable for your architecture. -.. image:: /images/docker_images.png - :align: center + Before building the docker locally from the Dockerfile, first you should clone Slips repo or download the code directly: