From b10a349bf9de43478817bd11cbb2006650cff9d0 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 7 Dec 2023 16:28:31 +0200 Subject: [PATCH 01/18] print not processed filtered evidence (for debugging) --- modules/rnn_cc_detection/rnn_cc_detection.py | 19 ++- .../threat_intelligence.py | 83 ++++++------ slips_files/common/slips_utils.py | 2 +- .../core/database/redis_db/alert_handler.py | 14 +- slips_files/core/evidence.py | 124 +++++++++++++----- 5 files changed, 154 insertions(+), 88 deletions(-) diff --git a/modules/rnn_cc_detection/rnn_cc_detection.py b/modules/rnn_cc_detection/rnn_cc_detection.py index 19efd6f43..4ee68c581 100644 --- a/modules/rnn_cc_detection/rnn_cc_detection.py +++ b/modules/rnn_cc_detection/rnn_cc_detection.py @@ -57,9 +57,22 @@ def set_evidence( f'score: {format(score, ".4f")}. {ip_identification}' ) victim = profileid.split('_')[-1] - self.db.setEvidence(evidence_type, attacker_direction, attacker, threat_level, confidence, description, - timestamp, categroy, source_target_tag=source_target_tag, port=port, proto=proto, - profileid=profileid, twid=twid, uid=uid, victim= victim) + self.db.setEvidence( + evidence_type, + attacker_direction, + attacker, + threat_level, + confidence, + description, + timestamp, + categroy, + source_target_tag=source_target_tag, + port=port, + proto=proto, + profileid=profileid, + twid=twid, + uid=uid, + victim= victim) diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index cbd696d5b..9f85c3e29 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -276,9 +276,19 @@ def set_evidence_malicious_domain( if tags: description += f'with tags: {tags}. ' - self.db.setEvidence(evidence_type, attacker_direction, attacker, threat_level, confidence, description, - timestamp, category, source_target_tag=source_target_tag, profileid=profileid, - twid=twid, uid=uid) + self.db.setEvidence( + evidence_type, + attacker_direction, + attacker, + threat_level, + confidence, + description, + timestamp, + category, + source_target_tag=source_target_tag, + profileid=profileid, + twid=twid, + uid=uid) def is_valid_threat_level(self, threat_level): return threat_level in utils.threat_levels @@ -327,63 +337,43 @@ def parse_local_ti_file(self, ti_file_path: str) -> bool: # default value threat_level = 'medium' + ioc_info = { + 'description': description, + 'source': data_file_name, + 'threat_level': threat_level, + 'tags': 'local TI file', + } + ioc_info: str = json.dumps(ioc_info) + data_type = utils.detect_data_type(ioc.strip()) if data_type == 'ip': ip_address = ipaddress.ip_address(ioc.strip()) - # Only use global addresses. Ignore multicast, broadcast, private, reserved and undefined + # Only use global addresses. Ignore multicast, + # broadcast, private, reserved and undefined if ip_address.is_global: - # Store the ip in our local dict - malicious_ips[str(ip_address)] = json.dumps( - { - 'description': description, - 'source': data_file_name, - 'threat_level': threat_level, - 'tags': 'local TI file', - } - ) + malicious_ips[str(ip_address)] = ioc_info + elif data_type == 'domain': - malicious_domains[ioc] = json.dumps( - { - 'description': description, - 'source': data_file_name, - 'threat_level': threat_level, - 'tags': 'local TI file', - } - ) + malicious_domains[ioc] = ioc_info + elif data_type == 'ip_range': net_addr = ioc[: ioc.index('/')] - ip_obj = ipaddress.ip_address(net_addr) if ( - ip_obj.is_multicast - or utils.is_private_ip(ip_obj) - or ip_obj.is_link_local + utils.is_ignored_ip(net_addr) or net_addr in utils.home_networks ): continue - malicious_ip_ranges[ioc] = json.dumps( - { - 'description': description, - 'source': data_file_name, - 'threat_level': threat_level, - 'tags': 'local TI file', - } - ) + malicious_ip_ranges[ioc] = ioc_info + elif data_type == 'asn': - malicious_asns[ioc] = json.dumps( - { - 'description': description, - 'source': data_file_name, - 'threat_level': threat_level, - 'tags': 'local TI file', - } - ) + malicious_asns[ioc] = ioc_info else: # invalid ioc, skip it self.print( f'Error while reading the TI file {ti_file_path}.' f' Line {line_number} has invalid data: {ioc}', - 0, 1, + 0, 1 ) # Add all loaded malicious ips to the database @@ -458,7 +448,8 @@ def parse_ja3_file(self, path): # "JA3 hash", "Threat level", "Description" data = line.replace('\n', '').replace('"', '').split(',') - # the column order is hardcoded because it's owr own ti file and we know the format, + # the column order is hardcoded because it's owr + # own ti file and we know the format, # we shouldn't be trying to find it ja3, threat_level, description = ( data[0].strip(), @@ -515,7 +506,9 @@ def parse_jarm_file(self, path): if len(data) < 3: # invalid line continue - # the column order is hardcoded because it's owr own ti file and we know the format, + + # the column order is hardcoded because + # it's owr own ti file and we know the format, # we shouldn't be trying to find it jarm, threat_level, description = ( data[0].strip(), @@ -650,7 +643,7 @@ def spamhaus(self, ip): return { 'source': source_dataset, 'description': description, - 'therat_level': 'medium', + 'threat_level': 'medium', 'tags': 'spam', } diff --git a/slips_files/common/slips_utils.py b/slips_files/common/slips_utils.py index a601010b5..bc5c40fe2 100644 --- a/slips_files/common/slips_utils.py +++ b/slips_files/common/slips_utils.py @@ -332,7 +332,7 @@ def is_private_ip(self, ip_obj:ipaddress) -> bool: r_value = True return r_value - def is_ignored_ip(self, ip) -> bool: + def is_ignored_ip(self, ip: str) -> bool: """ This function checks if an IP is a special list of IPs that should not be alerted for different reasons diff --git a/slips_files/core/database/redis_db/alert_handler.py b/slips_files/core/database/redis_db/alert_handler.py index 3cb92ba6e..1b390c0e8 100644 --- a/slips_files/core/database/redis_db/alert_handler.py +++ b/slips_files/core/database/redis_db/alert_handler.py @@ -191,7 +191,7 @@ def setEvidence( # every evidence should have an ID according to the IDEA format evidence_ID = str(uuid4()) - if type(uid) == list: + if isinstance(uid, list): # some evidence are caused by several uids, use the last one only # todo check why we have duplicates in the first place # remove duplicate uids @@ -201,9 +201,9 @@ def setEvidence( self.set_flow_causing_evidence(uids, evidence_ID) - if type(threat_level) != str: + if not isinstance(threat_level, str): # make sure we always store str threat levels in the db - threat_level = utils.threat_level_to_string(threat_level) + threat_level: str = utils.threat_level_to_string(threat_level) if timestamp: timestamp = utils.convert_format(timestamp, utils.alerts_format) @@ -283,13 +283,13 @@ def init_evidence_number(self): def get_evidence_number(self): return self.r.get('number_of_evidence') - def mark_evidence_as_processed(self, evidence_ID): + def mark_evidence_as_processed(self, evidence_ID: str): """ If an evidence was processed by the evidenceprocess, mark it in the db """ self.r.sadd('processed_evidence', evidence_ID) - def is_evidence_processed(self, evidence_ID): + def is_evidence_processed(self, evidence_ID: str) -> bool: return self.r.sismember('processed_evidence', evidence_ID) def set_evidence_for_profileid(self, evidence): @@ -386,10 +386,10 @@ def get_profileid_twid_alerts(self, profileid, twid) -> dict: The format for the returned dict is {profile123_twid1_: [ev_uuid1, ev_uuid2, ev_uuid3]} """ - alerts = self.r.hget(f'{profileid}{self.separator}{twid}', 'alerts') + alerts: str = self.r.hget(f'{profileid}_{twid}', 'alerts') if not alerts: return {} - alerts = json.loads(alerts) + alerts: dict = json.loads(alerts) return alerts def getEvidenceForTW(self, profileid: str, twid: str) -> str: diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index 12b021cba..2ef8ffa71 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -20,6 +20,7 @@ from slips_files.core.helpers.notify import Notify from slips_files.common.abstracts.core import ICore import json +from typing import Union from datetime import datetime from os import path from colorama import Fore, Style @@ -75,6 +76,9 @@ def init(self): self.print(f'Storing Slips logs in {self.output_dir}') # this list will have our local and public ips when using -i self.our_ips = utils.get_own_IPs() + filtered_log_path = os.path.join(self.output_dir, 'filtered_ev.log') + self.filtered_log = open(filtered_log_path, 'w') + self.discarded_bc_never_processed = {} def read_configuration(self): conf = ConfigParser() @@ -93,6 +97,8 @@ def read_configuration(self): if IS_IN_A_DOCKER_CONTAINER: self.popup_alerts = False + def log_filtered(self, txt): + self.filtered_log.write(txt+'\n') def format_evidence_string(self, ip, detection_module, attacker, description) -> str: @@ -396,40 +402,86 @@ def mark_as_blocked( def shutdown_gracefully(self): self.logfile.close() self.jsonfile.close() - - def delete_alerted_evidence(self, profileid, twid, tw_evidence:dict): + self.filtered_log.close() + self.print("@@@@@@@@@@@@@@@@ discarded_bc_never_processed") + self.print(self.discarded_bc_never_processed) + + def delete_alerted_evidence( + self, + profileid: str, + twid: str, + tw_evidence: dict) -> dict: """ if there was an alert in this tw before, remove the evidence that - were part of the past alert - from the current evidence + were part of the past alert from the current evidence. + + when blocking is not enabled, we can alert on a single profile many times + when we get all the tw evidence from the db, we get the once we + alerted, and the new once we need to alert + this method removes the already alerted evidence to avoid duplicates + + :param tw_evidence: dict with {: {evidence_details}} """ - # format of tw_evidence is {: {evidence_details}} - past_alerts = self.db.get_profileid_twid_alerts(profileid, twid) + past_alerts: dict = self.db.get_profileid_twid_alerts(profileid, twid) if not past_alerts: return tw_evidence for alert_id, evidence_IDs in past_alerts.items(): - evidence_IDs = json.loads(evidence_IDs) + alert_id: str + evidence_IDs: str # json serialized + + evidence_IDs: dict = json.loads(evidence_IDs) for ID in evidence_IDs: tw_evidence.pop(ID, None) return tw_evidence - def delete_whitelisted_evidence(self, evidence): + def delete_whitelisted_evidence(self, profileid, twid, evidence: dict) \ + -> dict: """ - delete the hash of all whitelisted evidence from the given dict of evidence ids + delete the hash of all whitelisted evidence + from the given dict of evidence ids + and the evidence that were returned from the db but not yet + processed by evidence.py """ res = {} for evidence_ID, evidence_info in evidence.items(): - # sometimes the db has evidence that didnt come yet to evidenceprocess + # sometimes the db has evidence that didnt come yet to evidence.py # and they are alerted without checking the whitelist! - # to fix this, we have processed evidence which are - # evidence that came to new_evidence channel and were processed by it + + # to fix this, we keep track of processed evidence + # that came to new_evidence channel and were processed by it. # so they are ready to be a part of an alerted - if ( - not self.db.is_whitelisted_evidence(evidence_ID) - and self.db.is_evidence_processed(evidence_ID) - ): + whitelisted: bool = self.db.is_whitelisted_evidence(evidence_ID) + processed: bool = self.db.is_evidence_processed(evidence_ID) + # client_ip = "192.168.1.113" + client_ip = "10.0.2.15" + if not processed and client_ip in profileid: + self.log_filtered(f"Evidence {evidence_ID} .." + f" {evidence_info} is not processed yet, " + f"discarding") + try: + if evidence_ID not in self.discarded_bc_never_processed[profileid][twid]: + self.discarded_bc_never_processed[profileid][twid].append(evidence_ID) + except KeyError: + self.discarded_bc_never_processed.update({ + profileid: {twid: [evidence_ID]} + }) + + if processed and not whitelisted: res[evidence_ID] = evidence_info + if client_ip in profileid: + try: + if evidence_ID in self.discarded_bc_never_processed[profileid][twid]: + self.log_filtered(f"Evidence Alerted after being " + f"discarded before {evidence_ID}") + self.discarded_bc_never_processed[profileid][ + twid].remove(evidence_ID) + except KeyError: + continue + + if client_ip in profileid: + self.log_filtered(f"done getting tw evidence {profileid} {twid}") + return res def delete_evidence_done_by_others(self, tw_evidence): @@ -450,24 +502,34 @@ def delete_evidence_done_by_others(self, tw_evidence): return res - def get_evidence_for_tw(self, profileid, twid): - # Get all the evidence for this profile in this TW - tw_evidence: str = self.db.getEvidenceForTW( - profileid, twid - ) + def get_evidence_for_tw(self, profileid: str, twid: str) \ + -> Union[dict, bool]: + """ + filters and returns all the evidence for this profile in this TW + filters the follwing: + * evidence that were part of a past alert in this same profileid twid + * evidence that weren't done by the given profileid + * evidence that are whitelisted + """ + tw_evidence: str = self.db.getEvidenceForTW(profileid, twid) if not tw_evidence: return False tw_evidence: dict = json.loads(tw_evidence) tw_evidence = self.delete_alerted_evidence(profileid, twid, tw_evidence) tw_evidence = self.delete_evidence_done_by_others(tw_evidence) - tw_evidence = self.delete_whitelisted_evidence(tw_evidence) + tw_evidence = self.delete_whitelisted_evidence(profileid, twid, + tw_evidence) return tw_evidence - def get_accumulated_threat_level(self, tw_evidence): + def get_accumulated_threat_level(self, tw_evidence: dict): + """ + return the sum of all threat levels of all evidence in + the given tw evidence dict + """ accumulated_threat_level = 0.0 - # to store all the ids causing this alerts in the database + # to store all the ids causing this alert in the database self.IDs_causing_an_alert = [] for evidence in tw_evidence.values(): evidence = json.loads(evidence) @@ -479,11 +541,11 @@ def get_accumulated_threat_level(self, tw_evidence): description = evidence.get('description') ID = evidence.get('ID') self.IDs_causing_an_alert.append(ID) + # each threat level is a string, get the numerical value of it try: - threat_level = utils.threat_levels[ - threat_level.lower() - ] + threat_level: float = \ + utils.threat_levels[threat_level.lower()] except KeyError: self.print( f'Error: Evidence of type {evidence_type} has ' @@ -493,17 +555,15 @@ def get_accumulated_threat_level(self, tw_evidence): threat_level = 0 # Compute the moving average of evidence - new_threat_level = threat_level * confidence - self.print( - f'\t\tWeighted Threat Level: {new_threat_level}', 3, 0 - ) + new_threat_level: float = threat_level * confidence + self.print(f'\t\tWeighted Threat Level: {new_threat_level}', 3, 0) accumulated_threat_level += new_threat_level self.print( f'\t\tAccumulated Threat Level: {accumulated_threat_level}', 3, 0, ) return accumulated_threat_level - def get_last_evidence_ID(self, tw_evidence) -> str: + def get_last_evidence_ID(self, tw_evidence: dict) -> str: return list(tw_evidence.keys())[-1] def send_to_exporting_module(self, tw_evidence): From 6ec87d36c7b3b2f8eb262ec86a1e91fe13352992 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 7 Dec 2023 16:36:16 +0200 Subject: [PATCH 02/18] change the client id --- slips_files/core/evidence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index 2ef8ffa71..a020aff46 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -453,8 +453,8 @@ def delete_whitelisted_evidence(self, profileid, twid, evidence: dict) \ # so they are ready to be a part of an alerted whitelisted: bool = self.db.is_whitelisted_evidence(evidence_ID) processed: bool = self.db.is_evidence_processed(evidence_ID) - # client_ip = "192.168.1.113" - client_ip = "10.0.2.15" + client_ip = "192.168.1.113" + # client_ip = "10.0.2.15" if not processed and client_ip in profileid: self.log_filtered(f"Evidence {evidence_ID} .." f" {evidence_info} is not processed yet, " From f131a3b65be22b7ed714d830a5e88df727426587 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 7 Dec 2023 17:56:55 +0200 Subject: [PATCH 03/18] evidence.py: optimize getting tw evidence --- slips_files/core/evidence.py | 116 ++++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 50 deletions(-) diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index a020aff46..ce82df0f5 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -20,7 +20,7 @@ from slips_files.core.helpers.notify import Notify from slips_files.common.abstracts.core import ICore import json -from typing import Union +from typing import Union, List from datetime import datetime from os import path from colorama import Fore, Style @@ -406,35 +406,6 @@ def shutdown_gracefully(self): self.print("@@@@@@@@@@@@@@@@ discarded_bc_never_processed") self.print(self.discarded_bc_never_processed) - def delete_alerted_evidence( - self, - profileid: str, - twid: str, - tw_evidence: dict) -> dict: - """ - if there was an alert in this tw before, remove the evidence that - were part of the past alert from the current evidence. - - when blocking is not enabled, we can alert on a single profile many times - when we get all the tw evidence from the db, we get the once we - alerted, and the new once we need to alert - this method removes the already alerted evidence to avoid duplicates - - :param tw_evidence: dict with {: {evidence_details}} - """ - past_alerts: dict = self.db.get_profileid_twid_alerts(profileid, twid) - if not past_alerts: - return tw_evidence - - for alert_id, evidence_IDs in past_alerts.items(): - alert_id: str - evidence_IDs: str # json serialized - - evidence_IDs: dict = json.loads(evidence_IDs) - for ID in evidence_IDs: - tw_evidence.pop(ID, None) - return tw_evidence - def delete_whitelisted_evidence(self, profileid, twid, evidence: dict) \ -> dict: """ @@ -484,22 +455,30 @@ def delete_whitelisted_evidence(self, profileid, twid, evidence: dict) \ return res - def delete_evidence_done_by_others(self, tw_evidence): - """ - given all the tw evidence, we should only consider evidence that makes this given - profile malicious, aka evidence of this profile attacking others. - """ - res = {} - for evidence_ID, evidence_info in tw_evidence.items(): - evidence_info = json.loads(evidence_info) - attacker_direction = evidence_info.get('attacker_direction', '') - # the following type detections are the ones - # expected to be seen when we are attacking others - # marking this profileid (srcip) as malicious - if attacker_direction in ('srcip', 'sport', 'srcport'): - res[evidence_ID] = json.dumps(evidence_info) + def get_evidence_that_were_part_of_a_past_alert( + self, profileid: str, twid: str) -> List[str]: - return res + past_alerts: dict = self.db.get_profileid_twid_alerts(profileid, twid) + try: + past_evidence_ids = list(past_alerts.values())[0] + past_evidence_ids: List[str] = json.loads(past_evidence_ids) + except IndexError: + # no past evidence + past_evidence_ids = [] + return past_evidence_ids + + def is_evidence_done_by_others(self, evidence: str) -> bool: + evidence: str = json.loads(evidence) + # given all the tw evidence, we should only + # consider evidence that makes this given + # profile malicious, aka evidence of this profile attacking others. + attacker_direction: str = evidence.get('attacker_direction', '') + # the following type detections are the ones + # expected to be seen when we are attacking others + # marking this profileid (srcip) as malicious + if attacker_direction in ('srcip', 'sport', 'srcport'): + return False + return True def get_evidence_for_tw(self, profileid: str, twid: str) \ @@ -510,17 +489,54 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ * evidence that were part of a past alert in this same profileid twid * evidence that weren't done by the given profileid * evidence that are whitelisted + * evidence that weren't processed by evidence.py yet """ tw_evidence: str = self.db.getEvidenceForTW(profileid, twid) if not tw_evidence: return False + # format of this is {ev_id, json_serialized(ev_details)} tw_evidence: dict = json.loads(tw_evidence) - tw_evidence = self.delete_alerted_evidence(profileid, twid, tw_evidence) - tw_evidence = self.delete_evidence_done_by_others(tw_evidence) - tw_evidence = self.delete_whitelisted_evidence(profileid, twid, - tw_evidence) - return tw_evidence + + past_evidence_ids: List[str] = \ + self.get_evidence_that_were_part_of_a_past_alert(profileid, twid) + + res = {} + for id, evidence in tw_evidence.items(): + id: str + evidence: str + + # delete already alerted evidence + # if there was an alert in this tw before, remove the evidence that + # were part of the past alert from the current evidence. + + # when blocking is not enabled, we can alert on a single profile many times + # when we get all the tw evidence from the db, we get the once we + # alerted, and the new once we need to alert + # this method removes the already alerted evidence to avoid duplicates + if id in past_evidence_ids: + continue + + if self.is_evidence_done_by_others(evidence): + continue + + whitelisted: bool = self.db.is_whitelisted_evidence(id) + if whitelisted: + continue + + # delete_not processed evidence + # sometimes the db has evidence that didn't come yet to evidence.py + # and they are alerted without checking the whitelist! + # to fix this, we keep track of processed evidence + # that came to new_evidence channel and were processed by it. + # so they are ready to be a part of an alerted + processed: bool = self.db.is_evidence_processed(id) + if not processed: + continue + + res[id] = evidence + + return res def get_accumulated_threat_level(self, tw_evidence: dict): From 22e057de291edf1e05d4172f9c45d6a7154f30f4 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 7 Dec 2023 18:11:52 +0200 Subject: [PATCH 04/18] evidence.py: optimize getting accummulated threat level --- slips_files/core/evidence.py | 98 ++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 50 deletions(-) diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index ce82df0f5..40603cf08 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -20,7 +20,7 @@ from slips_files.core.helpers.notify import Notify from slips_files.common.abstracts.core import ICore import json -from typing import Union, List +from typing import Union, List, Tuple from datetime import datetime from os import path from colorama import Fore, Style @@ -46,6 +46,7 @@ def init(self): self.separator = self.db.get_separator() self.read_configuration() self.detection_threshold_in_this_width = self.detection_threshold * self.width / 60 + self.running_non_stop = self.is_running_on_interface() # to keep track of the number of generated evidence self.db.init_evidence_number() if self.popup_alerts: @@ -467,8 +468,7 @@ def get_evidence_that_were_part_of_a_past_alert( past_evidence_ids = [] return past_evidence_ids - def is_evidence_done_by_others(self, evidence: str) -> bool: - evidence: str = json.loads(evidence) + def is_evidence_done_by_others(self, evidence: dict) -> bool: # given all the tw evidence, we should only # consider evidence that makes this given # profile malicious, aka evidence of this profile attacking others. @@ -482,7 +482,7 @@ def is_evidence_done_by_others(self, evidence: str) -> bool: def get_evidence_for_tw(self, profileid: str, twid: str) \ - -> Union[dict, bool]: + -> Tuple[dict, float]: """ filters and returns all the evidence for this profile in this TW filters the follwing: @@ -501,6 +501,7 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ past_evidence_ids: List[str] = \ self.get_evidence_that_were_part_of_a_past_alert(profileid, twid) + accumulated_threat_level = 0.0 res = {} for id, evidence in tw_evidence.items(): id: str @@ -517,6 +518,7 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ if id in past_evidence_ids: continue + evidence: dict = json.loads(evidence) if self.is_evidence_done_by_others(evidence): continue @@ -534,49 +536,48 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ if not processed: continue - res[id] = evidence - - return res + accumulated_threat_level: float = \ + self.accummulate_threat_level(evidence, accumulated_threat_level) + res[id] = json.dumps(evidence) + return res, accumulated_threat_level - def get_accumulated_threat_level(self, tw_evidence: dict): - """ - return the sum of all threat levels of all evidence in - the given tw evidence dict - """ - accumulated_threat_level = 0.0 + def accummulate_threat_level( + self, + evidence: dict, + accumulated_threat_level: float + ) -> float: # to store all the ids causing this alert in the database self.IDs_causing_an_alert = [] - for evidence in tw_evidence.values(): - evidence = json.loads(evidence) - # attacker_direction = evidence.get('attacker_direction') - # attacker = evidence.get('attacker') - evidence_type = evidence.get('evidence_type') - confidence = float(evidence.get('confidence')) - threat_level = evidence.get('threat_level') - description = evidence.get('description') - ID = evidence.get('ID') - self.IDs_causing_an_alert.append(ID) - - # each threat level is a string, get the numerical value of it - try: - threat_level: float = \ - utils.threat_levels[threat_level.lower()] - except KeyError: - self.print( - f'Error: Evidence of type {evidence_type} has ' - f'an invalid threat level {threat_level}', 0, 1 - ) - self.print(f'Description: {description}', 0, 1) - threat_level = 0 - - # Compute the moving average of evidence - new_threat_level: float = threat_level * confidence - self.print(f'\t\tWeighted Threat Level: {new_threat_level}', 3, 0) - accumulated_threat_level += new_threat_level + # attacker_direction = evidence.get('attacker_direction') + # attacker = evidence.get('attacker') + evidence_type: str = evidence.get('evidence_type') + confidence: float = float(evidence.get('confidence')) + threat_level: float = evidence.get('threat_level') + description: str = evidence.get('description') + id: str = evidence.get('ID') + #TODO + self.IDs_causing_an_alert.append(id) + + # each threat level is a string, get the numerical value of it + try: + threat_level: float = \ + utils.threat_levels[threat_level.lower()] + except KeyError: self.print( - f'\t\tAccumulated Threat Level: {accumulated_threat_level}', 3, 0, + f'Error: Evidence of type {evidence_type} has ' + f'an invalid threat level {threat_level}', 0, 1 ) + self.print(f'Description: {description}', 0, 1) + threat_level = 0 + + # Compute the moving average of evidence + new_threat_level: float = threat_level * confidence + self.print(f'\t\tWeighted Threat Level: {new_threat_level}', 3, 0) + accumulated_threat_level += new_threat_level + self.print( + f'\t\tAccumulated Threat Level: {accumulated_threat_level}', 3, 0, + ) return accumulated_threat_level def get_last_evidence_ID(self, tw_evidence: dict) -> str: @@ -648,7 +649,6 @@ def get_evidence_to_log( flow_datetime: str, profileid: str, ) -> str: - timewindow_number: int = twid.replace("timewindow", '') # to keep the alignment of alerts.json ip + hostname @@ -733,13 +733,12 @@ def main(self): proto, evidence_ID ) - - # Format the time to a common style given multiple type of time variables - if self.is_running_on_interface(): + # convert time to local timezone + if self.running_non_stop: timestamp: datetime = utils.convert_to_local_timezone(timestamp) flow_datetime = utils.convert_format(timestamp, 'iso') - evidence_to_log = self.get_evidence_to_log( + evidence_to_log: str = self.get_evidence_to_log( srcip, description, twid, @@ -749,10 +748,9 @@ def main(self): # Add the evidence to alerts.log self.add_to_log_file(evidence_to_log) - tw_evidence: dict = self.get_evidence_for_tw(profileid, twid) - # The accumulated threat level is for all the types of evidence for this profile - accumulated_threat_level: float = \ - self.get_accumulated_threat_level(tw_evidence) + tw_evidence: dict + accumulated_threat_level: float + tw_evidence, accumulated_threat_level= self.get_evidence_for_tw(profileid, twid) # add to alerts.json self.add_to_json_log_file( From 30783503326186b23a43d111995c0d6128ea7b6b Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 7 Dec 2023 18:33:18 +0200 Subject: [PATCH 05/18] evidence.py: move th labelling of flows causing an alert to the db --- slips_files/core/database/database_manager.py | 10 ++++++++++ slips_files/core/database/sqlite_db/database.py | 6 ++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/slips_files/core/database/database_manager.py b/slips_files/core/database/database_manager.py index e81a2e348..fb256423a 100644 --- a/slips_files/core/database/database_manager.py +++ b/slips_files/core/database/database_manager.py @@ -1,3 +1,5 @@ +from typing import List + from slips_files.core.database.redis_db.database import RedisDB from slips_files.core.database.sqlite_db.database import SQLiteDB from slips_files.common.parsers.config_parser import ConfigParser @@ -753,6 +755,14 @@ def set_ipv4_of_profile(self, *args, **kwargs): def get_mac_vendor_from_profile(self, *args, **kwargs): return self.rdb.get_mac_vendor_from_profile(*args, **kwargs) + def label_flows_causing_alert(self, evidence_ids: List[str]): + """ + :param evidence_ids: list of ids of evidence causing an alert + """ + for evidence_id in evidence_ids: + uids: List[str] = self.rdb.get_flows_causing_evidence(evidence_id) + self.set_flow_label(uids, 'malicious') + def set_mac_vendor_to_profile(self, *args, **kwargs): return self.rdb.set_mac_vendor_to_profile(*args, **kwargs) diff --git a/slips_files/core/database/sqlite_db/database.py b/slips_files/core/database/sqlite_db/database.py index 869d8ebde..eec15a3cb 100644 --- a/slips_files/core/database/sqlite_db/database.py +++ b/slips_files/core/database/sqlite_db/database.py @@ -1,3 +1,4 @@ +from typing import List import os.path import sqlite3 import json @@ -170,7 +171,7 @@ def get_all_flows(self): flow_list.append(json.loads(flow[1])) return flow_list - def set_flow_label(self, uids: list, new_label: str): + def set_flow_label(self, uids: List[str], new_label: str): """ sets the given new_label to each flow in the uids list """ @@ -301,7 +302,8 @@ def add_alert(self, alert: dict): :param alert: should contain alert_id, alert_ts, ip_alerted, twid, tw_start, tw_end, label alert_time is the local time slips detected this alert, not the network time """ - # 'alerts': 'alert_id TEXT PRIMARY KEY, alert_time TEXT, ip_alerted TEXT, timewindow TEXT, tw_start TEXT, tw_end TEXT, label TEXT' + # 'alerts': 'alert_id TEXT PRIMARY KEY, alert_time TEXT, ip_alerted TEXT, + # timewindow TEXT, tw_start TEXT, tw_end TEXT, label TEXT' self.execute( 'INSERT OR REPLACE INTO alerts (alert_id, ip_alerted, timewindow, tw_start, tw_end, label, alert_time) ' 'VALUES (?, ?, ?, ?, ?, ?, ?);', From 373fea3827d66665f2a1284f3eee6d66008bfc42 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 7 Dec 2023 18:34:21 +0200 Subject: [PATCH 06/18] evidence.py: minimize json dumping and loading tw_evidence --- slips_files/core/evidence.py | 82 +++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index 40603cf08..a490c0123 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -20,7 +20,7 @@ from slips_files.core.helpers.notify import Notify from slips_files.common.abstracts.core import ICore import json -from typing import Union, List, Tuple +from typing import Union, List, Tuple, Dict from datetime import datetime from os import path from colorama import Fore, Style @@ -256,15 +256,22 @@ def show_popup(self, alert_to_log: str): def format_evidence_causing_this_alert( - self, all_evidence, profileid, twid, flow_datetime + self, + all_evidence: Dict[str, dict], + profileid: str, + twid: str, + flow_datetime: str ) -> str: """ Function to format the string with all evidence causing an alert flow_datetime: time of the last evidence received """ - # alerts in slips consists of several evidence, each evidence has a threat_level - # once we reach a certain threshold of accumulated threat_levels, we produce an alert - # Now instead of printing the last evidence only, we print all of them + # alerts in slips consists of several evidence, + # each evidence has a threat_level + # once we reach a certain threshold of accumulated + # threat_levels, we produce an alert + # Now instead of printing the last evidence only, + # we print all of them try: twid_num = twid.split('timewindow')[1] srcip = profileid.split(self.separator)[1] @@ -274,7 +281,8 @@ def format_evidence_causing_this_alert( # give the database time to retreive the time twid_start_time = self.db.getTimeTW(profileid, twid) - tw_start_time_str = utils.convert_format(twid_start_time, '%Y/%m/%d %H:%M:%S') + tw_start_time_str = utils.convert_format(twid_start_time, + '%Y/%m/%d %H:%M:%S') # datetime obj tw_start_time_datetime = utils.convert_to_datetime(tw_start_time_str) @@ -304,14 +312,15 @@ def format_evidence_causing_this_alert( except Exception: exception_line = sys.exc_info()[2].tb_lineno self.print( - f'Problem on format_evidence_causing_this_alert() line {exception_line}',0,1, + f'Problem on format_evidence_causing_this_alert() ' + f'line {exception_line}',0,1, ) self.print(traceback.print_exc(),0,1) return True for evidence in all_evidence.values(): - evidence = json.loads(evidence) - description = evidence.get('description') + evidence: dict + description: str = evidence.get('description') evidence_string = f'Detected {description}' evidence_string = self.line_wrap(evidence_string) @@ -482,7 +491,7 @@ def is_evidence_done_by_others(self, evidence: dict) -> bool: def get_evidence_for_tw(self, profileid: str, twid: str) \ - -> Tuple[dict, float]: + -> Tuple[Dict[str, dict], float]: """ filters and returns all the evidence for this profile in this TW filters the follwing: @@ -490,6 +499,9 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ * evidence that weren't done by the given profileid * evidence that are whitelisted * evidence that weren't processed by evidence.py yet + + returns the dict with filtered evidence + and the accumulated threat levels of them """ tw_evidence: str = self.db.getEvidenceForTW(profileid, twid) if not tw_evidence: @@ -502,7 +514,10 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ self.get_evidence_that_were_part_of_a_past_alert(profileid, twid) accumulated_threat_level = 0.0 - res = {} + # to store all the ids causing this alert in the database + self.IDs_causing_an_alert = [] + + filtered_evidence = {} for id, evidence in tw_evidence.items(): id: str evidence: str @@ -537,27 +552,30 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ continue accumulated_threat_level: float = \ - self.accummulate_threat_level(evidence, accumulated_threat_level) - res[id] = json.dumps(evidence) + self.accummulate_threat_level( + evidence, + accumulated_threat_level + ) + id: str = evidence.get('ID') + # we keep track of these IDs to be able to label the flows of these + # evidence later if this was detected as an alert + self.IDs_causing_an_alert.append(id) - return res, accumulated_threat_level + filtered_evidence[id] = evidence + + return filtered_evidence, accumulated_threat_level def accummulate_threat_level( self, evidence: dict, accumulated_threat_level: float ) -> float: - # to store all the ids causing this alert in the database - self.IDs_causing_an_alert = [] # attacker_direction = evidence.get('attacker_direction') # attacker = evidence.get('attacker') evidence_type: str = evidence.get('evidence_type') confidence: float = float(evidence.get('confidence')) threat_level: float = evidence.get('threat_level') description: str = evidence.get('description') - id: str = evidence.get('ID') - #TODO - self.IDs_causing_an_alert.append(id) # each threat level is a string, get the numerical value of it try: @@ -583,13 +601,9 @@ def accummulate_threat_level( def get_last_evidence_ID(self, tw_evidence: dict) -> str: return list(tw_evidence.keys())[-1] - def send_to_exporting_module(self, tw_evidence): + def send_to_exporting_module(self, tw_evidence: Dict[str, str]): for evidence in tw_evidence.values(): - self.db.publish('export_evidence', evidence) - - def add_hostname_to_alert(self, alert_to_log, profileid, flow_datetime, evidence): - - return alert_to_log + self.db.publish('export_evidence', json.dumps(evidence)) def is_blocking_module_enabled(self) -> bool: """ @@ -601,12 +615,8 @@ def is_blocking_module_enabled(self) -> bool: custom_flows = '-im' in sys.argv or '--input-module' in sys.argv return (self.is_running_on_interface() and '-p' not in sys.argv) or custom_flows - def label_flows_causing_alert(self): - #todo should be moved to the db - """Add the label "malicious" to all flows causing this alert in our db """ - for evidence_id in self.IDs_causing_an_alert: - uids: list = self.db.get_flows_causing_evidence(evidence_id) - self.db.set_flow_label(uids, 'malicious') + + def handle_new_alert(self, alert_ID: str, tw_evidence: dict): """ @@ -638,7 +648,7 @@ def handle_new_alert(self, alert_ID: str, tw_evidence: dict): {'time_detected': utils.convert_format(datetime.now(), 'unixtimestamp'), 'label': 'malicious'}) self.db.add_alert(alert_details) - self.label_flows_causing_alert() + self.db.label_flows_causing_alert(self.IDs_causing_an_alert) self.send_to_exporting_module(tw_evidence) def get_evidence_to_log( @@ -748,9 +758,10 @@ def main(self): # Add the evidence to alerts.log self.add_to_log_file(evidence_to_log) - tw_evidence: dict + tw_evidence: Dict[str, dict] accumulated_threat_level: float - tw_evidence, accumulated_threat_level= self.get_evidence_for_tw(profileid, twid) + tw_evidence, accumulated_threat_level = \ + self.get_evidence_for_tw(profileid, twid) # add to alerts.json self.add_to_json_log_file( @@ -789,7 +800,8 @@ def main(self): self.handle_new_alert(alert_id, tw_evidence) # print the alert - alert_to_print = self.format_evidence_causing_this_alert( + alert_to_print: str = \ + self.format_evidence_causing_this_alert( tw_evidence, profileid, twid, From b2179ef6de2ab6711a0f13b310f7e4e8b5ab0b2e Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 8 Dec 2023 15:08:01 +0200 Subject: [PATCH 07/18] evidence.py: log filtered evidence (for debugging only) --- slips_files/core/evidence.py | 97 ++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 55 deletions(-) diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index a490c0123..2bb1e7328 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -416,55 +416,6 @@ def shutdown_gracefully(self): self.print("@@@@@@@@@@@@@@@@ discarded_bc_never_processed") self.print(self.discarded_bc_never_processed) - def delete_whitelisted_evidence(self, profileid, twid, evidence: dict) \ - -> dict: - """ - delete the hash of all whitelisted evidence - from the given dict of evidence ids - and the evidence that were returned from the db but not yet - processed by evidence.py - """ - res = {} - for evidence_ID, evidence_info in evidence.items(): - # sometimes the db has evidence that didnt come yet to evidence.py - # and they are alerted without checking the whitelist! - - # to fix this, we keep track of processed evidence - # that came to new_evidence channel and were processed by it. - # so they are ready to be a part of an alerted - whitelisted: bool = self.db.is_whitelisted_evidence(evidence_ID) - processed: bool = self.db.is_evidence_processed(evidence_ID) - client_ip = "192.168.1.113" - # client_ip = "10.0.2.15" - if not processed and client_ip in profileid: - self.log_filtered(f"Evidence {evidence_ID} .." - f" {evidence_info} is not processed yet, " - f"discarding") - try: - if evidence_ID not in self.discarded_bc_never_processed[profileid][twid]: - self.discarded_bc_never_processed[profileid][twid].append(evidence_ID) - except KeyError: - self.discarded_bc_never_processed.update({ - profileid: {twid: [evidence_ID]} - }) - - if processed and not whitelisted: - res[evidence_ID] = evidence_info - if client_ip in profileid: - try: - if evidence_ID in self.discarded_bc_never_processed[profileid][twid]: - self.log_filtered(f"Evidence Alerted after being " - f"discarded before {evidence_ID}") - self.discarded_bc_never_processed[profileid][ - twid].remove(evidence_ID) - except KeyError: - continue - - if client_ip in profileid: - self.log_filtered(f"done getting tw evidence {profileid} {twid}") - - return res - def get_evidence_that_were_part_of_a_past_alert( self, profileid: str, twid: str) -> List[str]: @@ -517,6 +468,9 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ # to store all the ids causing this alert in the database self.IDs_causing_an_alert = [] + client_ip = "192.168.1.113" + # client_ip = "192.168.1.129" + filtered_evidence = {} for id, evidence in tw_evidence.items(): id: str @@ -549,20 +503,52 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ # so they are ready to be a part of an alerted processed: bool = self.db.is_evidence_processed(id) if not processed: + if client_ip in profileid: + self.log_filtered(f"Evidence {id} .." + f" {evidence} is not processed yet, " + f"discarding") + try: + if id not in self.discarded_bc_never_processed[ + profileid][twid]: + self.discarded_bc_never_processed[profileid][ + twid].append(id) + except KeyError: + self.discarded_bc_never_processed.update({ + profileid: {twid: [id]} + }) continue - accumulated_threat_level: float = \ - self.accummulate_threat_level( - evidence, - accumulated_threat_level - ) id: str = evidence.get('ID') # we keep track of these IDs to be able to label the flows of these # evidence later if this was detected as an alert + # now this should be done in its' own function but this is more + # optimal so we don't loop through all evidence again. i'll + # just leave it like that:D self.IDs_causing_an_alert.append(id) + + accumulated_threat_level: float = \ + self.accummulate_threat_level( + evidence, + accumulated_threat_level + ) + filtered_evidence[id] = evidence + if client_ip in profileid: + try: + if id in self.discarded_bc_never_processed[ + profileid][twid]: + self.log_filtered(f"Evidence Alerted after being " + f"discarded before {id}") + self.discarded_bc_never_processed[profileid][ + twid].remove(id) + except KeyError: + pass + + if client_ip in profileid: + self.log_filtered(f"done getting tw evidence {profileid} {twid}") + return filtered_evidence, accumulated_threat_level def accummulate_threat_level( @@ -645,7 +631,8 @@ def handle_new_alert(self, alert_ID: str, tw_evidence: dict): #store the alerts in the alerts table in sqlite db alert_details.update( - {'time_detected': utils.convert_format(datetime.now(), 'unixtimestamp'), + {'time_detected': utils.convert_format(datetime.now(), + 'unixtimestamp'), 'label': 'malicious'}) self.db.add_alert(alert_details) self.db.label_flows_causing_alert(self.IDs_causing_an_alert) From 36c475f3a21c43450e3c79213c0f2620296a7c40 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 8 Dec 2023 17:13:11 +0200 Subject: [PATCH 08/18] CC: replace 'outTuple' attacker direction with "dstip" --- modules/http_analyzer/http_analyzer.py | 19 ++++++++++++++----- modules/p2ptrust/p2ptrust.py | 9 +++++++-- modules/rnn_cc_detection/rnn_cc_detection.py | 9 ++++----- .../threat_intelligence.py | 18 +++++++++++++++--- 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/modules/http_analyzer/http_analyzer.py b/modules/http_analyzer/http_analyzer.py index c168fcb35..3fb1f8474 100644 --- a/modules/http_analyzer/http_analyzer.py +++ b/modules/http_analyzer/http_analyzer.py @@ -424,11 +424,20 @@ def set_evidence_http_traffic(self, daddr, profileid, twid, uid, timestamp): attacker_direction = 'dstip' attacker = daddr saddr = profileid.split('_')[-1] - description = (f'Unencrypted HTTP traffic from {saddr} to {daddr}.') - - self.db.setEvidence(evidence_type, attacker_direction, attacker, threat_level, confidence, description, - timestamp, category, source_target_tag=source_target_tag, profileid=profileid, - twid=twid, uid=uid) + description = f'Unencrypted HTTP traffic from {saddr} to {daddr}.' + + self.db.setEvidence(evidence_type, + attacker_direction, + attacker, + threat_level, + confidence, + description, + timestamp, + category, + source_target_tag=source_target_tag, + profileid=profileid, + twid=twid, + uid=uid) return True diff --git a/modules/p2ptrust/p2ptrust.py b/modules/p2ptrust/p2ptrust.py index dc5eb609d..3e44f4639 100644 --- a/modules/p2ptrust/p2ptrust.py +++ b/modules/p2ptrust/p2ptrust.py @@ -395,7 +395,10 @@ def data_request_callback(self, msg: Dict): # # tell other peers that we're blocking this IP # utils.send_blame_to_go(ip_address, score, confidence, self.pygo_channel) - def set_evidence_malicious_ip(self, ip_info, threat_level, confidence): + def set_evidence_malicious_ip(self, + ip_info: dict, + threat_level: str, + confidence: float): """ Set an evidence for a malicious IP met in the timewindow ip_info format is json serialized { @@ -403,7 +406,9 @@ def set_evidence_malicious_ip(self, ip_info, threat_level, confidence): # 'profileid' : profile where the alert was generated. It includes the src ip # 'twid' : name of the timewindow when it happened. # 'proto' : protocol - # 'ip_state' : 'srcip/dstip', + # 'ip_state' : is basically the answer to "which one is the + # blacklisted IP"?'can be 'srcip' or + # 'dstip', # 'stime': Exact time when the evidence happened # 'uid': Zeek uid of the flow that generated the evidence, # 'cache_age': How old is the info about this ip diff --git a/modules/rnn_cc_detection/rnn_cc_detection.py b/modules/rnn_cc_detection/rnn_cc_detection.py index 4ee68c581..6948e18bd 100644 --- a/modules/rnn_cc_detection/rnn_cc_detection.py +++ b/modules/rnn_cc_detection/rnn_cc_detection.py @@ -40,14 +40,14 @@ def set_evidence( Set an evidence for malicious Tuple """ - attacker_direction = 'outTuple' - attacker = tupleid + tupleid = tupleid.split('-') + dstip, port, proto = tupleid[0], tupleid[1], tupleid[2] + attacker_direction = 'dstip' + attacker = dstip source_target_tag = 'Botnet' evidence_type = 'Command-and-Control-channels-detection' threat_level = 'high' categroy = 'Intrusion.Botnet' - tupleid = tupleid.split('-') - dstip, port, proto = tupleid[0], tupleid[1], tupleid[2] portproto = f'{port}/{proto}' port_info = self.db.get_port_info(portproto) ip_identification = self.db.get_ip_identification(dstip) @@ -75,7 +75,6 @@ def set_evidence( victim= victim) - def convert_input_for_module(self, pre_behavioral_model): """ Takes the input from the letters and converts them diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index 9f85c3e29..393bdf6be 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -149,7 +149,8 @@ def set_evidence_malicious_ip( :param ip_info: is all the info we have about that IP in the db source, confidence, description, etc. :param profileid: profile where the alert was generated. It includes the src ip :param twid: name of the timewindow when it happened. - :param ip_state: can be 'srcip' or 'dstip' + :param ip_state: is basically the answer to "which one is the + blacklisted IP"? can be 'srcip' or 'dstip' """ attacker_direction = ip_state @@ -852,8 +853,19 @@ def search_offline_for_domain(self, domain): def search_online_for_url(self, url): return self.urlhaus.urlhaus_lookup(url, 'url') - def is_malicious_ip(self, ip, uid, daddr, timestamp, profileid, twid, ip_state) -> bool: - """Search for this IP in our database of IoC""" + def is_malicious_ip(self, + ip: str, + uid: str, + daddr: str, + timestamp: str, + profileid: str, + twid: str, + ip_state: str) -> bool: + """ + Search for this IP in our database of IoC + :param ip_state: is basically the answer to "which one is the + blacklisted IP"? can be 'srcip' or 'dstip' + """ ip_info = self.search_offline_for_ip(ip) if not ip_info: ip_info = self.search_online_for_ip(ip) From a2027fa64440a0661ee06412ff552673e106e416 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 8 Dec 2023 17:14:12 +0200 Subject: [PATCH 09/18] remove debugging prints --- slips_files/core/evidence.py | 38 ------------------------------------ 1 file changed, 38 deletions(-) diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index 2bb1e7328..ef8fa7ca9 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -77,8 +77,6 @@ def init(self): self.print(f'Storing Slips logs in {self.output_dir}') # this list will have our local and public ips when using -i self.our_ips = utils.get_own_IPs() - filtered_log_path = os.path.join(self.output_dir, 'filtered_ev.log') - self.filtered_log = open(filtered_log_path, 'w') self.discarded_bc_never_processed = {} def read_configuration(self): @@ -98,9 +96,6 @@ def read_configuration(self): if IS_IN_A_DOCKER_CONTAINER: self.popup_alerts = False - def log_filtered(self, txt): - self.filtered_log.write(txt+'\n') - def format_evidence_string(self, ip, detection_module, attacker, description) -> str: """ @@ -412,9 +407,6 @@ def mark_as_blocked( def shutdown_gracefully(self): self.logfile.close() self.jsonfile.close() - self.filtered_log.close() - self.print("@@@@@@@@@@@@@@@@ discarded_bc_never_processed") - self.print(self.discarded_bc_never_processed) def get_evidence_that_were_part_of_a_past_alert( self, profileid: str, twid: str) -> List[str]: @@ -468,9 +460,6 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ # to store all the ids causing this alert in the database self.IDs_causing_an_alert = [] - client_ip = "192.168.1.113" - # client_ip = "192.168.1.129" - filtered_evidence = {} for id, evidence in tw_evidence.items(): id: str @@ -503,19 +492,6 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ # so they are ready to be a part of an alerted processed: bool = self.db.is_evidence_processed(id) if not processed: - if client_ip in profileid: - self.log_filtered(f"Evidence {id} .." - f" {evidence} is not processed yet, " - f"discarding") - try: - if id not in self.discarded_bc_never_processed[ - profileid][twid]: - self.discarded_bc_never_processed[profileid][ - twid].append(id) - except KeyError: - self.discarded_bc_never_processed.update({ - profileid: {twid: [id]} - }) continue id: str = evidence.get('ID') @@ -535,20 +511,6 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ filtered_evidence[id] = evidence - if client_ip in profileid: - try: - if id in self.discarded_bc_never_processed[ - profileid][twid]: - self.log_filtered(f"Evidence Alerted after being " - f"discarded before {id}") - self.discarded_bc_never_processed[profileid][ - twid].remove(id) - except KeyError: - pass - - if client_ip in profileid: - self.log_filtered(f"done getting tw evidence {profileid} {twid}") - return filtered_evidence, accumulated_threat_level def accummulate_threat_level( From 5bcd147cda4ebc896e594d2075e82f4a0c74258f Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 7 Dec 2023 16:28:31 +0200 Subject: [PATCH 10/18] print not processed filtered evidence (for debugging) --- modules/rnn_cc_detection/rnn_cc_detection.py | 19 ++- .../threat_intelligence.py | 83 +++++----- slips_files/common/slips_utils.py | 2 +- .../core/database/redis_db/alert_handler.py | 14 +- slips_files/core/evidence.py | 151 ++++++++++++++---- 5 files changed, 182 insertions(+), 87 deletions(-) diff --git a/modules/rnn_cc_detection/rnn_cc_detection.py b/modules/rnn_cc_detection/rnn_cc_detection.py index 19efd6f43..4ee68c581 100644 --- a/modules/rnn_cc_detection/rnn_cc_detection.py +++ b/modules/rnn_cc_detection/rnn_cc_detection.py @@ -57,9 +57,22 @@ def set_evidence( f'score: {format(score, ".4f")}. {ip_identification}' ) victim = profileid.split('_')[-1] - self.db.setEvidence(evidence_type, attacker_direction, attacker, threat_level, confidence, description, - timestamp, categroy, source_target_tag=source_target_tag, port=port, proto=proto, - profileid=profileid, twid=twid, uid=uid, victim= victim) + self.db.setEvidence( + evidence_type, + attacker_direction, + attacker, + threat_level, + confidence, + description, + timestamp, + categroy, + source_target_tag=source_target_tag, + port=port, + proto=proto, + profileid=profileid, + twid=twid, + uid=uid, + victim= victim) diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index cbd696d5b..9f85c3e29 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -276,9 +276,19 @@ def set_evidence_malicious_domain( if tags: description += f'with tags: {tags}. ' - self.db.setEvidence(evidence_type, attacker_direction, attacker, threat_level, confidence, description, - timestamp, category, source_target_tag=source_target_tag, profileid=profileid, - twid=twid, uid=uid) + self.db.setEvidence( + evidence_type, + attacker_direction, + attacker, + threat_level, + confidence, + description, + timestamp, + category, + source_target_tag=source_target_tag, + profileid=profileid, + twid=twid, + uid=uid) def is_valid_threat_level(self, threat_level): return threat_level in utils.threat_levels @@ -327,63 +337,43 @@ def parse_local_ti_file(self, ti_file_path: str) -> bool: # default value threat_level = 'medium' + ioc_info = { + 'description': description, + 'source': data_file_name, + 'threat_level': threat_level, + 'tags': 'local TI file', + } + ioc_info: str = json.dumps(ioc_info) + data_type = utils.detect_data_type(ioc.strip()) if data_type == 'ip': ip_address = ipaddress.ip_address(ioc.strip()) - # Only use global addresses. Ignore multicast, broadcast, private, reserved and undefined + # Only use global addresses. Ignore multicast, + # broadcast, private, reserved and undefined if ip_address.is_global: - # Store the ip in our local dict - malicious_ips[str(ip_address)] = json.dumps( - { - 'description': description, - 'source': data_file_name, - 'threat_level': threat_level, - 'tags': 'local TI file', - } - ) + malicious_ips[str(ip_address)] = ioc_info + elif data_type == 'domain': - malicious_domains[ioc] = json.dumps( - { - 'description': description, - 'source': data_file_name, - 'threat_level': threat_level, - 'tags': 'local TI file', - } - ) + malicious_domains[ioc] = ioc_info + elif data_type == 'ip_range': net_addr = ioc[: ioc.index('/')] - ip_obj = ipaddress.ip_address(net_addr) if ( - ip_obj.is_multicast - or utils.is_private_ip(ip_obj) - or ip_obj.is_link_local + utils.is_ignored_ip(net_addr) or net_addr in utils.home_networks ): continue - malicious_ip_ranges[ioc] = json.dumps( - { - 'description': description, - 'source': data_file_name, - 'threat_level': threat_level, - 'tags': 'local TI file', - } - ) + malicious_ip_ranges[ioc] = ioc_info + elif data_type == 'asn': - malicious_asns[ioc] = json.dumps( - { - 'description': description, - 'source': data_file_name, - 'threat_level': threat_level, - 'tags': 'local TI file', - } - ) + malicious_asns[ioc] = ioc_info else: # invalid ioc, skip it self.print( f'Error while reading the TI file {ti_file_path}.' f' Line {line_number} has invalid data: {ioc}', - 0, 1, + 0, 1 ) # Add all loaded malicious ips to the database @@ -458,7 +448,8 @@ def parse_ja3_file(self, path): # "JA3 hash", "Threat level", "Description" data = line.replace('\n', '').replace('"', '').split(',') - # the column order is hardcoded because it's owr own ti file and we know the format, + # the column order is hardcoded because it's owr + # own ti file and we know the format, # we shouldn't be trying to find it ja3, threat_level, description = ( data[0].strip(), @@ -515,7 +506,9 @@ def parse_jarm_file(self, path): if len(data) < 3: # invalid line continue - # the column order is hardcoded because it's owr own ti file and we know the format, + + # the column order is hardcoded because + # it's owr own ti file and we know the format, # we shouldn't be trying to find it jarm, threat_level, description = ( data[0].strip(), @@ -650,7 +643,7 @@ def spamhaus(self, ip): return { 'source': source_dataset, 'description': description, - 'therat_level': 'medium', + 'threat_level': 'medium', 'tags': 'spam', } diff --git a/slips_files/common/slips_utils.py b/slips_files/common/slips_utils.py index a601010b5..bc5c40fe2 100644 --- a/slips_files/common/slips_utils.py +++ b/slips_files/common/slips_utils.py @@ -332,7 +332,7 @@ def is_private_ip(self, ip_obj:ipaddress) -> bool: r_value = True return r_value - def is_ignored_ip(self, ip) -> bool: + def is_ignored_ip(self, ip: str) -> bool: """ This function checks if an IP is a special list of IPs that should not be alerted for different reasons diff --git a/slips_files/core/database/redis_db/alert_handler.py b/slips_files/core/database/redis_db/alert_handler.py index 260775615..8db0ec82b 100644 --- a/slips_files/core/database/redis_db/alert_handler.py +++ b/slips_files/core/database/redis_db/alert_handler.py @@ -209,7 +209,7 @@ def setEvidence( # every evidence should have an ID according to the IDEA format evidence_ID = str(uuid4()) - if type(uid) == list: + if isinstance(uid, list): # some evidence are caused by several uids, use the last one only # todo check why we have duplicates in the first place # remove duplicate uids @@ -219,9 +219,9 @@ def setEvidence( self.set_flow_causing_evidence(uids, evidence_ID) - if type(threat_level) != str: + if not isinstance(threat_level, str): # make sure we always store str threat levels in the db - threat_level = utils.threat_level_to_string(threat_level) + threat_level: str = utils.threat_level_to_string(threat_level) if timestamp: timestamp = utils.convert_format(timestamp, utils.alerts_format) @@ -301,13 +301,13 @@ def init_evidence_number(self): def get_evidence_number(self): return self.r.get('number_of_evidence') - def mark_evidence_as_processed(self, evidence_ID): + def mark_evidence_as_processed(self, evidence_ID: str): """ If an evidence was processed by the evidenceprocess, mark it in the db """ self.r.sadd('processed_evidence', evidence_ID) - def is_evidence_processed(self, evidence_ID): + def is_evidence_processed(self, evidence_ID: str) -> bool: return self.r.sismember('processed_evidence', evidence_ID) def set_evidence_for_profileid(self, evidence): @@ -404,10 +404,10 @@ def get_profileid_twid_alerts(self, profileid, twid) -> dict: The format for the returned dict is {profile123_twid1_: [ev_uuid1, ev_uuid2, ev_uuid3]} """ - alerts = self.r.hget(f'{profileid}{self.separator}{twid}', 'alerts') + alerts: str = self.r.hget(f'{profileid}_{twid}', 'alerts') if not alerts: return {} - alerts = json.loads(alerts) + alerts: dict = json.loads(alerts) return alerts def getEvidenceForTW(self, profileid: str, twid: str) -> str: diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index 3940e936f..90dbb06f1 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -20,6 +20,7 @@ from slips_files.core.helpers.notify import Notify from slips_files.common.abstracts.core import ICore import json +from typing import Union from datetime import datetime from os import path from colorama import Fore, Style @@ -75,6 +76,9 @@ def init(self): self.print(f'Storing Slips logs in {self.output_dir}') # this list will have our local and public ips when using -i self.our_ips = utils.get_own_IPs() + filtered_log_path = os.path.join(self.output_dir, 'filtered_ev.log') + self.filtered_log = open(filtered_log_path, 'w') + self.discarded_bc_never_processed = {} def read_configuration(self): conf = ConfigParser() @@ -93,6 +97,37 @@ def read_configuration(self): if IS_IN_A_DOCKER_CONTAINER: self.popup_alerts = False + + def format_evidence_string(self, ip, detection_module, attacker, + description) -> str: + """ + Function to add the dns resolution of the src and dst ips of + each evidence + :return : string with a correct evidence displacement + """ + evidence_string = '' + dns_resolution_attacker = self.db.get_dns_resolution(attacker) + dns_resolution_attacker = dns_resolution_attacker.get( + 'domains', [] + ) + dns_resolution_attacker = dns_resolution_attacker[ + :3] if dns_resolution_attacker else '' + + dns_resolution_ip = self.db.get_dns_resolution(ip) + dns_resolution_ip = dns_resolution_ip.get('domains', []) + if len(dns_resolution_ip) >= 1: + dns_resolution_ip = dns_resolution_ip[0] + elif len(dns_resolution_ip) == 0: + dns_resolution_ip = '' + + # dns_resolution_ip_final = f' DNS: {dns_resolution_ip[:3]}. ' if dns_resolution_attacker and len( + # dns_resolution_ip[:3] + # ) > 0 else '. ' + + + return f'{evidence_string}' + + def line_wrap(self, txt): """ is called for evidence that are goinng to be printed in the terminal @@ -365,40 +400,86 @@ def mark_as_blocked( def shutdown_gracefully(self): self.logfile.close() self.jsonfile.close() + self.filtered_log.close() + self.print("@@@@@@@@@@@@@@@@ discarded_bc_never_processed") + self.print(self.discarded_bc_never_processed) - def delete_alerted_evidence(self, profileid, twid, tw_evidence:dict): + def delete_alerted_evidence( + self, + profileid: str, + twid: str, + tw_evidence: dict) -> dict: """ if there was an alert in this tw before, remove the evidence that - were part of the past alert - from the current evidence + were part of the past alert from the current evidence. + + when blocking is not enabled, we can alert on a single profile many times + when we get all the tw evidence from the db, we get the once we + alerted, and the new once we need to alert + this method removes the already alerted evidence to avoid duplicates + + :param tw_evidence: dict with {: {evidence_details}} """ - # format of tw_evidence is {: {evidence_details}} - past_alerts = self.db.get_profileid_twid_alerts(profileid, twid) + past_alerts: dict = self.db.get_profileid_twid_alerts(profileid, twid) if not past_alerts: return tw_evidence for alert_id, evidence_IDs in past_alerts.items(): - evidence_IDs = json.loads(evidence_IDs) + alert_id: str + evidence_IDs: str # json serialized + + evidence_IDs: dict = json.loads(evidence_IDs) for ID in evidence_IDs: tw_evidence.pop(ID, None) return tw_evidence - def delete_whitelisted_evidence(self, evidence): + def delete_whitelisted_evidence(self, profileid, twid, evidence: dict) \ + -> dict: """ - delete the hash of all whitelisted evidence from the given dict of evidence ids + delete the hash of all whitelisted evidence + from the given dict of evidence ids + and the evidence that were returned from the db but not yet + processed by evidence.py """ res = {} for evidence_ID, evidence_info in evidence.items(): - # sometimes the db has evidence that didnt come yet to evidenceprocess + # sometimes the db has evidence that didnt come yet to evidence.py # and they are alerted without checking the whitelist! - # to fix this, we have processed evidence which are - # evidence that came to new_evidence channel and were processed by it + + # to fix this, we keep track of processed evidence + # that came to new_evidence channel and were processed by it. # so they are ready to be a part of an alerted - if ( - not self.db.is_whitelisted_evidence(evidence_ID) - and self.db.is_evidence_processed(evidence_ID) - ): + whitelisted: bool = self.db.is_whitelisted_evidence(evidence_ID) + processed: bool = self.db.is_evidence_processed(evidence_ID) + # client_ip = "192.168.1.113" + client_ip = "10.0.2.15" + if not processed and client_ip in profileid: + self.log_filtered(f"Evidence {evidence_ID} .." + f" {evidence_info} is not processed yet, " + f"discarding") + try: + if evidence_ID not in self.discarded_bc_never_processed[profileid][twid]: + self.discarded_bc_never_processed[profileid][twid].append(evidence_ID) + except KeyError: + self.discarded_bc_never_processed.update({ + profileid: {twid: [evidence_ID]} + }) + + if processed and not whitelisted: res[evidence_ID] = evidence_info + if client_ip in profileid: + try: + if evidence_ID in self.discarded_bc_never_processed[profileid][twid]: + self.log_filtered(f"Evidence Alerted after being " + f"discarded before {evidence_ID}") + self.discarded_bc_never_processed[profileid][ + twid].remove(evidence_ID) + except KeyError: + continue + + if client_ip in profileid: + self.log_filtered(f"done getting tw evidence {profileid} {twid}") + return res def delete_evidence_done_by_others(self, tw_evidence): @@ -419,24 +500,34 @@ def delete_evidence_done_by_others(self, tw_evidence): return res - def get_evidence_for_tw(self, profileid, twid): - # Get all the evidence for this profile in this TW - tw_evidence: str = self.db.getEvidenceForTW( - profileid, twid - ) + def get_evidence_for_tw(self, profileid: str, twid: str) \ + -> Union[dict, bool]: + """ + filters and returns all the evidence for this profile in this TW + filters the follwing: + * evidence that were part of a past alert in this same profileid twid + * evidence that weren't done by the given profileid + * evidence that are whitelisted + """ + tw_evidence: str = self.db.getEvidenceForTW(profileid, twid) if not tw_evidence: return False tw_evidence: dict = json.loads(tw_evidence) tw_evidence = self.delete_alerted_evidence(profileid, twid, tw_evidence) tw_evidence = self.delete_evidence_done_by_others(tw_evidence) - tw_evidence = self.delete_whitelisted_evidence(tw_evidence) + tw_evidence = self.delete_whitelisted_evidence(profileid, twid, + tw_evidence) return tw_evidence - def get_accumulated_threat_level(self, tw_evidence): + def get_accumulated_threat_level(self, tw_evidence: dict): + """ + return the sum of all threat levels of all evidence in + the given tw evidence dict + """ accumulated_threat_level = 0.0 - # to store all the ids causing this alerts in the database + # to store all the ids causing this alert in the database self.IDs_causing_an_alert = [] for evidence in tw_evidence.values(): evidence = json.loads(evidence) @@ -448,11 +539,11 @@ def get_accumulated_threat_level(self, tw_evidence): description = evidence.get('description') ID = evidence.get('ID') self.IDs_causing_an_alert.append(ID) + # each threat level is a string, get the numerical value of it try: - threat_level = utils.threat_levels[ - threat_level.lower() - ] + threat_level: float = \ + utils.threat_levels[threat_level.lower()] except KeyError: self.print( f'Error: Evidence of type {evidence_type} has ' @@ -462,17 +553,15 @@ def get_accumulated_threat_level(self, tw_evidence): threat_level = 0 # Compute the moving average of evidence - new_threat_level = threat_level * confidence - self.print( - f'\t\tWeighted Threat Level: {new_threat_level}', 3, 0 - ) + new_threat_level: float = threat_level * confidence + self.print(f'\t\tWeighted Threat Level: {new_threat_level}', 3, 0) accumulated_threat_level += new_threat_level self.print( f'\t\tAccumulated Threat Level: {accumulated_threat_level}', 3, 0, ) return accumulated_threat_level - def get_last_evidence_ID(self, tw_evidence) -> str: + def get_last_evidence_ID(self, tw_evidence: dict) -> str: return list(tw_evidence.keys())[-1] def send_to_exporting_module(self, tw_evidence): From b297ff6cb21f829277735666a94cffdd5bddc681 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 7 Dec 2023 16:36:16 +0200 Subject: [PATCH 11/18] change the client id --- slips_files/core/evidence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index 90dbb06f1..b2bc76e0e 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -451,8 +451,8 @@ def delete_whitelisted_evidence(self, profileid, twid, evidence: dict) \ # so they are ready to be a part of an alerted whitelisted: bool = self.db.is_whitelisted_evidence(evidence_ID) processed: bool = self.db.is_evidence_processed(evidence_ID) - # client_ip = "192.168.1.113" - client_ip = "10.0.2.15" + client_ip = "192.168.1.113" + # client_ip = "10.0.2.15" if not processed and client_ip in profileid: self.log_filtered(f"Evidence {evidence_ID} .." f" {evidence_info} is not processed yet, " From 840eb8a6f90d66610508bfd868beb2b7c7b0bc0f Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 7 Dec 2023 17:56:55 +0200 Subject: [PATCH 12/18] evidence.py: optimize getting tw evidence --- slips_files/core/evidence.py | 116 ++++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 50 deletions(-) diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index b2bc76e0e..577090158 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -20,7 +20,7 @@ from slips_files.core.helpers.notify import Notify from slips_files.common.abstracts.core import ICore import json -from typing import Union +from typing import Union, List from datetime import datetime from os import path from colorama import Fore, Style @@ -404,35 +404,6 @@ def shutdown_gracefully(self): self.print("@@@@@@@@@@@@@@@@ discarded_bc_never_processed") self.print(self.discarded_bc_never_processed) - def delete_alerted_evidence( - self, - profileid: str, - twid: str, - tw_evidence: dict) -> dict: - """ - if there was an alert in this tw before, remove the evidence that - were part of the past alert from the current evidence. - - when blocking is not enabled, we can alert on a single profile many times - when we get all the tw evidence from the db, we get the once we - alerted, and the new once we need to alert - this method removes the already alerted evidence to avoid duplicates - - :param tw_evidence: dict with {: {evidence_details}} - """ - past_alerts: dict = self.db.get_profileid_twid_alerts(profileid, twid) - if not past_alerts: - return tw_evidence - - for alert_id, evidence_IDs in past_alerts.items(): - alert_id: str - evidence_IDs: str # json serialized - - evidence_IDs: dict = json.loads(evidence_IDs) - for ID in evidence_IDs: - tw_evidence.pop(ID, None) - return tw_evidence - def delete_whitelisted_evidence(self, profileid, twid, evidence: dict) \ -> dict: """ @@ -482,22 +453,30 @@ def delete_whitelisted_evidence(self, profileid, twid, evidence: dict) \ return res - def delete_evidence_done_by_others(self, tw_evidence): - """ - given all the tw evidence, we should only consider evidence that makes this given - profile malicious, aka evidence of this profile attacking others. - """ - res = {} - for evidence_ID, evidence_info in tw_evidence.items(): - evidence_info = json.loads(evidence_info) - attacker_direction = evidence_info.get('attacker_direction', '') - # the following type detections are the ones - # expected to be seen when we are attacking others - # marking this profileid (srcip) as malicious - if attacker_direction in ('srcip', 'sport', 'srcport'): - res[evidence_ID] = json.dumps(evidence_info) + def get_evidence_that_were_part_of_a_past_alert( + self, profileid: str, twid: str) -> List[str]: - return res + past_alerts: dict = self.db.get_profileid_twid_alerts(profileid, twid) + try: + past_evidence_ids = list(past_alerts.values())[0] + past_evidence_ids: List[str] = json.loads(past_evidence_ids) + except IndexError: + # no past evidence + past_evidence_ids = [] + return past_evidence_ids + + def is_evidence_done_by_others(self, evidence: str) -> bool: + evidence: str = json.loads(evidence) + # given all the tw evidence, we should only + # consider evidence that makes this given + # profile malicious, aka evidence of this profile attacking others. + attacker_direction: str = evidence.get('attacker_direction', '') + # the following type detections are the ones + # expected to be seen when we are attacking others + # marking this profileid (srcip) as malicious + if attacker_direction in ('srcip', 'sport', 'srcport'): + return False + return True def get_evidence_for_tw(self, profileid: str, twid: str) \ @@ -508,17 +487,54 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ * evidence that were part of a past alert in this same profileid twid * evidence that weren't done by the given profileid * evidence that are whitelisted + * evidence that weren't processed by evidence.py yet """ tw_evidence: str = self.db.getEvidenceForTW(profileid, twid) if not tw_evidence: return False + # format of this is {ev_id, json_serialized(ev_details)} tw_evidence: dict = json.loads(tw_evidence) - tw_evidence = self.delete_alerted_evidence(profileid, twid, tw_evidence) - tw_evidence = self.delete_evidence_done_by_others(tw_evidence) - tw_evidence = self.delete_whitelisted_evidence(profileid, twid, - tw_evidence) - return tw_evidence + + past_evidence_ids: List[str] = \ + self.get_evidence_that_were_part_of_a_past_alert(profileid, twid) + + res = {} + for id, evidence in tw_evidence.items(): + id: str + evidence: str + + # delete already alerted evidence + # if there was an alert in this tw before, remove the evidence that + # were part of the past alert from the current evidence. + + # when blocking is not enabled, we can alert on a single profile many times + # when we get all the tw evidence from the db, we get the once we + # alerted, and the new once we need to alert + # this method removes the already alerted evidence to avoid duplicates + if id in past_evidence_ids: + continue + + if self.is_evidence_done_by_others(evidence): + continue + + whitelisted: bool = self.db.is_whitelisted_evidence(id) + if whitelisted: + continue + + # delete_not processed evidence + # sometimes the db has evidence that didn't come yet to evidence.py + # and they are alerted without checking the whitelist! + # to fix this, we keep track of processed evidence + # that came to new_evidence channel and were processed by it. + # so they are ready to be a part of an alerted + processed: bool = self.db.is_evidence_processed(id) + if not processed: + continue + + res[id] = evidence + + return res def get_accumulated_threat_level(self, tw_evidence: dict): From 6938381afb2b2f23fe93deb8b665313a85218d03 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 7 Dec 2023 18:11:52 +0200 Subject: [PATCH 13/18] evidence.py: optimize getting accummulated threat level --- slips_files/core/evidence.py | 100 +++++++++++++++++------------------ 1 file changed, 49 insertions(+), 51 deletions(-) diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index 577090158..04167d38d 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -20,7 +20,7 @@ from slips_files.core.helpers.notify import Notify from slips_files.common.abstracts.core import ICore import json -from typing import Union, List +from typing import Union, List, Tuple from datetime import datetime from os import path from colorama import Fore, Style @@ -46,6 +46,7 @@ def init(self): self.separator = self.db.get_separator() self.read_configuration() self.detection_threshold_in_this_width = self.detection_threshold * self.width / 60 + self.running_non_stop = self.is_running_on_interface() # to keep track of the number of generated evidence self.db.init_evidence_number() if self.popup_alerts: @@ -465,8 +466,7 @@ def get_evidence_that_were_part_of_a_past_alert( past_evidence_ids = [] return past_evidence_ids - def is_evidence_done_by_others(self, evidence: str) -> bool: - evidence: str = json.loads(evidence) + def is_evidence_done_by_others(self, evidence: dict) -> bool: # given all the tw evidence, we should only # consider evidence that makes this given # profile malicious, aka evidence of this profile attacking others. @@ -480,7 +480,7 @@ def is_evidence_done_by_others(self, evidence: str) -> bool: def get_evidence_for_tw(self, profileid: str, twid: str) \ - -> Union[dict, bool]: + -> Tuple[dict, float]: """ filters and returns all the evidence for this profile in this TW filters the follwing: @@ -499,6 +499,7 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ past_evidence_ids: List[str] = \ self.get_evidence_that_were_part_of_a_past_alert(profileid, twid) + accumulated_threat_level = 0.0 res = {} for id, evidence in tw_evidence.items(): id: str @@ -515,6 +516,7 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ if id in past_evidence_ids: continue + evidence: dict = json.loads(evidence) if self.is_evidence_done_by_others(evidence): continue @@ -532,49 +534,48 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ if not processed: continue - res[id] = evidence - - return res + accumulated_threat_level: float = \ + self.accummulate_threat_level(evidence, accumulated_threat_level) + res[id] = json.dumps(evidence) + return res, accumulated_threat_level - def get_accumulated_threat_level(self, tw_evidence: dict): - """ - return the sum of all threat levels of all evidence in - the given tw evidence dict - """ - accumulated_threat_level = 0.0 + def accummulate_threat_level( + self, + evidence: dict, + accumulated_threat_level: float + ) -> float: # to store all the ids causing this alert in the database self.IDs_causing_an_alert = [] - for evidence in tw_evidence.values(): - evidence = json.loads(evidence) - # attacker_direction = evidence.get('attacker_direction') - # attacker = evidence.get('attacker') - evidence_type = evidence.get('evidence_type') - confidence = float(evidence.get('confidence')) - threat_level = evidence.get('threat_level') - description = evidence.get('description') - ID = evidence.get('ID') - self.IDs_causing_an_alert.append(ID) - - # each threat level is a string, get the numerical value of it - try: - threat_level: float = \ - utils.threat_levels[threat_level.lower()] - except KeyError: - self.print( - f'Error: Evidence of type {evidence_type} has ' - f'an invalid threat level {threat_level}', 0, 1 - ) - self.print(f'Description: {description}', 0, 1) - threat_level = 0 - - # Compute the moving average of evidence - new_threat_level: float = threat_level * confidence - self.print(f'\t\tWeighted Threat Level: {new_threat_level}', 3, 0) - accumulated_threat_level += new_threat_level + # attacker_direction = evidence.get('attacker_direction') + # attacker = evidence.get('attacker') + evidence_type: str = evidence.get('evidence_type') + confidence: float = float(evidence.get('confidence')) + threat_level: float = evidence.get('threat_level') + description: str = evidence.get('description') + id: str = evidence.get('ID') + #TODO + self.IDs_causing_an_alert.append(id) + + # each threat level is a string, get the numerical value of it + try: + threat_level: float = \ + utils.threat_levels[threat_level.lower()] + except KeyError: self.print( - f'\t\tAccumulated Threat Level: {accumulated_threat_level}', 3, 0, + f'Error: Evidence of type {evidence_type} has ' + f'an invalid threat level {threat_level}', 0, 1 ) + self.print(f'Description: {description}', 0, 1) + threat_level = 0 + + # Compute the moving average of evidence + new_threat_level: float = threat_level * confidence + self.print(f'\t\tWeighted Threat Level: {new_threat_level}', 3, 0) + accumulated_threat_level += new_threat_level + self.print( + f'\t\tAccumulated Threat Level: {accumulated_threat_level}', 3, 0, + ) return accumulated_threat_level def get_last_evidence_ID(self, tw_evidence: dict) -> str: @@ -646,7 +647,6 @@ def get_evidence_to_log( flow_datetime: str, profileid: str, ) -> str: - timewindow_number: int = twid.replace("timewindow", '') # to keep the alignment of alerts.json ip + hostname @@ -668,6 +668,7 @@ def get_evidence_to_log( f'Detected {description}' return evidence + def increment_attack_counter( self, attacker: str, @@ -751,13 +752,12 @@ def main(self): proto, evidence_ID ) - - # Format the time to a common style given multiple type of time variables - if self.is_running_on_interface(): + # convert time to local timezone + if self.running_non_stop: timestamp: datetime = utils.convert_to_local_timezone(timestamp) flow_datetime = utils.convert_format(timestamp, 'iso') - evidence_to_log = self.get_evidence_to_log( + evidence_to_log: str = self.get_evidence_to_log( srcip, description, twid, @@ -766,13 +766,11 @@ def main(self): ) # Add the evidence to alerts.log self.add_to_log_file(evidence_to_log) - self.increment_attack_counter(profileid, victim, evidence_type) - tw_evidence: dict = self.get_evidence_for_tw(profileid, twid) - # The accumulated threat level is for all the types of evidence for this profile - accumulated_threat_level: float = \ - self.get_accumulated_threat_level(tw_evidence) + tw_evidence: dict + accumulated_threat_level: float + tw_evidence, accumulated_threat_level= self.get_evidence_for_tw(profileid, twid) # add to alerts.json self.add_to_json_log_file( From c2d57d2eabba8dbc9a7af8aee94423dfd0e09baf Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 7 Dec 2023 18:33:18 +0200 Subject: [PATCH 14/18] evidence.py: move th labelling of flows causing an alert to the db --- slips_files/core/database/database_manager.py | 10 ++++++++++ slips_files/core/database/sqlite_db/database.py | 6 ++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/slips_files/core/database/database_manager.py b/slips_files/core/database/database_manager.py index eec9c3892..4429b67bf 100644 --- a/slips_files/core/database/database_manager.py +++ b/slips_files/core/database/database_manager.py @@ -1,3 +1,5 @@ +from typing import List + from slips_files.core.database.redis_db.database import RedisDB from slips_files.core.database.sqlite_db.database import SQLiteDB from slips_files.common.parsers.config_parser import ConfigParser @@ -753,6 +755,14 @@ def set_ipv4_of_profile(self, *args, **kwargs): def get_mac_vendor_from_profile(self, *args, **kwargs): return self.rdb.get_mac_vendor_from_profile(*args, **kwargs) + def label_flows_causing_alert(self, evidence_ids: List[str]): + """ + :param evidence_ids: list of ids of evidence causing an alert + """ + for evidence_id in evidence_ids: + uids: List[str] = self.rdb.get_flows_causing_evidence(evidence_id) + self.set_flow_label(uids, 'malicious') + def set_mac_vendor_to_profile(self, *args, **kwargs): return self.rdb.set_mac_vendor_to_profile(*args, **kwargs) diff --git a/slips_files/core/database/sqlite_db/database.py b/slips_files/core/database/sqlite_db/database.py index 869d8ebde..eec15a3cb 100644 --- a/slips_files/core/database/sqlite_db/database.py +++ b/slips_files/core/database/sqlite_db/database.py @@ -1,3 +1,4 @@ +from typing import List import os.path import sqlite3 import json @@ -170,7 +171,7 @@ def get_all_flows(self): flow_list.append(json.loads(flow[1])) return flow_list - def set_flow_label(self, uids: list, new_label: str): + def set_flow_label(self, uids: List[str], new_label: str): """ sets the given new_label to each flow in the uids list """ @@ -301,7 +302,8 @@ def add_alert(self, alert: dict): :param alert: should contain alert_id, alert_ts, ip_alerted, twid, tw_start, tw_end, label alert_time is the local time slips detected this alert, not the network time """ - # 'alerts': 'alert_id TEXT PRIMARY KEY, alert_time TEXT, ip_alerted TEXT, timewindow TEXT, tw_start TEXT, tw_end TEXT, label TEXT' + # 'alerts': 'alert_id TEXT PRIMARY KEY, alert_time TEXT, ip_alerted TEXT, + # timewindow TEXT, tw_start TEXT, tw_end TEXT, label TEXT' self.execute( 'INSERT OR REPLACE INTO alerts (alert_id, ip_alerted, timewindow, tw_start, tw_end, label, alert_time) ' 'VALUES (?, ?, ?, ?, ?, ?, ?);', From 00025dc476546d3f63cd3457c2d684e7f5ce0021 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 7 Dec 2023 18:34:21 +0200 Subject: [PATCH 15/18] evidence.py: minimize json dumping and loading tw_evidence --- slips_files/core/evidence.py | 82 +++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index 04167d38d..8a11a3282 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -20,7 +20,7 @@ from slips_files.core.helpers.notify import Notify from slips_files.common.abstracts.core import ICore import json -from typing import Union, List, Tuple +from typing import Union, List, Tuple, Dict from datetime import datetime from os import path from colorama import Fore, Style @@ -254,15 +254,22 @@ def show_popup(self, alert_to_log: str): def format_evidence_causing_this_alert( - self, all_evidence, profileid, twid, flow_datetime + self, + all_evidence: Dict[str, dict], + profileid: str, + twid: str, + flow_datetime: str ) -> str: """ Function to format the string with all evidence causing an alert flow_datetime: time of the last evidence received """ - # alerts in slips consists of several evidence, each evidence has a threat_level - # once we reach a certain threshold of accumulated threat_levels, we produce an alert - # Now instead of printing the last evidence only, we print all of them + # alerts in slips consists of several evidence, + # each evidence has a threat_level + # once we reach a certain threshold of accumulated + # threat_levels, we produce an alert + # Now instead of printing the last evidence only, + # we print all of them try: twid_num = twid.split('timewindow')[1] srcip = profileid.split(self.separator)[1] @@ -272,7 +279,8 @@ def format_evidence_causing_this_alert( # give the database time to retreive the time twid_start_time = self.db.getTimeTW(profileid, twid) - tw_start_time_str = utils.convert_format(twid_start_time, '%Y/%m/%d %H:%M:%S') + tw_start_time_str = utils.convert_format(twid_start_time, + '%Y/%m/%d %H:%M:%S') # datetime obj tw_start_time_datetime = utils.convert_to_datetime(tw_start_time_str) @@ -302,14 +310,15 @@ def format_evidence_causing_this_alert( except Exception: exception_line = sys.exc_info()[2].tb_lineno self.print( - f'Problem on format_evidence_causing_this_alert() line {exception_line}',0,1, + f'Problem on format_evidence_causing_this_alert() ' + f'line {exception_line}',0,1, ) self.print(traceback.print_exc(),0,1) return True for evidence in all_evidence.values(): - evidence = json.loads(evidence) - description = evidence.get('description') + evidence: dict + description: str = evidence.get('description') evidence_string = f'Detected {description}' evidence_string = self.line_wrap(evidence_string) @@ -480,7 +489,7 @@ def is_evidence_done_by_others(self, evidence: dict) -> bool: def get_evidence_for_tw(self, profileid: str, twid: str) \ - -> Tuple[dict, float]: + -> Tuple[Dict[str, dict], float]: """ filters and returns all the evidence for this profile in this TW filters the follwing: @@ -488,6 +497,9 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ * evidence that weren't done by the given profileid * evidence that are whitelisted * evidence that weren't processed by evidence.py yet + + returns the dict with filtered evidence + and the accumulated threat levels of them """ tw_evidence: str = self.db.getEvidenceForTW(profileid, twid) if not tw_evidence: @@ -500,7 +512,10 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ self.get_evidence_that_were_part_of_a_past_alert(profileid, twid) accumulated_threat_level = 0.0 - res = {} + # to store all the ids causing this alert in the database + self.IDs_causing_an_alert = [] + + filtered_evidence = {} for id, evidence in tw_evidence.items(): id: str evidence: str @@ -535,27 +550,30 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ continue accumulated_threat_level: float = \ - self.accummulate_threat_level(evidence, accumulated_threat_level) - res[id] = json.dumps(evidence) + self.accummulate_threat_level( + evidence, + accumulated_threat_level + ) + id: str = evidence.get('ID') + # we keep track of these IDs to be able to label the flows of these + # evidence later if this was detected as an alert + self.IDs_causing_an_alert.append(id) - return res, accumulated_threat_level + filtered_evidence[id] = evidence + + return filtered_evidence, accumulated_threat_level def accummulate_threat_level( self, evidence: dict, accumulated_threat_level: float ) -> float: - # to store all the ids causing this alert in the database - self.IDs_causing_an_alert = [] # attacker_direction = evidence.get('attacker_direction') # attacker = evidence.get('attacker') evidence_type: str = evidence.get('evidence_type') confidence: float = float(evidence.get('confidence')) threat_level: float = evidence.get('threat_level') description: str = evidence.get('description') - id: str = evidence.get('ID') - #TODO - self.IDs_causing_an_alert.append(id) # each threat level is a string, get the numerical value of it try: @@ -581,13 +599,9 @@ def accummulate_threat_level( def get_last_evidence_ID(self, tw_evidence: dict) -> str: return list(tw_evidence.keys())[-1] - def send_to_exporting_module(self, tw_evidence): + def send_to_exporting_module(self, tw_evidence: Dict[str, str]): for evidence in tw_evidence.values(): - self.db.publish('export_evidence', evidence) - - def add_hostname_to_alert(self, alert_to_log, profileid, flow_datetime, evidence): - - return alert_to_log + self.db.publish('export_evidence', json.dumps(evidence)) def is_blocking_module_enabled(self) -> bool: """ @@ -599,12 +613,8 @@ def is_blocking_module_enabled(self) -> bool: custom_flows = '-im' in sys.argv or '--input-module' in sys.argv return (self.is_running_on_interface() and '-p' not in sys.argv) or custom_flows - def label_flows_causing_alert(self): - #todo should be moved to the db - """Add the label "malicious" to all flows causing this alert in our db """ - for evidence_id in self.IDs_causing_an_alert: - uids: list = self.db.get_flows_causing_evidence(evidence_id) - self.db.set_flow_label(uids, 'malicious') + + def handle_new_alert(self, alert_ID: str, tw_evidence: dict): """ @@ -636,7 +646,7 @@ def handle_new_alert(self, alert_ID: str, tw_evidence: dict): {'time_detected': utils.convert_format(datetime.now(), 'unixtimestamp'), 'label': 'malicious'}) self.db.add_alert(alert_details) - self.label_flows_causing_alert() + self.db.label_flows_causing_alert(self.IDs_causing_an_alert) self.send_to_exporting_module(tw_evidence) def get_evidence_to_log( @@ -768,9 +778,10 @@ def main(self): self.add_to_log_file(evidence_to_log) self.increment_attack_counter(profileid, victim, evidence_type) - tw_evidence: dict + tw_evidence: Dict[str, dict] accumulated_threat_level: float - tw_evidence, accumulated_threat_level= self.get_evidence_for_tw(profileid, twid) + tw_evidence, accumulated_threat_level = \ + self.get_evidence_for_tw(profileid, twid) # add to alerts.json self.add_to_json_log_file( @@ -809,7 +820,8 @@ def main(self): self.handle_new_alert(alert_id, tw_evidence) # print the alert - alert_to_print = self.format_evidence_causing_this_alert( + alert_to_print: str = \ + self.format_evidence_causing_this_alert( tw_evidence, profileid, twid, From a954b6c9c8ca2d35633c89e15d39873f19560aa7 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 8 Dec 2023 15:08:01 +0200 Subject: [PATCH 16/18] evidence.py: log filtered evidence (for debugging only) --- slips_files/core/evidence.py | 97 ++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 55 deletions(-) diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index 8a11a3282..3ff43c3e8 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -414,55 +414,6 @@ def shutdown_gracefully(self): self.print("@@@@@@@@@@@@@@@@ discarded_bc_never_processed") self.print(self.discarded_bc_never_processed) - def delete_whitelisted_evidence(self, profileid, twid, evidence: dict) \ - -> dict: - """ - delete the hash of all whitelisted evidence - from the given dict of evidence ids - and the evidence that were returned from the db but not yet - processed by evidence.py - """ - res = {} - for evidence_ID, evidence_info in evidence.items(): - # sometimes the db has evidence that didnt come yet to evidence.py - # and they are alerted without checking the whitelist! - - # to fix this, we keep track of processed evidence - # that came to new_evidence channel and were processed by it. - # so they are ready to be a part of an alerted - whitelisted: bool = self.db.is_whitelisted_evidence(evidence_ID) - processed: bool = self.db.is_evidence_processed(evidence_ID) - client_ip = "192.168.1.113" - # client_ip = "10.0.2.15" - if not processed and client_ip in profileid: - self.log_filtered(f"Evidence {evidence_ID} .." - f" {evidence_info} is not processed yet, " - f"discarding") - try: - if evidence_ID not in self.discarded_bc_never_processed[profileid][twid]: - self.discarded_bc_never_processed[profileid][twid].append(evidence_ID) - except KeyError: - self.discarded_bc_never_processed.update({ - profileid: {twid: [evidence_ID]} - }) - - if processed and not whitelisted: - res[evidence_ID] = evidence_info - if client_ip in profileid: - try: - if evidence_ID in self.discarded_bc_never_processed[profileid][twid]: - self.log_filtered(f"Evidence Alerted after being " - f"discarded before {evidence_ID}") - self.discarded_bc_never_processed[profileid][ - twid].remove(evidence_ID) - except KeyError: - continue - - if client_ip in profileid: - self.log_filtered(f"done getting tw evidence {profileid} {twid}") - - return res - def get_evidence_that_were_part_of_a_past_alert( self, profileid: str, twid: str) -> List[str]: @@ -515,6 +466,9 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ # to store all the ids causing this alert in the database self.IDs_causing_an_alert = [] + client_ip = "192.168.1.113" + # client_ip = "192.168.1.129" + filtered_evidence = {} for id, evidence in tw_evidence.items(): id: str @@ -547,20 +501,52 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ # so they are ready to be a part of an alerted processed: bool = self.db.is_evidence_processed(id) if not processed: + if client_ip in profileid: + self.log_filtered(f"Evidence {id} .." + f" {evidence} is not processed yet, " + f"discarding") + try: + if id not in self.discarded_bc_never_processed[ + profileid][twid]: + self.discarded_bc_never_processed[profileid][ + twid].append(id) + except KeyError: + self.discarded_bc_never_processed.update({ + profileid: {twid: [id]} + }) continue - accumulated_threat_level: float = \ - self.accummulate_threat_level( - evidence, - accumulated_threat_level - ) id: str = evidence.get('ID') # we keep track of these IDs to be able to label the flows of these # evidence later if this was detected as an alert + # now this should be done in its' own function but this is more + # optimal so we don't loop through all evidence again. i'll + # just leave it like that:D self.IDs_causing_an_alert.append(id) + + accumulated_threat_level: float = \ + self.accummulate_threat_level( + evidence, + accumulated_threat_level + ) + filtered_evidence[id] = evidence + if client_ip in profileid: + try: + if id in self.discarded_bc_never_processed[ + profileid][twid]: + self.log_filtered(f"Evidence Alerted after being " + f"discarded before {id}") + self.discarded_bc_never_processed[profileid][ + twid].remove(id) + except KeyError: + pass + + if client_ip in profileid: + self.log_filtered(f"done getting tw evidence {profileid} {twid}") + return filtered_evidence, accumulated_threat_level def accummulate_threat_level( @@ -643,7 +629,8 @@ def handle_new_alert(self, alert_ID: str, tw_evidence: dict): #store the alerts in the alerts table in sqlite db alert_details.update( - {'time_detected': utils.convert_format(datetime.now(), 'unixtimestamp'), + {'time_detected': utils.convert_format(datetime.now(), + 'unixtimestamp'), 'label': 'malicious'}) self.db.add_alert(alert_details) self.db.label_flows_causing_alert(self.IDs_causing_an_alert) From 1509ade861a611f97a3c47fd11a3a51c2cca4f83 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 8 Dec 2023 17:13:11 +0200 Subject: [PATCH 17/18] CC: replace 'outTuple' attacker direction with "dstip" --- modules/http_analyzer/http_analyzer.py | 19 ++++++++++++++----- modules/p2ptrust/p2ptrust.py | 9 +++++++-- modules/rnn_cc_detection/rnn_cc_detection.py | 9 ++++----- .../threat_intelligence.py | 18 +++++++++++++++--- 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/modules/http_analyzer/http_analyzer.py b/modules/http_analyzer/http_analyzer.py index c168fcb35..3fb1f8474 100644 --- a/modules/http_analyzer/http_analyzer.py +++ b/modules/http_analyzer/http_analyzer.py @@ -424,11 +424,20 @@ def set_evidence_http_traffic(self, daddr, profileid, twid, uid, timestamp): attacker_direction = 'dstip' attacker = daddr saddr = profileid.split('_')[-1] - description = (f'Unencrypted HTTP traffic from {saddr} to {daddr}.') - - self.db.setEvidence(evidence_type, attacker_direction, attacker, threat_level, confidence, description, - timestamp, category, source_target_tag=source_target_tag, profileid=profileid, - twid=twid, uid=uid) + description = f'Unencrypted HTTP traffic from {saddr} to {daddr}.' + + self.db.setEvidence(evidence_type, + attacker_direction, + attacker, + threat_level, + confidence, + description, + timestamp, + category, + source_target_tag=source_target_tag, + profileid=profileid, + twid=twid, + uid=uid) return True diff --git a/modules/p2ptrust/p2ptrust.py b/modules/p2ptrust/p2ptrust.py index dc5eb609d..3e44f4639 100644 --- a/modules/p2ptrust/p2ptrust.py +++ b/modules/p2ptrust/p2ptrust.py @@ -395,7 +395,10 @@ def data_request_callback(self, msg: Dict): # # tell other peers that we're blocking this IP # utils.send_blame_to_go(ip_address, score, confidence, self.pygo_channel) - def set_evidence_malicious_ip(self, ip_info, threat_level, confidence): + def set_evidence_malicious_ip(self, + ip_info: dict, + threat_level: str, + confidence: float): """ Set an evidence for a malicious IP met in the timewindow ip_info format is json serialized { @@ -403,7 +406,9 @@ def set_evidence_malicious_ip(self, ip_info, threat_level, confidence): # 'profileid' : profile where the alert was generated. It includes the src ip # 'twid' : name of the timewindow when it happened. # 'proto' : protocol - # 'ip_state' : 'srcip/dstip', + # 'ip_state' : is basically the answer to "which one is the + # blacklisted IP"?'can be 'srcip' or + # 'dstip', # 'stime': Exact time when the evidence happened # 'uid': Zeek uid of the flow that generated the evidence, # 'cache_age': How old is the info about this ip diff --git a/modules/rnn_cc_detection/rnn_cc_detection.py b/modules/rnn_cc_detection/rnn_cc_detection.py index 4ee68c581..6948e18bd 100644 --- a/modules/rnn_cc_detection/rnn_cc_detection.py +++ b/modules/rnn_cc_detection/rnn_cc_detection.py @@ -40,14 +40,14 @@ def set_evidence( Set an evidence for malicious Tuple """ - attacker_direction = 'outTuple' - attacker = tupleid + tupleid = tupleid.split('-') + dstip, port, proto = tupleid[0], tupleid[1], tupleid[2] + attacker_direction = 'dstip' + attacker = dstip source_target_tag = 'Botnet' evidence_type = 'Command-and-Control-channels-detection' threat_level = 'high' categroy = 'Intrusion.Botnet' - tupleid = tupleid.split('-') - dstip, port, proto = tupleid[0], tupleid[1], tupleid[2] portproto = f'{port}/{proto}' port_info = self.db.get_port_info(portproto) ip_identification = self.db.get_ip_identification(dstip) @@ -75,7 +75,6 @@ def set_evidence( victim= victim) - def convert_input_for_module(self, pre_behavioral_model): """ Takes the input from the letters and converts them diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index 9f85c3e29..393bdf6be 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -149,7 +149,8 @@ def set_evidence_malicious_ip( :param ip_info: is all the info we have about that IP in the db source, confidence, description, etc. :param profileid: profile where the alert was generated. It includes the src ip :param twid: name of the timewindow when it happened. - :param ip_state: can be 'srcip' or 'dstip' + :param ip_state: is basically the answer to "which one is the + blacklisted IP"? can be 'srcip' or 'dstip' """ attacker_direction = ip_state @@ -852,8 +853,19 @@ def search_offline_for_domain(self, domain): def search_online_for_url(self, url): return self.urlhaus.urlhaus_lookup(url, 'url') - def is_malicious_ip(self, ip, uid, daddr, timestamp, profileid, twid, ip_state) -> bool: - """Search for this IP in our database of IoC""" + def is_malicious_ip(self, + ip: str, + uid: str, + daddr: str, + timestamp: str, + profileid: str, + twid: str, + ip_state: str) -> bool: + """ + Search for this IP in our database of IoC + :param ip_state: is basically the answer to "which one is the + blacklisted IP"? can be 'srcip' or 'dstip' + """ ip_info = self.search_offline_for_ip(ip) if not ip_info: ip_info = self.search_online_for_ip(ip) From 3c32d79b74886e5ef12a7c960e6b77aaa62d2fbf Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 8 Dec 2023 17:14:12 +0200 Subject: [PATCH 18/18] remove debugging prints --- slips_files/core/evidence.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index 3ff43c3e8..ace0a1df4 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -77,8 +77,6 @@ def init(self): self.print(f'Storing Slips logs in {self.output_dir}') # this list will have our local and public ips when using -i self.our_ips = utils.get_own_IPs() - filtered_log_path = os.path.join(self.output_dir, 'filtered_ev.log') - self.filtered_log = open(filtered_log_path, 'w') self.discarded_bc_never_processed = {} def read_configuration(self): @@ -410,9 +408,6 @@ def mark_as_blocked( def shutdown_gracefully(self): self.logfile.close() self.jsonfile.close() - self.filtered_log.close() - self.print("@@@@@@@@@@@@@@@@ discarded_bc_never_processed") - self.print(self.discarded_bc_never_processed) def get_evidence_that_were_part_of_a_past_alert( self, profileid: str, twid: str) -> List[str]: @@ -466,9 +461,6 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ # to store all the ids causing this alert in the database self.IDs_causing_an_alert = [] - client_ip = "192.168.1.113" - # client_ip = "192.168.1.129" - filtered_evidence = {} for id, evidence in tw_evidence.items(): id: str @@ -501,19 +493,6 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ # so they are ready to be a part of an alerted processed: bool = self.db.is_evidence_processed(id) if not processed: - if client_ip in profileid: - self.log_filtered(f"Evidence {id} .." - f" {evidence} is not processed yet, " - f"discarding") - try: - if id not in self.discarded_bc_never_processed[ - profileid][twid]: - self.discarded_bc_never_processed[profileid][ - twid].append(id) - except KeyError: - self.discarded_bc_never_processed.update({ - profileid: {twid: [id]} - }) continue id: str = evidence.get('ID') @@ -533,20 +512,6 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ filtered_evidence[id] = evidence - if client_ip in profileid: - try: - if id in self.discarded_bc_never_processed[ - profileid][twid]: - self.log_filtered(f"Evidence Alerted after being " - f"discarded before {id}") - self.discarded_bc_never_processed[profileid][ - twid].remove(id) - except KeyError: - pass - - if client_ip in profileid: - self.log_filtered(f"done getting tw evidence {profileid} {twid}") - return filtered_evidence, accumulated_threat_level def accummulate_threat_level(