From c03c8d6f08e72d7c5f7da0e4496330030da56947 Mon Sep 17 00:00:00 2001 From: alya Date: Tue, 28 Nov 2023 15:02:40 +0200 Subject: [PATCH 1/2] add accummulated threat level to alerts.json --- slips_files/core/evidence.py | 60 +++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index 83ffd2553..949ded6e6 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -156,15 +156,23 @@ def clean_file(self, output_dir, file_to_clean): open(logfile_path, 'w').close() return open(logfile_path, 'a') - def add_to_json_log_file(self, IDEA_dict: dict, all_uids): + def add_to_json_log_file( + self, + IDEA_dict: dict, + all_uids, + accumulated_threat_level: float =0 + ): """ - Add a new evidence line to our alerts.json file in json IDEA format. + Add a new evidence line to our alerts.json file in json format. :param IDEA_dict: dict containing 1 alert :param all_uids: the uids of the flows causing this evidence """ try: # we add extra fields to alerts.json that are not in the IDEA format - IDEA_dict['uids'] = all_uids + IDEA_dict.update({ + 'uids': all_uids, + 'accumulated_threat_level': accumulated_threat_level + }) json.dump(IDEA_dict, self.jsonfile) self.jsonfile.write('\n') except KeyboardInterrupt: @@ -374,21 +382,18 @@ def mark_as_blocked( # log in alerts.log self.add_to_log_file(msg) - # log the alert - blocked_srcip_dict = { - 'type': 'alert', - 'profileid': profileid, - 'twid': twid, - 'threat_level': accumulated_threat_level, - } # Add a json field stating that this ip is blocked in alerts.json - # replace the evidence description with slip msg that this is a blocked profile - + # replace the evidence description with slips msg that this is a + # blocked profile IDEA_dict['Format'] = 'Json' IDEA_dict['Category'] = 'Alert' + IDEA_dict['profileid'] = profileid + IDEA_dict['twid'] = twid + IDEA_dict['threat_level'] = accumulated_threat_level IDEA_dict['Attach'][0]['Content'] = msg + # add to alerts.json - self.add_to_json_log_file(IDEA_dict, []) + self.add_to_json_log_file(IDEA_dict, [], accumulated_threat_level) def shutdown_gracefully(self): @@ -500,7 +505,7 @@ def get_accumulated_threat_level(self, tw_evidence): ) return accumulated_threat_level - def get_last_evidence_ID(self, tw_evidence): + def get_last_evidence_ID(self, tw_evidence) -> str: return list(tw_evidence.keys())[-1] def send_to_exporting_module(self, tw_evidence): @@ -627,9 +632,11 @@ def main(self): flow_datetime = utils.convert_format(timestamp, 'iso') # prepare evidence for text log file - evidence = self.format_evidence_string(srcip, evidence_type, attacker, description) + evidence_str = self.format_evidence_string(srcip, evidence_type, attacker, description) + + # prepare evidence for json log file - IDEA_dict = utils.IDEA_format( + IDEA_dict: dict = utils.IDEA_format( srcip, evidence_type, attacker_direction, @@ -645,27 +652,30 @@ def main(self): ) # to keep the alignment of alerts.json ip + hostname combined should take no more than 26 chars - alert_to_log = f'{flow_datetime}: Src IP {srcip:26}. {evidence}' - alert_to_log = self.add_hostname_to_alert(alert_to_log, profileid, flow_datetime, evidence) + alert_to_log = f'{flow_datetime}: Src IP {srcip:26}. {evidence_str}' + alert_to_log = self.add_hostname_to_alert(alert_to_log, profileid, flow_datetime, evidence_str) # Add the evidence to alerts.log self.add_to_log_file(alert_to_log) + tw_evidence: dict = self.get_evidence_for_tw(profileid, twid) + + # The accumulated threat level is for all the types of evidence for this profile + accumulated_threat_level: float = \ + self.get_accumulated_threat_level(tw_evidence) + # add to alerts.json - self.add_to_json_log_file(IDEA_dict, all_uids) + self.add_to_json_log_file(IDEA_dict, all_uids, accumulated_threat_level) self.db.set_evidence_for_profileid(IDEA_dict) self.db.publish('report_to_peers', json.dumps(data)) - if tw_evidence := self.get_evidence_for_tw(profileid, twid): + if tw_evidence: # self.print(f'Evidence: {tw_evidence}. Profileid {profileid}, twid {twid}') # Important! It may happen that the evidence is not related to a profileid and twid. # For example when the evidence is on some src IP attacking our home net, and we are not creating # profiles for attackers - # The accumulated threat level is for all the types of evidence for this profile - accumulated_threat_level = self.get_accumulated_threat_level(tw_evidence) - - ID = self.get_last_evidence_ID(tw_evidence) + id: str = self.get_last_evidence_ID(tw_evidence) # if the profile was already blocked in this twid, we shouldn't alert profile_already_blocked = self.db.checkBlockedProfTW(profileid, twid) @@ -680,7 +690,7 @@ def main(self): ): # store the alert in our database # the alert ID is profileid_twid + the ID of the last evidence causing this alert - alert_ID = f'{profileid}_{twid}_{ID}' + alert_ID = f'{profileid}_{twid}_{id}' self.handle_new_alert(alert_ID, tw_evidence) From cb0085c76157ef320629d3e163344ef0cd61f4a9 Mon Sep 17 00:00:00 2001 From: alya Date: Tue, 28 Nov 2023 15:34:09 +0200 Subject: [PATCH 2/2] add the tw number to alerts.log --- slips_files/core/evidence.py | 97 +++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 46 deletions(-) diff --git a/slips_files/core/evidence.py b/slips_files/core/evidence.py index 949ded6e6..c9531d9f1 100644 --- a/slips_files/core/evidence.py +++ b/slips_files/core/evidence.py @@ -94,9 +94,11 @@ def read_configuration(self): self.popup_alerts = False - def format_evidence_string(self, ip, detection_module, attacker, description): + def format_evidence_string(self, ip, detection_module, attacker, + description) -> str: """ - Function to format each evidence and enrich it with more data, to be displayed according to each detection module. + Function to add the dns resolution of the src and dst ips of + each evidence :return : string with a correct evidence displacement """ evidence_string = '' @@ -113,15 +115,11 @@ def format_evidence_string(self, ip, detection_module, attacker, description): dns_resolution_ip = dns_resolution_ip[0] elif len(dns_resolution_ip) == 0: dns_resolution_ip = '' + # dns_resolution_ip_final = f' DNS: {dns_resolution_ip[:3]}. ' if dns_resolution_attacker and len( # dns_resolution_ip[:3] # ) > 0 else '. ' - if detection_module == 'SSHSuccessful': - evidence_string = f'Did a successful SSH. {description}' - else: - evidence_string = f'Detected {description}' - return f'{evidence_string}' @@ -305,15 +303,9 @@ def format_evidence_causing_this_alert( for evidence in all_evidence.values(): evidence = json.loads(evidence) - attacker = evidence.get('attacker') - evidence_type = evidence.get('evidence_type') description = evidence.get('description') - # evidence_ID = evidence.get('ID') - # attacker_direction = evidence.get('attacker_direction') - # format the string of this evidence only: for example Detected C&C - # channels detection, destination IP:xyz - evidence_string = self.format_evidence_string(srcip, evidence_type, attacker, description) + evidence_string = f'Detected {description}' evidence_string = self.line_wrap(evidence_string) alert_to_print += ( @@ -513,16 +505,7 @@ def send_to_exporting_module(self, tw_evidence): self.db.publish('export_evidence', evidence) def add_hostname_to_alert(self, alert_to_log, profileid, flow_datetime, evidence): - # sometimes slips tries to get the hostname of a profile before ip_info stores it in the db - # there's nothing we can do about it - if hostname := self.db.get_hostname_from_profile(profileid): - srcip = profileid.split("_")[-1] - srcip = f'{srcip} ({hostname})' - # fill the rest of the 26 characters with spaces to keep the alignment - srcip = f'{srcip}{" "*(26-len(srcip))}' - alert_to_log = ( - f'{flow_datetime}: Src IP {srcip}. {evidence}' - ) + return alert_to_log def is_blocking_module_enabled(self) -> bool: @@ -575,6 +558,33 @@ def handle_new_alert(self, alert_ID: str, tw_evidence: dict): self.label_flows_causing_alert() self.send_to_exporting_module(tw_evidence) + def get_evidence_to_log( + self, + srcip: str, + description: str, + twid: str, + flow_datetime: str, + profileid: str, + ) -> str: + + timewindow_number: int = twid.replace("timewindow", '') + + # to keep the alignment of alerts.json ip + hostname + # combined should take no more than 26 chars + evidence = f'{flow_datetime} (TW {timewindow_number}): Src IP' \ + f' {srcip:26}. Detected {description}' + + # sometimes slips tries to get the hostname of a + # profile before ip_info stores it in the db + # there's nothing we can do about it + hostname: str = self.db.get_hostname_from_profile(profileid) + if not hostname: + return evidence + + # fill the rest of the 26 characters with spaces to keep the alignment + evidence = f'{flow_datetime}: Src IP {srcip} ({hostname}){" "*(26-len(srcip))}. {evidence}' + + return evidence def main(self): while not self.should_stop(): @@ -597,9 +607,7 @@ def main(self): timestamp = data.get('stime') # this is all the uids of the flows that cause this evidence all_uids = data.get('uid') - # tags = data.get('tags', False) confidence = data.get('confidence', False) - threat_level = data.get('threat_level', False) category = data.get('category', False) conn_count = data.get('conn_count', False) port = data.get('port', False) @@ -625,16 +633,6 @@ def main(self): ) continue - - # Format the time to a common style given multiple type of time variables - if self.is_running_on_interface(): - timestamp: datetime = utils.convert_to_local_timezone(timestamp) - flow_datetime = utils.convert_format(timestamp, 'iso') - - # prepare evidence for text log file - evidence_str = self.format_evidence_string(srcip, evidence_type, attacker, description) - - # prepare evidence for json log file IDEA_dict: dict = utils.IDEA_format( srcip, @@ -651,14 +649,22 @@ def main(self): evidence_ID ) - # to keep the alignment of alerts.json ip + hostname combined should take no more than 26 chars - alert_to_log = f'{flow_datetime}: Src IP {srcip:26}. {evidence_str}' - alert_to_log = self.add_hostname_to_alert(alert_to_log, profileid, flow_datetime, evidence_str) + # Format the time to a common style given multiple type of time variables + if self.is_running_on_interface(): + timestamp: datetime = utils.convert_to_local_timezone(timestamp) + flow_datetime = utils.convert_format(timestamp, 'iso') + evidence_to_log = self.get_evidence_to_log( + srcip, + description, + twid, + flow_datetime, + profileid + ) # Add the evidence to alerts.log - self.add_to_log_file(alert_to_log) - tw_evidence: dict = self.get_evidence_for_tw(profileid, twid) + self.add_to_log_file(evidence_to_log) + tw_evidence: dict = self.get_evidence_for_tw(profileid, twid) # The accumulated threat level is for all the types of evidence for this profile accumulated_threat_level: float = \ self.get_accumulated_threat_level(tw_evidence) @@ -690,19 +696,18 @@ def main(self): ): # store the alert in our database # the alert ID is profileid_twid + the ID of the last evidence causing this alert - alert_ID = f'{profileid}_{twid}_{id}' + alert_id: str = f'{profileid}_{twid}_{id}' - self.handle_new_alert(alert_ID, tw_evidence) + self.handle_new_alert(alert_id, tw_evidence) # print the alert - alert_to_print = ( - self.format_evidence_causing_this_alert( + alert_to_print = self.format_evidence_causing_this_alert( tw_evidence, profileid, twid, flow_datetime, ) - ) + self.print(f'{alert_to_print}', 1, 0) if self.popup_alerts: