Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

log accumulated threat lvl of and timewindow each evidence in alerts.log and alerts.json #421

Merged
merged 2 commits into from
Nov 28, 2023
+79 −64
Merged

log accumulated threat lvl of and timewindow each evidence in alerts.log and alerts.json #421

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c03c8d6
add accummulated threat level to alerts.json
AlyaGomaa Nov 28, 2023
cb0085c
add the tw number to alerts.log
AlyaGomaa Nov 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 79 additions & 64 deletions slips_files/core/evidence.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,11 @@ def read_configuration(self):
self.popup_alerts = False


def format_evidence_string(self, ip, detection_module, attacker, description):
def format_evidence_string(self, ip, detection_module, attacker,
description) -> str:
"""
Function to format each evidence and enrich it with more data, to be displayed according to each detection module.
Function to add the dns resolution of the src and dst ips of
each evidence
:return : string with a correct evidence displacement
"""
evidence_string = ''
Expand All @@ -113,15 +115,11 @@ def format_evidence_string(self, ip, detection_module, attacker, description):
dns_resolution_ip = dns_resolution_ip[0]
elif len(dns_resolution_ip) == 0:
dns_resolution_ip = ''

# dns_resolution_ip_final = f' DNS: {dns_resolution_ip[:3]}. ' if dns_resolution_attacker and len(
# dns_resolution_ip[:3]
# ) > 0 else '. '

if detection_module == 'SSHSuccessful':
evidence_string = f'Did a successful SSH. {description}'
else:
evidence_string = f'Detected {description}'


return f'{evidence_string}'

Expand Down Expand Up @@ -156,15 +154,23 @@ def clean_file(self, output_dir, file_to_clean):
open(logfile_path, 'w').close()
return open(logfile_path, 'a')

def add_to_json_log_file(self, IDEA_dict: dict, all_uids):
def add_to_json_log_file(
self,
IDEA_dict: dict,
all_uids,
accumulated_threat_level: float =0
):
"""
Add a new evidence line to our alerts.json file in json IDEA format.
Add a new evidence line to our alerts.json file in json format.
:param IDEA_dict: dict containing 1 alert
:param all_uids: the uids of the flows causing this evidence
"""
try:
# we add extra fields to alerts.json that are not in the IDEA format
IDEA_dict['uids'] = all_uids
IDEA_dict.update({
'uids': all_uids,
'accumulated_threat_level': accumulated_threat_level
})
json.dump(IDEA_dict, self.jsonfile)
self.jsonfile.write('\n')
except KeyboardInterrupt:
Expand Down Expand Up @@ -297,15 +303,9 @@ def format_evidence_causing_this_alert(

for evidence in all_evidence.values():
evidence = json.loads(evidence)
attacker = evidence.get('attacker')
evidence_type = evidence.get('evidence_type')
description = evidence.get('description')
# evidence_ID = evidence.get('ID')
# attacker_direction = evidence.get('attacker_direction')

# format the string of this evidence only: for example Detected C&C
# channels detection, destination IP:xyz
evidence_string = self.format_evidence_string(srcip, evidence_type, attacker, description)
evidence_string = f'Detected {description}'
evidence_string = self.line_wrap(evidence_string)

alert_to_print += (
Expand Down Expand Up @@ -374,21 +374,18 @@ def mark_as_blocked(
# log in alerts.log
self.add_to_log_file(msg)

# log the alert
blocked_srcip_dict = {
'type': 'alert',
'profileid': profileid,
'twid': twid,
'threat_level': accumulated_threat_level,
}
# Add a json field stating that this ip is blocked in alerts.json
# replace the evidence description with slip msg that this is a blocked profile

# replace the evidence description with slips msg that this is a
# blocked profile
IDEA_dict['Format'] = 'Json'
IDEA_dict['Category'] = 'Alert'
IDEA_dict['profileid'] = profileid
IDEA_dict['twid'] = twid
IDEA_dict['threat_level'] = accumulated_threat_level
IDEA_dict['Attach'][0]['Content'] = msg

# add to alerts.json
self.add_to_json_log_file(IDEA_dict, [])
self.add_to_json_log_file(IDEA_dict, [], accumulated_threat_level)


def shutdown_gracefully(self):
Expand Down Expand Up @@ -500,24 +497,15 @@ def get_accumulated_threat_level(self, tw_evidence):
)
return accumulated_threat_level

def get_last_evidence_ID(self, tw_evidence):
def get_last_evidence_ID(self, tw_evidence) -> str:
return list(tw_evidence.keys())[-1]

def send_to_exporting_module(self, tw_evidence):
for evidence in tw_evidence.values():
self.db.publish('export_evidence', evidence)

def add_hostname_to_alert(self, alert_to_log, profileid, flow_datetime, evidence):
# sometimes slips tries to get the hostname of a profile before ip_info stores it in the db
# there's nothing we can do about it
if hostname := self.db.get_hostname_from_profile(profileid):
srcip = profileid.split("_")[-1]
srcip = f'{srcip} ({hostname})'
# fill the rest of the 26 characters with spaces to keep the alignment
srcip = f'{srcip}{" "*(26-len(srcip))}'
alert_to_log = (
f'{flow_datetime}: Src IP {srcip}. {evidence}'
)

return alert_to_log

def is_blocking_module_enabled(self) -> bool:
Expand Down Expand Up @@ -570,6 +558,33 @@ def handle_new_alert(self, alert_ID: str, tw_evidence: dict):
self.label_flows_causing_alert()
self.send_to_exporting_module(tw_evidence)

def get_evidence_to_log(
self,
srcip: str,
description: str,
twid: str,
flow_datetime: str,
profileid: str,
) -> str:

timewindow_number: int = twid.replace("timewindow", '')

# to keep the alignment of alerts.json ip + hostname
# combined should take no more than 26 chars
evidence = f'{flow_datetime} (TW {timewindow_number}): Src IP' \
f' {srcip:26}. Detected {description}'

# sometimes slips tries to get the hostname of a
# profile before ip_info stores it in the db
# there's nothing we can do about it
hostname: str = self.db.get_hostname_from_profile(profileid)
if not hostname:
return evidence

# fill the rest of the 26 characters with spaces to keep the alignment
evidence = f'{flow_datetime}: Src IP {srcip} ({hostname}){" "*(26-len(srcip))}. {evidence}'

return evidence

def main(self):
while not self.should_stop():
Expand All @@ -592,9 +607,7 @@ def main(self):
timestamp = data.get('stime')
# this is all the uids of the flows that cause this evidence
all_uids = data.get('uid')
# tags = data.get('tags', False)
confidence = data.get('confidence', False)
threat_level = data.get('threat_level', False)
category = data.get('category', False)
conn_count = data.get('conn_count', False)
port = data.get('port', False)
Expand All @@ -620,16 +633,8 @@ def main(self):
)
continue


# Format the time to a common style given multiple type of time variables
if self.is_running_on_interface():
timestamp: datetime = utils.convert_to_local_timezone(timestamp)
flow_datetime = utils.convert_format(timestamp, 'iso')

# prepare evidence for text log file
evidence = self.format_evidence_string(srcip, evidence_type, attacker, description)
# prepare evidence for json log file
IDEA_dict = utils.IDEA_format(
IDEA_dict: dict = utils.IDEA_format(
srcip,
evidence_type,
attacker_direction,
Expand All @@ -644,28 +649,39 @@ def main(self):
evidence_ID
)

# to keep the alignment of alerts.json ip + hostname combined should take no more than 26 chars
alert_to_log = f'{flow_datetime}: Src IP {srcip:26}. {evidence}'
alert_to_log = self.add_hostname_to_alert(alert_to_log, profileid, flow_datetime, evidence)
# Format the time to a common style given multiple type of time variables
if self.is_running_on_interface():
timestamp: datetime = utils.convert_to_local_timezone(timestamp)
flow_datetime = utils.convert_format(timestamp, 'iso')

evidence_to_log = self.get_evidence_to_log(
srcip,
description,
twid,
flow_datetime,
profileid
)
# Add the evidence to alerts.log
self.add_to_log_file(alert_to_log)
self.add_to_log_file(evidence_to_log)

tw_evidence: dict = self.get_evidence_for_tw(profileid, twid)
# The accumulated threat level is for all the types of evidence for this profile
accumulated_threat_level: float = \
self.get_accumulated_threat_level(tw_evidence)

# add to alerts.json
self.add_to_json_log_file(IDEA_dict, all_uids)
self.add_to_json_log_file(IDEA_dict, all_uids, accumulated_threat_level)

self.db.set_evidence_for_profileid(IDEA_dict)
self.db.publish('report_to_peers', json.dumps(data))

if tw_evidence := self.get_evidence_for_tw(profileid, twid):
if tw_evidence:
# self.print(f'Evidence: {tw_evidence}. Profileid {profileid}, twid {twid}')
# Important! It may happen that the evidence is not related to a profileid and twid.
# For example when the evidence is on some src IP attacking our home net, and we are not creating
# profiles for attackers

# The accumulated threat level is for all the types of evidence for this profile
accumulated_threat_level = self.get_accumulated_threat_level(tw_evidence)

ID = self.get_last_evidence_ID(tw_evidence)
id: str = self.get_last_evidence_ID(tw_evidence)

# if the profile was already blocked in this twid, we shouldn't alert
profile_already_blocked = self.db.checkBlockedProfTW(profileid, twid)
Expand All @@ -680,19 +696,18 @@ def main(self):
):
# store the alert in our database
# the alert ID is profileid_twid + the ID of the last evidence causing this alert
alert_ID = f'{profileid}_{twid}_{ID}'
alert_id: str = f'{profileid}_{twid}_{id}'

self.handle_new_alert(alert_ID, tw_evidence)
self.handle_new_alert(alert_id, tw_evidence)

# print the alert
alert_to_print = (
self.format_evidence_causing_this_alert(
alert_to_print = self.format_evidence_causing_this_alert(
tw_evidence,
profileid,
twid,
flow_datetime,
)
)

self.print(f'{alert_to_print}', 1, 0)

if self.popup_alerts:
Expand Down
Loading