Merge pull request #431 from stratosphereips/alya/fix_accumulated_thr…

…eat_level

Optimize evidence.py

modules/http_analyzer/http_analyzer.py

@@ -424,11 +424,20 @@ def set_evidence_http_traffic(self, daddr, profileid, twid, uid, timestamp):
         attacker_direction = 'dstip'
         attacker = daddr
         saddr = profileid.split('_')[-1]
-        description = (f'Unencrypted HTTP traffic from {saddr} to {daddr}.')
-
-        self.db.setEvidence(evidence_type, attacker_direction, attacker, threat_level, confidence, description,
-                                 timestamp, category, source_target_tag=source_target_tag, profileid=profileid,
-                                 twid=twid, uid=uid)
+        description = f'Unencrypted HTTP traffic from {saddr} to {daddr}.'
+
+        self.db.setEvidence(evidence_type,
+                            attacker_direction,
+                            attacker,
+                            threat_level,
+                            confidence,
+                            description,
+                            timestamp,
+                            category,
+                            source_target_tag=source_target_tag,
+                            profileid=profileid,
+                            twid=twid,
+                            uid=uid)
         return True
 
 

modules/p2ptrust/p2ptrust.py

@@ -395,15 +395,20 @@ def data_request_callback(self, msg: Dict):
     #         # tell other peers that we're blocking this IP
     #         utils.send_blame_to_go(ip_address, score, confidence, self.pygo_channel)
 
-    def set_evidence_malicious_ip(self, ip_info, threat_level, confidence):
+    def set_evidence_malicious_ip(self,
+                                  ip_info: dict,
+                                  threat_level: str,
+                                  confidence: float):
         """
         Set an evidence for a malicious IP met in the timewindow
         ip_info format is json serialized {
         #             'ip': the source/dst ip
         #             'profileid' : profile where the alert was generated. It includes the src ip
         #             'twid' : name of the timewindow when it happened.
         #             'proto' : protocol
-        #             'ip_state' : 'srcip/dstip',
+        #             'ip_state' : is basically the answer to "which one is the
+        #                           blacklisted IP"?'can be 'srcip' or
+        #                            'dstip',
         #             'stime': Exact time when the evidence happened
         #             'uid': Zeek uid of the flow that generated the evidence,
         #             'cache_age': How old is the info about this ip

modules/rnn_cc_detection/rnn_cc_detection.py

@@ -40,14 +40,14 @@ def set_evidence(
         Set an evidence for malicious Tuple
         """
 
-        attacker_direction = 'outTuple'
-        attacker = tupleid
+        tupleid = tupleid.split('-')
+        dstip, port, proto = tupleid[0], tupleid[1], tupleid[2]
+        attacker_direction = 'dstip'
+        attacker = dstip
         source_target_tag = 'Botnet'
         evidence_type = 'Command-and-Control-channels-detection'
         threat_level = 'high'
         categroy = 'Intrusion.Botnet'
-        tupleid = tupleid.split('-')
-        dstip, port, proto = tupleid[0], tupleid[1], tupleid[2]
         portproto = f'{port}/{proto}'
         port_info = self.db.get_port_info(portproto)
         ip_identification = self.db.get_ip_identification(dstip)
@@ -57,10 +57,22 @@ def set_evidence(
             f'score: {format(score, ".4f")}. {ip_identification}'
         )
         victim = profileid.split('_')[-1]
-        self.db.setEvidence(evidence_type, attacker_direction, attacker, threat_level, confidence, description,
-                                 timestamp, categroy, source_target_tag=source_target_tag, port=port, proto=proto,
-                                 profileid=profileid, twid=twid, uid=uid, victim= victim)
-
+        self.db.setEvidence(
+            evidence_type,
+            attacker_direction,
+            attacker,
+            threat_level,
+            confidence,
+            description,
+            timestamp,
+            categroy,
+            source_target_tag=source_target_tag,
+            port=port,
+            proto=proto,
+            profileid=profileid,
+            twid=twid,
+            uid=uid,
+            victim= victim)
 
 
     def convert_input_for_module(self, pre_behavioral_model):

modules/threat_intelligence/threat_intelligence.py

@@ -149,7 +149,8 @@ def set_evidence_malicious_ip(
         :param ip_info: is all the info we have about that IP in the db source, confidence, description, etc.
         :param profileid: profile where the alert was generated. It includes the src ip
         :param twid: name of the timewindow when it happened.
-        :param ip_state: can be 'srcip' or 'dstip'
+        :param ip_state: is basically the answer to "which one is the
+        blacklisted IP"? can be 'srcip' or 'dstip'
         """
 
         attacker_direction = ip_state
@@ -276,9 +277,19 @@ def set_evidence_malicious_domain(
         if tags:
             description += f'with tags: {tags}. '
 
-        self.db.setEvidence(evidence_type, attacker_direction, attacker, threat_level, confidence, description,
-                                 timestamp, category, source_target_tag=source_target_tag, profileid=profileid,
-                                 twid=twid, uid=uid)
+        self.db.setEvidence(
+            evidence_type,
+            attacker_direction,
+            attacker,
+            threat_level,
+            confidence,
+            description,
+            timestamp,
+            category,
+            source_target_tag=source_target_tag,
+            profileid=profileid,
+            twid=twid,
+            uid=uid)
 
     def is_valid_threat_level(self, threat_level):
         return threat_level in utils.threat_levels
@@ -327,63 +338,43 @@ def parse_local_ti_file(self, ti_file_path: str) -> bool:
                     # default value
                     threat_level = 'medium'
 
+                ioc_info = {
+                    'description': description,
+                    'source': data_file_name,
+                    'threat_level': threat_level,
+                    'tags': 'local TI file',
+                }
+                ioc_info: str = json.dumps(ioc_info)
+
                 data_type = utils.detect_data_type(ioc.strip())
                 if data_type == 'ip':
                     ip_address = ipaddress.ip_address(ioc.strip())
-                    # Only use global addresses. Ignore multicast, broadcast, private, reserved and undefined
+                    # Only use global addresses. Ignore multicast,
+                    # broadcast, private, reserved and undefined
                     if ip_address.is_global:
-                        # Store the ip in our local dict
-                        malicious_ips[str(ip_address)] = json.dumps(
-                            {
-                                'description': description,
-                                'source': data_file_name,
-                                'threat_level': threat_level,
-                                'tags': 'local TI file',
-                            }
-                        )
+                        malicious_ips[str(ip_address)] = ioc_info
+
                 elif data_type == 'domain':
-                    malicious_domains[ioc] = json.dumps(
-                        {
-                            'description': description,
-                            'source': data_file_name,
-                            'threat_level': threat_level,
-                            'tags': 'local TI file',
-                        }
-                    )
+                    malicious_domains[ioc] = ioc_info
+
                 elif data_type == 'ip_range':
                     net_addr = ioc[: ioc.index('/')]
-                    ip_obj = ipaddress.ip_address(net_addr)
                     if (
-                        ip_obj.is_multicast
-                        or utils.is_private_ip(ip_obj)
-                        or ip_obj.is_link_local
+                        utils.is_ignored_ip(net_addr)
                         or net_addr in utils.home_networks
                     ):
                         continue
-                    malicious_ip_ranges[ioc] = json.dumps(
-                        {
-                            'description': description,
-                            'source': data_file_name,
-                            'threat_level': threat_level,
-                            'tags': 'local TI file',
-                        }
-                    )
+                    malicious_ip_ranges[ioc] = ioc_info
+
                 elif data_type == 'asn':
-                    malicious_asns[ioc] = json.dumps(
-                            {
-                                'description': description,
-                                'source': data_file_name,
-                                'threat_level': threat_level,
-                                'tags': 'local TI file',
-                            }
-                        )
+                    malicious_asns[ioc] = ioc_info
 
                 else:
                     # invalid ioc, skip it
                     self.print(
                         f'Error while reading the TI file {ti_file_path}.'
                         f' Line {line_number} has invalid data: {ioc}',
-                        0, 1,
+                        0, 1
                     )
 
         # Add all loaded malicious ips to the database
@@ -458,7 +449,8 @@ def parse_ja3_file(self, path):
                 # "JA3 hash", "Threat level", "Description"
                 data = line.replace('\n', '').replace('"', '').split(',')
 
-                # the column order is hardcoded because it's owr own ti file and we know the format,
+                # the column order is hardcoded because it's owr
+                # own ti file and we know the format,
                 # we shouldn't be trying to find it
                 ja3, threat_level, description = (
                     data[0].strip(),
@@ -515,7 +507,9 @@ def parse_jarm_file(self, path):
                 if len(data) < 3:
                     # invalid line
                     continue
-                # the column order is hardcoded because it's owr own ti file and we know the format,
+
+                # the column order is hardcoded because
+                # it's owr own ti file and we know the format,
                 # we shouldn't be trying to find it
                 jarm, threat_level, description = (
                     data[0].strip(),
@@ -650,7 +644,7 @@ def spamhaus(self, ip):
         return {
             'source': source_dataset,
             'description': description,
-            'therat_level': 'medium',
+            'threat_level': 'medium',
             'tags': 'spam',
         }
 
@@ -859,8 +853,19 @@ def search_offline_for_domain(self, domain):
     def search_online_for_url(self, url):
         return self.urlhaus.urlhaus_lookup(url, 'url')
 
-    def is_malicious_ip(self, ip, uid, daddr, timestamp, profileid, twid, ip_state) -> bool:
-        """Search for this IP in our database of IoC"""
+    def is_malicious_ip(self,
+                        ip: str,
+                        uid: str,
+                        daddr: str,
+                        timestamp: str,
+                        profileid: str,
+                        twid: str,
+                        ip_state: str) -> bool:
+        """
+        Search for this IP in our database of IoC
+        :param ip_state: is basically the answer to "which one is the
+        blacklisted IP"? can be 'srcip' or 'dstip'
+        """
         ip_info = self.search_offline_for_ip(ip)
         if not ip_info:
             ip_info = self.search_online_for_ip(ip)

slips_files/common/slips_utils.py

@@ -332,7 +332,7 @@ def is_private_ip(self, ip_obj:ipaddress) -> bool:
                 r_value = True
         return r_value
 
-    def is_ignored_ip(self, ip) -> bool:
+    def is_ignored_ip(self, ip: str) -> bool:
         """
         This function checks if an IP is a special list of IPs that
         should not be alerted for different reasons

slips_files/core/database/database_manager.py

@@ -1,3 +1,5 @@
+from typing import List
+
 from slips_files.core.database.redis_db.database import RedisDB
 from slips_files.core.database.sqlite_db.database import SQLiteDB
 from slips_files.common.parsers.config_parser import ConfigParser
@@ -753,6 +755,14 @@ def set_ipv4_of_profile(self, *args, **kwargs):
     def get_mac_vendor_from_profile(self, *args, **kwargs):
         return self.rdb.get_mac_vendor_from_profile(*args, **kwargs)
 
+    def label_flows_causing_alert(self, evidence_ids: List[str]):
+        """
+        :param evidence_ids: list of ids of evidence causing an alert
+        """
+        for evidence_id in evidence_ids:
+            uids: List[str] = self.rdb.get_flows_causing_evidence(evidence_id)
+            self.set_flow_label(uids, 'malicious')
+
     def set_mac_vendor_to_profile(self, *args, **kwargs):
         return self.rdb.set_mac_vendor_to_profile(*args, **kwargs)
 

slips_files/core/database/redis_db/alert_handler.py

@@ -209,7 +209,7 @@ def setEvidence(
         # every evidence should have an ID according to the IDEA format
         evidence_ID = str(uuid4())
 
-        if type(uid) == list:
+        if isinstance(uid, list):
             # some evidence are caused by several uids, use the last one only
             # todo check why we have duplicates in the first place
             # remove duplicate uids
@@ -219,9 +219,9 @@ def setEvidence(
 
         self.set_flow_causing_evidence(uids, evidence_ID)
 
-        if type(threat_level) != str:
+        if not isinstance(threat_level, str):
             # make sure we always store str threat levels in the db
-            threat_level = utils.threat_level_to_string(threat_level)
+            threat_level: str = utils.threat_level_to_string(threat_level)
 
         if timestamp:
             timestamp = utils.convert_format(timestamp, utils.alerts_format)
@@ -301,13 +301,13 @@ def init_evidence_number(self):
     def get_evidence_number(self):
         return self.r.get('number_of_evidence')
 
-    def mark_evidence_as_processed(self, evidence_ID):
+    def mark_evidence_as_processed(self, evidence_ID: str):
         """
         If an evidence was processed by the evidenceprocess, mark it in the db
         """
         self.r.sadd('processed_evidence', evidence_ID)
 
-    def is_evidence_processed(self, evidence_ID):
+    def is_evidence_processed(self, evidence_ID: str) -> bool:
         return self.r.sismember('processed_evidence', evidence_ID)
 
     def set_evidence_for_profileid(self, evidence):
@@ -404,10 +404,10 @@ def get_profileid_twid_alerts(self, profileid, twid) -> dict:
         The format for the returned dict is
             {profile123_twid1_<alert_uuid>: [ev_uuid1, ev_uuid2, ev_uuid3]}
         """
-        alerts = self.r.hget(f'{profileid}{self.separator}{twid}', 'alerts')
+        alerts: str = self.r.hget(f'{profileid}_{twid}', 'alerts')
         if not alerts:
             return {}
-        alerts = json.loads(alerts)
+        alerts: dict = json.loads(alerts)
         return alerts
 
     def getEvidenceForTW(self, profileid: str, twid: str) -> str:

slips_files/core/database/sqlite_db/database.py

@@ -1,3 +1,4 @@
+from typing import List
 import os.path
 import sqlite3
 import json
@@ -170,7 +171,7 @@ def get_all_flows(self):
                 flow_list.append(json.loads(flow[1]))
         return flow_list
 
-    def set_flow_label(self, uids: list, new_label: str):
+    def set_flow_label(self, uids: List[str], new_label: str):
         """
         sets the given new_label to each flow in the uids list
         """
@@ -301,7 +302,8 @@ def add_alert(self, alert: dict):
         :param alert: should contain alert_id, alert_ts, ip_alerted, twid, tw_start, tw_end, label
             alert_time is the local time slips detected this alert, not the network time
         """
-        # 'alerts': 'alert_id TEXT PRIMARY KEY, alert_time TEXT, ip_alerted TEXT, timewindow TEXT, tw_start TEXT, tw_end TEXT, label TEXT'
+        # 'alerts': 'alert_id TEXT PRIMARY KEY, alert_time TEXT, ip_alerted TEXT,
+        # timewindow TEXT, tw_start TEXT, tw_end TEXT, label TEXT'
         self.execute(
             'INSERT OR REPLACE INTO alerts (alert_id, ip_alerted, timewindow, tw_start, tw_end, label, alert_time) '
             'VALUES (?, ?, ?, ?, ?, ?, ?);',