Improve young domain detection by setting evidence for all the young domain DNS answers

modules/flowalerts/set_evidence.py

@@ -41,8 +41,9 @@ def young_domain(
         twid_number: int =  int(twid.replace("timewindow", ""))
         description: str =  (f'connection to a young domain: {domain} '
                              f'registered {age} days ago.')
-        if answers:
-            attacker = answers[0]
+        # set evidence for all the young domain dns answers
+        for attacker in answers:
+            attacker: str
             evidence = Evidence(
                     evidence_type=EvidenceType.YOUNG_DOMAIN,
                     attacker=Attacker(
@@ -1387,7 +1388,7 @@ def data_exfiltration(
                 value=saddr
             ),
             threat_level=ThreatLevel.INFO,
-            confidence=confidence,
+            confidence=0.6,
             description=description,
             profile=ProfileID(ip=saddr),
             timewindow=TimeWindow(number=twid_number),

modules/p2ptrust/p2ptrust.py

@@ -6,7 +6,9 @@
 import subprocess
 import time
 from pathlib import Path
-from typing import Dict
+from typing import Dict, \
+    Optional, \
+    Tuple
 import json
 import sys
 import socket
@@ -19,18 +21,16 @@
 from modules.p2ptrust.utils.go_director import GoDirector
 from slips_files.core.evidence_structure.evidence import \
     (
+        dict_to_evidence,
         Evidence,
         ProfileID,
         TimeWindow,
-        Victim,
         Attacker,
-        Proto,
         ThreatLevel,
         EvidenceType,
         IoCType,
         Direction,
         IDEACategory,
-        Tag
     )
 
 
@@ -262,84 +262,124 @@ def _configure(self):
 
 
             # print(f"[debugging] runnning pigeon: {executable}")
+
+    def extract_confidence(self, evidence: Evidence) -> Optional[float]:
+        """
+        returns the confidence of the given evidence or None if no
+        confidence was found
+        """
+        confidence: float = evidence.confidence
 
-    def new_evidence_callback(self, msg: Dict):
+        if confidence:
+            return confidence
+
+        attacker_ip: str = evidence.attacker.value
+        self.print(
+            f"IP {attacker_ip} doesn't have a confidence. "
+            f"not sharing to the network.", 0, 2,
+        )
+        return
+
+    def extract_threat_level(self, evidence: Evidence) -> Optional[ThreatLevel]:
         """
-        This function is called whenever a msg arrives to the report_to_peers channel,
-        It compares the score and confidence of the given IP and decides whether or not to
-        share it accordingly
+        returns the confidence of the given evidence or None if no
+        confidence was found
         """
-        try:
-            data = json.loads(msg['data'])
-        except json.decoder.JSONDecodeError:
-            # not a valid json dict
-            return
+        threat_level: ThreatLevel = evidence.threat_level
 
-        # example: dstip srcip dport sport dstdomain
-        attacker_direction = data.get('attacker_direction')
-        if 'ip' not in attacker_direction:   # and not 'domain' in attacker_direction:
-            # todo do we share domains too?
-            # the detection is a srcport, dstport, etc. don't share
-            return
+        if threat_level:
+            return threat_level
+
+        attacker_ip: str = evidence.attacker.value
+        self.print(
+                f"IP {attacker_ip} doesn't have a threat_level. "
+                f"not sharing to the network.", 0,2,
+        )
 
-        evidence_type = data.get('evidence_type')
-        if 'P2PReport' in evidence_type:
+        return
+
+    def should_share(self, evidence: Evidence) -> bool:
+        """
+        decides whether or not to report the given evidence to other
+        peers
+        """
+        if evidence.evidence_type == EvidenceType.P2P_REPORT:
             # we shouldn't re-share evidence reported by other peers
-            return
+            return False
 
+        if evidence.attacker.attacker_type != IoCType.IP.name:
+            # we only share ips with other peers.
+            return False
 
-        attacker = data.get('attacker')
-        confidence = data.get('confidence', False)
-        threat_level = data.get('threat_level', False)
+        confidence = self.extract_confidence(evidence)
+        if not confidence:
+            return False
+
+        threat_level: ThreatLevel = self.extract_threat_level(evidence)
         if not threat_level:
-            self.print(
-                f"IP {attacker} doesn't have a threat_level. "
-                f"not sharing to the network.", 0,2,
-            )
+            return False
+
+        return True
+
+
+    def new_evidence_callback(self, msg: Dict[str, str]):
+        """
+        Decides to share an evidence generated by slips to other peers or not
+        depending on whether we have info about this ip from the p2p
+        network or not
+        It is called whenever a msg arrives to the
+        report_to_peers channel,
+        """
+        try:
+            evidence: Dict[str, str] = json.loads(msg['data'])
+        except json.decoder.JSONDecodeError:
             return
-        if not confidence:
-            self.print(
-                f"IP {attacker} doesn't have a confidence. "
-                f"not sharing to the network.", 0, 2,
-            )
+        evidence: Evidence = dict_to_evidence(evidence)
+
+        if not self.should_share(evidence):
             return
 
-        # get the int representing this threat_level
-        score = self.threat_levels[threat_level]
-        # todo what we're currently sharing is the threat level(int) of the evidence caused by this ip
+        # todo what we're currently sharing is the threat level(int)
+        #  of the evidence caused by this ip
 
         # todo when we generate a new evidence,
-        #  we give it a score and a tl, but we don't update the IP_Info and give this ip a score in th db!
+        #  we give it a score and a tl, but we don't update the
+        #  IP_Info and give this ip a score in th db!
 
         # TODO: discuss - only share score if confidence is high enough?
         # compare slips data with data in go
         data_already_reported = True
         try:
-            cached_opinion = self.trust_db.get_cached_network_opinion(
-                'ip', attacker
+            cached_opinion: Tuple = self.trust_db.get_cached_network_opinion(
+                'ip', evidence.attacker.value
             )
+            # get the cached network opinion about this ip
             (
                 cached_score,
                 cached_confidence,
                 network_score,
                 timestamp,
             ) = cached_opinion
             # if we don't have info about this ip from the p2p network,
-            # report it to the p2p netwrok
+            # report it to the p2p network
             if not cached_score:
                 data_already_reported = False
         except KeyError:
             data_already_reported = False
         except IndexError:
-            # data saved in local db have wrong structure, this is an invalid state
+            # data saved in local db have wrong structure,
+            # this is an invalid state
             return
 
-        # TODO: in the future, be smarter and share only when needed.
-        #  For now, we will always share
         if not data_already_reported:
-            # Take data and send it to a peer as report.
+            # send the peer report to other peers
             p2p_utils.send_evaluation_to_go(
-                attacker, score, confidence, '*', self.pygo_channel, self.db
+                evidence.attacker.value,
+                evidence.threat_level.value,
+                evidence.confidence,
+                '*',
+                self.pygo_channel,
+                self.db
             )
 
     def gopy_callback(self, msg: Dict):
@@ -442,12 +482,14 @@ def set_evidence_malicious_ip(self,
 
     def handle_data_request(self, message_data: str) -> None:
         """
-        Process the request from Slips, ask the network and process the network response.
+        Process the request from Slips, ask the network and
+        process the network response.
 
         Three `arguments` are expected in the redis channel:
             ip_address: str,
             cache_age: int [seconds]
-        The return value is sent to the redis channel `p2p_data_response` in the format:
+        The return value is sent to the redis channel
+         `p2p_data_response` in the format:
             ip_address: str,
             timestamp: int [time of assembling the response],
             network_opinion: float,
@@ -458,20 +500,24 @@ def handle_data_request(self, message_data: str) -> None:
         This method will check if any data not older than `cache_age`
         is saved in cache. If yes, this data is returned.
         If not, the database is checked.
-        An ASK query is sent to the network and responses are collected and saved into
-        the redis database.
+        An ASK query is sent to the network and responses
+        are collected and saved into the redis database.
 
-        :param message_data: The data received from the redis channel `p2p_data_response`
-        :return: None, the result is saved into the redis database under key `p2p4slips`
+        :param message_data: The data received from the redis
+         channel `p2p_data_response`
+        :return: None, the result is saved into the redis
+            database under key `p2p4slips`
         """
 
         # make sure that IP address is valid
         # and cache age is a valid timestamp from the past
         ip_info = validate_slips_data(message_data)
         if ip_info is None:
             # IP address is not valid, aborting
-            # print(f"DEBUGGING: IP address is not valid: {ip_info}, not asking the network")
+            # print(f"DEBUGGING: IP address is not valid:
+            # {ip_info}, not asking the network")
             return
+
         # ip_info is {
         #             'ip': str(saddr),
         #             'profileid' : str(profileid),
@@ -497,10 +543,12 @@ def handle_data_request(self, message_data: str) -> None:
             # print("DEBUGGING:  cached value is ok, not asking the network.")
             return
 
-        # if cached value is old, ask the peers
+        # since cached value is old, ask the peers
 
-        # TODO: in some cases, it is not necessary to wait, specify that and implement it
-        #       I do not remember writing this comment. I have no idea in which cases there is no need to wait? Maybe
+        # TODO: in some cases, it is not necessary to wait, specify
+        #  that and implement it
+        #       I do not remember writing this comment. I have no idea
+        #       in which cases there is no need to wait? Maybe
         #       when everybody responds asap?
         p2p_utils.send_request_to_go(ip_address, self.pygo_channel, self.db)
         self.print(f'[Slips -> The Network] request about {ip_address}')
@@ -516,38 +564,53 @@ def handle_data_request(self, message_data: str) -> None:
             combined_score,
             combined_confidence,
         ) = self.reputation_model.get_opinion_on_ip(ip_address)
-
+
+        self.process_network_response(ip_address,
+                                      combined_score,
+                                      combined_confidence,
+                                      network_score,
+                                      confidence,
+                                      ip_info)
+
+    def process_network_response(
+        self, ip, combined_score, combined_confidence, network_score,
+        confidence, ip_info
+        ):
+        """
+        stores the reported score and confidence about the ip and adds an
+        evidence if necessary
+        """
         # no data in db - this happens when testing,
         # if there is not enough data on peers
         if combined_score is None:
             self.print(
                 f'No data received from the'
-                f' network about {ip_address}\n', 0, 2
-            )
-            # print(f"[DEBUGGING] No data received
-            # from the network about {ip_address}\n")
-        else:
-            self.print(
-                f'The Network shared some data about {ip_address}, '
-                f'Shared data: score={combined_score}, '
-                f'confidence={combined_confidence} saving it to  now!\n',
-                0,
-                2,
+                f' network about {ip}\n', 0, 2
             )
+            return
+
+        self.print(
+            f'The Network shared some data about {ip}, '
+            f'Shared data: score={combined_score}, '
+            f'confidence={combined_confidence} saving it to  now!\n',
+            0,
+            2,
+        )
 
-            # save it to IPsInfo hash in p2p4slips key in the db AND p2p_reports key
-            p2p_utils.save_ip_report_to_db(
-                ip_address,
-                combined_score,
-                combined_confidence,
-                network_score,
-                self.db,
-                self.storage_name,
+        # save it to IPsInfo hash in p2p4slips key in the db
+        # AND p2p_reports key
+        p2p_utils.save_ip_report_to_db(
+            ip,
+            combined_score,
+            combined_confidence,
+            network_score,
+            self.db,
+            self.storage_name,
+        )
+        if int(combined_score) * int(confidence) > 0:
+            self.set_evidence_malicious_ip(
+                ip_info, combined_score, confidence
             )
-            if int(combined_score) * int(confidence) > 0:
-                self.set_evidence_malicious_ip(
-                    ip_info, combined_score, confidence
-                )
 
     def respond_to_message_request(self, key, reporter):
         # todo do you mean another peer is asking me about
@@ -603,14 +666,13 @@ def pre_main(self):
         # self.c4 = self.db.subscribe(self.slips_update_channel)
 
     def main(self):
-        """main loop function"""
-        if msg:= self.get_msg('report_to_peers'):
+        if msg := self.get_msg('report_to_peers'):
             self.new_evidence_callback(msg)
 
-        if msg:= self.get_msg(self.p2p_data_request_channel):
+        if msg := self.get_msg(self.p2p_data_request_channel):
             self.data_request_callback(msg)
 
-        if msg:= self.get_msg(self.gopy_channel):
+        if msg := self.get_msg(self.gopy_channel):
             self.gopy_callback(msg)
 
         ret_code = self.pigeon.poll()