Fix domain resolved with no conn FP and datetime errors

docs/flowalerts.md

@@ -66,7 +66,7 @@ so we simply ignore alerts of this type when connected to well known organizatio
 Slips uses it's own lists of organizations and information about them (IPs, IP ranges, domains, and ASNs). They are stored in ```slips_files/organizations_info``` and they are used to check whether the IP/domain of each flow belong to a known org or not.
 
 Slips doesn't detect 'connection without DNS' when running
-on an interface except for when it's done by this instance's own IP.
+on an interface except for when it's done by this instance's own IP and only after 30 minutes has passed to avoid false positives (assuming the DNS resolution of these connections did happen before slips started).
 
 check [DoH section](https://stratospherelinuxips.readthedocs.io/en/develop/detection_modules.html#detect-doh)
 of the docs for info on how slips detects DoH.
@@ -91,6 +91,8 @@ The domains that are excepted are:
 - Ignore WPAD domain from Windows
 - Ignore domains without a TLD such as the Chrome test domains.
 
+Slips doesn't detect 'DNS resolutions without a connection' when running
+on an interface except for when it's done by this instance's own IP and only after 5 minutes has passed to avoid false positives (assuming the connection did happen and yet to be logged).
 
 
 ## Connection to unknown ports

managers/metadata_manager.py

@@ -74,7 +74,7 @@ def set_analysis_end_date(self, end_date):
         """
         if not self.main.conf.enable_metadata():
             return
-
+        end_date = utils.convert_format(datetime.now(), utils.alerts_format)
         self.main.db.set_input_metadata({"analysis_end": end_date})
 
         # add slips end date in the metadata dir

managers/process_manager.py

@@ -475,7 +475,7 @@ def get_analysis_time(self) -> Tuple[str, str]:
         returns analysis_time in minutes and slips end_time as a date
         """
         start_time = self.main.db.get_slips_start_time()
-        end_time = utils.convert_format(datetime.now(), utils.alerts_format)
+        end_time = utils.convert_format(datetime.now(), "unixtimestamp")
         return (
             utils.get_time_diff(start_time, end_time, return_type="minutes"),
             end_time,
@@ -711,11 +711,10 @@ def shutdown_gracefully(self):
             if self.main.conf.export_labeled_flows():
                 format_ = self.main.conf.export_labeled_flows_to().lower()
                 self.main.db.export_labeled_flows(format_)
-                
+
             self.main.profilers_manager.cpu_profiler_release()
             self.main.profilers_manager.memory_profiler_release()
 
-
             # if store_a_copy_of_zeek_files is set to yes in slips.yaml
             # copy the whole zeek_files dir to the output dir
             self.main.store_zeek_dir_copy()

modules/flowalerts/conn.py

@@ -28,21 +28,21 @@ def init(self):
         # thread (we waited for the dns resolution for these connections)
         self.connections_checked_in_conn_dns_timer_thread = []
         self.whitelist = self.flowalerts.whitelist
-        # Threshold how much time to wait when capturing in an interface,
-        # to start reporting connections without DNS
-        # Usually the computer resolved DNS already, so we need to wait a little to report
+        # how much time to wait when running on interface before reporting
+        # connections without DNS? Usually the computer resolved DNS
+        # already, so we need to wait a little to report
         # In mins
         self.conn_without_dns_interface_wait_time = 30
         self.dns_analyzer = DNS(self.db, flowalerts=self)
         self.is_running_non_stop: bool = self.db.is_running_non_stop()
         self.classifier = FlowClassifier()
+        self.our_ips = utils.get_own_ips()
 
     def read_configuration(self):
         conf = ConfigParser()
         self.long_connection_threshold = conf.long_connection_threshold()
         self.data_exfiltration_threshold = conf.data_exfiltration_threshold()
         self.data_exfiltration_threshold = conf.data_exfiltration_threshold()
-        self.our_ips = utils.get_own_ips()
         self.client_ips: List[str] = conf.client_ips()
 
     def name(self) -> str:
@@ -377,8 +377,6 @@ def should_ignore_conn_without_dns(self, flow) -> bool:
         """
         checks for the cases that we should ignore the connection without dns
         """
-        # we should ignore this evidence if the ip is ours, whether it's a
-        # private ip or in the list of client_ips
         return (
             flow.type_ != "conn"
             or flow.appproto in ("dns", "icmp")
@@ -391,6 +389,10 @@ def should_ignore_conn_without_dns(self, flow) -> bool:
             # because there's no dns.log to know if the dns was made
             or self.db.get_input_type() == "zeek_log_file"
             or self.db.is_doh_server(flow.daddr)
+            # connection without dns in case of an interface,
+            # should only be detected from the srcip of this device,
+            # not all ips, to avoid so many alerts of this type when port scanning
+            or (self.is_running_non_stop and flow.saddr not in self.our_ips)
         )
 
     def check_if_resolution_was_made_by_different_version(
@@ -416,6 +418,22 @@ def check_if_resolution_was_made_by_different_version(
             pass
         return False
 
+    def is_interface_timeout_reached(self) -> bool:
+        """
+        To avoid false positives in case of an interface
+        don't alert ConnectionWithoutDNS until 30 minutes has passed after
+        starting slips because the dns may have happened before starting slips
+        """
+        if not self.is_running_non_stop:
+            # no timeout
+            return True
+
+        start_time = self.db.get_slips_start_time()
+        now = datetime.now()
+        diff = utils.get_time_diff(start_time, now, return_type="minutes")
+        # 30 minutes have passed?
+        return diff >= self.conn_without_dns_interface_wait_time
+
     async def check_connection_without_dns_resolution(
         self, profileid, twid, flow
     ) -> bool:
@@ -434,26 +452,8 @@ async def check_connection_without_dns_resolution(
         # We dont have yet the dhcp in the redis, when is there check it
         # if self.db.get_dhcp_servers(daddr):
         # continue
-
-        # To avoid false positives in case of an interface
-        # don't alert ConnectionWithoutDNS
-        # until 30 minutes has passed
-        # after starting slips because the dns may have happened before
-        # starting slips
-        if self.is_running_non_stop:
-            # connection without dns in case of an interface,
-            # should only be detected from the srcip of this device,
-            # not all ips, to avoid so many alerts of this type when port scanning
-            saddr = profileid.split("_")[-1]
-            if saddr not in self.our_ips:
-                return False
-
-            start_time = self.db.get_slips_start_time()
-            now = datetime.now()
-            diff = utils.get_time_diff(start_time, now, return_type="minutes")
-            if diff < self.conn_without_dns_interface_wait_time:
-                # less than 30 minutes have passed
-                return False
+        if not self.is_interface_timeout_reached():
+            return False
 
         # search 24hs back for a dns resolution
         if self.db.is_ip_resolved(flow.daddr, 24):

modules/flowalerts/dns.py

@@ -2,6 +2,7 @@
 import collections
 import json
 import math
+from datetime import datetime
 from typing import List
 import validators
 
@@ -30,7 +31,11 @@ def init(self):
         self.dns_arpa_queries = {}
         # after this number of arpa queries, slips will detect an arpa scan
         self.arpa_scan_threshold = 10
+        self.is_running_non_stop: bool = self.db.is_running_non_stop()
         self.classifier = FlowClassifier()
+        self.our_ips = utils.get_own_ips()
+        # In mins
+        self.dns_without_conn_interface_wait_time = 5
 
     def name(self) -> str:
         return "DNS_analyzer"
@@ -39,8 +44,7 @@ def read_configuration(self):
         conf = ConfigParser()
         self.shannon_entropy_threshold = conf.get_entropy_threshold()
 
-    @staticmethod
-    def should_detect_dns_without_conn(flow) -> bool:
+    def should_detect_dns_without_conn(self, flow) -> bool:
         """
         returns False in the following cases
          - All reverse dns resolutions
@@ -65,6 +69,10 @@ def should_detect_dns_without_conn(flow) -> bool:
             or flow.query == "WPAD"
             or flow.rcode_name != "NOERROR"
             or not flow.answers
+            # dns without conn in case of an interface,
+            # should only be detected from the srcip of this device,
+            # not all ips, to avoid so many alerts of this type when port scanning
+            or (self.is_running_non_stop and flow.saddr not in self.our_ips)
         ):
             return False
         return True
@@ -216,6 +224,22 @@ def is_any_flow_answer_contacted(self, profileid, twid, flow) -> bool:
             # this is not a DNS without resolution
             return True
 
+    def is_interface_timeout_reached(self):
+        """
+        To avoid false positives in case of an interface
+        don't alert ConnectionWithoutDNS until 30 minutes has passed after
+        starting slips because the dns may have happened before starting slips
+        """
+        if not self.is_running_non_stop:
+            # no timeout
+            return True
+
+        start_time = self.db.get_slips_start_time()
+        now = datetime.now()
+        diff = utils.get_time_diff(start_time, now, return_type="minutes")
+        # 30 minutes have passed?
+        return diff >= self.dns_without_conn_interface_wait_time
+
     async def check_dns_without_connection(
         self, profileid, twid, flow
     ) -> bool:
@@ -225,6 +249,9 @@ async def check_dns_without_connection(
         if not self.should_detect_dns_without_conn(flow):
             return False
 
+        if not self.is_interface_timeout_reached():
+            return False
+
         if self.is_any_flow_answer_contacted(profileid, twid, flow):
             return False
 

modules/update_manager/update_manager.py

@@ -912,17 +912,17 @@ def parse_json_ti_feed(self, link_to_download, ti_file_path: str) -> bool:
                     return False
 
                 for ioc in file:
-                    date = ioc["InsertDate"]
-                    date = utils.convert_ts_to_tz_aware(date)
-                    diff = utils.get_time_diff(
-                        date, time.time(), return_type="days"
-                    )
+                    date = utils.convert_ts_to_tz_aware(ioc["InsertDate"])
+                    now = utils.convert_ts_to_tz_aware(time.time())
+                    diff = utils.get_time_diff(date, now, return_type="days")
 
                     if diff > self.interval:
                         continue
+
                     domain = ioc["DomainAddress"]
                     if not utils.is_valid_domain(domain):
                         continue
+
                     malicious_domains_dict[domain] = json.dumps(
                         {
                             "description": "",

slips_files/common/slips_utils.py

@@ -265,7 +265,7 @@ def convert_format(self, ts, required_format: str):
 
         # convert to the req format
         if required_format == "iso":
-            return datetime_obj.astimezone(self.local_tz).isoformat()
+            return datetime_obj.astimezone().isoformat()
         elif required_format == "unixtimestamp":
             return datetime_obj.timestamp()
         else:
@@ -302,7 +302,7 @@ def convert_to_datetime(self, ts):
 
         given_format = self.get_time_format(ts)
         return (
-            datetime.fromtimestamp(float(ts), tz=self.local_tz)
+            datetime.fromtimestamp(float(ts))
             if given_format == "unixtimestamp"
             else datetime.strptime(ts, given_format)
         )
@@ -415,7 +415,6 @@ def is_ignored_ip(self, ip: str) -> bool:
         except (ipaddress.AddressValueError, ValueError):
             return True
         # Is the IP multicast, private? (including localhost)
-        # local_link or reserved?
         # The broadcast address 255.255.255.255 is reserved.
         return bool(
             (

slips_files/core/database/redis_db/database.py

@@ -15,7 +15,6 @@
 import time
 import json
 import subprocess
-from datetime import datetime
 import ipaddress
 import sys
 import validators
@@ -204,10 +203,8 @@ def set_slips_internal_time(cls, timestamp):
 
     @classmethod
     def get_slips_start_time(cls) -> str:
-        """get the time slips started"""
-        if start_time := cls.r.get("slips_start_time"):
-            start_time = utils.convert_format(start_time, utils.alerts_format)
-            return start_time
+        """get the time slips started in unix format"""
+        return cls.r.get("slips_start_time")
 
     @classmethod
     def init_redis_server(cls) -> Tuple[bool, str]:
@@ -363,7 +360,7 @@ def change_redis_limits(cls, client: redis.StrictRedis):
     @classmethod
     def _set_slips_start_time(cls):
         """store the time slips started (datetime obj)"""
-        now = utils.convert_format(datetime.now(), utils.alerts_format)
+        now = time.time()
         cls.r.set("slips_start_time", now)
 
     def publish(self, channel, msg):

slips_files/core/database/redis_db/profile_handler.py

@@ -1229,8 +1229,8 @@ def add_mac_addr_to_profile(self, profileid: str, mac_addr: str):
             return False
 
         # get the ips that belong to this mac
-        cached_ip = self.r.hmget("MAC", mac_addr)[0]
-        if not cached_ip:
+        cached_ips: Optional[List] = self.r.hmget("MAC", mac_addr)[0]
+        if not cached_ips:
             # no mac info stored for profileid
             ip = json.dumps([incoming_ip])
             self.r.hset("MAC", mac_addr, ip)
@@ -1241,10 +1241,10 @@ def add_mac_addr_to_profile(self, profileid: str, mac_addr: str):
         else:
             # we found another profile that has the same mac as this one
             # get all the ips, v4 and 6, that are stored with this mac
-            cached_ips = json.loads(cached_ip)
+            cached_ips: List[str] = json.loads(cached_ips)
             # get the last one of them
             found_ip = cached_ips[-1]
-            cached_ips = set(cached_ips)
+            cached_ips: Set[str] = set(cached_ips)
 
             if incoming_ip in cached_ips:
                 # this is the case where we have the given ip already

tests/module_factory.py

@@ -130,8 +130,9 @@ def create_db_manager_obj(
                 flush_db=flush_db,
                 start_redis_server=start_redis_server,
             )
-        db.r = db.rdb.r
         db.print = Mock()
+        # for easier access to redis db
+        db.r = db.rdb.r
         assert db.get_used_redis_port() == port
         return db