From 82b59128869da8b3c40ebec73c80e258fd9b6ee1 Mon Sep 17 00:00:00 2001
From: alya <alyaggomaa@gmail.com>
Date: Wed, 6 Mar 2024 16:39:11 +0200
Subject: [PATCH 1/7] use snake case

---
 managers/metadata_manager.py                  |  2 +-
 managers/process_manager.py                   |  2 +-
 modules/virustotal/virustotal.py              |  2 +-
 slips/main.py                                 |  2 +-
 slips_files/core/database/database_manager.py | 59 +++++++++----------
 .../core/database/redis_db/ioc_handler.py     |  6 +-
 .../core/database/redis_db/profile_handler.py | 46 +++++++--------
 slips_files/core/helpers/symbols_handler.py   |  2 +-
 slips_files/core/profiler.py                  |  4 +-
 9 files changed, 61 insertions(+), 64 deletions(-)

diff --git a/managers/metadata_manager.py b/managers/metadata_manager.py
index 291fda15f..cb4550973 100644
--- a/managers/metadata_manager.py
+++ b/managers/metadata_manager.py
@@ -159,7 +159,7 @@ def update_slips_running_stats(self) -> Tuple[int, Set[str]] :
         # this is the modification time of the last timewindow
         last_modified_tw_time: float
         modified_profiles, last_modified_tw_time = (
-            self.main.db.getModifiedProfilesSince(slips_internal_time)
+            self.main.db.get_modified_profiles_since(slips_internal_time)
         )
         modified_ips_in_the_last_tw = len(modified_profiles)
         self.main.db.set_input_metadata(
diff --git a/managers/process_manager.py b/managers/process_manager.py
index 05f96dcdd..cd424b3f5 100644
--- a/managers/process_manager.py
+++ b/managers/process_manager.py
@@ -633,7 +633,7 @@ def shutdown_gracefully(self):
             timeout_seconds: float = timeout * 60
 
             # close all tws
-            self.main.db.check_TW_to_close(close_all=True)
+            self.main.db.check_tw_to_close(close_all=True)
             analysis_time = self.get_analysis_time()
             self.main.print(
                 f"Analysis of {self.main.input_information} "
diff --git a/modules/virustotal/virustotal.py b/modules/virustotal/virustotal.py
index 9d058ff13..55df97aa7 100644
--- a/modules/virustotal/virustotal.py
+++ b/modules/virustotal/virustotal.py
@@ -165,7 +165,7 @@ def set_url_data_in_URLInfo(self, url, cached_data):
         # Score of this url didn't change
         vtdata = {'URL': score, 'timestamp': time.time()}
         data = {'VirusTotal': vtdata}
-        self.db.setInfoForURLs(url, data)
+        self.db.set_info_for_urls(url, data)
 
     def set_domain_data_in_DomainInfo(self, domain, cached_data):
         """
diff --git a/slips/main.py b/slips/main.py
index 34cd552d3..5baa3e9b7 100644
--- a/slips/main.py
+++ b/slips/main.py
@@ -722,7 +722,7 @@ def sig_handler(sig, frame):
                 self.update_stats()
 
                 # Check if we need to close any TWs
-                self.db.check_TW_to_close()
+                self.db.check_tw_to_close()
 
                 modified_profiles: Set[str] = (
                     self.metadata_man.update_slips_running_stats()[1]
diff --git a/slips_files/core/database/database_manager.py b/slips_files/core/database/database_manager.py
index 4f3b2e5a0..478db3a7e 100644
--- a/slips_files/core/database/database_manager.py
+++ b/slips_files/core/database/database_manager.py
@@ -537,23 +537,23 @@ def setNewURL(self, *args, **kwargs):
     def get_domain_data(self, *args, **kwargs):
         return self.rdb.get_domain_data(*args, **kwargs)
 
-    def setNewDomain(self, *args, **kwargs):
-        return self.rdb.setNewDomain(*args, **kwargs)
+    def set_new_domain(self, *args, **kwargs):
+        return self.rdb.set_new_domain(*args, **kwargs)
 
     def set_info_for_domains(self, *args, **kwargs):
         return self.rdb.set_info_for_domains(*args, **kwargs)
 
-    def setInfoForURLs(self, *args, **kwargs):
-        return self.rdb.setInfoForURLs(*args, **kwargs)
+    def set_info_for_urls(self, *args, **kwargs):
+        return self.rdb.set_info_for_urls(*args, **kwargs)
 
     def get_data_from_profile_tw(self, *args, **kwargs):
         return self.rdb.get_data_from_profile_tw(*args, **kwargs)
 
-    def getOutTuplesfromProfileTW(self, *args, **kwargs):
-        return self.rdb.getOutTuplesfromProfileTW(*args, **kwargs)
+    def get_outtuples_from_profile_tw(self, *args, **kwargs):
+        return self.rdb.get_outtuples_from_profile_tw(*args, **kwargs)
 
-    def getInTuplesfromProfileTW(self, *args, **kwargs):
-        return self.rdb.getInTuplesfromProfileTW(*args, **kwargs)
+    def get_intuples_from_profile_tw(self, *args, **kwargs):
+        return self.rdb.get_intuples_from_profile_tw(*args, **kwargs)
 
     def get_dhcp_flows(self, *args, **kwargs):
         return self.rdb.get_dhcp_flows(*args, **kwargs)
@@ -573,8 +573,8 @@ def add_out_dns(self, *args, **kwargs):
     def add_port(self, *args, **kwargs):
         return self.rdb.add_port(*args, **kwargs)
 
-    def getFinalStateFromFlags(self, *args, **kwargs):
-        return self.rdb.getFinalStateFromFlags(*args, **kwargs)
+    def get_final_state_from_flags(self, *args, **kwargs):
+        return self.rdb.get_final_state_from_flags(*args, **kwargs)
 
     def add_ips(self, *args, **kwargs):
         return self.rdb.add_ips(*args, **kwargs)
@@ -642,14 +642,14 @@ def getTWsfromProfile(self, *args, **kwargs):
     def get_number_of_tws_in_profile(self, *args, **kwargs):
         return self.rdb.get_number_of_tws_in_profile(*args, **kwargs)
 
-    def getSrcIPsfromProfileTW(self, *args, **kwargs):
-        return self.rdb.getSrcIPsfromProfileTW(*args, **kwargs)
+    def get_srcips_from_profile_tw(self, *args, **kwargs):
+        return self.rdb.get_srcips_from_profile_tw(*args, **kwargs)
 
-    def getDstIPsfromProfileTW(self, *args, **kwargs):
-        return self.rdb.getDstIPsfromProfileTW(*args, **kwargs)
+    def get_dstips_from_profile_tw(self, *args, **kwargs):
+        return self.rdb.get_dstips_from_profile_tw(*args, **kwargs)
 
-    def getT2ForProfileTW(self, *args, **kwargs):
-        return self.rdb.getT2ForProfileTW(*args, **kwargs)
+    def get_t2_for_profile_tw(self, *args, **kwargs):
+        return self.rdb.get_t2_for_profile_tw(*args, **kwargs)
 
     def has_profile(self, *args, **kwargs):
         return self.rdb.has_profile(*args, **kwargs)
@@ -675,14 +675,14 @@ def add_new_tw(self, *args, **kwargs):
     def get_tw_start_time(self, *args, **kwargs):
         return self.rdb.get_tw_start_time(*args, **kwargs)
 
-    def getAmountTW(self, *args, **kwargs):
-        return self.rdb.getAmountTW(*args, **kwargs)
+    def get_number_of_tws(self, *args, **kwargs):
+        return self.rdb.get_number_of_tws(*args, **kwargs)
 
-    def getModifiedTWSinceTime(self, *args, **kwargs):
-        return self.rdb.getModifiedTWSinceTime(*args, **kwargs)
+    def get_modified_tw_since_time(self, *args, **kwargs):
+        return self.rdb.get_modified_tw_since_time(*args, **kwargs)
 
-    def getModifiedProfilesSince(self, *args, **kwargs):
-        return self.rdb.getModifiedProfilesSince(*args, **kwargs)
+    def get_modified_profiles_since(self, *args, **kwargs):
+        return self.rdb.get_modified_profiles_since(*args, **kwargs)
 
     def add_mac_addr_to_profile(self, *args, **kwargs):
         return self.rdb.add_mac_addr_to_profile(*args, **kwargs)
@@ -714,17 +714,17 @@ def add_profile(self, *args, **kwargs):
     def set_profile_module_label(self, *args, **kwargs):
         return self.rdb.set_profile_module_label(*args, **kwargs)
 
-    def check_TW_to_close(self, *args, **kwargs):
-        return self.rdb.check_TW_to_close(*args, **kwargs)
+    def check_tw_to_close(self, *args, **kwargs):
+        return self.rdb.check_tw_to_close(*args, **kwargs)
 
     def check_health(self):
         self.rdb.pubsub.check_health()
 
-    def markProfileTWAsClosed(self, *args, **kwargs):
-        return self.rdb.markProfileTWAsClosed(*args, **kwargs)
+    def mark_profile_tw_as_closed(self, *args, **kwargs):
+        return self.rdb.mark_profile_tw_as_closed(*args, **kwargs)
 
-    def markProfileTWAsModified(self, *args, **kwargs):
-        return self.rdb.markProfileTWAsModified(*args, **kwargs)
+    def mark_profile_tw_as_modified(self, *args, **kwargs):
+        return self.rdb.mark_profile_tw_as_modified(*args, **kwargs)
 
     def add_tuple(self, *args, **kwargs):
         return self.rdb.add_tuple(*args, **kwargs)
@@ -858,9 +858,6 @@ def delete(self, *args, **kwargs):
     def select(self, *args, **kwargs):
         return self.sqlite.select(*args, **kwargs)
 
-    def execute_query(self, *args, **kwargs):
-        return self.sqlite.execute_query(*args, **kwargs)
-
     def get_pid_of(self, *args, **kwargs):
         return self.rdb.get_pid_of(*args, **kwargs)
 
diff --git a/slips_files/core/database/redis_db/ioc_handler.py b/slips_files/core/database/redis_db/ioc_handler.py
index 6c3ce7df0..fd16b286d 100644
--- a/slips_files/core/database/redis_db/ioc_handler.py
+++ b/slips_files/core/database/redis_db/ioc_handler.py
@@ -361,7 +361,7 @@ def get_domain_data(self, domain):
         data = json.loads(data) if data or data == {} else False
         return data
 
-    def setNewDomain(self, domain: str):
+    def set_new_domain(self, domain: str):
         """
         1- Stores this new domain in the Domains hash
         2- Publishes in the channels that there is a new domain, and that we want
@@ -391,7 +391,7 @@ def set_info_for_domains(self, domain: str, info_to_set: dict, mode= 'leave'):
         domain_data = self.get_domain_data(domain)
         if not domain_data:
             # This domain is not in the dictionary, add it first:
-            self.setNewDomain(domain)
+            self.set_new_domain(domain)
             # Now get the data, which should be empty, but just in case
             domain_data = self.get_domain_data(domain)
 
@@ -451,7 +451,7 @@ def set_info_for_domains(self, domain: str, info_to_set: dict, mode= 'leave'):
             # Publish the changes
             self.r.publish('dns_info_change', domain)
 
-    def setInfoForURLs(self, url: str, urldata: dict):
+    def set_info_for_urls(self, url: str, urldata: dict):
         """
         Store information for this URL
         We receive a dictionary, such as {'VirusTotal': {'URL':score}} that we are
diff --git a/slips_files/core/database/redis_db/profile_handler.py b/slips_files/core/database/redis_db/profile_handler.py
index 30ea907ac..7c2b2265b 100644
--- a/slips_files/core/database/redis_db/profile_handler.py
+++ b/slips_files/core/database/redis_db/profile_handler.py
@@ -54,11 +54,11 @@ def print(self, text, verbose=1, debug=0):
             {"from": self.name, "txt": text, "verbose": verbose, "debug": debug}
         )
 
-    def getOutTuplesfromProfileTW(self, profileid, twid):
+    def get_outtuples_from_profile_tw(self, profileid, twid):
         """Get the out tuples"""
         return self.r.hget(profileid + self.separator + twid, "OutTuples")
 
-    def getInTuplesfromProfileTW(self, profileid, twid):
+    def get_intuples_from_profile_tw(self, profileid, twid):
         """Get the in tuples"""
         return self.r.hget(profileid + self.separator + twid, "InTuples")
 
@@ -333,7 +333,7 @@ def add_port(
         ip_key = "srcips" if role == "Server" else "dstips"
 
         # Get the state. Established, NotEstablished
-        summaryState = self.getFinalStateFromFlags(state, pkts)
+        summaryState = self.get_final_state_from_flags(state, pkts)
 
         old_profileid_twid_data = self.get_data_from_profile_tw(
             profileid, twid, port_type, summaryState, proto, role, "Ports"
@@ -374,9 +374,9 @@ def add_port(
         hash_key = f"{profileid}{self.separator}{twid}"
         key_name = f"{port_type}Ports{role}{proto}{summaryState}"
         self.r.hset(hash_key, key_name, str(data))
-        self.markProfileTWAsModified(profileid, twid, starttime)
+        self.mark_profile_tw_as_modified(profileid, twid, starttime)
 
-    def getFinalStateFromFlags(self, state, pkts):
+    def get_final_state_from_flags(self, state, pkts):
         """
         Analyze the flags given and return a summary of the state. Should work with Argus and Bro flags
         We receive the pakets to distinguish some Reset connections
@@ -702,7 +702,7 @@ def add_ips(self, profileid, twid, flow, role):
         self.update_times_contacted(ip, direction, profileid, twid)
 
         # Get the state. Established, NotEstablished
-        summaryState = self.getFinalStateFromFlags(flow.state, flow.pkts)
+        summaryState = self.get_final_state_from_flags(flow.state, flow.pkts)
         key_name = f"{direction}IPs{role}{flow.proto.upper()}{summaryState}"
         # Get the previous data about this key
         old_profileid_twid_data = self.get_data_from_profile_tw(
@@ -786,7 +786,7 @@ def add_flow(
         The profileid is the main profile that this flow is related too.
         : param new_profile_added : is set to True for everytime we see a new srcaddr
         """
-        summaryState = self.getFinalStateFromFlags(flow.state, flow.pkts)
+        summary_state = self.get_final_state_from_flags(flow.state, flow.pkts)
         flow_dict = {
             "ts": flow.starttime,
             "dur": flow.dur,
@@ -796,7 +796,7 @@ def add_flow(
             "dport": flow.dport,
             "proto": flow.proto,
             "origstate": flow.state,
-            "state": summaryState,
+            "state": summary_state,
             "pkts": flow.pkts,
             "allbytes": flow.bytes,
             "spkts": flow.spkts,
@@ -834,7 +834,7 @@ def add_flow(
             self.set_input_metadata({"file_start": flow.starttime})
             self.first_flow = False
 
-        self.set_local_network(flow.saddr)
+        self.db.set_local_network(flow.saddr)
 
         # dont send arp flows in this channel, they have their own new_arp channel
         if flow.type_ != "arp":
@@ -1089,19 +1089,19 @@ def get_number_of_tws_in_profile(self, profileid) -> int:
         """
         return len(self.getTWsfromProfile(profileid)) if profileid else 0
 
-    def getSrcIPsfromProfileTW(self, profileid, twid):
+    def get_srcips_from_profile_tw(self, profileid, twid):
         """
         Get the src ip for a specific TW for a specific profileid
         """
         return self.r.hget(profileid + self.separator + twid, "SrcIPs")
 
-    def getDstIPsfromProfileTW(self, profileid, twid):
+    def get_dstips_from_profile_tw(self, profileid, twid):
         """
         Get the dst ip for a specific TW for a specific profileid
         """
         return self.r.hget(profileid + self.separator + twid, "DstIPs")
 
-    def getT2ForProfileTW(self, profileid, twid, tupleid, tuple_key: str):
+    def get_t2_for_profile_tw(self, profileid, twid, tupleid, tuple_key: str):
         """
         Get T1 and the previous_time for this previous_time, twid and tupleid
         """
@@ -1254,11 +1254,11 @@ def get_tw_start_time(self, profileid, twid):
         # sorted set is encoded
         return self.r.zscore(f"tws{profileid}", twid.encode("utf-8"))
 
-    def getAmountTW(self, profileid):
+    def get_number_of_tws(self, profileid):
         """Return the number of tws for this profile id"""
         return self.r.zcard(f"tws{profileid}") if profileid else False
 
-    def getModifiedTWSinceTime(self, time: float) -> List[Tuple[str, float]]:
+    def get_modified_tw_since_time(self, time: float) -> List[Tuple[str, float]]:
         """
         Return the list of modified timewindows since a certain time
         """
@@ -1268,10 +1268,10 @@ def getModifiedTWSinceTime(self, time: float) -> List[Tuple[str, float]]:
         data = self.r.zrangebyscore("ModifiedTW", time, float("+inf"), withscores=True)
         return data or []
 
-    def getModifiedProfilesSince(self, time: float) -> Tuple[Set[str], float]:
+    def get_modified_profiles_since(self, time: float) -> Tuple[Set[str], float]:
         """Returns a set of modified profiles since a certain time and
         the time of the last modified profile"""
-        modified_tws: List[Tuple[str, float]] = self.getModifiedTWSinceTime(time)
+        modified_tws: List[Tuple[str, float]] = self.get_modified_tw_since_time(time)
         if not modified_tws:
             # no modified tws, and no time_of_last_modified_tw
             return [], 0
@@ -1573,7 +1573,7 @@ def set_profile_module_label(self, profileid, module, label):
         data = json.dumps(data)
         self.r.hset(profileid, "modules_labels", data)
 
-    def check_TW_to_close(self, close_all=False):
+    def check_tw_to_close(self, close_all=False):
         """
         Check if we should close some TW
         Search in the modifed tw list and compare when they
@@ -1605,9 +1605,9 @@ def check_TW_to_close(self, close_all=False):
                 3,
                 0,
             )
-            self.markProfileTWAsClosed(profile_tw_to_close_id)
+            self.mark_profile_tw_as_closed(profile_tw_to_close_id)
 
-    def markProfileTWAsClosed(self, profileid_tw):
+    def mark_profile_tw_as_closed(self, profileid_tw):
         """
         Mark the TW as closed so tools can work on its data
         """
@@ -1615,7 +1615,7 @@ def markProfileTWAsClosed(self, profileid_tw):
         self.r.zrem("ModifiedTW", profileid_tw)
         self.publish("tw_closed", profileid_tw)
 
-    def markProfileTWAsModified(self, profileid, twid, timestamp):
+    def mark_profile_tw_as_modified(self, profileid, twid, timestamp):
         """
         Mark a TW in a profile as modified
         This means:
@@ -1630,7 +1630,7 @@ def markProfileTWAsModified(self, profileid, twid, timestamp):
         self.r.zadd("ModifiedTW", data)
         self.publish("tw_modified", f"{profileid}:{twid}")
         # Check if we should close some TW
-        self.check_TW_to_close()
+        self.check_tw_to_close()
 
     def publish_new_letter(
         self, new_symbol: str, profileid: str, twid: str, tupleid: str, flow
@@ -1741,7 +1741,7 @@ def add_tuple(
 
             prev_symbols = json.dumps(prev_symbols)
             self.r.hset(profileid_twid, direction, prev_symbols)
-            self.markProfileTWAsModified(profileid, twid, flow.starttime)
+            self.mark_profile_tw_as_modified(profileid, twid, flow.starttime)
 
         except Exception:
             exception_line = sys.exc_info()[2].tb_lineno
@@ -1772,7 +1772,7 @@ def add_timeline_line(self, profileid, twid, data, timestamp):
         mapping = {data: timestamp}
         self.r.zadd(key, mapping)
         # Mark the tw as modified since the timeline line is new data in the TW
-        self.markProfileTWAsModified(profileid, twid, timestamp="")
+        self.mark_profile_tw_as_modified(profileid, twid, timestamp="")
 
     def get_timeline_last_lines(
         self, profileid, twid, first_index: int
diff --git a/slips_files/core/helpers/symbols_handler.py b/slips_files/core/helpers/symbols_handler.py
index 134aa4492..05019f6fd 100644
--- a/slips_files/core/helpers/symbols_handler.py
+++ b/slips_files/core/helpers/symbols_handler.py
@@ -95,7 +95,7 @@ def compute(
 
             # Get the time of the last flow in this tuple, and the last last
             # Implicitely this is converting what we stored as 'now' into 'last_ts' and what we stored as 'last_ts' as 'last_last_ts'
-            (last_last_ts, last_ts) = self.db.getT2ForProfileTW(
+            (last_last_ts, last_ts) = self.db.get_t2_for_profile_tw(
                 profileid, twid, tupleid, tuple_key
             )
             # self.print(f'Profileid: {profileid}. Data extracted from DB. last_ts: {last_ts}, last_last_ts: {last_last_ts}', 0, 5)
diff --git a/slips_files/core/profiler.py b/slips_files/core/profiler.py
index 2cfac61e1..ee2b3464d 100644
--- a/slips_files/core/profiler.py
+++ b/slips_files/core/profiler.py
@@ -226,7 +226,7 @@ def store_features_going_out(self):
 
         # if the flow type matched any of the ifs above,
         # mark this profile as modified
-        self.db.markProfileTWAsModified(self.profileid, self.twid, '')
+        self.db.mark_profile_tw_as_modified(self.profileid, self.twid, '')
 
     def store_features_going_in(self, profileid: str, twid: str):
         """
@@ -271,7 +271,7 @@ def store_features_going_in(self, profileid: str, twid: str):
             twid=twid,
             label=self.label,
         )
-        self.db.markProfileTWAsModified(profileid, twid, '')
+        self.db.mark_profile_tw_as_modified(profileid, twid, '')
 
     def handle_in_flows(self):
         """

From 4807f38a0eaa56135cbd1c37843c3c554d36598b Mon Sep 17 00:00:00 2001
From: alya <alyaggomaa@gmail.com>
Date: Wed, 6 Mar 2024 16:47:20 +0200
Subject: [PATCH 2/7] db: refactor set_local_network()

---
 .../core/database/redis_db/database.py        | 28 +++++++++++--------
 .../core/database/redis_db/profile_handler.py |  8 ++----
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py
index c0776f12f..ff981fb9e 100644
--- a/slips_files/core/database/redis_db/database.py
+++ b/slips_files/core/database/redis_db/database.py
@@ -474,23 +474,27 @@ def get_equivalent_tws(self, hrs: float):
         for example if the tw width is 1h, and hrs is 24, this function returns 24
         """
         return int(hrs*3600/self.width)
-
+    
+    def should_set_localnet(self, saddr) -> bool:
+        is_private_ip = validators.ipv4(saddr) and utils.is_private_ip(
+            ipaddress.ip_address(saddr))
+            
+        if (
+                self.is_localnet_set
+                or saddr in ('0.0.0.0', '255.255.255.255')
+                or not is_private_ip
+        ):
+            return False
+        return True
+    
     def set_local_network(self, saddr):
         # set the local network used in the db
         # For now the local network is only ipv4, but it
         # could be ipv6 in the future. Todo.
 
-        if self.is_localnet_set:
-            return
-
-        if saddr in ('0.0.0.0', '255.255.255.255'):
-            return
-
-        if not (
-                validators.ipv4(saddr)
-                and utils.is_private_ip(ipaddress.ip_address(saddr))
-        ):
-            return
+       if not self.should_set_localnet(saddr):
+           return False
+        
         # get the local network of this saddr
         if network_range := utils.get_cidr_of_ip(saddr):
             self.r.set("local_network", network_range)
diff --git a/slips_files/core/database/redis_db/profile_handler.py b/slips_files/core/database/redis_db/profile_handler.py
index 7c2b2265b..90bb7d012 100644
--- a/slips_files/core/database/redis_db/profile_handler.py
+++ b/slips_files/core/database/redis_db/profile_handler.py
@@ -309,10 +309,6 @@ def add_port(
         ip = str(flow.daddr)
         spkts = flow.spkts
         state_hist = flow.state_hist if hasattr(flow, "state_hist") else ""
-        # dpkts = columns['dpkts']
-        # daddr = columns['daddr']
-        # saddr = columns['saddr']
-        # sbytes = columns['sbytes']
 
         if "^" in state_hist:
             # The majority of the FP with horizontal port scan detection happen because a
@@ -737,7 +733,7 @@ def get_all_contacted_ips_in_profileid_twid(self, profileid, twid) -> dict:
         """
         Get all the contacted IPs in a given profile and TW
         """
-        all_flows: dict = self.db.get_all_flows_in_profileid_twid(profileid, twid)
+        all_flows: dict = self.get_all_flows_in_profileid_twid(profileid, twid)
         if not all_flows:
             return {}
         contacted_ips = {}
@@ -834,7 +830,7 @@ def add_flow(
             self.set_input_metadata({"file_start": flow.starttime})
             self.first_flow = False
 
-        self.db.set_local_network(flow.saddr)
+        self.set_local_network(flow.saddr)
 
         # dont send arp flows in this channel, they have their own new_arp channel
         if flow.type_ != "arp":

From b9aee6de5f258e952c56e970391f01ac099710dc Mon Sep 17 00:00:00 2001
From: alya <alyaggomaa@gmail.com>
Date: Wed, 6 Mar 2024 17:32:23 +0200
Subject: [PATCH 3/7] profiler: set the local network as the network of the
 first client_ip if available, or the localnet of the first private srcip seen

---
 slips_files/common/slips_utils.py |  9 ++--
 slips_files/core/profiler.py      | 77 ++++++++++++++++++++++++++++---
 2 files changed, 76 insertions(+), 10 deletions(-)

diff --git a/slips_files/common/slips_utils.py b/slips_files/common/slips_utils.py
index 4c9bf8c65..57e3d89c6 100644
--- a/slips_files/common/slips_utils.py
+++ b/slips_files/common/slips_utils.py
@@ -66,10 +66,10 @@ def __init__(self):
         self.local_tz = self.get_local_timezone()
         self.aid = aid_hash.AID()
 
-    def get_cidr_of_ip(self, ip):
+    def get_cidr_of_private_ip(self, ip):
         """
         returns the cidr/range of the given private ip
-        :param ip: should be  a private ips
+        :param ip: should be a private ip
         """
         if validators.ipv4(ip):
             first_octet = ip.split('.')[0]
@@ -322,7 +322,7 @@ def get_own_IPs(self) -> list:
     def convert_to_mb(self, bytes):
         return int(bytes)/(10**6)
 
-    def is_private_ip(self, ip_obj:ipaddress) -> bool:
+    def is_private_ip(self, ip_obj: ipaddress) -> bool:
         """
         This function replaces the ipaddress library 'is_private'
         because it does not work correctly and it does not ignore
@@ -331,7 +331,8 @@ def is_private_ip(self, ip_obj:ipaddress) -> bool:
         # Is it a well-formed ipv4 or ipv6?
         r_value = False
         if ip_obj and ip_obj.is_private:
-            if ip_obj != ipaddress.ip_address('0.0.0.0') and ip_obj != ipaddress.ip_address('255.255.255.255'):
+            if (ip_obj != ipaddress.ip_address('0.0.0.0')
+                    and ip_obj != ipaddress.ip_address('255.255.255.255')):
                 r_value = True
         return r_value
 
diff --git a/slips_files/core/profiler.py b/slips_files/core/profiler.py
index ee2b3464d..daeabc66d 100644
--- a/slips_files/core/profiler.py
+++ b/slips_files/core/profiler.py
@@ -19,10 +19,12 @@
 # stratosphere@aic.fel.cvut.cz
 from dataclasses import asdict
 import queue
-import sys
 import ipaddress
 import pprint
 from datetime import datetime
+from typing import List
+
+import validators
 
 from slips_files.common.imports import *
 from slips_files.common.abstracts.core import ICore
@@ -33,8 +35,7 @@
 from slips_files.core.input_profilers.nfdump import Nfdump
 from slips_files.core.input_profilers.suricata import Suricata
 from slips_files.core.input_profilers.zeek import ZeekJSON, ZeekTabs
-
-
+from slips_files.core.output import Output
 
 
 SUPPORTED_INPUT_TYPES = {
@@ -57,7 +58,7 @@
 class Profiler(ICore):
     """A class to create the profiles for IPs"""
     name = 'Profiler'
-
+    
     def init(self,
              is_profiler_done: multiprocessing.Semaphore = None,
              profiler_queue=None,
@@ -75,9 +76,9 @@ def init(self,
         self.input_type = False
         self.whitelisted_flows_ctr = 0
         self.rec_lines = 0
+        self.is_localnet_set = False
         self.has_pbar = has_pbar
         self.whitelist = Whitelist(self.logger, self.db)
-        # Read the configuration
         self.read_configuration()
         self.symbol = SymbolHandler(self.logger, self.db)
         # there has to be a timeout or it will wait forever and never
@@ -99,6 +100,8 @@ def read_configuration(self):
         self.analysis_direction = conf.analysis_direction()
         self.label = conf.label()
         self.width = conf.get_tw_width_as_float()
+        self.client_ips: List[str] = conf.client_ips()
+
 
     def convert_starttime_to_epoch(self):
         try:
@@ -284,6 +287,25 @@ def handle_in_flows(self):
             return
         rev_profileid, rev_twid = self.get_rev_profile()
         self.store_features_going_in(rev_profileid, rev_twid)
+    
+
+    def should_set_localnet(self) -> bool:
+        """
+        returns true only if the saddr of the current flow is ipv4, private
+        and we don't have the local_net set already
+        """
+        is_ipv4 = validators.ipv4(self.flow.saddr)
+        saddr_obj: ipaddress = ipaddress.ip_address(self.flow.saddr)
+        is_private_ip = utils.is_private_ip(saddr_obj)
+
+        if (
+                self.is_localnet_set
+                or not is_ipv4
+                or not is_private_ip
+        ):
+            return False
+        return True
+    
 
     def define_separator(self, line: dict, input_type: str):
         """
@@ -371,10 +393,52 @@ def init_pbar(self, total_flows:int):
             }
         })
         self.supported_pbar = True
+    
+    def get_private_client_ips(self) -> List[str]:
+        """
+        returns the private ips found in the client_ips param
+        in the config file
+        """
+        private_clients = []
+        for ip in self.client_ips:
+            if utils.is_private_ip(ipaddress.ip_address(ip)):
+                private_clients.append(ip)
+        return private_clients
+        
+
+    def get_local_net(self) -> str:
+        """
+        gets the local network from client_ip param in the config file,
+        or by using the localnetwork of the first private
+        srcip seen in the traffic
+        """
+        # For now the local network is only ipv4, but it
+        # could be ipv6 in the future. Todo.
+        private_client_ips: List[str] = self.get_private_client_ips()
+        if private_client_ips:
+            # all client ips should belong to the same local network,
+            # it doesn't make sense to have ips belonging to different
+            # networks in the config file!
+            ip = private_client_ips[0]
+        else:
+            ip = self.flow.saddr
 
+        self.is_localnet_set = True
+        return utils.get_cidr_of_private_ip(ip)
+        
+    def handle_setting_local_net(self):
+        """
+        stores the local network if possible
+        """
+        if not self.should_set_localnet():
+            return
+        
+        local_net: str = self.get_local_net()
+        self.db.set_local_network(local_net)
+    
     def pre_main(self):
         utils.drop_root_privs()
-
+    
     def main(self):
         while not self.should_stop():
             try:
@@ -426,6 +490,7 @@ def main(self):
             self.flow = self.input.process_line(line)
             if self.flow:
                 self.add_flow_to_profile()
+                self.handle_setting_local_net()
 
             # now that one flow is processed tell output.py
             # to update the bar

From 62b802fc18e32e9f0432f1140626a49f531c1425 Mon Sep 17 00:00:00 2001
From: alya <alyaggomaa@gmail.com>
Date: Wed, 6 Mar 2024 17:33:02 +0200
Subject: [PATCH 4/7] db: don't handle getting the local net by
 set_local_network(), handle it in profiler instead

---
 .../core/database/redis_db/database.py        | 33 +++++--------------
 1 file changed, 9 insertions(+), 24 deletions(-)

diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py
index ff981fb9e..e35509e17 100644
--- a/slips_files/core/database/redis_db/database.py
+++ b/slips_files/core/database/redis_db/database.py
@@ -178,6 +178,7 @@ def _read_configuration(cls):
         cls.deletePrevdb: bool = conf.deletePrevdb()
         cls.disabled_detections: List[str] = conf.disabled_detections()
         cls.width = conf.get_tw_width_as_float()
+        cls.client_ips: List[str] = conf.client_ips()
 
     @classmethod
     def set_slips_internal_time(cls, timestamp):
@@ -475,36 +476,20 @@ def get_equivalent_tws(self, hrs: float):
         """
         return int(hrs*3600/self.width)
     
-    def should_set_localnet(self, saddr) -> bool:
-        is_private_ip = validators.ipv4(saddr) and utils.is_private_ip(
-            ipaddress.ip_address(saddr))
-            
-        if (
-                self.is_localnet_set
-                or saddr in ('0.0.0.0', '255.255.255.255')
-                or not is_private_ip
-        ):
-            return False
-        return True
     
-    def set_local_network(self, saddr):
-        # set the local network used in the db
-        # For now the local network is only ipv4, but it
-        # could be ipv6 in the future. Todo.
+    
+    def set_local_network(self, cidr):
+        """
+        set the local network used in the db
+        """
+        self.r.set("local_network", cidr)
 
-       if not self.should_set_localnet(saddr):
-           return False
-        
-        # get the local network of this saddr
-        if network_range := utils.get_cidr_of_ip(saddr):
-            self.r.set("local_network", network_range)
-            self.is_localnet_set = True
+    def get_local_network(self):
+         return self.r.get("local_network")
 
     def get_used_port(self):
         return int(self.r.config_get('port')['port'])
 
-    def get_local_network(self):
-         return self.r.get("local_network")
 
     def get_label_count(self, label):
         """

From 975fa634a87f3ccd0a3e0bec9a975efe088795ae Mon Sep 17 00:00:00 2001
From: alya <alyaggomaa@gmail.com>
Date: Thu, 7 Mar 2024 12:58:53 +0200
Subject: [PATCH 5/7] flowalerts: set the threat level to low if the ip
 ouytside of local net is the srcip, otherwise set it to high

---
 modules/flowalerts/set_evidence.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/modules/flowalerts/set_evidence.py b/modules/flowalerts/set_evidence.py
index 8e5a38e3d..1b052a034 100644
--- a/modules/flowalerts/set_evidence.py
+++ b/modules/flowalerts/set_evidence.py
@@ -138,7 +138,8 @@ def different_localnet_usage(
         'srcip' outside the localnet or the 'dstip'?
         """
         srcip = profileid.split('_')[-1]
-        # the attacker here is the IP found to be private and outside the localnet
+        # the attacker here is the IP found to be
+        # private and outside the localnet
         if ip_outside_localnet == 'srcip':
             attacker = Attacker(
                 direction=Direction.SRC,
@@ -150,6 +151,7 @@ def different_localnet_usage(
                 victim_type=IoCType.IP,
                 value=daddr
             )
+            threat_level = ThreatLevel.LOW
             description = f'A connection from a private IP ({srcip}) ' \
                           f'outside of the used local network ' \
                           f'{self.db.get_local_network()}. To IP: {daddr} '
@@ -164,6 +166,7 @@ def different_localnet_usage(
                 victim_type=IoCType.IP,
                 value=srcip
             )
+            threat_level = ThreatLevel.HIGH
             description = f'A connection to a private IP ({daddr}) ' \
                           f'outside of the used local network ' \
                           f'{self.db.get_local_network()}. ' \
@@ -173,8 +176,8 @@ def different_localnet_usage(
 
 
         confidence = 1.0
-        threat_level = ThreatLevel.HIGH
-
+        
+        twid_number = int(twid.replace("timewindow", ""))
         evidence = Evidence(
             evidence_type=EvidenceType.DIFFERENT_LOCALNET,
             attacker=attacker,
@@ -183,7 +186,7 @@ def different_localnet_usage(
             description=description,
             victim=victim,
             profile=ProfileID(ip=srcip),
-            timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))),
+            timewindow=TimeWindow(number=twid_number),
             uid=[uid],
             timestamp=timestamp,
             conn_count=1,

From 70cd53a9e26a7740d2586184a1f585d06be58656 Mon Sep 17 00:00:00 2001
From: alya <alyaggomaa@gmail.com>
Date: Thu, 7 Mar 2024 14:36:47 +0200
Subject: [PATCH 6/7] update the docs of "Connection to private IPs outside the
 current local network"

---
 docs/flowalerts.md | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/docs/flowalerts.md b/docs/flowalerts.md
index 3173bc49c..c7ba3b57d 100644
--- a/docs/flowalerts.md
+++ b/docs/flowalerts.md
@@ -319,6 +319,17 @@ For example if the currently used local network is: 192.168.1.0/24
 
 and slips sees a forged packet going from 192.168.1.2 to 10.0.0.1, it will alert
 
+Slips detects the current local network by using the local network of the private
+ips specified in ```client_ips``` parameter in ```slips.conf```
+
+If no IPs are specified, slips uses the local network of the first private source ip 
+found in the traffic.
+
+This threat level of this detection is low if the source ip is the one outside of local network 
+because it's unlikely.
+and high if the destination ip is the one outside of local network. 
+
+
 ## High entropy DNS TXT answers 
 
 Slips check every DNS answer with TXT record for high entropy

From 96f9d576fcf68c26bd10e516d7da8f34419f53a6 Mon Sep 17 00:00:00 2001
From: alya <alyaggomaa@gmail.com>
Date: Thu, 7 Mar 2024 16:18:11 +0200
Subject: [PATCH 7/7] profiler: if we have client ips param set, don't check
 for private source ips to use

---
 slips_files/common/slips_utils.py             |  2 +-
 .../core/database/redis_db/profile_handler.py |  2 --
 slips_files/core/profiler.py                  | 24 +++++++++++--------
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/slips_files/common/slips_utils.py b/slips_files/common/slips_utils.py
index 57e3d89c6..e9afb2d05 100644
--- a/slips_files/common/slips_utils.py
+++ b/slips_files/common/slips_utils.py
@@ -69,7 +69,7 @@ def __init__(self):
     def get_cidr_of_private_ip(self, ip):
         """
         returns the cidr/range of the given private ip
-        :param ip: should be a private ip
+        :param ip: should be a private ipv4
         """
         if validators.ipv4(ip):
             first_octet = ip.split('.')[0]
diff --git a/slips_files/core/database/redis_db/profile_handler.py b/slips_files/core/database/redis_db/profile_handler.py
index 90bb7d012..90c33b721 100644
--- a/slips_files/core/database/redis_db/profile_handler.py
+++ b/slips_files/core/database/redis_db/profile_handler.py
@@ -830,8 +830,6 @@ def add_flow(
             self.set_input_metadata({"file_start": flow.starttime})
             self.first_flow = False
 
-        self.set_local_network(flow.saddr)
-
         # dont send arp flows in this channel, they have their own new_arp channel
         if flow.type_ != "arp":
             self.publish("new_flow", to_send)
diff --git a/slips_files/core/profiler.py b/slips_files/core/profiler.py
index daeabc66d..e91e68d0d 100644
--- a/slips_files/core/profiler.py
+++ b/slips_files/core/profiler.py
@@ -1,7 +1,5 @@
 # Stratosphere Linux IPS. A machine-learning Intrusion Detection System
 # Copyright (C) 2021 Sebastian Garcia
-import multiprocessing
-
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
@@ -294,16 +292,22 @@ def should_set_localnet(self) -> bool:
         returns true only if the saddr of the current flow is ipv4, private
         and we don't have the local_net set already
         """
-        is_ipv4 = validators.ipv4(self.flow.saddr)
+        if self.is_localnet_set:
+            return False
+        
+        if self.get_private_client_ips():
+            # if we have private client ips, we're ready to set the
+            # localnetwork
+            return True
+        
+        if not validators.ipv4(self.flow.saddr):
+            return False
+        
         saddr_obj: ipaddress = ipaddress.ip_address(self.flow.saddr)
         is_private_ip = utils.is_private_ip(saddr_obj)
-
-        if (
-                self.is_localnet_set
-                or not is_ipv4
-                or not is_private_ip
-        ):
+        if not is_private_ip:
             return False
+        
         return True
     
 
@@ -444,7 +448,7 @@ def main(self):
             try:
                 # this msg can be a str only when it's a 'stop' msg indicating
                 # that this module should stop
-                msg: dict = self.profiler_queue.get(timeout=1, block=False)
+                msg = self.profiler_queue.get(timeout=1, block=False)
                 # ALYA, DO NOT REMOVE THIS CHECK
                 # without it, there's no way thi module will know it's time to
                 # stop and no new fows are coming