diff --git a/docs/contributing.md b/docs/contributing.md index 0995c7918..1c807bf64 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -161,8 +161,14 @@ Evidence Handler is the only process that stops but keeps waiting in memory for Once all modules are done processing, EvidenceHandler is killed by the Process manager. -### How the tests work? +### How does the tests work? - Running the tests locally should be done using ./tests/run_all_tests.sh - It runs the unit tests first, then the integration tests. - Please get familiar with pytest first https://docs.pytest.org/en/stable/how-to/output.html + +### Where and how do we get the GW info? + +Using one of these 3 ways + + Optional[str]: """ Given the gw_ip, this function tries to get the MAC from arp.log or from arp tables @@ -352,6 +354,7 @@ def get_gateway_mac(self, gw_ip: str): return gw_mac if not self.is_running_non_stop: + # running on pcap or a given zeek file/dir # no MAC in arp.log (in the db) and can't use arp tables, # so it's up to the db.is_gw_mac() function to determine the gw mac # if it's seen associated with a public IP @@ -492,9 +495,14 @@ def pre_main(self): utils.drop_root_privs() self.wait_for_dbs() # the following method only works when running on an interface - if ip := self.get_gateway_ip(): + if ip := self.get_gateway_ip_if_interface(): self.db.set_default_gateway("IP", ip) + # whether we found the gw ip using dhcp in profiler + # or using ip route using self.get_gateway_ip() + # now that it's found, get and store the mac addr of it + self.get_gateway_mac(ip) + def handle_new_ip(self, ip: str): try: # make sure its a valid ip @@ -531,16 +539,6 @@ async def main(self): self.get_vendor(mac_addr, profileid) self.check_if_we_have_pending_offline_mac_queries() - # set the gw mac and ip if they're not set yet - if not self.is_gw_mac_set: - # whether we found the gw ip using dhcp in profiler - # or using ip route using self.get_gateway_ip() - # now that it's found, get and store the mac addr of it - if ip := self.db.get_gateway_ip(): - # now that we know the GW IP address, - # try to get the MAC of this IP (of the gw) - self.get_gateway_mac(ip) - self.is_gw_mac_set = True if msg := self.get_msg("new_dns"): msg = json.loads(msg["data"]) diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index 0123dd3e2..15ce295f8 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -1539,7 +1539,10 @@ def increment_processed_flows(self): return self.r.incr(self.constants.PROCESSED_FLOWS, 1) def get_processed_flows_so_far(self) -> int: - return int(self.r.get(self.constants.PROCESSED_FLOWS)) + processed_flows = self.r.get(self.constants.PROCESSED_FLOWS) + if not processed_flows: + return 0 + return int(processed_flows) def store_std_file(self, **kwargs): """ diff --git a/slips_files/core/helpers/flow_handler.py b/slips_files/core/helpers/flow_handler.py index ae14d50c1..3a04c3cb5 100644 --- a/slips_files/core/helpers/flow_handler.py +++ b/slips_files/core/helpers/flow_handler.py @@ -60,6 +60,10 @@ def new_software(self, profileid, flow): class FlowHandler: + """ + Each flow seen by slips will be a different instance of this class + """ + def __init__(self, db, symbol_handler, flow): self.db = db self.publisher = Publisher(self.db) @@ -161,9 +165,18 @@ def handle_notice(self): self.db.add_out_notice(self.profileid, self.twid, self.flow) if "Gateway_addr_identified" in self.flow.note: - # get the gw addr form the msg - gw_addr = self.flow.msg.split(": ")[-1].strip() - self.db.set_default_gateway("IP", gw_addr) + # foirst check if the gw ip and mac are set by + # profiler.get_gateway_info() or ip_info module + gw_ip = False + if not self.db.get_gateway_ip(): + # get the gw addr from the msg + gw_ip = self.flow.msg.split(": ")[-1].strip() + self.db.set_default_gateway("IP", gw_ip) + + if not self.db.get_gateway_mac() and gw_ip: + gw_mac = self.db.get_mac_addr_from_profile(f"profile_{gw_ip}") + if gw_mac: + self.db.set_default_gateway("MAC", gw_mac) self.db.add_altflow(self.flow, self.profileid, self.twid, "benign") diff --git a/slips_files/core/profiler.py b/slips_files/core/profiler.py index 0e9554477..83b177d7d 100644 --- a/slips_files/core/profiler.py +++ b/slips_files/core/profiler.py @@ -4,6 +4,7 @@ # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. +import json # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -22,6 +23,7 @@ import multiprocessing from typing import ( List, + Optional, ) import validators @@ -95,6 +97,8 @@ def init( # is set by this proc to tell input proc that we are done # processing and it can exit no issue self.is_profiler_done_event = is_profiler_done_event + self.gw_mac = None + self.gw_ip = None def read_configuration(self): conf = ConfigParser() @@ -140,6 +144,87 @@ def get_rev_profile(self): ) return rev_profileid, rev_twid + def get_gw_ip_using_gw_mac(self) -> Optional[str]: + """ + gets the ip of the given mac from the db + prioritizes returning the ipv4. if not found, the function returns + the ipv6. or none if both are not found. + """ + # the db returns a serialized list of IPs belonging to this mac + gw_ips: str = self.db.get_ip_of_mac(self.gw_mac) + + if not gw_ips: + return + + gw_ips: List[str] = json.loads(gw_ips) + # try to get the ipv4 if found in that list + for ip in gw_ips: + try: + ipaddress.IPv4Address(ip) + return ip + except ipaddress.AddressValueError: + continue + + # all of them are ipv6, return the first + return gw_ips[0] + + def is_gw_info_detected(self, info_type: str) -> bool: + """ + checks own attributes and the db for the gw mac/ip + :param info_type: can be 'mac' or 'ip' + """ + info_mapping = { + "mac": ("gw_mac", self.db.get_gateway_mac), + "ip": ("gw_ip", self.db.get_gateway_ip), + } + + if info_type not in info_mapping: + raise ValueError(f"Unsupported info_type: {info_type}") + + attr, check_db_method = info_mapping[info_type] + + if getattr(self, attr): + # the reason we don't just check the db is we don't want a db + # call per each flow + return True + + # did some other module manage to get it? + if info := check_db_method(): + setattr(self, attr, info) + return True + + return False + + def get_gateway_info(self): + """ + Gets the IP and MAC of the gateway and stores them in the db + + usually the mac of the flow going from a private ip -> a + public ip is the mac of the GW + """ + gw_mac_found: bool = self.is_gw_info_detected("mac") + if not gw_mac_found: + if utils.is_private_ip( + self.flow.saddr + ) and not utils.is_ignored_ip(self.flow.daddr): + self.gw_mac: str = self.flow.dmac + self.db.set_default_gateway("MAC", self.gw_mac) + self.print( + f"MAC address of the gateway detected: " + f"{green(self.gw_mac)}" + ) + gw_mac_found = True + + # we need the mac to be set to be able to find the ip using it + if not self.is_gw_info_detected("ip") and gw_mac_found: + self.gw_ip: Optional[str] = self.get_gw_ip_using_gw_mac() + if self.gw_ip: + self.db.set_default_gateway("IP", self.gw_ip) + self.print( + f"IP address of the gateway detected: " + f"{green(self.gw_ip)}" + ) + def add_flow_to_profile(self): """ This is the main function that takes the columns of a flow @@ -170,6 +255,8 @@ def add_flow_to_profile(self): # software and weird.log flows are allowed to not have a daddr return False + self.get_gateway_info() + # Check if the flow is whitelisted and we should not process it if self.whitelist.is_whitelisted_flow(self.flow): return True diff --git a/tests/test_flow_handler.py b/tests/test_flow_handler.py index bb2a03383..50eb76cda 100644 --- a/tests/test_flow_handler.py +++ b/tests/test_flow_handler.py @@ -280,12 +280,17 @@ def test_handle_notice(flow): flow.note = "Gateway_addr_identified: 192.168.1.1" flow.msg = "Gateway_addr_identified: 192.168.1.1" + flow_handler.db.get_gateway_ip.return_value = False + flow_handler.db.get_gateway_mac.return_value = False + flow_handler.db.get_mac_addr_from_profile.return_value = "xyz" + flow_handler.handle_notice() flow_handler.db.add_out_notice.assert_called_with( flow_handler.profileid, flow_handler.twid, flow ) - flow_handler.db.set_default_gateway.assert_called_with("IP", "192.168.1.1") + flow_handler.db.set_default_gateway.assert_any_call("IP", "192.168.1.1") + flow_handler.db.set_default_gateway.assert_any_call("MAC", "xyz") flow_handler.db.add_altflow.assert_called_with( flow, flow_handler.profileid, flow_handler.twid, "benign" ) diff --git a/tests/test_ip_info.py b/tests/test_ip_info.py index 143648cd0..fabbcd9c8 100644 --- a/tests/test_ip_info.py +++ b/tests/test_ip_info.py @@ -421,7 +421,7 @@ def test_get_gateway_ip( mocker.patch("platform.system", return_value=platform_system) mocker.patch("subprocess.check_output", return_value=subprocess_output) mocker.patch("sys.argv", ["-i", "eth0"]) - result = ip_info.get_gateway_ip() + result = ip_info.get_gateway_ip_if_interface() assert result == expected_ip diff --git a/tests/test_profiler.py b/tests/test_profiler.py index d2f1d2d74..4c4025960 100644 --- a/tests/test_profiler.py +++ b/tests/test_profiler.py @@ -643,3 +643,141 @@ def test_notify_observers_with_correct_message(): test_msg = {"action": "test_action"} profiler.notify_observers(test_msg) observer_mock.update.assert_called_once_with(test_msg) + + +@patch("slips_files.core.profiler.utils.is_private_ip") +@patch("slips_files.core.profiler.utils.is_ignored_ip") +def test_get_gateway_info_sets_mac_and_ip( + mock_is_ignored_ip, mock_is_private_ip +): + profiler = ModuleFactory().create_profiler_obj() + # mac not detected, ip not detected + profiler.is_gw_info_detected = Mock() + profiler.is_gw_info_detected.side_effect = [False, False] + mock_is_private_ip.return_value = True + mock_is_ignored_ip.return_value = False + profiler.get_gw_ip_using_gw_mac = Mock() + profiler.get_gw_ip_using_gw_mac.return_value = "8.8.8.1" + profiler.flow = Conn( + "1.0", + "1234", + "192.168.1.1", + "8.8.8.8", + 5, + "TCP", + "dhcp", + 80, + 88, + 20, + 20, + 20, + 20, + "", + "00:11:22:33:44:55", + "Established", + "", + ) + profiler.get_gateway_info() + + profiler.db.set_default_gateway.assert_any_call("MAC", profiler.flow.dmac) + profiler.db.set_default_gateway.assert_any_call("IP", "8.8.8.1") + + +@patch("slips_files.core.profiler.utils.is_private_ip") +def test_get_gateway_info_no_mac_detected(mock_is_private_ip): + profiler = ModuleFactory().create_profiler_obj() + + # mac not detected, ip not detected + profiler.is_gw_info_detected = Mock() + profiler.is_gw_info_detected.side_effect = [False, False] + mock_is_private_ip.return_value = False + profiler.flow = Conn( + "1.0", + "1234", + "192.168.1.1", + "8.8.8.8", + 5, + "TCP", + "dhcp", + 80, + 88, + 20, + 20, + 20, + 20, + "", + "00:11:22:33:44:55", + "Established", + "", + ) + profiler.get_gateway_info() + + # mac and ip should not be set + profiler.db.set_default_gateway.assert_not_called() + profiler.print.assert_not_called() + + +def test_get_gateway_info_mac_detected_but_no_ip(): + profiler = ModuleFactory().create_profiler_obj() + # mac detected, ip not detected + profiler.is_gw_info_detected = Mock() + profiler.is_gw_info_detected.side_effect = [True, False] + profiler.get_gw_ip_using_gw_mac = Mock() + profiler.get_gw_ip_using_gw_mac.return_value = None + + profiler.get_gateway_info() + + # assertions for mac + profiler.db.set_default_gateway.assert_not_called() + profiler.print.assert_not_called() + + +@pytest.mark.parametrize( + "info_type, attr_name, db_method, db_value", + [ + ("mac", "gw_mac", "get_gateway_mac", "00:1A:2B:3C:4D:5E"), + ("ip", "gw_ip", "get_gateway_ip", "192.168.1.1"), + ], +) +def test_is_gw_info_detected(info_type, attr_name, db_method, db_value): + # create a profiler object using the ModuleFactory + profiler = ModuleFactory().create_profiler_obj() + + # mock the profiler's database methods and attributes + setattr(profiler, attr_name, None) + getattr(profiler.db, db_method).return_value = db_value + + # test with info_type + result = profiler.is_gw_info_detected(info_type) + + # assertions + assert result + assert getattr(profiler, attr_name) == db_value + getattr(profiler.db, db_method).assert_called_once() + + +def test_is_gw_info_detected_unsupported_info_type(): + # create a profiler object using the ModuleFactory + profiler = ModuleFactory().create_profiler_obj() + + # test with an unsupported info_type + with pytest.raises(ValueError) as exc_info: + profiler.is_gw_info_detected("unsupported_type") + + # assertion + assert str(exc_info.value) == "Unsupported info_type: unsupported_type" + + +def test_is_gw_info_detected_when_attribute_is_already_set(): + # create a profiler object using the ModuleFactory + profiler = ModuleFactory().create_profiler_obj() + + # set gw_mac attribute to a value + profiler.gw_mac = "00:1A:2B:3C:4D:5E" + + # test with info_type "mac" + result = profiler.is_gw_info_detected("mac") + + # assertions + assert result + assert profiler.gw_mac == "00:1A:2B:3C:4D:5E"