diff --git a/CHANGELOG.md b/CHANGELOG.md index 82f3c8e9a..f62738d47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,20 @@ +- 1.0.12 (March 2024) +- Add an option to specify the current client IP in slips.conf to help avoid false positives. +- Better handling of URLhaus threat intelligence. +- Change how slips determines the local network of the current client IP. +- Fix issues with the progress bar. +- Fix problem logging alerts and errors to alerts.log and erros.log. +- Fix problem reporting evidence to other peers. +- Fix problem starting the web interface. +- Fix whitelists. +- Improve how the evidence for young domain detections is set. +- Remove the description of blacklisted IPs from the evidence description and add the source TI feed instead. +- Set evidence to all young domain IPs when a connection to a young domain is found. +- Set two evidence in some detections e.g. when the source address connects to a blacklisted IP, evidence is set for both. +- Use blacklist name instead of IP description in all evidence. +- Use the latest Redis and NodeJS version in all docker images. + + - 1.0.11 (February 2024) - Improve the logging of evidence in alerts.json and alerts.log. - Optimize the storing of evidence in the Redis database. diff --git a/README.md b/README.md index c1b0f5448..ea846b426 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

-Slips v1.0.11 +Slips v1.0.12

diff --git a/VERSION b/VERSION index 86844988e..492b167a6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.11 \ No newline at end of file +1.0.12 \ No newline at end of file diff --git a/config/slips.conf b/config/slips.conf index 24514146d..8ef2a9da8 100644 --- a/config/slips.conf +++ b/config/slips.conf @@ -137,6 +137,14 @@ export_labeled_flows = no # export_format can be tsv or json. this parameter is ignored if export_labeled_flows is set to no export_format = json +# These are the IPs that we see the majority of traffic going out of from. +# for example, this can be your own IP or some computer you’re monitoring +# when using slips on an interface, this client IP is automatically set as +# your own IP and is used to improve detections +# it would be useful to specify it when analyzing pcaps or zeek logs +#client_ips = [10.0.0.1, 172.16.0.9, 172.217.171.238] +client_ips = [] + ##################### # [2] Configuration for the detections [detection] diff --git a/docker/P2P-image/Dockerfile b/docker/P2P-image/Dockerfile index 7934f3d17..249556f5a 100644 --- a/docker/P2P-image/Dockerfile +++ b/docker/P2P-image/Dockerfile @@ -18,14 +18,18 @@ RUN apt update && apt install -y --no-install-recommends \ curl \ gnupg \ nano \ + lsb-release \ && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_20.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list \ - && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null + && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/security_zeek.gpg \ + && curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/redis.list + # Install Slips dependencies. -RUN apt update && apt install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y --no-install-recommends \ python3 \ - redis-server \ zeek \ + redis \ python3-pip \ python3-certifi \ python3-dev \ @@ -33,8 +37,8 @@ RUN apt update && apt install -y --no-install-recommends \ file \ lsof \ net-tools \ - iproute2 \ iptables \ + iproute2 \ python3-tzlocal \ nfdump \ tshark \ @@ -68,7 +72,7 @@ RUN pip3 install -r install/requirements.txt # For Kalipso: -RUN curl -fsSL https://deb.nodesource.com/setup_19.x | bash - && apt install -y --no-install-recommends nodejs +RUN curl -fsSL https://deb.nodesource.com/setup_21.x | bash - && apt install -y --no-install-recommends nodejs # Switch to kalipso dir to install node dependencies WORKDIR ${SLIPS_DIR}/modules/kalipso diff --git a/docker/dependency-image/Dockerfile b/docker/dependency-image/Dockerfile index ccbe368b7..9ea0b511b 100644 --- a/docker/dependency-image/Dockerfile +++ b/docker/dependency-image/Dockerfile @@ -31,14 +31,18 @@ RUN apt update && apt install -y --no-install-recommends \ git \ curl \ gnupg \ + lsb-release \ && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_20.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list \ - && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null + && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/security_zeek.gpg \ + && curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/redis.list + # Install Slips dependencies. -RUN apt update && apt install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y --no-install-recommends \ python3 \ - redis-server \ zeek \ + redis \ python3-pip \ python3-certifi \ python3-dev \ @@ -46,8 +50,8 @@ RUN apt update && apt install -y --no-install-recommends \ file \ lsof \ net-tools \ - iproute2 \ iptables \ + iproute2 \ python3-tzlocal \ nfdump \ tshark \ @@ -58,6 +62,7 @@ RUN apt update && apt install -y --no-install-recommends \ && ln -s /opt/zeek/bin/zeek /usr/local/bin/bro + # Install python dependencies # you should build the image using diff --git a/docker/macosm1-P2P-image/Dockerfile b/docker/macosm1-P2P-image/Dockerfile index cbde30a75..877dadb6e 100644 --- a/docker/macosm1-P2P-image/Dockerfile +++ b/docker/macosm1-P2P-image/Dockerfile @@ -18,14 +18,18 @@ RUN apt update && apt install -y --no-install-recommends \ git \ curl \ gnupg \ + lsb-release \ && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_20.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list \ - && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null + && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/security_zeek.gpg \ + && curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/redis.list + # Install Slips dependencies. -RUN apt update && apt install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y --no-install-recommends \ python3 \ - redis-server \ zeek \ + redis \ python3-pip \ python3-certifi \ python3-dev \ @@ -33,8 +37,8 @@ RUN apt update && apt install -y --no-install-recommends \ file \ lsof \ net-tools \ - iproute2 \ iptables \ + iproute2 \ python3-tzlocal \ nfdump \ tshark \ @@ -64,7 +68,7 @@ RUN pip3 install --upgrade pip RUN pip3 install -r ${SLIPS_DIR}/docker/macosm1-P2P-image/requirements-macos-m1-docker.txt # For Kalipso: -RUN curl -fsSL https://deb.nodesource.com/setup_19.x | bash - && apt install -y --no-install-recommends nodejs +RUN curl -fsSL https://deb.nodesource.com/setup_21.x | bash - && apt install -y --no-install-recommends nodejs # Switch to kalipso dir to install node dependencies WORKDIR ${SLIPS_DIR}/modules/kalipso diff --git a/docker/macosm1-P2P-image/requirements-macos-m1-docker.txt b/docker/macosm1-P2P-image/requirements-macos-m1-docker.txt index b4086debb..82752adb9 100644 --- a/docker/macosm1-P2P-image/requirements-macos-m1-docker.txt +++ b/docker/macosm1-P2P-image/requirements-macos-m1-docker.txt @@ -23,7 +23,7 @@ pytest-mock pytest-xdist slackclient scipy -sklearn +scikit-learn GitPython protobuf blinker diff --git a/docker/macosm1-image/Dockerfile b/docker/macosm1-image/Dockerfile index 04f935120..f54edc14c 100644 --- a/docker/macosm1-image/Dockerfile +++ b/docker/macosm1-image/Dockerfile @@ -18,14 +18,18 @@ RUN apt update && apt install -y --no-install-recommends \ git \ curl \ gnupg \ + lsb-release \ && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_20.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list \ - && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null + && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/security_zeek.gpg \ + && curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/redis.list + # Install Slips dependencies. -RUN apt update && apt install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y --no-install-recommends \ python3 \ - redis-server \ zeek \ + redis \ python3-pip \ python3-certifi \ python3-dev \ @@ -33,8 +37,8 @@ RUN apt update && apt install -y --no-install-recommends \ file \ lsof \ net-tools \ - iproute2 \ iptables \ + iproute2 \ python3-tzlocal \ nfdump \ tshark \ @@ -60,7 +64,7 @@ RUN pip3 install --upgrade pip RUN pip3 install -r ${SLIPS_DIR}/docker/macosm1-image/requirements-macos-m1-docker.txt # For Kalipso: -RUN curl -fsSL https://deb.nodesource.com/setup_19.x | bash - && apt install -y --no-install-recommends nodejs +RUN curl -fsSL https://deb.nodesource.com/setup_21.x | bash - && apt install -y --no-install-recommends nodejs # Switch to kalipso dir to install node dependencies WORKDIR ${SLIPS_DIR}/modules/kalipso diff --git a/docker/tensorflow-image/Dockerfile b/docker/tensorflow-image/Dockerfile index 0a80d418c..f64116b88 100644 --- a/docker/tensorflow-image/Dockerfile +++ b/docker/tensorflow-image/Dockerfile @@ -16,14 +16,17 @@ RUN apt update && apt install -y --no-install-recommends \ git \ curl \ gnupg \ + lsb-release \ && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_20.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list \ - && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null + && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/security_zeek.gpg \ + && curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/redis.list + # Install Slips dependencies. -RUN apt update && apt install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y --no-install-recommends \ python3 \ - curl \ - redis-server \ + redis \ zeek \ python3-pip \ python3-certifi \ @@ -70,7 +73,7 @@ RUN pip3 install -r ${SLIPS_DIR}/install/requirements.txt # For Kalipso: -RUN curl -fsSL https://deb.nodesource.com/setup_19.x | bash - && apt install -y --no-install-recommends nodejs +RUN curl -fsSL https://deb.nodesource.com/setup_21.x | bash - && apt install -y --no-install-recommends nodejs # Switch to kalipso dir to install node dependencies WORKDIR ${SLIPS_DIR}/modules/kalipso diff --git a/docker/ubuntu-image/Dockerfile b/docker/ubuntu-image/Dockerfile index 9fb211165..c2ad44ce7 100644 --- a/docker/ubuntu-image/Dockerfile +++ b/docker/ubuntu-image/Dockerfile @@ -9,21 +9,25 @@ ENV IS_IN_A_DOCKER_CONTAINER True # destionation dir for slips inside the container ENV SLIPS_DIR /StratosphereLinuxIPS -# Install wget and add Zeek repository to our sources. +# Install wget and add Zeek and redis repositories to our sources. RUN apt update && apt install -y --no-install-recommends \ wget \ ca-certificates \ git \ curl \ gnupg \ + lsb-release \ && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_20.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list \ - && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null + && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/security_zeek.gpg \ + && curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/redis.list + # Install Slips dependencies. -RUN apt update && apt install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y --no-install-recommends \ python3 \ - redis-server \ zeek \ + redis \ python3-pip \ python3-certifi \ python3-dev \ @@ -31,8 +35,8 @@ RUN apt update && apt install -y --no-install-recommends \ file \ lsof \ net-tools \ - iproute2 \ iptables \ + iproute2 \ python3-tzlocal \ nfdump \ tshark \ @@ -53,7 +57,7 @@ RUN pip3 install --no-cache-dir -r ${SLIPS_DIR}/install/requirements.txt # For Kalipso: -RUN curl -fsSL https://deb.nodesource.com/setup_19.x | bash - && apt install -y --no-install-recommends nodejs +RUN curl -fsSL https://deb.nodesource.com/setup_21.x | bash - && apt install -y --no-install-recommends nodejs # Switch to kalipso dir to install node dependencies WORKDIR ${SLIPS_DIR}/modules/kalipso diff --git a/docs/flowalerts.md b/docs/flowalerts.md index 3173bc49c..c7ba3b57d 100644 --- a/docs/flowalerts.md +++ b/docs/flowalerts.md @@ -319,6 +319,17 @@ For example if the currently used local network is: 192.168.1.0/24 and slips sees a forged packet going from 192.168.1.2 to 10.0.0.1, it will alert +Slips detects the current local network by using the local network of the private +ips specified in ```client_ips``` parameter in ```slips.conf``` + +If no IPs are specified, slips uses the local network of the first private source ip +found in the traffic. + +This threat level of this detection is low if the source ip is the one outside of local network +because it's unlikely. +and high if the destination ip is the one outside of local network. + + ## High entropy DNS TXT answers Slips check every DNS answer with TXT record for high entropy diff --git a/docs/images/slips.gif b/docs/images/slips.gif index a8cc42813..5a3a4e9b9 100644 Binary files a/docs/images/slips.gif and b/docs/images/slips.gif differ diff --git a/docs/installation.md b/docs/installation.md index 22d82e141..5e9cd5392 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -294,7 +294,7 @@ _Note: for those using a different base image, you need to also install tensorfl As we mentioned before, the GUI of Slips known as Kalipso relies on NodeJs v19. Make sure to use NodeJs greater than version 12. For Kalipso to work, we will install the following npm packages: - curl -fsSL https://deb.nodesource.com/setup_19.x | bash - && apt install -y --no-install-recommends nodejs + curl -fsSL https://deb.nodesource.com/setup_21.x | sudo -E bash - && sudo apt install -y --no-install-recommends nodejs cd modules/kalipso && npm install #### Installing Zeek diff --git a/docs/usage.md b/docs/usage.md index 82d90a13e..dd63b1b8d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -248,7 +248,7 @@ There are two options how to run Kalipso Locally: You can run Kalipso as a shell script in another terminal using the command: - ```./kalipso.sh``` + ./kalipso.sh In docker, you can open a new terminal inside the slips container and execute ```./kalipso.sh``` @@ -276,14 +276,14 @@ The traffic of IP is splitted into time windows. each time window is 1h long of You can press Enter of any of them to view the list of flows in the timewindow. - You can switch to the flows view in kalipso by pressing TAB, now you can scroll on flows using arrows On the very top you can see the ASN, the GEO location, and the virustotal score of each IP if available -Check how to setup virustotal in Slips here https://stratospherelinuxips.readthedocs.io/en/develop/usage.html#popup-notifications +Check how to setup virustotal in Slips [here](https://stratospherelinuxips.readthedocs.io/en/develop/usage.html#popup-notifications). ### The Web Interface diff --git a/install/install.sh b/install/install.sh index 51dc90bb1..6df042fc3 100755 --- a/install/install.sh +++ b/install/install.sh @@ -1,32 +1,29 @@ #!/bin/sh -echo "[+] Installing zeek ...\n" +sudo apt-get update +echo "[+] Installing slips dependencies ...\n" sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python3 python3-dev swig zlib1g-dev sudo apt install -y --no-install-recommends \ wget \ ca-certificates \ git \ curl \ - gnupg + gnupg \ + lsb-release + echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_20.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null -sudo apt update -sudo apt install zeek +curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list -# create a symlink to zeek so that slips can find it -echo "[+] Executing 'ln -s /opt/zeek/bin/zeek /usr/local/bin/bro'\n" -sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro -echo "[+] Executing 'export PATH=$PATH:/usr/local/zeek/bin'\n" -export PATH=$PATH:/usr/local/zeek/bin -echo "[+] Adding /usr/local/zeek/bin to ~/.bashrc\n" -echo "export PATH=$PATH:/usr/local/zeek/bin" >> ~/.bashrc +sudo apt-get update echo "[+] Installing Slips dependencies ...\n" sudo apt install -y --no-install-recommends \ python3 \ - redis-server \ + redis \ zeek \ python3-pip \ python3-certifi \ @@ -47,17 +44,23 @@ sudo apt install -y --no-install-recommends \ yara \ libnotify-bin +echo "[+] Installing zeek ..." +# create a symlink to zeek so that slips can find it +sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro +export PATH=$PATH:/usr/local/zeek/bin +echo "export PATH=$PATH:/usr/local/zeek/bin" >> ~/.bashrc + -echo "[+] Executing 'python3 -m pip install --upgrade pip'\n" +echo "[+] Executing 'python3 -m pip install --upgrade pip'" python3 -m pip install --upgrade pip -echo "[+] Executing 'pip3 install -r install/requirements.txt'\n" +echo "[+] Executing 'pip3 install -r install/requirements.txt'" pip3 install -r install/requirements.txt -echo "[+] Executing pip3 install --ignore-installed six\n" +echo "[+] Executing pip3 install --ignore-installed six" pip3 install --ignore-installed six # For Kalipso echo "[+] Downloading nodejs v19 and npm dependencies" -curl -fsSL https://deb.nodesource.com/setup_19.x | bash - && apt install -y --no-install-recommends nodejs +curl -fsSL https://deb.nodesource.com/setup_21.x | sudo -E bash - && sudo apt install -y --no-install-recommends nodejs cd ./modules/kalipso && npm install cd ../.. diff --git a/managers/metadata_manager.py b/managers/metadata_manager.py index 291fda15f..4ea005bb0 100644 --- a/managers/metadata_manager.py +++ b/managers/metadata_manager.py @@ -30,7 +30,8 @@ def get_host_ip(self): def get_pid_using_port(self, port): """ - Returns the PID of the process using the given port or False if no process is using it + Returns the PID of the process using the given port or + False if no process is using it """ port = int(port) for conn in psutil.net_connections(): @@ -148,7 +149,7 @@ def set_input_metadata(self): self.main.db.set_input_metadata(info) - def update_slips_running_stats(self) -> Tuple[int, Set[str]] : + def update_slips_stats_in_the_db(self) -> Tuple[int, Set[str]] : """ updates the number of processed ips, slips internal time, and modified tws so far in the db @@ -159,7 +160,7 @@ def update_slips_running_stats(self) -> Tuple[int, Set[str]] : # this is the modification time of the last timewindow last_modified_tw_time: float modified_profiles, last_modified_tw_time = ( - self.main.db.getModifiedProfilesSince(slips_internal_time) + self.main.db.get_modified_profiles_since(slips_internal_time) ) modified_ips_in_the_last_tw = len(modified_profiles) self.main.db.set_input_metadata( diff --git a/managers/process_manager.py b/managers/process_manager.py index b36b0b6f0..95d7065a1 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -14,18 +14,19 @@ Process, Semaphore, Pipe, - ) +) from typing import ( List, Tuple, - ) +) from exclusiveprocess import ( Lock, CannotAcquireLock, - ) +) import modules +from modules.progress_bar.progress_bar import PBar from modules.update_manager.update_manager import UpdateManager from slips_files.common.imports import * from slips_files.common.style import green @@ -38,7 +39,6 @@ class ProcessManager: def __init__(self, main): self.main = main - self.module_objects = {} # this is the queue that will be used by the input proces # to pass flows to the profiler self.profiler_queue = Queue() @@ -58,6 +58,7 @@ def __init__(self, main): # and inout stops and renders the profiler queue useless and profiler # cant get more lines anymore! self.is_profiler_done_event = Event() + self.read_config() # for the communication between output.py and the progress bar # Pipe(False) means the pipe is unidirectional. # aka only msgs can go from output -> pbar and not vice versa @@ -65,7 +66,12 @@ def __init__(self, main): # send_pipe use donly for sending self.pbar_recv_pipe, self.output_send_pipe = Pipe(False) self.pbar_finished: Event = Event() - + + def read_config(self): + self.modules_to_ignore: list = self.main.conf.get_disabled_modules( + self.main.input_type + ) + def is_pbar_supported(self) -> bool: """ When running on a pcap, interface, or taking flows from an @@ -74,27 +80,26 @@ def is_pbar_supported(self) -> bool: """ # input type can be false whne using -S or in unit tests if ( - not self.main.input_type - or self.main.input_type in ('interface', 'pcap', 'stdin') - or self.main.mode == 'daemonized' + not self.main.input_type + or self.main.input_type in ("interface", "pcap", "stdin") + or self.main.mode == "daemonized" ): return False - - if self.main.stdout != '': + if self.main.stdout != "": # this means that stdout was redirected to a file, # no need to print the progress bar return False - + if ( - self.main.args.growing - or self.main.args.input_module - or self.main.args.testing + self.main.args.growing + or self.main.args.input_module + or self.main.args.testing ): return False - + return True - + def start_output_process(self, current_stdout, stderr, slips_logfile): output_process = Output( stdout=current_stdout, @@ -106,13 +111,13 @@ def start_output_process(self, current_stdout, stderr, slips_logfile): sender_pipe=self.output_send_pipe, has_pbar=self.is_pbar_supported(), pbar_finished=self.pbar_finished, - stop_daemon=self.main.args.stopdaemon + stop_daemon=self.main.args.stopdaemon, ) self.slips_logfile = output_process.slips_logfile return output_process - - def start_progress_bar(self, cls): - pbar = cls( + + def start_progress_bar(self): + pbar = PBar( self.main.logger, self.main.args.output, self.main.redis_port, @@ -122,8 +127,12 @@ def start_progress_bar(self, cls): slips_mode=self.main.mode, pbar_finished=self.pbar_finished, ) + pbar.start() + self.main.db.store_pid(pbar.name, int(pbar.pid)) + self.main.print(f"Started {green('PBar')} process [" + f"PID {green(pbar.pid)}]") return pbar - + def start_profiler_process(self): profiler_process = Profiler( self.main.logger, @@ -132,15 +141,17 @@ def start_profiler_process(self): self.termination_event, is_profiler_done=self.is_profiler_done, profiler_queue=self.profiler_queue, - is_profiler_done_event= self.is_profiler_done_event, + is_profiler_done_event=self.is_profiler_done_event, has_pbar=self.is_pbar_supported(), ) profiler_process.start() self.main.print( f'Started {green("Profiler Process")} ' - f"[PID {green(profiler_process.pid)}]", 1, 0, + f"[PID {green(profiler_process.pid)}]", + 1, + 0, ) - self.main.db.store_process_PID("Profiler", int(profiler_process.pid)) + self.main.db.store_pid("Profiler", int(profiler_process.pid)) return profiler_process def start_evidence_process(self): @@ -157,7 +168,7 @@ def start_evidence_process(self): 1, 0, ) - self.main.db.store_process_PID("Evidence", int(evidence_process.pid)) + self.main.db.store_pid("Evidence", int(evidence_process.pid)) return evidence_process def start_input_process(self): @@ -178,15 +189,13 @@ def start_input_process(self): ) input_process.start() self.main.print( - f'Started {green("Input Process")} ' - f'[PID {green(input_process.pid)}]', + f'Started {green("Input Process")} ' f"[PID {green(input_process.pid)}]", 1, 0, ) - self.main.db.store_process_PID("Input", int(input_process.pid)) + self.main.db.store_pid("Input", int(input_process.pid)) return input_process - def kill_process_tree(self, pid: int): try: # Send SIGKILL signal to the process @@ -196,9 +205,7 @@ def kill_process_tree(self, pid: int): # Get the child processes of the current process try: - process_list = (os.popen(f'pgrep -P {pid}') - .read() - .splitlines()) + process_list = os.popen(f"pgrep -P {pid}").read().splitlines() except: process_list = [] @@ -222,28 +229,24 @@ def kill_all_children(self): self.kill_process_tree(process.pid) self.print_stopped_module(module_name) - def is_ignored_module( - self, module_name: str, to_ignore: list - )-> bool: + def is_ignored_module(self, module_name: str) -> bool: - for ignored_module in to_ignore: - ignored_module = (ignored_module - .replace(' ','') - .replace('_','') - .replace('-','') - .lower()) + for ignored_module in self.modules_to_ignore: + ignored_module = ( + ignored_module.replace(" ", "") + .replace("_", "") + .replace("-", "") + .lower() + ) # this version of the module name wont contain # _ or spaces so we can # easily match it with the ignored module name - curr_module_name = (module_name - .replace('_','') - .replace('-','') - .lower()) + curr_module_name = module_name.replace("_", "").replace("-", "").lower() if curr_module_name.__contains__(ignored_module): return True return False - def get_modules(self, to_ignore: list): + def get_modules(self): """ Get modules from the 'modules' folder. """ @@ -252,7 +255,6 @@ def get_modules(self, to_ignore: list): plugins = {} failed_to_load_modules = 0 - # __path__ is the current path of this python program look_for_modules_in = modules.__path__ prefix = f"{modules.__name__}." @@ -272,11 +274,9 @@ def get_modules(self, to_ignore: list): if dir_name != file_name: continue - - if self.is_ignored_module(module_name, to_ignore): + if self.is_ignored_module(module_name): continue - # Try to import the module, otherwise skip. try: # "level specifies whether to use absolute or relative imports. @@ -289,9 +289,11 @@ def get_modules(self, to_ignore: list): # module calling __import__()." module = importlib.import_module(module_name) except ImportError as e: - print(f"Something wrong happened while " - f"importing the module {module_name}: {e}") - print(traceback.print_stack()) + print( + f"Something wrong happened while " + f"importing the module {module_name}: {e}" + ) + print(traceback.format_exc()) failed_to_load_modules += 1 continue @@ -299,12 +301,8 @@ def get_modules(self, to_ignore: list): # Walk through all members of currently imported modules. for member_name, member_object in inspect.getmembers(module): # Check if current member is a class. - if ( - inspect.isclass(member_object) - and ( - issubclass(member_object, IModule) - and member_object is not IModule - ) + if inspect.isclass(member_object) and ( + issubclass(member_object, IModule) and member_object is not IModule ): plugins[member_object.name] = dict( obj=member_object, @@ -329,43 +327,44 @@ def get_modules(self, to_ignore: list): return plugins, failed_to_load_modules - def load_modules(self): - to_ignore: list = self.main.conf.get_disabled_modules( - self.main.input_type) + def print_disabled_modules(self): + print("-" * 27) + self.main.print(f"Disabled Modules: {self.modules_to_ignore}", 1, 0) - # Import all the modules - modules_to_call = self.get_modules(to_ignore)[0] - loaded_modules = [] + def load_modules(self): + """responsible for starting all the modules in the modules/ dir""" + modules_to_call = self.get_modules()[0] for module_name in modules_to_call: - if module_name in to_ignore: - continue - module_class = modules_to_call[module_name]["obj"] if module_name == "Progress Bar": - module = self.start_progress_bar(module_class) - else: - module = module_class( - self.main.logger, - self.main.args.output, - self.main.redis_port, - self.termination_event, - ) + # started it manually in main.py + # otherwise we miss some of the print right when slips + # starts, because when the pbar is supported, it handles + # all the printing + continue + + module = module_class( + self.main.logger, + self.main.args.output, + self.main.redis_port, + self.termination_event, + ) module.start() - self.main.db.store_process_PID(module_name, int(module.pid)) - self.module_objects[module_name] = module # maps name -> object - description = modules_to_call[module_name]["description"] - self.main.print( - f"\t\tStarting the module {green(module_name)} " - f"({description}) " - f"[PID {green(module.pid)}]", - 1, 0, + self.main.db.store_pid(module_name, int(module.pid)) + self.print_started_module( + module_name, module.pid, modules_to_call[module_name]["description"] ) - loaded_modules.append(module_name) - # give outputprocess time to print all the started modules - time.sleep(0.5) - print("-" * 27) - self.main.print(f"Disabled Modules: {to_ignore}", 1, 0) - return loaded_modules + + def print_started_module( + self, module_name: str, module_pid: int, module_description: str + ) -> None: + self.main.print( + f"\t\tStarting the module {green(module_name)} " + f"({module_description}) " + f"[PID {green(module_pid)}]", + 1, + 0, + ) def print_stopped_module(self, module): self.stopped_modules.append(module) @@ -374,9 +373,9 @@ def print_stopped_module(self, module): # to vertically align them when printing module += " " * (20 - len(module)) - self.main.print(f"\t{green(module)} \tStopped. " - f"" f"{green(modules_left)} left.") - + self.main.print( + f"\t{green(module)} \tStopped. " f"" f"{green(modules_left)} left." + ) def start_update_manager(self, local_files=False, TI_feeds=False): """ @@ -399,7 +398,7 @@ def start_update_manager(self, local_files=False, TI_feeds=False): self.main.logger, self.main.args.output, self.main.redis_port, - multiprocessing.Event() + multiprocessing.Event(), ) if local_files: @@ -441,7 +440,6 @@ def warn_about_pending_modules(self, pending_modules: List[Process]): self.warning_printed_once = True return True - def get_hitlist_in_order(self) -> Tuple[List[Process], List[Process]]: """ returns a list of PIDs that slips should terminate first, @@ -516,26 +514,42 @@ def get_analysis_time(self): end_date = self.main.metadata_man.set_analysis_end_date() start_time = self.main.db.get_slips_start_time() - return utils.get_time_diff( - start_time, end_date, return_type="minutes" - ) + return utils.get_time_diff(start_time, end_date, return_type="minutes") + + def stop_slips(self) -> bool: + """ + determines whether slips should stop + based on the following: + 1. is slips still receiving new flows? + 2. did slips the control channel recv the stop_slips + 3. is a debugger present? + """ + if self.should_run_non_stop(): + return False + + if ( + self.stop_slips_received() + or self.slips_is_done_receiving_new_flows() + ): + return True + + return False - def should_stop(self): + def stop_slips_received(self): """ - returns true if the channel received the stop msg + returns true if the channel received the 'stop_slips' msg """ message = self.main.c1.get_message(timeout=0.01) if ( - message - and utils.is_msg_intended_for(message, 'control_channel') - and message['data'] == 'stop_slips' + message + and utils.is_msg_intended_for(message, "control_channel") + and message["data"] == "stop_slips" ): return True - def is_debugger_active(self) -> bool: """Returns true if the debugger is currently active""" - gettrace = getattr(sys, 'gettrace', lambda: None) + gettrace = getattr(sys, "gettrace", lambda: None) return gettrace() is not None def should_run_non_stop(self) -> bool: @@ -547,9 +561,9 @@ def should_run_non_stop(self) -> bool: # when slips is reading from a special module other than the input process # this module should handle the stopping of slips if ( - self.is_debugger_active() - or self.main.input_type in ('stdin', 'cyst') - or self.main.is_interface + self.is_debugger_active() + or self.main.input_type in ("stdin", "cyst") + or self.main.is_interface ): return True return False @@ -590,7 +604,6 @@ def shutdown_interactive(self, to_kill_first, to_kill_last): # all of them are killed return None, None - def slips_is_done_receiving_new_flows(self) -> bool: """ this method will return True when the input and profiler release @@ -598,12 +611,8 @@ def slips_is_done_receiving_new_flows(self) -> bool: If they're still processing it will return False """ # try to acquire the semaphore without blocking - input_done_processing: bool = self.is_input_done.acquire( - block=False - ) - profiler_done_processing: bool = self.is_profiler_done.acquire( - block=False - ) + input_done_processing: bool = self.is_input_done.acquire(block=False) + profiler_done_processing: bool = self.is_profiler_done.acquire(block=False) if input_done_processing and profiler_done_processing: return True @@ -611,7 +620,6 @@ def slips_is_done_receiving_new_flows(self) -> bool: # can't acquire the semaphore, processes are still running return False - def shutdown_daemon(self): """ Shutdown slips modules in daemon mode @@ -636,7 +644,6 @@ def shutdown_gracefully(self): print("\n" + "-" * 27) self.main.print("Stopping Slips") - # by default, 15 mins from this time, all modules should be killed method_start_time = time.time() @@ -645,13 +652,15 @@ def shutdown_gracefully(self): timeout_seconds: float = timeout * 60 # close all tws - self.main.db.check_TW_to_close(close_all=True) + self.main.db.check_tw_to_close(close_all=True) analysis_time = self.get_analysis_time() - self.main.print(f"Analysis of {self.main.input_information} " - f"finished in {analysis_time:.2f} minutes") + self.main.print( + f"Analysis of {self.main.input_information} " + f"finished in {analysis_time:.2f} minutes" + ) graceful_shutdown = True - if self.main.mode == 'daemonized': + if self.main.mode == "daemonized": self.processes: dict = self.main.db.get_pids() self.shutdown_daemon() @@ -664,11 +673,13 @@ def shutdown_gracefully(self): else: flows_count: int = self.main.db.get_flows_count() - self.main.print(f"Total flows read (without altflows): " - f"{flows_count}", log_to_logfiles_only=True) + self.main.print( + f"Total flows read (without altflows): " f"{flows_count}", + log_to_logfiles_only=True, + ) hitlist: Tuple[List[Process], List[Process]] - hitlist = self.get_hitlist_in_order() + hitlist = self.get_hitlist_in_order() to_kill_first: List[Process] = hitlist[0] to_kill_last: List[Process] = hitlist[1] self.termination_event.set() @@ -677,13 +688,11 @@ def shutdown_gracefully(self): # modules self.warning_printed_once = False - try: # Wait timeout_seconds for all the processes to finish while time.time() - method_start_time < timeout_seconds: to_kill_first, to_kill_last = self.shutdown_interactive( - to_kill_first, - to_kill_last + to_kill_first, to_kill_last ) if not to_kill_first and not to_kill_last: # all modules are done @@ -704,8 +713,10 @@ def shutdown_gracefully(self): # getting here means we're killing them bc of the timeout # not getting here means we're killing them bc of double # ctr+c OR they terminated successfully - reason = (f"Killing modules that took more than {timeout}" - f" mins to finish.") + reason = ( + f"Killing modules that took more than {timeout}" + f" mins to finish." + ) self.main.print(reason) graceful_shutdown = False @@ -732,12 +743,16 @@ def shutdown_gracefully(self): self.main.db.close() if graceful_shutdown: - self.main.print("[Process Manager] Slips shutdown gracefully\n", - log_to_logfiles_only=True) + self.main.print( + "[Process Manager] Slips shutdown gracefully\n", + log_to_logfiles_only=True, + ) else: - self.main.print(f"[Process Manager] Slips didn't " - f"shutdown gracefully - {reason}\n", - log_to_logfiles_only=True) + self.main.print( + f"[Process Manager] Slips didn't " + f"shutdown gracefully - {reason}\n", + log_to_logfiles_only=True, + ) except KeyboardInterrupt: return False diff --git a/managers/redis_manager.py b/managers/redis_manager.py index 1a105af9a..19fbcc090 100644 --- a/managers/redis_manager.py +++ b/managers/redis_manager.py @@ -217,17 +217,12 @@ def check_if_port_is_in_use(self, port: int) -> bool: # even if it's already in use, slips should override it return False - # is it used by another app? - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if sock.connect_ex(("localhost", port)) != 0: - # not used - sock.close() - return False - - sock.close() - self.print_port_in_use(port) - self.main.terminate_slips() - return True + if utils.is_port_in_use(port): + self.print_port_in_use(port) + self.main.terminate_slips() + return True + + return False def get_pid_of_redis_server(self, port: int) -> int: diff --git a/managers/ui_manager.py b/managers/ui_manager.py index 8e5da4985..9f5896bf3 100644 --- a/managers/ui_manager.py +++ b/managers/ui_manager.py @@ -1,3 +1,4 @@ +from slips_files.common.slips_utils import utils from slips_files.common.style import green import subprocess @@ -34,25 +35,33 @@ def start_webinterface(self): """ def detach_child(): """ - Detach the web interface from the parent process group(slips.py), the child(web interface) - will no longer receive signals and should be manually killed in shutdown_gracefully() + Detach the web interface from the parent process group(slips.py), + the child(web interface) + will no longer receive signals and should be manually killed in + shutdown_gracefully() """ os.setpgrp() def run_webinterface(): - # starting the wbeinterface using the shell script results in slips not being able to + # starting the wbeinterface using the shell script results + # in slips not being able to # get the PID of the python proc started by the .sh script - command = ['python3', 'webinterface/app.py'] + # so we'll start it with python instead + command = ['python3', '-m', 'webinterface.app'] + webinterface = subprocess.Popen( command, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, stdin=subprocess.DEVNULL, - preexec_fn=detach_child + preexec_fn=detach_child, + cwd=os.getcwd() + ) - # self.webinterface_pid = webinterface.pid - self.main.db.store_process_PID('Web Interface', webinterface.pid) - # we'll assume that it started, and if not, the return value will immediately change and this thread will + + self.main.db.store_pid('Web Interface', webinterface.pid) + # we'll assume that it started, and if not, the return value will + # immediately change and this thread will # print an error self.webinterface_return_value.put(True) @@ -60,21 +69,23 @@ def run_webinterface(): # we will never get the return value of this thread error = webinterface.communicate()[1] if error: - # pop the True we just added + # pop the return value we just added self.webinterface_return_value.get() # set false as the return value of this thread self.webinterface_return_value.put(False) - self.main.print (f"Web interface error:\n") + self.main.print(f"Web interface error:") for line in error.strip().decode().splitlines(): - self.main.print (f"{line}") - - pid = self.main.metadata_man.get_pid_using_port(55000) - self.main.print(f"Port 55000 is used by PID {pid}") - - # if there's an error, this will be set to false, and the error will be printed - # otherwise we assume that the interface started - # self.webinterface_started = True + self.main.print(f"{line}") + + if utils.is_port_in_use(55000): + pid = self.main.metadata_man.get_pid_using_port(55000) + self.main.print(f"Failed to start web interface. Port 55000 is " + f"used by PID {pid}") + return + + # if there's an error, this webinterface_return_value will be set + # to false, and the error will be printed self.webinterface_return_value = Queue() self.webinterface_thread = threading.Thread( target=run_webinterface, diff --git a/modules/arp/arp.py b/modules/arp/arp.py index b620439b9..468f1aa6f 100644 --- a/modules/arp/arp.py +++ b/modules/arp/arp.py @@ -1,4 +1,3 @@ -from slips_files.common.abstracts._module import IModule import json import ipaddress import time @@ -379,7 +378,7 @@ def detect_unsolicited_arp( # We're sure this is unsolicited arp # it may be arp spoofing confidence: float = 0.8 - threat_level: ThreatLevel = ThreatLevel.INFO + threat_level: ThreatLevel = ThreatLevel.LOW description: str = 'broadcasting unsolicited ARP' saddr: str = profileid.split('_')[-1] diff --git a/modules/cyst/cyst.py b/modules/cyst/cyst.py index 17901a16a..2583f35d2 100644 --- a/modules/cyst/cyst.py +++ b/modules/cyst/cyst.py @@ -1,5 +1,4 @@ from slips_files.common.abstracts._module import IModule -import multiprocessing import socket import json import os @@ -149,7 +148,7 @@ def shutdown_gracefully(self): self.close_connection() # if slips is done, slips shouldn't expect more flows or send evidence # it should terminate - self.db.publish('control_channel', 'stop_slips') + self.db.publish_stop() return def pre_main(self): diff --git a/modules/flowalerts/flowalerts.py b/modules/flowalerts/flowalerts.py index 2582a1ca1..d9b4cd940 100644 --- a/modules/flowalerts/flowalerts.py +++ b/modules/flowalerts/flowalerts.py @@ -1,11 +1,6 @@ import contextlib - -from slips_files.common.abstracts._module import IModule from slips_files.common.imports import * -from .timer_thread import TimerThread -from .set_evidence import SetEvidnceHelper -from slips_files.core.helpers.whitelist import Whitelist -import multiprocessing + import json import threading import ipaddress @@ -15,7 +10,15 @@ import collections import math import time + +from slips_files.common.imports import * +from .timer_thread import TimerThread +from .set_evidence import SetEvidnceHelper +from slips_files.core.helpers.whitelist import Whitelist from slips_files.common.slips_utils import utils +from typing import List, \ + Tuple, \ + Dict class FlowAlerts(IModule): @@ -68,7 +71,8 @@ def init(self): # after this number of failed ssh logins, we alert pw guessing self.pw_guessing_threshold = 20 self.password_guessing_cache = {} - # in pastebin download detection, we wait for each conn.log flow of the seen ssl flow to appear + # in pastebin download detection, we wait for each conn.log flow + # of the seen ssl flow to appear # this is the dict of ssl flows we're waiting for self.pending_ssl_flows = multiprocessing.Queue() # thread that waits for ssl flows to appear in conn.log @@ -109,6 +113,7 @@ def read_configuration(self): self.pastebin_downloads_threshold = conf.get_pastebin_download_threshold() self.our_ips = utils.get_own_IPs() self.shannon_entropy_threshold = conf.get_entropy_threshold() + self.client_ips: List[str] = conf.client_ips() def check_connection_to_local_ip( self, @@ -364,8 +369,6 @@ def check_pastebin_download( """ Alerts on downloads from pastebin.com with more than 12000 bytes This function waits for the ssl.log flow to appear in conn.log before alerting - :param wait_time: the time we wait for the ssl conn to appear in conn.log in seconds - every time the timer is over, we wait extra 2 min and call the function again : param flow: this is the conn.log of the ssl flow we're currently checking """ @@ -384,54 +387,66 @@ def check_pastebin_download( # maybe an empty file is downloaded return False + def get_sent_bytes(self, all_flows: Dict[str, dict]) \ + -> Dict[str, Tuple[int, List[str], str]] : + """ + Returns a dict of sent bytes to all ips in the all_flows dict + { + contacted_ip: ( + sum_of_mbs_sent, + [uids], + last_ts_of_flow_containging_this_contacted_ip + ) + } + """ + bytes_sent = {} + for uid, flow in all_flows.items(): + daddr = flow['daddr'] + sbytes: int = flow.get('sbytes', 0) + ts: str = flow.get('starttime', '') + + if self.is_ignored_ip_data_upload(daddr) or not sbytes: + continue + if daddr in bytes_sent: + mbs_sent, uids, _= bytes_sent[daddr] + mbs_sent += sbytes + uids.append(uid) + bytes_sent[daddr] = (mbs_sent, uids, ts) + else: + bytes_sent[daddr] = (sbytes, [uid], ts) + + return bytes_sent + def detect_data_upload_in_twid(self, profileid, twid): """ For each contacted ip in this twid, check if the total bytes sent to this ip is >= data_exfiltration_threshold """ - def get_sent_bytes(all_flows: dict): - """Returns a dict of sent bytes to all ips {contacted_ip: (mbs_sent, [uids])}""" - bytes_sent = {} - for uid, flow in all_flows.items(): - daddr = flow['daddr'] - sbytes: int = flow.get('sbytes', 0) - - if self.is_ignored_ip_data_upload(daddr) or not sbytes: - continue - - if daddr in bytes_sent: - mbs_sent, uids = bytes_sent[daddr] - mbs_sent += sbytes - uids.append(uid) - bytes_sent[daddr] = (mbs_sent, uids) - else: - bytes_sent[daddr] = (sbytes, [uid]) - - return bytes_sent - - all_flows = self.db.get_all_flows_in_profileid( + all_flows: Dict[str, dict] = self.db.get_all_flows_in_profileid( profileid ) if not all_flows: return - bytes_sent: dict = get_sent_bytes(all_flows) - for ip, ip_info in bytes_sent.items(): - # ip_info is a tuple (bytes_sent, [uids]) - uids = ip_info[1] + bytes_sent: Dict[str, Tuple[int, List[str], str]] + bytes_sent = self.get_sent_bytes(all_flows) - bytes_uploaded = ip_info[0] + for ip, ip_info in bytes_sent.items(): + ip_info: Tuple[int, List[str], str] + bytes_uploaded, uids, ts = ip_info + mbs_uploaded = utils.convert_to_mb(bytes_uploaded) if mbs_uploaded < self.data_exfiltration_threshold: continue - + self.set_evidence.data_exfiltration( ip, mbs_uploaded, profileid, twid, uids, + ts ) @@ -599,7 +614,26 @@ def is_well_known_org(self, ip): # (fb, twitter, microsoft, etc.) if self.whitelist.is_ip_in_org(ip, org): return True - + + def should_ignore_conn_without_dns(self, flow_type, appproto, daddr) \ + -> bool: + """ + checks for the cases that we should ignore the connection without dns + """ + # we should ignore this evidence if the ip is ours, whether it's a + # private ip or in the list of client_ips + return ( + flow_type != 'conn' + or appproto == 'dns' + or utils.is_ignored_ip(daddr) + # if the daddr is a client ip, it means that this is a conn + # from the internet to our ip, the dns res was probably + # made on their side before connecting to us, + # so we shouldn't be doing this detection on this ip + or daddr in self.client_ips + # because there's no dns.log to know if the dns was made + or self.db.get_input_type() == 'zeek_log_file' + ) def check_connection_without_dns_resolution( self, flow_type, appproto, daddr, twid, profileid, timestamp, uid @@ -611,18 +645,9 @@ def check_connection_without_dns_resolution( # 1- Do not check for DNS requests # 2- Ignore some IPs like private IPs, multicast, and broadcast - if ( - flow_type != 'conn' - or appproto == 'dns' - or utils.is_ignored_ip(daddr) - ): + if self.should_ignore_conn_without_dns(flow_type, appproto, daddr): return - # disable this alert when running on a zeek conn.log file - # because there's no dns.log to know if the dns was made - if self.db.get_input_type() == 'zeek_log_file': - return False - # Ignore some IP ## - All dhcp servers. Since is ok to connect to # them without a DNS request. @@ -652,6 +677,7 @@ def check_connection_without_dns_resolution( # search 24hs back for a dns resolution if self.db.is_ip_resolved(daddr, 24): return False + # self.print(f'No DNS resolution in {answers_dict}') # There is no DNS resolution, but it can be that Slips is # still reading it from the files. @@ -714,15 +740,11 @@ def is_CNAME_contacted(self, answers, contacted_ips) -> bool: if ip in contacted_ips: return True return False - - def check_dns_without_connection( - self, domain, answers: list, rcode_name: str, - timestamp: str, profileid, twid, uid - ): - """ - Makes sure all cached DNS answers are used in contacted_ips - :param contacted_ips: dict of ips used in a specific tw {ip: uid} + + def should_detect_dns_without_conn(self, domain: str, rcode_name: str) \ + -> bool: """ + returns False in the following cases ## - All reverse dns resolutions ## - All .local domains ## - The wildcard domain * @@ -734,7 +756,7 @@ def check_dns_without_connection( ## - The WPAD domain of windows # - When there is an NXDOMAIN as answer, it means # the domain isn't resolved, so we should not expect any connection later - + """ if ( 'arpa' in domain or '.local' in domain @@ -746,6 +768,19 @@ def check_dns_without_connection( ): return False + return True + + + def check_dns_without_connection( + self, domain, answers: list, rcode_name: str, + timestamp: str, profileid, twid, uid + ): + """ + Makes sure all cached DNS answers are used in contacted_ips + """ + if not self.should_detect_dns_without_conn(domain, rcode_name): + return False + # One DNS query may not be answered exactly by UID, # but the computer can re-ask the domain, # and the next DNS resolution can be @@ -762,7 +797,7 @@ def check_dns_without_connection( # with AAAA, and the computer chooses the A address. # Therefore, the 2nd DNS resolution # would be treated as 'without connection', but this is false. - if prev_domain_resolutions := self.db.getDomainData(domain): + if prev_domain_resolutions := self.db.get_domain_data(domain): prev_domain_resolutions = prev_domain_resolutions.get('IPs',[]) # if there's a domain in the cache # (prev_domain_resolutions) that is not in the @@ -776,6 +811,7 @@ def check_dns_without_connection( # the computer to connect to anything # self.print(f'No ips in the answer, so ignoring') return False + # self.print(f'The extended DNS query to {domain} had as answers {answers} ') contacted_ips = self.db.get_all_contacted_ips_in_profileid_twid( @@ -789,7 +825,7 @@ def check_dns_without_connection( # every dns answer is a list of ips that correspond to 1 query, # one of these ips should be present in the contacted ips # check each one of the resolutions of this domain - for ip in answers: + for ip in self.extract_ips_from_dns_answers(answers): # self.print(f'Checking if we have a connection to ip {ip}') if ( ip in contacted_ips @@ -826,7 +862,7 @@ def check_dns_without_connection( else: # It means we already checked this dns with the Timer process # but still no connection for it. - self.set_evidence.DNS_without_conn( + self.set_evidence.dns_without_conn( domain, timestamp, profileid, twid, uid ) # This UID will never appear again, so we can remove it and @@ -846,7 +882,7 @@ def detect_successful_ssh_by_zeek(self, uid, timestamp, profileid, twid): ) daddr = ssh_flow_dict['daddr'] saddr = ssh_flow_dict['saddr'] - size = ssh_flow_dict['allbytes'] + size = ssh_flow_dict['sbytes'] + ssh_flow_dict['dbytes'] self.set_evidence.ssh_successful( twid, saddr, @@ -941,8 +977,6 @@ def check_successful_ssh( else: self.detect_successful_ssh_by_slips(uid, timestamp, profileid, twid, auth_success) - - def detect_incompatible_CN( self, daddr, @@ -959,6 +993,7 @@ def detect_incompatible_CN( """ if not issuer: return False + found_org_in_cn = '' for org in utils.supported_orgs: if org not in issuer.lower(): @@ -1096,9 +1131,9 @@ def check_invalid_dns_answers( for answer in answers: if answer in invalid_answers and domain != "localhost": - #blocked answer found + # blocked answer found self.set_evidence.invalid_dns_answer( - domain, answer, daddr, profileid, twid, stime, uid + domain, answer, profileid, twid, stime, uid ) # delete answer from redis cache to prevent # associating this dns answer with this domain/query and @@ -1251,22 +1286,37 @@ def check_multiple_reconnection_attempts( self.db.setReconnections( profileid, twid, current_reconnections ) - - def detect_young_domains(self, domain, stime, profileid, twid, uid): + + def should_detect_young_domain(self, domain): + """ + returns true if it's ok to detect young domains for the given + domain + """ + return ( + domain + and not domain.endswith(".local") + and not domain.endswith('.arpa') + ) + + def detect_young_domains( + self, + domain, + answers: List[str], + stime, + profileid, + twid, + uid + ): """ Detect domains that are too young. The threshold is 60 days """ - if not domain: + if not self.should_detect_young_domain(domain): return False age_threshold = 60 - # Ignore arpa and local domains - if domain.endswith('.arpa') or domain.endswith('.local'): - return False - - domain_info: dict = self.db.getDomainData(domain) + domain_info: dict = self.db.get_domain_data(domain) if not domain_info: return False @@ -1278,12 +1328,26 @@ def detect_young_domains(self, domain, stime, profileid, twid, uid): age = domain_info['Age'] if age >= age_threshold: return False - + + + ips_returned_in_answer: List[str] = ( + self.extract_ips_from_dns_answers(answers) + ) self.set_evidence.young_domain( - domain, age, stime, profileid, twid, uid + domain, age, stime, profileid, twid, uid, ips_returned_in_answer ) return True - + + def extract_ips_from_dns_answers(self, answers: List[str]) -> List[str]: + """ + extracts ipv4 and 6 from DNS answers + """ + ips = [] + for answer in answers: + if validators.ipv4(answer) or validators.ipv6(answer): + ips.append(answer) + return ips + def check_smtp_bruteforce( self, profileid, @@ -1465,7 +1529,6 @@ def detect_malicious_ja3( daddr, ja3, ja3s, - profileid, twid, uid, timestamp @@ -1483,21 +1546,20 @@ def detect_malicious_ja3( twid, uid, timestamp, - daddr, saddr, - type_='ja3', + daddr, ja3=ja3, - ) + ) + if ja3s in malicious_ja3_dict: - self.set_evidence.malicious_ja3( + self.set_evidence.malicious_ja3s( malicious_ja3_dict, twid, uid, timestamp, saddr, daddr, - type_='ja3s', ja3=ja3s, ) @@ -1585,25 +1647,6 @@ def check_malicious_ssl(self, ssl_info): ssl_info, ssl_info_from_db ) - def check_weird_http_method(self, msg): - """ - detect weird http methods in zeek's weird.log - """ - flow = msg['flow'] - profileid = msg['profileid'] - twid = msg['twid'] - - # what's the weird.log about - name = flow['name'] - - if 'unknown_HTTP_method' not in name: - return False - - self.set_evidence.weird_http_method( - profileid, - twid, - flow - ) def check_non_http_port_80_conns( self, @@ -1638,8 +1681,8 @@ def check_non_http_port_80_conns( def check_GRE_tunnel(self, tunnel_info: dict): """ Detects GRE tunnels - @param tunnel_flow: dict containing tunnel zeek flow - @return: None + :param tunnel_info: dict containing tunnel zeek flow + :return: None """ tunnel_flow = tunnel_info['flow'] tunnel_type = tunnel_flow['tunnel_type'] @@ -2065,7 +2108,6 @@ def main(self): daddr, ja3, ja3s, - profileid, twid, uid, timestamp @@ -2083,7 +2125,7 @@ def main(self): if msg := self.get_msg('tw_closed'): profileid_tw = msg['data'].split('_') - profileid = f'{profileid_tw[0]}_{profileid_tw[1]}', + profileid = f'{profileid_tw[0]}_{profileid_tw[1]}' twid = profileid_tw[-1] self.detect_data_upload_in_twid(profileid, twid) @@ -2129,7 +2171,7 @@ def main(self): # TODO: not sure how to make sure IP_info is # done adding domain age to the db or not self.detect_young_domains( - domain, stime, profileid, twid, uid + domain, answers, stime, profileid, twid, uid ) self.check_dns_arpa_scan( domain, stime, profileid, twid, uid @@ -2167,10 +2209,7 @@ def main(self): role='SSH::SERVER' ) - if msg := self.get_msg('new_weird'): - msg = json.loads(msg['data']) - self.check_weird_http_method(msg) - + if msg := self.get_msg('new_tunnel'): msg = json.loads(msg['data']) self.check_GRE_tunnel(msg) diff --git a/modules/flowalerts/set_evidence.py b/modules/flowalerts/set_evidence.py index 8d69a2ed4..3454c3d31 100644 --- a/modules/flowalerts/set_evidence.py +++ b/modules/flowalerts/set_evidence.py @@ -1,8 +1,8 @@ -import datetime import json import sys import time -from typing import List +from typing import List, \ + Dict from slips_files.common.slips_utils import utils from slips_files.core.evidence_structure.evidence import \ @@ -32,31 +32,47 @@ def young_domain( domain: str, age: int, stime: str, - profileid: ProfileID, + profileid: str, twid: str, - uid: str + uid: str, + answers: List[str] ): saddr: str = profileid.split("_")[-1] - victim = Victim( - direction=Direction.SRC, - victim_type=IoCType.IP, - value=saddr, - ) - attacker = Attacker( - direction=Direction.DST, - attacker_type=IoCType.DOMAIN, - value=domain, - ) twid_number: int = int(twid.replace("timewindow", "")) - description = f'connection to a young domain: {domain} ' \ - f'registered {age} days ago.', + description: str = (f'connection to a young domain: {domain} ' + f'registered {age} days ago.') + # set evidence for all the young domain dns answers + for attacker in answers: + attacker: str + evidence = Evidence( + evidence_type=EvidenceType.YOUNG_DOMAIN, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=attacker, + ), + threat_level=ThreatLevel.LOW, + category=IDEACategory.ANOMALY_TRAFFIC, + description=description, + profile=ProfileID(ip=attacker), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=stime, + conn_count=1, + confidence=1.0 + ) + self.db.set_evidence(evidence) + evidence = Evidence( evidence_type=EvidenceType.YOUNG_DOMAIN, - attacker=attacker, - threat_level=ThreatLevel.LOW, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr, + ), + threat_level=ThreatLevel.INFO, category=IDEACategory.ANOMALY_TRAFFIC, description=description, - victim=victim, profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=twid_number), uid=[uid], @@ -65,6 +81,7 @@ def young_domain( confidence=1.0 ) self.db.set_evidence(evidence) + def multiple_ssh_versions( self, @@ -82,22 +99,21 @@ def multiple_ssh_versions( :param role: can be 'SSH::CLIENT' or 'SSH::SERVER' as seen in zeek software.log flows """ - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) role = 'client' if 'CLIENT' in role.upper() else 'server' description = f'SSH {role} version changing from ' \ f'{cached_versions} to {current_versions}' evidence = Evidence( evidence_type=EvidenceType.MULTIPLE_SSH_VERSIONS, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), threat_level=ThreatLevel.MEDIUM, category=IDEACategory.ANOMALY_TRAFFIC, description=description, - profile=ProfileID(ip=attacker.value), + profile=ProfileID(ip=srcip), timewindow=TimeWindow(int(twid.replace("timewindow", ''))), uid=uid, timestamp=timestamp, @@ -111,7 +127,7 @@ def different_localnet_usage( self, daddr: str, portproto: str, - profileid: ProfileID, + profileid: str, timestamp: str, twid: str, uid: str, @@ -122,7 +138,8 @@ def different_localnet_usage( 'srcip' outside the localnet or the 'dstip'? """ srcip = profileid.split('_')[-1] - # the attacker here is the IP found to be private and outside the localnet + # the attacker here is the IP found to be + # private and outside the localnet if ip_outside_localnet == 'srcip': attacker = Attacker( direction=Direction.SRC, @@ -134,6 +151,7 @@ def different_localnet_usage( victim_type=IoCType.IP, value=daddr ) + threat_level = ThreatLevel.LOW description = f'A connection from a private IP ({srcip}) ' \ f'outside of the used local network ' \ f'{self.db.get_local_network()}. To IP: {daddr} ' @@ -148,6 +166,7 @@ def different_localnet_usage( victim_type=IoCType.IP, value=srcip ) + threat_level = ThreatLevel.HIGH description = f'A connection to a private IP ({daddr}) ' \ f'outside of the used local network ' \ f'{self.db.get_local_network()}. ' \ @@ -157,8 +176,8 @@ def different_localnet_usage( confidence = 1.0 - threat_level = ThreatLevel.HIGH - + + twid_number = int(twid.replace("timewindow", "")) evidence = Evidence( evidence_type=EvidenceType.DIFFERENT_LOCALNET, attacker=attacker, @@ -167,7 +186,7 @@ def different_localnet_usage( description=description, victim=victim, profile=ProfileID(ip=srcip), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_number), uid=[uid], timestamp=timestamp, conn_count=1, @@ -187,22 +206,18 @@ def device_changing_ips( confidence = 0.8 threat_level = ThreatLevel.MEDIUM saddr: str = profileid.split("_")[-1] - - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - description = f'A device changing IPs. IP {saddr} was found ' \ f'with MAC address {smac} but the MAC belongs ' \ f'originally to IP: {old_ip}. ' - twid_number = int(twid.replace("timewindow", "")) evidence = Evidence( evidence_type=EvidenceType.DEVICE_CHANGING_IP, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), threat_level=threat_level, category=IDEACategory.ANOMALY_TRAFFIC, description=description, @@ -226,17 +241,8 @@ def non_http_port_80_conn( uid: str ) -> None: confidence = 0.8 - threat_level = ThreatLevel.MEDIUM saddr: str = profileid.split("_")[-1] - - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) - description: str = f'non-HTTP established connection to port 80. ' \ f'destination IP: {daddr} {ip_identification}' @@ -244,8 +250,12 @@ def non_http_port_80_conn( evidence: Evidence = Evidence( evidence_type=EvidenceType.NON_HTTP_PORT_80_CONNECTION, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, category=IDEACategory.ANOMALY_TRAFFIC, description=description, profile=ProfileID(ip=saddr), @@ -255,6 +265,25 @@ def non_http_port_80_conn( conn_count=1, confidence=confidence ) + self.db.set_evidence(evidence) + + evidence: Evidence = Evidence( + evidence_type=EvidenceType.NON_HTTP_PORT_80_CONNECTION, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.MEDIUM, + category=IDEACategory.ANOMALY_TRAFFIC, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=timestamp, + conn_count=1, + confidence=confidence + ) self.db.set_evidence(evidence) @@ -267,20 +296,7 @@ def non_ssl_port_443_conn( uid: str ) -> None: confidence: float = 0.8 - threat_level: ThreatLevel = ThreatLevel.MEDIUM saddr: str = profileid.split("_")[-1] - - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = f'non-SSL established connection to port 443. ' \ f'destination IP: {daddr} {ip_identification}' @@ -289,9 +305,17 @@ def non_ssl_port_443_conn( evidence: Evidence = Evidence( evidence_type=EvidenceType.NON_SSL_PORT_443_CONNECTION, - attacker=attacker, - victim=victim, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.MEDIUM, category=IDEACategory.ANOMALY_TRAFFIC, description=description, profile=ProfileID(ip=saddr), @@ -304,55 +328,7 @@ def non_ssl_port_443_conn( self.db.set_evidence(evidence) - def weird_http_method( - self, - profileid: str, - twid: str, - flow: dict - ) -> None: - daddr: str = flow['daddr'] - weird_method: str = flow['addl'] - uid: str = flow['uid'] - timestamp: str = flow['starttime'] - - confidence = 0.9 - threat_level: ThreatLevel = ThreatLevel.MEDIUM - saddr: str = profileid.split("_")[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - - ip_identification: str = self.db.get_ip_identification(daddr) - description: str = f'Weird HTTP method "{weird_method}" to IP: ' \ - f'{daddr} {ip_identification}. by Zeek.' - - twid_number: int = int(twid.replace("timewindow", "")) - - evidence: Evidence = Evidence( - evidence_type=EvidenceType.WEIRD_HTTP_METHOD, - attacker=attacker, - victim=victim, - threat_level=threat_level, - category=IDEACategory.ANOMALY_TRAFFIC, - description=description, - profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=twid_number), - uid=[uid], - timestamp=timestamp, - conn_count=1, - confidence=confidence - ) - self.db.set_evidence(evidence) def incompatible_CN( self, @@ -364,21 +340,7 @@ def incompatible_CN( uid: str ) -> None: confidence: float = 0.9 - threat_level: ThreatLevel = ThreatLevel.MEDIUM saddr: str = profileid.split("_")[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = f'Incompatible certificate CN to IP: {daddr} ' \ f'{ip_identification} claiming to ' \ @@ -387,9 +349,17 @@ def incompatible_CN( twid_number: int = int(twid.replace("timewindow", "")) evidence: Evidence = Evidence( evidence_type=EvidenceType.INCOMPATIBLE_CN, - attacker=attacker, - victim=victim, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.MEDIUM, category=IDEACategory.ANOMALY_TRAFFIC, description=description, profile=ProfileID(ip=saddr), @@ -415,21 +385,18 @@ def DGA( # +1 ensures that the minimum confidence score is 1. confidence: float = max(0, (1 / 100) * (nxdomains - 100) + 1) confidence = round(confidence, 2) # for readability - threat_level = ThreatLevel.HIGH saddr = profileid.split("_")[-1] description = f'Possible DGA or domain scanning. {saddr} ' \ f'failed to resolve {nxdomains} domains' - attacker = Attacker( + evidence: Evidence = Evidence( + evidence_type=EvidenceType.DGA_NXDOMAINS, + attacker= Attacker( direction=Direction.SRC, attacker_type=IoCType.IP, value=saddr - ) - - evidence: Evidence = Evidence( - evidence_type=EvidenceType.DGA_NXDOMAINS, - attacker=attacker, - threat_level=threat_level, + ), + threat_level=ThreatLevel.HIGH, category=IDEACategory.ANOMALY_BEHAVIOUR, description=description, profile=ProfileID(ip=saddr), @@ -443,7 +410,7 @@ def DGA( self.db.set_evidence(evidence) - def DNS_without_conn( + def dns_without_conn( self, domain: str, timestamp: str, @@ -452,23 +419,18 @@ def DNS_without_conn( uid: str ) -> None: confidence: float = 0.8 - threat_level: ThreatLevel = ThreatLevel.LOW saddr: str = profileid.split("_")[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - description: str = f'domain {domain} resolved with no connection' - twid_number: int = int(twid.replace("timewindow", "")) evidence: Evidence = Evidence( evidence_type=EvidenceType.DNS_WITHOUT_CONNECTION, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, category=IDEACategory.ANOMALY_TRAFFIC, description=description, profile=ProfileID(ip=saddr), @@ -490,15 +452,8 @@ def pastebin_download( uid: str ) -> bool: - threat_level: ThreatLevel = ThreatLevel.INFO confidence: float = 1.0 saddr: str = profileid.split("_")[-1] - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - response_body_len: float = utils.convert_to_mb(bytes_downloaded) description: str = f'A downloaded file from pastebin.com. ' \ f'size: {response_body_len} MBs' @@ -506,8 +461,12 @@ def pastebin_download( twid_number: int = int(twid.replace("timewindow", "")) evidence: Evidence = Evidence( evidence_type=EvidenceType.PASTEBIN_DOWNLOAD, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.INFO, category=IDEACategory.ANOMALY_BEHAVIOUR, description=description, profile=ProfileID(ip=saddr), @@ -531,7 +490,7 @@ def conn_without_dns( uid: str ) -> None: confidence: float = 0.8 - threat_level: ThreatLevel = ThreatLevel.HIGH + threat_level: ThreatLevel = ThreatLevel.INFO saddr: str = profileid.split("_")[-1] attacker: Attacker = Attacker( direction=Direction.SRC, @@ -587,18 +546,16 @@ def dns_arpa_scan( description = f"Doing DNS ARPA scan. Scanned {arpa_scan_threshold}" \ f" hosts within 2 seconds." - # Store attacker details in a local variable - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) # Create Evidence object using local variables evidence = Evidence( evidence_type=EvidenceType.DNS_ARPA_SCAN, description=description, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), threat_level=threat_level, category=IDEACategory.RECON_SCANNING, profile=ProfileID(ip=saddr), @@ -629,18 +586,6 @@ def unknown_port( twid_number: int = int(twid.replace("timewindow", "")) saddr = profileid.split('_')[-1] - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = ( f'Connection to unknown destination port {dport}/{proto.upper()} ' @@ -649,8 +594,16 @@ def unknown_port( evidence: Evidence = Evidence( evidence_type=EvidenceType.UNKNOWN_PORT, - attacker=attacker, - victim=victim, + attacker= Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=ThreatLevel.HIGH, category=IDEACategory.ANOMALY_CONNECTION, description=description, @@ -677,24 +630,20 @@ def pw_guessing( # confidence = 1 because this detection is comming # from a zeek file so we're sure it's accurate confidence: float = 1.0 - threat_level: ThreatLevel = ThreatLevel.HIGH twid_number: int = int(twid.replace("timewindow", "")) scanning_ip: str = msg.split(' appears')[0] description: str = f'password guessing. {msg}. by {by}.' - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=scanning_ip - ) - conn_count: int = int(msg.split('in ')[1].split('connections')[0]) evidence: Evidence = Evidence( evidence_type=EvidenceType.PASSWORD_GUESSING, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=scanning_ip + ), + threat_level=ThreatLevel.HIGH, category= IDEACategory.ATTEMPT_LOGIN, description=description, profile=ProfileID(ip=scanning_ip), @@ -718,7 +667,6 @@ def horizontal_portscan( uid: str ) -> None: confidence: float = 1.0 - threat_level: ThreatLevel = ThreatLevel.HIGH twid_number: int = int(twid.replace("timewindow", "")) saddr = profileid.split('_')[-1] @@ -726,16 +674,14 @@ def horizontal_portscan( # get the number of unique hosts scanned on a specific port conn_count: int = int(msg.split('least')[1].split('unique')[0]) - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - evidence: Evidence = Evidence( evidence_type=EvidenceType.HORIZONTAL_PORT_SCAN, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.HIGH, category=IDEACategory.RECON_SCANNING, description=description, profile=ProfileID(ip=saddr), @@ -754,14 +700,13 @@ def conn_to_private_ip( self, proto: str, daddr: str, - dport: str, + dport: int, saddr: str, twid: str, uid: str, timestamp: str ) -> None: confidence: float = 1.0 - threat_level: ThreatLevel = ThreatLevel.INFO twid_number: int = int(twid.replace("timewindow", "")) description: str = f'Connecting to private IP: {daddr} ' @@ -772,21 +717,14 @@ def conn_to_private_ip( else: description += f'on destination port: {dport}' - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - evidence: Evidence = Evidence( evidence_type=EvidenceType.CONNECTION_TO_PRIVATE_IP, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.INFO, category=IDEACategory.RECON, description=description, profile=ProfileID(ip=saddr), @@ -795,7 +733,11 @@ def conn_to_private_ip( timestamp=timestamp, conn_count=1, confidence=confidence, - victim=victim + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ) ) self.db.set_evidence(evidence) @@ -803,7 +745,7 @@ def conn_to_private_ip( def GRE_tunnel( self, - tunnel_info: dict + tunnel_info: Dict[str, str] ) -> None: profileid: str = tunnel_info['profileid'] twid: str = tunnel_info['twid'] @@ -825,22 +767,18 @@ def GRE_tunnel( f'to {daddr} {ip_identification} ' \ f'tunnel action: {action}' - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - evidence: Evidence = Evidence( evidence_type=EvidenceType.GRE_TUNNEL, - attacker=attacker, - victim=victim, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, category=IDEACategory.INFO, description=description, @@ -866,29 +804,24 @@ def vertical_portscan( # confidence = 1 because this detection is coming # from a Zeek file so we're sure it's accurate confidence: float = 1.0 - threat_level: ThreatLevel = ThreatLevel.HIGH twid: int = int(twid.replace("timewindow", "")) # msg example: 192.168.1.200 has scanned 60 ports of 192.168.1.102 description: str = f'vertical port scan by Zeek engine. {msg}' conn_count: int = int(msg.split('least ')[1].split(' unique')[0]) - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=scanning_ip - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=msg.split('ports of host ')[-1].split(" in")[0] - ) - + evidence: Evidence = Evidence( evidence_type=EvidenceType.VERTICAL_PORT_SCAN, - attacker=attacker, - victim=victim, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=scanning_ip + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=msg.split('ports of host ')[-1].split(" in")[0] + ), + threat_level=ThreatLevel.HIGH, category=IDEACategory.RECON_SCANNING, description=description, profile=ProfileID(ip=scanning_ip), @@ -908,7 +841,7 @@ def ssh_successful( twid: str, saddr: str, daddr: str, - size, + size: int, uid: str, timestamp: str, by='', @@ -918,34 +851,32 @@ def ssh_successful( This is not strictly a detection, but we don't have a better way to show it. The threat_level is 0.01 to show that this is not a detection + :param size: src and dst bytes sent and recieved """ confidence: float = 0.8 threat_level: ThreatLevel = ThreatLevel.INFO twid: int = int(twid.replace("timewindow", "")) - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = ( f'SSH successful to IP {daddr}. {ip_identification}. ' - f'From IP {saddr}. Size: {str(size)}. Detection model {by}.' + f'From IP {saddr}. Sent bytes: {str(size)}. Detection model {by}.' f' Confidence {confidence}' ) evidence: Evidence = Evidence( evidence_type=EvidenceType.SSH_SUCCESSFUL, - attacker=attacker, - victim=victim, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, confidence=confidence, description=description, @@ -970,7 +901,6 @@ def long_connection( """ Set an evidence for a long connection. """ - threat_level: ThreatLevel = ThreatLevel.LOW twid: int = int(twid.replace("timewindow", "")) # Confidence depends on how long the connection. # Scale the confidence from 0 to 1; 1 means 24 hours long. @@ -980,18 +910,6 @@ def long_connection( duration_minutes: int = int(duration / 60) srcip: str = profileid.split('_')[1] - attacker_obj: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) - - victim_obj: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = ( f'Long Connection. Connection from {srcip} ' @@ -1001,8 +919,12 @@ def long_connection( evidence: Evidence = Evidence( evidence_type=EvidenceType.LONG_CONNECTION, - attacker=attacker_obj, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), + threat_level=ThreatLevel.LOW, confidence=confidence, description=description, profile=ProfileID(ip=srcip), @@ -1010,7 +932,11 @@ def long_connection( uid=[uid], timestamp=timestamp, category=IDEACategory.ANOMALY_CONNECTION, - victim=victim_obj + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ) ) self.db.set_evidence(evidence) @@ -1028,7 +954,6 @@ def self_signed_certificates( Set evidence for self-signed certificates. """ confidence: float = 0.5 - threat_level: ThreatLevel = ThreatLevel.LOW saddr: str = profileid.split("_")[-1] twid: int = int(twid.replace("timewindow", "")) @@ -1048,7 +973,7 @@ def self_signed_certificates( evidence: Evidence = Evidence( evidence_type=EvidenceType.SELF_SIGNED_CERTIFICATE, attacker=attacker, - threat_level=threat_level, + threat_level=ThreatLevel.LOW, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -1057,7 +982,25 @@ def self_signed_certificates( timestamp=timestamp, category=IDEACategory.ANOMALY_BEHAVIOUR ) + self.db.set_evidence(evidence) + attacker: Attacker = Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ) + evidence: Evidence = Evidence( + evidence_type=EvidenceType.SELF_SIGNED_CERTIFICATE, + attacker=attacker, + threat_level=ThreatLevel.LOW, + confidence=confidence, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid), + uid=[uid], + timestamp=timestamp, + category=IDEACategory.ANOMALY_BEHAVIOUR + ) self.db.set_evidence(evidence) def multiple_reconnection_attempts( @@ -1077,18 +1020,6 @@ def multiple_reconnection_attempts( saddr: str = profileid.split("_")[-1] twid: int = int(twid.replace("timewindow", "")) - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - ip_identification = self.db.get_ip_identification(daddr) description = ( f'Multiple reconnection attempts to Destination IP:' @@ -1097,8 +1028,16 @@ def multiple_reconnection_attempts( ) evidence: Evidence = Evidence( evidence_type=EvidenceType.MULTIPLE_RECONNECTION_ATTEMPTS, - attacker=attacker, - victim = victim, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim = Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, confidence=confidence, description=description, @@ -1125,7 +1064,6 @@ def connection_to_multiple_ports( Set evidence for connection to multiple ports. """ confidence: float = 0.5 - threat_level: ThreatLevel = ThreatLevel.INFO twid: int = int(twid.replace("timewindow", "")) ip_identification = self.db.get_ip_identification(attacker) description = f'Connection to multiple ports {dstports} of ' \ @@ -1140,22 +1078,19 @@ def connection_to_multiple_ports( victim_direction = Direction.SRC profile_ip = victim - victim: Victim = Victim( + evidence = Evidence( + evidence_type=EvidenceType.CONNECTION_TO_MULTIPLE_PORTS, + attacker=Attacker( + direction=attacker_direction, + attacker_type=IoCType.IP, + value=attacker + ), + victim=Victim( direction=victim_direction, victim_type=IoCType.IP, value=victim - ) - attacker: Attacker = Attacker( - direction=attacker_direction, - attacker_type=IoCType.IP, - value=attacker - ) - - evidence = Evidence( - evidence_type=EvidenceType.CONNECTION_TO_MULTIPLE_PORTS, - attacker=attacker, - victim=victim, - threat_level=threat_level, + ), + threat_level=ThreatLevel.INFO, confidence=confidence, description=description, profile=ProfileID(ip=profile_ip), @@ -1179,30 +1114,21 @@ def suspicious_dns_answer( uid: str ) -> None: confidence: float = 0.6 - threat_level: ThreatLevel = ThreatLevel.MEDIUM twid: int = int(twid.replace("timewindow", "")) saddr: str = profileid.split("_")[-1] - attacker: Attacker = Attacker( - direction=Direction.DST, - attacker_type=IoCType.IP, - value=daddr - ) - victim: Victim = Victim( - direction=Direction.SRC, - victim_type=IoCType.IP, - value=saddr - ) - description: str = f'A DNS TXT answer with high entropy. ' \ f'query: {query} answer: "{answer}" ' \ f'entropy: {round(entropy, 2)} ' evidence: Evidence = Evidence( evidence_type=EvidenceType.HIGH_ENTROPY_DNS_ANSWER, - attacker=attacker, - victim=victim, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.MEDIUM, confidence=confidence, description=description, profile=ProfileID(ip=daddr), @@ -1213,40 +1139,49 @@ def suspicious_dns_answer( ) self.db.set_evidence(evidence) + + + evidence: Evidence = Evidence( + evidence_type=EvidenceType.HIGH_ENTROPY_DNS_ANSWER, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, + confidence=confidence, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid), + uid=[uid], + timestamp=stime, + category=IDEACategory.ANOMALY_TRAFFIC + ) + self.db.set_evidence(evidence) def invalid_dns_answer( self, query: str, answer: str, - daddr: str, profileid: str, twid: str, stime: str, - uid: str + uid: str, ) -> None: - threat_level: ThreatLevel = ThreatLevel.INFO confidence: float = 0.7 twid: int = int(twid.replace("timewindow", "")) saddr: str = profileid.split("_")[-1] - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) - description: str = f"The DNS query {query} was resolved to {answer}" evidence: Evidence = Evidence( evidence_type=EvidenceType.INVALID_DNS_RESOLUTION, - attacker=attacker, - victim=victim, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.INFO, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -1257,7 +1192,7 @@ def invalid_dns_answer( ) self.db.set_evidence(evidence) - + def for_port_0_connection( self, @@ -1284,26 +1219,22 @@ def for_port_0_connection( victim_direction = Direction.SRC profile_ip = victim - victim: Victim = Victim( - direction=victim_direction, - victim_type=IoCType.IP, - value=victim - ) - attacker: Attacker = Attacker( - direction=attacker_direction, - attacker_type=IoCType.IP, - value=attacker - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = f'Connection on port 0 from {saddr}:{sport} ' \ f'to {daddr}:{dport}. {ip_identification}.' - evidence: Evidence = Evidence( evidence_type=EvidenceType.PORT_0_CONNECTION, - attacker=attacker, - victim=victim, + attacker=Attacker( + direction=attacker_direction, + attacker_type=IoCType.IP, + value=attacker + ), + victim=Victim( + direction=victim_direction, + victim_type=IoCType.IP, + value=victim + ), threat_level=threat_level, confidence=confidence, description=description, @@ -1317,7 +1248,75 @@ def for_port_0_connection( ) self.db.set_evidence(evidence) + + + def malicious_ja3s( + self, + malicious_ja3_dict: dict, + twid: str, + uid: str, + timestamp: str, + saddr: str, + daddr: str, + ja3: str = '', + ) -> None: + ja3_info: dict = json.loads(malicious_ja3_dict[ja3]) + + threat_level: str = ja3_info['threat_level'].upper() + threat_level: ThreatLevel = ThreatLevel[threat_level] + + tags: str = ja3_info.get('tags', '') + ja3_description: str = ja3_info['description'] + + ip_identification: str = self.db.get_ip_identification(daddr) + description = ( + f'Malicious JA3s: (possible C&C server): {ja3} to server ' + f'{daddr} {ip_identification} ' + ) + if ja3_description != 'None': + description += f'description: {ja3_description} ' + description += f'tags: {tags}' + confidence: float = 1 + twid_number: int = int(twid.replace("timewindow", "")) + evidence: Evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_JA3S, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=timestamp, + category=IDEACategory.INTRUSION_BOTNET, + source_target_tag=Tag.CC + ) + + self.db.set_evidence(evidence) + evidence: Evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_JA3S, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, + confidence=confidence, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=timestamp, + category=IDEACategory.INTRUSION_BOTNET, + source_target_tag=Tag.CC + ) + self.db.set_evidence(evidence) + def malicious_ja3( self, @@ -1325,13 +1324,10 @@ def malicious_ja3( twid: str, uid: str, timestamp: str, - victim: str, - attacker: str, - type_: str = '', + daddr: str, + saddr: str, ja3: str = '', ) -> None: - """ - """ ja3_info: dict = json.loads(malicious_ja3_dict[ja3]) threat_level: str = ja3_info['threat_level'].upper() @@ -1340,58 +1336,34 @@ def malicious_ja3( tags: str = ja3_info.get('tags', '') ja3_description: str = ja3_info['description'] - if type_ == 'ja3': - description = f'Malicious JA3: {ja3} from source address ' \ - f'{attacker} ' - evidence_type: EvidenceType = EvidenceType.MALICIOUS_JA3 - source_target_tag: Tag = Tag.BOTNET - attacker_direction: Direction = Direction.SRC - victim_direction: Direction = Direction.DST - - elif type_ == 'ja3s': - description = ( - f'Malicious JA3s: (possible C&C server): {ja3} to server ' - f'{attacker} ' - ) - - evidence_type: EvidenceType = EvidenceType.MALICIOUS_JA3S - source_target_tag: Tag = Tag.CC - attacker_direction: Direction = Direction.DST - victim_direction: Direction = Direction.SRC - else: - return - - # append daddr identification to the description - ip_identification: str = self.db.get_ip_identification(attacker) - description += f'{ip_identification} ' + ip_identification: str = self.db.get_ip_identification(saddr) + description = f'Malicious JA3: {ja3} from source address ' \ + f'{saddr} {ip_identification}' if ja3_description != 'None': - description += f'description: {ja3_description} ' - description += f'tags: {tags}' + description += f' description: {ja3_description} ' + description += f' tags: {tags}' - attacker: Attacker = Attacker( - direction=attacker_direction, - attacker_type=IoCType.IP, - value=attacker - ) - victim: Victim = Victim( - direction=victim_direction, - victim_type=IoCType.IP, - value=victim - ) - confidence: float = 1 evidence: Evidence = Evidence( - evidence_type=evidence_type, - attacker=attacker, - victim=victim, + evidence_type=EvidenceType.MALICIOUS_JA3, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction= Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, - confidence=confidence, + confidence=1, description=description, - profile=ProfileID(ip=attacker.value), + profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), uid=[uid], timestamp=timestamp, category=IDEACategory.INTRUSION_BOTNET, - source_target_tag=source_target_tag + source_target_tag=Tag.BOTNET ) self.db.set_evidence(evidence) @@ -1405,27 +1377,45 @@ def data_exfiltration( uid: List[str], timestamp ) -> None: - confidence: float = 0.6 - threat_level: ThreatLevel = ThreatLevel.HIGH saddr: str = profileid.split("_")[-1] - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) ip_identification: str = self.db.get_ip_identification(daddr) description: str = f'Large data upload. {src_mbs} MBs ' \ f'sent to {daddr} {ip_identification}' timestamp: str = utils.convert_format(timestamp, utils.alerts_format) - + twid_number = int(twid.replace("timewindow", "")) + evidence: Evidence = Evidence( evidence_type=EvidenceType.DATA_UPLOAD, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.INFO, + confidence=0.6, description=description, profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_number), + uid=uid, + timestamp=timestamp, + category=IDEACategory.MALWARE, + source_target_tag=Tag.ORIGIN_MALWARE + ) + + self.db.set_evidence(evidence) + + evidence: Evidence = Evidence( + evidence_type=EvidenceType.DATA_UPLOAD, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.HIGH, + confidence=0.6, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_number), uid=uid, timestamp=timestamp, category=IDEACategory.MALWARE, @@ -1445,24 +1435,22 @@ def bad_smtp_login( confidence: float = 1.0 threat_level: ThreatLevel = ThreatLevel.HIGH - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) ip_identification: str = self.db.get_ip_identification(daddr) description: str = f'doing bad SMTP login to {daddr} ' \ f'{ip_identification}' evidence: Evidence = Evidence( evidence_type=EvidenceType.BAD_SMTP_LOGIN, - attacker=attacker, - victim=victim, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + victim=Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, confidence=confidence, description=description, @@ -1527,11 +1515,12 @@ def smtp_bruteforce( def malicious_ssl( self, ssl_info: dict, - ssl_info_from_db: dict + ssl_info_from_db: str ) -> None: flow: dict = ssl_info['flow'] ts: str = flow.get('starttime', '') daddr: str = flow.get('daddr', '') + saddr: str = flow.get('saddr', '') uid: str = flow.get('uid', '') twid: str = ssl_info.get('twid', '') @@ -1550,17 +1539,34 @@ def malicious_ssl( f'{ip_identification} description: ' \ f'{cert_description} {tags} ' - - attacker: Attacker = Attacker( - direction=Direction.DST, - attacker_type=IoCType.IP, - value=daddr + evidence: Evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_SSL_CERT, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + uid=[uid], + timestamp=ts, + category=IDEACategory.INTRUSION_BOTNET, + source_target_tag=Tag.CC ) + self.db.set_evidence(evidence) + evidence: Evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_SSL_CERT, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, confidence=confidence, description=description, profile=ProfileID(ip=daddr), diff --git a/modules/flowmldetection/flowmldetection.py b/modules/flowmldetection/flowmldetection.py index f1d2c1af4..e0195dc09 100644 --- a/modules/flowmldetection/flowmldetection.py +++ b/modules/flowmldetection/flowmldetection.py @@ -95,7 +95,7 @@ def train(self): ) except Exception: self.print('Error while calling clf.train()') - self.print(traceback.print_stack()) + self.print(traceback.format_exc(), 0, 1) # See score so far in training score = self.clf.score(X_flow, y_flow) @@ -115,7 +115,7 @@ def train(self): except Exception: self.print('Error in train()', 0 , 1) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) def process_features(self, dataset): @@ -216,7 +216,7 @@ def process_features(self, dataset): except Exception: # Stop the timer self.print('Error in process_features()') - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) def process_flows(self): """ @@ -295,7 +295,7 @@ def process_flows(self): except Exception: # Stop the timer self.print('Error in process_flows()') - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) def process_flow(self): """ @@ -312,7 +312,7 @@ def process_flow(self): except Exception: # Stop the timer self.print('Error in process_flow()') - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) def detect(self): """ @@ -333,7 +333,7 @@ def detect(self): # Stop the timer self.print('Error in detect() X_flow:') self.print(X_flow) - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) def store_model(self): """ @@ -384,14 +384,6 @@ def set_evidence_malicious_flow( uid: str ): confidence: float = 0.1 - threat_level: ThreatLevel = ThreatLevel.LOW - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - ip_identification = self.db.get_ip_identification(daddr) description = f'Malicious flow by ML. Src IP {saddr}:{sport} to ' \ f'{daddr}:{dport} {ip_identification}' @@ -403,8 +395,12 @@ def set_evidence_malicious_flow( evidence: Evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_FLOW, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, confidence=confidence, description=description, profile=ProfileID(ip=saddr), diff --git a/modules/http_analyzer/http_analyzer.py b/modules/http_analyzer/http_analyzer.py index f502387ed..677a669ba 100644 --- a/modules/http_analyzer/http_analyzer.py +++ b/modules/http_analyzer/http_analyzer.py @@ -1,9 +1,10 @@ -from slips_files.common.abstracts._module import IModule import json import urllib import requests -from typing import Union - +from typing import ( + Union, + Dict + ) from slips_files.common.imports import * from slips_files.core.evidence_structure.evidence import \ ( @@ -29,8 +30,10 @@ class HTTPAnalyzer(IModule): def init(self): self.c1 = self.db.subscribe('new_http') + self.c2 = self.db.subscribe('new_weird') self.channels = { - 'new_http': self.c1 + 'new_http': self.c1, + 'new_weird': self.c2 } self.connections_counter = {} self.empty_connections_threshold = 4 @@ -89,21 +92,19 @@ def check_suspicious_user_agents( ) for suspicious_ua in suspicious_user_agents: if suspicious_ua.lower() in user_agent.lower(): - threat_level: ThreatLevel = ThreatLevel.HIGH confidence: float = 1 saddr = profileid.split('_')[1] description: str = (f'Suspicious user-agent: ' f'{user_agent} while ' f'connecting to {host}{uri}') - attacker = Attacker( + evidence: Evidence = Evidence( + evidence_type=EvidenceType.SUSPICIOUS_USER_AGENT, + attacker=Attacker( direction=Direction.SRC, attacker_type=IoCType.IP, value=saddr - ) - evidence: Evidence = Evidence( - evidence_type=EvidenceType.SUSPICIOUS_USER_AGENT, - attacker=attacker, - threat_level=threat_level, + ), + threat_level=ThreatLevel.HIGH, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -162,21 +163,18 @@ def check_multiple_empty_connections( uids, connections = self.connections_counter[host] if connections == self.empty_connections_threshold: - threat_level: ThreatLevel = ThreatLevel.MEDIUM confidence: float = 1 saddr: str = profileid.split('_')[-1] description: str = f'Multiple empty HTTP connections to {host}' - attacker = Attacker( + evidence: Evidence = Evidence( + evidence_type=EvidenceType.EMPTY_CONNECTIONS, + attacker=Attacker( direction=Direction.SRC, attacker_type=IoCType.IP, value=saddr - ) - - evidence: Evidence = Evidence( - evidence_type=EvidenceType.EMPTY_CONNECTIONS, - attacker=attacker, - threat_level=threat_level, + ), + threat_level=ThreatLevel.MEDIUM, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -203,9 +201,7 @@ def set_evidence_incompatible_user_agent( twid, uid: str ): - threat_level: ThreatLevel = ThreatLevel.HIGH saddr = profileid.split('_')[1] - confidence: float = 1 os_type: str = user_agent.get('os_type', '').lower() os_name: str = user_agent.get('os_name', '').lower() @@ -219,17 +215,15 @@ def set_evidence_incompatible_user_agent( f'IP has MAC vendor: {vendor.capitalize()}' ) - attacker: Attacker = Attacker( - direction= Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - evidence: Evidence = Evidence( evidence_type=EvidenceType.INCOMPATIBLE_USER_AGENT, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction= Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.HIGH, + confidence=1, description=description, profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), @@ -250,30 +244,46 @@ def set_evidence_executable_mime_type( timestamp: str, daddr: str ): - confidence: float = 1 - threat_level: ThreatLevel = ThreatLevel.LOW saddr: str = profileid.split('_')[1] - attacker_obj: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - ip_identification: str = self.db.get_ip_identification(daddr) description: str = ( f'Download of an executable with MIME type: {mime_type} ' f'by {saddr} from {daddr} {ip_identification}.' ) - + twid_number = int(twid.replace("timewindow", "")) evidence: Evidence = Evidence( evidence_type=EvidenceType.EXECUTABLE_MIME_TYPE, - attacker=attacker_obj, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, + confidence=1, description=description, profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=timestamp, + category=IDEACategory.ANOMALY_FILE, + source_target_tag=Tag.EXECUTABLE_MIME_TYPE + ) + + self.db.set_evidence(evidence) + + evidence: Evidence = Evidence( + evidence_type=EvidenceType.EXECUTABLE_MIME_TYPE, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=ThreatLevel.LOW, + confidence=1, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_number), uid=[uid], timestamp=timestamp, category=IDEACategory.ANOMALY_FILE, @@ -512,24 +522,20 @@ def check_multiple_UAs( # 'Linux' in both UAs, so we shouldn't alert return False - threat_level: ThreatLevel = ThreatLevel.INFO - confidence: float = 1 saddr: str = profileid.split('_')[1] - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - ua: str = cached_ua.get('user_agent', '') description: str = (f'Using multiple user-agents:' f' "{ua}" then "{user_agent}"') evidence: Evidence = Evidence( evidence_type=EvidenceType.MULTIPLE_USER_AGENT, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.INFO, + confidence=1, description=description, profile=ProfileID(ip=saddr), timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), @@ -553,26 +559,17 @@ def set_evidence_http_traffic( timestamp: str ): confidence: float = 1 - threat_level: ThreatLevel = ThreatLevel.LOW saddr = profileid.split('_')[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - - victim: Victim = Victim( - direction=Direction.DST, - victim_type=IoCType.IP, - value=daddr - ) description = f'Unencrypted HTTP traffic from {saddr} to {daddr}.' evidence: Evidence = Evidence( evidence_type=EvidenceType.HTTP_TRAFFIC, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, confidence=confidence, description=description, profile=ProfileID(ip=saddr), @@ -581,7 +578,11 @@ def set_evidence_http_traffic( timestamp=timestamp, category=IDEACategory.ANOMALY_TRAFFIC, source_target_tag=Tag.SENDING_UNENCRYPTED_DATA, - victim=victim + victim= Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ) ) self.db.set_evidence(evidence) @@ -637,8 +638,79 @@ def check_pastebin_downloads( self.db.set_evidence(evidence) return True + + def set_evidence_weird_http_method( + self, + profileid: str, + twid: str, + flow: dict + ) -> None: + daddr: str = flow['daddr'] + weird_method: str = flow['addl'] + uid: str = flow['uid'] + timestamp: str = flow['starttime'] + + confidence = 0.9 + threat_level: ThreatLevel = ThreatLevel.MEDIUM + saddr: str = profileid.split("_")[-1] + + attacker: Attacker = Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ) + + victim: Victim = Victim( + direction=Direction.DST, + victim_type=IoCType.IP, + value=daddr + ) + + ip_identification: str = self.db.get_ip_identification(daddr) + description: str = f'Weird HTTP method "{weird_method}" to IP: ' \ + f'{daddr} {ip_identification}. by Zeek.' + + twid_number: int = int(twid.replace("timewindow", "")) + + evidence: Evidence = Evidence( + evidence_type=EvidenceType.WEIRD_HTTP_METHOD, + attacker=attacker, + victim=victim, + threat_level=threat_level, + category=IDEACategory.ANOMALY_TRAFFIC, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=timestamp, + conn_count=1, + confidence=confidence + ) + self.db.set_evidence(evidence) + + + def check_weird_http_method(self, msg: Dict[str, str]): + """ + detect weird http methods in zeek's weird.log + """ + flow = msg['flow'] + profileid = msg['profileid'] + twid = msg['twid'] + + # what's the weird.log about + name = flow['name'] + + if 'unknown_HTTP_method' not in name: + return False + + self.set_evidence_weird_http_method( + profileid, + twid, + flow + ) + def pre_main(self): utils.drop_root_privs() @@ -736,3 +808,7 @@ def main(self): uid, timestamp ) + + if msg := self.get_msg('new_weird'): + msg = json.loads(msg['data']) + self.check_weird_http_method(msg) diff --git a/modules/ip_info/ip_info.py b/modules/ip_info/ip_info.py index adefe6390..411aba2b3 100644 --- a/modules/ip_info/ip_info.py +++ b/modules/ip_info/ip_info.py @@ -358,7 +358,7 @@ def get_age(self, domain): # tld not supported return False - cached_data = self.db.getDomainData(domain) + cached_data = self.db.get_domain_data(domain) if cached_data and 'Age' in cached_data: # we already have age info about this domain return False @@ -385,8 +385,7 @@ def get_age(self, domain): today, return_type='days' ) - - self.db.setInfoForDomains(domain, {'Age': age}) + self.db.set_info_for_domains(domain, { 'Age': age}) return age def shutdown_gracefully(self): @@ -511,17 +510,9 @@ def set_evidence_malicious_jarm_hash( dport: int = flow['dport'] dstip: str = flow['daddr'] saddr: str = flow['saddr'] - timestamp: float = flow['starttime'] + timestamp = flow['starttime'] protocol: str = flow['proto'] - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - threat_level = ThreatLevel.MEDIUM - confidence = 0.7 - portproto = f'{dport}/{protocol}' port_info = self.db.get_port_info(portproto) or "" port_info = f'({port_info.upper()})' if port_info else "" @@ -529,17 +520,22 @@ def set_evidence_malicious_jarm_hash( dstip_id = self.db.get_ip_identification(dstip) description = ( f"Malicious JARM hash detected for destination IP: {dstip}" - f" on port: {portproto} {port_info}. {dstip_id}" + f" on port: {portproto} {port_info}. {dstip_id}" ) - + twid_number = int(twid.replace("timewindow", "")) + evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_JARM, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=dstip + ), + threat_level=ThreatLevel.MEDIUM, + confidence=0.7, description=description, - profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + profile=ProfileID(ip=dstip), + timewindow=TimeWindow(number=twid_number), uid=[flow['uid']], timestamp=timestamp, category=IDEACategory.ANOMALY_TRAFFIC, @@ -549,6 +545,28 @@ def set_evidence_malicious_jarm_hash( ) self.db.set_evidence(evidence) + + evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_JARM, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.LOW, + confidence=0.7, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_number), + uid=[flow['uid']], + timestamp=timestamp, + category=IDEACategory.ANOMALY_TRAFFIC, + proto=Proto(protocol.lower()), + port=dport, + source_target_tag=Tag.MALWARE + ) + + self.db.set_evidence(evidence) def pre_main(self): diff --git a/modules/leak_detector/leak_detector.py b/modules/leak_detector/leak_detector.py index 3471dd6bd..50792e851 100644 --- a/modules/leak_detector/leak_detector.py +++ b/modules/leak_detector/leak_detector.py @@ -163,78 +163,100 @@ def get_packet_info(self, offset: int): def set_evidence_yara_match(self, info: dict): """ This function is called when yara finds a match - :param info: a dict with info about the matched rule, example keys 'vars_matched', 'index', + :param info: a dict with info about the matched rule, + example keys 'vars_matched', 'index', 'rule', 'srings_matched' """ rule = info.get('rule').replace('_', ' ') offset = info.get('offset') # vars_matched = info.get('vars_matched') strings_matched = info.get('strings_matched') - # we now know there's a match at offset x, we need to know offset x belongs to which packet - if packet_info := self.get_packet_info(offset): - srcip, dstip, proto, sport, dport, ts = ( - packet_info[0], - packet_info[1], - packet_info[2], - packet_info[3], - packet_info[4], - packet_info[5], - ) - - portproto = f'{dport}/{proto}' - port_info = self.db.get_port_info(portproto) - - # generate a random uid - uid = base64.b64encode(binascii.b2a_hex(os.urandom(9))).decode( - 'utf-8' - ) - profileid = f'profile_{srcip}' - # sometimes this module tries to find the profile before it's created. so - # wait a while before alerting. - time.sleep(4) - - ip_identification = self.db.get_ip_identification(dstip) - description = f"{rule} to destination address: {dstip} " \ - f"{ip_identification} port: {portproto} " \ - f"{port_info or ''}. " \ - f"Leaked location: {strings_matched}" - - # in which tw is this ts? - twid = self.db.get_tw_of_ts(profileid, ts) - # convert ts to a readable format - ts = utils.convert_format(ts, utils.alerts_format) - - if twid: - twid = twid[0] - source_target_tag = Tag.CC - confidence = 0.9 - threat_level = ThreatLevel.HIGH - - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) - + # we now know there's a match at offset x, we need + # to know offset x belongs to which packet + packet_info = self.get_packet_info(offset) + if not packet_info: + return + + srcip, dstip, proto, sport, dport, ts = ( + packet_info[0], + packet_info[1], + packet_info[2], + packet_info[3], + packet_info[4], + packet_info[5], + ) - evidence = Evidence( - evidence_type=EvidenceType.NETWORK_GPS_LOCATION_LEAKED, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, - description=description, - profile=ProfileID(ip=srcip), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), - uid=[uid], - timestamp=ts, - proto=Proto(proto.lower()), - port=dport, - source_target_tag=source_target_tag, - category=IDEACategory.MALWARE - ) + portproto = f'{dport}/{proto}' + port_info = self.db.get_port_info(portproto) - self.db.set_evidence(evidence) + # generate a random uid + uid = base64.b64encode(binascii.b2a_hex(os.urandom(9))).decode( + 'utf-8' + ) + profileid = f'profile_{srcip}' + # sometimes this module tries to find the profile before it's created. so + # wait a while before alerting. + time.sleep(4) + + ip_identification = self.db.get_ip_identification(dstip) + description = f"{rule} to destination address: {dstip} " \ + f"{ip_identification} port: {portproto} " \ + f"{port_info or ''}. " \ + f"Leaked location: {strings_matched}" + + # in which tw is this ts? + twid = self.db.get_tw_of_ts(profileid, ts) + # convert ts to a readable format + ts = utils.convert_format(ts, utils.alerts_format) + + if not twid: + return + + twid_number = int(twid[0].replace("timewindow", "")) + evidence = Evidence( + evidence_type=EvidenceType.NETWORK_GPS_LOCATION_LEAKED, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), + threat_level=ThreatLevel.LOW, + confidence=0.9, + description=description, + profile=ProfileID(ip=srcip), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=ts, + proto=Proto(proto.lower()), + port=dport, + source_target_tag=Tag.CC, + category=IDEACategory.MALWARE + ) + self.db.set_evidence(evidence) + + evidence = Evidence( + evidence_type=EvidenceType.NETWORK_GPS_LOCATION_LEAKED, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=dstip + ), + threat_level=ThreatLevel.HIGH, + confidence=0.9, + description=description, + profile=ProfileID(ip=dstip), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=ts, + proto=Proto(proto.lower()), + port=dport, + source_target_tag=Tag.CC, + category=IDEACategory.MALWARE + ) + + self.db.set_evidence(evidence) + def compile_and_save_rules(self): """ diff --git a/modules/network_discovery/horizontal_portscan.py b/modules/network_discovery/horizontal_portscan.py index 6b73ae772..abdd89334 100644 --- a/modules/network_discovery/horizontal_portscan.py +++ b/modules/network_discovery/horizontal_portscan.py @@ -304,7 +304,6 @@ def set_evidence_horizontal_portscan(self, evidence: dict): f'Horizontal port scan to port {port_info} {portproto}. ' f'From {srcip} to {evidence["amount_of_dips"]} unique destination IPs. ' f'Total packets sent: {evidence["pkts_sent"]}. ' - f'Threat Level: {threat_level}. ' f'Confidence: {confidence}. by Slips' ) diff --git a/modules/network_discovery/network_discovery.py b/modules/network_discovery/network_discovery.py index bcdebf0da..7632ec5ba 100644 --- a/modules/network_discovery/network_discovery.py +++ b/modules/network_discovery/network_discovery.py @@ -74,7 +74,6 @@ def check_icmp_sweep( Use our own Zeek scripts to detect ICMP scans. Threshold is on the scripts and it is 25 ICMP flows """ - if 'TimestampScan' in note: evidence_type = EvidenceType.ICMP_TIMESTAMP_SCAN elif 'ICMPAddressScan' in note: @@ -88,21 +87,17 @@ def check_icmp_sweep( hosts_scanned = int(msg.split('on ')[1].split(' hosts')[0]) # get the confidence from 0 to 1 based on the number of hosts scanned confidence = 1 / (255 - 5) * (hosts_scanned - 255) + 1 - threat_level = ThreatLevel.MEDIUM saddr = profileid.split('_')[1] - # this is the last IP scanned - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) - # this one is detected by Zeek, so we can't track the UIDs causing it evidence = Evidence( evidence_type=evidence_type, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=ThreatLevel.MEDIUM, confidence=confidence, description=msg, profile=ProfileID(ip=saddr), @@ -321,29 +316,25 @@ def set_evidence_dhcp_scan( uids, number_of_requested_addrs ): - threat_level = ThreatLevel.MEDIUM - confidence = 0.8 srcip = profileid.split('_')[-1] - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) description = ( f'Performing a DHCP scan by requesting ' f'{number_of_requested_addrs} different IP addresses. ' - f'Threat Level: {threat_level}. ' f'Confidence: {confidence}. by Slips' ) - + twid_number = int(twid.replace("timewindow", "")) evidence = Evidence( evidence_type=EvidenceType.DHCP_SCAN, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), + threat_level=ThreatLevel.MEDIUM, + confidence=0.8, description=description, profile=ProfileID(ip=srcip), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_number), uid=uids, timestamp=timestamp, category=IDEACategory.RECON_SCANNING, @@ -363,7 +354,8 @@ def check_dhcp_scan(self, flow_info): flow = flow_info['flow'] requested_addr = flow['requested_addr'] if not requested_addr: - # we are only interested in DHCPREQUEST flows, where a client is requesting an IP + # we are only interested in DHCPREQUEST flows, + # where a client is requesting an IP return profileid = flow_info['profileid'] @@ -400,7 +392,8 @@ def check_dhcp_scan(self, flow_info): # we alert every 4,8,12, etc. requested IPs number_of_requested_addrs = len(dhcp_flows) if number_of_requested_addrs % self.minimum_requested_addrs == 0: - # get the uids of all the flows where this client was requesting an addr in this tw + # get the uids of all the flows where this client + # was requesting an addr in this tw for uids_list in dhcp_flows.values(): uids.append(uids_list[0]) @@ -430,15 +423,20 @@ def main(self): # 1. Vertical port scan: # (single IP being scanned for multiple ports) - # - 1 srcip sends not established flows to > 3 dst ports in the same dst ip. Any number of packets + # - 1 srcip sends not established flows to > 3 dst ports in the + # same dst ip. Any number of packets # 2. Horizontal port scan: # (scan against a group of IPs for a single port) - # - 1 srcip sends not established flows to the same dst ports in > 3 dst ip. + # - 1 srcip sends not established flows to the same dst ports in + # > 3 dst ip. # 3. Too many connections???: - # - 1 srcip sends not established flows to the same dst ports, > 3 pkts, to the same dst ip - # 4. Slow port scan. Same as the others but distributed in multiple time windows + # - 1 srcip sends not established flows to the same dst ports, + # > 3 pkts, to the same dst ip + # 4. Slow port scan. Same as the others but distributed in + # multiple time windows - # Remember that in slips all these port scans can happen for traffic going IN to an IP or going OUT from the IP. + # Remember that in slips all these port scans can happen + # for traffic going IN to an IP or going OUT from the IP. self.horizontal_ps.check(profileid, twid) self.vertical_ps.check(profileid, twid) diff --git a/modules/p2ptrust/p2ptrust.py b/modules/p2ptrust/p2ptrust.py index 4c0ea7e38..5914c9861 100644 --- a/modules/p2ptrust/p2ptrust.py +++ b/modules/p2ptrust/p2ptrust.py @@ -6,7 +6,9 @@ import subprocess import time from pathlib import Path -from typing import Dict +from typing import Dict, \ + Optional, \ + Tuple import json import sys import socket @@ -19,18 +21,16 @@ from modules.p2ptrust.utils.go_director import GoDirector from slips_files.core.evidence_structure.evidence import \ ( + dict_to_evidence, Evidence, ProfileID, TimeWindow, - Victim, Attacker, - Proto, ThreatLevel, EvidenceType, IoCType, Direction, IDEACategory, - Tag ) @@ -48,7 +48,8 @@ def validate_slips_data(message_data: str) -> (str, int): 'cache_age': cache_age } - If the message is correct, the two values are returned as a tuple (str, int). + If the message is correct, the two values are + returned as a tuple (str, int). If not, (None, None) is returned. :param message_data: data from slips request channel :return: the received msg or None tuple @@ -62,7 +63,8 @@ def validate_slips_data(message_data: str) -> (str, int): except ValueError: # message has wrong format print( - f'The message received from p2p_data_request channel has incorrect format: {message_data}' + f'The message received from p2p_data_request channel' + f' has incorrect format: {message_data}' ) return None @@ -78,7 +80,8 @@ class Trust(IModule): gopy_channel_raw='p2p_gopy' pygo_channel_raw='p2p_pygo' start_pigeon=True - pigeon_binary= os.path.join(os.getcwd(),'p2p4slips/p2p4slips') # or make sure the binary is in $PATH + # or make sure the binary is in $PATH + pigeon_binary= os.path.join(os.getcwd(),'p2p4slips/p2p4slips') pigeon_key_file='pigeon.keys' rename_redis_ip_info=False rename_sql_db_file=False @@ -122,7 +125,8 @@ def init(self, *args, **kwargs): if self.rename_redis_ip_info: self.storage_name += str(self.port) self.c1 = self.db.subscribe('report_to_peers') - # channel to send msgs to whenever slips needs info from other peers about an ip + # channel to send msgs to whenever slips needs + # info from other peers about an ip self.c2 = self.db.subscribe(self.p2p_data_request_channel) # this channel receives peers requests/updates self.c3 = self.db.subscribe(self.gopy_channel) @@ -144,7 +148,7 @@ def init(self, *args, **kwargs): self.sql_db_name = f'{self.data_dir}trustdb.db' if self.rename_sql_db_file: - self.sql_db_name += str(pigeon_port) + self.sql_db_name += str(self.pigeon_port) # todo don't duplicate this dict, move it to slips_utils # all evidence slips detects has threat levels of strings # each string should have a corresponding int value to be able to calculate @@ -200,8 +204,10 @@ def _configure(self): self.sql_db_name, drop_tables_on_startup=True ) - self.reputation_model = reputation_model.BaseModel(self.logger, self.trust_db) - # print(f"[DEBUGGING] Starting godirector with pygo_channel: {self.pygo_channel}") + self.reputation_model = reputation_model.BaseModel( + self.logger, self.trust_db) + # print(f"[DEBUGGING] Starting godirector with + # pygo_channel: {self.pygo_channel}") self.go_director = GoDirector( self.logger, self.trust_db, @@ -256,80 +262,124 @@ def _configure(self): # print(f"[debugging] runnning pigeon: {executable}") - - def new_evidence_callback(self, msg: Dict): + + def extract_confidence(self, evidence: Evidence) -> Optional[float]: """ - This function is called whenever a msg arrives to the report_to_peers channel, - It compares the score and confidence of the given IP and decides whether or not to - share it accordingly + returns the confidence of the given evidence or None if no + confidence was found """ - try: - data = json.loads(msg['data']) - except json.decoder.JSONDecodeError: - # not a valid json dict - return - - # example: dstip srcip dport sport dstdomain - attacker_direction = data.get('attacker_direction') - if 'ip' not in attacker_direction: # and not 'domain' in attacker_direction: - # todo do we share domains too? - # the detection is a srcport, dstport, etc. don't share - return + confidence: float = evidence.confidence + + if confidence: + return confidence + + attacker_ip: str = evidence.attacker.value + self.print( + f"IP {attacker_ip} doesn't have a confidence. " + f"not sharing to the network.", 0, 2, + ) + return + + def extract_threat_level(self, evidence: Evidence) -> Optional[ThreatLevel]: + """ + returns the confidence of the given evidence or None if no + confidence was found + """ + threat_level: ThreatLevel = evidence.threat_level + + if threat_level: + return threat_level + + attacker_ip: str = evidence.attacker.value + self.print( + f"IP {attacker_ip} doesn't have a threat_level. " + f"not sharing to the network.", 0,2, + ) - evidence_type = data.get('evidence_type') - if 'P2PReport' in evidence_type: + return + + def should_share(self, evidence: Evidence) -> bool: + """ + decides whether or not to report the given evidence to other + peers + """ + if evidence.evidence_type == EvidenceType.P2P_REPORT: # we shouldn't re-share evidence reported by other peers - return + return False + if evidence.attacker.attacker_type != IoCType.IP.name: + # we only share ips with other peers. + return False - attacker = data.get('attacker') - confidence = data.get('confidence', False) - threat_level = data.get('threat_level', False) + confidence = self.extract_confidence(evidence) + if not confidence: + return False + + threat_level: ThreatLevel = self.extract_threat_level(evidence) if not threat_level: - self.print( - f"IP {attacker} doesn't have a threat_level. not sharing to the network.", 0,2, - ) + return False + + return True + + + def new_evidence_callback(self, msg: Dict[str, str]): + """ + Decides to share an evidence generated by slips to other peers or not + depending on whether we have info about this ip from the p2p + network or not + It is called whenever a msg arrives to the + report_to_peers channel, + """ + try: + evidence: Dict[str, str] = json.loads(msg['data']) + except json.decoder.JSONDecodeError: return - if not confidence: - self.print( - f"IP {attacker} doesn't have a confidence. not sharing to the network.", 0, 2, - ) + evidence: Evidence = dict_to_evidence(evidence) + + if not self.should_share(evidence): return - # get the int representing this threat_level - score = self.threat_levels[threat_level] - # todo what we're currently sharing is the threat level(int) of the evidence caused by this ip + # todo what we're currently sharing is the threat level(int) + # of the evidence caused by this ip # todo when we generate a new evidence, - # we give it a score and a tl, but we don't update the IP_Info and give this ip a score in th db! + # we give it a score and a tl, but we don't update the + # IP_Info and give this ip a score in th db! # TODO: discuss - only share score if confidence is high enough? # compare slips data with data in go data_already_reported = True try: - cached_opinion = self.trust_db.get_cached_network_opinion( - 'ip', attacker + cached_opinion: Tuple = self.trust_db.get_cached_network_opinion( + 'ip', evidence.attacker.value ) + # get the cached network opinion about this ip ( cached_score, cached_confidence, network_score, timestamp, ) = cached_opinion - # if we don't have info about this ip from the p2p network, report it to the p2p netwrok + # if we don't have info about this ip from the p2p network, + # report it to the p2p network if not cached_score: data_already_reported = False except KeyError: data_already_reported = False except IndexError: - # data saved in local db have wrong structure, this is an invalid state + # data saved in local db have wrong structure, + # this is an invalid state return - # TODO: in the future, be smarter and share only when needed. For now, we will always share if not data_already_reported: - # Take data and send it to a peer as report. + # send the peer report to other peers p2p_utils.send_evaluation_to_go( - attacker, score, confidence, '*', self.pygo_channel, self.db + evidence.attacker.value, + evidence.threat_level.value, + evidence.confidence, + '*', + self.pygo_channel, + self.db ) def gopy_callback(self, msg: Dict): @@ -360,58 +410,6 @@ def data_request_callback(self, msg: Dict): except Exception as e: self.print(f'Exception {e} in data_request_callback', 0, 1) - # def handle_update(self, ip_address: str) -> None: - # """ - # Handle IP scores changing in Slips received from the ip_info_change channel - # - # This method checks if Slips has a new score that are different - # from the scores known to the network, and if so, it means that it is worth - # sharing and it will be shared. - # Additionally, if the score is serious, the node will be blamed(blocked) - # :param ip_address: The IP address sent through the ip_info_change channel (if it is not valid IP, it returns) - # """ - # - # # abort if the IP is not valid - # if not utils.validate_ip_address(ip_address): - # self.print("IP validation failed") - # return - # - # score, confidence = utils.get_ip_info_from_slips(ip_address) - # if score is None: - # self.print("IP doesn't have any score/confidence values in DB") - # return - # - # # insert data from slips to database - # self.trust_db.insert_slips_score(ip_address, score, confidence) - # - # # TODO: discuss - only share score if confidence is high enough? - # - # # compare slips data with data in go - # data_already_reported = True - # try: - # cached_opinion = self.trust_db.get_cached_network_opinion("ip", ip_address) - # cached_score, cached_confidence, network_score, timestamp = cached_opinion - # if cached_score is None: - # data_already_reported = False - # elif abs(score - cached_score) < 0.1: - # data_already_reported = False - # except KeyError: - # data_already_reported = False - # except IndexError: - # # data saved in local db have wrong structure, this is an invalid state - # return - # - # # TODO: in the future, be smarter and share only when needed. For now, we will always share - # if not data_already_reported: - # utils.send_evaluation_to_go(ip_address, score, confidence, "*", self.pygo_channel) - # - # # TODO: discuss - based on what criteria should we start blaming? - # # decide whether or not to block - # if score > 0.8 and confidence > 0.6: - # #todo finish the blocking logic and actually block the ip - # - # # tell other peers that we're blocking this IP - # utils.send_blame_to_go(ip_address, score, confidence, self.pygo_channel) def set_evidence_malicious_ip(self, ip_info: dict, @@ -420,79 +418,78 @@ def set_evidence_malicious_ip(self, """ Set an evidence for a malicious IP met in the timewindow ip_info format is json serialized { - # 'ip': the source/dst ip - # 'profileid' : profile where the alert was generated. It includes the src ip - # 'twid' : name of the timewindow when it happened. - # 'proto' : protocol - # 'ip_state' : is basically the answer to "which one is the - # blacklisted IP"?'can be 'srcip' or - # 'dstip', - # 'stime': Exact time when the evidence happened - # 'uid': Zeek uid of the flow that generated the evidence, - # 'cache_age': How old is the info about this ip - # } + 'ip': the source/dst ip + 'profileid' : profile where the alert was generated. + It includes the src ip + 'twid' : name of the timewindow when it happened. + 'proto' : protocol + 'ip_state' : is basically the answer to "which one is the + blacklisted IP"?'can be 'srcip' or + 'dstip', + 'stime': Exact time when the evidence happened + 'uid': Zeek uid of the flow that generated the evidence, + 'cache_age': How old is the info about this ip + } :param threat_level: the threat level we learned form the network :param confidence: how confident the network opinion is about this opinion """ - + attacker_ip: str = ip_info.get('ip') - ip_state = ip_info.get('ip_state') - uid = ip_info.get('uid') profileid = ip_info.get('profileid') - twid = ip_info.get('twid') - timestamp = str(ip_info.get('stime')) saddr = profileid.split("_")[-1] - - category = IDEACategory.ANOMALY_TRAFFIC - + + threat_level = utils.threat_level_to_string(threat_level) + threat_level = ThreatLevel[threat_level.upper()] + twid_int = int(ip_info.get('twid').replace("timewindow", "")) + + # add this ip to our MaliciousIPs hash in the database + self.db.set_malicious_ip(attacker_ip, profileid, ip_info.get('twid')) + ip_identification = self.db.get_ip_identification(attacker_ip) - if 'src' in ip_state: + + if 'src' in ip_info.get('ip_state'): description = ( f'Connection from blacklisted IP {attacker_ip} ' f'({ip_identification}) to {saddr} Source: Slips P2P network.' ) - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=attacker_ip - ) else: description = ( f'Connection to blacklisted IP {attacker_ip} ' f'({ip_identification}) ' f'from {saddr} Source: Slips P2P network.' ) - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr + + for ip in (saddr, attacker_ip): + evidence = Evidence( + evidence_type= EvidenceType.MALICIOUS_IP_FROM_P2P_NETWORK, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=ip + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=attacker_ip), + timewindow=TimeWindow(number=twid_int), + uid=[ip_info.get('uid')], + timestamp=str(ip_info.get('stime')), + category=IDEACategory.ANOMALY_TRAFFIC, ) + + self.db.set_evidence(evidence) - evidence = Evidence( - evidence_type= EvidenceType.MALICIOUS_IP_FROM_P2P_NETWORK, - attacker=attacker, - threat_level=threat_level, - confidence=confidence, - description=description, - profile=ProfileID(ip=attacker.value), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), - uid=[uid], - timestamp=timestamp, - category=category, - ) - - self.db.set_evidence(evidence) - # add this ip to our MaliciousIPs hash in the database - self.db.set_malicious_ip(attacker, profileid, twid) def handle_data_request(self, message_data: str) -> None: """ - Process the request from Slips, ask the network and process the network response. + Process the request from Slips, ask the network and + process the network response. Three `arguments` are expected in the redis channel: ip_address: str, cache_age: int [seconds] - The return value is sent to the redis channel `p2p_data_response` in the format: + The return value is sent to the redis channel + `p2p_data_response` in the format: ip_address: str, timestamp: int [time of assembling the response], network_opinion: float, @@ -503,11 +500,13 @@ def handle_data_request(self, message_data: str) -> None: This method will check if any data not older than `cache_age` is saved in cache. If yes, this data is returned. If not, the database is checked. - An ASK query is sent to the network and responses are collected and saved into - the redis database. + An ASK query is sent to the network and responses + are collected and saved into the redis database. - :param message_data: The data received from the redis channel `p2p_data_response` - :return: None, the result is saved into the redis database under key `p2p4slips` + :param message_data: The data received from the redis + channel `p2p_data_response` + :return: None, the result is saved into the redis + database under key `p2p4slips` """ # make sure that IP address is valid @@ -515,8 +514,10 @@ def handle_data_request(self, message_data: str) -> None: ip_info = validate_slips_data(message_data) if ip_info is None: # IP address is not valid, aborting - # print(f"DEBUGGING: IP address is not valid: {ip_info}, not asking the network") + # print(f"DEBUGGING: IP address is not valid: + # {ip_info}, not asking the network") return + # ip_info is { # 'ip': str(saddr), # 'profileid' : str(profileid), @@ -542,10 +543,12 @@ def handle_data_request(self, message_data: str) -> None: # print("DEBUGGING: cached value is ok, not asking the network.") return - # if cached value is old, ask the peers + # since cached value is old, ask the peers - # TODO: in some cases, it is not necessary to wait, specify that and implement it - # I do not remember writing this comment. I have no idea in which cases there is no need to wait? Maybe + # TODO: in some cases, it is not necessary to wait, specify + # that and implement it + # I do not remember writing this comment. I have no idea + # in which cases there is no need to wait? Maybe # when everybody responds asap? p2p_utils.send_request_to_go(ip_address, self.pygo_channel, self.db) self.print(f'[Slips -> The Network] request about {ip_address}') @@ -561,37 +564,57 @@ def handle_data_request(self, message_data: str) -> None: combined_score, combined_confidence, ) = self.reputation_model.get_opinion_on_ip(ip_address) - - # no data in db - this happens when testing, if there is not enough data on peers + + self.process_network_response(ip_address, + combined_score, + combined_confidence, + network_score, + confidence, + ip_info) + + def process_network_response( + self, ip, combined_score, combined_confidence, network_score, + confidence, ip_info + ): + """ + stores the reported score and confidence about the ip and adds an + evidence if necessary + """ + # no data in db - this happens when testing, + # if there is not enough data on peers if combined_score is None: self.print( - f'No data received from the network about {ip_address}\n', 0, 2 - ) - # print(f"[DEBUGGING] No data received from the network about {ip_address}\n") - else: - self.print( - f'The Network shared some data about {ip_address}, ' - f'Shared data: score={combined_score}, confidence={combined_confidence} saving it to now!\n', - 0, - 2, + f'No data received from the' + f' network about {ip}\n', 0, 2 ) + return + + self.print( + f'The Network shared some data about {ip}, ' + f'Shared data: score={combined_score}, ' + f'confidence={combined_confidence} saving it to now!\n', + 0, + 2, + ) - # save it to IPsInfo hash in p2p4slips key in the db AND p2p_reports key - p2p_utils.save_ip_report_to_db( - ip_address, - combined_score, - combined_confidence, - network_score, - self.db, - self.storage_name, + # save it to IPsInfo hash in p2p4slips key in the db + # AND p2p_reports key + p2p_utils.save_ip_report_to_db( + ip, + combined_score, + combined_confidence, + network_score, + self.db, + self.storage_name, + ) + if int(combined_score) * int(confidence) > 0: + self.set_evidence_malicious_ip( + ip_info, combined_score, confidence ) - if int(combined_score) * int(confidence) > 0: - self.set_evidence_malicious_ip( - ip_info, combined_score, confidence - ) def respond_to_message_request(self, key, reporter): - # todo do you mean another peer is asking me about an ip? yes. in override mode + # todo do you mean another peer is asking me about + # an ip? yes. in override mode """ Handle data request from a peer (in overriding p2p mode) (set to false by defualt) :param key: The ip requested by the peer @@ -629,7 +652,8 @@ def pre_main(self): # check if it was possible to start up pigeon if self.start_pigeon and self.pigeon is None: self.print( - 'Module was supposed to start up pigeon but it was not possible to start pigeon! Exiting...' + 'Module was supposed to start up pigeon but it was not' + ' possible to start pigeon! Exiting...' ) return 1 @@ -642,20 +666,20 @@ def pre_main(self): # self.c4 = self.db.subscribe(self.slips_update_channel) def main(self): - """main loop function""" - if msg:= self.get_msg('report_to_peers'): + if msg := self.get_msg('report_to_peers'): self.new_evidence_callback(msg) - if msg:= self.get_msg(self.p2p_data_request_channel): + if msg := self.get_msg(self.p2p_data_request_channel): self.data_request_callback(msg) - if msg:= self.get_msg(self.gopy_channel): + if msg := self.get_msg(self.gopy_channel): self.gopy_callback(msg) ret_code = self.pigeon.poll() if ret_code is not None: self.print( - f'Pigeon process suddenly terminated with return code {ret_code}. Stopping module.' + f'Pigeon process suddenly terminated with ' + f'return code {ret_code}. Stopping module.' ) return 1 diff --git a/modules/p2ptrust/utils/go_director.py b/modules/p2ptrust/utils/go_director.py index f15a8703e..9641e84b2 100644 --- a/modules/p2ptrust/utils/go_director.py +++ b/modules/p2ptrust/utils/go_director.py @@ -314,20 +314,20 @@ def respond_to_message_request(self, key, reporter): key, score, confidence, reporter, self.pygo_channel, self.db ) self.print( - f'[Slips -> The Network] Slips responded with info score={score} confidence={confidence} about IP: {key} to {reporter}.', + f'[Slips -> The Network] Slips responded with info score={score} ' + f'confidence={confidence} about IP: {key} to {reporter}.', 2, 0, ) - # print(f"[Slips -> The Network] Slips responded with info score={score} confidence={confidence} about IP: {key} to {reporter}.") + # print(f"[Slips -> The Network] Slips responded with info score={score} + # confidence={confidence} about IP: {key} to {reporter}.") else: # send_empty_evaluation_to_go(key, reporter, self.pygo_channel) - # self.print(f"[Slips -> The Network] Slips has no info about IP: {key}. Responded with empty report to {reporter}", 2, 0) self.print( f'[Slips -> The Network] Slips has no info about IP: {key}. Not responding to {reporter}', 2, 0, ) - # self.print(f"[DEBUGGING] [Slips -> The Network] Slips has no info about IP: {key}. Responded with empty report to {reporter}") def process_message_report( self, reporter: str, report_time: int, data: dict @@ -488,11 +488,6 @@ def set_evidence_p2p_report( set evidence for the newly created attacker profile stating that it attacked another peer """ - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=ip - ) threat_level = utils.threat_level_to_string(score) # confidence depends on how long the connection @@ -507,7 +502,7 @@ def set_evidence_p2p_report( reporter_ip = '' description = f'attacking another peer: {reporter_ip} ' \ - f'({reporter}). threat level: {threat_level} ' \ + f'({reporter}). ' \ f'confidence: {confidence} {ip_identification}' # get the tw of this report time @@ -521,7 +516,11 @@ def set_evidence_p2p_report( timestamp = utils.convert_format(timestamp, utils.alerts_format) evidence = Evidence( evidence_type=EvidenceType.P2P_REPORT, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=ip + ), threat_level=threat_level, confidence=confidence, description=description, diff --git a/modules/p2ptrust/utils/utils.py b/modules/p2ptrust/utils/utils.py index d55a8ca40..e9899fe11 100644 --- a/modules/p2ptrust/utils/utils.py +++ b/modules/p2ptrust/utils/utils.py @@ -142,7 +142,8 @@ def read_data_from_ip_info(ip_info: dict) -> (float, float): -def save_ip_report_to_db(ip, score, confidence, network_trust, db, timestamp=None): +def save_ip_report_to_db(ip, score, confidence, network_trust, + db, timestamp=None): if timestamp is None: timestamp = time.time() @@ -154,7 +155,8 @@ def save_ip_report_to_db(ip, score, confidence, network_trust, db, timestamp=Non } # store it in p2p_reports key - # print(f"*** [debugging p2p] *** stored a report about {ip} in p2p_Reports and IPsInfo keys") + # print(f"*** [debugging p2p] *** stored a report about + # {ip} in p2p_Reports and IPsInfo keys") db.store_p2p_report(ip, report_data) # store it in IPsInfo key diff --git a/modules/progress_bar/progress_bar.py b/modules/progress_bar/progress_bar.py index 405c6c102..9f8eed432 100644 --- a/modules/progress_bar/progress_bar.py +++ b/modules/progress_bar/progress_bar.py @@ -124,8 +124,6 @@ def shutdown_gracefully(self): # to tell output.py to no longer send prints here self.pbar_finished.set() - - def main(self): """ keeps receiving events until pbar reaches 100% diff --git a/modules/rnn_cc_detection/rnn_cc_detection.py b/modules/rnn_cc_detection/rnn_cc_detection.py index bebaa6e8f..e6f28f915 100644 --- a/modules/rnn_cc_detection/rnn_cc_detection.py +++ b/modules/rnn_cc_detection/rnn_cc_detection.py @@ -56,14 +56,6 @@ def set_evidence_cc_channel( tupleid = tupleid.split('-') dstip, port, proto = tupleid[0], tupleid[1], tupleid[2] srcip = profileid.split("_")[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) - - threat_level: ThreatLevel = ThreatLevel.HIGH portproto: str = f'{port}/{proto}' port_info: str = self.db.get_port_info(portproto) ip_identification: str = self.db.get_ip_identification(dstip) @@ -74,14 +66,19 @@ def set_evidence_cc_channel( ) timestamp: str = utils.convert_format(timestamp, utils.alerts_format) + twid_int = int(twid.replace("timewindow", "")) evidence: Evidence = Evidence( evidence_type=EvidenceType.COMMAND_AND_CONTROL_CHANNEL, - attacker=attacker, - threat_level=threat_level, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), + threat_level=ThreatLevel.HIGH, confidence=confidence, description=description, profile=ProfileID(ip=srcip), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_int), uid=[uid], timestamp=timestamp, category=IDEACategory.INTRUSION_BOTNET, @@ -89,7 +86,28 @@ def set_evidence_cc_channel( port=int(port), proto=Proto(proto.lower()) if proto else None, ) + self.db.set_evidence(evidence) + evidence: Evidence = Evidence( + evidence_type=EvidenceType.COMMAND_AND_CONTROL_CHANNEL, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=dstip + ), + threat_level=ThreatLevel.HIGH, + confidence=confidence, + description=description, + profile=ProfileID(ip=dstip), + timewindow=TimeWindow(number=twid_int), + uid=[uid], + timestamp=timestamp, + category=IDEACategory.INTRUSION_BOTNET, + source_target_tag=Tag.CC, + port=int(port), + proto=Proto(proto.lower()) if proto else None, + ) + self.db.set_evidence(evidence) diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index b3eeeea92..e53921275 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -6,7 +6,8 @@ import requests import threading import time -from typing import Dict +from typing import Dict, \ + List from slips_files.common.slips_utils import utils from slips_files.common.imports import * @@ -110,16 +111,18 @@ def __read_configuration(self): def set_evidence_malicious_asn( self, - attacker: str, + daddr: str, uid: str, timestamp: str, profileid: str, twid: str, asn: str, asn_info: dict, + is_dns_response: bool = False ): """ - :param asn_info: the malicious ASN info taken from own_malicious_iocs.csv + :param asn_info: the malicious ASN info taken from + own_malicious_iocs.csv """ confidence: float = 0.8 @@ -133,27 +136,35 @@ def set_evidence_malicious_asn( threat_level: ThreatLevel = ThreatLevel(threat_level) tags = asn_info.get('tags', '') - identification: str = self.db.get_ip_identification(attacker) - - description: str = ( - f'Connection to IP: {attacker} with blacklisted ASN: {asn} ' + identification: str = self.db.get_ip_identification(daddr) + if is_dns_response: + description: str = ( + f'Connection to IP: {daddr} with blacklisted ASN: {asn} ' + ) + else: + description: str = ( + f'DNS response with IP: {daddr} with blacklisted ASN: {asn} ' + ) + + description += ( f'Description: {asn_info["description"]}, ' f'Found in feed: {asn_info["source"]}, ' - f'Confidence: {confidence}. Tags: {tags} {identification}' + f'Confidence: {confidence}. ' + f'Tags: {tags} {identification}' ) - attacker = Attacker( + twid_int = int(twid.replace("timewindow", "")) + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_ASN, + attacker=Attacker( direction=Direction.SRC, attacker_type=IoCType.IP, value=saddr - ) - evidence = Evidence( - evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_ASN, - attacker=attacker, + ), threat_level=threat_level, confidence=confidence, description=description, profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_int), uid=[uid], timestamp=utils.convert_format(timestamp, utils.alerts_format), category=IDEACategory.ANOMALY_TRAFFIC, @@ -161,13 +172,119 @@ def set_evidence_malicious_asn( ) self.db.set_evidence(evidence) + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_ASN, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_int), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_ASN, + ) + + self.db.set_evidence(evidence) + + + def set_evidence_malicious_ip_in_dns_response( + self, + ip: str, + uid: str, + timestamp: str, + ip_info: dict, + dns_query: str, + profileid: str, + twid: str, + ): + """ + Set an evidence for a blacklisted IP found in one of the TI files + :param ip: the ip source file + :param uid: Zeek uid of the flow that generated the evidence + :param timestamp: Exact time when the evidence happened + :param ip_info: is all the info we have about that IP + in the db source, confidence, description, etc. + :param profileid: profile where the alert was generated. It includes the src ip + :param twid: name of the timewindow when it happened. + """ + threat_level: float = utils.threat_levels[ + ip_info.get('threat_level', 'medium') + ] + threat_level: ThreatLevel = ThreatLevel(threat_level) + saddr = profileid.split("_")[-1] + + ip_identification: str = self.db.get_ip_identification( + ip, get_ti_data=False + ).strip() + description: str = (f'DNS answer with a blacklisted ' + f'IP: {ip} for query: {dns_query}' + f'{ip_identification} Description: ' + f'{ip_info["description"]}. ' + f'Source: {ip_info["source"]}.') + + twid_int = int(twid.replace("timewindow", "")) + evidence = Evidence( + evidence_type=EvidenceType + .THREAT_INTELLIGENCE_BLACKLISTED_DNS_ANSWER, + attacker= Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=ip + ), + threat_level=threat_level, + confidence=1.0, + description=description, + profile=ProfileID(ip=ip), + timewindow=TimeWindow(number=twid_int), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_IP, + ) + + self.db.set_evidence(evidence) + + evidence = Evidence( + evidence_type=EvidenceType + .THREAT_INTELLIGENCE_BLACKLISTED_DNS_ANSWER, + attacker= Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=threat_level, + confidence=1.0, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_int), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_IP, + ) + + self.db.set_evidence(evidence) + + # mark this ip as malicious in our database + ip_info = {'threatintelligence': ip_info} + self.db.setInfoForIPs(ip, ip_info) + # add this ip to our MaliciousIPs hash in the database + self.db.set_malicious_ip(ip, profileid, twid) + + def set_evidence_malicious_ip( self, ip: str, uid: str, - dstip: str, + daddr: str, timestamp: str, ip_info: dict, profileid: str = '', @@ -179,7 +296,9 @@ def set_evidence_malicious_ip( :param ip: the ip source file :param uid: Zeek uid of the flow that generated the evidence :param timestamp: Exact time when the evidence happened - :param ip_info: is all the info we have about that IP in the db source, confidence, description, etc. + :param daddr: dst address of the flow + :param ip_info: is all the info we have about that IP + in the db source, confidence, description, etc. :param profileid: profile where the alert was generated. It includes the src ip :param twid: name of the timewindow when it happened. :param ip_state: is basically the answer to "which one is the @@ -189,30 +308,14 @@ def set_evidence_malicious_ip( ip_info.get('threat_level', 'medium') ] threat_level: ThreatLevel = ThreatLevel(threat_level) - confidence: float = 1.0 - srcip = profileid.split("_")[-1] + saddr = profileid.split("_")[-1] if 'src' in ip_state: description: str = f'connection from blacklisted ' \ - f'IP: {ip} to {dstip}. ' - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=ip - ) + f'IP: {ip} to {daddr}. ' elif 'dst' in ip_state: - if self.is_dns_response: - description: str = f'DNS answer with a blacklisted ' \ - f'IP: {ip} for query: {self.dns_query}' - else: - description: str = f'connection to blacklisted ' \ - f'IP: {ip} from {srcip}. ' - - attacker = Attacker( - direction=Direction.DST, - attacker_type=IoCType.IP, - value=ip - ) + description: str = (f'connection to blacklisted ' + f'IP: {ip} from {saddr}. ') else: # ip_state is not specified? return @@ -224,23 +327,44 @@ def set_evidence_malicious_ip( f'{ip_info["description"]}. ' f'Source: {ip_info["source"]}.') - + twid_int = int(twid.replace("timewindow", "")) evidence = Evidence( evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_IP, - attacker=attacker, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, - confidence=confidence, + confidence=1.0, description=description, - profile=ProfileID(ip=attacker.value), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_int), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_IP, + ) + self.db.set_evidence(evidence) + + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_IP, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=threat_level, + confidence=1.0, + description=description, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_int), uid=[uid], timestamp=utils.convert_format(timestamp, utils.alerts_format), category=IDEACategory.ANOMALY_TRAFFIC, source_target_tag=Tag.BLACKLISTED_IP, ) - self.db.set_evidence(evidence) - # mark this ip as malicious in our database ip_info = {'threatintelligence': ip_info} @@ -260,7 +384,7 @@ def set_evidence_malicious_domain( twid: str = '', ): """ - Set an evidence for a malicious domain met in the timewindow + Set an evidence for a malicious domain :param source_file: is the domain source file :param domain_info: is all the info we have about this domain in the db source, confidence , description etc... @@ -281,37 +405,27 @@ def set_evidence_malicious_domain( domain_info.get('threat_level', 'high') ] threat_level: ThreatLevel = ThreatLevel(threat_level) - - - if self.is_dns_response: - description: str = (f'DNS answer with a blacklisted ' - f'CNAME: {domain} ' - f'for query: {self.dns_query} ') - else: - description: str = f'connection to a blacklisted domain {domain}. ' - - description += f'Description: {domain_info.get("description", "")},' \ - f' Found in feed: {domain_info["source"]}, ' \ - f'Confidence: {confidence}. ' + description: str = (f'connection to a blacklisted domain {domain}. ' + f'Description: {domain_info.get("description", "")},' + f'Found in feed: {domain_info["source"]}, ' + f'Confidence: {confidence}. ') tags = domain_info.get('tags', None) if tags: description += f'with tags: {tags}. ' - - attacker = Attacker( + twid_number = int(twid.replace("timewindow", "")) + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN, + attacker=Attacker( direction=Direction.SRC, attacker_type=IoCType.IP, value=srcip - ) - - evidence = Evidence( - evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN, - attacker=attacker, + ), threat_level=threat_level, confidence=confidence, description=description, profile=ProfileID(ip=srcip), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_number), uid=[uid], timestamp=utils.convert_format(timestamp, utils.alerts_format), category=IDEACategory.ANOMALY_TRAFFIC, @@ -319,13 +433,36 @@ def set_evidence_malicious_domain( ) self.db.set_evidence(evidence) + domain_resolution: List[str] = self.db.get_domain_resolution(domain) + if domain_resolution: + domain_resolution: str = domain_resolution[0] + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.DOMAIN, + value=domain + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=domain_resolution), + timewindow=TimeWindow(number=twid_number), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_DOMAIN, + ) + + self.db.set_evidence(evidence) def is_valid_threat_level(self, threat_level): return threat_level in utils.threat_levels def parse_local_ti_file(self, ti_file_path: str) -> bool: """ - Read all the files holding IP addresses and a description and store in the db. + Read all the files holding IP addresses and a description + and store in the db. This also helps in having unique ioc across files Returns nothing, but the dictionary should be filled :param ti_file_path: full path_to local threat intel file @@ -719,11 +856,6 @@ def set_evidence_malicious_hash(self, file_info: Dict[str, any]): f'Detected by: {file_info["blacklist"]}. ' f'Score: {confidence}. {ip_identification}' ) - attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=srcip - ) ts = utils.convert_format( file_info['flow']["starttime"], utils.alerts_format ) @@ -732,7 +864,11 @@ def set_evidence_malicious_hash(self, file_info: Dict[str, any]): )) evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_DOWNLOADED_FILE, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ), threat_level=threat_level, confidence=confidence, description=description, @@ -744,6 +880,25 @@ def set_evidence_malicious_hash(self, file_info: Dict[str, any]): ) self.db.set_evidence(evidence) + + evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_DOWNLOADED_FILE, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=srcip), + timewindow=twid, + uid=[file_info['flow']["uid"]], + timestamp=ts, + category=IDEACategory.MALWARE + ) + + self.db.set_evidence(evidence) def circl_lu(self, flow_info: dict): """ @@ -825,7 +980,8 @@ def search_online_for_ip(self, ip): return spamhaus_res def ip_has_blacklisted_ASN( - self, ip, uid, timestamp, profileid, twid, ip_state + self, ip, uid, timestamp, profileid, twid, + is_dns_response: bool = False, ): """ Check if this ip has any of our blacklisted ASNs. @@ -853,6 +1009,7 @@ def ip_has_blacklisted_ASN( twid, asn, asn_info, + is_dns_response=is_dns_response, ) def ip_belongs_to_blacklisted_range( @@ -870,6 +1027,7 @@ def ip_belongs_to_blacklisted_range( ranges_starting_with_octet = self.cached_ipv6_ranges.get(first_octet, []) else: return False + for range in ranges_starting_with_octet: if ip_obj in ipaddress.ip_network(range): # ip was found in one of the blacklisted ranges @@ -911,10 +1069,18 @@ def is_malicious_ip(self, timestamp: str, profileid: str, twid: str, - ip_state: str) -> bool: + ip_state: str, + is_dns_response: bool=False, + dns_query: str=False + ) -> bool: """ Search for this IP in our database of IoC :param ip_state: is basically the answer to "which one is the + :param is_dns_response: set to true if the ip we're + looking up is a dns response + :param dns_query: is the dns query if the ip we're + looking up is a dns response + blacklisted IP"? can be 'srcip' or 'dstip' """ ip_info = self.search_offline_for_ip(ip) @@ -923,19 +1089,31 @@ def is_malicious_ip(self, if not ip_info: # not malicious return False + self.db.add_ips_to_IoC({ ip: json.dumps(ip_info) }) - self.set_evidence_malicious_ip( - ip, - uid, - daddr, - timestamp, - ip_info, - profileid, - twid, - ip_state, - ) + if is_dns_response: + self.set_evidence_malicious_ip_in_dns_response( + ip, + uid, + timestamp, + ip_info, + dns_query, + profileid, + twid, + ) + else: + self.set_evidence_malicious_ip( + ip, + uid, + daddr, + timestamp, + ip_info, + profileid, + twid, + ip_state, + ) return True def is_malicious_hash(self, flow_info: dict): @@ -944,9 +1122,12 @@ def is_malicious_hash(self, flow_info: dict): """ if not flow_info['flow']['md5']: # some lines in the zeek files.log doesn't have a hash for example - # {"ts":293.713187,"fuid":"FpvjEj3U0Qoj1fVCQc","tx_hosts":["94.127.78.125"],"rx_hosts":["10.0.2.19"], - # "conn_uids":["CY7bgw3KI8QyV67jqa","CZEkWx4wAvHJv0HTw9","CmM1ggccDvwnwPCl3","CBwoAH2RcIueFH4eu9","CZVfkc4BGLqRR7wwD5"], - # "source":"HTTP","depth":0,"analyzers":["SHA1","SHA256","MD5"] .. } + # {"ts":293.713187,"fuid":"FpvjEj3U0Qoj1fVCQc", + # "tx_hosts":["94.127.78.125"],"rx_hosts":["10.0.2.19"], + # "conn_uids":["CY7bgw3KI8QyV67jqa","CZEkWx4wAvHJv0HTw9", + # "CmM1ggccDvwnwPCl3","CBwoAH2RcIueFH4eu9","CZVfkc4BGLqRR7wwD5"], + # "source":"HTTP","depth":0,"analyzers":["SHA1","SHA256","MD5"] + # .. } return if blacklist_details := self.search_online_for_hash(flow_info): @@ -966,6 +1147,7 @@ def is_malicious_url( url, uid, timestamp, + daddr, profileid, twid ): @@ -974,14 +1156,122 @@ def is_malicious_url( if not url_info: # not malicious return False - self.set_evidence_malicious_url( + + self.urlhaus.set_evidence_malicious_url( + daddr, url_info, uid, timestamp, profileid, twid ) + + def set_evidence_malicious_cname_in_dns_response(self, + cname: str, + dns_query: str, + uid: str, + timestamp: str, + cname_info: dict, + is_subdomain: bool, + profileid: str = '', + twid: str = '' + ): + """ + :param cname: the dns answer that we looked up and turned out to be + malicious + :param dns_query: the query we asked the DNS server for when the + server returned the given cname + """ + if not cname_info: + return + + srcip = profileid.split("_")[-1] + # in case of finding a subdomain in our blacklists + # print that in the description of the alert and change the + # confidence accordingly in case of a domain, confidence=1 + confidence: float = 0.7 if is_subdomain else 1 + + # when we comment ti_files and run slips, we + # get the error of not being able to get feed threat_level + threat_level: float = utils.threat_levels[ + cname_info.get('threat_level', 'high') + ] + threat_level: ThreatLevel = ThreatLevel(threat_level) + description: str = (f'blacklisted CNAME: {cname} when resolving ' + f'{dns_query}' + f'Description: {cname_info.get("description", "")},' + f'Found in feed: {cname_info["source"]}, ' + f'Confidence: {confidence}. ') + + tags = cname_info.get('tags', None) + if tags: + description += f'with tags: {tags}. ' + + attacker = Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=srcip + ) + + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_BLACKLISTED_DNS_ANSWER, + attacker=attacker, + threat_level=threat_level, + confidence=confidence, + description=description, + profile=ProfileID(ip=srcip), + timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + uid=[uid], + timestamp=utils.convert_format(timestamp, utils.alerts_format), + category=IDEACategory.ANOMALY_TRAFFIC, + source_target_tag=Tag.BLACKLISTED_DOMAIN, + ) + self.db.set_evidence(evidence) + + + + def is_malicious_cname(self, + dns_query, + cname, + uid, + timestamp, + profileid, + twid, + ): + """ + looks up the given CNAME + :param cname: is the dns answer we're looking up + :param dns_query: the query we asked the DNS server for when the + server returned the given cname + """ + + if self.is_ignored_domain(cname): + return False + + domain_info, is_subdomain = self.search_offline_for_domain(cname) + if not domain_info: + return False + + self.set_evidence_malicious_cname_in_dns_response( + cname, + dns_query, + uid, + timestamp, + domain_info, + is_subdomain, + profileid, + twid, + ) + # mark this domain as malicious in our database + domain_info = { + 'threatintelligence': domain_info + } + self.db.set_info_for_domains(cname, domain_info) + + # add this domain to our MaliciousDomains hash in the database + self.db.set_malicious_domain(cname, profileid, twid) + def is_malicious_domain( self, @@ -989,7 +1279,7 @@ def is_malicious_domain( uid, timestamp, profileid, - twid + twid, ): if self.is_ignored_domain(domain): return False @@ -997,7 +1287,7 @@ def is_malicious_domain( domain_info, is_subdomain = self.search_offline_for_domain(domain) if not domain_info: return False - + self.set_evidence_malicious_domain( domain, uid, @@ -1012,20 +1302,17 @@ def is_malicious_domain( domain_info = { 'threatintelligence': domain_info } - self.db.setInfoForDomains( - domain, domain_info - ) + self.db.set_info_for_domains(domain, domain_info) # add this domain to our MaliciousDomains hash in the database - self.db.set_malicious_domain( - domain, profileid, twid - ) + self.db.set_malicious_domain(domain, profileid, twid) def update_local_file(self, filename): """ Updates the given local ti file if the hash of it has changed - : param filename: local ti file, has to be plased in config/local_ti_files/ dir + : param filename: local ti file, has to be plased in + config/local_ti_files/ dir """ fullpath = os.path.join(self.path_to_local_ti_files, filename) if filehash := self.should_update_local_ti_file(fullpath): @@ -1057,13 +1344,17 @@ def pre_main(self): self.update_local_file(local_file) self.circllu_calls_thread.start() - + + def should_lookup(self, ip: str, protocol: str, ip_state: str) \ + -> bool: + """return whther slips should lookup the given ip or notd""" + return (utils.is_ignored_ip(ip) or + self.is_outgoing_icmp_packet(protocol, ip_state)) + def main(self): - # The channel now can receive an IP address or a domain name + # The channel can receive an IP address or a domain name if msg:= self.get_msg('give_threat_intelligence'): - # Data is sent in the channel as a json dict so we need to deserialize it first data = json.loads(msg['data']) - # Extract data from dict profileid = data.get('profileid') twid = data.get('twid') timestamp = data.get('stime') @@ -1073,11 +1364,12 @@ def main(self): # these 2 are only available when looking up dns answers # the query is needed when a malicious answer is found, # for more detailed description of the evidence - self.is_dns_response = data.get('is_dns_response') - self.dns_query = data.get('dns_query') + is_dns_response = data.get('is_dns_response') + dns_query = data.get('dns_query') # IP is the IP that we want the TI for. It can be a SRC or DST IP to_lookup = data.get('to_lookup', '') - # detect the type given because sometimes, http.log host field has ips OR domains + # detect the type given because sometimes, + # http.log host field has ips OR domains type_ = utils.detect_data_type(to_lookup) # ip_state will say if it is a srcip or if it was a dst_ip @@ -1085,39 +1377,51 @@ def main(self): # If given an IP, ask for it # Block only if the traffic isn't outgoing ICMP port unreachable packet + if type_ == 'ip': ip = to_lookup - if not ( - utils.is_ignored_ip(ip) - or self.is_outgoing_icmp_packet(protocol, ip_state) - ): + if not self.should_lookup(ip, protocol, ip_state): self.is_malicious_ip( - ip, uid, daddr, timestamp, profileid, twid, ip_state + ip, uid, daddr, timestamp, profileid, twid, + ip_state, + dns_query=dns_query, + is_dns_response=is_dns_response, ) self.ip_belongs_to_blacklisted_range( ip, uid, daddr, timestamp, profileid, twid, ip_state ) self.ip_has_blacklisted_ASN( - ip, uid, timestamp, profileid, twid, ip_state + ip, uid, timestamp, profileid, twid, + is_dns_response=is_dns_response ) elif type_ == 'domain': - self.is_malicious_domain( - to_lookup, - uid, - timestamp, - profileid, - twid - ) + if is_dns_response: + self.is_malicious_cname( + dns_query, + to_lookup, + uid, + timestamp, + profileid, + twid) + else: + self.is_malicious_domain( + to_lookup, + uid, + timestamp, + profileid, + twid + ) elif type_ == 'url': self.is_malicious_url( to_lookup, uid, timestamp, + daddr, profileid, twid ) - if msg:= self.get_msg('new_downloaded_file'): + if msg := self.get_msg('new_downloaded_file'): file_info: dict = json.loads(msg['data']) # the format of file_info is as follows # { diff --git a/modules/threat_intelligence/urlhaus.py b/modules/threat_intelligence/urlhaus.py index 58f81b1ab..89c35076d 100644 --- a/modules/threat_intelligence/urlhaus.py +++ b/modules/threat_intelligence/urlhaus.py @@ -80,7 +80,8 @@ def parse_urlhaus_url_response(self, response, url): threat_level = virustotal_percent # virustotal_result = virustotal_info.get("result", "") # virustotal_result.replace('\',''') - description += f'and was marked by {virustotal_percent}% of virustotal\'s AVs as malicious' + description += (f'and was marked by {virustotal_percent}% ' + f'of virustotal\'s AVs as malicious') except (KeyError, IndexError): # no payloads available @@ -133,7 +134,10 @@ def urlhaus_lookup(self, ioc, type_of_ioc: str): if urlhaus_api_response.status_code != 200: return - response: dict = json.loads(urlhaus_api_response.text) + try: + response: dict = json.loads(urlhaus_api_response.text) + except json.decoder.JSONDecodeError: + return if response['query_status'] in ['no_results', 'invalid_url']: # no response or empty response @@ -145,7 +149,6 @@ def urlhaus_lookup(self, ioc, type_of_ioc: str): return self.parse_urlhaus_url_response(response, ioc) def set_evidence_malicious_hash(self, file_info: Dict[str, Any]) -> None: - flow: Dict[str, Any] = file_info['flow'] daddr: str = flow["daddr"] @@ -174,47 +177,77 @@ def set_evidence_malicious_hash(self, file_info: Dict[str, Any]) -> None: f" by URLhaus." ) - threat_level: float = file_info.get("threat_level", 0) + threat_level: float = file_info.get("threat_level") if threat_level: # Threat level here is the VT percentage from URLhaus description += f" Virustotal score: {threat_level}% malicious" threat_level: str = utils.threat_level_to_string(float( threat_level) / 100) + threat_level: ThreatLevel = ThreatLevel[threat_level.upper()] else: - threat_level = 'high' - - threat_level: ThreatLevel= ThreatLevel[threat_level.upper()] + threat_level: ThreatLevel = ThreatLevel.HIGH confidence: float = 0.7 saddr: str = file_info['profileid'].split("_")[-1] - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr - ) timestamp: str = flow["starttime"] - twid: str = file_info["twid"] - - # Assuming you have an instance of the Evidence class in your class + twid_int = int(file_info["twid"].replace("timewindow", "")) evidence = Evidence( evidence_type=EvidenceType.MALICIOUS_DOWNLOADED_FILE, - attacker=attacker, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), threat_level=threat_level, confidence=confidence, description=description, timestamp=timestamp, category=IDEACategory.MALWARE, profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_int), uid=[flow["uid"]] ) self.db.set_evidence(evidence) - + + evidence = Evidence( + evidence_type=EvidenceType.MALICIOUS_DOWNLOADED_FILE, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), + threat_level=threat_level, + confidence=confidence, + description=description, + timestamp=timestamp, + category=IDEACategory.MALWARE, + profile=ProfileID(ip=daddr), + timewindow=TimeWindow(number=twid_int), + uid=[flow["uid"]] + ) + + self.db.set_evidence(evidence) + + def get_threat_level(self, url_info: dict) -> ThreatLevel: + threat_level = url_info.get('threat_level', '') + if not threat_level: + return ThreatLevel.MEDIUM + + # Convert percentage reported by URLhaus (VirusTotal) to + # a valid SLIPS confidence + try: + threat_level = int(threat_level) / 100 + threat_level: str = utils.threat_level_to_string(threat_level) + return ThreatLevel[threat_level.upper()] + except ValueError: + return ThreatLevel.MEDIUM + def set_evidence_malicious_url( self, + daddr: str, url_info: Dict[str, Any], uid: str, timestamp: str, @@ -224,42 +257,42 @@ def set_evidence_malicious_url( """ Set evidence for a malicious URL based on the provided URL info """ - threat_level: str = url_info.get('threat_level', '') + threat_level: ThreatLevel = self.get_threat_level(url_info) description: str = url_info.get('description', '') - - confidence: float = 0.7 - - if not threat_level: - threat_level = 'medium' - else: - # Convert percentage reported by URLhaus (VirusTotal) to - # a valid SLIPS confidence - try: - threat_level = int(threat_level) / 100 - threat_level = utils.threat_level_to_string(threat_level) - except ValueError: - threat_level = 'medium' - - threat_level: ThreatLevel = ThreatLevel[threat_level.upper()] saddr: str = profileid.split("_")[-1] - - attacker: Attacker = Attacker( - direction=Direction.SRC, - attacker_type=IoCType.IP, - value=saddr + twid_int = int(twid.replace("timewindow", "")) + evidence = Evidence( + evidence_type=EvidenceType.THREAT_INTELLIGENCE_MALICIOUS_URL, + attacker=Attacker( + direction=Direction.SRC, + attacker_type=IoCType.IP, + value=saddr + ), + threat_level=threat_level, + confidence=0.7, + description=description, + timestamp=timestamp, + category=IDEACategory.MALWARE, + profile=ProfileID(ip=saddr), + timewindow=TimeWindow(number=twid_int), + uid=[uid] ) + self.db.set_evidence(evidence) - # Assuming you have an instance of the Evidence class in your class evidence = Evidence( - evidence_type=EvidenceType.MALICIOUS_URL, - attacker=attacker, + evidence_type=EvidenceType.THREAT_INTELLIGENCE_MALICIOUS_URL, + attacker=Attacker( + direction=Direction.DST, + attacker_type=IoCType.IP, + value=daddr + ), threat_level=threat_level, - confidence=confidence, + confidence=0.7, description=description, timestamp=timestamp, category=IDEACategory.MALWARE, profile=ProfileID(ip=saddr), - timewindow=TimeWindow(number=int(twid.replace("timewindow", ""))), + timewindow=TimeWindow(number=twid_int), uid=[uid] ) diff --git a/modules/timeline/timeline.py b/modules/timeline/timeline.py index 591cac9f9..e1d73b166 100644 --- a/modules/timeline/timeline.py +++ b/modules/timeline/timeline.py @@ -358,7 +358,7 @@ def process_flow(self, profileid, twid, flow, timestamp: float): self.print( f'Problem on process_flow() line {exception_line}', 0, 1 ) - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) return True def pre_main(self): diff --git a/modules/update_manager/update_manager.py b/modules/update_manager/update_manager.py index 52dcbc2e3..ed9876a53 100644 --- a/modules/update_manager/update_manager.py +++ b/modules/update_manager/update_manager.py @@ -426,7 +426,7 @@ def check_if_update(self, file_to_download: str, update_period) -> bool: except Exception: exception_line = sys.exc_info()[2].tb_lineno self.print(f"Problem on update_TI_file() line {exception_line}", 0, 1) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) return False def get_e_tag(self, response): @@ -635,7 +635,7 @@ async def update_TI_file(self, link_to_download: str) -> bool: except Exception: exception_line = sys.exc_info()[2].tb_lineno self.print(f"Problem on update_TI_file() line {exception_line}", 0, 1) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) return False def update_riskiq_feed(self): @@ -865,7 +865,7 @@ def parse_ja3_feed(self, url, ja3_feed_path: str) -> bool: except Exception: self.print("Problem in parse_ja3_feed()", 0, 1) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) return False def parse_json_ti_feed(self, link_to_download, ti_file_path: str) -> bool: @@ -1347,7 +1347,7 @@ def parse_ti_feed(self, link_to_download, ti_file_path: str) -> bool: 0, 1, ) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) return False def check_if_update_org(self, file): @@ -1509,18 +1509,22 @@ async def update(self) -> bool: # every function call to update_TI_file is now running concurrently instead of serially # so when a server's taking a while to give us the TI feed, we proceed # to download the next file instead of being idle - task = asyncio.create_task(self.update_TI_file(file_to_download)) + task = asyncio.create_task( + self.update_TI_file(file_to_download) + ) ####################################################### # in case of riskiq files, we don't have a link for them in ti_files, We update these files using their API # check if we have a username and api key and a week has passed since we last updated - if self.check_if_update("riskiq_domains", self.riskiq_update_period): + if self.check_if_update("riskiq_domains", + self.riskiq_update_period): self.update_riskiq_feed() # wait for all TI files to update try: await task except UnboundLocalError: - # in case all our files are updated, we don't have task defined, skip + # in case all our files are updated, we don't + # have task defined, skip pass self.db.set_loaded_ti_files(self.loaded_ti_files) @@ -1533,10 +1537,12 @@ async def update_ti_files(self): """ Update TI files and store them in database before slips starts """ - # create_task is used to run update() function concurrently instead of serially + # create_task is used to run update() function + # concurrently instead of serially self.update_finished = asyncio.create_task(self.update()) await self.update_finished - self.print(f"{self.db.get_loaded_ti_files()} TI files successfully loaded.") + self.print(f"{self.db.get_loaded_ti_files()} " + f"TI files successfully loaded.") def shutdown_gracefully(self): # terminating the timer for the process to be killed diff --git a/modules/virustotal/virustotal.py b/modules/virustotal/virustotal.py index 466989e58..55df97aa7 100644 --- a/modules/virustotal/virustotal.py +++ b/modules/virustotal/virustotal.py @@ -165,7 +165,7 @@ def set_url_data_in_URLInfo(self, url, cached_data): # Score of this url didn't change vtdata = {'URL': score, 'timestamp': time.time()} data = {'VirusTotal': vtdata} - self.db.setInfoForURLs(url, data) + self.db.set_info_for_urls(url, data) def set_domain_data_in_DomainInfo(self, domain, cached_data): """ @@ -188,7 +188,7 @@ def set_domain_data_in_DomainInfo(self, domain, cached_data): data['asn'] = { 'number': f'AS{as_owner}' } - self.db.setInfoForDomains(domain, data) + self.db.set_info_for_domains(domain, data) def API_calls_thread(self): """ @@ -226,7 +226,7 @@ def API_calls_thread(self): self.set_vt_data_in_IPInfo(ioc, cached_data) elif ioc_type == 'domain': - cached_data = self.db.getDomainData(ioc) + cached_data = self.db.get_domain_data(ioc) if not cached_data or 'VirusTotal' not in cached_data: self.set_domain_data_in_DomainInfo(ioc, cached_data) @@ -286,7 +286,7 @@ def get_ip_vt_data(self, ip: str): self.print( f'Problem in the get_ip_vt_data() line {exception_line}', 0, 1 ) - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) def get_domain_vt_data(self, domain: str): """ @@ -313,7 +313,7 @@ def get_domain_vt_data(self, domain: str): f'Problem in the get_domain_vt_data() ' f'line {exception_line}',0,1, ) - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) return False def get_ioc_type(self, ioc): @@ -601,7 +601,7 @@ def main(self): ) # this is a dict {'uid':json flow data} domain = flow_data.get('query', False) - cached_data = self.db.getDomainData(domain) + cached_data = self.db.get_domain_data(domain) # If VT data of this domain is not in the DomainInfo, ask VT # If 'Virustotal' key is not in the DomainInfo if domain and ( diff --git a/slips/main.py b/slips/main.py index 570141a1a..7534b3b8a 100644 --- a/slips/main.py +++ b/slips/main.py @@ -37,7 +37,6 @@ def __init__(self, testing=False): self.redis_man = RedisManager(self) self.ui_man = UIManager(self) self.metadata_man = MetadataManager(self) - self.proc_man = ProcessManager(self) self.conf = ConfigParser() self.version = self.get_slips_version() # will be filled later @@ -45,6 +44,7 @@ def __init__(self, testing=False): self.branch = "None" self.last_updated_stats_time = datetime.now() self.input_type = False + self.proc_man = ProcessManager(self) # in testing mode we manually set the following params if not testing: self.args = self.conf.get_args() @@ -498,11 +498,11 @@ def update_stats(self): self.last_updated_stats_time = now now = utils.convert_format(now, "%Y/%m/%d %H:%M:%S") modified_ips_in_the_last_tw = self.db.get_modified_ips_in_the_last_tw() - profilesLen = self.db.get_profiles_len() + profiles_len = self.db.get_profiles_len() evidence_number = self.db.get_evidence_number() or 0 msg = ( f"Total analyzed IPs so far: " - f"{green(profilesLen)}. " + f"{green(profiles_len)}. " f"Evidence Added: {green(evidence_number)}. " f"IPs sending traffic in the last " f"{self.twid_width}: {green(modified_ips_in_the_last_tw)}. " @@ -510,16 +510,16 @@ def update_stats(self): ) self.print(msg) - def update_host_ip(self, hostIP: str, modified_profiles: Set[str]) -> str: + def update_host_ip(self, host_ip: str, modified_profiles: Set[str]) -> str: """ when running on an interface we keep track of the host IP. If there was no modified TWs in the host IP, we check if the network was changed. """ - if self.is_interface and hostIP not in modified_profiles: - if hostIP := self.metadata_man.get_host_ip(): - self.db.set_host_ip(hostIP) - return hostIP + if self.is_interface and host_ip not in modified_profiles: + if host_ip := self.metadata_man.get_host_ip(): + self.db.set_host_ip(host_ip) + return host_ip def is_total_flows_unknown(self) -> bool: """ @@ -554,6 +554,7 @@ def start(self): current_stdout, stderr, slips_logfile ) self.add_observer(self.logger) + # get the port that is going to be used for this instance of slips if self.args.port: @@ -580,6 +581,17 @@ def start(self): "branch": self.branch, } ) + self.print( + f"Using redis server on " f"port: " + f"{green(self.redis_port)}", 1, 0 + ) + self.print( + f'Started {green("Main")} process ' f"[PID" + f" {green(self.pid)}]", 1, 0 + ) + # start progress bar before all modules so it doesn't miss + # any prints in its queue and slips wouldn't seem like it's frozen + self.proc_man.start_progress_bar() self.cpu_profiler_init() self.memory_profiler_init() @@ -593,7 +605,7 @@ def start(self): ) else: self.print( - f"Running on a growing zeek dir:" f" {self.input_information}" + f"Running on a growing zeek dir: {self.input_information}" ) self.db.set_growing_zeek_dir() @@ -620,13 +632,7 @@ def start(self): self.db.store_std_file(**std_files) - self.print( - f"Using redis server on " f"port: {green(self.redis_port)}", 1, 0 - ) - self.print( - f'Started {green("Main")} process ' f"[PID {green(self.pid)}]", 1, 0 - ) - self.print("Starting modules", 1, 0) + # if slips is given a .rdb file, don't load the # modules as we don't need them @@ -638,7 +644,11 @@ def start(self): self.proc_man.start_update_manager( local_files=True, TI_feeds=self.conf.wait_for_TI_to_finish() ) + self.print("Starting modules",1, 0) self.proc_man.load_modules() + # give outputprocess time to print all the started modules + time.sleep(0.5) + self.proc_man.print_disabled_modules() if self.args.webinterface: self.ui_man.start_webinterface() @@ -663,7 +673,7 @@ def sig_handler(sig, frame): # obtain the list of active processes self.proc_man.processes = multiprocessing.active_children() - self.db.store_process_PID("slips.py", int(self.pid)) + self.db.store_pid("slips.py", int(self.pid)) self.metadata_man.set_input_metadata() if self.conf.use_p2p() and not self.args.interface: @@ -686,7 +696,7 @@ def sig_handler(sig, frame): "of traffic by querying TI sites." ) - hostIP = self.metadata_man.store_host_ip() + host_ip = self.metadata_man.store_host_ip() # Don't try to stop slips if it's capturing from # an interface or a growing zeek dir @@ -694,10 +704,7 @@ def sig_handler(sig, frame): self.args.interface or self.db.is_growing_zeek_dir() ) - while ( - not self.proc_man.should_stop() - and not self.proc_man.slips_is_done_receiving_new_flows() - ): + while not self.proc_man.stop_slips(): # Sleep some time to do routine checks and give time for # more traffic to come time.sleep(5) @@ -705,26 +712,18 @@ def sig_handler(sig, frame): # if you remove the below logic anywhere before the # above sleep() statement, it will try to get the return # value very quickly before - # the webinterface thread sets it. so don't + # the webinterface thread sets it. so don't:D self.ui_man.check_if_webinterface_started() - # update the text we show in the cli self.update_stats() - # Check if we need to close any TWs - self.db.check_TW_to_close() + self.db.check_tw_to_close() modified_profiles: Set[str] = ( - self.metadata_man.update_slips_running_stats()[1] + self.metadata_man.update_slips_stats_in_the_db()[1] ) - hostIP: str = self.update_host_ip(hostIP, modified_profiles) - - # don't move this line up because we still need to print the - # stats and check tws anyway - if self.proc_man.should_run_non_stop(): - continue - - self.db.check_health() + + self.update_host_ip(host_ip, modified_profiles) except KeyboardInterrupt: # the EINTR error code happens if a signal occurred while diff --git a/slips_files/common/abstracts/_module.py b/slips_files/common/abstracts/_module.py index 1b0ba2d0c..bbc817386 100644 --- a/slips_files/common/abstracts/_module.py +++ b/slips_files/common/abstracts/_module.py @@ -134,7 +134,7 @@ def run(self): except Exception: exception_line = sys.exc_info()[2].tb_lineno self.print(f'Problem in pre_main() line {exception_line}', 0, 1) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) return True try: @@ -150,10 +150,6 @@ def run(self): except KeyboardInterrupt: self.shutdown_gracefully() except Exception: - exception_line = sys.exc_info()[2].tb_lineno - self.print(f'Problem in {self.name}\'s main() ' - f'line {exception_line}', - 0, 1) - traceback.print_stack() - + self.print(f'Problem in {self.name}',0, 1) + self.print(traceback.format_exc(), 0, 1) return True diff --git a/slips_files/common/abstracts/core.py b/slips_files/common/abstracts/core.py index 2d36f2418..31f324eed 100644 --- a/slips_files/common/abstracts/core.py +++ b/slips_files/common/abstracts/core.py @@ -56,9 +56,7 @@ def run(self): except KeyboardInterrupt: self.shutdown_gracefully() except Exception: - exception_line = sys.exc_info()[2].tb_lineno - self.print(f'Problem in main() line {exception_line}', 0, 1) - self.print(traceback.print_stack(), 0, 1) - + self.print(f'Problem in {self.name}',0, 1) + self.print(traceback.format_exc(), 0, 1) return True diff --git a/slips_files/common/idea_format.py b/slips_files/common/idea_format.py index c1d796af8..fa3087367 100644 --- a/slips_files/common/idea_format.py +++ b/slips_files/common/idea_format.py @@ -163,4 +163,4 @@ def idea_format(evidence: Evidence): return idea_dict except Exception as e: print(f"Error in idea_format(): {e}") - print(traceback.print_stack()) + print(traceback.format_exc()) diff --git a/slips_files/common/parsers/config_parser.py b/slips_files/common/parsers/config_parser.py index 91c3deabf..ac4da2ba7 100644 --- a/slips_files/common/parsers/config_parser.py +++ b/slips_files/common/parsers/config_parser.py @@ -1,6 +1,7 @@ from datetime import timedelta import sys import ipaddress +from typing import List import configparser from slips_files.common.parsers.arg_parser import ArgumentParser from slips_files.common.slips_utils import utils @@ -607,7 +608,25 @@ def rotation_period(self): 'parameters', 'rotation_period', '1 day' ) return utils.sanitize(rotation_period) - + + + def client_ips(self) -> List[str]: + client_ips: str = self.read_configuration( + 'parameters', 'client_ips', '[]' + ) + client_ips: str = utils.sanitize(client_ips) + client_ips: List[str] = (client_ips + .replace('[', '') + .replace(']', '') + .split(",") + ) + client_ips: List[str] = [client_ip.strip().strip("'") for client_ip + in client_ips] + # Remove empty strings if any + client_ips: List[str] = [client_ip for client_ip in client_ips if + client_ip] + return client_ips + def keep_rotated_files_for(self) -> int: """ returns period in seconds""" keep_rotated_files_for = self.read_configuration( diff --git a/slips_files/common/slips_utils.py b/slips_files/common/slips_utils.py index 4c9bf8c65..4580a1796 100644 --- a/slips_files/common/slips_utils.py +++ b/slips_files/common/slips_utils.py @@ -11,9 +11,10 @@ import sys import ipaddress import aid_hash -from typing import Any, Union -from dataclasses import is_dataclass, asdict, fields -from enum import Enum, auto +from typing import Any, \ + Optional +from dataclasses import is_dataclass, asdict +from enum import Enum IS_IN_A_DOCKER_CONTAINER = os.environ.get('IS_IN_A_DOCKER_CONTAINER', False) @@ -66,10 +67,10 @@ def __init__(self): self.local_tz = self.get_local_timezone() self.aid = aid_hash.AID() - def get_cidr_of_ip(self, ip): + def get_cidr_of_private_ip(self, ip): """ returns the cidr/range of the given private ip - :param ip: should be a private ips + :param ip: should be a private ipv4 """ if validators.ipv4(ip): first_octet = ip.split('.')[0] @@ -250,7 +251,7 @@ def convert_to_datetime(self, ts): ) - def define_time_format(self, time: str) -> str: + def define_time_format(self, time: str) -> Optional[str]: if self.is_datetime_obj(time): return 'datetimeobj' @@ -321,8 +322,22 @@ def get_own_IPs(self) -> list: def convert_to_mb(self, bytes): return int(bytes)/(10**6) - - def is_private_ip(self, ip_obj:ipaddress) -> bool: + + + def is_port_in_use(self, port: int) -> bool: + """ + return True if the given port is used by another app + """ + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if sock.connect_ex(("localhost", port)) != 0: + # not used + sock.close() + return False + + sock.close() + return True + + def is_private_ip(self, ip_obj: ipaddress) -> bool: """ This function replaces the ipaddress library 'is_private' because it does not work correctly and it does not ignore @@ -331,7 +346,8 @@ def is_private_ip(self, ip_obj:ipaddress) -> bool: # Is it a well-formed ipv4 or ipv6? r_value = False if ip_obj and ip_obj.is_private: - if ip_obj != ipaddress.ip_address('0.0.0.0') and ip_obj != ipaddress.ip_address('255.255.255.255'): + if (ip_obj != ipaddress.ip_address('0.0.0.0') + and ip_obj != ipaddress.ip_address('255.255.255.255')): r_value = True return r_value diff --git a/slips_files/core/database/database_manager.py b/slips_files/core/database/database_manager.py index 9689c2cfa..3bb2b5044 100644 --- a/slips_files/core/database/database_manager.py +++ b/slips_files/core/database/database_manager.py @@ -306,8 +306,8 @@ def set_asn_cache(self, *args, **kwargs): def get_asn_cache(self, *args, **kwargs): return self.rdb.get_asn_cache(*args, **kwargs) - def store_process_PID(self, *args, **kwargs): - return self.rdb.store_process_PID(*args, **kwargs) + def store_pid(self, *args, **kwargs): + return self.rdb.store_pid(*args, **kwargs) def get_pids(self, *args, **kwargs): return self.rdb.get_pids(*args, **kwargs) @@ -534,26 +534,26 @@ def getURLData(self, *args, **kwargs): def setNewURL(self, *args, **kwargs): return self.rdb.setNewURL(*args, **kwargs) - def getDomainData(self, *args, **kwargs): - return self.rdb.getDomainData(*args, **kwargs) + def get_domain_data(self, *args, **kwargs): + return self.rdb.get_domain_data(*args, **kwargs) - def setNewDomain(self, *args, **kwargs): - return self.rdb.setNewDomain(*args, **kwargs) + def set_new_domain(self, *args, **kwargs): + return self.rdb.set_new_domain(*args, **kwargs) - def setInfoForDomains(self, *args, **kwargs): - return self.rdb.setInfoForDomains(*args, **kwargs) + def set_info_for_domains(self, *args, **kwargs): + return self.rdb.set_info_for_domains(*args, **kwargs) - def setInfoForURLs(self, *args, **kwargs): - return self.rdb.setInfoForURLs(*args, **kwargs) + def set_info_for_urls(self, *args, **kwargs): + return self.rdb.set_info_for_urls(*args, **kwargs) def get_data_from_profile_tw(self, *args, **kwargs): return self.rdb.get_data_from_profile_tw(*args, **kwargs) - def getOutTuplesfromProfileTW(self, *args, **kwargs): - return self.rdb.getOutTuplesfromProfileTW(*args, **kwargs) + def get_outtuples_from_profile_tw(self, *args, **kwargs): + return self.rdb.get_outtuples_from_profile_tw(*args, **kwargs) - def getInTuplesfromProfileTW(self, *args, **kwargs): - return self.rdb.getInTuplesfromProfileTW(*args, **kwargs) + def get_intuples_from_profile_tw(self, *args, **kwargs): + return self.rdb.get_intuples_from_profile_tw(*args, **kwargs) def get_dhcp_flows(self, *args, **kwargs): return self.rdb.get_dhcp_flows(*args, **kwargs) @@ -573,8 +573,8 @@ def add_out_dns(self, *args, **kwargs): def add_port(self, *args, **kwargs): return self.rdb.add_port(*args, **kwargs) - def getFinalStateFromFlags(self, *args, **kwargs): - return self.rdb.getFinalStateFromFlags(*args, **kwargs) + def get_final_state_from_flags(self, *args, **kwargs): + return self.rdb.get_final_state_from_flags(*args, **kwargs) def add_ips(self, *args, **kwargs): return self.rdb.add_ips(*args, **kwargs) @@ -642,14 +642,14 @@ def getTWsfromProfile(self, *args, **kwargs): def get_number_of_tws_in_profile(self, *args, **kwargs): return self.rdb.get_number_of_tws_in_profile(*args, **kwargs) - def getSrcIPsfromProfileTW(self, *args, **kwargs): - return self.rdb.getSrcIPsfromProfileTW(*args, **kwargs) + def get_srcips_from_profile_tw(self, *args, **kwargs): + return self.rdb.get_srcips_from_profile_tw(*args, **kwargs) - def getDstIPsfromProfileTW(self, *args, **kwargs): - return self.rdb.getDstIPsfromProfileTW(*args, **kwargs) + def get_dstips_from_profile_tw(self, *args, **kwargs): + return self.rdb.get_dstips_from_profile_tw(*args, **kwargs) - def getT2ForProfileTW(self, *args, **kwargs): - return self.rdb.getT2ForProfileTW(*args, **kwargs) + def get_t2_for_profile_tw(self, *args, **kwargs): + return self.rdb.get_t2_for_profile_tw(*args, **kwargs) def has_profile(self, *args, **kwargs): return self.rdb.has_profile(*args, **kwargs) @@ -675,14 +675,14 @@ def add_new_tw(self, *args, **kwargs): def get_tw_start_time(self, *args, **kwargs): return self.rdb.get_tw_start_time(*args, **kwargs) - def getAmountTW(self, *args, **kwargs): - return self.rdb.getAmountTW(*args, **kwargs) + def get_number_of_tws(self, *args, **kwargs): + return self.rdb.get_number_of_tws(*args, **kwargs) - def getModifiedTWSinceTime(self, *args, **kwargs): - return self.rdb.getModifiedTWSinceTime(*args, **kwargs) + def get_modified_tw_since_time(self, *args, **kwargs): + return self.rdb.get_modified_tw_since_time(*args, **kwargs) - def getModifiedProfilesSince(self, *args, **kwargs): - return self.rdb.getModifiedProfilesSince(*args, **kwargs) + def get_modified_profiles_since(self, *args, **kwargs): + return self.rdb.get_modified_profiles_since(*args, **kwargs) def add_mac_addr_to_profile(self, *args, **kwargs): return self.rdb.add_mac_addr_to_profile(*args, **kwargs) @@ -714,17 +714,17 @@ def add_profile(self, *args, **kwargs): def set_profile_module_label(self, *args, **kwargs): return self.rdb.set_profile_module_label(*args, **kwargs) - def check_TW_to_close(self, *args, **kwargs): - return self.rdb.check_TW_to_close(*args, **kwargs) + def check_tw_to_close(self, *args, **kwargs): + return self.rdb.check_tw_to_close(*args, **kwargs) def check_health(self): self.rdb.pubsub.check_health() - def markProfileTWAsClosed(self, *args, **kwargs): - return self.rdb.markProfileTWAsClosed(*args, **kwargs) + def mark_profile_tw_as_closed(self, *args, **kwargs): + return self.rdb.mark_profile_tw_as_closed(*args, **kwargs) - def markProfileTWAsModified(self, *args, **kwargs): - return self.rdb.markProfileTWAsModified(*args, **kwargs) + def mark_profile_tw_as_modified(self, *args, **kwargs): + return self.rdb.mark_profile_tw_as_modified(*args, **kwargs) def add_tuple(self, *args, **kwargs): return self.rdb.add_tuple(*args, **kwargs) @@ -858,9 +858,6 @@ def delete(self, *args, **kwargs): def select(self, *args, **kwargs): return self.sqlite.select(*args, **kwargs) - def execute_query(self, *args, **kwargs): - return self.sqlite.execute_query(*args, **kwargs) - def get_pid_of(self, *args, **kwargs): return self.rdb.get_pid_of(*args, **kwargs) diff --git a/slips_files/core/database/redis_db/alert_handler.py b/slips_files/core/database/redis_db/alert_handler.py index 74aaae601..7d663fe97 100644 --- a/slips_files/core/database/redis_db/alert_handler.py +++ b/slips_files/core/database/redis_db/alert_handler.py @@ -167,20 +167,11 @@ def set_evidence(self, evidence: Evidence): # an evidence is generated for this profile # update the threat level of this profile - if evidence.attacker.direction == Direction.SRC: - # the srcip is the malicious one - self.update_threat_level( - str(evidence.profile), - str(evidence.threat_level), - evidence.confidence - ) - elif evidence.attacker.direction == Direction.DST: - # the dstip is the malicious one - self.update_threat_level( - str(evidence.attacker.profile), - str(evidence.threat_level), - evidence.confidence - ) + self.update_threat_level( + str(evidence.attacker.profile), + str(evidence.threat_level), + evidence.confidence + ) return True @@ -318,7 +309,7 @@ def update_max_threat_level( ) -> float: """ given the current threat level of a profileid, this method sets the - max_threaty_level value to the given val if that max is less than + max_threat_level value to the given val if that max is less than the given :returns: the numerical val of the max threat level """ @@ -341,33 +332,27 @@ def update_max_threat_level( return threat_level_float return old_max_threat_level_float - - - def update_threat_level( - self, profileid: str, threat_level: str, confidence: float - ): + + def update_past_threat_levels( + self, profileid, threat_level, confidence + ): """ - Update the threat level of a certain profile - Updates the profileid key and the IPsInfo key with the - new score and confidence of this profile - :param threat_level: available options are 'low', 'medium' 'critical' etc + updates the past_threat_levels key of the given profileid + if the past threat level and confidence + are the same as the ones we wanna store, we replace the timestamp only """ - - self.r.hset(profileid, 'threat_level', threat_level) - now = utils.convert_format(time.time(), utils.alerts_format) confidence = f'confidence: {confidence}' - # this is what we'll be storing in the db, tl, ts, and confidence threat_level_data = (threat_level, now, confidence) - past_threat_levels: List[Tuple] = self.r.hget( + past_threat_levels: str = self.r.hget( profileid, 'past_threat_levels' ) if past_threat_levels: # get the list of ts and past threat levels - past_threat_levels = json.loads(past_threat_levels) + past_threat_levels: List[Tuple] = json.loads(past_threat_levels) latest: Tuple = past_threat_levels[-1] latest_threat_level: str = latest[0] @@ -391,19 +376,16 @@ def update_threat_level( past_threat_levels = json.dumps(past_threat_levels) self.r.hset(profileid, 'past_threat_levels', past_threat_levels) - - max_threat_lvl: float = self.update_max_threat_level( - profileid, threat_level - ) - + + + def update_ips_info(self, profileid, max_threat_lvl, confidence): + # set the score and confidence of the given ip in the db + # when it causes an evidence + # these 2 values will be needed when sharing with peers score_confidence = { - # get the numerical value of this threat level 'score': max_threat_lvl, 'confidence': confidence } - # set the score and confidence of the given ip in the db - # when it causes an evidence - # these 2 values will be needed when sharing with peers ip = profileid.split('_')[-1] if cached_ip_info := self.get_ip_info(ip): @@ -412,4 +394,28 @@ def update_threat_level( score_confidence = cached_ip_info self.rcache.hset('IPsInfo', ip, json.dumps(score_confidence)) + + + def update_threat_level( + self, profileid: str, threat_level: str, confidence: float + ): + """ + Update the threat level of a certain profile + Updates the profileid key and the IPsInfo key with the + new score and confidence of this profile + Stores the max threat level of the given profile as the score + in IPsInfo + :param threat_level: available options are 'low', + 'medium' 'critical' etc + """ + + self.r.hset(profileid, 'threat_level', threat_level) + + self.update_past_threat_levels(profileid, threat_level, confidence) + + max_threat_lvl: float = self.update_max_threat_level( + profileid, threat_level + ) + + self.update_ips_info(profileid, max_threat_lvl, confidence) diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index e8d951cf7..06abbabb8 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -15,7 +15,8 @@ import ipaddress import sys import validators -from typing import List +from typing import List, \ + Dict RUNNING_IN_DOCKER = os.environ.get('IS_IN_A_DOCKER_CONTAINER', False) @@ -177,6 +178,7 @@ def _read_configuration(cls): cls.deletePrevdb: bool = conf.deletePrevdb() cls.disabled_detections: List[str] = conf.disabled_detections() cls.width = conf.get_tw_width_as_float() + cls.client_ips: List[str] = conf.client_ips() @classmethod def set_slips_internal_time(cls, timestamp): @@ -204,12 +206,6 @@ def start(cls) -> bool: if not cls.connect_to_redis_server(): return False - # Set the memory limits of the output buffer, For normal clients: no limits - # for pub-sub 4GB maximum buffer size - # and 2GB for soft limit - # The original values were 50MB for maxmem and 8MB for soft limit. - # don't flush the loaded db when using '-db' - # don't flush the db when starting or stopping the daemon, or when testing if ( cls.deletePrevdb and not ('-S' in sys.argv @@ -221,6 +217,10 @@ def start(cls) -> bool: # the PIDS to close slips files cls.r.flushdb() + # Set the memory limits of the output buffer, + # For normal clients: no limits + # for pub-sub 4GB maximum buffer size and 2GB for soft limit + # The original values were 50MB for maxmem and 8MB for soft limit. cls.change_redis_limits(cls.r) cls.change_redis_limits(cls.rcache) @@ -238,7 +238,7 @@ def start(cls) -> bool: @staticmethod def start_redis_instance(port: int, db: int) -> redis.StrictRedis: # set health_check_interval to avoid redis ConnectionReset errors: - # if the connection is idle for more than 30 seconds, + # if the connection is idle for more than health_check_interval seconds, # a round trip PING/PONG will be attempted before next redis cmd. # If the PING/PONG fails, the connection will re-established @@ -290,9 +290,10 @@ def close_redis_server(cls, redis_port): os.kill(int(server_pid), signal.SIGKILL) @classmethod - def change_redis_limits(cls, client): + def change_redis_limits(cls, client: redis.StrictRedis): """ - To fix redis closing/resetting the pub/sub connection, change redis soft and hard limits + changes redis soft and hard limits to fix redis closing/resetting + the pub/sub connection, """ # maximum buffer size for pub/sub clients: = 4294967296 Bytes = 4GBs, # when msgs in queue reach this limit, Redis will @@ -301,9 +302,11 @@ def change_redis_limits(cls, client): # soft limit for pub/sub clients: 2147483648 Bytes = 2GB over 10 mins, # means if the client has an output buffer bigger than 2GB # for, continuously, 10 mins, the connection gets closed. - client.config_set('client-output-buffer-limit', "normal 0 0 0 " - "slave 268435456 67108864 60 " - "pubsub 4294967296 2147483648 600") + # format is client hard_limit soft_limit + client.config_set('client-output-buffer-limit', + "normal 0 0 0 " + "slave 268435456 67108864 60 " + "pubsub 4294967296 2147483648 600") @classmethod def _set_slips_start_time(cls): @@ -350,7 +353,9 @@ def get_message(self, channel, timeout=0.0000001): if self.connection_retry >= self.max_retries: self.publish_stop() - self.print(f'Stopping slips due to redis.exceptions.ConnectionError: {ex}', 1, 1) + self.print(f'Stopping slips due to ' + f'redis.exceptions.ConnectionError: {ex}', + 1, 1) else: # don't log this each retry if self.connection_retry % 10 == 0: @@ -473,32 +478,21 @@ def get_equivalent_tws(self, hrs: float): for example if the tw width is 1h, and hrs is 24, this function returns 24 """ return int(hrs*3600/self.width) + + + + def set_local_network(self, cidr): + """ + set the local network used in the db + """ + self.r.set("local_network", cidr) - def set_local_network(self, saddr): - # set the local network used in the db - # For now the local network is only ipv4, but it could be ipv6 in the future. Todo. - - if self.is_localnet_set: - return - - if saddr in ('0.0.0.0', '255.255.255.255'): - return - - if not ( - validators.ipv4(saddr) - and utils.is_private_ip(ipaddress.ip_address(saddr)) - ): - return - # get the local network of this saddr - if network_range := utils.get_cidr_of_ip(saddr): - self.r.set("local_network", network_range) - self.is_localnet_set = True + def get_local_network(self): + return self.r.get("local_network") def get_used_port(self): return int(self.r.config_get('port')['port']) - def get_local_network(self): - return self.r.get("local_network") def get_label_count(self, label): """ @@ -608,18 +602,22 @@ def get_p2p_reports_about_ip(self, ip) -> dict: def store_p2p_report(self, ip: str, report_data: dict): """ stores answers about IPs slips asked other peers for. + updates the p2p_reports key only """ - # reports in the db are sorted by reporter bydefault + # reports in the db are sorted by reporter by default reporter = report_data['reporter'] del report_data['reporter'] # if we have old reports about this ip, append this one to them # cached_p2p_reports is a dict - if cached_p2p_reports := self.get_p2p_reports_about_ip(ip): + cached_p2p_reports: Dict[str, List[dict]] = ( + self.get_p2p_reports_about_ip(ip)) + if cached_p2p_reports: # was this ip reported by the same peer before? if reporter in cached_p2p_reports: # ip was reported before, by the same peer - # did the same peer report the same score and confidence about the same ip twice in a row? + # did the same peer report the same score and + # confidence about the same ip twice in a row? last_report_about_this_ip = cached_p2p_reports[reporter][-1] score = report_data['score'] confidence = report_data['confidence'] @@ -628,10 +626,12 @@ def store_p2p_report(self, ip: str, report_data: dict): and last_report_about_this_ip['confidence'] == confidence ): report_time = report_data['report_time'] - # score and confidence are the same as the last report, only update the time + # score and confidence are the same as the last report, + # only update the time last_report_about_this_ip['report_time'] = report_time else: - # score and confidence are the different from the last report, add report to the list + # score and confidence are the different from the last + # report, add report to the list cached_p2p_reports[reporter].append(report_data) else: # ip was reported before, but not by the same peer @@ -744,7 +744,8 @@ def set_dns_resolution( # get stored DNS resolution from our db ip_info_from_db = self.get_dns_resolution(answer) if ip_info_from_db == {}: - # if the domain(query) we have isn't already in DNSresolution in the db + # if the domain(query) we have isn't already in + # DNSresolution in the db resolved_by = [srcip] domains = [] timewindows = [profileid_twid] @@ -760,14 +761,17 @@ def set_dns_resolution( if profileid_twid not in timewindows: timewindows.append(profileid_twid) - # we'll be appending the current answer to these cached domains + # we'll be appending the current answer + # to these cached domains domains = ip_info_from_db.get('domains', []) - # if the domain(query) we have isn't already in DNSresolution in the db, add it + # if the domain(query) we have isn't already in + # DNSresolution in the db, add it if query not in domains: domains.append(query) - # domains should be a list, not a string!, so don't use json.dumps here + # domains should be a list, not a string!, + # so don't use json.dumps here ip_info = { 'ts': ts, 'uid': uid, @@ -797,7 +801,7 @@ def set_dns_resolution( # no CNAME came with this query pass - self.setInfoForDomains(query, domaindata, mode='add') + self.set_info_for_domains(query, domaindata, mode='add') self.set_domain_resolution(query, ips_to_add) def set_domain_resolution(self, domain, ips): @@ -842,7 +846,8 @@ def get_modified_ips_in_the_last_tw(self): this number is updated in the db every 5s by slips.py used for printing running stats in slips.py or outputprocess """ - if modified_ips := self.r.hget('analysis', 'modified_ips_in_the_last_tw'): + if modified_ips := self.r.hget('analysis', + 'modified_ips_in_the_last_tw'): return modified_ips else: return 0 @@ -852,7 +857,8 @@ def is_connection_error_logged(self): def mark_connection_error_as_logged(self): """ - When redis connection error occurs, to prevent every module from logging it to slips.log and the console, + When redis connection error occurs, to prevent + every module from logging it to slips.log and the console, set this variable in the db """ self.r.set('logged_connection_error', 'True') @@ -873,13 +879,15 @@ def mark_srcip_as_seen_in_connlog(self, ip): """ Marks the given ip as seen in conn.log keeps track of private ipv4 only. - if an ip is not present in this set, it means we may have seen it but not in conn.log + if an ip is not present in this set, it means we may + have seen it but not in conn.log """ self.r.sadd("srcips_seen_in_connlog", ip) def is_gw_mac(self, mac_addr: str, ip: str) -> bool: """ - Detects the MAC of the gateway if 1 mac is seen assigned to 1 public destination IP + Detects the MAC of the gateway if 1 mac is seen + assigned to 1 public destination IP :param ip: dst ip that should be associated with the given MAC info """ @@ -890,7 +898,8 @@ def is_gw_mac(self, mac_addr: str, ip: str) -> bool: # gateway MAC already set using this function return self.get_gateway_mac() == mac_addr - # since we don't have a mac gw in the db, see eif this given mac is the gw mac + # since we don't have a mac gw in the db, see if + # this given mac is the gw mac ip_obj = ipaddress.ip_address(ip) if not utils.is_private_ip(ip_obj): # now we're given a public ip and a MAC that's supposedly belongs to it @@ -930,7 +939,8 @@ def is_whitelisted_tranco_domain(self, domain): def set_growing_zeek_dir(self): """ - Mark a dir as growing so it can be treated like the zeek logs generated by an interface + Mark a dir as growing so it can be treated like the zeek + logs generated by an interface """ self.r.set('growing_zeek_dir', 'yes') @@ -938,50 +948,42 @@ def is_growing_zeek_dir(self): """ Did slips mark the given dir as growing?""" return 'yes' in str(self.r.get('growing_zeek_dir')) - def get_ip_identification(self, ip: str, get_ti_data=True): + def get_ip_identification(self, ip: str, get_ti_data=True) -> str: """ Return the identification of this IP based - on the data stored so far + on the AS, rDNS, and SNI of the IP. + + :param ip: The IP address to retrieve information for. :param get_ti_data: do we want to get info about this IP from out TI lists? + :return: string containing AS, rDNS, and SNI of the IP. """ - current_data = self.get_ip_info(ip) - identification = '' - if current_data: - if 'asn' in current_data: - asn_details = '' - if asnorg := current_data['asn'].get('org', ''): - asn_details += f'{asnorg} ' - - if number := current_data['asn'].get('number', ''): - asn_details += f'{number} ' - - if len(asn_details) > 1: - identification += f'AS: {asn_details}' - - if 'SNI' in current_data: - sni = current_data['SNI'] - if type(sni) == list: - sni = sni[0] - identification += 'SNI: ' + sni['server_name'] + ', ' - - if 'reverse_dns' in current_data: - identification += 'rDNS: ' + current_data['reverse_dns'] + ', ' - - if 'threatintelligence' in current_data and get_ti_data: - identification += ( - 'Description: ' - + current_data['threatintelligence']['description'] - + ', ' - ) - - tags: list = current_data['threatintelligence'].get('tags', False) - # remove brackets - if tags: - identification += f'tags= {tags} ' - - identification = identification[:-2] - return identification + ip_info = self.get_ip_info(ip) + id = '' + if not ip_info: + return id + + asn = ip_info.get('asn', '') + if asn: + asn_org = asn.get('org', '') + asn_number = asn.get('number', '') + id += f'AS: {asn_org} {asn_number}' + + sni = ip_info.get('SNI', '') + if sni: + sni = sni[0] if isinstance(sni, list) else sni + id += f'SNI: {sni["server_name"]}, ' + + rdns = ip_info.get('reverse_dns', '') + if rdns: + id += f'rDNS: {rdns}, ' + + threat_intel = ip_info.get('threatintelligence', '') + if threat_intel and get_ti_data: + id += f"IP seen in blacklist: {threat_intel['source']}." + id = id.rstrip(', ') + return id + def get_multiaddr(self): """ this is can only be called when p2p is enabled, this value is set by p2p pigeon @@ -1023,12 +1025,14 @@ def is_ftp_port(self, port): def set_organization_of_port(self, organization, ip: str, portproto: str): """ - Save in the DB a port with its organization and the ip/ range used by this organization + Save in the DB a port with its organization and the ip/ + range used by this organization :param portproto: portnumber + / + protocol.lower() :param ip: can be a single org ip, or a range or '' """ if org_info := self.get_organization_of_port(portproto): - # this port and proto was used with another organization, append to it + # this port and proto was used with another + # organization, append to it org_info = json.loads(org_info) org_info['ip'].append(ip) org_info['org_name'].append(organization) @@ -1078,7 +1082,7 @@ def set_default_gateway(self, address_type: str, address: str): self.r.hset('default_gateway', address_type, address) - def get_domain_resolution(self, domain): + def get_domain_resolution(self, domain) -> List[str]: """ Returns the IPs resolved by this domain """ @@ -1159,7 +1163,8 @@ def set_asn_cache(self, org: str, asn_range: str, asn_number: str) -> None: } """ if cached_asn := self.get_asn_cache(first_octet=first_octet): - # we already have a cached asn of a range that starts with the same first octet + # we already have a cached asn of a range that + # starts with the same first octet cached_asn: dict = json.loads(cached_asn) cached_asn.update(range_info) self.rcache.hset('cached_asn', first_octet, json.dumps(cached_asn)) @@ -1177,7 +1182,7 @@ def get_asn_cache(self, first_octet=False): else: return self.rcache.hgetall('cached_asn') - def store_process_PID(self, process, pid): + def store_pid(self, process, pid): """ Stores each started process or module with it's PID :param pid: int @@ -1203,17 +1208,20 @@ def get_name_of_module_at(self, given_pid): def set_org_info(self, org, org_info, info_type): """ store ASN, IP and domains of an org in the db - :param org: supported orgs are ('google', 'microsoft', 'apple', 'facebook', 'twitter') + :param org: supported orgs are ('google', 'microsoft', + 'apple', 'facebook', 'twitter') : param org_info: a json serialized list of asns or ips or domains :param info_type: supported types are 'asn', 'domains', 'IPs' """ - # info will be stored in OrgInfo key {'facebook_asn': .., 'twitter_domains': ...} + # info will be stored in OrgInfo key {'facebook_asn': .., + # 'twitter_domains': ...} self.rcache.hset('OrgInfo', f'{org}_{info_type}', org_info) def get_org_info(self, org, info_type) -> str: """ get the ASN, IP and domains of an org from the db - :param org: supported orgs are ('google', 'microsoft', 'apple', 'facebook', 'twitter') + :param org: supported orgs are ('google', 'microsoft', 'apple', + 'facebook', 'twitter') :param info_type: supported types are 'asn', 'domains' " returns a json serialized dict with info """ @@ -1246,8 +1254,10 @@ def get_all_whitelist(self): def get_whitelist(self, key): """ - Whitelist supports different keys like : IPs domains and organizations - this function is used to check if we have any of the above keys whitelisted + Whitelist supports different keys like : IPs domains + and organizations + this function is used to check if we have any of the + above keys whitelisted """ if whitelist := self.r.hget('whitelist', key): return json.loads(whitelist) @@ -1294,7 +1304,8 @@ def save(self, backup_file): return True print( - f'[DB] Error Saving: Cannot find the redis database directory {redis_db_path}' + f'[DB] Error Saving: Cannot find the redis ' + f'database directory {redis_db_path}' ) return False @@ -1336,7 +1347,8 @@ def is_valid_rdb_file(): # Stop the server first in order for redis to load another db os.system(f'{self.sudo}service redis-server stop') - # Start the server again, but make sure it's flushed and doesnt have any keys + # Start the server again, but make sure it's flushed + # and doesnt have any keys os.system('redis-server redis.conf > /dev/null 2>&1') return True except Exception: @@ -1383,12 +1395,15 @@ def end_profiling(profile): def store_blame_report(self, ip, network_evaluation): """ - :param network_evaluation: a dict with {'score': ..,'confidence': .., 'ts': ..} taken from a blame report + :param network_evaluation: a dict with {'score': .., + 'confidence': + .., 'ts': ..} taken from a blame report """ self.rcache.hset('p2p-received-blame-reports', ip, network_evaluation) def store_zeek_path(self, path): - """used to store the path of zeek log files slips is currently using""" + """used to store the path of zeek log + files slips is currently using""" self.r.set('zeek_path', path) def get_zeek_path(self) -> str: diff --git a/slips_files/core/database/redis_db/ioc_handler.py b/slips_files/core/database/redis_db/ioc_handler.py index 7c17b3480..fd16b286d 100644 --- a/slips_files/core/database/redis_db/ioc_handler.py +++ b/slips_files/core/database/redis_db/ioc_handler.py @@ -348,7 +348,7 @@ def setNewURL(self, url: str): # We use the empty dictionary to find if an URL exists or not self.rcache.hset('URLsInfo', url, '{}') - def getDomainData(self, domain): + def get_domain_data(self, domain): """ Return information about this domain Returns a dictionary or False if there is no domain in the database @@ -361,13 +361,13 @@ def getDomainData(self, domain): data = json.loads(data) if data or data == {} else False return data - def setNewDomain(self, domain: str): + def set_new_domain(self, domain: str): """ 1- Stores this new domain in the Domains hash 2- Publishes in the channels that there is a new domain, and that we want data from the Threat Intelligence modules """ - data = self.getDomainData(domain) + data = self.get_domain_data(domain) if data is False: # If there is no data about this domain # Set this domain for the first time in the DomainsInfo @@ -376,7 +376,7 @@ def setNewDomain(self, domain: str): # We use the empty dictionary to find if a domain exists or not self.rcache.hset('DomainsInfo', domain, '{}') - def setInfoForDomains(self, domain: str, info_to_set: dict, mode='leave'): + def set_info_for_domains(self, domain: str, info_to_set: dict, mode= 'leave'): """ Store information for this domain :param info_to_set: a dictionary, such as {'geocountry': 'rumania'} that we are @@ -388,12 +388,12 @@ def setInfoForDomains(self, domain: str, info_to_set: dict, mode='leave'): """ # Get the previous info already stored - domain_data = self.getDomainData(domain) + domain_data = self.get_domain_data(domain) if not domain_data: # This domain is not in the dictionary, add it first: - self.setNewDomain(domain) + self.set_new_domain(domain) # Now get the data, which should be empty, but just in case - domain_data = self.getDomainData(domain) + domain_data = self.get_domain_data(domain) # Let's check each key stored for this domain for key in iter(info_to_set): @@ -451,7 +451,7 @@ def setInfoForDomains(self, domain: str, info_to_set: dict, mode='leave'): # Publish the changes self.r.publish('dns_info_change', domain) - def setInfoForURLs(self, url: str, urldata: dict): + def set_info_for_urls(self, url: str, urldata: dict): """ Store information for this URL We receive a dictionary, such as {'VirusTotal': {'URL':score}} that we are diff --git a/slips_files/core/database/redis_db/profile_handler.py b/slips_files/core/database/redis_db/profile_handler.py index ffc1a8f7c..90c33b721 100644 --- a/slips_files/core/database/redis_db/profile_handler.py +++ b/slips_files/core/database/redis_db/profile_handler.py @@ -54,11 +54,11 @@ def print(self, text, verbose=1, debug=0): {"from": self.name, "txt": text, "verbose": verbose, "debug": debug} ) - def getOutTuplesfromProfileTW(self, profileid, twid): + def get_outtuples_from_profile_tw(self, profileid, twid): """Get the out tuples""" return self.r.hget(profileid + self.separator + twid, "OutTuples") - def getInTuplesfromProfileTW(self, profileid, twid): + def get_intuples_from_profile_tw(self, profileid, twid): """Get the in tuples""" return self.r.hget(profileid + self.separator + twid, "InTuples") @@ -309,10 +309,6 @@ def add_port( ip = str(flow.daddr) spkts = flow.spkts state_hist = flow.state_hist if hasattr(flow, "state_hist") else "" - # dpkts = columns['dpkts'] - # daddr = columns['daddr'] - # saddr = columns['saddr'] - # sbytes = columns['sbytes'] if "^" in state_hist: # The majority of the FP with horizontal port scan detection happen because a @@ -333,7 +329,7 @@ def add_port( ip_key = "srcips" if role == "Server" else "dstips" # Get the state. Established, NotEstablished - summaryState = self.getFinalStateFromFlags(state, pkts) + summaryState = self.get_final_state_from_flags(state, pkts) old_profileid_twid_data = self.get_data_from_profile_tw( profileid, twid, port_type, summaryState, proto, role, "Ports" @@ -374,9 +370,9 @@ def add_port( hash_key = f"{profileid}{self.separator}{twid}" key_name = f"{port_type}Ports{role}{proto}{summaryState}" self.r.hset(hash_key, key_name, str(data)) - self.markProfileTWAsModified(profileid, twid, starttime) + self.mark_profile_tw_as_modified(profileid, twid, starttime) - def getFinalStateFromFlags(self, state, pkts): + def get_final_state_from_flags(self, state, pkts): """ Analyze the flags given and return a summary of the state. Should work with Argus and Bro flags We receive the pakets to distinguish some Reset connections @@ -505,7 +501,7 @@ def getFinalStateFromFlags(self, state, pkts): 0, 1, ) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) def get_data_from_profile_tw( self, @@ -558,7 +554,7 @@ def get_data_from_profile_tw( self.print( f"Error in getDataFromProfileTW database.py line {exception_line}", 0, 1 ) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) def update_ip_info( self, @@ -702,7 +698,7 @@ def add_ips(self, profileid, twid, flow, role): self.update_times_contacted(ip, direction, profileid, twid) # Get the state. Established, NotEstablished - summaryState = self.getFinalStateFromFlags(flow.state, flow.pkts) + summaryState = self.get_final_state_from_flags(flow.state, flow.pkts) key_name = f"{direction}IPs{role}{flow.proto.upper()}{summaryState}" # Get the previous data about this key old_profileid_twid_data = self.get_data_from_profile_tw( @@ -737,7 +733,7 @@ def get_all_contacted_ips_in_profileid_twid(self, profileid, twid) -> dict: """ Get all the contacted IPs in a given profile and TW """ - all_flows: dict = self.db.get_all_flows_in_profileid_twid(profileid, twid) + all_flows: dict = self.get_all_flows_in_profileid_twid(profileid, twid) if not all_flows: return {} contacted_ips = {} @@ -786,7 +782,7 @@ def add_flow( The profileid is the main profile that this flow is related too. : param new_profile_added : is set to True for everytime we see a new srcaddr """ - summaryState = self.getFinalStateFromFlags(flow.state, flow.pkts) + summary_state = self.get_final_state_from_flags(flow.state, flow.pkts) flow_dict = { "ts": flow.starttime, "dur": flow.dur, @@ -796,7 +792,7 @@ def add_flow( "dport": flow.dport, "proto": flow.proto, "origstate": flow.state, - "state": summaryState, + "state": summary_state, "pkts": flow.pkts, "allbytes": flow.bytes, "spkts": flow.spkts, @@ -834,8 +830,6 @@ def add_flow( self.set_input_metadata({"file_start": flow.starttime}) self.first_flow = False - self.set_local_network(flow.saddr) - # dont send arp flows in this channel, they have their own new_arp channel if flow.type_ != "arp": self.publish("new_flow", to_send) @@ -1089,19 +1083,19 @@ def get_number_of_tws_in_profile(self, profileid) -> int: """ return len(self.getTWsfromProfile(profileid)) if profileid else 0 - def getSrcIPsfromProfileTW(self, profileid, twid): + def get_srcips_from_profile_tw(self, profileid, twid): """ Get the src ip for a specific TW for a specific profileid """ return self.r.hget(profileid + self.separator + twid, "SrcIPs") - def getDstIPsfromProfileTW(self, profileid, twid): + def get_dstips_from_profile_tw(self, profileid, twid): """ Get the dst ip for a specific TW for a specific profileid """ return self.r.hget(profileid + self.separator + twid, "DstIPs") - def getT2ForProfileTW(self, profileid, twid, tupleid, tuple_key: str): + def get_t2_for_profile_tw(self, profileid, twid, tupleid, tuple_key: str): """ Get T1 and the previous_time for this previous_time, twid and tupleid """ @@ -1119,13 +1113,12 @@ def getT2ForProfileTW(self, profileid, twid, tupleid, tuple_key: str): except Exception as e: exception_line = sys.exc_info()[2].tb_lineno self.print( - f"Error in getT2ForProfileTW in database.py line " f"{exception_line}", + f"Error in getT2ForProfileTW in database.py line {exception_line}", 0, 1, ) self.print(type(e), 0, 1) self.print(e, 0, 1) - self.print(traceback.print_stack(), 0, 1) def has_profile(self, profileid): """Check if we have the given profile""" @@ -1216,7 +1209,7 @@ def add_new_older_tw(self, profileid: str, tw_start_time: float, tw_number: int) self.print("error in addNewOlderTW in database.py", 0, 1) self.print(type(e), 0, 1) self.print(e, 0, 1) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) def add_new_tw(self, profileid, timewindow: str, startoftw: float): """ @@ -1246,8 +1239,7 @@ def add_new_tw(self, profileid, timewindow: str, startoftw: float): self.update_threat_level(profileid, "info", 0.5) except redis.exceptions.ResponseError as e: self.print("Error in addNewTW", 0, 1) - self.print(traceback.print_stack(), 0, 1) - self.print(e, 0, 1) + self.print(traceback.format_exc(), 0, 1) def get_tw_start_time(self, profileid, twid): """Return the time when this TW in this profile was created""" @@ -1256,11 +1248,11 @@ def get_tw_start_time(self, profileid, twid): # sorted set is encoded return self.r.zscore(f"tws{profileid}", twid.encode("utf-8")) - def getAmountTW(self, profileid): + def get_number_of_tws(self, profileid): """Return the number of tws for this profile id""" return self.r.zcard(f"tws{profileid}") if profileid else False - def getModifiedTWSinceTime(self, time: float) -> List[Tuple[str, float]]: + def get_modified_tw_since_time(self, time: float) -> List[Tuple[str, float]]: """ Return the list of modified timewindows since a certain time """ @@ -1270,10 +1262,10 @@ def getModifiedTWSinceTime(self, time: float) -> List[Tuple[str, float]]: data = self.r.zrangebyscore("ModifiedTW", time, float("+inf"), withscores=True) return data or [] - def getModifiedProfilesSince(self, time: float) -> Tuple[Set[str], float]: + def get_modified_profiles_since(self, time: float) -> Tuple[Set[str], float]: """Returns a set of modified profiles since a certain time and the time of the last modified profile""" - modified_tws: List[Tuple[str, float]] = self.getModifiedTWSinceTime(time) + modified_tws: List[Tuple[str, float]] = self.get_modified_tw_since_time(time) if not modified_tws: # no modified tws, and no time_of_last_modified_tw return [], 0 @@ -1548,7 +1540,6 @@ def add_profile(self, profileid, starttime, duration): self.r.hset(profileid, "duration", duration) # When a new profiled is created assign threat level = 0 # and confidence = 0.05 - # self.r.hset(profileid, 'threat_level', 0) confidence = 0.05 self.update_threat_level(profileid, "info", confidence) self.r.hset(profileid, "confidence", confidence) @@ -1576,7 +1567,7 @@ def set_profile_module_label(self, profileid, module, label): data = json.dumps(data) self.r.hset(profileid, "modules_labels", data) - def check_TW_to_close(self, close_all=False): + def check_tw_to_close(self, close_all=False): """ Check if we should close some TW Search in the modifed tw list and compare when they @@ -1608,9 +1599,9 @@ def check_TW_to_close(self, close_all=False): 3, 0, ) - self.markProfileTWAsClosed(profile_tw_to_close_id) + self.mark_profile_tw_as_closed(profile_tw_to_close_id) - def markProfileTWAsClosed(self, profileid_tw): + def mark_profile_tw_as_closed(self, profileid_tw): """ Mark the TW as closed so tools can work on its data """ @@ -1618,7 +1609,7 @@ def markProfileTWAsClosed(self, profileid_tw): self.r.zrem("ModifiedTW", profileid_tw) self.publish("tw_closed", profileid_tw) - def markProfileTWAsModified(self, profileid, twid, timestamp): + def mark_profile_tw_as_modified(self, profileid, twid, timestamp): """ Mark a TW in a profile as modified This means: @@ -1633,7 +1624,7 @@ def markProfileTWAsModified(self, profileid, twid, timestamp): self.r.zadd("ModifiedTW", data) self.publish("tw_modified", f"{profileid}:{twid}") # Check if we should close some TW - self.check_TW_to_close() + self.check_tw_to_close() def publish_new_letter( self, new_symbol: str, profileid: str, twid: str, tupleid: str, flow @@ -1744,12 +1735,12 @@ def add_tuple( prev_symbols = json.dumps(prev_symbols) self.r.hset(profileid_twid, direction, prev_symbols) - self.markProfileTWAsModified(profileid, twid, flow.starttime) + self.mark_profile_tw_as_modified(profileid, twid, flow.starttime) except Exception: exception_line = sys.exc_info()[2].tb_lineno self.print(f"Error in add_tuple in database.py line {exception_line}", 0, 1) - self.print(traceback.print_stack(), 0, 1) + self.print(traceback.format_exc(), 0, 1) def get_tws_to_search(self, go_back): tws_to_search = float("inf") @@ -1775,7 +1766,7 @@ def add_timeline_line(self, profileid, twid, data, timestamp): mapping = {data: timestamp} self.r.zadd(key, mapping) # Mark the tw as modified since the timeline line is new data in the TW - self.markProfileTWAsModified(profileid, twid, timestamp="") + self.mark_profile_tw_as_modified(profileid, twid, timestamp="") def get_timeline_last_lines( self, profileid, twid, first_index: int diff --git a/slips_files/core/database/sqlite_db/database.py b/slips_files/core/database/sqlite_db/database.py index eec15a3cb..766843a4b 100644 --- a/slips_files/core/database/sqlite_db/database.py +++ b/slips_files/core/database/sqlite_db/database.py @@ -1,4 +1,5 @@ -from typing import List +from typing import List, \ + Dict import os.path import sqlite3 import json @@ -143,14 +144,14 @@ def get_all_flows_in_profileid_twid(self, profileid, twid): res[uid] = json.loads(flow) return res - def get_all_flows_in_profileid(self, profileid): + def get_all_flows_in_profileid(self, profileid) -> Dict[str, dict]: """ Return a list of all the flows in this profileid [{'uid':flow},...] """ condition = f'profileid = "{profileid}"' flows = self.select('flows', condition=condition) - all_flows = {} + all_flows: Dict[str, dict] = {} if flows: for flow in flows: uid = flow[0] diff --git a/slips_files/core/evidence_structure/evidence.py b/slips_files/core/evidence_structure/evidence.py index 8e8c23628..0bb2b22f4 100644 --- a/slips_files/core/evidence_structure/evidence.py +++ b/slips_files/core/evidence_structure/evidence.py @@ -83,9 +83,10 @@ class EvidenceType(Enum): COMMAND_AND_CONTROL_CHANNEL = auto() THREAT_INTELLIGENCE_BLACKLISTED_ASN = auto() THREAT_INTELLIGENCE_BLACKLISTED_IP = auto() + THREAT_INTELLIGENCE_BLACKLISTED_DNS_ANSWER = auto() THREAT_INTELLIGENCE_BLACKLISTED_DOMAIN = auto() MALICIOUS_DOWNLOADED_FILE = auto() - MALICIOUS_URL = auto() + THREAT_INTELLIGENCE_MALICIOUS_URL = auto() def __str__(self): return self.name @@ -321,7 +322,7 @@ def dict_to_evidence(evidence: dict): 'evidence_type': EvidenceType[evidence["evidence_type"]], 'description': evidence['description'], 'attacker': Attacker(**evidence['attacker']), - 'threat_level': ThreatLevel[evidence['threat_level']], + 'threat_level': ThreatLevel[evidence['threat_level'].upper()], 'category': IDEACategory[evidence['category']], 'victim': Victim(**evidence['victim']) if 'victim' in evidence and evidence['victim'] else None, diff --git a/slips_files/core/evidencehandler.py b/slips_files/core/evidencehandler.py index 7d50e7178..4c9534952 100644 --- a/slips_files/core/evidencehandler.py +++ b/slips_files/core/evidencehandler.py @@ -27,7 +27,7 @@ import platform import traceback from slips_files.common.idea_format import idea_format -from slips_files.common.style import red, green, cyan +from slips_files.common.style import red, cyan from slips_files.common.imports import * from slips_files.core.helpers.whitelist import Whitelist from slips_files.core.helpers.notify import Notify @@ -37,14 +37,9 @@ evidence_to_dict, ProfileID, Evidence, - Direction, Victim, - IoCType, EvidenceType, - IDEACategory, TimeWindow, - Proto, - Tag ) IS_IN_A_DOCKER_CONTAINER = os.environ.get('IS_IN_A_DOCKER_CONTAINER', False) @@ -173,7 +168,12 @@ def clean_file(self, output_dir, file_to_clean): if path.exists(logfile_path): open(logfile_path, 'w').close() return open(logfile_path, 'a') + + def handle_unable_to_log_evidence(self): + self.print('Error in add_to_json_log_file()') + self.print(traceback.format_exc(), 0, 1) + def add_to_json_log_file( self, idea_dict: dict, @@ -186,6 +186,10 @@ def add_to_json_log_file( :param idea_dict: dict containing 1 alert :param all_uids: the uids of the flows causing this evidence """ + if not idea_dict: + self.handle_unable_to_log_evidence() + return + try: # we add extra fields to alerts.json that are not in the IDEA format idea_dict.update({ @@ -198,8 +202,7 @@ def add_to_json_log_file( except KeyboardInterrupt: return True except Exception: - self.print('Error in add_to_json_log_file()') - self.print(traceback.print_stack(), 0, 1) + self.handle_unable_to_log_evidence() def add_to_log_file(self, data): """ @@ -215,7 +218,7 @@ def add_to_log_file(self, data): return True except Exception: self.print('Error in add_to_log_file()') - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) def get_domains_of_flow(self, flow: dict): """ @@ -241,8 +244,6 @@ def get_domains_of_flow(self, flow: dict): except (KeyError, TypeError): pass try: - # self.print(f"DNS of src IP {self.column_values['saddr']}: - # {self.db.get_dns_resolution(self.column_values['saddr'])}") src_dns_domains = self.db.get_dns_resolution(flow['saddr']) src_dns_domains = src_dns_domains.get('domains', []) @@ -260,21 +261,6 @@ def get_domains_of_flow(self, flow: dict): return domains_to_check_dst, domains_to_check_src - def show_popup(self, alert_to_log: str): - """ - Function to display a popup with the alert depending on the OS - """ - if platform.system() == 'Linux': - # is notify_cmd is set in setup_notifications function - # depending on the user - os.system(f'{self.notify_cmd} "Slips" "{alert_to_log}"') - elif platform.system() == 'Darwin': - os.system( - f'osascript -e \'display notification "{alert_to_log}" ' - f'with title "Slips"\' ' - ) - - def format_evidence_causing_this_alert( self, all_evidence: Dict[str, Evidence], @@ -322,8 +308,10 @@ def format_evidence_causing_this_alert( for evidence in all_evidence.values(): evidence: Evidence - description: str = evidence.description - evidence_string = self.line_wrap(f'Detected {description}') + evidence: Evidence = ( + self.add_threat_level_to_evidence_description(evidence) + ) + evidence_string = self.line_wrap(f'Detected {evidence.description}') alert_to_print += cyan(f'\t- {evidence_string}\n') # Add the timestamp to the alert. @@ -342,7 +330,6 @@ def is_running_on_interface(self): def decide_blocking(self, profileid) -> bool: """ Decide whether to block or not and send to the blocking module - :param ip: IP to block """ # now since this source ip(profileid) caused an alert, @@ -390,19 +377,19 @@ def mark_as_blocked( now = utils.convert_format(now, utils.alerts_format) ip = profileid.split('_')[-1].strip() - msg = f'{flow_datetime}: Src IP {ip:26}. ' + line = f'{flow_datetime}: Src IP {ip:26}. ' if blocked: self.db.markProfileTWAsBlocked(profileid, twid) # Add to log files that this srcip is being blocked - msg += 'Blocked ' + line += 'Blocked ' else: - msg += 'Generated an alert ' + line += 'Generated an alert ' - msg += (f'given enough evidence on timewindow ' + line += (f'given enough evidence on timewindow ' f'{twid.split("timewindow")[1]}. (real time {now})') # log in alerts.log - self.add_to_log_file(msg) + self.add_to_log_file(line) # Add a json field stating that this ip is blocked in alerts.json # replace the evidence description with slips msg that this is a @@ -411,7 +398,7 @@ def mark_as_blocked( IDEA_dict['Category'] = 'Alert' IDEA_dict['profileid'] = profileid IDEA_dict['threat_level'] = accumulated_threat_level - IDEA_dict['Attach'][0]['Content'] = msg + IDEA_dict['Attach'][0]['Content'] = line # add to alerts.json self.add_to_json_log_file( @@ -446,7 +433,7 @@ def is_evidence_done_by_others(self, evidence: Evidence) -> bool: return evidence.attacker.direction != 'SRC' def get_evidence_for_tw(self, profileid: str, twid: str) \ - -> Dict[str, dict]: + -> Optional[Dict[str, Evidence]]: """ filters and returns all the evidence for this profile in this TW returns the dict with filtered evidence @@ -455,7 +442,7 @@ def get_evidence_for_tw(self, profileid: str, twid: str) \ profileid, twid ) if not tw_evidence: - return False + return past_evidence_ids: List[str] = \ self.get_evidence_that_were_part_of_a_past_alert(profileid, twid) @@ -613,31 +600,36 @@ def handle_new_alert(self, alert_ID: str, tw_evidence: dict): def get_evidence_to_log( self, evidence: Evidence, flow_datetime ) -> str: - timewindow_number: int = evidence.timewindow.number - - # to keep the alignment of alerts.json ip + hostname - # combined should take no more than 26 chars - evidence_str = f'{flow_datetime} (TW {timewindow_number}): Src ' \ - f'IP {evidence.profile.ip:26}. Detected ' \ - f' {evidence.description}' - - # sometimes slips tries to get the hostname of a - # profile before ip_info stores it in the db - # there's nothing we can do about it - hostname: Optional[str] = self.db.get_hostname_from_profile( - str(evidence.profile) - ) - if not hostname: - return evidence_str + """ + returns the line of evidence that we log to alerts logfiles only. + not the cli + """ + timewindow_number: int = evidence.timewindow.number - padding_len = 26 - len(evidence.profile.ip) - len(hostname) - 3 - # fill the rest of the 26 characters with spaces to keep the alignment - evidence_str = f'{flow_datetime} (TW {timewindow_number}): Src IP' \ - f' {evidence.profile.ip} ({hostname}) {" "*padding_len}. ' \ - f'Detected {evidence.description}' + # to keep the alignment of alerts.json ip + hostname + # combined should take no more than 26 chars + evidence_str = (f'{flow_datetime} (TW {timewindow_number}): Src ' + f'IP {evidence.profile.ip:26}. Detected' + f' {evidence.description} ') + + # sometimes slips tries to get the hostname of a + # profile before ip_info stores it in the db + # there's nothing we can do about it + hostname: Optional[str] = self.db.get_hostname_from_profile( + str(evidence.profile) + ) + if not hostname: return evidence_str + padding_len = 26 - len(evidence.profile.ip) - len(hostname) - 3 + # fill the rest of the 26 characters with spaces to keep the alignment + evidence_str = (f'{flow_datetime} (TW {timewindow_number}): Src IP' + f' {evidence.profile.ip} ({hostname})' + f' {" "*padding_len}. ' + f'Detected {evidence.description} ') + return evidence_str + def increment_attack_counter( self, attacker: str, @@ -685,6 +677,13 @@ def show_popup(self, alert: str): .replace(Style.RESET_ALL, '') ) self.notify.show_popup(alert) + + + def add_threat_level_to_evidence_description( + self, evidence: Evidence) -> Evidence: + evidence.description += (f' threat level: ' + f'{evidence.threat_level.name.lower()}.') + return evidence def main(self): while not self.should_stop(): @@ -703,7 +702,7 @@ def main(self): # below. # to avoid this, we only alert about processed evidence self.db.mark_evidence_as_processed(evidence.id) - # Ignore alert if IP is whitelisted + # Ignore evidence if IP is whitelisted if self.whitelist.is_whitelisted_evidence(evidence): self.db.cache_whitelisted_evidence_ID(evidence.id) # Modules add evidence to the db before @@ -721,12 +720,17 @@ def main(self): ) flow_datetime = utils.convert_format(timestamp, 'iso') + evidence: Evidence = ( + self.add_threat_level_to_evidence_description(evidence) + ) + evidence_to_log: str = self.get_evidence_to_log( evidence, flow_datetime, ) # Add the evidence to alerts.log self.add_to_log_file(evidence_to_log) + self.increment_attack_counter( evidence.profile.ip, evidence.victim, diff --git a/slips_files/core/helpers/checker.py b/slips_files/core/helpers/checker.py index b8134474a..53d8d684f 100644 --- a/slips_files/core/helpers/checker.py +++ b/slips_files/core/helpers/checker.py @@ -1,6 +1,7 @@ import os import subprocess import sys +from typing import Tuple import psutil @@ -187,7 +188,7 @@ def input_module_exists(self, module): return True - def check_output_redirection(self) -> tuple: + def check_output_redirection(self) -> Tuple[str,str,str]: """ Determine where slips will place stdout, stderr and logfile based on slips mode diff --git a/slips_files/core/helpers/filemonitor.py b/slips_files/core/helpers/filemonitor.py index d50d99cb8..443e7eeac 100644 --- a/slips_files/core/helpers/filemonitor.py +++ b/slips_files/core/helpers/filemonitor.py @@ -69,8 +69,8 @@ def on_modified(self, event): with open(os.path.join(self.dir_to_monitor, file), 'r') as f: while line := f.readline(): if 'termination' in line: - # this is how modules tell slips to terminate - self.db.publish('control_channel', 'stop_slips') + # tell slips to terminate + self.db.publish_stop() break elif 'whitelist' in filename: self.db.publish('reload_whitelist', 'reload') diff --git a/slips_files/core/helpers/notify.py b/slips_files/core/helpers/notify.py index 05adcbf7a..a77fdccaf 100644 --- a/slips_files/core/helpers/notify.py +++ b/slips_files/core/helpers/notify.py @@ -36,8 +36,10 @@ def setup_notifications(self): self.notify_cmd = 'notify-send -t 5000 ' return False - # Get the used display (if the user has only 1 screen it will be set to 0), if not we should know which screen is slips running on. - # A "display" is the address for your screen. Any program that wants to write to your screen has to know the address. + # Get the used display (if the user has only 1 screen it will be + # set to 0), if not we should know which screen is slips running on. + # A "display" is the address for your screen. Any program that + # wants to write to your screen has to know the address. used_display = psutil.Process().environ()['DISPLAY'] # when you login as user x in linux, no user other than x is authorized to write to your display, not even root @@ -60,15 +62,18 @@ def setup_notifications(self): # run notify-send as user using the used_display and give it the dbus addr self.notify_cmd = f'sudo -u {user} DISPLAY={used_display} ' \ f'DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/{uid}/bus notify-send -t 5000 ' - + + def show_popup(self, alert_to_log: str): """ Function to display a popup with the alert depending on the OS """ if platform.system() == 'Linux': - # is notify_cmd is set in setup_notifications function depending on the user + # is notify_cmd is set in + # setup_notifications function depending on the user os.system(f'{self.notify_cmd} "Slips" "{alert_to_log}"') elif platform.system() == 'Darwin': os.system( - f'osascript -e \'display notification "{alert_to_log}" with title "Slips"\' ' + f'osascript -e \'display notification' + f' "{alert_to_log}" with title "Slips"\' ' ) diff --git a/slips_files/core/helpers/symbols_handler.py b/slips_files/core/helpers/symbols_handler.py index 0b5fadafb..05019f6fd 100644 --- a/slips_files/core/helpers/symbols_handler.py +++ b/slips_files/core/helpers/symbols_handler.py @@ -95,7 +95,7 @@ def compute( # Get the time of the last flow in this tuple, and the last last # Implicitely this is converting what we stored as 'now' into 'last_ts' and what we stored as 'last_ts' as 'last_last_ts' - (last_last_ts, last_ts) = self.db.getT2ForProfileTW( + (last_last_ts, last_ts) = self.db.get_t2_for_profile_tw( profileid, twid, tupleid, tuple_key ) # self.print(f'Profileid: {profileid}. Data extracted from DB. last_ts: {last_ts}, last_last_ts: {last_last_ts}', 0, 5) @@ -290,4 +290,4 @@ def compute_timechar(): # For some reason we can not use the output queue here.. check self.print('Error in compute_symbol in Profiler Process.', 0, 1) - self.print(traceback.print_stack(), 0, 1) \ No newline at end of file + self.print(traceback.format_exc(), 0, 1) \ No newline at end of file diff --git a/slips_files/core/helpers/whitelist.py b/slips_files/core/helpers/whitelist.py index 41f950451..4293f1fa5 100644 --- a/slips_files/core/helpers/whitelist.py +++ b/slips_files/core/helpers/whitelist.py @@ -1,5 +1,8 @@ import json import ipaddress +from typing import Optional, \ + Dict + import validators from slips_files.common.imports import * from slips_files.common.abstracts.observer import IObservable @@ -7,14 +10,9 @@ import tld import os from slips_files.core.evidence_structure.evidence import ( - dict_to_evidence, Evidence, Direction, IoCType, - EvidenceType, - IDEACategory, - Proto, - Tag, Attacker, Victim ) @@ -765,32 +763,35 @@ def parse_whitelist(self, whitelist): return whitelisted_IPs, whitelisted_domains, whitelisted_orgs, \ whitelisted_macs - - def is_whitelisted_evidence( - self, evidence: Evidence - ) -> bool: - """ - Checks if an evidence is whitelisted - """ - - # self.print(f'Checking the whitelist of {srcip}: {data} - # {attacker_direction} {description} ') - + + def get_all_whitelist(self) -> Optional[Dict[str, dict]]: whitelist = self.db.get_all_whitelist() max_tries = 10 - # if this module is loaded before profilerProcess or before we're - # done processing the whitelist in general - # the database won't return the whitelist - # so we need to try several times until the db returns the - # populated whitelist - # empty dicts evaluate to False + # if this module is loaded before profilerProcess or before we're + # done processing the whitelist in general + # the database won't return the whitelist + # so we need to try several times until the db returns the + # populated whitelist + # empty dicts evaluate to False while not bool(whitelist) and max_tries != 0: # try max 10 times to get the whitelist, if it's still empty # hen it's not empty by mistake max_tries -= 1 whitelist = self.db.get_all_whitelist() + if max_tries == 0: # we tried 10 times to get the whitelist, it's probably empty. + return + + return whitelist + + def is_whitelisted_evidence( + self, evidence: Evidence + ) -> bool: + """ + Checks if an evidence is whitelisted + """ + if not self.get_all_whitelist(): return False if self.check_whitelisted_attacker(evidence.attacker): @@ -800,15 +801,13 @@ def is_whitelisted_evidence( hasattr(evidence, 'victim') and self.check_whitelisted_victim(evidence.victim) ): - return True + return True - def check_whitelisted_victim(self, victim: Victim): + + def check_whitelisted_victim(self, victim: Victim) -> bool: if not victim: return False - whitelist = self.db.get_all_whitelist() - whitelisted_orgs = self.parse_whitelist(whitelist)[2] - if ( victim.victim_type == IoCType.IP.name and self.is_ip_whitelisted(victim.value, victim.direction) @@ -822,11 +821,7 @@ def check_whitelisted_victim(self, victim: Victim): ): return True - - if( - whitelisted_orgs - and self.is_part_of_a_whitelisted_org(victim) - ): + if self.is_part_of_a_whitelisted_org(victim): return True @@ -880,7 +875,7 @@ def load_org_asn(self, org) -> list: asn_cache: dict = self.db.get_asn_cache() org_asn = [] # asn_cache is a dict sorted by first octet - for octet, range_info in asn_cache.items: + for octet, range_info in asn_cache.items(): # range_info is a serialized dict of ranges range_info = json.loads(range_info) for range, asn_info in range_info.items(): @@ -973,33 +968,83 @@ def is_ip_whitelisted(self, ip: str, direction: Direction): whitelist_direction: str = whitelisted_ips[ip]['from'] what_to_ignore = whitelisted_ips[ip]['what_to_ignore'] ignore_alerts = self.should_ignore_alerts(what_to_ignore) - - ignore_alerts_from_ip = ( - ignore_alerts - and direction == Direction.SRC - and self.should_ignore_from(whitelist_direction) - ) - ignore_alerts_to_ip = ( - ignore_alerts - and direction == Direction.DST - and self.should_ignore_to(whitelist_direction) - ) - if ignore_alerts_from_ip or ignore_alerts_to_ip: + + if self.ignore_alert( + direction, + ignore_alerts, + whitelist_direction + ): # self.print(f'Whitelisting src IP {srcip} for evidence' # f' about {ip}, due to a connection related to {data} ' # f'in {description}') return True - # Now we know this ipv4 or ipv6 isn't whitelisted - # is the mac address of this ip whitelisted? + # Now we know this ipv4 or ipv6 isn't whitelisted + # is the mac address of this ip whitelisted? if whitelisted_macs and self.profile_has_whitelisted_mac( ip, whitelisted_macs, direction ): return True + + def ignore_alert(self, direction, ignore_alerts, whitelist_direction) -> bool: + """ + determines whether or not we should ignore the given alert based + on the ip's direction and the whitelist direction + """ + if ( + self.ignore_alerts_from_ip( + direction, ignore_alerts, whitelist_direction + ) + or self.ignore_alerts_to_ip( + direction, ignore_alerts, whitelist_direction + ) + or self.ignore_alerts_from_both_directions( + ignore_alerts, whitelist_direction + ) + ): + return True + + def ignore_alerts_from_both_directions( + self, + ignore_alerts: bool, + whitelist_direction: str + ) -> bool: + return ignore_alerts and 'both' in whitelist_direction + + def ignore_alerts_from_ip( + self, + direction: Direction, + ignore_alerts: bool, + whitelist_direction: str + ) -> bool: + if not ignore_alerts: + return False + + if ( + direction == Direction.SRC + and self.should_ignore_from(whitelist_direction) + ): + return True + + def ignore_alerts_to_ip( + self, + direction: Direction, + ignore_alerts: bool, + whitelist_direction: str + ) -> bool: + + if not ignore_alerts: + return False + + if ( + direction == Direction.DST + and self.should_ignore_to(whitelist_direction) + ): + return True + def is_domain_whitelisted(self, domain: str, direction: Direction): # todo differentiate between this and is_whitelisted_Domain() - # extract the top level domain try: domain = tld.get_fld(domain, fix_protocol=True) @@ -1069,8 +1114,7 @@ def is_part_of_a_whitelisted_org(self, ioc): ioc_type: IoCType = ioc.attacker_type if isinstance(ioc, Attacker) \ - else \ - ioc.victim_type + else ioc.victim_type # Check if the IP in the alert belongs to a whitelisted organization if ioc_type == IoCType.DOMAIN.name: # Method 3 Check if the domains of this flow belong to this org domains diff --git a/slips_files/core/input.py b/slips_files/core/input.py index 3aa2a2c70..45fe9acb2 100644 --- a/slips_files/core/input.py +++ b/slips_files/core/input.py @@ -660,7 +660,7 @@ def handle_pcap_and_interface(self) -> int: # Give Zeek some time to generate at least 1 file. time.sleep(3) - self.db.store_process_PID("Zeek", self.zeek_pid) + self.db.store_pid("Zeek", self.zeek_pid) if not hasattr(self, "is_zeek_tabs"): self.is_zeek_tabs = False self.lines = self.read_zeek_files() diff --git a/slips_files/core/input_profilers/argus.py b/slips_files/core/input_profilers/argus.py index 9a2cff195..4ea97541a 100644 --- a/slips_files/core/input_profilers/argus.py +++ b/slips_files/core/input_profilers/argus.py @@ -133,5 +133,5 @@ def define_columns(self, new_line: dict) -> dict: self.print( f'\tProblem in define_columns() line {exception_line}', 0, 1 ) - self.print(traceback.print_stack(),0,1) + self.print(traceback.format_exc(),0,1) sys.exit(1) diff --git a/slips_files/core/output.py b/slips_files/core/output.py index 9af1003c2..c2d1097a9 100644 --- a/slips_files/core/output.py +++ b/slips_files/core/output.py @@ -286,6 +286,24 @@ def tell_pbar(self, msg: dict): def is_pbar_finished(self )-> bool: return self.pbar_finished.is_set() + + def forward_progress_bar_msgs(self, msg: dict): + """ + passes init and update msgs to pbar module + """ + pbar_event: str = msg['bar'] + if pbar_event == 'init': + self.tell_pbar({ + 'event': pbar_event, + 'total_flows': msg['bar_info']['total_flows'], + }) + return + + if pbar_event == 'update' and not self.is_pbar_finished(): + self.tell_pbar({ + 'event': 'update_bar', + }) + def update(self, msg: dict): """ gets called whenever any module need to print something @@ -303,28 +321,22 @@ def update(self, msg: dict): total_flows: int, } """ - try: - if 'init' in msg.get('bar', ''): - self.tell_pbar({ - 'event': 'init', - 'total_flows': msg['bar_info']['total_flows'], - }) - - elif ( - 'update' in msg.get('bar', '') - and not self.is_pbar_finished() - ): - # if pbar wasn't supported, inputproc won't send update msgs - self.tell_pbar({ - 'event': 'update_bar', - }) - else: - # output to terminal and logs or logs only? - if msg.get('log_to_logfiles_only', False): - self.log_line(msg) - else: - # output to terminal - self.output_line(msg) - except Exception as e: - print(f"Error in output.py: {e}") - print(traceback.print_stack()) + # if pbar wasn't supported, inputproc won't send update msgs + + # try: + if 'bar' in msg: + self.forward_progress_bar_msgs(msg) + return + + # output to terminal and logs or logs only? + if msg.get('log_to_logfiles_only', False): + self.log_line(msg) + else: + # output to terminal + self.output_line(msg) + + +# except Exception as e: +# print(f"Error in output.py: {e} {type(e)}") +# traceback.print_stack() + diff --git a/slips_files/core/profiler.py b/slips_files/core/profiler.py index 2cfac61e1..e91e68d0d 100644 --- a/slips_files/core/profiler.py +++ b/slips_files/core/profiler.py @@ -1,7 +1,5 @@ # Stratosphere Linux IPS. A machine-learning Intrusion Detection System # Copyright (C) 2021 Sebastian Garcia -import multiprocessing - # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 @@ -19,10 +17,12 @@ # stratosphere@aic.fel.cvut.cz from dataclasses import asdict import queue -import sys import ipaddress import pprint from datetime import datetime +from typing import List + +import validators from slips_files.common.imports import * from slips_files.common.abstracts.core import ICore @@ -33,8 +33,7 @@ from slips_files.core.input_profilers.nfdump import Nfdump from slips_files.core.input_profilers.suricata import Suricata from slips_files.core.input_profilers.zeek import ZeekJSON, ZeekTabs - - +from slips_files.core.output import Output SUPPORTED_INPUT_TYPES = { @@ -57,7 +56,7 @@ class Profiler(ICore): """A class to create the profiles for IPs""" name = 'Profiler' - + def init(self, is_profiler_done: multiprocessing.Semaphore = None, profiler_queue=None, @@ -75,9 +74,9 @@ def init(self, self.input_type = False self.whitelisted_flows_ctr = 0 self.rec_lines = 0 + self.is_localnet_set = False self.has_pbar = has_pbar self.whitelist = Whitelist(self.logger, self.db) - # Read the configuration self.read_configuration() self.symbol = SymbolHandler(self.logger, self.db) # there has to be a timeout or it will wait forever and never @@ -99,6 +98,8 @@ def read_configuration(self): self.analysis_direction = conf.analysis_direction() self.label = conf.label() self.width = conf.get_tw_width_as_float() + self.client_ips: List[str] = conf.client_ips() + def convert_starttime_to_epoch(self): try: @@ -226,7 +227,7 @@ def store_features_going_out(self): # if the flow type matched any of the ifs above, # mark this profile as modified - self.db.markProfileTWAsModified(self.profileid, self.twid, '') + self.db.mark_profile_tw_as_modified(self.profileid, self.twid, '') def store_features_going_in(self, profileid: str, twid: str): """ @@ -271,7 +272,7 @@ def store_features_going_in(self, profileid: str, twid: str): twid=twid, label=self.label, ) - self.db.markProfileTWAsModified(profileid, twid, '') + self.db.mark_profile_tw_as_modified(profileid, twid, '') def handle_in_flows(self): """ @@ -284,6 +285,31 @@ def handle_in_flows(self): return rev_profileid, rev_twid = self.get_rev_profile() self.store_features_going_in(rev_profileid, rev_twid) + + + def should_set_localnet(self) -> bool: + """ + returns true only if the saddr of the current flow is ipv4, private + and we don't have the local_net set already + """ + if self.is_localnet_set: + return False + + if self.get_private_client_ips(): + # if we have private client ips, we're ready to set the + # localnetwork + return True + + if not validators.ipv4(self.flow.saddr): + return False + + saddr_obj: ipaddress = ipaddress.ip_address(self.flow.saddr) + is_private_ip = utils.is_private_ip(saddr_obj) + if not is_private_ip: + return False + + return True + def define_separator(self, line: dict, input_type: str): """ @@ -371,16 +397,58 @@ def init_pbar(self, total_flows:int): } }) self.supported_pbar = True + + def get_private_client_ips(self) -> List[str]: + """ + returns the private ips found in the client_ips param + in the config file + """ + private_clients = [] + for ip in self.client_ips: + if utils.is_private_ip(ipaddress.ip_address(ip)): + private_clients.append(ip) + return private_clients + + + def get_local_net(self) -> str: + """ + gets the local network from client_ip param in the config file, + or by using the localnetwork of the first private + srcip seen in the traffic + """ + # For now the local network is only ipv4, but it + # could be ipv6 in the future. Todo. + private_client_ips: List[str] = self.get_private_client_ips() + if private_client_ips: + # all client ips should belong to the same local network, + # it doesn't make sense to have ips belonging to different + # networks in the config file! + ip = private_client_ips[0] + else: + ip = self.flow.saddr + self.is_localnet_set = True + return utils.get_cidr_of_private_ip(ip) + + def handle_setting_local_net(self): + """ + stores the local network if possible + """ + if not self.should_set_localnet(): + return + + local_net: str = self.get_local_net() + self.db.set_local_network(local_net) + def pre_main(self): utils.drop_root_privs() - + def main(self): while not self.should_stop(): try: # this msg can be a str only when it's a 'stop' msg indicating # that this module should stop - msg: dict = self.profiler_queue.get(timeout=1, block=False) + msg = self.profiler_queue.get(timeout=1, block=False) # ALYA, DO NOT REMOVE THIS CHECK # without it, there's no way thi module will know it's time to # stop and no new fows are coming @@ -426,6 +494,7 @@ def main(self): self.flow = self.input.process_line(line) if self.flow: self.add_flow_to_profile() + self.handle_setting_local_net() # now that one flow is processed tell output.py # to update the bar diff --git a/tests/common_test_utils.py b/tests/common_test_utils.py index 632e27be3..a29cfbdde 100644 --- a/tests/common_test_utils.py +++ b/tests/common_test_utils.py @@ -71,19 +71,19 @@ def check_for_text(txt, output_dir): return False -def check_error_keywords(line): +def has_error_keywords(line): """ these keywords indicate that an error needs to be fixed and should fail the integration tests when found """ error_keywords = ('