Merge pull request #438 from stratosphereips/develop

Slips v1.0.10

.github/workflows/CI-production-testing.yml

@@ -23,7 +23,7 @@ jobs:
         fetch-depth: ''
 
     - name: Install slips dependencies
-      run: sudo apt-get -y --no-install-recommends install python3 redis-server python3-pip python3-certifi python3-dev build-essential file lsof net-tools iproute2 iptables python3-tzlocal nfdump tshark git whois golang nodejs notify-osd yara libnotify-bin
+      run: sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install python3 redis-server python3-pip python3-certifi python3-dev build-essential file lsof net-tools iproute2 iptables python3-tzlocal nfdump tshark git whois golang nodejs notify-osd yara libnotify-bin
 
     - name: Install Zeek
       run: |
@@ -47,7 +47,7 @@ jobs:
       run: redis-server --daemonize yes
 
     - name: Run unit tests
-      run: python3  -m pytest tests/  --ignore="tests/test_daemon.py" --ignore="tests/test_database.py" --ignore="tests/integration_tests" -n 7 -p no:warnings -vv -s
+      run: python3  -m pytest tests/ --ignore="tests/test_database.py" --ignore="tests/integration_tests" -n 7 -p no:warnings -vv -s
 
     - name: Run database unit tests
       run: python3  -m pytest tests/test_database.py -p no:warnings -vv
@@ -121,29 +121,21 @@ jobs:
         image: stratosphereips/slips:latest
         run: |
           git reset --hard
-          git pull & git checkout origin/develop
+          git pull & git checkout -f origin/develop
           redis-server --daemonize yes
-          python3  -m pytest tests/ --ignore="tests/test_daemon.py" --ignore="tests/test_database.py" --ignore="tests/integration_tests" -n 7 -p no:warnings -vv -s
+          python3  -m pytest tests/ --ignore="tests/test_database.py" --ignore="tests/integration_tests" -n 7 -p no:warnings -vv -s
 
     - name: Run database tests inside docker
       uses: addnab/docker-run-action@v3
       with:
         image: stratosphereips/slips:latest
         run: |
           git reset --hard
-          git pull & git checkout origin/develop
+          git pull & git checkout -f origin/develop
           redis-server --daemonize yes
           python3  -m pytest tests/test_database.py -p no:warnings -vv
 
-    - name: Run daemon tests inside docker
-      uses: addnab/docker-run-action@v3
-      with:
-        image: stratosphereips/slips:latest
-        run: |
-          git reset --hard
-          git pull & git checkout origin/develop
-          redis-server --daemonize yes
-          python3  -m pytest tests/test_daemon.py -p no:warnings -vv
+
 
     - name: Run integration tests inside docker
       uses: addnab/docker-run-action@v3
@@ -154,7 +146,7 @@ jobs:
         options: -v ${{ github.workspace }}/output:/StratosphereLinuxIPS/output
         run: |
           git reset --hard
-          git pull & git checkout origin/develop
+          git pull & git checkout -f origin/develop
           redis-server --daemonize yes
           python3 -m pytest -s tests/integration_tests/test_dataset.py -p no:warnings -vv
 
@@ -167,7 +159,7 @@ jobs:
         options: -v ${{ github.workspace }}/output:/StratosphereLinuxIPS/output
         run: |
           git reset --hard
-          git pull & git checkout origin/develop
+          git pull & git checkout -f origin/develop
           redis-server --daemonize yes
           python3 -m pytest -s tests/integration_tests/test_config_files.py -p no:warnings -vv
 

CHANGELOG.md

@@ -1,3 +1,12 @@
+- 1.0.10 (January 2024)
+- Faster ensembling of evidence.
+- Log accumulated threat levels of each evidence in alerts.json.
+- Better handling of the termination of the progress bar.
+- Re-add support for tensorflow to the dockers for macOS M1 and macOS M1 P2P.
+- Fix problem setting 'vertical portscan' evidence detected by Zeek.
+- Fix unable to do RDAP lookups
+- Fix stopping Slips daemon.
+
 -1.0.9 (December 2023)
 - Fix using -k to kill opened redis servers.
 - Better README and docs.

README.md

@@ -1,5 +1,5 @@
 <h1 align="center"> 
-Slips v1.0.9
+Slips v1.0.10
 </h1>
 
 
@@ -51,8 +51,7 @@ Slips v1.0.9
 
 Slips is a powerful endpoint behavioral intrusion prevention and detection system that uses machine learning to detect malicious behaviors in network traffic. Slips can work with network traffic in real-time, PCAP files, and network flows from popular tools like Suricata, Zeek/Bro, and Argus. Slips threat detection is based on a combination of machine learning models trained to detect malicious behaviors, 40+ threat intelligence feeds, and expert heuristics. Slips gathers evidence of malicious behavior and uses extensively trained thresholds to trigger alerts when enough evidence is accumulated.
 
-<img src="https://raw.githubusercontent.com/stratosphereips/StratosphereLinuxIPS/develop/docs/images/slips.gif" width="850px"
-title="Slips in action.">
+<img src="https://raw.githubusercontent.com/stratosphereips/StratosphereLinuxIPS/develop/docs/images/slips.gif" width="850px" title="Slips in action.">
 
 
 

VERSION

@@ -1 +1 @@
-1.0.9
+1.0.10

docker/P2P-image/Dockerfile

@@ -10,8 +10,6 @@ ENV IS_IN_A_DOCKER_CONTAINER True
 ENV SLIPS_DIR /StratosphereLinuxIPS
 
 
-
-
 # Install wget and add Zeek repository to our sources.
 RUN apt update && apt install -y --no-install-recommends \
     wget \
@@ -48,9 +46,10 @@ RUN apt update && apt install -y --no-install-recommends \
  && ln -s /opt/zeek/bin/zeek /usr/local/bin/bro
 
 
-RUN git clone --recurse-submodules --remote-submodules https://github.com/stratosphereips/StratosphereLinuxIPS/ ${SLIPS_DIR}/
-# Switch to Slips installation dir when login.
+RUN git clone https://github.com/stratosphereips/StratosphereLinuxIPS/ ${SLIPS_DIR}/
 WORKDIR ${SLIPS_DIR}
+RUN git submodule sync && git pull --recurse-submodules
+# Switch to Slips installation dir when login.
 RUN chmod 774 slips.py &&  git submodule init && git submodule update
 
 

managers/process_manager.py

@@ -547,8 +547,6 @@ def shutdown_gracefully(self):
             analysis_time = self.get_analysis_time()
             self.main.print(f"Analysis of {self.main.input_information} "
                             f"finished in {analysis_time:.2f} minutes")
-            flows_count: int = self.main.db.get_flows_count()
-            self.main.print(f"Total flows read (without altflows): {flows_count}", log_to_logfiles_only=True)
 
             graceful_shutdown = True
             if self.main.mode == 'daemonized':
@@ -563,6 +561,9 @@ def shutdown_gracefully(self):
                 self.main.daemon.delete_pidfile()
 
             else:
+                flows_count: int = self.main.db.get_flows_count()
+                self.main.print(f"Total flows read (without altflows): {flows_count}", log_to_logfiles_only=True)
+
                 hitlist: Tuple[List[Process], List[Process]] = self.get_hitlist_in_order()
                 to_kill_first: List[Process] = hitlist[0]
                 to_kill_last: List[Process] = hitlist[1]

modules/flowalerts/flowalerts.py

@@ -2042,10 +2042,12 @@ def main(self):
 
         if msg := self.get_msg('tw_closed'):
             profileid_tw = msg['data'].split('_')
-            profileid, twid = f'{profileid_tw[0]}_{profileid_tw[1]}', profileid_tw[-1]
+            profileid = f'{profileid_tw[0]}_{profileid_tw[1]}',
+            twid = profileid_tw[-1]
             self.detect_data_upload_in_twid(profileid, twid)
 
-        # --- Detect DNS issues: 1) DNS resolutions without connection, 2) DGA, 3) young domains, 4) ARPA SCANs
+        # --- Detect DNS issues: 1) DNS resolutions without
+        # connection, 2) DGA, 3) young domains, 4) ARPA SCANs
         if msg:= self.get_msg('new_dns'):
             data = json.loads(msg['data'])
             profileid = data['profileid']
@@ -2060,8 +2062,10 @@ def main(self):
             rcode_name = flow_data.get('rcode_name', False)
             stime = data.get('stime', False)
 
-            # only check dns without connection if we have answers(we're sure the query is resolved)
-            # sometimes we have 2 dns flows, 1 for ipv4 and 1 fo ipv6, both have the
+            # only check dns without connection if we have
+            # answers(we're sure the query is resolved)
+            # sometimes we have 2 dns flows, 1 for ipv4 and
+            # 1 fo ipv6, both have the
             # same uid, this causes FP dns without connection,
             # so make sure we only check the uid once
             if answers and uid not in self.connections_checked_in_dns_conn_timer_thread:
@@ -2089,12 +2093,12 @@ def main(self):
                 domain, stime, profileid, twid, uid
             )
 
-        if msg:= self.get_msg('new_downloaded_file'):
+        if msg := self.get_msg('new_downloaded_file'):
             ssl_info = json.loads(msg['data'])
             self.check_malicious_ssl(ssl_info)
 
         # --- Detect Bad SMTP logins ---
-        if msg:= self.get_msg('new_smtp'):
+        if msg := self.get_msg('new_smtp'):
             smtp_info = json.loads(msg['data'])
             profileid = smtp_info['profileid']
             twid = smtp_info['twid']
@@ -2106,7 +2110,7 @@ def main(self):
                 flow
             )
         # --- Detect multiple used SSH versions ---
-        if msg:= self.get_msg('new_software'):
+        if msg := self.get_msg('new_software'):
             msg = json.loads(msg['data'])
             flow:dict = msg['sw_flow']
             twid = msg['twid']
@@ -2121,10 +2125,10 @@ def main(self):
                 role='SSH::SERVER'
             )
 
-        if msg:=self.get_msg('new_weird'):
+        if msg := self.get_msg('new_weird'):
             msg = json.loads(msg['data'])
             self.check_weird_http_method(msg)
 
-        if msg:= self.get_msg('new_tunnel'):
+        if msg := self.get_msg('new_tunnel'):
             msg = json.loads(msg['data'])
             self.check_GRE_tunnel(msg)

modules/flowalerts/set_evidence.py

@@ -672,7 +672,7 @@ def set_evidence_vertical_portscan(
         source_target_tag = 'Recon'
         conn_count = int(msg.split('least ')[1].split(' unique')[0])
         attacker = scanning_ip
-        victim = msg.splt('ports of ')[-1]
+        victim = msg.split('ports of ')[-1]
         self.db.setEvidence(
             evidence_type,
 			attacker_direction,

modules/http_analyzer/http_analyzer.py

@@ -421,7 +421,7 @@ def set_evidence_http_traffic(self, daddr, profileid, twid, uid, timestamp):
         source_target_tag = 'SendingUnencryptedData'
         category = 'Anomaly.Traffic'
         evidence_type = 'HTTPtraffic'
-        attacker_direction = 'dstip'
+        attacker_direction = 'srcip'
         attacker = daddr
         saddr = profileid.split('_')[-1]
         description = f'Unencrypted HTTP traffic from {saddr} to {daddr}.'

modules/ip_info/asn_info.py

@@ -91,14 +91,17 @@ def get_asn_info_from_geolite(self, ip) -> dict:
 
         return ip_info
 
-    def cache_ip_range(self, ip):
+    def cache_ip_range(self, ip: str):
         """
         Get the range of the given ip and
         cache the asn of the whole ip range
         """
+        if not ip:
+            return False
+
         try:
             # Cache the range of this ip
-            whois_info = ipwhois.IPWhois(address=ip).lookup_rdap()
+            whois_info: dict = ipwhois.IPWhois(address=ip).lookup_rdap()
             asnorg = whois_info.get('asn_description', False)
             asn_cidr = whois_info.get('asn_cidr', False)
             asn_number = whois_info.get('asn', False)
@@ -115,12 +118,12 @@ def cache_ip_range(self, ip):
         except (
             ipwhois.exceptions.IPDefinedError,
             ipwhois.exceptions.HTTPLookupError,
+            ipwhois.exceptions.ASNRegistryError,
+            ipwhois.exceptions.ASNParseError,
         ):
             # private ip or RDAP lookup failed. don't cache
+            # or ASN lookup failed with no more methods to try
             return False
-        except ipwhois.exceptions.ASNRegistryError:
-            # ASN lookup failed with no more methods to try
-            pass
 
 
     def get_asn_online(self, ip):

modules/rnn_cc_detection/rnn_cc_detection.py

@@ -42,7 +42,7 @@ def set_evidence(
 
         tupleid = tupleid.split('-')
         dstip, port, proto = tupleid[0], tupleid[1], tupleid[2]
-        attacker_direction = 'dstip'
+        attacker_direction = 'srcip'
         attacker = dstip
         source_target_tag = 'Botnet'
         evidence_type = 'Command-and-Control-channels-detection'

modules/threat_intelligence/threat_intelligence.py

@@ -106,7 +106,7 @@ def set_evidence_malicious_asn(
         """
         :param asn_info: the malicious asn info taken from own_malicious_iocs.csv
         """
-        attacker_direction = 'dstip'
+        attacker_direction = 'srcip'
         category = 'Anomaly.Traffic'
         evidence_type = 'ThreatIntelligenceBlacklistedASN'
         confidence = 0.8
@@ -161,16 +161,18 @@ def set_evidence_malicious_ip(
 
         confidence = 1
         category = 'Anomaly.Traffic'
-        if 'src' in attacker_direction:
+        if 'src' in ip_state:
             direction = 'from'
             opposite_dir = 'to'
             victim = daddr
-        elif 'dst' in attacker_direction:
+            attacker_direction = 'srcip'
+        elif 'dst' in ip_state:
             direction = 'to'
             opposite_dir = 'from'
             victim = profileid.split("_")[-1]
+            attacker_direction = 'srcip'
         else:
-            # attacker_dir is not specified?
+            # ip_state is not specified?
             return
 
 
@@ -997,9 +999,14 @@ def pre_main(self):
         # Load the local Threat Intelligence files that are
         # stored in the local folder self.path_to_local_ti_files
         # The remote files are being loaded by the update_manager
-        self.update_local_file('own_malicious_iocs.csv')
-        self.update_local_file('own_malicious_JA3.csv')
-        self.update_local_file('own_malicious_JARM.csv')
+        local_files = (
+            'own_malicious_iocs.csv',
+            'own_malicious_JA3.csv',
+            'own_malicious_JARM.csv',
+        )
+        for local_file in local_files:
+            self.update_local_file(local_file)
+
         self.circllu_calls_thread.start()
 
     def main(self):

slips_files/core/database/database_manager.py

@@ -127,6 +127,18 @@ def get_output_dir(self, *args, **kwargs):
     def get_input_file(self, *args, **kwargs):
         return self.rdb.get_input_file(*args, **kwargs)
 
+
+    def get_accumulated_threat_level(self, *args, **kwargs):
+        return self.rdb.get_accumulated_threat_level(*args, **kwargs)
+
+
+    def set_accumulated_threat_level(self, *args, **kwargs):
+        return self.rdb.set_accumulated_threat_level(*args, **kwargs)
+
+
+    def update_accumulated_threat_level(self, *args, **kwargs):
+        return self.rdb.update_accumulated_threat_level(*args, **kwargs)
+
     def setInfoForIPs(self, *args, **kwargs):
         return self.rdb.setInfoForIPs(*args, **kwargs)