stratosphereips · AlyaGomaa · Sep 15, 2023 · Jun 30, 2023 · Jul 11, 2023 · Jul 13, 2023
diff --git a/README.md b/README.md
@@ -32,7 +32,7 @@ Slips v1.0.6
 # Slips: Behavioral Machine Learning-Based Intrusion Prevention System
 
 
-Slips is a behavioral intrusion prevention system that uses machine learning to detect malicious behaviors in network traffic. Slips focus on targeted attacks, detection of command and control channels, and providing a good visualization for the analyst. It can analyze network traffic in real-time, network captures such as pcap files, and network flows produced by Suricata, Zeek/Bro, and Argus. Slips processes the input data, analyzes it, and highlights suspicious behavior that needs the analyst's attention.
+Slips is a powerful endpoint behavioral intrusion prevention and detection system that uses machine learning to detect malicious behaviors in network traffic. Slips can work with network traffic in real-time, pcap files, and network flows from popular tools like Suricata, Zeek/Bro, and Argus. Slips threat detection is based on a combination of machine learning models trained to detect malicious behaviors, 40+ threat intelligence feeds and expert heuristics. Slips gathers evidence of malicious behavior and uses extensively trained thresholds to trigger alerts when enough evidence is accumulated.
 
 <img src="https://raw.githubusercontent.com/stratosphereips/StratosphereLinuxIPS/develop/docs/images/slips.gif" width="850px"
 title="Slips in action.">

diff --git a/config/slips.conf b/config/slips.conf
@@ -403,8 +403,19 @@ cpu_profiler_output_limit = 20
 # set the wait time between sampling sequences in seconds (live mode only)
 cpu_profiler_sampling_interval = 20
 
+# [12] Memory Profiling
+
+# enable memory profiling [yes,no]
+memory_profiler_enable = no
+
+# set profiling mode [dev,live]
+memory_profiler_mode = live
+
+# profile all subprocesses [yes,no]
+memory_profiler_multiprocess = yes
+
 ####################
-# [12] enable or disable p2p for slips
+# [13] enable or disable p2p for slips
 [P2P]
 
 # create p2p.log with additional info about peer communications? yes or no

diff --git a/conftest.py b/conftest.py
@@ -19,7 +19,7 @@
 
 
 @pytest.fixture
-def mock_db():
+def mock_rdb():
     # Create a mock version of the database object
     with patch('slips_files.core.database.database_manager.DBManager') as mock:
         yield mock.return_value

diff --git a/docs/features.md b/docs/features.md
@@ -1045,8 +1045,19 @@ Options to enable cpu profiling can be found under the [Profiling] section of th
 ```cpu_profiler_output_limit``` is set to an integer value and only affects the live mode profiling. This option sets the limit on the number of processes output for live mode profiling updates.
 ```cpu_profiler_sampling_interval``` is set to an integer value and only affects the live mode profiling. This option sets the duration in seconds of live mode sampling intervals. It is recommended to set this option greater than 10 seconds otherwise there won't be much useful information captured during sampling.
 
+### Memory Profiling
+Memory profiling can be found in ```slips_files/common/memory_profiler.py```
 
+Just like CPU profiling, it also has supports live and development mode.
+Set ```memory_profiler_enable``` to ```yes``` to enable this feature.
+Set ```memory_profiler_mode``` to ```live``` to use live mode or ```dev``` to use development mode profiling.
 
+#### Live Mode
+This mode shows memory usage stats during the runtime of the program.
+```memory_profiler_multiprocess``` controls whether live mode tracks all processes or only the main process. If set to no, the program will wait for you to connect from a different terminal using the command ```memray live <port_number>```, where port_number is 5000 by default. After connection, the program will continue with its run and the terminal that is connected will receive a feed of the memory statistics. If set to yes, the redis channel "memory_profile" can be used to set pid of the process to be tracked. Only a single process can be tracked at a time. The interface is cumbersome to use from the command line so multiprocess live profiling is intended to be used primarily from the web interface.
+
+#### Development Mode
+When enabled, the profiler will output the profile data into the output directory. The data will be in the ```memoryprofile``` directory of the output directory of the run. Each process during the run of the program will have an associated binary file. Each of the generated binaries will automatically be converted to viewable html files, with each process converted to a flamegraph and table format. All generated files will be denoted by their PID.
 
 ---
 

diff --git a/docs/images/alternate_mem_profiler_testing.png b/docs/images/alternate_mem_profiler_testing.png
diff --git a/docs/images/cpu-profiler-config.png b/docs/images/cpu-profiler-config.png
diff --git a/docs/images/cpu-profiler-live-mode.png b/docs/images/cpu-profiler-live-mode.png
diff --git a/docs/images/cpu-profiler-live-results.png b/docs/images/cpu-profiler-live-results.png
diff --git a/docs/images/cpu-profiler-starting.png b/docs/images/cpu-profiler-starting.png
diff --git a/docs/images/cpu-profiler-termination.png b/docs/images/cpu-profiler-termination.png
diff --git a/docs/images/cpu=profiler-results.png b/docs/images/cpu=profiler-results.png
diff --git a/docs/images/flamegraph.png b/docs/images/flamegraph.png
diff --git a/docs/images/live-mem-profiler.png b/docs/images/live-mem-profiler.png
diff --git a/docs/images/mem-profiler-ending.png b/docs/images/mem-profiler-ending.png
diff --git a/docs/images/mem-profiler-running.png b/docs/images/mem-profiler-running.png
diff --git a/docs/images/mem-profiler-starting.png b/docs/images/mem-profiler-starting.png
diff --git a/docs/images/mem-profiler-table-view.png b/docs/images/mem-profiler-table-view.png
diff --git a/docs/images/memory_multi_process_profiler_structure.png b/docs/images/memory_multi_process_profiler_structure.png
diff --git a/docs/images/memory_profiler_interface.png b/docs/images/memory_profiler_interface.png
diff --git a/docs/images/memory_profiler_structure.png b/docs/images/memory_profiler_structure.png
diff --git a/docs/images/running-vizviewer.png b/docs/images/running-vizviewer.png
diff --git a/docs/images/testing_live_cpu_profiler.png b/docs/images/testing_live_cpu_profiler.png
diff --git a/docs/images/testinig_mem_profiler_live_mode.png b/docs/images/testinig_mem_profiler_live_mode.png
diff --git a/docs/images/testinig_mem_profiler_live_mode_step3_1.png b/docs/images/testinig_mem_profiler_live_mode_step3_1.png
diff --git a/docs/images/testinig_mem_profiler_live_mode_step3_2.png b/docs/images/testinig_mem_profiler_live_mode_step3_2.png
diff --git a/docs/profiling_slips.md b/docs/profiling_slips.md
diff --git a/modules/flowalerts/flowalerts.py b/modules/flowalerts/flowalerts.py
@@ -290,10 +290,9 @@ def check_data_upload(self, sbytes, daddr, uid, profileid, twid):
         """
         Set evidence when 1 flow is sending >= the flow_upload_threshold bytes
         """
-
-
         if (
-            self.is_ignored_ip_data_upload(daddr)
+            not daddr
+            or self.is_ignored_ip_data_upload(daddr)
             or not sbytes
         ):
             return False

diff --git a/process_manager.py b/process_manager.py
@@ -29,9 +29,9 @@ def __init__(self, main):
 
     def start_output_process(self, current_stdout, stderr, slips_logfile):
         output_process = OutputProcess(
-            self.main.db,
             self.main.output_queue,
             self.main.args.output,
+            self.main.redis_port,
             self.termination_event,
             verbose=self.main.args.verbose,
             debug=self.main.args.debug,
@@ -46,9 +46,9 @@ def start_output_process(self, current_stdout, stderr, slips_logfile):
 
     def start_profiler_process(self):
         profiler_process = ProfilerProcess(
-            self.main.db,
             self.main.output_queue,
             self.main.args.output,
+            self.main.redis_port,
             self.termination_event,
             profiler_queue=self.profiler_queue,
         )
@@ -64,9 +64,9 @@ def start_profiler_process(self):
 
     def start_evidence_process(self):
         evidence_process = EvidenceProcess(
-            self.main.db,
             self.main.output_queue,
             self.main.args.output,
+            self.main.redis_port,
             self.termination_event,
         )
         evidence_process.start()
@@ -81,9 +81,9 @@ def start_evidence_process(self):
 
     def start_input_process(self):
         input_process = InputProcess(
-            self.main.db,
             self.main.output_queue,
             self.main.args.output,
+            self.main.redis_port,
             self.termination_event,
             profiler_queue=self.profiler_queue,
             input_type=self.main.input_type,
@@ -229,7 +229,8 @@ def load_modules(self):
             module_class = modules_to_call[module_name]["obj"]
             module = module_class(
                 self.main.output_queue,
-                self.main.db,
+                self.main.args.output,
+                self.main.redis_port,
                 self.termination_event,
             )
             module.start()

diff --git a/requirements.txt b/requirements.txt
@@ -40,3 +40,5 @@ termcolor
 communityid
 viztracer
 yappi
+pytest-sugar
+memray
diff --git a/slips.py b/slips.py
@@ -21,6 +21,7 @@
 import multiprocessing
 from slips_files.common.imports import *
 from slips_files.common.cpu_profiler import CPUProfiler
+from slips_files.common.memory_profiler import MemoryProfiler
 from exclusiveprocess import Lock, CannotAcquireLock
 from redis_manager import RedisManager
 from metadata_manager import MetadataManager
@@ -124,6 +125,51 @@ def cpu_profiler_release(self):
         if self.cpuProfilerEnabled and not self.cpuProfilerMultiprocess:
             self.cpuProfiler.stop()
             self.cpuProfiler.print()
+
+    def memory_profiler_init(self):
+        self.memoryProfilerEnabled = slips.conf.get_memory_profiler_enable() == "yes"
+        memoryProfilerMode = slips.conf.get_memory_profiler_mode()
+        memoryProfilerMultiprocess = slips.conf.get_memory_profiler_multiprocess() == "yes"
+        if self.memoryProfilerEnabled:
+            output_dir = os.path.join(slips.args.output,'memoryprofile/')
+            if not os.path.exists(output_dir):
+                os.makedirs(output_dir)
+            output_file = os.path.join(output_dir, 'memory_profile.bin')
+            self.memoryProfiler = MemoryProfiler(output_file, db=self.db, mode=memoryProfilerMode, multiprocess=memoryProfilerMultiprocess)
+            self.memoryProfiler.start()
+
+    def memory_profiler_release(self):
+        if self.memoryProfilerEnabled:
+            self.memoryProfiler.stop()
+
+    def memory_profiler_multiproc_test(self):
+        def target_function():
+            print("Target function started")
+            time.sleep(5)
+
+        def mem_function():
+            print("Mem function started")
+            while True:
+                time.sleep(1)
+                array = []
+                for i in range(1000000):
+                    array.append(i)
+        processes = []
+        num_processes = 3
+
+        for _ in range(num_processes):
+            process = multiprocessing.Process(target=target_function if _%2 else mem_function)
+            process.start()
+            processes.append(process)
+
+        # Message passing
+        self.db.publish("memory_profile", processes[1].pid) # successful
+        # subprocess.Popen(["memray", "live", "1234"])
+        time.sleep(5) # target_function will timeout and tracker will be cleared
+        self.db.publish("memory_profile", processes[0].pid) # end but maybe don't start
+        time.sleep(5) # mem_function will get tracker started
+        self.db.publish("memory_profile", processes[0].pid) # start successfully
+        input()
 
     def get_slips_version(self):
         version_file = 'VERSION'
@@ -175,7 +221,10 @@ def update_local_TI_files(self):
             # so this function will only be allowed to run from 1 slips instance.
             with Lock(name="slips_ports_and_orgs"):
                 # pass a dummy termination event for update manager to update orgs and ports info
-                update_manager = UpdateManager(self.output_queue, self.db, multiprocessing.Event())
+                update_manager = UpdateManager(self.output_queue,
+                                               self.args.output,
+                                               self.redis_port,
+                                               multiprocessing.Event())
                 update_manager.update_ports_info()
                 update_manager.update_org_files()
         except CannotAcquireLock:
@@ -501,6 +550,10 @@ def start(self):
                 })
 
             self.cpu_profiler_init()
+            self.memory_profiler_init()
+            # uncomment line to see that memory profiler works correctly
+            # Should print out red text if working properly
+            # self.memory_profiler_multiproc_test()
 
             # if stdout is redirected to a file,
             # tell outputProcess.py to redirect it's output as well
@@ -705,5 +758,7 @@ def sig_handler(sig, frame):
             daemon.start()
     else:
         # interactive mode
+        pass
         slips.start()
     slips.cpu_profiler_release()
+    slips.memory_profiler_release()