Merge remote-tracking branch 'origin/develop' into fork/patch-7

# Conflicts:
#	modules/threat_intelligence/threat_intelligence.py

.dockerignore

@@ -1,26 +1,28 @@
-# slips generated alerts
-output/alerts.log
-output/alerts.json
-2021-*/
-2022-*/
+# Ignore all .log files in output/ dir
+output/*.log
+**/alerts.json
+**/.git
 
+#node modules
+**/node_modules
 
-# zeek log files
-zeek_files/*.log
 
-yara-python/*
+# files with API keys/tokens
+config/slack_bot_token_secret
+config/warden.conf
+config/RiskIQ_credentials
+
 
 Datasett/
+Dataset/
 
 # virtual env files
 ENV/*
+env/*
 
 # compiled yara rules
 modules/leak_detector/yara_rules/compiled/
 
-# api keys
-modules/RiskIQ/api_key_secret
-modules/virustotal/api_key_secret
 
 .git/
 .cache/
@@ -30,16 +32,12 @@ __pycache__/
 
 # CESNET module logs
 *.log
-modules/CESNET/*.log
 
 # redis backups
-redis_backups/*.rdb
+**/*.rdb
 
 # for IntelliJ IDE
 .idea/
 
 # for vscode
 .vscode/
-
-
-

.github/pull_request_template.md

@@ -18,6 +18,15 @@
 [ ] - Not correct; marked as **not** done
 -->
 
+
+## Steps you followed to test the changes purposed in this PR:
+
+
+* 
+*
+
+
+
 ## Check List (Check all the applicable boxes) <!-- Follow the above conventions to check the box -->
 
 - [ ] My code follows the code style of this project.
@@ -31,6 +40,8 @@
 
 <!-- Add all the screenshots which support your changes -->
 
+<!-- Add screenshots with the passing unit and integration tests locally -->
+
 ## Note to reviewers
 
 <!-- Add notes to reviewers if applicable -->

.github/workflows/CI-production-testing.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
 
-  test_slips_locally:
+  unit_and_integration_tests:
     # runs the tests on a GH VM
     runs-on: ubuntu-20.04
     # 2 hours timeout
@@ -39,31 +39,148 @@ jobs:
         python-version: "3.8"
 
     - name: Install Python dependencies
-      run:
+      run: |
         python -m pip install --upgrade pip
-        pip install -r install/requirements.txt
+        grep -v 'black' install/requirements.txt | xargs pip3 install  --no-cache-dir
+        pip install coverage
 
     - name: Start redis server
       run: redis-server --daemonize yes
 
     - name: Run unit tests
-      run: python3  -m pytest tests/ --ignore="tests/test_database.py" --ignore="tests/integration_tests" -n 7 -p no:warnings -vv -s
+      run: coverage run --source=./ -m pytest tests/ --ignore="tests/test_database.py" --ignore="tests/integration_tests" -n 7 -p no:warnings -vv -s
+
 
     - name: Run database unit tests
-      run: python3  -m pytest tests/test_database.py -p no:warnings -vv
+      run: |
+        coverage run --source=./ -m pytest tests/test_database.py -p no:warnings -vv
+        coverage report --include="slips_files/core/database/*"
+        coverage html --include="slips_files/core/database/*" -d coverage_reports/database
 
     - name: Clear redis cache
       run: ./slips.py -cc
 
     - name: Portscan tests
-      run: python3 -m pytest -s tests/integration_tests/test_portscans.py  -p no:warnings -vv
+      run: |
+        coverage run --source=./ -m pytest -s tests/integration_tests/test_portscans.py -p no:warnings -vv
+        coverage report --include="modules/network_discovery/*"
+        coverage html --include="modules/network_discovery/*" -d coverage_reports/network_discovery
 
     - name: Integration tests
-      run: python3 -m pytest -s tests/integration_tests/test_dataset.py  -p no:warnings -vv
+      run: |
+        python3 -m pytest -s tests/integration_tests/test_dataset.py -p no:warnings -vv
+#        coverage run --source=./ -m pytest -s tests/integration_tests/test_dataset.py -p no:warnings -vv
+#        coverage report --include="dataset/*"
+#        coverage html --include="dataset/*" -d coverage_reports/dataset
 
-    - name: Config file tests
-      run: python3 -m pytest -s tests/integration_tests/test_config_files.py  -p no:warnings -vv
+    - name: Flowalerts test
+      run: |
+        coverage run --source=./ -m pytest tests/test_flowalerts.py -p no:warnings -vv
+        coverage report  --include="modules/flowalerts/*"
+        coverage html  --include="modules/flowalerts/*" -d coverage_reports/flowalerts
+
+    - name: Whitelist test
+      run: |
+        coverage run --source=./ -m pytest tests/test_whitelist.py -p no:warnings -vv
+        coverage report  --include="slips_files/core/helpers/whitelist.py*"
+        coverage html  --include="slips_files/core/helpers/whitelist.py*" -d coverage_reports/whitelist
+
+    - name: arp test
+      run: |
+        coverage run --source=./ -m pytest tests/test_arp.py -p no:warnings -vv
+        coverage report  --include="modules/arp/*"
+        coverage html  --include="modules/arp/*" -d coverage_reports/arp
+
+    - name: blocking test
+      run: |
+        coverage run --source=./ -m pytest tests/test_blocking.py -p no:warnings -vv
+        coverage report  --include="modules/blocking/*"
+        coverage html  --include="modules/blocking/*" -d coverage_reports/blocking
+
+    - name: flowhandler test
+      run: |
+        coverage run --source=./ -m pytest tests/test_flow_handler.py -p no:warnings -vv
+        coverage report  --include="slips_files/core/helpers/flow_handler.py*"
+        coverage html  --include="slips_files/core/helpers/flow_handler.py*" -d coverage_reports/flowhandler
+
+    - name: horizontal_portscans test
+      run: |
+        coverage run --source=./ -m pytest tests/test_horizontal_portscans.py -p no:warnings -vv
+        coverage report  --include="modules/network_discovery/horizontal_portscan.py*"
+        coverage html  --include="modules/network_discovery/horizontal_portscan.py*" -d coverage_reports/horizontal_portscan
+
+    - name: http_analyzer test
+      run: |
+        coverage run --source=./ -m pytest tests/test_http_analyzer.py -p no:warnings -vv
+        coverage report  --include="modules/http_analyzer/http_analyzer.py*"
+        coverage html  --include="modules/http_analyzer/http_analyzer.py*" -d coverage_reports/http_analyzer
+
+    - name: vertical_portscans test
+      run: |
+        coverage run --source=./ -m pytest tests/test_vertical_portscan.py -p no:warnings -vv
+        coverage report  --include="modules/network_discovery/vertical_portscan.py*"
+        coverage html  --include="modules/network_discovery/vertical_portscan.py*" -d coverage_reports/vertical_portscan
 
+    - name: virustotal test
+      run: |
+        coverage run --source=./ -m pytest tests/test_virustotal.py -p no:warnings -vv
+        coverage report  --include="modules/virustotal/virustotal.py*"
+        coverage html  --include="modules/virustotal/virustotal.py*" -d coverage_reports/virustotal
+
+    - name: updatemanager test
+      run: |
+        coverage run --source=./ -m pytest tests/test_update_file_manager.py -p no:warnings -vv
+        coverage report  --include="modules/update_manager/update_manager.py*"
+        coverage html  --include="modules/update_manager/update_manager.py*" -d coverage_reports/updatemanager
+
+    - name: threatintelligence test
+      run: |
+        coverage run --source=./ -m pytest tests/test_threat_intelligence.py -p no:warnings -vv
+        coverage report  --include="modules/threat_intelligence/threat_intelligence.py*"
+        coverage html  --include="modules/threat_intelligence/threat_intelligence.py*" -d coverage_reports/threat_intelligence
+
+    - name: slipsutils test
+      run: |
+        coverage run --source=./ -m pytest tests/test_slips_utils.py -p no:warnings -vv
+        coverage report  --include="slips_files/common/slips_utils.py*"
+        coverage html  --include="slips_files/common/slips_utils.py*" -d coverage_reports/slips_utils
+
+    - name: slips test
+      run: |
+        coverage run --source=./ -m pytest tests/test_slips.py -p no:warnings -vv
+        coverage report  --include="slips.py*"
+        coverage html  --include="slips.py*" -d coverage_reports/slips
+
+    - name: profiler test
+      run: |
+        coverage run --source=./ -m pytest tests/test_profiler.py -p no:warnings -vv
+        coverage report  --include="slips_files/core/profiler.py*"
+        coverage html  --include="slips_files/core/profiler.py*" -d coverage_reports/profiler
+
+    - name: leak detector test
+      run: |
+        coverage run --source=./ -m pytest tests/test_leak_detector.py -p no:warnings -vv
+        coverage report  --include="modules/leak_detector/leak_detector.py*"
+        coverage html  --include="modules/leak_detector/leak_detector.py*" -d coverage_reports/leak_detector
+
+    - name: ipinfo test
+      run: |
+        coverage run --source=./ -m pytest tests/test_ip_info.py -p no:warnings -vv
+        coverage report  --include="modules/ip_info/ip_info.py*"
+        coverage html  --include="modules/ip_info/ip_info.py*" -d coverage_reports/ip_info
+
+    - name: input test
+      run: |
+        coverage run --source=./ -m pytest tests/test_inputProc.py -p no:warnings -vv
+        coverage report  --include="slips_files/core/input.py*"
+        coverage html  --include="slips_files/core/input.py*" -d coverage_reports/input
+
+    - name: Config file tests
+      run: |
+        python3 -m pytest -s tests/integration_tests/test_config_files.py  -p no:warnings -vv
+#       coverage run --source=./ -m pytest -s tests/integration_tests/test_config_files.py -p no:warnings -vv
+#       coverage report --include="dataset/*"
+#       coverage html --include="dataset/*" -d coverage_reports/dataset
 
     - name: Upload Artifact
       # run this job whether the above jobs failed or passed
@@ -73,4 +190,4 @@ jobs:
         name: test_slips_locally-integration-tests-output
         path: |
           output/integration_tests
-
+          coverage_reports/

.github/workflows/CI-publishing.yml

@@ -90,6 +90,8 @@ jobs:
         ref: 'master'
         # Fetch all history for all tags and branches
         fetch-depth: ''
+        submodules: true
+
 
     - name: Login to DockerHub
       uses: docker/login-action@v2
@@ -102,8 +104,7 @@ jobs:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v2
 
-    # build p2p image for ubuntu from dockerfile
-    - name: Build our p2p image from dockerfile
+    - name: Build and push p2p image using dockerfile
       id: docker_build_p2p_for_slips
       uses: docker/build-push-action@v3
       with:
@@ -164,4 +165,4 @@ jobs:
         context: ./
         file: ./docker/dependency-image/Dockerfile
         tags: stratosphereips/slips_dependencies:latest
-        push: true
+        push: true