-
Notifications
You must be signed in to change notification settings - Fork 181
199 lines (164 loc) · 8.68 KB
/
CI-staging.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
# This workflow will install Slips dependencies and run unit tests
name: CI-staging
on:
push:
branches:
- '!develop'
- '!master'
pull_request:
branches:
- 'develop'
- '!master'
jobs:
test_slips_locally:
# runs the tests on a GH VM
runs-on: ubuntu-20.04
# 2 hours timeout
timeout-minutes: 7200
steps:
- uses: actions/checkout@v3
with:
ref: 'develop'
# Fetch all history for all tags and branches
fetch-depth: ''
- name: Install slips dependencies
run: sudo apt-get update --fix-missing && sudo apt-get -y --no-install-recommends install python3 redis-server python3-pip python3-certifi python3-dev build-essential file lsof net-tools iproute2 iptables python3-tzlocal nfdump tshark git whois golang nodejs notify-osd yara libnotify-bin
- name: Install Zeek
run: |
sudo echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_20.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list
curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null
sudo apt update
sudo apt install -y --no-install-recommends zeek
sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro
- name: Set up Python 3.8
uses: actions/setup-python@v2
with:
python-version: "3.8"
- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
# exclude black when installing slips dependencies due to dependency conflict with tensorflow
grep -v 'black' install/requirements.txt | xargs pip3 install --no-cache-dir
pip install coverage
- name: Start redis server
run: redis-server --daemonize yes
- name: Run unit tests
run: coverage run --source=./ -m pytest tests/ --ignore="tests/test_database.py" --ignore="tests/integration_tests" -n 7 -p no:warnings -vv -s
- name: Run database unit tests
run: |
coverage run --source=./ -m pytest tests/test_database.py -p no:warnings -vv
coverage report --include="slips_files/core/database/*"
coverage html --include="slips_files/core/database/*" -d coverage_reports/database
- name: Clear redis cache
run: ./slips.py -cc
- name: Portscan tests
run: |
coverage run --source=./ -m pytest -s tests/integration_tests/test_portscans.py -p no:warnings -vv
coverage report --include="modules/network_discovery/*"
coverage html --include="modules/network_discovery/*" -d coverage_reports/network_discovery
- name: Integration tests
run: |
python3 -m pytest -s tests/integration_tests/test_dataset.py -p no:warnings -vv
# coverage run --source=./ -m pytest -s tests/integration_tests/test_dataset.py -p no:warnings -vv
# coverage report --include="dataset/*"
# coverage html --include="dataset/*" -d coverage_reports/dataset
- name: Flowalerts test
run: |
coverage run --source=./ -m pytest tests/test_flowalerts.py -p no:warnings -vv
coverage report --include="modules/flowalerts/*"
coverage html --include="modules/flowalerts/*" -d coverage_reports/flowalerts
- name: Whitelist test
run: |
coverage run --source=./ -m pytest tests/test_whitelist.py -p no:warnings -vv
coverage report --include="slips_files/core/helpers/whitelist.py*"
coverage html --include="slips_files/core/helpers/whitelist.py*" -d coverage_reports/whitelist
- name: arp test
run: |
coverage run --source=./ -m pytest tests/test_arp.py -p no:warnings -vv
coverage report --include="modules/arp/*"
coverage html --include="modules/arp/*" -d coverage_reports/arp
- name: blocking test
run: |
coverage run --source=./ -m pytest tests/test_blocking.py -p no:warnings -vv
coverage report --include="modules/blocking/*"
coverage html --include="modules/blocking/*" -d coverage_reports/blocking
- name: flowhandler test
run: |
coverage run --source=./ -m pytest tests/test_flow_handler.py -p no:warnings -vv
coverage report --include="slips_files/core/helpers/flow_handler.py*"
coverage html --include="slips_files/core/helpers/flow_handler.py*" -d coverage_reports/flowhandler
- name: horizontal_portscans test
run: |
coverage run --source=./ -m pytest tests/test_horizontal_portscans.py -p no:warnings -vv
coverage report --include="modules/network_discovery/horizontal_portscan.py*"
coverage html --include="modules/network_discovery/horizontal_portscan.py*" -d coverage_reports/horizontal_portscan
- name: http_analyzer test
run: |
coverage run --source=./ -m pytest tests/test_http_analyzer.py -p no:warnings -vv
coverage report --include="modules/http_analyzer/http_analyzer.py*"
coverage html --include="modules/http_analyzer/http_analyzer.py*" -d coverage_reports/http_analyzer
- name: vertical_portscans test
run: |
coverage run --source=./ -m pytest tests/test_vertical_portscans.py -p no:warnings -vv
coverage report --include="modules/network_discovery/vertical_portscan.py*"
coverage html --include="modules/network_discovery/vertical_portscan.py*" -d coverage_reports/vertical_portscan
- name: virustotal test
run: |
coverage run --source=./ -m pytest tests/test_virustotal.py -p no:warnings -vv
coverage report --include="modules/virustotal/virustotal.py*"
coverage html --include="modules/virustotal/virustotal.py*" -d coverage_reports/virustotal
- name: updatemanager test
run: |
coverage run --source=./ -m pytest tests/test_update_file_manager.py -p no:warnings -vv
coverage report --include="modules/update_manager/update_manager.py*"
coverage html --include="modules/update_manager/update_manager.py*" -d coverage_reports/updatemanager
- name: threatintelligence test
run: |
coverage run --source=./ -m pytest tests/test_threat_intelligence.py -p no:warnings -vv
coverage report --include="modules/threat_intelligence/threat_intelligence.py*"
coverage html --include="modules/threat_intelligence/threat_intelligence.py*" -d coverage_reports/threat_intelligence
- name: slipsutils test
run: |
coverage run --source=./ -m pytest tests/test_slips_utils.py -p no:warnings -vv
coverage report --include="slips_files/common/slips_utils.py*"
coverage html --include="slips_files/common/slips_utils.py*" -d coverage_reports/slips_utils
- name: slips test
run: |
coverage run --source=./ -m pytest tests/test_slips.py -p no:warnings -vv
coverage report --include="slips.py*"
coverage html --include="slips.py*" -d coverage_reports/slips
- name: profiler test
run: |
coverage run --source=./ -m pytest tests/test_profiler.py -p no:warnings -vv
coverage report --include="slips_files/core/profiler.py*"
coverage html --include="slips_files/core/profiler.py*" -d coverage_reports/profiler
- name: leak detector test
run: |
coverage run --source=./ -m pytest tests/test_leak_detector.py -p no:warnings -vv
coverage report --include="modules/leak_detector/leak_detector.py*"
coverage html --include="modules/leak_detector/leak_detector.py*" -d coverage_reports/leak_detector
- name: ipinfo test
run: |
coverage run --source=./ -m pytest tests/test_ip_info.py -p no:warnings -vv
coverage report --include="modules/ip_info/ip_info.py*"
coverage html --include="modules/ip_info/ip_info.py*" -d coverage_reports/ip_info
- name: input test
run: |
coverage run --source=./ -m pytest tests/test_inputProc.py -p no:warnings -vv
coverage report --include="slips_files/core/input.py*"
coverage html --include="slips_files/core/input.py*" -d coverage_reports/input
- name: Config file tests
run: |
python3 -m pytest -s tests/integration_tests/test_config_files.py -p no:warnings -vv
# coverage run --source=./ -m pytest -s tests/integration_tests/test_config_files.py -p no:warnings -vv
# coverage report --include="dataset/*"
# coverage html --include="dataset/*" -d coverage_reports/dataset
- name: Upload Artifact
# run this job whether the above jobs failed or passed
if: success() || failure()
uses: actions/upload-artifact@v3
with:
name: test_slips_locally-integration-tests-output
path: |
output/integration_tests
coverage_reports/