[dev] Add sample data ingestor

elkserver/mounts/sample-data/filebeat.yml

@@ -0,0 +1,179 @@
+filebeat.inputs:
+
+# BEGIN REDIRECTORS CONFIG
+- type: log
+  enabled: true
+  fields_under_root: true
+  paths:
+    - /var/log/sample-data/haproxy.log
+  fields:
+    infra:
+      log:
+        type: redirtraffic
+    redir:
+      program: haproxy
+- type: log
+  enabled: true
+  fields_under_root: true
+  paths:
+    - /var/log/sample-data/apache2.log
+  fields:
+    infra:
+      log:
+        type: redirtraffic
+    redir:
+      program: apache
+- type: log
+  enabled: true
+  fields_under_root: true
+  paths:
+    - /var/log/sample-data/nginx.log
+  fields:
+    infra:
+      log:
+        type: redirtraffic
+    redir:
+      program: nginx
+# END REDIRECTORS CONFIG
+
+# BEGIN COBALT STRIKE CONFIG
+- type: log
+  scan_frequency: 5s
+  enabled: true
+  fields_under_root: true
+  paths:
+    - /var/log/sample-data/cobaltstrike/logs/*/events.log
+  fields:
+    infra:
+      log:
+        type: rtops
+    c2:
+      program: cobaltstrike
+      log:
+        type: events
+
+- type: log
+  scan_frequency: 5s
+  enabled: true
+  fields_under_root: true
+  paths:
+    - /var/log/sample-data/cobaltstrike/logs/*/weblog.log
+  fields:
+    infra:
+      log:
+        type: rtops
+    c2:
+      program: cobaltstrike
+      log:
+        type: weblog
+
+- type: log
+  scan_frequency: 5s
+  enabled: true
+  fields_under_root: true
+  paths:
+    - /var/log/sample-data/cobaltstrike/logs/*/downloads.log
+  fields:
+    infra:
+      log:
+        type: rtops
+    c2:
+      program: cobaltstrike
+      log:
+        type: downloads
+
+- type: log
+  scan_frequency: 5s
+  enabled: true
+  fields_under_root: true
+  paths:
+    - /var/log/sample-data/cobaltstrike/data/export_credentials.tsv
+  fields:
+    infra:
+      log:
+        type: rtops
+    c2:
+      program: cobaltstrike
+      log:
+        type: credentials
+
+- type: log
+  scan_frequency: 5s
+  enabled: true
+  fields_under_root: true
+  paths:
+    - /var/log/sample-data/cobaltstrike/logs/*/*/beacon_*.log
+    - /var/log/sample-data/cobaltstrike/logs/*/*/ssh_*.log
+  # Since Cobalt Strike version 3.14 the time format in the logs is changed. Here we use regex 'or' function (expr1)|(expr2) to match new or old format
+  multiline.pattern: '(^\d\d\/\d\d\s\d\d\:\d\d\:\d\d\sUTC\s\[)|(^\d\d\/\d\d\s\d\d\:\d\d\:\d\d\s\[)' # match "06/19 12:32:56 UTC [" or "06/19 12:32:56 ["
+  multiline.negate: true
+  multiline.match: after
+  multiline.max_lines: 100000
+  fields:
+    infra:
+      log:
+        type: rtops
+    c2:
+      program: cobaltstrike
+      log:
+        type: beacon
+
+- type: log
+  scan_frequency: 5s
+  enabled: true
+  fields_under_root: true
+  paths:
+    - /var/log/sample-data/cobaltstrike/logs/*/*/keystrokes/keystrokes_*.txt
+  # Since Cobalt Strike version 3.14 the time format in the logs is changed. Here we use regex 'or' function (expr1)|(expr2) to match new or old format
+  multiline.pattern: '(^\d\d\/\d\d\s\d\d\:\d\d\:\d\d\sUTC\s\[)|(^\d\d\/\d\d\s\d\d\:\d\d\:\d\d\s\[)' # match "06/19 12:32:56 UTC [" or "06/19 12:32:56 ["
+  multiline.negate: true
+  multiline.match: after
+  multiline.max_lines: 100000
+  fields:
+    infra:
+      log:
+        type: rtops
+    c2:
+      program: cobaltstrike
+      log:
+        type: keystrokes
+
+- type: log
+  scan_frequency: 5s
+  enabled: true
+  fields_under_root: true
+  paths:
+    - /var/log/sample-data/cobaltstrike/logs/*/*/screenshots.log
+  # Since Cobalt Strike version 3.14 the time format in the logs is changed. Here we use regex 'or' function (expr1)|(expr2) to match new or old format
+  multiline.pattern: '(^\d\d\/\d\d\s\d\d\:\d\d\:\d\d\sUTC\s\[)|(^\d\d\/\d\d\s\d\d\:\d\d\:\d\d\s\[)' # match "06/19 12:32:56 UTC [" or "06/19 12:32:56 ["
+  multiline.negate: true
+  multiline.match: after
+  multiline.max_lines: 100000
+  fields:
+    infra:
+      log:
+        type: rtops
+    c2:
+      program: cobaltstrike
+      log:
+        type: screenshots
+
+# END COBALT STRIKE CONFIG
+
+filebeat.config.modules:
+  path: ${path.config}/modules.d/*.yml
+  reload.enabled: false
+
+setup.template.settings:
+  index.number_of_shards: 3
+
+name: "sample-client"
+fields_under_root: true
+fields:
+  infra:
+    attack_scenario: sample-scenario
+
+output.logstash:
+  hosts: ["redelk-logstash:5044"]
+  ssl.certificate_authorities: ["/usr/share/filebeat/redelkCA.crt"]
+  ssl.verification_mode: none

elkserver/mounts/sample-data/logs/cobaltstrike/logs/200329/10.99.1.4/beacon_1233239984.log

@@ -0,0 +1,53 @@
+03/29 15:18:46 UTC [metadata] 13.80.254.1 <- 10.99.1.4; computer: LabMaster; user: outflank *; process: testbeacon.exe; pid: 2004; os: Windows; version: 6.2; beacon arch: x64 (x64)
+03/29 15:19:00 UTC [input] <neo> upload
+03/29 15:19:15 UTC [input] <neo> pwd
+03/29 15:19:15 UTC [task] <> Tasked beacon to print working directory
+03/29 15:19:19 UTC [checkin] host called home, sent: 8 bytes
+03/29 15:19:20 UTC [output]
+Current directory is C:\Users\outflank\Desktop
+
+03/29 15:19:39 UTC [input] <neo> cd \windows\temp
+03/29 15:19:39 UTC [task] <> cd \windows\temp
+03/29 15:19:39 UTC [checkin] host called home, sent: 21 bytes
+03/29 15:19:40 UTC [input] <neo> pwd
+03/29 15:19:40 UTC [task] <> Tasked beacon to print working directory
+03/29 15:19:43 UTC [checkin] host called home, sent: 8 bytes
+03/29 15:19:43 UTC [output]
+Current directory is C:\windows\temp
+
+03/29 15:19:49 UTC [input] <neo> upload
+03/29 15:19:56 UTC [task] <> Tasked beacon to upload C:\Users\outflank\Desktop\OfferNr2020F6592_salary.doc as OfferNr2020F6592_salary.doc
+03/29 15:19:56 UTC [indicator] file: f06d1ae4cbde03cde3898f05b841850f 150016 bytes OfferNr2020F6592_salary.doc
+03/29 15:19:58 UTC [checkin] host called home, sent: 150055 bytes
+03/29 15:20:21 UTC [input] <neo> ls
+03/29 15:20:21 UTC [task] <> Tasked beacon to list files in .
+03/29 15:20:21 UTC [checkin] host called home, sent: 19 bytes
+03/29 15:20:21 UTC [output]
+C:\windows\temp\*
+D	0	03/29/2020 15:20:01	.
+D	0	03/29/2020 15:20:01	..
+D	0	03/29/2020 11:18:45	C4663637-44E3-43AA-9240-B6235C0B5998-Sigs
+F	33311	03/29/2020 09:50:19	chrome_installer.log
+D	0	03/29/2020 09:37:54	Crashpad
+F	0	03/29/2020 09:27:39	DMI6A08.tmp
+D	0	03/29/2020 09:43:18	hsperfdata_LabMaster$
+F	0	03/29/2020 11:24:19	LabMaster-20200329-1124.log
+F	8670	03/29/2020 11:25:55	LabMaster-20200329-1125.log
+F	14480	03/29/2020 11:25:59	LabMaster-20200329-1125a.log
+F	11410	03/29/2020 11:25:59	LabMaster-20200329-1125b.log
+F	32790	03/29/2020 11:26:29	LabMaster-20200329-1126.log
+F	10400	03/29/2020 11:18:45	MpCmdRun.log
+F	18736	03/29/2020 11:18:45	MpSigStub.log
+F	150016	03/29/2020 15:19:58	OfferNr2020F6592_salary.doc
+F	0	03/29/2020 11:24:19	officeclicktorun.exe_streamserver(202003291124191078).log
+F	102	03/29/2020 09:17:42	silconfig.log
+D	0	03/29/2020 09:18:28	winrmdone
+D	0	03/29/2020 09:18:24	winrmrunning
+
+
+03/29 15:29:24 UTC [input] <neo> exit
+03/29 15:29:24 UTC [task] <> Tasked beacon to exit
+03/29 15:29:28 UTC [checkin] host called home, sent: 8 bytes
+03/29 15:29:28 UTC [output]
+beacon exit.
+

elkserver/mounts/sample-data/logs/cobaltstrike/logs/200329/10.99.1.4/beacon_2019412980.log

@@ -0,0 +1,95 @@
+03/29 14:54:56 UTC [metadata] 13.80.254.1 <- 10.99.1.4; computer: LabMaster; user: outflank *; process: rundll32.exe; pid: 3568; os: Windows; version: 10.0; beacon arch: x86 (x64)
+03/29 15:00:18 UTC [input] <MarcS> ps
+03/29 15:00:18 UTC [task] <T1057> Tasked beacon to list processes
+03/29 15:00:22 UTC [checkin] host called home, sent: 12 bytes
+03/29 15:00:22 UTC [output]
+[System Process]	0	0
+System	0	4	x64		0
+smss.exe	4	304	x64	NT AUTHORITY\SYSTEM	0
+csrss.exe	416	428
+wininit.exe	416	492	x64	NT AUTHORITY\SYSTEM	0
+csrss.exe	484	500
+winlogon.exe	484	552	x64	NT AUTHORITY\SYSTEM	1
+services.exe	492	620	x64	NT AUTHORITY\SYSTEM	0
+lsass.exe	492	628	x64	NT AUTHORITY\SYSTEM	0
+svchost.exe	620	720	x64	NT AUTHORITY\SYSTEM	0
+svchost.exe	620	764	x64	NT AUTHORITY\NETWORK SERVICE	0
+svchost.exe	620	876	x64	NT AUTHORITY\SYSTEM	0
+dwm.exe	552	908
+svchost.exe	620	956	x64	NT AUTHORITY\LOCAL SERVICE	0
+svchost.exe	620	1008	x64	NT AUTHORITY\SYSTEM	0
+svchost.exe	620	1408	x64	NT AUTHORITY\LOCAL SERVICE	0
+svchost.exe	620	1416	x64	NT AUTHORITY\LOCAL SERVICE	0
+svchost.exe	620	1560	x64	NT AUTHORITY\NETWORK SERVICE	0
+svchost.exe	620	1700	x64	NT AUTHORITY\SYSTEM	0
+svchost.exe	620	1840	x64	NT AUTHORITY\SYSTEM	0
+svchost.exe	620	2016	x64	NT AUTHORITY\LOCAL SERVICE	0
+VSSVC.exe	620	2024	x64	NT AUTHORITY\SYSTEM	0
+spoolsv.exe	620	1168	x64	NT AUTHORITY\SYSTEM	0
+svchost.exe	620	1360	x64	NT AUTHORITY\SYSTEM	0
+svchost.exe	620	1960	x64	NT AUTHORITY\SYSTEM	0
+MsMpEng.exe	620	1988	x64	NT AUTHORITY\SYSTEM	0
+svchost.exe	620	2436	x64	NT AUTHORITY\NETWORK SERVICE	0
+WaAppAgent.exe	620	3244	x64	NT AUTHORITY\SYSTEM	0
+LogonUI.exe	552	3412	x64	NT AUTHORITY\SYSTEM	1
+rundll32.exe	876	3656	x64	NT AUTHORITY\SYSTEM	0
+rundll32.exe	876	3772	x64	NT AUTHORITY\SYSTEM	0
+WindowsAzureTelemetryService.exe	620	3980	x64	NT AUTHORITY\SYSTEM	0
+WindowsAzureGuestAgent.exe	620	2384	x64	NT AUTHORITY\SYSTEM	0
+WaSecAgentProv.exe	3244	688	x64	NT AUTHORITY\SYSTEM	0
+conhost.exe	688	3900	x64	NT AUTHORITY\SYSTEM	0
+svchost.exe	620	1504	x64	NT AUTHORITY\NETWORK SERVICE	0
+msdtc.exe	620	2236	x64	NT AUTHORITY\NETWORK SERVICE	0
+WindowsAzureNetAgent.exe	620	3896	x64	NT AUTHORITY\SYSTEM	0
+VFPlugin.exe	3896	2932	x64	NT AUTHORITY\SYSTEM	0
+conhost.exe	2932	3576	x64	NT AUTHORITY\SYSTEM	0
+csrss.exe	440	1312
+winlogon.exe	440	2980	x64	NT AUTHORITY\SYSTEM	2
+dwm.exe	2980	1288
+rdpclip.exe	2436	2920	x64	LabMaster\outflank	2
+RuntimeBroker.exe	720	808	x64	LabMaster\outflank	2
+sihost.exe	876	2368	x64	LabMaster\outflank	2
+explorer.exe	2136	3636	x64	LabMaster\outflank	2
+svchost.exe	620	2632	x64	LabMaster\outflank	2
+taskhostw.exe	876	3368	x64	LabMaster\outflank	2
+ShellExperienceHost.exe	720	4664	x64	LabMaster\outflank	2
+SearchUI.exe	720	4760	x64	LabMaster\outflank	2
+powershell.exe	808	3084	x64	LabMaster\outflank	2
+conhost.exe	3084	4844	x64	LabMaster\outflank	2
+svchost.exe	620	3812	x64	NT AUTHORITY\LOCAL SERVICE	0
+taskhostw.exe	876	5548	x64	LabMaster\outflank	2
+OfficeClickToRun.exe	620	4216	x64	NT AUTHORITY\SYSTEM	0
+fontdrvhost.exe	2980	4908
+AppVShNotify.exe	4216	2172	x64	LabMaster\outflank	2
+mstsc.exe	3636	2196	x64	LabMaster\outflank	2
+mstsc.exe	3636	4900	x64	LabMaster\outflank	2
+javaw.exe	3272	3832	x64	LabMaster\outflank	2
+testbeacon.exe	3636	3324	x64	LabMaster\outflank	2
+testbeacon-longhaul.exe	3636	5792	x64	LabMaster\outflank	2
+notepad.exe	3636	4804	x64	LabMaster\outflank	2
+chrome.exe	808	1060	x64	LabMaster\outflank	2
+chrome.exe	1060	2888	x64	LabMaster\outflank	2
+chrome.exe	1060	4640	x64	LabMaster\outflank	2
+chrome.exe	1060	4240	x64	LabMaster\outflank	2
+chrome.exe	1060	5788	x64	LabMaster\outflank	2
+chrome.exe	1060	4052	x64	LabMaster\outflank	2
+WmiPrvSE.exe	720	5300
+chrome.exe	1060	5380	x64	LabMaster\outflank	2
+chrome.exe	1060	2508	x64	LabMaster\outflank	2
+WINWORD.EXE	5648	5096	x86	LabMaster\outflank	2
+rundll32.exe	5096	3568	x86	LabMaster\outflank	2
+WmiPrvSE.exe	720	2868
+
+
+03/29 15:00:35 UTC [input] <MarcS> screenshot
+03/29 15:00:35 UTC [task] <T1113, T1093> Tasked beacon to take screenshot
+03/29 15:00:36 UTC [checkin] host called home, sent: 162370 bytes
+03/29 15:00:37 UTC [output]
+received screenshot (253367 bytes)
+
+03/29 15:01:00 UTC [input] <MarcS> exit
+03/29 15:01:00 UTC [task] <> Tasked beacon to exit
+03/29 15:01:01 UTC [checkin] host called home, sent: 8 bytes
+03/29 15:01:01 UTC [output]
+beacon exit.
+

elkserver/mounts/sample-data/logs/cobaltstrike/logs/200329/10.99.1.4/beacon_496538698.log

@@ -0,0 +1,34 @@
+03/29 13:16:36 UTC [metadata] 13.80.254.1 <- 10.99.1.4; computer: LabMaster; user: outflank *; process: testbeacon.exe; pid: 3324; os: Windows; version: 6.2; beacon arch: x64 (x64)
+03/29 13:26:57 UTC [input] <MarcS> screenshot
+03/29 13:26:57 UTC [task] <T1113, T1093> Tasked beacon to take screenshot
+03/29 13:27:01 UTC [checkin] host called home, sent: 197186 bytes
+03/29 13:27:02 UTC [output]
+received screenshot (166469 bytes)
+
+03/29 13:54:31 UTC [input] <neo> keylogger
+03/29 13:54:31 UTC [task] <T1056, T1093> Tasked beacon to log keystrokes
+03/29 13:54:35 UTC [checkin] host called home, sent: 81474 bytes
+03/29 13:55:09 UTC [output]
+received keystrokes
+
+03/29 13:55:18 UTC [output]
+received keystrokes
+
+03/29 13:55:34 UTC [input] <neo> jobkill
+03/29 13:55:34 UTC [error] jobkill error: not enough arguments
+03/29 13:55:36 UTC [input] <neo> jobs
+03/29 13:55:36 UTC [task] <> Tasked beacon to list jobs
+03/29 13:55:37 UTC [checkin] host called home, sent: 8 bytes
+03/29 13:55:37 UTC [output]
+1	0	keystroke logger
+
+
+03/29 13:55:42 UTC [input] <neo> jobkill 1
+03/29 13:55:42 UTC [task] <> Tasked beacon to kill job 1
+03/29 13:55:42 UTC [checkin] host called home, sent: 10 bytes
+03/29 15:01:06 UTC [input] <MarcS> exit
+03/29 15:01:06 UTC [task] <> Tasked beacon to exit
+03/29 15:01:08 UTC [checkin] host called home, sent: 8 bytes
+03/29 15:01:08 UTC [output]
+beacon exit.
+

elkserver/mounts/sample-data/logs/cobaltstrike/logs/200329/10.99.1.4/keystrokes/keystrokes_496538698.txt

@@ -0,0 +1,23 @@
+03/29 13:55:09 UTC Received keystrokes
+
+
+
+CCobalt Strike
+E=======
+r2[command]
+
+CRun
+E=======
+notepad
+
+C
+E=======
+
+
+
+CUntitled - Notepad
+E=======
+test for keylogger
+03/29 13:55:18 UTC Received keystrokes
+
+ - hello :-)