LME V0.3 Release (#68)

* Updating winlogbeat to support the Elastic Common Schema * updating documentation for new winlogbeat files * Update to ES 7.8 * Updating deployment questions to make them more clear * Providing instructions to update winlogbeat * Adding syslog support * Adding directory creation command * Removing reporting settings as this is currently broken, Fixed in the unreleased 7.8.1 elastic/kibana#69621 * The module files are now actually shipped with ES 7.8.0 so not including them in the git, Removing the version named directory for install as this would break updates. * Removing module files that are no longer needed * changing install script to reflect the fact that network and hashing is now suitable in our recommended sysmon config * Updating winlogbeat to support ECS with forwarded events, Fixes the multi script issues * Updating logstash config to support syslog via pipelines * Adding pipeline upgrade paths and updating documentation * Updating Documentation and screenshots for v0.3 * Adding more screenshots of v0.3 * Adding kibana saved objects encryption key settings * Correcting variable name * adding screenshots for siem activation * Updating siem instructions and adding more screenshots * Removing NGINX mention as this was removed in v0.2
ukncsc · Jul 9, 2020 · d2caa4a · d2caa4a
1 parent e1a4cd4
commit d2caa4a
Show file tree

Hide file tree

Showing 21 changed files with 558 additions and 52 deletions.
diff --git a/Chapter 2 Files/GPO Deployment/update.bat b/Chapter 2 Files/GPO Deployment/update.bat
@@ -53,7 +53,7 @@ xcopy %GLBSYSMONBIN% %SYSMONDIR% /y
 xcopy %GLBSYSMONCONFIG% %SYSMONDIR% /y
 xcopy %GLBSIGCHECK% %SYSMONDIR% /y
 chdir %SYSMONDIR%
-%SYSMONBIN% -i %SYSMONCONF% -accepteula -h md5,sha256 -n -l
+%SYSMONBIN% -i %SYSMONCONF% -accepteula
 sc config Sysmon64 start= auto
 goto :checkversion
 

diff --git a/Chapter 2 Files/SCCM Deployment/Install_Sysmon64.ps1 b/Chapter 2 Files/SCCM Deployment/Install_Sysmon64.ps1
@@ -1 +1 @@
-C:\WINDOWS\Sysmon64.exe -i c:\WINDOWS\sysmonconf.xml -accepteula -h sha1,sha256 -n
+C:\WINDOWS\Sysmon64.exe -i c:\WINDOWS\sysmonconf.xml -accepteula
diff --git a/Chapter 3 Files/deploy.sh b/Chapter 3 Files/deploy.sh
@@ -23,6 +23,8 @@ kibana_system_pass=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -
 logstash_system_pass=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
 logstash_writer=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
 update_user_pass=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+kibanakey=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 42 | head -n 1)
+
 
 echo -e "\e[32m[x]\e[0m Updating logstash configuration with logstash writer"
 cp /opt/lme/Chapter\ 3\ Files/logstash.conf /opt/lme/Chapter\ 3\ Files/logstash.edited.conf
@@ -304,6 +306,7 @@ sed -i "s/ram-count/$ES_RAM/g" /opt/lme/Chapter\ 3\ Files/docker-compose-stack-l
 
 sed -i "s/insertkibanapasswordhere/$kibana_system_pass/g" /opt/lme/Chapter\ 3\ Files/docker-compose-stack-live.yml
 
+sed -i "s/kibanakey/$kibanakey/g" /opt/lme/Chapter\ 3\ Files/docker-compose-stack-live.yml
 
 
 
@@ -355,6 +358,57 @@ chmod 700 /opt/lme/lme_update.sh
 echo -e "\e[32m[x]\e[0m Creating LME update crontab"
 crontab -l | { cat; echo "30 1 * * * /opt/lme/lme_update.sh"; } | crontab -
 
+}
+
+function pipelineupdate(){
+  #ask user for password
+read -e -p "Enter the password for the existing elastic user: " pipeline_elastic_user
+
+curl --cacert certs/root-ca.crt --user elastic:$pipeline_elastic_user -X PUT "https://127.0.0.1:9200/_ingest/pipeline/syslog" -H 'Content-Type: application/json' -d'
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "host",
+        "target_field": "old.provider"
+      }
+    }
+  ]
+}
+'
+
+#create syslog pipeline
+curl --cacert certs/root-ca.crt --user elastic:$pipeline_elastic_user -X PUT "https://127.0.0.1:9200/_ingest/pipeline/winlogbeat" -H 'Content-Type: application/json' -d'
+{
+  "processors": []
+}
+'
+}
+
+function pipelines(){
+#create beats pipeline
+curl --cacert certs/root-ca.crt --user elastic:$elastic_user_pass -X PUT "https://127.0.0.1:9200/_ingest/pipeline/syslog" -H 'Content-Type: application/json' -d'
+{
+  "processors": [
+    {
+      "rename": {
+        "field": "host",
+        "target_field": "old.provider"
+      }
+    }
+  ]
+}
+'
+
+#create syslog pipeline
+curl --cacert certs/root-ca.crt --user elastic:$elastic_user_pass -X PUT "https://127.0.0.1:9200/_ingest/pipeline/winlogbeat" -H 'Content-Type: application/json' -d'
+{
+  "processors": []
+}
+'
+
+
+
 }
 
 function data_retention(){
@@ -490,9 +544,9 @@ DEFAULT_IF="$(route | grep '^default' | grep -o '[^ ]*$')"
 #get ip of the interface
 EXT_IP="$(/sbin/ifconfig $DEFAULT_IF| awk -F ' *|:' '/inet /{print $3}')"
 
-read -e -p "Enter the IP that winlogbeat will use to communicate with this box: " -i "$EXT_IP" logstaship
+read -e -p "Enter the IP of this linux server: " -i "$EXT_IP" logstaship
 
-read -e -p "Enter the DNS name that winlogbeat uses to communicate with this box: " logstashcn
+read -e -p "Enter the DNS name of this linux server, This needs to be resolvable from the Windows Event Collector: " logstashcn
 echo "[x] Configuring winlogbeat config and certificates to use $logstaship as the IP and $logstashcn as the DNS"
 
 #enable auto updates if ubuntu
@@ -579,6 +633,8 @@ fi
 #ILM
 data_retention
 
+#pipeline
+pipeline
 
 echo "##################################################################################"
 echo "## KIBANA/Elasticsearch Credentials are (these will not be accesible again!!!!) ##"
@@ -621,7 +677,7 @@ function update(){
         cp /opt/lme/Chapter\ 3\ Files/logstash.conf /opt/lme/Chapter\ 3\ Files/logstash.edited.conf
 
         # copy pass from old config into var
-        Logstash_Config_Pass="$(awk '{if(/password/) print $3}' < /opt/lme/Chapter\ 3\ Files/logstash.edited.conf.old)"
+        Logstash_Config_Pass="$(awk '{if(/password/) print $3}' < /opt/lme/Chapter\ 3\ Files/logstash.edited.conf.old | head -1)"
 
         # Insert var into new config
         sed -i "s/insertlogstashwriterpasswordhere/$Logstash_Config_Pass/g" /opt/lme/Chapter\ 3\ Files/logstash.edited.conf
@@ -650,7 +706,13 @@ function update(){
 
         #update config with kibana password
         sed -i "s/insertkibanapasswordhere/$Kibanapass_from_conf/g" /opt/lme/Chapter\ 3\ Files/docker-compose-stack-live.yml
-
+
+	      #copy kibana encryption key
+        kibanakey="$(grep -P -o "(?<=XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: ).*" /opt/lme/Chapter\ 3\ Files/docker-compose-stack-live.yml.old)"
+
+        #update config with kibana key
+        sed -i "s/kibanakey/$kibanakey/g" /opt/lme/Chapter\ 3\ Files/docker-compose-stack-live.yml
+
         customlogstashconf
 
 
@@ -671,7 +733,7 @@ function update(){
 
 if [ "$1" == "" ]; then
         echo "No operation specified"
-        echo "Usage:            ./deploy.sh (install/uninstall/update)"
+        echo "Usage:            ./deploy.sh (install/uninstall/update/pipelineupdate)"
         echo "Example:  ./deploy.sh install"
         exit
 elif [ "$1" == "install" ]; then
@@ -680,9 +742,11 @@ elif [ "$1" == "uninstall" ]; then
         uninstall
 elif [ "$1" == "update" ]; then
         update
+elif [ "$1" == "pipelineupdate" ]; then
+        pipelineupdate     
 else
         echo "Invalid operation specified"
-        echo "Usage:            ./deploy.sh (install/uninstall/update)"
+        echo "Usage:            ./deploy.sh (install/uninstall/update/pipelineupdate)"
         echo "Example:  ./deploy.sh install"
         exit
 fi
diff --git a/Chapter 3 Files/docker-compose-stack.yml b/Chapter 3 Files/docker-compose-stack.yml
@@ -1,12 +1,12 @@
 version: '3.3'
 ##########################
 # LME Stack deploy file  #
-# Version 0.1 - 27/03/19 #
+# Version 0.3 - 30/06/19 #
 ##########################
 services:
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.4.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.8.0
     environment:
       - node.name=es01
       - discovery.seed_hosts=es01
@@ -51,7 +51,7 @@ services:
 
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:7.4.0
+    image: docker.elastic.co/kibana/kibana:7.8.0
     environment:
       SERVER_NAME: kibana
       ELASTICSEARCH_HOSTS: https://elasticsearch:9200
@@ -61,9 +61,7 @@ services:
       SERVER_SSL_CERTIFICATE: /usr/share/kibana/certificates/kibana.crt
       ELASTICSEARCH_USERNAME: kibana
       ELASTICSEARCH_PASSWORD: insertkibanapasswordhere
-      XPACK_ML_ENABLED: "false"
-      XPACK_REPORTING_ENABLED: "true"
-      XPACK_APM_UI_ENABLED: "false"
+      XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: kibanakey
     command: /bin/bash -c "cp -r /run/secrets /usr/share/kibana/certificates && /usr/local/bin/kibana-docker"
     secrets:
       - ca.crt
@@ -75,11 +73,12 @@ services:
       - 443:5601
 
   logstash:
-    image: docker.elastic.co/logstash/logstash:7.4.0
+    image: docker.elastic.co/logstash/logstash:7.8.0
     environment:
       XPACK_MONITORING_ENABLED: "false"
     ports:
       - 5044:5044
+      - 12514:12514
     networks:
       - esnet
     configs:

diff --git a/Chapter 3 Files/logstash.conf b/Chapter 3 Files/logstash.conf
@@ -1,6 +1,6 @@
 ##########################
 # LME Logstash Config    #
-# Version 0.2 - 28/05/19 #
+# Version 0.3 - 06/07/2020 #
 ##########################
 
 input {
@@ -11,6 +11,12 @@ input {
     ssl_key => "/run/secrets/logstash.key"
     ssl_certificate_authorities => ["/run/secrets/ca.crt"]
     ssl_verify_mode => "force_peer"
+    tags => "beats"
+
+  }
+  syslog {
+    port => 12514
+    tags => "syslog"
   }
 }
 
@@ -19,6 +25,7 @@ filter {
     clones => ['cloned_logs']
     add_tag => ["cloned_logs"]
     }
+
 # If a windows update event that contains an updateTitle field (these sometimes contain KB numbers) then extract the KB
   if [updateTitle][winlog][event_data] {
     grok { match => {"[winlog][event_data][updateTitle]" => "(?<KB_number>KB[0-9]{5,})"} }
@@ -33,25 +40,6 @@ filter {
             }
    }
 
-# If hashes are present extract MD5 (Proc launch)
-  if [winlog][event_data][Hashes] {
-    grok { match => {"[winlog][event_data][Hashes]" => "(?<MD5_hash>[A-F0-9]{32})"} }
-    }
-
-# If hashes are present extract SHA256 (Proc launch)
-  if [winlog][event_data][Hashes] {
-    grok { match => {"[winlog][event_data][Hashes]" => "(?<SHA256_hash>[A-F0-9]{64})"} }
-    }
-
-# If hashes are present extract MD5 (downloads)
-  if [winlog][event_data][Hash] {
-    grok { match => {"[winlog][event_data][Hash]" => "(?<MD5_hash>[A-F0-9]{32})"} }
-    }
-
-# If hashes are present extract SHA256 (downloads)
-  if [winlog][event_data][Hash] {
-    grok { match => {"[winlog][event_data][Hash]" => "(?<SHA256_hash>[A-F0-9]{64})"} }
-    }
 
 
 # Add IP Geo
@@ -73,13 +61,38 @@ filter {
 
 output {
         if "cloned_logs" not in [tags] {
-  elasticsearch {
-    hosts => "https://elasticsearch:9200"
-    index => "winlogbeat-%{+dd.MM.YYYY}"
-    user => logstash_writer
-    password => insertlogstashwriterpasswordhere
-    ssl => true
-    cacert => '/run/secrets/ca.crt'
-  }
+
+  if "beats" in [tags] {
+    elasticsearch {
+      hosts => "https://elasticsearch:9200"
+      index => "winlogbeat-%{+dd.MM.YYYY}"
+      user => logstash_writer
+      password => insertlogstashwriterpasswordhere
+      ssl => true
+      cacert => '/run/secrets/ca.crt'
+      pipeline => "winlogbeat"
+      }
+  #end wlb
+       }
+
+  if "syslog" in [tags] {
+    elasticsearch {
+      hosts => "https://elasticsearch:9200"
+      index => "winlogbeat-%{+dd.MM.YYYY}"
+      user => logstash_writer
+      password => insertlogstashwriterpasswordhere
+      ssl => true
+      cacert => '/run/secrets/ca.crt'
+      pipeline => "syslog"
+      }
+
+
+#end type syslog
+}
+
+
+#end cloned logs if
   }
+
+#end output
 }
diff --git a/Chapter 3 Files/winlogbeat.yml b/Chapter 3 Files/winlogbeat.yml
@@ -1,5 +1,18 @@
 winlogbeat.event_logs:
   - name: ForwardedEvents
+    tags: [forwarded]
+    processors:
+      - script:
+          when.equals.winlog.channel: Security
+          lang: javascript
+          id: security
+          file: ${path.home}/module/security/config/winlogbeat-security.js
+      - script:
+          when.equals.winlog.channel: Microsoft-Windows-Sysmon/Operational
+          lang: javascript
+          id: sysmon
+          file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js
+
 output.logstash:
   hosts: ["logstash_dns_name:5044"]
   ssl.certificate_authorities: ["C:\\Program Files\\lme\\root-ca.crt"]

diff --git a/Chapter 4 Files/dashboards v0.3.0.ndjson b/Chapter 4 Files/dashboards v0.3.0.ndjson
diff --git a/docs/chapter2.md b/docs/chapter2.md
@@ -54,7 +54,7 @@ Import the [LME-Sysmon-Task](/Chapter%202%20Files/GPO%20Deployment/sysmon_gpo.zi
 2. Edit the Lme-Sysmon-Task GPO
 3. Change the setting for the batch file network location by navigating to: ```Computer Configuration\Preferences\Control Panel Settings\Scheduled Tasks\lme-sysmon-deploy\Actions``` and then select ```"Start a program" > Edit > Change the Location.```
 
-For example \\ad.testme.local\SYSVOL\ad.testme.local\Sysmon\update.bat
+For example ```\\testme.local\SYSVOL\testme.local\Sysmon\update.bat```
 
 
 ## 2.3 SCCM Deployment

diff --git a/docs/chapter3-easy.md b/docs/chapter3-easy.md
@@ -29,7 +29,7 @@ Figure 1: Elastic Stack components
 ## 3.1 Getting Started
 During the installation guide below you will see that the steps can been carried out automatically using the [Easy method](chapter3-easy.md). Commands are highlighted in grayboxes.
 
-You will need a linux box for this portion, **The deploy script is only tested on Ubuntu 18.04 Long Term Support (LTS).** In addition, only installation on a single server is supported. Please see [the resilience documentation](resilience.md) for more details.
+You will need a linux box for this portion, **The deploy script is only tested on Ubuntu Long Term Support (LTS) editions.** In addition, only installation on a single server is supported. Please see [the resilience documentation](resilience.md) for more details.
 
 ### 3.1.1 Firewall Rules
 You will need port 5044 open for the event collector to send data into the database (on the Linux server), To be able to access the web interface you will need to have firewall rules in place to allow access to port 443 (HTTPS) on the Linux server.
@@ -118,6 +118,8 @@ In 'files_for_windows.zip', copied in [step 3.2.2](#323-download-files-for-windo
 * wlbclient.key
 * wlbclient.crt
 * winlogbeat.yml 
+* module/security/config/winlogbeat-security.js
+* /module/sysmon/config/winlogbeat-sysmon.js
 
 In the zip file obtained from https://www.elastic.co/downloads/beats/winlogbeat
 * install-service-winlogbeat.ps1
@@ -127,13 +129,19 @@ In the zip file obtained from https://www.elastic.co/downloads/beats/winlogbeat
 On the windows event collector server unzip the winlogbeat file and replace 'winlogbeat.yml' with the one that came in 'files_for_windows.zip'.
 If your certificates are not in the same place as the locations in the file please change this too. Pay attention to the double slashes, these are required!
 
+* Copy winlogbeat-sysmon.js and winlogbeat-security.js file from the latest winlogbeat download and place them in the directories listed below
+```
+C:\\Program Files\\lme\\winlogbeat-7.6.1-windows-x86_64\\module\\sysmon\\config\\winlogbeat-sysmon.js
+C:\\Program Files\\lme\\winlogbeat-7.6.1-windows-x86_64\\module\\security\\config\\winlogbeat-security.js
+``` 
+
 Now open PowerShell as an administrator and run the following command from the unzipped folder: ```./install-service-winlogbeat.ps1```
 
 
 # Chapter 3 - Checklist
 
 1. Check Services.msc on the Windows box, Does the winlogbeat show as running and automatic?
-2. On the Linux machine, check the output of ```docker stack ps lme``` , You should see lme_elasticsearch/lme_nginx/lme_kibana and lme_logstash all in the 'current' state of ‘running’
+2. On the Linux machine, check the output of ```docker stack ps lme``` , You should see lme_elasticsearch/lme_kibana and lme_logstash all in the 'current' state of ‘running’
 3. You can now visit the website https://your_Linux_server/ and access Kibana. The username and password is provided from the script in [Chapter 3.2](#32-install-lme-the-easy-way-using-our-script).
 
 ## Now move onto [Chapter 4 - Post Install Actions ](chapter4.md)
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		C:\WINDOWS\Sysmon64.exe -i c:\WINDOWS\sysmonconf.xml -accepteula -h sha1,sha256 -n
		C:\WINDOWS\Sysmon64.exe -i c:\WINDOWS\sysmonconf.xml -accepteula