Release v2.0.0 🚀 - Merge pull request #43 from sherifabdlnaby/v2.0.0

Release `v2.0.0` 🚀 - ⬆️ Default Stack Version `7.12.0` => `7.16.2` ⬆️ - ✨Enabled SSL on HTTP Layer - Enables the usage of ELK Alerts - Enables the usage of ML Jobs - Enables the usage of SIEM features - ✨Add `make collect-docker-logs` option that will ship Host docker containers to ELK. - Removed `ElastAlert` & `Curator` from bundled tools, replaced with native ELK Features. - No Longer use Custom Built Images instead pull images directly from Registry.
sherifabdlnaby · Dec 25, 2021 · 3dcdb1e · 3dcdb1e
2 parents 04e694a + 05b65ca
commit 3dcdb1e
Show file tree

Hide file tree

Showing 39 changed files with 175 additions and 903 deletions.
diff --git a/.env b/.env
@@ -1,5 +1,5 @@
 COMPOSE_PROJECT_NAME=elastic
-ELK_VERSION=7.12.0
+ELK_VERSION=7.16.2
 
 #----------- Resources --------------------------#
 ELASTICSEARCH_HEAP=1024m

diff --git a/Makefile b/Makefile
@@ -1,15 +1,17 @@
 .DEFAULT_GOAL:=help
 
-COMPOSE_ALL_FILES := -f docker-compose.yml -f docker-compose.monitor.yml -f docker-compose.tools.yml -f docker-compose.nodes.yml
+COMPOSE_ALL_FILES := -f docker-compose.yml -f docker-compose.monitor.yml -f docker-compose.tools.yml -f docker-compose.nodes.yml -f docker-compose.logs.yml
 COMPOSE_MONITORING := -f docker-compose.yml -f docker-compose.monitor.yml
+COMPOSE_LOGGING := -f docker-compose.yml -f docker-compose.logs.yml
 COMPOSE_TOOLS := -f docker-compose.yml -f docker-compose.tools.yml
 COMPOSE_NODES := -f docker-compose.yml -f docker-compose.nodes.yml
 ELK_SERVICES   := elasticsearch logstash kibana
+ELK_LOG_COLLECTION := filebeat
 ELK_MONITORING := elasticsearch-exporter logstash-exporter filebeat-cluster-logs
-ELK_TOOLS  := curator elastalert rubban
+ELK_TOOLS  := rubban
 ELK_NODES := elasticsearch-1 elasticsearch-2
 ELK_MAIN_SERVICES := ${ELK_SERVICES} ${ELK_MONITORING} ${ELK_TOOLS}
-ELK_ALL_SERVICES := ${ELK_MAIN_SERVICES} ${ELK_NODES}
+ELK_ALL_SERVICES := ${ELK_MAIN_SERVICES} ${ELK_NODES} ${ELK_LOG_COLLECTION}
 # --------------------------
 
 # load .env so that Docker Swarm Commands has .env values too. (https://github.com/moby/moby/issues/29133)
@@ -41,6 +43,9 @@ up:
 monitoring:		## Start ELK Monitoring.
 	@docker-compose ${COMPOSE_MONITORING} up -d --build ${ELK_MONITORING}
 
+collect-docker-logs:		## Start Filebeat that collects all Host Docker Logs and ship it to ELK
+	@docker-compose ${COMPOSE_LOGGING} up -d --build ${ELK_LOG_COLLECTION}
+
 tools:		    ## Start ELK Tools (ElastAlert, Curator).
 	@docker-compose ${COMPOSE_TOOLS} up -d --build ${ELK_TOOLS}
 
@@ -49,6 +54,8 @@ nodes:		    ## Start Two Extra Elasticsearch Nodes
 
 build:			## Build ELK and all its extra components.
 	@docker-compose ${COMPOSE_ALL_FILES} build ${ELK_ALL_SERVICES}
+ps:				## Show all running containers.
+	@docker-compose ${COMPOSE_ALL_FILES} ps
 
 down:			## Down ELK and all its extra components.
 	@docker-compose ${COMPOSE_ALL_FILES} down

diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@
 <h4 align="center">With tools like Curator, Rubban, ElastAlert for Alerting.</h4>
 <p align="center">
    <a>
-      <img src="https://img.shields.io/badge/Elastic%20Stack-7.12.0-blue?style=flat&logo=elasticsearch" alt="Elastic Stack Version 7^^">
+      <img src="https://img.shields.io/badge/Elastic%20Stack-7.16.2-blue?style=flat&logo=elasticsearch" alt="Elastic Stack Version 7^^">
    </a>
    <a>
       <img src="https://img.shields.io/github/v/tag/sherifabdlnaby/elastdocker?label=release&amp;sort=semver">
@@ -29,7 +29,7 @@ Elastic Stack (**ELK**) Docker Composition, preconfigured with **Security**, **M
 
 Based on [Official Elastic Docker Images](https://www.docker.elastic.co/)
 
-Stack Version: [7.12.0](https://www.elastic.co/blog/elastic-stack-7-12-0-released)
+Stack Version: [7.16.2](https://www.elastic.co/blog/elastic-stack-7-16-2-released)
 > You can change Elastic Stack version by setting `ELK_VERSION` in `.env` file and rebuild your images. Any version >= 7.0.0 is compatible with this template.
 
 ### Main Features 📜
@@ -171,7 +171,7 @@ $ make prune
 
 * Some Configuration are parameterized in the `.env` file.
   * `ELASTIC_PASSWORD`, user `elastic`'s password (default: `changeme` _pls_).
-  * `ELK_VERSION` Elastic Stack Version (default: `7.12.0`)
+  * `ELK_VERSION` Elastic Stack Version (default: `7.16.2`)
   * `ELASTICSEARCH_HEAP`, how much Elasticsearch allocate from memory (default: 1GB -good for development only-)
   * `LOGSTASH_HEAP`, how much Logstash allocate from memory.
   * Other configurations which their such as cluster name, and node name, etc.
@@ -193,22 +193,10 @@ To Re-generate Keystore:
 make keystore
 ```
 
-### Enable SSL on HTTP
-
-By default, only Transport Layer has SSL Enabled, to enable SSL on HTTP layer, add the following lines to `elasticsearch.yml`
-```yaml
-## - http
-xpack.security.http.ssl.enabled: true
-xpack.security.http.ssl.key: certs/elasticsearch.key
-xpack.security.http.ssl.certificate: certs/elasticsearch.crt
-xpack.security.http.ssl.certificate_authorities: certs/ca.crt
-xpack.security.http.ssl.client_authentication: optional
-```
-
-> ⚠️ Enabling SSL on HTTP layer will require all clients that connect to Elasticsearch to configure SSL connection for HTTP, this includes all the current configured parts of the stack (e.g Logstash, Kibana, Curator, etc) plus any library/binding that connects to Elasticsearch from your application code.
+### Notes
 
 
-### Notes
+- ⚠️ Elasticsearch HTTP layer is using SSL, thus mean you need to configure your elasticsearch clients with the `CA` in `secrets/certs/ca/ca.crt`, or configure client to ignore SSL Certificate Verification (e.g `--insecure` in `curl`).
 
 - Adding Two Extra Nodes to the cluster will make the cluster depending on them and won't start without them again.
 

diff --git a/docker-compose.logs.yml b/docker-compose.logs.yml
@@ -0,0 +1,24 @@
+version: '3.5'
+
+# will contain all elasticsearch data.
+volumes:
+  filebeat-data:
+
+services:
+  # Docker Logs Shipper ------------------------------
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:${ELK_VERSION}
+    restart: always
+    # -e flag to log to stderr and disable syslog/file output
+    command: -e --strict.perms=false
+    user: root
+    environment:
+      ELASTIC_USERNAME: ${ELASTIC_USERNAME}
+      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
+      KIBANA_HOST_PORT: ${KIBANA_HOST}:${KIBANA_PORT}
+      ELASTICSEARCH_HOST_PORT: https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}
+    volumes:
+      - ./filebeat/filebeat.docker.logs.yml:/usr/share/filebeat/filebeat.yml:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - filebeat-data:/var/lib/filebeat/data
diff --git a/docker-compose.monitor.yml b/docker-compose.monitor.yml
@@ -6,7 +6,7 @@ services:
   elasticsearch-exporter:
     image: justwatch/elasticsearch_exporter:1.1.0
     restart: always
-    command: ["--es.uri", "http://${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}@${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}",
+    command: ["--es.uri", "https://${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}@${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}",
               "--es.all",
               "--es.snapshots",
               "--es.indices"]
@@ -22,11 +22,7 @@ services:
 
   # Cluster Logs Shipper ------------------------------
   filebeat-cluster-logs:
-    image: filebeat:elastdocker-${ELK_VERSION}
-    build:
-      context: tools/filebeat/
-      args:
-        ELK_VERSION: $ELK_VERSION
+    image: docker.elastic.co/beats/filebeat:${ELK_VERSION}
     restart: always
     # -e flag to log to stderr and disable syslog/file output
     command: -e --strict.perms=false
@@ -35,8 +31,8 @@ services:
       ELASTIC_USERNAME: ${ELASTIC_USERNAME}
       ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
       KIBANA_HOST_PORT: ${KIBANA_HOST}:${KIBANA_PORT}
-      ELASTICSEARCH_HOST_PORT: ${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}
+      ELASTICSEARCH_HOST_PORT: https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}
     volumes:
-      - ./tools/filebeat/monitor/filebeat_cluster_logs.yml:/usr/share/filebeat/filebeat.yml:ro
+      - ./filebeat/filebeat.monitoring.yml:/usr/share/filebeat/filebeat.yml:ro
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro
diff --git a/docker-compose.nodes.yml b/docker-compose.nodes.yml
@@ -7,12 +7,8 @@ volumes:
 
 services:
   elasticsearch-1:
-    image: elasticsearch:elastdocker-${ELK_VERSION}
-    build:
-      context: elasticsearch/
-      args:
-        ELK_VERSION: ${ELK_VERSION}
-#    restart: unless-stopped
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
+    restart: unless-stopped
     environment:
       ELASTIC_USERNAME: ${ELASTIC_USERNAME}
       ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
@@ -43,12 +39,8 @@ services:
         soft: 200000
         hard: 200000
   elasticsearch-2:
-    image: elasticsearch:elastdocker-${ELK_VERSION}
-    build:
-      context: elasticsearch/
-      args:
-        ELK_VERSION: ${ELK_VERSION}
-    #    restart: unless-stopped
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
+    restart: unless-stopped
     environment:
       ELASTIC_USERNAME: ${ELASTIC_USERNAME}
       ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}

diff --git a/docker-compose.setup.yml b/docker-compose.setup.yml
@@ -2,26 +2,17 @@ version: '3.5'
 
 services:
   keystore:
-    build:
-      context: elasticsearch/
-      args:
-        ELK_VERSION: ${ELK_VERSION}
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
     command: bash /setup/setup-keystore.sh
     user: "0"
     volumes:
         - ./secrets:/secrets
         - ./setup/:/setup/
     environment:
       ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
-      # Add keystore values used in `keystore.sh` here. (e.g AMAZON S3 Repo Creds)
-      AWS_ACCESS_KEY_ID:      ${AWS_ACCESS_KEY_ID}
-      AWS_SECRET_ACCESS_KEY:  ${AWS_SECRET_ACCESS_KEY}
 
   certs:
-    build:
-      context: elasticsearch/
-      args:
-        ELK_VERSION: ${ELK_VERSION}
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
     command: bash /setup/setup-certs.sh
     user: "0"
     volumes:

diff --git a/docker-compose.tools.yml b/docker-compose.tools.yml
@@ -1,36 +1,6 @@
 version: '3.5'
 
 services:
-  curator:
-    image: curator:elastdocker-${ELK_VERSION}
-    build:
-      context: tools/curator/.
-    restart: unless-stopped
-    environment:
-      ELASTICSEARCH_HOST_PORT: ${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}
-      ELASTICSEARCH_HTTP_AUTH: ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}
-    volumes:
-      - ./tools/curator/curator.yml:/root/.curator/curator.yml:ro
-      - ./tools/curator/crontab:/etc/crontab:ro
-      - ./tools/curator/actions/:/actions/:ro
-
-  elastalert:
-    image: sherifabdlnaby/elastalert:3.1.1
-    restart: unless-stopped
-    volumes:
-      - ./tools/elastalert/config/elastalert.yaml:/opt/config/config.yaml
-      - ./tools/elastalert/config/elastalert-test.yaml:/opt/config/config-test.yaml
-      - ./tools/elastalert/config/config.json:/opt/config/config.json
-      - ./tools/elastalert/rules:/opt/elastalert/rules
-      - ./tools/elastalert/rule_templates:/opt/elastalert/rule_templates
-    environment:
-      ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST}
-      ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT}
-      ELASTIC_USERNAME: ${ELASTIC_USERNAME}
-      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
-    ports:
-      - 3030:3030
-
   rubban:
     image: sherifabdlnaby/rubban:latest
     restart: unless-stopped
@@ -41,3 +11,6 @@ services:
       RUBBAN_REFRESHINDEXPATTERN_ENABLED: 'true'
       RUBBAN_REFRESHINDEXPATTERN_SCHEDULE: '*/5 * * * *'
       RUBBAN_REFRESHINDEXPATTERN_PATTERNS: '*'
+      RUBBAN_AUTOINDEXPATTERN_ENABLED: true
+      RUBBAN_AUTOINDEXPATTERN_SCHEDULE: '*/5 * * * *'
+      RUBBAN_AUTOINDEXPATTERN_GENERALPATTERNS: '[{"pattern":"filebeat?","timeFieldName":"@timestamp"},{"pattern":"logstash?","timeFieldName":"@timestamp"}]'
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -26,11 +26,7 @@ secrets:
 
 services:
   elasticsearch:
-    image: elasticsearch:elastdocker-${ELK_VERSION}
-    build:
-      context: elasticsearch/
-      args:
-        ELK_VERSION: ${ELK_VERSION}
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
     restart: unless-stopped
     environment:
       ELASTIC_USERNAME: ${ELASTIC_USERNAME}
@@ -64,40 +60,39 @@ services:
       nofile:
         soft: 200000
         hard: 200000
+    healthcheck:
+      test: ["CMD", "sh", "-c", "curl -sf --insecure https://$ELASTIC_USERNAME:$ELASTIC_PASSWORD@localhost:9200/_cat/health | grep -ioE 'green|yellow' || echo 'not green/yellow cluster status'"]
 
   logstash:
-    image: logstash:elastdocker-${ELK_VERSION}
-    build:
-      context: logstash/
-      args:
-        ELK_VERSION: $ELK_VERSION
+    image: docker.elastic.co/logstash/logstash:${ELK_VERSION}
     restart: unless-stopped
     volumes:
       - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
       - ./logstash/config/pipelines.yml:/usr/share/logstash/config/pipelines.yml:ro
       - ./logstash/pipeline:/usr/share/logstash/pipeline:ro
+    secrets:
+      - source: elastic.ca
+        target: /certs/ca.crt
     environment:
       ELASTIC_USERNAME: ${ELASTIC_USERNAME}
       ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
-      ELASTICSEARCH_HOST_PORT: ${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}
+      ELASTICSEARCH_HOST_PORT: https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}
       LS_JAVA_OPTS: "-Xmx${LOGSTASH_HEAP} -Xms${LOGSTASH_HEAP}"
     ports:
       - "5044:5044"
       - "9600:9600"
+    healthcheck:
+      test: ["CMD", "curl", "-s" ,"-XGET", "http://127.0.0.1:9600"]
 
   kibana:
-    image: kibana:elastdocker-${ELK_VERSION}
-    build:
-      context: kibana/
-      args:
-        ELK_VERSION: $ELK_VERSION
+    image: docker.elastic.co/kibana/kibana:${ELK_VERSION}
     restart: unless-stopped
     volumes:
       - ./kibana/config/:/usr/share/kibana/config:ro
     environment:
       ELASTIC_USERNAME: ${ELASTIC_USERNAME}
       ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
-      ELASTICSEARCH_HOST_PORT: ${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}
+      ELASTICSEARCH_HOST_PORT: https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}
     secrets:
       - source: elastic.ca
         target: /certs/ca.crt

diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile
diff --git a/elasticsearch/config/elasticsearch.yml b/elasticsearch/config/elasticsearch.yml
@@ -24,11 +24,11 @@ xpack.security.transport.ssl.certificate: certs/elasticsearch.crt
 xpack.security.transport.ssl.certificate_authorities: certs/ca.crt
 
 ## - http
-#xpack.security.http.ssl.enabled: true
-#xpack.security.http.ssl.key: certs/elasticsearch.key
-#xpack.security.http.ssl.certificate: certs/elasticsearch.crt
-#xpack.security.http.ssl.certificate_authorities: certs/ca.crt
-#xpack.security.http.ssl.client_authentication: optional
+xpack.security.http.ssl.enabled: true
+xpack.security.http.ssl.key: certs/elasticsearch.key
+xpack.security.http.ssl.certificate: certs/elasticsearch.crt
+xpack.security.http.ssl.certificate_authorities: certs/ca.crt
+xpack.security.http.ssl.client_authentication: optional
 
 # Monitoring
 xpack.monitoring.collection.enabled: true
diff --git a/elasticsearch/scripts/docker-healthcheck b/elasticsearch/scripts/docker-healthcheck