From 93b5b436284bba30178c32399bca1712c870a148 Mon Sep 17 00:00:00 2001 From: AlfonsoRBJ Date: Wed, 3 Apr 2019 18:11:37 +0200 Subject: [PATCH 01/11] first approach --- elasticsearch/Dockerfile | 13 ++++++++++++ elasticsearch/config/config_cluster.sh | 28 ++++++++++++++++++++++++++ elasticsearch/config/entrypoint.sh | 2 ++ 3 files changed, 43 insertions(+) create mode 100644 elasticsearch/config/config_cluster.sh diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index 30a2cd0a..76649af2 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -13,6 +13,16 @@ ENV ENABLE_CONFIGURE_S3="false" ENV TEMPLATE_VERSION=v3.8.2 +ENV ELASTIC_CLUSTER="false" \ + CLUSTER_NAME="wazuh" \ + NODE_MASTER="true" \ + NODE_DATA="true" \ + NODE_INGEST="true" \ + NODE_NAME="wazuh-elasticsearch" \ + MEMORY_LOCK="true" \ + DISCOVERY_SERVICE="wazuh-elasticsearch" \ + NUMBER_OF_MASTERS="2" + ADD https://raw.githubusercontent.com/wazuh/wazuh/$TEMPLATE_VERSION/extensions/elasticsearch/wazuh-elastic6-template-alerts.json /usr/share/elasticsearch/config COPY config/entrypoint.sh /entrypoint.sh @@ -28,5 +38,8 @@ RUN elasticsearch-plugin install --batch repository-s3 COPY config/configure_s3.sh ./config/configure_s3.sh RUN chmod 755 ./config/configure_s3.sh +COPY --chown=elasticsearch:elasticsearch ./config/config_cluster.sh ./ +RUN chmod +x ./config_cluster.sh + ENTRYPOINT ["/entrypoint.sh"] CMD ["elasticsearch"] diff --git a/elasticsearch/config/config_cluster.sh b/elasticsearch/config/config_cluster.sh new file mode 100644 index 00000000..3c3b9b10 --- /dev/null +++ b/elasticsearch/config/config_cluster.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Wazuh App Copyright (C) 2019 Wazuh Inc. (License GPLv2) + +elastic_config_file="/usr/share/elasticsearch/config/elasticsearch.yml" + +if [[ $ELASTIC_CLUSTER == "true" ]] +then + + sed -i "s/cluster.name: \"docker-cluster\"/cluster.name: \"${CLUSTER_NAME}\":g" $elastic_config_file + sed -i "s/discovery:zen:minimum_master_nodes: 1/discovery:zen:minimum_master_nodes: ${NUMBER_OF_MASTERS}/g" $elastic_config_file + + echo " +#cluster node +node: + master: ${NODE_MASTER} + data: ${NODE_DATA} + ingest: ${NODE_INGEST} + name: ${NODE_NAME} + +bootstrap: + memory_lock: ${MEMORY_LOCK} + +discovery: + zen: + ping.unicast.hosts: ${DISCOVERY_SERVICE} + +" >> $elastic_config_file +fi diff --git a/elasticsearch/config/entrypoint.sh b/elasticsearch/config/entrypoint.sh index 2c394cbd..c57703f1 100644 --- a/elasticsearch/config/entrypoint.sh +++ b/elasticsearch/config/entrypoint.sh @@ -43,6 +43,8 @@ fi # Run load settings script. +./config_cluster.sh + ./load_settings.sh & # Execute elasticsearch From e4e3abbc6227fa2e41fe648f800d827e98327ca7 Mon Sep 17 00:00:00 2001 From: AlfonsoRBJ Date: Wed, 3 Apr 2019 18:57:36 +0200 Subject: [PATCH 02/11] fix sed --- elasticsearch/config/config_cluster.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/elasticsearch/config/config_cluster.sh b/elasticsearch/config/config_cluster.sh index 3c3b9b10..ae8c1036 100644 --- a/elasticsearch/config/config_cluster.sh +++ b/elasticsearch/config/config_cluster.sh @@ -6,8 +6,9 @@ elastic_config_file="/usr/share/elasticsearch/config/elasticsearch.yml" if [[ $ELASTIC_CLUSTER == "true" ]] then - sed -i "s/cluster.name: \"docker-cluster\"/cluster.name: \"${CLUSTER_NAME}\":g" $elastic_config_file - sed -i "s/discovery:zen:minimum_master_nodes: 1/discovery:zen:minimum_master_nodes: ${NUMBER_OF_MASTERS}/g" $elastic_config_file + sed -i 's:cluster.name\: "docker-cluster":cluster.name\: "'$CLUSTER_NAME'":g' $elastic_config_file + sed -i 's:discovery:zen:minimum_master_nodes: 1:discovery:zen:minimum_master_nodes: '$NUMBER_OF_MASTERS':g' $elastic_config_file + echo " #cluster node From d1f92d7acb1f0ac93f907eca1817678bb26a3c1f Mon Sep 17 00:00:00 2001 From: AlfonsoRBJ Date: Wed, 3 Apr 2019 19:10:27 +0200 Subject: [PATCH 03/11] fix sed for minimun master nodes --- elasticsearch/config/config_cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch/config/config_cluster.sh b/elasticsearch/config/config_cluster.sh index ae8c1036..4b911fd6 100644 --- a/elasticsearch/config/config_cluster.sh +++ b/elasticsearch/config/config_cluster.sh @@ -7,7 +7,7 @@ if [[ $ELASTIC_CLUSTER == "true" ]] then sed -i 's:cluster.name\: "docker-cluster":cluster.name\: "'$CLUSTER_NAME'":g' $elastic_config_file - sed -i 's:discovery:zen:minimum_master_nodes: 1:discovery:zen:minimum_master_nodes: '$NUMBER_OF_MASTERS':g' $elastic_config_file + sed -i 's:discovery.zen.minimum_master_nodes\: 1:discovery.zen.minimum_master_nodes\: '$NUMBER_OF_MASTERS':g' $elastic_config_file echo " From bcc3bd33c752ce410ec6a986a0b7eddcb97ec6e6 Mon Sep 17 00:00:00 2001 From: AlfonsoRBJ Date: Thu, 4 Apr 2019 18:34:33 +0200 Subject: [PATCH 04/11] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61093b6a..db5a29eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file. - Adding env variables for alerts data flow. ([#118](https://github.com/wazuh/wazuh-docker/pull/118)) - New Logstash entrypoint added. ([#135](https://github.com/wazuh/wazuh-docker/pull/135/files)) - Welcome screen management. ([#133](https://github.com/wazuh/wazuh-docker/pull/133)) +- Add Elasticsearch cluster configuration. ([#146](https://github.com/wazuh/wazuh-docker/pull/146)) ### Changed From 8886560ccadb6cfc44c02b9a8a017f2e454bf2f9 Mon Sep 17 00:00:00 2001 From: AlfonsoRBJ Date: Fri, 5 Apr 2019 15:49:44 +0200 Subject: [PATCH 05/11] Add options for Elastic Cluster based on sts replicas --- elasticsearch/Dockerfile | 4 +++- elasticsearch/config/config_cluster.sh | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index 76649af2..ba4ed1b4 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -21,7 +21,9 @@ ENV ELASTIC_CLUSTER="false" \ NODE_NAME="wazuh-elasticsearch" \ MEMORY_LOCK="true" \ DISCOVERY_SERVICE="wazuh-elasticsearch" \ - NUMBER_OF_MASTERS="2" + NUMBER_OF_MASTERS="2" \ + MAX_NODES="1" \ + DELAYED_TIMEOUT="1" ADD https://raw.githubusercontent.com/wazuh/wazuh/$TEMPLATE_VERSION/extensions/elasticsearch/wazuh-elastic6-template-alerts.json /usr/share/elasticsearch/config diff --git a/elasticsearch/config/config_cluster.sh b/elasticsearch/config/config_cluster.sh index 4b911fd6..38232875 100644 --- a/elasticsearch/config/config_cluster.sh +++ b/elasticsearch/config/config_cluster.sh @@ -17,6 +17,7 @@ node: data: ${NODE_DATA} ingest: ${NODE_INGEST} name: ${NODE_NAME} + max_local_storage_nodes: ${MAX_NODES} bootstrap: memory_lock: ${MEMORY_LOCK} @@ -24,6 +25,9 @@ bootstrap: discovery: zen: ping.unicast.hosts: ${DISCOVERY_SERVICE} + +index: + unassigned.node_left.delayed_timeout: ${DELAYED_TIMEOUT} " >> $elastic_config_file fi From a2240d6be1401fc4d21f1cd8a2dfead94cd33a14 Mon Sep 17 00:00:00 2001 From: AlfonsoRBJ Date: Fri, 5 Apr 2019 16:17:14 +0200 Subject: [PATCH 06/11] fix index level from elastic config --- elasticsearch/config/config_cluster.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/elasticsearch/config/config_cluster.sh b/elasticsearch/config/config_cluster.sh index 38232875..755b13b5 100644 --- a/elasticsearch/config/config_cluster.sh +++ b/elasticsearch/config/config_cluster.sh @@ -26,8 +26,7 @@ discovery: zen: ping.unicast.hosts: ${DISCOVERY_SERVICE} -index: - unassigned.node_left.delayed_timeout: ${DELAYED_TIMEOUT} +index.unassigned.node_left.delayed_timeout: ${DELAYED_TIMEOUT} " >> $elastic_config_file fi From 770bd9271fac50ba9380de0f769267f869908997 Mon Sep 17 00:00:00 2001 From: AlfonsoRBJ Date: Fri, 5 Apr 2019 16:29:01 +0200 Subject: [PATCH 07/11] remove index from yml and add it as curl --- elasticsearch/Dockerfile | 2 +- elasticsearch/config/config_cluster.sh | 2 -- elasticsearch/config/load_settings.sh | 8 ++++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index ba4ed1b4..c417c079 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -23,7 +23,7 @@ ENV ELASTIC_CLUSTER="false" \ DISCOVERY_SERVICE="wazuh-elasticsearch" \ NUMBER_OF_MASTERS="2" \ MAX_NODES="1" \ - DELAYED_TIMEOUT="1" + DELAYED_TIMEOUT="1m" ADD https://raw.githubusercontent.com/wazuh/wazuh/$TEMPLATE_VERSION/extensions/elasticsearch/wazuh-elastic6-template-alerts.json /usr/share/elasticsearch/config diff --git a/elasticsearch/config/config_cluster.sh b/elasticsearch/config/config_cluster.sh index 755b13b5..0d10bc28 100644 --- a/elasticsearch/config/config_cluster.sh +++ b/elasticsearch/config/config_cluster.sh @@ -26,7 +26,5 @@ discovery: zen: ping.unicast.hosts: ${DISCOVERY_SERVICE} -index.unassigned.node_left.delayed_timeout: ${DELAYED_TIMEOUT} - " >> $elastic_config_file fi diff --git a/elasticsearch/config/load_settings.sh b/elasticsearch/config/load_settings.sh index 17154c29..e016d331 100644 --- a/elasticsearch/config/load_settings.sh +++ b/elasticsearch/config/load_settings.sh @@ -94,5 +94,13 @@ curl -XPUT "$el_url/_cluster/settings" -H 'Content-Type: application/json' -d' } ' +curl -X PUT "$el_url/_all/_settings" -H 'Content-Type: application/json' -d' +{ + "settings": { + "index.unassigned.node_left.delayed_timeout": "'"$DELAYED_TIMEOUT"'" + } +} +' + echo "Elasticsearch is ready." From 8366ff7f74567415b3c7780adf4391b7450d1a5f Mon Sep 17 00:00:00 2001 From: AlfonsoRBJ Date: Wed, 17 Apr 2019 10:03:57 +0200 Subject: [PATCH 08/11] add Elasticsearch url as ENV variable --- elasticsearch/Dockerfile | 2 ++ elasticsearch/config/load_settings.sh | 6 +----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index c417c079..3e455ca7 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -1,6 +1,8 @@ # Wazuh App Copyright (C) 2019 Wazuh Inc. (License GPLv2) FROM docker.elastic.co/elasticsearch/elasticsearch:6.5.4 +ENV ELASTICSEARCH_URL="http://elasticsearch:9200" + ENV ALERTS_SHARDS="1" \ ALERTS_REPLICAS="0" diff --git a/elasticsearch/config/load_settings.sh b/elasticsearch/config/load_settings.sh index e016d331..766dd080 100644 --- a/elasticsearch/config/load_settings.sh +++ b/elasticsearch/config/load_settings.sh @@ -3,11 +3,7 @@ set -e -if [ "x${ELASTICSEARCH_URL}" = "x" ]; then - el_url="http://elasticsearch:9200" -else - el_url="${ELASTICSEARCH_URL}" -fi +el_url=${ELASTICSEARCH_URL} if [ "x${WAZUH_API_URL}" = "x" ]; then wazuh_url="https://wazuh" From 59e25c98b6793dcb2e832184e11b3826506516ed Mon Sep 17 00:00:00 2001 From: AlfonsoRBJ Date: Wed, 24 Apr 2019 16:39:40 +0200 Subject: [PATCH 09/11] improve cluster variables configuration --- elasticsearch/Dockerfile | 20 +++++++++++--------- elasticsearch/config/config_cluster.sh | 21 ++++++++++++--------- elasticsearch/config/load_settings.sh | 3 ++- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index 3e455ca7..56c44115 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -15,17 +15,19 @@ ENV ENABLE_CONFIGURE_S3="false" ENV TEMPLATE_VERSION=v3.8.2 +# Elasticearch cluster configuration environment variables +# If ELASTIC_CLUSTER is set to "true" the following variables will be added to the Elasticsearch configuration ENV ELASTIC_CLUSTER="false" \ CLUSTER_NAME="wazuh" \ - NODE_MASTER="true" \ - NODE_DATA="true" \ - NODE_INGEST="true" \ - NODE_NAME="wazuh-elasticsearch" \ - MEMORY_LOCK="true" \ - DISCOVERY_SERVICE="wazuh-elasticsearch" \ - NUMBER_OF_MASTERS="2" \ - MAX_NODES="1" \ - DELAYED_TIMEOUT="1m" + CLUSTER_NODE_MASTER="true" \ + CLUSTER_NODE_DATA="true" \ + CLUSTER_NODE_INGEST="true" \ + CLUSTER_NODE_NAME="wazuh-elasticsearch" \ + CLUSTER_MEMORY_LOCK="true" \ + CLUSTER_DISCOVERY_SERVICE="wazuh-elasticsearch" \ + CLUSTER_NUMBER_OF_MASTERS="2" \ + CLUSTER_MAX_NODES="1" \ + CLUSTER_DELAYED_TIMEOUT="1m" ADD https://raw.githubusercontent.com/wazuh/wazuh/$TEMPLATE_VERSION/extensions/elasticsearch/wazuh-elastic6-template-alerts.json /usr/share/elasticsearch/config diff --git a/elasticsearch/config/config_cluster.sh b/elasticsearch/config/config_cluster.sh index 0d10bc28..b4063825 100644 --- a/elasticsearch/config/config_cluster.sh +++ b/elasticsearch/config/config_cluster.sh @@ -3,28 +3,31 @@ elastic_config_file="/usr/share/elasticsearch/config/elasticsearch.yml" + +# If Elasticsearch cluster is enable if [[ $ELASTIC_CLUSTER == "true" ]] then + # Set the cluster.name and discovery.zen.minimun_master_nodes variables sed -i 's:cluster.name\: "docker-cluster":cluster.name\: "'$CLUSTER_NAME'":g' $elastic_config_file - sed -i 's:discovery.zen.minimum_master_nodes\: 1:discovery.zen.minimum_master_nodes\: '$NUMBER_OF_MASTERS':g' $elastic_config_file - + sed -i 's:discovery.zen.minimum_master_nodes\: 1:discovery.zen.minimum_master_nodes\: '$CLUSTER_NUMBER_OF_MASTERS':g' $elastic_config_file + # Add the cluster configuration echo " #cluster node node: - master: ${NODE_MASTER} - data: ${NODE_DATA} - ingest: ${NODE_INGEST} - name: ${NODE_NAME} - max_local_storage_nodes: ${MAX_NODES} + master: ${CLUSTER_NODE_MASTER} + data: ${CLUSTER_NODE_DATA} + ingest: ${CLUSTER_NODE_INGEST} + name: ${CLUSTER_NODE_NAME} + max_local_storage_nodes: ${CLUSTER_MAX_NODES} bootstrap: - memory_lock: ${MEMORY_LOCK} + memory_lock: ${CLUSTER_MEMORY_LOCK} discovery: zen: - ping.unicast.hosts: ${DISCOVERY_SERVICE} + ping.unicast.hosts: ${CLUSTER_DISCOVERY_SERVICE} " >> $elastic_config_file fi diff --git a/elasticsearch/config/load_settings.sh b/elasticsearch/config/load_settings.sh index 766dd080..cccc104d 100644 --- a/elasticsearch/config/load_settings.sh +++ b/elasticsearch/config/load_settings.sh @@ -90,10 +90,11 @@ curl -XPUT "$el_url/_cluster/settings" -H 'Content-Type: application/json' -d' } ' +# Set cluster delayed timeout when node falls curl -X PUT "$el_url/_all/_settings" -H 'Content-Type: application/json' -d' { "settings": { - "index.unassigned.node_left.delayed_timeout": "'"$DELAYED_TIMEOUT"'" + "index.unassigned.node_left.delayed_timeout": "'"$CLUSTER_DELAYED_TIMEOUT"'" } } ' From a3c9e08092d5541cc38689e3a97fdbd39167360b Mon Sep 17 00:00:00 2001 From: AlfonsoRBJ Date: Thu, 25 Apr 2019 12:59:38 +0200 Subject: [PATCH 10/11] Add Elasticsearch cluster information --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 3760f61e..5f243eb8 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ In this repository you will find the containers to run: In addition, a docker-compose file is provided to launch the containers mentioned above. It also launches an Elasticsearch container (working as a single-node cluster) using Elastic Stack Docker images. +* Elasticsearch cluster. In the Elasticsearch Dockerfile we can visualize variables to configure an Elasticsearch Cluster. These variables are used in the file *config_cluster.sh* to set them in the *elasticsearch.yml* configuration file. You can see the meaning of the variables [here](https://gist.github.com/zsprackett/8546403). + ## Documentation * [Wazuh full documentation](http://documentation.wazuh.com) From 9cf6c15af94654b934f3656d8185610ca6ad70ae Mon Sep 17 00:00:00 2001 From: AlfonsoRBJ Date: Thu, 25 Apr 2019 15:48:47 +0200 Subject: [PATCH 11/11] Adde official links --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f243eb8..467d7fa1 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ In this repository you will find the containers to run: In addition, a docker-compose file is provided to launch the containers mentioned above. It also launches an Elasticsearch container (working as a single-node cluster) using Elastic Stack Docker images. -* Elasticsearch cluster. In the Elasticsearch Dockerfile we can visualize variables to configure an Elasticsearch Cluster. These variables are used in the file *config_cluster.sh* to set them in the *elasticsearch.yml* configuration file. You can see the meaning of the variables [here](https://gist.github.com/zsprackett/8546403). +* Elasticsearch cluster. In the Elasticsearch Dockerfile we can visualize variables to configure an Elasticsearch Cluster. These variables are used in the file *config_cluster.sh* to set them in the *elasticsearch.yml* configuration file. You can see the meaning of the node variables [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html) and other cluster settings [here](https://github.com/elastic/elasticsearch/blob/master/distribution/src/config/elasticsearch.yml). ## Documentation