From 3ef08ccf6687c09d16b47f1a5e9471f27e7dd273 Mon Sep 17 00:00:00 2001 From: Mayte Ariza Date: Sun, 10 Mar 2019 16:38:51 +0100 Subject: [PATCH 1/4] Elasticsearch dockerfile: added repository-s3 plugin --- elasticsearch/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index 1a41604b..b8b200f4 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -21,5 +21,7 @@ COPY --chown=elasticsearch:elasticsearch ./config/load_settings.sh ./ RUN chmod +x ./load_settings.sh +RUN elasticsearch-plugin install --batch repository-s3 + ENTRYPOINT ["/entrypoint.sh"] CMD ["elasticsearch"] From b15d61cda79735418852ac7c6ab1e3cb3a031313 Mon Sep 17 00:00:00 2001 From: Mayte Ariza Date: Tue, 26 Mar 2019 14:20:38 +0100 Subject: [PATCH 2/4] Added script configure_s3.sh to Elasticsearch Dockerfile --- elasticsearch/Dockerfile | 5 +++ elasticsearch/config/configure_s3.sh | 53 +++++++++++++++++++++++++++ elasticsearch/config/load_settings.sh | 19 ++++++++++ 3 files changed, 77 insertions(+) create mode 100644 elasticsearch/config/configure_s3.sh diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index b8b200f4..30a2cd0a 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -9,6 +9,8 @@ ENV API_USER="foo" \ ENV XPACK_ML="true" +ENV ENABLE_CONFIGURE_S3="false" + ENV TEMPLATE_VERSION=v3.8.2 ADD https://raw.githubusercontent.com/wazuh/wazuh/$TEMPLATE_VERSION/extensions/elasticsearch/wazuh-elastic6-template-alerts.json /usr/share/elasticsearch/config @@ -23,5 +25,8 @@ RUN chmod +x ./load_settings.sh RUN elasticsearch-plugin install --batch repository-s3 +COPY config/configure_s3.sh ./config/configure_s3.sh +RUN chmod 755 ./config/configure_s3.sh + ENTRYPOINT ["/entrypoint.sh"] CMD ["elasticsearch"] diff --git a/elasticsearch/config/configure_s3.sh b/elasticsearch/config/configure_s3.sh new file mode 100644 index 00000000..ca5db9a6 --- /dev/null +++ b/elasticsearch/config/configure_s3.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Check arguments +function CheckArgs() +{ + if [ $1 != 4 ] && [ $1 != 5 ];then + echo "Use: configure_s3.sh (By default /elasticsearch/ is added to the path)" + echo "or use: configure_s3.sh " + exit 1 + + fi +} + +# Create repository from base_path /elasticsearch/ (this last one is automatically added by the script itself, no arg version needed) +# Repository name would be "s3-repository-" plus the current elasticsearch_major_version +function CreateRepo() +{ + + elastic_ip_port="$2" + bucket_name="$3" + path="$4" + repository_name="$5" + + if [ $1 == 5 ];then + version="$6" + else + version=`curl -s $elastic_ip_port | grep number | cut -d"\"" -f4 | cut -c1` + fi + + repository="$repository_name-$version" + s3_path="$path/elasticsearch/$version" + + curl -X PUT "$elastic_ip_port/_snapshot/$repository" -H 'Content-Type: application/json' -d' + { + "type": "s3", + "settings": { + "bucket": "'$bucket_name'", + "base_path": "'$s3_path'" + } + } + ' + +} + + +function Main() +{ + CheckArgs $1 + + CreateRepo $1 $2 $3 $4 $5 $6 +} + + Main $# $1 $2 $3 $4 $5 \ No newline at end of file diff --git a/elasticsearch/config/load_settings.sh b/elasticsearch/config/load_settings.sh index bac61378..bce7dd90 100644 --- a/elasticsearch/config/load_settings.sh +++ b/elasticsearch/config/load_settings.sh @@ -23,6 +23,25 @@ done >&2 echo "Elastic is up - executing command" +if [ $ENABLE_CONFIGURE_S3 ]; then + + sleep 10 + IP_PORT="${ELASTICSEARCH_IP}:${ELASTICSEARCH_PORT}" + + if [ "x$S3_PATH" != "x" ]; then + + if [ "x$S3_ELASTIC_MAJOR" != "x" ]; then + ./config/configure_s3.sh $IP_PORT $S3_BUCKET_NAME $S3_PATH $S3_REPOSITORY_NAME $S3_ELASTIC_MAJOR + + else + ./config/configure_s3.sh $IP_PORT $S3_BUCKET_NAME $S3_PATH $S3_REPOSITORY_NAME + + fi + + fi + +fi + #Insert default templates sed -i 's| "index.refresh_interval": "5s"| "index.refresh_interval": "5s", "number_of_shards" : '"${ALERTS_SHARDS}"', "number_of_replicas" : '"${ALERTS_REPLICAS}"'|' /usr/share/elasticsearch/config/wazuh-elastic6-template-alerts.json From 1bc6ecca67c31e52c8d9efd492e0a76f5c1c0205 Mon Sep 17 00:00:00 2001 From: Mayte Ariza Date: Wed, 27 Mar 2019 10:04:24 +0100 Subject: [PATCH 3/4] Elasticsearch major version checked and repository base path changed --- elasticsearch/config/configure_s3.sh | 17 ++++++++++++----- elasticsearch/config/load_settings.sh | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/elasticsearch/config/configure_s3.sh b/elasticsearch/config/configure_s3.sh index ca5db9a6..5ec80832 100644 --- a/elasticsearch/config/configure_s3.sh +++ b/elasticsearch/config/configure_s3.sh @@ -1,18 +1,20 @@ #!/bin/bash +set -e + # Check arguments function CheckArgs() { if [ $1 != 4 ] && [ $1 != 5 ];then - echo "Use: configure_s3.sh (By default /elasticsearch/ is added to the path)" + echo "Use: configure_s3.sh (By default is added to the path and the repository name)" echo "or use: configure_s3.sh " exit 1 fi } -# Create repository from base_path /elasticsearch/ (this last one is automatically added by the script itself, no arg version needed) -# Repository name would be "s3-repository-" plus the current elasticsearch_major_version +# Create repository from base_path / (if there is no argument, current version is added) +# Repository name would be - (if there is no argument, current version is added) function CreateRepo() { @@ -27,8 +29,13 @@ function CreateRepo() version=`curl -s $elastic_ip_port | grep number | cut -d"\"" -f4 | cut -c1` fi + if ! [[ "$version" =~ ^[0-9]+$ ]];then + echo "Elasticsearch major version must be an integer" + exit 1 + fi + repository="$repository_name-$version" - s3_path="$path/elasticsearch/$version" + s3_path="$path/$version" curl -X PUT "$elastic_ip_port/_snapshot/$repository" -H 'Content-Type: application/json' -d' { @@ -50,4 +57,4 @@ function Main() CreateRepo $1 $2 $3 $4 $5 $6 } - Main $# $1 $2 $3 $4 $5 \ No newline at end of file +Main $# $1 $2 $3 $4 $5 \ No newline at end of file diff --git a/elasticsearch/config/load_settings.sh b/elasticsearch/config/load_settings.sh index bce7dd90..17154c29 100644 --- a/elasticsearch/config/load_settings.sh +++ b/elasticsearch/config/load_settings.sh @@ -24,7 +24,7 @@ done >&2 echo "Elastic is up - executing command" if [ $ENABLE_CONFIGURE_S3 ]; then - + #Wait for Elasticsearch to be ready to create the repository sleep 10 IP_PORT="${ELASTICSEARCH_IP}:${ELASTICSEARCH_PORT}" From 91e70da2b8793a29e1cfdb7aec0550c344bf343b Mon Sep 17 00:00:00 2001 From: Mayte Ariza Date: Wed, 27 Mar 2019 12:48:12 +0100 Subject: [PATCH 4/4] Configure_s3.sh: comments added --- elasticsearch/config/configure_s3.sh | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/elasticsearch/config/configure_s3.sh b/elasticsearch/config/configure_s3.sh index 5ec80832..49c88a25 100644 --- a/elasticsearch/config/configure_s3.sh +++ b/elasticsearch/config/configure_s3.sh @@ -2,7 +2,9 @@ set -e -# Check arguments +# Check number of arguments passed to configure_s3.sh. If it is different from 4 or 5, the process will finish with error. +# param 1: number of arguments passed to configure_s3.sh + function CheckArgs() { if [ $1 != 4 ] && [ $1 != 5 ];then @@ -13,8 +15,15 @@ function CheckArgs() fi } -# Create repository from base_path / (if there is no argument, current version is added) +# Create S3 repository from base_path / (if there is no argument, current version is added) # Repository name would be - (if there is no argument, current version is added) +# param 1: +# param 2: +# param 3: +# param 4: +# param 5: Optional +# output: It will show "acknowledged" if the repository has been successfully created + function CreateRepo() { @@ -49,6 +58,13 @@ function CreateRepo() } +# Run functions CheckArgs and CreateRepo +# param 1: number of arguments passed to configure_s3.sh +# param 2: +# param 3: +# param 4: +# param 5: +# param 6: Optional function Main() {