From 904ca4152fa3acddcf391cbabca596f8f37984b7 Mon Sep 17 00:00:00 2001 From: ruflin Date: Thu, 7 Jan 2016 17:48:57 +0100 Subject: [PATCH] Update tests to elasticsearch-220 and fix Shield * Fix Shield setup by using local elasticsearch image again * Introduce check if Shield is installed correctly * Update dependency to elasticsearch-2.2.0 * Remove additional build options as not used --- .travis.yml | 5 +- libbeat/docker-compose.yml | 18 ++--- libbeat/scripts/Makefile | 4 +- scripts/docker-entrypoint.sh | 6 ++ scripts/docker/elasticsearch/Dockerfile-2.2.0 | 70 +++++++++++++++++++ .../docker/elasticsearch/config/logging.yml | 15 ++++ .../docker/elasticsearch/docker-entrypoint.sh | 20 ++++++ scripts/docker/test.env | 2 +- 8 files changed, 126 insertions(+), 14 deletions(-) create mode 100644 scripts/docker/elasticsearch/Dockerfile-2.2.0 create mode 100644 scripts/docker/elasticsearch/config/logging.yml create mode 100755 scripts/docker/elasticsearch/docker-entrypoint.sh diff --git a/.travis.yml b/.travis.yml index bedd52a6810..6dd4e438faf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,8 +15,7 @@ os: env: matrix: - TARGETS="check" - # - TARGETS="testsuite ES_HOST=elasticsearch-173" - - TARGETS="-C libbeat testsuite ES_HOST=elasticsearch-210" + - TARGETS="-C libbeat testsuite" - TARGETS="-C topbeat testsuite" - TARGETS="-C filebeat testsuite" - TARGETS="-C winlogbeat testsuite" @@ -40,7 +39,7 @@ matrix: - os: osx env: TARGETS="-C winlogbeat crosscompile" - os: osx - env: TARGETS="-C libbeat testsuite ES_HOST=elasticsearch-210" + env: TARGETS="-C libbeat testsuite" fast_finish: true allow_failures: - env: TARGETS="-C libbeat crosscompile" diff --git a/libbeat/docker-compose.yml b/libbeat/docker-compose.yml index 33a201b3c70..8c99bd9fb4c 100644 --- a/libbeat/docker-compose.yml +++ b/libbeat/docker-compose.yml @@ -2,8 +2,7 @@ libbeat: build: .. links: - redis - - elasticsearch-173 - - elasticsearch-210 + - elasticsearch - logstash environment: - REDIS_HOST=redis @@ -11,12 +10,13 @@ libbeat: - LS_HOST=logstash - LS_TCP_PORT=5044 - LS_TLS_PORT=5055 + - ES_USER=beats + - ES_PASS=testing env_file: - build/test.env -elasticsearch-173: - image: elasticsearch:1.7.3 -elasticsearch-210: - image: elasticsearch:2.1.0 +elasticsearch: + build: ../scripts/docker/elasticsearch + dockerfile: Dockerfile-2.2.0 command: elasticsearch -Des.network.host=0.0.0.0 redis: image: redis @@ -27,5 +27,7 @@ logstash: env_file: - build/test.env links: - - elasticsearch-173 - - elasticsearch-210 + - elasticsearch + environment: + - ES_USER=beats + - ES_PASS=testing diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 7ce7c297f96..83bc1d9ae7b 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -7,7 +7,7 @@ export PATH := ./bin:$(PATH) export GO15VENDOREXPERIMENT=1 GOFILES = $(shell find . -type f -name '*.go') SHELL=/bin/bash -ES_HOST?="elasticsearch-210" +ES_HOST?="elasticsearch" BEAT_DIR?=github.com/elastic/beats BUILD_DIR=build COVERAGE_DIR=${BUILD_DIR}/coverage @@ -170,7 +170,7 @@ build-image: write-environment # To use it for running the test, set ES_HOST and REDIS_HOST environment variable to the ip of your docker-machine. .PHONY: start-environment start-environment: stop-environment - docker-compose up -d redis elasticsearch-173 elasticsearch-210 logstash + docker-compose up -d redis elasticsearch logstash .PHONY: stop-environment stop-environment: diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh index 0db355d43f4..90f125187b1 100755 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker-entrypoint.sh @@ -38,6 +38,12 @@ es_url() { auth="$auth@" fi + code=$(curl --write-out "%{http_code}\n" --silent --output /dev/null "http://${ES_HOST}:${ES_PORT}/") + + if [ $code != 401 ]; then + echo "Shield does not seem to be running" + exit 1 + fi echo "http://${auth}${ES_HOST}:${ES_PORT}" } diff --git a/scripts/docker/elasticsearch/Dockerfile-2.2.0 b/scripts/docker/elasticsearch/Dockerfile-2.2.0 new file mode 100644 index 00000000000..954afb5dfcf --- /dev/null +++ b/scripts/docker/elasticsearch/Dockerfile-2.2.0 @@ -0,0 +1,70 @@ +FROM java:8-jre + +# grab gosu for easy step-down from root +RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 +RUN arch="$(dpkg --print-architecture)" \ + && set -x \ + && curl -o /usr/local/bin/gosu -fSL "https://github.com/tianon/gosu/releases/download/1.3/gosu-$arch" \ + && curl -o /usr/local/bin/gosu.asc -fSL "https://github.com/tianon/gosu/releases/download/1.3/gosu-$arch.asc" \ + && gpg --verify /usr/local/bin/gosu.asc \ + && rm /usr/local/bin/gosu.asc \ + && chmod +x /usr/local/bin/gosu + +RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4 + +ENV ELASTICSEARCH_MAJOR 2.2 +ENV ELASTICSEARCH_VERSION 2.2.0 +ENV ELASTICSEARCH_REPO_BASE http://download.elasticsearch.org/elasticsearch/staging/2.2.0-848ea4e/repos/2.x/debian/ + +RUN echo "deb $ELASTICSEARCH_REPO_BASE stable main" > /etc/apt/sources.list.d/elasticsearch.list + +RUN set -x \ + && apt-get update \ + && apt-get install -y --no-install-recommends elasticsearch=$ELASTICSEARCH_VERSION \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH /usr/share/elasticsearch/bin:$PATH + +RUN set -ex \ + && for path in \ + /usr/share/elasticsearch/data \ + /usr/share/elasticsearch/logs \ + /usr/share/elasticsearch/config \ + /usr/share/elasticsearch/config/scripts \ + ; do \ + mkdir -p "$path"; \ + chown -R elasticsearch:elasticsearch "$path"; \ + done + +COPY config /usr/share/elasticsearch/config + +VOLUME /usr/share/elasticsearch/data + +COPY docker-entrypoint.sh / + +ENTRYPOINT ["/docker-entrypoint.sh"] + +EXPOSE 9200 9300 + +CMD ["elasticsearch"] + + + + + + +ENV ES_USER=beats +ENV ES_PASS=testing + +ENV ES_HOME=/usr/share/elasticsearch +ENV PATH=$ES_HOME/bin/shield:$ES_HOME/bin:$PATH + +RUN rm -fR /etc/elasticsearch && \ + ln -s $ES_HOME/config /etc/elasticsearch && \ + plugin -Des.plugins.staging=true install license && \ + plugin -Des.plugins.staging=true install shield + +COPY docker-entrypoint-shield.sh / + + +ENTRYPOINT ["/docker-entrypoint-shield.sh"] diff --git a/scripts/docker/elasticsearch/config/logging.yml b/scripts/docker/elasticsearch/config/logging.yml new file mode 100644 index 00000000000..c2681ac28db --- /dev/null +++ b/scripts/docker/elasticsearch/config/logging.yml @@ -0,0 +1,15 @@ +# you can override this using by setting a system property, for example -Des.logger.level=DEBUG +es.logger.level: INFO +rootLogger: ${es.logger.level}, console +logger: + # log action execution errors for easier debugging + action: DEBUG + # reduce the logging for aws, too much is logged under the default INFO + com.amazonaws: WARN + +appender: + console: + type: console + layout: + type: consolePattern + conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" diff --git a/scripts/docker/elasticsearch/docker-entrypoint.sh b/scripts/docker/elasticsearch/docker-entrypoint.sh new file mode 100755 index 00000000000..a8fca33969a --- /dev/null +++ b/scripts/docker/elasticsearch/docker-entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +# Add elasticsearch as command if needed +if [ "${1:0:1}" = '-' ]; then + set -- elasticsearch "$@" +fi + +# Drop root privileges if we are running elasticsearch +if [ "$1" = 'elasticsearch' ]; then + # Change the ownership of /usr/share/elasticsearch/data to elasticsearch + chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/data + exec gosu elasticsearch "$@" +fi + +# As argument is not related to elasticsearch, +# then assume that user wants to run his own process, +# for example a `bash` shell to explore this image +exec "$@" diff --git a/scripts/docker/test.env b/scripts/docker/test.env index 5801a621014..5856225ceaf 100644 --- a/scripts/docker/test.env +++ b/scripts/docker/test.env @@ -1,2 +1,2 @@ -ES_HOST=elasticsearch-210 +ES_HOST=elasticsearch ES_PORT=9200