WIZZIE-1267 Refactor druid-docker and update to 0.12 version (#11)

* WIZZIE-1267 Refactor druid-docker and update to 0.12 version

* Update README.md

* Update README.md

Dockerfile

@@ -0,0 +1,18 @@
+FROM openjdk:8-jdk
+
+ARG DRUID_VERSION=0.12.0
+ENV DRUID_VERSION ${DRUID_VERSION}
+ENV LOG_LEVEL info
+RUN apt-get update
+RUN apt-get install -y curl gettext-base
+RUN curl http://static.druid.io/artifacts/releases/druid-${DRUID_VERSION}-bin.tar.gz > /opt/druid-${DRUID_VERSION}-bin.tar.gz
+RUN tar -xvf /opt/druid-${DRUID_VERSION}-bin.tar.gz -C /opt/ && rm -f /opt/druid-${DRUID_VERSION}-bin.tar.gz
+RUN mv /opt/druid-${DRUID_VERSION} /opt/druid && mkdir -p /var/log/druid && mkdir -p /opt/druid/data
+RUN rm -f -r /opt/druid/conf /opt/druid/conf-quickstart
+
+COPY conf /opt/druid/conf/
+COPY druid-start.sh /usr/bin/
+COPY druid-service-checker.sh /usr/bin/
+COPY druid-pull-deps.sh /usr/bin/
+
+CMD druid-start.sh

README.md

@@ -1,20 +1,71 @@
 # Druid Docker
 
+## Build
+
+You can do the docker build using
+
+```
+docker build -t . druid-image:latest
+```
+
+This use the default value to DRUID_VERSION, that you can read inside Dockerfile. If you want to download other druid version you can configure using docker build args:
+
+```
+docker build --build-arg DRUID_VERSION=0.11.0 -t druid:latest .
+```
+
+## Druid Configuration
+
 The druid docker by default use derby metadata-storage and local deep storage. You can change it, to do it you could mount a volume with the new configuration into the folder `/opt/druid/conf`.
 
-## Enviroments
+## Environments
 
 * **ZOOKEEPER_SERVER**
 The zookeeper server address.
 
+* **DRUID_SERVICE**
+The name of the druid service [`broker`, `historical`, `coordinator`, `overlord`, `middleManager`]
+
 * **DRUID_HOST**
 The advertiser address that uses druid to expose the service on zookeeper.
 
+* **DRUID_SERVICE_PORT**
+The service port where bind the druid service.
+
 * **DRUID_JVM_ARGS**
 The JVM arguments to execute the druid services.
 
-## Example
+* **DRUID_PULL_EXTENSION**
+The druid extension to download, this download is performed at running time. You need to pass the extension with whitespace: `extension1:extension1:version extension2:extension2:version`
+
+## Examples
+
+* **Coordinator**
+
+```
+docker run -it -e ZOOKEEPER_SERVER=192.168.0.102 -e DRUID_SERVICE=coordinator -e DRUID_HOST=192.168.0.102 -e DRUID_SERVICE_PORT=8081 -e DRUID_JVM_ARGS="-server -Xms256m -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dderby.stream.error.file=var/druid/derby.log"
+```
+
+* **Broker**
+
+```
+docker run -it -e ZOOKEEPER_SERVER=192.168.0.102 -e DRUID_SERVICE=broker -e DRUID_HOST=192.168.0.102 -e DRUID_SERVICE_PORT=8080 -e DRUID_JVM_ARGS="-server -Xms6g -Xmx6g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -XX:NewSize=512m -XX:MaxNewSize=512m -XX:MaxDirectMemorySize=6g -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"
+```
+
+* **Historical**
+
+```
+docker run -it -e ZOOKEEPER_SERVER=192.168.0.102 -e DRUID_SERVICE=historical -e DRUID_HOST=192.168.0.102 -e DRUID_SERVICE_PORT=8081 -e DRUID_JVM_ARGS="-server -Xms2g -Xmx2g -XX:MaxDirectMemorySize=3g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -XX:NewSize=1g -XX:MaxNewSize=1g -XX:+UseConcMarkSweepGC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"
+```
+
+* **Overlord**
+
+```
+docker run -it -e ZOOKEEPER_SERVER=192.168.0.102 -e DRUID_SERVICE=overlord -e DRUID_HOST=192.168.0.102 -e DRUID_SERVICE_PORT=8084 -e DRUID_JVM_ARGS="-server -Xms256m -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"
+```
+
+* **MiddleManager**
 
 ```
-docker run -it -e ZOOKEEPER_SERVER=192.168.0.102 -e DRUID_HOST=192.168.0.102 druid-coordinator:latest
+docker run -it -e ZOOKEEPER_SERVER=192.168.0.102 -e DRUID_SERVICE=middleManager -e DRUID_HOST=192.168.0.102 -e DRUID_SERVICE_PORT=8091 -e DRUID_JVM_ARGS="-server -Xms64m -Xmx64m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"
 ```

conf/broker/runtime.properties_env

@@ -1,9 +1,9 @@
 # General
 druid.service=${DRUID_SERVICE}
 druid.host=${DRUID_HOST}
-druid.port=8080
+druid.port=${DRUID_SERVICE_PORT}
 
-# HTTP 
+# HTTP
 druid.server.http.numThreads=10
 druid.broker.http.numConnections=5
 

conf/coordinator/runtime.properties_env

@@ -1,5 +1,5 @@
 # General
 druid.service=${DRUID_SERVICE}
 druid.host=${DRUID_HOST}
-druid.port=8081
+druid.port=${DRUID_SERVICE_PORT}
 druid.coordinator.startDelay=PT60S

conf/historical/runtime.properties_env

@@ -1,11 +1,11 @@
 # General
 druid.service=${DRUID_SERVICE}
 druid.host=${DRUID_HOST}
-druid.port=8083
+druid.port=${DRUID_SERVICE_PORT}
 druid.server.http.numThreads=10
 
-druid.server.maxSize=10737418240
-druid.segmentCache.locations=[{"path": "/tmp", "maxSize": 10737418240}]
+druid.server.maxSize=5368709120
+druid.segmentCache.locations=[{"path": "/opt/druid/data", "maxSize": 5368709120}]
 
 druid.historical.cache.useCache=true
 druid.historical.cache.populateCache=true

conf/middleManager/runtime.properties_env

@@ -2,7 +2,7 @@
 druid.service=${DRUID_SERVICE}
 druid.host=${DRUID_HOST}
 druid.worker.ip=${DRUID_HOST}
-druid.port=8091
+druid.port=${DRUID_SERVICE_PORT}
 
 # Indexing
 druid.worker.capacity=2

conf/overlord/runtime.properties_env

@@ -1,7 +1,7 @@
 # General
 druid.service=${DRUID_SERVICE}
 druid.host=${DRUID_HOST}
-druid.port=8084
+druid.port=${DRUID_SERVICE_PORT}
 
 # Indexing
 druid.indexer.runner.type=remote

druid-pull-deps.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+pushd /opt/druid
+if [[ ! -z "${DRUID_PULL_EXTENSION}" ]]; then
+  java -cp "lib/*" -Ddruid.extensions.directory="extensions" -Ddruid.extensions.hadoopDependenciesDir="hadoop-dependencies" io.druid.cli.Main tools pull-deps --no-default-hadoop `for n in $(echo ${DRUID_PULL_EXTENSION} ); do echo -n "-c $n "; done`
+fi
+popd

druid-service-checker.sh

@@ -0,0 +1,6 @@
+OK=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:${DRUID_SERVICE_PORT}/status)
+if [ "$OK" == "200" ]; then
+   exit 0
+else
+   exit 1
+fi

druid-start.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+/usr/bin/druid-pull-deps.sh
+
 envsubst < /opt/druid/conf/_common/common.runtime.properties_env > /opt/druid/conf/_common/common.runtime.properties
 envsubst < /opt/druid/conf/_common/log4j2.xml_env > /opt/druid/conf/_common/log4j2.xml
 envsubst < /opt/druid/conf/_common/jets3t.properties_env > /opt/druid/conf/_common/jets3t.properties