Use Docker Healthcheck to wait for services to start (#3652)

Let Docker manager the start order of containers based on the dependencies declared in the docker-compose configuration files. The removes the need to have custom Docker entrypoint scripts that test services and wait for them to start before proceeding. Added 127.0.0.1 to the ports in local.yml so that the containers cannot be connected to remotely. Updated to Elastic stack 5.2.1 for latest.yml environment. Use docker-compose 1.11 on Travis CI. Remove docker-entrypoint.sh scripts.
elastic · Feb 24, 2017 · 651012c · 651012c
1 parent 5b2d6d7
commit 651012c
Show file tree

Hide file tree

Showing 39 changed files with 135 additions and 270 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -12,7 +12,7 @@ env:
   global:
     # Cross-compile for amd64 only to speed up testing.
     - GOX_FLAGS="-arch amd64"
-    - DOCKER_COMPOSE_VERSION: 1.9.0
+    - DOCKER_COMPOSE_VERSION: 1.11.1
     - &go_version 1.7.4
 
 matrix:

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -97,8 +97,8 @@ You can run the whole testsuite with the following command:
 Running the testsuite has the following requirements:
 
 * Python >=2.7.9
-* Docker >=1.10.0
-* Docker-compose >= 1.8.0
+* Docker >=1.12
+* Docker-compose >= 1.11
 
 
 ## Documentation

diff --git a/filebeat/docker-compose.yml b/filebeat/docker-compose.yml
@@ -1,16 +1,22 @@
-version: '2'
+version: '2.1'
 services:
   beat:
     build: ${PWD}/.
     depends_on:
-      - elasticsearch
+      - proxy_dep
     env_file:
       - ${PWD}/build/test.env
     working_dir: /go/src/github.com/elastic/beats/filebeat
     volumes:
       - ${PWD}/..:/go/src/github.com/elastic/beats/
     command: make
-    entrypoint: /go/src/github.com/elastic/beats/filebeat/docker-entrypoint.sh
+
+  # This is a proxy used to block beats until all services are healthy.
+  # See: https://github.com/docker/compose/issues/4369
+  proxy_dep:
+    image: busybox
+    depends_on:
+      elasticsearch: { condition: service_healthy }
 
   elasticsearch:
     extends:

diff --git a/filebeat/docker-entrypoint.sh b/filebeat/docker-entrypoint.sh
diff --git a/filebeat/tests/open-file-handlers/Makefile b/filebeat/tests/open-file-handlers/Makefile
@@ -1,4 +1,3 @@
-
 ES_USER?=admin
 ES_PASSWORD?=secure
 ES_HOST?=http://localhost:9200/

diff --git a/libbeat/docker-compose.yml b/libbeat/docker-compose.yml
@@ -1,13 +1,9 @@
-version: '2'
+version: '2.1'
 services:
   beat:
     build: ${PWD}/.
     depends_on:
-      - logstash
-      - elasticsearch
-      - redis
-      - sredis
-      - kafka
+      - proxy_dep
     environment:
       - REDIS_HOST=redis
       - REDIS_PORT=6379
@@ -28,39 +24,54 @@ services:
       - ${PWD}/..:/go/src/github.com/elastic/beats/
     working_dir: /go/src/github.com/elastic/beats/libbeat
     command: make
-    entrypoint: /go/src/github.com/elastic/beats/libbeat/scripts/docker-entrypoint.sh
+
+  # This is a proxy used to block beats until all services are healthy.
+  # See: https://github.com/docker/compose/issues/4369
+  proxy_dep:
+    image: busybox
+    depends_on:
+      elasticsearch: { condition: service_healthy }
+      logstash:      { condition: service_healthy }
+      kafka:         { condition: service_healthy }
+      redis:         { condition: service_healthy }
+      sredis:        { condition: service_healthy }
+      kafka:         { condition: service_healthy }
 
   elasticsearch:
     extends:
       file: ../testing/environments/${TESTING_ENVIRONMENT}.yml
       service: elasticsearch
 
-  # This host name is fixed because of the certificate
+  # This host name is static because of the certificate.
   logstash:
     extends:
       file: ../testing/environments/${TESTING_ENVIRONMENT}.yml
       service: logstash
     env_file:
       - ${PWD}/build/test.env
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
 
   redis:
-    image: redis:3.2.4-alpine
+    build: ${PWD}/../testing/environments/docker/redis
 
-  # This host name is fixed because of the certificate
+  # This host name is static because of the certificate.
   sredis: # stunnel proxy for redis
     build: ${PWD}/../testing/environments/docker/sredis
     expose:
       - 6380
-    links:
-      - redis
+    depends_on:
+      redis:
+        condition: service_healthy
     environment:
       - REDIS_HOST=redis
       - REDIS_PORT=6379
     env_file:
       - ${PWD}/build/test.env
 
   kafka:
-    image: spotify/kafka
+    build: ${PWD}/../testing/environments/docker/kafka
     expose:
       - 9092
       - 2181

diff --git a/libbeat/scripts/docker-entrypoint.sh b/libbeat/scripts/docker-entrypoint.sh
diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
@@ -1,4 +1,4 @@
-version: '2'
+version: '2.1'
 services:
   beat:
     build: ${PWD}/.
@@ -10,24 +10,10 @@ services:
       # This is required for the docker module tests
       - /var/run/docker.sock:/var/run/docker.sock
     command: make
-    entrypoint: /go/src/github.com/elastic/beats/metricbeat/docker-entrypoint.sh
 
-    # Module specific dependencies
+    # Wait for all services to be up and healthy.
     depends_on:
-      - apache
-      - ceph
-      - couchbase
-      - haproxy
-      - jolokia
-      - kafka
-      - mongodb
-      - mysql
-      - nginx
-      - phpfpm
-      - postgresql
-      - prometheus
-      - redis
-      - zookeeper
+      - proxy_dep
 
     env_file:
       - ${PWD}/module/apache/_meta/env
@@ -45,6 +31,26 @@ services:
       - ${PWD}/module/redis/_meta/env
       - ${PWD}/module/zookeeper/_meta/env
 
+  # This is a proxy used to block beats until all services are healthy.
+  # See: https://github.com/docker/compose/issues/4369
+  proxy_dep:
+    image: busybox
+    depends_on:
+      apache:     { condition: service_healthy }
+      ceph:       { condition: service_healthy }
+      couchbase:  { condition: service_healthy }
+      haproxy:    { condition: service_healthy }
+      jolokia:    { condition: service_healthy }
+      kafka:      { condition: service_healthy }
+      mongodb:    { condition: service_healthy }
+      mysql:      { condition: service_healthy }
+      nginx:      { condition: service_healthy }
+      phpfpm:     { condition: service_healthy }
+      postgresql: { condition: service_healthy }
+      prometheus: { condition: service_healthy }
+      redis:      { condition: service_healthy }
+      zookeeper:  { condition: service_healthy }
+
   # Modules
   apache:
     build: ${PWD}/module/apache/_meta

diff --git a/metricbeat/docker-entrypoint.sh b/metricbeat/docker-entrypoint.sh
diff --git a/metricbeat/module/apache/_meta/Dockerfile b/metricbeat/module/apache/_meta/Dockerfile
@@ -1,2 +1,4 @@
 FROM httpd:2.4.20
+RUN apt-get update && apt-get install -y curl
+HEALTHCHECK CMD curl -f http://localhost
 COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf
diff --git a/metricbeat/module/ceph/_meta/Dockerfile b/metricbeat/module/ceph/_meta/Dockerfile
@@ -3,4 +3,6 @@ FROM ceph/demo:tag-build-master-jewel-centos-7
 ENV MON_IP 0.0.0.0
 ENV CEPH_PUBLIC_NETWORK 0.0.0.0/0
 
+RUN yum install -y nc && yum clean all
+HEALTHCHECK CMD nc -w 1 -v 127.0.0.1 5000 </dev/null
 EXPOSE 5000
diff --git a/metricbeat/module/couchbase/_meta/Dockerfile b/metricbeat/module/couchbase/_meta/Dockerfile
@@ -1,4 +1,5 @@
 FROM couchbase:4.5.1
+HEALTHCHECK CMD curl -f http://localhost:8091
 
 COPY configure-node.sh /opt/couchbase
 

diff --git a/metricbeat/module/haproxy/_meta/Dockerfile b/metricbeat/module/haproxy/_meta/Dockerfile
@@ -1,2 +1,5 @@
 FROM haproxy:1.6
-COPY ./haproxy.conf /usr/local/etc/haproxy/haproxy.cfg
+RUN apt-get update && apt-get install -y netcat
+HEALTHCHECK CMD nc -z localhost 14567
+COPY ./haproxy.conf /usr/local/etc/haproxy/haproxy.cfg
+EXPOSE 14567
diff --git a/metricbeat/module/jolokia/_meta/Dockerfile b/metricbeat/module/jolokia/_meta/Dockerfile
@@ -3,6 +3,7 @@ FROM jolokia/java-jolokia:7
 ENV TOMCAT_VERSION 7.0.55
 ENV TC apache-tomcat-${TOMCAT_VERSION}
 
+HEALTHCHECK CMD curl -f curl localhost:8778/jolokia/
 EXPOSE 8778
 RUN wget http://archive.apache.org/dist/tomcat/tomcat-7/v${TOMCAT_VERSION}/bin/${TC}.tar.gz
 RUN tar xzf ${TC}.tar.gz -C /opt