EZP-27788: Update and cleanup docker config for Docker 17.06 features (…

…#192)

* [Docker] Update to Docker 17.06 & improve Mac performance with :cached

Now that Docker 17.06 stable release is out, `:cached` and other features
are now available in stable versions and can be used (17.04/5: edge).

Bumps compose version all the way up to 3.3 which came in 17.06 for
future use and hopefully not having to bump in in a while.

* [Docker] Use healtcheck images for mariadb and redis

Should theoretically get rid of the issue where app image starts up
before mariadb is ready for connections, leading to user reaching 500 page.

Note: The previous commit adding cached mounts for the sql file mostly fixes the
      issue, but this commit should in theory avoid app contianer starting to early.

* [Docker] Add prod image for nginx

Based on code from ezsystems/ezplatform-ee-demo#33

* [Docker] Update Varnish to use stretch & build config into image

* [Travis] Use Docker Compose 1.14 and Docker 17.06

* [Docker] Rename folder to doc/docker now that it contains more then just compose files

As a bonus this also shorten the paths when working with the files.

* [Docker] Swithc to dev by default now that we have :cached folders

* [Doc] Update doc to better reflect what the bundled docker blueprints are for

Aslo rather add links to Launchpad for people wanting to get up and running easily.

* [Docker] Minor fixes in Dockerfiles

* [Docker] Add env variable for malloc size

.dockerignore

@@ -41,6 +41,7 @@ web/bundles/
 web/css/
 web/js/
 web/uploads/
+web/fonts/
 
 # Assets managed by Bower
 web/assets/vendor/

.env

@@ -1,6 +1,5 @@
 # Allows us to set default values of env variables, see: https://docs.docker.com/compose/env-file/
-# On v1.7.0 file separator if defining several is different from Win and nix, so we just use one for default prod setup
-COMPOSE_FILE=doc/docker-compose/base-prod.yml
+COMPOSE_FILE=doc/docker/base-dev.yml
 COMPOSE_DIR=.
 
 # You'll need to adjust this for Windows and XDB Linux systems: https://getcomposer.org/doc/03-cli.md#composer-home
@@ -13,9 +12,9 @@ DATABASE_NAME=ezp
 PHP_IMAGE=ezsystems/php:7.0-v1
 PHP_IMAGE_DEV=ezsystems/php:7.0-v1-dev
 NGINX_IMAGE=nginx:stable
-MYSQL_IMAGE=mariadb:10.0
+MYSQL_IMAGE=healthcheck/mariadb
 SELENIUM_IMAGE=selenium/standalone-firefox:2.53.1
-REDIS_IMAGE=redis
+REDIS_IMAGE=healthcheck/redis
 
 # App image name for use if you intend to push it to docker registry/hub.
 APP_PROD_IMAGE=my-ez-app

.gitignore

@@ -11,13 +11,14 @@
 /ezpublish_legacy
 /app/config/parameters.yml
 /app/bootstrap.php.cache
-/doc/docker-compose/entrypoint/*/*.sql
+/doc/docker/entrypoint/*/*.sql
 /web/index_treemenu.php
 /web/index_rest.php
 /web/index_cluster.php
 /web/bundles/
 /web/css/
 /web/js/
+web/fonts/
 /web/design
 /web/extension
 /web/share

.travis.yml

@@ -11,16 +11,16 @@ cache:
 
 env:
   global:
-    - COMPOSE_FILE="doc/docker-compose/base-dev.yml:doc/docker-compose/selenium.yml"
+    - COMPOSE_FILE="doc/docker/base-dev.yml:doc/docker/selenium.yml"
     - SYMFONY_ENV=behat
     - SYMFONY_DEBUG=1
   # list of behat arguments to test
   matrix:
-    - TEST_CMD="bin/behat -vv --profile=rest --suite=fullJson --tags=~@broken" COMPOSE_FILE="doc/docker-compose/base-prod.yml:doc/docker-compose/varnish.yml:doc/docker-compose/selenium.yml" WEB_HOST="varnish"
+    - TEST_CMD="bin/behat -vv --profile=rest --suite=fullJson --tags=~@broken" COMPOSE_FILE="doc/docker/base-prod.yml:doc/docker/varnish.yml:doc/docker/selenium.yml" WEB_HOST="varnish"
     - TEST_CMD="bin/behat -vv --profile=rest --suite=fullXml --tags=~@broken"
     - TEST_CMD="bin/behat -vv --profile=core --tags=~@broken"
     - TEST_CMD="bin/phpunit -v vendor/ezsystems/ezpublish-kernel/eZ/Bundle/EzPublishRestBundle/Tests/Functional"
-    - TEST_CMD="bin/behat -vv --profile=platformui --tags='@common'" COMPOSE_FILE="doc/docker-compose/base-dev.yml:doc/docker-compose/redis.yml:doc/docker-compose/selenium.yml"
+    - TEST_CMD="bin/behat -vv --profile=platformui --tags='@common'" COMPOSE_FILE="doc/docker/base-dev.yml:doc/docker/redis.yml:doc/docker/selenium.yml"
 
 # test only master (+ Pull requests)
 branches:
@@ -35,14 +35,14 @@ before_script:
   # Internal auth token dedicated to testing with travis+composer on ezsystems repos, not for reuse!
   - echo "{\"github-oauth\":{\"github.com\":\"d0285ed5c8644f30547572ead2ed897431c1fc09\"}}" > auth.json
   # In case of dev mode we'll need to install composer packages first
-  - docker-compose -f doc/docker-compose/install.yml up --abort-on-container-exit
+  - docker-compose -f doc/docker/install.yml up --abort-on-container-exit
   # Run (start containers and execute install command)
   - docker-compose up -d
   #- docker ps
   #- docker-compose logs
 
 # Execute test command, need to use sh to get right exit code (docker/compose/issues/3379)
-# Behat will use behat.yml which is a copy of behat.yml.dist with hostnames update by doc/docker-compose/selenium.yml
+# Behat will use behat.yml which is a copy of behat.yml.dist with hostnames update by doc/docker/selenium.yml
 script: docker-compose exec --user www-data app sh -c "php /scripts/wait_for_db.php; php $TEST_CMD"
 
 after_failure:

RUNNING_BEHAT.md

@@ -1,7 +1,7 @@
 # Running the behat features
 
 *Note: If you want there is a way to run Behat using Docker setup, where you won't have to install any software other
-then Docker. See `doc/docker-compose/README.md` for further info.*
+then Docker. See `doc/docker/README.md` for further info.*
 
 ## Install selenium server
 Download the last version of the selenium server on the [download page](http://www.seleniumhq.org/download/).

bin/.travis/trusty/setup_from_external_repo.sh

@@ -9,7 +9,7 @@
 ## Example use:
 #
 # env:
-#  - COMPOSE_FILE="doc/docker-compose/prod.yml:doc/docker-compose/selenium.yml"
+#  - COMPOSE_FILE="doc/docker/prod.yml:doc/docker/selenium.yml"
 #
 # before_install:
 #  - git fetch --unshallow && git checkout -b tmp_travis_branch
@@ -39,6 +39,10 @@ ls -al .
 echo "> Modify composer.json to point to local checkout"
 composer config repositories.tmp_travis_folder git ${HOME}/build/ezplatform/tmp_travis_folder
 
+
+# Setup symlink for doc/docker-compose folder for compatibility with older package branches using this
+ln -s docker doc/docker-compose
+
 if [ "$RUN_INSTALL" = "1" ] ; then
   # TODO: avoid using composer on host so image don't need to be PHP image, needed atm as .
   # TODO: dockerignore or something strips info needed for composer to be able to find tmp_travis_branch
@@ -62,7 +66,7 @@ if [ "$RUN_INSTALL" = "1" ] ; then
   find app/cache app/logs web/var -type f | xargs chmod -R 664
   # Do NOT use this for your prod setup, this is done like this for behat
   sudo chown -R www-data:www-data app/config src
-  #docker-compose -f doc/docker-compose/install.yml up --abort-on-container-exit
+  #docker-compose -f doc/docker/install.yml up --abort-on-container-exit
 fi
 
 INSTALL_EZ_INSTALL_TYPE=${INSTALL_EZ_INSTALL_TYPE:-clean}

bin/.travis/trusty/update_docker.sh

@@ -3,7 +3,7 @@
 # Update package info and selectively update docker-engine (and keep old travis specific config file)
 docker -v
 sudo apt-get update
-sudo apt-get --reinstall -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install docker-engine
+sudo apt-get --reinstall -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install docker-ce
 docker -v
 
 # If we need to pin it to a given version:
@@ -12,7 +12,7 @@ docker -v
 
 
 docker-compose -v
-DOCKER_COMPOSE_VERSION="1.13.0"
+DOCKER_COMPOSE_VERSION="1.14.0"
 echo "\nUpdating Docker Compose to ${DOCKER_COMPOSE_VERSION}"
 sudo rm -f /usr/local/bin/docker-compose
 curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose

doc/docker/Dockerfile-nginx

@@ -0,0 +1,40 @@
+FROM ezsystems/php:7.1-v1 as web-build
+
+ENV SYMFONY_ENV=prod
+
+# Copy in project files into work dir
+COPY . /var/www
+
+# Create asset directories that might not exists
+RUN if [ ! -d /var/www/web/fonts ]; then mkdir /var/www/web/fonts; fi
+RUN if [ ! -d /var/www/web/assets ]; then mkdir /var/www/web/assets; fi
+
+# Generate assets using hard copy as we need to copy them over to resulting image
+RUN composer config extra.symfony-assets-install hard
+RUN composer run-script post-install-cmd --no-interaction
+
+
+# Copy over just the files we want in second stage, so resulting stage only has assets
+# and vhost config in as few layers as possible
+FROM nginx:stable as web-multilayers
+
+COPY bin/vhost.sh /var/www/bin/vhost.sh
+COPY doc/nginx/vhost.template /var/www/doc/nginx/vhost.template
+
+# Auto generated assets
+COPY --from=web-build /var/www/web/bundles /var/www/web/bundles
+COPY --from=web-build /var/www/web/css /var/www/web/css
+COPY --from=web-build /var/www/web/fonts /var/www/web/fonts
+COPY --from=web-build /var/www/web/js /var/www/web/js
+
+# User provided assets
+COPY --from=web-build /var/www/web/assets /var/www/web/assets
+
+
+# In third stage build the resulting image
+FROM nginx:stable
+
+COPY --from=web-multilayers /var/www /var/www
+COPY doc/nginx/ez_params.d /etc/nginx/ez_params.d
+
+CMD /bin/bash -c "cd /var/www && bin/vhost.sh --template-file=doc/nginx/vhost.template > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'"

doc/docker/Dockerfile-varnish

@@ -0,0 +1,20 @@
+FROM debian:stretch-slim
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Use packages from Varnish to get Varnsih 5.1, 5.0 packages with debian:stretch gives segmentation faults
+RUN apt-get update -q -y && \
+    apt-get install -q -y --no-install-recommends ca-certificates curl && \
+    curl -s https://packagecloud.io/install/repositories/varnishcache/varnish5/script.deb.sh | bash && \
+    apt-get install -q -y --allow-unauthenticated --no-install-recommends varnish && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY doc/varnish/vcl/varnish4.vcl /etc/varnish/default.vcl
+COPY doc/docker/entrypoint/varnish/parameters.vcl /etc/varnish/parameters.vcl
+
+ENV VARNISH_MALLOC_SIZE="256M"
+
+EXPOSE 80 6082
+
+# CMD in shell form to be able to use env variables here
+CMD varnishd -F -a :80 -T :6082 -f /etc/varnish/default.vcl -s malloc,${VARNISH_MALLOC_SIZE}

doc/docker-compose/README.md → doc/docker/README.md

@@ -14,8 +14,9 @@ Mac/Windows use of shared folders. This is a know issue and nothing we intend to
 
 ## Overview
 
-This setup requires Docker Compose 1.9 or higher, and Docker 1.12 or higher. Defaults are set in `.env`, and
-files to ignore are set in `.dockerignore`. By default `.env` specifies that production image is built and setup for use.
+This setup currently requires Docker Compose 1.14 and Docker 17.06 or higher. Defaults are set in `.env`, and
+files to ignore are set in `.dockerignore`. By default `.env` specifies that dev setup is used.
+
 _**NB:** For this and other reasons all docker-compose commands **must** be executed from root of your project directory._
 
 #### Before you begin: Install Docker & Docker-Compose
@@ -40,7 +41,7 @@ The current Docker Compose files are made to be mixed and matched together for Q
 
 These can be used with `-f` argument on docker-compose, like:
 ```bash
-docker-compose -f doc/docker-compose/base-prod.yml -f doc/docker-compose/redis.yml up -d --force-recreate
+docker-compose -f doc/docker/base-prod.yml -f doc/docker/redis.yml up -d --force-recreate
 ```
 
 However below environment variable `COMPOSE_FILE` is used instead since this is also what is used to have a default in
@@ -53,11 +54,13 @@ However below environment variable `COMPOSE_FILE` is used instead since this is
 
 From root of your projects clone of this distribution, [setup composer auth.json](#composer) and execute the following:
 ```sh
+export COMPOSE_FILE=doc/docker/base-prod.yml
+
 # Optional step if you'd like to use blackfire with the setup, change <id> and <token> with your own values
-#export COMPOSE_FILE=doc/docker-compose/base-prod.yml:doc/docker-compose/blackfire.yml BLACKFIRE_SERVER_ID=<id> BLACKFIRE_SERVER_TOKEN=<token>
+#export COMPOSE_FILE=doc/docker/base-prod.yml:doc/docker/blackfire.yml BLACKFIRE_SERVER_ID=<id> BLACKFIRE_SERVER_TOKEN=<token>
 
 # First time: Install setup, and generate database dump:
-docker-compose -f doc/docker-compose/install.yml up --abort-on-container-exit
+docker-compose -f doc/docker/install.yml up --abort-on-container-exit
 
 # Boot up full setup:
 docker-compose up -d --force-recreate
@@ -73,13 +76,11 @@ by default under the hood, which leads to much slower IO performance.*
 
 From root of your projects clone of this distribution, [setup composer auth.json](#composer) and execute the following:
 ```sh
-export COMPOSE_FILE=doc/docker-compose/base-dev.yml
-
 # Optional: If you use Docker Machine with NFS, you'll need to specify where project is, & give composer a valid directory.
-#export COMPOSE_DIR=/data/SOURCES/MYPROJECTS/ezplatform/doc/docker-compose COMPOSER_HOME=/tmp
+#export COMPOSE_DIR=/data/SOURCES/MYPROJECTS/ezplatform/doc/docker COMPOSER_HOME=/tmp
 
 # First time: Install setup, and generate database dump:
-docker-compose -f doc/docker-compose/install.yml up --abort-on-container-exit
+docker-compose -f doc/docker/install.yml up --abort-on-container-exit
 
 # Boot up full setup:
 docker-compose up -d --force-recreate
@@ -97,10 +98,10 @@ image to Docker Hub/Registry.*
 
 From root of your projects clone of this distribution, [setup composer auth.json](#composer) and execute the following:
 ```sh
-export COMPOSE_FILE=doc/docker-compose/base-prod.yml:doc/docker-compose/selenium.yml
+export COMPOSE_FILE=doc/docker/base-prod.yml:doc/docker/selenium.yml
 
 # First time: Install setup, and generate database dump:
-docker-compose -f doc/docker-compose/install.yml up --abort-on-container-exit
+docker-compose -f doc/docker/install.yml up --abort-on-container-exit
 
 # Boot up full setup:
 docker-compose up -d --force-recreate
@@ -153,11 +154,10 @@ docker-compose ps
 
 ### Database dumps
 
-Database dump is placed in `doc/docker-compose/entrypoint/mysql/`, this folder is used my mysql/mariadb which will execute
+Database dump is placed in `doc/docker/entrypoint/mysql/`, this folder is used my mysql/mariadb which will execute
 everything inside the folder. This means there should only be data represent one install in the folder at any given time.
 
 
-
 ### Updating service images
 
 To updated the used service images, you can run:

doc/docker-compose/base-dev.yml → doc/docker/base-dev.yml

@@ -1,12 +1,12 @@
-version: '2.1'
+version: '3.3'
 # Single server setup for dev
 
 services:
   app:
     image: ${PHP_IMAGE}
     volumes:
-     - ${COMPOSE_DIR}/../../:/var/www
-     - ${COMPOSER_HOME}:/root/.composer
+     - ${COMPOSE_DIR}/../../:/var/www:cached
+     - ${COMPOSER_HOME}:/root/.composer:cached
     depends_on:
      - db
     environment:
@@ -19,11 +19,15 @@ services:
      - DATABASE_PASSWORD
      - DATABASE_NAME
      - DATABASE_HOST=db
+    networks:
+     - backend
 
   web:
     image: ${NGINX_IMAGE}
-    volumes_from:
-     - app:ro
+    volumes:
+     - ${COMPOSE_DIR}/../../:/var/www:ro,cached
+    depends_on:
+     - app
     ports:
      - "8080:80"
     environment:
@@ -33,14 +37,23 @@ services:
      - TIMEOUT=190
      - DOCKER0NET
     command: /bin/bash -c "cd /var/www && cp -a doc/nginx/ez_params.d /etc/nginx && bin/vhost.sh --template-file=doc/nginx/vhost.template > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'"
+    networks:
+     - frontend
+     - backend
 
   db:
     image: ${MYSQL_IMAGE}
     volumes:
-     - ${COMPOSE_DIR}/entrypoint/mysql:/docker-entrypoint-initdb.d/:ro
+     - ${COMPOSE_DIR}/entrypoint/mysql:/docker-entrypoint-initdb.d/:ro,cached
     environment:
      - MYSQL_RANDOM_ROOT_PASSWORD=1
      - MYSQL_USER=$DATABASE_USER
      - MYSQL_PASSWORD=$DATABASE_PASSWORD
      - MYSQL_DATABASE=$DATABASE_NAME
      - TERM=dumb
+    networks:
+     - backend
+
+networks:
+  frontend:
+  backend: