Mutagen + code improvements (#81)

* Fixing certificate revoked on Mac

* Use Mutagen to bring up core config

* Wait for necessary files to be present when
building PWA theme.

* Remove whitespaces

* Mac optimizations
- Add volume core-frontend-data for core frontend container
- Implement mutagen configuration for both core and frontend dc configs
- FIx node container not having wget
- Keep maildev configuration in local setup only

* Optimize frontend deploy script acquiring process
Make them be mounted into frontend container on deploy
instead of copied on build

* Add mutagen config without frontend container
Optimize sync making it two-way-resolved
Propagate www-data as default file owner

* Move core-frontend-data volume from main compose
to core compose

* Fix permission issue in app and nginx containers

* Rename mutagen compose files to keep originals

* Add modified original compose files back

* Move core frontend volume declaration
to corresponding compose file

* Fix user being set as file owner on alpha

* Refactor sync sessions
- remove nginx session by introducing named volume
- extend mutagen permission config with default file mode

* Revert not working Mac certificate fix

* Typo and ping removal

* Remove ping from installed packages

* Add mutagen-powered-app container name
to resolve container naming issue in yml files

* Refactor frontend container compose files
Move dependency installation to build script
Replace deploy script with single command
Remove core-frontend-data volume, mount app-data directly
Rename start-with-frontend script

* Mount all app-data into nginx

* Return magento se:up success check to deploy

* Update process file names to the relevant ones

* Return dependency installation to FE deploy

* Revert demo deploy script changes

* Use separate deploy script for Mutagen
Remove waiting for composer.json files in main script

* Use start-with-mutagen script to wait for fs

* Map Mutagen start scripts into containers

* Build scripts into containers instead of map

* Mount app-data into  frontend container

* Remove mounting start script from host

* Move waiting logic to separate script for FE cont

* Apply proper permissions to waiting script

* Implement script to wait for fs

* Removed links

* Moved additional buildstep to separate folder, for quicker build.

* Moving global installs to build

* Adding new lines in the end to most mutagen files

* Added ignores for node_modules and vendor

* Added missing max-body size nginx setting

* Merged mutagen with ScandiPWA, fixed multiple issues

* Updated gitignore with a proper file name

* Fix for mutagen lock files

* Added proper labels to containers

Co-authored-by: negzu <[email protected]>

.gitignore

@@ -4,3 +4,4 @@ src/*
 !src/composer.*
 !.gitignore
 !**/.gitkeep
+mutagen.*.lock

Dockerfile

@@ -1,7 +1,5 @@
 # hadolint ignore=DL3007
 FROM scandipwa/php:latest
-LABEL maintainer="Scandiweb <[email protected]>"
-LABEL authors="Jurijs Jegorovs [email protected]; Ilja Lapkovskis [email protected]"
 
 # Set bash by default
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -60,7 +58,7 @@ RUN { \
         echo 'from no-reply@docker'; \
         echo 'account default'; \
     } | tee /etc/msmtprc
-    
+
 # Start script, executed upon container creation from image
 COPY deploy/start.sh /start.sh
 RUN chmod +x /start.sh
@@ -85,4 +83,4 @@ CMD ["/start.sh"]
 RUN echo "$(tput setaf 3)php, composer$(tput sgr0)";\
     composer diagnose; printf "\n";\
     echo "$(tput setaf 3)nodejs, npm$(tput sgr0)";\
-    npm doctor; printf "\n";
+    npm doctor; printf "\n";

Makefile

@@ -1,57 +1,11 @@
 # Define available commands
-.PHONY: build full-rebuild push up recreate down cert pull \
- down-rm-volumes applogs logs flushall core-up core-down core-logs \
- exec
+.PHONY: cert
 
 # Variables
 current_dir := $(shell pwd)
 uid := $(shell id -u)
 gid := $(shell id -g)
 
-# Warning! Do not use soft tabs!
-up:
-	docker-compose -f docker-compose.yml -f docker-compose.local.yml -f docker-compose.ssl.yml up -d
-
-core-up:
-	docker-compose -f docker-compose.yml -f docker-compose.local.yml -f docker-compose.core.yml -f docker-compose.ssl.yml up -d
-
-build:
-	docker-compose -f docker-compose.yml -f docker-compose.local.yml -f docker-compose.ssl.yml build
-
-pull:
-	docker-compose -f docker-compose.yml -f docker-compose.local.yml -f docker-compose.ssl.yml pull
-
-down:
-	docker-compose -f docker-compose.yml -f docker-compose.local.yml -f docker-compose.ssl.yml down --remove-orphans
-
-core-down:
-	docker-compose -f docker-compose.yml -f docker-compose.local.yml -f docker-compose.core.yml -f docker-compose.ssl.yml down --remove-orphans
-
-down-rm-volumes:
-	docker-compose -f docker-compose.yml -f docker-compose.local.yml -f docker-compose.ssl.yml down -v
-
-applogs:
-	docker-compose -f docker-compose.yml -f docker-compose.local.yml -f docker-compose.ssl.yml logs -f app
-
-logs:
-	docker-compose -f docker-compose.yml -f docker-compose.local.yml -f docker-compose.ssl.yml logs -f
-
-core-logs:
-	docker-compose -f docker-compose.yml -f docker-compose.local.yml -f docker-compose.core.yml -f docker-compose.ssl.yml logs -f --tail 100
-
-full-rebuild:
-	docker-compose -f docker-compose.yml -f docker-compose.local.yml -f docker-compose.ssl.yml build --pull --no-cache
-
-flushall:
-	docker-compose -f docker-compose.yml -f docker-compose.local.yml exec varnish varnishadm "ban req.url ~ /"
-	docker-compose -f docker-compose.yml -f docker-compose.local.yml exec redis redis-cli FLUSHALL
-
-exec:
-	docker-compose -f docker-compose.yml -f docker-compose.local.yml -f docker-compose.core.yml -f docker-compose.ssl.yml exec -u user app $(filter-out $@,$(MAKECMDGOALS))
-
-%:
-	@true
-
 cert:
 	mkdir -p opt/cert
 	docker run -it --rm --init \

build/frontend/Dockerfile

@@ -1,16 +1,11 @@
 ARG NODEJS_VERSION=10
 
 FROM node:$NODEJS_VERSION
-LABEL authors="Alfred Genkins, Ilja Lapkovskis [email protected]"
 
-ARG BASEPATH=/var/www/public
+LABEL maintainer="Alfreds Genkins [email protected]"
+LABEL author="Ilja Lapkovskis [email protected]"
 
-# Set working directory so any relative configuration or scripts wont fail
-WORKDIR $BASEPATH
+RUN npm install pm2 forever -g
 
 COPY start.sh /start.sh
-COPY start-core.sh /start-core.sh
 RUN chmod +x /start.sh
-RUN chmod +x /start-core.sh
-
-CMD ["/bin/bash", "/start.sh"]

build/frontend/start.sh

@@ -1,4 +1,58 @@
 #!/bin/bash
+
+mutagen="0"
+core="0"
+
+helpFunction()
+{
+   echo ""
+   echo "Usage: $0 [-M] [-C]"
+   echo -e "\t-M Wait for Mutagen to sync files before start"
+   echo -e "\t-C Start core watching script"
+   exit 1 # Exit script after printing help
+}
+
+while getopts "MC" opt
+do
+   case "$opt" in
+      M ) mutagen="1" ;;
+      C ) core="1" ;;
+      ? ) helpFunction ;;
+   esac
+done
+
+PATH_TO_THEME="/var/www/public"
+
+if [ "$core" = "1" ]
+then
+  PATH_TO_THEME="$PATH_TO_THEME/localmodules/base-theme/"
+else
+  PATH_TO_THEME="$PATH_TO_THEME/app/design/frontend/Scandiweb/pwa/"
+fi
+
+if [ -d $PATH_TO_THEME ]
+then
+  cd $PATH_TO_THEME;
+else
+  echo "ERROR: $PATH_TO_THEME is empty!"
+  exit 1
+fi
+
+if [ "$mutagen" = "1" ]
+then
+    echo "Waiting for Mutagen to sync initial files"
+    while ! [ -f ./package.json -a -f ./package-lock.json ]
+    do
+      sleep 2
+    done
+fi
+
+echo "Installing node modules"
 npm ci
-npm install pm2 forever -g
-pm2-runtime process.yml
+
+if [ "$core" = "1" ]
+then
+  pm2-runtime process-core.yml
+else
+  pm2-runtime process.yml
+fi

build/php/Dockerfile

@@ -1,5 +1,6 @@
 FROM ubuntu:bionic
-LABEL maintainer="Ilja Lapkovskis [email protected]"
+LABEL author="Ilja Lapkovskis [email protected]"
+LABEL maintainer="Alfreds Genkins [email protected]"
 
 ARG COMPOSER_HOME=/var/lib/composer
 ARG COMPOSER_VERSION=latest
@@ -55,7 +56,7 @@ ENV PHPIZE_DEPS \
   libxslt1-dev \
   libsodium-dev \
   libargon2-0-dev
-  
+
 # Update server packages to latest versions
 RUN apt-get -qq update &&\
     apt-get -qq dist-upgrade -y &&\

build/rendertron/Dockerfile

@@ -1,5 +1,7 @@
 FROM node:10
 
+LABEL maintainer="Alfreds Genkins [email protected]"
+
 # Install latest chrome dev package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others)
 # Note: this installs the necessary libs to make the bundled version of Chromium that Puppeteer
 # installs, work.

build/varnish/Dockerfile

@@ -1,6 +1,7 @@
 FROM debian:stretch-slim
-LABEL maintainer="Ilja Lapkovskis admin@scandiweb.com"
+LABEL maintainer="Alfreds Genkins alfreds@scandipwa.com"
 LABEL author="Jurijs Jegorovs [email protected]"
+
 ENV DEBIAN_FRONTEND=noninteractive \
     VARNISH_VERSION=5.2 \
     VARNISH_VERSION_FLATTEN=52\

build/xdebug/Dockerfile

@@ -1,6 +1,7 @@
 FROM scandipwa/base
-LABEL authors="Jurijs Jegorovs [email protected]"
-LABEL maintainer="Jurijs Jegorovs [email protected]"
+
+LABEL maintainer="Alfreds Genkins [email protected]"
+
 ENV DEBIAN_FRONTEND=noninteractive
 
 # Install required PHP extensions

deploy/entrypoint.sh

@@ -7,16 +7,26 @@ chmod augo+rwx /dev/stdout /dev/stderr /proc/self/fd/1 /proc/self/fd/2
 
 # Check if set UID variable, if empty, execute as root
 if [[ -n "$LOCAL" ]]; then
-
   HOST_UID=$(stat -c "%u" $BASEPATH)
-
   echo "Starting with UID : $HOST_UID"
   useradd --shell /bin/bash -u $HOST_UID -o -c "" -m user
   export HOME=/home/user
-
   exec gosu user "$@"
-
 else
   echo "Starting as root"
   exec "$@"
-fi 
+fi
+
+# Add every user, to every group, to eliminate permission issue
+adduser root www-data
+adduser root user
+adduser user www-data
+adduser user root
+adduser www-data root
+adduser www-data user
+usermod -aG user root
+usermod -aG user www-data
+usermod -aG root user
+usermod -aG root www-data
+usermod -aG www-data user
+usermod -aG www-data root

deploy/frontend/conf/nginx/cache-router.conf

@@ -2,6 +2,7 @@ server {
   listen 80;
   server_name _;
   resolver 127.0.0.11;
+  client_max_body_size 64M;
 
   # Route all traffic to app
   location / {

deploy/shared/conf/nginx/cache-router.conf

@@ -2,18 +2,16 @@ server {
   listen 80;
   server_name _;
   resolver 127.0.0.11;
+  client_max_body_size 64M;
 
   # Route all traffic to varnish
   location / {
     set $prerender 0;
-    if ( $http_user_agent ~* 'Googlebot|slackbot') {
+
+    if ($http_user_agent ~* 'Googlebot|slackbot') {
       set $prerender 1;
     }
-
-    if ( $http_x_renderer ~* 'rendertron' ) {
-      set $prerender 0;
-    }
-
+
     if ($uri ~* "\.(js|css|xml|less|png|jpg|jpeg|webp|gif|pdf|doc|txt|ico|rss|zip|mp3|rar|exe|wmv|doc|avi|ppt|mpg|mpeg|tif|wav|mov|psd|ai|xls|mp4|m4a|swf|dat|dmg|iso|flv|m4v|torrent|ttf|woff|svg|eot)") {
       set $prerender 0;
     }
@@ -27,6 +25,7 @@ server {
     if ($prerender = 1) {
       proxy_pass http://rendertron:8083/render/https://$host$request_uri;
     }
+
     if ($prerender = 0) {
       proxy_pass http://varnish:80;
     }

deploy/shared/conf/nginx/ssl-terminator.conf

@@ -13,6 +13,8 @@ server {
   listen 443 ssl http2;
   listen [::]:443 ssl http2;
   server_name _;
+  client_max_body_size 64M;
+
   ssl_certificate /etc/nginx/cert/server_crt.pem;
   ssl_certificate_key /etc/nginx/cert/server_key.pem;
   ssl_trusted_certificate /etc/nginx/cert/scandipwa-fullchain.pem;