Merge after #1160 - Docker - update to using Ubuntu 20.04 + minor changes related to recent PRs

Dockerfile

@@ -40,37 +40,23 @@
 FROM alpine/git AS base
 
 # build args specifying the branches for webwork2 and pg used to build the image
-
-# To use the master branches of webwork2 and pg 
-#ARG WEBWORK2_BRANCH=master
-#ARG PG_BRANCH=master
-# To use the 2.15 branches of webwork2 and pg from the "official" GitHub repositories
-ARG WEBWORK2_GIT_URL=https://github.com/openwebwork/webwork2.git
-ARG WEBWORK2_BRANCH=develop
-ARG PG_GIT_URL=https://github.com/openwebwork/pg.git
-ARG PG_BRANCH=develop
-
-# assign the build args to the ENV variables
-ENV WEBWORK2_GIT_URL_ENV ${WEBWORK2_GIT_URL}
-ENV WEBWORK2_BRANCH_ENV ${WEBWORK2_BRANCH}
-ENV PG_GIT_URL_ENV ${PG_GIT_URL}
-ENV PG_BRANCH_ENV ${PG_BRANCH}
+ARG WEBWORK2_GIT_URL
+ARG WEBWORK2_BRANCH
+ARG PG_GIT_URL
+ARG PG_BRANCH
 
 WORKDIR /opt/base
 
-RUN echo Cloning branch $WEBWORK2_BRANCH_ENV from $WEBWORK2_GIT_URL_ENV \
-  && echo git clone --single-branch --branch ${WEBWORK2_BRANCH_ENV} --depth 1 $WEBWORK2_GIT_URL_ENV \
-  && git clone --single-branch --branch ${WEBWORK2_BRANCH_ENV} --depth 1 $WEBWORK2_GIT_URL_ENV \
+RUN echo Cloning branch $WEBWORK2_BRANCH from $WEBWORK2_GIT_URL \
+  && echo git clone --single-branch --branch ${WEBWORK2_BRANCH} --depth 1 $WEBWORK2_GIT_URL \
+  && git clone --single-branch --branch ${WEBWORK2_BRANCH} --depth 1 $WEBWORK2_GIT_URL \
   && rm -rf webwork2/.git webwork2/{*ignore,Dockerfile,docker-compose.yml,docker-config}
 
-RUN echo Cloning branch $PG_BRANCH_ENV branch from $PG_GIT_URL_ENV \
-  && echo git clone --single-branch --branch ${PG_BRANCH_ENV} --depth 1 $PG_GIT_URL_ENV \
-  && git clone --single-branch --branch ${PG_BRANCH_ENV} --depth 1 $PG_GIT_URL_ENV \
+RUN echo Cloning branch $PG_BRANCH branch from $PG_GIT_URL \
+  && echo git clone --single-branch --branch ${PG_BRANCH} --depth 1 $PG_GIT_URL \
+  && git clone --single-branch --branch ${PG_BRANCH} --depth 1 $PG_GIT_URL \
   && rm -rf  pg/.git
 
-RUN git clone --single-branch --branch master --depth 1 https://github.com/mathjax/MathJax \
-  && rm -rf MathJax/.git
-
 # Optional - include OPL (also need to uncomment further below when an included OPL is desired):
 #RUN git clone --single-branch --branch master --depth 1 https://github.com/openwebwork/webwork-open-problem-library.git \
 #  && rm -rf  webwork-open-problem-library/.git
@@ -81,9 +67,7 @@ RUN git clone --single-branch --branch master --depth 1 https://github.com/mathj
 
 # we need to change FROM before setting the ENV variables
 
-FROM ubuntu:18.04
-# Once ubuntu 20.04 is used, CGI.pm and CGI::Cookie will be new enough to
-# drop the cpanm install of CGI::Cookie which is needed to upgrade it.
+FROM ubuntu:20.04
 
 ENV WEBWORK_URL=/webwork2 \
     WEBWORK_ROOT_URL=http://localhost \
@@ -111,14 +95,10 @@ ENV WEBWORK_ROOT=$APP_ROOT/webwork2 \
 
 # ==================================================================
 
-# Phase 3 - Ubuntu 18.04 base image + required packages
+# Phase 3 - Ubuntu 20.04 base image + required packages
 
-# Packages changes/added for ubuntu 18.04:
-
-# For ubuntu 18.04 libemail-address-xs-perl installed from Ubuntu, for 16.04 it would be installed using cpamn
-#
-#    texlive-generic-recommended # For ubuntu 16.04 - contains path.sty
-#    texlive-plain-generic       # For ubuntu 18.04 - contains path.sty
+# Packages changes/added for ubuntu 20.04:
+#       libcgi-pm-perl (for CGI::Cookie), libdbd-mariadb-perl
 
 # Do NOT include "apt-get -y upgrade"
 # see: https://docs.docker.com/develop/develop-images/dockerfile_best-practices/
@@ -131,9 +111,11 @@ RUN apt-get update \
 	gcc \
 	libapache2-request-perl \
 	libarchive-zip-perl \
+	libcgi-pm-perl \
 	libcrypt-ssleay-perl \
 	libdatetime-perl \
 	libdbd-mysql-perl \
+	libdbd-mariadb-perl \
 	libemail-address-xs-perl \
 	libexception-class-perl \
 	libextutils-xsbuilder-perl \
@@ -200,6 +182,7 @@ RUN apt-get update \
 	libcpanel-json-xs-perl \
 	make \
 	netpbm \
+	patch \
 	preview-latex-style \
 	texlive \
 	texlive-latex-extra \
@@ -221,6 +204,8 @@ RUN apt-get update \
 	fonts-linuxlibertine \
 	lmodern \
 	zip \
+	iputils-ping \
+	imagemagick \
 	jq \
 	npm \
     && apt-get clean \
@@ -231,14 +216,13 @@ RUN apt-get update \
 
 # ==================================================================
 
-# Phase 4 - Install webwork2, pg, MathJaX which were downloaded to /opt/base/ in phase 1
+# Phase 4 - Install webwork2 and pg which were downloaded to /opt/base/ in phase 1
 #   Option: Install the OPL in the image also (about 850 MB)
 
 RUN mkdir -p $APP_ROOT/courses $APP_ROOT/libraries $APP_ROOT/libraries/webwork-open-problem-library $APP_ROOT/webwork2 /www/www/html
 
 COPY --from=base /opt/base/webwork2 $APP_ROOT/webwork2
 COPY --from=base /opt/base/pg $APP_ROOT/pg
-COPY --from=base /opt/base/MathJax $APP_ROOT/MathJax
 
 # Optional - include OPL (also need to uncomment above to clone from GitHub when needed):
 # ??? could/should this include the main OPL = /opt/base/webwork-open-problem-library/OpenProblemLibrary and not Contrib and Pending ???
@@ -278,16 +262,9 @@ RUN echo "PATH=$PATH:$APP_ROOT/webwork2/bin" >> /root/.bashrc \
 
 # Phase 6 - install additional Perl modules from CPAN (not packaged for Ubuntu or outdated in Ubuntu)
 
-# Ubuntu 18.04 has CGI.pm 4.38-1 which is too old to support the cookie samesite attribute added in CGI.pm 4.45 - so install CGI::Cookie here to get an upgraded version.
-
-RUN cpanm install Statistics::R::IO CGI::Cookie \
+RUN cpanm install Statistics::R::IO \
     && rm -fr ./cpanm /root/.cpanm /tmp/*
 
-# Now installed from Ubuntu packages:
-#     XML::Parser::EasyTree Iterator Iterator::Util Pod::WSDL Array::Utils HTML::Template Mail::Sender Email::Sender::Simple Data::Dump
-# For Ubuntu 16.04 would also need:
-#     Email::Address::XS
-
 # ==================================================================
 
 # Phase 7 - setup apache
@@ -298,6 +275,10 @@ RUN cpanm install Statistics::R::IO CGI::Cookie \
 # Always provide the dummy default-ssl.conf file:
 COPY docker-config/ssl/default-ssl.conf /etc/apache2/sites-available/default-ssl.conf
 
+# Patch files that are applied below
+COPY docker-config/xmlrpc-lite-utf8-fix.patch /tmp
+COPY docker-config/imagemagick-allow-pdf-read.patch /tmp
+
 # However SSL will only be enabled at container startup via docker-entrypoint.sh.
 
 RUN cd $APP_ROOT/webwork2/conf \
@@ -316,13 +297,20 @@ RUN cd $APP_ROOT/webwork2/conf \
     && sed -i -e 's/^<Perl>$/\
 	PerlPassEnv WEBWORK_URL\n\
 	PerlPassEnv WEBWORK_ROOT_URL\n\
-	PerlPassEnv WEBWORK_DB_DSN\n\
+	PerlPassEnv WEBWORK_DB_DRIVER\n\
+	PerlPassEnv WEBWORK_DB_NAME\n\
+	PerlPassEnv WEBWORK_DB_HOST\n\
+	PerlPassEnv WEBWORK_DB_PORT\n\
 	PerlPassEnv WEBWORK_DB_USER\n\
 	PerlPassEnv WEBWORK_DB_PASSWORD\n\
 	PerlPassEnv WEBWORK_SMTP_SERVER\n\
 	PerlPassEnv WEBWORK_SMTP_SENDER\n\
 	PerlPassEnv WEBWORK_TIMEZONE\n\
-	\n<Perl>/' /etc/apache2/conf-enabled/webwork.conf
+	\n<Perl>/' /etc/apache2/conf-enabled/webwork.conf \
+	&& patch -p1 -d / < /tmp/xmlrpc-lite-utf8-fix.patch \
+	&& rm /tmp/xmlrpc-lite-utf8-fix.patch \
+	&& patch -p1 -d / < /tmp/imagemagick-allow-pdf-read.patch \
+	&& rm /tmp/imagemagick-allow-pdf-read.patch
 
 EXPOSE 80
 WORKDIR $APP_ROOT

docker-compose.yml

@@ -2,12 +2,39 @@ version: '3.5'
 services:
   db:
     image: mariadb:10.4
+
+    # Set a fixed container name, so it does not depend on the directory name
+    container_name: webwork2_db_1
+
     volumes:
       - mysql:/var/lib/mysql
 
       # Set up UTF8MB4 in config file for the container.
       # Needs to be done BEFORE the database is created.
+      # *** Some other MariaDB settings were modified in the sample file:
+      #        wait_timeout,  interactive_timeout, max_connections, net_read_timeout
+      #        + skip-name-resolve
+      # Note: It seems different versions of the MariaDB container need
+      #       this file in different locations. Put in in both places.
       - "./docker-config/db/mariadb.cnf:/etc/mysql/conf.d/mariadb.cnf"
+      - "./docker-config/db/mariadb.cnf:/etc/mysql/mariadb.cnf"
+
+      # Tuning MariaDB config - sample file
+      # The sample file modifies wait_timeout, and adds skip-name-resolve
+      #- "./docker-config/db/my.cnf:/etc/mysql/my.cnf"
+
+      # Adjust open file limits for MariaDB/mySQL - sample file
+      #- "./docker-config/db/limits.conf:/etc/security/limits.conf"
+
+      # Provides read only access to the host system's /etc/localtime - tested on Linux hosts
+      #- "/etc/localtime:/etc/localtime:ro"
+
+      # The ulimits lines were only tested on Linux hosts in conjuction woth the limits.conf file
+#    ulimits:
+#      nofile:
+#        soft: 4096
+#        hard: 4096
+
     restart: always
     environment:
       # When the MariaDB container is first started it will set the
@@ -21,30 +48,50 @@ services:
       MYSQL_USER: ${WEBWORK_DB_USER}
       MYSQL_PASSWORD: ${WEBWORK_DB_PASSWORD}
 
+      # A timezone for the DB server can be set:
+      #TZ: zone/city
+      # where zone/city must be a valid setting.
+      # "/usr/bin/timedatectl list-timezones" on an Ubuntu system with
+      # that tool installed will find valid values.
+      # See: https://stackoverflow.com/questions/39172652/using-docker-compose-to-set-containers-timezones
+
   app:
+    # Modifying the image name can be helpful to create a new image instead of
+    # overwriting the "standard" one during testing.
     image: webwork
 
-    # Select the appropriate "build:" block:
-
-    # For use/building when docker-compose.yml is in the webwork2 directory
-    build: .
 
-    # For use/building when docker-compose.yml is OUTSIDE the webwork2 directory.
-    # For example, if multiple hosts use a NFS shared webwork2/ directory, and
-    # each one needs customized values in docker-compose.yml.
-    # Under typical use, the Dockerfile should not need to be customized per host,
-    # but may contain some changes/additions relative to the standard webwork image.
-    #
-    #build:
-    #  context: /Path_To/webwork2/
-    #  dockerfile: /Path_To/Dockerfile
+    # Set a fixed container name, so it does not depend on the directory name
+    container_name: webwork2_app_1
+
+    # Set up the "build:" configuration:
+    build:
+        # For use/building when docker-compose.yml is in the webwork2 directory
+        context: .
+        # For use/building when docker-compose.yml is OUTSIDE the webwork2 directory.
+        # For example, if multiple hosts use a NFS shared webwork2/ directory, and
+        # each one needs customized values in docker-compose.yml.
+        # Under typical use, the Dockerfile should not need to be customized per host,
+        # but may contain some changes/additions relative to the standard webwork image.
+        #
+        #  context: /Path_To/webwork2/
+        #  dockerfile: /Path_To/Dockerfile
+        args:
+            # build args specifying the branches for webwork2 and pg used to build the image
+            # To use the master branches of webwork2 and pg 
+            #- ARG WEBWORK2_BRANCH=master
+            #- ARG PG_BRANCH=master
+            # To use the 2.15 branches of webwork2 and pg from the "official" GitHub repositories
+            - WEBWORK2_GIT_URL=https://github.com/openwebwork/webwork2.git
+            - WEBWORK2_BRANCH=develop
+            - PG_GIT_URL=https://github.com/openwebwork/pg.git
+            - PG_BRANCH=develop
 
     depends_on:
       - db
       - r
 
     volumes:
-
       # ======================================================================
 
       # If you are using locally modified webwork2 files, then
@@ -138,15 +185,14 @@ services:
       APACHE_RUN_GROUP: www-data
 
       # Standard database environment variables needed by WeBWorK:
+
+      # Select which DBD driver to use
+      WEBWORK_DB_DRIVER: MariaDB
+      # WEBWORK_DB_DRIVER: mysql
+
       WEBWORK_DB_HOST: db
       WEBWORK_DB_PORT: 3306
       WEBWORK_DB_NAME: webwork
-      WEBWORK_DB_DSN: DBI:mysql:webwork:db:3306
-      # We currently need to put the same data in the WEBWORK_DB_DSN line above
-      # as we cannot use the following form, as it would be done before the values
-      # needed are available.
-      # NO GOOD # WEBWORK_DB_DSN: DBI:mysql${WEBWORK_DB_NAME}:${WEBWORK_DB_HOST}:${WEBWORK_DB_PORT}
-
 
       # These are set in the .env file and import values from there
       WEBWORK_DB_PASSWORD: ${WEBWORK_DB_PASSWORD}
@@ -196,4 +242,8 @@ services:
 
 volumes:
   oplVolume:
+    driver: local
+    name: webwork2_oplVolume
   mysql:
+    driver: local
+    name: webwork2_mysql

docker-config/db/limits.conf

@@ -0,0 +1,58 @@
+# /etc/security/limits.conf
+#
+#Each line describes a limit for a user in the form:
+#
+#<domain>        <type>  <item>  <value>
+#
+#Where:
+#<domain> can be:
+#        - a user name
+#        - a group name, with @group syntax
+#        - the wildcard *, for default entry
+#        - the wildcard %, can be also used with %group syntax,
+#                 for maxlogin limit
+#        - NOTE: group and wildcard limits are not applied to root.
+#          To apply a limit to the root user, <domain> must be
+#          the literal username root.
+#
+#<type> can have the two values:
+#        - "soft" for enforcing the soft limits
+#        - "hard" for enforcing hard limits
+#
+#<item> can be one of the following:
+#        - core - limits the core file size (KB)
+#        - data - max data size (KB)
+#        - fsize - maximum filesize (KB)
+#        - memlock - max locked-in-memory address space (KB)
+#        - nofile - max number of open file descriptors
+#        - rss - max resident set size (KB)
+#        - stack - max stack size (KB)
+#        - cpu - max CPU time (MIN)
+#        - nproc - max number of processes
+#        - as - address space limit (KB)
+#        - maxlogins - max number of logins for this user
+#        - maxsyslogins - max number of logins on the system
+#        - priority - the priority to run user process with
+#        - locks - max number of file locks the user can hold
+#        - sigpending - max number of pending signals
+#        - msgqueue - max memory used by POSIX message queues (bytes)
+#        - nice - max nice priority allowed to raise to values: [-20, 19]
+#        - rtprio - max realtime priority
+#        - chroot - change root to directory (Debian-specific)
+#
+#<domain>      <type>  <item>         <value>
+#
+
+#*               soft    core            0
+#root            hard    core            100000
+#*               hard    rss             10000
+#@student        hard    nproc           20
+#@faculty        soft    nproc           20
+#@faculty        hard    nproc           50
+#ftp             hard    nproc           0
+#ftp             -       chroot          /ftp
+#@student        -       maxlogins       4
+
+mysql soft nofile 4096
+mysql hard nofile 4096
+# End of file

docker-config/db/mariadb.cnf

@@ -39,6 +39,9 @@ max_connections = 500
 #   wait_timeout (default usually 28800),
 #   interactive_timeout (default usually 28800),
 #   net_read_timeout (default usually 60)
-wait_timeout = 28800
+wait_timeout = 86400
 interactive_timeout = 28800
 net_read_timeout = 3600
+
+# WW add skip-name-resolve (saves time)
+skip-name-resolve