Make log files into volume (#420)

* Make log files into volume * Make log files into volume
CanastaWiki · Jul 18, 2024 · 06e3949 · 06e3949
1 parent 7e57acf
commit 06e3949
Show file tree

Hide file tree

Showing 6 changed files with 95 additions and 48 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -7,6 +7,7 @@ ENV MW_VERSION=REL1_39 \
 	MW_CORE_VERSION=1.39.8 \
 	WWW_ROOT=/var/www/mediawiki \
 	MW_HOME=/var/www/mediawiki/w \
+	MW_LOG=/var/log/mediawiki \
 	MW_ORIGIN_FILES=/mw_origin_files \
 	MW_VOLUME=/mediawiki \
 	WWW_USER=www-data \
@@ -84,6 +85,7 @@ RUN set -x; \
 	&& a2enmod proxy_fcgi \
     # Create directories
     && mkdir -p $MW_HOME \
+	&& mkdir -p $MW_LOG \
     && mkdir -p $MW_ORIGIN_FILES \
     && mkdir -p $MW_VOLUME
 
@@ -223,6 +225,7 @@ COPY _sources/configs/mpm_event.conf /etc/apache2/mods-available/mpm_event.conf
 
 RUN set -x; \
 	chmod -v +x /*.sh \
+	&& chmod -v +x /maintenance-scripts/*.sh \
 	# Sitemap directory
 	&& ln -s $MW_VOLUME/sitemap $MW_HOME/sitemap \
 	# Comment out ErrorLog and CustomLog parameters, we use rotatelogs in mediawiki.conf for the log files

diff --git a/_sources/canasta/getMediawikiSettings.php b/_sources/canasta/getMediawikiSettings.php
@@ -1,6 +1,7 @@
 <?php
 
 use MediaWiki\MediaWikiServices;
+use MediaWiki\Settings\SettingsBuilder;
 
 $mwHome = getenv( 'MW_HOME' );
 
@@ -139,8 +140,8 @@ public function getDbType() {
 	 * (for example they can check user and etc..)
 	 * but this script can be used for getting parameters when database is not initialized yet
 	 */
-	public function finalSetup() {
-		parent::finalSetup();
+	public function finalSetup(SettingsBuilder $settingsBuilder = null) {
+		parent::finalSetup($settingsBuilder);
 
 		global $wgShowExceptionDetails, $wgHooks;
 

diff --git a/_sources/scripts/maintenance-scripts/mw_job_runner.sh b/_sources/scripts/maintenance-scripts/mw_job_runner.sh
@@ -1,8 +1,9 @@
 #!/bin/bash
 
 RJ=$MW_HOME/maintenance/runJobs.php
+logfileName=mwjobrunner_log
 
-echo Starting job runner...
+echo "Starting job runner (in 10 seconds)..."
 
 # Wait 10 seconds after the server starts up to give other processes time to get started
 sleep 10
@@ -20,23 +21,30 @@ if [ -f "$MW_VOLUME/config/wikis.yaml" ]; then
 
         {
             while true; do
+                logFilePrev="$logfileNow"
+                logfileNow="$MW_LOG/$logfileName"_$(date +%Y%m%d)
+                if [ -n "$logFilePrev" ] && [ "$logFilePrev" != "$logfileNow" ]; then
+                    /rotatelogs-compress.sh "$logfileNow" "$logFilePrev" &
+                fi
+
+                date >> "$logfileNow"
                 # Job types that need to be run ASAP no matter how many of them are in the queue
                 # Those jobs should be very "cheap" to run
-                php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="enotifNotify" --server="https://$wiki_url" --wiki="$wiki_id" 
+                php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="enotifNotify" --server="https://$wiki_url" --wiki="$wiki_id" >> "$logfileNow" 2>&1
                 sleep 1
-                php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="createPage" --server="https://$wiki_url" --wiki="$wiki_id" 
+                php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="createPage" --server="https://$wiki_url" --wiki="$wiki_id" >> "$logfileNow" 2>&1
                 sleep 1
-                php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="refreshLinks" --server="https://$wiki_url" --wiki="$wiki_id" 
+                php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="refreshLinks" --server="https://$wiki_url" --wiki="$wiki_id" >> "$logfileNow" 2>&1
                 sleep 1
-                php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="htmlCacheUpdate" --maxjobs=500 --server="https://$wiki_url" --wiki="$wiki_id" 
+                php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="htmlCacheUpdate" --maxjobs=500 --server="https://$wiki_url" --wiki="$wiki_id" >> "$logfileNow" 2>&1
                 sleep 1
                 # Everything else, limit the number of jobs on each batch
                 # The --wait parameter will pause the execution here until new jobs are added,
                 # to avoid running the loop without anything to do
-                php $RJ --maxjobs=10 --server="https://$wiki_url" --wiki="$wiki_id" 
+                php $RJ --maxjobs=10 --server="https://$wiki_url" --wiki="$wiki_id" >> "$logfileNow" 2>&1
 
                 # Wait some seconds to let the CPU do other things, like handling web requests, etc
-                echo mwjobrunner waits for "$MW_JOB_RUNNER_PAUSE" seconds...
+                echo mwjobrunner waits for "$MW_JOB_RUNNER_PAUSE" seconds... >> "$logfileNow"
                 sleep "$MW_JOB_RUNNER_PAUSE"
             done
         } &

diff --git a/_sources/scripts/maintenance-scripts/mw_sitemap_generator.sh b/_sources/scripts/maintenance-scripts/mw_sitemap_generator.sh
@@ -1,53 +1,61 @@
 #!/bin/bash
 
-# read variables from LocalSettings.php
-if [ -z "$MW_SCRIPT_PATH" ]; then
-    MW_SCRIPT_PATH=$(php /getMediawikiSettings.php --variable=wgScriptPath --format=string)
-fi
-if [ -z "$MW_SCRIPT_PATH" ]; then
-    MW_SCRIPT_PATH="/w"
-fi
+. /functions.sh
 
 SCRIPT=$MW_HOME/maintenance/generateSitemap.php
+logfileName=mwsitemapgen_log
+
 # Verify the delay is >= 1, otherwise fall back to 1
 if [ "$MW_SITEMAP_PAUSE_DAYS" -lt "1" ]; then
     MW_SITEMAP_PAUSE_DAYS=1
 fi
 # Convert to seconds (suffixed sleep command has issues on OSX)
-SLEEPDAYS=$(expr $MW_SITEMAP_PAUSE_DAYS \* 60 \* 60 \* 24)
+SLEEP_DAYS=$((MW_SITEMAP_PAUSE_DAYS * 60 * 60 * 24))
 
-SITE_SERVER=$MW_SITE_SERVER
+SITE_SERVER=$(get_mediawiki_variable wgServer)
 # Fallback to https:// scheme if it's protocol-relative
 if [[ $SITE_SERVER == "//"* ]]; then
     SITE_SERVER="https:$SITE_SERVER"
 fi
 
+SCRIPT_PATH=$(get_mediawiki_variable wgScriptPath)
+
 # Adds slash to sitemap dir if it's not empty and has no starting slash
 SITEMAP_DIR=$MW_SITEMAP_SUBDIR
 if [[ -n "$SITEMAP_DIR" && "$SITEMAP_DIR" != "/"* ]]; then
   SITEMAP_DIR="/$SITEMAP_DIR"
 fi
 
-GOOGLE_PING_URL="https://www.google.com/ping?sitemap=${SITE_SERVER}${MW_SCRIPT_PATH}/sitemap${SITEMAP_DIR}/sitemap-index-${MW_SITEMAP_IDENTIFIER}.xml"
+GOOGLE_PING_URL="https://www.google.com/ping?sitemap=${SITE_SERVER}${SCRIPT_PATH}/sitemap${SITEMAP_DIR}/sitemap-index-${MW_SITEMAP_IDENTIFIER}.xml"
 
-echo Starting sitemap generator...
+echo "Starting sitemap generator (in 30 seconds)..."
 # Wait three minutes after the server starts up to give other processes time to get started
 sleep 30
 echo Sitemap generator started.
 while true; do
-    php $SCRIPT \
-      --fspath=$MW_HOME/sitemap/$MW_SITEMAP_SUBDIR \
-      --urlpath=$MW_SCRIPT_PATH/sitemap/$MW_SITEMAP_SUBDIR \
+    logFilePrev="$logfileNow"
+    logfileNow="$MW_LOG/$logfileName"_$(date +%Y%m%d)
+    if [ -n "$logFilePrev" ] && [ "$logFilePrev" != "$logfileNow" ]; then
+        /rotatelogs-compress.sh "$logfileNow" "$logFilePrev" &
+    fi
+
+    date >> "$logfileNow"
+
+    # generate the sitemap
+    php "$SCRIPT" \
+      --fspath="$MW_HOME/sitemap/$MW_SITEMAP_SUBDIR" \
+      --urlpath="$SCRIPT_PATH/sitemap/$MW_SITEMAP_SUBDIR" \
       --compress yes \
-      --server=$MW_SITE_SERVER \
+      --server="$MW_SITE_SERVER" \
       --skip-redirects \
-      --identifier=$MW_SITEMAP_IDENTIFIER
+      --identifier="$MW_SITEMAP_IDENTIFIER" \
+      >> "$logfileNow" 2>&1
 
     # sending the sitemap to google
     echo "sending to Google -> $GOOGLE_PING_URL"
     curl --silent "$GOOGLE_PING_URL" > /dev/null
 
     # Wait some seconds to let the CPU do other things, like handling web requests, etc
-    echo mwsitemapgen waits for "$SLEEPDAYS" seconds...
-    sleep "$SLEEPDAYS"
+    echo mwsitemapgen waits for "$SLEEP_DAYS" seconds... >> "$logfileNow"
+    sleep "$SLEEP_DAYS"
 done
diff --git a/_sources/scripts/maintenance-scripts/mw_transcoder.sh b/_sources/scripts/maintenance-scripts/mw_transcoder.sh
@@ -1,8 +1,9 @@
 #!/bin/bash
 
 RJ=$MW_HOME/maintenance/runJobs.php
+logfileName=mwtranscoder_log
 
-echo Starting transcoder...
+echo "Starting transcoder (in 180 seconds)..."
 
 # Wait three minutes after the server starts up to give other processes time to get started
 sleep 180
@@ -20,12 +21,19 @@ if [ -f "$MW_VOLUME/config/wikis.yaml" ]; then
         echo "$wiki_id transcoder started."
         {
             while true; do
-                php $RJ --type=webVideoTranscodePrioritized --maxjobs=10 --wiki="$wiki_id" --server="https://$wiki_url"
+                logFilePrev="$logfileNow"
+                logfileNow="$MW_LOG/$logfileName"_$(date +%Y%m%d)
+                if [ -n "$logFilePrev" ] && [ "$logFilePrev" != "$logfileNow" ]; then
+                    /rotatelogs-compress.sh "$logfileNow" "$logFilePrev" &
+                fi
+
+                date >> "$logfileNow"
+                php "$RJ" --type webVideoTranscodePrioritized --maxjobs=10 >> "$logfileNow" 2>&1
                 sleep 1
-                php $RJ --type=webVideoTranscode --maxjobs=1 --wiki="$wiki_id" --server="https://$wiki_url"
+                php "$RJ" --type webVideoTranscode --maxjobs=1 >> "$logfileNow" 2>&1
 
                 # Wait some seconds to let the CPU do other things, like handling web requests, etc
-                echo mwtranscoder waits for "$MW_JOB_TRANSCODER_PAUSE" seconds...
+                echo mwtranscoder waits for "$MW_JOB_TRANSCODER_PAUSE" seconds... >> "$logfileNow"
                 sleep "$MW_JOB_TRANSCODER_PAUSE"
             done
         } &

diff --git a/_sources/scripts/run-all.sh b/_sources/scripts/run-all.sh
@@ -4,6 +4,14 @@ set -x
 
 . /functions.sh
 
+if ! mountpoint -q -- "$MW_VOLUME"; then
+    echo "Folder $MW_VOLUME contains important data and must be mounted to persistent storage!"
+    if ! isTrue "$MW_ALLOW_UNMOUNTED_VOLUME"; then
+        exit 1
+    fi
+    echo "You allowed to continue because MW_ALLOW_UNMOUNTED_VOLUME is set as true"
+fi
+
 # Symlink all extensions and skins (both bundled and user)
 /create-symlinks.sh
 
@@ -13,8 +21,8 @@ set -x
 # $MW_VOLUME (./extensions, ./skins, ./config, ./images),
 # note that this command will also set all the necessary permissions
 echo "Syncing files..."
-rsync -ah --inplace --ignore-existing --remove-source-files \
-  -og --chown=$WWW_GROUP:$WWW_USER --chmod=Fg=rw,Dg=rwx \
+rsync -ah --inplace --ignore-existing \
+  -og --chown="$WWW_GROUP:$WWW_USER" --chmod=Fg=rw,Dg=rwx \
   "$MW_ORIGIN_FILES"/ "$MW_VOLUME"/
 
 # We don't need it anymore
@@ -30,20 +38,27 @@ rm -rf "$MW_ORIGIN_FILES"
 # hence it does not perform any recursive checks and may lead to files
 # or directories down the tree having incorrect permissions left untouched
 
-echo "Checking permissions of $MW_VOLUME..."
-if dir_is_writable $MW_VOLUME; then
-  echo "Permissions are OK!"
+# Write log files to $MW_VOLUME/log directory if target folders are not mounted
+echo "Checking permissions of Apache log dir $APACHE_LOG_DIR..."
+if ! mountpoint -q -- "$APACHE_LOG_DIR/"; then
+    mkdir -p "$MW_VOLUME/log/httpd"
+    rsync -avh --ignore-existing "$APACHE_LOG_DIR/" "$MW_VOLUME/log/httpd/"
+    mv "$APACHE_LOG_DIR" "${APACHE_LOG_DIR}_old"
+    ln -s "$MW_VOLUME/log/httpd" "$APACHE_LOG_DIR"
 else
-  chown -R "$WWW_GROUP":"$WWW_GROUP" "$MW_VOLUME"
-  chmod -R g=rwX "$MW_VOLUME"
+    chgrp -R "$WWW_GROUP" "$APACHE_LOG_DIR"
+    chmod -R g=rwX "$APACHE_LOG_DIR"
 fi
 
-echo "Checking permissions of $APACHE_LOG_DIR..."
-if dir_is_writable $APACHE_LOG_DIR; then
-  echo "Permissions are OK!"
+echo "Checking permissions of PHP-FPM log dir $PHP_LOG_DIR..."
+if ! mountpoint -q -- "$PHP_LOG_DIR/"; then
+    mkdir -p "$MW_VOLUME/log/php-fpm"
+    rsync -avh --ignore-existing "$PHP_LOG_DIR/" "$MW_VOLUME/log/php-fpm/"
+    mv "$PHP_LOG_DIR" "${PHP_LOG_DIR}_old"
+    ln -s "$MW_VOLUME/log/php-fpm" "$PHP_LOG_DIR"
 else
-  chown -R "$WWW_GROUP":"$WWW_GROUP" $APACHE_LOG_DIR
-  chmod -R g=rwX $APACHE_LOG_DIR
+    chgrp -R "$WWW_GROUP" "$PHP_LOG_DIR"
+    chmod -R g=rwX "$PHP_LOG_DIR"
 fi
 
 config_subdir_wikis() {
@@ -99,12 +114,16 @@ echo "Starting services..."
 touch "$WWW_ROOT/.maintenance"
 /run-maintenance-scripts.sh &
 
-echo "Checking permissions of $MW_VOLUME/sitemap..."
-if dir_is_writable "$MW_VOLUME/sitemap"; then
-  echo "Permissions are OK!"
+echo "Checking permissions of Mediawiki log dir $MW_LOG..."
+if ! mountpoint -q -- "$MW_LOG"; then
+    mkdir -p "$MW_VOLUME/log/mediawiki"
+    rsync -avh --ignore-existing "$MW_LOG/" "$MW_VOLUME/log/mediawiki/"
+    mv "$MW_LOG" "${MW_LOG}_old"
+    ln -s "$MW_VOLUME/log/mediawiki" "$MW_LOG"
+    chmod -R o=rwX "$MW_VOLUME/log/mediawiki"
 else
-  chown -R "$WWW_GROUP":"$WWW_GROUP" $MW_VOLUME/sitemap
-  chmod -R g=rwX $MW_VOLUME/sitemap
+    chgrp -R "$WWW_GROUP" "$MW_LOG"
+    chmod -R go=rwX "$MW_LOG"
 fi
 
 echo "Checking permissions of MediaWiki volume dir $MW_VOLUME except $MW_VOLUME/images..."