From ac3eb4a864d168d31b50f22aea21dcfd685def9f Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Tue, 26 Mar 2024 16:19:02 +0100 Subject: [PATCH 1/7] WIP: systemctl throws an error --- parts/24-gunicorn.sh | 47 ++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/parts/24-gunicorn.sh b/parts/24-gunicorn.sh index ef69fb6..fc5b79a 100644 --- a/parts/24-gunicorn.sh +++ b/parts/24-gunicorn.sh @@ -13,29 +13,34 @@ gunicorn_active-users() { ## : Shows active users in last 10 minutes grep '/history/current_history_json' | awk '{print $11}' | sort -u | wc -l)" } -gunicorn_handler-restart() { ## : Restart all handlers +gunicorn_handler-restart() { ## : Restart all handlers handle_help "$@" <<-EOF - Checks if service is inactive and restarts otherwise, if at least service 0 or 1 is running + Reads all running gunicorn instances in an array and starts first half and once it is back and serves requests, the second half EOF - for i in {0..9}; do - if systemctl status galaxy-gunicorn@$i | grep inactive >/dev/null - then - echo "not restarting galaxy-gunicorn@$i, bacause it is inactive" - elif systemctl status galaxy-gunicorn@0 | grep "GET /history/current_history_json" >/dev/null || \ - systemctl status galaxy-gunicorn@1 | grep "GET /history/current_history_json" >/dev/null - then - systemctl restart galaxy-gunicorn@$i - while true - do - if systemctl status galaxy-gunicorn@$i | grep "GET /history/current_history_json" >/dev/null - then - break - else - sleep 10 - fi - done - fi - done + readarray -t gunicorns < <(systemctl list-units --state=running | grep galaxy-gunicorn | cut -d '@' -f2 | cut -d '.' -f1) + gunicorn_len=${#gunicorns[@]} + last_unicorn=$((gunicorn_len-1)) + gunicorn_half=$((gunicorn_len/2)) + gunicorn_lower=$(printf ",%s" "${gunicorns[@]:0:$gunicorn_half}") + gunicorn_upper=$(printf ",%s" "${gunicorns[@]:$gunicorn_half:$last_unicorn}") + gunicorn_lower=${gunicorn_lower:1} + gunicorn_upper=${gunicorn_upper:1} + if systemctl status galaxy-gunicorn@{"$gunicorn_upper"} | grep "GET /history/current_history_json" >/dev/null + then + echo "First restarting: galaxy_gunicorn@{$gunicorn_lower}" + systemctl restart galaxy-gunicorn@{"$gunicorn_lower"} + while true + do + if systemctl status galaxy-gunicorn@{"$gunicorn_lower"} | grep "GET /history/current_history_json" >/dev/null + then + break + else + sleep 10 + fi + done + echo "Now restarting: galaxy_gunicorn@{$gunicorn_upper}" + systemctl restart galaxy-gunicorn@{"$gunicorn_upper"} + fi } From 3b132388243c3dcda2327d14d9abdb4d081be29c Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 27 Mar 2024 09:35:47 +0100 Subject: [PATCH 2/7] construct whole service name, by @sanjaysrikakulam --- parts/24-gunicorn.sh | 90 +++++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/parts/24-gunicorn.sh b/parts/24-gunicorn.sh index fc5b79a..c5a85c6 100644 --- a/parts/24-gunicorn.sh +++ b/parts/24-gunicorn.sh @@ -1,52 +1,58 @@ -registered_subcommands="$registered_subcommands gunicorn" -_gunicorn_short_help="Gunicorn WSGI statistics" +registered_subcommands="$registered_subcommands gunicorn" +_gunicorn_short_help="Gunicorn WSGI statistics" _gunicorn_long_help=" - Various commands that contain the gunicorn service, such as active users... Please help to extend + Various commands that contain the gunicorn service, such as active users... Please help to extend " -gunicorn_active-users() { ## : Shows active users in last 10 minutes - handle_help "$@" <<-EOF - See unique sorts IP adresses from 'GET /history/current_history_json' from last 10 minutes and prints it in influx line format +gunicorn_active-users() { ## : Shows active users in last 10 minutes + handle_help "$@" <<-EOF + See unique sorts IP adresses from 'GET /history/current_history_json' from last 10 minutes and prints it in influx line format EOF - echo "active_users,timespan=last_10_min users=$(journalctl -u galaxy-gunicorn@*.service --since '10 minutes ago' | \ - grep '/history/current_history_json' | awk '{print $11}' | sort -u | wc -l)" + echo "active_users,timespan=last_10_min users=$(journalctl -u galaxy-gunicorn@*.service --since '10 minutes ago' | \ + grep '/history/current_history_json' | awk '{print $11}' | sort -u | wc -l)" } -gunicorn_handler-restart() { ## : Restart all handlers - handle_help "$@" <<-EOF - Reads all running gunicorn instances in an array and starts first half and once it is back and serves requests, the second half - EOF +gunicorn_handler-restart() { + # Retrieve running gunicorn services matching "galaxy-gunicorn" readarray -t gunicorns < <(systemctl list-units --state=running | grep galaxy-gunicorn | cut -d '@' -f2 | cut -d '.' -f1) - gunicorn_len=${#gunicorns[@]} - last_unicorn=$((gunicorn_len-1)) - gunicorn_half=$((gunicorn_len/2)) - gunicorn_lower=$(printf ",%s" "${gunicorns[@]:0:$gunicorn_half}") - gunicorn_upper=$(printf ",%s" "${gunicorns[@]:$gunicorn_half:$last_unicorn}") - gunicorn_lower=${gunicorn_lower:1} - gunicorn_upper=${gunicorn_upper:1} - if systemctl status galaxy-gunicorn@{"$gunicorn_upper"} | grep "GET /history/current_history_json" >/dev/null + + # Calculate batch size + # batch_size=$(( (${#gunicorns[@]} + 1) / 2 )) + batch_size=$(( (${#gunicorns[@]} - 1) / 2 + 1)) + + # Construct service name ranges for each batch + batch1="" + batch2="" + for ((i = 0; i < ${#gunicorns[@]} - 1; i++)); do + if ((i < batch_size)); then + batch1+="galaxy-gunicorn@${gunicorns[i]}.service " + else + batch2+="galaxy-gunicorn@${gunicorns[i]}.service " + fi + done + # Restart each batch + if systemctl status $batch1 | grep "GET" | grep "200" >/dev/null then - echo "First restarting: galaxy_gunicorn@{$gunicorn_lower}" - systemctl restart galaxy-gunicorn@{"$gunicorn_lower"} - while true - do - if systemctl status galaxy-gunicorn@{"$gunicorn_lower"} | grep "GET /history/current_history_json" >/dev/null - then - break - else - sleep 10 - fi - done - echo "Now restarting: galaxy_gunicorn@{$gunicorn_upper}" - systemctl restart galaxy-gunicorn@{"$gunicorn_upper"} + echo "First restarting: $batch2" + systemctl restart $batch2 + while true + do + if systemctl status $batch2 | grep "GET" | grep "200" >/dev/null + then + break + else + sleep 10 + fi + done + echo "Now restarting: $batch1" + systemctl restart $batch1 fi - } -gunicorn_lastlog(){ ## : Fetch the number of seconds since the last log message was written - handle_help "$@" <<-EOF - Lets you know if any of your workers or handlers have maybe stopped processing jobs. +gunicorn_lastlog(){ ## : Fetch the number of seconds since the last log message was written + handle_help "$@" <<-EOF + Lets you know if any of your workers or handlers have maybe stopped processing jobs. $ gxadmin gunicorn lastlog journalctl.lastlog,service=galaxy-gunicorn@0 seconds=0 @@ -57,11 +63,11 @@ gunicorn_lastlog(){ ## : Fetch the number of seconds since the last log message NOW=$(date +%s) - for i in {0..10}; do - lines=$(journalctl -u galaxy-gunicorn@$i -n 1 --no-pager) - if [[ ! $lines == *"No entries"* ]]; then - timestamp=$(journalctl -u galaxy-handler@$i -n 1 --no-pager | grep -v 'Logs begin' | awk '{print $1" "$2" "$3}'); - unix=$(date -d "$timestamp" +%s) + for i in {0..10}; do + lines=$(journalctl -u galaxy-gunicorn@$i -n 1 --no-pager) + if [[ ! $lines == *"No entries"* ]]; then + timestamp=$(journalctl -u galaxy-handler@$i -n 1 --no-pager | grep -v 'Logs begin' | awk '{print $1" "$2" "$3}'); + unix=$(date -d "$timestamp" +%s) date_diff=$((NOW - unix)); echo "journalctl.lastlog,service=galaxy-handler@$i seconds=$date_diff"; fi From 66dc81319274c6dc139b47732bdbd326db4f60b5 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 27 Mar 2024 11:41:59 +0100 Subject: [PATCH 3/7] added handling for 0 and 1 handlers --- parts/24-gunicorn.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/parts/24-gunicorn.sh b/parts/24-gunicorn.sh index c5a85c6..003aa1d 100644 --- a/parts/24-gunicorn.sh +++ b/parts/24-gunicorn.sh @@ -18,23 +18,26 @@ gunicorn_handler-restart() { readarray -t gunicorns < <(systemctl list-units --state=running | grep galaxy-gunicorn | cut -d '@' -f2 | cut -d '.' -f1) # Calculate batch size - # batch_size=$(( (${#gunicorns[@]} + 1) / 2 )) - batch_size=$(( (${#gunicorns[@]} - 1) / 2 + 1)) - + batch_size=$(( (${#gunicorns[@]} + 1) / 2 )) + if ((${#gunicorns[@]} < 2)); then + echo "You have too few Gunicorn handlers to restart in batch mode. Please restart manually." + exit 1 + fi # Construct service name ranges for each batch batch1="" batch2="" - for ((i = 0; i < ${#gunicorns[@]} - 1; i++)); do + for ((i = 0; i < ${#gunicorns[@]}; i++)); do if ((i < batch_size)); then batch1+="galaxy-gunicorn@${gunicorns[i]}.service " else batch2+="galaxy-gunicorn@${gunicorns[i]}.service " fi done + echo "Found handlers: $batch1 and $batch2" # Restart each batch if systemctl status $batch1 | grep "GET" | grep "200" >/dev/null then - echo "First restarting: $batch2" + echo "First restarting: $batch2" systemctl restart $batch2 while true do From 5f5514c385613270ea97949a7519ffa64b9e0564 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 27 Mar 2024 11:44:42 +0100 Subject: [PATCH 4/7] whitespace --- parts/24-gunicorn.sh | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/parts/24-gunicorn.sh b/parts/24-gunicorn.sh index 003aa1d..3924c24 100644 --- a/parts/24-gunicorn.sh +++ b/parts/24-gunicorn.sh @@ -1,16 +1,16 @@ -registered_subcommands="$registered_subcommands gunicorn" -_gunicorn_short_help="Gunicorn WSGI statistics" +registered_subcommands="$registered_subcommands gunicorn" +_gunicorn_short_help="Gunicorn WSGI statistics" _gunicorn_long_help=" - Various commands that contain the gunicorn service, such as active users... Please help to extend + Various commands that contain the gunicorn service, such as active users... Please help to extend " -gunicorn_active-users() { ## : Shows active users in last 10 minutes - handle_help "$@" <<-EOF - See unique sorts IP adresses from 'GET /history/current_history_json' from last 10 minutes and prints it in influx line format +gunicorn_active-users() { ## : Shows active users in last 10 minutes + handle_help "$@" <<-EOF + See unique sorts IP adresses from 'GET /history/current_history_json' from last 10 minutes and prints it in influx line format EOF - echo "active_users,timespan=last_10_min users=$(journalctl -u galaxy-gunicorn@*.service --since '10 minutes ago' | \ - grep '/history/current_history_json' | awk '{print $11}' | sort -u | wc -l)" + echo "active_users,timespan=last_10_min users=$(journalctl -u galaxy-gunicorn@*.service --since '10 minutes ago' | \ + grep '/history/current_history_json' | awk '{print $11}' | sort -u | wc -l)" } gunicorn_handler-restart() { @@ -53,9 +53,9 @@ gunicorn_handler-restart() { fi } -gunicorn_lastlog(){ ## : Fetch the number of seconds since the last log message was written - handle_help "$@" <<-EOF - Lets you know if any of your workers or handlers have maybe stopped processing jobs. +gunicorn_lastlog(){ ## : Fetch the number of seconds since the last log message was written + handle_help "$@" <<-EOF + Lets you know if any of your workers or handlers have maybe stopped processing jobs. $ gxadmin gunicorn lastlog journalctl.lastlog,service=galaxy-gunicorn@0 seconds=0 @@ -66,14 +66,13 @@ gunicorn_lastlog(){ ## : Fetch the number of seconds since the last log message NOW=$(date +%s) - for i in {0..10}; do - lines=$(journalctl -u galaxy-gunicorn@$i -n 1 --no-pager) - if [[ ! $lines == *"No entries"* ]]; then - timestamp=$(journalctl -u galaxy-handler@$i -n 1 --no-pager | grep -v 'Logs begin' | awk '{print $1" "$2" "$3}'); - unix=$(date -d "$timestamp" +%s) + for i in {0..10}; do + lines=$(journalctl -u galaxy-gunicorn@$i -n 1 --no-pager) + if [[ ! $lines == *"No entries"* ]]; then + timestamp=$(journalctl -u galaxy-handler@$i -n 1 --no-pager | grep -v 'Logs begin' | awk '{print $1" "$2" "$3}'); + unix=$(date -d "$timestamp" +%s) date_diff=$((NOW - unix)); echo "journalctl.lastlog,service=galaxy-handler@$i seconds=$date_diff"; fi done -} - +} \ No newline at end of file From 4764b0404df10b5fb07489c63cb6f8cfc4ef08e0 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 27 Mar 2024 12:21:09 +0100 Subject: [PATCH 5/7] try what linter suggested --- parts/25-galaxy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/25-galaxy.sh b/parts/25-galaxy.sh index f06acea..425141e 100644 --- a/parts/25-galaxy.sh +++ b/parts/25-galaxy.sh @@ -430,7 +430,7 @@ galaxy_fix-conda-env() { ## : Fix broken conda environments done } -galaxy_fav_tools() { ## : Favourite tools in Galaxy DB +galaxy_fav-tools() { ## : Favourite tools in Galaxy DB meta <<-EOF ADDED: 15 EOF From 0ce9d7066b3848a94affb6873dbb27b604380d50 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 27 Mar 2024 12:47:19 +0100 Subject: [PATCH 6/7] linting --- parts/24-gunicorn.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/24-gunicorn.sh b/parts/24-gunicorn.sh index 3924c24..719feab 100644 --- a/parts/24-gunicorn.sh +++ b/parts/24-gunicorn.sh @@ -75,4 +75,4 @@ gunicorn_lastlog(){ ## : Fetch the number of seconds since the last log message echo "journalctl.lastlog,service=galaxy-handler@$i seconds=$date_diff"; fi done -} \ No newline at end of file +} From 970f1a2f8e8ec3aac7d4f176e34c520422d62f1a Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 27 Mar 2024 13:53:06 +0100 Subject: [PATCH 7/7] gunicorn batches --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56615bf..96fd960 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Updated: - Add summary and limit options to tool-metrics query, by @natefoo. - Updated the Wonderful Argument Parser with a fancier version, @hexylena + - Gunicorn handler-restart uses now a two-batches-approach instead of restarting handlers one-by-one @sanjaysrikakulam @mira-miracoli - Fixed: - Wonderful Argument Parser arg values could not contain spaces, @natefoo - Removed: