Skip to content

Commit

Permalink
Merge pull request #150 from mira-miracoli/gunicon-fast-restart
Browse files Browse the repository at this point in the history
Fast Gunicorn handler restart
  • Loading branch information
hexylena authored Mar 27, 2024
2 parents cc11a7b + 970f1a2 commit cf32ab5
Show file tree
Hide file tree
Showing 3 changed files with 37 additions and 23 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
- Updated:
- Add summary and limit options to tool-metrics query, by @natefoo.
- Updated the Wonderful Argument Parser with a fancier version, @hexylena
- Gunicorn handler-restart uses now a two-batches-approach instead of restarting handlers one-by-one @sanjaysrikakulam @mira-miracoli
- Fixed:
- Wonderful Argument Parser arg values could not contain spaces, @natefoo
- Removed:
Expand Down
57 changes: 35 additions & 22 deletions parts/24-gunicorn.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,30 +13,44 @@ gunicorn_active-users() { ## : Shows active users in last 10 minutes
grep '/history/current_history_json' | awk '{print $11}' | sort -u | wc -l)"
}

gunicorn_handler-restart() { ## : Restart all handlers
handle_help "$@" <<-EOF
Checks if service is inactive and restarts otherwise, if at least service 0 or 1 is running
EOF
for i in {0..9}; do
if systemctl status galaxy-gunicorn@$i | grep inactive >/dev/null
then
echo "not restarting galaxy-gunicorn@$i, bacause it is inactive"
elif systemctl status galaxy-gunicorn@0 | grep "GET /history/current_history_json" >/dev/null || \
systemctl status galaxy-gunicorn@1 | grep "GET /history/current_history_json" >/dev/null
then
systemctl restart galaxy-gunicorn@$i
gunicorn_handler-restart() {
# Retrieve running gunicorn services matching "galaxy-gunicorn"
readarray -t gunicorns < <(systemctl list-units --state=running | grep galaxy-gunicorn | cut -d '@' -f2 | cut -d '.' -f1)

# Calculate batch size
batch_size=$(( (${#gunicorns[@]} + 1) / 2 ))
if ((${#gunicorns[@]} < 2)); then
echo "You have too few Gunicorn handlers to restart in batch mode. Please restart manually."
exit 1
fi
# Construct service name ranges for each batch
batch1=""
batch2=""
for ((i = 0; i < ${#gunicorns[@]}; i++)); do
if ((i < batch_size)); then
batch1+="galaxy-gunicorn@${gunicorns[i]}.service "
else
batch2+="galaxy-gunicorn@${gunicorns[i]}.service "
fi
done
echo "Found handlers: $batch1 and $batch2"
# Restart each batch
if systemctl status $batch1 | grep "GET" | grep "200" >/dev/null
then
echo "First restarting: $batch2"
systemctl restart $batch2
while true
do
if systemctl status galaxy-gunicorn@$i | grep "GET /history/current_history_json" >/dev/null
then
break
else
sleep 10
fi
if systemctl status $batch2 | grep "GET" | grep "200" >/dev/null
then
break
else
sleep 10
fi
done
fi
done

echo "Now restarting: $batch1"
systemctl restart $batch1
fi
}

gunicorn_lastlog(){ ## : Fetch the number of seconds since the last log message was written
Expand All @@ -62,4 +76,3 @@ gunicorn_lastlog(){ ## : Fetch the number of seconds since the last log message
fi
done
}

2 changes: 1 addition & 1 deletion parts/25-galaxy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -430,7 +430,7 @@ galaxy_fix-conda-env() { ## <conda_dir/envs/>: Fix broken conda environments
done
}

galaxy_fav_tools() { ## : Favourite tools in Galaxy DB
galaxy_fav-tools() { ## : Favourite tools in Galaxy DB
meta <<-EOF
ADDED: 15
EOF
Expand Down

0 comments on commit cf32ab5

Please sign in to comment.