-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add cinder volumes attachment clean-up scripts
- Loading branch information
root
committed
Jan 31, 2015
1 parent
46cdc3e
commit 3ebcf5a
Showing
1 changed file
with
184 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,184 @@ | ||
| #!/usr/bin/env bash | ||
| set -o pipefail | ||
|
|
||
| OPENRC_FILE=${HOME}/openrc | ||
| DB_NOVA_USER=nova | ||
| LOG_FILE="fix-volume-issues-$(date +'%F_%H-%M-%S').log" | ||
| MYSQL_DUMP_FILE="nova-db-$(date +'%F_%H-%M-%S').mysqldump" | ||
| DRY_RUN_MODE=1 | ||
| while getopts ":u:p:H:l:m:o:Pdh" FLAG; do | ||
| case $FLAG in | ||
| u) | ||
| DB_NOVA_USER=$OPTARG | ||
| ;; | ||
| p) | ||
| DB_NOVA_PASS=$OPTARG | ||
| ;; | ||
| H) | ||
| DB_HOST=$OPTARG | ||
| ;; | ||
| l) | ||
| LOG_FILE=$OPTARG | ||
| ;; | ||
| m) | ||
| MYSQL_DUMP_FILE=$OPTARG | ||
| ;; | ||
| P) | ||
| DRY_RUN_MODE=0 | ||
| ;; | ||
| o) | ||
| OPENRC_FILE=$OPTARG | ||
| ;; | ||
| d) | ||
| set -x | ||
| ;; | ||
| \?) | ||
| echo "Invalid option: -$OPTARG. Use -h for help" >&2 | ||
| exit 1 | ||
| ;; | ||
| :) | ||
| echo "Option -$OPTARG requires an argument." >&2 | ||
| exit 1 | ||
| ;; | ||
| h) | ||
| echo -e "This script cleans up Cinder volumes entries for really deleted volumes in Nova database\n\nUsage: $(basename $0) OPTIONS\n\n\ | ||
| [-u <nova_db_user>] - Optional. Nova database username. Default value: nova\n\ | ||
| -p <nova_db_pass> - Mandatory. Password for nova database. Can be provided as DB_NOVA_PASS env variable\n\ | ||
| -H <database_host> - Mandatory. IP or hostname of Nova database host. Can be provided as DB_HOST env variable\n\ | ||
| [-P] - Optional. Disable dry-run mode (default) and go to production mode. Allows changes saving to nova database.\n\ | ||
| [-l <log_file_name>] - Optional. Name of log file. Default value: fix-volume-issues-<current_date>\n\ | ||
| [-m <mysql_dump_file>] - Optional. Name of file for storing Nova DB dump prior to make any changes. Default value: nova-db-<current_date>.mysqldump\n\ | ||
| [-d] - Optional. Enable debug mode (e.g. run bash with -x key)\n\ | ||
| [-o <openrc_file>] - Optional. Location of Openrc file with Openstack credentials. Default value: ${HOME}/openrc" | ||
| exit | ||
| ;; | ||
| esac | ||
| done | ||
| shift $((OPTIND-1)) | ||
|
|
||
| if [[ -z "$DB_HOST" || -z "$DB_NOVA_PASS" ]]; then | ||
| echo "Invalid usage. <nova_db_password> and <database_host> arguments are mandatory. Use -h fo help." | ||
| exit 1 | ||
| fi | ||
|
|
||
| source $OPENRC_FILE | ||
|
|
||
| # Grab list of instances ids | ||
| INSTANCES=$(nova list --all-tenants | awk -F '|' '{print $2}' | grep -v -E "ID|^$") | ||
| if [ "$?" != "0" -o -z "$INSTANCES" ]; then | ||
| echo "Cannot retreive list of instances! Aborting..." | tee -a $LOG_FILE | ||
| exit 1 | ||
| else | ||
| echo -e ">>>>>>>>>>>> The total amount of instances to proceed: $(echo "$INSTANCES" | wc -l) <<<<<<<<<<<<\n" | ||
| fi | ||
|
|
||
| # Check if dry-run mode disabled | ||
| if [ "$DRY_RUN_MODE" == "0" ]; then | ||
| echo -e "---------- ATTENTION! Script runs in production mode! All changes to nova database will be recorded! ----------\n" | ||
| commit_changes() { | ||
| bdm_ids_to_fix=$(mysql -BN -u nova -p$DB_NOVA_PASS -h $DB_HOST nova -e "select id from block_device_mapping where volume_id='$id' and instance_uuid='$instance' and deleted='0'") | ||
|
|
||
| if [ "$?" == "0" ]; then | ||
|
|
||
| # Iterate over volume incremental ids retreived from block_device_mapping table | ||
| for bdm_id in $bdm_ids_to_fix; do | ||
| if mysql -BN -u nova -p$DB_NOVA_PASS -h $DB_HOST nova -e "update block_device_mapping set deleted='$bdm_id' where id='$bdm_id' and volume_id='$id' and instance_uuid='$instance'"; then | ||
| echo "SUCCESS: Volume $id with internal id $bdm_id was cleaned up!" | tee -a $LOG_FILE | ||
| FIXED_VOLUMES="$FIXED_VOLUMES""FIXED: volume $id, inernal volume id $bdm_id, instance $instance\n"; | ||
| else | ||
| echo "CRITICAL: Impossible to issue mysql UPDATE command for volume $id, internal id $bdm_id, instance $instance" | tee -a $LOG_FILE | ||
| NON_FIXED_VOLUMES="$NON_FIXED_VOLUMES""NOT FIXED: volume $id, inernal volume id $bdm_id, instance $instance\n"; | ||
| EXIT_CODE=1 | ||
| fi # end of mysql update if | ||
| done # end of loop over $bdm_ids_to_fix | ||
|
|
||
| else | ||
| echo "ERROR: mysql SELECT query for volume $id, instance $instance failed!" | tee -a $LOG_FILE | ||
| NON_FIXED_VOLUMES="$NON_FIXED_VOLUMES""NOT FIXED: volume $id, inernal volume id $bdm_id, instance $instance\n"; | ||
| EXIT_CODE=1 | ||
| fi # end of mysql select exit code if | ||
| } | ||
| else | ||
| echo -e "---------- INFO: Script runs in dry-run mode. Changes won't be saved ----------\n" | tee -a $LOG_FILE | ||
| commit_changes(){ | ||
| VOLUMES_TO_FIX="$VOLUMES_TO_FIX""FIXME: volume $id, inernal volume id $bdm_id, instance $instance\n"; | ||
| } | ||
| fi | ||
|
|
||
|
|
||
| # Check if we have mysql-client utilities | ||
| which mysqldump &> /dev/null && which mysql &>/dev/null | ||
| if [ "$?" != "0" ]; then | ||
| echo "mysql client and mysqldump utility aren't installed. Trying to isntall them..." | tee -a $LOG_FILE | ||
| if ! apt-get install -y -q mysql-client; then echo "mysql-client package cannot be installed. Aborting..." | tee -a $LOG_FILE; exit 1; fi | ||
| fi | ||
|
|
||
| # Create backup of nova DB: | ||
| if ! mysqldump -u nova -p$DB_NOVA_PASS -h $DB_HOST nova > $MYSQL_DUMP_FILE; then | ||
| echo "ERROR: Couldn't create backup of nova database. Exitting..." | tee -a $LOG_FILE | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Main loop tries to fix as many volume entries as possible and don't break on any errors | ||
| # But if at least one volume won't be fixed, script will exit witn non-zero code | ||
| EXIT_CODE=0 | ||
|
|
||
| for instance in $INSTANCES; do | ||
| vol_ids=$(nova show $instance | grep os-extended-volumes:volumes_attached | awk -F '|' '{print $3}') | ||
| if [ "$?" != "0" ]; then | ||
| printf '=%.0s' {1..178} | tee -a $LOG_FILE | ||
| echo -e "\nERROR: Instance $instance won't be processed: cannot determine volumes attached to it! 'nova show $instance' command failed" | tee -a $LOG_FILE | ||
| printf '=%.0s' {1..178} | tee -a $LOG_FILE | ||
| echo -e "\n\n"| tee -a $LOG_FILE | ||
| NOVA_SHOW_FAILS="$NOVA_SHOW_FAILS""Instance $instance wasn't processed due to 'nova show' fail\n" | ||
| EXIT_CODE=1 | ||
| continue | ||
| fi | ||
| if [[ ! "$vol_ids" =~ "[]" ]]; then | ||
| printf '=%.0s' {1..139} | tee -a $LOG_FILE | ||
| for id in $(echo $vol_ids |grep -o -E "[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}"); do | ||
| echo -e "\nNova shows that volume $id attached to instance $instance" | tee -a $LOG_FILE | ||
| cinder_status_mes=$(cinder show $id 2>&1) | ||
| if [ "$?" != "0" ] ; then | ||
| if [ "$cinder_status_mes" == "ERROR: No volume with a name or ID of '$id' exists." ]; then | ||
| echo "WARNING: Volume $id marked as attached to instance $instance in nova block_device_mapping table, but realy doesn't exist" | tee -a $LOG_FILE | ||
| commit_changes | ||
| else | ||
| echo "ERROR: Impossible to check volume status: 'cinder show $id' command was failed. Error message: $cinder_status_mes" | tee -a $LOG_FILE | ||
| CINDER_SHOW_FAILS="$CINDER_SHOW_FAILS""Volume $id wasn't checked due to of 'cinder show' failure\n" | ||
| EXIT_CODE=1 | ||
| fi # cinder show status message check | ||
| else | ||
| echo "Volume $id is attached to instance $instance correctly and exists in cinder DB" | tee -a $LOG_FILE | ||
| fi # cinder show exit code check | ||
| done | ||
| printf '=%.0s' {1..139} | tee -a $LOG_FILE | ||
| echo -e "\n\n"| tee -a $LOG_FILE | ||
| else | ||
| printf '=%.0s' {1..97} | tee -a $LOG_FILE | ||
| echo -e "\nInstance $instance is already OK and doesn't have any attached volumes" | tee -a $LOG_FILE | ||
| printf '=%.0s' {1..97} | tee -a $LOG_FILE | ||
| echo -e "\n\n"| tee -a $LOG_FILE | ||
| fi | ||
| done | ||
|
|
||
| if [ "$EXIT_CODE" != "0" ]; then | ||
| echo "CRITICAL: Not all volumes were checked or fixed! Check logfile $LOG_FILE for more information!\ | ||
| You can restore Nova DB from file $MYSQL_DUMP_FILE if needed" | ||
| fi | ||
|
|
||
| # Some statistics | ||
| if [ "$DRY_RUN_MODE" != "0" ]; then | ||
| echo -e ">>>>>>>>>>>>DRY RUN MODE RESULTS<<<<<<<<<<<<\n\n" | ||
| echo -e "NON-CHECKED INSTANCES: $(echo -e "$NOVA_SHOW_FAILS" | grep -c -v "^$") from $(echo "$INSTANCES" | wc -l)\n\n"$NOVA_SHOW_FAILS"\n\n" | tee -a $LOG_FILE | ||
| echo -e "NON-CHECKED VOLUMES: $(echo -e "$CINDER_SHOW_FAILS" | grep -c -v "^$")\n\n"$CINDER_SHOW_FAILS"\n\n" | tee -a $LOG_FILE | ||
| echo -e "VOLUMES TO FIX: $(echo -e "$VOLUMES_TO_FIX" | grep -c -v "^$")\n\n"$VOLUMES_TO_FIX"\n\n" | tee -a $LOG_FILE | ||
| else | ||
| echo -e ">>>>>>>>>>>>PRODUCTION MODE RESULTS<<<<<<<<<<<<\n\n" | ||
| echo -e "NON-CHECKED INSTANCES: $(echo -e "$NOVA_SHOW_FAILS" | grep -c -v "^$") from $(echo "$INSTANCES" | wc -l)\n\n"$NOVA_SHOW_FAILS"\n\n" | tee -a $LOG_FILE | ||
| echo -e "NON-CHECKED VOLUMES: $(echo -e "$CINDER_SHOW_FAILS" | grep -c -v "^$")\n\n"$CINDER_SHOW_FAILS"\n\n" | tee -a $LOG_FILE | ||
| echo -e "NUMBER OF FIXED VOLUMES: $(echo -e "$FIXED_VOLUMES" | grep -c -v "^$")\n\n"$FIXED_VOLUMES"\n\n\ | ||
| NUMBER OF NON-FIXED VOLUMES: $(echo -e "$NON_FIXED_VOLUMES" | grep -c -v "^$")\n\n"$NON_FIXED_VOLUMES"" | tee -a $LOG_FILE | ||
| fi | ||
|
|
||
| exit $EXIT_CODE |