diff --git a/scripts/generate_dump b/scripts/generate_dump index 53adbda8ba96..c182d69f62c8 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -6,6 +6,10 @@ set -u +ERROR_TAR_FAILED=5 +ERROR_PROCFS_SAVE_FAILED=6 +ERROR_INVALID_ARGUMENT=10 + TAR=tar MKDIR=mkdir RM=rm @@ -14,10 +18,13 @@ GZIP=gzip CP=cp MV=mv GREP=grep +TOUCH=touch V= NOOP=false DO_COMPRESS=true CMD_PREFIX= +SINCE_DATE="@0" # default is set to January 1, 1970 at 00:00:00 GMT +REFERENCE_FILE=/tmp/reference BASE=sonic_dump_`hostname`_`date +%Y%m%d_%H%M%S` DUMPDIR=/var/dump TARDIR=$DUMPDIR/$BASE @@ -72,7 +79,7 @@ save_cmd() { fi fi ($TAR $V -rhf $TARFILE -C $DUMPDIR "$tarpath" \ - || abort 5 "tar append operation failed. Aborting to prevent data loss.") \ + || abort "${ERROR_TAR_FAILED}" "tar append operation failed. Aborting to prevent data loss.") \ && $RM $V -rf "$filepath" } @@ -151,7 +158,7 @@ save_proc() { local procfiles="$@" $MKDIR $V -p $TARDIR/proc \ && $CP $V -r $procfiles $TARDIR/proc \ - && $TAR $V -rhf $TARFILE -C $DUMPDIR --mode=+r $BASE/proc \ + && $TAR $V -rhf $TARFILE -C $DUMPDIR --mode=+rw $BASE/proc \ && $RM $V -rf $TARDIR/proc } @@ -212,10 +219,54 @@ save_file() { fi fi ($TAR $V -rhf $TARFILE -C $DUMPDIR "$tar_path" \ - || abort 5 "tar append operation failed. Aborting to prevent data loss.") \ + || abort "${ERROR_PROCFS_SAVE_FAILED}" "tar append operation failed. Aborting to prevent data loss.") \ && $RM $V -f "$gz_path" } +############################################################################### +# find_files routine +# Globals: +# SINCE_DATE: list files only newer than given date +# REFERENCE_FILE: the file to be created as a reference to compare modification time +# Arguments: +# directory: directory to search files in +# Returns: +# None +############################################################################### +find_files() { + local -r directory=$1 + $TOUCH --date="${SINCE_DATE}" "${REFERENCE_FILE}" + local -r find_command="find -L $directory -type f -newer ${REFERENCE_FILE}" + + echo $($find_command) +} + +############################################################################### +# disable_logrotate routine +# Globals: +# None +# Arguments: +# None +# Returns: +# None +############################################################################### +disable_logrotate() { + sed -i '/logrotate/s/^/#/g' /etc/cron.d/logrotate +} + +############################################################################### +# enable_logrotate routine +# Globals: +# None +# Arguments: +# None +# Returns: +# None +############################################################################### +enable_logrotate() { + sed -i '/logrotate/s/^#*//g' /etc/cron.d/logrotate +} + ############################################################################### # Main generate_dump routine # Globals: @@ -251,7 +302,7 @@ main() { /proc/softirqs /proc/stat /proc/swaps /proc/sysvipc /proc/timer_list \ /proc/uptime /proc/version /proc/vmallocinfo /proc/vmstat \ /proc/zoneinfo \ - || abort 6 "Proc saving operation failed. Aborting for safety." + || abort "${ERROR_PROCFS_SAVE_FAILED}" "Proc saving operation failed. Aborting for safety." save_cmd "show version" "version" save_cmd "show platform summary" "platform.summary" @@ -303,9 +354,16 @@ main() { local platform="$(/usr/local/bin/sonic-cfggen -H -v DEVICE_METADATA.localhost.platform)" if [[ $platform == *"mlnx"* ]]; then local sai_dump_filename="/tmp/sai_sdk_dump_$(date +"%m_%d_%Y_%I_%M_%p")" - docker exec -it syncd saisdkdump -f $sai_dump_filename - docker exec syncd tar Ccf $(dirname $sai_dump_filename) - $(basename $sai_dump_filename) | tar Cxf /tmp/ - + ${CMD_PREFIX}docker exec -it syncd saisdkdump -f $sai_dump_filename + ${CMD_PREFIX}docker exec syncd tar Ccf $(dirname $sai_dump_filename) - $(basename $sai_dump_filename) | tar Cxf /tmp/ - save_file $sai_dump_filename sai_sdk_dump true + + local mst_dump_filename="/tmp/mstdump" + local max_dump_count="3" + for i in $(seq 1 $max_dump_count); do + ${CMD_PREFIX}/usr/bin/mstdump /dev/mst/mt*conf0 > "${mst_dump_filename}${i}" + save_file "${mst_dump_filename}${i}" mstdump true + done fi local asic="$(/usr/local/bin/sonic-cfggen -y /etc/sonic/sonic_version.yml -v asic_type)" @@ -326,7 +384,7 @@ main() { $MKDIR $V -p $LOGDIR $LN $V -s /etc $TARDIR/etc - ($TAR $V -rhf $TARFILE -C $DUMPDIR --mode=+r \ + ($TAR $V -rhf $TARFILE -C $DUMPDIR --mode=+rw \ --exclude="etc/alternatives" \ --exclude="*/etc/passwd*" \ --exclude="*/etc/shadow*" \ @@ -335,12 +393,17 @@ main() { --exclude="*/etc/ssh*" \ --exclude="*get_creds*" \ --exclude="*snmpd.conf*" \ + --exclude="/etc/mlnx" \ + --exclude="/etc/mft" \ $BASE/etc \ - || abort 5 "Tar append operation failed. Aborting for safety.") \ + || abort "${ERROR_TAR_FAILED}" "Tar append operation failed. Aborting for safety.") \ && $RM $V -rf $TARDIR + disable_logrotate + trap enable_logrotate HUP INT QUIT TERM KILL ABRT ALRM + # gzip up all log files individually before placing them in the incremental tarball - for file in $(find -L /var/log -type f); do + for file in $(find_files "/var/log/"); do # ignore the sparse file lastlog if [ "$file" = "/var/log/lastlog" ]; then continue @@ -353,8 +416,10 @@ main() { fi done + enable_logrotate + # archive core dump files - for file in $(find -L /var/core -type f); do + for file in $(find_files "/var/core/"); do # don't gzip already-gzipped log files :) if [ -z "${file##*.gz}" ]; then save_file $file core false @@ -422,10 +487,15 @@ OPTIONS Noop mode. Don't actually create anything, just echo what would happen -z Don't compress the tar at the end. + -s DATE + Collect logs since DATE; + The argument is a mostly free format human readable string such as + "24 March", "yesterday", etc. + EOF } -while getopts ":xnvhz" opt; do +while getopts ":xnvhzs:" opt; do case $opt in x) # enable bash debugging @@ -450,11 +520,17 @@ while getopts ":xnvhz" opt; do CMD_PREFIX="echo " MV="echo mv" CP="echo cp" + TOUCH="echo touch" NOOP=true ;; z) DO_COMPRESS=false ;; + s) + SINCE_DATE="${OPTARG}" + # validate date expression + date --date="${SINCE_DATE}" &> /dev/null || abort "${ERROR_INVALID_ARGUMENT}" "Invalid date expression passed: '${SINCE_DATE}'" + ;; /?) echo "Invalid option: -$OPTARG" >&2 exit 1 diff --git a/show/main.py b/show/main.py index 2475cf29ec95..5147700e5b25 100755 --- a/show/main.py +++ b/show/main.py @@ -1282,10 +1282,13 @@ def users(verbose): # @cli.command() +@click.option('--since', required=False, help="Collect logs and core files since given date") @click.option('--verbose', is_flag=True, help="Enable verbose output") -def techsupport(verbose): +def techsupport(since, verbose): """Gather information for troubleshooting""" cmd = "sudo generate_dump -v" + if since: + cmd += " -s {}".format(since) run_command(cmd, display_cmd=verbose)