nhc

#!/bin/bash
#
# LBNL Node Health Check Script
#
# Michael Jennings <mej@lbl.gov>
# 13 December 2010
#
# Copyright (c) 2010-2016, Michael Jennings <mej@lbl.gov>/<mej@eterm.org>
#
# LBNL Node Health Check (NHC), Copyright (c) 2015, The Regents of the
# University of California, through Lawrence Berkeley National
# Laboratory (subject to receipt of any required approvals from the
# U.S. Dept. of Energy).  All rights reserved.
#
# If you have questions about your rights to use or distribute this
# software, please contact Berkeley Lab's Innovation & Partnerships
# Office at IPO@lbl.gov.
#
# NOTICE.  This Software was developed under funding from the
# U.S. Department of Energy and the U.S. Government consequently
# retains certain rights. As such, the U.S. Government has been
# granted for itself and others acting on its behalf a paid-up,
# nonexclusive, irrevocable, worldwide license in the Software to
# reproduce, distribute copies to the public, prepare derivative
# works, and perform publicly and display publicly, and to permit
# other to do so.
#
#
# Copyright (c) 2016-2021, Michael Jennings <mej@eterm.org>
# No additional restrictions apply; see LICENSE for terms.
#

# This is the driver program for the node health check script
# subsystem.  The include directory (/etc/nhc/scripts by default)
# contains a series of bash scripts which, when sourced, should define
# bash functions which will later be invoked to check node health.
#
# The configuration file (/etc/nhc/nhc.conf by default) is then read a
# line at a time.  Any lines beginning with a mask that matches the
# current hostname will invoke the specified check (usually one of the
# bash functions loaded above, but could also be an external command
# or script).  Failure of any check will result in the node being
# flagged as "unhealthy" and the termination of further checks.

### Library functions

# Declare a print-error-and-exit function.
function die() {
    IFS=$' \t\n'
    local RET="$1"
    shift

    CHECK_DIED=1
    [[ -z "$CHECK" ]] && CHECK="$FUNCNAME $*"
    log "ERROR:  $NAME:  Health check failed:  $*"
    syslog "Health check failed:  $*"
    syslog_flush
    if [[ -n "$NHC_RM" && "$MARK_OFFLINE" -eq 1 && "$FAIL_CNT" -eq 0 ]]; then
        if [[ "${CHECK:0:4}" == "die " ]]; then
            CHECK="$OFFLINE_NODE '$HOSTNAME' '$*'"
            eval $OFFLINE_NODE "'$HOSTNAME'" "'$*'"
            CHECK="$FUNCNAME $*"
        else
            eval $OFFLINE_NODE "'$HOSTNAME'" "'$*'"
        fi
    fi
    if [[ -n "$NHC_DETACHED" ]]; then
        if [[ -w "$RESULTFILE" || (! -e "$RESULTFILE" && -w "${RESULTFILE%/*}") ]] && echo "$RET $*" > $RESULTFILE ; then
            dbg "Wrote results file $RESULTFILE:  $RET $*"
        else
            log "ERROR:  $NAME:  Unable to write to \"$RESULTFILE\" -- is ${RESULTFILE%/*} missing/read-only?"
            syslog "ERROR:  $NAME:  Unable to write to \"$RESULTFILE\" -- is ${RESULTFILE%/*} missing/read-only?"
            syslog_flush
        fi
    elif [[ "$NHC_RM" == "sge" ]]; then
        echo "begin" >&$NHC_FD_OUT
        echo "$HOSTNAME:healthy:false" >&$NHC_FD_OUT
        echo "$HOSTNAME:diagnosis:NHC: $*" >&$NHC_FD_OUT
        echo "end" >&$NHC_FD_OUT
        return 77
    elif [[ -n "$LOGFILE" ]]; then
        oecho "ERROR:  $NAME:  Health check failed:  $*"
    fi
    if [[ "$NHC_CHECK_ALL" == "1" ]]; then
        ((FAIL_CNT++))
        return 0
    fi
    if [[ $RET -ne 142 && $RET -ne 143 ]]; then
        # Don't kill the watchdog if we're in die() because it is terminating us.
        kill_watchdog
    fi
    [[ $NHC_FD_OUT -eq 3 ]] && exec 1>&3- 2>&4-
    [[ "$NHC_CHECK_FORKED" == "1" ]] && ((RET+=100))
    exit $RET
}

# Quick-and-dirty debugging output
function dbg() {
    local PREFIX=""

    if [[ "$DEBUG" == "1" ]]; then
        [[ $TS -ne 0 ]] && PREFIX="[$SECONDS] - "
        echo "${PREFIX}DEBUG:  $*"
    fi
}

# Quick-and-dirty log output
function log() {
    local PREFIX=""

    if [[ "$SILENT" == "0" ]]; then
        [[ $TS -ne 0 ]] && PREFIX="[$SECONDS] - "
        echo "$PREFIX$@"
    fi
}

# Log output only in VERBOSE mode
function vlog() {
    local PREFIX=""

    if [[ "$VERBOSE" == "1" ]]; then
        [[ $TS -ne 0 ]] && PREFIX="[$SECONDS] - "
        echo "$PREFIX$@"
    fi
}

# Send information to previous stdout
function oecho() {
    local PREFIX=""

    if [[ "$SILENT" == "0" ]]; then
        [[ $TS -ne 0 ]] && PREFIX="[$SECONDS] - "
        echo "$PREFIX$@" >&$NHC_FD_OUT
    fi
}

# Send information to previous stderr
function eecho() {
    local PREFIX=""

    if [[ "$SILENT" == "0" ]]; then
        [[ $TS -ne 0 ]] && PREFIX="[$SECONDS] - "
        echo "$PREFIX$@" >&$NHC_FD_ERR
    fi
}

# Echo only if VERBOSE
function vecho() {
    local PREFIX=""

    if [[ "$VERBOSE" == "1" ]]; then
        [[ $TS -ne 0 ]] && PREFIX="[$SECONDS] - "
        echo "$PREFIX$@" >&$NHC_FD_OUT
    fi
}

# Store syslog output, send at end of script execution.
function syslog() {
    if [[ -z "$LOGGER_TEXT" ]]; then
        LOGGER_TEXT="$*"
    else
        LOGGER_TEXT="$LOGGER_TEXT"$'\n'"$*"
    fi
}

function syslog_flush() {
    if [[ -n "$LOGGER_TEXT" ]]; then
        echo "$LOGGER_TEXT" | logger -p daemon.err -t "$NAME[${BASHPID:-$$}]"
    fi
    LOGGER_TEXT=""
}

function kill_watchdog() {
    local i

    dbg "$FUNCNAME:  Watchdog PID is $WATCHDOG_PID."
    if [[ $WATCHDOG_PID -ne 0 ]]; then
        for i in 1 2 3 4 ; do
            dbg "Killing watchdog timer $WATCHDOG_PID -- Attempt #$i"
            kill -s INT -- -$WATCHDOG_PID $WATCHDOG_PID 2>/dev/null || return 0
            sleep 0.01 || sleep 1
        done
        kill -s KILL -- -$WATCHDOG_PID $WATCHDOG_PID 2>/dev/null
    fi
    return 0
}

function toggle_bash_tracing() {
    if [[ "$-" == *x* ]]; then
        : "BASH tracing mode (-x) turned off via SIGUSR1."
        set +x
    else
        set -x
        : "BASH tracing mode (-x) turned on via SIGUSR1."
    fi
}

function toggle_debug_mode() {
    if [[ -z "$DEBUG" || "$DEBUG" -eq 0 ]]; then
        DEBUG=1
        dbg "NHC DEBUG mode turned on via SIGUSR2."
    else
        dbg "NHC DEBUG mode turned off via SIGUSR2."
        DEBUG=0
    fi
}

#########################

function nhcmain_init_env() {
    ### Variable declarations

    # Static variables
    PATH="/sbin:/usr/sbin:/bin:/usr/bin"
    PS4='>[{L${SHLVL}/S${BASH_SUBSHELL}/D$((${#FUNCNAME[*]}==0?1:${#FUNCNAME[*]}))/R$?}@${BASH_SOURCE##*/}:${LINENO}:${FUNCNAME[0]}()]> '
    if [[ -f "/etc/debian_version" ]]; then
        SYSCONFIGDIR="/etc/default"
        LIBEXECDIR="/usr/lib"
    else
        if [[ -d "/etc/sysconfig" ]]; then
            SYSCONFIGDIR="/etc/sysconfig"
        elif [[ -d "/etc/default" ]]; then
            SYSCONFIGDIR="/etc/default"
        else
            SYSCONFIGDIR="/etc/nhc/sysconfig"
            if [[ ! -d "${SYSCONFIGDIR}" ]]; then
                mkdir -p -m 0700 "${SYSCONFIGDIR}" >&/dev/null
            fi
        fi
        if [[ -d "/usr/libexec" ]]; then
            LIBEXECDIR="/usr/libexec"
        else
            LIBEXECDIR="/usr/lib"
        fi
    fi
    if [[ -r /proc/sys/kernel/hostname ]]; then
        read HOSTNAME < /proc/sys/kernel/hostname
    elif [[ -z "$HOSTNAME" ]]; then
        HOSTNAME="localhost"
    fi
    HOSTNAME_S=${HOSTNAME/%.*}
    RET=0
    LOGGER_TEXT=""
    NHC_PID=$$
    if [[ -z "$BASHPID" || "$BASHPID" != "$NHC_PID" ]]; then
        unset BASHPID
    fi
    NHC_START_TS=0
    WATCHDOG_PID=0
    FAIL_CNT=0
    FORCE_SETSID=1
    NHC_FD_OUT=1
    NHC_FD_ERR=2
    export PATH PS4 SYSCONFIGDIR LIBEXECDIR HOSTNAME HOSTNAME_S RET LOGGER_TEXT
    export NHC_PID NHC_START_TS WATCHDOG_PID FAIL_CNT FORCE_SETSID NHC_FD_OUT NHC_FD_ERR

    # Users may override this in /etc/sysconfig/nhc.
    NAME=${0/#*\/}

    # Don't allow previous environment to leak in.  Must be done from /etc/sysconfig/nhc only.
    unset CONFDIR CONFFILE INCDIR HELPERDIR ONLINE_NODE OFFLINE_NODE LOGFILE DEBUG SILENT VERBOSE TIMEOUT
    unset RESULTFILE MAX_SYS_UID NHC_RM NHC_CHECK_ALL NHC_CHECK_FORKED EVAL_LINE

    if [[ -n "$NHC_DETACHED" ]]; then
        # We're running detached.
        export NHC_DETACHED
        DETACHED_MODE=1
    fi

    # Disable pathname expansion by default so no unintended globbing happens.
    # We'll turn it back on only where, and only for as long as, we need it.
    set -f
}

function nhcmain_help() {
    local PROGNAME=$0
    local TITLE UNDERLINE

    PROGNAME="${PROGNAME/#*\/}"
    TITLE="$PROGNAME Usage"
    UNDERLINE="${TITLE//?/-}"

    cat <<EOF

$TITLE
$UNDERLINE

  Syntax:  $PROGNAME [<options>] [<var>=<value> [...]]

 OPTION            DESCRIPTION
-------------------------------------------------------------------------------
 -h                Show command line help (this info)
 -D <confdir>      Use config directory <confdir> (default: /etc/<name>)
 -a                Run ALL checks; don't exit on first failure (NHC_CHECK_ALL=1)
 -c <conffile>     Load config from <conffile> (default: <confdir>/<name>.conf)
 -d                Activate debugging output (i.e., DEBUG=1)
 -e <check>        Evaluate check line <check> and exit immediately
 -f                Run checks in forked subprocesses (i.e., NHC_CHECK_FORKED=1)
 -l <logspec>      Log output to <logspec> (i.e., LOGFILE=<logspec>)
 -n <name>         Set program name to <name> (default: nhc); see -D & -c above
 -q                Run quietly (i.e., SILENT=1)
 -t <timeout>      Use timeout of <timeout> seconds (default: 30)
 -v                Run verbosely (i.e., VERBOSE=1)
 -x                Run in eXtreme debug/trace mode (same as "bash -x")

 All other command line parameters, if any, must be environment variable
 settings in the form VARNAME=value.

EXAMPLES:
---------
 To run in debug mode with a timeout of 60 seconds:
    # $PROGNAME -d -t 60
  OR
    # $PROGNAME DEBUG=1 TIMEOUT=60

 To run with the name "nhc-cron" which will alter default config paths:
    # $PROGNAME -n nhc-cron
  OR
    # $PROGNAME NAME=nhc-cron

EOF
}

function nhcmain_parse_cmdline() {
    local OPTION

    OPTIND=1
    while getopts ":D:ac:de:fhl:n:qt:vx" OPTION ; do
        case "$OPTION" in
            D) CONFDIR="$OPTARG" ; dbg "\$CONFDIR set to $CONFDIR." ;;
            a) NHC_CHECK_ALL=1 ; dbg "Force running of all checks." ;;
            c) CONFFILE="$OPTARG" ; dbg "\$CONFFILE set to $CONFFILE." ;;
            d) DEBUG=1 ; dbg "Debugging activated via -d option." ;;
            e) EVAL_LINE="$OPTARG" ; dbg "Evaluating single check line:  $EVAL_LINE" ;;
            f) NHC_CHECK_FORKED=1 ; dbg "Checks will be run in forked subprocesses." ;;
            h) nhcmain_help ; exit 0 ;;
            l) LOGFILE="$OPTARG" ; dbg "\$LOGFILE set to $LOGFILE." ;;
            n) NAME="$OPTARG" ; dbg "\$NAME set to $NAME." ;;
            q) SILENT=1 ; dbg "Silent mode activated via -q option." ;;
            t) TIMEOUT="$OPTARG" ; dbg "Timeout set to $TIMEOUT." ;;
            v) VERBOSE=1 ; dbg "Verbose mode activated via -v option." ;;
            x) set -x ; dbg "BASH tracing active." ;;
            :) nhcmain_help ; eecho "$NAME:  ERROR:  Option -$OPTARG requires an argument." ; return 8 ;;
            \?) nhcmain_help ; eecho "$NAME:  ERROR:  Invalid option:  -$OPTARG" ; return 9 ;;
        esac
    done
    shift $((OPTIND-1))
    while [[ ! -z "$1" ]]; do
        eval "$1"
        shift
    done
    return 0
}

function nhcmain_load_sysconfig() {
    # Load settings from system-wide location.  NOTE:  To change value of $NAME
    # here, the driver script must be renamed to something other than "nhc"
    # or "-n foo" or "NAME=foo" must be passed on the command line.
    if [[ -f $SYSCONFIGDIR/$NAME ]]; then
        . $SYSCONFIGDIR/$NAME
    fi
}

function nhcmain_finalize_env() {
    # Set some variables relative to possible /etc/sysconfig/nhc
    # modifications.  Users may have overridden some of these.
    CONFDIR="${CONFDIR:-/etc/nhc}"
    CONFFILE="${CONFFILE:-$CONFDIR/$NAME.conf}"
    INCDIR="${INCDIR:-$CONFDIR/scripts}"
    HELPERDIR="${HELPERDIR:-$LIBEXECDIR/nhc}"
    ONLINE_NODE="${ONLINE_NODE:-$HELPERDIR/node-mark-online}"
    OFFLINE_NODE="${OFFLINE_NODE:-$HELPERDIR/node-mark-offline}"
    LOGFILE="${LOGFILE:->>/var/log/$NAME.log 2>&1}"
    RESULTFILE="${RESULTFILE:-/var/run/nhc/$NAME.status}"
    DEBUG=${DEBUG:-0}
    TS=${TS:-$DEBUG}
    SILENT=${SILENT:-0}
    VERBOSE=${VERBOSE:-0}
    MARK_OFFLINE=${MARK_OFFLINE:-1}
    DETACHED_MODE=${DETACHED_MODE:-0}
    DETACHED_MODE_FAIL_NODATA=${DETACHED_MODE_FAIL_NODATA:-0}
    TIMEOUT=${TIMEOUT:-30}
    NHC_CHECK_ALL=${NHC_CHECK_ALL:-0}
    NHC_CHECK_FORKED=${NHC_CHECK_FORKED:-0}
    export NHC_SID=0

    # Check for session leader.
    kill -s 0 -- -$NHC_PID >/dev/null 2>&1
    if [[ $? -eq 0 ]]; then
        # We're already a session leader, possibly after setsid, possibly not.
        dbg "NHC process $NHC_PID is session leader."
        NHC_SID=-$NHC_PID
    elif [[ -n "$NHC_FORCED_SESSION" && "$NHC_FORCED_SESSION" == "$NHC_PID" ]]; then
        # We tried to become a session leader via setsid.  Apparently we failed.
        # Don't try again since that would create an infinite loop.
        dbg "NHC failed to become session leader."
    elif [[ "$FORCE_SETSID" == "1" && -z "$NHC_LOAD_ONLY" ]]; then
        # We are not yet a session leader, but we haven't tried setsid.  Try now.
        dbg "NHC process $NHC_PID is not session leader."
        dbg "Restarting via setsid:  setsid $0 ${NHC_ARGS_LIST[*]} FORCE_SETSID=0"
        export NHC_FORCED_SESSION=$NHC_PID
        exec setsid "$0" "${NHC_ARGS_LIST[@]}" "FORCE_SETSID=0"
    fi

    if [[ -n "$EVAL_LINE" ]]; then
        LOGFILE=""
        ONLINE_NODE=:
        OFFLINE_NODE=:
        MARK_OFFLINE=0
    elif [[ "${LOGFILE##/}" != "$LOGFILE" ]]; then
        # If the log file looks like a path rather than a redirect, fix it.
        LOGFILE=">>$LOGFILE 2>&1"
    elif [[ "$LOGFILE" == "-" ]]; then
        # Since bash doesn't distinguish between unset and empty, we use "-" instead.
        unset LOGFILE
    fi

    if [[ -z "$NHC_RM" ]]; then
        if ! nhcmain_find_rm ; then
            ONLINE_NODE=:
            OFFLINE_NODE=:
            MARK_OFFLINE=0
        fi
    fi
    if [[ "$NHC_RM" == "sge" ]]; then
        # With SGE, we return the status and note directly from NHC.
        ONLINE_NODE=:
        OFFLINE_NODE=:
        MARK_OFFLINE=0
        # SGE's looping model is incompatible with detached mode and the watchdog timer.
        DETACHED_MODE=0
        TIMEOUT=0
    fi

    # If timestamps are desired, initialize them here.
    if [[ $TS -ne 0 ]]; then
        TS=$(date '+%s')
        ELAPSED_SECONDS=$SECONDS
        SECONDS=$((ELAPSED_SECONDS+TS))
        NHC_START_TS=$((TS-ELAPSED_SECONDS))
    fi

    if [[ -n "$NHC_DETACHED" ]]; then
        dbg "This session is running detached from $NHC_DETACHED."
    elif [[ $DETACHED_MODE -eq 1 ]]; then
        dbg "Activating detached mode."
        nhcmain_detach
        return
    fi

    export NAME CONFDIR CONFFILE INCDIR HELPERDIR ONLINE_NODE OFFLINE_NODE LOGFILE DEBUG TS SILENT TIMEOUT NHC_RM
}

function nhcmain_find_rm() {
    local DIR
    local -a DIRLIST

    if [[ -d /var/spool/torque ]]; then
        NHC_RM="pbs"
        return 0
    elif [[ -n "$SGE_ROOT" && -x "$SGE_ROOT/util/arch" ]]; then
        # SGE binaries typically won't be on the path defined above in the
        # load sensor environment, but SGE_ROOT will be there.
        NHC_RM="sge"
    fi

    # Search PATH for commands
    if type -a -p -f -P scontrol >&/dev/null ; then
        NHC_RM="slurm"
        return 0
    elif type -a -p -f -P pbsnodes >&/dev/null ; then
        NHC_RM="pbs"
        return 0
    elif type -a -p -f -P qselect >&/dev/null ; then
        NHC_RM="sge"
        return 0
    elif type -a -p -f -P badmin >&/dev/null || type -a -p -f -P sbatchd >&/dev/null ; then
        NHC_RM="lsf"
        return 0
    fi

    if [[ -z "$NHC_RM" ]]; then
        dbg "Unable to detect resource manager."
        return 1
    fi
}

function nhcmain_redirect_output() {
    if [[ -n "$LOGFILE" ]]; then
        exec 3>&1- 4>&2-
        eval exec $LOGFILE
        if [[ $? -ne 0 ]]; then
            exec 1>&3- 2>&4-
            echo "ERROR NHC:  FATAL:  Can't write $LOGFILE as $USER (uid $EUID) -- Read-only filesystem/device failure?"
            syslog "NHC:  FATAL:  Can't write $LOGFILE as $USER (uid $EUID) -- Read-only filesystem/device failure?"
            syslog_flush
            exit 1
        else
            dbg "Output redirected per LOGFILE variable $LOGFILE"
            NHC_FD_OUT=3
            NHC_FD_ERR=4
        fi
    fi
}

function nhcmain_check_conffile() {
    # Check for config file before we do too much work.
    if [[ ! -f "$CONFFILE" ]]; then
        # Missing config means no checks.  No checks means no failures.
        return 1
    fi
    return 0
}

function nhcmain_load_scripts() {
    local -a NHC_SCRIPTS
    local IFS=''

    vlog "Node Health Check starting."

    # Load all include scripts.
    dbg "Loading scripts from $INCDIR..."
    set +f
    NHC_SCRIPTS=( $INCDIR/*.nhc )
    set -f
    for SCRIPT in "${NHC_SCRIPTS[@]}" ; do
	if [[ -e "$SCRIPT" ]]; then
            dbg "Loading ${SCRIPT/#*\/}"
            . $SCRIPT
        else
            dbg "No scripts found in $INCDIR"
        fi
    done
}

function nhcmain_watchdog_timer() {
    local TIMEOUT="$1" NHC_PID="$2"
    local SLEEP_PID MAIN_NHC_HUNG=0

    # Start sleep process in background first so we can obtain its PID and set traps to handle signals.
    sleep $TIMEOUT &
    SLEEP_PID=$!
    trap '' ALRM TERM
    trap "kill -9 $SLEEP_PID >&/dev/null ; exit 0" EXIT HUP INT QUIT ILL ABRT FPE KILL SEGV PIPE USR1 USR2 CONT STOP TSTP TTIN TTOU PWR
    # Now we wait for the sleep to finish or for our process to be killed by the main NHC process.
    wait

    # If we get here, the watchdog timer has expired, and we need to kill the NHC process (group).  Start gently.
    log "NHC watchdog timer ${BASHPID:-$$} ($TIMEOUT secs) has expired.  Signaling NHC:  kill -s ALRM -- $NHC_PID"
    kill -s ALRM -- $NHC_PID || return 0
    sleep 1

    # Send a stronger signal this time.  If there's nothing left to receive our signal, we're done.
    log "NHC watchdog timer ${BASHPID:-$$} terminating NHC:  kill -s TERM -- $NHC_PID"
    kill -s TERM -- $NHC_PID 2>/dev/null || return 0
    sleep 3

    # If we still had things to kill, check one last time and kill by force.
    log "NHC watchdog timer ${BASHPID:-$$} terminating NHC with extreme prejudice:  kill -s KILL -- $NHC_PID"
    if kill -0 ${NHC_PID#-} 2>/dev/null ; then
        # Setting this to true means it's the main NHC that's still around, not merely a hung child process.  That
        # means the main NHC process hasn't been able to terminate gracefully and write the appropriate message to
        # stdout or the status file.  Thus the watchdog has to be the one to call die().  If it's just a hung child,
        # we don't want to call die() again and overwrite the status message.
        MAIN_NHC_HUNG=1
    fi
    kill -s KILL -- $NHC_PID 2>/dev/null
    sleep 1
    if kill -0 ${NHC_PID#-} 2>/dev/null ; then
        # If the main NHC survived a "kill -9" it must be defunct.  Assume it called die() and exited cleanly.
        log "NHC process ${NHC_PID#-} is defunct.  Considering it terminated."
    elif [[ $MAIN_NHC_HUNG -eq 1 ]]; then
        # If the main NHC process survived SIGTERM but not SIGKILL, it likely didn't exit cleanly.  Call die().
        WATCHDOG_PID=0
        die 142 "Watchdog timer unable to terminate hung NHC process ${NHC_PID#-}."
    fi
    return 0
}

function nhcmain_set_watchdog() {
    # Set ALARM to timeout script.
    if [[ $TIMEOUT -gt 0 ]]; then
        # Start watchdog timer process in its own session.
        set -m
        eval nhcmain_watchdog_timer $TIMEOUT $((NHC_SID==0?NHC_PID:NHC_SID)) &
        WATCHDOG_PID=$!
        export WATCHDOG_PID
        set +m
        dbg "In ${BASHPID:-$$}:  Watchdog PID is $WATCHDOG_PID, NHC PID is $NHC_PID"
    else
        dbg "In ${BASHPID:-$$}:  No watchdog, NHC PID is $NHC_PID"
    fi
}

function nhcmain_spawn_detached() {
    export NHC_DETACHED=${BASHPID:-$$}
    exec -a ${NAME}-detached $0 </dev/null >/dev/null 2>&1
}

function nhcmain_detach() {
    local RC MSG ELAPSED

    # If the results file exists but the system has rebooted since its
    # creation, assume it's stale and remove it.
    if [[ -e "$RESULTFILE" && -d "/proc/1" && "$RESULTFILE" -ot "/proc" ]]; then
        rm -f "$RESULTFILE"
    fi
    if [[ -r "$RESULTFILE" ]]; then
        read RC MSG < "$RESULTFILE"
    elif [[ "$DETACHED_MODE_FAIL_NODATA" == "1" ]]; then
        RC=1
        MSG="Detached mode -- pending checks (no data found)"
    else
        RC=0
        MSG=""
    fi

    rm -f "$RESULTFILE" >/dev/null 2>&1
    if [[ -e "$RESULTFILE" ]]; then
        die 1 "Unable to overwrite/remove results file \"$RESULTFILE\" -- is ${RESULTFILE%/*} read-only?"
    else
        # Launch detached process (hopefully in its own session)
        set -m
        nhcmain_spawn_detached &
        set +m
    fi

    syslog_flush
    if [[ "$RC" != "0" ]]; then
        echo "ERROR Health check failed:  $MSG"
        exit $RC
    fi

    # FIXME:  TODO or !TODO, that is the question....
    nhcmain_redirect_output
    ELAPSED=$((SECONDS-NHC_START_TS))
    vlog "Node Health Check detached parent completed successfully (${ELAPSED}s)."
    [[ $NHC_FD_OUT -eq 3 ]] && exec 1>&3- 2>&4-
    exit 0
}

function nhcmain_run_checks() {
    CHECKS=( )
    nhc_load_conf "$CONFFILE"
    for ((CNUM=0; CNUM<${#CHECKS[*]}; CNUM++)); do
        CHECK="${CHECKS[$CNUM]}"
        CHECK_DIED=0

        # Run the check.
        if [[ "$NHC_CHECK_FORKED" == 0 ]] || mcheck "$CHECK" '/( |^)(export )?[A-Za-z_][A-Za-z0-9_]*=[^ =]/'; then
            vlog "Running check:  \"$CHECK\""
            eval $CHECK
            RET=$?
        else
            # Run the check in a separate forked process.
            eval $CHECK &
            vlog "Running check:  \"$CHECK\" (forked PID $!)"
            wait $! >/dev/null 2>&1
            RET=$?
        fi

        # Check for failure.
        if [[ "$RET" != "0" && "$NHC_CHECK_ALL" != "1" ]]; then
            if [[ "$NHC_CHECK_FORKED" == "1" && "$RET" -gt 100 ]]; then
                # The forked check did successfully call die(), so we don't have to.
                exit $((RET-100))
            elif [[ "$CHECK_DIED" == "0" ]]; then
                # If the check failed but didn't call die(), use this fallback.
                log "Node Health Check failed.  Check $CHECK returned $RET"
                die $RET "Check $CHECK returned $RET"
            fi
            return $RET
        fi
    done
    CHECK=""
    if [[ "$FAIL_CNT" > 0 && "$NHC_CHECK_ALL" == "1" ]]; then
        # One or more checks have failed.  Give final report and terminate.
        NHC_CHECK_ALL=0
        log "ERROR:  $NAME:  $FAIL_CNT health checks failed."
        syslog "$FAIL_CNT health checks failed."
        [[ -n "$LOGFILE" ]] && oecho "ERROR:  $NAME:  $FAIL_CNT health checks failed."
        syslog_flush
        kill_watchdog
        exit $FAIL_CNT
    fi
}

function nhcmain_mark_online() {
    if [[ -n "$NHC_RM" && "$MARK_OFFLINE" -eq 1 ]]; then
        eval $ONLINE_NODE "'$HOSTNAME'"
    fi
}

function nhcmain_finish() {
    local ELAPSED

    syslog_flush
    ELAPSED=$((SECONDS-NHC_START_TS))
    vlog "Node Health Check completed successfully (${ELAPSED}s)."
    if [[ "$NHC_RM" == "sge" ]]; then
        echo "begin" >&$NHC_FD_OUT
        echo "$HOSTNAME:healthy:true" >&$NHC_FD_OUT
        echo "$HOSTNAME:diagnosis:HEALTHY" >&$NHC_FD_OUT
        echo "end" >&$NHC_FD_OUT
        return 0
    fi
    kill_watchdog
    [[ $NHC_FD_OUT -eq 3 ]] && exec 1>&3- 2>&4-
    #[[ $DEBUG -eq 1 ]] && times >&2
    exit 0
}

### Script guts begin here.
NHC_ARGS_LIST=( "$@" )
if [[ -n "$NHC_LOAD_ONLY" ]]; then
    # We're only supposed to define functions, not actually run anything.
    return 0 || exit 0
fi

trap 'toggle_bash_tracing' SIGUSR1
trap 'toggle_debug_mode' SIGUSR2
trap 'die 129 "Terminated by signal SIGHUP." ; exit 129' 1
trap 'die 130 "Terminated by signal SIGINT." ; exit 130' 2
trap 'die 143 "Terminated by signal SIGTERM." ; exit 143' 15
trap 'die 142 "Script timed out${CHECK:+ while executing \"$CHECK\"}." ; exit 142' 14

nhcmain_init_env
nhcmain_parse_cmdline "$@" || exit 10
nhcmain_load_sysconfig
nhcmain_finalize_env
if [[ -n "$EVAL_LINE" ]]; then
    nhcmain_load_scripts
    eval $EVAL_LINE
    exit $?
fi
nhcmain_redirect_output
nhcmain_check_conffile || exit 0
if [[ "$NHC_RM" == "sge" ]]; then
    read INPUT
    if [[ $? != 0 || "$INPUT" == "quit" ]]; then
        exit 0
    fi
    # We need to do this each time to flush the caches which the
    # scripts mostly keep (filesystems, process trees, etc.).
    # Also the scripts can be changed without having to kill the
    # sensor on all the hosts to pick that up.
    nhcmain_load_scripts
    if nhcmain_run_checks ; then
        nhcmain_finish
    fi
    if [[ "$*" == *NHC_RM=sge* ]]; then
        exec $0 "$@"
    else
        exec $0 "$@" NHC_RM=sge
    fi
else
    nhcmain_load_scripts
    nhcmain_set_watchdog
    nhcmain_run_checks
    nhcmain_mark_online
    nhcmain_finish
fi