condor_node_check

#!/usr/bin/env python

#  Worker node health check script, to run from condor startd

from mc_util import *
import sys, os, stat
import memcache
import socket
import subprocess


## get host info
hostname = socket.gethostname() 
node = shortname(hostname)


## Define directories to test
SCRATCH_DIR = "/scratch"
TMP_DIR = "/tmp"
ROOT_DIR = "/"
TMP_FILE = "node_check"
EXE_DIR = "/var/lib/condor/execute"


## Memcache subroutines
def setOffline(msg):
   print """NodeOnline = false"""
   print """NodeOnlineReason = '%s'""" % (msg)

def setOnline():
   print """NodeOnline = true"""


## Test if the node has been set offline manually
STATUS = mc_get( node+".manualstatus" )
if STATUS == 'offline':
    sys.exit()


## Tests
def can_write( mypath, myfile ):
  if os.path.exists( "%s" % (mypath) ):
    if not os.path.isfile( "%s/%s" % (mypath, myfile) ):
      if ( open( "%s/%s" % (mypath, myfile), 'w').close() ):
        setOffline("Cannot write to %s" % mypath) 
        sys.exit()
      else:
        os.remove( "%s/%s" % (mypath, myfile))

def correct_perms( mypath, perms):
  if os.path.exists( "%s" % (mypath) ):
    PERMS = oct(os.stat(mypath).st_mode)[-4:]
 
    if ( int(PERMS) & 0777  == perms ):
      setOffline("Permissions on %s should be 1777, are %s" % (mypath, PERMS))
      sys.exit()

def disk_free ( mypath, amount ):  
  if os.path.exists( "%s" % (mypath) ):
    df = subprocess.Popen(["/bin/df", "-Ph", "%s" % mypath], stdout=subprocess.PIPE)
    output = df.communicate()[0]
    device, size, used, available, percent, mountpoint = output.split("\n")[1].split()

    if available.find('T') >= 0:
      next
    else:
      available = available.split(".")[0].replace('G', '') 
      if ( int(available) < int(amount) ):
        setOffline("Disk space available on %s is %s GB" % (mypath, available))
        sys.exit()


## run the checks
can_write( TMP_FILE, EXE_DIR )
correct_perms( EXE_DIR, 1777)
disk_free( ROOT_DIR, 5 ) # unit is GB
disk_free( EXE_DIR, 5 )  # unit is GB


## if we pass all the checks, set the node to online status
setOnline()
sys.exit()