dq-restart

#!/bin/bash

D=$(cd `dirname $0` && pwd)
umask 002

BDIR=$D/jobs/backend
QDIR=$D/jobs/queue
ADIR=$D/jobs/active
CDIR=$D/jobs/complete

jobspec=$1

OLDJOB=$(ls -1 $CDIR | grep "^$jobspec-")

if [ "x$OLDJOB" = "x" ]; then
    echo "No such job '$jobspec'"
    exit 1
fi


KRB5CCNAME=$BDIR/$OLDJOB/ticket.krb5 klist -s || {
    echo "Cannot restart (ticket expired?). Please submit new job instead."
    exit 1
}

echo -n "Locking..."
mkdir -p $QDIR $ADIR $CDIR $BDIR && touch $BDIR

(
    flock -xw 10 200 || {
	echo " Timeout!"
	echo "Please inspect system with 'dq-status' 'dq-free' etc."
	exit 1
    }

    echo " locked."

    touch $BDIR/.jobid # refresh NFS cache
    n=$(cat $BDIR/.jobid | sed -e 's/[^0-9]//g')
    if [ "x$n" = "x" ] ; then 
	N=1
    else
	N=$(( $n + 1 ))
    fi

    JOBID=$N-$USER
    echo $N > $BDIR/.jobid

    mkdir $BDIR/$JOBID
    cp -a $CDIR/$OLDJOB/runme.sh $BDIR/$JOBID/runme.sh
    cp -a $CDIR/$OLDJOB/cpwd.txt $BDIR/$JOBID/cpwd.txt
    cp -a $CDIR/$OLDJOB/cpath.txt $BDIR/$JOBID/cpath.txt
    cp -a $CDIR/$OLDJOB/user.txt $BDIR/$JOBID/user.txt
    cp -a $CDIR/$OLDJOB/script.txt $BDIR/$JOBID/script.txt
    cp -a $CDIR/$OLDJOB/env.txt $BDIR/$JOBID/env.txt
    cp -a $CDIR/$OLDJOB/ticket.krb5 $BDIR/$JOBID/ticket.krb5

    ln -s "../backend/$JOBID" $QDIR/$JOBID

    echo "Job ID: '$JOBID'"

) 200>$QDIR/.lock


$D/dq-churn