#!/bin/sh
#
#   Periodically scan for finished jobs
#
# usage: scan-remote-job CONTROL_DIR ...

#################
# General setup #
#################

NORDUGRID_LOCATION=${NORDUGRID_LOCATION:-/opt/nordugrid}

# Include remote LRMS functions
REMOTE_LRMS_FUNCTIONS="${NORDUGRID_LOCATION}/libexec/remote-lrms-functions"
if [ ! -r ${REMOTE_LRMS_FUNCTIONS} ]; then
	errormsg "${REMOTE_LRMS_FUNCTIONS} not readable!"
  	errormsg "Delaying automatic reexecution of this script" && sleep 60 
	exit 1
fi

. ${REMOTE_LRMS_FUNCTIONS}
if [ $? -ne '0' ]; then
	errormsg "remote LRMS configuration disabled! (source failed)"
  	errormsg "Delaying automatic reexecution of this script" && sleep 60 
	exit 1
fi

read_gw_conf $@
LOCAL_LRMS=$?

if [ ${LOCAL_LRMS} -ne '0' ]; then
	errormsg "remote LRMS configuration disabled! (${LOCAL_LRMS})"
  	errormsg "Delaying automatic reexecution of this script" && sleep 60 
	exit 1
fi

debugmsg "scan-remote-job: $@"


#################################
# Set up reasonable environment #
#################################

OLD_MASK=`umask`
umask 002

# Slow down GM repeating script execution in case of errors
DELAY_REEXEC="$SLEEP 60"

# DL_WAIT is the number of seconds to wait between downloading status updates
# while scan is running remotely
DL_WAIT="60"

# Read LRMS type from nordugrid.conf into LRMS_TYPE variable
LRMS_TYPE=""
read_lrms

# Grab LRMS arg before continuing as usual
SCAN_LRMS_KIND="$LRMS_TYPE"

if [ -z "${SCAN_LRMS_KIND}" ]; then 
	errormsg "Missing both arguments to $0 !!"
  	errormsg "Delaying reexecution of this script ($DELAY_REEXEC)" && $DELAY_REEXEC
	exit 1 
fi

# first CONTROL_DIR is used for storing own files
CONTROL_DIR=$1
if [ -z "${CONTROL_DIR}" ]; then 
	errormsg "Missing control dir argument to $0 !!"
  	errormsg "Delaying reexecution of this script ($DELAY_REEXEC)" && $DELAY_REEXEC
	exit 1 
fi


################################################
# Set up other variables we need at this point #
################################################

# Build name of submit script by replacing remote with supplied LRMS name
SCAN_LRMS_SCRIPT=`echo $0 | $SED "s/-remote-/-${SCAN_LRMS_KIND}-/"`

# Strip the path
SCAN_SCRIPT_NAME=`$BASENAME ${SCAN_LRMS_SCRIPT}`

# Restore script is in same dir as this script (libexec)
RESTORE_STATE_SCRIPT=`$DIRNAME ${SCAN_LRMS_SCRIPT}`/restore-remote-state

# Temporary scratch dir used locally and remotely
REMOTE_SCRATCH=`$MKTEMP -d ${TMP_DIR}/remote-scratch.XXXXXX`
# We need to allow grid user read access to dir
#debugmsg "$CHMOD -R 755 ${REMOTE_SCRATCH}"
$CHMOD -R 755 ${REMOTE_SCRATCH}

# Store copies of relevant files in REMOTE_SCRATCH
LRMS_SCRIPT=${REMOTE_SCRATCH}/`$BASENAME $SCAN_LRMS_SCRIPT`
STATE_SCRIPT=${REMOTE_SCRATCH}/`$BASENAME $RESTORE_STATE_SCRIPT`
# Save current environment in a file to be sourced before remote submit
ENV_FILE="${LRMS_SCRIPT}.environment"

# Copy options
CP_OPTS="$UPDATE"

# Command to wake up grid-manager
WAKE_GM="$PKILL -HUP grid-manager"


########################################################
# Build environment file to be sourced at the resource #
########################################################
store_environment $ENV_FILE


##################################################
# Prepare job directory for transfer to resource #
##################################################

# We need the scan script, environment file and the control directory at the remote 
# host. The scan script is copied to the scratch directory for a single
# transfer of both script and environment file.

# TODO: in priciple we should handle *all* control directories supplied in input

#debugmsg "$CP $CP_OPTS ${SCAN_LRMS_SCRIPT} ${LRMS_SCRIPT}"
$CP $CP_OPTS ${SCAN_LRMS_SCRIPT} ${LRMS_SCRIPT}
ret=$?
if [ $ret -ne '0' ]; then
	errormsg "$CP $CP_OPTS ${SCAN_LRMS_SCRIPT} ${LRMS_SCRIPT} failed!"
   	errormsg "Delaying reexecution of this script ($DELAY_REEXEC)" && $DELAY_REEXEC
	exit 1
fi

#debugmsg "$CP $CP_OPTS ${RESTORE_STATE_SCRIPT} ${STATE_SCRIPT}"
$CP $CP_OPTS ${RESTORE_STATE_SCRIPT} ${STATE_SCRIPT}
ret=$?
if [ $ret -ne '0' ]; then
	errormsg "$CP $CP_OPTS ${RESTORE_STATE_SCRIPT} ${STATE_SCRIPT} failed!"
   	errormsg "Delaying reexecution of this script ($DELAY_REEXEC)" && $DELAY_REEXEC
	exit 1
fi


#################################################################################
# Upload the scratch directory and control files to resource and execute script #
#################################################################################

# Please note that trailing slash is important
#debugmsg "upload/sync scratch dir (${REMOTE_SCRATCH}) to remote resource"
upload ${REMOTE_SCRATCH}/ ${REMOTE_SCRATCH}
ret=$?
if [ $ret -ne '0' ]; then
	errormsg "upsync command failed ($ret)! (upload/sync ${REMOTE_SCRATCH}/ ${REMOTE_SCRATCH}"
	exit 1
fi


# Backup scan state_file to catch jobs that got caught in transfer job.x.local race:
# If scan greps for localid in before job.x.local is uploaded it will fail to 
# finish job and remember that it tried in state_file.
# Restoring the previous state_file after each scan causes a second attempt to take 
# place without starting from scratch each time.

debugmsg "remote reset state file: ${STATE_SCRIPT} ${SCAN_LRMS_KIND} $@"
remote_exec "${STATE_SCRIPT} ${SCAN_LRMS_KIND} $@"
ret=$?
if [ $ret -ne '0' ]; then
	errormsg "executing remote restore state failed!"
   	errormsg "Delaying reexecution of this script ($DELAY_REEXEC)" && $DELAY_REEXEC
	exit $ret
fi

# We need to upsync job.x.local because it does not get updated until after submit 
# returns. Thus it won't contain localid until some later point.
# TODO: isn't this actually a race in GM?
#debugmsg "upload/sync job.*.{local,status} from control dir ($CONTROL_DIR) to remote resource"

# Only single '\' escapes here since local rsync needs to interpret names
#debugmsg "upload/sync $CONTROL_DIR/\*\.\{local,status\} $CONTROL_DIR/"
upsync $CONTROL_DIR/job\.\*\.\{local,status\} $CONTROL_DIR/
ret=$?
if [ $ret -ne '0' ]; then
	errormsg "upload/sync command failed! (upload/sync $CONTROL_DIR/job\.\*\.\{local,status\} $CONTROL_DIR/)"
   	errormsg "Delaying reexecution of this script ($DELAY_REEXEC)" && $DELAY_REEXEC
	exit 1
fi

# Spawn scan script execution to allow concurrent status updates
debugmsg "spawning remote execution of ${LRMS_SCRIPT} $CONTROL_DIR:"
remote_exec ". ${ENV_FILE} && ${LRMS_SCRIPT} $CONTROL_DIR" &
ret=$?
child_id=$!
#debugmsg "spawned remote execution with pid $child_id"
if [ $ret -ne '0' ]; then
	errormsg "spawning remote command failed!"
   	errormsg "Delaying reexecution of this script ($DELAY_REEXEC)" && $DELAY_REEXEC
	exit $ret
fi

dl_count=0

####################################################################
# Keep downloading generated lrms_done files while scan is running #
####################################################################

# We can not download full controldir since GM may have modified job.x.local now!
while $TRUE; do
	debugmsg "download/sync done and arg files from remote resource (loop $dl_count)"

	# Please note triple '\'s required to delay interpretation for remote host
	#debugmsg "download/sync $CONTROL_DIR/ $CONTROL_DIR/job\.\*\.\\\{lrms_done,grami\\\}"
	downsync $CONTROL_DIR/ $CONTROL_DIR/job\.\*\.\\\{lrms_done,grami\\\}
	ret=$?
	if [ $ret -ne '0' ]; then
		errormsg "download/sync command failed! (download/sync $CONTROL_DIR/ $CONTROL_DIR/job\.\*\.\\\{lrms_done,grami\\\})"
   		errormsg "Delaying reexecution of this script ($DELAY_REEXEC)" && $DELAY_REEXEC
		break
	fi

	dl_count=$((dl_count + 1)) 

	# TODO: scan-LRMS-job runtime and short jobs may miss recently done jobs
	# since state_file is only reset before exec so lrms_done may be up to 
	# 'runtime' minutes old for fast jobs that wait s for reset
	# TODO: how about using '-5' in first loop and '-1' there after?
	$FIND $CONTROL_DIR -maxdepth 1 -type f -name job.*.lrms_done -mmin -1|$EGREP \. >& /dev/null

	if [ $? -eq '0' ]; then
	#	# Wake up grid-manager to speed up reaction
		debugmsg "TODO: Found done jobs - wake up grid manager ${WAKE_GM}"
	#	${WAKE_GM}
	fi
	
	#debugmsg "/bin/ps -p $child_id >& /dev/null"
	$PS -p $child_id >& /dev/null 
	scan_done=$?
	#debugmsg "ps returned $scan_done"

	if [ $scan_done -ne 0 ]; then
		break
	fi
	#debugmsg "scan loop sleeping $DL_WAIT seconds ($SLEEP $DL_WAIT)"
	$SLEEP $DL_WAIT
done

debugmsg "download/sync loop done after $dl_count downloads"

# Make sure server doesn't throttle on error
if [ $dl_count -lt 2 ]; then
	infomsg "download/sync loop done after only $dl_count downloads - sleeping"
	$SLEEP $DL_WAIT
fi

wait $child_id
$scan_ret=$?

#debugmsg "scan done ($scan_ret)"

exit $scan_ret
