#!/bin/sh
#
#  Submits job to remote resource.
#  Input: path to grami file.
#
# usage: submit-remote-job grami_file

#################
# General setup #
#################

NORDUGRID_LOCATION=${NORDUGRID_LOCATION:-/opt/nordugrid}

# Include remote LRMS functions
REMOTE_LRMS_FUNCTIONS="${NORDUGRID_LOCATION}/libexec/remote-lrms-functions"
if [ ! -r ${REMOTE_LRMS_FUNCTIONS} ]; then
	logger "ERROR: ${REMOTE_LRMS_FUNCTIONS} not readable!"
	exit 1
fi

. ${REMOTE_LRMS_FUNCTIONS}

read_gw_conf $@
LOCAL_LRMS=$?

if [ ${LOCAL_LRMS} -ne '0' ]; then
	errormsg "remote LRMS configuration disabled! (${LOCAL_LRMS})"
	exit 1
fi

debugmsg "submit-remote-job: $@"


#################################
# Set up reasonable environment #
#################################

# Make sure we are not in /root or somewhere else with no access! 
# IMPORTANT! since sync won't work otherwise!!!
cd $TMP_DIR

# Please note that this script gets executed as 'su grid' (no minus!)
# Thus we will be located in /root/ and have some partial leftover root environment
OLD_MASK=`umask`
umask 007

# TODO: we should probably reset umask to OLD_UMASK before exit

# Grab LRMS arg before continuing as usual
if [ -z "$1" ] ; then 
	errormsg "No arg file supplied!"
	exit 1
fi

# Read LRMS type from nordugrid.conf into CONF_VALUE variable
read_conf_value "resource_lrms"
SUBMIT_LRMS_KIND="$CONF_VALUE"

#debugmsg "wrapping submit script for $SUBMIT_LRMS_KIND at resource"

SUBMIT_ARG_FILE=$1
##################################################################
# Source the argument file. (this will set job and control dirs) #
##################################################################
if [ -z "$SUBMIT_ARG_FILE" ] ; then
   errormsg "Arguments file should be specified"
   exit 1
fi
if [ ! -f $SUBMIT_ARG_FILE ] ; then
   errormsg "Missing arguments file"
   exit 1
fi
. $SUBMIT_ARG_FILE


################################################
# Set up other variables we need at this point #
################################################

# Build name of submit script by replacing remote with supplied LRMS name
SUBMIT_LRMS_SCRIPT=`echo $0 | $SED "s/-remote-/-${SUBMIT_LRMS_KIND}-/"`

# Strip the path
SUBMIT_SCRIPT_NAME=`$BASENAME ${SUBMIT_LRMS_SCRIPT}`

# Temporary scratch dir used locally and remotely
REMOTE_SCRATCH=`$MKTEMP -d ${TMP_DIR}/remote-scratch.XXXXXX`

# Store copies of relevant files in REMOTE_SCRATCH 
LRMS_SCRIPT=${REMOTE_SCRATCH}/`$BASENAME $SUBMIT_LRMS_SCRIPT`
#ARG_FILE=${REMOTE_SCRATCH}/`$BASENAME $SUBMIT_ARG_FILE`
# Save current environment in a file to be sourced before remote submit
ENV_FILE="${LRMS_SCRIPT}.environment"

# Copy options
CP_OPTS=$UPDATE


########################################################
# Build environment file to be sourced at the resource #
########################################################

store_environment $ENV_FILE


######################################################
# Prepare scratch directory for transfer to resource #
######################################################

# We need the submit script, the argument file, and the environment file all in a 
# directory at the remote host. The three files are placed in REMOTE_SCRATCH
# before being copied there in a single transfer.

#debugmsg "$CP $CP_OPTS ${SUBMIT_LRMS_SCRIPT} ${LRMS_SCRIPT}"
$CP $CP_OPTS ${SUBMIT_LRMS_SCRIPT} ${LRMS_SCRIPT}
ret=$?
if [ $ret -ne '0' ]; then
	errmsg "$CP $CP_OPTS ${SUBMIT_LRMS_SCRIPT} ${LRMS_SCRIPT} failed!"
	exit 1
fi

#debugmsg "$CP $CP_OPTS ${SUBMIT_ARG_FILE} ${ARG_FILE}"
#$CP $CP_OPTS ${SUBMIT_ARG_FILE} ${ARG_FILE}
#ret=$?
#if [ $ret -ne '0' ]; then
#	errmsg "$CP $CP_OPTS ${SUBMIT_ARG_FILE} ${ARG_FILE} failed!"
#	exit 1
#fi

# Now all files are inside REMOTE_SCRATCH

######################################################
# Transfer scratch directory to resource and execute #
######################################################

#debugmsg "upload/sync arg file (${SUBMIT_ARG_FILE}) to remote resource"
upload ${SUBMIT_ARG_FILE} ${SUBMIT_ARG_FILE}
ret=$?
if [ $ret -ne '0' ]; then
       errormsg "upload/sync command failed ($ret)! (upload/sync ${SUBMIT_ARG_FILE} ${SUBMIT_ARG_FILE})"
       exit 1
fi

#debugmsg "upload/sync scratch dir (${REMOTE_SCRATCH}) to remote resource"
upload ${REMOTE_SCRATCH}/ ${REMOTE_SCRATCH}
ret=$?
if [ $ret -ne '0' ]; then
	errormsg "upload/sync command failed ($ret)! (upload/sync ${REMOTE_SCRATCH}/ ${REMOTE_SCRATCH})"
	exit 1
fi


#debugmsg "Remote execution of $LRMS_SCRIPT with arg ${SUBMIT_ARG_FILE}"
remote_exec ". ${ENV_FILE} && ${LRMS_SCRIPT} ${SUBMIT_ARG_FILE}"
ret=$?
if [ $ret -ne '0' ]; then
	errormsg "remote job submission failed ($ret)! (${LRMS_SCRIPT} ${SUBMIT_ARG_FILE})"
	exit $ret
fi

debugmsg "remote job submission succeeded (${LRMS_SCRIPT} ${SUBMIT_ARG_FILE})"


# submit-pbs-job writes jobid to arg file before leaving. This must be propagated
# to GM, so we downsync it here. Please note that this is *not* a race since write 
# is done before job submission finishes and not inside actual job.

# We may not have permissions to create a new file in the control dir as rsync does.
# Thus we download rather than downsync.
#debugmsg "download/sync ${SUBMIT_ARG_FILE} ${SUBMIT_ARG_FILE}"
download ${SUBMIT_ARG_FILE} ${SUBMIT_ARG_FILE}
ret=$?
if [ $ret -ne '0' ]; then
	errormsg "download/sync command failed! (download/sync ${SUBMIT_ARG_FILE} ${SUBMIT_ARG_FILE})"
	exit 1
fi

if [ -z "$DEBUG_FILES" ]; then
	debugmsg "cleaning up local and remote scratch dir (${REMOTE_SCRATCH})"
	#debugmsg "local execution of $RM $FORCE $RECURSIVE ${REMOTE_SCRATCH}"
	$RM $FORCE $RECURSIVE ${REMOTE_SCRATCH}
	ret=$?
	if [ $ret -ne '0' ]; then
		errormsg "local clean up failed ($ret)! - ignoring"
	fi
	#debugmsg "remote execution of $RM $FORCE $RECURSIVE ${REMOTE_SCRATCH}"
	remote_exec "$RM $FORCE $RECURSIVE ${REMOTE_SCRATCH}"
	ret=$?
	if [ $ret -ne '0' ]; then
		errormsg "remote clean up failed ($ret)! - ignoring"
	fi
fi

exit 0
