#!/bin/bash -f
# set -x
#
#  Based on globus submission script for pbs
#
#  Submits job to SLURM.
#  Input: path to grami file (same as Globus).
#
# The temporary job script is created for the submission and then removed 
# at the end of this script. 

# TODO: check and use --get-user-env

echo "----- starting submit_slurm_job -----" 1>&2
joboption_lrms=SLURM
walltime_ratio='1'

if [ -z "${NORDUGRID_LOCATION}" ] ; then
    echo "NORDUGRID_LOCATION not set."  1>&2
    exit 1
fi
if [ ! -f "${NORDUGRID_LOCATION}/libexec/configure-${joboption_lrms}-env.sh" ] ; then
    echo "${NORDUGRID_LOCATION}/libexec/configure-${joboption_lrms}-env.sh not found." 1>&2
    exit 1
fi
. ${NORDUGRID_LOCATION}/libexec/configure-${joboption_lrms}-env.sh


# Where to store temporary files on gatekeeper
TMP_DIR=${TMP_DIR:-/tmp}
# Where runtime scripts can be found on computing nodes (empty if does not exist)
RUNTIME_CONFIG_DIR=${RUNTIME_CONFIG_DIR:-}
export RUNTIME_CONFIG_DIR
# Where GNU time utility is located on computing nodes (empty if does not exist)
GNU_TIME=${GNU_TIME:-/usr/bin/time}
# Command to get name of executing node
NODENAME=${NODENAME:-/bin/hostname -f}
# Description of (cross-)mounted disc space on cluster
RUNTIME_FRONTEND_SEES_NODE=${RUNTIME_FRONTEND_SEES_NODE:-}
RUNTIME_NODE_SEES_FRONTEND=${RUNTIME_NODE_SEES_FRONTEND:-}
RUNTIME_ARC_LOCATION="$ARC_LOCATION"
RUNTIME_ARC_LOCATION="${RUNTIME_ARC_LOCATION:-$NORDUGRID_LOCATION}"
RUNTIME_GLOBUS_LOCATION="$GLOBUS_LOCATION"

#default is NFS
if [ -z "${RUNTIME_NODE_SEES_FRONTEND}" ] ; then
  RUNTIME_NODE_SEES_FRONTEND=yes
fi
# locally empty means no
if [ "${RUNTIME_NODE_SEES_FRONTEND}" = 'no' ] ; then
  RUNTIME_NODE_SEES_FRONTEND=
fi

arg_file="$1"
##############################################################
# Source the argument file. 
##############################################################
if [ -z "$arg_file" ] ; then
   echo "Arguments file should be specified" 1>&2
   exit 1
fi
if [ ! -f "$arg_file" ] ; then
   echo "Missing arguments file" 1>&2
   exit 1
fi
. "$arg_file"

##############################################################
# Read queue_node_string from ARC config file
##############################################################

if [ ! -f "$ARC_LOCATION/libexec/config_parser.sh" ] ; then
    echo "$ARC_LOCATION/libexec/config_parser.sh not found." 1>&2
    exit 1
fi

ARC_CONFIG=${ARC_CONFIG:-/etc/arc.conf}
source $ARC_LOCATION/libexec/config_parser.sh

config_parse_file $ARC_CONFIG >&2 || exit $?
config_import_section "common"
config_import_section "cluster"

# Also read queue section
if [ ! -z "$joboption_queue" ]; then
  config_import_section "queue/$joboption_queue"
fi

queue_node_string=$CONFIG_queue_node_string

##############################################################

if [ -z "$joboption_controldir" ] ; then
  joboption_controldir=`dirname "$arg_file"`
  if [ "$joboption_controldir" = '.' ] ; then
    joboption_controldir="$PWD"
  fi
fi
if [ -z "$joboption_gridid" ] ; then
  joboption_gridid=`basename "$arg_file" | sed 's/^job\.\(.*\)\.grami$/\1/'`
fi
failures_file="$joboption_controldir/job.$joboption_gridid.failed"

if [ -z "${RUNTIME_NODE_SEES_FRONTEND}" ] ; then
  if [ -z "${RUNTIME_LOCAL_SCRATCH_DIR}" ] ; then
    echo "Need to know at which directory to run job: RUNTIME_LOCAL_SCRATCH_DIR must be set if RUNTIME_NODE_SEES_FRONTEND is empty" 1>&2
    echo "Submission: Configuration error.">>"$failures_file"
    exit 1
  fi
fi

##############################################################
# combine arguments to command -  easier to use
##############################################################
i=0
joboption_args=
eval "var_is_set=\${joboption_arg_$i+yes}"
while [ ! -z "${var_is_set}" ] ; do
  eval "var_value=\${joboption_arg_$i}"
  var_value=`echo "$var_value" | sed 's/\\\\/\\\\\\\\/g' | sed 's/"/\\\"/g'`
  joboption_args="$joboption_args \"${var_value}\""
  i=$(( i + 1 ))
  eval "var_is_set=\${joboption_arg_$i+yes}"
done

##############################################################
# Zero stage of runtime environments
##############################################################
joboption_num=0
eval "var_is_set=\${joboption_runtime_$joboption_num+yes}"
while [ ! -z "${var_is_set}" ] ; do
  eval "var_value=\${joboption_runtime_$joboption_num}"
  if [ -r "$RUNTIME_CONFIG_DIR/${var_value}" ] ; then
    . "$RUNTIME_CONFIG_DIR/${var_value}" "0"
    if [ $? -ne '0' ] ; then
      echo "ERROR: runtime script ${var_value} failed" 1>&2
      echo "Submission: runtime script ${var_value} failed.">>"$failures_file"
      exit 1
    fi
  else
    echo "ERROR: runtime script ${var_value} is missing" 1>&2
    echo "Submission: runtime script ${var_value} is missing.">>"$failures_file"
    exit 1
  fi
  joboption_num=$(( joboption_num + 1 ))
  eval "var_is_set=\${joboption_runtime_$joboption_num+yes}"
done

tmpdir=${TMP_DIR:-/tmp}

# File name to be used for temporary job script
SLURM_JOB_SCRIPT=`mktemp ${tmpdir}/slurm_job_script.XXXXXX`
if [ -z "$SLURM_JOB_SCRIPT" ] ; then 
  echo "Creation of temporary file failed" 1>&2
  echo "Submission: System error.">>"$failures_file"
  exit 1
fi
SLURM_JOB_OUT="${SLURM_JOB_SCRIPT}.out"
touch $SLURM_JOB_OUT
SLURM_JOB_ERR="${SLURM_JOB_SCRIPT}.err"
touch $SLURM_JOB_ERR
if [ ! -f "$SLURM_JOB_SCRIPT" ] || [ ! -f "$SLURM_JOB_OUT" ] || [ ! -f "$SLURM_JOB_ERR" ] ; then 
  echo "Something is wrong. Either somebody is playing bad or can't write to ${tmpdir}" 1>&2
  rm -f "$SLURM_JOB_SCRIPT" "$SLURM_JOB_OUT" "$SLURM_JOB_ERR"
  echo "Submission: System error.">>"$failures_file"
  exit 1
fi

is_cluster=true
##############################################################
# Start job script
##############################################################
echo "#!/bin/bash -l" > $SLURM_JOB_SCRIPT
echo "# SLURM batch job script built by grid-manager" >> $SLURM_JOB_SCRIPT
# rerun is handled by GM, do not let SLURM rqueue jobs itself.
echo "#SBATCH --no-requeue" >> $SLURM_JOB_SCRIPT

# write SLURM output to 'comment' file
echo "#SBATCH -e '${joboption_directory}.comment'">> $SLURM_JOB_SCRIPT
echo "#SBATCH -o '${joboption_directory}.comment'">> $SLURM_JOB_SCRIPT
echo "" >> $SLURM_JOB_SCRIPT
# choose queue
if [ ! -z "${joboption_queue}" ] ; then
  echo "#SBATCH -p $joboption_queue" >> $SLURM_JOB_SCRIPT
fi
# project name for accounting
if [ ! -z "${joboption_rsl_project}" ] ; then
  echo "#SBATCH -U $joboption_rsl_project" >> $SLURM_JOB_SCRIPT
fi
# job name for convenience
if [ ! -z "${joboption_jobname}" ] ; then
    #TODO is this necessary? do parts of the infosys need these limitations?
  jobname=`echo "$joboption_jobname" | \
           sed 's/^\([^[:alpha:]]\)/N\1/' | \
           sed 's/[^[:alnum:]]/_/g' | \
	   sed 's/\(...............\).*/\1/'`
  echo "#SBATCH -J '$jobname'" >> $SLURM_JOB_SCRIPT
else
    jobname="gridjob"
    echo "#SBATCH -J '$jobname'" >> $SLURM_JOB_SCRIPT
fi
echo "SLURM jobname: $jobname" 1>&2

##############################################################
# (non-)parallel jobs
##############################################################
if [ -z "$joboption_count" ] ; then 
  joboption_count=1
elif [ "$joboption_count" -le 0 ] ; then
  joboption_count=1
fi

nodes_string="#SBATCH -n ${joboption_count}"

i=0
eval "var_is_set=\${joboption_nodeproperty_$i+yes}"
while [ ! -z "${var_is_set}" ] ; do
  eval "var_value=\${joboption_nodeproperty_$i}"
  nodes_string="${nodes_string}:${var_value}"
  i=$(( i + 1 ))
  eval "var_is_set=\${joboption_nodeproperty_$i+yes}"
done
echo "$nodes_string" >> $SLURM_JOB_SCRIPT

##############################################################
# Execution times (minutes)
##############################################################
if [ ! -z "$joboption_cputime" ] ; then
  if [ $joboption_cputime -lt 0 ] ; then
    joboption_cputime=0
  fi
  maxcputime="$joboption_cputime"
  cputime_min=$(( $maxcputime / 60 ))
  cputime_sec=$(( $maxcputime - $cputime_min * 60 ))
  echo "#SBATCH -t ${cputime_min}:${cputime_sec}" >> $SLURM_JOB_SCRIPT
fi  
  
if [ -z "$joboption_walltime" ] ; then
  if [ ! -z "$joboption_cputime" ] ; then
    # Set walltime for backward compatibility or incomplete requests
    joboption_walltime=$(( $joboption_cputime * $walltime_ratio ))
  fi
fi

if [ ! -z "$joboption_walltime" ] ; then
  if [ $joboption_walltime -lt 0 ] ; then
    joboption_walltime=0
  fi
  maxwalltime="$joboption_walltime"
  walltime_min=$(( $maxwalltime / 60 ))
  walltime_sec=$(( $maxwalltime - $walltime_min * 60 ))
  echo "#SBATCH -t ${walltime_min}:${walltime_sec}" >> $SLURM_JOB_SCRIPT
fi

##############################################################
# Requested memory (mb)
##############################################################
if [ "$joboption_localtransfer" = 'yes' ] ; then
  # downloader and uploader can take a lot of memory
  if [ -z "$joboption_memory" ] ; then
    joboption_memory=1000
  else
    if [ "$joboption_memory" -lt 1000 ] ; then
      joboption_memory=1000
    fi
  fi
fi
if [ ! -z "$joboption_memory" ] ; then
  echo "#SBATCH --mem ${joboption_memory}mb" >> $SLURM_JOB_SCRIPT
fi

gate_host=`uname -n`
if [ -z "$gate_host" ] ; then 
  echo "Can't get own hostname" 1>&2
  rm -f "$SLURM_JOB_SCRIPT" "$SLURM_JOB_OUT" "$SLURM_JOB_ERR"
  echo "Submission: Configuration error.">>"$failures_file"
  exit 1
fi

echo "" >> $SLURM_JOB_SCRIPT
echo "# Overide umask of execution node (sometime values are really strange)" >> $SLURM_JOB_SCRIPT
echo "umask 077" >> $SLURM_JOB_SCRIPT

##############################################################
# Add environment variables
##############################################################
echo "# Setting environment variables as specified by user" >> $SLURM_JOB_SCRIPT
i=0
eval "var_is_set=\${joboption_env_$i+yes}"
while [ ! -z "${var_is_set}" ] ; do
  eval "var_value=\${joboption_env_$i}"
  echo "export ${var_value}" >> $SLURM_JOB_SCRIPT
  i=$(( i + 1 ))
  eval "var_is_set=\${joboption_env_$i+yes}"
done
echo "" >> $SLURM_JOB_SCRIPT


##############################################################
# Check for existance of executable,
# there is no sense to check for executable if files are 
# downloaded directly to computing node
##############################################################
if [ -z "${joboption_arg_0}" ] ; then
  echo 'Executable is not specified' 1>&2
  rm -f "$SLURM_JOB_SCRIPT" "$SLURM_JOB_OUT" "$SLURM_JOB_ERR"
  echo "Submission: Job description error.">>"$failures_file"
  exit 1
fi
#######################################################################
# copy information useful for transfering files to/from node directly
#######################################################################
if [ "$joboption_localtransfer" = 'yes' ] ; then
  RUNTIME_CONTROL_DIR=`mktemp ${joboption_directory}/control.XXXXXX`
  if [ -z "$RUNTIME_CONTROL_DIR" ] ; then
    echo 'Failed to choose name for temporary control directory' 1>&2
    rm -f "$SLURM_JOB_SCRIPT" "$SLURM_JOB_OUT" "$SLURM_JOB_ERR"
    echo "Submission: System error.">>"$failures_file"
    exit 1
  fi
  rm -f "$RUNTIME_CONTROL_DIR"
  mkdir "$RUNTIME_CONTROL_DIR"
  if [ $? -ne '0' ] ; then
    echo 'Failed to create temporary control directory' 1>&2
    rm -f "$SLURM_JOB_SCRIPT" "$SLURM_JOB_OUT" "$SLURM_JOB_ERR"
    echo "Submission: System error.">>"$failures_file"
    exit 1
  fi
  chmod go-rwx,u+rwx "${RUNTIME_CONTROL_DIR}"
  echo '' >"${RUNTIME_CONTROL_DIR}/job.local.proxy"
  chmod go-rw,u+r,a-x "${RUNTIME_CONTROL_DIR}/job.local.proxy"
  cat "${joboption_controldir}/job.${joboption_gridid}.proxy" >"${RUNTIME_CONTROL_DIR}/job.local.proxy"
  cat "${joboption_controldir}/job.${joboption_gridid}.input" >"${RUNTIME_CONTROL_DIR}/job.local.input"
  cat "${joboption_controldir}/job.${joboption_gridid}.output" >"${RUNTIME_CONTROL_DIR}/job.local.output"
  RUNTIME_CONTROL_DIR_REL=`basename "$RUNTIME_CONTROL_DIR"`
  echo "$RUNTIME_CONTROL_DIR_REL *.*" >>"${RUNTIME_CONTROL_DIR}/job.local.input"
  echo "$RUNTIME_CONTROL_DIR_REL" >>"${RUNTIME_CONTROL_DIR}/job.local.output"
  echo "$RUNTIME_CONTROL_DIR_REL" >>"${joboption_controldir}/job.${joboption_gridid}.output"
  RUNTIME_STDOUT_REL=`echo "${joboption_stdout}" | sed "s#^${joboption_directory}##"`
  RUNTIME_STDERR_REL=`echo "${joboption_stderr}" | sed "s#^${joboption_directory}##"`
  echo "$RUNTIME_STDOUT_REL *.*" >>"${RUNTIME_CONTROL_DIR}/job.local.input"
  echo "$RUNTIME_STDERR_REL *.*" >>"${RUNTIME_CONTROL_DIR}/job.local.input"
  echo "RUNTIME_CONTROL_DIR=$RUNTIME_CONTROL_DIR" >> $SLURM_JOB_SCRIPT
fi

######################################################################
# Adjust working directory for tweaky nodes
# RUNTIME_GRIDAREA_DIR should be defined by external means on nodes
######################################################################
if [ ! -z "${RUNTIME_NODE_SEES_FRONTEND}" ] ; then
  echo "RUNTIME_JOB_DIR=$joboption_directory" >> $SLURM_JOB_SCRIPT
  echo "RUNTIME_JOB_DIAG=${joboption_directory}.diag" >> $SLURM_JOB_SCRIPT
  echo "RUNTIME_JOB_STDIN=$joboption_stdin" >> $SLURM_JOB_SCRIPT
  echo "RUNTIME_JOB_STDOUT=$joboption_stdout" >> $SLURM_JOB_SCRIPT
  echo "RUNTIME_JOB_STDERR=$joboption_stderr" >> $SLURM_JOB_SCRIPT
  echo "if [ ! -z \"\$RUNTIME_GRIDAREA_DIR\" ] ; then" >> $SLURM_JOB_SCRIPT
  echo "  RUNTIME_JOB_DIR=\$RUNTIME_GRIDAREA_DIR/\`basename \$RUNTIME_JOB_DIR\`" >> $SLURM_JOB_SCRIPT
  echo "  RUNTIME_JOB_STDIN=\`echo \"\$RUNTIME_JOB_STDIN\" | sed \"s#^$joboption_directory#\$RUNTIME_JOB_DIR#\"\`" >> $SLURM_JOB_SCRIPT
  echo "  RUNTIME_JOB_STDOUT=\`echo \"\$RUNTIME_JOB_STDOUT\" | sed \"s#^$joboption_directory#\$RUNTIME_JOB_DIR#\"\`" >> $SLURM_JOB_SCRIPT
  echo "  RUNTIME_JOB_STDERR=\`echo \"\$RUNTIME_JOB_STDERR\" | sed \"s#^$joboption_directory#\$RUNTIME_JOB_DIR#\"\`" >> $SLURM_JOB_SCRIPT
  echo "  RUNTIME_JOB_DIAG=\`echo \"\$RUNTIME_JOB_DIAG\" | sed \"s#^$joboption_directory#\$RUNTIME_JOB_DIR#\"\`" >> $SLURM_JOB_SCRIPT
  echo "  RUNTIME_CONTROL_DIR=\`echo \"\$RUNTIME_CONTROL_DIR\" | sed \"s#^$joboption_directory#\$RUNTIME_JOB_DIR#\"\`" >> $SLURM_JOB_SCRIPT
  echo "fi" >> $SLURM_JOB_SCRIPT
else
  echo "RUNTIME_JOB_DIR=$RUNTIME_LOCAL_SCRATCH_DIR/$joboption_gridid" >> $SLURM_JOB_SCRIPT
  echo "RUNTIME_JOB_DIAG=$RUNTIME_LOCAL_SCRATCH_DIR/${joboption_gridid}.diag" >> $SLURM_JOB_SCRIPT
  echo "RUNTIME_GRIDAREA_DIR=" >> $SLURM_JOB_SCRIPT
  RUNTIME_STDIN_REL=`echo "${joboption_stdin}" | sed "s#^${joboption_directory}/*##"`
  RUNTIME_STDOUT_REL=`echo "${joboption_stdout}" | sed "s#^${joboption_directory}/*##"`
  RUNTIME_STDERR_REL=`echo "${joboption_stderr}" | sed "s#^${joboption_directory}/*##"`
  if [ "$RUNTIME_STDIN_REL" = "${joboption_stdin}" ] ; then
    echo "RUNTIME_JOB_STDIN=\"${joboption_stdin}\"" >> $SLURM_JOB_SCRIPT
  else
    echo "RUNTIME_JOB_STDIN=\"$RUNTIME_LOCAL_SCRATCH_DIR/$joboption_gridid/$RUNTIME_STDIN_REL\"" >> $SLURM_JOB_SCRIPT
  fi
  if [ "$RUNTIME_STDOUT_REL" = "${joboption_stdout}" ] ; then
    echo "RUNTIME_JOB_STDOUT=\"${joboption_stdout}\"" >> $SLURM_JOB_SCRIPT
  else
    echo "RUNTIME_JOB_STDOUT=\"$RUNTIME_LOCAL_SCRATCH_DIR/$joboption_gridid/$RUNTIME_STDOUT_REL\"" >> $SLURM_JOB_SCRIPT
  fi
  if [ "$RUNTIME_STDERR_REL" = "${joboption_stderr}" ] ; then
    echo "RUNTIME_JOB_STDERR=\"${joboption_stderr}\"" >> $SLURM_JOB_SCRIPT
  else
    echo "RUNTIME_JOB_STDERR=\"$RUNTIME_LOCAL_SCRATCH_DIR/$joboption_gridid/$RUNTIME_STDERR_REL\"" >> $SLURM_JOB_SCRIPT
  fi
fi

##############################################################
# Add std... to job arguments
##############################################################
if [ ! -z "$joboption_stdin" ] ; then
  joboption_args="$joboption_args <\$RUNTIME_JOB_STDIN"
fi
if [ ! -z "$joboption_stdout" ] ; then
  joboption_args="$joboption_args 1>\$RUNTIME_JOB_STDOUT"
fi
if [ ! -z "$joboption_stderr" ] ; then
  if [ X"$joboption_stderr" = X"$joboption_stdout" ] ; then
    joboption_args="$joboption_args 2>&1"
  else
    joboption_args="$joboption_args 2>\$RUNTIME_JOB_STDERR"
  fi
fi

##############################################################
#  Move files to local working directory (job is done on node only)
#  RUNTIME_JOB_DIR -> RUNTIME_LOCAL_SCRATCH_DIR/job_id
##############################################################
echo "RUNTIME_LOCAL_SCRATCH_DIR=\${RUNTIME_LOCAL_SCRATCH_DIR:-$RUNTIME_LOCAL_SCRATCH_DIR}" >> $SLURM_JOB_SCRIPT
echo "RUNTIME_FRONTEND_SEES_NODE=\${RUNTIME_FRONTEND_SEES_NODE:-$RUNTIME_FRONTEND_SEES_NODE}" >> $SLURM_JOB_SCRIPT
echo "RUNTIME_NODE_SEES_FRONTEND=\${RUNTIME_NODE_SEES_FRONTEND:-$RUNTIME_NODE_SEES_FRONTEND}" >> $SLURM_JOB_SCRIPT
cat >> $SLURM_JOB_SCRIPT <<'EOSCR'
if [ ! -z "$RUNTIME_LOCAL_SCRATCH_DIR" ] && [ ! -z "$RUNTIME_NODE_SEES_FRONTEND" ] ; then
  # moving (!!!!! race condition here - while there is no job directory
  # gridftp can create the one with the same name !!!!!)
  RUNTIME_NODE_JOB_DIR="$RUNTIME_LOCAL_SCRATCH_DIR"/`basename "$RUNTIME_JOB_DIR"`
  rm -rf "$RUNTIME_NODE_JOB_DIR"
  mv "$RUNTIME_JOB_DIR" "$RUNTIME_LOCAL_SCRATCH_DIR"
  if [ ! -z "$RUNTIME_FRONTEND_SEES_NODE" ] ; then
    # creating link for whole directory
    ln -s "$RUNTIME_FRONTEND_SEES_NODE"/`basename "$RUNTIME_JOB_DIR"` "$RUNTIME_JOB_DIR"
  else
    # keep stdout, stderr and control directory on frontend
    # recreate job directory
    mkdir "$RUNTIME_JOB_DIR"
    # make those files
    mkdir -p `dirname "$RUNTIME_JOB_STDOUT"`
    mkdir -p `dirname "$RUNTIME_JOB_STDERR"`
    touch "$RUNTIME_JOB_STDOUT"
    touch "$RUNTIME_JOB_STDERR"
    RUNTIME_JOB_STDOUT__=`echo "$RUNTIME_JOB_STDOUT" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
    RUNTIME_JOB_STDERR__=`echo "$RUNTIME_JOB_STDERR" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
    rm "$RUNTIME_JOB_STDOUT__" 2>/dev/null
    rm "$RUNTIME_JOB_STDERR__" 2>/dev/null
    if [ ! -z "$RUNTIME_JOB_STDOUT__" ] ; then
      ln -s "$RUNTIME_JOB_STDOUT" "$RUNTIME_JOB_STDOUT__"
    fi
    if [ "$RUNTIME_JOB_STDOUT__" != "$RUNTIME_JOB_STDERR__" ] ; then
      if [ ! -z "$RUNTIME_JOB_STDOUT__" ] ; then
        ln -s "$RUNTIME_JOB_STDERR" "$RUNTIME_JOB_STDERR__"
      fi
    fi
    if [ ! -z "$RUNTIME_CONTROL_DIR" ] ; then
      # move control directory back to frontend
      RUNTIME_CONTROL_DIR__=`echo "$RUNTIME_CONTROL_DIR" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
      mv "$RUNTIME_CONTROL_DIR__" "$RUNTIME_CONTROL_DIR"
    fi    
  fi
  # adjust stdin,stdout & stderr pointers
  RUNTIME_JOB_STDIN=`echo "$RUNTIME_JOB_STDIN" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
  RUNTIME_JOB_STDOUT=`echo "$RUNTIME_JOB_STDOUT" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
  RUNTIME_JOB_STDERR=`echo "$RUNTIME_JOB_STDERR" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
  RUNTIME_FRONTEND_JOB_DIR="$RUNTIME_JOB_DIR"
  RUNTIME_JOB_DIR="$RUNTIME_NODE_JOB_DIR"
fi
if [ -z "$RUNTIME_NODE_SEES_FRONTEND" ] ; then
  mkdir -p "$RUNTIME_JOB_DIR"
fi
EOSCR
echo "" >> $SLURM_JOB_SCRIPT
echo "RESULT=0" >> $SLURM_JOB_SCRIPT
echo "" >> $SLURM_JOB_SCRIPT


#####################################################
#  Download input files
####################################################
if [ "$joboption_localtransfer" = 'yes' ] ; then
echo "ARC_LOCATION=\${ARC_LOCATION:-$RUNTIME_ARC_LOCATION}" >> $SLURM_JOB_SCRIPT
echo "GLOBUS_LOCATION=\${GLOBUS_LOCATION:-$RUNTIME_GLOBUS_LOCATION}" >> $SLURM_JOB_SCRIPT
cat >> $SLURM_JOB_SCRIPT <<'EOSCR'
if [ -z "$ARC_LOCATION" ] ; then
  echo 'Variable ARC_LOCATION is not set' 1>&2
  exit 1
fi
if [ -z "$GLOBUS_LOCATION" ] ; then
  echo 'Variable GLOBUS_LOCATION is not set' 1>&2
  exit 1
fi
NORDUGRID_LOCATION="$ARC_LOCATION"
export GLOBUS_LOCATION
export NORDUGRID_LOCATION
export LD_LIBRARY_PATH="$GLOBUS_LOCATION/lib:$LD_LIBRARY_PATH"
export SASL_PATH="$GLOBUS_LOCATION/lib/sasl"
export X509_USER_KEY="${RUNTIME_CONTROL_DIR}/job.local.proxy"
export X509_USER_CERT="${RUNTIME_CONTROL_DIR}/job.local.proxy"
export X509_USER_PROXY="${RUNTIME_CONTROL_DIR}/job.local.proxy"
unset X509_RUN_AS_SERVER
$ARC_LOCATION/libexec/downloader -p -c 'local' "$RUNTIME_CONTROL_DIR" "$RUNTIME_JOB_DIR" 2>>${RUNTIME_CONTROL_DIR}/job.local.errors
if [ $? -ne '0' ] ; then
  echo 'ERROR: Downloader failed.' 1>&2
  RESULT=1
fi
EOSCR
fi
#####################################################
#  Go to working dir and start job
####################################################
echo "" >> $SLURM_JOB_SCRIPT
echo "# Changing to session directory" >> $SLURM_JOB_SCRIPT
echo "cd \$RUNTIME_JOB_DIR" >> $SLURM_JOB_SCRIPT
echo "export HOME=\$RUNTIME_JOB_DIR" >> $SLURM_JOB_SCRIPT

##############################################################
#  Skip execution if something already failed
##############################################################
echo "if [ \"\$RESULT\" = '0' ] ; then" >> $SLURM_JOB_SCRIPT

##############################################################
#  Runtime configuration at computing node
##############################################################
echo "# Running runtime scripts" >> $SLURM_JOB_SCRIPT
echo "RUNTIME_CONFIG_DIR=\${RUNTIME_CONFIG_DIR:-$RUNTIME_CONFIG_DIR}" >> $SLURM_JOB_SCRIPT
i=0
eval "var_is_set=\${joboption_runtime_$i+yes}"
echo "runtimeenvironments=" >> $SLURM_JOB_SCRIPT
while [ ! -z "${var_is_set}" ] ; do
  if [ "$i" = '0' ] ; then
    echo "if [ ! -z \"\$RUNTIME_CONFIG_DIR\" ] ; then" >> $SLURM_JOB_SCRIPT
  fi
  eval "  var_value=\"\${joboption_runtime_$i}\""
  echo "  if [ -r \"\${RUNTIME_CONFIG_DIR}/${var_value}\" ] ; then" >> $SLURM_JOB_SCRIPT
  echo "    runtimeenvironments=\"\${runtimeenvironments}${var_value};\"" >> $SLURM_JOB_SCRIPT
  echo "    source \${RUNTIME_CONFIG_DIR}/${var_value} 1 " >> $SLURM_JOB_SCRIPT
  echo "    if [ \$? -ne '0' ] ; then " >> $SLURM_JOB_SCRIPT
  echo "      echo \"Runtime ${var_value} script failed \" 1>&2 " >> $SLURM_JOB_SCRIPT
  echo "      echo \"Runtime ${var_value} script failed \" 1>\"\$RUNTIME_JOB_DIAG\" " >> $SLURM_JOB_SCRIPT
  echo "      RESULT=1 " >> $SLURM_JOB_SCRIPT
  echo "    fi " >> $SLURM_JOB_SCRIPT
  echo "  fi" >> $SLURM_JOB_SCRIPT
  i=$(( i + 1 ))
  eval "var_is_set=\${joboption_runtime_$i+yes}"
done
if [ ! "$i" = '0' ] ; then
  echo "fi" >> $SLURM_JOB_SCRIPT
fi
echo "" >> $SLURM_JOB_SCRIPT


##############################################################
#  Diagnostics
##############################################################
echo "echo \"runtimeenvironments=\$runtimeenvironments\" >> \"\$RUNTIME_JOB_DIAG\"" >> $SLURM_JOB_SCRIPT
cat >> $SLURM_JOB_SCRIPT <<'EOSCR'
if [ ! "X$SLURM_NODEFILE" = 'X' ] ; then
  if [ -r "$SLURM_NODEFILE" ] ; then
    cat "$SLURM_NODEFILE" | sed 's/\(.*\)/nodename=\1/' >> "$RUNTIME_JOB_DIAG"
  else
    SLURM_NODEFILE=
  fi
fi
EOSCR
if [ ! -z "$NODENAME" ] ; then
  echo 'if [ "X$SLURM_NODEFILE" = X ] ; then' >> $SLURM_JOB_SCRIPT
  echo "  nodename=\`$NODENAME\`" >> $SLURM_JOB_SCRIPT
  echo "  echo \"nodename=\$nodename\" >> \"\$RUNTIME_JOB_DIAG\"" >> $SLURM_JOB_SCRIPT
  echo 'fi' >> $SLURM_JOB_SCRIPT
fi

##############################################################
#  Check intermediate result again
##############################################################
echo "if [ \"\$RESULT\" = '0' ] ; then" >> $SLURM_JOB_SCRIPT

##############################################################
#  Execution
##############################################################
if [ -z "$GNU_TIME" ] ; then 
  echo "$joboption_args" >> $SLURM_JOB_SCRIPT
else
  echo "if [ ! -x \"$GNU_TIME\" ] ; then" >> $SLURM_JOB_SCRIPT
  echo "  echo \"warning='GNU time is missing - resource usage not available'\">\"\$RUNTIME_JOB_DIAG\" " >> $SLURM_JOB_SCRIPT
  echo "  $joboption_args" >> $SLURM_JOB_SCRIPT
  echo "else" >> $SLURM_JOB_SCRIPT
  echo "  \"$GNU_TIME\" -o \"\$RUNTIME_JOB_DIAG\" -a -f '\
WallTime=%es\nKernelTime=%Ss\nUserTime=%Us\nCPUUsage=%P\n\
MaxResidentMemory=%MkB\nAverageResidentMemory=%tkB\n\
AverageTotalMemory=%KkB\nAverageUnsharedMemory=%DkB\n\
AverageUnsharedStack=%pkB\nAverageSharedMemory=%XkB\n\
PageSize=%ZB\nMajorPageFaults=%F\nMinorPageFaults=%R\n\
Swaps=%W\nForcedSwitches=%c\nWaitSwitches=%w\n\
Inputs=%I\nOutputs=%O\nSocketReceived=%r\nSocketSent=%s\n\
Signals=%k\n' \
$joboption_args" >> $SLURM_JOB_SCRIPT
  echo "fi" >> $SLURM_JOB_SCRIPT
fi

echo "RESULT=\$?" >> $SLURM_JOB_SCRIPT

##############################################################
#  End of RESULT checks
##############################################################
echo "fi" >> $SLURM_JOB_SCRIPT
echo "fi" >> $SLURM_JOB_SCRIPT

##############################################################
#  Runtime (post)configuration at computing node
##############################################################
i=0
eval "var_is_set=\${joboption_runtime_$i+yes}"
while [ ! -z "${var_is_set}" ] ; do
  if [ "$i" = '0' ] ; then
    echo "if [ ! -z \"\$RUNTIME_CONFIG_DIR\" ] ; then" >> $SLURM_JOB_SCRIPT
  fi
  eval "var_value=\"\${joboption_runtime_$i}\""
  echo "  if [ -r \"\${RUNTIME_CONFIG_DIR}/${var_value}\" ] ; then" >> $SLURM_JOB_SCRIPT
  echo "    source \${RUNTIME_CONFIG_DIR}/${var_value} 2 " >> $SLURM_JOB_SCRIPT
  echo "  fi" >> $SLURM_JOB_SCRIPT
  i=$(( i + 1 ))
  eval "var_is_set=\${joboption_runtime_$i+yes}"
done
if [ ! "$i" = '0' ] ; then
  echo "fi" >> $SLURM_JOB_SCRIPT
fi
echo "" >> $SLURM_JOB_SCRIPT

#####################################################
#  Upload output files
####################################################
if [ "$joboption_localtransfer" = 'yes' ] ; then
cat >> $SLURM_JOB_SCRIPT <<'EOSCR'
if [ "$RESULT" = '0' ] ; then
  $ARC_LOCATION/libexec/uploader -p -c 'local' "$RUNTIME_CONTROL_DIR" "$RUNTIME_JOB_DIR" 2>>${RUNTIME_CONTROL_DIR}/job.local.errors
  if [ $? -ne '0' ] ; then
    echo 'ERROR: Uploader failed.' 1>&2
    if [ "$RESULT" = '0' ] ; then RESULT=1 ; fi
  fi
fi
rm -f "${RUNTIME_CONTROL_DIR}/job.local.proxy"
EOSCR
else
# There is no sense to keep trash till GM runs uploader
  echo 'if [ ! -z  "$RUNTIME_LOCAL_SCRATCH_DIR" ] ; then' >> $SLURM_JOB_SCRIPT
# Delete all files except listed in job.#.output
  echo '  find ./ -type l -exec rm -f "{}" ";"' >> $SLURM_JOB_SCRIPT
  echo '  find ./ -type f -exec chmod u+w "{}" ";"' >> $SLURM_JOB_SCRIPT
  if [ -f "$joboption_controldir/job.$joboption_gridid.output" ] ; then
    cat "$joboption_controldir/job.$joboption_gridid.output" | \
    sed 's/^ *"\([^"]*\)".*/\1/g;t suc;s/^ *//;s/\([^\\]\) .*/\1/;:suc;s/\\\(.\)/\1/g' | \
    { while true ; do
      read name
      if [ $? -ne '0' ] ; then break ; fi
      echo "  chmod -R u-w \"$RUNTIME_LOCAL_SCRATCH_DIR/$joboption_gridid/$name\" 2>/dev/null" >> $SLURM_JOB_SCRIPT
    done; }
  fi
  echo '  find ./ -type f -perm +200 -exec rm -f "{}" ";"' >> $SLURM_JOB_SCRIPT
  echo '  find ./ -type f -exec chmod u+w "{}" ";"' >> $SLURM_JOB_SCRIPT
  echo 'fi' >> $SLURM_JOB_SCRIPT
fi
echo "" >> $SLURM_JOB_SCRIPT

##############################################################
#  Move files back to session directory (job is done on node only)
#  RUNTIME_JOB_DIR -> RUNTIME_LOCAL_SCRATCH_DIR/job_id
# !!!!!!!!!!!!!!!!!!! would be better to know the names of files !!!!!!!!!!!
##############################################################
cat >> $SLURM_JOB_SCRIPT <<'EOSCR'
if [ ! -z "$RUNTIME_LOCAL_SCRATCH_DIR" ] && [ ! -z "$RUNTIME_NODE_SEES_FRONTEND" ] ; then
  if [ ! -z "$RUNTIME_FRONTEND_SEES_NODE" ] ; then
    # just move it
    rm "$RUNTIME_FRONTEND_JOB_DIR"
    mv "$RUNTIME_NODE_JOB_DIR" `dirname "$RUNTIME_FRONTEND_JOB_DIR"`
  else
    # remove links
    rm "$RUNTIME_JOB_STDOUT" 2>/dev/null
    rm "$RUNTIME_JOB_STDERR" 2>/dev/null
    # move whole directory
    cp -pR "$RUNTIME_NODE_JOB_DIR" `dirname "$RUNTIME_FRONTEND_JOB_DIR"`
    rm -rf "$RUNTIME_NODE_JOB_DIR"
  fi
fi
echo "exitcode=$RESULT" >> "$RUNTIME_JOB_DIAG"
exit $RESULT
EOSCR

#######################################
#  Submit the job
#######################################
echo "SLURM job script built" 1>&2
# Execute sbatch command
cd "$joboption_directory"
echo "SLURM script follows:" 1>&2
echo "-------------------------------------------------------------------" 1>&2
cat "$SLURM_JOB_SCRIPT" 1>&2
echo "-------------------------------------------------------------------" 1>&2
echo "" 1>&2
SLURM_RESULT=1
SLURM_TRIES=0
while [ "$SLURM_TRIES" -lt '10' ] ; do

  # Unset all environment variables before calling sbatch. Otherwise
  # SLURM will forward them to the job and leak information about
  # the grid-manager.
  # TODO: Maybe we only should unset $ARC_*, $CONFIG_*, $GLOBUS_* etc?
  (for i in $(env|cut -d= -f1);do unset $i;done; ${sbatch}) < \
    $SLURM_JOB_SCRIPT 1>$SLURM_JOB_OUT 2>$SLURM_JOB_ERR
  SLURM_RESULT="$?"
  if [ "$SLURM_RESULT" -eq '0' ] ; then break ; fi 
  if [ "$SLURM_RESULT" -eq '198' ] ; then 
    echo "Waiting for queue to decrease" 1>&2
    sleep 60
    SLURM_TRIES=0
    continue
  fi
  grep 'maximum number of jobs' "$SLURM_JOB_OUT" "$SLURM_JOB_ERR"
  if [ $? -eq '0' ] ; then 
    echo "Waiting for queue to decrease" 1>&2
    sleep 60
    SLURM_TRIES=0
    continue
  fi 
  SLURM_TRIES=$(( SLURM_TRIES + 1 ))
  sleep 2
done
if [ $SLURM_RESULT -eq '0' ] ; then

#TODO test what happens when the jobqueue is full or when the slurm ctld is not responding

   job_id=`cat $SLURM_JOB_ERR | sed 's/sbatch: Submitted batch job \([0-9]*\)$/\1/'`
   if [ "${job_id}" = "" ] ; then
      echo "job *NOT* submitted successfully!" 1>&2
      echo "failed getting the slurm jobid for the job!" 1>&2
      echo "Submission: Local submission client behaved unexpectedly.">>"$failures_file"
   else
      echo "joboption_jobid=$job_id" >> $arg_file
      echo "job submitted successfully!" 1>&2
      echo "local job id: $job_id" 1>&2
      # Remove temporary job script file
      rm -f $SLURM_JOB_SCRIPT $SLURM_JOB_OUT $SLURM_JOB_ERR
      echo "----- exiting submit_slurm_job -----" 1>&2
      echo "" 1>&2
      exit 0
   fi
else
  echo "job *NOT* submitted successfully!" 1>&2
  echo "got error code from sbatch: $SLURM_RESULT !" 1>&2
  echo "Submission: Local submission client failed.">>"$failures_file"
fi
echo "Output is:" 1>&2
cat $SLURM_JOB_OUT 1>&2
echo "Error output is:"
cat $SLURM_JOB_ERR 1>&2
rm -f "$SLURM_JOB_SCRIPT" "$SLURM_JOB_OUT" "$SLURM_JOB_ERR"
echo "----- exiting submit_slurm_job -----" 1>&2
echo "" 1>&2
exit 1
