#!/bin/sh
#
#   Periodically read log files of PBS and put mark files
# for job, which finished.
#
# usage: parse_pbs_log control_dir ...

if [ -z "$1" ] ; then exit 1 ; fi


# Path to pbs commands
PBS_BIN_PATH=${PBS_BIN_PATH:-/usr/bin}
# Where to store temporary files on gatekeeper
TMP_DIR=${TMP_DIR:-/tmp}


# Get all running jobs
pids=`${PBS_BIN_PATH}/qstat -a 2>/dev/null | grep '^[0-9]*\.' | sed 's/^\([^ ]*\).*/\1/'`
# Go through directories
for ctr_dir in "$@" ; do
  # Obtain ids stored in job.*.local
  ids=`grep -h '^localid=' ${ctr_dir}/job.*.local 2>/dev/null | sed 's/^localid=\([^ ]*\)/\1/'`
  if [ -z "$ids" ] ; then continue ; fi
  # compare them to running jobs and find missing
  bids=
  for id in $ids ; do
    found=`echo "$pids" | grep "^$id"`
    if [ -z "$found" ] ; then
      bids="$bids $id"
    fi
  done
  # go through missing ids
  for id in $bids ; do
    # find grid job corresponding to curent local id
    jobfile=`grep -F -l "localid=$id" ${ctr_dir}/job.*.local 2>/dev/null`
    if [ -z "$jobfile" ] ; then continue ; fi
    # extract grid id
    gridid=`basename "$jobfile" '.local' | sed 's/^job\.//'`
    donefile="${ctr_dir}/job.${gridid}.lrms_done"
    if [ -f "$donefile" ] ; then continue ; fi
    statusfile="${ctr_dir}/job.${gridid}.status"
    if [ ! -f "$statusfile" ] ; then continue ; fi
    status=`cat "$statusfile"`
    if [ ! "$status" = "INLRMS" ] ; then continue ; fi
    # get session directory of this job
    session=`grep -h '^sessiondir=' "$jobfile" | sed 's/^sessiondir=\(.*\)/\1/'`
    if [ ! -z "$session" ] ; then
      # have chance to obtain exit code
      diagfile="${session}.diag"
      exitcode=`grep '^exitcode=' "$diagfile" | sed 's/^exitcode=//'`
      if [ ! -z "$exitcode" ] ; then
        # job finished and exit code is known
        echo "$exitcode" > "$donefile"
        echo "Job $gridid finished with exit code $exitcode"
        continue
      fi
    fi
    # job is probaly finished and exit code is not known
    exitcode=1
    countfile="${ctr_dir}/job.${gridid}.lrms_job"
    counter=0
    if [ -f "$countfile" ] ; then
      counter=`cat "$countfile"`
      counter=$(( $counter + 1 ))
    fi
    if [ "$counter" -gt 5 ] ; then
      rm -f "$countfile"
      echo "$exitcode" > "$donefile"
        echo "Job $gridid finished with unknown exit code"
    else
      echo "$counter" > "$countfile"
    fi
  done
done
sleep 60
exit 0

fi
