#!/bin/sh
#
#  Periodically monitor for jobs which has finished or failed but not
#  reported an exitcode
#

id=`id -u`

#debug='eval echo >> /tmp/parse-fork-log.$id'
debug=:

$debug "run at `date`"
$debug "options = $@"

if [ -z "$1" ] ; then echo "Argument missing" 1>&2 ; exit 1 ; fi

basedir=`dirname $0`
basedir=`cd $basedir > /dev/null && pwd` || exit $?

pkglibdir="$basedir/../lib"
pkglibdir=`cd $pkglibdir > /dev/null && pwd` || exit $?

. "${pkglibdir}/scan_common.sh" || exit $?

for control_dir in "$@" ; do

    if [ ! -d "${control_dir}" ]; then 
        echo "No control dir $control_dir" 1>&2
        continue
    fi

    # iterate over all jobs known in the control directory
    find ${control_dir} -name 'job.[0-9]*.status' \
    | xargs egrep -l "INLRMS|CANCELING" \
    | sed -e 's/.*job\.//' -e 's/\.status$//' \
    | while read job; do
        $debug "scanning job = $job"
        unset joboption_jobid
        unset joboption_directory

        # this job was already completed, nothing remains to be done
        [ -f "${control_dir}/job.${job}.lrms_done" ] && continue

        # a grami file exists for all jobs that GM thinks are running.
        # proceed to next job if this file is missing.
        if [ ! -f "${control_dir}/job.${job}.grami" ]; then
            continue
        fi

        # extract process IDs of the grami file
        [ ! -f "${control_dir}/job.${job}.grami" ] && continue
        . "${control_dir}/job.${job}.grami"

        # process IDs could not be learned, proceeding to next
        [ -z "$joboption_jobid" ] && continue
    
        $debug "local jobid = $joboption_jobid"

        # checking if process is still running
        if ps -ouser= -p$joboption_jobid > /dev/null; then
            $debug "ps returned $? - process $joboption_jobid of job $job is still running. Continuing to next"
            continue
        else
            $debug "ps returned $? - process $joboption_jobid of job $job has exited!"
        fi
	name="${control_dir}/job.${job}.local"
        uid=$(get_owner_uid "$name")
        [ -z "$uid" ] && { log "Failed to stat $name"; continue; }

        $debug "local user id = $uid"
        diagfile=${joboption_directory}.diag
        $debug "checking $diagfile"
        exitcode=$(do_as_uid "$uid" "cat '$diagfile'" | sed -n 's/^exitcode=\([0-9]*\).*/\1/p')
        $debug "exitcode = [$exitcode] extracted from $diagfile"
        fork_comment=""
        if [ -z "$exitcode" ]; then
            echo "Job $job with PID $joboption_jobid died unexpectedly" 1>&2
            fork_comment="Job died unexpectedly" 1>&2
            exitcode=-1
        elif [ "$exitcode" -ne '0' ]; then
            fork_comment="Job finished with non-zero exit code" 1>&2
        fi
        $debug "got exitcode=$exitcode"
        save_commentfile "$uid" "${joboption_directory}.comment" "${control_dir}/job.${job}.errors"
        echo "$exitcode $fork_comment" > "${control_dir}/job.${job}.lrms_done"
    done

done

$debug "done, going to sleep"

sleep 10
exit 0
