#!/bin/sh
#
#   Periodically read log files of PBS and put mark files
# for job, which finished.
#
# usage: parse_pbs_log control_dir ...

if [ -z "$1" ] ; then exit 1 ; fi


# This should be set automatically
PBS_LOG_PATH=${PBS_LOG_PATH:-/var/spool/pbs/server_logs}
# Where to store temporary files on gatekeeper
TMP_DIR=${TMP_DIR:-/tmp}

pbs_log_dir=$PBS_LOG_PATH


# first control_dir is used for storing own files
control_dir=$1
control_dirs=
while [ $# -gt 0 ] ; do
  control_dirs="${control_dirs} \"$1\""
  shift
done

my_id=`id -u`

state_file=$control_dir/pbs_log_scan.`id -un`

lines=`cat "$state_file" 2>/dev/null`
ldt=`echo $lines | awk '{split($0,field," ");print field[1]}' `
lines=`echo $lines | awk '{split($0,field," ");print field[2]}'`
lines_skip=$(( lines + 0 ))
ldate=$(( ldt + 0 ))
if [ -z "$lines_skip" ] ; then lines_skip='0' ; fi
if [ -z "$ldate" ] ; then ldate='0' ; fi

process_log_file () {
# extract log lines with code 0010 (job exited)
eval "set -- $control_dirs"
egrep '^[^;]*;0010;[^;]*;Job;|^[^;]*;0008;PBS_Server;Job;[^;]*;Exit_status=' ${lname} | \
{
  # parse by ;
  IFS=';'
  lines_processed=0
  while true ; do
    if [ $lines_skip -gt '0' ] ; then
      read rest_line
      if [ $? != '0' ] ; then break ; fi
#      (( lines_skip-- ))
#      (( lines_processed++ ))
      lines_skip=$(( lines_skip - 1 ))
      lines_processed=$(( lines_processed + 1 ))
      continue
    fi
    read pbs_time pbs_code pbs_server pbs_job job_id job_message rest_line
    if [ "$?" != '0' ] ; then
      break
    fi
    exit_code=`echo "$job_message" | sed 's/^.*Exit_status=\([-0-9]*\).*/\1/;t leave;s/.*//;:leave'`
#    (( lines_processed++ ))
    lines_processed=$(( lines_processed + 1 ))
    job_id=`echo "$job_id" | awk '{split($0,field,".");print field[1]"."field[2]}'`
#    echo "new id:" $job_id 1>&2
    # look for this id in job.ID.local
    {
      for ctr_dir in "$@" ; do 
        grep -F -l "localid=$job_id" ${ctr_dir}/job.*.local 2>/dev/null
      done
    } | {
      while true ; do
        read -r name
        if [ -z "$name" ] ; then break ; fi
        if [ "$my_id" != '0' ] ; then
          if [ ! -O "$name" ] ; then continue ; fi
        fi
        base_name=`echo "$name" 2>/dev/null | sed 's/\.local$//;t leave;s/.*//;:leave'`
        if [ -z "${base_name}" ] ; then continue ; fi
        if [ -f "${base_name}.lrms_done" ] ; then continue ; fi
        echo "$exit_code" > "${base_name}.lrms_done"
        break 	  
      done
    }
    
#     for jobs in ${ctr_dir}/job.*.local ; do
#      if [ "$my_id" != '0' ] ; then
#        if [ ! -O "$jobs" ] ; then
#	  continue
#	fi
#      fi
#      base_name=`echo "$jobs" 2>/dev/null | sed 's/\.local$//;t leave;s/.*//;:leave'`
#      if [ -z "$base_name" ] ; then continue ; fi
#      if [ -f "${base_name}.lrms_done" ] ; then continue ; fi
#      job_is=`grep -F -c -h "localid=$job_id" "$jobs"`
#      if [ -z "$job_is" ] ; then continue ; fi
#      if [ "$job_is" -ge '1' ] ; then
#        echo "$exit_code" > "${base_name}.lrms_done"
#      fi
#     done
#    done
  done
  echo "$cname $lines_processed"> $state_file
}
}

for cname in `ls -1 ${pbs_log_dir}/ | grep '^[0-9]*$'` ; do
  lname="${pbs_log_dir}/$cname"
  if [ ! -f "$lname" ] ; then continue ; fi
  if [ "$cname" -lt "$ldate" ] ; then
    continue
  elif [ "$cname" -gt "$ldate" ] ; then
    lines_skip=0
  fi
  echo "Date: " $cname
  last_modified=`stat $lname | grep Modify`
  process_log_file
done
time_count=0
while true ; do
  new_modified=`stat $lname | grep Modify`
  if [ "$new_modified" != "$last_modified" ] ; then
    last_modified="$new_modified"
    lines=`cat "$state_file" 2>/dev/null`
    ldt=`echo $lines | awk '{split($0,field," ");print field[1]}' `
    lines=`echo $lines | awk '{split($0,field," ");print field[2]}'`
    lines_skip=$(( lines + 0 ))
    ldate=$(( ldt + 0 ))
    process_log_file
  fi
  sleep 10
#  (( time_count++ ))
  time_count=$(( time_count + 1 ))
  if [ "$time_count" -gt 60 ] ; then break ; fi
done
#sleep 120
exit 0

