#! /bin/sh
np=1
polling_mode=1
max_time=15
mvback=""
leavescript=0
leaveoutput=0
echoscript=0
hasmvback=0
while [ 1 -le $# ] ; do
  arg=$1
  #echo $arg
  #echo $#
  shift
  case $arg in 
    -np)
	np="$1"
        nodigits=`echo $np | sed 's/^[0-9]*$//'`
        if [ "$nodigits" != "" -o $np -lt 1 ] ; then
           echo np: $np is an invalid number of processors.  Exiting.
           exit 1
	fi
	eval np$narch=$1
	shift
	;;
    -v)
       	mpirun_verbose=1
	;;
    -t)
      	just_testing=1
	Show=echo
	;;
    -leave) 
	leavescript=1 
	leaveoutput=1
	;;
    -maxtime|-cpu)
        max_time="$1"
	shift
	;;
    -nopoll)
	polling_mode=0
	;;
    -mvhome)
	# discard
	;;
    -mvback)
	mvback="$mvback $1"
	hasmvback=1
	shift
	;;
    -echo)
	# this is for debugging
	set -x
	;;
    -echoscript)
	echoscript=1
	;;
    *)
	# The first unrecognized argument is assumed to be the name of
      	# the program, but only if it is executable
	proginstance=`echo $arg | sed "s/%a/$arch/g"`
      	if [ "$progname" = "" -a "$fake_progname" = "" -a \
		! -x "$proginstance" ] ; then
	    fake_progname="$arg"
      	elif [ "$progname" = "" -a -x "$proginstance" ] ; then 
	    progname="$arg"
            # any following unrecognized arguments are assumed to be arguments
            # to be passed to the program, as well as all future args
	    # We can't use use_execer here to decide how to store the
	    # arguments, since we may later change it (see the switch on arch)
	    # Thus we store the args in tow forms
	    while [ 1 -le $# ] ; do
               cmdLineArgsExecer="$cmdLineArgsExecer -arg=$1"
               cmdLineArgs="$cmdLineArgs $1"
	       shift
            done
         else
	    echo "Warning: Command line arguments for program should be given"
            echo "after the program name.  Assuming that $arg is a"
            echo "command line argument for the program."
            cmdLineArgsExecer="$cmdLineArgsExecer -arg=$arg"
            cmdLineArgs="$cmdLineArgs $arg"
         fi
      ;;
  esac
done
#
/bin/rm -f PIrun$$
CUR_DIR=`pwd`
# The absence of a ip name command is very painful.
HostName=`hostname`.larc.nasa.gov
#
# Generate the script file
cat >PIrun$$ <<EOF
#! /bin/sh
EOF
#
# Handle any files that we are moving back 
if [ -n "$mvback" ] ; then
    for file in $mvback ; do
        echo "#PBS -W stageout=/scratch1/$LOGNAME/${file}@${HostName}:$CUR_DIR/$file" >> \
		PIrun$$
    done
fi
if [ $echoscript = 1 ] ; then
    echo "set -x" >> PIrun$$
    echo "hostname" >> PIrun$$
fi
cat >>PIrun$$ <<EOF
# We can't do a cd cur_dir if the job has been migrated!
#if [ -d $CUR_DIR ] ; then
#    cd $CUR_DIR
#fi
# Since the HPM performance summary includes activities not related to the
# users program (such as this script and job startup), it isn't very
# useful.  The following test ensures that there is a file that suppresses
# this extraneous data
if [ ! -f /tmp/\$PBS_JOBID.hpmsilent ] ; then
    echo "quiet" > /tmp/\$PBS_JOBID.hpmsilent
fi
#
# Move to the execution directory
cd /scratch1/$LOGNAME
# Set up the MP parameters
MP_EUILIB=us
#MP_RMPOOL=0
#MP_HOSTFILE=$MPI_PARTITION
MP_PROCS=$np
MP_INFOLEVEL=0
MP_HOLD_STDIN=YES
MP_PULSE=0
# If labelio is not set, each output line is labeled with the processor
# that generated it.  A nice feature, but not the correct default if you
# want to make programs act as if they aren't special for being parallel.
MP_LABELIO=no
MP_STDINMODE=0
export MP_EUILIB
#export MP_RMPOOL
#export MP_HOSTFILE
export MP_PROCS
export MP_INFOLEVEL
export MP_HOLD_STDIN
export MP_PULSE
export MP_LABELIO
export MP_STDINMODE
if [ $polling_mode = 0 ] ; then
   MP_CSS_INTERRUPT=yes
else
   MP_CSS_INTERRUPT=no
fi
export MP_CSS_INTERRUPT
#PATH=".:$PATH"
#export PATH
# It is possible that this program is not yet copied over.
cnt=1
while [ \$cnt -lt 60 -a ! -x /scratch1/$LOGNAME/$progname$$ ] ; do
    sleep 1
    cnt=\`expr \$cnt + 1\`
done
if [ ! -x /scratch1/$LOGNAME/$progname$$ ] ; then
    echo "Could not find program /scratch1/$LOGNAME/$progname$$ on \`hostname\`"
    exit 1
fi
if [ \$cnt -gt 1 ] ; then
    echo "WARNING: Queueing system did not stagein file before attempting to"
    echo "run program.  Please report to system admin (waited \$cnt seconds)"
fi
#
# This script may be running on a  HOST that is different from the nodes.
# Use poe with test to check that the program is present
# 
#poe test -x /scratch1/$LOGNAME/$progname$$ < /dev/null
#rc=\$?
#if [ \$rc -ne 0 ] ; then
#    echo "test -x /scratch1/$LOGNAME/$progname$$ failed on some nodes in "
#    echo "partition containing \`cat \$MP_HOSTFILE\`"
#    exit 1
#fi
#
# Without the redirection of stdin, poe might try to read from stdin, and
# depending on how the job manager is operating, may cause problems.
poe /scratch1/$LOGNAME/$progname$$ $cmdLineArgs < /dev/null
rc=\$?
if [ $hasmvback = 1 ] ; then
    /bin/sync
fi
if [ \$rc -ne 0 ] ; then
    if [ -n "\$MP_HOSTFILE" -a -s "\$MP_HOSTFILE" ] ; then
	cat \$MP_HOSTFILE | tr '\012' ' '
    fi
    exit \$rc
fi
# Handle the possible output of log file data.
if [ $hasmvback = 1 ] ; then
    if [ -s /scratch1/$LOGNAME/$progname$$_profile.log ] ; then
        /bin/mv /scratch1/$LOGNAME/$progname$$_profile.log /scratch1/$LOGNAME/${progname}_profile.log
    fi
fi
EOF
#
#
chmod a+x PIrun$$
# Use -z to suppress the jobid as output from qsub
jobid=`qsub -l nodes=${np}:noserver -l walltime=${max_time}:00 -S /bin/sh -V \
    -W stagein=/scratch1/$LOGNAME/$progname$$@${HostName}:$CUR_DIR/$progname \
        PIrun$$ 2>&1`
if [ $? -ne 0 ] ; then
    echo "qsub failed: $jobid"
    echo "mpirun aborting"
    exit 1
fi
trap "qdel $jobid"
#
# Exponentially back off on the sleep time until we reach 1 minute.
# Backoff every 60 tests
sleeptime=1
cnt=1
while [ 1 ] ; do
    resp=`qstat $jobid 2>&1`
    # Look for "qstat: Unknown" in the response
    check=`expr "$resp" : '\(..............\)'`
    if [ "$check" = "qstat: Unknown" ] ; then break ; fi
    sleep $sleeptime
    if [ $cnt -ge 60 ] ; then 
	if [ $sleeptime -lt 60 ] ; then 
  	    sleeptime=`expr $sleeptime \* 2`
	    if [ $sleeptime -gt 60 ] ; then sleeptime=60 ; fi
	fi
	cnt=0
    fi
    cnt=`expr $cnt + 1`
done
# Recover the data
# Sequence number is at the head of the jobid
seqnum=`echo $jobid | sed -e 's/^\([0-9]*\)\..*$/\1/g'`
#
# Since the job is removed from the queue BEFORE the files are returned, 
# we need to wait for them to arrive
#
cnt=30
/bin/sync
while [ ! -s PIrun$$.o$seqnum -a $cnt -gt 0 ] ; do
    cnt=`expr $cnt - 1`
    sleep 2
done
sleep 2
if [ -s PIrun$$.o$seqnum ] ; then 
    cat PIrun$$.o$seqnum >&1
    if [ $leaveoutput = 0 ] ; then 
        /bin/rm -f PIrun$$.o$seqnum
    fi
fi
if [ -s PIrun$$.e$seqnum ] ; then 
    cat PIrun$$.e$seqnum >&2
    if [ $leaveoutput = 0 ] ; then
        /bin/rm -f PIrun$$.e$seqnum
    fi
fi
if [ $leavescript = 0 ] ; then
    /bin/rm -f PIrun$$
fi
# We'd like to use qsub with redirected stdin, but it won't let us.
# And there isn't any easy way to implement "wait for batch job to finish"
#qsub -l nodes=$np -l walltime=5:00 -I <<EOF
#PIrun$$
#exit
#EOF
# that's why there's a while [ ] sleep 1 ; done loop above
