#! /bin/sh
if [ "$MPIR_HOME" = "" ] ; then 
    MPIR_HOME=#MPIR_HOME#
fi
if [ "$MPIR_HOME" = "#""MPIR_HOME""#" ] ; then
    MPIR_HOME=`pwd`/..
fi
EXECER=#EXECER_DIR#/execer
if [ "#MPIRUN_BIN#" = "#""MPIRUN_BIN""#" ] ; then 
    MPIRUN_HOME=$MPIR_HOME/bin
else
    MPIRUN_HOME=$MPIR_HOME/#MPIRUN_BIN#
fi
MEIKO_RUN=prun
SP1_RUN=/usr/lpp/euih/eui/cotb0
jobid=$$
progname=
fake_progname=
np=1
arch=
default_arch='#DEFAULT_ARCH#'
machine='#DEFAULT_MACHINE#'
default_device='#DEFAULT_DEVICE#'
execer_machine=1
cmdLineArgs=
cmdLineArgsExecer=
cmdline=
CAC=
use_execer=0
mpirun_verbose=0
nolocal=0
leavePGFile=0
p4pgfile=
tcppgfile=
nexuspgfile=
nexusdbfile=
p4ssport=
just_testing=0
machineFile=
debugger=""
commandfile=""
rmcommandfile=""
rshcmd=#RSH_COMMAND#
mvhome=0
mvback=
narch=0
stdinfile=
stdoutfile=
stderrfile=
batch=
Show=eval
# Kind of paragon submisstion (default/mkpart/nqs)
paragon_type="default"
# paragon partition name, if any
paragon_pn=""
Machines="chameleon meiko paragon p4 sp1 ibmspx anlspx ksr sgi_mp ipsc860 inteldelta cray_t3d execer smp symm_ptx"
# Usage limits
max_time=15
max_mem=32
#
# polling_mode is for systems that can select between polling and 
# interrupt-driven operation.  Currently, only IBM POE is so supported
# (TMC CMMD has some support for this choice of mode, but we do not use it.)
polling_mode=1
#
PrintHelp() {
#
# If you change this, make the corresponding changes in README so that
# the man pages are updated.
#
cat <<EOF
mpirun [mpirun_options...] <progname> [options...]

  mpirun_options:
    -arch <architecture>
            specify the architecture (must have matching machines.<arch>
            file in ${MPIR_HOME}/bin/machines) if using the execer
    -h      This help
    -machine <machine name>
            use startup procedure for <machine name>
            Currently supported:
EOF
for machine in $Machines ; do
  echo "              $machine"
done
cat <<EOF

    -machinefile <machine-file name>
            Take the list of possible machines to run on from the
            file <machine-file name>.  This is a list of all available
            machines; use -np <np> to request a specific number of machines.
    -np <np>
            specify the number of processors to run on
    -nolocal
            don't run on the local machine (only works for 
            p4 and ch_p4 jobs)
    -stdin filename
            Use filename as the standard input for the program.  This
            is needed for programs that must be run as batch jobs, such
            as some IBM SP systems and Intel Paragons using NQS (see 
            -paragontype below).
    -t      Testing - do not actually run, just print what would be
            executed
    -v      Verbose - throw in some comments
    -dbx    Start the first process under dbx where possible
    -gdb    Start the first process under gdb where possible
             (on the Meiko, selecting either -dbx or -gdb starts prun
             under totalview instead)
    -xxgdb  Start the first process under xxgdb where possible (-xdbx
            does not work)
    -tv     Start under totalview
	          
      Special Options for NEC - CENJU-3:

    -batch  Excecute program as a batch job (using cjbr)

    -stdout filename
            Use filename as the standard output for the program.
    -stderr filename
            Use filename as the standard error  for the program.
	          
      Special Options for Nexus device:
  
    -nexuspg filename
	    Use the given Nexus startup file instead of creating one.
	    Overrides -np and -nolocal, selects -leave_pg.
    
    -nexusdb filename
	    Use the given Nexus resource database.

    Special Options for Workstation Clusters:

    -e      Use execer to start the program on workstation
            clusters
    -pg     Use a procgroup file to start the p4 programs, not execer
            (default)
    -leave_pg
            Don't delete the P4 procgroup file after running
    -p4pg filename
            Use the given p4 procgroup file instead of creating one.
            Overrides -np and -nolocal, selects -leave_pg.
    -tcppg filename
            Use the given tcp procgroup file instead of creating one.
            Overrides -np and -nolocal, selects -leave_pg.
    -p4ssport num
            Use the p4 secure server with port number num to start the
            programs.  If num is 0, use the value of the 
            environment variable MPI_P4SSPORT.  Using the server can
            speed up process startup.  If MPI_USEP4SSPORT as well as
            MPI_P4SSPORT are set, then that has the effect of giving
            mpirun the -p4ssport 0 parameters.

    Special Options for Batch Environments:

    -mvhome Move the executable to the home directory.  This 
            is needed when all file systems are not cross-mounted
            Currently only used by anlspx
    -mvback files
            Move the indicated files back to the current directory.
            Needed only when using -mvhome; has no effect otherwise.
    -maxtime min
            Maximum job run time in minutes.  Currently used only
            by anlspx.  Default value is $max_time minutes.
    -nopoll Do not use a polling-mode communication.
            Available only on IBM SPx.
    -mem value
	   This is the per node memory request (in Mbytes).  Needed for some
           CM-5s. ( Default $max_mem. )

    -cpu time
	   This is the the hard cpu limit used for some CM-5s in
	   minutes. (Default $maxtime minutes.)

    Special Options for IBM SP2:

    -cac name
            CAC for ANL scheduler.  Currently used only by anlspx.
            If not provided will choose some valid CAC.

    Special Options for Intel Paragon:

    -paragontype name
           Selects one of default, mkpart, NQS, depending on how you want
           to submit jobs to a Paragon.

    -paragonname name 
           Remote shells to name to run the job (using the -sz method) on 
           a Paragon.

    -paragonpn name
           Name of partition to run on in a Paragon (using the -pn name 
           command-line argument)

On exit, mpirun returns a status of zero unless mpirun detected a problem, in
which case it returns a non-zero status (currently, all are one, but this 
may change in the future).

Multiple architectures may be handled by giving multiple -arch and -np 
arguments.  For example, to run a program on 2 sun4s and 3 rs6000s, with 
the local machine being a sun4, use

    $0 -arch sun4 -np 2 -arch rs6000 -np 3 program

This assumes that program will run on both architectures.  If different
executables are needed, the string '%a' will be replaced with the arch name.
For example, if the programs are program.sun4 and program.rs6000, then the
command is

    $0 -arch sun4 -np 2 -arch rs6000 -np 3 program.%a

If instead the executables are in different directories; for example, 
/tmp/me/sun4 and /tmp/me/rs6000, then the command is

    $0 -arch sun4 -np 2 -arch rs6000 -np 3 /tmp/me/%a/program

It is important to specify the architecture with -arch BEFORE specifying
the number of processors.  Also, the FIRST -arch command must refer to the
processor on which the job will be started.  Specifically, if -nolocal is 
NOT specified, then the first -arch must refer to the processor from which
mpirun is running.

EOF
}

#    -xdbx   Start the first process under xdbx where possible
while [ 1 -le $# ] ; do
  arg=$1
  #echo $arg
  #echo $#
  shift
  case $arg in 
     -arch) 
     	arch=$1
	narch=`expr $narch + 1`
	eval arch$narch=$1
	shift
	;;
    -np)
	np="$1"
        nodigits=`echo $np | sed 's/^[0-9]*$//'`
        if [ "$nodigits" != "" -o $np -lt 1 ] ; then
           echo np: $np is an invalid number of processors.  Exiting.
           exit 1
	fi
	eval np$narch=$1
	shift
	;;
    -machine)
	machine="$1"
	shift
	;;
    -machinefile)
        machineFile="$1"
	eval machineFile$narch="$1"
	shift
	;;
    -device)
 	default_device="$1"
	shift
	;;
    -stdin)
	# Name of a file to use for standard input.
	stdinfile="$1"
	shift
	;;
    -stdout)
        # Name of a file to use for standard output.
        stdoutfile="$1"
        shift
        ;;
    -stderr)
        # Name of a file to use for standard error.
        stderrfile="$1"
        shift
        ;;
    -nolocal)
        nolocal=1
	;;	
    -h)
	if [ "$hasprinthelp" = 1 ] ; then
	   PrintHelp
        fi
      	exit 1
	;;	
    -e)
      	use_execer=1
	;;
    -pg)
      	use_execer=0
      	;;
    -leave_pg|-keep_pg)
      	leavePGFile=1
	;;
    -batch)
        batch=1
        ;;
    -nexuspg)
	if [ $np -gt 1 ] ; then
	    echo "-nexuspg and -np are mutally exlusive; -np $np being ignored."
	    np=1
	fi
	nexuspgfile="$1"
	shift
	leavePGFile=1
	;;
    -nexusdb)
	nexusdbfile="$1"
	shift
	;;
    -p4pg)
      	if [ $np -gt  1 ] ; then
	    echo "-p4pg and -np are mutually exclusive; -np $np being ignored."
            np=1
	fi
	p4pgfile="$1"
	shift
      	leavePGFile=1
	;;
    -tcppg)
      	if [ $np -gt  1 ] ; then
	    echo "-tcppg and -np are mutually exclusive; -np $np being ignored."
            np=1
	fi
	tcppgfile="$1"
	shift
      	leavePGFile=1
	;;
    -p4ssport)
      	p4ssport="$1"
	shift
        if [ $p4ssport = "0" -a "$MPI_P4SSPORT" != "" ] ; then
	    p4ssport=$MPI_P4SSPORT
      	fi
	;;
    -paragontype)
	paragon_type="$1"
	shift
	;;
    -paragonname)
	paragon_type=crossrun
	pgonname="$1"
	shift
	;;
    -paragonpn)
	paragon_pn="-pn $1"
	shift
	;;
    -v)
       	mpirun_verbose=1
	;;
    -t)
      	just_testing=1
	Show=echo
	;;
    -tv|-totalview)
        debugger="totalview"
	;;
    -dbx)
      	debugger="dbx"
 	commandfile="-sr %f"
	# For some systems, commandfile="-c %f" will ALMOST work (it
        # will leave the file behind)
      	;;
    -gdb)
      	debugger="gdb"
      	commandfile="-x %f"
	;;
    -xxgdb)
      	debugger="xxgdb"
      	commandfile="--command=%f"
	;;
    -nopoll)
	polling_mode=0
	;;
    -maxtime|=cpu)
        max_time="$1"
	shift
	;;
    -mem)
	mem="$1"
	if [ $mem -gt $max_mem ];then
echo "mem: Memory requested ($mem MB/node) greater than maximum available ($max_mem MB/node).  Exiting."
	    exit 1
	fi
	shift
	;;
    -mvhome)
        mvhome=1
	;;
    -mvback)
	mvback="$1"
	shift
	;;
    -cac)
        CAC="$1"
	shift
	;;
    -echo)
	# this is for debugging
	set -x
	echo "default_arch   = $default_arch"
	echo "default_device = $default_device"
	echo "machine	     = $machine"
	;;
    -usage|-help|-\?)
      	# Accept these for help until the program name is provided.
      	if [ "$progname" = "" ] ; then
	   if [ "$hasprinthelp" = "1" ] ; then
	      PrintHelp
           fi
	   exit 1
      	fi
	;;
    *)
	# The first unrecognized argument is assumed to be the name of
      	# the program, but only if it is executable
	proginstance=`echo $arg | sed "s/%a/$arch/g"`
      	if [ "$progname" = "" -a "$fake_progname" = "" -a \
		! -x "$proginstance" ] ; then
	    fake_progname="$arg"
      	elif [ "$progname" = "" -a -x "$proginstance" ] ; then 
	    progname="$arg"
            # any following unrecognized arguments are assumed to be arguments
            # to be passed to the program, as well as all future args
	    # We can't use use_execer here to decide how to store the
	    # arguments, since we may later change it (see the switch on arch)
	    # Thus we store the args in tow forms
	    while [ 1 -le $# ] ; do
               cmdLineArgsExecer="$cmdLineArgsExecer -arg=$1"
               cmdLineArgs="$cmdLineArgs $1"
	       shift
            done
         else
	    echo "Warning: Command line arguments for program should be given"
            echo "after the program name.  Assuming that $arg is a"
            echo "command line argument for the program."
            cmdLineArgsExecer="$cmdLineArgsExecer -arg=$arg"
            cmdLineArgs="$cmdLineArgs $arg"
         fi
      ;;
  esac
done
if [ "$machine" = "#""DEFAULT_MACHINE""#" ] ; then
    machine=""
fi
if [ "$default_arch" = "#""DEFAULT_ARCH""#" ] ; then
    default_arch=sun4
fi
if [ "$default_device" = "#""DEFAULT_DEVICE""#" ] ; then
    default_device=ch_p4
fi

# We need at least the program name

if [ "$progname" = "" ] ; then
  echo "Missing: program name"
  if [ "$fake_progname" != "" ] ; then
      echo "Program $fake_progname either does not exist, is not "
      echo "executable, or is an erroneous argument to mpirun."
  fi
  exit 1
fi
# 
# Warning: Some systems (OSF 1 at least) have a command called dbx
# with different command-line arguments.  In particular, it needs
# -c here (and that does not implement the remove part of the -sr).
#
# Turns out that this doesn't matter so much, because the dbx is terribly,
# terribly crippled.  
#
if [ "$debugger" = "dbx" -a "$arch" = "alpha" ] ; then
    commandfile="-c"
    rmcommandfile="/bin/rm -f"
fi
#
# Fill out relative program pathnames
#
# Get the current directory
# Note that PWD may NOT be `pwd` if an explicit "cd", outside of the csh
# shell, is used.  A downside of this is that PWD may have a more reasonable
# version of the path.  To try and fix this, we create a test file and see
# if it is accessible from PWD and pwd; if so, we take the PWD value
# 
# Why can't automounters work!???!!
#
# For those unfamiliar with the problem, the automounter creates
# paths like /tmp_mnt/.... .  But if you try to open a file with that path, 
# and the ACTUAL path has not been accessed before, the automounter fails
# to mount the partition.  In otherwords, the automounter is so mind-bogglingly
# stupid as to not recognize its OWN MOUNTS.  Network computing indeed.
#
# Note that PWD may ALSO be damaged, so we need to sed PWD as well...
#
# We should move this to the point in the code where it needs to know a
# particular filename, since this will fail if the directory is not
# writable by the user.
# 
if [ -n "#AUTOMOUNTFIX#" ] ; then
    PWDtest=`pwd | #AUTOMOUNTFIX#`
    if [ ! -d $PWDtest ] ; then
        PWDtest=`pwd`
    fi
    if [ -n "$PWD" ] ; then
	# Use a process-specific filename
        /bin/rm -f $PWDtest/.mpirtmp$$ $PWD/.mpirtmp$$
        if `eval "echo test > $PWD/.mpirtmp$$" 2>/dev/null` ; then
            if [ ! -s $PWDtest/.mpirtmp$$ ] ; then
	        /bin/rm -f $PWD/.mpirtmp$$
                PWD=$PWDtest
	    fi
	else
	    # Current directory is not writable.  Hope for the best
            PWD=$PWDtest        
        fi
        /bin/rm -f $PWDtest/.mpirtmp$$ $PWD/.mpirtmp$$
    else 
        PWD=$PWDtest
    fi
else
    PWD=`pwd`
fi
#
if [ -n "$PWD" ] ; then
    PWD_TRIAL=$PWD
else
    PWD_TRIAL=$PWDtest
fi
# The test for PWD_TRIAL usable is within the p4/execer/tcp test (since that
# is where it matters)
#
tail=`echo $progname | sed 's/\/.*//'` 
if  [ "$tail" = "" ] ; then
  #echo Absolute path, don't need to change anything
  true
else
  #echo Relative path
  progname="$PWD_TRIAL/$progname"
fi

#
# Get the architecture and the device
# Note that some architecture/device combinations have their own, private
# devices
#
if [ "$arch" = "" ] ; then
  arch=`$MPIR_HOME/bin/tarch`
fi

archlist="$arch"
case $arch in 
    HP-UX) archlist="hpux hp9000s700 hp9000s735 HP-UX" 
	;;
    IRIX|sgi|SGI) archlist="IRIX sgi"
	;;
    i386|symm_ptx) archlist="i386 symm_ptx"
	;;
    IP19|IP21) archlist="IRIX IP19 IP21"
	arch="IRIX"
	# Was sgi_mp instead of IRIX
	;;
    ipsc860|i860|ipsc)
      	arch="ipsc860"
      	archlist="ipsc860 i860 ipsc"
	;;
    freebsd|386BSD)
	arch="386BSD"
	archlist="386BSD freebsd"
	;;
    hpux|hp9000s700)
	arch="hpux"
	archlist="hpux hp9000s700"
	;;
esac

#
# Try to find the machine
if [ "$machine" = "" ] ; then 
  machine=$arch
  case $arch in 
    sun4|solaris)
      # This device should only exits on sun4s that are actually
      # MEIKO machines.
      if [ -r /dev/elan ] ; then
	machine="meiko"
      elif [ "$default_device" = "chameleon" ] ; then
	machine="chameleon"
      elif [ "$default_device" = "ch_p4" -a $use_execer = 0 ] ; then
	machine="p4"
      elif [ "$default_device" = "ch_tcp" -a $use_execer = 0 ] ; then
	machine="ch_tcp"
      elif [ "$default_device" = "ch_nexus" ] ; then
	machine="nexus"
      else
        machine="execer"
      fi
      ;;
    
    rs6000)
      # This only works on the ANL sp system
      # -e for "exists" isn't available on all systems
      echo `hostname` | grep spnode > /dev/null
      if [ $? = 0 ] ; then
        # This is the correct version to use once we're on a node
	machine="ibmspx"
      elif [ "$default_device" = "ch_nexus" ] ; then
        # We need this before the anlspx check, because the Nexus device
        # can be used on the ANL SP, as well, but the mpirun script for
        # it does not use Nexus by default.
	machine="nexus"
      elif [ -d /etc/FRAMES -a -d /mcs ] ; then
        machine="anlspx"
      elif [ "$default_device" = "chameleon" ] ; then
	machine="chameleon"
      elif [ "$default_device" = "ch_p4" -a $use_execer = 0 ] ; then
	machine="p4"
      elif [ "$default_device" = "ch_p4" -a $use_execer = 1 ] ; then
	machine="execer"
      elif [ "$default_device" = "ch_tcp" -a $use_execer = 0 ] ; then
	machine="ch_tcp"
      elif [ -d /usr/lpp/euih ] ; then
        machine="sp1"
      elif [ -x /usr/bin/poe -o -x /usr/lpp/poe/bin/poe ] ; then
        # should work for other users
        machine="ibmspx"
      else
	 echo "Can't determine the type of the machine this is."
         echo "Specifiy it with -machine <machine_type>."
      fi
      ;;
    ipsc860|i860|ipsc)
        machine="ipsc860"
	;;
    symm_ptx|i386)
        machine="symm_ptx"
	;;
    *)
      if [ "$default_device" = "chameleon" ] ; then
	machine="chameleon"
      elif [ "$default_device" = "ch_p4" -a $use_execer = 0 ] ; then
	machine="p4"
      elif [ "$default_device" = "ch_tcp" -a $use_execer = 0 ] ; then
	machine="ch_tcp"
      elif [ "$default_device" = "ch_nexus" ] ; then
	machine="nexus"
      else
        machine="execer"
      fi
      #echo "Can't determine the type of the machine this is."
      #echo "it with -machine <machine_type>."
      ;;
  esac
fi

# Get value of host if necessary
if [ "$machine" = p4 -o "$machine" = execer -o "$machine" = ch_tcp -o \
	"$machine" = sgi_mp -o "$machine" = ch_p4 -o \
	"$machine" = ch_p4-2 -o "$machine" = ch_nexus -o \
        "$machine" = nexus ] ; then
  if [ "$MPI_HOST" = "" ] ;  then
    MPI_HOST=""
    if [ "$arch" = "ipsc860" ] ; then
	MPI_HOST=`hostname`
    else
        # Which is not supported in many sh's
#	HOSTNAMEPGM="`which hostname | head -1`"
#        if [ -x "$HOSTNAMEPGM" ] ; then MPI_HOST="`hostname`" ; fi
	MPI_HOST=`hostname`
    fi
    # Note that uname -n may not produce a usable hostname.  Any suggestions?
    if [ "$MPI_HOST" = "" ] ; then MPI_HOST="`uname -n`" ; fi
  fi
  #
  # We don't need this for most systems; for the moment, we'll suppress the
  # message about the automounter unless we're using ch_p4 or ch_tcp. 
  #
  if [ "$PWD_TRIAL" != "" ] ; then
    PWD_TRIAL=`pwd | sed -e 's%/tmp_mnt/%/%g'`
    if [ ! -d $PWD_TRIAL ] ; then 
        echo "Warning: your default path uses the automounter; this may"
        echo "cause some problems if you use other NFS-connected systems."
        PWD_TRIAL=`pwd`
    fi
  fi
fi


if [ $mpirun_verbose = 1 ] ; then
  cnt=1
  if [ $narch -gt 1 ] ; then
      while [ $cnt -le $narch ] ; do
        eval mp=$"np$cnt"
        eval archm=$"arch$cnt"
	proginstance=`echo $progname | sed "s/%a/$archm/"`
	echo "running $proginstance on $mp $archm $machine processors"
	cnt=`expr $cnt + 1`
      done
  else
      echo "running $progname on $np $arch $machine processors"
  fi
fi

