#! /bin/sh
#
# This file is used to generate a p4-style procgroup file
#
# This should be used only for pg and execer runs.  It gnerates a list from
# the machines.$arch file
#
# The original csh code made extensive use of csh arrays, which are quite
# nice for reading parts of a file into a variable.
#
# Format of machines list is lines containing either a single
#    machinename
# or
#    machinename:n
# where n is the number of available processors.
# We use : separators instead of spaces to simplify parsing of the file
#
# I have a note to make mpirun handle hostnames with and without domainnames
# This may have been fixed below; I'm leaving this marker here in case it
# comes up again.
# 
#
if [ "$argsset" = "" ] ; then
   . mpirun.args
fi
#
# Get the defaults (this is in preparation for multi-architecture versions)
# 
if [ $narch = 0 ] ; then
    narch=1
    arch1=$arch
    archlist1="$archlist"
    archlocal=$arch
    np1=$np
fi
if [ $nolocal = 1 ] ; then
    # echo don\'t run local instance
    # if the job is not to be run locally, then don't include this machine
    # in the list
    procFound=0
else
    # echo run local instance
    # Note that in the SMP cluster case, we may need to increase this
    procFound=1
fi
#
machinelist=""
archuselist=""
nprocuselist=""
curarch=1
nolocalsave=$nolocal
archlocal=$arch1
#
# The first time through the loop, we use the default machinefile, unless
# one has been specified.
# 
# We use \$ instead of $ since some systems seem to require \$ and that
# appears to be correct.
#
while [ $curarch -le $narch ] ; do
    eval arch=\$"arch$curarch"
    eval archlist=\$"archlist$curarch"
    if [ -z "$archlist" ] ; then archlist=$arch ; fi
    eval np=\$"np$curarch"
    if [ -z "$np" ] ; then 
	echo "Warning: unspecified -np value for $arch.  Assuming -np 1."
	np=1
    fi
    eval mFile=\$"machineFile$curarch"
    if [ -n "$mFile" -a -r "$mFile" ] ; then machineFile=$mFile ; fi
    #
    # Get default machine file
    if [ -z "$machineFile" ] ; then
        # If on the ANL SPx, use getjid to get the machine list...
        if [ "$machine" = "ibmspx" -a -x /usr/local/bin/getjid ] ; then
	    machineFile="/sphome/$LOGNAME/SPnodes.`/usr/local/bin/getjid`"
        else
	    machineDir=$datadir
            for march in $archlist ; do
                machineFile="${machineDir}/machines.${march}"
	        if [ -r $machineFile ] ; then break ; fi
    	    done
        fi
    fi
    #
    if [ -z "$machineFile" -o ! -s "$machineFile" -o \
	! -r "$machineFile" ] ; then
        if [ $nolocal != 0 -o $np != 1 ] ; then
            echo Cannot read $machineFile.  
	    echo Looked for files with extension $archlist in 
	    echo directory $datadir .
            exit 1
        fi
	# If we did NOT issue a message because np == 1, we need 
	# to be careful that machineFile is not empty.
	machineFile=/dev/null
    fi
    #
    # Find the machines to use (build in file).  Note that "local"
    # refers only to the first architecture.
    # 
    MPI_HOSTLeader=`expr "$MPI_HOST" : '\([^\.]*\).*'`
    #
    # If MPIRUN_RANDOM is set to yes, randomly permute the machines file.
    # This can be set by the environment variable MPIRUN_RANDOM being set
    # to yes, or by the mpirun option -randommachines.
    # The machines file created by configure will stay the same, the
    # permuted one, mpirand-${LOGNAME}$$, will be located in 
    # $TEMPDIR ( if $TMPDIR is set, then TEMPDIR=$TMPDIR, 
    # else TEMPDIR=/tmp ).  
    if [ "$MPIRUN_RANDOM" = "yes" ] ; then
        # If mpirun.rand is located in the mpirun/bin directory
        if [ -f $MPIRUN_HOME/mpirun.rand ] ; then
	    if [ $mpirun_verbose = 1 ] ; then
		echo "Permuting machines file"
	    fi
            # Check to see if TMPDIR is set, if not, use /tmp
            if [ -n "$TMPDIR" ] ; then
                TEMPDIR=$TMPDIR
            else
                TEMPDIR="/tmp"
            fi
	    export TEMPDIR
	    # Set machines random file
	    # machines_randFile is the permuted machines file
            machines_randFile="$TEMPDIR/mpirand-${LOGNAME}$$"
            if [ -x $MPIRUN_HOME/mpirun.rand ] ; then
                $MPIRUN_HOME/mpirun.rand $machineFile $machines_randFile 
	    else
	        chmod +x $MPIRUN_HOME/mpirun.rand 
                $MPIRUN_HOME/mpirun.rand $machineFile $machines_randFile 
            fi
        else
	    echo "$MPIRUN_HOME/mpirun.rand does not exist"
	    echo "Cannot permute machine file"
	fi
	# If a permuted machines file has been created, use it
	if [ -s "$machines_randFile" ] ; then
	    machineFile=$machines_randFile
	    if [ $mpirun_verbose = 1 ] ; then
                echo "using the permuted machine file $machineFile"
	    fi
	fi
    fi
    if [ $nolocal = 0 -o $curarch -gt 1 ] ; then
        machineavail=`cat $machineFile | \
		sed -e '/^#/d' -e 's/#.*^//g' | head -${np} | \
		tr '\012' ' '`
    else
        if [ $np -gt 1 -o $nolocal = 1 ] ; then
            # Remove host from the list of available machines....
	    # Thanks to Bjarne Herland for modification to 
	    # remove EXACTLY the host machine (eg., bar and not bark)
	    machineavail=`cat $machineFile | sed -e '/^#/d' -e 's/#.*^//g' | \
		grep -v "^$MPI_HOST\([ -\.:]\)" | head -${np} | tr '\012' ' '`
	else
	    machineavail=""
        fi
    fi
    #
    # One final refinement.  We could make each job start with
    # different processors with code like
    #r=`date +%S`
    #r=`expr 1 + $r / 10`
    #m1=`echo $machineavail | cut -c $r- -d ' '`
    #r=`expr $r - 1`
    #m2=`echo $machineavail | cut -c 1-$r -d ' '`
    #machineavail="$m1 $m2"
    #
    # Get the machine list for the job
    # KeepHost suggested by Marc A. Viredaz; allows the same machine
    # to be listed multiple times in the machines file.
    # This will loop around the machines list until it
    # finds enough machines (the nfound is used to detect infinite loops)
    # Allow the host to be used if it is the only system found
    #
    KeepHost=0
    loopcnt=0
    # If we have an SMP cluster, then first find the number that we can run 
    # locally.
    if [ -z "$MPI_MAX_CLUSTER_SIZE" ] ; then MPI_MAX_CLUSTER_SIZE=1 ; fi
    if [ $nolocal = 0 -a $MPI_MAX_CLUSTER_SIZE -gt 1 ] ; then
        for machineName in $machineavail ; do
	    # Split machineName into name and number
	    ntest=`expr $machineName : '.*:\([0-9]*\)'`
	    # if ntest is blank, there is no : in the entry
	    if [ -z "$ntest" ] ; then ntest=1 ; fi
	    mtest=`expr $machineName : '\(.*\):.*'`
	    if [ -z "$mtest" ] ; then mtest=$machineName ; fi
	    machineName=$mtest
	    machineNameLeader=`expr $machineName : '\([^\.]*\).*'`
	    if [ $machineName = $MPI_HOST -o \
	         \( $machineName != $machineNameLeader -a \
	           $machineNameLeader = "$MPI_HOSTLeader" \) ] ; then
		nprocmachine=1
 	        # Some systems return empty for no match; others return 0.  
		# Empty is what they should use (0 is ambiguous), but we 
		# have to be prepared for either.
		if [ -n "$ntest" -a "$ntest" != "0" ] ; then
		    nprocmachine=$ntest
		    # Limit the allowed number
		    if [ "$MPI_MAX_CLUSTER_SIZE" -lt $nprocmachine ] ; then
			nprocmachine=$MPI_MAX_CLUSTER_SIZE
		    fi
		    machineName=`expr $machineName : '\(.*\):.*'`
		    nproclocal="$nprocmachine"
		    procFound=$nproclocal
		fi
	        break;
	    fi
        done
    fi
    nleft=`expr $np - $procFound`
# This if test is needed (req #5332) for -comm=shared configure option
    if [ $nleft -lt 0 ] ; then
	nproclocal=$np
    fi
    while [ $procFound -lt $np ] ; do
        nfound=0
        for machineName in $machineavail ; do
	    # Split off the number of processors, if present
	    nprocmachine=1
	    # Some systems return empty for no match; others return 0.  Empty 
	    # is what they should use (0 is ambiguous), but we have to be 
	    # prepared for either.
	    ntest=`expr $machineName : '.*:\([0-9]*\)'`
	    if [ -n "$ntest" -a "$ntest" != "0" ] ; then
		nprocmachine=$ntest
		# Limit the allowed number
		if [ -z "$MPI_MAX_CLUSTER_SIZE" ] ; then
		    nprocmachine=1
		elif [ "$MPI_MAX_CLUSTER_SIZE" -lt $nprocmachine ] ; then
		    nprocmachine=$MPI_MAX_CLUSTER_SIZE
		fi
	        machineName=`expr $machineName : '\(.*\):.*'`
	    fi
	    # If nolocal = 0 (local machine to be included), 
	    # the first time around, we don't accept the local host.
	    machineNameLeader=`expr $machineName : '\([^\.]*\).*'`
            if [ $nolocal = 1 -o $KeepHost = 1 -o \
		 \( $machineName != "$MPI_HOST" -a \
		    $machineNameLeader != "$MPI_HOSTLeader" \
	       \) ] ; then
		# Should we convert this the ${machineName}:$nprocmachine
                # (with a care taken for nprocmachine > np - procFound).
		if [ $nprocmachine -gt $nleft ] ; then
		    nprocmachine=$nleft
		fi
		# Even better, we should use 
		# machinename:nproc:arch, so that everything is together.
	        machinelist="$machinelist $machineName"
		archuselist="$archuselist $arch"
		nprocuselist="$nprocuselist $nprocmachine"
	        procFound=`expr $procFound + $nprocmachine`
	        nfound=`expr $nfound + $nprocmachine`
		nleft=`expr $nleft - $nprocmachine`
            fi
            if [ $procFound = $np ] ; then break ; fi
	    if [ $machineName = "$MPI_HOST" ] ; then KeepHost=1 ; fi
        done
        loopcnt=`expr $loopcnt + 1`
        # After the first time, we may have found no hosts BUT now
        # allow KeepHost.  
        if [ $nfound = 0 -a $loopcnt -gt 1 ] ; then
	    echo "Could not find enough machines for architecture $arch"
	    exit 1
        fi
    done
    curarch=`expr $curarch + 1`
    procFound=0
    nolocal=1
    machineFile=""
done
nolocal=$nolocalsave
#
if [ "$argsset" != "1" ] ; then 
    echo $machinelist
    echo $archuselist
    machinehead=""
else
    for machinehead in $machinelist ; do 
	break
    done
fi
