\documentstyle{article}
\topmargin -0.13 true in
\headheight 0 true in
\headsep 0 true in
\topskip 0 true in
\footheight 0.2 true in
\footskip 0.2 true in
\textheight 9.7 true in
\textwidth 6.75 true in
\oddsidemargin 0 true in
\evensidemargin 0 true in
\begin{document}
\bibliographystyle{ieeetr}
\title{The Fortran/PARMACS Array Communication Library}
\author{James Allwright \\ University of Southampton}
\maketitle
\section{Introduction}
This document describes an implementation of the Array Communication Library
in Fortran using PARMACS \cite{parmacs} as the underlying message-passing
system. The library was created by porting the original Occam implementation
of the library into Fortran and PARMACS.
This document does not attempt to explain the use of the library; for
this, the reader should consult the documentation for the original Occam
libraries \cite{aclref,aclintro,aclbackground}.

The new library attempts to preserve the
structure and format of the old library. However, fundamental differences
between Occam and Fortran, and between Channels and the PARMACS messaging
system, meant that some changes had to be made :
\begin{itemize}
\item Renaming of Routines. Fortran 77 requires identifiers to be no more
than 6 characters long and does not allow them to contain the ``.''
character. Almost all of the original Occam names have been shortened and
the new names are given here.
\item Variable Length Arrays. Occam allows arrays of unknown size to be
passed into a procedure. Their size can then be determined using the
{\tt SIZE} operator. Fortran provides no similar facility, so the length
of an array needs to be passed into the Fortran together with the array.
In general, the size of an array appears immediately after the array
itself in the parameter list. However, for some routines the size is not
passed over in this manner as it appears elsewhere in the parameter list.
\item Arrays of {\tt BYTE}. In the original Occam routines, the primitive
communication routines pass arrays of bytes. Other types can be passed by
{\tt RETYPEing} them as byte arrays. This is not possible with PARMACS, so
the primitive communication routines pass arrays of type {\tt INTEGER}
instead.
\item Lack of Dynamic Spawning. PARMACS only allows the host to spawn a
set of subordinate processes. These subordinate processes cannot then
spawn other processes (as they would be able to in VCR). Hence these
libraries have not inherited the dynamic process creation features of the
original.
\item Communication mechanism. The communication model of Occam is
somewhat different to that of PARMACS. This is reflected in the way
distributed channels have been implemented; in the original Occam they
were uni-directional, whereas in this version they are bi-directional.
\item Bitwise logical operations. Fortran 77 does not support bitwise
logical operations, which Application library I makes use of. This
library assumes the existence of intrinsic functions IEOR, IAND and IOR,
which will have to be provided if they do not exist in the target
Fortran.
\end{itemize}

There are 4 parts to the ported libraries. Two of these are application
level libraries, intended to be used directly by the applications
programmer, and two are base level libraries intended mainly to support
the application libraries, though they are available to applications
programmer.
\section{Application Library I : aciapp.u}
This library performs synchronization, data re-arrangement (broadcasts and
permuations) and partial and global operations (maximum, minimum, sum,
product, boolean-or and boolean-and) This library is smaller than the
original Occam library as the prefix and global operations are provided
only for integer data.

\begin{tabular}{||l|l||} \hline
Occam Procedure/Function & Fortran Subroutine/Function \\ \hline \hline
SYNCH & SYNCH \\ \hline
BROADCAST & BCAST \\ \hline
BROADCAST.EL & BCSTEL \\ \hline
BROADCAST.ARR & BCSTAR \\ \hline
PERM.DS & PDS \\ \hline
PERM.SHIFT & PSHFT \\ \hline
PERM.REFLECT & PREFL \\ \hline
PERM.XOR & PXOR \\ \hline
PERM.SWAP & PSWAP \\ \hline
PERM.DS.IN & PDSI \\ \hline
PERM.SHIFT.IN & PSHFTI \\ \hline
PERM.REFLECT.IN & PREFLI \\ \hline
PERM.XOR.IN & PXORI \\ \hline
PERM.SWAP.IN & PSWAPI \\ \hline
PREFIX.INT.SUM & INSUM \\ \hline
PREFIX.INT.PROD & INPRD \\ \hline
PREFIX.INT.MAX & INMAX \\ \hline
PREFIX.INT.MIN & INMIN \\ \hline
PREFIX.BOOL.AND & PFXAND \\ \hline
PREFIX.BOOL.OR & PFXOR \\ \hline
GLOB.INT.SUM & GINSUM \\ \hline
GLOB.INT.PROD & GINPRD \\ \hline
GLOB.INT.MAX & GINMAX \\ \hline
GLOB.INT.MIN & GINMIN \\ \hline
GLOB.BOOL.AND & GAND \\ \hline
GLOB.BOOL.OR & GOR \\ \hline
\end{tabular}

\begin{verbatim}
SUBROUTINE SYNCH (rank, rs)
INTEGER rs, rank(0:rs-1)
\end{verbatim}
- synchronises all the processes in the index set supplied in the array
{\tt rank}. This doesn't terminate on any of the sites connected by rank
until it has been called on all of them.
\begin{verbatim}
      SUBROUTINE BCAST (x, result, rlen, data, dlen)
      IMPLICIT NONE
      INTEGER x, dlen, data(0:dlen-1), rlen, result(0:dlen-1, 0:rlen-1)
\end{verbatim}
- converts a distribution into a global array. The algorithm is
fan-in to origin, followed by broadcast.
\begin{verbatim}
SUBROUTINE BCSTEL (x, result, rlen, data, dlen, src)
INTEGER x, rlen, dlen, src, result(0:rlen-1), data(0:dlen-1)
\end{verbatim}
\begin{verbatim}
      SUBROUTINE BCSTAR (x, result, rlen, tlen, segsize, data, dlen)
      INTEGER x, rlen, tlen, segsize, dlen
      INTEGER result(0:tlen-1, 0:rlen-1), data(0:tlen-1, 0:dlen-1)
\end{verbatim}
- converts a distribution of arrays into a global array. The algorithm is
fan-in to origin, followed by broadcast.
\begin{verbatim}
SUBROUTINE PDS (x, result, rlen, data, dlen, src, dst)
INTEGER x, rlen, dlen, result(0:rlen-1), data(0:dlen-1)
\end{verbatim}
- performs a general permutation. The process sends its piece of data
to {\tt dst} and receives a new piece of data from {\tt src}.
\begin{verbatim}
SUBROUTINE PSHFT (x, result, rlen, data, dlen, shift)
INTEGER x, rlen, dlen, shift, result(0:rlen-1), data(0:dlen-1)
\end{verbatim}
- shifts data cyclically in dimension given by index {\tt x}.
\begin{verbatim}
SUBROUTINE PREFL (x, result, rlen, data, dlen, a)
INTEGER x, rlen, dlen, a, result(0:rlen-1), data(0:dlen-1)
\end{verbatim}
- performs a reflection in the dimension given by index {\tt x}.
\begin{verbatim}
SUBROUTINE PXOR (x, result, rlen, data, dlen, p)
INTEGER x, rlen, dlen, p, result(0:rlen-1), data(0:dlen-1)
\end{verbatim}
- performs a generalised butterfly permutation.
\begin{verbatim}
      SUBROUTINE PSWAP (x, result, rlen, data, dlen, i1, i2)
      IMPLICIT NONE
      INTEGER x, rlen, dlen, i1, i2, result(0:rlen-1), data(0:dlen-1)
\end{verbatim}
- exchanges data between pairs of processes.
\begin{verbatim}
SUBROUTINE PDSI (x, data, dlen, src, dst)
INTEGER x, dlen, src, dst, data(0:dlen-1)
\end{verbatim}
- permutes {\tt data} array ``in situ''; analagous to {\tt PDS}.
\begin{verbatim}
SUBROUTINE PSHFTI (x, data, dlen, shift)
INTEGER x, dlen, shift, data(0:dlen-1)
\end{verbatim}
- permutes {\tt data} array ``in situ''; analagous to {\tt PSHFT}.
\begin{verbatim}
SUBROUTINE PREFLI (x, data, dlen, a)
INTEGER x, dlen, a, data(0:dlen-1)
\end{verbatim}
- permutes {\tt data} array ``in situ''; analagous to {\tt PREFL}.
\begin{verbatim}
SUBROUTINE PXORI (x, data, dlen, p)
INTEGER x, dlen, p, data(0:dlen-1)
\end{verbatim}
- permutes {\tt data} array ``in situ''; analagous to {\tt PXOR}.
\begin{verbatim}
SUBROUTINE PSWAPI (x, data, dlen, i1, i2)
INTEGER x, dlen, i1, i2, data(0:dlen-1)
\end{verbatim}
- permutes {\tt data} array ``in situ''; analagous to {\tt PSWAP}.
\begin{verbatim}
SUBROUTINE INSUM (x, result, data)
INTEGER x, result, data
\end{verbatim}
- calculates a partial sum of integer data in dimension {\tt x}.
\begin{verbatim}
SUBROUTINE INPRD (x, result, data)
INTEGER x, result, data
\end{verbatim}
- calculates a partial product of integer data in dimension {\tt x}.
\begin{verbatim}
SUBROUTINE INMAX (x, result, restag, data, datatg)
INTEGER x, result, restag, data, datatg
\end{verbatim}
- the partial maximum of integer data in dimension {\tt x}.
\begin{verbatim}
SUBROUTINE INMIN (x, result, restag, data, datatg)
INTEGER x, result, restag, data, datatg
\end{verbatim}
- the partial minimum of integer data in dimension {\tt x}.
\begin{verbatim}
SUBROUTINE PFXAND (x, result, data)
INTEGER x
LOGICAL result, data
\end{verbatim}
- performs a partial {\tt AND} of the boolean data in dimension {\tt x}.
\begin{verbatim}
SUBROUTINE PFXOR (x, result, data)
INTEGER x
LOGICAL result, data
\end{verbatim}
- performs a  partial {\tt OR} of the boolean data in dimension {\tt x}.
\begin{verbatim}
INTEGER FUNCTION GINSUM (x, data)
INTEGER x, data
\end{verbatim}
- calculates a global sum of integer data in dimension {\tt x}.
\begin{verbatim}
INTEGER FUNCTION GINPRD (x, data)
INTEGER x, data
\end{verbatim}
- calculates a global product of integer data in dimension {\tt x}.
\begin{verbatim}
SUBROUTINE GINMAX (res, restg, x, data, datatg)
INTEGER x, data, datatg, res, restg
\end{verbatim}
- calculates a global maximum of integer data in dimension {\tt x}.
It returns res and restg as results.
\begin{verbatim}
SUBROUTINE GINMIN (res, restg, x, data, datatg)
INTEGER x, data, datatg, res, restg
\end{verbatim}
- calculates a global minimum of integer data in dimension {\tt x}.
It returns res and restg as results.
\begin{verbatim}
LOGICAL FUNCTION GAND (x, data)
INTEGER x
LOGICAL data
\end{verbatim}
- calculates a global {\tt AND} of boolean data in dimension {\tt x}.
\begin{verbatim}
LOGICAL FUNCTION GOR (x, data)
INTEGER x
LOGICAL data
\end{verbatim}
- calculates a global {\tt OR} of boolean data in dimension {\tt x}.
\section{Library Accapp - Application comms library}
\subsection{Introduction}
This library supports communication between arrays of processes. The first
parameter {\tt chan} of each routine is the channel which is to be used for
communication. The last parameter {\tt tag} is a single integer value that
is passed over with each message. This can be used for signalling some piece
of information or for error-checking, much like tagged protocols in Occam.
If {\tt tag} is set to the special value {\tt tuntagged} $ = -1$, then no
tag is sent.

The operation of the input and output routines is not immediately obvious.
The routine {\tt OUTPUT} is called on every process in an array, but causes
only a single piece of data to be send over the channel. Thus each process
should output the same data and these are merged together. The corresponding
{\tt INPUT} routine broadcasts a single piece of data received on the
channel to all processes.

The array input and output routines provide a way of performing the
communication so that different processes input or output different things.
These routines are supplied with one or more index structures. Processes
with the same co-ordinates within those indexes behave identically (e.g.
outputting a 1D array to a 2D process array causes the data to be broadcast
across one of the dimension). However, processes at different co-ordinates
in the indexes can send different data. This data is sent over the channel
in an ordered sequence. Likewise, processes at different co-ordinates in
the indexes can input different components of an array being sent over a
channel.

As there are so many of these routines, no table is provided to show the
correspondence between Occam and Fortran routines. However, they follow a
straightforward pattern and the naming scheme is systemmatic and relatively
simple. Where ``INPUT.'' appeared in the original name it is replaced by
``IN''. Likewise, ``OUTPUT.'' is replaced by ``OU'', ``.EL'' is replaced by
``EL'' and ``.ARR'' is replaced by ``AR''. The routines are not all
individually described as the behaviour of most of them should be apparent
by comparison with other routines.

\subsection{Simple I/O}
\begin{verbatim}
SUBROUTINE INPUT (chan, result, rlen, tag)
INTEGER chan, rlen, tag, result(0:rlen-1)
\end{verbatim}
- inputs global variable {\tt result}.
\begin{verbatim}
SUBROUTINE INTAG (chan, tag)
INTEGER chan, tag
\end{verbatim}
- inputs a tag.
\begin{verbatim}
SUBROUTINE OUTPUT (chan, data, dlen, tag)
INTEGER chan, dlen, tag, data(0:dlen-1)
\end{verbatim}
- outputs global variable {\tt data}.
\begin{verbatim}
SUBROUTINE OUTAG (chan, tag)
INTEGER chan, tag
\end{verbatim}
- outputs a tag.
\subsection{Process-oriented I/O}
\begin{verbatim}
SUBROUTINE INDD (chan, rank, rs, result, rlen, tag)
INTEGER chan, rs, rlen, tag, rank(0:rs-1), result(0:rlen-1)
\end{verbatim}
- inputs a whole distribution over the set of indexes contained in the
array {\tt rank}. Each process inputs a single item, which is the array
{\tt result}.
\begin{verbatim}
SUBROUTINE IN1D (chan, x, result, rlen, tag)
INTEGER chan, x, rlen, tag, result(0:rlen-1)
\end{verbatim}
- inputs a whole distribution over the index {\tt x}. Each process inputs
a single item, which is the array {\tt result}.
\begin{verbatim}
SUBROUTINE IN2D (chan, x, y, result, rlen, tag)
INTEGER chan, x, y, rlen, tag, result(0:rlen-1)
\end{verbatim}
\begin{verbatim}
SUBROUTINE IN3D (chan, x, y, z, result, rlen, tag)
INTEGER chan, x, y, z, rlen, tag, result(0:rlen-1)
\end{verbatim}
\begin{verbatim}
SUBROUTINE OUDD (chan, rank, rs, data, dlen, tag)
INTEGER chan, rs, dlen, tag, rank(0:rs-1), data(0:dlen-1)
\end{verbatim}
- outputs a whole distribution over the set of indexes contained in the
array {\tt rank}. Each process outputs a single item, which is the array
{\tt result}.
\begin{verbatim}
SUBROUTINE OU1D (chan, x, data, dlen, tag)
INTEGER chan, x, dlen, tag, data(0:dlen-1)
\end{verbatim}
\begin{verbatim}
SUBROUTINE OU2D (chan, x, y, data, dlen, tag)
INTEGER chan, x, y, dlen, tag, data(0:dlen-1)
\end{verbatim}
\begin{verbatim}
SUBROUTINE OU3D (chan, x, y, z, data, dlen, tag)
INTEGER chan, x, y, z, dlen, tag, data(0:dlen-1)
\end{verbatim}
\subsection{Data-oriented I/O}
\begin{verbatim}
SUBROUTINE INDDAR (chan, rank, rs, ssizes, strides, result, unitlen, tag)
INTEGER chan, rs, unitlen, tag, result(0:*)
INTEGER rank(0:rs-1), ssizes(0:rs-1), strides(0:rs-1)
\end{verbatim}
- inputs an array distributed in the indexes specified in array
{\tt rank}. Each element in the array contains {\tt unitlen} integers and
they are read into the array {\tt result}. The array {\tt strides} specifies
the change of subscript into {\tt result} corresponding to incrementing the
co-ordinate in each dimension. The array {\tt ssizes} specifies the number
of elements that the calling process will read in each dimension.
Thus the total number of items read is the product of the values in
{\tt sizes}. Supplying the extra array {\tt strides} means that the array
{\tt result} need not be declared to exactly fit the sata it will contain.
This is useful when the size of an array is not an exact multiple of the
size of the index over which it will be distributed. Also, this routine may
be used to input partial arrays by setting up {\tt ssizes} appropriately.
\begin{verbatim}
SUBROUTINE IN1DAR (chan, x, ssx, result, unitlen, ddx, tag)
INTEGER chan, x, ssx, unitlen, ddx, tag, result(0:unitlen-1, 0:ddx-1)
\end{verbatim}
- inputs an array distributed over index {\tt x}. On the
calling process, this routine will input {\tt ssx} items, each {\tt unitlen}
integers long, into the first {\tt ssx} elements of {\tt result}.
\begin{verbatim}
SUBROUTINE IN2DAR (chan, y, x, ssy, ssx, result, unitlen, ddy, ddx, tag)
INTEGER chan, x, y, ssx, ssy, unitlen, ddx, ddy, tag
INTEGER result(0:unitlen-1, 0:ddy-1, 0:ddx-1)
\end{verbatim}
- analagous to IN1DAR
\begin{verbatim}
SUBROUTINE IN3DAR (chan, x, y, z, ssx, ssy, ssz,
                   result, unitlen, ddx, ddy, ddz, tag)
INTEGER chan, x, y, z, ssx, ssy, ssz, unitlen, tag
INTEGER ddx, ddy, ddz, result(0:unitlen-1, 0:ddx-1, 0:ddy-1, 0:ddz-1)
\end{verbatim}
\begin{verbatim}
SUBROUTINE OUDDAR (chan, rank, rs, ssizes, strides, data, unitlen, tag)
INTEGER chan, rs, unitlen, tag, rank(0:rs-1), ssizes(0:rs-1)
INTEGER strides(0:rs-1), data(0:*)
\end{verbatim}
\begin{verbatim}
SUBROUTINE OU1DAR (chan, x, ssx, data, unitlen, ddx, tag)
INTEGER chan, x, ssx, unitlen, ddx, tag, data(0:unitlen-1, 0:ddx-1)
\end{verbatim}
\begin{verbatim}
SUBROUTINE OU2DAR (chan, y, x, ssy, ssx, data, unitlen, ddy, ddx, tag)
INTEGER chan, x, y, ssx, ssy, unitlen, ddx, ddy, tag
INTEGER data(0:unitlen-1, 0:ddy-1, 0:ddx-1)
\end{verbatim}
\begin{verbatim}
SUBROUTINE OU3DAR (chan, x, y, z, ssx, ssy, ssz,
                   data, unitlen, ddx, ddy, ddz, tag)
INTEGER chan, x, y, z, ssx, ssy, ssz, unitlen, tag
INTEGER ddx, ddy, ddz, data(0:unitlen-1, 0:ddx-1, 0:ddy-1, 0:ddz-1)
\end{verbatim}
\subsection{I/O for specific data elements}
\begin{verbatim}
SUBROUTINE INDDEL (chan, rank, rs, result, rlen, dsts, dlen, tag)
INTEGER chan, rs, rlen, dlen, tag
INTEGER rank(0:rs-1), result(0:rlen-1), dsts(0:dlen-1)
\end{verbatim}
- inputs into the array {\tt result} at the element given by the
co-ordinates in array {\tt dsts} within the indexes in {\tt rank}.
The array {\tt dsts} should be the same for every process calling it.
\begin{verbatim}
SUBROUTINE IN1DEL (chan, x, result, rlen, dstx, tag)
INTEGER chan, x, rlen, dstx, tag, result(0:rlen-1)
\end{verbatim}
- inputs an element at {\tt dstx} of index {\tt x}.
\begin{verbatim}
SUBROUTINE IN2DEL (chan, x, y, result, rlen, dstx, dsty, tag)
INTEGER chan, x, y, rlen, dstx, dsty, tag, result(0:rlen-1), table(0:1)
\end{verbatim}
- analagous to IN1DEL
\begin{verbatim}
SUBROUTINE IN3DEL (chan, x, y, z, result, rlen, dstx, dsty, dstz, tag)
INTEGER chan, x, y, z, rlen, dstx, dsty, dstz, tag
INTEGER result(0:rlen-1)
\end{verbatim}
- inputs element at {\tt (dstx, dsty, dstz)} of a three-dimensional
distribution over {\tt x, y, z}.
\begin{verbatim}
SUBROUTINE OUDDEL (chan, rank, rs, data, dlen, srcs, slen, tag)
INTEGER chan, rs, dlen, slen, tag
INTEGER rank(0:rs-1), data(0:dlen-1), srcs(0:slen-1)
\end{verbatim}
- outputs from the array {\tt data} on the process at
co-ordinates {\tt dsts} within the indexes in {\tt rank}.
The array {\tt dsts} should be the same for every process calling it.
\begin{verbatim}
SUBROUTINE OU1DEL (chan, x, data, dlen, srcx, tag)
INTEGER chan, x, dlen, srcx, tag, data(0:dlen-1)
\end{verbatim}
\begin{verbatim}
SUBROUTINE OU2DEL (chan, x, y, data, dlen, srcx, srcy, tag)
INTEGER chan, x, y, dlen, srcx, srcy, tag, data(0:dlen-1)
\end{verbatim}
\begin{verbatim}
SUBROUTINE OU3DEL (chan, x, y, z, data, dlen, srcx, srcy, srcz, tag)
INTEGER chan, x, y, z, dlen, srcx, srcy, srcz, tag, data(0:dlen-1)
\end{verbatim}
\section{Base Library I : acibas.u}
This is a low-level library of index operations. They are provided mainly
for use by the higher level libraries. However, the application programmer
is likely to want to use the functions {\tt Isize} and {\tt Icoord}
directly.

\begin{tabular}{||l|l||} \hline
Occam Procedure/Function & Fortran Subroutine/Function \\ \hline \hline
INDEX.define & Idefn \\ \hline
INDEX.desc & Idesc \\ \hline
INDEX.size & Isize \\ \hline
INDEX.coord & Icoord \\ \hline
INDEX.in & Iin \\ \hline
INDEX.in (asynchronous) & IAin \\ \hline
INDEX.out & Iout \\ \hline
INDEX.out (asynchronous) & IAout \\ \hline
INDEX.broadcast & Ibroad \\ \hline
INDEX.merge & Imerge \\ \hline
\end{tabular}

\begin{verbatim}
SUBROUTINE Idefn (index, pids, desc, size, coord)
INTEGER index, desc, size, coord, pids(0:size-1)
\end{verbatim}
Define an index structure.
\begin{verbatim}
INTEGER FUNCTION Idesc (index)
INTEGER index
\end{verbatim}
- returns the descriptor of the index. This is a number uniquely
specifying the index, which will be between 0 and one less than the number
of index structures.
\begin{verbatim}
INTEGER FUNCTION Isize (index)
INTEGER index
\end{verbatim}
- returns the size of the index.
\begin{verbatim}
INTEGER FUNCTION Icoord (index)
INTEGER index
\end{verbatim}
- returns the co-ordinate of the process within the given index. This will
be an integer between 0 and one less than the size of the index.
\begin{verbatim}
SUBROUTINE Iin (index, data, dlen, src)
INTEGER index, dlen, data(0:dlen-1), src
\end{verbatim}
- receives the array {\tt data} from the process at co-ordinate {\tt src}
in the specified index (and at the same co-ordinate as the calling process
in all other indexes).
\begin{verbatim}
SUBROUTINE IAin (index, data, dlen, src)
INTEGER index, dlen, data(0:dlen-1), src
\end{verbatim}
- an asynchronous version of {\tt Iin}.
\begin{verbatim}
SUBROUTINE Iout (index, data, dlen, dst)
INTEGER index, dlen, data(0:dlen-1), dst
\end{verbatim}
- sends the array {\tt data} to the process at co-ordinate {\tt dst}
in the specified index (and at the same co-ordinate as the calling process
in all other indexes).
\begin{verbatim}
SUBROUTINE IAout (index, data, dlen, dst)
INTEGER index, dlen, data(0:dlen-1), dst
\end{verbatim}
- an asynchronous version of {\tt Iout}.
\begin{verbatim}
SUBROUTINE Ibroad (index, data, length, src)
INTEGER index, length, src, data(0:length-1)
\end{verbatim}
- broadcasts the value in {\tt data} from the process at location
{\tt src} within {\tt index} to all processes at a different location
within that index (but at the same location within other indexes).
\begin{verbatim}
SUBROUTINE Imerge (index, data, length, dst, checkglobal)
INTEGER index, dst, length, data(0:length-1)
LOGICAL checkglobal
\end{verbatim}
- causes all processes other than the one at location {\tt dst} in {\tt index}
send {\tt data} to {\tt dst}. This performs a synchronization on that index.
If the variable {\tt checkglobal} is set to {\tt .TRUE.} then the destination
process checks to see that all data received is identical.
\section{Base Library II : accbas.u}
This library contains low-level routines to handle distributed channels.
The routines are primarily for use by the higher-level applications
library. Normally, the applications programmer will not use them.

\begin{tabular}{||l|l||} \hline
Occam Procedure/Function & Fortran Subroutine/Function \\ \hline \hline
IO.REC.set.external & IOwext \\ \hline
IO.REC.set.corank & IOwcor \\ \hline
IO.define  & IOdef2 \\ \hline
IO.input.define  & IOdefn \\ \hline
IO.output.define  & IOdefn \\ \hline
IO.rank  & IOrank \\ \hline
IO.at.root & IOatR \\ \hline
IO.at.coords & IOatC \\ \hline
IO.local.set & IOlset \\ \hline
IO.root.in & IOrti \\ \hline
IO.forward.in & IOfwdi \\ \hline
IO.root.out & IOrto \\ \hline
IO.forward.out & IOfwdo \\ \hline
IO.in & IOin \\ \hline
IO.tag.in & IOtin \\ \hline
IO.out & IOout \\ \hline
IO.tag.out & IOtout \\ \hline
IO.pre.synch & IOpre \\ \hline
IO.post.synch & IOpost \\ \hline
IO.REC.external & IOrext \\ \hline
IO.REC.corank & IOrcor \\ \hline
\end{tabular}

\begin{verbatim}
SUBROUTINE IOwext( data, extpid)
INTEGER data(0:IOmemsize-1), extpid
\end{verbatim}
- writes the value {\tt extpid} into the channel's external processor id
field.
\begin{verbatim}
SUBROUTINE IOwcor(data, corank, dim)
INTEGER data(0:IOmemsize-1), corank(0:dim-1)
\end{verbatim}
- writes the array {\tt corank} in the corank field of a channel.
\begin{verbatim}
SUBROUTINE IOdefn (chan, type, extpid, corank, length)
INTEGER chan, type, extpid, length, corank(0:length-1)
\end{verbatim}
- procedure creates a channel structure.
\begin{verbatim}
SUBROUTINE IOrank (chan, rank, rs)
INTEGER chan, rs, rank(0:rs-1)
\end{verbatim}
- defines the rank to be used prior to a data transfer taking place.
\begin{verbatim}
SUBROUTINE IOatR (chan, atroot)
INTEGER chan
LOGICAL atroot
\end{verbatim}
- sets the variable {\tt atroot} to be true or false depending on whether
the calling process is at the root of the specified channel.
\begin{verbatim}
SUBROUTINE IOatC (chan, atcoords, coords, clen)
INTEGER chan, clen, coords(0:clen-1), i, ranksize
LOGICAL atcoords
\end{verbatim}
- sets the variable {\tt atccords} to true or false depending on whether
the calling process is at the specified co-ordinates.
\begin{verbatim}
SUBROUTINE IOlset (chan, locset, glset, length)
INTEGER chan, length, locset(0:1, 0:length-1), glset(0:1, 0:length-1)
\end{verbatim}
This routine is provided for optimisation of procedures
for inputting or outputting multiple elements of a distribution.
{\tt glset} defines a volume of sites from which elements are to be
the input or output.  Both it and {\tt locset} are arrays sized
according to the current rank.  Each element defines a coordinate
range by a pair -- a base and count.
The {\tt locset} returned is a subset of {\tt glset}, namely
that set of sites for which the local process may have to
perform forwarding.  If the current transfer is to or from a site
which is in the global set but not the local set, the local process
needn't engage in a call to the corresponding ``forward'' operation.
\begin{verbatim}
SUBROUTINE IOrti  (chan, result, length)
INTEGER chan, length, result(0:length-1)
\end{verbatim}
- performs input on the root process.
\begin{verbatim}
SUBROUTINE IOfwdi (chan, data, dlen, coords, clen)
INTEGER chan, dlen, clen, data(0:dlen-1), coords(0:clen-1)
\end{verbatim}
- broadcasts the value of {\tt data} from root along the forwarding path
to the sites with {\tt coords} relative to the current
rank.  The size of the {\tt data} array must be global across all the
processes involved.
\begin{verbatim}
SUBROUTINE IOrto  (chan, data, length)
INTEGER chan, length, data(0:length-1)
\end{verbatim}
- performs an output on the root process.
\begin{verbatim}
SUBROUTINE IOfwdo (chan, data, dlen, coords, clen)
INTEGER chan, dlen, clen, data(0:dlen-1), coords(0:clen-1)
\end{verbatim}
- forwards data prior to it being output by the root process.
\begin{verbatim}
SUBROUTINE IOin (chan, result, rlen, coords, clen)
IMPLICIT NONE
INTEGER chan, rlen, clen, result(0:rlen-1), coords(0:clen-1)
\end{verbatim}
- performs input on a channel.
\begin{verbatim}
SUBROUTINE IOtin (chan, tag)
INTEGER chan, tag
\end{verbatim}
- inputs a tag on a channel.
\begin{verbatim}
SUBROUTINE IOout (chan, data, dlen, coords, clen)
INTEGER chan, dlen, clen, data(0:dlen-1), coords(0:clen-1)
\end{verbatim}
- outputs on a channel.
\begin{verbatim}
SUBROUTINE IOtout (chan, tag)
INTEGER chan, tag
\end{verbatim}
- outputs a tag on a channel.
\begin{verbatim}
SUBROUTINE IOpre (chan)
INTEGER chan
\end{verbatim}
- performs partial synchronization. This guarantees not to terminate on the
root until it has been called on all processes.
\begin{verbatim}
SUBROUTINE IOpost (chan)
INTEGER chan
\end{verbatim}
- performs partial synchronization. This guarantees not to terminate on any
site until after it has been called at the root.
\begin{verbatim}
SUBROUTINE IOrext (data, external)
INTEGER data(0:IOmemsize-1), external
\end{verbatim}
- returns the value in the {\em external id} field of a channel.
\begin{verbatim}
SUBROUTINE IOrcor (data, coranksize, corank, length)
INTEGER data(0:IOmemsize-1), coranksize, length, corank(0:length-1)
\end{verbatim}
- returns the corank field of a channel.
\section{Configuration routines}
The array communication library supports an array-based parallel computation
model. The user's program consists of multidimensional arrays of identical
processes; a typical configuration might be a host (conceptually a
zero-dimensional array) and a 2-D grid of worker processes. Associated with
each dimension is an object called an {\em index}. This has only one
important attribute, a {\em size}, which is the number of processes in that
dimension. In addition, there are objects called {\em channels} which allow
communication between arrays of processes. Unlike Occam channels and unlike
the channels in the Occam version of this library, these channels allow
bi-directional communication.

The user's program consists of code for the processes that are to be run and
a configuration level that initiates these processes. Below is an example
configuration level :

\begin{verbatim}
INTEGER xid, yid, zid
INTEGER xsize, ysize, zsize
INTEGER hostarr, gridarr, linearr
INTEGER griddim(3), linedim(1)
INTEGER hostch(1), gridch(2), linech(1)

PARAMETER (xid   = 0, yid   = 1, zid   = 2)
PARAMETER (xsize = 4, ysize = 4, zsize = 4)

PARAMETER (hostarr = 0, gridarr = 1, linearr = 2)

PARAMETER (hgchan = 0, glchan = 1)

DATA griddim /xid, yid, zid/, linedim /xid/
DATA hostch /hgchan/, gridch /hgchan, glchan/, linech /glchan/

CALL acbegin ()

CALL acindx (xid, xsize)
CALL acindx (yid, ysize)
CALL acindx (zid, zsize)

CALL acchan (hgchan, 0)
CALL acchan (glchan, 1, linedim)

CALL acarry (hostarr, 1, hostch, 0)
CALL acarry (gridarr, 2, gridch, 3, griddim)
CALL acarry (linearr, 2, linech, 3, linedim)
\end{verbatim}

The routines are :
\begin{verbatim}
SUBROUTINE acbegin()
\end{verbatim}
- performs housekeeping tasks. This should be the first
configuration routine to be called.
\begin{verbatim}
SUBROUTINE acindx(label, size)
INTEGER label, size
\end{verbatim}
- sets up an index structure of size {\em size}. The
label is a number uniquely indentifying that index, used solely within the
configuration level.
\begin{verbatim}
SUBROUTINE acchan(name, dimension, indexes)
INTEGER dimension, indexes(dimension)
\end{verbatim}
- sets up a channel or array of
channels. If only one channel is to be created, the parameter {\em dimension}
is set to zero and the last parameter may be ommited. If an array of channels
is required, {\em indexes} should be an array of {\em dimension} channels.
The number of parallel created is the product of the sizes of the indexes in
this array.
\begin{verbatim}
SUBROUTINE accarry(subroutine, nchans, channels, nindexes, indexes)
INTEGER nchans, nindexes, chans(nchans), indexes(nindexes)
\end{verbatim}
- spawns a
process array. The number of processes spawned is the product of the sizes of
the indexes. The parameter {\em subroutine} is the name of the routine to be
spawned. This subroutine must have a parameter list of the form {\tt nchans,
channels, nindexes, indexes}, where {\tt channels} is an array of
{\em nchans} channels and {\tt indexes} is an array of {\em nindexes}
indexes.
\begin{verbatim}
SUBROUTINE acend()
\end{verbatim}
- performs housekeeping tasks. This should be the last
configuration routine to be called.
\section{Example: matrix transpose}
Below is an example program using the array communication library.
The configuration level sets up a master process and a 2-D array of
3 by 3 processes. The master sets up a 3 by 3 matrix and sends it
to the workers. The workers transpose it using three ``in situ''
permuation operations and send it back to the master, which prints
it out.
\begin{verbatim}
C master
      SUBROUTINE display (source, rows, columns)
C  Read a stream of 'rows' * 'columns' real numbers from 'source',
C  and print as a table.
      IMPLICIT NONE
      INTEGER source, rows, columns
      INTEGER i, j, tag, data(0:40)
      INTEGER tend, tint, tr32, tstr, tuntagged
      PARAMETER (tuntagged = -1)
      PARAMETER (tend = 0)
      PARAMETER (tint = 1)
      PARAMETER (tr32 = 2)
      PARAMETER (tstr = 3)
      DO 10, i = 0, rows-1
        DO 20, j = 0, columns-1
           call INTAG(source, tag)
           IF (tag .EQ. tr32) THEN
             call INPUT(source, data(j), 1, tuntagged)
           ELSE
             STOP "Unexpected Tag value"
           ENDIF
20      CONTINUE
        print '(50(i3, ''  ''))', (data(j), j = 0, columns-1)
10    CONTINUE
      END
C
      SUBROUTINE transpose (chan, size)
      IMPLICIT NONE
      INTEGER chan, size, i, j
      INTEGER tend, tint, tr32, tstr, tuntagged
      PARAMETER (tuntagged = -1)
      PARAMETER (tend = 0)
      PARAMETER (tint = 1)
      PARAMETER (tr32 = 2)
      PARAMETER (tstr = 3)
      DO 30, i = 0, size-1
        DO 40, j = 0, size-1
          call OUTAG(chan, tr32)
          call OUTPUT(chan, ((size * i) + j), 1, tuntagged)
40      CONTINUE
30    CONTINUE
      print *, 'matrix'
      call display(chan, size, size)
      print *, 'transpose'
      call display (chan, size, size)
      END
C
      SUBROUTINE master (nchans, chan)
      IMPLICIT NONE
      INTEGER nchans
      INTEGER chan
      INTEGER tend, tint, tr32, tstr, tuntagged
      PARAMETER (tuntagged = -1)
      PARAMETER (tend = 0)
      PARAMETER (tint = 1)
      PARAMETER (tr32 = 2)
      PARAMETER (tstr = 3)
      INTEGER size, tag
50    CONTINUE
      call INTAG(chan, tag)
      IF (tag .EQ. tint) THEN
        call INPUT(chan, size, 1, tuntagged)
        print *, 'Calling tranpose'
        call transpose(chan, size)
        GOTO 50
      ENDIF
      END
C
C worker
      SUBROUTINE worker (nchans, chan, dim, rank)
      IMPLICIT NONE
      INTEGER dim, nchans
      INTEGER chan
      INTEGER rank (0 : dim - 1)
      INTEGER x, y
      INTEGER Isize, Icoord
      INTEGER tend, tint, tr32, tstr, tuntagged
      PARAMETER (tuntagged = -1)
      PARAMETER (tend = 0)
      PARAMETER (tint = 1)
      PARAMETER (tr32 = 2)
      PARAMETER (tstr = 3)
      INTEGER L, A
      x = rank(0)
      y = rank(1)
C  3 * 3 problem (one element per process).
      L = Isize (x)
      call OUTPUT (chan, L, 1, tint)
      call IN2D (chan, x, y, A, 1, tr32)
      call OU2D (chan, x, y, A, 1, tr32)
      call PSHFTI (x, A, 1, Icoord (y))
      call PREFLI (y, A, 1, Icoord (x))
      call PSHFTI (x, A, 1,-Icoord (y))
      call OU2D (chan, x, y, A, 1, tr32)
      call OUTAG (chan, tend)
      END
C
C configuration
      IMPLICIT NONE
      EXTERNAL master, worker
      INTEGER xid, yid
      INTEGER xsize, ysize
      INTEGER chan, null
      INTEGER rank (2)
      PARAMETER (xid   = 0, yid   = 1)
      PARAMETER (xsize = 3, ysize = 3)
      PARAMETER (chan = 0)
      DATA rank /xid, yid/
      CALL acbegn ()
      CALL acindx (xid, xsize)
      CALL acindx (yid, ysize)
      CALL acchan (chan, 0)
      CALL acarry (master, 1, chan, 0, null)
      CALL acarry (worker, 1, chan, 2, rank)
      CALL acdone ()
      END
\end{verbatim}
\bibliography{acl}
\end{document}
