\documentstyle[11 pt]{article}

\textwidth=6.0in
\textheight=8.0in
\voffset=-0.5in
\hoffset=-0.5in

\begin{document}
\title{Annotated node programs for the HPF \\
program template for FORALL statement}

\author{Xiaoming Li}

\date{May 24, 1996}

\maketitle

\section*{First, some general words}

This is a followup document to "The HPF program templates for
initial FORALL statement compilation". As an initial guide for
the compilation and runtime construction process, 
detailed node programs for each of the three templates are
suggested in this document. Basic considerations is: simple
enough for a rapid prototyping, while leaving room for 
future generalization and performance tuning.
\paragraph{}
As I can see, node program may take the following general framework.
\begin{itemize}
\item
Data declaration. This is a result of processing HPF data alignment
and distribution directives, and an analysis of temporary storage
requirement by the program.
\item
Initialization, including initialization of underline communication 
system and the DAD for distributed arrays.
\item
Statement translation. Since we are doing a limited compilation,
not much effort is expected in global analysis and optimization. As
a result, compiler is very much working in a linear fashion, namely 
statement by statement translation.
\end{itemize}

\section*{Node program for level one template}

The following was the proposed HPF program template.

\begin{verbatim}
       PROGRAM MAIN
       REAL X(2:99), Y(100)
 !HPF$ PROCESSORS P(4)
 !HPF$ TEMPLATE TX(400),TY(-200:199)
 !HPF$ DISTRIBUTE TX(BLOCK) ONTO P
 !HPF$ DISTRIBUTE TY(BLOCK) ONTO P
 !HPF$ ALIGN X(i) WITH TX(2*i+1)   
 !HPF$ ALIGN Y(i) WITH TY(3*i-150)
       FORALL (i=2:99) X(i) = 0
       FORALL (i=1:100) Y(i) = i
       FORALL (i=2:99) X(i) = Y(i+1) + Y(i-1)
       PRINT *, (X(i),i=2,99)       
       END
\end{verbatim}

We see it consists of
\begin{itemize}
\item
Three FORALL statements, involving one or two distributed arrays, to be
executed on four processors. Each FORALL presents some distinct feature
for compiler to handle.
\item
A PRINT statement involving a distributed array. This may give rise to
some parallel I/O issue, namely, do we want the array to be printed by
one processor, or by four processor collectively ?
\end{itemize}

Here is a possible node program corresponding to it. Illustration follows.
\begin{verbatim}
1       PROGRAM MAIN
2       REAL X(100), Y(100), t1(100), t2(100), t3(400), t4(400)
3       REAL u1,v1,u2,v2
4       INTEGER pDim(1),comm,myid,xDAD,yDAD,t1DAD,t2DAD
5       INTEGER i,ll,lu,ls,amount1,amount2,status1,status2
6  C     REAL X(2:99), Y(100)
7  CHPF$ PROCESSORS P(4)
8  CHPF$ TEMPLATE TX(400),TY(-200:199)
9  CHPF$ DISTRIBUTE TX(BLOCK) ONTO P
10 CHPF$ DISTRIBUTE TY(BLOCK) ONTO P
11 CHPF$ ALIGN X(i) WITH TX(2*i+1)  
12 CHPF$ ALIGN Y(i) WITH TY(3*i-150)
14 C 
15      pDim(1) = 4
16      CALL MPI_INIT(ierror)
17      CALL gridInit(1,pDim,comm)
18      xDAD = setPerArrayInfo(X,2,4,1,comm,1)
19      CALL setCommonValues(xDAD,1,2,99,1,0,1,400,1,2,1,4)
20      CALL setDiffValues(xDAD,1)
21      yDAD = setPerArrayInfo(Y,2,4,1,comm,1)
22      CALL setCommonValues(yDAD,1,1,100,1,0,-200,199,-150,3,1,4)
23      CALL setDiffValues(yDAD,1)
24
25 C     FORALL (i=2:99) X(i) = 0
26      CALL loopBounds(xDAD,2,99,1,ll,lu,ls,1)
27      DO i = ll,lu,ls
28         X(i) = 0
29      END
30
31 C     FORALL (i=1:100) Y(i) = i
32      CALL loopBounds(yDAD,1,100,1,ll,lu,ls,1)
33      DO i = ll,lu,ls
34         Y(i) = localToGlobal(yDAD,i,1)
35      END
36
37 C     FORALL (i=2:99) X(i) = Y(i+1) + Y(i-1)
38      CALL loopBounds(xDAD,2,99,1,ll,lu,ls,1)
39      status1 = detectCommunication(xDAD,yDAD,1,2,99,1,1,1,u1,v1,amount1
40     +)
41 
42      IF (status1.EQ.1) THEN
43         t1DAD = DAD_copy(t1,yDAD)
45         CALL tshift(yDAD,t1DAD,1,amount1)
46      ELSE IF(status1.EQ.2) THEN
47         CALL concatenate(yDAD,t3)
48      END IF
49
50      status2 = detectCommunication(xDAD,yDAD,1,2,99,1,1,-1,u2,v2,amount
51     +2)
52
53      IF (status2.EQ.1) THEN
54         t2DAD = DAD_copy(t2,yDAD)
56         CALL tshift(yDAD,t2DAD,1,amount2)
57      ELSE IF(status2.EQ.2) THEN
58         CALL concatenate(yDAD,t4)
59      END IF
60
61      DO i = ll,lu,ls
62         i1 = NINT(u1*i+v1)
63         i2 = NINT(u2*i+v2)
64         IF(status1.EQ.0.AND.status2.EQ.0) THEN
65            X(i) = Y(i1) + Y(i2)
66         ELSE IF(status1.EQ.0.AND.status2.EQ.1) THEN
67            X(i) = Y(i1) + t2(i2)
68         ELSE IF(status1.EQ.0.AND.status2.EQ.2) THEN
69            X(i) = Y(i1) + t4(i2)
70         ELSE IF(status1.EQ.1.AND.status2.EQ.0) THEN
71            X(i) = t1(i1) + Y(i2)
72         ELSE IF(status1.EQ.1.AND.status2.EQ.1) THEN
73            X(i) = t1(i1) + t2(i2)
74         ELSE IF(status1.EQ.1.AND.status2.EQ.2) THEN
75            X(i) = t1(i1) + t4(i2)
76         ELSE IF(status1.EQ.2.AND.status2.EQ.0) THEN
77            X(i) = t3(i1) + Y(i2)
78         ELSE IF(status1.EQ.2.AND.status2.EQ.1) THEN
79            X(i) = t3(i1) + t2(i2)
80         ELSE IF(status1.EQ.2.AND.status2.EQ.2) THEN
81            X(i) = t3(i1) + t4(i2)
82         ELSE
83            PRINT *, 'something is wrong !'
84         END IF
85      END DO
87 C 
89      CALL MPI_COMM_RANK(comm,myid,ierror)
90      CALL loopBounds(xDAD,2,99,1,ll,lu,ls)
91      PRINT *, 'From process',myid
92      PRINT *, (X(i),i=ll,lu,ls) 
93      END PROGRAM
\end{verbatim}

\paragraph{Annotation}
\begin{itemize}
\item
First, we notice 10 runtime functions are called, besides two direct MPI calls.
  \begin{itemize}
  \item gridInit()
  \item setPerArrayInfo()
  \item setCommonValues()
  \item setDiffValues()
  \item loopBounds()
  \item localToGlobal()
  \item detectCommunication()
  \item DAD\_copy(), it copies a DAD, except baseAddress.
  \item tshift()
  \item concatenate()
  \end{itemize}
They are described in my CRSI draft in detail. In fact, the 3 {\em set} functions 
can be considered as one. They construct a DAD collectively.
\item
Line 15-17: Initialize MPI and processor grid. An array {\em pDim} is used
to tell gridInit() the number of processors in each dimension of the grid.
{\em comm} is the id corresponding to this processor grid, in terms of
MPI communicator. If MPICH has no problem, the call to MPI\_INIT() should be
inside gridInit(), eliminated from the node program.
\item
Line 18-23: Create the DADs for distributed arrays declared in HPF program.
\item
Line 25-28: Initialize array X with constant, only need to identify {\em
effective} elements in local array. This is accomplished by determining
proper loop bounds and stride.
\item
Line 31-35: Initialize array Y with FORALL index. Besides determining loop
bounds, we also need to perform local-to-global index conversion, in order
to get correct values for Y elements.
\item
Line 37-85 is for the FORALL statement involving two arrays. There are
basically four general steps involved.
  \begin{itemize}
  \item Determine loop bounds as for other FORALL statements.
  \item Analysis communication needed for each term at the right hand side
        (rhs) of the assignment.
  \item Perform the communication, if needed.
  \item Perform the computation defined by rhs expression on
        local elements.
  \end{itemize}
where the first item is not a prerequisite of others, while the other three
should be done in sequence. We employ {\em owner computes rule} in the scheme. 
\paragraph{}
detectCommunication() is a major runtime function, which analyzes the
data alignment relation between one rhs term and the lhs. It employs an
aggressive algorithm to figure out whether communication is needed, and
the form of communication, if needed. It also produces the adjustment
for rhs subscript expression.

detectCommunication() classifies communication patterns into three cases,
   \begin{itemize}
   \item No communication.
   \item A global shift. rhs term is shifted to a temporary storage, which
         has the same distribution as the rhs term.
   \item A concatenation. Let each node program have a copy of entire
         global array.
   \end{itemize}

The DO loop from line 61 to 85 enumerates all cases. In general, if there
are m terms in the rhs, the big IF statement will have $3^m$ cases.
This is the situation when communication detection is done at runtime.
As anticipated, a version of the communication detection function will
be used at compile time in our next step, which will result in much more concise
node program. something like following may be expected:

\begin{verbatim}
1       PROGRAM MAIN
10      ......
37 C    FORALL (i=2:99) X(i) = Y(i+1) + Y(i-1)
38      ll = ...
39      lu = ...
40      ls = ...
41 C ll and lu are generally processor dependent.
42      t1DAD = setPerArrayInfo(t1,2,4,1,comm,1)
43      CALL setCommonValues(t1DAD,1,1,100,1,0,-200,199,-150,3,1,4)
44      CALL setDiffValues(t1DAD,1)
45      CALL tshift(yDAD,t1DAD,1,amount1)
46 C amount1 is a constant figured out at compile time
49
58      CALL concatenate(yDAD,t4)
59
60      DO i = ll,lu,ls
61 C u1,v1,u2,v2 are real constants figured out at compile time
62         i1 = NINT(u1*i+v1)
63         i2 = NINT(u2*i+v2)
75         X(i) = t1(i1) + t4(i2)
85      END DO
86      ......
93      END PROGRAM
\end{verbatim}

\item
Line 89-91: We let each processor print the elements it owns. Another possible
approach could be to collect (gather) all elements into one processor and let the
processor print them. 
\end{itemize}

\section*{Node program for level two template}
Here was the proposed HPF program template. 
\begin{verbatim}
      PROGRAM MAIN
      REAL X(2:99), Y(100)
!HPF$ PROCESSORS P(4)
!HPF$ TEMPLATE TX(400),TY(-200:199)
!HPF$ DISTRIBUTE TX(BLOCK) ONTO P
!HPF$ DISTRIBUTE TY(BLOCK) ONTO P
!HPF$ ALIGN X(i) WITH TX(2*i+1) 
!HPF$ ALIGN Y(i) WITH TY(3*i-150)
      FORALL (i=2:99) X(i) = 0
      FORALL (i=1:100) Y(i) = i
      FORALL (i=2:99) X(i) = Y(i+1) + Y(i-1)

      CALL FOO(X(2:98:2),Y(1:99:2))        !!! new
      CALL FOO(X(3:99:2),Y(2:100:2))       !!! new

      PRINT *, (X(i),i=2,99)
      END

      SUBROUTINE FOO(X,Y)                           !!! new
      REAL X(49), Y(50)                             !!! new
!HPF$ INHERIT X,Y                                   !!! new
      FORALL (i=1:49) X(i) = X(i) - Y(i) - Y(i+1)   !!! new
      RETURN                                        !!! new
      END                                           !!! new
\end{verbatim}

To concentrate, we only address the node program segment
for the ``new'' stuff. The issues are: 
\begin{itemize}
\item
Whether we want to support separate compilation. Temporary storage
allocation is the major problem for separate compilation. Since
Fortran 77 does not support dynamic storage allocation, compiler
in general has no way to know how much space should be allocated
for those temporary storage related to dummy arguments (such as t1
and t3 for X in the first node program.).
\item
How to handle the case where underline array for an actual argument 
has different rank than corresponding dummy argument. DAD should be
adjusted to reflect this situation. For instance, to pass
A(i,1:n) of A(n,n) to a subroutine, the caller should prepare a DAD
(by DAD\_inherit\_caller) so that the callee sees a strided one
dimensional array. 
\item
Even for equal ranks, caller may see a strided memory locations, while 
callee sees a consecutive memory block. Thus, X(i) in callee HPF
program really means X(f(i)) for strided actual argument, which then
becomes X(globalToLocal(f(i))) in node program. This implies that
the shape of a dummy array in node program will be determined by 
caller. Thus, a global dummy array X (as in HPF program) will
corresponds to (X,xDAD,d1,d2,...,dm) in node program dummy list,
where d1, d2, ..., dm are used in node subroutine for dimensioning
X.
\end{itemize}

The thought is,
\begin{itemize}
\item
Support (or prepare to support) separate compilation. But now in some
restricted fashion due to Fortran 77, namely, we take care of other
aspects of separate compilation while imposing some limit on maximum
storage requirement. This may be implemented in two ways. One is assuming
maximum array size and processor number used in HPF programs. The other
is allocating a big COMMON block, from which program units to request 
temporary storage. For now, we take the first approach in this document.

\item
INHERIT may be implemented in two ways. One is the `true' INHERIT, the
other is some kind of copy-in/copy-out by caller. From callee point of view,
they are both INHERIT. We'll try the `true' one, whenever it's possible.
\end{itemize}

The following is a possible node program following the above considerations.

\begin{verbatim}
      PROGRAM MAIN
      INTEGER TRIPLES(7,3),NX(7),NY(7)
      ......
C     CALL FOO(X(2:98:2),Y(1:99:2))
      TRIPLES(1,1) = 2
      TRIPLES(1,2) = 98
      TRIPLES(1,3) = 2
C     (other entries = something for null)
      CALL DAD_inheritCaller(xDAD,xsDAD,TRIPLES,NX)
      TRIPLES(1,1) = 1
      TRIPLES(1,2) = 99
      TRIPLES(1,3) = 2
C     (other entries = something for null)
      CALL DAD_inheritCaller(yDAD,ysDAD,TRIPLES,NY)
      CALL FOO(X,xsDAD,NX(1),Y,ysDAD,NY(1))

C     CALL FOO(X(3:99:2),Y(2:100:2))
      TRIPLES(1,1) = 3
      TRIPLES(1,2) = 99
      TRIPLES(1,3) = 2
C     (other entries = something for null)
      CALL DAD_inheritCaller(xDAD,xsDAD,TRLPLES,NX)
      TRIPLES(1,1) = 2
      TRIPLES(1,2) = 100
      TRIPLES(1,3) = 2
C     (other entries = something for null)
      CALL DAD_inheritCaller(yDAD,ysDAD,TRIPLES,NY)
      CALL FOO(X,xsDAD,NX(1),Y,ysDAD,NY(1))
      ......
      END

      SUBROUTINE FOO(X,xDAD,NX,Y,yDAD,NY)
      REAL X(NX), Y(NY), t1(100),t2(100),t3(400),t4(400)
      INTEGER xDAD,yDAD,t1DAD,t2DAD
CHPF$ INHERIT X,Y
      CALL DAD_inheritCallee(xDAD,xDAD,1,1,49,1)
      CALL DAD_inheritCallee(yDAD,yDAD,1,1,50,1)
            
C     FORALL (i=1:49) X(i) = X(i) - Y(i) - Y(i+1)
      CALL loopBounds(xDAD,1,49,1,ll,lu,ls,1)
      status1 = detectCommunication(xDAD,yDAD,1,1,NX,1,1,0,u1,v1,
     +amount1)
      IF (status1.EQ.1) THEN
         t1DAD = DAD_copy(t1,yDAD)
         CALL tshift(yDAD,t1DAD,1,amount1)
      ELSE IF(status1.EQ.2) THEN
         CALL concatenate(yDAD,t3)
      END IF

      status2 = detectCommunication(xDAD,yDAD,1,2,99,1,1,1,u2,v2,
     +amount2)
      IF (status2.EQ.1) THEN
         t2DAD = DAD_copy(t2,yDAD)
         CALL tshift(yDAD,t2DAD,1,amount2)
      ELSE IF(status2.EQ.2) THEN
         CALL concatenate(yDAD,t4)
      END IF

      DO i = ll,lu,ls
         i1 = NINT(u1*i+v1)
         i2 = NINT(u2*i+v2)
         IF(status1.EQ.0.AND.status2.EQ.0) THEN
            X(i) = X(i) - Y(i1) - Y(i2)
         ELSE IF(status1.EQ.0.AND.status2.EQ.1) THEN
            X(i) = X(i) - Y(i1) - t2(i2)
         ELSE IF(status1.EQ.0.AND.status2.EQ.2) THEN
            X(i) = X(i) - Y(i1) - t4(i2)
         ELSE IF(status1.EQ.1.AND.status2.EQ.0) THEN
            X(i) = X(i) - t1(i1) - Y(i2)
         ELSE IF(status1.EQ.1.AND.status2.EQ.1) THEN
            X(i) = X(i) - t1(i1) - t2(i2)
         ELSE IF(status1.EQ.1.AND.status2.EQ.2) THEN
            X(i) = X(i) - t1(i1) - t4(i2)
         ELSE IF(status1.EQ.2.AND.status2.EQ.0) THEN
            X(i) = X(i) - t3(i1) - Y(i2)
         ELSE IF(status1.EQ.2.AND.status2.EQ.1) THEN
            X(i) = X(i) - t3(i1) - t2(i2)
         ELSE IF(status1.EQ.2.AND.status2.EQ.2) THEN
            X(i) = X(i) - t3(i1) - t4(i2)
         ELSE
            PRINT *, 'something is wrong !'
         END IF
      END DO
      RETURN
      END SUBROUTINE
\end{verbatim}

\section*{Node program for level three template}
We allow two-dimensional arrays in this level.
The proposed sample program was:
\begin{verbatim}
      PROGRAM MAIN
      PARAMETER (N=100)
      REAL X(N,N), Y(N,N)
!HPF$ PROCESSORS P(4)
!HPF$ TEMPLATE TX(200,200),TY(100,100)
!HPF$ DISTRIBUTE TX(BLOCK,*) ONTO P
!HPF$ DISTRIBUTE TY(BLOCK,*) ONTO P
!HPF$ ALIGN X(i,j) WITH TX(2*i,2*j)
!HPF$ ALIGN Y(i,j) WITH TY(i,j)

      FORALL (i=1:N,j=1:100) X(i,j) = 0
      FORALL (i=1:100,j=1:N) Y(i,j) = i + j

      FORALL (i=26:50,j=26:50) X(i,j) = Y(i+1,j) + Y(i,j+1)
      FORALL (i=1:100:2,j=1:N:2,X(i,j)/=0) X(i,j) = 1

      CALL FOO(X(1:100:2,1))

      PRINT *, (X(i,1),i=1,100,2)
      END

      SUBROUTINE FOO(X)
      REAL X(50)
!HPF$ INHERIT X
      REAL Y(50)
!HPF$ PROCESSORS P(4)
!HPF$ DISTRIBUTE Y(BLOCK) ONTO P

      FORALL (i=1:50) Y(i) = 2
      FORALL (i=1:50) X(i) = Y(i) + 1

      RETURN
      END
\end{verbatim}

\paragraph{The node program.}

\begin{verbatim}
      PROGRAM MAIN
C     PARAMETER (N=100)
      REAL X(50,200), Y(25,100)
CHPF$ PROCESSORS P(4)
CHPF$ TEMPLATE TX(200,200),TY(100,100)
CHPF$ DISTRIBUTE TX(BLOCK,*) ONTO P
CHPF$ DISTRIBUTE TY(BLOCK,*) ONTO P
CHPF$ ALIGN X(i,j) WITH TX(2*i,2*j)
CHPF$ ALIGN Y(i,j) WITH TY(i,j)
      INTEGER TRIPLES(7,3), NX(7)

      ......
C     FORALL (i=1:N,j=1:100) X(i,j) = 0
      call loopBounds(xDAD,1,100,1,ll1,lu1,ls1,1)
      call loopBounds(xDAD,1,100,1,ll2,lu2,ls2,2)
      DO i=ll1,lu1,ls1
        DO j=ll2,lu2,ls2
          X(i,j) = 0
        END DO
      END DO

C     FORALL (i=1:100,j=1:N) Y(i,j) = i + j
      call loopBounds(yDAD,1,100,1,ll1,lu1,ls1,1)
      call loopBounds(yDAD,1,100,1,ll2,lu2,ls2,2)
      DO i=ll1,lu1,ls1
        DO j=ll2,lu2,ls2
          Y(i,j) = localToGlobal(yDAD,i,1) + localToGlobal(yDAD,j,2)
        END DO
      END DO

C     FORALL (i=26:50,j=26:50) X(i,j) = Y(i+1,j) + Y(i,j+1)
      call loopBounds(xDAD,26,50,1,ll1,lu1,ls1,1)
      call loopBounds(xDAD,26,50,1,ll2,lu2,ls2,2)
      status1 = detectCommunication(xDAD,yDAD,1,26,50,1,1,1,u1,v1,
     +amount1)
      status2 = detectCommunication(xDAD,yDAD,2,26,50,1,1,0,u2,v2,
     +amount2)
      IF (status1.EQ.2.OR.status2.EQ.2) THEN
        concatenate(yDAD,t3)
      ELSE 
        IF (status1.EQ.1.AND.status2.EQ.0) THEN
          tshift(yDAD,t1DAD,1,amount1)
        ELSE IF (status2.EQ.1.AND.status1.EQ.0) THEN
          tshift(yDAD,t1DAD,2,amount2)
        ELSE IF (status2.EQ.1.AND.status1.EQ.1) THEN
          tshift(yDAD,t1DAD,1,amount1)
          tshift(t1DAD,t1DAD,2,amount2)
        END IF
      END IF

      status1 = detectCommunication(xDAD,yDAD,1,26,50,1,1,0,u3,v3,
     +amount1)
      status2 = detectCommunication(xDAD,yDAD,2,26,50,1,1,1,u4,v4,
     +amount2)
      IF (status1.EQ.2.OR.status2.EQ.2) THEN
        concatenate(yDAD,t4)
      ELSE 
        IF (status1.EQ.1.AND.status2.EQ.0) THEN
          tshift(yDAD,t2DAD,1,amount1)
        ELSE IF (status2.EQ.1.AND.status1.EQ.0) THEN
          tshift(yDAD,t2DAD,2,amount2)
        ELSE IF (status2.EQ.1.AND.status1.EQ.1) THEN
          tshift(yDAD,t2DAD,1,amount1)
          tshift(t1DAD,t2DAD,2,amount2)
        END IF
      END IF

      DO i=ll1,lu1,ls1
        DO j=ll2,lu2,ls2
C         a big IF-THEN-ELSE IF-END IF
        END DO
      END DO

C     FORALL (i=1:100:2,j=1:N:2,X(i,j)/=0) X(i,j) = 1
      call loopBounds(xDAD,1,100,2,ll1,lu1,ls1,1)
      call loopBounds(xDAD,1,100,2,ll2,lu2,ls2,2)
      DO i=ll1,lu1,ls1
        DO j=ll2,lu2,ls2
          IF(X(i,j).NE.0) X(i,j) = 1
        END DO
      END DO

C     CALL FOO(X(1:100:2,1))
      TRIPLES(1,1)=1
      TRIPLES(1,2)=100
      TRIPLES(1,3)=2
      TRIPLES(2,1)=1
C     (other TRIPLES entries = somethign for null)
      CALL DAD_inheritCaller(xDAD,xsDAD,TRIPLES,NX)
      CALL FOO(X,xsDAD,NX(1))

C     PRINT *, (X(i,1),i=1,100,2)
      CALL MPI_COMM_RANK(comm,myid,ierror)
      call loopBounds(xDAD,1,100,2,ll1,lu1,ls1,1)
      PRINT *, 'From process',myid
      PRINT *, (X(i),i=ll,lu,ls)
      END

      SUBROUTINE FOO(X,xDAD,N)
      REAL X(N)
CHPF$ INHERIT X
      REAL Y(13),t1(13),t3(52)
CHPF$ PROCESSORS P(4)
CHPF$ DISTRIBUTE Y(BLOCK) ONTO P
      CALL DAD_inheritCallee(xDAD,xDAD,1,1,50,1)
      pDim(1) = 4
      CALL gridInit(1,pDim,comm)
      yDAD = setPerArrayInfo(Y,2,4,1,comm,1)
      CALL setCommonValues(yDAD,1,1,50,1,0,1,50,0,1,1,4)
      CALL setDiffValues(yDAD,1)
C     FORALL (i=1:50) Y(i) = 2
      CALL loopBounds(yDAD,1,50,1,ll,lu,ls,1)
      DO i = ll, lu,ls
        Y(i) = 2
      END DO

C     FORALL (i=1:50) X(i) = Y(i) + 1
      CALL loopBounds(xDAD,1,50,1,ll,lu,ls,1)
      status1 = detectCommunication(xDAD,yDAD,1,1,50,1,1,0,u1,v1,
     +amount1)
      IF (status1.EQ.1) THEN
         t1DAD = DAD_copy(t1,yDAD)
         CALL tshift(yDAD,t1DAD,1,amount1)
      ELSE IF(status1.EQ.2) THEN
         CALL concatenate(yDAD,t3)
      END IF

      DO i = ll,lu,ls
        i1 = NINT(u1*i+v1)
        IF (status1.EQ.0) THEN
          X(i) = Y(i1) + 1
        ELSE IF (status1.EQ.1) THEN
          X(i) = t1(i1) + 1
        ELSE IF (status1.EQ.2) THEN
          X(i) = t3(i1) + 1
        ELSE
          PRINT *, 'something is wrong'
        END IF
      END DO
      RETURN
      END
\end{verbatim}
\end{document}
