1c1
< c @(#) lsdsr.f 1.2 (IDS) 1/19/95
---
> c %Z% %M% %I% (IDS) %G%
3c3,54
< C--- Southern Comm
---
> C+
> C*
> C*   This file contains the following routines in the nodal part 
> C*   of the host-node program :
> C*   
> C*   1) subroutine comm_south - Gathers elements from southern nodes.
> C*   2) subroutine comm_north - Gathers elements from northern nodes.
> C*   3) subroutine readjust_jj - Converts global jj's to local jj's 
> C*                              
> C-
> 
> C+ 
> C  Subroutine - comm_south
> C  This routine first copies the partition into a larger buffer 
> C  which hold a segment of the original wind component. It then          
> C  iterates for the number of southern hops as follows,         
> C         1) It fetches the lower bound of the to be received partition
> C         2) It verifies if the to be fetched partition is "wrapped"
> C            around the globe, i.e. crosses the south pole.          
> C         3) If the there is a pole crossover, then the book-keeping
> C            is done differently. Else, update the new upper bound to
> C            to be one less than current lower bound. Current lower
> C            bound changes to the newly received lower bound. 
> C         4) It then moves the data in the segment by the width
> C            of the to be received partition.
> C         5) It receives new partition.
> C         6) It copies new partition into segment.
> C
> C         When there is a pole crossover, the lower bound of the
> C         wrapped region is also noted, else the lower bound of
> C         of the segment alone needs to be known. Additionally the
> C         the width of the wrapped region is also noted in addition
> C         to the width of the segment.  These are needed for correct
> C         conversion to local indices.  Please see SCCS 694 for 
> C         additional information.
> C
> C  Inputs : 
> C
> C           win_d         - decomposed wind component
> C           lower, upper  - partition bounds on each node
> C           max_iter      - number of hops in each direction
> C  
> C  Outputs : 
> C           win_n         - buffer to hold the segment
> C           wr_south      - whether a pole cross-over occured in the
> C                           south direction.
> C           wr_lower      - lower bound of the wrapped region
> C           co_lower      - lower bound of the non-wrapped region
> C           wrapd         - width of wrapped region
> C           width         - width of segment
> C-
> 
18c69,72
<       
---
> 
> C
> C     Layout commons
> C      
22c76,83
<       integer mp_send_and_rcvint
---
> C
> C     Timing Code.
> C
>          real*8 gcbtime, lcbtime, cabtime, llitime, loitime,
>      &          brtime, cstime, cntime, reatime, fbtime, intime
>          common /ntime/ cabtime, gcbtime, lcbtime, llitime, 
>      &          loitime, brtime, cstime, cntime, reatime, fbtime,
>      &          intime
23a85,101
> 
> CT      Function prototypes
>         integer mp_send_and_rcvint
>         integer mp_timer_clear
>         integer mp_timer_start
>         integer mp_timer_stop
>         real*8 mp_elapsed_time
>         real*8 mp_busy_time
> CT
> 
> C
> C     c_range - Width of current partition 
> C     c_upper - upper bound of the partition
> C     c_lower - lower bound "   "      "
> C     t_lower - lower bound "  to be received partition
> C     wr_first - is the first time, pole is crossed
> C
26c104
<       integer i, j, k, tmp
---
>       integer i, j, k, tmp        !counters and temporary variables
27a106,108
> C
> C     Buffers for sending and receiving
> C
30a112,135
> 
> D      print *, "In subrt. comm_south"
> CT    Timing Code for measuring comm. south
> 	tmp = mp_timer_clear(7)
>         if (tmp .eq. -1) then
>            print *, 'Failure in clearing timer'
>            stop
>         end if
>         tmp = mp_timer_start(7)
>         if (tmp .eq. -1) then
>            print *, 'Failure in starting timer'
>            stop
>         end if
> 
> 	tmp = mp_timer_clear(8)
>         if (tmp .eq. -1) then
>            print *, 'Failure in clearing timer'
>            stop
>         end if
> CT    Timing Code 
> 
> C
> C     Initialization of variables.
> C
43a149,151
> C
> C     Set up the segment and sending buffers
> C
51a160,162
> C
> C     For the number of southern hops.
> C
53a165,175
> 
>           tmp = mp_timer_start(8)
>           if (tmp .eq. -1) then
>              print *, 'Failure in starting timer'
>              stop
>           end if
> 
> C
> C     Send lower bound to northern neighbor and receive 
> C                    from southern 
> C
56d177
< C          width = width + c_range
57a179,190
> CT    Timing Code 
>         tmp = mp_timer_stop(8)
>         if (tmp .eq. -1) then
>            print *, 'Failure in stopping timer'
>            stop
>         end if
> 
> C          width = width + c_range      ! not used
> 
> C
> C     If there is a pole crossover (c_lower is 1 the first time)
> C
59,63c192,196
< 	    wr_south = 1		
<             wr_first = 1
< 	    c_upper = jmp3
< 	    c_lower = t_lower
< 	    c_range = jmp3 - t_lower + 1
---
> 	    wr_south = 1	    ! pole crossover ocurred
>             wr_first = 1            ! first time 
> 	    c_upper = jmp3          ! upper bound is jmp3 now (slgcm.slgprm)
> 	    c_lower = t_lower       ! lower bound is new lower bound
> 	    c_range = jmp3 - t_lower + 1  ! update range
65a199,201
> C
> C     If a pole crossover occurred
> C
67,68c203,204
< 	    wr_lower = t_lower
<             wrapd = jmp3 + 1 - wr_lower
---
> 	    wr_lower = t_lower           ! Record lower bound for wrap region
>             wrapd = jmp3 + 1 - wr_lower  ! and the width
71,72c207,208
< 	  if (wr_south .eq. 0) then
< 	    co_lower = t_lower
---
> 	  if (wr_south .eq. 0) then      ! No Pole crossover
> 	    co_lower = t_lower           ! Record lower bound for segment
75,76c211,216
< 	if (wr_first .ne. 1) then
< 	  c_upper = c_lower - 1
---
> C
> C     If no pole crossover or not the first pole crossover, the 
> C     current upper bound is  the previous lower bound less one.
> C
> 	if (wr_first .ne. 1) then        
> 	  c_upper = c_lower - 1          
81c221
< 	wr_first = 0
---
> 	wr_first = 0              ! reset first time pole crossover
82a223,225
> C
> C     shift the data in the segment to make place for new partition
> C
89c232
<           width = width + c_range
---
>           width = width + c_range     ! update width of segment
91a235,251
> 
> CT    Timing Code for subraction of comm.
> D	tmp = mp_timer_clear(8)
> D        if (tmp .eq. -1) then
> D           print *, 'Failure in clearing timer'
> D           stop
> D        end if
>         tmp = mp_timer_start(8)
>         if (tmp .eq. -1) then
>            print *, 'Failure in starting timer'
>            stop
>         end if
> CT    Timing Code 
> 
> C
> C     Circulate or shift the partitions of wind components
> C
93a254,265
> 
> CT    Timing Code 
>         tmp = mp_timer_stop(8)
>         if (tmp .eq. -1) then
>            print *, 'Failure in stopping timer'
>            stop
>         end if
> C
> C      Subtract time for communication
> C
> D        cstime = cstime - mp_busy_time(8) 
> CT    Timing Code 
94a267,270
> C
> C     Copy the newly received partition to the segment. 
> C     Copy the receiving buffer to the sending buffer.
> C
102c278
< 	 k = k + 1
---
> 	 k = k + 1                          ! update no. of hops
104a281,296
> CT    Timing Code 
>         tmp = mp_timer_stop(7)
>         if (tmp .eq. -1) then
>            print *, 'Failure in stopping timer'
>            stop
>         end if
> C
> C       Accumulate times for measuring comm. south
> C
>         cstime = cstime + mp_busy_time(7) 
> C
> C      Subtract time for communication
> C
>         cstime = cstime - mp_busy_time(8) 
> CT    Timing Code 
> 
107a300,314
> 
> C+ 
> C  Subroutine - comm_north
> C  This routine first copies the partition into the send buffer.
> C  It then iterates for the number of northern hops as follows,         
> C         1) It fetches the upper bound of the to be received partition
> C         2) It verifies if the to be fetched partition is "wrapped"
> C            around the globe, i.e. crosses the northern pole.          
> C         3) If the there is a pole crossover, then the book-keeping
> C            is done differently. Else, update the new lower bound to
> C            to be one more than current upper bound. Current upper
> C            bound changes to the newly received upper bound. 
> C         4) It receives new partition from the northern neigbor.
> C         6) It copies new partition into segment at the northern end.
> C         7) Update width of segment.
108a316,319
> C         When there is a pole crossover for the first time, the
> C         lower bound is set to '1' and upper bound is set to newly
> C         received bound.  Please see SCCS 694 for 
> C         additional information.
109a321
> C  Inputs : 
111c323,332
< C--- Northern Comm
---
> C           win_d         - decomposed wind component
> C           lower, upper  - partition bounds on each node
> C           max_iter      - number of hops in each direction
> C  
> C  Outputs : 
> C           win_n         - buffer to hold the segment
> C           wr_north      - whether a pole cross-over occured in the
> C                           northern direction.
> C           width         - width of segment
> C-
125a347,350
> 
> C
> C     Layout commons
> C      
129c354,361
<       integer mp_send_and_rcvint
---
> C
> C     Timing Code.
> C
>           real*8 gcbtime, lcbtime, cabtime, llitime, loitime,
>      &          brtime, cstime, cntime, reatime, fbtime, intime
>          common /ntime/ cabtime, gcbtime, lcbtime, llitime, 
>      &          loitime, brtime, cstime, cntime, reatime, fbtime,
>      &          intime
130a363,379
> 
> CT      Function prototypes
>         integer mp_send_and_rcvint
>         integer mp_timer_clear
>         integer mp_timer_start
>         integer mp_timer_stop
>         real*8 mp_elapsed_time
>         real*8 mp_busy_time
> CT
> 
> C
> C     c_range - Width of current partition 
> C     c_upper - upper bound of the partition
> C     c_lower - lower bound "   "      "
> C     t_lower - lower bound "  to be received partition
> C     wr_first - is the first time, pole is crossed
> C
133c382
<       integer i, j, k, tmp
---
>       integer i, j, k, tmp      !counters and temporary variables
134a384,386
> C
> C     Buffers for sending and receiving
> C
137a390,412
> D      print *, "In subrt. comm_north"
> CT    Timing Code for measuring comm. north
> 	tmp = mp_timer_clear(7)
>         if (tmp .eq. -1) then
>            print *, 'Failure in clearing timer'
>            stop
>         end if
>         tmp = mp_timer_start(7)
>         if (tmp .eq. -1) then
>            print *, 'Failure in starting timer'
>            stop
>         end if
> 
> 	tmp = mp_timer_clear(8)
>         if (tmp .eq. -1) then
>            print *, 'Failure in clearing timer'
>            stop
>         end if
> CT    Timing Code 
> 
> C
> C     Initialization of variables.
> C
146a422,424
> C
> C     Set up the sending buffer
> C
152a431,433
> C
> C     For the number of northern hops.
> C
154a436,446
> 
>            tmp = mp_timer_start(8)
>            if (tmp .eq. -1) then
>               print *, 'Failure in starting timer'
>               stop
>            end if
> 
> C
> C     Send lower bound to southern neighbor and receive 
> C                    from northern 
> C
157a450,455
>            tmp = mp_timer_stop(8)
>            if (tmp .eq. -1) then
>               print *, 'Failure in stopping timer'
>               stop
>            end if
> 
159a458,460
> C
> C     If there is a pole crossover (c_upper equals jmp3 the first time)
> C
161,165c462,466
< 	    wr_north = 1		
<             wr_first = 1
< 	    c_upper = t_upper
< 	    c_lower = 1
< 	    c_range = t_upper - c_lower + 1
---
> 	    wr_north = 1		! pole crossover ocurred
>             wr_first = 1                ! first time
> 	    c_upper = t_upper           ! update upper bound
> 	    c_lower = 1                 ! set lower bound to 1
> 	    c_range = t_upper - c_lower + 1  ! update range
167a469,472
> C
> C     If no pole crossover or not the first pole crossover, the 
> C     current lower bound is the previous lower bound plus one.
> C
174c479
< 	wr_first = 0
---
> 	wr_first = 0                     ! reset first time
176a482,498
> 
> CT    Timing Code for subraction of comm.
> D	tmp = mp_timer_clear(8)
> D        if (tmp .eq. -1) then
> D           print *, 'Failure in clearing timer'
> D           stop
> D        end if
>         tmp = mp_timer_start(8)
>         if (tmp .eq. -1) then
>            print *, 'Failure in starting timer'
>            stop
>         end if
> CT    Timing Code 
> 
> C
> C     Circulate or shift the partitions of wind components
> C
179a502,512
> CT    Timing Code 
>         tmp = mp_timer_stop(8)
>         if (tmp .eq. -1) then
>            print *, 'Failure in stopping timer'
>            stop
>         end if
> C
> C      Subtract time for communication
> C
> D        cntime = cntime - mp_busy_time(8) 
> CT    Timing Code 
180a514,517
> C
> C     Copy the newly received partition to the segment at northern edge.
> C     Copy the receiving buffer to the sending buffer.
> C
188,189c525,526
<             width = width + c_range
<             k = k + 1
---
>             width = width + c_range       ! update width of segment
>             k = k + 1                     ! update no. of hops
191a529,544
> CT    Timing Code 
>         tmp = mp_timer_stop(7)
>         if (tmp .eq. -1) then
>            print *, 'Failure in stopping timer'
>            stop
>         end if
> C
> C       Accumulate times for measuring comm. north
> C
>         cntime = cntime + mp_busy_time(7) 
> C
> C      Subtract time for communication
> C
>         cntime = cntime - mp_busy_time(8) 
> CT    Timing Code 
> 
195a549,562
> C+ 
> C  Subroutine - readjust_jj
> C  This routine first ascertains if there was a "wrap" in either 
> C  of the directions.  According to that there are 3 cases.
> C     1) No Wrap
> C     2) Southern Wrap (South pole crossover)
> C     3) Northern Wrap (North pole crossover)
> C     
> C  If there is no wrap, then the local index 
> C                          = global index - lower bound of segment + 1
> C  If there is a southern wrap, it further decides, if the index is
> C  falls within the wrapped part or not. If it is in the wrapped part,
> C  local index = global index - lower bound of wrapped region + 1
> C  Else it is global index + width of wrapped part.
196a564,567
> C  If there is a northern wrap, it further decides, if the index is
> C  falls within the wrapped part or not. If it is in the wrapped part,
> C  local index = global index + width of non-wrapped part
> C     Else it is global index - lower bound of non-wrapped part +1
199c570,583
< C--- Northern Comm
---
> C  Inputs : 
> C
> C           jj                  - global indices
> C           wr_south, wr_north  - did a wrap occur ?
> C                                 0 - No ; 1 - Yes
> C                                 
> C           width         - width of segment
> C           wr_lower      - lower bound of wrapped part (southern wrap)
> C           co_lower      - lower bound of segment
> C           wrapd         - width of wrapped part (southern wrap)
> C
> C  Outputs : 
> C           njj           - local indices
> C-
207a592,597
>          real*8 gcbtime, lcbtime, cabtime, llitime, loitime,
>      &          brtime, cstime, cntime, reatime, fbtime, intime
>          common /ntime/ cabtime, gcbtime, lcbtime, llitime, 
>      &          loitime, brtime, cstime, cntime, reatime, fbtime,
>      &          intime
> 
213c603
<       integer i, j
---
>       integer i, j, tmp
214a605,630
> CT      Function prototypes
>         integer mp_timer_clear
>         integer mp_timer_start
>         integer mp_timer_stop
>         real*8 mp_elapsed_time
>         real*8 mp_busy_time
> CT
> 
> D      print *, "In subrt. read_jj"
> CT    Timing Code for measuring calcn. of local indices
> 	tmp = mp_timer_clear(7)
>         if (tmp .eq. -1) then
>            print *, 'Failure in clearing timer'
>            stop
>         end if
>         tmp = mp_timer_start(7)
>         if (tmp .eq. -1) then
>            print *, 'Failure in starting timer'
>            stop
>         end if
> CT    Timing Code 
> 
> C
> C     If no wrapped region, 
> C
> 
223a640,642
> C
> C     If wrapped over the south pole
> C
227c646
< C               if (jj(i,j) .ge. wr_lower) then
---
> C               if (jj(i,j) .ge. wr_lower) then        ! if in wrapped part
236a656,658
> C
> C       If wrapped over North pole
> C
240c662,664
<                if (jj(i,j) .lt. co_lower) then
---
>                if (jj(i,j) .lt. co_lower) then            ! if in wrapped part
> 
> C                 jmp3 - co_lower + 1 is width of non-wrapped part
247a672,684
> 
> CT    Timing Code 
>         tmp = mp_timer_stop(7)
>         if (tmp .eq. -1) then
>            print *, 'Failure in stopping timer'
>            stop
>         end if
> C
> C      Accumulate times for calculating local indices
> C
>         reatime = reatime + mp_busy_time(7) 
> CT    Timing Code 
> 
